clarity 0.9.0

data/public/stylesheets/app.css

@@ -0,0 +1,54 @@
+/* ----------------------------------------------------------------------------
+ * Log Search Server CSS
+ * by John Tajima
+ * ----------------------------------------------------------------------------
+ */
+
+/* 960 reset */ 
+html,body,div,span,applet,object,iframe,h1,h2,h3,h4,h5,h6,p,blockquote,pre,a,abbr,acronym,address,big,cite,code,del,dfn,em,font,img,ins,kbd,q,s,samp,small,strike,strong,sub,sup,tt,var,b,u,i,center,dl,dt,dd,ol,ul,li,fieldset,form,label,legend,table,caption,tbody,tfoot,thead,tr,th,td{margin:0;padding:0;border:0;outline:0;font-size:100%;vertical-align:baseline;background:transparent}body{line-height:1}ol,ul{list-style:none}blockquote,q{quotes:none}blockquote:before,blockquote:after,q:before,q:after{content:'';content:none}:focus{outline:0}ins{text-decoration:none}del{text-decoration:line-through}table{border-collapse:collapse;border-spacing:0}
+/* 960 text */
+body{font:13px/1.5 'Helvetica Neue',Arial,'Liberation Sans',FreeSans,sans-serif}a:focus{outline:1px dotted invert}hr{border:0 #ccc solid;border-top-width:1px;clear:both;height:0}h1{font-size:25px}h2{font-size:23px}h3{font-size:21px}h4{font-size:19px}h5{font-size:17px}h6{font-size:15px}ol{list-style:decimal}ul{list-style:disc}li{margin-left:30px}p,dl,hr,h1,h2,h3,h4,h5,h6,ol,ul,pre,table,address,fieldset{margin-bottom:20px}
+
+/* 
+ * Toolbar Styling
+ *
+ */
+
+
+#toolbar { color: #333; font-size: 12px; min-width: 980px; height: 130px; 
+           padding: 0 10px; background: #f0f0f0; border-bottom: 2px solid #000;
+           position:fixed; top: 0; margin: 0 -10px; width: 100%; 
+           font-family: 'Helvetica Neue',Arial,'Liberation Sans',FreeSans,sans-serif;
+           }
+#toolbar a { color: blue; background: none; }
+#toolbar a:hover { background: none; }
+
+#toolbar #header { height: 20px; padding: 3px 5px; margin: 0 -10px 10px; overflow:hidden; background: #333; border-bottom: 2px solid #222; color: #eee; }
+#toolbar #header h1 { font-size: 16px; line-height: 20px; }
+#toolbar #header a { text-decoration: none; color: #eee; }
+#toolbar #header a:hover { text-decoration: none; color: #fff;}
+#toolbar .small { font-size: 11px; color: #333; font-weight: bold;}
+#toolbar input.time { width: 20px;}
+
+table.actions { width: auto; border-collapse: collapse; }
+table.actions th { font-size: 11px; color: #333; text-align: left; min-width: 100px; padding: 0 5px; }
+table.actions td { padding: 2px 5px; text-align: left; vertical-align: middle;}
+table.actions span.note { font-weight: normal; color: #999; }
+table.actions span.and { font-size: 11px; color: red; font-weight: bold; }
+table.actions span.label { font-weight: bold; cursor:pointer;}
+table.actions input[type=text] { width: 200px; padding: 2px; border: 1px solid #aaa;}
+
+
+div#option-ctrl { position: fixed; z-index: 100; bottom: 0; right: 0; min-width: 60px; min-height: 20px; height: 20px; padding: 5px; border: 1px solid #ddd; background: #f0f0f0; font-size: 11px; color: #000;
+                             font-family: 'Helvetica Neue',Arial,'Liberation Sans',FreeSans,sans-serif; }
+div#option-ctrl ul { list-style:none; margin: 0; padding: 0;}
+div#option-ctrl ul li { list-style:none; margin: 0 5px 0 0; float:left; }
+
+
+
+/* output */    
+body {  margin: 10px; padding-top: 130px; font-family: 'Monaco', 'Deja Vu Sans Mono', 'Inconsolata' ,'Consolas',monospace; background:#111 none repeat scroll 0 0; color:#fff; font-size:10px;}
+a { color: #0f0; }
+a:hover { background-color: #03c; color: white; text-decoration: none; }
+
+

data/script/console

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+# File: script/console
+irb = RUBY_PLATFORM =~ /(:?mswin|mingw)/ ? 'irb.bat' : 'irb'
+
+libs =  " -r irb/completion"
+# Perhaps use a console_lib to store any extra methods I may want available in the cosole
+# libs << " -r #{File.dirname(__FILE__) + '/../lib/console_lib/console_logger.rb'}"
+libs <<  " -r #{File.dirname(__FILE__) + '/../lib/clarity.rb'}"
+puts "Loading clarity gem"
+exec "#{irb} #{libs} --simple-prompt"

data/script/destroy

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+APP_ROOT = File.expand_path(File.join(File.dirname(__FILE__), '..'))
+
+begin
+  require 'rubigen'
+rescue LoadError
+  require 'rubygems'
+  require 'rubigen'
+end
+require 'rubigen/scripts/destroy'
+
+ARGV.shift if ['--help', '-h'].include?(ARGV[0])
+RubiGen::Base.use_component_sources! [:rubygems, :newgem, :newgem_theme, :test_unit]
+RubiGen::Scripts::Destroy.new.run(ARGV)

data/script/generate

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+APP_ROOT = File.expand_path(File.join(File.dirname(__FILE__), '..'))
+
+begin
+  require 'rubigen'
+rescue LoadError
+  require 'rubygems'
+  require 'rubigen'
+end
+require 'rubigen/scripts/generate'
+
+ARGV.shift if ['--help', '-h'].include?(ARGV[0])
+RubiGen::Base.use_component_sources! [:rubygems, :newgem, :newgem_theme, :test_unit]
+RubiGen::Scripts::Generate.new.run(ARGV)

data/test/commands/command_builder_test.rb

@@ -0,0 +1,100 @@
+require "test/unit"
+require File.dirname(__FILE__) + "/../../lib/commands/command_builder.rb"
+
+class CommandBuilderTest < Test::Unit::TestCase
+  
+  def setup
+    @params = { 
+      "file" => "testfile.log", 
+      "tool" => "grep",
+      "term1" => "foo",
+      "term2" => nil,
+      "term3" => nil
+    }
+    @command = CommandBuilder.new(@params)
+  end
+  
+  def test_create_new_builder
+    assert @command.is_a?(CommandBuilder)
+    assert_equal "testfile.log", @command.filename
+    assert_equal 1, @command.terms.size
+  end
+  
+  def test_raises_error_if_no_file
+    assert_raises CommandBuilder::InvalidParameterError do
+      command = CommandBuilder.new(@params.merge("file" => nil))
+    end
+  end
+  
+  def test_exec_functions_for_log
+    command = CommandBuilder.new(@params)
+    assert_equal 1, command.exec_functions.size
+    assert_match /^grep/, command.exec_functions.first
+  end
+  
+  def test_exec_functions_with_multiple_terms_for_log
+    command = CommandBuilder.new(@params.merge("term2" => "bar", "term3" => "baz"))
+    assert_equal 3, command.exec_functions.size
+    assert_match /^grep/, command.exec_functions[0]
+    assert_match /^grep/, command.exec_functions[1]
+    assert_match /^grep/, command.exec_functions[2]    
+  end
+  
+  def test_exec_function_with_no_terms_for_log
+    command = CommandBuilder.new(@params.merge("term1" => nil))
+    assert_equal 1, command.exec_functions.size
+    assert_match /^cat/, command.exec_functions[0]
+  end
+
+  def test_exec_funcations_for_gzip
+    command = CommandBuilder.new(@params.merge("file" => "testfile.gz"))
+    assert_equal 1, command.exec_functions.size
+    assert_match /^zgrep/, command.exec_functions.first
+  end
+    
+  def test_exec_functions_with_multiple_terms_for_gzip
+    command = CommandBuilder.new(@params.merge("file" => "testfile.gz", "term2" => "bar", "term3" => "baz"))
+    assert_equal 3, command.exec_functions.size
+    assert_match /^zgrep/, command.exec_functions[0]
+    assert_match /^grep/, command.exec_functions[1]
+    assert_match /^grep/, command.exec_functions[2]    
+  end
+  
+  def test_exec_function_with_no_terms_for_gzip
+    command = CommandBuilder.new(@params.merge("file" => "testfile.gz", "term1" => nil))
+    assert_equal 1, command.exec_functions.size
+    assert_match "gzcat", command.exec_functions[0]
+  end
+
+  def test_command_for_log
+    command = CommandBuilder.new(@params)
+    assert_equal "sh -c 'grep  -e foo testfile.log'", command.command
+  end
+
+  def test_command_with_no_terms_for_log
+    command = CommandBuilder.new(@params.merge("term1" => nil))
+    assert_equal "sh -c 'cat testfile.log'", command.command    
+  end
+  
+  def test_command_with_multiple_terms_for_log
+    command = CommandBuilder.new(@params.merge("term2" => "bar", "term3" => "baz"))
+    assert_equal "sh -c 'grep  -e foo testfile.log | grep  -e bar | grep  -e baz'", command.command
+  end
+
+  def test_command_for_gzip
+    command = CommandBuilder.new(@params.merge("file" => "testfile.gz"))
+    assert_equal "sh -c 'zgrep  -e foo testfile.gz'", command.command
+  end
+
+  def test_command_with_no_terms_for_gzip
+    command = CommandBuilder.new(@params.merge("file" => "testfile.gz","term1" => nil))
+    assert_equal "sh -c 'gzcat testfile.gz'", command.command    
+  end
+  
+  def test_command_with_multiple_terms_for_gzip
+    command = CommandBuilder.new(@params.merge("file" => "testfile.gz","term2" => "bar", "term3" => "baz"))
+    assert_equal "sh -c 'zgrep  -e foo testfile.gz | grep  -e bar | grep  -e baz'", command.command
+  end
+
+
+end

data/test/commands/tail_command_builder_test.rb

@@ -0,0 +1,43 @@
+require "test/unit"
+require File.dirname(__FILE__) + "/../../lib/commands/tail_command_builder.rb"
+
+class TailCommandBuilderTest < Test::Unit::TestCase
+  
+  def setup
+    @params = { 
+      "file" => "testfile.log", 
+      "tool" => "tail",
+      "term1" => "foo",
+      "term2" => nil,
+      "term3" => nil
+    }
+    @command = TailCommandBuilder.new(@params)
+  end
+  
+  def test_create_new_builder
+    assert @command.is_a?(TailCommandBuilder)
+    assert_equal "testfile.log", @command.filename
+    assert_equal 1, @command.terms.size
+  end
+  
+  def test_raises_error_if_invalid_file
+    assert_raises CommandBuilder::InvalidParameterError do
+      command = TailCommandBuilder.new(@params.merge("file" => "testfile.gz"))
+    end
+  end
+  
+  def test_command_for_log
+    command = TailCommandBuilder.new(@params)
+    assert_equal "sh -c 'tail -n 250 -f testfile.log | grep  -e foo'", command.command
+  end
+
+  def test_command_with_no_terms_for_log
+    command = TailCommandBuilder.new(@params.merge("term1" => nil))
+    assert_equal "sh -c 'tail -n 250 -f testfile.log'", command.command    
+  end
+  
+  def test_command_with_multiple_terms_for_log
+    command = TailCommandBuilder.new(@params.merge("term2" => "bar", "term3" => "baz"))
+    assert_equal "sh -c 'tail -n 250 -f testfile.log | grep  -e foo | grep  -e bar | grep  -e baz'", command.command
+  end
+end

data/test/files/logfile.log

@@ -0,0 +1,17 @@
+Jul  7 20:12:11 staging rails.shopify[27975]: [jessetesting.staging.shopify.com]   Processing Admin::ProductsController#index (for 67.70.29.242 at 2009-07-07 20:12:11) [GET]
+Jul  7 20:12:11 staging rails.shopify[27975]: [jessetesting.staging.shopify.com]   Parameters: {"action"=>"index", "controller"=>"admin/products"}
+Jul  7 20:12:12 staging rails.shopify[27975]: [jessetesting.staging.shopify.com]   Shop Load (44.5ms)   SELECT shops.* FROM shops JOIN domains ON domains.host = 'jessetesting.staging.shopify.com' AND domains.shop_id = shops.id LIMIT 1
+Jul  7 20:12:12 staging rails.shopify[27975]: [jessetesting.staging.shopify.com]   Subscription Columns (1.4ms)   SHOW FIELDS FROM `subscriptions`
+Jul  7 20:12:12 staging rails.shopify[27975]: [jessetesting.staging.shopify.com]   Subscription Load (11.3ms)   SELECT * FROM `subscriptions` WHERE (`subscriptions`.`id` = 500852) 
+Jul  7 20:12:13 staging rails.shopify[27975]: [jessetesting.staging.shopify.com] Redirected to http://jessetesting.staging.shopify.com/admin/auth/login
+Jul  7 20:12:13 staging rails.shopify[27975]: [jessetesting.staging.shopify.com] Filter chain halted as [:ensure_proper_user] rendered_or_redirected.
+Jul  7 20:12:13 staging rails.shopify[27975]: [jessetesting.staging.shopify.com] Completed in 118ms (DB: 57) | 302 Found [http://jessetesting.staging.shopify.com/admin/products]
+Jul  7 20:20:00 staging rails.shopify[27975]: [jessetesting.staging.shopify.com]   Processing Admin::AuthController#login (for 67.70.29.242 at 2009-07-07 20:12:11) [GET]
+Jul  7 20:20:00 staging rails.shopify[27975]: [jessetesting.staging.shopify.com]   Parameters: {"action"=>"login", "controller"=>"admin/auth"}
+Jul  7 20:30:11 staging rails.shopify[27975]: [jessetesting.staging.shopify.com]   Shop Load (0.2ms)   SELECT shops.* FROM shops JOIN domains ON domains.host = 'jessetesting.staging.shopify.com' AND domains.shop_id = shops.id LIMIT 1
+Jul  7 20:40:11 staging rails.shopify[27975]: [jessetesting.staging.shopify.com] Rendering template within layouts/admin/auth
+Jul  7 20:50:11 staging rails.shopify[27975]: [jessetesting.staging.shopify.com] Rendering admin/auth/login
+Jul  7 21:00:11 staging rails.shopify[27975]: [jessetesting.staging.shopify.com] Completed in 17ms (View: 10, DB: 0) | 200 OK [http://jessetesting.staging.shopify.com/admin/auth/login]
+Jul  7 22:12:11 10.1.1.1 rails.shopify[27975]: [john.staging.shopify.com]   Processing Admin::ProductsController#index (for 67.70.29.242 at 2009-07-07 20:12:11) [GET]
+Jul  7 22:12:11 192.168.5.1 rails.shopify[27975]: [john.staging.shopify.com]   Processing Admin::ProductsController#index (for 67.70.29.242 at 2009-07-07 20:12:11) [GET]
+Jul  7 22:12:11 192.168.5.1 rails.shopify[27975]: [john.staging.shopify.com]   Processing Admin::ProductsController#index (for 67.70.29.242 at 2009-07-07 20:12:11) [GET]

data/test/parsers/hostname_parser_test.rb

@@ -0,0 +1,40 @@
+require "test/unit"
+require File.dirname(__FILE__) + "/../../lib/parsers/hostname_parser.rb"
+
+class HostnameParserTest < Test::Unit::TestCase
+  
+  def setup
+    @params = { "sh" => "12", "sm" => "00", "ss" => "10", "eh" => "12", "em" => "00", "es" => "59" }    
+    @lines = %|Jul 24 14:58:21 app3 rails.shopify[9855]: [wadedemt.myshopify.com]   Processing ShopController#products (for 192.168.1.230 at 2009-07-24 14:58:21) [GET]
+Jul 24 12:00:21 192.168.5.1 rails.shopify[9855]: [wadedemt.myshopify.com]   Processing ShopController#products (for 192.168.1.230 at 2009-07-24 14:58:21) [GET]
+Jul 24 12:00:31 192.168.5.1 rails.shopify[9855]: [test.myshopify.com]   Processing ShopController#products (for 192.168.1.230 at 2009-07-24 14:58:21) [GET]
+Jul 24 12:00:41 192.168.5.1 rails.shopify[9855]: [test2.myshopify.com]   Processing some other line of text 
+|.split("\n")
+    @parser = HostnameParser.new(nil)
+  end
+  
+  
+  def test_parse_strips_out_ip_and_appname
+    line = @lines[1][16..-1] # strip out first 16 lines
+    out = @parser.parse(line)
+    
+    assert out.has_key?(:line)
+    assert_equal "[wadedemt.myshopify.com]   Processing ShopController#products (for 192.168.1.230 at 2009-07-24 14:58:21) [GET]", out[:line]
+  end
+  
+  def test_parse_strips_out_host_and_appname
+    line = @lines.first[16..-1] # strip out first 16 lines
+    out = @parser.parse(line)
+    
+    assert out.has_key?(:line)
+    assert_equal "[wadedemt.myshopify.com]   Processing ShopController#products (for 192.168.1.230 at 2009-07-24 14:58:21) [GET]", out[:line]    
+  end
+  
+  def test_parse_returns_line_if_no_match
+    line = "in@valid rails.shopify[9855]: [wadedemt.myshopify.com]   Processing ShopController#products (for 192.168.1.230 at 2009-07-24 14:58:21) [GET]"
+    out = @parser.parse(line)
+    assert out.has_key?(:line)
+    assert_equal line, out[:line]
+  end
+  
+end

data/test/parsers/shop_parser_test.rb

@@ -0,0 +1,54 @@
+require "test/unit"
+require File.dirname(__FILE__) + "/../../lib/parsers/shop_parser.rb"
+
+class ShopParserTest < Test::Unit::TestCase
+  
+  def setup
+    @parser = ShopParser.new(nil)
+    @lines = %|[wadedemt.myshopify.com]   Processing ShopController#products (for 192.168.1.230 at 2009-07-24 14:58:21) [GET]
+ [wadedemt.myshopify.com]   Processing ShopController#products (for 192.168.1.230 at 2009-07-24 14:58:21) [GET]
+[test.myshopify.com]   Processing ShopController#products (for 192.168.1.230 at 2009-07-24 14:58:21) [GET]
+[my-shop.myshopify.com]   Processing some other line of text
+Jul  7 20:12:11 staging rails.shopify[27975]: [jessetesting.staging.shopify.com]   Processing Admin::ProductsController#index (for 67.70.29.242 at 2009-07-07 20:12:11) [GET]
+|.split("\n")
+  end
+  
+  def test_parse_shop_name
+    line = @lines.first
+    out  = @parser.parse(line)
+    
+    assert out.has_key?(:shop)
+    assert out.has_key?(:line)
+    assert_equal "wadedemt.myshopify.com", out[:shop]
+    assert_equal "Processing ShopController#products (for 192.168.1.230 at 2009-07-24 14:58:21) [GET]", out[:line]
+  end
+  
+  def test_parse_shop_name_with_extra_space
+    line = @lines[1]
+    out  = @parser.parse(line)
+    
+    assert out.has_key?(:shop)
+    assert out.has_key?(:line)
+    assert_equal "wadedemt.myshopify.com", out[:shop]
+    assert_equal "Processing ShopController#products (for 192.168.1.230 at 2009-07-24 14:58:21) [GET]", out[:line]    
+  end
+  
+  def test_parse_shop_name_with_dashes
+    line = @lines[3]
+    out  = @parser.parse(line)
+    
+    assert out.has_key?(:shop)
+    assert out.has_key?(:line)
+    assert_equal "my-shop.myshopify.com", out[:shop]
+    assert_equal "Processing some other line of text", out[:line]        
+  end
+  
+  def test_parse_with_no_shop_returns_line
+    line = @lines[4]
+    out  = @parser.parse(line)
+    assert !out.has_key?(:shop)
+    assert_equal line, out[:line]
+  end
+  
+end
+  

data/test/parsers/time_parser_test.rb

@@ -0,0 +1,84 @@
+require "test/unit"
+require File.dirname(__FILE__) + "/../../lib/parsers/time_parser.rb"
+
+class TimeParserTest < Test::Unit::TestCase
+  
+  def setup
+    @params = { "sh" => "12", "sm" => "00", "ss" => "10", "eh" => "12", "em" => "00", "es" => "59" }    
+    @lines = %|Jul 24 14:58:21 app3 rails.shopify[9855]: [wadedemt.myshopify.com]   Processing ShopController#products (for 192.168.1.230 at 2009-07-24 14:58:21) [GET]
+Jul 24 12:00:21 192.168.5.1 rails.shopify[9855]: [wadedemt.myshopify.com]   Processing ShopController#products (for 192.168.1.230 at 2009-07-24 14:58:21) [GET] 
+Jul 24 12:00:31 192.168.5.1 rails.shopify[9855]: [test.myshopify.com]   Processing ShopController#products (for 192.168.1.230 at 2009-07-24 14:58:21) [GET] 
+Jul 24 12:00:41 192.168.5.1 rails.shopify[9855]: [test2.myshopify.com]   Processing some other line of text 
+|.split("\n")
+  end
+  
+  def test_parse_timestamp
+    @parser = TimeParser.new(nil, {})
+    line = @lines.first
+    elements = @parser.parse(line)
+    
+    assert elements.has_key?(:timestamp)
+    assert_equal "Jul 24 14:58:21", elements[:timestamp]
+    assert_equal "app3 rails.shopify[9855]: [wadedemt.myshopify.com]   Processing ShopController#products (for 192.168.1.230 at 2009-07-24 14:58:21) [GET]", elements[:line]
+  end
+  
+  def test_parse_with_start_time_and_early_timestamp
+    @parser = TimeParser.new(nil, {"sh" => "15", "sm" => "00", "ss" => "10"})
+    line = @lines.first
+    elements = @parser.parse(line)
+
+    # does not match - reject line all together
+    assert elements.empty?    
+  end
+  
+  def test_parse_with_start_time_and_later_timestamp
+    @parser = TimeParser.new(nil, {"sh" => "12", "sm" => "00", "ss" => "10"})
+    line = @lines[1]
+    elements = @parser.parse(line)
+    
+    assert elements.has_key?(:timestamp)
+    assert elements.has_key?(:line)    
+  end
+
+  def test_parse_with_end_time_and_early_timestamp
+    @parser = TimeParser.new(nil, {"eh" => "12", "em" => "00", "es" => "00"})
+    line = @lines.first
+    elements = @parser.parse(line)
+    
+    assert elements.empty?    
+  end
+  
+  def test_parse_with_end_time_and_later_timestamp
+    @parser = TimeParser.new(nil, {"eh" => "13", "em" => "00", "es" => "00"})
+    line = @lines[1]
+    elements = @parser.parse(line)
+    
+    assert elements.has_key?(:timestamp)
+    assert elements.has_key?(:line)    
+  end
+  
+  def test_parse_with_entry_between_start_and_end
+    @parser = TimeParser.new(nil, {"sh" => "14", "sm" => "00", "ss" => "00", "eh" => "15", "em" => "00", "es" => "00"})
+    line = @lines.first
+    elements = @parser.parse(line)
+    
+    assert elements.has_key?(:timestamp)
+    assert elements.has_key?(:line)    
+  end
+  
+  def test_parse_with_entry_outside_start_and_end
+    @parser = TimeParser.new(nil, {"sh" => "14", "sm" => "00", "ss" => "00", "eh" => "15", "em" => "00", "es" => "00"})
+    line = @lines[1]
+    elements = @parser.parse(line)
+    
+    assert elements.empty?    
+  end
+  
+  def test_parse_returns_entry_if_doesnt_match_parser
+    line = "July 24, 1999 12:00:21 192.168.5.1 rails.shopify[9855]: [wadedemt.myshopify.com]   Processing ShopController#products (for 192.168.1.230 at 2009-07-24 14:58:21) [GET] "
+    @parser = TimeParser.new(nil)
+    elements = @parser.parse(line)
+    assert_equal line, elements[:line]
+  end
+  
+end

data/test/test_helper.rb

@@ -0,0 +1,3 @@
+require 'stringio'
+require 'test/unit'
+require File.dirname(__FILE__) + '/../lib/clarity'