edouard-clarity 0.9.9

data/script/console

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+# File: script/console
+irb = RUBY_PLATFORM =~ /(:?mswin|mingw)/ ? 'irb.bat' : 'irb'
+
+libs =  " -r irb/completion"
+# Perhaps use a console_lib to store any extra methods I may want available in the cosole
+# libs << " -r #{File.dirname(__FILE__) + '/../lib/console_lib/console_logger.rb'}"
+libs <<  " -r #{File.dirname(__FILE__) + '/../lib/clarity.rb'}"
+puts "Loading clarity gem"
+exec "#{irb} #{libs} --simple-prompt"

data/script/destroy

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+APP_ROOT = File.expand_path(File.join(File.dirname(__FILE__), '..'))
+
+begin
+  require 'rubigen'
+rescue LoadError
+  require 'rubygems'
+  require 'rubigen'
+end
+require 'rubigen/scripts/destroy'
+
+ARGV.shift if ['--help', '-h'].include?(ARGV[0])
+RubiGen::Base.use_component_sources! [:rubygems, :newgem, :newgem_theme, :test_unit]
+RubiGen::Scripts::Destroy.new.run(ARGV)

data/script/generate

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+APP_ROOT = File.expand_path(File.join(File.dirname(__FILE__), '..'))
+
+begin
+  require 'rubigen'
+rescue LoadError
+  require 'rubygems'
+  require 'rubigen'
+end
+require 'rubigen/scripts/generate'
+
+ARGV.shift if ['--help', '-h'].include?(ARGV[0])
+RubiGen::Base.use_component_sources! [:rubygems, :newgem, :newgem_theme, :test_unit]
+RubiGen::Scripts::Generate.new.run(ARGV)

data/test/commands/grep_command_builder_test.rb

@@ -0,0 +1,99 @@
+require 'test_helper'
+
+class GrepCommandBuilderTest < Test::Unit::TestCase
+  
+  def setup
+    @params = { 
+      "file" => "testfile.log", 
+      "tool" => "grep",
+      "term1" => "foo",
+      "term2" => nil,
+      "term3" => nil
+    }
+    @command = GrepCommandBuilder.new(@params)
+  end
+  
+  def test_create_new_builder
+    assert @command.is_a?(GrepCommandBuilder)
+    assert_equal "testfile.log", @command.filename
+    assert_equal 1, @command.terms.size
+  end
+  
+  def test_raises_error_if_no_file
+    assert_raises GrepCommandBuilder::InvalidParameterError do
+      command = GrepCommandBuilder.new(@params.merge("file" => nil))
+    end
+  end
+  
+  def test_exec_functions_for_log
+    command = GrepCommandBuilder.new(@params)
+    assert_equal 1, command.exec_functions.size
+    assert_match(/^grep/, command.exec_functions.first)
+  end
+  
+  def test_exec_functions_with_multiple_terms_for_log
+    command = GrepCommandBuilder.new(@params.merge("term2" => "bar", "term3" => "baz"))
+    assert_equal 3, command.exec_functions.size
+    assert_match(/^grep/, command.exec_functions[0])
+    assert_match(/^grep/, command.exec_functions[1])
+    assert_match(/^grep/, command.exec_functions[2])    
+  end
+  
+  def test_exec_function_with_no_terms_for_log
+    command = GrepCommandBuilder.new(@params.merge("term1" => nil))
+    assert_equal 1, command.exec_functions.size
+    assert_match(/^cat/, command.exec_functions[0])
+  end
+
+  def test_exec_funcations_for_gzip
+    command = GrepCommandBuilder.new(@params.merge("file" => "testfile.gz"))
+    assert_equal 1, command.exec_functions.size
+    assert_match(/^zgrep/, command.exec_functions.first)
+  end
+    
+  def test_exec_functions_with_multiple_terms_for_gzip
+    command = GrepCommandBuilder.new(@params.merge("file" => "testfile.gz", "term2" => "bar", "term3" => "baz"))
+    assert_equal 3, command.exec_functions.size
+    assert_match(/^zgrep/, command.exec_functions[0])
+    assert_match(/^grep/, command.exec_functions[1])
+    assert_match(/^grep/, command.exec_functions[2])
+  end
+  
+  def test_exec_function_with_no_terms_for_gzip
+    command = GrepCommandBuilder.new(@params.merge("file" => "testfile.gz", "term1" => nil))
+    assert_equal 1, command.exec_functions.size
+    assert_match "gzcat", command.exec_functions[0]
+  end
+
+  def test_command_for_log
+    command = GrepCommandBuilder.new(@params)
+    assert_equal "sh -c 'grep  -e foo testfile.log'", command.command
+  end
+
+  def test_command_with_no_terms_for_log
+    command = GrepCommandBuilder.new(@params.merge("term1" => nil))
+    assert_equal "sh -c 'cat testfile.log'", command.command    
+  end
+  
+  def test_command_with_multiple_terms_for_log
+    command = GrepCommandBuilder.new(@params.merge("term2" => "bar", "term3" => "baz"))
+    assert_equal "sh -c 'grep  -e foo testfile.log | grep  -e bar | grep  -e baz'", command.command
+  end
+
+  def test_command_for_gzip
+    command = GrepCommandBuilder.new(@params.merge("file" => "testfile.gz"))
+    assert_equal "sh -c 'zgrep  -e foo testfile.gz'", command.command
+  end
+
+  def test_command_with_no_terms_for_gzip
+    command = GrepCommandBuilder.new(@params.merge("file" => "testfile.gz","term1" => nil))
+    assert_equal "sh -c 'gzcat testfile.gz'", command.command    
+  end
+  
+  def test_command_with_multiple_terms_for_gzip
+    command = GrepCommandBuilder.new(@params.merge("file" => "testfile.gz","term2" => "bar", "term3" => "baz"))
+    assert_equal "sh -c 'zgrep  -e foo testfile.gz | grep  -e bar | grep  -e baz'", command.command
+  end
+
+
+end

data/test/commands/tail_command_builder_test.rb

@@ -0,0 +1,42 @@
+require 'test_helper'
+
+class TailCommandBuilderTest < Test::Unit::TestCase
+  
+  def setup
+    @params = { 
+      "file" => "testfile.log", 
+      "tool" => "tail",
+      "term1" => "foo",
+      "term2" => nil,
+      "term3" => nil
+    }
+    @command = TailCommandBuilder.new(@params)
+  end
+  
+  def test_create_new_builder
+    assert @command.is_a?(TailCommandBuilder)
+    assert_equal "testfile.log", @command.filename
+    assert_equal 1, @command.terms.size
+  end
+  
+  def test_raises_error_if_invalid_file
+    assert_raises GrepCommandBuilder::InvalidParameterError do
+      command = TailCommandBuilder.new(@params.merge("file" => "testfile.gz"))
+    end
+  end
+  
+  def test_command_for_log
+    command = TailCommandBuilder.new(@params)
+    assert_equal "sh -c 'tail -n 250 -f testfile.log | grep  -e foo'", command.command
+  end
+
+  def test_command_with_no_terms_for_log
+    command = TailCommandBuilder.new(@params.merge("term1" => nil))
+    assert_equal "sh -c 'tail -n 250 -f testfile.log'", command.command    
+  end
+  
+  def test_command_with_multiple_terms_for_log
+    command = TailCommandBuilder.new(@params.merge("term2" => "bar", "term3" => "baz"))
+    assert_equal "sh -c 'tail -n 250 -f testfile.log | grep  -e foo | grep  -e bar | grep  -e baz'", command.command
+  end
+end

data/test/files/logfile.log

@@ -0,0 +1,17 @@
+Jul  7 20:12:11 staging rails.shopify[27975]: [jessetesting.staging.shopify.com]   Processing Admin::ProductsController#index (for 67.70.29.242 at 2009-07-07 20:12:11) [GET]
+Jul  7 20:12:11 staging rails.shopify[27975]: [jessetesting.staging.shopify.com]   Parameters: {"action"=>"index", "controller"=>"admin/products"}
+Jul  7 20:12:12 staging rails.shopify[27975]: [jessetesting.staging.shopify.com]   Shop Load (44.5ms)   SELECT shops.* FROM shops JOIN domains ON domains.host = 'jessetesting.staging.shopify.com' AND domains.shop_id = shops.id LIMIT 1
+Jul  7 20:12:12 staging rails.shopify[27975]: [jessetesting.staging.shopify.com]   Subscription Columns (1.4ms)   SHOW FIELDS FROM `subscriptions`
+Jul  7 20:12:12 staging rails.shopify[27975]: [jessetesting.staging.shopify.com]   Subscription Load (11.3ms)   SELECT * FROM `subscriptions` WHERE (`subscriptions`.`id` = 500852) 
+Jul  7 20:12:13 staging rails.shopify[27975]: [jessetesting.staging.shopify.com] Redirected to http://jessetesting.staging.shopify.com/admin/auth/login
+Jul  7 20:12:13 staging rails.shopify[27975]: [jessetesting.staging.shopify.com] Filter chain halted as [:ensure_proper_user] rendered_or_redirected.
+Jul  7 20:12:13 staging rails.shopify[27975]: [jessetesting.staging.shopify.com] Completed in 118ms (DB: 57) | 302 Found [http://jessetesting.staging.shopify.com/admin/products]
+Jul  7 20:20:00 staging rails.shopify[27975]: [jessetesting.staging.shopify.com]   Processing Admin::AuthController#login (for 67.70.29.242 at 2009-07-07 20:12:11) [GET]
+Jul  7 20:20:00 staging rails.shopify[27975]: [jessetesting.staging.shopify.com]   Parameters: {"action"=>"login", "controller"=>"admin/auth"}
+Jul  7 20:30:11 staging rails.shopify[27975]: [jessetesting.staging.shopify.com]   Shop Load (0.2ms)   SELECT shops.* FROM shops JOIN domains ON domains.host = 'jessetesting.staging.shopify.com' AND domains.shop_id = shops.id LIMIT 1
+Jul  7 20:40:11 staging rails.shopify[27975]: [jessetesting.staging.shopify.com] Rendering template within layouts/admin/auth
+Jul  7 20:50:11 staging rails.shopify[27975]: [jessetesting.staging.shopify.com] Rendering admin/auth/login
+Jul  7 21:00:11 staging rails.shopify[27975]: [jessetesting.staging.shopify.com] Completed in 17ms (View: 10, DB: 0) | 200 OK [http://jessetesting.staging.shopify.com/admin/auth/login]
+Jul  7 22:12:11 10.1.1.1 rails.shopify[27975]: [john.staging.shopify.com]   Processing Admin::ProductsController#index (for 67.70.29.242 at 2009-07-07 20:12:11) [GET]
+Jul  7 22:12:11 192.168.5.1 rails.shopify[27975]: [john.staging.shopify.com]   Processing Admin::ProductsController#index (for 67.70.29.242 at 2009-07-07 20:12:11) [GET]
+Jul  7 22:12:11 192.168.5.1 rails.shopify[27975]: [john.staging.shopify.com]   Processing Admin::ProductsController#index (for 67.70.29.242 at 2009-07-07 20:12:11) [GET]

data/test/test_helper.rb

@@ -0,0 +1,4 @@
+$LOAD_PATH.unshift File.join(File.dirname(__FILE__), "..", "lib")
+
+require 'test/unit'
+require 'clarity'

data/test/test_string_scanner.rb

@@ -0,0 +1,16 @@
+require 'test_helper'
+require "strscan"
+
+class TestLibraryFileName < Test::Unit::TestCase
+  
+  NewLine = /\n/
+  def test_case_name
+    s = StringScanner.new('')
+    s << "abc\nd"
+    assert_equal "abc\n", s.scan_until(NewLine)
+    assert_equal nil, s.scan_until(NewLine)
+    s << "ef\ng"
+    assert_equal "def\n", s.scan_until(NewLine)
+    assert_equal nil, s.scan_until(NewLine)    
+  end
+end

data/views/_header.html.erb

@@ -0,0 +1,5 @@
+<meta http-equiv="Content-type" content="text/html; charset=utf-8" />
+<title>Clarity on <%= @hostname %> ~ A Log Search Tool<%= relative_root %></title>
+<link rel="stylesheet" href="<%= relative_root %>/stylesheets/app.css" type="text/css" media="screen">
+<script src="http://ajax.googleapis.com/ajax/libs/jquery/1.3.2/jquery.min.js" type="text/javascript"></script>
+<script src="<%= relative_root %>/javascripts/app.js" type="text/javascript"></script>

data/views/_toolbar.html.erb

@@ -0,0 +1,64 @@
+    <div id='toolbar'>
+      <div id='header'>
+        <h1><a href="/">Clarity</a>  <span class='small'><%= @hostname %></span></h1>
+      </div>
+
+      <form id='search' method='get' action='<%= relative_root %>/perform'>
+      <table class='actions'>
+        <tr>
+          <th>Action</th>
+          <th>Log File</th>
+          <th colspan=5>
+            Search Terms
+            <span class='note'><a href='#' onclick='Search.clear(); return false;'>clear</a></span>
+          </th>
+          <th> </th>
+        </tr>
+        <tr>
+          <td id='tool-selector'>
+            <input type='radio' name='tool' value='grep' id='grep-tool' checked='checked'> <span class='label' id='grep-label'>Search</span> <br/>
+            <input type='radio' name='tool' value='tail' id='tail-tool'> <span class='label' id='tail-label'>Tail</span> <br/>
+          </td>
+          <td id='file-selector'>
+            <select id='file-list' name='file'>
+              <% logfiles.map do |f| %>
+                <%= "<option value='#{f}'>#{f}</option>\n" %>
+              <% end %>
+            </select>
+          </td>
+          <td>
+            <input type='text' id='term1' name='term1' />
+          </td>
+          <td> <span class='and'>AND</span> </td>
+          <td>
+            <input type='text' id='term2' name='term2' />
+          </td>
+          <td> <span class='and'>AND</span> </td>
+          <td>
+            <input type='text' id='term3' name='term3' />
+          </td>
+          <td>
+            <input type='submit' value='Submit' onclick="Search.submit(); return false;"/>
+          </td>
+          <td>
+          </td>
+        </tr>
+      </table>
+      </form>
+    </div>
+
+
+    <div id="option-ctrl">
+      <ul>
+        <li><input type='checkbox' name='enable_scrolling' id='auto-scroll'/><label for='auto-scroll'>Auto scroll?</label></li>
+      </ul>
+    </div>
+
+    <script>
+        Search.url = "<%= relative_root %>" + Search.url;
+        Search.init({ 'grep': <%= logfiles.map {|f| f }.to_json %>,
+                      'tail': <%= logfiles.map {|f| f }.to_json %> },
+                      <%= params.empty? ? 'null' : params.to_json %> );
+
+
+    </script>

data/views/error.html.erb

@@ -0,0 +1,11 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html>
+  <head>
+    <%= @content_for_header %>
+  </head>
+  <body>
+    <%= @toolbar %>
+    <h1>Error</h1>    
+    <p><%= @error %></p>
+  </body>
+</html>

data/views/index.html.erb

@@ -0,0 +1,9 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html>
+  <head>
+    <%= @content_for_header %>
+  </head>
+
+  <body>
+    <%= @toolbar %>
+    <div id="results">

metadata

@@ -0,0 +1,163 @@
+--- !ruby/object:Gem::Specification 
+name: edouard-clarity
+version: !ruby/object:Gem::Version 
+  version: 0.9.9
+platform: ruby
+authors: 
+- "Tobias L\xC3\xBCtke"
+- John Tajima
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-03-23 00:00:00 +01:00
+default_executable: clarity
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: eventmachine
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 0.12.10
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: eventmachine_httpserver
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 0.2.0
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: json
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.0.0
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: rubyforge
+  type: :development
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 2.0.3
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: gemcutter
+  type: :development
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 0.3.0
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: hoe
+  type: :development
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 2.5.0
+    version: 
+description: "Clarity - a log search tool\n\
+  By John Tajima & Tobi L\xC3\xBCtke\n\n\
+  Clarity is a Splunk like web interface for your server log files. It supports \n\
+  searching (using grep) as well as trailing log files in realtime. It has been written \n\
+  using the event based architecture based on EventMachine and so allows real-time search\n\
+  of very large log files. If you hit the browser Stop button it will also kill \n\
+  the grep / tail utility. \n\n\
+  We wrote Clarity to allow our support staff to use a simple interface to look\n\
+  through the various log files in our server farm. The application was such a \n\
+  big success internally that we decided to release it as open source."
+email: 
+- tobi@shopify.com
+- john@shopify.com
+executables: 
+- clarity
+extensions: []
+
+extra_rdoc_files: 
+- History.txt
+- Manifest.txt
+- PostInstall.txt
+files: 
+- History.txt
+- Manifest.txt
+- PostInstall.txt
+- README.rdoc
+- Rakefile
+- bin/clarity
+- config/config.yml.sample
+- lib/clarity.rb
+- lib/clarity/cli.rb
+- lib/clarity/commands/grep_command_builder.rb
+- lib/clarity/commands/hostname_command_builder.rb
+- lib/clarity/commands/tail_command_builder.rb
+- lib/clarity/grep_renderer.rb
+- lib/clarity/process_tree.rb
+- lib/clarity/renderers/log_renderer.rb
+- lib/clarity/server.rb
+- lib/clarity/server/basic_auth.rb
+- lib/clarity/server/chunk_http.rb
+- lib/clarity/server/mime_types.rb
+- public/images/spinner_big.gif
+- public/javascripts/app.js
+- public/stylesheets/app.css
+- script/console
+- script/destroy
+- script/generate
+- test/commands/grep_command_builder_test.rb
+- test/commands/tail_command_builder_test.rb
+- test/files/logfile.log
+- test/test_helper.rb
+- test/test_string_scanner.rb
+- views/_header.html.erb
+- views/_toolbar.html.erb
+- views/error.html.erb
+- views/index.html.erb
+has_rdoc: true
+homepage: http://github.com/tobi/clarity
+licenses: []
+
+post_install_message: PostInstall.txt
+rdoc_options: 
+- --main
+- README.rdoc
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: edouard-clarity
+rubygems_version: 1.3.5
+signing_key: 
+specification_version: 3
+summary: Web interface for grep and tail -f
+test_files: 
+- test/commands/grep_command_builder_test.rb
+- test/commands/tail_command_builder_test.rb