clarity 0.9.2 → 0.9.5

data/Manifest.txt

@@ -10,9 +10,7 @@ lib/clarity/cli.rb
 lib/clarity/commands/command_builder.rb
 lib/clarity/commands/tail_command_builder.rb
 lib/clarity/grep_renderer.rb
-lib/clarity/parsers/hostname_parser.rb
-lib/clarity/parsers/shop_parser.rb
-lib/clarity/parsers/time_parser.rb
+lib/clarity/process_tree.rb
 lib/clarity/renderers/log_renderer.rb
 lib/clarity/server.rb
 lib/clarity/server/basic_auth.rb

data/README.rdoc

@@ -23,7 +23,7 @@ big success internally that we decided to release it as open source.
 
 == SCREENSHOT: 
 
-  http://img.skitch.com/20091104-je9kq1a2gfr586ia8y246bq4n8.png
+http://img.skitch.com/20091104-je9kq1a2gfr586ia8y246bq4n8.png
 
 == REQUIREMENTS:
 

data/Rakefile

@@ -1,27 +1,21 @@
+$LOAD_PATH.unshift File.join(File.dirname(__FILE__), "lib")
+
 require 'rubygems'
 gem 'hoe', '>= 2.1.0'
+gem 'newgem'
+
 require 'hoe'
-require 'fileutils'
-require './lib/clarity'
+require 'clarity'
 
 Hoe.plugin :newgem
-# Hoe.plugin :website
-# Hoe.plugin :cucumberfeatures
 
-# Generate all the Rake tasks
-# Run 'rake -T' to see list of generated tasks (from gem root directory)
 $hoe = Hoe.spec 'clarity' do
   self.developer 'Tobias Lütke', 'tobi@shopify.com'
   self.developer 'John Tajima', 'john@shopify.com'
-  self.post_install_message = 'PostInstall.txt' # TODO remove if post-install message not required
-  # self.rubyforge_name       = self.name # TODO this is default value
-  
+  self.post_install_message = 'PostInstall.txt'  
   self.extra_deps         = [['eventmachine','>= 0.12.10'], ['eventmachine_httpserver','>= 0.2.0']]
+  self.test_globs = ['test/**/*_test.rb']
 end
 
 require 'newgem/tasks'
-Dir['tasks/**/*.rake'].each { |t| load t }
-
-# TODO - want other tests/tasks run by default? Add them to the list
-# remove_task :default
-# task :default => [:spec, :features]
+Dir['tasks/**/*.rake'].each { |t| load t }

data/lib/clarity.rb

@@ -13,7 +13,7 @@ require 'clarity/parsers/shop_parser'
 require 'clarity/renderers/log_renderer'
 
 module Clarity
-  VERSION = '0.9.2'  
+  VERSION = '0.9.5'  
   
   Templates = File.dirname(__FILE__) + '/../views'
   Public    = File.dirname(__FILE__) + '/../public'

data/lib/clarity/cli.rb

@@ -1,6 +1,4 @@
 require 'optparse'
-#require File.dirname(__FILE__) + '/../clarity'
-
 
 module Clarity
   class CLI
@@ -9,9 +7,11 @@ module Clarity
       options = {
         :username => nil,
         :password => nil,
-        :log_files => ['**/*.log*'],
+        :log_files => nil,
         :port => 8080,
-        :address => "0.0.0.0"
+        :address => "0.0.0.0",
+        :user => nil,
+        :group => nil
       }
       
       mandatory_options = %w(  )
@@ -23,7 +23,10 @@ module Clarity
         opts.separator "Specific options:"
 
         opts.on( "-f", "--config=FILE", String, "Config file (yml)" ) do |opt|
-          options.update YAML.load_file( opt )
+          config = YAML.load_file( opt )
+          config.keys.each do |key|
+            options[key.to_sym] = config[key]
+          end
         end
 
         opts.on( "-p", "--port=PORT", Integer, "Port to listen on" ) do |opt|
@@ -38,7 +41,11 @@ module Clarity
           options[:log_files] ||= []
           options[:log_files] += opt
         end
-
+        
+        opts.on( "--user=USER", String, "User to run as" ) do |opt|
+          options[:user] = opt
+        end
+        
         opts.separator " "
         opts.separator "Password protection:"
 
@@ -67,6 +74,8 @@ module Clarity
             Dir.chdir(arguments.first)
           end
           
+          options[:log_files] ||= ['**/*.log*']
+          
           ::Clarity::Server.run(options)
           
         #rescue

data/lib/clarity/grep_renderer.rb

@@ -1,10 +1,7 @@
 module Clarity
-  module GrepRenderer  
-    attr_accessor :response, :parser, :marker, :params
-
-    def parser
-      @parser ||= TimeParser.new( HostnameParser.new(ShopParser.new), params)
-    end
+  module GrepRenderer
+    attr_accessor :response
+    attr_writer :renderer
 
     def renderer
       @renderer ||= LogRenderer.new
@@ -15,22 +12,25 @@ module Clarity
       @buffer ||= StringScanner.new("")
       @buffer << data
 
-      html = ""
       while line = @buffer.scan_until(/\n/)
-        tokens = parser.parse(line)
-        html << renderer.render(tokens)
-      end
-
-      return if html.empty?
-
-      response.chunk html
+        response.chunk renderer.render(line)
+        flush
+      end      
+    end
+            
+    def flush
       response.send_chunks
     end
+    
+    def close
+      ProcessTree.kill(get_status.pid)      
+    end    
 
-    def unbind
-      response.chunk '</div><hr><p id="done">Done</p></body></html>'
+    def unbind          
+      response.chunk renderer.finalize
       response.chunk ''
-      response.send_chunks
+      close
+      flush
       puts 'Done'
     end
   end

data/lib/clarity/process_tree.rb

@@ -0,0 +1,23 @@
+module ProcessTree
+  
+  def self.kill(ppid)
+    return if ppid.nil?
+    all_pids = [ppid] + child_pids_of(ppid).flatten.uniq.compact
+    all_pids.each do |pid|
+      Process.kill('TERM',pid.to_i) rescue nil
+    end
+  end
+
+  def self.child_pids_of(ppid)
+    out = `ps -opid,ppid | grep #{ppid.to_s}`
+    ids = out.split("\n").map {|line| $1 if line =~ /^\s*([0-9]+)\s.*/ }.compact
+    ids.delete(ppid.to_s)
+    if ids.empty?
+      ids
+    else
+      ids << ids.map {|id| child_pids_of(id) }
+    end
+  end
+  
+  
+end

data/lib/clarity/renderers/log_renderer.rb

@@ -1,47 +1,36 @@
-require 'action_view'
 require 'uri'
+require 'erb'
 
-class LogRenderer
-  include ActionView::Helpers::TagHelper
-  include ActionView::Helpers::UrlHelper
+class LogRenderer  
   
+  # Thank you to http://daringfireball.net/2009/11/liberal_regex_for_matching_urls
+  #
+  UrlParser = %r{\b(([\w-]+://?|www[.])[^\s()<>]+(?:\([\w\d]+\)|([^[:punct:]\s]|/)))}
   Prefix   = ""
   Suffix   = "<br/>\n"
-  TagOrder = [ :timestamp, :shop, :labels, :line ]
-  MarkTime = 60 * 5 # 5 minutes
-
-  def initialize()
-    @last_timestamp = nil
-  end
-  
-  def render(elements = {})
-    @elements = elements
-    @tags = []
-    TagOrder.each do |tag|
-      if content = @elements.fetch(tag, nil)
-        method  = ("tag_"+tag.to_s).to_sym
-        @tags << self.send(method, content)
-      end
-    end
-    
-    @tags.empty? ? "" : Prefix + @tags.join(" ").to_s + Suffix
-  end
   
+  def render(line = {})
+    # Escape
+    output = ERB::Util.h(line)
     
-  def tag_timestamp(content, options = {})
-    content + " : "
+    # Transform urls into html links
+    output.gsub!(UrlParser) do |match|
+      html_link(match)
+    end
+        
+    # Return with formatting
+    "#{Prefix}#{output}#{Suffix}"
   end
   
-  def tag_line(content, options = {})
-    ERB::Util.h(content)
+  def finalize
+    '</div><hr><p id="done">Done</p></body></html>'
   end
   
-  def tag_shop(content, options = {})
-    "[<a href='http://#{URI.escape(content)}'>#{content}</a>]"
+  private
+    
+  def html_link(url)
+    uri = URI.parse(url)
+    "<a href='#{uri}'>#{url}</a>"
   end
   
-  def tag_labels(content, options = {})
-    "[#{content}]"
-  end
-
 end

data/lib/clarity/server.rb

@@ -3,6 +3,7 @@ require File.dirname(__FILE__) + '/server/basic_auth'
 require File.dirname(__FILE__) + '/server/mime_types'
 require File.dirname(__FILE__) + '/server/chunk_http'
 require File.dirname(__FILE__) + '/grep_renderer'
+require File.dirname(__FILE__) + '/process_tree'
 
 module Clarity
   class NotFoundError < StandardError; end
@@ -18,6 +19,7 @@ module Clarity
     attr_accessor :log_files    
     
     def self.run(options)
+      
       EventMachine::run do
         EventMachine.epoll
         EventMachine::start_server(options[:address], options[:port], self) do |a|
@@ -25,7 +27,14 @@ module Clarity
           a.required_username = options[:username]
           a.required_password = options[:password]
         end
+
         STDERR.puts "Listening #{options[:address]}:#{options[:port]}..."
+        
+        if options[:user]
+          STDERR.puts "Running as user #{options[:user]}"
+          EventMachine.set_effective_user(options[:user])
+        end
+        
         STDERR.puts "Adding log files: #{options[:log_files].inspect}"
       end      
     end
@@ -54,10 +63,9 @@ module Clarity
           response.chunk results_page # display page header
 
           puts "Running: #{command}"
+          
           EventMachine::popen(command, GrepRenderer) do |grepper|
             @grepper = grepper          
-            @grepper.marker = 0
-            @grepper.params = params
             @grepper.response = response 
           end
         end
@@ -97,34 +105,10 @@ module Clarity
     end
 
     def unbind
-      return unless @grepper
-      kill_processes(@grepper.get_status.pid)
+      @grepper.close_connection if @grepper
       close_connection
     end
 
-    def kill_processes(ppid)
-      return if ppid.nil?
-      all_pids = [ppid] + get_child_pids(ppid).flatten.uniq.compact
-      puts "=== pids are #{all_pids.inspect}"
-      all_pids.each do |pid|
-        Process.kill('TERM',pid.to_i)
-        puts "=== killing #{pid}"
-      end
-    rescue Exception => e
-      puts "!Error killing processes: #{e}"
-    end
-
-    def get_child_pids(ppid)
-      out = `ps -opid,ppid | grep #{ppid.to_s}`
-      ids = out.split("\n").map {|line| $1 if line =~ /^\s*([0-9]+)\s.*/ }.compact
-      ids.delete(ppid.to_s)
-      if ids.empty?
-        ids
-      else
-        ids << ids.map {|id| get_child_pids(id) }
-      end
-    end
-
     private
 
     def authenticate!

data/lib/clarity/server/chunk_http.rb

@@ -1,3 +1,5 @@
+require 'erb'
+
 module Clarity
   
   module ChunkHttp

data/test/commands/command_builder_test.rb

@@ -1,5 +1,4 @@
-require "test/unit"
-require File.dirname(__FILE__) + "/../../lib/commands/command_builder.rb"
+require 'test_helper'
 
 class CommandBuilderTest < Test::Unit::TestCase
   
@@ -29,35 +28,35 @@ class CommandBuilderTest < Test::Unit::TestCase
   def test_exec_functions_for_log
     command = CommandBuilder.new(@params)
     assert_equal 1, command.exec_functions.size
-    assert_match /^grep/, command.exec_functions.first
+    assert_match(/^grep/, command.exec_functions.first)
   end
   
   def test_exec_functions_with_multiple_terms_for_log
     command = CommandBuilder.new(@params.merge("term2" => "bar", "term3" => "baz"))
     assert_equal 3, command.exec_functions.size
-    assert_match /^grep/, command.exec_functions[0]
-    assert_match /^grep/, command.exec_functions[1]
-    assert_match /^grep/, command.exec_functions[2]    
+    assert_match(/^grep/, command.exec_functions[0])
+    assert_match(/^grep/, command.exec_functions[1])
+    assert_match(/^grep/, command.exec_functions[2])    
   end
   
   def test_exec_function_with_no_terms_for_log
     command = CommandBuilder.new(@params.merge("term1" => nil))
     assert_equal 1, command.exec_functions.size
-    assert_match /^cat/, command.exec_functions[0]
+    assert_match(/^cat/, command.exec_functions[0])
   end
 
   def test_exec_funcations_for_gzip
     command = CommandBuilder.new(@params.merge("file" => "testfile.gz"))
     assert_equal 1, command.exec_functions.size
-    assert_match /^zgrep/, command.exec_functions.first
+    assert_match(/^zgrep/, command.exec_functions.first)
   end
     
   def test_exec_functions_with_multiple_terms_for_gzip
     command = CommandBuilder.new(@params.merge("file" => "testfile.gz", "term2" => "bar", "term3" => "baz"))
     assert_equal 3, command.exec_functions.size
-    assert_match /^zgrep/, command.exec_functions[0]
-    assert_match /^grep/, command.exec_functions[1]
-    assert_match /^grep/, command.exec_functions[2]    
+    assert_match(/^zgrep/, command.exec_functions[0])
+    assert_match(/^grep/, command.exec_functions[1])
+    assert_match(/^grep/, command.exec_functions[2])
   end
   
   def test_exec_function_with_no_terms_for_gzip

data/test/commands/tail_command_builder_test.rb

@@ -1,5 +1,4 @@
-require "test/unit"
-require File.dirname(__FILE__) + "/../../lib/commands/tail_command_builder.rb"
+require 'test_helper'
 
 class TailCommandBuilderTest < Test::Unit::TestCase
   

data/test/parsers/hostname_parser_test.rb

@@ -1,5 +1,4 @@
-require "test/unit"
-require File.dirname(__FILE__) + "/../../lib/parsers/hostname_parser.rb"
+require 'test_helper'
 
 class HostnameParserTest < Test::Unit::TestCase
   

data/test/parsers/shop_parser_test.rb

@@ -1,5 +1,4 @@
-require "test/unit"
-require File.dirname(__FILE__) + "/../../lib/parsers/shop_parser.rb"
+require 'test_helper'
 
 class ShopParserTest < Test::Unit::TestCase
   

data/test/parsers/time_parser_test.rb

@@ -1,5 +1,4 @@
-require "test/unit"
-require File.dirname(__FILE__) + "/../../lib/parsers/time_parser.rb"
+require 'test_helper'
 
 class TimeParserTest < Test::Unit::TestCase
   

data/test/test_helper.rb

@@ -1,3 +1,4 @@
-require 'stringio'
+$LOAD_PATH.unshift File.join(File.dirname(__FILE__), "..", "lib")
+
 require 'test/unit'
-require File.dirname(__FILE__) + '/../lib/clarity'
+require 'clarity'

data/test/test_string_scanner.rb

@@ -1,5 +1,4 @@
-require "test/unit"
-
+require 'test_helper'
 require "strscan"
 
 class TestLibraryFileName < Test::Unit::TestCase

data/views/_toolbar.html.erb

@@ -81,9 +81,9 @@
     </div>
     
     <script>
-        Search.init({ 'grep': <%= logfiles.map {|f| f }.to_json %>, 
-                      'tail': <%= logfiles.map {|f| f if f =~ /log$/ }.compact.to_json %> },
-                      <%= params.empty? ? nil.to_json : params.to_json %> );
+        Search.init({ 'grep': <%= logfiles.map {|f| f }.inspect %>, 
+                      'tail': <%= logfiles.map {|f| f if f =~ /log$/ }.compact.inspect %> },
+                      <%= params.empty? ? 'null' : params.inspect %> );
                       
         
     </script>

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: clarity
 version: !ruby/object:Gem::Version 
-  version: 0.9.2
+  version: 0.9.5
 platform: ruby
 authors: 
 - "Tobias L\xC3\xBCtke"
@@ -10,7 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-11-04 00:00:00 -05:00
+date: 2009-12-04 00:00:00 -05:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -77,9 +77,7 @@ files:
 - lib/clarity/commands/command_builder.rb
 - lib/clarity/commands/tail_command_builder.rb
 - lib/clarity/grep_renderer.rb
-- lib/clarity/parsers/hostname_parser.rb
-- lib/clarity/parsers/shop_parser.rb
-- lib/clarity/parsers/time_parser.rb
+- lib/clarity/process_tree.rb
 - lib/clarity/renderers/log_renderer.rb
 - lib/clarity/server.rb
 - lib/clarity/server/basic_auth.rb
@@ -133,5 +131,8 @@ signing_key:
 specification_version: 3
 summary: "Clarity - a log search tool By John Tajima & Tobi L\xC3\xBCtke  Clarity is a Splunk like web interface for your server log files"
 test_files: 
-- test/test_helper.rb
-- test/test_string_scanner.rb
+- test/commands/command_builder_test.rb
+- test/commands/tail_command_builder_test.rb
+- test/parsers/hostname_parser_test.rb
+- test/parsers/shop_parser_test.rb
+- test/parsers/time_parser_test.rb

data/lib/clarity/parsers/hostname_parser.rb

@@ -1,43 +0,0 @@
-
-class HostnameParser
-
-  # given a string in format:
-  #
-  # app3 rails.shopify[9855]: [wadedemt.myshopify.com]   Processing ShopController#products (for 192.168.1.230 at 2009-07-24 14:58:21) [GET] 
-  # 129.123.2.1 rails.shopify[9855]: [wadedemt.myshopify.com]   Processing ShopController#products (for 192.168.1.230 at 2009-07-24 14:58:21) [GET] 
-  #
-  # strips out the hostname/IP and appname
-  #
-  # result => [wadedemt.myshopify.com]   Processing ShopController#products (for 192.168.1.230 at 2009-07-24 14:58:21) [GET] 
-  
-  
-  LineRegexp   = /^([\w-]+|\d+\.\d+\.\d+\.\d+)\s([^:]*):\s*(.*)/
-  
-  attr_accessor :elements, :next_parser
-  
-  def initialize(next_renderer = nil)
-    @next_renderer = next_renderer
-  end
-  
-  def parse(line, elements = {})
-    @elements = elements
-    # parse line into elements and put into element
-    next_line = parse_line(line)
-    if @next_renderer && next_line
-      @elements = @next_renderer.parse(next_line, @elements)
-    end
-    @elements
-  end
-  
-  # parse line and break into pieces
-  def parse_line(line)
-    results = LineRegexp.match(line)
-    if results 
-      @elements[:line] = results[-1]
-      results[-1] # remaining line      
-    else
-      @elements[:line] = line
-      line      
-    end
-  end
-end

data/lib/clarity/parsers/shop_parser.rb

@@ -1,48 +0,0 @@
-class ShopParser
-  
-  # given a string in format:
-  #
-  # [wadedemt.myshopify.com]   Processing ShopController#products (for 192.168.1.230 at 2009-07-24 14:58:21) [GET] 
-  #
-  # strips out the shop name
-  #
-  # result => :shop => wadedemt.myshopify.com
-  #           :line => Processing ShopController#products (for 192.168.1.230 at 2009-07-24 14:58:21) [GET] 
-
-
-  LineRegexp   = /^\s*\[([a-zA-Z0-9\-.]+)\]\s*(.*)/
-  
-  attr_accessor :elements
-  
-  def initialize(next_renderer = nil)
-    @next_renderer = next_renderer
-  end
-  
-  def parse(line, elements = {})
-    @elements = elements
-    # parse line into elements and put into element
-    next_line = parse_line(line)
-    if @next_renderer && next_line
-      @elements = @next_renderer.parse(next_line, @elements)
-    end
-    @elements
-  end
-  
-  # parse line and break into pieces
-  def parse_line(line)
-    results = LineRegexp.match(line)
-    if results
-      if results[1] =~ /\./
-        @elements[:shop] = results[1]
-        @elements[:line] = results[-1]
-        results[-1]
-      else
-        @elements[:line] = line
-        line
-      end
-    else
-      @elements[:line] = line
-      line
-    end
-  end
-end

data/lib/clarity/parsers/time_parser.rb

@@ -1,93 +0,0 @@
-
-class TimeParser
-  
-  # strips out timestamp and if start/end times are defined, will reject lines that don't fall within proper time periods
-  #
-  # entry format:
-  # Jul 24 14:58:21 app3 rails.shopify[9855]: [wadedemt.myshopify.com]   Processing ShopController#products (for 192.168.1.230 at 2009-07-24 14:58:21) [GET] 
-  #
-  # params = {
-  #  'sh' => start hour
-  #  'sm' => start minute
-  #  'ss' => start second
-  #  'eh' => end hour
-  #  'em' => end minute
-  #  'es' => end second
-  # }
-  # 
-  # if 'sh' is defined, reject any lines where timestamp is earlier than start time
-  # if 'eh' is defined, reject any lines where timestamp is later than end time
-  # if 'sh' && 'eh' is defined, reject any lines where timestamp is not between start time and end time
-
-  LineRegexp   = /^(\w+\s+\d+\s\d\d:\d\d:\d\d)\s(.*)/
-
-  attr_accessor :elements, :params
-  
-  def initialize(next_renderer = nil, params = {})
-    @next_renderer = next_renderer
-    @params = params
-  end
-  
-  def parse(line, elements = {})
-    @elements = elements
-    next_line = parse_line(line)
-    
-    # reject line if we filter by time
-    if check_time? 
-      if !start_time_valid? || !end_time_valid?
-        # reject this entry
-        @elements = {}
-        return @elements
-      end
-    else
-      if @next_renderer && next_line
-        @elements = @next_renderer.parse(next_line, @elements)
-      end
-    end
-    @elements
-  end
-
-  def check_time?
-    (params['sh'] && !params['sh'].empty?) || (params['eh'] && !params['eh'].empty?)
-  end
-  
-  # check if current line's time is >= start time, if it was set
-  def start_time_valid?
-    line_time  = parse_time_from_string(@elements[:timestamp])
-    start_time = Time.utc(line_time.year, line_time.month, line_time.day, params.fetch('sh',0).to_i, params.fetch('sm', 0).to_i, params.fetch('ss', 0).to_i )    
-    line_time >= start_time ? true : false
-  rescue Exception => e
-    puts "Error! #{e}"
-  end
-
-  def end_time_valid?
-    line_time = parse_time_from_string(@elements[:timestamp])
-    end_time  = Time.utc(line_time.year, line_time.month, line_time.day, params.fetch('eh',23).to_i, params.fetch('em', 59).to_i, params.fetch('es', 59).to_i )
-    line_time <= end_time ? true : false
-  rescue Exception => e
-    puts "Error! #{e}"
-  end
-
-  def parse_time_from_string(text)
-    # Jul 24 14:58:21
-    time = nil
-    if text =~ /(\w+)\s+(\d+)\s+(\d+):(\d+):(\d+)/
-      time = Time.utc(Time.now.year, $1, $2, $3, $4, $5)
-    end
-    time
-  end
-  
-  # parse line and break into pieces
-  def parse_line(line)
-    results = LineRegexp.match(line)
-    if results 
-      @elements[:timestamp] = results[1]
-      @elements[:line]      = results[-1]
-      results[-1] # remaining line      
-    else
-      @elements[:line] = line
-      line      
-    end
-  end  
-  
-end