kastner-clarity 0.9.7

data/History.txt

@@ -0,0 +1,4 @@
+=== 0.0.1 2009-10-31
+
+* 1 major enhancement:
+  * Initial release

data/Manifest.txt

@@ -0,0 +1,33 @@
+History.txt
+Manifest.txt
+PostInstall.txt
+README.rdoc
+Rakefile
+bin/clarity
+config/config.yml.sample
+lib/clarity.rb
+lib/clarity/cli.rb
+lib/clarity/commands/grep_command_builder.rb
+lib/clarity/commands/tail_command_builder.rb
+lib/clarity/grep_renderer.rb
+lib/clarity/process_tree.rb
+lib/clarity/renderers/log_renderer.rb
+lib/clarity/server.rb
+lib/clarity/server/basic_auth.rb
+lib/clarity/server/chunk_http.rb
+lib/clarity/server/mime_types.rb
+public/images/spinner_big.gif
+public/javascripts/app.js
+public/stylesheets/app.css
+script/console
+script/destroy
+script/generate
+test/commands/grep_command_builder_test.rb
+test/commands/tail_command_builder_test.rb
+test/files/logfile.log
+test/test_helper.rb
+test/test_string_scanner.rb
+views/_header.html.erb
+views/_toolbar.html.erb
+views/error.html.erb
+views/index.html.erb

data/PostInstall.txt

@@ -0,0 +1,8 @@
+
+For more information on clarity, see http://github/tobi/clarity
+
+You can try clarity by running: 
+  clarity -p 3000 /var/log
+
+
+

data/README.rdoc

@@ -0,0 +1,78 @@
+= Clarity
+
+* http://github.com/tobi/clarity
+
+== DESCRIPTION:
+
+Clarity - a log search tool
+By John Tajima & Tobi Lütke
+
+Clarity is a Splunk like web interface for your server log files. It supports 
+searching (using grep) as well as trailing log files in realtime. It has been written 
+using the event based architecture based on EventMachine and so allows real-time search
+of very large log files. If you hit the browser Stop button it will also kill 
+the grep / tail utility. 
+
+We wrote Clarity to allow our support staff to use a simple interface to look
+through the various log files in our server farm. The application was such a 
+big success internally that we decided to release it as open source. 
+
+== USAGE:
+
+  clarity --username=admin --password=secret --port=8989 /var/log
+  
+== COMMANDLINE:
+  
+  Specific options:
+      -f, --config=FILE                Config file (yml)
+      -p, --port=PORT                  Port to listen on
+      -b, --address=ADDRESS            Address to bind to (default 0.0.0.0)
+          --include=MASK               File mask of logs to add (default: **/*.log*)
+          --user=USER                  User to run as   
+  Password protection:
+          --username=USER              Enable httpauth username
+          --password=PASS              Enable httpauth password   
+  Misc:
+      -h, --help                       Show this message.
+  
+
+== SCREENSHOT: 
+
+http://img.skitch.com/20091104-je9kq1a2gfr586ia8y246bq4n8.png
+
+== REQUIREMENTS:
+
+  * eventmachine
+  * eventmachine_httpserver
+  * json
+
+== INSTALL:
+
+  * sudo gem install clarity
+
+== LICENSE:
+
+(The MIT License)
+
+Copyright (c) 2009 Tobias Lütke
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OT
+
+

data/Rakefile

@@ -0,0 +1,24 @@
+$LOAD_PATH.unshift File.join(File.dirname(__FILE__), "lib")
+
+require 'rubygems'
+gem 'hoe', '>= 2.1.0'
+gem 'newgem'
+
+require 'hoe'
+require 'clarity'
+
+Hoe.plugin :newgem
+
+$hoe = Hoe.spec 'kastner-clarity' do
+  self.developer 'Tobias Lütke', 'tobi@shopify.com'
+  self.developer 'John Tajima', 'john@shopify.com'
+  self.developer 'Erik Kastner', 'kastner@gmail.com'
+  self.summary = 'Web interface for grep and tail -f'  
+  self.post_install_message = 'PostInstall.txt'  
+  self.readme_file          = 'README.rdoc'  
+  self.extra_deps           = [['eventmachine','>= 0.12.10'], ['eventmachine_httpserver','>= 0.2.0'], ["json", ">= 1.0.0"]]
+  self.test_globs           = ['test/**/*_test.rb']
+end
+
+require 'newgem/tasks'
+Dir['tasks/**/*.rake'].each { |t| load t }

data/bin/clarity

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+#
+#  Created on 2009-10-31.
+#  Copyright (c) 2009. All rights reserved.
+
+require 'rubygems'
+require File.expand_path(File.dirname(__FILE__) + "/../lib/clarity")
+require "clarity/cli"
+
+Clarity::CLI.execute(STDOUT, ARGV)

data/config/config.yml.sample

@@ -0,0 +1,7 @@
+# sample config file. Copy to config.yml since we are ignoring config.yml in .gitignore
+
+log_files:
+ - /var/log/apps/user.log.*
+ - /var/log/mail.*
+
+port: 8080

data/lib/clarity.rb

@@ -0,0 +1,23 @@
+$:.unshift(File.dirname(__FILE__)) unless $:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname(__FILE__)))
+
+begin
+  require File.join(File.dirname(__FILE__), *%w[.. vendor gems environment])
+rescue LoadError
+end
+
+require 'eventmachine'
+require 'evma_httpserver'
+require 'json'
+require 'yaml'
+require 'base64'
+require 'clarity/server'
+require 'clarity/commands/grep_command_builder'
+require 'clarity/commands/tail_command_builder'
+require 'clarity/renderers/log_renderer'
+
+module Clarity
+  VERSION = '0.9.7'  
+  
+  Templates = File.dirname(__FILE__) + '/../views'
+  Public    = File.dirname(__FILE__) + '/../public'
+end

data/lib/clarity/cli.rb

@@ -0,0 +1,94 @@
+require 'optparse'
+
+module Clarity
+  class CLI
+    def self.execute(stdout, arguments=[])
+      
+      options = {
+        :username => nil,
+        :password => nil,
+        :log_files => nil,
+        :port => 8080,
+        :address => "0.0.0.0",
+        :user => nil,
+        :group => nil,
+        :relative_root => nil
+      }
+      
+      mandatory_options = %w(  )
+      
+      ARGV.options do |opts|
+        opts.banner = "Usage:  #{File.basename($PROGRAM_NAME)} [options] directory"
+
+        opts.separator " "
+        opts.separator "Specific options:"
+
+        opts.on( "-f", "--config=FILE", String, "Config file (yml)" ) do |opt|
+          config = YAML.load_file( opt )
+          config.keys.each do |key|
+            options[key.to_sym] = config[key]
+          end
+        end
+
+        opts.on( "-p", "--port=PORT", Integer, "Port to listen on" ) do |opt|
+          options[:port] = opt
+        end  
+
+        opts.on( "-b", "--address=ADDRESS", String, "Address to bind to (default 0.0.0.0)" ) do |opt|
+          options[:address] = opt
+        end  
+
+        opts.on( "-r", "--relative=ROOT", String, "Run under a relative root" ) do |opt|
+          options[:relative_root] = opt
+        end
+
+        opts.on( "--include=MASK", String, "File mask of logs to add (default: **/*.log*)" ) do |opt|
+          options[:log_files] ||= []
+          options[:log_files] += opt
+        end
+        
+        opts.on( "--user=USER", String, "User to run as" ) do |opt|
+          options[:user] = opt
+        end
+        
+        opts.separator " "
+        opts.separator "Password protection:"
+
+        opts.on( "--username=USER", String, "Enable httpauth username" ) do |opt|
+          options[:username] = opt
+        end
+
+        opts.on( "--password=PASS", String, "Enable httpauth password" ) do |opt|
+          options[:password] = opt
+        end
+
+        opts.separator " "
+        opts.separator "Misc:"
+
+        opts.on( "-h", "--help", "Show this message." ) do
+          puts opts
+          exit
+        end
+
+        opts.separator " "
+
+        begin
+          opts.parse!(arguments)
+   
+          options[:log_files] ||= ['**/*.log*']
+          
+          if arguments.first
+            Dir.chdir(arguments.first)
+            
+            ::Clarity::Server.run(options)
+            
+          else
+            puts opts
+            exit(1)
+          end
+        end
+      end
+            
+    end
+  end
+end

data/lib/clarity/commands/grep_command_builder.rb

@@ -0,0 +1,56 @@
+class GrepCommandBuilder
+
+  # parameter names
+  TermParameters = ['term1', 'term2', 'term3']
+  FileParameter  = 'file'
+
+  attr_accessor :params
+  attr_reader   :terms, :filename, :options
+
+  def initialize(params)
+    @params   = params
+    @filename = params.fetch(FileParameter)
+    @terms    = TermParameters.map {|term| params.fetch(term, nil) }.compact.reject {|term| term.empty? }
+    @options  = ""
+    valid?
+  end
+
+  def valid?
+    raise InvalidParameterError, "Log file parameter not supplied" unless filename && !filename.empty?
+    true
+  end
+  
+  def command
+    results = []
+    exec_functions.each_with_index do |cmd, index|
+      results << cmd.gsub('filename', filename.to_s).gsub('options', options.to_s).gsub('term', terms[index].to_s)
+    end
+    %[sh -c '#{results.join(" | ")}']
+  end
+
+
+  def exec_functions
+    case File.extname(filename)
+    when '.gz' then gzip_tools
+    when '.bz2' then bzip_tools
+    else default_tools
+    end
+  end
+
+  
+  def gzip_tools
+    terms.empty? ? ['gzcat filename'] : ['zgrep options -e term filename'] + ['grep options -e term'] * (terms.size-1)
+  end  
+  
+  def bzip_tools
+    terms.empty? ? ['bzcat filename'] : ['bzgrep options -e term filename'] + ['grep options -e term'] * (terms.size-1)
+  end
+  
+  def default_tools
+    terms.empty? ? ['cat filename'] : ['grep options -e term filename'] + ['grep options -e term']* (terms.size-1)
+  end
+
+
+  class InvalidParameterError < StandardError; end
+  
+end

data/lib/clarity/commands/tail_command_builder.rb

@@ -0,0 +1,27 @@
+# 
+# Handles tailing of log files
+#
+class TailCommandBuilder < GrepCommandBuilder
+  
+  def valid?
+    raise InvalidParameterError, "Log file parameter not supplied or invalid log file" unless filename && !filename.empty? && File.extname(filename) == ".log"
+    true
+  end
+  
+    def command
+    results = []
+    exec_functions.each_with_index do |cmd, index|
+      if index == 0
+        results << cmd.gsub('filename', filename.to_s)
+      else
+        results << cmd.gsub('filename', filename.to_s).gsub('options', options.to_s).gsub('term', terms[index-1].to_s)
+      end
+    end
+    %[sh -c '#{results.join(" | ")}']
+  end
+
+  
+  def default_tools
+    terms.empty? ? ['tail -n 250 -f filename'] : ['tail -n 250 -f filename'] + ['grep options -e term'] * (terms.size)
+  end
+end

data/lib/clarity/grep_renderer.rb

@@ -0,0 +1,37 @@
+module Clarity
+  module GrepRenderer
+    attr_accessor :response
+    attr_writer :renderer
+
+    def renderer
+      @renderer ||= LogRenderer.new
+    end
+
+    # once download is complete, send it to client
+    def receive_data(data)
+      @buffer ||= StringScanner.new("")
+      @buffer << data
+
+      while line = @buffer.scan_until(/\n/)
+        response.chunk renderer.render(line)
+        flush
+      end      
+    end
+            
+    def flush
+      response.send_chunks
+    end
+    
+    def close
+      ProcessTree.kill(get_status.pid)      
+    end    
+
+    def unbind          
+      response.chunk renderer.finalize
+      response.chunk ''
+      close
+      flush
+      puts 'Done'
+    end
+  end
+end

data/lib/clarity/process_tree.rb

@@ -0,0 +1,23 @@
+module ProcessTree
+  
+  def self.kill(ppid)
+    return if ppid.nil?
+    all_pids = [ppid] + child_pids_of(ppid).flatten.uniq.compact
+    all_pids.each do |pid|
+      Process.kill('TERM',pid.to_i) rescue nil
+    end
+  end
+
+  def self.child_pids_of(ppid)
+    out = `ps -opid,ppid | grep #{ppid.to_s}`
+    ids = out.split("\n").map {|line| $1 if line =~ /^\s*([0-9]+)\s.*/ }.compact
+    ids.delete(ppid.to_s)
+    if ids.empty?
+      ids
+    else
+      ids << ids.map {|id| child_pids_of(id) }
+    end
+  end
+  
+  
+end

data/lib/clarity/renderers/log_renderer.rb

@@ -0,0 +1,36 @@
+require 'uri'
+require 'erb'
+
+class LogRenderer  
+  
+  # Thank you to http://daringfireball.net/2009/11/liberal_regex_for_matching_urls
+  #
+  UrlParser = %r{\b(([\w-]+://?|www[.])[^\s()<>]+(?:\([\w\d]+\)|([^[:punct:]\s]|/)))}
+  Prefix   = ""
+  Suffix   = "<br/>\n"
+  
+  def render(line = {})
+    # Escape
+    output = ERB::Util.h(line)
+    
+    # Transform urls into html links
+    output.gsub!(UrlParser) do |match|
+      html_link(match)
+    end
+        
+    # Return with formatting
+    "#{Prefix}#{output}#{Suffix}"
+  end
+  
+  def finalize
+    '</div><hr><p id="done">Done</p></body></html>'
+  end
+  
+  private
+    
+  def html_link(url)
+    uri = URI.parse(url) rescue url
+    "<a href='#{uri}'>#{url}</a>"
+  end
+  
+end