clarity 0.9.0

data/History.txt

@@ -0,0 +1,4 @@
+=== 0.0.1 2009-10-31
+
+* 1 major enhancement:
+  * Initial release

data/Manifest.txt

@@ -0,0 +1,37 @@
+History.txt
+Manifest.txt
+PostInstall.txt
+README.rdoc
+Rakefile
+bin/clarity
+config/config.yml.sample
+lib/clarity.rb
+lib/clarity/cli.rb
+lib/clarity/commands/command_builder.rb
+lib/clarity/commands/tail_command_builder.rb
+lib/clarity/parsers/hostname_parser.rb
+lib/clarity/parsers/shop_parser.rb
+lib/clarity/parsers/time_parser.rb
+lib/clarity/renderers/log_renderer.rb
+lib/clarity/server.rb
+lib/clarity/server/basic_auth.rb
+lib/clarity/server/chunk_http.rb
+lib/clarity/server/mime_types.rb
+public/images/spinner_big.gif
+public/javascripts/app.js
+public/stylesheets/app.css
+script/console
+script/destroy
+script/generate
+test/commands/command_builder_test.rb
+test/commands/tail_command_builder_test.rb
+test/files/logfile.log
+test/parsers/hostname_parser_test.rb
+test/parsers/shop_parser_test.rb
+test/parsers/time_parser_test.rb
+test/test_helper.rb
+test/test_string_scanner.rb
+views/_header.html.erb
+views/_toolbar.html.erb
+views/error.html.erb
+views/index.html.erb

data/PostInstall.txt

@@ -0,0 +1,8 @@
+
+For more information on clarity, see http://github/tobi/clarity
+
+You can try clarity by running: 
+  clarity -p 3000 /var/log
+
+
+

data/README.rdoc

@@ -0,0 +1,56 @@
+= Clarity
+
+* http://github.com/#{github_username}/#{project_name}
+
+== DESCRIPTION:
+
+Clarity - a log search tool
+By John Tajima & Tobi Lütke
+
+Clarity is an eventmachine-based web application that is used at Shopify to
+search log files on production servers.
+
+We wrote Clarity to allow authorized users to use a simple interface to look
+through the various log files in our server farm, without requiring access to
+production servers.
+
+Clarity requires eventmachine and eventmachine/evma_httpserver.
+
+  sudo gem install eventmachine eventmachine_httpserver
+
+
+== REQUIREMENTS:
+
+* eventmachine
+* eventmachine_httpserver
+
+== INSTALL:
+
+* sudo gem install clarity
+
+== LICENSE:
+
+(The MIT License)
+
+Copyright (c) 2009 Tobias Lütke
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OT
+
+

data/Rakefile

@@ -0,0 +1,27 @@
+require 'rubygems'
+gem 'hoe', '>= 2.1.0'
+require 'hoe'
+require 'fileutils'
+require './lib/clarity'
+
+Hoe.plugin :newgem
+# Hoe.plugin :website
+# Hoe.plugin :cucumberfeatures
+
+# Generate all the Rake tasks
+# Run 'rake -T' to see list of generated tasks (from gem root directory)
+$hoe = Hoe.spec 'clarity' do
+  self.developer 'Tobias Lütke', 'tobi@shopify.com'
+  self.developer 'John Tajima', 'john@shopify.com'
+  self.post_install_message = 'PostInstall.txt' # TODO remove if post-install message not required
+  # self.rubyforge_name       = self.name # TODO this is default value
+  
+  self.extra_deps         = [['eventmachine','>= 0.12.10'], ['eventmachine_httpserver','>= 0.2.0']]
+end
+
+require 'newgem/tasks'
+Dir['tasks/**/*.rake'].each { |t| load t }
+
+# TODO - want other tests/tasks run by default? Add them to the list
+# remove_task :default
+# task :default => [:spec, :features]

data/bin/clarity

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+#
+#  Created on 2009-10-31.
+#  Copyright (c) 2009. All rights reserved.
+
+require 'rubygems'
+require File.expand_path(File.dirname(__FILE__) + "/../lib/clarity")
+require "clarity/cli"
+
+Clarity::CLI.execute(STDOUT, ARGV)

data/config/config.yml.sample

@@ -0,0 +1,7 @@
+# sample config file. Copy to config.yml since we are ignoring config.yml in .gitignore
+
+log_files:
+ - /var/log/apps/user.log.*
+ - /var/log/mail.*
+
+port: 8080

data/lib/clarity.rb

@@ -0,0 +1,20 @@
+$:.unshift(File.dirname(__FILE__)) unless $:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname(__FILE__)))
+
+require 'eventmachine'
+require 'evma_httpserver'
+require 'yaml'
+require 'base64'
+require 'clarity/server'
+require 'clarity/commands/command_builder'
+require 'clarity/commands/tail_command_builder'
+require 'clarity/parsers/time_parser'
+require 'clarity/parsers/hostname_parser'
+require 'clarity/parsers/shop_parser'
+require 'clarity/renderers/log_renderer'
+
+module Clarity
+  VERSION = '0.9.0'  
+  
+  Templates = File.dirname(__FILE__) + '/../views'
+  Public    = File.dirname(__FILE__) + '/../public'
+end

data/lib/clarity/cli.rb

@@ -0,0 +1,80 @@
+require 'optparse'
+#require File.dirname(__FILE__) + '/../clarity'
+
+
+module Clarity
+  class CLI
+    def self.execute(stdout, arguments=[])
+      
+      options = {
+        :username => nil,
+        :password => nil,
+        :log_files => ['**/*.log*'],
+        :port => 8080,
+        :address => "0.0.0.0"
+      }
+      
+      mandatory_options = %w(  )
+      
+      ARGV.options do |opts|
+        opts.banner = "Usage:  #{File.basename($PROGRAM_NAME)} [options] directory"
+
+        opts.separator " "
+        opts.separator "Specific options:"
+
+        opts.on( "-f", "--config=FILE", String, "Config file (yml)" ) do |opt|
+          options.update YAML.load_file( opt )
+        end
+
+        opts.on( "-p", "--port=PORT", Integer, "Port to listen on" ) do |opt|
+          options[:port] = opt
+        end  
+
+        opts.on( "-b", "--address=ADDRESS", String, "Address to bind to (default 0.0.0.0)" ) do |opt|
+          options[:address] = opt
+        end  
+
+        opts.on( "--include=MASK", String, "File mask of logs to add (default: **/*.log*)" ) do |opt|
+          options[:log_files] ||= []
+          options[:log_files] += opt
+        end
+
+        opts.separator " "
+        opts.separator "Password protection:"
+
+        opts.on( "--username=USER", String, "Enable httpauth username" ) do |opt|
+          options[:username] = opt
+        end
+
+        opts.on( "--password=PASS", String, "Enable httpauth password" ) do |opt|
+          options[:password] = opt
+        end
+
+        opts.separator " "
+        opts.separator "Misc:"
+
+        opts.on( "-h", "--help", "Show this message." ) do
+          puts opts
+          exit
+        end
+
+        opts.separator " "
+
+        begin
+          opts.parse!(arguments)
+          
+          if arguments.first
+            Dir.chdir(arguments.first)
+          end
+          
+          ::Clarity::Server.run(options)
+          
+        #rescue
+        #  puts opts
+        #  exit
+        end
+      end
+            
+    end
+  end
+end

data/lib/clarity/commands/command_builder.rb

@@ -0,0 +1,56 @@
+class CommandBuilder
+
+  # parameter names
+  TermParameters = ['term1', 'term2', 'term3']
+  FileParameter  = 'file'
+
+  attr_accessor :params
+  attr_reader   :terms, :filename, :options
+
+  def initialize(params)
+    @params   = params
+    @filename = params.fetch(FileParameter)
+    @terms    = TermParameters.map {|term| params.fetch(term, nil) }.compact.reject {|term| term.empty? }
+    @options  = ""
+    valid?
+  end
+
+  def valid?
+    raise InvalidParameterError, "Log file parameter not supplied" unless filename && !filename.empty?
+    true
+  end
+  
+  def command
+    results = []
+    exec_functions.each_with_index do |cmd, index|
+      results << cmd.gsub('filename', filename.to_s).gsub('options', options.to_s).gsub('term', terms[index].to_s)
+    end
+    %[sh -c '#{results.join(" | ")}']
+  end
+
+
+  def exec_functions
+    case File.extname(filename)
+    when '.gz' then gzip_tools
+    when '.bz2' then bzip_tools
+    else default_tools
+    end
+  end
+
+  
+  def gzip_tools
+    terms.empty? ? ['gzcat filename'] : ['zgrep options -e term filename'] + ['grep options -e term'] * (terms.size-1)
+  end  
+  
+  def bzip_tools
+    terms.empty? ? ['bzcat filename'] : ['bzgrep options -e term filename'] + ['grep options -e term'] * (terms.size-1)
+  end
+  
+  def default_tools
+    terms.empty? ? ['cat filename'] : ['grep options -e term filename'] + ['grep options -e term']* (terms.size-1)
+  end
+
+
+  class InvalidParameterError < StandardError; end
+  
+end

data/lib/clarity/commands/tail_command_builder.rb

@@ -0,0 +1,27 @@
+# 
+# Handles tailing of log files
+#
+class TailCommandBuilder < CommandBuilder
+  
+  def valid?
+    raise InvalidParameterError, "Log file parameter not supplied or invalid log file" unless filename && !filename.empty? && File.extname(filename) == ".log"
+    true
+  end
+  
+    def command
+    results = []
+    exec_functions.each_with_index do |cmd, index|
+      if index == 0
+        results << cmd.gsub('filename', filename.to_s)
+      else
+        results << cmd.gsub('filename', filename.to_s).gsub('options', options.to_s).gsub('term', terms[index-1].to_s)
+      end
+    end
+    %[sh -c '#{results.join(" | ")}']
+  end
+
+  
+  def default_tools
+    terms.empty? ? ['tail -n 250 -f filename'] : ['tail -n 250 -f filename'] + ['grep options -e term'] * (terms.size)
+  end
+end

data/lib/clarity/parsers/hostname_parser.rb

@@ -0,0 +1,43 @@
+
+class HostnameParser
+
+  # given a string in format:
+  #
+  # app3 rails.shopify[9855]: [wadedemt.myshopify.com]   Processing ShopController#products (for 192.168.1.230 at 2009-07-24 14:58:21) [GET] 
+  # 129.123.2.1 rails.shopify[9855]: [wadedemt.myshopify.com]   Processing ShopController#products (for 192.168.1.230 at 2009-07-24 14:58:21) [GET] 
+  #
+  # strips out the hostname/IP and appname
+  #
+  # result => [wadedemt.myshopify.com]   Processing ShopController#products (for 192.168.1.230 at 2009-07-24 14:58:21) [GET] 
+  
+  
+  LineRegexp   = /^([\w-]+|\d+\.\d+\.\d+\.\d+)\s([^:]*):\s*(.*)/
+  
+  attr_accessor :elements, :next_parser
+  
+  def initialize(next_renderer = nil)
+    @next_renderer = next_renderer
+  end
+  
+  def parse(line, elements = {})
+    @elements = elements
+    # parse line into elements and put into element
+    next_line = parse_line(line)
+    if @next_renderer && next_line
+      @elements = @next_renderer.parse(next_line, @elements)
+    end
+    @elements
+  end
+  
+  # parse line and break into pieces
+  def parse_line(line)
+    results = LineRegexp.match(line)
+    if results 
+      @elements[:line] = results[-1]
+      results[-1] # remaining line      
+    else
+      @elements[:line] = line
+      line      
+    end
+  end
+end

data/lib/clarity/parsers/shop_parser.rb

@@ -0,0 +1,48 @@
+class ShopParser
+  
+  # given a string in format:
+  #
+  # [wadedemt.myshopify.com]   Processing ShopController#products (for 192.168.1.230 at 2009-07-24 14:58:21) [GET] 
+  #
+  # strips out the shop name
+  #
+  # result => :shop => wadedemt.myshopify.com
+  #           :line => Processing ShopController#products (for 192.168.1.230 at 2009-07-24 14:58:21) [GET] 
+
+
+  LineRegexp   = /^\s*\[([a-zA-Z0-9\-.]+)\]\s*(.*)/
+  
+  attr_accessor :elements
+  
+  def initialize(next_renderer = nil)
+    @next_renderer = next_renderer
+  end
+  
+  def parse(line, elements = {})
+    @elements = elements
+    # parse line into elements and put into element
+    next_line = parse_line(line)
+    if @next_renderer && next_line
+      @elements = @next_renderer.parse(next_line, @elements)
+    end
+    @elements
+  end
+  
+  # parse line and break into pieces
+  def parse_line(line)
+    results = LineRegexp.match(line)
+    if results
+      if results[1] =~ /\./
+        @elements[:shop] = results[1]
+        @elements[:line] = results[-1]
+        results[-1]
+      else
+        @elements[:line] = line
+        line
+      end
+    else
+      @elements[:line] = line
+      line
+    end
+  end
+end

data/lib/clarity/parsers/time_parser.rb

@@ -0,0 +1,93 @@
+
+class TimeParser
+  
+  # strips out timestamp and if start/end times are defined, will reject lines that don't fall within proper time periods
+  #
+  # entry format:
+  # Jul 24 14:58:21 app3 rails.shopify[9855]: [wadedemt.myshopify.com]   Processing ShopController#products (for 192.168.1.230 at 2009-07-24 14:58:21) [GET] 
+  #
+  # params = {
+  #  'sh' => start hour
+  #  'sm' => start minute
+  #  'ss' => start second
+  #  'eh' => end hour
+  #  'em' => end minute
+  #  'es' => end second
+  # }
+  # 
+  # if 'sh' is defined, reject any lines where timestamp is earlier than start time
+  # if 'eh' is defined, reject any lines where timestamp is later than end time
+  # if 'sh' && 'eh' is defined, reject any lines where timestamp is not between start time and end time
+
+  LineRegexp   = /^(\w+\s+\d+\s\d\d:\d\d:\d\d)\s(.*)/
+
+  attr_accessor :elements, :params
+  
+  def initialize(next_renderer = nil, params = {})
+    @next_renderer = next_renderer
+    @params = params
+  end
+  
+  def parse(line, elements = {})
+    @elements = elements
+    next_line = parse_line(line)
+    
+    # reject line if we filter by time
+    if check_time? 
+      if !start_time_valid? || !end_time_valid?
+        # reject this entry
+        @elements = {}
+        return @elements
+      end
+    else
+      if @next_renderer && next_line
+        @elements = @next_renderer.parse(next_line, @elements)
+      end
+    end
+    @elements
+  end
+
+  def check_time?
+    (params['sh'] && !params['sh'].empty?) || (params['eh'] && !params['eh'].empty?)
+  end
+  
+  # check if current line's time is >= start time, if it was set
+  def start_time_valid?
+    line_time  = parse_time_from_string(@elements[:timestamp])
+    start_time = Time.utc(line_time.year, line_time.month, line_time.day, params.fetch('sh',0).to_i, params.fetch('sm', 0).to_i, params.fetch('ss', 0).to_i )    
+    line_time >= start_time ? true : false
+  rescue Exception => e
+    puts "Error! #{e}"
+  end
+
+  def end_time_valid?
+    line_time = parse_time_from_string(@elements[:timestamp])
+    end_time  = Time.utc(line_time.year, line_time.month, line_time.day, params.fetch('eh',23).to_i, params.fetch('em', 59).to_i, params.fetch('es', 59).to_i )
+    line_time <= end_time ? true : false
+  rescue Exception => e
+    puts "Error! #{e}"
+  end
+
+  def parse_time_from_string(text)
+    # Jul 24 14:58:21
+    time = nil
+    if text =~ /(\w+)\s+(\d+)\s+(\d+):(\d+):(\d+)/
+      time = Time.utc(Time.now.year, $1, $2, $3, $4, $5)
+    end
+    time
+  end
+  
+  # parse line and break into pieces
+  def parse_line(line)
+    results = LineRegexp.match(line)
+    if results 
+      @elements[:timestamp] = results[1]
+      @elements[:line]      = results[-1]
+      results[-1] # remaining line      
+    else
+      @elements[:line] = line
+      line      
+    end
+  end  
+  
+end