danielsdeleo-teeth 0.0.2 → 0.1.0

data/lib/rule_statement.rb

@@ -0,0 +1,61 @@
+module Teeth
+  
+  class DuplicateRuleError < ScannerError
+  end
+
+  class RuleStatement
+    attr_reader :name, :regex, :strip_ends, :skip_line, :begin
+    
+    def initialize(name, regex, options={})
+      @name, @regex = name, regex
+      @strip_ends, @skip_line, @begin = options[:strip_ends], options[:skip_line], options[:begin]
+      @ignore = options[:ignore]
+    end
+    
+    def ==(other)
+      other.kind_of?(RuleStatement) && other.name == name && other.regex == regex
+    end
+    
+    def scanner_code
+      if @ignore
+        regex
+      else
+        "#{regex} {\n" + function_body + "}"
+      end
+    end
+        
+    def function_body
+      code = ""
+      code += "  BEGIN(#{@begin});\n" if @begin
+      if skip_line
+        code += "  return EOF_KVPAIR;\n"
+      else
+        code += "  KVPAIR #{name.to_s} = {\"#{name.to_s}\", #{yytext_statement}};\n" +
+        "  return #{name.to_s};\n"
+      end
+      code
+    end
+    
+    def yytext_statement
+      strip_ends ? "strip_ends(yytext)" : "yytext"
+    end
+    
+  end
+  
+  class RuleStatementGroup < Array
+    
+    def add(name, regex, options={})
+      push RuleStatement.new(name, regex, options)
+    end
+    
+    def rule_names
+      map { |rule_statement| rule_statement.name.to_s }
+    end
+    
+    def method_missing(called_method_name, *args, &block)
+      args[1] ||={}
+      add(called_method_name, args[0], args[1])
+    end
+    
+  end
+end

data/lib/scanner.rb

@@ -0,0 +1,98 @@
+require "erb"
+module Teeth
+  class ScannerError < StandardError
+  end
+  
+  class InvalidExtensionDirectory < ScannerError
+  end
+  
+  class Scanner
+    TEMPLATE = File.dirname(__FILE__) + "/../templates/tokenizer.yy.erb"
+    attr_reader :scanner_defns, :scanner_rules, :rdoc
+    
+    def initialize(name, ext_dir=nil)
+      @scanner_base_name, @ext_dir = name, ext_dir
+      @scanner_defns, @scanner_rules = ScannerDefinitionGroup.new, RuleStatementGroup.new
+      ensure_ext_dir_exists if ext_dir
+    end
+    
+    def scanner_name
+      "scan_" + @scanner_base_name.to_s
+    end
+    
+    def main_function_name
+      "t_" + scanner_name
+    end
+    
+    def init_function_name
+      "Init_" + scanner_name
+    end
+    
+    def function_prefix
+      @scanner_base_name.to_s + "_yy"
+    end
+    
+    def entry_point
+      "scan_" + @scanner_base_name.to_s
+    end
+    
+    def extconf
+      'require "mkmf"' + "\n" + '$CFLAGS += " -Wall"' + "\n" + "create_makefile " +
+      %Q|"teeth/#{scanner_name}", "./"\n| 
+    end
+    
+    def rdoc=(rdoc_text)
+      lines_of_rdoc_text = rdoc_text.split("\n").map { |line| " * " + line.strip}
+      lines_of_rdoc_text.first[0] = "/"
+      lines_of_rdoc_text[-1] = lines_of_rdoc_text.last + " */"
+      @rdoc = lines_of_rdoc_text.join("\n")
+    end
+    
+    def define(*args)
+      @scanner_defns.add(*args)
+    end
+    
+    def definitions
+      yield @scanner_defns
+    end
+    
+    def load_default_definitions_for(*defn_types)
+      @scanner_defns.defaults_for(*defn_types)
+    end
+    
+    def rule(*args)
+      scanner_rules.add(*args)
+    end
+    
+    def rules
+      yield scanner_rules
+    end
+    
+    def generate
+      template = ERB.new(IO.read(TEMPLATE))
+      scanner = self
+      b = binding
+      template.result(b)
+    end
+    
+    def write!
+      raise InvalidExtensionDirectory, "no extension directory specified" unless @ext_dir
+      File.open(@ext_dir + "/extconf.rb", "w") do |extconf_rb|
+        extconf_rb.write extconf
+      end
+      File.open(@ext_dir + "/" + scanner_name + ".yy", "w") do |scanner|
+        scanner.write generate
+      end
+    end
+    
+    private
+    
+    def ensure_ext_dir_exists
+      unless File.exist?(@ext_dir)
+        Dir.mkdir @ext_dir
+      end
+    end
+    
+  end
+  
+end

data/lib/scanner_definition.rb

@@ -0,0 +1,116 @@
+module Teeth
+  class DuplicateDefinitionError < ScannerError
+  end
+  
+  class InvalidDefaultDefinitionName < ScannerError
+  end
+  
+  class ScannerDefinitionArgumentError < ScannerError
+  end
+  
+  class ScannerDefinition
+    attr_reader :name, :regex
+    
+    def initialize(name, regex, opts={})
+      if regex.kind_of?(Hash)
+        regex, opts = nil, regex
+      end
+      @name, @regex, @start_condition = name, regex, opts[:start_condition]
+      assert_valid_argument_combination
+    end
+    
+    def scanner_code
+      start_condition_string + @name.to_s + regex_to_s
+    end
+    
+    def regex_to_s
+      unless @regex.to_s == ""
+        " " + @regex.to_s
+      else
+        ""
+      end
+    end
+    
+    def start_condition_string
+      case @start_condition.to_s
+      when /^inc/
+        "%s "
+      when /^exc/
+        "%x "
+      else
+        ""
+      end
+    end
+    
+    private
+    
+    def assert_valid_argument_combination
+      if @start_condition
+        if @regex.to_s != "" # (nil or "").to_s == ""
+          raise ScannerDefinitionArgumentError, "a scanner definition cannot define both a regex and start condition"
+        end
+      end
+    end
+    
+  end
+  
+  class ScannerDefinitionGroup < Array
+    
+    DEFAULT_DEFINITIONS = {}
+    DEFAULT_DEFINITIONS[:whitespace] = [["WS",  '[[:space:]]'],
+                                        ["NON_WS", "([a-z]|[0-9]|[:punct:])"]]
+    DEFAULT_DEFINITIONS[:ip]    = [ ["IP4_OCT", "[0-9]|[0-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5]"],
+                                    ["HOST", '[a-z0-9][a-z0-9\-]*\.[a-z0-9][a-z0-9\-]*.[a-z0-9][a-z0-9\-\.]*[a-z]+(\:[0-9]+)?']]
+    DEFAULT_DEFINITIONS[:time]  = [ ["WDAY", "mon|tue|wed|thu|fri|sat|sun"],
+                                    ["MON", "jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec"],
+                                    ["MONTH_NUM", "0[1-9]|1[0-2]"],
+                                    ["MDAY", "3[0-1]|[1-2][0-9]|0[1-9]"],
+                                    ["HOUR", "2[0-3]|[0-1][0-9]"],
+                                    ["MINSEC", "[0-5][0-9]|60"],
+                                    ["YEAR", "[0-9][0-9][0-9][0-9]"],
+                                    ["PLUSMINUS", '(\+|\-)']]
+    DEFAULT_DEFINITIONS[:web]   = [ ["REL_URL", %q{(\/|\\\\|\.)[a-z0-9\._\~\-\/\?&;#=\%\:\+\[\]\\\\]*}],
+                                    ["PROTO", "(http:|https:)"],
+                                    ["ERR_LVL", "(emerg|alert|crit|err|error|warn|warning|notice|info|debug)"],
+                                    ["HTTP_VERS", 'HTTP\/(1.0|1.1)'],
+                                    ["HTTP_VERB", "(get|head|put|post|delete|trace|connect)"],
+                                    ["HTTPCODE", "(100|101|20[0-6]|30[0-5]|307|40[0-9]|41[0-7]|50[0-5])"],
+                                    ["BROWSER_STR", '\"(moz|msie|lynx).+\"']]
+
+    def add(name, regex, options={})
+      assert_defn_has_unique_name(name)
+      push ScannerDefinition.new(name, regex, options)
+    end
+    
+    def assert_defn_has_unique_name(name)
+      if defn_names.include?(name.to_s)
+        raise DuplicateDefinitionError, "a definition for #{name.to_s} has already been defined"
+      end
+    end
+    
+    def defn_names
+      map { |defn_statement| defn_statement.name.to_s }
+    end
+    
+    def method_missing(called_method_name, *args, &block)
+      args[1] ||={}
+      add(called_method_name, args[0], args[1])
+    end
+    
+    def defaults_for(*default_types)
+      default_types.each do |default_type|
+        unless default_definitions = DEFAULT_DEFINITIONS[default_type]
+          raise InvalidDefaultDefinitionName, "no default definitions found for #{default_type.to_s}"
+        end
+        default_definitions.each do |defn|
+          begin
+            add(defn.first, defn.last)
+          rescue DuplicateDefinitionError
+          end
+        end
+      end
+    end
+    
+    
+  end
+end

data/lib/teeth.rb

@@ -1 +1,5 @@
-require "teeth/tokenize_apache_logs"
+require "teeth/scan_apache_logs"
+$:.unshift File.dirname(__FILE__) + "/"
+require "scanner"
+require "scanner_definition"
+require "rule_statement"

data/scanners/scan_apache_logs.rb

@@ -0,0 +1,27 @@
+require File.dirname(__FILE__) + "/../lib/teeth"
+scanner = Teeth::Scanner.new(:apache_logs, File.dirname(__FILE__) + '/../ext/scan_apache_logs/')
+scanner.load_default_definitions_for(:whitespace, :ip, :time, :web)
+scanner.rdoc = <<-RDOC
+Scans self, which is expected to be a single line from an Apache error or 
+access log, and returns a Hash of the components of the log message.  The
+following parts of the log message are returned if they are present:
+IPv4 address, datetime, HTTP Version used, the browser string given by the 
+client, any absolute or relative URLs, the error level, HTTP response code,
+HTTP Method (verb), and any other uncategorized strings present.
+RDOC
+scanner.rules do |r|
+  r.ipv4_addr '{IP4_OCT}"."{IP4_OCT}"."{IP4_OCT}"."{IP4_OCT}'
+  r.apache_err_datetime '{WDAY}{WS}{MON}{WS}{MDAY}{WS}{HOUR}":"{MINSEC}":"{MINSEC}{WS}{YEAR}'
+  r.apache_access_datetime '{MDAY}\/{MON}\/{YEAR}":"{HOUR}":"{MINSEC}":"{MINSEC}{WS}{PLUSMINUS}{YEAR}'
+  r.http_version '{HTTP_VERS}'
+  r.browser_string '{BROWSER_STR}', :strip_ends => true
+  r.absolute_url '{PROTO}"\/\/"({HOST}|{IP4_OCT}"."{IP4_OCT}"."{IP4_OCT}"."{IP4_OCT})({REL_URL}|"\/")?'
+  r.host '{HOST}'
+  r.relative_url '{REL_URL}'
+  r.error_level '{ERR_LVL}'
+  r.http_response '{HTTPCODE}'
+  r.http_method '{HTTP_VERB}'
+  r.strings '{NON_WS}{NON_WS}*'
+end
+
+scanner.write!

data/scanners/scan_rails_logs.rb

@@ -0,0 +1,70 @@
+require File.dirname(__FILE__) + "/../lib/teeth"
+scanner = Teeth::Scanner.new(:rails_logs, File.dirname(__FILE__) + '/../ext/scan_rails_logs/')
+scanner.load_default_definitions_for(:whitespace, :ip, :time, :web)
+scanner.rdoc = <<-RDOC
+Scans self, which is expected to be a line from a Rails production or dev log,
+and returns a Hash of the significant features in the log message, including 
+the IP address of the client, the Controller and Action, any partials rendered,
+and the time spent rendering them, the duration of the DB request(s), the HTTP
+verb, etc.
+RDOC
+scanner.definitions do |define|
+  define.RAILS_TEASER '(processing|filter\ chain\ halted|rendered)'
+  define.CONTROLLER_ACTION '[a-z0-9]+#[a-z0-9]+'
+  define.RAILS_SKIP_LINES '(session\ id)'
+  define.CACHE_HIT 'actioncontroller"::"caching"::"actions"::"actioncachefilter":"0x[0-9a-f]+'
+  define.PARTIAL_SESSION_ID '^([a-z0-9]+"="*"-"+[a-z0-9]+)'
+  define.RAILS_ERROR_CLASS '([a-z]+\:\:)*[a-z]+error'
+  define.REQUEST_COMPLETED :start_condition => :exclusive
+  define.COMPLETED_REQ_VIEW_STATS :start_condition => :exclusive
+  define.COMPLETED_REQ_DB_STATS :start_condition => :exclusive
+end
+scanner.rules do |r|
+  # Processing DashboardController#index (for 1.1.1.1 at 2008-08-14 21:16:25) [GET]
+  r.teaser '{RAILS_TEASER}'
+  r.controller_action '{CONTROLLER_ACTION}'
+  r.ipv4_addr '{IP4_OCT}"."{IP4_OCT}"."{IP4_OCT}"."{IP4_OCT}'
+  r.datetime '{YEAR}"-"{MONTH_NUM}"-"{MDAY}{WS}{HOUR}":"{MINSEC}":"{MINSEC}'
+  r.http_method '{HTTP_VERB}'
+  # Session ID: BAh7CToMcmVmZXJlciIbL3ByaXNjaWxsYS9wZW9wbGUvMjM1MCIKZmxhc2hJ
+  # QzonQWN0aW9uQ29udHJvbGxlcjo6Rmxhc2g6OkZsYXNoSGFzaHsABjoKQHVz ...
+  r.skip_lines '{RAILS_SKIP_LINES}', :skip_line => true
+  r.end_session_id '{PARTIAL_SESSION_ID}'
+  # RuntimeError (Cannot destroy employee):  /app/models/employee.rb:198:in `before_destroy' 
+  # ActionController::RoutingError (no route found to match "/favicon.ico" with {:method=>:get}):
+  # ActionView::TemplateError (No rhtml, rxml, rjs or delegate template found for /shared/_ids_modal_selection_panel in script/../config/../app/views) on line #2 of app/views/events/index.rhtml:
+  # ActionView::TemplateError (You have a nil object when you didn't expect it!
+  # NoMethodError (undefined method `find' for ActionController::Filters::Filter:Class):
+  r.error '{RAILS_ERROR_CLASS}'
+  r.error_message '\(({WS}|{NON_WS})+\)', :strip_ends => true
+  r.line_number '"#"[0-9]+{WS}', :strip_ends => true
+  r.file_and_line '{WS}{REL_URL}":"', :strip_ends => true
+  # Filter chain halted as [#<ActionController::Caching::Actions::ActionCacheFilter:0x2a999ad620 @check=nil, @options={:store_options=>{}, :layout=>nil, :cache_path=>#<Proc:0x0000002a999b8890@/app/controllers/cached_controller.rb:8>}>] rendered_or_redirected.    
+  r.cache_hit '{CACHE_HIT}'
+  # Rendered shared/_analytics (0.2ms)
+  # Rendered layouts/_doc_type (0.00001)
+  r.partial '[a-z0-9]+{REL_URL}/\ \('
+  r.render_duration_ms '[0-9\.]+/ms\)'
+  r.render_duration_s '\([0-9\.]+\)', :strip_ends => true
+  # Completed in 0.21665 (4 reqs/sec) | Rendering: 0.00926 (4%) | DB: 0.00000 (0%) | 200 OK [http://demo.nu/employees]
+  # Completed in 614ms (View: 120, DB: 31) | 200 OK [http://floorplanner.local/demo]
+  r.teaser 'completed\ in', :begin => "REQUEST_COMPLETED"
+  r.duration_s '<REQUEST_COMPLETED>[0-9]+\.[0-9]+'
+  r.duration_ms '<REQUEST_COMPLETED>[0-9]+/ms'
+  r.start_view_stats '<REQUEST_COMPLETED>(View":"|Rendering":")', :begin => "COMPLETED_REQ_VIEW_STATS"
+  r.view_s '<COMPLETED_REQ_VIEW_STATS>([0-9]+\.[0-9]+)', :begin => "REQUEST_COMPLETED"
+  r.view_ms '<COMPLETED_REQ_VIEW_STATS>[0-9]+', :begin => "REQUEST_COMPLETED"
+  r.view_throwaway_tokens '<COMPLETED_REQ_VIEW_STATS>{CATCHALL}', :ignore => true
+  r.start_db_stats '<REQUEST_COMPLETED>DB":"', :begin => "COMPLETED_REQ_DB_STATS"
+  r.db_s '<COMPLETED_REQ_DB_STATS>[0-9]+\.[0-9]+', :begin => "REQUEST_COMPLETED"
+  r.db_ms '<COMPLETED_REQ_DB_STATS>[0-9]+', :begin => "REQUEST_COMPLETED"
+  r.db_throwaway_tokens '<COMPLETED_REQ_DB_STATS>{CATCHALL}', :ignore => true
+  r.url '<REQUEST_COMPLETED>\[{PROTO}"\/\/"({HOST}|{IP4_OCT}"."{IP4_OCT}"."{IP4_OCT}"."{IP4_OCT})({REL_URL}|"\/")?\]', :strip_ends => true
+  r.http_response '<REQUEST_COMPLETED>{HTTPCODE}'
+  r.strings '<REQUEST_COMPLETED>{NON_WS}{NON_WS}*'
+  r.ignore_others '<REQUEST_COMPLETED>{CATCHALL}', :ignore => true
+  # fallback to collecting strings
+  r.strings '{NON_WS}{NON_WS}*'
+end
+
+scanner.write!

data/spec/fixtures/rails_1x.log

@@ -0,0 +1,59 @@
+Processing DashboardController#index (for 1.1.1.1 at 2008-08-14 21:16:25) [GET]
+  Session ID: BAh7CToMcmVmZXJlciIbL3ByaXNjaWxsYS9wZW9wbGUvMjM1MCIKZmxhc2hJ
+QzonQWN0aW9uQ29udHJvbGxlcjo6Rmxhc2g6OkZsYXNoSGFzaHsABjoKQHVz
+ZWR7ADoNbGFuZ3VhZ2VvOhNMb2NhbGU6Ok9iamVjdBI6CUB3aW4wOg1AY291
+bnRyeSIHTkw6CkBoYXNoaf3L2Js6DkBvcmlnX3N0ciIKbmwtTkw6DUBpc28z
+MDY2MDoNQGNoYXJzZXQiClVURi04Og5AbGFuZ3VhZ2UiB25sOg5AbW9kaWZp
+ZXIwOgtAcG9zaXgiCm5sX05MOg1AZ2VuZXJhbCIKbmxfTkw6DUB2YXJpYW50
+MDoOQGZhbGxiYWNrMDoMQHNjcmlwdDA6DnBlcnNvbl9pZGkCMgc=--7918aed37151c13360cd370c37b541f136146fbd
+  Parameters: {"action"=>"index", "controller"=>"dashboard"}
+Set language to: nl_NL
+Rendering template within layouts/priscilla
+Rendering dashboard/index
+Completed in 0.22699 (4 reqs/sec) | Rendering: 0.02667 (11%) | DB: 0.03057 (13%) | 200 OK [https://www.example.com/]
+
+
+Processing PeopleController#index (for 1.1.1.1 at 2008-08-14 21:16:30) [GET]
+  Session ID: BAh7CSIKZmxhc2hJQzonQWN0aW9uQ29udHJvbGxlcjo6Rmxhc2g6OkZsYXNo
+SGFzaHsABjoKQHVzZWR7ADoMcmVmZXJlciIQL3ByaXNjaWxsYS86DnBlcnNv
+bl9pZGkCMgc6DWxhbmd1YWdlbzoTTG9jYWxlOjpPYmplY3QSOg1AY291bnRy
+eSIHTkw6CUB3aW4wOg5Ab3JpZ19zdHIiCm5sLU5MOgpAaGFzaGn9y9ibOg5A
+bGFuZ3VhZ2UiB25sOg1AY2hhcnNldCIKVVRGLTg6DUBpc28zMDY2MDoOQG1v
+ZGlmaWVyMDoLQHBvc2l4IgpubF9OTDoNQHZhcmlhbnQwOg1AZ2VuZXJhbCIK
+bmxfTkw6DEBzY3JpcHQwOg5AZmFsbGJhY2sw--48cbe3788ef27f6005f8e999610a42af6e90ffb3
+  Parameters: {"commit"=>"Zoek", "action"=>"index", "q"=>"gaby", "controller"=>"people"}
+Set language to: nl_NL
+Redirected to https://www.example.com/people/2545
+Completed in 0.04759 (21 reqs/sec) | DB: 0.03719 (78%) | 302 Found [https://www.example.com/people?q=gaby&commit=Zoek]
+
+
+Processing PeopleController#show (for 1.1.1.1 at 2008-08-14 21:16:30) [GET]
+  Session ID: BAh7CToMcmVmZXJlciIpL3ByaXNjaWxsYS9wZW9wbGU/cT1nYWJ5JmNvbW1p
+dD1ab2VrIgpmbGFzaElDOidBY3Rpb25Db250cm9sbGVyOjpGbGFzaDo6Rmxh
+c2hIYXNoewAGOgpAdXNlZHsAOg1sYW5ndWFnZW86E0xvY2FsZTo6T2JqZWN0
+EjoJQHdpbjA6DUBjb3VudHJ5IgdOTDoKQGhhc2hp/cvYmzoOQG9yaWdfc3Ry
+IgpubC1OTDoNQGlzbzMwNjYwOg1AY2hhcnNldCIKVVRGLTg6DkBsYW5ndWFn
+ZSIHbmw6DkBtb2RpZmllcjA6C0Bwb3NpeCIKbmxfTkw6DUBnZW5lcmFsIgpu
+bF9OTDoNQHZhcmlhbnQwOg5AZmFsbGJhY2swOgxAc2NyaXB0MDoOcGVyc29u
+X2lkaQIyBw==--3ad1948559448522a49d289a2a89dc7ccbe8847a
+  Parameters: {"action"=>"show", "id"=>"2545", "controller"=>"people"}
+Set language to: nl_NL
+Rendering template within layouts/priscilla
+Rendering people/show
+person: John Doe, study_year: 2008/2009
+Completed in 0.29077 (3 reqs/sec) | Rendering: 0.24187 (83%) | DB: 0.04030 (13%) | 200 OK [https://www.example.com/people/2545]
+
+
+Processing PeopleController#picture (for 1.1.1.1 at 2008-08-14 21:16:35) [GET]
+  Session ID: BAh7CSIKZmxhc2hJQzonQWN0aW9uQ29udHJvbGxlcjo6Rmxhc2g6OkZsYXNo
+SGFzaHsABjoKQHVzZWR7ADoMcmVmZXJlciIbL3ByaXNjaWxsYS9wZW9wbGUv
+MjU0NToOcGVyc29uX2lkaQIyBzoNbGFuZ3VhZ2VvOhNMb2NhbGU6Ok9iamVj
+dBI6DUBjb3VudHJ5IgdOTDoJQHdpbjA6DkBvcmlnX3N0ciIKbmwtTkw6CkBo
+YXNoaf3L2Js6DkBsYW5ndWFnZSIHbmw6DUBjaGFyc2V0IgpVVEYtODoNQGlz
+bzMwNjYwOg5AbW9kaWZpZXIwOgtAcG9zaXgiCm5sX05MOg1AdmFyaWFudDA6
+DUBnZW5lcmFsIgpubF9OTDoMQHNjcmlwdDA6DkBmYWxsYmFjazA=--797a33f280a482647111397d138d0918f2658167
+  Parameters: {"action"=>"picture", "id"=>"2545", "controller"=>"people"}
+Set language to: nl_NL
+Rendering template within layouts/priscilla
+Rendering people/picture
+Completed in 0.05383 (18 reqs/sec) | Rendering: 0.04622 (85%) | DB: 0.00206 (3%) | 200 OK [https://www.example.com/people/2545/picture]

data/spec/fixtures/rails_22.log

@@ -0,0 +1,12 @@
+Processing PageController#demo (for 127.0.0.1 at 2008-12-10 16:28:09) [GET]
+  Parameters: {"action"=>"demo", "controller"=>"page"}
+Logging in from session data...
+Logged in as test@example.com
+Using locale: en-US, http-accept: ["en-US"], session: , det browser: en-US, det domain: 
+Rendering template within layouts/demo
+Rendering page/demo
+Rendered shared/_analytics (0.2ms)
+Rendered layouts/_actions (0.6ms)
+Rendered layouts/_menu (2.2ms)
+Rendered layouts/_tabbar (0.5ms)
+Completed in 614ms (View: 120, DB: 31) | 200 OK [http://www.example.coml/demo]

data/spec/fixtures/rails_22_cached.log

@@ -0,0 +1,10 @@
+Processing CachedController#cached (for 1.1.1.1 at 2008-12-24 07:36:53) [GET]
+  Parameters: {"action"=>"cached", "controller"=>"cached"}
+Logging in from session data...
+Logging in using cookie...
+Using locale: zh-Hans, http-accept: ["zh-CN", "zh-HK", "zh-TW", "en-US"], session: , det browser: zh-Hans, det domain: , user pref locale: 
+Referer: http://www.example.com/referer
+Cached fragment hit: views/zh-Hans-www-cached-cached-all-CN--- (0.0ms)
+Filter chain halted as [#<ActionController::Caching::Actions::ActionCacheFilter:0x2a999ad620 @check=nil, @options={:store_options=>{}, :layout=>nil, :cache_path=>#<Proc:0x0000002a999b8890@/app/controllers/cached_controller.rb:8>}>] rendered_or_redirected.
+Filter chain halted as [#<ActionController::Filters::AroundFilter:0x2a999ad120 @identifier=nil, @kind=:filter, @options={:only=>#<Set: {"cached"}>, :if=>:not_logged_in?, :unless=>nil}, @method=#<ActionController::Caching::Actions::ActionCacheFilter:0x2a999ad620 @check=nil, @options={:store_options=>{}, :layout=>nil, :cache_path=>#<Proc:0x0000002a999b8890@/app/controllers/cached_controller.rb:8>}>>] did_not_yield.
+Completed in 3ms (View: 0, DB: 0) | 200 OK [http://www.example.com/cached/cached/]

data/spec/fixtures/rails_unordered.log

@@ -0,0 +1,24 @@
+Processing AccountController#dashboard (for 1.1.1.1 at 2008-12-24 07:36:49) [GET]
+  Parameters: {"action"=>"dashboard", "controller"=>"account", "first_use"=>"true"}
+Logging in from session data...
+
+
+Processing ProjectsController#new (for 1.1.1.1 at 2008-12-24 07:36:49) [GET]
+  Parameters: {"action"=>"new", "controller"=>"projects"}
+Rendering template within layouts/default
+Rendering account/dashboard
+Logging in from session data...
+Logging in using cookie...
+Using locale: en-US, http-accept: [], session: , det browser: , det domain: , user pref locale: 
+Rendered shared/_maintenance (0.6ms)
+Rendering template within layouts/templates/general_default/index.html.erb
+Rendered projects/_recent_designs (4.3ms)
+Rendered projects/_project (13.6ms)
+Rendered projects/_projects (18.7ms)
+Rendered layouts/_menu (1.4ms)
+Completed in 36ms (View: 30, DB: 3) | 200 OK [http://www.example.com/projects/new]
+Rendered layouts/_actions (0.3ms)
+Rendered layouts/_menu (1.6ms)
+Rendered layouts/_tabbar (1.9ms)
+Rendered layouts/_footer (3.2ms)
+Completed in 50ms (View: 41, DB: 4) | 200 OK [http://www.example.com/dashboard?first_use=true]

data/spec/playground/show_apache_processing.rb

@@ -0,0 +1,13 @@
+require File.dirname(__FILE__) + "/../spec_helper" # loads libs
+
+error_line = %q{[Sun Nov 30 14:23:45 2008] [error] [client 10.0.1.197] Invalid URI in request GET .\\.\\.\\.\\.\\.\\.\\.\\.\\.\\/winnt/win.ini HTTP/1.1}
+access_line = %q{127.81.248.53 - - [14/Jan/2009:11:49:43 -0500] "GET /reports/REPORT7_1ART02.pdf HTTP/1.1" 206 255404}
+
+mangled_error_line = error_line + "more words"
+
+puts "Processed Error Message:"
+puts error_line.tokenize_apache_logs.inspect
+puts "Error Message with extras:"
+puts mangled_error_line.tokenize_apache_logs.inspect
+puts "Processed Access Message"
+puts access_line.tokenize_apache_logs.inspect

data/spec/spec_helper.rb

@@ -1,4 +1,7 @@
-require 'teeth/tokenize_apache_logs'
+require 'teeth/scan_apache_logs'
+#require 'teeth/scan_rails_logs'
+
+require File.dirname(__FILE__) + "/../lib/teeth"
 
 def be_greater_than(expected)
   simple_matcher("be greater than #{expected.to_s}") do |given, matcher|
@@ -8,3 +11,5 @@ def be_greater_than(expected)
   end
   
 end
+
+include Teeth

data/spec/unit/rule_statement_spec.rb

@@ -0,0 +1,60 @@
+require File.dirname(__FILE__) + '/../spec_helper'
+
+describe RuleStatement do
+  
+  it "should generate a rule to short circuit scanner processing when given option :skip_line => true" do
+    rs = RuleStatement.new :rails_session_id_start, '{WS}*Session ID":"', :skip_line => true
+    expected = 
+%q|{WS}*Session ID":" {
+  return EOF_KVPAIR;
+}|
+    rs.scanner_code.should == expected
+  end
+
+  it "should use strip_ends(yytext) in the rule when given option :strip_ends => true" do
+    rs = RuleStatement.new :browser_string, '{BROWSER_STR}', :strip_ends => true
+    expected = 
+%q|{BROWSER_STR} {
+  KVPAIR browser_string = {"browser_string", strip_ends(yytext)};
+  return browser_string;
+}|
+    rs.scanner_code.should == expected
+  end
+  
+  it "should include a call to the BEGIN() macro if given the :begin option" do
+    rs = RuleStatement.new :start_special_state, '{SPECIAL_STATE_REGEX}', :begin => "SPECIAL_STATE"
+    expected = 
+%q|{SPECIAL_STATE_REGEX} {
+  BEGIN(SPECIAL_STATE);
+  KVPAIR start_special_state = {"start_special_state", yytext};
+  return start_special_state;
+}|
+    rs.scanner_code.should == expected
+  end
+  
+  it "should not include any C code if given :ignore => true" do
+    rs = RuleStatement.new :catchall_rule_for_special_state, '<SPECIAL_STATE>{CATCHALL}', :ignore => true
+    expected = %q|<SPECIAL_STATE>{CATCHALL}|
+    rs.scanner_code.should == expected
+  end
+  
+end
+
+describe RuleStatementGroup do
+  
+  before(:each) do
+    @statement_group = RuleStatementGroup.new
+  end
+  
+  it "should not reject duplicate rule definitions" do
+    @statement_group.add :explode_on_2nd_try, '{WS}'
+    lambda {@statement_group.add :explode_on_2nd_try, '{WS}'}.should_not raise_error DuplicateRuleError
+  end
+  
+  it "should use method missing magic to define rules with sugary syntax" do
+    @statement_group.http_version "{HTTP_VERSION}"
+    @statement_group.first.should == RuleStatement.new(:http_version, "{HTTP_VERSION}")
+  end
+  
+  
+end

data/spec/unit/{tokenize_apache_spec.rb → scan_apache_spec.rb}

@@ -1,15 +1,16 @@
 require File.dirname(__FILE__) + '/../spec_helper'
+require "teeth/scan_apache_logs"
 $INCLUDE_SLOW_TESTS = true
 
 describe "Apache Lexer Extension", "when lexing apache errors" do
 
   before(:each) do
     str = "[Sun Nov 30 14:23:45 2008] [error] [client 10.0.1.197] Invalid URI in request GET .\\.\\.\\.\\.\\.\\.\\.\\.\\.\\/winnt/win.ini HTTP/1.1"
-    @tokens = str.tokenize_apache_logs
+    @tokens = str.scan_apache_logs
   end
   
   it "should return an uuid and empty message for an empty string" do
-    tokens = "".tokenize_apache_logs
+    tokens = "".scan_apache_logs
     tokens[:message].should == ""
     tokens[:id].should match(/[0-9A-F]{32}/)
   end
@@ -30,9 +31,13 @@ describe "Apache Lexer Extension", "when lexing apache errors" do
     @tokens[:relative_url].first.should == ".\\.\\.\\.\\.\\.\\.\\.\\.\\.\\/winnt/win.ini"
   end
   
+  it "should group unknown tokens into strings" do
+    @tokens[:strings].should == ["client", "Invalid URI in request"]
+  end
+  
   it "should error out if the string is longer than 1M chars" do
     str = ((("abcDE" * 2) * 1000) * 100) + "X"
-    lambda {str.tokenize_apache_logs[:word]}.should raise_error(ArgumentError, "string too long for tokenize_apache_logs! max length is 1,000,000 chars")
+    lambda {str.scan_apache_logs}.should raise_error(ArgumentError, "string too long for scan_apache_logs! max length is 1,000,000 chars")
   end
   
 end
@@ -40,19 +45,19 @@ end
 describe "Apache Lexer Extension", "when lexing apache access logs" do
   before(:each) do
     str = %q{couchdb.localdomain:80 172.16.115.1 - - [13/Dec/2008:19:26:11 -0500] "GET /favicon.ico HTTP/1.1" 404 241 "http://172.16.115.130/" "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_4_11; en) AppleWebKit/525.27.1 (KHTML, like Gecko) Version/3.2.1 Safari/525.27.1"}
-    @tokens = str.tokenize_apache_logs
+    @tokens = str.scan_apache_logs
     str2 = %q{127.162.219.29 - - [14/Jan/2009:15:32:32 -0500] "GET /reports//ee_commerce/paypalcart.php?toroot=http://www.shenlishi.com//skin/fxid1.txt?? HTTP/1.1" 404 5636}
-    @tokens2 = str2.tokenize_apache_logs
+    @tokens2 = str2.scan_apache_logs
     str3 = %q{127.81.248.53 - - [14/Jan/2009:11:49:43 -0500] "GET /reports/REPORT7_1ART02.pdf HTTP/1.1" 206 255404}
-    @tokens3 = str3.tokenize_apache_logs
+    @tokens3 = str3.scan_apache_logs
     str4 = %q{127.140.136.56 - - [23/Jan/2009:12:59:24 -0500] "GET /scripts/..%255c%255c../winnt/system32/cmd.exe?/c+dir" 404 5607}
-    @tokens4 = str4.tokenize_apache_logs
+    @tokens4 = str4.scan_apache_logs
     str5 = %q{127.254.43.205 - - [26/Jan/2009:08:32:08 -0500] "GET /reports/REPORT9_3.pdf//admin/includes/footer.php?admin_template_default=../../../../../../../../../../../../../etc/passwd%00 HTTP/1.1" 404 5673}
-    @tokens5 = str5.tokenize_apache_logs
+    @tokens5 = str5.scan_apache_logs
     str6 = %q{127.218.234.82 - - [26/Jan/2009:08:32:19 -0500] "GET /reports/REPORT9_3.pdf//admin/includes/header.php?bypass_installed=1&bypass_restrict=1&row_secure[account_theme]=../../../../../../../../../../../../../etc/passwd%00 HTTP/1.1" 404 5721}
-    @tokens6 = str6.tokenize_apache_logs
+    @tokens6 = str6.scan_apache_logs
     str_naked_url = %q{127.218.234.82 - - [26/Jan/2009:08:32:19 -0500] "GET / HTTP/1.1" 404 5721}
-    @tokens_naked_url = str_naked_url.tokenize_apache_logs
+    @tokens_naked_url = str_naked_url.scan_apache_logs
   end
   
   it "provides hints for testing" do
@@ -74,7 +79,7 @@ describe "Apache Lexer Extension", "when lexing apache access logs" do
     (300 .. 305).map { |n| n.to_s } + ['307'] + (400 .. 417).map { |n| n.to_s } +
     (500 .. 505).map { |n| n.to_s }
     codes.each do |code|
-      code.tokenize_apache_logs[:http_response].first.should == code
+      code.scan_apache_logs[:http_response].first.should == code
     end
   end