teeth 0.2.0

data/lib/teeth.rb

@@ -0,0 +1,14 @@
+# encoding: utf-8
+
+TEETH_EXT_DIR = File.expand_path(File.dirname(__FILE__) + '/../ext')
+
+require "teeth/scanner"
+require "teeth/scanner_definition"
+require "teeth/rule_statement"
+
+begin
+  require "teeth/scanners/scan_apache_logs"
+  require "teeth/scanners/scan_rails_logs"
+rescue LoadError => e
+  STDERR.puts "WARNING: could not load extensions. This is okay if you are creating them from source for the first time."
+end

data/lib/teeth/rule_statement.rb

@@ -0,0 +1,61 @@
+module Teeth
+  
+  class DuplicateRuleError < ScannerError
+  end
+
+  class RuleStatement
+    attr_reader :name, :regex, :strip_ends, :skip_line, :begin
+    
+    def initialize(name, regex, options={})
+      @name, @regex = name, regex
+      @strip_ends, @skip_line, @begin = options[:strip_ends], options[:skip_line], options[:begin]
+      @ignore = options[:ignore]
+    end
+    
+    def ==(other)
+      other.kind_of?(RuleStatement) && other.name == name && other.regex == regex
+    end
+    
+    def scanner_code
+      if @ignore
+        regex
+      else
+        "#{regex} {\n" + function_body + "}"
+      end
+    end
+        
+    def function_body
+      code = ""
+      code += "  BEGIN(#{@begin});\n" if @begin
+      if skip_line
+        code += "  return EOF_KVPAIR;\n"
+      else
+        code += "  KVPAIR #{name.to_s} = {\"#{name.to_s}\", #{yytext_statement}};\n" +
+        "  return #{name.to_s};\n"
+      end
+      code
+    end
+    
+    def yytext_statement
+      strip_ends ? "strip_ends(yytext)" : "yytext"
+    end
+    
+  end
+  
+  class RuleStatementGroup < Array
+    
+    def add(name, regex, options={})
+      push RuleStatement.new(name, regex, options)
+    end
+    
+    def rule_names
+      map { |rule_statement| rule_statement.name.to_s }
+    end
+    
+    def method_missing(called_method_name, *args, &block)
+      args[1] ||={}
+      add(called_method_name, args[0], args[1])
+    end
+    
+  end
+end

data/lib/teeth/scanner.rb

@@ -0,0 +1,101 @@
+require "erb"
+
+module Teeth
+  class ScannerError < StandardError
+  end
+  
+  class InvalidExtensionDirectory < ScannerError
+  end
+  
+  class Scanner
+    TEMPLATE = File.dirname(__FILE__) + "/templates/tokenizer.yy.erb"
+    attr_reader :scanner_defns, :scanner_rules, :rdoc
+    
+    def initialize(name, ext_dir=nil)
+      @scanner_base_name, @ext_dir = name, ext_dir
+      @scanner_defns, @scanner_rules = ScannerDefinitionGroup.new, RuleStatementGroup.new
+      ensure_ext_dir_exists if ext_dir
+    end
+    
+    def scanner_name
+      "scan_" + @scanner_base_name.to_s
+    end
+    
+    def main_function_name
+      "t_" + scanner_name
+    end
+    
+    def init_function_name
+      "Init_" + scanner_name
+    end
+    
+    def function_prefix
+      @scanner_base_name.to_s + "_yy"
+    end
+    
+    def entry_point
+      "scan_" + @scanner_base_name.to_s
+    end
+    
+    def extconf
+      'require "mkmf"' + "\n" + '$CFLAGS += " -Wall"' + "\n" + 
+      'have_library("uuid", "uuid_generate_time")' + "\n" +
+      "create_makefile " +
+      %Q|"teeth/#{scanner_name}", "./"\n| 
+    end
+    
+    def rdoc=(rdoc_text)
+      lines_of_rdoc_text = rdoc_text.split("\n").map { |line| " * " + line.strip}
+      lines_of_rdoc_text.first[0] = "/"
+      lines_of_rdoc_text[-1] = lines_of_rdoc_text.last + " */"
+      @rdoc = lines_of_rdoc_text.join("\n")
+    end
+    
+    def define(*args)
+      @scanner_defns.add(*args)
+    end
+    
+    def definitions
+      yield @scanner_defns
+    end
+    
+    def load_default_definitions_for(*defn_types)
+      @scanner_defns.defaults_for(*defn_types)
+    end
+    
+    def rule(*args)
+      scanner_rules.add(*args)
+    end
+    
+    def rules
+      yield scanner_rules
+    end
+    
+    def generate
+      template = ERB.new(IO.read(TEMPLATE))
+      scanner = self
+      b = binding
+      template.result(b)
+    end
+    
+    def write!
+      raise InvalidExtensionDirectory, "no extension directory specified" unless @ext_dir
+      File.open(@ext_dir + "/extconf.rb", "w") do |extconf_rb|
+        extconf_rb.write extconf
+      end
+      File.open(@ext_dir + "/" + scanner_name + ".yy", "w") do |scanner|
+        scanner.write generate
+      end
+    end
+    
+    private
+    
+    def ensure_ext_dir_exists
+      unless File.exist?(@ext_dir)
+        Dir.mkdir @ext_dir
+      end
+    end
+    
+  end
+  
+end

data/lib/teeth/scanner_definition.rb

@@ -0,0 +1,117 @@
+module Teeth
+  class DuplicateDefinitionError < ScannerError
+  end
+  
+  class InvalidDefaultDefinitionName < ScannerError
+  end
+  
+  class ScannerDefinitionArgumentError < ScannerError
+  end
+  
+  class ScannerDefinition
+    attr_reader :name, :regex
+    
+    def initialize(name, regex, opts={})
+      if regex.kind_of?(Hash)
+        regex, opts = nil, regex
+      end
+      @name, @regex, @start_condition = name, regex, opts[:start_condition]
+      assert_valid_argument_combination
+    end
+    
+    def scanner_code
+      start_condition_string + @name.to_s + regex_to_s
+    end
+    
+    def regex_to_s
+      unless @regex.to_s == ""
+        " " + @regex.to_s
+      else
+        ""
+      end
+    end
+    
+    def start_condition_string
+      case @start_condition.to_s
+      when /^inc/
+        "%s "
+      when /^exc/
+        "%x "
+      else
+        ""
+      end
+    end
+    
+    private
+    
+    def assert_valid_argument_combination
+      if @start_condition
+        if @regex.to_s != "" # (nil or "").to_s == ""
+          raise ScannerDefinitionArgumentError, "a scanner definition cannot define both a regex and start condition"
+        end
+      end
+    end
+    
+  end
+  
+  class ScannerDefinitionGroup < Array
+    
+    DEFAULT_DEFINITIONS = {}
+    DEFAULT_DEFINITIONS[:whitespace] = [["WS",  '[[:space:]]'],
+                                        ["NON_WS", "([a-z]|[0-9]|[:punct:])"]]
+    DEFAULT_DEFINITIONS[:ip]    = [ ["IP4_OCT", "[0-9]|[0-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5]"],
+                                    ["HOST", '([a-z0-9][a-z0-9\-]*\.[a-z0-9][a-z0-9\-]*.[a-z0-9][a-z0-9\-\.]*[a-z]+(\:[0-9]+)?)|localhost']]
+    DEFAULT_DEFINITIONS[:time]  = [ ["WDAY", "mon|tue|wed|thu|fri|sat|sun"],
+                                    ["MON", "jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec"],
+                                    ["MONTH_NUM", "0[1-9]|1[0-2]"],
+                                    ["MDAY", "3[0-1]|[1-2][0-9]|0[1-9]"],
+                                    ["HOUR", "2[0-3]|[0-1][0-9]"],
+                                    ["MINSEC", "[0-5][0-9]|60"],
+                                    ["YEAR", "[0-9][0-9][0-9][0-9]"],
+                                    ["PLUSMINUS", '(\+|\-)']]
+    DEFAULT_DEFINITIONS[:web]   = [ ["TIMING", %q{[0-9]+\.[0-9]+}],
+                                    ["REL_URL", %q{(\/|\\\\|\.)[a-z0-9\._\~\-\/\?&;#=\%\:\+\[\]\\\\]*}],
+                                    ["PROTO", "(http:|https:)"],
+                                    ["ERR_LVL", "(emerg|alert|crit|err|error|warn|warning|notice|info|debug)"],
+                                    ["HTTP_VERS", 'HTTP\/(1.0|1.1)'],
+                                    ["HTTP_VERB", "(get|head|put|post|delete|trace|connect)"],
+                                    ["HTTPCODE", "(100|101|20[0-6]|30[0-5]|307|40[0-9]|41[0-7]|50[0-5])"],
+                                    ["BROWSER_STR", '\"(moz|msie|lynx|reconnoiter|pingdom)[^"]+\"']]
+
+    def add(name, regex, options={})
+      assert_defn_has_unique_name(name)
+      push ScannerDefinition.new(name, regex, options)
+    end
+    
+    def assert_defn_has_unique_name(name)
+      if defn_names.include?(name.to_s)
+        raise DuplicateDefinitionError, "a definition for #{name.to_s} has already been defined"
+      end
+    end
+    
+    def defn_names
+      map { |defn_statement| defn_statement.name.to_s }
+    end
+    
+    def method_missing(called_method_name, *args, &block)
+      args[1] ||={}
+      add(called_method_name, args[0], args[1])
+    end
+    
+    def defaults_for(*default_types)
+      default_types.each do |default_type|
+        unless default_definitions = DEFAULT_DEFINITIONS[default_type]
+          raise InvalidDefaultDefinitionName, "no default definitions found for #{default_type.to_s}"
+        end
+        default_definitions.each do |defn|
+          begin
+            add(defn.first, defn.last)
+          rescue DuplicateDefinitionError
+          end
+        end
+      end
+    end
+    
+    
+  end
+end

data/lib/teeth/scanner_definitions/scan_apache_logs.rb

@@ -0,0 +1,28 @@
+require "teeth"
+scanner = Teeth::Scanner.new(:apache_logs, TEETH_EXT_DIR + '/scan_apache_logs/')
+scanner.load_default_definitions_for(:whitespace, :ip, :time, :web)
+scanner.rdoc = <<-RDOC
+Scans self, which is expected to be a single line from an Apache error or 
+access log, and returns a Hash of the components of the log message.  The
+following parts of the log message are returned if they are present:
+IPv4 address, datetime, HTTP Version used, the browser string given by the 
+client, any absolute or relative URLs, the error level, HTTP response code,
+HTTP Method (verb), and any other uncategorized strings present.
+RDOC
+scanner.rules do |r|
+  r.timing '{TIMING}'
+  r.ipv4_addr '{IP4_OCT}"."{IP4_OCT}"."{IP4_OCT}"."{IP4_OCT}'
+  r.apache_err_datetime '{WDAY}{WS}{MON}{WS}{MDAY}{WS}{HOUR}":"{MINSEC}":"{MINSEC}{WS}{YEAR}'
+  r.apache_access_datetime '{MDAY}\/{MON}\/{YEAR}":"{HOUR}":"{MINSEC}":"{MINSEC}{WS}{PLUSMINUS}{YEAR}'
+  r.http_version '{HTTP_VERS}'
+  r.browser_string '{BROWSER_STR}', :strip_ends => true
+  r.absolute_url '{PROTO}"\/\/"({HOST}|{IP4_OCT}"."{IP4_OCT}"."{IP4_OCT}"."{IP4_OCT})({REL_URL}|"\/")?'
+  r.host '{HOST}'
+  r.relative_url '{REL_URL}'
+  r.error_level '{ERR_LVL}'
+  r.http_response '{HTTPCODE}'
+  r.http_method '{HTTP_VERB}'
+  r.strings '{NON_WS}{NON_WS}*'
+end
+
+scanner.write!

data/lib/teeth/scanner_definitions/scan_rails_logs.rb

@@ -0,0 +1,70 @@
+require "teeth"
+scanner = Teeth::Scanner.new(:rails_logs, TEETH_EXT_DIR + '/scan_rails_logs/')
+scanner.load_default_definitions_for(:whitespace, :ip, :time, :web)
+scanner.rdoc = <<-RDOC
+Scans self, which is expected to be a line from a Rails production or dev log,
+and returns a Hash of the significant features in the log message, including 
+the IP address of the client, the Controller and Action, any partials rendered,
+and the time spent rendering them, the duration of the DB request(s), the HTTP
+verb, etc.
+RDOC
+scanner.definitions do |define|
+  define.RAILS_TEASER '(processing|filter\ chain\ halted|rendered)'
+  define.CONTROLLER_ACTION '[a-z0-9]+#[a-z0-9]+'
+  define.RAILS_SKIP_LINES '(session\ id)'
+  define.CACHE_HIT 'actioncontroller"::"caching"::"actions"::"actioncachefilter":"0x[0-9a-f]+'
+  define.PARTIAL_SESSION_ID '^([a-z0-9]+"="*"-"+[a-z0-9]+)'
+  define.RAILS_ERROR_CLASS '([a-z]+\:\:)*[a-z]+error'
+  define.REQUEST_COMPLETED :start_condition => :exclusive
+  define.COMPLETED_REQ_VIEW_STATS :start_condition => :exclusive
+  define.COMPLETED_REQ_DB_STATS :start_condition => :exclusive
+end
+scanner.rules do |r|
+  # Processing DashboardController#index (for 1.1.1.1 at 2008-08-14 21:16:25) [GET]
+  r.teaser '{RAILS_TEASER}'
+  r.controller_action '{CONTROLLER_ACTION}'
+  r.ipv4_addr '{IP4_OCT}"."{IP4_OCT}"."{IP4_OCT}"."{IP4_OCT}'
+  r.datetime '{YEAR}"-"{MONTH_NUM}"-"{MDAY}{WS}{HOUR}":"{MINSEC}":"{MINSEC}'
+  r.http_method '{HTTP_VERB}'
+  # Session ID: BAh7CToMcmVmZXJlciIbL3ByaXNjaWxsYS9wZW9wbGUvMjM1MCIKZmxhc2hJ
+  # QzonQWN0aW9uQ29udHJvbGxlcjo6Rmxhc2g6OkZsYXNoSGFzaHsABjoKQHVz ...
+  r.skip_lines '{RAILS_SKIP_LINES}', :skip_line => true
+  r.end_session_id '{PARTIAL_SESSION_ID}'
+  # RuntimeError (Cannot destroy employee):  /app/models/employee.rb:198:in `before_destroy' 
+  # ActionController::RoutingError (no route found to match "/favicon.ico" with {:method=>:get}):
+  # ActionView::TemplateError (No rhtml, rxml, rjs or delegate template found for /shared/_ids_modal_selection_panel in script/../config/../app/views) on line #2 of app/views/events/index.rhtml:
+  # ActionView::TemplateError (You have a nil object when you didn't expect it!
+  # NoMethodError (undefined method `find' for ActionController::Filters::Filter:Class):
+  r.error '{RAILS_ERROR_CLASS}'
+  r.error_message '\(({WS}|{NON_WS})+\)', :strip_ends => true
+  r.line_number '"#"[0-9]+{WS}', :strip_ends => true
+  r.file_and_line '{WS}{REL_URL}":"', :strip_ends => true
+  # Filter chain halted as [#<ActionController::Caching::Actions::ActionCacheFilter:0x2a999ad620 @check=nil, @options={:store_options=>{}, :layout=>nil, :cache_path=>#<Proc:0x0000002a999b8890@/app/controllers/cached_controller.rb:8>}>] rendered_or_redirected.    
+  r.cache_hit '{CACHE_HIT}'
+  # Rendered shared/_analytics (0.2ms)
+  # Rendered layouts/_doc_type (0.00001)
+  r.partial '[a-z0-9]+{REL_URL}/\ \('
+  r.render_duration_ms '[0-9\.]+/ms\)'
+  r.render_duration_s '\([0-9\.]+\)', :strip_ends => true
+  # Completed in 0.21665 (4 reqs/sec) | Rendering: 0.00926 (4%) | DB: 0.00000 (0%) | 200 OK [http://demo.nu/employees]
+  # Completed in 614ms (View: 120, DB: 31) | 200 OK [http://floorplanner.local/demo]
+  r.teaser 'completed\ in', :begin => "REQUEST_COMPLETED"
+  r.duration_s '<REQUEST_COMPLETED>[0-9]+\.[0-9]+'
+  r.duration_ms '<REQUEST_COMPLETED>[0-9]+/ms'
+  r.start_view_stats '<REQUEST_COMPLETED>(View":"|Rendering":")', :begin => "COMPLETED_REQ_VIEW_STATS"
+  r.view_s '<COMPLETED_REQ_VIEW_STATS>([0-9]+\.[0-9]+)', :begin => "REQUEST_COMPLETED"
+  r.view_ms '<COMPLETED_REQ_VIEW_STATS>[0-9]+', :begin => "REQUEST_COMPLETED"
+  r.view_throwaway_tokens '<COMPLETED_REQ_VIEW_STATS>{CATCHALL}', :ignore => true
+  r.start_db_stats '<REQUEST_COMPLETED>DB":"', :begin => "COMPLETED_REQ_DB_STATS"
+  r.db_s '<COMPLETED_REQ_DB_STATS>[0-9]+\.[0-9]+', :begin => "REQUEST_COMPLETED"
+  r.db_ms '<COMPLETED_REQ_DB_STATS>[0-9]+', :begin => "REQUEST_COMPLETED"
+  r.db_throwaway_tokens '<COMPLETED_REQ_DB_STATS>{CATCHALL}', :ignore => true
+  r.url '<REQUEST_COMPLETED>\[{PROTO}"\/\/"({HOST}|({IP4_OCT}"."{IP4_OCT}"."{IP4_OCT}"."{IP4_OCT}))({REL_URL}|"/"|"\\\\")?\]', :strip_ends => true
+  r.http_response '<REQUEST_COMPLETED>{HTTPCODE}'
+  r.strings '<REQUEST_COMPLETED>{NON_WS}{NON_WS}*'
+  r.ignore_others '<REQUEST_COMPLETED>{CATCHALL}', :ignore => true
+  # fallback to collecting strings
+  r.strings '{NON_WS}{NON_WS}*'
+end
+
+scanner.write!

data/lib/teeth/templates/tokenizer.yy.erb

@@ -0,0 +1,168 @@
+%option prefix="<%= scanner.function_prefix %>"
+%option full
+%option never-interactive
+%option read
+%option nounput
+%option noyywrap noreject noyymore nodefault
+%{
+#include <ruby.h>
+#include <uuid/uuid.h>
+/* Data types */
+typedef struct {
+  char *key;
+  char *value;
+} KVPAIR;
+const KVPAIR EOF_KVPAIR = {"EOF", "EOF"};
+/* prototypes */
+char *strip_ends(char *);
+VALUE <%= scanner.main_function_name %>(VALUE);
+void new_uuid(char *str_ptr);
+void raise_error_for_string_too_long(VALUE string);
+void include_message_in_token_hash(VALUE message, VALUE token_hash);
+void add_uuid_to_token_hash(VALUE token_hash);
+void push_kv_pair_to_hash(KVPAIR key_value, VALUE token_hash);
+void concat_word_to_string(KVPAIR key_value, VALUE token_hash);
+/* Set the scanner name, and return type */
+#define YY_DECL KVPAIR <%= scanner.entry_point %>(void)
+#define yyterminate() return EOF_KVPAIR
+/* Ruby 1.8 and 1.9 compatibility */
+#if !defined(RSTRING_LEN) 
+# define RSTRING_LEN(x) (RSTRING(x)->len) 
+# define RSTRING_PTR(x) (RSTRING(x)->ptr) 
+#endif 
+
+%}
+
+/* Definitions */
+
+CATCHALL (.|"\n")
+
+<% scanner.scanner_defns.each do |scanner_defn| %>
+<%= scanner_defn.scanner_code %>
+<% end %>
+
+%%
+  /* 
+    Actions 
+ */
+  
+<% scanner.scanner_rules.each do |scanner_rule| %>
+<%= scanner_rule.scanner_code %>
+<% end %>
+{CATCHALL} /* ignore */
+%%
+
+char *strip_ends(char *string) {
+  string[yyleng-1] = '\0';
+  ++string;
+  return string;
+}
+
+void uuid_unparse_upper_sans_dash(const uuid_t uu, char *out)
+{
+        sprintf(out,
+                "%02X%02X%02X%02X"
+                "%02X%02X"
+                "%02X%02X"
+                "%02X%02X"
+                "%02X%02X%02X%02X%02X%02X",
+                uu[0], uu[1], uu[2], uu[3],
+                uu[4], uu[5],
+                uu[6], uu[7],
+                uu[8], uu[9],
+                uu[10], uu[11], uu[12], uu[13], uu[14], uu[15]);
+}
+
+void new_uuid(char *str_ptr){
+  uuid_t new_uuid;
+  uuid_generate_time(new_uuid);
+  uuid_unparse_upper_sans_dash(new_uuid, str_ptr);
+}
+
+void raise_error_for_string_too_long(VALUE string){
+  if( RSTRING_LEN(string) > 1000000){
+    rb_raise(rb_eArgError, "string too long for <%=scanner.scanner_name %>! max length is 1,000,000 chars"); 
+  }
+}
+
+<%= scanner.rdoc %>
+VALUE <%= scanner.main_function_name %>(VALUE self) {
+  KVPAIR kv_result;
+  int scan_complete = 0;
+  int building_words_to_string = 0;
+  VALUE token_hash = rb_hash_new();
+  
+  BEGIN(INITIAL);
+  
+  /* error out on absurdly large strings */
+  raise_error_for_string_too_long(self);
+  /* {:message => self()} */
+  include_message_in_token_hash(self, token_hash);
+  /* {:id => UUID} */
+  add_uuid_to_token_hash(token_hash);
+  yy_scan_string(RSTRING_PTR(self));
+  while (scan_complete == 0) {
+    kv_result = <%= scanner.entry_point %>();
+    if (kv_result.key == "EOF"){
+      scan_complete = 1;
+    }
+    else if (kv_result.key == "strings"){
+      /* build a string until we get a non-word */
+      if (building_words_to_string == 0){
+        building_words_to_string = 1;
+        push_kv_pair_to_hash(kv_result, token_hash);
+      }
+      else{
+        concat_word_to_string(kv_result, token_hash);
+      }    
+    }    
+    else {
+      building_words_to_string = 0;
+      push_kv_pair_to_hash(kv_result, token_hash);
+    }
+  }
+  yy_delete_buffer(YY_CURRENT_BUFFER);
+  return rb_obj_dup(token_hash);
+}
+
+void add_uuid_to_token_hash(VALUE token_hash) {
+  char new_uuid_str[33];
+  new_uuid(new_uuid_str);
+  VALUE hsh_key_id = ID2SYM(rb_intern("id"));
+  VALUE hsh_val_id = rb_tainted_str_new2(new_uuid_str);
+  rb_hash_aset(token_hash, hsh_key_id, hsh_val_id);
+}
+
+void include_message_in_token_hash(VALUE message, VALUE token_hash) {
+  /* {:message => self()} */  
+  VALUE hsh_key_msg = ID2SYM(rb_intern("message"));
+  rb_hash_aset(token_hash, hsh_key_msg, message);
+}
+
+void concat_word_to_string(KVPAIR key_value, VALUE token_hash) {
+  char * space = " ";
+  VALUE hsh_key = ID2SYM(rb_intern(key_value.key));
+  VALUE hsh_value = rb_hash_aref(token_hash, hsh_key);
+  VALUE string = rb_ary_entry(hsh_value, -1);
+  rb_str_cat(string, space, 1);
+  rb_str_cat(string, key_value.value, yyleng);
+}
+
+void push_kv_pair_to_hash(KVPAIR key_value, VALUE token_hash) {
+  VALUE hsh_key = ID2SYM(rb_intern(key_value.key));
+  VALUE hsh_value = rb_hash_aref(token_hash, hsh_key);
+  VALUE ary_for_token_type = rb_ary_new();
+  switch (TYPE(hsh_value)) {
+    case T_NIL:
+      rb_ary_push(ary_for_token_type, rb_tainted_str_new2(key_value.value));
+      rb_hash_aset(token_hash, hsh_key, ary_for_token_type);
+      break;
+    case T_ARRAY:
+      rb_ary_push(hsh_value, rb_tainted_str_new2(key_value.value));
+      break;
+  }
+}
+
+void <%=scanner.init_function_name %>() {
+  rb_define_method(rb_cString, "<%= scanner.scanner_name %>", <%= scanner.main_function_name %>, 0);
+}