teeth 0.2.0

data/LICENSE

@@ -0,0 +1,11 @@
+Copyright (c) 2009, Daniel Stephen DeLeo
+Portions Copyright (c) 2004 Apple Computer, Inc. All rights reserved. 
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+
+Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+Neither the name of the author nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

data/README.rdoc

@@ -0,0 +1,123 @@
+= Teeth
+Teeth is a library for fast parsing of log files such as Apache access and error logs.  It uses C extensions generated by flex[http://flex.sourceforge.net/index.html] (as in Flex and Bison).  If you only want to use the built-in scanners, you don't need flex.  If you want to add support for new/different log formats, you'll need to have flex installed.
+
+= Example
+ require "teeth"
+ 
+ access_log = %q{myhost.localdomain:80 172.16.115.1 - - [13/Dec/2008:19:26:11 -0500] "GET /favicon.ico HTTP/1.1" 404 241 "http://172.16.115.130/" "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_4_11; en) AppleWebKit/525.27.1 (KHTML, like Gecko) Version/3.2.1 Safari/525.27.1"}
+ access_log.scan_apache_logs
+	=>   {:strings=>["241"], 
+				:apache_access_datetime=>["13/Dec/2008:19:26:11 -0500"], 
+				:absolute_url=>["http://172.16.115.130/"], 
+				:message=>"myhost.localdomain:80 172.16.115.1 - - [13/Dec/2008:19:26:11 -0500] \"GET /favicon.ico HTTP/1.1\" 404 241 \"http://172.16.115.130/\" \"Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_4_11; en) AppleWebKit/525.27.1 (KHTML, like Gecko) Version/3.2.1 Safari/525.27.1\"", 
+				:http_method=>["GET"], 
+				:browser_string=>["Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_4_11; en) AppleWebKit/525.27.1 (KHTML, like Gecko) Version/3.2.1 Safari/525.27.1"], 
+				:relative_url=>["/favicon.ico"], 
+				:http_version=>["HTTP/1.1"], 
+				:host=>["myhost.localdomain:80"], 
+				:id=>"8AD5CBCC1CB011DE8CE10017F22FF48F", 
+				:http_response=>["404"], 
+				:ipv4_addr=>["172.16.115.1"]}
+	
+= Supported Log Formats
+* Apache (access and error logs)
+* Rails
+
+Support for other web servers, app servers, and applications as well as other types of servers (e.g., SMTP, etc.) and generic syslog logs is planned for the future.
+
+== Creating Your Own Scanners
+Teeth includes a library that can generate a flex scanner definition using a simplified definition written in ruby.  This cuts down on the repetition involved in writing all the C code by hand.  The included scanners for Apache and Rails logs are defined this way.  You can find them in the scanners directory.
+
+Here's an example based on the definition for the Rails log scanner:
+
+	require File.dirname(__FILE__) + "/../lib/teeth"
+	scanner = Teeth::Scanner.new(:rails_logs, File.dirname(__FILE__) + '/../ext/scan_rails_logs/')
+	
+Flex definitions are kinda like macros for regular expressions.
+We include some of the available defaults here to make writing 
+the scanner easier
+	
+	scanner.load_default_definitions_for(:whitespace, :ip, :time, :web)
+	
+Add some more definitions
+	scanner.definitions do |define|
+		define.RAILS_TEASER '(processing|filter\ chain\ halted|rendered)'
+		define.CONTROLLER_ACTION '[a-z0-9]+#[a-z0-9]+'
+		
+Scanner is case insensitive
+		define.RAILS_ERROR_CLASS '([a-z]+\:\:)*[a-z]+error'
+	  
+"start conditions" are a feature of flex that allows
+us to have some regular expressions that are only active
+when we tell the scanner to enter a certain state.  Here
+we define the ``REQUEST_COMPLETED'' state, and specify
+that it is exclusive.  This means that if the scanner is
+in this state, it only matches rules written for this state
+		define.REQUEST_COMPLETED :start_condition => :exclusive
+	end
+	
+Define rules.  These are the actions that the scanner executes when
+it sees text that matches a regular expression.  The default action
+is to add :action_name => [matched_text] to the results Hash, or push
+the matched text on the end of the array if it already exists.
+	scanner.rules do |r|
+		
+This will add something like :teaser => ["Processing"] to the results
+		r.teaser '{RAILS_TEASER}'
+		r.controller_action '{CONTROLLER_ACTION}'
+		
+Use some of the default definitions we added above.
+		r.datetime '{YEAR}"-"{MONTH_NUM}"-"{MDAY}{WS}{HOUR}":"{MINSEC}":"{MINSEC}'
+		r.http_method '{HTTP_VERB}'
+		
+With :skip_line => true, scanner stops processing the line immediately
+		r.skip_lines '{RAILS_SKIP_LINES}', :skip_line => true
+	  r.error '{RAILS_ERROR_CLASS}'
+		
+with :strip_ends => true, scanner removes first and last characters from matched text
+		r.error_message '\(({WS}|{NON_WS})+\)', :strip_ends => true
+		
+Puts scanner in the ``REQUEST_COMPLETED'' state we defined above.
+The scanner only matches rules beginning with ``<REQUEST_COMPLETED>''
+now
+		r.teaser 'completed\ in', :begin => "REQUEST_COMPLETED"
+	  
+These rules only apply to the ``REQUEST_COMPLETED'' State
+		r.duration_s '<REQUEST_COMPLETED>[0-9]+\.[0-9]+'
+		r.duration_ms '<REQUEST_COMPLETED>[0-9]+/ms'
+		r.http_response '<REQUEST_COMPLETED>{HTTPCODE}'
+		
+Need a "catchall" rule -- flex scanner "jams" if there isn't a default rule (the
+catchall rule for the default/INITIAL state is automatically included).
+note that :ignore => true makes the scanner ignore what it matches but doesn't
+stop processing of the line.
+		r.ignore_others '<REQUEST_COMPLETED>{CATCHALL}', :ignore => true
+	  
+The "strings" action is special.  It keeps track of whether the last token 
+was also a string, and if it was, the new string is appended to the 
+last string instead of being pushed to the array.  For example, when scanning
+an apache error log, ``Invalid URI in request'' will be extracted as a complete
+string (instead of ["Invalid", "URI", "in", "request"])
+		r.strings '{NON_WS}{NON_WS}*'
+	end
+	
+Writes the generated scanner and an extconf.rb for it to the directory
+we specified when we initialized the scanner.
+	scanner.write!
+	
+There's not much in the way of documentation for the scanner generator, but you can 
+refer to the specs and the definitions for Apache and Rails logs to get a sense
+of how it works.  It would probably help to learn about flex's regex syntax
+and other features.
+
+= Ruby 1.9
+Ruby 1.9 is supported on the master branch.  Don't use the ruby1.9 branch, it is orphaned.
+
+= Shortcomings and Known Issues
+In addition to the lack of support for formats other than Apache and Rails described above:
+* It's a new project, lots of API changes
+* Does not convert datetimes to Ruby Time objects
+* Does not always use context or knowledge of the log format to its advantage.  This is improving now that the scanner can utilize start conditions.
+
+= Performance
+On my laptop, a white MacBook 2.0 GHz Intel Core Duo, teeth can process more than 30k lines of Apache access logs per second.  So it's pretty fast.  If modified to not create a UUID or keep the full message, this can be increased to around 45k lines/sec.  One could potentially do pretty well on the wide_finder2[http://www.tbray.org/ongoing/When/200x/2008/05/01/Wide-Finder-2]...

data/Rakefile

@@ -0,0 +1,112 @@
+require "spec/rake/spectask"
+require "rake/clean"
+require "rake/rdoctask"
+
+desc "Run all of the specs"
+Spec::Rake::SpecTask.new do |t|
+  t.spec_opts = ['--options', "\"spec/spec.opts\""]
+  t.fail_on_error = false
+end
+
+task :default => :spec
+
+namespace :spec do
+
+  desc "Generate HTML report for failing examples"
+  Spec::Rake::SpecTask.new('report') do |t|
+    t.spec_files = FileList['failing_examples/**/*.rb']
+    t.spec_opts = ["--format", "html:doc/tools/reports/failing_examples.html", "--diff", '--options', '"spec/spec.opts"']
+    t.fail_on_error = false
+  end
+  
+end
+
+Rake::RDocTask.new do |rdt|
+  rdt.rdoc_dir = "doc"
+  rdt.main = "README.rdoc"
+  rdt.rdoc_files.include("README.rdoc", "lib/*", "ext/*/*.yy.c")
+end
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |s|
+    s.name = 'teeth'
+    s.summary = 'Fast log file parsing in Ruby'
+    s.description = s.summary
+    s.email = 'ddeleo@basecommander.net'
+    s.homepage = "http://github.com/danielsdeleo/teeth"
+    s.platform = Gem::Platform::RUBY 
+    s.has_rdoc = true
+    s.extra_rdoc_files = ["README.rdoc"]
+    s.require_path = ["lib"]
+    s.authors = ["Daniel DeLeo"]
+    s.extensions = ["ext/scan_apache_logs/extconf.rb", "ext/scan_rails_logs/extconf.rb"]
+
+    # ruby -rpp -e' pp `git ls-files`.split("\n") '
+    s.files = `git ls-files`.split("\n").reject {|f| f =~ /git/}
+  end
+rescue LoadError
+  puts "Jeweler not available. Install it with: sudo gem install technicalpickles-jeweler -s http://gems.github.com"
+end
+
+desc "outputs a list of files suitable for use with the gemspec"
+task :list_files do
+  sh %q{ruby -rpp -e' pp `git ls-files`.split("\n").reject {|f| f =~ /git/} '}
+end
+
+
+CLEAN.add ["ext/*/*.bundle", "ext/*/*.so", "ext/*/*.o"]
+CLOBBER.add ["ext/*/Makefile", "ext/*/*.c"]
+
+namespace :ext do
+  desc "Installs the C extensions.  Usually requires root."
+  task :install => :build do
+    Dir.glob("ext/*/").each do |ext_dir|
+      Dir.chdir(ext_dir) {sh "make install"}
+    end
+  end
+
+  desc "Compiles the C extensions"
+  task :build do |t|
+    Dir.glob("ext/*/").each do |ext_dir|
+      cd(ext_dir) {sh "make"}
+    end
+  end
+  
+  desc "Generates Makefiles with extconf/mkmf"
+  task :makefiles
+  
+  FileList["ext/*/*.yy"].each do |flex_file|
+    flex_generated_c = flex_file.ext("yy.c")
+    file flex_generated_c => flex_file do |t|
+      sh "flex -i -s -o #{flex_generated_c} #{flex_file}"
+    end
+    task :build => flex_generated_c
+    file flex_file
+  end
+  
+  FileList["ext/*/extconf.rb"].each do |extconf_file|
+    extension_dir = extconf_file.sub("extconf.rb", '')
+    makefile = extension_dir + "Makefile"
+    file makefile => extconf_file do |t|
+      Dir.chdir(extension_dir) {ruby "./extconf.rb"}
+    end
+    file extconf_file
+    task :build => makefile
+  end
+  
+  desc "Compiles Teeth::Scanner scanner definitions into flex scanner definition"
+  task :scanners do
+    FileList["scanners/*"].each do |scanner|
+      ruby scanner
+    end
+  end
+  
+    
+end
+
+desc "deletes generated gem"
+task :clobber_gem do
+  FileList['pkg/*gem'].each { |gemfile| rm gemfile }
+end
+task :clobber => :clobber_gem

data/VERSION.yml

@@ -0,0 +1,5 @@
+--- 
+:minor: 2
+:build: 
+:patch: 0
+:major: 0

data/ext/scan_apache_logs/extconf.rb

@@ -0,0 +1,4 @@
+require "mkmf"
+$CFLAGS += " -Wall"
+have_library("uuid", "uuid_generate_time")
+create_makefile "teeth/scanners/scan_apache_logs", "./"

data/ext/scan_apache_logs/scan_apache_logs.yy

@@ -0,0 +1,274 @@
+%option prefix="apache_logs_yy"
+%option full
+%option never-interactive
+%option read
+%option nounput
+%option noyywrap noreject noyymore nodefault
+%{
+#include <ruby.h>
+#include <uuid/uuid.h>
+/* Data types */
+typedef struct {
+  char *key;
+  char *value;
+} KVPAIR;
+const KVPAIR EOF_KVPAIR = {"EOF", "EOF"};
+/* prototypes */
+char *strip_ends(char *);
+VALUE t_scan_apache_logs(VALUE);
+void new_uuid(char *str_ptr);
+void raise_error_for_string_too_long(VALUE string);
+void include_message_in_token_hash(VALUE message, VALUE token_hash);
+void add_uuid_to_token_hash(VALUE token_hash);
+void push_kv_pair_to_hash(KVPAIR key_value, VALUE token_hash);
+void concat_word_to_string(KVPAIR key_value, VALUE token_hash);
+/* Set the scanner name, and return type */
+#define YY_DECL KVPAIR scan_apache_logs(void)
+#define yyterminate() return EOF_KVPAIR
+/* Ruby 1.8 and 1.9 compatibility */
+#if !defined(RSTRING_LEN) 
+# define RSTRING_LEN(x) (RSTRING(x)->len) 
+# define RSTRING_PTR(x) (RSTRING(x)->ptr) 
+#endif 
+
+%}
+
+/* Definitions */
+
+CATCHALL (.|"\n")
+
+
+WS [[:space:]]
+
+NON_WS ([a-z]|[0-9]|[:punct:])
+
+IP4_OCT [0-9]|[0-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5]
+
+HOST ([a-z0-9][a-z0-9\-]*\.[a-z0-9][a-z0-9\-]*.[a-z0-9][a-z0-9\-\.]*[a-z]+(\:[0-9]+)?)|localhost
+
+WDAY mon|tue|wed|thu|fri|sat|sun
+
+MON jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec
+
+MONTH_NUM 0[1-9]|1[0-2]
+
+MDAY 3[0-1]|[1-2][0-9]|0[1-9]
+
+HOUR 2[0-3]|[0-1][0-9]
+
+MINSEC [0-5][0-9]|60
+
+YEAR [0-9][0-9][0-9][0-9]
+
+PLUSMINUS (\+|\-)
+
+TIMING [0-9]+\.[0-9]+
+
+REL_URL (\/|\\|\.)[a-z0-9\._\~\-\/\?&;#=\%\:\+\[\]\\]*
+
+PROTO (http:|https:)
+
+ERR_LVL (emerg|alert|crit|err|error|warn|warning|notice|info|debug)
+
+HTTP_VERS HTTP\/(1.0|1.1)
+
+HTTP_VERB (get|head|put|post|delete|trace|connect)
+
+HTTPCODE (100|101|20[0-6]|30[0-5]|307|40[0-9]|41[0-7]|50[0-5])
+
+BROWSER_STR \"(moz|msie|lynx|reconnoiter|pingdom)[^"]+\"
+
+
+%%
+  /* 
+    Actions 
+ */
+  
+
+{TIMING} {
+  KVPAIR timing = {"timing", yytext};
+  return timing;
+}
+
+{IP4_OCT}"."{IP4_OCT}"."{IP4_OCT}"."{IP4_OCT} {
+  KVPAIR ipv4_addr = {"ipv4_addr", yytext};
+  return ipv4_addr;
+}
+
+{WDAY}{WS}{MON}{WS}{MDAY}{WS}{HOUR}":"{MINSEC}":"{MINSEC}{WS}{YEAR} {
+  KVPAIR apache_err_datetime = {"apache_err_datetime", yytext};
+  return apache_err_datetime;
+}
+
+{MDAY}\/{MON}\/{YEAR}":"{HOUR}":"{MINSEC}":"{MINSEC}{WS}{PLUSMINUS}{YEAR} {
+  KVPAIR apache_access_datetime = {"apache_access_datetime", yytext};
+  return apache_access_datetime;
+}
+
+{HTTP_VERS} {
+  KVPAIR http_version = {"http_version", yytext};
+  return http_version;
+}
+
+{BROWSER_STR} {
+  KVPAIR browser_string = {"browser_string", strip_ends(yytext)};
+  return browser_string;
+}
+
+{PROTO}"\/\/"({HOST}|{IP4_OCT}"."{IP4_OCT}"."{IP4_OCT}"."{IP4_OCT})({REL_URL}|"\/")? {
+  KVPAIR absolute_url = {"absolute_url", yytext};
+  return absolute_url;
+}
+
+{HOST} {
+  KVPAIR host = {"host", yytext};
+  return host;
+}
+
+{REL_URL} {
+  KVPAIR relative_url = {"relative_url", yytext};
+  return relative_url;
+}
+
+{ERR_LVL} {
+  KVPAIR error_level = {"error_level", yytext};
+  return error_level;
+}
+
+{HTTPCODE} {
+  KVPAIR http_response = {"http_response", yytext};
+  return http_response;
+}
+
+{HTTP_VERB} {
+  KVPAIR http_method = {"http_method", yytext};
+  return http_method;
+}
+
+{NON_WS}{NON_WS}* {
+  KVPAIR strings = {"strings", yytext};
+  return strings;
+}
+
+{CATCHALL} /* ignore */
+%%
+
+char *strip_ends(char *string) {
+  string[yyleng-1] = '\0';
+  ++string;
+  return string;
+}
+
+void uuid_unparse_upper_sans_dash(const uuid_t uu, char *out)
+{
+        sprintf(out,
+                "%02X%02X%02X%02X"
+                "%02X%02X"
+                "%02X%02X"
+                "%02X%02X"
+                "%02X%02X%02X%02X%02X%02X",
+                uu[0], uu[1], uu[2], uu[3],
+                uu[4], uu[5],
+                uu[6], uu[7],
+                uu[8], uu[9],
+                uu[10], uu[11], uu[12], uu[13], uu[14], uu[15]);
+}
+
+void new_uuid(char *str_ptr){
+  uuid_t new_uuid;
+  uuid_generate_time(new_uuid);
+  uuid_unparse_upper_sans_dash(new_uuid, str_ptr);
+}
+
+void raise_error_for_string_too_long(VALUE string){
+  if( RSTRING_LEN(string) > 1000000){
+    rb_raise(rb_eArgError, "string too long for scan_apache_logs! max length is 1,000,000 chars"); 
+  }
+}
+
+/* Scans self, which is expected to be a single line from an Apache error or
+ * access log, and returns a Hash of the components of the log message.  The
+ * following parts of the log message are returned if they are present:
+ * IPv4 address, datetime, HTTP Version used, the browser string given by the
+ * client, any absolute or relative URLs, the error level, HTTP response code,
+ * HTTP Method (verb), and any other uncategorized strings present. */
+VALUE t_scan_apache_logs(VALUE self) {
+  KVPAIR kv_result;
+  int scan_complete = 0;
+  int building_words_to_string = 0;
+  VALUE token_hash = rb_hash_new();
+  
+  BEGIN(INITIAL);
+  
+  /* error out on absurdly large strings */
+  raise_error_for_string_too_long(self);
+  /* {:message => self()} */
+  include_message_in_token_hash(self, token_hash);
+  /* {:id => UUID} */
+  add_uuid_to_token_hash(token_hash);
+  yy_scan_string(RSTRING_PTR(self));
+  while (scan_complete == 0) {
+    kv_result = scan_apache_logs();
+    if (kv_result.key == "EOF"){
+      scan_complete = 1;
+    }
+    else if (kv_result.key == "strings"){
+      /* build a string until we get a non-word */
+      if (building_words_to_string == 0){
+        building_words_to_string = 1;
+        push_kv_pair_to_hash(kv_result, token_hash);
+      }
+      else{
+        concat_word_to_string(kv_result, token_hash);
+      }    
+    }    
+    else {
+      building_words_to_string = 0;
+      push_kv_pair_to_hash(kv_result, token_hash);
+    }
+  }
+  yy_delete_buffer(YY_CURRENT_BUFFER);
+  return rb_obj_dup(token_hash);
+}
+
+void add_uuid_to_token_hash(VALUE token_hash) {
+  char new_uuid_str[33];
+  new_uuid(new_uuid_str);
+  VALUE hsh_key_id = ID2SYM(rb_intern("id"));
+  VALUE hsh_val_id = rb_tainted_str_new2(new_uuid_str);
+  rb_hash_aset(token_hash, hsh_key_id, hsh_val_id);
+}
+
+void include_message_in_token_hash(VALUE message, VALUE token_hash) {
+  /* {:message => self()} */  
+  VALUE hsh_key_msg = ID2SYM(rb_intern("message"));
+  rb_hash_aset(token_hash, hsh_key_msg, message);
+}
+
+void concat_word_to_string(KVPAIR key_value, VALUE token_hash) {
+  char * space = " ";
+  VALUE hsh_key = ID2SYM(rb_intern(key_value.key));
+  VALUE hsh_value = rb_hash_aref(token_hash, hsh_key);
+  VALUE string = rb_ary_entry(hsh_value, -1);
+  rb_str_cat(string, space, 1);
+  rb_str_cat(string, key_value.value, yyleng);
+}
+
+void push_kv_pair_to_hash(KVPAIR key_value, VALUE token_hash) {
+  VALUE hsh_key = ID2SYM(rb_intern(key_value.key));
+  VALUE hsh_value = rb_hash_aref(token_hash, hsh_key);
+  VALUE ary_for_token_type = rb_ary_new();
+  switch (TYPE(hsh_value)) {
+    case T_NIL:
+      rb_ary_push(ary_for_token_type, rb_tainted_str_new2(key_value.value));
+      rb_hash_aset(token_hash, hsh_key, ary_for_token_type);
+      break;
+    case T_ARRAY:
+      rb_ary_push(hsh_value, rb_tainted_str_new2(key_value.value));
+      break;
+  }
+}
+
+void Init_scan_apache_logs() {
+  rb_define_method(rb_cString, "scan_apache_logs", t_scan_apache_logs, 0);
+}