danielsdeleo-teeth 0.0.2 → 0.1.0

data/doc/rdoc-style.css

@@ -0,0 +1,208 @@
+
+body {
+    font-family: Verdana,Arial,Helvetica,sans-serif;
+    font-size:   90%;
+    margin: 0;
+    margin-left: 40px;
+    padding: 0;
+    background: white;
+}
+
+h1,h2,h3,h4 { margin: 0; color: #efefef; background: transparent; }
+h1 { font-size: 150%; }
+h2,h3,h4 { margin-top: 1em; }
+
+a { background: #eef; color: #039; text-decoration: none; }
+a:hover { background: #039; color: #eef; }
+
+/* Override the base stylesheet's Anchor inside a table cell */
+td > a {
+  background: transparent;
+  color: #039;
+  text-decoration: none;
+}
+
+/* and inside a section title */
+.section-title > a {
+  background: transparent;
+  color: #eee;
+  text-decoration: none;
+}
+
+/* === Structural elements =================================== */
+
+div#index {
+    margin: 0;
+    margin-left: -40px;
+    padding: 0;
+    font-size: 90%;
+}
+
+
+div#index a {
+    margin-left: 0.7em;
+}
+
+div#index .section-bar {
+   margin-left: 0px;
+   padding-left: 0.7em;
+   background: #ccc;
+   font-size: small;
+}
+
+
+div#classHeader, div#fileHeader {
+    width: auto;
+    color: white;
+    padding: 0.5em 1.5em 0.5em 1.5em;
+    margin: 0;
+    margin-left: -40px;
+    border-bottom: 3px solid #006;
+}
+
+div#classHeader a, div#fileHeader a {
+    background: inherit;
+    color: white;
+}
+
+div#classHeader td, div#fileHeader td {
+    background: inherit;
+    color: white;
+}
+
+
+div#fileHeader {
+    background: #057;
+}
+
+div#classHeader {
+    background: #048;
+}
+
+
+.class-name-in-header {
+  font-size:  180%;
+  font-weight: bold;
+}
+
+
+div#bodyContent {
+    padding: 0 1.5em 0 1.5em;
+}
+
+div#description {
+    padding: 0.5em 1.5em;
+    background: #efefef;
+    border: 1px dotted #999;
+}
+
+div#description h1,h2,h3,h4,h5,h6 {
+    color: #125;;
+    background: transparent;
+}
+
+div#validator-badges {
+    text-align: center;
+}
+div#validator-badges img { border: 0; }
+
+div#copyright {
+    color: #333;
+    background: #efefef;
+    font: 0.75em sans-serif;
+    margin-top: 5em;
+    margin-bottom: 0;
+    padding: 0.5em 2em;
+}
+
+
+/* === Classes =================================== */
+
+table.header-table {
+    color: white;
+    font-size: small;
+}
+
+.type-note {
+    font-size: small;
+    color: #DEDEDE;
+}
+
+.xxsection-bar {
+    background: #eee;
+    color: #333;
+    padding: 3px;
+}
+
+.section-bar {
+   color: #333;
+   border-bottom: 1px solid #999;
+    margin-left: -20px;
+}
+
+
+.section-title {
+    background: #79a;
+    color: #eee;
+    padding: 3px;
+    margin-top: 2em;
+    margin-left: -30px;
+    border: 1px solid #999;
+}
+
+.top-aligned-row {  vertical-align: top }
+.bottom-aligned-row { vertical-align: bottom }
+
+/* --- Context section classes ----------------------- */
+
+.context-row { }
+.context-item-name { font-family: monospace; font-weight: bold; color: black; }
+.context-item-value { font-size: small; color: #448; }
+.context-item-desc { color: #333; padding-left: 2em; }
+
+/* --- Method classes -------------------------- */
+.method-detail {
+    background: #efefef;
+    padding: 0;
+    margin-top: 0.5em;
+    margin-bottom: 1em;
+    border: 1px dotted #ccc;
+}
+.method-heading {
+  color: black;
+  background: #ccc;
+  border-bottom: 1px solid #666;
+  padding: 0.2em 0.5em 0 0.5em;
+}
+.method-signature { color: black; background: inherit; }
+.method-name { font-weight: bold; }
+.method-args { font-style: italic; }
+.method-description { padding: 0 0.5em 0 0.5em; }
+
+/* --- Source code sections -------------------- */
+
+a.source-toggle { font-size: 90%; }
+div.method-source-code {
+    background: #262626;
+    color: #ffdead;
+    margin: 1em;
+    padding: 0.5em;
+    border: 1px dashed #999;
+    overflow: hidden;
+}
+
+div.method-source-code pre { color: #ffdead; overflow: hidden; }
+
+/* --- Ruby keyword styles --------------------- */
+
+.standalone-code { background: #221111; color: #ffdead; overflow: hidden; }
+
+.ruby-constant  { color: #7fffd4; background: transparent; }
+.ruby-keyword { color: #00ffff; background: transparent; }
+.ruby-ivar    { color: #eedd82; background: transparent; }
+.ruby-operator  { color: #00ffee; background: transparent; }
+.ruby-identifier { color: #ffdead; background: transparent; }
+.ruby-node    { color: #ffa07a; background: transparent; }
+.ruby-comment { color: #b22222; font-weight: bold; background: transparent; }
+.ruby-regexp  { color: #ffa07a; background: transparent; }
+.ruby-value   { color: #7fffd4; background: transparent; }

data/ext/scan_apache_logs/Makefile

@@ -0,0 +1,158 @@
+
+SHELL = /bin/sh
+
+#### Start of system configuration section. ####
+
+srcdir = ./
+topdir = /opt/local/lib/ruby/1.8/i686-darwin8.11.1
+hdrdir = $(topdir)
+VPATH = $(srcdir):$(topdir):$(hdrdir)
+exec_prefix = $(prefix)
+prefix = $(DESTDIR)/opt/local
+sharedstatedir = $(prefix)/com
+mandir = $(DESTDIR)/opt/local/share/man
+psdir = $(docdir)
+oldincludedir = $(DESTDIR)/usr/include
+localedir = $(datarootdir)/locale
+bindir = $(exec_prefix)/bin
+libexecdir = $(exec_prefix)/libexec
+sitedir = $(libdir)/ruby/site_ruby
+htmldir = $(docdir)
+vendorarchdir = $(vendorlibdir)/$(sitearch)
+includedir = $(prefix)/include
+infodir = $(datarootdir)/info
+vendorlibdir = $(vendordir)/$(ruby_version)
+sysconfdir = $(prefix)/etc
+libdir = $(exec_prefix)/lib
+sbindir = $(exec_prefix)/sbin
+rubylibdir = $(libdir)/ruby/$(ruby_version)
+docdir = $(datarootdir)/doc/$(PACKAGE)
+dvidir = $(docdir)
+vendordir = $(DESTDIR)/opt/local/lib/ruby/vendor_ruby
+datarootdir = $(prefix)/share
+pdfdir = $(docdir)
+archdir = $(rubylibdir)/$(arch)
+sitearchdir = $(sitelibdir)/$(sitearch)
+datadir = $(datarootdir)
+localstatedir = $(prefix)/var
+sitelibdir = $(sitedir)/$(ruby_version)
+
+CC = /usr/bin/gcc-4.0
+LIBRUBY = $(LIBRUBY_SO)
+LIBRUBY_A = lib$(RUBY_SO_NAME)-static.a
+LIBRUBYARG_SHARED = -l$(RUBY_SO_NAME)
+LIBRUBYARG_STATIC = -l$(RUBY_SO_NAME)-static
+
+RUBY_EXTCONF_H = 
+CFLAGS   =  -fno-common -O2  -fno-common -pipe -fno-common $(cflags) -Wall 
+INCFLAGS = -I. -I$(topdir) -I$(hdrdir) -I$(srcdir)
+DEFS     = 
+CPPFLAGS =  -I/opt/local/include -D_XOPEN_SOURCE -D_DARWIN_C_SOURCE $(DEFS) $(cppflags)
+CXXFLAGS = $(CFLAGS) 
+ldflags  = -L. -L/opt/local/lib
+dldflags = 
+archflag = 
+DLDFLAGS = $(ldflags) $(dldflags) $(archflag)
+LDSHARED = cc -dynamic -bundle -undefined suppress -flat_namespace
+AR = ar
+EXEEXT = 
+
+RUBY_INSTALL_NAME = ruby
+RUBY_SO_NAME = ruby
+arch = i686-darwin8.11.1
+sitearch = i686-darwin8.11.1
+vendorarch = i686-darwin8.11.1
+ruby_version = 1.8
+ruby = /opt/local/bin/ruby
+RUBY = $(ruby)
+RM = rm -f
+MAKEDIRS = mkdir -p
+INSTALL = /usr/bin/install
+INSTALL_PROG = $(INSTALL) -m 0755
+INSTALL_DATA = $(INSTALL) -m 644
+COPY = cp
+
+#### End of system configuration section. ####
+
+preload = 
+
+libpath = . $(libdir)
+LIBPATH =  -L. -L$(libdir)
+DEFFILE = 
+
+CLEANFILES = mkmf.log
+DISTCLEANFILES = 
+
+extout = 
+extout_prefix = 
+target_prefix = /teeth
+LOCAL_LIBS = 
+LIBS = $(LIBRUBYARG_SHARED)  -lpthread -ldl -lobjc  
+SRCS = scan_apache_logs.yy.c
+OBJS = scan_apache_logs.yy.o
+TARGET = scan_apache_logs
+DLLIB = $(TARGET).bundle
+EXTSTATIC = 
+STATIC_LIB = 
+
+BINDIR        = $(bindir)
+RUBYCOMMONDIR = $(sitedir)$(target_prefix)
+RUBYLIBDIR    = $(sitelibdir)$(target_prefix)
+RUBYARCHDIR   = $(sitearchdir)$(target_prefix)
+
+TARGET_SO     = $(DLLIB)
+CLEANLIBS     = $(TARGET).bundle $(TARGET).il? $(TARGET).tds $(TARGET).map
+CLEANOBJS     = *.o *.a *.s[ol] *.pdb *.exp *.bak
+
+all:		$(DLLIB)
+static:		$(STATIC_LIB)
+
+clean:
+		@-$(RM) $(CLEANLIBS) $(CLEANOBJS) $(CLEANFILES)
+
+distclean:	clean
+		@-$(RM) Makefile $(RUBY_EXTCONF_H) conftest.* mkmf.log
+		@-$(RM) core ruby$(EXEEXT) *~ $(DISTCLEANFILES)
+
+realclean:	distclean
+install: install-so install-rb
+
+install-so: $(RUBYARCHDIR)
+install-so: $(RUBYARCHDIR)/$(DLLIB)
+$(RUBYARCHDIR)/$(DLLIB): $(DLLIB)
+	$(INSTALL_PROG) $(DLLIB) $(RUBYARCHDIR)
+install-rb: pre-install-rb install-rb-default
+install-rb-default: pre-install-rb-default
+pre-install-rb: Makefile
+pre-install-rb-default: Makefile
+$(RUBYARCHDIR):
+	$(MAKEDIRS) $@
+
+site-install: site-install-so site-install-rb
+site-install-so: install-so
+site-install-rb: install-rb
+
+.SUFFIXES: .c .m .cc .cxx .cpp .C .o
+
+.cc.o:
+	$(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) -c $<
+
+.cxx.o:
+	$(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) -c $<
+
+.cpp.o:
+	$(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) -c $<
+
+.C.o:
+	$(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) -c $<
+
+.c.o:
+	$(CC) $(INCFLAGS) $(CPPFLAGS) $(CFLAGS) -c $<
+
+$(DLLIB): $(OBJS)
+	@-$(RM) $@
+	$(LDSHARED) -o $@ $(OBJS) $(LIBPATH) $(DLDFLAGS) $(LOCAL_LIBS) $(LIBS)
+
+
+
+$(OBJS): ruby.h defines.h

data/ext/scan_apache_logs/extconf.rb

@@ -0,0 +1,3 @@
+require "mkmf"
+$CFLAGS += " -Wall"
+create_makefile "teeth/scan_apache_logs", "./"

data/ext/scan_apache_logs/scan_apache_logs.yy

@@ -0,0 +1,267 @@
+%option prefix="apache_logs_yy"
+%option full
+%option never-interactive
+%option read
+%option nounput
+%option noyywrap noreject noyymore nodefault
+%{
+#include <ruby.h>
+#include <uuid/uuid.h>
+/* Data types */
+typedef struct {
+  char *key;
+  char *value;
+} KVPAIR;
+const KVPAIR EOF_KVPAIR = {"EOF", "EOF"};
+/* prototypes */
+char *strip_ends(char *);
+VALUE t_scan_apache_logs(VALUE);
+void new_uuid(char *str_ptr);
+void raise_error_for_string_too_long(VALUE string);
+void include_message_in_token_hash(VALUE message, VALUE token_hash);
+void add_uuid_to_token_hash(VALUE token_hash);
+void push_kv_pair_to_hash(KVPAIR key_value, VALUE token_hash);
+void concat_word_to_string(KVPAIR key_value, VALUE token_hash);
+/* Set the scanner name, and return type */
+#define YY_DECL KVPAIR scan_apache_logs(void)
+#define yyterminate() return EOF_KVPAIR
+/* Ruby 1.8 and 1.9 compatibility */
+#if !defined(RSTRING_LEN) 
+# define RSTRING_LEN(x) (RSTRING(x)->len) 
+# define RSTRING_PTR(x) (RSTRING(x)->ptr) 
+#endif 
+
+%}
+
+/* Definitions */
+
+CATCHALL (.|"\n")
+
+
+WS [[:space:]]
+
+NON_WS ([a-z]|[0-9]|[:punct:])
+
+IP4_OCT [0-9]|[0-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5]
+
+HOST [a-z0-9][a-z0-9\-]*\.[a-z0-9][a-z0-9\-]*.[a-z0-9][a-z0-9\-\.]*[a-z]+(\:[0-9]+)?
+
+WDAY mon|tue|wed|thu|fri|sat|sun
+
+MON jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec
+
+MONTH_NUM 0[1-9]|1[0-2]
+
+MDAY 3[0-1]|[1-2][0-9]|0[1-9]
+
+HOUR 2[0-3]|[0-1][0-9]
+
+MINSEC [0-5][0-9]|60
+
+YEAR [0-9][0-9][0-9][0-9]
+
+PLUSMINUS (\+|\-)
+
+REL_URL (\/|\\|\.)[a-z0-9\._\~\-\/\?&;#=\%\:\+\[\]\\]*
+
+PROTO (http:|https:)
+
+ERR_LVL (emerg|alert|crit|err|error|warn|warning|notice|info|debug)
+
+HTTP_VERS HTTP\/(1.0|1.1)
+
+HTTP_VERB (get|head|put|post|delete|trace|connect)
+
+HTTPCODE (100|101|20[0-6]|30[0-5]|307|40[0-9]|41[0-7]|50[0-5])
+
+BROWSER_STR \"(moz|msie|lynx).+\"
+
+
+%%
+  /* 
+    Actions 
+ */
+  
+
+{IP4_OCT}"."{IP4_OCT}"."{IP4_OCT}"."{IP4_OCT} {
+  KVPAIR ipv4_addr = {"ipv4_addr", yytext};
+  return ipv4_addr;
+}
+
+{WDAY}{WS}{MON}{WS}{MDAY}{WS}{HOUR}":"{MINSEC}":"{MINSEC}{WS}{YEAR} {
+  KVPAIR apache_err_datetime = {"apache_err_datetime", yytext};
+  return apache_err_datetime;
+}
+
+{MDAY}\/{MON}\/{YEAR}":"{HOUR}":"{MINSEC}":"{MINSEC}{WS}{PLUSMINUS}{YEAR} {
+  KVPAIR apache_access_datetime = {"apache_access_datetime", yytext};
+  return apache_access_datetime;
+}
+
+{HTTP_VERS} {
+  KVPAIR http_version = {"http_version", yytext};
+  return http_version;
+}
+
+{BROWSER_STR} {
+  KVPAIR browser_string = {"browser_string", strip_ends(yytext)};
+  return browser_string;
+}
+
+{PROTO}"\/\/"({HOST}|{IP4_OCT}"."{IP4_OCT}"."{IP4_OCT}"."{IP4_OCT})({REL_URL}|"\/")? {
+  KVPAIR absolute_url = {"absolute_url", yytext};
+  return absolute_url;
+}
+
+{HOST} {
+  KVPAIR host = {"host", yytext};
+  return host;
+}
+
+{REL_URL} {
+  KVPAIR relative_url = {"relative_url", yytext};
+  return relative_url;
+}
+
+{ERR_LVL} {
+  KVPAIR error_level = {"error_level", yytext};
+  return error_level;
+}
+
+{HTTPCODE} {
+  KVPAIR http_response = {"http_response", yytext};
+  return http_response;
+}
+
+{HTTP_VERB} {
+  KVPAIR http_method = {"http_method", yytext};
+  return http_method;
+}
+
+{NON_WS}{NON_WS}* {
+  KVPAIR strings = {"strings", yytext};
+  return strings;
+}
+
+{CATCHALL} /* ignore */
+%%
+
+char *strip_ends(char *string) {
+  string[yyleng-1] = '\0';
+  ++string;
+  return string;
+}
+
+void uuid_unparse_upper_sans_dash(const uuid_t uu, char *out)
+{
+        sprintf(out,
+                "%02X%02X%02X%02X"
+                "%02X%02X"
+                "%02X%02X"
+                "%02X%02X"
+                "%02X%02X%02X%02X%02X%02X",
+                uu[0], uu[1], uu[2], uu[3],
+                uu[4], uu[5],
+                uu[6], uu[7],
+                uu[8], uu[9],
+                uu[10], uu[11], uu[12], uu[13], uu[14], uu[15]);
+}
+
+void new_uuid(char *str_ptr){
+  uuid_t new_uuid;
+  uuid_generate_time(new_uuid);
+  uuid_unparse_upper_sans_dash(new_uuid, str_ptr);
+}
+
+void raise_error_for_string_too_long(VALUE string){
+  if( RSTRING_LEN(string) > 1000000){
+    rb_raise(rb_eArgError, "string too long for scan_apache_logs! max length is 1,000,000 chars"); 
+  }
+}
+
+/* Scans self, which is expected to be a single line from an Apache error or
+ * access log, and returns a Hash of the components of the log message.  The
+ * following parts of the log message are returned if they are present:
+ * IPv4 address, datetime, HTTP Version used, the browser string given by the
+ * client, any absolute or relative URLs, the error level, HTTP response code,
+ * HTTP Method (verb), and any other uncategorized strings present. */
+VALUE t_scan_apache_logs(VALUE self) {
+  KVPAIR kv_result;
+  int scan_complete = 0;
+  int building_words_to_string = 0;
+  VALUE token_hash = rb_hash_new();
+  
+  BEGIN(INITIAL);
+  
+  /* error out on absurdly large strings */
+  raise_error_for_string_too_long(self);
+  /* {:message => self()} */
+  include_message_in_token_hash(self, token_hash);
+  /* {:id => UUID} */
+  add_uuid_to_token_hash(token_hash);
+  yy_scan_string(RSTRING_PTR(self));
+  while (scan_complete == 0) {
+    kv_result = scan_apache_logs();
+    if (kv_result.key == "EOF"){
+      scan_complete = 1;
+    }
+    else if (kv_result.key == "strings"){
+      /* build a string until we get a non-word */
+      if (building_words_to_string == 0){
+        building_words_to_string = 1;
+        push_kv_pair_to_hash(kv_result, token_hash);
+      }
+      else{
+        concat_word_to_string(kv_result, token_hash);
+      }    
+    }    
+    else {
+      building_words_to_string = 0;
+      push_kv_pair_to_hash(kv_result, token_hash);
+    }
+  }
+  yy_delete_buffer(YY_CURRENT_BUFFER);
+  return rb_obj_dup(token_hash);
+}
+
+void add_uuid_to_token_hash(VALUE token_hash) {
+  char new_uuid_str[33];
+  new_uuid(new_uuid_str);
+  VALUE hsh_key_id = ID2SYM(rb_intern("id"));
+  VALUE hsh_val_id = rb_tainted_str_new2(new_uuid_str);
+  rb_hash_aset(token_hash, hsh_key_id, hsh_val_id);
+}
+
+void include_message_in_token_hash(VALUE message, VALUE token_hash) {
+  /* {:message => self()} */  
+  VALUE hsh_key_msg = ID2SYM(rb_intern("message"));
+  rb_hash_aset(token_hash, hsh_key_msg, message);
+}
+
+void concat_word_to_string(KVPAIR key_value, VALUE token_hash) {
+  char * space = " ";
+  VALUE hsh_key = ID2SYM(rb_intern(key_value.key));
+  VALUE hsh_value = rb_hash_aref(token_hash, hsh_key);
+  VALUE string = rb_ary_entry(hsh_value, -1);
+  rb_str_cat(string, space, 1);
+  rb_str_cat(string, key_value.value, yyleng);
+}
+
+void push_kv_pair_to_hash(KVPAIR key_value, VALUE token_hash) {
+  VALUE hsh_key = ID2SYM(rb_intern(key_value.key));
+  VALUE hsh_value = rb_hash_aref(token_hash, hsh_key);
+  VALUE ary_for_token_type = rb_ary_new();
+  switch (TYPE(hsh_value)) {
+    case T_NIL:
+      rb_ary_push(ary_for_token_type, rb_tainted_str_new2(key_value.value));
+      rb_hash_aset(token_hash, hsh_key, ary_for_token_type);
+      break;
+    case T_ARRAY:
+      rb_ary_push(hsh_value, rb_tainted_str_new2(key_value.value));
+      break;
+  }
+}
+
+void Init_scan_apache_logs() {
+  rb_define_method(rb_cString, "scan_apache_logs", t_scan_apache_logs, 0);
+}