teeth 0.2.0

data/spec/unit/scaner_definition_spec.rb

@@ -0,0 +1,65 @@
+require File.dirname(__FILE__) + '/../spec_helper'
+
+require File.dirname(__FILE__) + '/../spec_helper'
+
+describe ScannerDefinition do
+  
+  it "should generate text for flex definitions" do
+    defn = ScannerDefinition.new "IP4_OCTET", "[0-9]|[0-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5]"
+    defn.scanner_code.should == "IP4_OCTET [0-9]|[0-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5]"
+  end
+  
+  it "should complain if given a :start_condition option and a regex" do
+    fail = lambda {ScannerDefinition.new "FAIL", "FAIL", :start_condition => :inclusive}
+    fail.should raise_error ScannerDefinitionArgumentError
+  end
+  
+  it "should add %s to the beginning of the definition for option :start_condition => :inclusive" do
+    defn = ScannerDefinition.new "SPECIAL_STATE", :start_condition => :inclusive
+    defn.scanner_code.should == "%s SPECIAL_STATE"
+  end
+  
+end
+
+describe ScannerDefinitionGroup do
+  before(:each) do
+    @defn_group = ScannerDefinitionGroup.new
+  end
+  
+  it "should hold multiple definitions" do
+    @defn_group.add "IP4_OCTET", "[0-9]|[0-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5]"
+    @defn_group.add "WDAY", "mon|tue|wed|thu|fri|sat|sun"
+    @defn_group.should have(2).definitions
+  end
+  
+  it "should use method missing magic for sugary definitions" do
+    @defn_group.IP4_OCTET "[0-9]|[0-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5]"
+    @defn_group.scanner_defns.first.scanner_code.should == "IP4_OCTET [0-9]|[0-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5]"
+  end
+  
+  it "should reject duplicate definitions" do
+    @defn_group.add("WS",  '[\000-\s]')
+    lambda { @defn_group.add("WS",  '[\000-\s]') }.should raise_error DuplicateDefinitionError
+  end
+  
+  it "should add a default set of definitions" do
+    @defn_group.defaults_for(:whitespace, :ip)
+    @defn_group.defn_names.should == ["WS", "NON_WS", "IP4_OCT", "HOST"]
+  end
+  
+  it "should explode if requested to add default definitions that don't exist" do
+    lambda { @defn_group.defaults_for(:foobarbaz) }.should raise_error InvalidDefaultDefinitionName
+  end
+  
+  it "should not error if conflicting user definitions exist when loading defaults" do
+    @defn_group.add "WS", '[\000-\s]'
+    lambda {@defn_group.load_default_definitions_for(:whitespace)}.should_not raise_error
+  end
+  
+  it "should not override user definitions when loading defaults" do
+    @defn_group.add "WS", "a-telltale-sign"
+    @defn_group.load_default_definitions_for(:whitespace)
+    @defn_group.reject { |defn| defn.name != "WS" }.first.regex.should == "a-telltale-sign"
+  end
+  
+end

data/spec/unit/scanner_spec.rb

@@ -0,0 +1,108 @@
+require File.dirname(__FILE__) + '/../spec_helper'
+require "fileutils"
+include FileUtils
+describe Scanner do
+  TEST_EXT_DIR = File.dirname(__FILE__) + "/test_ext_dir"
+  
+IPV4_ACTION_TEXT = 
+%q|{IP4_OCT}"."{IP4_OCT}"."{IP4_OCT}"."{IP4_OCT} {
+  KVPAIR ipv4_addr = {"ipv4_addr", yytext};
+  return ipv4_addr;
+}| 
+  
+  def clean_test_scanner_dir
+    rm_rf TEST_EXT_DIR
+  end
+  
+  before(:all) do
+    clean_test_scanner_dir
+  end
+  
+  before(:each) do
+    @scanner = Scanner.new(:rspec_awesome)
+  end
+  
+  after(:all) do
+    clean_test_scanner_dir
+  end
+  
+  it "should generate the code for an extconf file" do
+    expected = %Q{require "mkmf"\n$CFLAGS += " -Wall"\nhave_library("uuid", "uuid_generate_time")\ncreate_makefile "teeth/scan_rspec_awesome\", "./"\n}
+    @scanner.extconf.should == expected
+  end
+  
+  it "should create a new directory for the extension" do
+    scanner = Scanner.new(:rspec_awesomeness, TEST_EXT_DIR)
+    File.exist?(TEST_EXT_DIR).should be_true
+  end
+  
+  it "should initialize with a name for for the tokenizer function and method" do
+    scanner = Scanner.new(:rails_dev_logs)
+    scanner.scanner_name.should == "scan_rails_dev_logs"
+    scanner.main_function_name.should == "t_scan_rails_dev_logs"
+    scanner.init_function_name.should == "Init_scan_rails_dev_logs"
+    scanner.function_prefix.should == "rails_dev_logs_yy"
+    scanner.entry_point.should == "scan_rails_dev_logs"
+  end
+  
+  it "should format rdoc for the C function which corresponds to the ruby method" do
+    @scanner.rdoc = <<-RDOC
+    Premature optimization
+    is the root of
+    all evil.
+    RDOC
+    @scanner.rdoc.should == "/* Premature optimization\n * is the root of\n * all evil. */"
+  end
+  
+  it "should accept a ``global'' option to include UUID generation or not"
+  
+  it "should store scanner definitions" do
+    @scanner.define "IP4_OCTET", "[0-9]|[0-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5]"
+    @scanner.scanner_defns.should have(1).definition
+  end
+  
+  it "should accept definitions in a block" do
+    @scanner.definitions do |d|
+      d.add "IP4_OCTET", "[0-9]|[0-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5]"
+      d.add "WDAY", "mon|tue|wed|thu|fri|sat|sun"
+    end
+    @scanner.scanner_defns.should have(2).definitions
+  end
+    
+  it "should generate scanner rule statements, defaulting to returning a KVPAIR of rule name, yytext" do
+    @scanner.rule :ipv4_addr, '{IP4_OCT}"."{IP4_OCT}"."{IP4_OCT}"."{IP4_OCT}'
+    @scanner.scanner_rules.first.scanner_code.should == IPV4_ACTION_TEXT
+  end
+  
+  it "should accept rule statments in a block" do
+    @scanner.rules do |rules|
+      rules.add  :ipv4_addr, '{IP4_OCT}"."{IP4_OCT}"."{IP4_OCT}"."{IP4_OCT}'
+    end
+    #@scanner.scanner_rules.first.scanner_code.should == IPV4_ACTION_TEXT
+    @scanner.scanner_rules.should have(1).scanner_rule
+  end
+    
+  it "should render a scanner scanner from the template" do
+    @scanner.load_default_definitions_for(:whitespace, :ip, :web)
+    @scanner.rules do |rule|
+      rule.ipv4_addr '{IP4_OCT}"."{IP4_OCT}"."{IP4_OCT}"."{IP4_OCT}'
+      rule.relative_url '{REL_URL}'
+    end
+    result = @scanner.generate
+    # real test is if scanner compiles and passes its own tests
+    result.should_not match(Regexp.new(Regexp.quote '<%='))
+  end
+  
+  it "should write the scanner and extconf files in the ext directory" do
+    scanner = Scanner.new(:rspec_awesomeness, TEST_EXT_DIR)
+    scanner.load_default_definitions_for(:whitespace, :ip, :web)
+    scanner.rules do |rule|
+      rule.ipv4_addr '{IP4_OCT}"."{IP4_OCT}"."{IP4_OCT}"."{IP4_OCT}'
+      rule.relative_url '{REL_URL}'
+    end
+    scanner.write!
+    File.exist?(TEST_EXT_DIR + "/extconf.rb").should be_true
+    File.exist?(TEST_EXT_DIR + "/scan_rspec_awesomeness.yy").should be_true
+  end
+  
+end

data/teeth.gemspec

@@ -0,0 +1,78 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{teeth}
+  s.version = "0.2.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Daniel DeLeo"]
+  s.date = %q{2010-05-14}
+  s.description = %q{Fast log file parsing in Ruby}
+  s.email = %q{ddeleo@basecommander.net}
+  s.extensions = ["ext/scan_apache_logs/extconf.rb", "ext/scan_rails_logs/extconf.rb"]
+  s.extra_rdoc_files = [
+    "README.rdoc"
+  ]
+  s.files = [
+    "LICENSE",
+     "README.rdoc",
+     "Rakefile",
+     "VERSION.yml",
+     "ext/scan_apache_logs/extconf.rb",
+     "ext/scan_apache_logs/scan_apache_logs.yy",
+     "ext/scan_apache_logs/scan_apache_logs.yy.c",
+     "ext/scan_rails_logs/extconf.rb",
+     "ext/scan_rails_logs/scan_rails_logs.yy",
+     "ext/scan_rails_logs/scan_rails_logs.yy.c",
+     "lib/teeth.rb",
+     "lib/teeth/rule_statement.rb",
+     "lib/teeth/scanner.rb",
+     "lib/teeth/scanner_definition.rb",
+     "lib/teeth/scanner_definitions/scan_apache_logs.rb",
+     "lib/teeth/scanner_definitions/scan_rails_logs.rb",
+     "lib/teeth/templates/tokenizer.yy.erb",
+     "spec/fixtures/rails_1x.log",
+     "spec/fixtures/rails_22.log",
+     "spec/fixtures/rails_22_cached.log",
+     "spec/fixtures/rails_unordered.log",
+     "spec/playground/scan_rails_logs.rb",
+     "spec/playground/show_apache_processing.rb",
+     "spec/spec.opts",
+     "spec/spec_helper.rb",
+     "spec/unit/rule_statement_spec.rb",
+     "spec/unit/scan_apache_spec.rb",
+     "spec/unit/scan_rails_logs_spec.rb",
+     "spec/unit/scaner_definition_spec.rb",
+     "spec/unit/scanner_spec.rb",
+     "teeth.gemspec"
+  ]
+  s.homepage = %q{http://github.com/danielsdeleo/teeth}
+  s.rdoc_options = ["--charset=UTF-8"]
+  s.require_paths = [["lib"]]
+  s.rubygems_version = %q{1.3.6}
+  s.summary = %q{Fast log file parsing in Ruby}
+  s.test_files = [
+    "spec/playground/scan_rails_logs.rb",
+     "spec/playground/show_apache_processing.rb",
+     "spec/spec_helper.rb",
+     "spec/unit/rule_statement_spec.rb",
+     "spec/unit/scan_apache_spec.rb",
+     "spec/unit/scan_rails_logs_spec.rb",
+     "spec/unit/scaner_definition_spec.rb",
+     "spec/unit/scanner_spec.rb"
+  ]
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+    else
+    end
+  else
+  end
+end
+

metadata

@@ -0,0 +1,100 @@
+--- !ruby/object:Gem::Specification 
+name: teeth
+version: !ruby/object:Gem::Version 
+  prerelease: false
+  segments: 
+  - 0
+  - 2
+  - 0
+  version: 0.2.0
+platform: ruby
+authors: 
+- Daniel DeLeo
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-05-14 00:00:00 -07:00
+default_executable: 
+dependencies: []
+
+description: Fast log file parsing in Ruby
+email: ddeleo@basecommander.net
+executables: []
+
+extensions: 
+- ext/scan_apache_logs/extconf.rb
+- ext/scan_rails_logs/extconf.rb
+extra_rdoc_files: 
+- README.rdoc
+files: 
+- LICENSE
+- README.rdoc
+- Rakefile
+- VERSION.yml
+- ext/scan_apache_logs/extconf.rb
+- ext/scan_apache_logs/scan_apache_logs.yy
+- ext/scan_apache_logs/scan_apache_logs.yy.c
+- ext/scan_rails_logs/extconf.rb
+- ext/scan_rails_logs/scan_rails_logs.yy
+- ext/scan_rails_logs/scan_rails_logs.yy.c
+- lib/teeth.rb
+- lib/teeth/rule_statement.rb
+- lib/teeth/scanner.rb
+- lib/teeth/scanner_definition.rb
+- lib/teeth/scanner_definitions/scan_apache_logs.rb
+- lib/teeth/scanner_definitions/scan_rails_logs.rb
+- lib/teeth/templates/tokenizer.yy.erb
+- spec/fixtures/rails_1x.log
+- spec/fixtures/rails_22.log
+- spec/fixtures/rails_22_cached.log
+- spec/fixtures/rails_unordered.log
+- spec/playground/scan_rails_logs.rb
+- spec/playground/show_apache_processing.rb
+- spec/spec.opts
+- spec/spec_helper.rb
+- spec/unit/rule_statement_spec.rb
+- spec/unit/scan_apache_spec.rb
+- spec/unit/scan_rails_logs_spec.rb
+- spec/unit/scaner_definition_spec.rb
+- spec/unit/scanner_spec.rb
+- teeth.gemspec
+has_rdoc: true
+homepage: http://github.com/danielsdeleo/teeth
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- - lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.6
+signing_key: 
+specification_version: 3
+summary: Fast log file parsing in Ruby
+test_files: 
+- spec/playground/scan_rails_logs.rb
+- spec/playground/show_apache_processing.rb
+- spec/spec_helper.rb
+- spec/unit/rule_statement_spec.rb
+- spec/unit/scan_apache_spec.rb
+- spec/unit/scan_rails_logs_spec.rb
+- spec/unit/scaner_definition_spec.rb
+- spec/unit/scanner_spec.rb