teeth 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/LICENSE +11 -0
- data/README.rdoc +123 -0
- data/Rakefile +112 -0
- data/VERSION.yml +5 -0
- data/ext/scan_apache_logs/extconf.rb +4 -0
- data/ext/scan_apache_logs/scan_apache_logs.yy +274 -0
- data/ext/scan_apache_logs/scan_apache_logs.yy.c +9345 -0
- data/ext/scan_rails_logs/extconf.rb +4 -0
- data/ext/scan_rails_logs/scan_rails_logs.yy +378 -0
- data/ext/scan_rails_logs/scan_rails_logs.yy.c +11528 -0
- data/lib/teeth.rb +14 -0
- data/lib/teeth/rule_statement.rb +61 -0
- data/lib/teeth/scanner.rb +101 -0
- data/lib/teeth/scanner_definition.rb +117 -0
- data/lib/teeth/scanner_definitions/scan_apache_logs.rb +28 -0
- data/lib/teeth/scanner_definitions/scan_rails_logs.rb +70 -0
- data/lib/teeth/templates/tokenizer.yy.erb +168 -0
- data/spec/fixtures/rails_1x.log +59 -0
- data/spec/fixtures/rails_22.log +12 -0
- data/spec/fixtures/rails_22_cached.log +10 -0
- data/spec/fixtures/rails_unordered.log +24 -0
- data/spec/playground/scan_rails_logs.rb +56 -0
- data/spec/playground/show_apache_processing.rb +13 -0
- data/spec/spec.opts +1 -0
- data/spec/spec_helper.rb +17 -0
- data/spec/unit/rule_statement_spec.rb +60 -0
- data/spec/unit/scan_apache_spec.rb +110 -0
- data/spec/unit/scan_rails_logs_spec.rb +100 -0
- data/spec/unit/scaner_definition_spec.rb +65 -0
- data/spec/unit/scanner_spec.rb +108 -0
- data/teeth.gemspec +78 -0
- metadata +100 -0
@@ -0,0 +1,65 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/../spec_helper'
|
2
|
+
|
3
|
+
require File.dirname(__FILE__) + '/../spec_helper'
|
4
|
+
|
5
|
+
describe ScannerDefinition do
|
6
|
+
|
7
|
+
it "should generate text for flex definitions" do
|
8
|
+
defn = ScannerDefinition.new "IP4_OCTET", "[0-9]|[0-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5]"
|
9
|
+
defn.scanner_code.should == "IP4_OCTET [0-9]|[0-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5]"
|
10
|
+
end
|
11
|
+
|
12
|
+
it "should complain if given a :start_condition option and a regex" do
|
13
|
+
fail = lambda {ScannerDefinition.new "FAIL", "FAIL", :start_condition => :inclusive}
|
14
|
+
fail.should raise_error ScannerDefinitionArgumentError
|
15
|
+
end
|
16
|
+
|
17
|
+
it "should add %s to the beginning of the definition for option :start_condition => :inclusive" do
|
18
|
+
defn = ScannerDefinition.new "SPECIAL_STATE", :start_condition => :inclusive
|
19
|
+
defn.scanner_code.should == "%s SPECIAL_STATE"
|
20
|
+
end
|
21
|
+
|
22
|
+
end
|
23
|
+
|
24
|
+
describe ScannerDefinitionGroup do
|
25
|
+
before(:each) do
|
26
|
+
@defn_group = ScannerDefinitionGroup.new
|
27
|
+
end
|
28
|
+
|
29
|
+
it "should hold multiple definitions" do
|
30
|
+
@defn_group.add "IP4_OCTET", "[0-9]|[0-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5]"
|
31
|
+
@defn_group.add "WDAY", "mon|tue|wed|thu|fri|sat|sun"
|
32
|
+
@defn_group.should have(2).definitions
|
33
|
+
end
|
34
|
+
|
35
|
+
it "should use method missing magic for sugary definitions" do
|
36
|
+
@defn_group.IP4_OCTET "[0-9]|[0-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5]"
|
37
|
+
@defn_group.scanner_defns.first.scanner_code.should == "IP4_OCTET [0-9]|[0-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5]"
|
38
|
+
end
|
39
|
+
|
40
|
+
it "should reject duplicate definitions" do
|
41
|
+
@defn_group.add("WS", '[\000-\s]')
|
42
|
+
lambda { @defn_group.add("WS", '[\000-\s]') }.should raise_error DuplicateDefinitionError
|
43
|
+
end
|
44
|
+
|
45
|
+
it "should add a default set of definitions" do
|
46
|
+
@defn_group.defaults_for(:whitespace, :ip)
|
47
|
+
@defn_group.defn_names.should == ["WS", "NON_WS", "IP4_OCT", "HOST"]
|
48
|
+
end
|
49
|
+
|
50
|
+
it "should explode if requested to add default definitions that don't exist" do
|
51
|
+
lambda { @defn_group.defaults_for(:foobarbaz) }.should raise_error InvalidDefaultDefinitionName
|
52
|
+
end
|
53
|
+
|
54
|
+
it "should not error if conflicting user definitions exist when loading defaults" do
|
55
|
+
@defn_group.add "WS", '[\000-\s]'
|
56
|
+
lambda {@defn_group.load_default_definitions_for(:whitespace)}.should_not raise_error
|
57
|
+
end
|
58
|
+
|
59
|
+
it "should not override user definitions when loading defaults" do
|
60
|
+
@defn_group.add "WS", "a-telltale-sign"
|
61
|
+
@defn_group.load_default_definitions_for(:whitespace)
|
62
|
+
@defn_group.reject { |defn| defn.name != "WS" }.first.regex.should == "a-telltale-sign"
|
63
|
+
end
|
64
|
+
|
65
|
+
end
|
@@ -0,0 +1,108 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/../spec_helper'
|
2
|
+
require "fileutils"
|
3
|
+
include FileUtils
|
4
|
+
describe Scanner do
|
5
|
+
TEST_EXT_DIR = File.dirname(__FILE__) + "/test_ext_dir"
|
6
|
+
|
7
|
+
IPV4_ACTION_TEXT =
|
8
|
+
%q|{IP4_OCT}"."{IP4_OCT}"."{IP4_OCT}"."{IP4_OCT} {
|
9
|
+
KVPAIR ipv4_addr = {"ipv4_addr", yytext};
|
10
|
+
return ipv4_addr;
|
11
|
+
}|
|
12
|
+
|
13
|
+
def clean_test_scanner_dir
|
14
|
+
rm_rf TEST_EXT_DIR
|
15
|
+
end
|
16
|
+
|
17
|
+
before(:all) do
|
18
|
+
clean_test_scanner_dir
|
19
|
+
end
|
20
|
+
|
21
|
+
before(:each) do
|
22
|
+
@scanner = Scanner.new(:rspec_awesome)
|
23
|
+
end
|
24
|
+
|
25
|
+
after(:all) do
|
26
|
+
clean_test_scanner_dir
|
27
|
+
end
|
28
|
+
|
29
|
+
it "should generate the code for an extconf file" do
|
30
|
+
expected = %Q{require "mkmf"\n$CFLAGS += " -Wall"\nhave_library("uuid", "uuid_generate_time")\ncreate_makefile "teeth/scan_rspec_awesome\", "./"\n}
|
31
|
+
@scanner.extconf.should == expected
|
32
|
+
end
|
33
|
+
|
34
|
+
it "should create a new directory for the extension" do
|
35
|
+
scanner = Scanner.new(:rspec_awesomeness, TEST_EXT_DIR)
|
36
|
+
File.exist?(TEST_EXT_DIR).should be_true
|
37
|
+
end
|
38
|
+
|
39
|
+
it "should initialize with a name for for the tokenizer function and method" do
|
40
|
+
scanner = Scanner.new(:rails_dev_logs)
|
41
|
+
scanner.scanner_name.should == "scan_rails_dev_logs"
|
42
|
+
scanner.main_function_name.should == "t_scan_rails_dev_logs"
|
43
|
+
scanner.init_function_name.should == "Init_scan_rails_dev_logs"
|
44
|
+
scanner.function_prefix.should == "rails_dev_logs_yy"
|
45
|
+
scanner.entry_point.should == "scan_rails_dev_logs"
|
46
|
+
end
|
47
|
+
|
48
|
+
it "should format rdoc for the C function which corresponds to the ruby method" do
|
49
|
+
@scanner.rdoc = <<-RDOC
|
50
|
+
Premature optimization
|
51
|
+
is the root of
|
52
|
+
all evil.
|
53
|
+
RDOC
|
54
|
+
@scanner.rdoc.should == "/* Premature optimization\n * is the root of\n * all evil. */"
|
55
|
+
end
|
56
|
+
|
57
|
+
it "should accept a ``global'' option to include UUID generation or not"
|
58
|
+
|
59
|
+
it "should store scanner definitions" do
|
60
|
+
@scanner.define "IP4_OCTET", "[0-9]|[0-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5]"
|
61
|
+
@scanner.scanner_defns.should have(1).definition
|
62
|
+
end
|
63
|
+
|
64
|
+
it "should accept definitions in a block" do
|
65
|
+
@scanner.definitions do |d|
|
66
|
+
d.add "IP4_OCTET", "[0-9]|[0-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5]"
|
67
|
+
d.add "WDAY", "mon|tue|wed|thu|fri|sat|sun"
|
68
|
+
end
|
69
|
+
@scanner.scanner_defns.should have(2).definitions
|
70
|
+
end
|
71
|
+
|
72
|
+
it "should generate scanner rule statements, defaulting to returning a KVPAIR of rule name, yytext" do
|
73
|
+
@scanner.rule :ipv4_addr, '{IP4_OCT}"."{IP4_OCT}"."{IP4_OCT}"."{IP4_OCT}'
|
74
|
+
@scanner.scanner_rules.first.scanner_code.should == IPV4_ACTION_TEXT
|
75
|
+
end
|
76
|
+
|
77
|
+
it "should accept rule statments in a block" do
|
78
|
+
@scanner.rules do |rules|
|
79
|
+
rules.add :ipv4_addr, '{IP4_OCT}"."{IP4_OCT}"."{IP4_OCT}"."{IP4_OCT}'
|
80
|
+
end
|
81
|
+
#@scanner.scanner_rules.first.scanner_code.should == IPV4_ACTION_TEXT
|
82
|
+
@scanner.scanner_rules.should have(1).scanner_rule
|
83
|
+
end
|
84
|
+
|
85
|
+
it "should render a scanner scanner from the template" do
|
86
|
+
@scanner.load_default_definitions_for(:whitespace, :ip, :web)
|
87
|
+
@scanner.rules do |rule|
|
88
|
+
rule.ipv4_addr '{IP4_OCT}"."{IP4_OCT}"."{IP4_OCT}"."{IP4_OCT}'
|
89
|
+
rule.relative_url '{REL_URL}'
|
90
|
+
end
|
91
|
+
result = @scanner.generate
|
92
|
+
# real test is if scanner compiles and passes its own tests
|
93
|
+
result.should_not match(Regexp.new(Regexp.quote '<%='))
|
94
|
+
end
|
95
|
+
|
96
|
+
it "should write the scanner and extconf files in the ext directory" do
|
97
|
+
scanner = Scanner.new(:rspec_awesomeness, TEST_EXT_DIR)
|
98
|
+
scanner.load_default_definitions_for(:whitespace, :ip, :web)
|
99
|
+
scanner.rules do |rule|
|
100
|
+
rule.ipv4_addr '{IP4_OCT}"."{IP4_OCT}"."{IP4_OCT}"."{IP4_OCT}'
|
101
|
+
rule.relative_url '{REL_URL}'
|
102
|
+
end
|
103
|
+
scanner.write!
|
104
|
+
File.exist?(TEST_EXT_DIR + "/extconf.rb").should be_true
|
105
|
+
File.exist?(TEST_EXT_DIR + "/scan_rspec_awesomeness.yy").should be_true
|
106
|
+
end
|
107
|
+
|
108
|
+
end
|
data/teeth.gemspec
ADDED
@@ -0,0 +1,78 @@
|
|
1
|
+
# Generated by jeweler
|
2
|
+
# DO NOT EDIT THIS FILE DIRECTLY
|
3
|
+
# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
|
4
|
+
# -*- encoding: utf-8 -*-
|
5
|
+
|
6
|
+
Gem::Specification.new do |s|
|
7
|
+
s.name = %q{teeth}
|
8
|
+
s.version = "0.2.0"
|
9
|
+
|
10
|
+
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
|
+
s.authors = ["Daniel DeLeo"]
|
12
|
+
s.date = %q{2010-05-14}
|
13
|
+
s.description = %q{Fast log file parsing in Ruby}
|
14
|
+
s.email = %q{ddeleo@basecommander.net}
|
15
|
+
s.extensions = ["ext/scan_apache_logs/extconf.rb", "ext/scan_rails_logs/extconf.rb"]
|
16
|
+
s.extra_rdoc_files = [
|
17
|
+
"README.rdoc"
|
18
|
+
]
|
19
|
+
s.files = [
|
20
|
+
"LICENSE",
|
21
|
+
"README.rdoc",
|
22
|
+
"Rakefile",
|
23
|
+
"VERSION.yml",
|
24
|
+
"ext/scan_apache_logs/extconf.rb",
|
25
|
+
"ext/scan_apache_logs/scan_apache_logs.yy",
|
26
|
+
"ext/scan_apache_logs/scan_apache_logs.yy.c",
|
27
|
+
"ext/scan_rails_logs/extconf.rb",
|
28
|
+
"ext/scan_rails_logs/scan_rails_logs.yy",
|
29
|
+
"ext/scan_rails_logs/scan_rails_logs.yy.c",
|
30
|
+
"lib/teeth.rb",
|
31
|
+
"lib/teeth/rule_statement.rb",
|
32
|
+
"lib/teeth/scanner.rb",
|
33
|
+
"lib/teeth/scanner_definition.rb",
|
34
|
+
"lib/teeth/scanner_definitions/scan_apache_logs.rb",
|
35
|
+
"lib/teeth/scanner_definitions/scan_rails_logs.rb",
|
36
|
+
"lib/teeth/templates/tokenizer.yy.erb",
|
37
|
+
"spec/fixtures/rails_1x.log",
|
38
|
+
"spec/fixtures/rails_22.log",
|
39
|
+
"spec/fixtures/rails_22_cached.log",
|
40
|
+
"spec/fixtures/rails_unordered.log",
|
41
|
+
"spec/playground/scan_rails_logs.rb",
|
42
|
+
"spec/playground/show_apache_processing.rb",
|
43
|
+
"spec/spec.opts",
|
44
|
+
"spec/spec_helper.rb",
|
45
|
+
"spec/unit/rule_statement_spec.rb",
|
46
|
+
"spec/unit/scan_apache_spec.rb",
|
47
|
+
"spec/unit/scan_rails_logs_spec.rb",
|
48
|
+
"spec/unit/scaner_definition_spec.rb",
|
49
|
+
"spec/unit/scanner_spec.rb",
|
50
|
+
"teeth.gemspec"
|
51
|
+
]
|
52
|
+
s.homepage = %q{http://github.com/danielsdeleo/teeth}
|
53
|
+
s.rdoc_options = ["--charset=UTF-8"]
|
54
|
+
s.require_paths = [["lib"]]
|
55
|
+
s.rubygems_version = %q{1.3.6}
|
56
|
+
s.summary = %q{Fast log file parsing in Ruby}
|
57
|
+
s.test_files = [
|
58
|
+
"spec/playground/scan_rails_logs.rb",
|
59
|
+
"spec/playground/show_apache_processing.rb",
|
60
|
+
"spec/spec_helper.rb",
|
61
|
+
"spec/unit/rule_statement_spec.rb",
|
62
|
+
"spec/unit/scan_apache_spec.rb",
|
63
|
+
"spec/unit/scan_rails_logs_spec.rb",
|
64
|
+
"spec/unit/scaner_definition_spec.rb",
|
65
|
+
"spec/unit/scanner_spec.rb"
|
66
|
+
]
|
67
|
+
|
68
|
+
if s.respond_to? :specification_version then
|
69
|
+
current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
|
70
|
+
s.specification_version = 3
|
71
|
+
|
72
|
+
if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
|
73
|
+
else
|
74
|
+
end
|
75
|
+
else
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
metadata
ADDED
@@ -0,0 +1,100 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: teeth
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
prerelease: false
|
5
|
+
segments:
|
6
|
+
- 0
|
7
|
+
- 2
|
8
|
+
- 0
|
9
|
+
version: 0.2.0
|
10
|
+
platform: ruby
|
11
|
+
authors:
|
12
|
+
- Daniel DeLeo
|
13
|
+
autorequire:
|
14
|
+
bindir: bin
|
15
|
+
cert_chain: []
|
16
|
+
|
17
|
+
date: 2010-05-14 00:00:00 -07:00
|
18
|
+
default_executable:
|
19
|
+
dependencies: []
|
20
|
+
|
21
|
+
description: Fast log file parsing in Ruby
|
22
|
+
email: ddeleo@basecommander.net
|
23
|
+
executables: []
|
24
|
+
|
25
|
+
extensions:
|
26
|
+
- ext/scan_apache_logs/extconf.rb
|
27
|
+
- ext/scan_rails_logs/extconf.rb
|
28
|
+
extra_rdoc_files:
|
29
|
+
- README.rdoc
|
30
|
+
files:
|
31
|
+
- LICENSE
|
32
|
+
- README.rdoc
|
33
|
+
- Rakefile
|
34
|
+
- VERSION.yml
|
35
|
+
- ext/scan_apache_logs/extconf.rb
|
36
|
+
- ext/scan_apache_logs/scan_apache_logs.yy
|
37
|
+
- ext/scan_apache_logs/scan_apache_logs.yy.c
|
38
|
+
- ext/scan_rails_logs/extconf.rb
|
39
|
+
- ext/scan_rails_logs/scan_rails_logs.yy
|
40
|
+
- ext/scan_rails_logs/scan_rails_logs.yy.c
|
41
|
+
- lib/teeth.rb
|
42
|
+
- lib/teeth/rule_statement.rb
|
43
|
+
- lib/teeth/scanner.rb
|
44
|
+
- lib/teeth/scanner_definition.rb
|
45
|
+
- lib/teeth/scanner_definitions/scan_apache_logs.rb
|
46
|
+
- lib/teeth/scanner_definitions/scan_rails_logs.rb
|
47
|
+
- lib/teeth/templates/tokenizer.yy.erb
|
48
|
+
- spec/fixtures/rails_1x.log
|
49
|
+
- spec/fixtures/rails_22.log
|
50
|
+
- spec/fixtures/rails_22_cached.log
|
51
|
+
- spec/fixtures/rails_unordered.log
|
52
|
+
- spec/playground/scan_rails_logs.rb
|
53
|
+
- spec/playground/show_apache_processing.rb
|
54
|
+
- spec/spec.opts
|
55
|
+
- spec/spec_helper.rb
|
56
|
+
- spec/unit/rule_statement_spec.rb
|
57
|
+
- spec/unit/scan_apache_spec.rb
|
58
|
+
- spec/unit/scan_rails_logs_spec.rb
|
59
|
+
- spec/unit/scaner_definition_spec.rb
|
60
|
+
- spec/unit/scanner_spec.rb
|
61
|
+
- teeth.gemspec
|
62
|
+
has_rdoc: true
|
63
|
+
homepage: http://github.com/danielsdeleo/teeth
|
64
|
+
licenses: []
|
65
|
+
|
66
|
+
post_install_message:
|
67
|
+
rdoc_options:
|
68
|
+
- --charset=UTF-8
|
69
|
+
require_paths:
|
70
|
+
- - lib
|
71
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - ">="
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
segments:
|
76
|
+
- 0
|
77
|
+
version: "0"
|
78
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - ">="
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
segments:
|
83
|
+
- 0
|
84
|
+
version: "0"
|
85
|
+
requirements: []
|
86
|
+
|
87
|
+
rubyforge_project:
|
88
|
+
rubygems_version: 1.3.6
|
89
|
+
signing_key:
|
90
|
+
specification_version: 3
|
91
|
+
summary: Fast log file parsing in Ruby
|
92
|
+
test_files:
|
93
|
+
- spec/playground/scan_rails_logs.rb
|
94
|
+
- spec/playground/show_apache_processing.rb
|
95
|
+
- spec/spec_helper.rb
|
96
|
+
- spec/unit/rule_statement_spec.rb
|
97
|
+
- spec/unit/scan_apache_spec.rb
|
98
|
+
- spec/unit/scan_rails_logs_spec.rb
|
99
|
+
- spec/unit/scaner_definition_spec.rb
|
100
|
+
- spec/unit/scanner_spec.rb
|