teeth 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/LICENSE +11 -0
- data/README.rdoc +123 -0
- data/Rakefile +112 -0
- data/VERSION.yml +5 -0
- data/ext/scan_apache_logs/extconf.rb +4 -0
- data/ext/scan_apache_logs/scan_apache_logs.yy +274 -0
- data/ext/scan_apache_logs/scan_apache_logs.yy.c +9345 -0
- data/ext/scan_rails_logs/extconf.rb +4 -0
- data/ext/scan_rails_logs/scan_rails_logs.yy +378 -0
- data/ext/scan_rails_logs/scan_rails_logs.yy.c +11528 -0
- data/lib/teeth.rb +14 -0
- data/lib/teeth/rule_statement.rb +61 -0
- data/lib/teeth/scanner.rb +101 -0
- data/lib/teeth/scanner_definition.rb +117 -0
- data/lib/teeth/scanner_definitions/scan_apache_logs.rb +28 -0
- data/lib/teeth/scanner_definitions/scan_rails_logs.rb +70 -0
- data/lib/teeth/templates/tokenizer.yy.erb +168 -0
- data/spec/fixtures/rails_1x.log +59 -0
- data/spec/fixtures/rails_22.log +12 -0
- data/spec/fixtures/rails_22_cached.log +10 -0
- data/spec/fixtures/rails_unordered.log +24 -0
- data/spec/playground/scan_rails_logs.rb +56 -0
- data/spec/playground/show_apache_processing.rb +13 -0
- data/spec/spec.opts +1 -0
- data/spec/spec_helper.rb +17 -0
- data/spec/unit/rule_statement_spec.rb +60 -0
- data/spec/unit/scan_apache_spec.rb +110 -0
- data/spec/unit/scan_rails_logs_spec.rb +100 -0
- data/spec/unit/scaner_definition_spec.rb +65 -0
- data/spec/unit/scanner_spec.rb +108 -0
- data/teeth.gemspec +78 -0
- metadata +100 -0
@@ -0,0 +1,59 @@
|
|
1
|
+
Processing DashboardController#index (for 1.1.1.1 at 2008-08-14 21:16:25) [GET]
|
2
|
+
Session ID: BAh7CToMcmVmZXJlciIbL3ByaXNjaWxsYS9wZW9wbGUvMjM1MCIKZmxhc2hJ
|
3
|
+
QzonQWN0aW9uQ29udHJvbGxlcjo6Rmxhc2g6OkZsYXNoSGFzaHsABjoKQHVz
|
4
|
+
ZWR7ADoNbGFuZ3VhZ2VvOhNMb2NhbGU6Ok9iamVjdBI6CUB3aW4wOg1AY291
|
5
|
+
bnRyeSIHTkw6CkBoYXNoaf3L2Js6DkBvcmlnX3N0ciIKbmwtTkw6DUBpc28z
|
6
|
+
MDY2MDoNQGNoYXJzZXQiClVURi04Og5AbGFuZ3VhZ2UiB25sOg5AbW9kaWZp
|
7
|
+
ZXIwOgtAcG9zaXgiCm5sX05MOg1AZ2VuZXJhbCIKbmxfTkw6DUB2YXJpYW50
|
8
|
+
MDoOQGZhbGxiYWNrMDoMQHNjcmlwdDA6DnBlcnNvbl9pZGkCMgc=--7918aed37151c13360cd370c37b541f136146fbd
|
9
|
+
Parameters: {"action"=>"index", "controller"=>"dashboard"}
|
10
|
+
Set language to: nl_NL
|
11
|
+
Rendering template within layouts/priscilla
|
12
|
+
Rendering dashboard/index
|
13
|
+
Completed in 0.22699 (4 reqs/sec) | Rendering: 0.02667 (11%) | DB: 0.03057 (13%) | 200 OK [https://www.example.com/]
|
14
|
+
|
15
|
+
|
16
|
+
Processing PeopleController#index (for 1.1.1.1 at 2008-08-14 21:16:30) [GET]
|
17
|
+
Session ID: BAh7CSIKZmxhc2hJQzonQWN0aW9uQ29udHJvbGxlcjo6Rmxhc2g6OkZsYXNo
|
18
|
+
SGFzaHsABjoKQHVzZWR7ADoMcmVmZXJlciIQL3ByaXNjaWxsYS86DnBlcnNv
|
19
|
+
bl9pZGkCMgc6DWxhbmd1YWdlbzoTTG9jYWxlOjpPYmplY3QSOg1AY291bnRy
|
20
|
+
eSIHTkw6CUB3aW4wOg5Ab3JpZ19zdHIiCm5sLU5MOgpAaGFzaGn9y9ibOg5A
|
21
|
+
bGFuZ3VhZ2UiB25sOg1AY2hhcnNldCIKVVRGLTg6DUBpc28zMDY2MDoOQG1v
|
22
|
+
ZGlmaWVyMDoLQHBvc2l4IgpubF9OTDoNQHZhcmlhbnQwOg1AZ2VuZXJhbCIK
|
23
|
+
bmxfTkw6DEBzY3JpcHQwOg5AZmFsbGJhY2sw--48cbe3788ef27f6005f8e999610a42af6e90ffb3
|
24
|
+
Parameters: {"commit"=>"Zoek", "action"=>"index", "q"=>"gaby", "controller"=>"people"}
|
25
|
+
Set language to: nl_NL
|
26
|
+
Redirected to https://www.example.com/people/2545
|
27
|
+
Completed in 0.04759 (21 reqs/sec) | DB: 0.03719 (78%) | 302 Found [https://www.example.com/people?q=gaby&commit=Zoek]
|
28
|
+
|
29
|
+
|
30
|
+
Processing PeopleController#show (for 1.1.1.1 at 2008-08-14 21:16:30) [GET]
|
31
|
+
Session ID: BAh7CToMcmVmZXJlciIpL3ByaXNjaWxsYS9wZW9wbGU/cT1nYWJ5JmNvbW1p
|
32
|
+
dD1ab2VrIgpmbGFzaElDOidBY3Rpb25Db250cm9sbGVyOjpGbGFzaDo6Rmxh
|
33
|
+
c2hIYXNoewAGOgpAdXNlZHsAOg1sYW5ndWFnZW86E0xvY2FsZTo6T2JqZWN0
|
34
|
+
EjoJQHdpbjA6DUBjb3VudHJ5IgdOTDoKQGhhc2hp/cvYmzoOQG9yaWdfc3Ry
|
35
|
+
IgpubC1OTDoNQGlzbzMwNjYwOg1AY2hhcnNldCIKVVRGLTg6DkBsYW5ndWFn
|
36
|
+
ZSIHbmw6DkBtb2RpZmllcjA6C0Bwb3NpeCIKbmxfTkw6DUBnZW5lcmFsIgpu
|
37
|
+
bF9OTDoNQHZhcmlhbnQwOg5AZmFsbGJhY2swOgxAc2NyaXB0MDoOcGVyc29u
|
38
|
+
X2lkaQIyBw==--3ad1948559448522a49d289a2a89dc7ccbe8847a
|
39
|
+
Parameters: {"action"=>"show", "id"=>"2545", "controller"=>"people"}
|
40
|
+
Set language to: nl_NL
|
41
|
+
Rendering template within layouts/priscilla
|
42
|
+
Rendering people/show
|
43
|
+
person: John Doe, study_year: 2008/2009
|
44
|
+
Completed in 0.29077 (3 reqs/sec) | Rendering: 0.24187 (83%) | DB: 0.04030 (13%) | 200 OK [https://www.example.com/people/2545]
|
45
|
+
|
46
|
+
|
47
|
+
Processing PeopleController#picture (for 1.1.1.1 at 2008-08-14 21:16:35) [GET]
|
48
|
+
Session ID: BAh7CSIKZmxhc2hJQzonQWN0aW9uQ29udHJvbGxlcjo6Rmxhc2g6OkZsYXNo
|
49
|
+
SGFzaHsABjoKQHVzZWR7ADoMcmVmZXJlciIbL3ByaXNjaWxsYS9wZW9wbGUv
|
50
|
+
MjU0NToOcGVyc29uX2lkaQIyBzoNbGFuZ3VhZ2VvOhNMb2NhbGU6Ok9iamVj
|
51
|
+
dBI6DUBjb3VudHJ5IgdOTDoJQHdpbjA6DkBvcmlnX3N0ciIKbmwtTkw6CkBo
|
52
|
+
YXNoaf3L2Js6DkBsYW5ndWFnZSIHbmw6DUBjaGFyc2V0IgpVVEYtODoNQGlz
|
53
|
+
bzMwNjYwOg5AbW9kaWZpZXIwOgtAcG9zaXgiCm5sX05MOg1AdmFyaWFudDA6
|
54
|
+
DUBnZW5lcmFsIgpubF9OTDoMQHNjcmlwdDA6DkBmYWxsYmFjazA=--797a33f280a482647111397d138d0918f2658167
|
55
|
+
Parameters: {"action"=>"picture", "id"=>"2545", "controller"=>"people"}
|
56
|
+
Set language to: nl_NL
|
57
|
+
Rendering template within layouts/priscilla
|
58
|
+
Rendering people/picture
|
59
|
+
Completed in 0.05383 (18 reqs/sec) | Rendering: 0.04622 (85%) | DB: 0.00206 (3%) | 200 OK [https://www.example.com/people/2545/picture]
|
@@ -0,0 +1,12 @@
|
|
1
|
+
Processing PageController#demo (for 127.0.0.1 at 2008-12-10 16:28:09) [GET]
|
2
|
+
Parameters: {"action"=>"demo", "controller"=>"page"}
|
3
|
+
Logging in from session data...
|
4
|
+
Logged in as test@example.com
|
5
|
+
Using locale: en-US, http-accept: ["en-US"], session: , det browser: en-US, det domain:
|
6
|
+
Rendering template within layouts/demo
|
7
|
+
Rendering page/demo
|
8
|
+
Rendered shared/_analytics (0.2ms)
|
9
|
+
Rendered layouts/_actions (0.6ms)
|
10
|
+
Rendered layouts/_menu (2.2ms)
|
11
|
+
Rendered layouts/_tabbar (0.5ms)
|
12
|
+
Completed in 614ms (View: 120, DB: 31) | 200 OK [http://www.example.coml/demo]
|
@@ -0,0 +1,10 @@
|
|
1
|
+
Processing CachedController#cached (for 1.1.1.1 at 2008-12-24 07:36:53) [GET]
|
2
|
+
Parameters: {"action"=>"cached", "controller"=>"cached"}
|
3
|
+
Logging in from session data...
|
4
|
+
Logging in using cookie...
|
5
|
+
Using locale: zh-Hans, http-accept: ["zh-CN", "zh-HK", "zh-TW", "en-US"], session: , det browser: zh-Hans, det domain: , user pref locale:
|
6
|
+
Referer: http://www.example.com/referer
|
7
|
+
Cached fragment hit: views/zh-Hans-www-cached-cached-all-CN--- (0.0ms)
|
8
|
+
Filter chain halted as [#<ActionController::Caching::Actions::ActionCacheFilter:0x2a999ad620 @check=nil, @options={:store_options=>{}, :layout=>nil, :cache_path=>#<Proc:0x0000002a999b8890@/app/controllers/cached_controller.rb:8>}>] rendered_or_redirected.
|
9
|
+
Filter chain halted as [#<ActionController::Filters::AroundFilter:0x2a999ad120 @identifier=nil, @kind=:filter, @options={:only=>#<Set: {"cached"}>, :if=>:not_logged_in?, :unless=>nil}, @method=#<ActionController::Caching::Actions::ActionCacheFilter:0x2a999ad620 @check=nil, @options={:store_options=>{}, :layout=>nil, :cache_path=>#<Proc:0x0000002a999b8890@/app/controllers/cached_controller.rb:8>}>>] did_not_yield.
|
10
|
+
Completed in 3ms (View: 0, DB: 0) | 200 OK [http://www.example.com/cached/cached/]
|
@@ -0,0 +1,24 @@
|
|
1
|
+
Processing AccountController#dashboard (for 1.1.1.1 at 2008-12-24 07:36:49) [GET]
|
2
|
+
Parameters: {"action"=>"dashboard", "controller"=>"account", "first_use"=>"true"}
|
3
|
+
Logging in from session data...
|
4
|
+
|
5
|
+
|
6
|
+
Processing ProjectsController#new (for 1.1.1.1 at 2008-12-24 07:36:49) [GET]
|
7
|
+
Parameters: {"action"=>"new", "controller"=>"projects"}
|
8
|
+
Rendering template within layouts/default
|
9
|
+
Rendering account/dashboard
|
10
|
+
Logging in from session data...
|
11
|
+
Logging in using cookie...
|
12
|
+
Using locale: en-US, http-accept: [], session: , det browser: , det domain: , user pref locale:
|
13
|
+
Rendered shared/_maintenance (0.6ms)
|
14
|
+
Rendering template within layouts/templates/general_default/index.html.erb
|
15
|
+
Rendered projects/_recent_designs (4.3ms)
|
16
|
+
Rendered projects/_project (13.6ms)
|
17
|
+
Rendered projects/_projects (18.7ms)
|
18
|
+
Rendered layouts/_menu (1.4ms)
|
19
|
+
Completed in 36ms (View: 30, DB: 3) | 200 OK [http://www.example.com/projects/new]
|
20
|
+
Rendered layouts/_actions (0.3ms)
|
21
|
+
Rendered layouts/_menu (1.6ms)
|
22
|
+
Rendered layouts/_tabbar (1.9ms)
|
23
|
+
Rendered layouts/_footer (3.2ms)
|
24
|
+
Completed in 50ms (View: 41, DB: 4) | 200 OK [http://www.example.com/dashboard?first_use=true]
|
@@ -0,0 +1,56 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
# encoding: utf-8
|
3
|
+
require "uri"
|
4
|
+
require File.dirname(__FILE__) + "/../spec_helper"
|
5
|
+
|
6
|
+
# A very basic log stats program to show the use of Teeth. Gives the top 20
|
7
|
+
# URLs and the top 10 Errors. Supports Ruby 1.9, though it seems a bit
|
8
|
+
# slower than 1.8 (?)
|
9
|
+
|
10
|
+
results = Hash.new(0)
|
11
|
+
error_results = Hash.new(0)
|
12
|
+
if RUBY_VERSION >= "1.9.0"
|
13
|
+
filename = ARGV[0].dup.force_encoding("ASCII-8BIT") # 1.9 is weird on Mac
|
14
|
+
else
|
15
|
+
filename = ARGV[0]
|
16
|
+
end
|
17
|
+
|
18
|
+
File.open(filename, "r") do |f|
|
19
|
+
n_processed = n_completed = n_errors = 0
|
20
|
+
while line = f.gets
|
21
|
+
tokens = line.scan_rails_logs
|
22
|
+
if tokens[:teaser] && tokens[:teaser].first == "Processing"
|
23
|
+
n_processed += 1
|
24
|
+
results[tokens[:controller_action].first] += 1
|
25
|
+
end
|
26
|
+
if tokens[:error]
|
27
|
+
n_errors += 1
|
28
|
+
error_results[tokens[:error].first] += 1
|
29
|
+
end
|
30
|
+
if tokens[:url] && tokens[:teaser].first == "Completed in"
|
31
|
+
n_completed +=1
|
32
|
+
end
|
33
|
+
end
|
34
|
+
puts "=" * 80
|
35
|
+
puts "Totals"
|
36
|
+
puts "#{n_processed} requests processed"
|
37
|
+
puts "#{n_completed} requests completed"
|
38
|
+
results_ary = results.map { |url, hits| [url, hits] }.sort { |a, b| b.last <=> a.last }
|
39
|
+
puts "=" * 80
|
40
|
+
puts "\nTop 20 URLs"
|
41
|
+
puts "=" * 80
|
42
|
+
puts " URL".ljust(40) + " | " + "Hits"
|
43
|
+
puts "-" * 80
|
44
|
+
results_ary[0, 20].each do |url_hits|
|
45
|
+
puts url_hits.first.ljust(40) + " | " + url_hits.last.to_s
|
46
|
+
end
|
47
|
+
errors_ary = error_results.map { |error, hits| [error, hits] }.sort { |a, b| b.last <=> a.last }
|
48
|
+
puts "=" * 80
|
49
|
+
puts "\nTop 10 Errors"
|
50
|
+
puts "=" * 80
|
51
|
+
puts " Error".ljust(40) + " | " + "Hits"
|
52
|
+
puts "-" * 80
|
53
|
+
errors_ary[0, 10].each do |error_hits|
|
54
|
+
puts error_hits.first.ljust(40) + " | " + error_hits.last.to_s
|
55
|
+
end
|
56
|
+
end
|
@@ -0,0 +1,13 @@
|
|
1
|
+
require File.dirname(__FILE__) + "/../spec_helper" # loads libs
|
2
|
+
|
3
|
+
error_line = %q{[Sun Nov 30 14:23:45 2008] [error] [client 10.0.1.197] Invalid URI in request GET .\\.\\.\\.\\.\\.\\.\\.\\.\\.\\/winnt/win.ini HTTP/1.1}
|
4
|
+
access_line = %q{127.81.248.53 - - [14/Jan/2009:11:49:43 -0500] "GET /reports/REPORT7_1ART02.pdf HTTP/1.1" 206 255404}
|
5
|
+
|
6
|
+
mangled_error_line = error_line + "more words"
|
7
|
+
|
8
|
+
puts "Processed Error Message:"
|
9
|
+
puts error_line.tokenize_apache_logs.inspect
|
10
|
+
puts "Error Message with extras:"
|
11
|
+
puts mangled_error_line.tokenize_apache_logs.inspect
|
12
|
+
puts "Processed Access Message"
|
13
|
+
puts access_line.tokenize_apache_logs.inspect
|
data/spec/spec.opts
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
-c
|
data/spec/spec_helper.rb
ADDED
@@ -0,0 +1,17 @@
|
|
1
|
+
$:.unshift File.expand_path(File.dirname(__FILE__) + '/../lib')
|
2
|
+
$:.unshift File.expand_path(File.dirname(__FILE__) + '/../ext')
|
3
|
+
|
4
|
+
require "teeth"
|
5
|
+
require 'scan_apache_logs/scan_apache_logs'
|
6
|
+
require 'scan_rails_logs/scan_rails_logs'
|
7
|
+
|
8
|
+
|
9
|
+
def be_greater_than(expected)
|
10
|
+
simple_matcher("be greater than #{expected.to_s}") do |given, matcher|
|
11
|
+
matcher.failure_message = "expected #{given.to_s} to be greater than #{expected.to_s}"
|
12
|
+
matcher.negative_failure_message = "expected #{given.to_s} to not be greater than #{expected.to_s}"
|
13
|
+
given > expected
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
include Teeth
|
@@ -0,0 +1,60 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/../spec_helper'
|
2
|
+
|
3
|
+
describe RuleStatement do
|
4
|
+
|
5
|
+
it "should generate a rule to short circuit scanner processing when given option :skip_line => true" do
|
6
|
+
rs = RuleStatement.new :rails_session_id_start, '{WS}*Session ID":"', :skip_line => true
|
7
|
+
expected =
|
8
|
+
%q|{WS}*Session ID":" {
|
9
|
+
return EOF_KVPAIR;
|
10
|
+
}|
|
11
|
+
rs.scanner_code.should == expected
|
12
|
+
end
|
13
|
+
|
14
|
+
it "should use strip_ends(yytext) in the rule when given option :strip_ends => true" do
|
15
|
+
rs = RuleStatement.new :browser_string, '{BROWSER_STR}', :strip_ends => true
|
16
|
+
expected =
|
17
|
+
%q|{BROWSER_STR} {
|
18
|
+
KVPAIR browser_string = {"browser_string", strip_ends(yytext)};
|
19
|
+
return browser_string;
|
20
|
+
}|
|
21
|
+
rs.scanner_code.should == expected
|
22
|
+
end
|
23
|
+
|
24
|
+
it "should include a call to the BEGIN() macro if given the :begin option" do
|
25
|
+
rs = RuleStatement.new :start_special_state, '{SPECIAL_STATE_REGEX}', :begin => "SPECIAL_STATE"
|
26
|
+
expected =
|
27
|
+
%q|{SPECIAL_STATE_REGEX} {
|
28
|
+
BEGIN(SPECIAL_STATE);
|
29
|
+
KVPAIR start_special_state = {"start_special_state", yytext};
|
30
|
+
return start_special_state;
|
31
|
+
}|
|
32
|
+
rs.scanner_code.should == expected
|
33
|
+
end
|
34
|
+
|
35
|
+
it "should not include any C code if given :ignore => true" do
|
36
|
+
rs = RuleStatement.new :catchall_rule_for_special_state, '<SPECIAL_STATE>{CATCHALL}', :ignore => true
|
37
|
+
expected = %q|<SPECIAL_STATE>{CATCHALL}|
|
38
|
+
rs.scanner_code.should == expected
|
39
|
+
end
|
40
|
+
|
41
|
+
end
|
42
|
+
|
43
|
+
describe RuleStatementGroup do
|
44
|
+
|
45
|
+
before(:each) do
|
46
|
+
@statement_group = RuleStatementGroup.new
|
47
|
+
end
|
48
|
+
|
49
|
+
it "should not reject duplicate rule definitions" do
|
50
|
+
@statement_group.add :explode_on_2nd_try, '{WS}'
|
51
|
+
lambda {@statement_group.add :explode_on_2nd_try, '{WS}'}.should_not raise_error DuplicateRuleError
|
52
|
+
end
|
53
|
+
|
54
|
+
it "should use method missing magic to define rules with sugary syntax" do
|
55
|
+
@statement_group.http_version "{HTTP_VERSION}"
|
56
|
+
@statement_group.first.should == RuleStatement.new(:http_version, "{HTTP_VERSION}")
|
57
|
+
end
|
58
|
+
|
59
|
+
|
60
|
+
end
|
@@ -0,0 +1,110 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/../spec_helper'
|
2
|
+
$INCLUDE_SLOW_TESTS = true
|
3
|
+
|
4
|
+
describe "Apache Lexer Extension", "when lexing apache errors" do
|
5
|
+
|
6
|
+
before(:each) do
|
7
|
+
str = "[Sun Nov 30 14:23:45 2008] [error] [client 10.0.1.197] Invalid URI in request GET .\\.\\.\\.\\.\\.\\.\\.\\.\\.\\/winnt/win.ini HTTP/1.1"
|
8
|
+
@tokens = str.scan_apache_logs
|
9
|
+
end
|
10
|
+
|
11
|
+
it "should return an uuid and empty message for an empty string" do
|
12
|
+
tokens = "".scan_apache_logs
|
13
|
+
tokens[:message].should == ""
|
14
|
+
tokens[:id].should match(/[0-9A-F]{32}/)
|
15
|
+
end
|
16
|
+
|
17
|
+
it "should extract an IP address" do
|
18
|
+
@tokens[:ipv4_addr].first.should == "10.0.1.197"
|
19
|
+
end
|
20
|
+
|
21
|
+
it "should extract an apache datetime" do
|
22
|
+
@tokens[:apache_err_datetime].first.should == "Sun Nov 30 14:23:45 2008"
|
23
|
+
end
|
24
|
+
|
25
|
+
it "should extract the error level" do
|
26
|
+
@tokens[:error_level].first.should == "error"
|
27
|
+
end
|
28
|
+
|
29
|
+
it "should extract the URI" do
|
30
|
+
@tokens[:relative_url].first.should == ".\\.\\.\\.\\.\\.\\.\\.\\.\\.\\/winnt/win.ini"
|
31
|
+
end
|
32
|
+
|
33
|
+
it "should group unknown tokens into strings" do
|
34
|
+
@tokens[:strings].should == ["client", "Invalid URI in request"]
|
35
|
+
end
|
36
|
+
|
37
|
+
it "should error out if the string is longer than 1M chars" do
|
38
|
+
str = ((("abcDE" * 2) * 1000) * 100) + "X"
|
39
|
+
lambda {str.scan_apache_logs}.should raise_error(ArgumentError, "string too long for scan_apache_logs! max length is 1,000,000 chars")
|
40
|
+
end
|
41
|
+
|
42
|
+
end
|
43
|
+
|
44
|
+
describe "Apache Lexer Extension", "when lexing apache access logs" do
|
45
|
+
before(:each) do
|
46
|
+
str = %q{couchdb.localdomain:80 172.16.115.1 - - [13/Dec/2008:19:26:11 -0500] "GET /favicon.ico HTTP/1.1" 404 241 "http://172.16.115.130/" "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_4_11; en) AppleWebKit/525.27.1 (KHTML, like Gecko) Version/3.2.1 Safari/525.27.1"}
|
47
|
+
@tokens = str.scan_apache_logs
|
48
|
+
str2 = %q{127.162.219.29 - - [14/Jan/2009:15:32:32 -0500] "GET /reports//ee_commerce/paypalcart.php?toroot=http://www.shenlishi.com//skin/fxid1.txt?? HTTP/1.1" 404 5636}
|
49
|
+
@tokens2 = str2.scan_apache_logs
|
50
|
+
str3 = %q{127.81.248.53 - - [14/Jan/2009:11:49:43 -0500] "GET /reports/REPORT7_1ART02.pdf HTTP/1.1" 206 255404}
|
51
|
+
@tokens3 = str3.scan_apache_logs
|
52
|
+
str4 = %q{127.140.136.56 - - [23/Jan/2009:12:59:24 -0500] "GET /scripts/..%255c%255c../winnt/system32/cmd.exe?/c+dir" 404 5607}
|
53
|
+
@tokens4 = str4.scan_apache_logs
|
54
|
+
str5 = %q{127.254.43.205 - - [26/Jan/2009:08:32:08 -0500] "GET /reports/REPORT9_3.pdf//admin/includes/footer.php?admin_template_default=../../../../../../../../../../../../../etc/passwd%00 HTTP/1.1" 404 5673}
|
55
|
+
@tokens5 = str5.scan_apache_logs
|
56
|
+
str6 = %q{127.218.234.82 - - [26/Jan/2009:08:32:19 -0500] "GET /reports/REPORT9_3.pdf//admin/includes/header.php?bypass_installed=1&bypass_restrict=1&row_secure[account_theme]=../../../../../../../../../../../../../etc/passwd%00 HTTP/1.1" 404 5721}
|
57
|
+
@tokens6 = str6.scan_apache_logs
|
58
|
+
str_naked_url = %q{127.218.234.82 - - [26/Jan/2009:08:32:19 -0500] "GET / HTTP/1.1" 404 5721}
|
59
|
+
@tokens_naked_url = str_naked_url.scan_apache_logs
|
60
|
+
end
|
61
|
+
|
62
|
+
it "provides hints for testing" do
|
63
|
+
#puts "\n" + @tokens.inspect + "\n"
|
64
|
+
end
|
65
|
+
|
66
|
+
it "should extract the vhost name" do
|
67
|
+
@tokens[:host].first.should == "couchdb.localdomain:80"
|
68
|
+
end
|
69
|
+
|
70
|
+
it "should extract the datetime" do
|
71
|
+
@tokens[:apache_access_datetime].first.should == "13/Dec/2008:19:26:11 -0500"
|
72
|
+
end
|
73
|
+
|
74
|
+
it "should extract the HTTP response code" do
|
75
|
+
@tokens[:http_response].first.should == "404"
|
76
|
+
#(100|101|20[0-6]|30[0-5]|307|40[0-9]|41[0-7]|50[0-5])
|
77
|
+
codes = ['100', '101'] + (200 .. 206).map { |n| n.to_s } +
|
78
|
+
(300 .. 305).map { |n| n.to_s } + ['307'] + (400 .. 417).map { |n| n.to_s } +
|
79
|
+
(500 .. 505).map { |n| n.to_s }
|
80
|
+
codes.each do |code|
|
81
|
+
code.scan_apache_logs[:http_response].first.should == code
|
82
|
+
end
|
83
|
+
end
|
84
|
+
|
85
|
+
it "should extract the HTTP version" do
|
86
|
+
@tokens[:http_version].first.should == "HTTP/1.1"
|
87
|
+
end
|
88
|
+
|
89
|
+
it "should extract the browser string with quotes removed" do
|
90
|
+
@tokens[:browser_string].first.should == "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_4_11; en) AppleWebKit/525.27.1 (KHTML, like Gecko) Version/3.2.1 Safari/525.27.1"
|
91
|
+
end
|
92
|
+
|
93
|
+
it "should not extract an HTTP code when a HTTP response code number appears in the bytes transferred" do
|
94
|
+
#puts "\nTOKENS3:\n" + @tokens3.inspect
|
95
|
+
@tokens3[:http_response].include?("404").should_not be_true
|
96
|
+
end
|
97
|
+
|
98
|
+
it "should correctly identify gnarly URLs from web attacks as URLs" do
|
99
|
+
#puts "\nTOKENS2:\n" + @tokens2.inspect
|
100
|
+
@tokens2[:relative_url].first.should == "/reports//ee_commerce/paypalcart.php?toroot=http://www.shenlishi.com//skin/fxid1.txt??"
|
101
|
+
@tokens4[:relative_url].first.should == "/scripts/..%255c%255c../winnt/system32/cmd.exe?/c+dir"
|
102
|
+
@tokens5[:relative_url].first.should == "/reports/REPORT9_3.pdf//admin/includes/footer.php?admin_template_default=../../../../../../../../../../../../../etc/passwd%00"
|
103
|
+
@tokens6[:relative_url].first.should == "/reports/REPORT9_3.pdf//admin/includes/header.php?bypass_installed=1&bypass_restrict=1&row_secure[account_theme]=../../../../../../../../../../../../../etc/passwd%00"
|
104
|
+
end
|
105
|
+
|
106
|
+
it "should correctly extract ``/'' as a URL" do
|
107
|
+
@tokens_naked_url[:relative_url].should == ["/"]
|
108
|
+
end
|
109
|
+
|
110
|
+
end
|
@@ -0,0 +1,100 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/../spec_helper'
|
2
|
+
#require "teeth/scan_rails_logs"
|
3
|
+
# special shout out to Willem van Bergen, author of request-log-analyzer:
|
4
|
+
# http://github.com/wvanbergen/request-log-analyzer/
|
5
|
+
# Thanks for the log samples!
|
6
|
+
|
7
|
+
describe "Rails Request Log Lexer", "when lexing Rails 1.x logs" do
|
8
|
+
|
9
|
+
it "should extract the Controller, action, IP, timestamp and HTTP verb from a ``Processing'' line" do
|
10
|
+
line = %q{Processing PageController#demo (for 127.0.0.1 at 2008-12-10 16:28:09) [GET]}
|
11
|
+
result = line.scan_rails_logs
|
12
|
+
result[:teaser].first.should == "Processing"
|
13
|
+
result[:controller_action].first.should == "PageController#demo"
|
14
|
+
result[:ipv4_addr].first.should == "127.0.0.1"
|
15
|
+
result[:http_method].first.should == "GET"
|
16
|
+
result[:datetime].first.should == "2008-12-10 16:28:09"
|
17
|
+
end
|
18
|
+
|
19
|
+
it "should give a hash with :cache_hit => not falsy value for a cache hit" do
|
20
|
+
cache_hit = %q{Filter chain halted as [#<ActionController::Caching::Actions::ActionCacheFilter:0x2a999ad620 @check=nil, @options={:store_options=>{}, :layout=>nil, :cache_path=>#<Proc:0x0000002a999b8890@/app/controllers/cached_controller.rb:8>}>] rendered_or_redirected.}
|
21
|
+
cache_hit.scan_rails_logs[:cache_hit].should be_true
|
22
|
+
cache_hit.scan_rails_logs[:teaser].first.should == "Filter chain halted"
|
23
|
+
end
|
24
|
+
|
25
|
+
it "should extract an error, error_message, line of code, source code file, and stack_trace from a ``RuntimeError'' line" do
|
26
|
+
line = %q{RuntimeError (Cannot destroy employee): /app/models/employee.rb:198:in `before_destroy' }
|
27
|
+
result = line.scan_rails_logs
|
28
|
+
#puts "###\n" + result.inspect
|
29
|
+
result[:error].first.should == "RuntimeError"
|
30
|
+
result[:error_message].first.should == "Cannot destroy employee"
|
31
|
+
result[:file_and_line].first.should == "/app/models/employee.rb:198"
|
32
|
+
end
|
33
|
+
|
34
|
+
it "should extract the duration, view duration, db duration, HTTP status code, and url from a ``Completed'' line for Rails 1.x" do
|
35
|
+
rails_1x = %q{Completed in 0.21665 (4 reqs/sec) | Rendering: 0.00926 (4%) | DB: 0.00000 (0%) | 200 OK [http://demo.nu/employees]}
|
36
|
+
result = rails_1x.scan_rails_logs
|
37
|
+
result[:teaser].first.should == "Completed in"
|
38
|
+
result[:duration_s].first.should == "0.21665"
|
39
|
+
result[:view_s].first.should == "0.00926"
|
40
|
+
result[:db_s].first.should == "0.00000"
|
41
|
+
result[:http_response].first.should == "200"
|
42
|
+
result[:url].first.should == "http://demo.nu/employees"
|
43
|
+
end
|
44
|
+
|
45
|
+
it "should also process a different 1.x ``Completed'' line correctly" do
|
46
|
+
alt_1x = %q{Completed in 0.04180 (23 reqs/sec) | Rendering: 0.02667 (63%) | DB: 0.00259 (6%) | 200 OK [http://localhost/]}
|
47
|
+
result = alt_1x.scan_rails_logs
|
48
|
+
#puts result.inspect
|
49
|
+
result[:teaser].first.should == "Completed in"
|
50
|
+
result[:duration_s].first.should == "0.04180"
|
51
|
+
result[:view_s].first.should == "0.02667"
|
52
|
+
result[:db_s].first.should == "0.00259"
|
53
|
+
result[:http_response].first.should == "200"
|
54
|
+
result[:url].first.should == "http://localhost/"
|
55
|
+
end
|
56
|
+
|
57
|
+
it "should extract the relevant components from a ``Completed'' line for Rails 2.x" do
|
58
|
+
rails_2x = %q{Completed in 614ms (View: 120, DB: 31) | 200 OK [http://floorplanner.local/demo]}
|
59
|
+
result = rails_2x.scan_rails_logs
|
60
|
+
result[:teaser].first.should == "Completed in"
|
61
|
+
result[:duration_ms].first.should == "614"
|
62
|
+
result[:view_ms].first.should == "120"
|
63
|
+
result[:db_ms].first.should == "31"
|
64
|
+
result[:http_response].first.should == "200"
|
65
|
+
result[:url].first.should == "http://floorplanner.local/demo"
|
66
|
+
end
|
67
|
+
|
68
|
+
it "should extract the duration and partial from a ``Rendered'' line for rails 2.x" do
|
69
|
+
rendered_2x = "Rendered shared/_analytics (0.2ms)"
|
70
|
+
#puts "(rendered 2.x): " + rendered_2x.scan_rails_logs.inspect
|
71
|
+
rendered_2x.scan_rails_logs[:partial].first.should == "shared/_analytics"
|
72
|
+
rendered_2x.scan_rails_logs[:render_duration_ms].first.should == "0.2"
|
73
|
+
end
|
74
|
+
|
75
|
+
it "should extract the duration and partial from a ``Rendered'' line for rails 1.x" do
|
76
|
+
rendered_1x = "Rendered layouts/_doc_type (0.00001)"
|
77
|
+
#puts "(rendered 1.x): " + rendered_1x.scan_rails_logs.inspect
|
78
|
+
rendered_1x.scan_rails_logs[:partial].first.should == "layouts/_doc_type"
|
79
|
+
rendered_1x.scan_rails_logs[:render_duration_s].first.should == "0.00001"
|
80
|
+
end
|
81
|
+
|
82
|
+
it "should skip session id lines" do
|
83
|
+
session_id = %q{Session ID: BAh7CToMcmVmZXJlciIbL3ByaXNjaWxsYS9wZW9wbGUvMjM1MCIKZmxhc2hJ}
|
84
|
+
session_id.scan_rails_logs.keys.map { |k| k.to_s}.sort.should == ["id", "message"]
|
85
|
+
end
|
86
|
+
|
87
|
+
it "should not return a teaser for session id continuation lines" do
|
88
|
+
session_id_contd = "ZWR7ADoNbGFuZ3VhZ2VvOhNMb2NhbGU6Ok9iamVjdBI6CUB3aW4wOg1AY291"
|
89
|
+
session_id_contd.scan_rails_logs[:teaser].should be_nil
|
90
|
+
end
|
91
|
+
|
92
|
+
it "should give a non falsy value for :end_session_id at for the last line of a session id" do
|
93
|
+
session_id_end_1 = "bmxfTkw6DEBzY3JpcHQwOg5AZmFsbGJhY2sw--48cbe3788ef27f6005f8e999610a42af6e90ffb3"
|
94
|
+
session_id_end_1.scan_rails_logs[:end_session_id].should_not be_nil
|
95
|
+
session_id_end_2 = "X2lkaQIyBw==--3ad1948559448522a49d289a2a89dc7ccbe8847a"
|
96
|
+
session_id_end_2.scan_rails_logs[:end_session_id].should_not be_nil
|
97
|
+
end
|
98
|
+
|
99
|
+
|
100
|
+
end
|