danielsdeleo-teeth 0.1.1 → 0.1.2
Sign up to get free protection for your applications and to get access to all the features.
- data/VERSION.yml +1 -1
- data/doc/created.rid +1 -1
- data/doc/files/README_rdoc.html +5 -5
- data/doc/files/ext/scan_apache_logs/scan_apache_logs_yy_c.html +1 -1
- data/doc/files/ext/scan_rails_logs/scan_rails_logs_yy_c.html +1 -1
- data/doc/files/lib/scanner_definition_rb.html +1 -1
- data/doc/files/lib/teeth_rb.html +8 -1
- data/ext/scan_apache_logs/scan_apache_logs.yy +1 -1
- data/ext/scan_apache_logs/scan_apache_logs.yy.c +3799 -3236
- data/ext/scan_rails_logs/scan_rails_logs.yy +2 -2
- data/ext/scan_rails_logs/scan_rails_logs.yy.c +1960 -1558
- data/lib/scanner_definition.rb +1 -1
- data/lib/teeth.rb +8 -2
- data/scanners/scan_rails_logs.rb +1 -1
- data/spec/playground/scan_rails_logs.rb +56 -0
- data/spec/unit/scan_rails_logs_spec.rb +12 -2
- data/teeth.gemspec +2 -2
- metadata +2 -1
data/lib/scanner_definition.rb
CHANGED
|
@@ -60,7 +60,7 @@ module Teeth
|
|
|
60
60
|
DEFAULT_DEFINITIONS[:whitespace] = [["WS", '[[:space:]]'],
|
|
61
61
|
["NON_WS", "([a-z]|[0-9]|[:punct:])"]]
|
|
62
62
|
DEFAULT_DEFINITIONS[:ip] = [ ["IP4_OCT", "[0-9]|[0-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5]"],
|
|
63
|
-
["HOST", '[a-z0-9][a-z0-9\-]*\.[a-z0-9][a-z0-9\-]*.[a-z0-9][a-z0-9\-\.]*[a-z]+(\:[0-9]+)?']]
|
|
63
|
+
["HOST", '([a-z0-9][a-z0-9\-]*\.[a-z0-9][a-z0-9\-]*.[a-z0-9][a-z0-9\-\.]*[a-z]+(\:[0-9]+)?)|localhost']]
|
|
64
64
|
DEFAULT_DEFINITIONS[:time] = [ ["WDAY", "mon|tue|wed|thu|fri|sat|sun"],
|
|
65
65
|
["MON", "jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec"],
|
|
66
66
|
["MONTH_NUM", "0[1-9]|1[0-2]"],
|
data/lib/teeth.rb
CHANGED
|
@@ -1,5 +1,11 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
1
|
+
# encoding: utf-8
|
|
2
|
+
begin
|
|
3
|
+
require "teeth/scan_apache_logs"
|
|
4
|
+
require "teeth/scan_rails_logs"
|
|
5
|
+
rescue LoadError => e
|
|
6
|
+
STDERR.puts "WARNING: could not load extensions. This is okay if you are creating them from\n" +
|
|
7
|
+
"source for the first time. If that isn't the case, you're screwed"
|
|
8
|
+
end
|
|
3
9
|
$:.unshift File.dirname(__FILE__) + "/"
|
|
4
10
|
require "scanner"
|
|
5
11
|
require "scanner_definition"
|
data/scanners/scan_rails_logs.rb
CHANGED
|
@@ -59,7 +59,7 @@ scanner.rules do |r|
|
|
|
59
59
|
r.db_s '<COMPLETED_REQ_DB_STATS>[0-9]+\.[0-9]+', :begin => "REQUEST_COMPLETED"
|
|
60
60
|
r.db_ms '<COMPLETED_REQ_DB_STATS>[0-9]+', :begin => "REQUEST_COMPLETED"
|
|
61
61
|
r.db_throwaway_tokens '<COMPLETED_REQ_DB_STATS>{CATCHALL}', :ignore => true
|
|
62
|
-
r.url '<REQUEST_COMPLETED>\[{PROTO}"\/\/"({HOST}|{IP4_OCT}"."{IP4_OCT}"."{IP4_OCT}"."{IP4_OCT})({REL_URL}|"
|
|
62
|
+
r.url '<REQUEST_COMPLETED>\[{PROTO}"\/\/"({HOST}|({IP4_OCT}"."{IP4_OCT}"."{IP4_OCT}"."{IP4_OCT}))({REL_URL}|"/"|"\\\\")?\]', :strip_ends => true
|
|
63
63
|
r.http_response '<REQUEST_COMPLETED>{HTTPCODE}'
|
|
64
64
|
r.strings '<REQUEST_COMPLETED>{NON_WS}{NON_WS}*'
|
|
65
65
|
r.ignore_others '<REQUEST_COMPLETED>{CATCHALL}', :ignore => true
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
#!/usr/bin/env ruby
|
|
2
|
+
# encoding: utf-8
|
|
3
|
+
require "uri"
|
|
4
|
+
require File.dirname(__FILE__) + "/../spec_helper"
|
|
5
|
+
|
|
6
|
+
# A very basic log stats program to show the use of Teeth. Gives the top 20
|
|
7
|
+
# URLs and the top 10 Errors. Supports Ruby 1.9, though it seems a bit
|
|
8
|
+
# slower than 1.8 (?)
|
|
9
|
+
|
|
10
|
+
results = Hash.new(0)
|
|
11
|
+
error_results = Hash.new(0)
|
|
12
|
+
if RUBY_VERSION >= "1.9.0"
|
|
13
|
+
filename = ARGV[0].dup.force_encoding("ASCII-8BIT") # 1.9 is weird on Mac
|
|
14
|
+
else
|
|
15
|
+
filename = ARGV[0]
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
File.open(filename, "r") do |f|
|
|
19
|
+
n_processed = n_completed = n_errors = 0
|
|
20
|
+
while line = f.gets
|
|
21
|
+
tokens = line.scan_rails_logs
|
|
22
|
+
if tokens[:teaser] && tokens[:teaser].first == "Processing"
|
|
23
|
+
n_processed += 1
|
|
24
|
+
results[tokens[:controller_action].first] += 1
|
|
25
|
+
end
|
|
26
|
+
if tokens[:error]
|
|
27
|
+
n_errors += 1
|
|
28
|
+
error_results[tokens[:error].first] += 1
|
|
29
|
+
end
|
|
30
|
+
if tokens[:url] && tokens[:teaser].first == "Completed in"
|
|
31
|
+
n_completed +=1
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
puts "=" * 80
|
|
35
|
+
puts "Totals"
|
|
36
|
+
puts "#{n_processed} requests processed"
|
|
37
|
+
puts "#{n_completed} requests completed"
|
|
38
|
+
results_ary = results.map { |url, hits| [url, hits] }.sort { |a, b| b.last <=> a.last }
|
|
39
|
+
puts "=" * 80
|
|
40
|
+
puts "\nTop 20 URLs"
|
|
41
|
+
puts "=" * 80
|
|
42
|
+
puts " URL".ljust(40) + " | " + "Hits"
|
|
43
|
+
puts "-" * 80
|
|
44
|
+
results_ary[0, 20].each do |url_hits|
|
|
45
|
+
puts url_hits.first.ljust(40) + " | " + url_hits.last.to_s
|
|
46
|
+
end
|
|
47
|
+
errors_ary = error_results.map { |error, hits| [error, hits] }.sort { |a, b| b.last <=> a.last }
|
|
48
|
+
puts "=" * 80
|
|
49
|
+
puts "\nTop 10 Errors"
|
|
50
|
+
puts "=" * 80
|
|
51
|
+
puts " Error".ljust(40) + " | " + "Hits"
|
|
52
|
+
puts "-" * 80
|
|
53
|
+
errors_ary[0, 10].each do |error_hits|
|
|
54
|
+
puts error_hits.first.ljust(40) + " | " + error_hits.last.to_s
|
|
55
|
+
end
|
|
56
|
+
end
|
|
@@ -34,7 +34,6 @@ describe "Rails Request Log Lexer", "when lexing Rails 1.x logs" do
|
|
|
34
34
|
it "should extract the duration, view duration, db duration, HTTP status code, and url from a ``Completed'' line for Rails 1.x" do
|
|
35
35
|
rails_1x = %q{Completed in 0.21665 (4 reqs/sec) | Rendering: 0.00926 (4%) | DB: 0.00000 (0%) | 200 OK [http://demo.nu/employees]}
|
|
36
36
|
result = rails_1x.scan_rails_logs
|
|
37
|
-
puts "\n==completed (1.x): " + result.inspect
|
|
38
37
|
result[:teaser].first.should == "Completed in"
|
|
39
38
|
result[:duration_s].first.should == "0.21665"
|
|
40
39
|
result[:view_s].first.should == "0.00926"
|
|
@@ -43,10 +42,21 @@ describe "Rails Request Log Lexer", "when lexing Rails 1.x logs" do
|
|
|
43
42
|
result[:url].first.should == "http://demo.nu/employees"
|
|
44
43
|
end
|
|
45
44
|
|
|
45
|
+
it "should also process a different 1.x ``Completed'' line correctly" do
|
|
46
|
+
alt_1x = %q{Completed in 0.04180 (23 reqs/sec) | Rendering: 0.02667 (63%) | DB: 0.00259 (6%) | 200 OK [http://localhost/]}
|
|
47
|
+
result = alt_1x.scan_rails_logs
|
|
48
|
+
puts result.inspect
|
|
49
|
+
result[:teaser].first.should == "Completed in"
|
|
50
|
+
result[:duration_s].first.should == "0.04180"
|
|
51
|
+
result[:view_s].first.should == "0.02667"
|
|
52
|
+
result[:db_s].first.should == "0.00259"
|
|
53
|
+
result[:http_response].first.should == "200"
|
|
54
|
+
result[:url].first.should == "http://localhost/"
|
|
55
|
+
end
|
|
56
|
+
|
|
46
57
|
it "should extract the relevant components from a ``Completed'' line for Rails 2.x" do
|
|
47
58
|
rails_2x = %q{Completed in 614ms (View: 120, DB: 31) | 200 OK [http://floorplanner.local/demo]}
|
|
48
59
|
result = rails_2x.scan_rails_logs
|
|
49
|
-
puts "\n==completed (2.x): " + result.inspect
|
|
50
60
|
result[:teaser].first.should == "Completed in"
|
|
51
61
|
result[:duration_ms].first.should == "614"
|
|
52
62
|
result[:view_ms].first.should == "120"
|
data/teeth.gemspec
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
Gem::Specification.new do |s|
|
|
4
4
|
s.name = %q{teeth}
|
|
5
|
-
s.version = "0.1.
|
|
5
|
+
s.version = "0.1.2"
|
|
6
6
|
|
|
7
7
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
|
8
8
|
s.authors = ["Daniel DeLeo"]
|
|
@@ -11,7 +11,7 @@ Gem::Specification.new do |s|
|
|
|
11
11
|
s.email = %q{ddeleo@basecommander.net}
|
|
12
12
|
s.extensions = ["ext/scan_apache_logs/extconf.rb", "ext/scan_rails_logs/extconf.rb"]
|
|
13
13
|
s.extra_rdoc_files = ["README.rdoc"]
|
|
14
|
-
s.files = ["LICENSE", "README.rdoc", "Rakefile", "VERSION.yml", "doc/classes/String.html", "doc/classes/Teeth.html", "doc/classes/Teeth/DuplicateDefinitionError.html", "doc/classes/Teeth/DuplicateRuleError.html", "doc/classes/Teeth/InvalidDefaultDefinitionName.html", "doc/classes/Teeth/InvalidExtensionDirectory.html", "doc/classes/Teeth/RuleStatement.html", "doc/classes/Teeth/RuleStatementGroup.html", "doc/classes/Teeth/Scanner.html", "doc/classes/Teeth/ScannerDefinition.html", "doc/classes/Teeth/ScannerDefinitionArgumentError.html", "doc/classes/Teeth/ScannerDefinitionGroup.html", "doc/classes/Teeth/ScannerError.html", "doc/created.rid", "doc/files/README_rdoc.html", "doc/files/ext/scan_apache_logs/scan_apache_logs_yy_c.html", "doc/files/ext/scan_rails_logs/scan_rails_logs_yy_c.html", "doc/files/lib/rule_statement_rb.html", "doc/files/lib/scanner_definition_rb.html", "doc/files/lib/scanner_rb.html", "doc/files/lib/teeth_rb.html", "doc/fr_class_index.html", "doc/fr_file_index.html", "doc/fr_method_index.html", "doc/index.html", "doc/rdoc-style.css", "ext/scan_apache_logs/Makefile", "ext/scan_apache_logs/extconf.rb", "ext/scan_apache_logs/scan_apache_logs.yy", "ext/scan_apache_logs/scan_apache_logs.yy.c", "ext/scan_rails_logs/Makefile", "ext/scan_rails_logs/extconf.rb", "ext/scan_rails_logs/scan_rails_logs.yy", "ext/scan_rails_logs/scan_rails_logs.yy.c", "lib/rule_statement.rb", "lib/scanner.rb", "lib/scanner_definition.rb", "lib/teeth.rb", "scanners/scan_apache_logs.rb", "scanners/scan_rails_logs.rb", "spec/fixtures/rails_1x.log", "spec/fixtures/rails_22.log", "spec/fixtures/rails_22_cached.log", "spec/fixtures/rails_unordered.log", "spec/playground/show_apache_processing.rb", "spec/spec.opts", "spec/spec_helper.rb", "spec/unit/rule_statement_spec.rb", "spec/unit/scan_apache_spec.rb", "spec/unit/scan_rails_logs_spec.rb", "spec/unit/scaner_definition_spec.rb", "spec/unit/scanner_spec.rb", "teeth.gemspec", "templates/tokenizer.yy.erb"]
|
|
14
|
+
s.files = ["LICENSE", "README.rdoc", "Rakefile", "VERSION.yml", "doc/classes/String.html", "doc/classes/Teeth.html", "doc/classes/Teeth/DuplicateDefinitionError.html", "doc/classes/Teeth/DuplicateRuleError.html", "doc/classes/Teeth/InvalidDefaultDefinitionName.html", "doc/classes/Teeth/InvalidExtensionDirectory.html", "doc/classes/Teeth/RuleStatement.html", "doc/classes/Teeth/RuleStatementGroup.html", "doc/classes/Teeth/Scanner.html", "doc/classes/Teeth/ScannerDefinition.html", "doc/classes/Teeth/ScannerDefinitionArgumentError.html", "doc/classes/Teeth/ScannerDefinitionGroup.html", "doc/classes/Teeth/ScannerError.html", "doc/created.rid", "doc/files/README_rdoc.html", "doc/files/ext/scan_apache_logs/scan_apache_logs_yy_c.html", "doc/files/ext/scan_rails_logs/scan_rails_logs_yy_c.html", "doc/files/lib/rule_statement_rb.html", "doc/files/lib/scanner_definition_rb.html", "doc/files/lib/scanner_rb.html", "doc/files/lib/teeth_rb.html", "doc/fr_class_index.html", "doc/fr_file_index.html", "doc/fr_method_index.html", "doc/index.html", "doc/rdoc-style.css", "ext/scan_apache_logs/Makefile", "ext/scan_apache_logs/extconf.rb", "ext/scan_apache_logs/scan_apache_logs.yy", "ext/scan_apache_logs/scan_apache_logs.yy.c", "ext/scan_rails_logs/Makefile", "ext/scan_rails_logs/extconf.rb", "ext/scan_rails_logs/scan_rails_logs.yy", "ext/scan_rails_logs/scan_rails_logs.yy.c", "lib/rule_statement.rb", "lib/scanner.rb", "lib/scanner_definition.rb", "lib/teeth.rb", "scanners/scan_apache_logs.rb", "scanners/scan_rails_logs.rb", "spec/fixtures/rails_1x.log", "spec/fixtures/rails_22.log", "spec/fixtures/rails_22_cached.log", "spec/fixtures/rails_unordered.log", "spec/playground/scan_rails_logs.rb", "spec/playground/show_apache_processing.rb", "spec/spec.opts", "spec/spec_helper.rb", "spec/unit/rule_statement_spec.rb", "spec/unit/scan_apache_spec.rb", "spec/unit/scan_rails_logs_spec.rb", "spec/unit/scaner_definition_spec.rb", "spec/unit/scanner_spec.rb", "teeth.gemspec", "templates/tokenizer.yy.erb"]
|
|
15
15
|
s.has_rdoc = true
|
|
16
16
|
s.homepage = %q{http://github.com/danielsdeleo/teeth}
|
|
17
17
|
s.rdoc_options = ["--inline-source", "--charset=UTF-8"]
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: danielsdeleo-teeth
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.1.
|
|
4
|
+
version: 0.1.2
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Daniel DeLeo
|
|
@@ -71,6 +71,7 @@ files:
|
|
|
71
71
|
- spec/fixtures/rails_22.log
|
|
72
72
|
- spec/fixtures/rails_22_cached.log
|
|
73
73
|
- spec/fixtures/rails_unordered.log
|
|
74
|
+
- spec/playground/scan_rails_logs.rb
|
|
74
75
|
- spec/playground/show_apache_processing.rb
|
|
75
76
|
- spec/spec.opts
|
|
76
77
|
- spec/spec_helper.rb
|