wvanbergen-request-log-analyzer 0.3.4 → 1.0.0
Sign up to get free protection for your applications and to get access to all the features.
- data/README.textile +23 -10
- data/TODO +5 -0
- data/bin/request-log-analyzer +96 -20
- data/lib/request_log_analyzer.rb +14 -161
- data/spec/controller_spec.rb +13 -6
- data/spec/database_inserter_spec.rb +17 -14
- data/spec/line_definition_spec.rb +94 -4
- data/spec/log_parser_spec.rb +9 -11
- data/spec/merb_format_spec.rb +58 -58
- data/spec/rails_format_spec.rb +4 -4
- data/spec/request_spec.rb +6 -2
- data/spec/spec_helper.rb +22 -4
- data/spec/summarizer_spec.rb +0 -94
- data/tasks/github-gem.rake +1 -1
- data/test/rails_log_parser_test.rb +0 -5
- metadata +2 -2
- data/lib/base/summarizer.rb +0 -85
- data/lib/bashcolorizer.rb +0 -60
- data/lib/command_line/exceptions.rb +0 -37
- data/lib/merb_analyzer/summarizer.rb +0 -61
- data/lib/rails_analyzer/summarizer.rb +0 -81
- data/lib/rails_analyzer/virtual_mongrel.rb +0 -91
- data/lib/ruby-progressbar/progressbar.en.rd +0 -103
- data/lib/ruby-progressbar/progressbar.ja.rd +0 -100
- data/lib/ruby-progressbar/progressbar.rb +0 -236
- data/output/blockers.rb +0 -9
- data/output/errors.rb +0 -8
- data/output/hourly_spread.rb +0 -28
- data/output/mean_db_time.rb +0 -6
- data/output/mean_rendering_time.rb +0 -6
- data/output/mean_time.rb +0 -6
- data/output/most_requested.rb +0 -5
- data/output/timespan.rb +0 -18
- data/output/total_db_time.rb +0 -5
- data/output/total_time.rb +0 -5
- data/output/usage.rb +0 -16
- data/test/base_summarizer_test.rb +0 -30
data/README.textile
CHANGED
@@ -17,16 +17,29 @@ h2. Installation
|
|
17
17
|
h2. Usage
|
18
18
|
|
19
19
|
<pre>
|
20
|
-
Usage: request-log-analyzer [
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
20
|
+
Usage: request-log-analyzer [LOGFILES*] <OPTIONS>
|
21
|
+
|
22
|
+
Input options:
|
23
|
+
--format <format>, -f: Uses the specified log file format. Defaults to rails.
|
24
|
+
--after <date> Only consider requests from <date> or later.
|
25
|
+
--before <date> Only consider requests before <date>.
|
26
|
+
--select <field> <value> Only consider requests where <field> matches <value>.
|
27
|
+
--reject <field> <value> Only consider requests where <field> does not match <value>.
|
28
|
+
|
29
|
+
Output options:
|
30
|
+
--boring, -b Output reports without ASCII colors.
|
31
|
+
--database <filename>, -d: Creates an SQLite3 database of all the parsed request information.
|
32
|
+
--debug Print debug information while parsing.
|
33
|
+
|
34
|
+
Examples:
|
35
|
+
request-log-analyzer development.log
|
36
|
+
request-log-analyzer -z mongrel.0.log mongrel.1.log mongrel.2.log
|
37
|
+
request-log-analyzer --format merb -d requests.db production.log
|
38
|
+
|
39
|
+
To install rake tasks in your Rails application,
|
40
|
+
run the following command in your application's root directory:
|
41
|
+
|
42
|
+
request-log-analyzer install rails
|
30
43
|
</pre>
|
31
44
|
|
32
45
|
|
data/TODO
CHANGED
@@ -5,6 +5,11 @@ Contact willem AT vanbergen DOT org if you want to help out with the development
|
|
5
5
|
General:
|
6
6
|
- Add more tests / specs
|
7
7
|
|
8
|
+
Parameters:
|
9
|
+
- Add commandline parameters
|
10
|
+
--version, -v -> This should support github - last version checking
|
11
|
+
--help, -h -> Show usage
|
12
|
+
|
8
13
|
Datamining:
|
9
14
|
- Add query functionality for the resulting database file (interactive reports?)
|
10
15
|
- Link request processing line to request completed line (VirtualMongrel?)
|
data/bin/request-log-analyzer
CHANGED
@@ -1,31 +1,94 @@
|
|
1
1
|
#!/usr/bin/ruby
|
2
2
|
require File.dirname(__FILE__) + '/../lib/request_log_analyzer'
|
3
|
-
require File.dirname(__FILE__) + '/../lib/
|
3
|
+
require File.dirname(__FILE__) + '/../lib/cli/command_line_arguments'
|
4
4
|
|
5
|
-
|
5
|
+
def terminal_width(default = 81)
|
6
|
+
IO.popen('stty -a') do |pipe|
|
7
|
+
column_line = pipe.detect { |line| /(\d+) columns/ =~ line }
|
8
|
+
width = column_line ? $1.to_i : default
|
9
|
+
end
|
10
|
+
rescue
|
11
|
+
default
|
12
|
+
end
|
6
13
|
|
7
14
|
# Parse the arguments given via commandline
|
8
15
|
begin
|
9
16
|
arguments = CommandLine::Arguments.parse do |command_line|
|
10
|
-
|
11
|
-
command_line.
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
command_line.
|
16
|
-
|
17
|
-
|
17
|
+
|
18
|
+
command_line.command(:install) do |install|
|
19
|
+
install.parameters = 1
|
20
|
+
end
|
21
|
+
|
22
|
+
command_line.command(:strip) do |strip|
|
23
|
+
strip.minimum_parameters = 1
|
24
|
+
strip.option(:format, :alias => :f, :default => 'rails')
|
25
|
+
strip.option(:output, :alias => :o)
|
26
|
+
strip.switch(:discard_teaser_lines, :t)
|
27
|
+
strip.switch(:keep_junk_lines, :j)
|
28
|
+
end
|
29
|
+
|
30
|
+
command_line.command(:anonymize) do |anonymize|
|
31
|
+
anonymize.minimum_parameters = 1
|
32
|
+
anonymize.option(:format, :alias => :f, :default => 'rails')
|
33
|
+
anonymize.option(:output, :alias => :o)
|
34
|
+
anonymize.switch(:discard_teaser_lines, :t)
|
35
|
+
anonymize.switch(:keep_junk_lines, :j)
|
36
|
+
end
|
37
|
+
|
38
|
+
command_line.option(:format, :alias => :f, :default => 'rails')
|
39
|
+
command_line.option(:file, :alias => :e)
|
40
|
+
command_line.switch(:single_lines, :s)
|
41
|
+
command_line.switch(:assume_correct_order)
|
42
|
+
|
43
|
+
command_line.option(:aggregator, :alias => :a, :multiple => true)
|
44
|
+
command_line.option(:database, :alias => :d)
|
45
|
+
|
46
|
+
# filtering options
|
47
|
+
command_line.option(:select, :multiple => true, :parameters => 2)
|
48
|
+
command_line.option(:reject, :multiple => true, :parameters => 2)
|
49
|
+
command_line.option(:after)
|
50
|
+
command_line.option(:before)
|
51
|
+
|
52
|
+
command_line.switch(:boring, :b)
|
53
|
+
command_line.option(:report_width, :default => terminal_width - 1)
|
54
|
+
|
55
|
+
command_line.switch(:debug)
|
56
|
+
|
57
|
+
command_line.minimum_parameters = 1
|
18
58
|
end
|
19
59
|
|
20
60
|
rescue CommandLine::Error => e
|
21
|
-
puts "ARGUMENT ERROR: " + e.message
|
61
|
+
puts "ARGUMENT ERROR: " + e.message if e.message
|
22
62
|
puts
|
23
|
-
|
63
|
+
puts "Usage: request-log-analyzer [LOGFILES*] <OPTIONS>"
|
64
|
+
puts
|
65
|
+
puts "Input options:"
|
66
|
+
puts " --format <format>, -f: Uses the specified log file format. Defaults to rails."
|
67
|
+
puts " --after <date> Only consider requests from <date> or later."
|
68
|
+
puts " --before <date> Only consider requests before <date>."
|
69
|
+
puts " --select <field> <value> Only consider requests where <field> matches <value>."
|
70
|
+
puts " --reject <field> <value> Only consider requests where <field> does not match <value>."
|
71
|
+
puts
|
72
|
+
puts "Output options:"
|
73
|
+
puts " --boring, -b Output reports without ASCII colors."
|
74
|
+
puts " --database <filename>, -d: Creates an SQLite3 database of all the parsed request information."
|
75
|
+
puts " --debug Print debug information while parsing."
|
76
|
+
puts " --file <filename> Output to file."
|
77
|
+
puts
|
78
|
+
puts "Examples:"
|
79
|
+
puts " request-log-analyzer development.log"
|
80
|
+
puts " request-log-analyzer -b mongrel.0.log mongrel.1.log mongrel.2.log "
|
81
|
+
puts " request-log-analyzer --format merb -d requests.db production.log"
|
82
|
+
puts
|
83
|
+
puts "To install rake tasks in your Rails application, "
|
84
|
+
puts "run the following command in your application's root directory:"
|
85
|
+
puts
|
86
|
+
puts " request-log-analyzer install rails"
|
24
87
|
exit(0)
|
25
88
|
end
|
26
89
|
|
27
|
-
|
28
|
-
if
|
90
|
+
def install_rake_tasks(install_type)
|
91
|
+
if install_type == 'rails'
|
29
92
|
require 'ftools'
|
30
93
|
if File.directory?('./lib/tasks/')
|
31
94
|
File.copy(File.dirname(__FILE__) + '/../tasks/request_log_analyzer.rake', './lib/tasks/request_log_analyze.rake')
|
@@ -35,14 +98,27 @@ if arguments[:install]
|
|
35
98
|
puts "Cannot find /lib/tasks folder. Are you in your Rails directory?"
|
36
99
|
puts "Installation aborted."
|
37
100
|
end
|
101
|
+
else
|
102
|
+
raise "Cannot perform this install type! (#{install_type})"
|
38
103
|
end
|
39
|
-
exit(0)
|
40
104
|
end
|
41
105
|
|
42
|
-
$colorize = true if arguments[:colorize]
|
43
|
-
|
44
|
-
# Run the request_log_analyzer!
|
45
|
-
request_log_analyzer = RequestLogAnalyzer.new(arguments)
|
46
|
-
request_log_analyzer.analyze_this(arguments.files)
|
47
106
|
|
107
|
+
case arguments.command
|
108
|
+
when :install
|
109
|
+
install_rake_tasks(arguments.parameters[0])
|
110
|
+
when :strip
|
111
|
+
require File.dirname(__FILE__) + '/../lib/request_log_analyzer/log_processor'
|
112
|
+
RequestLogAnalyzer::LogProcessor.build(:strip, arguments).run!
|
113
|
+
when :anonymize
|
114
|
+
require File.dirname(__FILE__) + '/../lib/request_log_analyzer/log_processor'
|
115
|
+
RequestLogAnalyzer::LogProcessor.build(:anonymize, arguments).run!
|
116
|
+
else
|
117
|
+
puts "Request log analyzer, by Willem van Bergen and Bart ten Brinke - Version 0.4.0\n\n"
|
118
|
+
|
119
|
+
# Run the request_log_analyzer!
|
120
|
+
RequestLogAnalyzer::Controller.build(arguments, terminal_width).run!
|
48
121
|
|
122
|
+
puts
|
123
|
+
puts "Thanks for using request-log-analyzer"
|
124
|
+
end
|
data/lib/request_log_analyzer.rb
CHANGED
@@ -1,161 +1,14 @@
|
|
1
|
-
require
|
2
|
-
require File.dirname(__FILE__) + '
|
3
|
-
require File.dirname(__FILE__) + '
|
4
|
-
|
5
|
-
require File.dirname(__FILE__) + '
|
6
|
-
require File.dirname(__FILE__) + '
|
7
|
-
require File.dirname(__FILE__) + '
|
8
|
-
require File.dirname(__FILE__) + '
|
9
|
-
require File.dirname(__FILE__) + '
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
attr_reader :arguments
|
16
|
-
attr_reader :line_types
|
17
|
-
attr_reader :amount
|
18
|
-
attr_reader :fast
|
19
|
-
|
20
|
-
# Initializer. Sets global variables
|
21
|
-
# Options
|
22
|
-
# *<tt>:fast</tt> Only look at request initializers. Faster, but not not all outputs are shown.
|
23
|
-
# *<tt>:guess_database_time</tt> Guess database time if it it not in the log (default for Rails produciton).
|
24
|
-
# *<tt>:merb</tt> Use merb summarizer and parser classes instead of rails.
|
25
|
-
# *<tt>:output_reports</tt> Comma separated string of requested output reports
|
26
|
-
# *<tt>:amount</tt> Amount of lines shown for each result table. Defaults to 10.
|
27
|
-
def initialize(options = {})
|
28
|
-
@fast = options[:fast] || false
|
29
|
-
@guess_database_time = options[:guess_database_time] || false
|
30
|
-
|
31
|
-
if options[:merb]
|
32
|
-
@summarizer = MerbAnalyzer::Summarizer.new(:calculate_database => @guess_database_time)
|
33
|
-
@log_parser_class = MerbAnalyzer::LogParser
|
34
|
-
else
|
35
|
-
@summarizer = RailsAnalyzer::Summarizer.new(:calculate_database => @guess_database_time)
|
36
|
-
@log_parser_class = RailsAnalyzer::LogParser
|
37
|
-
end
|
38
|
-
|
39
|
-
@output_reports = options[:output].split(', ') rescue [:timespan, :most_requested, :total_time, :mean_time, :total_db_time, :mean_db_time, :mean_rendering_time, :blockers, :hourly_spread, :errors]
|
40
|
-
@amount = options[:amount] || 10
|
41
|
-
|
42
|
-
@line_types = @log_parser_class::LOG_LINES.keys
|
43
|
-
@line_types = [:completed] if @fast
|
44
|
-
end
|
45
|
-
|
46
|
-
|
47
|
-
# Substitutes variable elements in a url (like the id field) with a fixed string (like ":id")
|
48
|
-
# This is used to aggregate simular requests.
|
49
|
-
# <tt>request</tt> The request to evaluate.
|
50
|
-
# Returns uniformed url string.
|
51
|
-
# Raises on mailformed request.
|
52
|
-
def request_hasher(request)
|
53
|
-
if request[:url]
|
54
|
-
url = request[:url].downcase.split(/^http[s]?:\/\/[A-z0-9\.-]+/).last.split('?').first # only the relevant URL part
|
55
|
-
url << '/' if url[-1] != '/'[0] && url.length > 1 # pad a trailing slash for consistency
|
56
|
-
|
57
|
-
url.gsub!(/\/\d+-\d+-\d+(\/|$)/, '/:date') # Combine all (year-month-day) queries
|
58
|
-
url.gsub!(/\/\d+-\d+(\/|$)/, '/:month') # Combine all date (year-month) queries
|
59
|
-
url.gsub!(/\/\d+[\w-]*/, '/:id') # replace identifiers in URLs
|
60
|
-
|
61
|
-
return url
|
62
|
-
elsif request[:controller] && request[:action]
|
63
|
-
return "#{request[:controller]}##{request[:action]}"
|
64
|
-
else
|
65
|
-
raise 'Cannot hash this request! ' + request.inspect
|
66
|
-
end
|
67
|
-
end
|
68
|
-
|
69
|
-
# Print results using a ASCII table.
|
70
|
-
# <tt>summarizer</tt> The summarizer containg information to draw the table.
|
71
|
-
# <tt>field</tt> The field containing the data to be printed
|
72
|
-
# <tt>amount</tt> The length of the table (defaults to 20)
|
73
|
-
def print_table(summarizer, field, amount = @amount)
|
74
|
-
summarizer.sort_actions_by(field).reverse[0, amount.to_i].each do |a|
|
75
|
-
# As we show count by default, show totaltime if we sort by count
|
76
|
-
field = :total_time if field == :count
|
77
|
-
|
78
|
-
puts "%-50s: %10.03fs [#{green("%d requests")}]" % [a[0], a[1][field], a[1][:count]]
|
79
|
-
end
|
80
|
-
end
|
81
|
-
|
82
|
-
# Execute the analyze
|
83
|
-
def analyze_this(files = [])
|
84
|
-
# Walk through al the files given via the arguments.
|
85
|
-
files.each do |log_file|
|
86
|
-
puts "Processing #{@line_types.join(', ')} log lines from #{log_file}..."
|
87
|
-
|
88
|
-
parser = @log_parser_class.new(log_file)
|
89
|
-
|
90
|
-
# add progress bar
|
91
|
-
unless @fast
|
92
|
-
pbar = ProgressBar.new(green(log_file), File.size(log_file))
|
93
|
-
parser.progress { |pos, total| (pos == :finished) ? pbar.finish : pbar.set(pos) }
|
94
|
-
end
|
95
|
-
|
96
|
-
parser.each(*line_types) do |request|
|
97
|
-
@summarizer.group(request) { |r| request_hasher(r) }
|
98
|
-
end
|
99
|
-
end
|
100
|
-
|
101
|
-
# Select the reports to output and generate them.
|
102
|
-
@output_reports.each do |report|
|
103
|
-
report_location = "#{File.dirname(__FILE__)}/../output/#{report}.rb"
|
104
|
-
|
105
|
-
if File.exist?(report_location)
|
106
|
-
eval File.read(report_location)
|
107
|
-
else
|
108
|
-
puts "\nERROR: Output report #{report} not found!"
|
109
|
-
end
|
110
|
-
end
|
111
|
-
end
|
112
|
-
|
113
|
-
def analyze_with_virtual_mongrels(files = [])
|
114
|
-
# Walk through al the files given via the arguments.
|
115
|
-
files.each do |log_file|
|
116
|
-
puts "Processing #{@line_types.join(', ')} log lines from #{log_file}..."
|
117
|
-
|
118
|
-
parser = @log_parser_class.new(log_file)
|
119
|
-
|
120
|
-
virtual_mongrels = []
|
121
|
-
|
122
|
-
line = 0
|
123
|
-
|
124
|
-
parser.each(*line_types) do |request|
|
125
|
-
line += 1
|
126
|
-
|
127
|
-
puts "Number of mongrels: #{virtual_mongrels.length}"
|
128
|
-
puts "Line number: #{line}"
|
129
|
-
|
130
|
-
case request[:type]
|
131
|
-
when :started
|
132
|
-
puts 'Spawned new virtual mongrel'
|
133
|
-
new_mongrel = VirtualMongrel.new(:start_line => line, :calculate_database => @guess_database_time, :running_mongrels => virtual_mongrels.length + 1)
|
134
|
-
new_mongrel.group(request)
|
135
|
-
virtual_mongrels << new_mongrel
|
136
|
-
else
|
137
|
-
completed_mongrel = virtual_mongrels.first
|
138
|
-
completed_mongrel.group(request)
|
139
|
-
completed_mongrel.update_running_mongrels(virtual_mongrels.length)
|
140
|
-
completed_mongrel.save
|
141
|
-
end
|
142
|
-
|
143
|
-
keep_virtual_mongrels = []
|
144
|
-
|
145
|
-
virtual_mongrels.each do |mongrel|
|
146
|
-
if mongrel.die_line >= line && mongrel.status == :started
|
147
|
-
keep_virtual_mongrels << mongrel
|
148
|
-
else
|
149
|
-
puts 'Destroyed virtual mongrel!'
|
150
|
-
puts ""
|
151
|
-
|
152
|
-
end
|
153
|
-
end
|
154
|
-
|
155
|
-
virtual_mongrels = keep_virtual_mongrels
|
156
|
-
|
157
|
-
end
|
158
|
-
end
|
159
|
-
|
160
|
-
end
|
161
|
-
end
|
1
|
+
require 'date'
|
2
|
+
require File.dirname(__FILE__) + '/cli/progressbar'
|
3
|
+
require File.dirname(__FILE__) + '/cli/bashcolorizer'
|
4
|
+
|
5
|
+
require File.dirname(__FILE__) + '/request_log_analyzer/file_format'
|
6
|
+
require File.dirname(__FILE__) + '/request_log_analyzer/line_definition'
|
7
|
+
require File.dirname(__FILE__) + '/request_log_analyzer/request'
|
8
|
+
require File.dirname(__FILE__) + '/request_log_analyzer/log_parser'
|
9
|
+
require File.dirname(__FILE__) + '/request_log_analyzer/aggregator/base'
|
10
|
+
require File.dirname(__FILE__) + '/request_log_analyzer/aggregator/summarizer'
|
11
|
+
require File.dirname(__FILE__) + '/request_log_analyzer/filter/base'
|
12
|
+
require File.dirname(__FILE__) + '/request_log_analyzer/controller'
|
13
|
+
require File.dirname(__FILE__) + '/request_log_analyzer/source/base'
|
14
|
+
require File.dirname(__FILE__) + '/request_log_analyzer/source/log_file'
|
data/spec/controller_spec.rb
CHANGED
@@ -4,14 +4,15 @@ describe RequestLogAnalyzer::Controller do
|
|
4
4
|
|
5
5
|
include RequestLogAnalyzerSpecHelper
|
6
6
|
|
7
|
-
it "should include the file format module" do
|
8
|
-
|
9
|
-
|
10
|
-
end
|
7
|
+
# it "should include the file format module" do
|
8
|
+
# controller = RequestLogAnalyzer::Controller.new(:rails)
|
9
|
+
# (class << controller; self; end).ancestors.include?(RequestLogAnalyzer::FileFormat::Rails)
|
10
|
+
# end
|
11
11
|
|
12
12
|
it "should call the aggregators when run" do
|
13
|
-
|
14
|
-
|
13
|
+
file_format = RequestLogAnalyzer::FileFormat.load(:rails)
|
14
|
+
source = RequestLogAnalyzer::Source::LogFile.new(file_format, :source_files => log_fixture(:rails_1x))
|
15
|
+
controller = RequestLogAnalyzer::Controller.new(source)
|
15
16
|
|
16
17
|
mock_aggregator = mock('aggregator')
|
17
18
|
mock_aggregator.should_receive(:prepare).once.ordered
|
@@ -30,4 +31,10 @@ describe RequestLogAnalyzer::Controller do
|
|
30
31
|
controller.run!
|
31
32
|
end
|
32
33
|
|
34
|
+
it "should run well from the command line" do
|
35
|
+
temp_file = "#{File.dirname(__FILE__)}/fixtures/temp.txt"
|
36
|
+
system("#{File.dirname(__FILE__)}/../bin/request-log-analyzer #{log_fixture(:rails_1x)} > #{temp_file}").should be_true
|
37
|
+
File.unlink(temp_file)
|
38
|
+
end
|
39
|
+
|
33
40
|
end
|
@@ -8,7 +8,8 @@ describe RequestLogAnalyzer::Aggregator::Database, "schema creation" do
|
|
8
8
|
include RequestLogAnalyzerSpecHelper
|
9
9
|
|
10
10
|
before(:each) do
|
11
|
-
|
11
|
+
log_parser = RequestLogAnalyzer::LogParser.new(spec_format)
|
12
|
+
@database_inserter = RequestLogAnalyzer::Aggregator::Database.new(log_parser, :database => TEST_DATABASE_FILE)
|
12
13
|
end
|
13
14
|
|
14
15
|
after(:each) do
|
@@ -26,7 +27,7 @@ describe RequestLogAnalyzer::Aggregator::Database, "schema creation" do
|
|
26
27
|
it "should create the default table names" do
|
27
28
|
@database_inserter.prepare
|
28
29
|
@database_inserter.file_format.line_definitions.each do |name, definition|
|
29
|
-
klass =
|
30
|
+
klass = SpecFormat.const_get("#{name}_line".camelize)
|
30
31
|
klass.column_names.should include('id')
|
31
32
|
klass.column_names.should include('lineno')
|
32
33
|
klass.column_names.should include('request_id')
|
@@ -36,21 +37,23 @@ describe RequestLogAnalyzer::Aggregator::Database, "schema creation" do
|
|
36
37
|
it "should create the correct fields in the table" do
|
37
38
|
@database_inserter.prepare
|
38
39
|
|
39
|
-
|
40
|
-
|
41
|
-
|
40
|
+
SpecFormat::FirstLine.column_names.should include('request_no')
|
41
|
+
SpecFormat::LastLine.column_names.should include('request_no')
|
42
|
+
SpecFormat::TestLine.column_names.should include('test_capture')
|
42
43
|
end
|
43
44
|
|
44
45
|
end
|
45
46
|
|
46
47
|
describe RequestLogAnalyzer::Aggregator::Database, "record insertion" do
|
47
|
-
|
48
|
+
include RequestLogAnalyzerSpecHelper
|
49
|
+
|
48
50
|
before(:each) do
|
49
|
-
|
51
|
+
log_parser = RequestLogAnalyzer::LogParser.new(spec_format)
|
52
|
+
@database_inserter = RequestLogAnalyzer::Aggregator::Database.new(log_parser, :database => TEST_DATABASE_FILE)
|
50
53
|
@database_inserter.prepare
|
51
54
|
|
52
|
-
@single = RequestLogAnalyzer::Request.create(
|
53
|
-
@combined = RequestLogAnalyzer::Request.create(
|
55
|
+
@single = RequestLogAnalyzer::Request.create(spec_format, {:line_type => :first, :request_no => 564})
|
56
|
+
@combined = RequestLogAnalyzer::Request.create(spec_format,
|
54
57
|
{:line_type => :first, :request_no => 564},
|
55
58
|
{:line_type => :test, :test_capture => "awesome"},
|
56
59
|
{:line_type => :test, :test_capture => "indeed"},
|
@@ -62,19 +65,19 @@ describe RequestLogAnalyzer::Aggregator::Database, "record insertion" do
|
|
62
65
|
end
|
63
66
|
|
64
67
|
it "should insert a record in the relevant table" do
|
65
|
-
|
68
|
+
SpecFormat::FirstLine.should_receive(:create!).with(hash_including(:request_no => 564))
|
66
69
|
@database_inserter.aggregate(@single)
|
67
70
|
end
|
68
71
|
|
69
72
|
it "should insert records in all relevant tables" do
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
+
SpecFormat::FirstLine.should_receive(:create!).with(hash_including(:request_no => 564)).once
|
74
|
+
SpecFormat::TestLine.should_receive(:create!).twice
|
75
|
+
SpecFormat::LastLine.should_receive(:create!).with(hash_including(:request_no => 564)).once
|
73
76
|
@database_inserter.aggregate(@combined)
|
74
77
|
end
|
75
78
|
|
76
79
|
it "should log a warning in the warnings table" do
|
77
|
-
|
80
|
+
SpecFormat::Warning.should_receive(:create!).with(hash_including(:warning_type => 'test_warning'))
|
78
81
|
@database_inserter.warning(:test_warning, "Testing the warning system", 12)
|
79
82
|
end
|
80
83
|
|
@@ -1,12 +1,12 @@
|
|
1
1
|
require File.dirname(__FILE__) + '/spec_helper'
|
2
2
|
|
3
|
-
describe RequestLogAnalyzer::
|
3
|
+
describe RequestLogAnalyzer::LineDefinition, :parsing do
|
4
4
|
|
5
5
|
before(:each) do
|
6
|
-
@line_definition = RequestLogAnalyzer::
|
6
|
+
@line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
|
7
7
|
:teaser => /Testing /,
|
8
8
|
:regexp => /Testing (\w+), tries\: (\d+)/,
|
9
|
-
:captures => [{:what => :string}, {:tries => :integer}]
|
9
|
+
:captures => [{ :name => :what, :type => :string }, { :name => :tries, :type => :integer }]
|
10
10
|
})
|
11
11
|
end
|
12
12
|
|
@@ -31,4 +31,94 @@ describe RequestLogAnalyzer::FileFormat::LineDefinition do
|
|
31
31
|
it "should return a hash with :line_type set" do
|
32
32
|
@line_definition.matches("Testing LineDefinition, tries: 123")[:line_type].should == :test
|
33
33
|
end
|
34
|
-
end
|
34
|
+
end
|
35
|
+
|
36
|
+
describe RequestLogAnalyzer::LineDefinition, :anonymizing_basics do
|
37
|
+
before(:each) do
|
38
|
+
@line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
|
39
|
+
:teaser => /Anonymize /,
|
40
|
+
:regexp => /Anonymize (\w+)!/,
|
41
|
+
:captures => [{ :name => :what, :type => :string }]
|
42
|
+
})
|
43
|
+
end
|
44
|
+
|
45
|
+
it "should return nil if the teaser does not match" do
|
46
|
+
@line_definition.anonymize("Nonsense").should be_nil
|
47
|
+
end
|
48
|
+
|
49
|
+
it "should return nil if no teaser exists and the regexp doesn't match" do
|
50
|
+
line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
|
51
|
+
:regexp => /Anonymize!/, :captures => []})
|
52
|
+
|
53
|
+
line_definition.anonymize('nonsense').should be_nil
|
54
|
+
end
|
55
|
+
|
56
|
+
it "should return itself if only the teaser matches" do
|
57
|
+
@line_definition.anonymize("Anonymize 456").should == "Anonymize 456"
|
58
|
+
end
|
59
|
+
|
60
|
+
it "should return an empty string if the teaser matches and discard_teaser_lines is set" do
|
61
|
+
@line_definition.anonymize("Anonymize 456", :discard_teaser_lines => true).should == ""
|
62
|
+
end
|
63
|
+
|
64
|
+
it "should return a string if the line matches" do
|
65
|
+
@line_definition.anonymize("Anonymize anonymizing!").should be_kind_of(String)
|
66
|
+
end
|
67
|
+
|
68
|
+
it "should not anonymize :what" do
|
69
|
+
@line_definition.anonymize("Anonymize anonymizing!").should == "Anonymize anonymizing!"
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
describe RequestLogAnalyzer::LineDefinition, :anonymizing_specifics do
|
74
|
+
|
75
|
+
it "should anonymize completely if anonymize is true" do
|
76
|
+
@line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
|
77
|
+
:regexp => /Anonymize (.+)!/, :captures => [{ :name => :what, :type => :string, :anonymize => true }]})
|
78
|
+
|
79
|
+
@line_definition.anonymize("Anonymize 1.2.3.4!").should == "Anonymize ***!"
|
80
|
+
end
|
81
|
+
|
82
|
+
it "should anonymize a URL" do
|
83
|
+
@line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
|
84
|
+
:regexp => /Anonymize (.+)!/, :captures => [{ :name => :what, :type => :string, :anonymize => :url }]})
|
85
|
+
|
86
|
+
@line_definition.anonymize("Anonymize https://www.not-anonymous.com/path/to/file.html!").should == "Anonymize http://example.com/path/to/file.html!"
|
87
|
+
end
|
88
|
+
|
89
|
+
it "should anonymize an IP address" do
|
90
|
+
@line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
|
91
|
+
:regexp => /Anonymize (.+)!/, :captures => [{ :name => :what, :type => :string, :anonymize => :ip }]})
|
92
|
+
|
93
|
+
@line_definition.anonymize("Anonymize 1.2.3.4!").should == "Anonymize 127.0.0.1!"
|
94
|
+
end
|
95
|
+
|
96
|
+
it "should anonymize completely if the anonymizer is unknown" do
|
97
|
+
@line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
|
98
|
+
:regexp => /Anonymize (.+)!/, :captures => [{ :name => :what, :type => :string, :anonymize => :unknown }]})
|
99
|
+
|
100
|
+
@line_definition.anonymize("Anonymize 1.2.3.4!").should == "Anonymize ***!"
|
101
|
+
end
|
102
|
+
|
103
|
+
it "should anonymize an integer slightly" do
|
104
|
+
@line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
|
105
|
+
:regexp => /Anonymize (.+)!/, :captures => [{ :name => :what, :type => :integer, :anonymize => :slightly }]})
|
106
|
+
|
107
|
+
@line_definition.anonymize("Anonymize 1234!").should =~ /Anonymize \d{3,4}\!/
|
108
|
+
end
|
109
|
+
|
110
|
+
it "should anonymize an integer slightly" do
|
111
|
+
@line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
|
112
|
+
:regexp => /Anonymize (.+)!/, :captures => [{ :name => :what, :type => :integer, :anonymize => :slightly }]})
|
113
|
+
|
114
|
+
@line_definition.anonymize("Anonymize 1234!").should =~ /Anonymize \d{3,4}\!/
|
115
|
+
end
|
116
|
+
|
117
|
+
it "should anonymize an double slightly" do
|
118
|
+
@line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
|
119
|
+
:regexp => /Anonymize (.+)!/, :captures => [{ :name => :what, :type => :double, :anonymize => :slightly }]})
|
120
|
+
|
121
|
+
@line_definition.anonymize("Anonymize 1.3!").should =~ /Anonymize 1\.\d+\!/
|
122
|
+
end
|
123
|
+
|
124
|
+
end
|
data/spec/log_parser_spec.rb
CHANGED
@@ -5,18 +5,18 @@ describe RequestLogAnalyzer::LogParser, :single_line_requests do
|
|
5
5
|
include RequestLogAnalyzerSpecHelper
|
6
6
|
|
7
7
|
before(:each) do
|
8
|
-
@log_parser = RequestLogAnalyzer::LogParser.new(
|
8
|
+
@log_parser = RequestLogAnalyzer::LogParser.new(spec_format)
|
9
9
|
end
|
10
10
|
|
11
11
|
it "should have line definitions" do
|
12
12
|
@log_parser.file_format.line_definitions.should_not be_empty
|
13
13
|
end
|
14
14
|
|
15
|
-
it "should have include the language specific hooks in the instance, not in the class" do
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
end
|
15
|
+
# it "should have include the language specific hooks in the instance, not in the class" do
|
16
|
+
# metaclass = (class << @log_parser; self; end)
|
17
|
+
# metaclass.ancestors.should include(TestFileFormat::LogParser)
|
18
|
+
# @log_parser.class.ancestors.should_not include(TestFileFormat::LogParser)
|
19
|
+
# end
|
20
20
|
|
21
21
|
it "should parse a stream and find valid requests" do
|
22
22
|
io = File.new(log_fixture(:test_file_format), 'r')
|
@@ -38,7 +38,7 @@ describe RequestLogAnalyzer::LogParser, :combined_requests do
|
|
38
38
|
include RequestLogAnalyzerSpecHelper
|
39
39
|
|
40
40
|
before(:each) do
|
41
|
-
@log_parser = RequestLogAnalyzer::LogParser.new(
|
41
|
+
@log_parser = RequestLogAnalyzer::LogParser.new(spec_format, :combined_requests => true)
|
42
42
|
end
|
43
43
|
|
44
44
|
it "should have multiple line definitions" do
|
@@ -74,19 +74,17 @@ describe RequestLogAnalyzer::LogParser, :warnings do
|
|
74
74
|
include RequestLogAnalyzerSpecHelper
|
75
75
|
|
76
76
|
it "should warn about teaser matching problems" do
|
77
|
-
@log_parser = RequestLogAnalyzer::LogParser.new(
|
77
|
+
@log_parser = RequestLogAnalyzer::LogParser.new(spec_format)
|
78
78
|
@log_parser.should_receive(:warn).with(:teaser_check_failed, anything).exactly(5).times
|
79
79
|
@log_parser.parse_file(log_fixture(:test_file_format))
|
80
80
|
end
|
81
81
|
|
82
82
|
it "should warn about unmatching request headers and footers" do
|
83
|
-
@log_parser = RequestLogAnalyzer::LogParser.new(
|
83
|
+
@log_parser = RequestLogAnalyzer::LogParser.new(spec_format, :combined_requests => true)
|
84
84
|
|
85
85
|
@log_parser.should_receive(:warn).with(:unclosed_request, anything).at_least(1).times
|
86
86
|
@log_parser.should_receive(:warn).with(:no_current_request, anything).at_least(1).times
|
87
87
|
@log_parser.should_not_receive(:handle_request)
|
88
88
|
@log_parser.parse_file(log_fixture(:test_order))
|
89
|
-
|
90
89
|
end
|
91
|
-
|
92
90
|
end
|