wvanbergen-request-log-analyzer 0.3.4 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README.textile +23 -10
- data/TODO +5 -0
- data/bin/request-log-analyzer +96 -20
- data/lib/request_log_analyzer.rb +14 -161
- data/spec/controller_spec.rb +13 -6
- data/spec/database_inserter_spec.rb +17 -14
- data/spec/line_definition_spec.rb +94 -4
- data/spec/log_parser_spec.rb +9 -11
- data/spec/merb_format_spec.rb +58 -58
- data/spec/rails_format_spec.rb +4 -4
- data/spec/request_spec.rb +6 -2
- data/spec/spec_helper.rb +22 -4
- data/spec/summarizer_spec.rb +0 -94
- data/tasks/github-gem.rake +1 -1
- data/test/rails_log_parser_test.rb +0 -5
- metadata +2 -2
- data/lib/base/summarizer.rb +0 -85
- data/lib/bashcolorizer.rb +0 -60
- data/lib/command_line/exceptions.rb +0 -37
- data/lib/merb_analyzer/summarizer.rb +0 -61
- data/lib/rails_analyzer/summarizer.rb +0 -81
- data/lib/rails_analyzer/virtual_mongrel.rb +0 -91
- data/lib/ruby-progressbar/progressbar.en.rd +0 -103
- data/lib/ruby-progressbar/progressbar.ja.rd +0 -100
- data/lib/ruby-progressbar/progressbar.rb +0 -236
- data/output/blockers.rb +0 -9
- data/output/errors.rb +0 -8
- data/output/hourly_spread.rb +0 -28
- data/output/mean_db_time.rb +0 -6
- data/output/mean_rendering_time.rb +0 -6
- data/output/mean_time.rb +0 -6
- data/output/most_requested.rb +0 -5
- data/output/timespan.rb +0 -18
- data/output/total_db_time.rb +0 -5
- data/output/total_time.rb +0 -5
- data/output/usage.rb +0 -16
- data/test/base_summarizer_test.rb +0 -30
data/README.textile
CHANGED
@@ -17,16 +17,29 @@ h2. Installation
|
|
17
17
|
h2. Usage
|
18
18
|
|
19
19
|
<pre>
|
20
|
-
Usage: request-log-analyzer [
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
20
|
+
Usage: request-log-analyzer [LOGFILES*] <OPTIONS>
|
21
|
+
|
22
|
+
Input options:
|
23
|
+
--format <format>, -f: Uses the specified log file format. Defaults to rails.
|
24
|
+
--after <date> Only consider requests from <date> or later.
|
25
|
+
--before <date> Only consider requests before <date>.
|
26
|
+
--select <field> <value> Only consider requests where <field> matches <value>.
|
27
|
+
--reject <field> <value> Only consider requests where <field> does not match <value>.
|
28
|
+
|
29
|
+
Output options:
|
30
|
+
--boring, -b Output reports without ASCII colors.
|
31
|
+
--database <filename>, -d: Creates an SQLite3 database of all the parsed request information.
|
32
|
+
--debug Print debug information while parsing.
|
33
|
+
|
34
|
+
Examples:
|
35
|
+
request-log-analyzer development.log
|
36
|
+
request-log-analyzer -z mongrel.0.log mongrel.1.log mongrel.2.log
|
37
|
+
request-log-analyzer --format merb -d requests.db production.log
|
38
|
+
|
39
|
+
To install rake tasks in your Rails application,
|
40
|
+
run the following command in your application's root directory:
|
41
|
+
|
42
|
+
request-log-analyzer install rails
|
30
43
|
</pre>
|
31
44
|
|
32
45
|
|
data/TODO
CHANGED
@@ -5,6 +5,11 @@ Contact willem AT vanbergen DOT org if you want to help out with the development
|
|
5
5
|
General:
|
6
6
|
- Add more tests / specs
|
7
7
|
|
8
|
+
Parameters:
|
9
|
+
- Add commandline parameters
|
10
|
+
--version, -v -> This should support github - last version checking
|
11
|
+
--help, -h -> Show usage
|
12
|
+
|
8
13
|
Datamining:
|
9
14
|
- Add query functionality for the resulting database file (interactive reports?)
|
10
15
|
- Link request processing line to request completed line (VirtualMongrel?)
|
data/bin/request-log-analyzer
CHANGED
@@ -1,31 +1,94 @@
|
|
1
1
|
#!/usr/bin/ruby
|
2
2
|
require File.dirname(__FILE__) + '/../lib/request_log_analyzer'
|
3
|
-
require File.dirname(__FILE__) + '/../lib/
|
3
|
+
require File.dirname(__FILE__) + '/../lib/cli/command_line_arguments'
|
4
4
|
|
5
|
-
|
5
|
+
def terminal_width(default = 81)
|
6
|
+
IO.popen('stty -a') do |pipe|
|
7
|
+
column_line = pipe.detect { |line| /(\d+) columns/ =~ line }
|
8
|
+
width = column_line ? $1.to_i : default
|
9
|
+
end
|
10
|
+
rescue
|
11
|
+
default
|
12
|
+
end
|
6
13
|
|
7
14
|
# Parse the arguments given via commandline
|
8
15
|
begin
|
9
16
|
arguments = CommandLine::Arguments.parse do |command_line|
|
10
|
-
|
11
|
-
command_line.
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
command_line.
|
16
|
-
|
17
|
-
|
17
|
+
|
18
|
+
command_line.command(:install) do |install|
|
19
|
+
install.parameters = 1
|
20
|
+
end
|
21
|
+
|
22
|
+
command_line.command(:strip) do |strip|
|
23
|
+
strip.minimum_parameters = 1
|
24
|
+
strip.option(:format, :alias => :f, :default => 'rails')
|
25
|
+
strip.option(:output, :alias => :o)
|
26
|
+
strip.switch(:discard_teaser_lines, :t)
|
27
|
+
strip.switch(:keep_junk_lines, :j)
|
28
|
+
end
|
29
|
+
|
30
|
+
command_line.command(:anonymize) do |anonymize|
|
31
|
+
anonymize.minimum_parameters = 1
|
32
|
+
anonymize.option(:format, :alias => :f, :default => 'rails')
|
33
|
+
anonymize.option(:output, :alias => :o)
|
34
|
+
anonymize.switch(:discard_teaser_lines, :t)
|
35
|
+
anonymize.switch(:keep_junk_lines, :j)
|
36
|
+
end
|
37
|
+
|
38
|
+
command_line.option(:format, :alias => :f, :default => 'rails')
|
39
|
+
command_line.option(:file, :alias => :e)
|
40
|
+
command_line.switch(:single_lines, :s)
|
41
|
+
command_line.switch(:assume_correct_order)
|
42
|
+
|
43
|
+
command_line.option(:aggregator, :alias => :a, :multiple => true)
|
44
|
+
command_line.option(:database, :alias => :d)
|
45
|
+
|
46
|
+
# filtering options
|
47
|
+
command_line.option(:select, :multiple => true, :parameters => 2)
|
48
|
+
command_line.option(:reject, :multiple => true, :parameters => 2)
|
49
|
+
command_line.option(:after)
|
50
|
+
command_line.option(:before)
|
51
|
+
|
52
|
+
command_line.switch(:boring, :b)
|
53
|
+
command_line.option(:report_width, :default => terminal_width - 1)
|
54
|
+
|
55
|
+
command_line.switch(:debug)
|
56
|
+
|
57
|
+
command_line.minimum_parameters = 1
|
18
58
|
end
|
19
59
|
|
20
60
|
rescue CommandLine::Error => e
|
21
|
-
puts "ARGUMENT ERROR: " + e.message
|
61
|
+
puts "ARGUMENT ERROR: " + e.message if e.message
|
22
62
|
puts
|
23
|
-
|
63
|
+
puts "Usage: request-log-analyzer [LOGFILES*] <OPTIONS>"
|
64
|
+
puts
|
65
|
+
puts "Input options:"
|
66
|
+
puts " --format <format>, -f: Uses the specified log file format. Defaults to rails."
|
67
|
+
puts " --after <date> Only consider requests from <date> or later."
|
68
|
+
puts " --before <date> Only consider requests before <date>."
|
69
|
+
puts " --select <field> <value> Only consider requests where <field> matches <value>."
|
70
|
+
puts " --reject <field> <value> Only consider requests where <field> does not match <value>."
|
71
|
+
puts
|
72
|
+
puts "Output options:"
|
73
|
+
puts " --boring, -b Output reports without ASCII colors."
|
74
|
+
puts " --database <filename>, -d: Creates an SQLite3 database of all the parsed request information."
|
75
|
+
puts " --debug Print debug information while parsing."
|
76
|
+
puts " --file <filename> Output to file."
|
77
|
+
puts
|
78
|
+
puts "Examples:"
|
79
|
+
puts " request-log-analyzer development.log"
|
80
|
+
puts " request-log-analyzer -b mongrel.0.log mongrel.1.log mongrel.2.log "
|
81
|
+
puts " request-log-analyzer --format merb -d requests.db production.log"
|
82
|
+
puts
|
83
|
+
puts "To install rake tasks in your Rails application, "
|
84
|
+
puts "run the following command in your application's root directory:"
|
85
|
+
puts
|
86
|
+
puts " request-log-analyzer install rails"
|
24
87
|
exit(0)
|
25
88
|
end
|
26
89
|
|
27
|
-
|
28
|
-
if
|
90
|
+
def install_rake_tasks(install_type)
|
91
|
+
if install_type == 'rails'
|
29
92
|
require 'ftools'
|
30
93
|
if File.directory?('./lib/tasks/')
|
31
94
|
File.copy(File.dirname(__FILE__) + '/../tasks/request_log_analyzer.rake', './lib/tasks/request_log_analyze.rake')
|
@@ -35,14 +98,27 @@ if arguments[:install]
|
|
35
98
|
puts "Cannot find /lib/tasks folder. Are you in your Rails directory?"
|
36
99
|
puts "Installation aborted."
|
37
100
|
end
|
101
|
+
else
|
102
|
+
raise "Cannot perform this install type! (#{install_type})"
|
38
103
|
end
|
39
|
-
exit(0)
|
40
104
|
end
|
41
105
|
|
42
|
-
$colorize = true if arguments[:colorize]
|
43
|
-
|
44
|
-
# Run the request_log_analyzer!
|
45
|
-
request_log_analyzer = RequestLogAnalyzer.new(arguments)
|
46
|
-
request_log_analyzer.analyze_this(arguments.files)
|
47
106
|
|
107
|
+
case arguments.command
|
108
|
+
when :install
|
109
|
+
install_rake_tasks(arguments.parameters[0])
|
110
|
+
when :strip
|
111
|
+
require File.dirname(__FILE__) + '/../lib/request_log_analyzer/log_processor'
|
112
|
+
RequestLogAnalyzer::LogProcessor.build(:strip, arguments).run!
|
113
|
+
when :anonymize
|
114
|
+
require File.dirname(__FILE__) + '/../lib/request_log_analyzer/log_processor'
|
115
|
+
RequestLogAnalyzer::LogProcessor.build(:anonymize, arguments).run!
|
116
|
+
else
|
117
|
+
puts "Request log analyzer, by Willem van Bergen and Bart ten Brinke - Version 0.4.0\n\n"
|
118
|
+
|
119
|
+
# Run the request_log_analyzer!
|
120
|
+
RequestLogAnalyzer::Controller.build(arguments, terminal_width).run!
|
48
121
|
|
122
|
+
puts
|
123
|
+
puts "Thanks for using request-log-analyzer"
|
124
|
+
end
|
data/lib/request_log_analyzer.rb
CHANGED
@@ -1,161 +1,14 @@
|
|
1
|
-
require
|
2
|
-
require File.dirname(__FILE__) + '
|
3
|
-
require File.dirname(__FILE__) + '
|
4
|
-
|
5
|
-
require File.dirname(__FILE__) + '
|
6
|
-
require File.dirname(__FILE__) + '
|
7
|
-
require File.dirname(__FILE__) + '
|
8
|
-
require File.dirname(__FILE__) + '
|
9
|
-
require File.dirname(__FILE__) + '
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
attr_reader :arguments
|
16
|
-
attr_reader :line_types
|
17
|
-
attr_reader :amount
|
18
|
-
attr_reader :fast
|
19
|
-
|
20
|
-
# Initializer. Sets global variables
|
21
|
-
# Options
|
22
|
-
# *<tt>:fast</tt> Only look at request initializers. Faster, but not not all outputs are shown.
|
23
|
-
# *<tt>:guess_database_time</tt> Guess database time if it it not in the log (default for Rails produciton).
|
24
|
-
# *<tt>:merb</tt> Use merb summarizer and parser classes instead of rails.
|
25
|
-
# *<tt>:output_reports</tt> Comma separated string of requested output reports
|
26
|
-
# *<tt>:amount</tt> Amount of lines shown for each result table. Defaults to 10.
|
27
|
-
def initialize(options = {})
|
28
|
-
@fast = options[:fast] || false
|
29
|
-
@guess_database_time = options[:guess_database_time] || false
|
30
|
-
|
31
|
-
if options[:merb]
|
32
|
-
@summarizer = MerbAnalyzer::Summarizer.new(:calculate_database => @guess_database_time)
|
33
|
-
@log_parser_class = MerbAnalyzer::LogParser
|
34
|
-
else
|
35
|
-
@summarizer = RailsAnalyzer::Summarizer.new(:calculate_database => @guess_database_time)
|
36
|
-
@log_parser_class = RailsAnalyzer::LogParser
|
37
|
-
end
|
38
|
-
|
39
|
-
@output_reports = options[:output].split(', ') rescue [:timespan, :most_requested, :total_time, :mean_time, :total_db_time, :mean_db_time, :mean_rendering_time, :blockers, :hourly_spread, :errors]
|
40
|
-
@amount = options[:amount] || 10
|
41
|
-
|
42
|
-
@line_types = @log_parser_class::LOG_LINES.keys
|
43
|
-
@line_types = [:completed] if @fast
|
44
|
-
end
|
45
|
-
|
46
|
-
|
47
|
-
# Substitutes variable elements in a url (like the id field) with a fixed string (like ":id")
|
48
|
-
# This is used to aggregate simular requests.
|
49
|
-
# <tt>request</tt> The request to evaluate.
|
50
|
-
# Returns uniformed url string.
|
51
|
-
# Raises on mailformed request.
|
52
|
-
def request_hasher(request)
|
53
|
-
if request[:url]
|
54
|
-
url = request[:url].downcase.split(/^http[s]?:\/\/[A-z0-9\.-]+/).last.split('?').first # only the relevant URL part
|
55
|
-
url << '/' if url[-1] != '/'[0] && url.length > 1 # pad a trailing slash for consistency
|
56
|
-
|
57
|
-
url.gsub!(/\/\d+-\d+-\d+(\/|$)/, '/:date') # Combine all (year-month-day) queries
|
58
|
-
url.gsub!(/\/\d+-\d+(\/|$)/, '/:month') # Combine all date (year-month) queries
|
59
|
-
url.gsub!(/\/\d+[\w-]*/, '/:id') # replace identifiers in URLs
|
60
|
-
|
61
|
-
return url
|
62
|
-
elsif request[:controller] && request[:action]
|
63
|
-
return "#{request[:controller]}##{request[:action]}"
|
64
|
-
else
|
65
|
-
raise 'Cannot hash this request! ' + request.inspect
|
66
|
-
end
|
67
|
-
end
|
68
|
-
|
69
|
-
# Print results using a ASCII table.
|
70
|
-
# <tt>summarizer</tt> The summarizer containg information to draw the table.
|
71
|
-
# <tt>field</tt> The field containing the data to be printed
|
72
|
-
# <tt>amount</tt> The length of the table (defaults to 20)
|
73
|
-
def print_table(summarizer, field, amount = @amount)
|
74
|
-
summarizer.sort_actions_by(field).reverse[0, amount.to_i].each do |a|
|
75
|
-
# As we show count by default, show totaltime if we sort by count
|
76
|
-
field = :total_time if field == :count
|
77
|
-
|
78
|
-
puts "%-50s: %10.03fs [#{green("%d requests")}]" % [a[0], a[1][field], a[1][:count]]
|
79
|
-
end
|
80
|
-
end
|
81
|
-
|
82
|
-
# Execute the analyze
|
83
|
-
def analyze_this(files = [])
|
84
|
-
# Walk through al the files given via the arguments.
|
85
|
-
files.each do |log_file|
|
86
|
-
puts "Processing #{@line_types.join(', ')} log lines from #{log_file}..."
|
87
|
-
|
88
|
-
parser = @log_parser_class.new(log_file)
|
89
|
-
|
90
|
-
# add progress bar
|
91
|
-
unless @fast
|
92
|
-
pbar = ProgressBar.new(green(log_file), File.size(log_file))
|
93
|
-
parser.progress { |pos, total| (pos == :finished) ? pbar.finish : pbar.set(pos) }
|
94
|
-
end
|
95
|
-
|
96
|
-
parser.each(*line_types) do |request|
|
97
|
-
@summarizer.group(request) { |r| request_hasher(r) }
|
98
|
-
end
|
99
|
-
end
|
100
|
-
|
101
|
-
# Select the reports to output and generate them.
|
102
|
-
@output_reports.each do |report|
|
103
|
-
report_location = "#{File.dirname(__FILE__)}/../output/#{report}.rb"
|
104
|
-
|
105
|
-
if File.exist?(report_location)
|
106
|
-
eval File.read(report_location)
|
107
|
-
else
|
108
|
-
puts "\nERROR: Output report #{report} not found!"
|
109
|
-
end
|
110
|
-
end
|
111
|
-
end
|
112
|
-
|
113
|
-
def analyze_with_virtual_mongrels(files = [])
|
114
|
-
# Walk through al the files given via the arguments.
|
115
|
-
files.each do |log_file|
|
116
|
-
puts "Processing #{@line_types.join(', ')} log lines from #{log_file}..."
|
117
|
-
|
118
|
-
parser = @log_parser_class.new(log_file)
|
119
|
-
|
120
|
-
virtual_mongrels = []
|
121
|
-
|
122
|
-
line = 0
|
123
|
-
|
124
|
-
parser.each(*line_types) do |request|
|
125
|
-
line += 1
|
126
|
-
|
127
|
-
puts "Number of mongrels: #{virtual_mongrels.length}"
|
128
|
-
puts "Line number: #{line}"
|
129
|
-
|
130
|
-
case request[:type]
|
131
|
-
when :started
|
132
|
-
puts 'Spawned new virtual mongrel'
|
133
|
-
new_mongrel = VirtualMongrel.new(:start_line => line, :calculate_database => @guess_database_time, :running_mongrels => virtual_mongrels.length + 1)
|
134
|
-
new_mongrel.group(request)
|
135
|
-
virtual_mongrels << new_mongrel
|
136
|
-
else
|
137
|
-
completed_mongrel = virtual_mongrels.first
|
138
|
-
completed_mongrel.group(request)
|
139
|
-
completed_mongrel.update_running_mongrels(virtual_mongrels.length)
|
140
|
-
completed_mongrel.save
|
141
|
-
end
|
142
|
-
|
143
|
-
keep_virtual_mongrels = []
|
144
|
-
|
145
|
-
virtual_mongrels.each do |mongrel|
|
146
|
-
if mongrel.die_line >= line && mongrel.status == :started
|
147
|
-
keep_virtual_mongrels << mongrel
|
148
|
-
else
|
149
|
-
puts 'Destroyed virtual mongrel!'
|
150
|
-
puts ""
|
151
|
-
|
152
|
-
end
|
153
|
-
end
|
154
|
-
|
155
|
-
virtual_mongrels = keep_virtual_mongrels
|
156
|
-
|
157
|
-
end
|
158
|
-
end
|
159
|
-
|
160
|
-
end
|
161
|
-
end
|
1
|
+
require 'date'
|
2
|
+
require File.dirname(__FILE__) + '/cli/progressbar'
|
3
|
+
require File.dirname(__FILE__) + '/cli/bashcolorizer'
|
4
|
+
|
5
|
+
require File.dirname(__FILE__) + '/request_log_analyzer/file_format'
|
6
|
+
require File.dirname(__FILE__) + '/request_log_analyzer/line_definition'
|
7
|
+
require File.dirname(__FILE__) + '/request_log_analyzer/request'
|
8
|
+
require File.dirname(__FILE__) + '/request_log_analyzer/log_parser'
|
9
|
+
require File.dirname(__FILE__) + '/request_log_analyzer/aggregator/base'
|
10
|
+
require File.dirname(__FILE__) + '/request_log_analyzer/aggregator/summarizer'
|
11
|
+
require File.dirname(__FILE__) + '/request_log_analyzer/filter/base'
|
12
|
+
require File.dirname(__FILE__) + '/request_log_analyzer/controller'
|
13
|
+
require File.dirname(__FILE__) + '/request_log_analyzer/source/base'
|
14
|
+
require File.dirname(__FILE__) + '/request_log_analyzer/source/log_file'
|
data/spec/controller_spec.rb
CHANGED
@@ -4,14 +4,15 @@ describe RequestLogAnalyzer::Controller do
|
|
4
4
|
|
5
5
|
include RequestLogAnalyzerSpecHelper
|
6
6
|
|
7
|
-
it "should include the file format module" do
|
8
|
-
|
9
|
-
|
10
|
-
end
|
7
|
+
# it "should include the file format module" do
|
8
|
+
# controller = RequestLogAnalyzer::Controller.new(:rails)
|
9
|
+
# (class << controller; self; end).ancestors.include?(RequestLogAnalyzer::FileFormat::Rails)
|
10
|
+
# end
|
11
11
|
|
12
12
|
it "should call the aggregators when run" do
|
13
|
-
|
14
|
-
|
13
|
+
file_format = RequestLogAnalyzer::FileFormat.load(:rails)
|
14
|
+
source = RequestLogAnalyzer::Source::LogFile.new(file_format, :source_files => log_fixture(:rails_1x))
|
15
|
+
controller = RequestLogAnalyzer::Controller.new(source)
|
15
16
|
|
16
17
|
mock_aggregator = mock('aggregator')
|
17
18
|
mock_aggregator.should_receive(:prepare).once.ordered
|
@@ -30,4 +31,10 @@ describe RequestLogAnalyzer::Controller do
|
|
30
31
|
controller.run!
|
31
32
|
end
|
32
33
|
|
34
|
+
it "should run well from the command line" do
|
35
|
+
temp_file = "#{File.dirname(__FILE__)}/fixtures/temp.txt"
|
36
|
+
system("#{File.dirname(__FILE__)}/../bin/request-log-analyzer #{log_fixture(:rails_1x)} > #{temp_file}").should be_true
|
37
|
+
File.unlink(temp_file)
|
38
|
+
end
|
39
|
+
|
33
40
|
end
|
@@ -8,7 +8,8 @@ describe RequestLogAnalyzer::Aggregator::Database, "schema creation" do
|
|
8
8
|
include RequestLogAnalyzerSpecHelper
|
9
9
|
|
10
10
|
before(:each) do
|
11
|
-
|
11
|
+
log_parser = RequestLogAnalyzer::LogParser.new(spec_format)
|
12
|
+
@database_inserter = RequestLogAnalyzer::Aggregator::Database.new(log_parser, :database => TEST_DATABASE_FILE)
|
12
13
|
end
|
13
14
|
|
14
15
|
after(:each) do
|
@@ -26,7 +27,7 @@ describe RequestLogAnalyzer::Aggregator::Database, "schema creation" do
|
|
26
27
|
it "should create the default table names" do
|
27
28
|
@database_inserter.prepare
|
28
29
|
@database_inserter.file_format.line_definitions.each do |name, definition|
|
29
|
-
klass =
|
30
|
+
klass = SpecFormat.const_get("#{name}_line".camelize)
|
30
31
|
klass.column_names.should include('id')
|
31
32
|
klass.column_names.should include('lineno')
|
32
33
|
klass.column_names.should include('request_id')
|
@@ -36,21 +37,23 @@ describe RequestLogAnalyzer::Aggregator::Database, "schema creation" do
|
|
36
37
|
it "should create the correct fields in the table" do
|
37
38
|
@database_inserter.prepare
|
38
39
|
|
39
|
-
|
40
|
-
|
41
|
-
|
40
|
+
SpecFormat::FirstLine.column_names.should include('request_no')
|
41
|
+
SpecFormat::LastLine.column_names.should include('request_no')
|
42
|
+
SpecFormat::TestLine.column_names.should include('test_capture')
|
42
43
|
end
|
43
44
|
|
44
45
|
end
|
45
46
|
|
46
47
|
describe RequestLogAnalyzer::Aggregator::Database, "record insertion" do
|
47
|
-
|
48
|
+
include RequestLogAnalyzerSpecHelper
|
49
|
+
|
48
50
|
before(:each) do
|
49
|
-
|
51
|
+
log_parser = RequestLogAnalyzer::LogParser.new(spec_format)
|
52
|
+
@database_inserter = RequestLogAnalyzer::Aggregator::Database.new(log_parser, :database => TEST_DATABASE_FILE)
|
50
53
|
@database_inserter.prepare
|
51
54
|
|
52
|
-
@single = RequestLogAnalyzer::Request.create(
|
53
|
-
@combined = RequestLogAnalyzer::Request.create(
|
55
|
+
@single = RequestLogAnalyzer::Request.create(spec_format, {:line_type => :first, :request_no => 564})
|
56
|
+
@combined = RequestLogAnalyzer::Request.create(spec_format,
|
54
57
|
{:line_type => :first, :request_no => 564},
|
55
58
|
{:line_type => :test, :test_capture => "awesome"},
|
56
59
|
{:line_type => :test, :test_capture => "indeed"},
|
@@ -62,19 +65,19 @@ describe RequestLogAnalyzer::Aggregator::Database, "record insertion" do
|
|
62
65
|
end
|
63
66
|
|
64
67
|
it "should insert a record in the relevant table" do
|
65
|
-
|
68
|
+
SpecFormat::FirstLine.should_receive(:create!).with(hash_including(:request_no => 564))
|
66
69
|
@database_inserter.aggregate(@single)
|
67
70
|
end
|
68
71
|
|
69
72
|
it "should insert records in all relevant tables" do
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
+
SpecFormat::FirstLine.should_receive(:create!).with(hash_including(:request_no => 564)).once
|
74
|
+
SpecFormat::TestLine.should_receive(:create!).twice
|
75
|
+
SpecFormat::LastLine.should_receive(:create!).with(hash_including(:request_no => 564)).once
|
73
76
|
@database_inserter.aggregate(@combined)
|
74
77
|
end
|
75
78
|
|
76
79
|
it "should log a warning in the warnings table" do
|
77
|
-
|
80
|
+
SpecFormat::Warning.should_receive(:create!).with(hash_including(:warning_type => 'test_warning'))
|
78
81
|
@database_inserter.warning(:test_warning, "Testing the warning system", 12)
|
79
82
|
end
|
80
83
|
|
@@ -1,12 +1,12 @@
|
|
1
1
|
require File.dirname(__FILE__) + '/spec_helper'
|
2
2
|
|
3
|
-
describe RequestLogAnalyzer::
|
3
|
+
describe RequestLogAnalyzer::LineDefinition, :parsing do
|
4
4
|
|
5
5
|
before(:each) do
|
6
|
-
@line_definition = RequestLogAnalyzer::
|
6
|
+
@line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
|
7
7
|
:teaser => /Testing /,
|
8
8
|
:regexp => /Testing (\w+), tries\: (\d+)/,
|
9
|
-
:captures => [{:what => :string}, {:tries => :integer}]
|
9
|
+
:captures => [{ :name => :what, :type => :string }, { :name => :tries, :type => :integer }]
|
10
10
|
})
|
11
11
|
end
|
12
12
|
|
@@ -31,4 +31,94 @@ describe RequestLogAnalyzer::FileFormat::LineDefinition do
|
|
31
31
|
it "should return a hash with :line_type set" do
|
32
32
|
@line_definition.matches("Testing LineDefinition, tries: 123")[:line_type].should == :test
|
33
33
|
end
|
34
|
-
end
|
34
|
+
end
|
35
|
+
|
36
|
+
describe RequestLogAnalyzer::LineDefinition, :anonymizing_basics do
|
37
|
+
before(:each) do
|
38
|
+
@line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
|
39
|
+
:teaser => /Anonymize /,
|
40
|
+
:regexp => /Anonymize (\w+)!/,
|
41
|
+
:captures => [{ :name => :what, :type => :string }]
|
42
|
+
})
|
43
|
+
end
|
44
|
+
|
45
|
+
it "should return nil if the teaser does not match" do
|
46
|
+
@line_definition.anonymize("Nonsense").should be_nil
|
47
|
+
end
|
48
|
+
|
49
|
+
it "should return nil if no teaser exists and the regexp doesn't match" do
|
50
|
+
line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
|
51
|
+
:regexp => /Anonymize!/, :captures => []})
|
52
|
+
|
53
|
+
line_definition.anonymize('nonsense').should be_nil
|
54
|
+
end
|
55
|
+
|
56
|
+
it "should return itself if only the teaser matches" do
|
57
|
+
@line_definition.anonymize("Anonymize 456").should == "Anonymize 456"
|
58
|
+
end
|
59
|
+
|
60
|
+
it "should return an empty string if the teaser matches and discard_teaser_lines is set" do
|
61
|
+
@line_definition.anonymize("Anonymize 456", :discard_teaser_lines => true).should == ""
|
62
|
+
end
|
63
|
+
|
64
|
+
it "should return a string if the line matches" do
|
65
|
+
@line_definition.anonymize("Anonymize anonymizing!").should be_kind_of(String)
|
66
|
+
end
|
67
|
+
|
68
|
+
it "should not anonymize :what" do
|
69
|
+
@line_definition.anonymize("Anonymize anonymizing!").should == "Anonymize anonymizing!"
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
describe RequestLogAnalyzer::LineDefinition, :anonymizing_specifics do
|
74
|
+
|
75
|
+
it "should anonymize completely if anonymize is true" do
|
76
|
+
@line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
|
77
|
+
:regexp => /Anonymize (.+)!/, :captures => [{ :name => :what, :type => :string, :anonymize => true }]})
|
78
|
+
|
79
|
+
@line_definition.anonymize("Anonymize 1.2.3.4!").should == "Anonymize ***!"
|
80
|
+
end
|
81
|
+
|
82
|
+
it "should anonymize a URL" do
|
83
|
+
@line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
|
84
|
+
:regexp => /Anonymize (.+)!/, :captures => [{ :name => :what, :type => :string, :anonymize => :url }]})
|
85
|
+
|
86
|
+
@line_definition.anonymize("Anonymize https://www.not-anonymous.com/path/to/file.html!").should == "Anonymize http://example.com/path/to/file.html!"
|
87
|
+
end
|
88
|
+
|
89
|
+
it "should anonymize an IP address" do
|
90
|
+
@line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
|
91
|
+
:regexp => /Anonymize (.+)!/, :captures => [{ :name => :what, :type => :string, :anonymize => :ip }]})
|
92
|
+
|
93
|
+
@line_definition.anonymize("Anonymize 1.2.3.4!").should == "Anonymize 127.0.0.1!"
|
94
|
+
end
|
95
|
+
|
96
|
+
it "should anonymize completely if the anonymizer is unknown" do
|
97
|
+
@line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
|
98
|
+
:regexp => /Anonymize (.+)!/, :captures => [{ :name => :what, :type => :string, :anonymize => :unknown }]})
|
99
|
+
|
100
|
+
@line_definition.anonymize("Anonymize 1.2.3.4!").should == "Anonymize ***!"
|
101
|
+
end
|
102
|
+
|
103
|
+
it "should anonymize an integer slightly" do
|
104
|
+
@line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
|
105
|
+
:regexp => /Anonymize (.+)!/, :captures => [{ :name => :what, :type => :integer, :anonymize => :slightly }]})
|
106
|
+
|
107
|
+
@line_definition.anonymize("Anonymize 1234!").should =~ /Anonymize \d{3,4}\!/
|
108
|
+
end
|
109
|
+
|
110
|
+
it "should anonymize an integer slightly" do
|
111
|
+
@line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
|
112
|
+
:regexp => /Anonymize (.+)!/, :captures => [{ :name => :what, :type => :integer, :anonymize => :slightly }]})
|
113
|
+
|
114
|
+
@line_definition.anonymize("Anonymize 1234!").should =~ /Anonymize \d{3,4}\!/
|
115
|
+
end
|
116
|
+
|
117
|
+
it "should anonymize an double slightly" do
|
118
|
+
@line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
|
119
|
+
:regexp => /Anonymize (.+)!/, :captures => [{ :name => :what, :type => :double, :anonymize => :slightly }]})
|
120
|
+
|
121
|
+
@line_definition.anonymize("Anonymize 1.3!").should =~ /Anonymize 1\.\d+\!/
|
122
|
+
end
|
123
|
+
|
124
|
+
end
|
data/spec/log_parser_spec.rb
CHANGED
@@ -5,18 +5,18 @@ describe RequestLogAnalyzer::LogParser, :single_line_requests do
|
|
5
5
|
include RequestLogAnalyzerSpecHelper
|
6
6
|
|
7
7
|
before(:each) do
|
8
|
-
@log_parser = RequestLogAnalyzer::LogParser.new(
|
8
|
+
@log_parser = RequestLogAnalyzer::LogParser.new(spec_format)
|
9
9
|
end
|
10
10
|
|
11
11
|
it "should have line definitions" do
|
12
12
|
@log_parser.file_format.line_definitions.should_not be_empty
|
13
13
|
end
|
14
14
|
|
15
|
-
it "should have include the language specific hooks in the instance, not in the class" do
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
end
|
15
|
+
# it "should have include the language specific hooks in the instance, not in the class" do
|
16
|
+
# metaclass = (class << @log_parser; self; end)
|
17
|
+
# metaclass.ancestors.should include(TestFileFormat::LogParser)
|
18
|
+
# @log_parser.class.ancestors.should_not include(TestFileFormat::LogParser)
|
19
|
+
# end
|
20
20
|
|
21
21
|
it "should parse a stream and find valid requests" do
|
22
22
|
io = File.new(log_fixture(:test_file_format), 'r')
|
@@ -38,7 +38,7 @@ describe RequestLogAnalyzer::LogParser, :combined_requests do
|
|
38
38
|
include RequestLogAnalyzerSpecHelper
|
39
39
|
|
40
40
|
before(:each) do
|
41
|
-
@log_parser = RequestLogAnalyzer::LogParser.new(
|
41
|
+
@log_parser = RequestLogAnalyzer::LogParser.new(spec_format, :combined_requests => true)
|
42
42
|
end
|
43
43
|
|
44
44
|
it "should have multiple line definitions" do
|
@@ -74,19 +74,17 @@ describe RequestLogAnalyzer::LogParser, :warnings do
|
|
74
74
|
include RequestLogAnalyzerSpecHelper
|
75
75
|
|
76
76
|
it "should warn about teaser matching problems" do
|
77
|
-
@log_parser = RequestLogAnalyzer::LogParser.new(
|
77
|
+
@log_parser = RequestLogAnalyzer::LogParser.new(spec_format)
|
78
78
|
@log_parser.should_receive(:warn).with(:teaser_check_failed, anything).exactly(5).times
|
79
79
|
@log_parser.parse_file(log_fixture(:test_file_format))
|
80
80
|
end
|
81
81
|
|
82
82
|
it "should warn about unmatching request headers and footers" do
|
83
|
-
@log_parser = RequestLogAnalyzer::LogParser.new(
|
83
|
+
@log_parser = RequestLogAnalyzer::LogParser.new(spec_format, :combined_requests => true)
|
84
84
|
|
85
85
|
@log_parser.should_receive(:warn).with(:unclosed_request, anything).at_least(1).times
|
86
86
|
@log_parser.should_receive(:warn).with(:no_current_request, anything).at_least(1).times
|
87
87
|
@log_parser.should_not_receive(:handle_request)
|
88
88
|
@log_parser.parse_file(log_fixture(:test_order))
|
89
|
-
|
90
89
|
end
|
91
|
-
|
92
90
|
end
|