request-log-analyzer 1.2.9 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/bin/request-log-analyzer +33 -19
- data/lib/cli/database_console.rb +26 -0
- data/lib/cli/database_console_init.rb +42 -0
- data/lib/cli/tools.rb +1 -1
- data/lib/request_log_analyzer/aggregator/database_inserter.rb +81 -0
- data/lib/request_log_analyzer/aggregator/summarizer.rb +2 -2
- data/lib/request_log_analyzer/aggregator.rb +4 -0
- data/lib/request_log_analyzer/controller.rb +23 -7
- data/lib/request_log_analyzer/database/base.rb +114 -0
- data/lib/request_log_analyzer/database/connection.rb +38 -0
- data/lib/request_log_analyzer/database.rb +177 -0
- data/lib/request_log_analyzer/file_format.rb +6 -3
- data/lib/request_log_analyzer/mailer.rb +46 -0
- data/lib/request_log_analyzer/request.rb +2 -1
- data/lib/request_log_analyzer/source/{database.rb → database_loader.rb} +1 -1
- data/lib/request_log_analyzer/source/log_parser.rb +28 -15
- data/lib/request_log_analyzer/source.rb +7 -2
- data/lib/request_log_analyzer.rb +5 -8
- data/request-log-analyzer.gemspec +8 -8
- data/spec/database.yml +17 -0
- data/spec/fixtures/rails.db +0 -0
- data/spec/integration/command_line_usage_spec.rb +14 -9
- data/spec/lib/macros.rb +16 -0
- data/spec/lib/mocks.rb +18 -6
- data/spec/unit/aggregator/database_inserter_spec.rb +93 -0
- data/spec/unit/database/base_class_spec.rb +190 -0
- data/spec/unit/database/connection_spec.rb +34 -0
- data/spec/unit/database/database_spec.rb +138 -0
- data/spec/unit/source/log_parser_spec.rb +12 -0
- metadata +29 -16
- data/lib/request_log_analyzer/aggregator/database.rb +0 -220
- data/spec/spec.opts +0 -3
- data/spec/unit/aggregator/database_spec.rb +0 -245
@@ -0,0 +1,177 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'activerecord'
|
3
|
+
|
4
|
+
class RequestLogAnalyzer::Database
|
5
|
+
|
6
|
+
def self.const_missing(const) # :nodoc:
|
7
|
+
RequestLogAnalyzer::load_default_class_file(self, const)
|
8
|
+
end
|
9
|
+
|
10
|
+
include RequestLogAnalyzer::Database::Connection
|
11
|
+
|
12
|
+
attr_accessor :file_format
|
13
|
+
attr_reader :request_class, :warning_class, :source_class, :line_classes
|
14
|
+
|
15
|
+
def initialize(connection_identifier = nil)
|
16
|
+
@line_classes = []
|
17
|
+
RequestLogAnalyzer::Database::Base.database = self
|
18
|
+
connect(connection_identifier)
|
19
|
+
end
|
20
|
+
|
21
|
+
# Returns the ORM class for the provided line type
|
22
|
+
def get_class(line_type)
|
23
|
+
line_type = line_type.name if line_type.respond_to?(:name)
|
24
|
+
Object.const_get("#{line_type}_line".camelize)
|
25
|
+
end
|
26
|
+
|
27
|
+
# Returns the Request ORM class for the current database.
|
28
|
+
#
|
29
|
+
# It will create the class if not previously done so. The class will
|
30
|
+
# include a create_table! method the migrate the database.
|
31
|
+
def request_class
|
32
|
+
@request_class ||= begin
|
33
|
+
klass = Class.new(RequestLogAnalyzer::Database::Base) do
|
34
|
+
|
35
|
+
def lines
|
36
|
+
@lines ||= begin
|
37
|
+
lines = []
|
38
|
+
self.class.reflections.each { |r, d| lines += self.send(r).all }
|
39
|
+
lines.sort
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
# Creates the requests table
|
44
|
+
def self.create_table!
|
45
|
+
unless database.connection.table_exists?(:requests)
|
46
|
+
database.connection.create_table(:requests) do |t|
|
47
|
+
t.column :first_lineno, :integer
|
48
|
+
t.column :last_lineno, :integer
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
Object.const_set('Request', klass)
|
55
|
+
Object.const_get('Request')
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
# Returns the Source ORM class for the current database.
|
60
|
+
#
|
61
|
+
# It will create the class if not previously done so. The class will
|
62
|
+
# include a create_table! method the migrate the database.
|
63
|
+
def source_class
|
64
|
+
@source_class ||= begin
|
65
|
+
klass = Class.new(RequestLogAnalyzer::Database::Base) do
|
66
|
+
|
67
|
+
# Creates the sources table
|
68
|
+
def self.create_table!
|
69
|
+
unless database.connection.table_exists?(:sources)
|
70
|
+
database.connection.create_table(:sources) do |t|
|
71
|
+
t.column :filename, :string
|
72
|
+
t.column :mtime, :datetime
|
73
|
+
t.column :filesize, :integer
|
74
|
+
end
|
75
|
+
end
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
79
|
+
Object.const_set('Source', klass)
|
80
|
+
Object.const_get('Source')
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
|
85
|
+
# Returns the Warning ORM class for the current database.
|
86
|
+
#
|
87
|
+
# It will create the class if not previously done so. The class will
|
88
|
+
# include a create_table! method the migrate the database.
|
89
|
+
def warning_class
|
90
|
+
@warning_class ||= begin
|
91
|
+
klass = Class.new(RequestLogAnalyzer::Database::Base) do
|
92
|
+
|
93
|
+
# Creates the warnings table
|
94
|
+
def self.create_table!
|
95
|
+
unless database.connection.table_exists?(:warnings)
|
96
|
+
database.connection.create_table(:warnings) do |t|
|
97
|
+
t.column :warning_type, :string, :limit => 30, :null => false
|
98
|
+
t.column :message, :string
|
99
|
+
t.column :source_id, :integer
|
100
|
+
t.column :lineno, :integer
|
101
|
+
end
|
102
|
+
end
|
103
|
+
end
|
104
|
+
end
|
105
|
+
|
106
|
+
Object.const_set('Warning', klass)
|
107
|
+
Object.const_get('Warning')
|
108
|
+
end
|
109
|
+
end
|
110
|
+
|
111
|
+
# Loads the ORM classes by inspecting the tables in the current database
|
112
|
+
def load_database_schema!
|
113
|
+
connection.tables.map do |table|
|
114
|
+
case table.to_sym
|
115
|
+
when :warnings then warning_class
|
116
|
+
when :sources then source_class
|
117
|
+
when :requests then request_class
|
118
|
+
else load_activerecord_class(table)
|
119
|
+
end
|
120
|
+
end
|
121
|
+
end
|
122
|
+
|
123
|
+
# Returns an array of all the ActiveRecord-bases ORM classes for this database
|
124
|
+
def orm_classes
|
125
|
+
[warning_class, request_class, source_class] + line_classes
|
126
|
+
end
|
127
|
+
|
128
|
+
# Loads an ActiveRecord-based class that correspond to the given parameter, which can either be
|
129
|
+
# a table name or a LineDefinition instance.
|
130
|
+
def load_activerecord_class(linedefinition_or_table)
|
131
|
+
|
132
|
+
case linedefinition_or_table
|
133
|
+
when String, Symbol
|
134
|
+
klass_name = linedefinition_or_table.to_s.singularize.camelize
|
135
|
+
klass = RequestLogAnalyzer::Database::Base.subclass_from_table(linedefinition_or_table)
|
136
|
+
when RequestLogAnalyzer::LineDefinition
|
137
|
+
klass_name = "#{linedefinition_or_table.name}_line".camelize
|
138
|
+
klass = RequestLogAnalyzer::Database::Base.subclass_from_line_definition(linedefinition_or_table)
|
139
|
+
end
|
140
|
+
|
141
|
+
Object.const_set(klass_name, klass)
|
142
|
+
klass = Object.const_get(klass_name)
|
143
|
+
@line_classes << klass
|
144
|
+
return klass
|
145
|
+
end
|
146
|
+
|
147
|
+
def fileformat_classes
|
148
|
+
raise "No file_format provided!" unless file_format
|
149
|
+
|
150
|
+
default_classes = [request_class, source_class, warning_class]
|
151
|
+
line_classes = file_format.line_definitions.map { |(name, definition)| load_activerecord_class(definition) }
|
152
|
+
return default_classes + line_classes
|
153
|
+
end
|
154
|
+
|
155
|
+
# Creates the database schema and related ActiveRecord::Base subclasses that correspond to the
|
156
|
+
# file format definition. These ORM classes will later be used to create records in the database.
|
157
|
+
def create_database_schema!
|
158
|
+
fileformat_classes.each { |klass| klass.create_table! }
|
159
|
+
end
|
160
|
+
|
161
|
+
# Drops the table of all the ORM classes, and unregisters the classes
|
162
|
+
def drop_database_schema!
|
163
|
+
file_format ? fileformat_classes.map(&:drop_table!) : orm_classes.map(&:drop_table!)
|
164
|
+
remove_orm_classes!
|
165
|
+
end
|
166
|
+
|
167
|
+
# Unregisters every ORM class constant
|
168
|
+
def remove_orm_classes!
|
169
|
+
orm_classes.each do |klass|
|
170
|
+
if klass.respond_to?(:name) && !klass.name.blank?
|
171
|
+
# klass_base_name = klass.name.split('::').last
|
172
|
+
Object.send(:remove_const, klass.name) if Object.const_defined?(klass.name)
|
173
|
+
end
|
174
|
+
end
|
175
|
+
end
|
176
|
+
|
177
|
+
end
|
@@ -161,9 +161,12 @@ module RequestLogAnalyzer::FileFormat
|
|
161
161
|
|
162
162
|
# Parses a line by trying to parse it using every line definition in this file format
|
163
163
|
def parse_line(line, &warning_handler)
|
164
|
-
|
165
|
-
|
166
|
-
|
164
|
+
self.line_definitions.each do |lt, definition|
|
165
|
+
match = definition.matches(line, &warning_handler)
|
166
|
+
return match if match
|
167
|
+
end
|
168
|
+
|
169
|
+
return nil
|
167
170
|
end
|
168
171
|
end
|
169
172
|
end
|
@@ -0,0 +1,46 @@
|
|
1
|
+
module RequestLogAnalyzer
|
2
|
+
|
3
|
+
class Mailer
|
4
|
+
|
5
|
+
attr_accessor :data, :to, :host
|
6
|
+
|
7
|
+
# Initialize a mailer
|
8
|
+
# <tt>to</tt> to address
|
9
|
+
# <tt>host</tt> the mailer host
|
10
|
+
# <tt>options</tt> Specific style options
|
11
|
+
def initialize(to, host = 'localhost', options = {})
|
12
|
+
require 'net/smtp'
|
13
|
+
@to = to
|
14
|
+
@host = host
|
15
|
+
@options = options
|
16
|
+
@data = []
|
17
|
+
end
|
18
|
+
|
19
|
+
def mail
|
20
|
+
from = @options[:from] || 'contact@railsdoctors.com'
|
21
|
+
from_alias = @options[:from_alias] || 'Request-log-analyzer reporter'
|
22
|
+
to_alias = @options[:to_alias] || to
|
23
|
+
subject = @options[:subjeect] || "Request log analyzer report - generated on #{Time.now.to_s}"
|
24
|
+
msg = <<END_OF_MESSAGE
|
25
|
+
From: #{from_alias} <#{from}>
|
26
|
+
To: #{to_alias} <#{@to}>
|
27
|
+
Subject: #{subject}
|
28
|
+
|
29
|
+
#{@data.to_s}
|
30
|
+
END_OF_MESSAGE
|
31
|
+
|
32
|
+
Net::SMTP.start(@host) do |smtp|
|
33
|
+
smtp.send_message msg, from, to
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
def << string
|
38
|
+
data << string
|
39
|
+
end
|
40
|
+
|
41
|
+
def puts string
|
42
|
+
data << string
|
43
|
+
end
|
44
|
+
|
45
|
+
end
|
46
|
+
end
|
@@ -80,12 +80,13 @@ module RequestLogAnalyzer
|
|
80
80
|
value_hash = parsed_line[:line_definition].convert_captured_values(parsed_line[:captures], self)
|
81
81
|
value_hash[:line_type] = parsed_line[:line_definition].name
|
82
82
|
value_hash[:lineno] = parsed_line[:lineno]
|
83
|
+
value_hash[:source] = parsed_line[:source]
|
83
84
|
add_line_hash(value_hash)
|
84
85
|
end
|
85
86
|
|
86
87
|
def add_line_hash(value_hash)
|
87
88
|
@lines << value_hash
|
88
|
-
@attributes = value_hash.merge(@attributes)
|
89
|
+
@attributes = value_hash.merge(@attributes)
|
89
90
|
end
|
90
91
|
|
91
92
|
|
@@ -11,6 +11,8 @@ module RequestLogAnalyzer::Source
|
|
11
11
|
# that deal differently with this problem.
|
12
12
|
class LogParser < Base
|
13
13
|
|
14
|
+
include Enumerable
|
15
|
+
|
14
16
|
# The default parse strategy that will be used to parse the input.
|
15
17
|
DEFAULT_PARSE_STRATEGY = 'assume-correct'
|
16
18
|
|
@@ -46,11 +48,11 @@ module RequestLogAnalyzer::Source
|
|
46
48
|
# lines specified in the FileFormat. This lines will be combined into Request instances,
|
47
49
|
# that will be yielded. The actual parsing occurs in the parse_io method.
|
48
50
|
# <tt>options</tt>:: A Hash of options that will be pased to parse_io.
|
49
|
-
def each_request(options = {}, &block) # :yields: request
|
51
|
+
def each_request(options = {}, &block) # :yields: :request, request
|
50
52
|
|
51
53
|
case @source_files
|
52
54
|
when IO;
|
53
|
-
puts "Parsing from the standard input. Press CTRL+C to finish."
|
55
|
+
puts "Parsing from the standard input. Press CTRL+C to finish." # FIXME: not here
|
54
56
|
parse_stream(@source_files, options, &block)
|
55
57
|
when String
|
56
58
|
parse_file(@source_files, options, &block)
|
@@ -60,6 +62,9 @@ module RequestLogAnalyzer::Source
|
|
60
62
|
raise "Unknown source provided"
|
61
63
|
end
|
62
64
|
end
|
65
|
+
|
66
|
+
# Make sure the Enumerable methods work as expected
|
67
|
+
alias_method :each, :each_request
|
63
68
|
|
64
69
|
# Parses a list of subsequent files of the same format, by calling parse_file for every
|
65
70
|
# file in the array.
|
@@ -89,18 +94,22 @@ module RequestLogAnalyzer::Source
|
|
89
94
|
# TODO: Fix progress bar that is broken for IO.popen, as it returns a single string.
|
90
95
|
#
|
91
96
|
# <tt>file</tt>:: The file that should be parsed.
|
92
|
-
# <tt>options</tt>:: A Hash of options that will be pased to parse_io.
|
97
|
+
# <tt>options</tt>:: A Hash of options that will be pased to parse_io.
|
93
98
|
def parse_file(file, options = {}, &block)
|
94
99
|
|
95
|
-
@
|
100
|
+
@current_source = File.expand_path(file)
|
101
|
+
@progress_handler.call(:started, file) if @progress_handler
|
102
|
+
@source_changes_handler.call(:started, @current_source) if @source_changes_handler
|
96
103
|
|
97
104
|
if decompress_file?(file).empty?
|
98
105
|
File.open(file, 'r') { |f| parse_io(f, options, &block) }
|
99
106
|
else
|
100
107
|
IO.popen(decompress_file?(file), 'r') { |f| parse_io(f, options, &block) }
|
101
108
|
end
|
102
|
-
|
103
|
-
@
|
109
|
+
|
110
|
+
@source_changes_handler.call(:finished, @current_source) if @source_changes_handler
|
111
|
+
@progress_handler.call(:finished, file) if @progress_handler
|
112
|
+
@current_source = nil
|
104
113
|
end
|
105
114
|
|
106
115
|
# Parses an IO stream. It will simply call parse_io. This function does not support progress updates
|
@@ -124,21 +133,19 @@ module RequestLogAnalyzer::Source
|
|
124
133
|
# <tt>io</tt>:: The IO instance to use as source
|
125
134
|
# <tt>options</tt>:: A hash of options that can be used by the parser.
|
126
135
|
def parse_io(io, options = {}, &block) # :yields: request
|
127
|
-
|
128
|
-
@current_lineno = 0
|
136
|
+
@current_lineno = 1
|
129
137
|
io.each_line do |line|
|
130
138
|
@progress_handler.call(:progress, io.pos) if @progress_handler && io.kind_of?(File)
|
131
|
-
|
139
|
+
|
132
140
|
if request_data = file_format.parse_line(line) { |wt, message| warn(wt, message) }
|
133
141
|
@parsed_lines += 1
|
134
|
-
update_current_request(request_data.merge(:lineno => @current_lineno), &block)
|
142
|
+
update_current_request(request_data.merge(:source => @current_source, :lineno => @current_lineno), &block)
|
135
143
|
end
|
136
144
|
|
137
145
|
@current_lineno += 1
|
138
146
|
end
|
139
|
-
|
147
|
+
|
140
148
|
warn(:unfinished_request_on_eof, "End of file reached, but last request was not completed!") unless @current_request.nil?
|
141
|
-
|
142
149
|
@current_lineno = nil
|
143
150
|
end
|
144
151
|
|
@@ -149,10 +156,16 @@ module RequestLogAnalyzer::Source
|
|
149
156
|
end
|
150
157
|
|
151
158
|
# Add a block to this method to install a warning handler while parsing,
|
152
|
-
# <tt>proc</tt>:: The proc that will be called to handle parse warning messages
|
159
|
+
# <tt>proc</tt>:: The proc that will be called to handle parse warning messages
|
153
160
|
def warning=(proc)
|
154
161
|
@warning_handler = proc
|
155
162
|
end
|
163
|
+
|
164
|
+
# Add a block to this method to install a source change handler while parsing,
|
165
|
+
# <tt>proc</tt>:: The proc that will be called to handle source changes
|
166
|
+
def source_changes=(proc)
|
167
|
+
@source_changes_handler = proc
|
168
|
+
end
|
156
169
|
|
157
170
|
# This method is called by the parser if it encounteres any parsing problems.
|
158
171
|
# It will call the installed warning handler if any.
|
@@ -229,11 +242,11 @@ module RequestLogAnalyzer::Source
|
|
229
242
|
# - It will update the parsed_requests and skipped_requests variables accordingly
|
230
243
|
#
|
231
244
|
# <tt>request</tt>:: The parsed request instance (RequestLogAnalyzer::Request)
|
232
|
-
def handle_request(request, &block) # :yields: request
|
245
|
+
def handle_request(request, &block) # :yields: :request, request
|
233
246
|
@parsed_requests += 1
|
234
247
|
request.validate
|
235
248
|
accepted = block_given? ? yield(request) : true
|
236
|
-
@skipped_requests += 1
|
249
|
+
@skipped_requests += 1 unless accepted
|
237
250
|
end
|
238
251
|
|
239
252
|
# Checks whether a given line hash is a header line according to the current file format.
|
@@ -33,11 +33,16 @@ module RequestLogAnalyzer::Source
|
|
33
33
|
# The total number of parsed lines
|
34
34
|
attr_reader :parsed_lines
|
35
35
|
|
36
|
+
# The number of skipped lines because of warnings
|
37
|
+
attr_reader :skipped_lines
|
38
|
+
|
36
39
|
# The total number of parsed requests.
|
37
40
|
attr_reader :parsed_requests
|
38
41
|
|
39
|
-
# The number of skipped
|
40
|
-
attr_reader :
|
42
|
+
# The total number of skipped requests because of filters.
|
43
|
+
attr_reader :skipped_requests
|
44
|
+
|
45
|
+
|
41
46
|
|
42
47
|
# Initializer, which will register the file format and save any options given as a hash.
|
43
48
|
# <tt>format</tt>:: The file format instance
|
data/lib/request_log_analyzer.rb
CHANGED
@@ -11,23 +11,20 @@ module RequestLogAnalyzer
|
|
11
11
|
|
12
12
|
# The current version of request-log-analyzer.
|
13
13
|
# This will be diplayed in output reports etc.
|
14
|
-
VERSION = "1.
|
15
|
-
|
14
|
+
VERSION = "1.3.0"
|
15
|
+
|
16
16
|
# Loads constants in the RequestLogAnalyzer namespace using self.load_default_class_file(base, const)
|
17
17
|
# <tt>const</tt>:: The constant that is not yet loaded in the RequestLogAnalyzer namespace. This should be passed as a string or symbol.
|
18
18
|
def self.const_missing(const)
|
19
19
|
load_default_class_file(RequestLogAnalyzer, const)
|
20
20
|
end
|
21
|
-
|
21
|
+
|
22
22
|
# Loads constants that reside in the RequestLogAnalyzer tree using the constant name
|
23
23
|
# and its base constant to determine the filename.
|
24
24
|
# <tt>base</tt>:: The base constant to load the constant from. This should be Foo when the constant Foo::Bar is being loaded.
|
25
25
|
# <tt>const</tt>:: The constant to load from the base constant as a string or symbol. This should be 'Bar' or :Bar when the constant Foo::Bar is being loaded.
|
26
26
|
def self.load_default_class_file(base, const)
|
27
|
-
|
28
|
-
basename = to_underscore(const.to_s)
|
29
|
-
filename = "#{File.dirname(__FILE__)}/#{path}/#{basename}"
|
30
|
-
require filename
|
27
|
+
require "#{to_underscore("#{base.name}::#{const}")}"
|
31
28
|
base.const_get(const)
|
32
29
|
end
|
33
30
|
|
@@ -36,7 +33,7 @@ module RequestLogAnalyzer
|
|
36
33
|
# <tt>str</tt>:: The string to convert in the following format: <tt>ModuleName::ClassName</tt>
|
37
34
|
def self.to_underscore(str)
|
38
35
|
str.to_s.gsub(/::/, '/').gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').gsub(/([a-z\d])([A-Z])/,'\1_\2').tr("-", "_").downcase
|
39
|
-
end
|
36
|
+
end
|
40
37
|
|
41
38
|
# Convert a string/symbol in underscores (<tt>request_log_analyzer/controller</tt>) to camelcase
|
42
39
|
# (<tt>RequestLogAnalyzer::Controller</tt>). This can be used to find the class that is defined in a given filename.
|
@@ -1,7 +1,7 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
|
-
s.name =
|
3
|
-
s.version = "1.
|
4
|
-
s.date = "2009-09-
|
2
|
+
s.name = "request-log-analyzer"
|
3
|
+
s.version = "1.3.0"
|
4
|
+
s.date = "2009-09-12"
|
5
5
|
|
6
6
|
s.rubyforge_project = 'r-l-a'
|
7
7
|
|
@@ -9,14 +9,14 @@ Gem::Specification.new do |s|
|
|
9
9
|
s.executables = ['request-log-analyzer']
|
10
10
|
s.default_executable = 'request-log-analyzer'
|
11
11
|
|
12
|
-
s.summary = "A command line tool to analyze request logs for Rails, Merb and other application servers"
|
12
|
+
s.summary = "A command line tool to analyze request logs for Apache, Rails, Merb and other application servers"
|
13
13
|
s.description = <<-eos
|
14
14
|
Request log analyzer's purpose is to find ot how your web application is being used and to focus your optimization efforts.
|
15
15
|
This tool will parse all requests in the application's log file and aggregate the information. Once it is finished parsing
|
16
16
|
the log file(s), it will show the requests that take op most server time using various metrics. It can also insert all
|
17
17
|
parsed request information into a database so you can roll your own analysis. It supports Rails- and Merb-based applications
|
18
|
-
out of the box, but file formats of other applications can easily be supported by supplying an
|
19
|
-
definition.
|
18
|
+
and Apache access log files out of the box, but file formats of other applications can easily be supported by supplying an
|
19
|
+
easy to write log file format definition.
|
20
20
|
eos
|
21
21
|
|
22
22
|
s.rdoc_options << '--title' << s.name << '--main' << 'README.rdoc' << '--line-numbers' << '--inline-source'
|
@@ -31,6 +31,6 @@ Gem::Specification.new do |s|
|
|
31
31
|
s.email = ['willem@railsdoctors.com', 'bart@railsdoctors.com']
|
32
32
|
s.homepage = 'http://railsdoctors.com'
|
33
33
|
|
34
|
-
s.files = %w(spec/unit/filter/anonymize_filter_spec.rb lib/request_log_analyzer/line_definition.rb lib/request_log_analyzer/output/html.rb lib/request_log_analyzer/controller.rb spec/fixtures/rails_22_cached.log lib/request_log_analyzer/file_format/rails_development.rb spec/
|
35
|
-
s.test_files = %w(spec/unit/filter/anonymize_filter_spec.rb spec/unit/file_format/file_format_api_spec.rb spec/unit/file_format/apache_format_spec.rb spec/integration/command_line_usage_spec.rb spec/unit/filter/timespan_filter_spec.rb spec/unit/tracker/tracker_api_spec.rb spec/unit/tracker/duration_tracker_spec.rb spec/unit/controller/log_processor_spec.rb spec/unit/filter/filter_spec.rb spec/unit/filter/field_filter_spec.rb spec/unit/tracker/timespan_tracker_spec.rb spec/unit/tracker/hourly_spread_spec.rb spec/unit/file_format/merb_format_spec.rb spec/unit/file_format/line_definition_spec.rb spec/unit/controller/controller_spec.rb spec/unit/source/request_spec.rb spec/unit/source/log_parser_spec.rb spec/unit/
|
34
|
+
s.files = %w(spec/unit/filter/anonymize_filter_spec.rb lib/request_log_analyzer/line_definition.rb lib/request_log_analyzer/output/html.rb lib/request_log_analyzer/controller.rb spec/fixtures/rails_22_cached.log spec/lib/macros.rb lib/request_log_analyzer/file_format/rails_development.rb spec/fixtures/apache_combined.log spec/fixtures/apache_common.log spec/fixtures/merb_prefixed.log tasks/request_log_analyzer.rake spec/unit/file_format/file_format_api_spec.rb spec/unit/file_format/apache_format_spec.rb spec/integration/command_line_usage_spec.rb lib/request_log_analyzer/database.rb spec/fixtures/decompression.log.bz2 lib/request_log_analyzer/log_processor.rb lib/request_log_analyzer/tracker.rb lib/request_log_analyzer/filter.rb spec/fixtures/rails_unordered.log bin/request-log-analyzer request-log-analyzer.gemspec DESIGN.rdoc spec/unit/filter/timespan_filter_spec.rb spec/unit/aggregator/database_inserter_spec.rb spec/lib/matchers.rb lib/request_log_analyzer/filter/field.rb lib/request_log_analyzer/tracker/frequency.rb spec/fixtures/decompression.log.gz spec/fixtures/decompression.log spec/lib/testing_format.rb spec/fixtures/test_order.log spec/fixtures/rails.db lib/request_log_analyzer/output/fixed_width.rb lib/request_log_analyzer/filter/anonymize.rb lib/request_log_analyzer/tracker/timespan.rb lib/request_log_analyzer/database/base.rb lib/request_log_analyzer/aggregator.rb lib/cli/progressbar.rb lib/request_log_analyzer/mailer.rb README.rdoc spec/fixtures/merb.log lib/request_log_analyzer/tracker/hourly_spread.rb .gitignore spec/unit/tracker/tracker_api_spec.rb spec/unit/tracker/duration_tracker_spec.rb lib/request_log_analyzer/aggregator/echo.rb spec/unit/controller/log_processor_spec.rb spec/spec_helper.rb lib/request_log_analyzer.rb spec/database.yml Rakefile lib/request_log_analyzer/database/connection.rb spec/unit/filter/filter_spec.rb spec/fixtures/test_language_combined.log lib/request_log_analyzer/aggregator/database_inserter.rb lib/request_log_analyzer/aggregator/summarizer.rb lib/request_log_analyzer/file_format/rails.rb spec/fixtures/decompression.tar.gz spec/unit/filter/field_filter_spec.rb spec/unit/database/base_class_spec.rb lib/request_log_analyzer/filter/timespan.rb lib/request_log_analyzer/source/log_parser.rb spec/fixtures/decompression.tgz spec/unit/tracker/timespan_tracker_spec.rb spec/unit/tracker/hourly_spread_spec.rb spec/fixtures/header_and_footer.log lib/cli/tools.rb lib/request_log_analyzer/file_format/merb.rb spec/fixtures/multiple_files_1.log spec/unit/file_format/merb_format_spec.rb spec/unit/file_format/line_definition_spec.rb lib/request_log_analyzer/source.rb lib/request_log_analyzer/request.rb lib/cli/database_console.rb spec/unit/database/connection_spec.rb spec/unit/controller/controller_spec.rb spec/lib/mocks.rb spec/lib/helpers.rb lib/cli/database_console_init.rb lib/request_log_analyzer/output.rb lib/request_log_analyzer/file_format/apache.rb spec/fixtures/rails_1x.log spec/fixtures/decompression.log.zip spec/unit/source/request_spec.rb spec/unit/source/log_parser_spec.rb spec/fixtures/test_file_format.log tasks/github-gem.rake spec/unit/database/database_spec.rb lib/request_log_analyzer/tracker/duration.rb lib/request_log_analyzer/file_format.rb spec/unit/aggregator/summarizer_spec.rb spec/fixtures/rails_22.log spec/fixtures/multiple_files_2.log spec/fixtures/syslog_1x.log LICENSE lib/request_log_analyzer/source/database_loader.rb spec/unit/tracker/frequency_tracker_spec.rb spec/unit/file_format/rails_format_spec.rb lib/cli/command_line_arguments.rb)
|
35
|
+
s.test_files = %w(spec/unit/filter/anonymize_filter_spec.rb spec/unit/file_format/file_format_api_spec.rb spec/unit/file_format/apache_format_spec.rb spec/integration/command_line_usage_spec.rb spec/unit/filter/timespan_filter_spec.rb spec/unit/aggregator/database_inserter_spec.rb spec/unit/tracker/tracker_api_spec.rb spec/unit/tracker/duration_tracker_spec.rb spec/unit/controller/log_processor_spec.rb spec/unit/filter/filter_spec.rb spec/unit/filter/field_filter_spec.rb spec/unit/database/base_class_spec.rb spec/unit/tracker/timespan_tracker_spec.rb spec/unit/tracker/hourly_spread_spec.rb spec/unit/file_format/merb_format_spec.rb spec/unit/file_format/line_definition_spec.rb spec/unit/database/connection_spec.rb spec/unit/controller/controller_spec.rb spec/unit/source/request_spec.rb spec/unit/source/log_parser_spec.rb spec/unit/database/database_spec.rb spec/unit/aggregator/summarizer_spec.rb spec/unit/tracker/frequency_tracker_spec.rb spec/unit/file_format/rails_format_spec.rb)
|
36
36
|
end
|
data/spec/database.yml
ADDED
@@ -0,0 +1,17 @@
|
|
1
|
+
sqlite3:
|
2
|
+
adapter: "sqlite3"
|
3
|
+
database: ":memory:"
|
4
|
+
|
5
|
+
# mysql:
|
6
|
+
# adapter: "mysql"
|
7
|
+
# host: "localhost"
|
8
|
+
# username: "root"
|
9
|
+
# password:
|
10
|
+
# database: "rla_test"
|
11
|
+
|
12
|
+
# postgresql:
|
13
|
+
# adapter: "postgresql"
|
14
|
+
# host: "localhost"
|
15
|
+
# username: "rla"
|
16
|
+
# password: "rla"
|
17
|
+
# database: "rla_test"
|
Binary file
|
@@ -12,17 +12,17 @@ describe RequestLogAnalyzer, 'running from command line' do
|
|
12
12
|
|
13
13
|
it "should find 4 requests in default mode" do
|
14
14
|
output = run("#{log_fixture(:rails_1x)}")
|
15
|
-
output.
|
15
|
+
output.any? { |line| /^Parsed requests\:\s*4\s/ =~ line }.should be_true
|
16
16
|
end
|
17
17
|
|
18
|
-
it "should
|
18
|
+
it "should skip 1 requests with a --select option" do
|
19
19
|
output = run("#{log_fixture(:rails_1x)} --select controller PeopleController")
|
20
|
-
output.
|
20
|
+
output.any? { |line| /^Skipped requests\:\s*1\s/ =~ line }.should be_true
|
21
21
|
end
|
22
22
|
|
23
|
-
it "should
|
23
|
+
it "should skip 3 requests with a --reject option" do
|
24
24
|
output = run("#{log_fixture(:rails_1x)} --reject controller PeopleController")
|
25
|
-
output.
|
25
|
+
output.any? { |line| /^Skipped requests\:\s*3\s/ =~ line }.should be_true
|
26
26
|
end
|
27
27
|
|
28
28
|
it "should write output to a file with the --file option" do
|
@@ -37,7 +37,7 @@ describe RequestLogAnalyzer, 'running from command line' do
|
|
37
37
|
|
38
38
|
it "should write HTML if --output HTML is provided" do
|
39
39
|
output = run("#{log_fixture(:rails_1x)} --output HTML")
|
40
|
-
output.any? { |line| /<html
|
40
|
+
output.any? { |line| /<html[^>]*>/ =~ line}.should be_true
|
41
41
|
end
|
42
42
|
|
43
43
|
it "should run with the --database option" do
|
@@ -57,17 +57,22 @@ describe RequestLogAnalyzer, 'running from command line' do
|
|
57
57
|
|
58
58
|
it "should parse a Merb file if --format merb is set" do
|
59
59
|
output = run("#{log_fixture(:merb)} --format merb")
|
60
|
-
output.
|
60
|
+
output.any? { |line| /Parsed requests\:\s*11/ =~ line }.should be_true
|
61
61
|
end
|
62
62
|
|
63
63
|
it "should parse a Apache access log file if --apache-format is set" do
|
64
64
|
output = run("#{log_fixture(:apache_combined)} --apache-format combined")
|
65
|
-
output.
|
65
|
+
output.any? { |line| /Parsed requests\:\s*5/ =~ line }.should be_true
|
66
66
|
end
|
67
67
|
|
68
68
|
it "should dump the results to a YAML file" do
|
69
69
|
run("#{log_fixture(:rails_1x)} --dump #{temp_output_file(:dump)}")
|
70
70
|
File.exist?(temp_output_file(:dump)).should be_true
|
71
71
|
YAML::load(File.read(temp_output_file(:dump))).should have_at_least(1).item
|
72
|
-
end
|
72
|
+
end
|
73
|
+
|
74
|
+
it "should parse 4 requests from the standard input" do
|
75
|
+
output = run("- < #{log_fixture(:rails_1x)}")
|
76
|
+
output.any? { |line| /^Parsed requests\:\s*4\s/ =~ line }.should be_true
|
77
|
+
end
|
73
78
|
end
|
data/spec/lib/macros.rb
CHANGED
@@ -1,2 +1,18 @@
|
|
1
1
|
module RequestLogAnalyzer::Spec::Macros
|
2
|
+
|
3
|
+
def test_databases
|
4
|
+
require 'yaml'
|
5
|
+
hash = YAML.load(File.read("#{File.dirname(__FILE__)}/../database.yml"))
|
6
|
+
hash.inject({}) { |res, (name, h)| res[name] = h.map { |(k,v)| "#{k}=#{v}" }.join(';'); res }
|
7
|
+
end
|
8
|
+
|
9
|
+
# Create or return a new TestingFormat
|
10
|
+
def testing_format
|
11
|
+
@testing_format ||= TestingFormat.create
|
12
|
+
end
|
13
|
+
|
14
|
+
def default_orm_class_names
|
15
|
+
['Warning', 'Request', 'Source']
|
16
|
+
end
|
17
|
+
|
2
18
|
end
|
data/spec/lib/mocks.rb
CHANGED
@@ -9,6 +9,7 @@ module RequestLogAnalyzer::Spec::Mocks
|
|
9
9
|
|
10
10
|
source.stub!(:warning=)
|
11
11
|
source.stub!(:progress=)
|
12
|
+
source.stub!(:source_changes=)
|
12
13
|
|
13
14
|
source.stub!(:prepare)
|
14
15
|
source.stub!(:finalize)
|
@@ -43,15 +44,26 @@ module RequestLogAnalyzer::Spec::Mocks
|
|
43
44
|
return output
|
44
45
|
end
|
45
46
|
|
46
|
-
def
|
47
|
+
def mock_database(*stubs)
|
48
|
+
database = mock('RequestLogAnalyzer::Database')
|
49
|
+
database.stub!(:connect)
|
50
|
+
database.stub!(:disconnect)
|
51
|
+
database.stub!(:connection).and_return(mock_connection)
|
52
|
+
stubs.each { |s| database.stub!(s)}
|
53
|
+
return database
|
54
|
+
end
|
55
|
+
|
56
|
+
def mock_connection
|
47
57
|
table_creator = mock('ActiveRecord table creator')
|
48
58
|
table_creator.stub!(:column)
|
49
59
|
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
60
|
+
connection = mock('ActiveRecord::Base.connection')
|
61
|
+
connection.stub!(:add_index)
|
62
|
+
connection.stub!(:remove_index)
|
63
|
+
connection.stub!(:table_exists?).and_return(false)
|
64
|
+
connection.stub!(:create_table).and_yield(table_creator).and_return(true)
|
65
|
+
connection.stub!(:table_creator).and_return(table_creator)
|
66
|
+
return connection
|
55
67
|
end
|
56
68
|
|
57
69
|
|