request-log-analyzer 1.2.9 → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (33) hide show
  1. data/bin/request-log-analyzer +33 -19
  2. data/lib/cli/database_console.rb +26 -0
  3. data/lib/cli/database_console_init.rb +42 -0
  4. data/lib/cli/tools.rb +1 -1
  5. data/lib/request_log_analyzer/aggregator/database_inserter.rb +81 -0
  6. data/lib/request_log_analyzer/aggregator/summarizer.rb +2 -2
  7. data/lib/request_log_analyzer/aggregator.rb +4 -0
  8. data/lib/request_log_analyzer/controller.rb +23 -7
  9. data/lib/request_log_analyzer/database/base.rb +114 -0
  10. data/lib/request_log_analyzer/database/connection.rb +38 -0
  11. data/lib/request_log_analyzer/database.rb +177 -0
  12. data/lib/request_log_analyzer/file_format.rb +6 -3
  13. data/lib/request_log_analyzer/mailer.rb +46 -0
  14. data/lib/request_log_analyzer/request.rb +2 -1
  15. data/lib/request_log_analyzer/source/{database.rb → database_loader.rb} +1 -1
  16. data/lib/request_log_analyzer/source/log_parser.rb +28 -15
  17. data/lib/request_log_analyzer/source.rb +7 -2
  18. data/lib/request_log_analyzer.rb +5 -8
  19. data/request-log-analyzer.gemspec +8 -8
  20. data/spec/database.yml +17 -0
  21. data/spec/fixtures/rails.db +0 -0
  22. data/spec/integration/command_line_usage_spec.rb +14 -9
  23. data/spec/lib/macros.rb +16 -0
  24. data/spec/lib/mocks.rb +18 -6
  25. data/spec/unit/aggregator/database_inserter_spec.rb +93 -0
  26. data/spec/unit/database/base_class_spec.rb +190 -0
  27. data/spec/unit/database/connection_spec.rb +34 -0
  28. data/spec/unit/database/database_spec.rb +138 -0
  29. data/spec/unit/source/log_parser_spec.rb +12 -0
  30. metadata +29 -16
  31. data/lib/request_log_analyzer/aggregator/database.rb +0 -220
  32. data/spec/spec.opts +0 -3
  33. data/spec/unit/aggregator/database_spec.rb +0 -245
@@ -1,220 +0,0 @@
1
- require 'rubygems'
2
- require 'activerecord'
3
-
4
- module RequestLogAnalyzer::Aggregator
5
-
6
- # The database aggregator will create an SQLite3 database with all parsed request information.
7
- #
8
- # The prepare method will create a database schema according to the file format definitions.
9
- # It will also create ActiveRecord::Base subclasses to interact with the created tables.
10
- # Then, the aggregate method will be called for every parsed request. The information of
11
- # these requests is inserted into the tables using the ActiveRecord classes.
12
- #
13
- # A requests table will be created, in which a record is inserted for every parsed request.
14
- # For every line type, a separate table will be created with a request_id field to point to
15
- # the request record, and a field for every parsed value. Finally, a warnings table will be
16
- # created to log all parse warnings.
17
- class Database < Base
18
-
19
- attr_reader :request_class, :request_count, :orm_module, :warning_class
20
-
21
- # Establishes a connection to the database and creates the necessary database schema for the
22
- # current file format
23
- def prepare
24
- initialize_orm_module!
25
- establish_database_connection!
26
- File.unlink(options[:database]) if File.exist?(options[:database]) # TODO: keep old database?
27
- create_database_schema!
28
- end
29
-
30
- # Aggregates a request into the database
31
- # This will create a record in the requests table and create a record for every line that has been parsed,
32
- # in which the captured values will be stored.
33
- def aggregate(request)
34
- @request_object = request_class.new(:first_lineno => request.first_lineno, :last_lineno => request.last_lineno)
35
- request.lines.each do |line|
36
- class_columns = orm_module.const_get("#{line[:line_type]}_line".classify).column_names.reject { |column| ['id'].include?(column) }
37
- attributes = Hash[*line.select { |(k, v)| class_columns.include?(k.to_s) }.flatten]
38
- @request_object.send("#{line[:line_type]}_lines").build(attributes)
39
- end
40
- @request_object.save!
41
- rescue SQLite3::SQLException => e
42
- raise Interrupt, e.message
43
- end
44
-
45
- # Finalizes the aggregator by closing the connection to the database
46
- def finalize
47
- @request_count = orm_module::Request.count
48
- remove_database_connection!
49
- deinitialize_orm_module!
50
- end
51
-
52
- # Records w warining in the warnings table.
53
- def warning(type, message, lineno)
54
- warning_class.create!(:warning_type => type.to_s, :message => message, :lineno => lineno)
55
- end
56
-
57
- # Prints a short report of what has been inserted into the database
58
- def report(output)
59
- output.title('Request database created')
60
-
61
- output << "A database file has been created with all parsed request information.\n"
62
- output << "#{@request_count} requests have been added to the database.\n"
63
- output << "To execute queries on this database, run the following command:\n"
64
- output << output.colorize(" $ sqlite3 #{options[:database]}\n", :bold)
65
- output << "\n"
66
- end
67
-
68
- # Retreives the connection that is used for the database inserter
69
- def connection
70
- orm_module::Base.connection
71
- end
72
-
73
- protected
74
-
75
- # Create a module and a default subclass of ActiveRecord::Base on which to establish the database connection
76
- def initialize_orm_module!
77
-
78
- # Create a Database module in the file format if it does not yet exists
79
- file_format.class.const_set('Database', Module.new) unless file_format.class.const_defined?('Database')
80
- @orm_module = file_format.class.const_get('Database')
81
-
82
- # Register the base activerecord class
83
- unless orm_module.const_defined?('Base')
84
- orm_base_class = Class.new(ActiveRecord::Base)
85
- orm_base_class.abstract_class = true
86
- orm_module.const_set('Base', orm_base_class)
87
- end
88
- end
89
-
90
- # Deinitializes the ORM module and the ActiveRecord::Base subclass.
91
- def deinitialize_orm_module!
92
- file_format.class.send(:remove_const, 'Database') if file_format.class.const_defined?('Database')
93
- @orm_module = nil
94
- end
95
-
96
- # Established a connection with the database for this session
97
- def establish_database_connection!
98
- orm_module::Base.establish_connection(:adapter => 'sqlite3', :database => options[:database])
99
- #ActiveRecord::Migration.class_eval("def self.connection; #{@orm_module.to_s}::Base.connection; end ")
100
- end
101
-
102
- def remove_database_connection!
103
- #ActiveRecord::Migration.class_eval("def self.connection; ActiveRecord::Base.connection; end ")
104
- orm_module::Base.remove_connection
105
- end
106
-
107
- # This function creates a database table for a given line definition.
108
- # It will create a field for every capture in the line, and adds a lineno field to indicate at
109
- # what line in the original file the line was found, and a request_id to link lines related
110
- # to the same request. It will also create an index in the request_id field to speed up queries.
111
- def create_database_table(definition)
112
- connection.create_table("#{definition.name}_lines") do |t|
113
-
114
- # Add default fields for every line type
115
- t.column(:request_id, :integer)
116
- t.column(:lineno, :integer)
117
-
118
- definition.captures.each do |capture|
119
- # Add a field for every capture
120
- t.column(capture[:name], column_type(capture[:type]))
121
-
122
- # If the capture provides other field as well, create columns for them, too
123
- capture[:provides].each { |field, field_type| t.column(field, column_type(field_type)) } if capture[:provides].kind_of?(Hash)
124
- end
125
- end
126
-
127
- # Create an index on the request_id column to support faster querying
128
- connection.add_index("#{definition.name}_lines", [:request_id])
129
- end
130
-
131
- # Creates an ActiveRecord class for a given line definition.
132
- # A subclass of ActiveRecord::Base is created and an association with the Request class is
133
- # created using belongs_to / has_many. This association will later be used to create records
134
- # in the corresponding table. This table should already be created before this method is called.
135
- def create_activerecord_class(definition)
136
- class_name = "#{definition.name}_line".camelize
137
- klass = Class.new(orm_module::Base)
138
- klass.send(:belongs_to, :request)
139
-
140
- definition.captures.select { |c| c.has_key?(:provides) }.each do |capture|
141
- klass.send(:serialize, capture[:name], Hash)
142
- end
143
-
144
- orm_module.const_set(class_name, klass) unless orm_module.const_defined?(class_name)
145
- request_class.send(:has_many, "#{definition.name}_lines".to_sym)
146
- end
147
-
148
- # Creates a requests table, in which a record is created for every parsed request.
149
- # It also creates an ActiveRecord::Base class to communicate with this table.
150
- def create_request_table_and_class
151
- connection.create_table("requests") do |t|
152
- t.column :first_lineno, :integer
153
- t.column :last_lineno, :integer
154
- end
155
-
156
- orm_module.const_set('Request', Class.new(orm_module::Base)) unless orm_module.const_defined?('Request')
157
- @request_class = orm_module.const_get('Request')
158
- end
159
-
160
- # Creates a sources table, in which a record is created for every file that is parsed.
161
- # It also creates an ActiveRecord::Base ORM class for the table.
162
- def create_source_table_and_class
163
- connection.create_table('sources') do |t|
164
- t.column :filename, :string
165
- t.column :mtime, :datetime
166
- t.column :filesize, :integer
167
- end
168
-
169
- orm_module.const_set('Source', Class.new(orm_module::Base)) unless orm_module.const_defined?('Source')
170
- @source_class = orm_module.const_get('Source')
171
- end
172
-
173
- # Creates a warnings table and a corresponding Warning class to communicate with this table using ActiveRecord.
174
- def create_warning_table_and_class
175
- connection.create_table("warnings") do |t|
176
- t.column :warning_type, :string, :limit => 30, :null => false
177
- t.column :message, :string
178
- t.column :lineno, :integer
179
- end
180
-
181
- orm_module.const_set('Warning', Class.new(orm_module::Base)) unless orm_module.const_defined?('Warning')
182
- @warning_class = orm_module.const_get('Warning')
183
- end
184
-
185
- # Creates the database schema and related ActiveRecord::Base subclasses that correspond to the
186
- # file format definition. These ORM classes will later be used to create records in the database.
187
- def create_database_schema!
188
- create_source_table_and_class
189
- create_request_table_and_class
190
- create_warning_table_and_class
191
-
192
- file_format.line_definitions.each do |name, definition|
193
- create_database_table(definition)
194
- create_activerecord_class(definition)
195
- end
196
- end
197
-
198
- # Function to determine the column type for a field
199
- # TODO: make more robust / include in file-format definition
200
- def column_type(type_indicator)
201
- case type_indicator
202
- when :eval; :text
203
- when :hash; :text
204
- when :text; :text
205
- when :string; :string
206
- when :sec; :double
207
- when :msec; :double
208
- when :duration; :double
209
- when :float; :double
210
- when :double; :double
211
- when :integer; :integer
212
- when :int; :int
213
- when :timestamp; :datetime
214
- when :datetime; :datetime
215
- when :date; :date
216
- else :string
217
- end
218
- end
219
- end
220
- end
data/spec/spec.opts DELETED
@@ -1,3 +0,0 @@
1
- --colour
2
- --format specdoc
3
- --reverse
@@ -1,245 +0,0 @@
1
- require File.dirname(__FILE__) + '/../../spec_helper'
2
-
3
- describe RequestLogAnalyzer::Aggregator::Database do
4
-
5
- before(:all) do
6
- log_parser = RequestLogAnalyzer::Source::LogParser.new(testing_format)
7
- @database_inserter = RequestLogAnalyzer::Aggregator::Database.new(log_parser, :database => ':memory:')
8
-
9
- @line_definition = RequestLogAnalyzer::LineDefinition.new(:test, { :regexp => /Testing (\w+), tries\: (\d+)/,
10
- :captures => [{ :name => :what, :type => :string }, { :name => :tries, :type => :integer },
11
- { :name => :evaluated, :type => :hash, :provides => {:evaluated_field => :duration} }]})
12
- end
13
-
14
- # The prepare method is called before the parsing starts. It should establish a connection
15
- # to a database that is suitable for inserting requests later on.
16
- describe '#prepare' do
17
-
18
- before(:each) do
19
- @database_inserter.stub!(:initialize_orm_module!)
20
- @database_inserter.stub!(:establish_database_connection!)
21
- @database_inserter.stub!(:create_database_schema!)
22
- end
23
-
24
- it 'should create the ORM mdoule in which the classes can be created' do
25
- @database_inserter.should_receive(:initialize_orm_module!)
26
- @database_inserter.prepare
27
- end
28
-
29
- it 'should establish the database connection' do
30
- @database_inserter.should_receive(:establish_database_connection!)
31
- @database_inserter.prepare
32
- end
33
-
34
- it 'should create the database schema during preparation' do
35
- @database_inserter.should_receive(:create_database_schema!)
36
- @database_inserter.prepare
37
- end
38
- end
39
-
40
- # The database inserter creates is own "Database" module within the file format
41
- # class to create all the classes that are needed.
42
- describe '#initialize_orm_module!' do
43
-
44
- before(:all) { @database_inserter.send(:deinitialize_orm_module!) }
45
- after(:all) { @database_inserter.send(:initialize_orm_module!) }
46
-
47
- before(:each) { @database_inserter.send(:initialize_orm_module!) }
48
- after(:each) { @database_inserter.send(:deinitialize_orm_module!) }
49
-
50
- it "should create a Database module under the file format's class" do
51
- testing_format.class.should be_const_defined('Database')
52
- end
53
-
54
- it "should define a Base class in the Database module" do
55
- testing_format.class::Database.should be_const_defined('Base')
56
- end
57
-
58
- it "should create a ActiveRecord::Base class in the Database module" do
59
- testing_format.class::Database::Base.ancestors.should include(ActiveRecord::Base)
60
- end
61
-
62
- end
63
-
64
- # The create_database_table method should create a database table according to the line definition,
65
- # so that parsed lines can be stored in it later on.
66
- describe '#create_database_table' do
67
-
68
- before(:each) do
69
- @connection = mock_migrator
70
- @database_inserter.stub!(:connection).and_return(@connection)
71
- end
72
-
73
- it "should create a table based on the line type name" do
74
- @connection.should_receive(:create_table).with('test_lines')
75
- @database_inserter.send(:create_database_table, @line_definition)
76
- end
77
-
78
- it "should create an index on the request_id field" do
79
- @connection.should_receive(:add_index).with('test_lines', [:request_id])
80
- @database_inserter.send(:create_database_table, @line_definition)
81
- end
82
-
83
- it "should create a request_id field to link the requests together" do
84
- @connection.table_creator.should_receive(:column).with(:request_id, :integer)
85
- @database_inserter.send(:create_database_table, @line_definition)
86
- end
87
-
88
- it "should create a lineno field to save the location of the line in the original file" do
89
- @connection.table_creator.should_receive(:column).with(:lineno, :integer)
90
- @database_inserter.send(:create_database_table, @line_definition)
91
- end
92
-
93
- it "should create a field of the correct type for every defined field" do
94
- @connection.table_creator.should_receive(:column).with(:what, :string)
95
- @connection.table_creator.should_receive(:column).with(:tries, :integer)
96
- # :hash capture type should map on a :text field type
97
- @connection.table_creator.should_receive(:column).with(:evaluated, :text)
98
- @database_inserter.send(:create_database_table, @line_definition)
99
- end
100
-
101
- it "should create a field of the correct type for every provided field" do
102
- # :duration capture type should map on a :double field type
103
- @connection.table_creator.should_receive(:column).with(:evaluated_field, :double)
104
- @database_inserter.send(:create_database_table, @line_definition)
105
- end
106
- end
107
-
108
- describe '#create_activerecord_class' do
109
- before(:each) do
110
- # Make sure the ORM module exists
111
- @database_inserter.send(:initialize_orm_module!)
112
-
113
- # Mockthe request ORM class
114
- @request_class = mock('Request ActiveRecord::Base class')
115
- @request_class.stub!(:has_many)
116
- @database_inserter.stub!(:request_class).and_return(@request_class)
117
- end
118
-
119
- it "should register the constant for the line type AR class" do
120
- @database_inserter.send(:create_activerecord_class, @line_definition)
121
- @database_inserter.orm_module.const_defined?('TestLine').should be_true
122
- end
123
-
124
- it "should create a class that inherits from ActiveRecord::Base and the base class of the ORM module" do
125
- @database_inserter.send(:create_activerecord_class, @line_definition)
126
- @database_inserter.orm_module.const_get('TestLine').ancestors.should include(ActiveRecord::Base, @database_inserter.orm_module::Base)
127
- end
128
-
129
- describe 'defining the new ORM class' do
130
- before(:each) do
131
- # Mock the newly created ORM class for the test_line
132
- @orm_class = mock('Line ActiveRecord::Base class')
133
- @orm_class.stub!(:belongs_to)
134
- @orm_class.stub!(:serialize)
135
- Class.stub!(:new).and_return(@orm_class)
136
- end
137
-
138
- it "should create a has_many relation on the Request class" do
139
- @request_class.should_receive(:has_many).with(:test_lines)
140
- @database_inserter.send(:create_activerecord_class, @line_definition)
141
- end
142
-
143
- it "should create a belongs_to relation to the request class" do
144
- @orm_class.should_receive(:belongs_to).with(:request)
145
- @database_inserter.send(:create_activerecord_class, @line_definition)
146
- end
147
-
148
- it "should serialize the :evaluate field into the database" do
149
- @orm_class.should_receive(:serialize).with(:evaluated, Hash)
150
- @database_inserter.send(:create_activerecord_class, @line_definition)
151
- end
152
- end
153
- end
154
-
155
- describe '#create_database_schema!' do
156
-
157
- before(:each) do
158
- @line_type_cnt = testing_format.line_definitions.length
159
-
160
- @connection = mock_migrator
161
- @database_inserter.stub!(:connection).and_return(@connection)
162
-
163
- # Stub the expected method calls for the preparation
164
- @database_inserter.stub!(:create_database_table)
165
- @database_inserter.stub!(:create_activerecord_class)
166
-
167
- # Make sure the ORM module exists
168
- @database_inserter.send(:initialize_orm_module!)
169
- end
170
-
171
- it "should create a requests table to join request lines" do
172
- @connection.should_receive(:create_table).with("requests")
173
- @database_inserter.send :create_database_schema!
174
- end
175
-
176
- it "should create a Request class inheriting from ActiveRecord and the base class of the ORM module" do
177
- @database_inserter.send :create_database_schema!
178
- @database_inserter.request_class.ancestors.should include(ActiveRecord::Base, @database_inserter.orm_module::Base)
179
- end
180
-
181
- it "should create a warnings table for logging parse warnings" do
182
- @connection.should_receive(:create_table).with("warnings")
183
- @database_inserter.send :create_database_schema!
184
- end
185
-
186
- it "should create a Warning class inheriting from ActiveRecord and the base class of the ORM module" do
187
- @database_inserter.send :create_database_schema!
188
- @database_inserter.warning_class.ancestors.should include(ActiveRecord::Base, @database_inserter.orm_module::Base)
189
- end
190
-
191
- it "should create a sources table to track parsed files" do
192
- @connection.should_receive(:create_table).with("sources")
193
- @database_inserter.send :create_database_schema!
194
- end
195
-
196
- it "should create a Source ORM class" do
197
- @database_inserter.send :create_database_schema!
198
- @database_inserter.orm_module::Source.ancestors.should include(ActiveRecord::Base, @database_inserter.orm_module::Base)
199
- end
200
-
201
-
202
- it "should create a table for every line type" do
203
- @database_inserter.should_receive(:create_database_table).with(an_instance_of(RequestLogAnalyzer::LineDefinition)).exactly(@line_type_cnt).times
204
- @database_inserter.send :create_database_schema!
205
- end
206
-
207
- it "should create a ORM for every line type" do
208
- @database_inserter.should_receive(:create_activerecord_class).with(an_instance_of(RequestLogAnalyzer::LineDefinition)).exactly(@line_type_cnt).times
209
- @database_inserter.send :create_database_schema!
210
- end
211
- end
212
-
213
- describe '#aggregate' do
214
- before(:each) do
215
- @database_inserter.prepare
216
-
217
- @incomplete_request = testing_format.request( {:line_type => :first, :request_no => 564})
218
- @completed_request = testing_format.request( {:line_type => :first, :request_no => 564},
219
- {:line_type => :test, :test_capture => "awesome"},
220
- {:line_type => :test, :test_capture => "indeed"},
221
- {:line_type => :eval, :evaluated => { :greating => 'howdy'}, :greating => 'howdy' },
222
- {:line_type => :last, :request_no => 564})
223
- end
224
-
225
- it "should insert a record in the request table" do
226
- TestingFormat::Database::Request.count.should == 0
227
- @database_inserter.aggregate(@incomplete_request)
228
- TestingFormat::Database::Request.count.should == 1
229
- end
230
-
231
- it "should insert records in all relevant line tables" do
232
- @database_inserter.aggregate(@completed_request)
233
- request = TestingFormat::Database::Request.first
234
- request.should have(2).test_lines
235
- request.should have(1).first_lines
236
- request.should have(1).eval_lines
237
- request.should have(1).last_lines
238
- end
239
-
240
- it "should log a warning in the warnings table" do
241
- TestingFormat::Database::Warning.should_receive(:create!).with(hash_including(:warning_type => 'test_warning'))
242
- @database_inserter.warning(:test_warning, "Testing the warning system", 12)
243
- end
244
- end
245
- end