request-log-analyzer 1.0.2 → 1.6.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (144) hide show
  1. data/.gitignore +10 -0
  2. data/DESIGN.rdoc +41 -0
  3. data/LICENSE +4 -4
  4. data/README.rdoc +38 -0
  5. data/Rakefile +6 -3
  6. data/bin/request-log-analyzer +70 -72
  7. data/lib/cli/command_line_arguments.rb +53 -53
  8. data/lib/cli/database_console.rb +26 -0
  9. data/lib/cli/database_console_init.rb +43 -0
  10. data/lib/cli/progressbar.rb +166 -189
  11. data/lib/cli/tools.rb +49 -0
  12. data/lib/request_log_analyzer/aggregator/database_inserter.rb +83 -0
  13. data/lib/request_log_analyzer/aggregator/echo.rb +17 -12
  14. data/lib/request_log_analyzer/aggregator/summarizer.rb +101 -63
  15. data/lib/request_log_analyzer/{aggregator/base.rb → aggregator.rb} +17 -13
  16. data/lib/request_log_analyzer/controller.rb +251 -98
  17. data/lib/request_log_analyzer/database/base.rb +114 -0
  18. data/lib/request_log_analyzer/database/connection.rb +38 -0
  19. data/lib/request_log_analyzer/database/request.rb +22 -0
  20. data/lib/request_log_analyzer/database/source.rb +13 -0
  21. data/lib/request_log_analyzer/database/warning.rb +14 -0
  22. data/lib/request_log_analyzer/database.rb +102 -0
  23. data/lib/request_log_analyzer/file_format/amazon_s3.rb +74 -0
  24. data/lib/request_log_analyzer/file_format/apache.rb +147 -0
  25. data/lib/request_log_analyzer/file_format/delayed_job.rb +55 -0
  26. data/lib/request_log_analyzer/file_format/merb.rb +65 -29
  27. data/lib/request_log_analyzer/file_format/mysql.rb +101 -0
  28. data/lib/request_log_analyzer/file_format/postgresql.rb +68 -0
  29. data/lib/request_log_analyzer/file_format/rack.rb +9 -0
  30. data/lib/request_log_analyzer/file_format/rails.rb +164 -78
  31. data/lib/request_log_analyzer/file_format/rails3.rb +86 -0
  32. data/lib/request_log_analyzer/file_format/rails_development.rb +12 -0
  33. data/lib/request_log_analyzer/file_format.rb +252 -58
  34. data/lib/request_log_analyzer/filter/anonymize.rb +39 -0
  35. data/lib/request_log_analyzer/filter/field.rb +19 -13
  36. data/lib/request_log_analyzer/filter/timespan.rb +25 -12
  37. data/lib/request_log_analyzer/filter.rb +30 -0
  38. data/lib/request_log_analyzer/line_definition.rb +69 -96
  39. data/lib/request_log_analyzer/log_processor.rb +31 -53
  40. data/lib/request_log_analyzer/mailer.rb +65 -0
  41. data/lib/request_log_analyzer/output/fancy_html.rb +49 -0
  42. data/lib/request_log_analyzer/output/fixed_width.rb +220 -0
  43. data/lib/request_log_analyzer/output/html.rb +187 -0
  44. data/lib/request_log_analyzer/output.rb +117 -0
  45. data/lib/request_log_analyzer/request.rb +125 -40
  46. data/lib/request_log_analyzer/source/database_loader.rb +87 -0
  47. data/lib/request_log_analyzer/source/log_parser.rb +297 -0
  48. data/lib/request_log_analyzer/source.rb +72 -0
  49. data/lib/request_log_analyzer/tracker/duration.rb +28 -64
  50. data/lib/request_log_analyzer/tracker/frequency.rb +108 -0
  51. data/lib/request_log_analyzer/tracker/hourly_spread.rb +76 -49
  52. data/lib/request_log_analyzer/tracker/numeric_value.rb +223 -0
  53. data/lib/request_log_analyzer/tracker/timespan.rb +54 -27
  54. data/lib/request_log_analyzer/tracker/traffic.rb +40 -0
  55. data/lib/request_log_analyzer/tracker.rb +101 -0
  56. data/lib/request_log_analyzer.rb +43 -13
  57. data/request-log-analyzer.gemspec +41 -0
  58. data/spec/database.yml +23 -0
  59. data/spec/fixtures/apache_combined.log +5 -0
  60. data/spec/fixtures/apache_common.log +10 -0
  61. data/spec/fixtures/decompression.log +12 -0
  62. data/spec/fixtures/decompression.log.bz2 +0 -0
  63. data/spec/fixtures/decompression.log.gz +0 -0
  64. data/spec/fixtures/decompression.log.zip +0 -0
  65. data/spec/fixtures/decompression.tar.gz +0 -0
  66. data/spec/fixtures/decompression.tgz +0 -0
  67. data/spec/fixtures/header_and_footer.log +6 -0
  68. data/spec/fixtures/merb_prefixed.log +9 -0
  69. data/spec/fixtures/mysql_slow_query.log +110 -0
  70. data/spec/fixtures/postgresql.log +2980 -0
  71. data/spec/fixtures/rails.db +0 -0
  72. data/spec/fixtures/rails_22.log +1 -1
  73. data/spec/fixtures/sinatra.log +99 -0
  74. data/spec/integration/command_line_usage_spec.rb +84 -0
  75. data/spec/integration/mailer_spec.rb +179 -0
  76. data/spec/integration/munin_plugins_rails_spec.rb +58 -0
  77. data/spec/integration/scout_spec.rb +152 -0
  78. data/spec/lib/helpers.rb +72 -0
  79. data/spec/lib/macros.rb +18 -0
  80. data/spec/lib/matchers.rb +77 -0
  81. data/spec/lib/mocks.rb +77 -0
  82. data/spec/lib/testing_format.rb +46 -0
  83. data/spec/spec_helper.rb +16 -59
  84. data/spec/unit/aggregator/database_inserter_spec.rb +93 -0
  85. data/spec/unit/aggregator/summarizer_spec.rb +26 -0
  86. data/spec/unit/controller/controller_spec.rb +41 -0
  87. data/spec/unit/controller/log_processor_spec.rb +18 -0
  88. data/spec/unit/database/base_class_spec.rb +183 -0
  89. data/spec/unit/database/connection_spec.rb +34 -0
  90. data/spec/unit/database/database_spec.rb +133 -0
  91. data/spec/unit/file_format/amazon_s3_format_spec.rb +67 -0
  92. data/spec/unit/file_format/apache_format_spec.rb +203 -0
  93. data/spec/unit/file_format/common_regular_expressions_spec.rb +53 -0
  94. data/spec/unit/file_format/delayed_job_format_spec.rb +83 -0
  95. data/spec/unit/file_format/file_format_api_spec.rb +69 -0
  96. data/spec/unit/file_format/format_autodetection_spec.rb +40 -0
  97. data/spec/unit/file_format/line_definition_spec.rb +75 -0
  98. data/spec/unit/file_format/merb_format_spec.rb +52 -0
  99. data/spec/unit/file_format/mysql_format_spec.rb +154 -0
  100. data/spec/unit/file_format/postgresql_format_spec.rb +65 -0
  101. data/spec/unit/file_format/rack_format_spec.rb +50 -0
  102. data/spec/unit/file_format/rails3_format_spec.rb +120 -0
  103. data/spec/unit/file_format/rails_format_spec.rb +180 -0
  104. data/spec/unit/filter/anonymize_filter_spec.rb +21 -0
  105. data/spec/unit/filter/field_filter_spec.rb +66 -0
  106. data/spec/unit/filter/filter_spec.rb +17 -0
  107. data/spec/unit/filter/timespan_filter_spec.rb +58 -0
  108. data/spec/unit/mailer_spec.rb +42 -0
  109. data/spec/unit/request_spec.rb +111 -0
  110. data/spec/unit/source/log_parser_spec.rb +119 -0
  111. data/spec/unit/tracker/duration_tracker_spec.rb +49 -0
  112. data/spec/unit/tracker/frequency_tracker_spec.rb +88 -0
  113. data/spec/unit/tracker/hourly_spread_spec.rb +79 -0
  114. data/spec/unit/tracker/numeric_value_tracker_spec.rb +166 -0
  115. data/spec/unit/tracker/timespan_tracker_spec.rb +73 -0
  116. data/spec/unit/tracker/tracker_api_spec.rb +125 -0
  117. data/spec/unit/tracker/traffic_tracker_spec.rb +28 -0
  118. data/tasks/github-gem.rake +290 -139
  119. data/tasks/request_log_analyzer.rake +23 -7
  120. metadata +221 -94
  121. data/DESIGN +0 -14
  122. data/HACKING +0 -7
  123. data/README.textile +0 -36
  124. data/lib/cli/bashcolorizer.rb +0 -60
  125. data/lib/request_log_analyzer/aggregator/database.rb +0 -148
  126. data/lib/request_log_analyzer/filter/base.rb +0 -29
  127. data/lib/request_log_analyzer/log_parser.rb +0 -173
  128. data/lib/request_log_analyzer/source/base.rb +0 -42
  129. data/lib/request_log_analyzer/source/log_file.rb +0 -170
  130. data/lib/request_log_analyzer/tracker/base.rb +0 -54
  131. data/lib/request_log_analyzer/tracker/category.rb +0 -71
  132. data/spec/controller_spec.rb +0 -40
  133. data/spec/database_inserter_spec.rb +0 -101
  134. data/spec/file_format_spec.rb +0 -78
  135. data/spec/file_formats/spec_format.rb +0 -26
  136. data/spec/filter_spec.rb +0 -137
  137. data/spec/line_definition_spec.rb +0 -124
  138. data/spec/log_parser_spec.rb +0 -68
  139. data/spec/log_processor_spec.rb +0 -57
  140. data/spec/merb_format_spec.rb +0 -38
  141. data/spec/rails_format_spec.rb +0 -76
  142. data/spec/request_spec.rb +0 -72
  143. data/spec/summarizer_spec.rb +0 -9
  144. data/tasks/rspec.rake +0 -6
@@ -1,14 +1,12 @@
1
1
  module RequestLogAnalyzer
2
-
2
+
3
3
  # The RequestLogAnalyzer::Controller class creates a LogParser instance for the
4
- # requested file format, and connect it with sources and aggregators.
4
+ # requested file format and connect it with sources and aggregators.
5
5
  #
6
6
  # Sources are streams or files from which the requests will be parsed.
7
7
  # Aggregators will handle every passed request to yield a meaningfull results.
8
8
  #
9
- # - Use the build-function to build a controller instance using command line arguments.
10
- # - Use add_aggregator to register a new aggregator
11
- # - Use add_source to register a new aggregator
9
+ # - Use the build -function to build a new controller instance.
12
10
  # - Use the run! method to start the parser and send the requests to the aggregators.
13
11
  #
14
12
  # Note that the order of sources can be imported if you have log files than succeed
@@ -17,35 +15,50 @@ module RequestLogAnalyzer
17
15
  # from several logrotated log files.
18
16
  class Controller
19
17
 
20
- include RequestLogAnalyzer::FileFormat::Awareness
21
-
22
- attr_reader :aggregators
23
- attr_reader :filters
24
- attr_reader :log_parser
25
- attr_reader :source
26
- attr_reader :output
27
- attr_reader :options
18
+ attr_reader :source, :filters, :aggregators, :output, :options
28
19
 
29
20
  # Builds a RequestLogAnalyzer::Controller given parsed command line arguments
30
21
  # <tt>arguments<tt> A CommandLine::Arguments hash containing parsed commandline parameters.
31
- # <rr>report_with</tt> Width of the report. Defaults to 80.
32
- def self.build(arguments, report_width = 80)
33
-
34
- options = { :report_width => arguments[:report_width].to_i, :output => STDOUT}
35
-
36
- options[:database] = arguments[:database] if arguments[:database]
37
- options[:debug] = arguments[:debug]
38
- options[:colorize] = !arguments[:boring]
39
-
40
- if arguments[:file]
41
- options[:output] = File.new(arguments[:file], "w+")
42
- options[:colorize] = false
22
+ def self.build_from_arguments(arguments)
23
+
24
+ options = {}
25
+
26
+ # Copy fields
27
+ options[:database] = arguments[:database]
28
+ options[:reset_database] = arguments[:reset_database]
29
+ options[:debug] = arguments[:debug]
30
+ options[:yaml] = arguments[:yaml] || arguments[:dump]
31
+ options[:mail] = arguments[:mail]
32
+ options[:no_progress] = arguments[:no_progress]
33
+ options[:format] = arguments[:format]
34
+ options[:output] = arguments[:output].downcase
35
+ options[:file] = arguments[:file]
36
+ options[:after] = arguments[:after]
37
+ options[:before] = arguments[:before]
38
+ options[:reject] = arguments[:reject]
39
+ options[:select] = arguments[:select]
40
+ options[:boring] = arguments[:boring]
41
+ options[:aggregator] = arguments[:aggregator]
42
+ options[:report_width] = arguments[:report_width]
43
+ options[:report_sort] = arguments[:report_sort]
44
+ options[:report_amount] = arguments[:report_amount]
45
+ options[:mailhost] = arguments[:mailhost]
46
+
47
+ # Apache format workaround
48
+ if arguments[:rails_format]
49
+ options[:format] = {:rails => arguments[:rails_format]}
50
+ elsif arguments[:apache_format]
51
+ options[:format] = {:apache => arguments[:apache_format]}
43
52
  end
44
-
45
- # Create the controller with the correct file format
46
- file_format = RequestLogAnalyzer::FileFormat.load(arguments[:format])
47
-
48
- # register sources
53
+
54
+ # Handle output format casing
55
+ if options[:output].class == String
56
+ options[:output] = 'FancyHTML' if options[:output] =~ /^fancy_?html$/i
57
+ options[:output] = 'HTML' if options[:output] =~ /^html$/i
58
+ options[:output] = 'FixedWidth' if options[:output] =~ /^fixed_?width$/i
59
+ end
60
+
61
+ # Register sources
49
62
  if arguments.parameters.length == 1
50
63
  file = arguments.parameters[0]
51
64
  if file == '-' || file == 'STDIN'
@@ -60,50 +73,157 @@ module RequestLogAnalyzer
60
73
  options.store(:source_files, arguments.parameters)
61
74
  end
62
75
 
63
- controller = Controller.new(RequestLogAnalyzer::Source::LogFile.new(file_format, options), options)
76
+ # Guess file format
77
+ if !options[:format] && options[:source_files]
78
+ options[:format] = :rails # Default
79
+
80
+ if options[:source_files] != $stdin
81
+ if options[:source_files].class == String
82
+ options[:format] = RequestLogAnalyzer::FileFormat.autodetect(options[:source_files])
64
83
 
65
- options[:assume_correct_order] = arguments[:assume_correct_order]
84
+ elsif options[:source_files].class == Array && options[:source_files].first != $stdin
85
+ options[:format] = RequestLogAnalyzer::FileFormat.autodetect(options[:source_files].first)
86
+ end
87
+ end
88
+ end
66
89
 
67
- # register filters
68
- if arguments[:after] || arguments[:before]
69
- filter_options = {}
70
- filter_options[:after] = DateTime.parse(arguments[:after])
71
- filter_options[:before] = DateTime.parse(arguments[:before]) if arguments[:before]
72
- controller.add_filter(:timespan, filter_options)
90
+ build(options)
91
+ end
92
+
93
+ # Build a new controller.
94
+ # Returns a new RequestLogAnalyzer::Controller object.
95
+ #
96
+ # Options
97
+ # * <tt>:after</tt> Drop all requests after this date (Date, DateTime, Time, or a String in "YYYY-MM-DD hh:mm:ss" format)
98
+ # * <tt>:aggregator</tt> Array of aggregators (ATM: STRINGS OR SYMBOLS ONLY! - Defaults to [:summarizer]).
99
+ # * <tt>:boring</tt> Do not show color on STDOUT (Defaults to false).
100
+ # * <tt>:before</tt> Drop all requests before this date (Date, DateTime, Time or a String in "YYYY-MM-DD hh:mm:ss" format)
101
+ # * <tt>:database</tt> Database file to insert encountered requests to.
102
+ # * <tt>:debug</tt> Enables echo aggregator which will echo each request analyzed.
103
+ # * <tt>:file</tt> Filestring, File or StringIO.
104
+ # * <tt>:format</tt> :rails, {:apache => 'FORMATSTRING'}, :merb, :amazon_s3, :mysql or RequestLogAnalyzer::FileFormat class. (Defaults to :rails).
105
+ # * <tt>:mail</tt> Email the results to this email address.
106
+ # * <tt>:mailhost</tt> Email the results to this mail server.
107
+ # * <tt>:no_progress</tt> Do not display the progress bar (increases parsing speed).
108
+ # * <tt>:output</tt> 'FixedWidth', 'HTML' or RequestLogAnalyzer::Output class. Defaults to 'FixedWidth'.
109
+ # * <tt>:reject</tt> Reject specific {:field => :value} combination (expects a single hash).
110
+ # * <tt>:report_width</tt> Width of reports in characters for FixedWidth reports. (Defaults to 80)
111
+ # * <tt>:reset_database</tt> Reset the database before starting.
112
+ # * <tt>:select</tt> Select specific {:field => :value} combination (expects a single hash).
113
+ # * <tt>:source_files</tt> Source files to analyze. Provide either File, array of files or STDIN.
114
+ # * <tt>:yaml</tt> Output to YAML file.
115
+ #
116
+ # === Example
117
+ # RequestLogAnalyzer::Controller.build(
118
+ # :output => :HTML,
119
+ # :mail => 'root@localhost',
120
+ # :after => Time.now - 24*60*60,
121
+ # :source_files => '/var/log/passenger.log'
122
+ # ).run!
123
+ #
124
+ # === Todo
125
+ # * Check if defaults work (Aggregator defaults seem wrong).
126
+ # * Refactor :database => options[:database], :dump => options[:dump] away from contoller intialization.
127
+ def self.build(options)
128
+ # Defaults
129
+ options[:output] ||= 'FixedWidth'
130
+ options[:format] ||= :rails
131
+ options[:aggregator] ||= [:summarizer]
132
+ options[:report_width] ||= 80
133
+ options[:report_amount] ||= 20
134
+ options[:report_sort] ||= 'sum,mean'
135
+ options[:boring] ||= false
136
+
137
+ # Deprecation warnings
138
+ if options[:dump]
139
+ warn "[DEPRECATION] `:dump` is deprecated. Please use `:yaml` instead."
140
+ options[:yaml] = options[:dump]
73
141
  end
74
142
 
75
- arguments[:reject].each do |(field, value)|
76
- controller.add_filter(:field, :mode => :reject, :field => field, :value => value)
143
+ # Set the output class
144
+ output_args = {}
145
+ output_object = nil
146
+ if options[:output].is_a?(Class)
147
+ output_class = options[:output]
148
+ else
149
+ output_class = RequestLogAnalyzer::Output::const_get(options[:output])
150
+ end
151
+
152
+ output_sort = options[:report_sort].split(',').map { |s| s.to_sym }
153
+ output_amount = options[:report_amount] == 'all' ? :all : options[:report_amount].to_i
154
+
155
+ if options[:file]
156
+ output_object = %w[File StringIO].include?(options[:file].class.name) ? options[:file] : File.new(options[:file], "w+")
157
+ output_args = {:width => 80, :color => false, :characters => :ascii, :sort => output_sort, :amount => output_amount }
158
+ elsif options[:mail]
159
+ output_object = RequestLogAnalyzer::Mailer.new(options[:mail], options[:mailhost])
160
+ output_args = {:width => 80, :color => false, :characters => :ascii, :sort => output_sort, :amount => output_amount }
161
+ else
162
+ output_object = STDOUT
163
+ output_args = {:width => options[:report_width].to_i, :color => !options[:boring],
164
+ :characters => (options[:boring] ? :ascii : :utf), :sort => output_sort, :amount => output_amount }
77
165
  end
78
166
 
79
- arguments[:select].each do |(field, value)|
80
- controller.add_filter(:field, :mode => :select, :field => field, :value => value)
167
+ output_instance = output_class.new(output_object, output_args)
168
+
169
+ # Create the controller with the correct file format
170
+ if options[:format].kind_of?(Hash)
171
+ file_format = RequestLogAnalyzer::FileFormat.load(options[:format].keys[0], options[:format].values[0])
172
+ else
173
+ file_format = RequestLogAnalyzer::FileFormat.load(options[:format])
174
+ end
175
+
176
+ # Kickstart the controller
177
+ controller = Controller.new( RequestLogAnalyzer::Source::LogParser.new(file_format, :source_files => options[:source_files]),
178
+ { :output => output_instance,
179
+ :database => options[:database], # FUGLY!
180
+ :yaml => options[:yaml],
181
+ :reset_database => options[:reset_database],
182
+ :no_progress => options[:no_progress]})
183
+
184
+ # register filters
185
+ if options[:after] || options[:before]
186
+ filter_options = {}
187
+ [:after, :before].each do |filter|
188
+ case options[filter]
189
+ when Date, DateTime, Time
190
+ filter_options[filter] = options[filter]
191
+ when String
192
+ filter_options[filter] = DateTime.parse(options[filter])
193
+ end
194
+ end
195
+ controller.add_filter(:timespan, filter_options)
196
+ end
197
+
198
+ if options[:reject]
199
+ options[:reject].each do |(field, value)|
200
+ controller.add_filter(:field, :mode => :reject, :field => field, :value => value)
201
+ end
202
+ end
203
+
204
+ if options[:reject]
205
+ options[:select].each do |(field, value)|
206
+ controller.add_filter(:field, :mode => :select, :field => field, :value => value)
207
+ end
81
208
  end
82
209
 
83
210
  # register aggregators
84
- arguments[:aggregator].each { |agg| controller.add_aggregator(agg.to_sym) }
85
-
86
- # register the database
87
- controller.add_aggregator(:database) if arguments[:database] && !arguments[:aggregator].include?('database')
88
- controller.add_aggregator(:summarizer) if arguments[:aggregator].empty?
89
-
90
- # register the echo aggregator in debug mode
91
- controller.add_aggregator(:echo) if arguments[:debug]
92
-
211
+ options[:aggregator].each { |agg| controller.add_aggregator(agg.to_sym) }
212
+ controller.add_aggregator(:summarizer) if options[:aggregator].empty?
213
+ controller.add_aggregator(:echo) if options[:debug]
214
+ controller.add_aggregator(:database_inserter) if options[:database] && !options[:aggregator].include?('database')
215
+
93
216
  file_format.setup_environment(controller)
94
-
95
217
  return controller
96
- end
218
+ end
97
219
 
98
220
  # Builds a new Controller for the given log file format.
99
221
  # <tt>format</tt> Logfile format. Defaults to :rails
100
222
  # Options are passd on to the LogParser.
101
- # * <tt>:aggregator</tt> Aggregator array.
102
223
  # * <tt>:database</tt> Database the controller should use.
103
- # * <tt>:echo</tt> Output debug information.
104
- # * <tt>:silent</tt> Do not output any warnings.
105
- # * <tt>:colorize</tt> Colorize output
224
+ # * <tt>:yaml</tt> Yaml Dump the contrller should use.
106
225
  # * <tt>:output</tt> All report outputs get << through this output.
226
+ # * <tt>:no_progress</tt> No progress bar
107
227
  def initialize(source, options = {})
108
228
 
109
229
  @source = source
@@ -111,17 +231,17 @@ module RequestLogAnalyzer
111
231
  @aggregators = []
112
232
  @filters = []
113
233
  @output = options[:output]
234
+ @interrupted = false
114
235
 
115
- # Requester format through RequestLogAnalyzer::FileFormat and construct the parser
116
- register_file_format(@source.file_format)
117
-
118
- # Pass all warnings to every aggregator so they can do something useful with them.
119
- @source.warning = lambda { |type, message, lineno| @aggregators.each { |agg| agg.warning(type, message, lineno) } } if @source
236
+ # Register the request format for this session after checking its validity
237
+ raise "Invalid file format!" unless @source.file_format.valid?
120
238
 
121
- # Handle progress messagess
122
- @source.progress = lambda { |message, value| handle_progress(message, value) } if @source
239
+ # Install event handlers for wrnings, progress updates and source changes
240
+ @source.warning = lambda { |type, message, lineno| @aggregators.each { |agg| agg.warning(type, message, lineno) } }
241
+ @source.progress = lambda { |message, value| handle_progress(message, value) } unless options[:no_progress]
242
+ @source.source_changes = lambda { |change, filename| handle_source_change(change, filename) }
123
243
  end
124
-
244
+
125
245
  # Progress function.
126
246
  # Expects :started with file, :progress with current line and :finished or :interrupted when done.
127
247
  # <tt>message</tt> Current state (:started, :finished, :interupted or :progress).
@@ -129,7 +249,7 @@ module RequestLogAnalyzer
129
249
  def handle_progress(message, value = nil)
130
250
  case message
131
251
  when :started
132
- @progress_bar = ProgressBar.new(green(File.basename(value), options[:colorize]), File.size(value))
252
+ @progress_bar = CommandLine::ProgressBar.new(File.basename(value), File.size(value), STDOUT)
133
253
  when :finished
134
254
  @progress_bar.finish
135
255
  @progress_bar = nil
@@ -142,30 +262,46 @@ module RequestLogAnalyzer
142
262
  @progress_bar.set(value)
143
263
  end
144
264
  end
145
-
146
- # Adds an aggregator to the controller. The aggregator will be called for every request
265
+
266
+ # Source change handler
267
+ def handle_source_change(change, filename)
268
+ @aggregators.each { |agg| agg.source_change(change, File.expand_path(filename, Dir.pwd)) }
269
+ end
270
+
271
+ # Adds an aggregator to the controller. The aggregator will be called for every request
147
272
  # that is parsed from the provided sources (see add_source)
148
273
  def add_aggregator(agg)
149
- if agg.kind_of?(Symbol)
150
- require File.dirname(__FILE__) + "/aggregator/#{agg}"
151
- agg = RequestLogAnalyzer::Aggregator.const_get(agg.to_s.split(/[^a-z0-9]/i).map{ |w| w.capitalize }.join(''))
152
- end
153
-
274
+ agg = RequestLogAnalyzer::Aggregator.const_get(RequestLogAnalyzer::to_camelcase(agg)) if agg.kind_of?(Symbol)
154
275
  @aggregators << agg.new(@source, @options)
155
276
  end
156
-
277
+
157
278
  alias :>> :add_aggregator
158
-
279
+
159
280
  # Adds a request filter to the controller.
160
281
  def add_filter(filter, filter_options = {})
161
- if filter.kind_of?(Symbol)
162
- require File.dirname(__FILE__) + "/filter/#{filter}"
163
- filter = RequestLogAnalyzer::Filter.const_get(filter.to_s.split(/[^a-z0-9]/i).map{ |w| w.capitalize }.join(''))
282
+ filter = RequestLogAnalyzer::Filter.const_get(RequestLogAnalyzer::to_camelcase(filter)) if filter.kind_of?(Symbol)
283
+ @filters << filter.new(source.file_format, @options.merge(filter_options))
284
+ end
285
+
286
+ # Push a request through the entire filterchain (@filters).
287
+ # <tt>request</tt> The request to filter.
288
+ # Returns the filtered request or nil.
289
+ def filter_request(request)
290
+ @filters.each do |filter|
291
+ request = filter.filter(request)
292
+ return nil if request.nil?
164
293
  end
165
-
166
- @filters << filter.new(file_format, @options.merge(filter_options))
294
+ return request
295
+ end
296
+
297
+ # Push a request to all the aggregators (@aggregators).
298
+ # <tt>request</tt> The request to push to the aggregators.
299
+ def aggregate_request(request)
300
+ return false unless request
301
+ @aggregators.each { |agg| agg.aggregate(request) }
302
+ return true
167
303
  end
168
-
304
+
169
305
  # Runs RequestLogAnalyzer
170
306
  # 1. Call prepare on every aggregator
171
307
  # 2. Generate requests from source object
@@ -175,27 +311,44 @@ module RequestLogAnalyzer
175
311
  # 5. Call report on every aggregator
176
312
  # 6. Finalize Source
177
313
  def run!
178
-
179
- @filters.each { |filter| filter.prepare }
314
+
315
+ # @aggregators.each{|agg| p agg}
316
+
180
317
  @aggregators.each { |agg| agg.prepare }
181
-
182
- begin
183
- @source.requests do |request|
184
- #@filters.each { |filter| request = filter.filter(request) }
185
- @aggregators.each { |agg| agg.aggregate(request) } if request
186
- end
187
- rescue Interrupt => e
188
- handle_progress(:interrupted)
189
- puts "Caught interrupt! Stopped parsing."
318
+ install_signal_handlers
319
+
320
+ @source.each_request do |request|
321
+ break if @interrupted
322
+ aggregate_request(filter_request(request))
190
323
  end
191
324
 
192
- puts "\n"
193
-
194
325
  @aggregators.each { |agg| agg.finalize }
195
- @aggregators.each { |agg| agg.report(@output, options[:report_width], options[:colorize]) }
196
-
326
+
327
+ @output.header
328
+ @aggregators.each { |agg| agg.report(@output) }
329
+ @output.footer
330
+
197
331
  @source.finalize
332
+
333
+ if @output.io.kind_of?(File)
334
+ puts
335
+ puts "Report written to: " + File.expand_path(@output.io.path)
336
+ puts "Need an expert to analyze your application?"
337
+ puts "Mail to contact@railsdoctors.com or visit us at http://railsdoctors.com"
338
+ puts "Thanks for using request-log-analyzer!"
339
+ @output.io.close
340
+ elsif @output.io.kind_of?(RequestLogAnalyzer::Mailer)
341
+ @output.io.mail
342
+ end
198
343
  end
199
-
344
+
345
+ def install_signal_handlers
346
+ Signal.trap("INT") do
347
+ handle_progress(:interrupted)
348
+ puts "Caught interrupt! Stopping parsing..."
349
+ @interrupted = true
350
+ end
351
+ end
352
+
200
353
  end
201
- end
354
+ end
@@ -0,0 +1,114 @@
1
+ class RequestLogAnalyzer::Database::Base < ActiveRecord::Base
2
+
3
+ self.abstract_class = true
4
+
5
+ def <=>(other)
6
+ if (source_id.nil? && other.source_id.nil?) || (source_comparison = source_id <=> other.source_id) == 0
7
+ lineno <=> other.lineno
8
+ else
9
+ source_comparison
10
+ end
11
+ end
12
+
13
+ def line_type
14
+ self.class.name.underscore.gsub(/_line$/, '').to_sym
15
+ end
16
+
17
+ class_inheritable_accessor :line_definition
18
+ cattr_accessor :database
19
+
20
+ def self.subclass_from_line_definition(definition)
21
+ klass = Class.new(RequestLogAnalyzer::Database::Base)
22
+ klass.set_table_name("#{definition.name}_lines")
23
+
24
+ klass.line_definition = definition
25
+
26
+ # Set relations with requests and sources table
27
+ klass.belongs_to :request
28
+ klass.belongs_to :source
29
+
30
+ # Serialize complex fields into the database
31
+ definition.captures.select { |c| c.has_key?(:provides) }.each do |capture|
32
+ klass.send(:serialize, capture[:name], Hash)
33
+ end
34
+
35
+ RequestLogAnalyzer::Database::Request.has_many "#{definition.name}_lines".to_sym
36
+ RequestLogAnalyzer::Database::Source.has_many "#{definition.name}_lines".to_sym
37
+
38
+ return klass
39
+ end
40
+
41
+ def self.subclass_from_table(table)
42
+ raise "Table #{table} not found!" unless database.connection.table_exists?(table)
43
+
44
+ klass = Class.new(RequestLogAnalyzer::Database::Base)
45
+ klass.set_table_name(table)
46
+
47
+ if klass.column_names.include?('request_id')
48
+ klass.belongs_to :request
49
+ RequestLogAnalyzer::Database::Request.has_many table.to_sym
50
+ end
51
+
52
+ if klass.column_names.include?('source_id')
53
+ klass.belongs_to :source
54
+ RequestLogAnalyzer::Database::Source.has_many table.to_sym
55
+ end
56
+
57
+ return klass
58
+ end
59
+
60
+ def self.drop_table!
61
+ database.connection.remove_index(self.table_name, [:source_id]) rescue nil
62
+ database.connection.remove_index(self.table_name, [:request_id]) rescue nil
63
+ database.connection.drop_table(self.table_name) if database.connection.table_exists?(self.table_name)
64
+ end
65
+
66
+ def self.create_table!
67
+ raise "No line_definition available to base table schema on!" unless self.line_definition
68
+
69
+ unless table_exists?
70
+ database.connection.create_table(table_name.to_sym) do |t|
71
+
72
+ # Default fields
73
+ t.column :request_id, :integer
74
+ t.column :source_id, :integer
75
+ t.column :lineno, :integer
76
+
77
+ line_definition.captures.each do |capture|
78
+ # Add a field for every capture
79
+ t.column(capture[:name], column_type(capture[:type]))
80
+
81
+ # If the capture provides other field as well, create columns for them, too
82
+ capture[:provides].each { |field, field_type| t.column(field, column_type(field_type)) } if capture[:provides].kind_of?(Hash)
83
+ end
84
+ end
85
+
86
+ # Add indices to table for more speedy querying
87
+ database.connection.add_index(self.table_name.to_sym, [:request_id]) # rescue
88
+ database.connection.add_index(self.table_name.to_sym, [:source_id]) # rescue
89
+ end
90
+ end
91
+
92
+ # Function to determine the column type for a field
93
+ # TODO: make more robust / include in file-format definition
94
+ def self.column_type(type_indicator)
95
+ case type_indicator
96
+ when :eval; :text
97
+ when :hash; :text
98
+ when :text; :text
99
+ when :string; :string
100
+ when :sec; :double
101
+ when :msec; :double
102
+ when :duration; :double
103
+ when :float; :double
104
+ when :double; :double
105
+ when :integer; :integer
106
+ when :int; :int
107
+ when :timestamp; :datetime
108
+ when :datetime; :datetime
109
+ when :date; :date
110
+ else :string
111
+ end
112
+ end
113
+
114
+ end
@@ -0,0 +1,38 @@
1
+ module RequestLogAnalyzer::Database::Connection
2
+
3
+ def self.from_string(string)
4
+ hash = {}
5
+ if string =~ /^(?:\w+=(?:[^;])*;)*\w+=(?:[^;])*$/
6
+ string.scan(/(\w+)=([^;]*);?/) { |variable, value| hash[variable.to_sym] = value }
7
+ elsif string =~ /^(\w+)\:\/\/(?:(?:([^:]+)(?:\:([^:]+))?\@)?([\w\.-]+)\/)?([\w\:\-\.\/]+)$/
8
+ hash[:adapter], hash[:username], hash[:password], hash[:host], hash[:database] = $1, $2, $3, $4, $5
9
+ hash.delete_if { |k, v| v.nil? }
10
+ end
11
+ return hash.empty? ? nil : hash
12
+ end
13
+
14
+ def connect(connection_identifier)
15
+ if connection_identifier.kind_of?(Hash)
16
+ RequestLogAnalyzer::Database::Base.establish_connection(connection_identifier)
17
+ elsif connection_identifier == ':memory:'
18
+ RequestLogAnalyzer::Database::Base.establish_connection(:adapter => 'sqlite3', :database => ':memory:')
19
+ elsif connection_hash = RequestLogAnalyzer::Database::Connection.from_string(connection_identifier)
20
+ RequestLogAnalyzer::Database::Base.establish_connection(connection_hash)
21
+ elsif connection_identifier.kind_of?(String) # Normal SQLite 3 database file
22
+ RequestLogAnalyzer::Database::Base.establish_connection(:adapter => 'sqlite3', :database => connection_identifier)
23
+ elsif connection_identifier.nil?
24
+ nil
25
+ else
26
+ raise "Cannot connect with this connection_identifier: #{connection_identifier.inspect}"
27
+ end
28
+ end
29
+
30
+ def disconnect
31
+ RequestLogAnalyzer::Database::Base.remove_connection
32
+ end
33
+
34
+ def connection
35
+ RequestLogAnalyzer::Database::Base.connection
36
+ end
37
+
38
+ end
@@ -0,0 +1,22 @@
1
+ class RequestLogAnalyzer::Database::Request < RequestLogAnalyzer::Database::Base
2
+
3
+ # Returns an array of all the Line objects of this request in the correct order.
4
+ def lines
5
+ @lines ||= begin
6
+ lines = []
7
+ self.class.reflections.each { |r, d| lines += self.send(r).all }
8
+ lines.sort
9
+ end
10
+ end
11
+
12
+ # Creates the table to store requests in.
13
+ def self.create_table!
14
+ unless database.connection.table_exists?(:requests)
15
+ database.connection.create_table(:requests) do |t|
16
+ t.column :first_lineno, :integer
17
+ t.column :last_lineno, :integer
18
+ end
19
+ end
20
+ end
21
+
22
+ end
@@ -0,0 +1,13 @@
1
+ class RequestLogAnalyzer::Database::Source < RequestLogAnalyzer::Database::Base
2
+
3
+ def self.create_table!
4
+ unless database.connection.table_exists?(:sources)
5
+ database.connection.create_table(:sources) do |t|
6
+ t.column :filename, :string
7
+ t.column :mtime, :datetime
8
+ t.column :filesize, :integer
9
+ end
10
+ end
11
+ end
12
+
13
+ end
@@ -0,0 +1,14 @@
1
+ class RequestLogAnalyzer::Database::Warning < RequestLogAnalyzer::Database::Base
2
+
3
+ def self.create_table!
4
+ unless database.connection.table_exists?(:warnings)
5
+ database.connection.create_table(:warnings) do |t|
6
+ t.column :warning_type, :string, :limit => 30, :null => false
7
+ t.column :message, :string
8
+ t.column :source_id, :integer
9
+ t.column :lineno, :integer
10
+ end
11
+ end
12
+ end
13
+
14
+ end