request-log-analyzer 1.7.0 → 1.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (88) hide show
  1. data/.gitignore +4 -1
  2. data/DESIGN.rdoc +0 -1
  3. data/Gemfile +2 -0
  4. data/README.rdoc +16 -5
  5. data/bin/request-log-analyzer +4 -2
  6. data/lib/cli/database_console_init.rb +1 -1
  7. data/lib/cli/tools.rb +2 -1
  8. data/lib/mixins/gets_memory_protection.rb +1 -1
  9. data/lib/request_log_analyzer/aggregator/summarizer.rb +2 -2
  10. data/lib/request_log_analyzer/aggregator.rb +3 -3
  11. data/lib/request_log_analyzer/controller.rb +42 -36
  12. data/lib/request_log_analyzer/database.rb +5 -3
  13. data/lib/request_log_analyzer/file_format/apache.rb +19 -28
  14. data/lib/request_log_analyzer/file_format/delayed_job2.rb +38 -0
  15. data/lib/request_log_analyzer/file_format/delayed_job21.rb +39 -0
  16. data/lib/request_log_analyzer/file_format/haproxy.rb +138 -0
  17. data/lib/request_log_analyzer/file_format/oink.rb +96 -0
  18. data/lib/request_log_analyzer/file_format/postgresql.rb +7 -6
  19. data/lib/request_log_analyzer/file_format/rails.rb +4 -4
  20. data/lib/request_log_analyzer/file_format/rails3.rb +34 -11
  21. data/lib/request_log_analyzer/file_format/w3c.rb +56 -0
  22. data/lib/request_log_analyzer/file_format.rb +33 -10
  23. data/lib/request_log_analyzer/filter.rb +3 -5
  24. data/lib/request_log_analyzer/output/fancy_html.rb +0 -5
  25. data/lib/request_log_analyzer/output.rb +6 -7
  26. data/lib/request_log_analyzer/request.rb +17 -3
  27. data/lib/request_log_analyzer/source/log_parser.rb +3 -1
  28. data/lib/request_log_analyzer/source.rb +2 -6
  29. data/lib/request_log_analyzer/tracker/duration.rb +7 -0
  30. data/lib/request_log_analyzer/tracker/numeric_value.rb +123 -12
  31. data/lib/request_log_analyzer/tracker/traffic.rb +7 -0
  32. data/lib/request_log_analyzer/tracker.rb +6 -5
  33. data/lib/request_log_analyzer.rb +36 -17
  34. data/request-log-analyzer.gemspec +9 -6
  35. data/spec/fixtures/oink_22.log +19 -0
  36. data/spec/fixtures/oink_22_failure.log +17 -0
  37. data/spec/fixtures/postgresql.log +29 -2980
  38. data/spec/integration/command_line_usage_spec.rb +4 -4
  39. data/spec/integration/mailer_spec.rb +20 -4
  40. data/spec/integration/munin_plugins_rails_spec.rb +1 -1
  41. data/spec/integration/scout_spec.rb +1 -1
  42. data/spec/lib/helpers.rb +2 -2
  43. data/spec/lib/macros.rb +1 -1
  44. data/spec/lib/matchers.rb +1 -1
  45. data/spec/lib/mocks.rb +1 -1
  46. data/spec/spec_helper.rb +10 -10
  47. data/spec/unit/aggregator/database_inserter_spec.rb +1 -1
  48. data/spec/unit/aggregator/summarizer_spec.rb +1 -1
  49. data/spec/unit/controller/controller_spec.rb +1 -1
  50. data/spec/unit/controller/log_processor_spec.rb +1 -1
  51. data/spec/unit/database/base_class_spec.rb +1 -1
  52. data/spec/unit/database/connection_spec.rb +1 -1
  53. data/spec/unit/database/database_spec.rb +1 -1
  54. data/spec/unit/file_format/amazon_s3_format_spec.rb +1 -1
  55. data/spec/unit/file_format/apache_format_spec.rb +18 -1
  56. data/spec/unit/file_format/common_regular_expressions_spec.rb +1 -1
  57. data/spec/unit/file_format/delayed_job21_format_spec.rb +60 -0
  58. data/spec/unit/file_format/delayed_job2_format_spec.rb +60 -0
  59. data/spec/unit/file_format/delayed_job_format_spec.rb +2 -2
  60. data/spec/unit/file_format/file_format_api_spec.rb +2 -2
  61. data/spec/unit/file_format/format_autodetection_spec.rb +7 -7
  62. data/spec/unit/file_format/haproxy_format_spec.rb +84 -0
  63. data/spec/unit/file_format/line_definition_spec.rb +1 -1
  64. data/spec/unit/file_format/merb_format_spec.rb +1 -1
  65. data/spec/unit/file_format/mysql_format_spec.rb +1 -1
  66. data/spec/unit/file_format/oink_format_spec.rb +105 -0
  67. data/spec/unit/file_format/postgresql_format_spec.rb +24 -46
  68. data/spec/unit/file_format/rack_format_spec.rb +1 -1
  69. data/spec/unit/file_format/rails3_format_spec.rb +41 -14
  70. data/spec/unit/file_format/rails_format_spec.rb +11 -1
  71. data/spec/unit/file_format/w3c_format_spec.rb +46 -0
  72. data/spec/unit/filter/anonymize_filter_spec.rb +1 -1
  73. data/spec/unit/filter/field_filter_spec.rb +1 -1
  74. data/spec/unit/filter/filter_spec.rb +1 -1
  75. data/spec/unit/filter/timespan_filter_spec.rb +1 -1
  76. data/spec/unit/mailer_spec.rb +1 -1
  77. data/spec/unit/request_spec.rb +1 -1
  78. data/spec/unit/source/log_parser_spec.rb +1 -1
  79. data/spec/unit/tracker/duration_tracker_spec.rb +4 -2
  80. data/spec/unit/tracker/frequency_tracker_spec.rb +1 -1
  81. data/spec/unit/tracker/hourly_spread_spec.rb +1 -1
  82. data/spec/unit/tracker/numeric_value_tracker_spec.rb +38 -17
  83. data/spec/unit/tracker/timespan_tracker_spec.rb +1 -1
  84. data/spec/unit/tracker/tracker_api_spec.rb +1 -1
  85. data/spec/unit/tracker/traffic_tracker_spec.rb +1 -1
  86. data/tasks/github-gem.rake +97 -60
  87. data/tasks/request_log_analyzer.rake +102 -0
  88. metadata +198 -153
@@ -1,4 +1,4 @@
1
- require File.dirname(__FILE__) + '/../spec_helper.rb'
1
+ require 'spec_helper'
2
2
 
3
3
  describe RequestLogAnalyzer, 'running from command line' do
4
4
 
@@ -38,7 +38,7 @@ describe RequestLogAnalyzer, 'running from command line' do
38
38
 
39
39
  it "should not write output with the --silent option" do
40
40
  output = run("#{log_fixture(:rails_1x)} --silent --file #{temp_output_file(:report)}")
41
- output.to_s.should eql("")
41
+ output.should be_empty
42
42
  File.exist?(temp_output_file(:report)).should be_true
43
43
  end
44
44
 
@@ -83,9 +83,9 @@ describe RequestLogAnalyzer, 'running from command line' do
83
83
  end
84
84
 
85
85
  it "should dump the results to a YAML file" do
86
- run("#{log_fixture(:rails_1x)} --dump #{temp_output_file(:yaml)}")
86
+ run("#{log_fixture(:rails_1x)} --yaml #{temp_output_file(:yaml)}")
87
87
  File.exist?(temp_output_file(:yaml)).should be_true
88
- YAML::load(File.read(temp_output_file(:yaml))).should have_at_least(1).item
88
+ YAML.load(File.read(temp_output_file(:yaml))).should have_at_least(1).item
89
89
  end
90
90
 
91
91
  it "should parse 4 requests from the standard input" do
@@ -1,4 +1,4 @@
1
- require File.dirname(__FILE__) + '/../spec_helper.rb'
1
+ require 'spec_helper'
2
2
  require 'socket'
3
3
 
4
4
  describe RequestLogAnalyzer, 'running mailer integration' do
@@ -30,8 +30,24 @@ describe RequestLogAnalyzer, 'running mailer integration' do
30
30
  find_string_in_file("From: <contact@railsdoctors.com>", @log_file).should_not be_nil
31
31
  find_string_in_file("To: <root@localhost>", @log_file).should_not be_nil
32
32
  find_string_in_file("From: Request-log-analyzer reporter <contact@railsdoctors.com>", @log_file).should_not be_nil
33
+ find_string_in_file("Subject: Request log analyzer report - generated on", @log_file).should_not be_nil
33
34
  find_string_in_file("Request summary", @log_file).should_not be_nil
34
- find_string_in_file("PeopleController#show.html [ | 1 | 0.29s | 0.29s | 0.00s | 0.29s | 0.29s", @log_file).should_not be_nil
35
+ find_string_in_file("PeopleControll | 1 | 0.04s | 0.04s | 0.00s | 0.04s | 0.04s | 0.04s-0.04s", @log_file).should_not be_nil
36
+ end
37
+
38
+ it "should allow a custom mail subject" do
39
+ RequestLogAnalyzer::Controller.build(
40
+ :mail => 'root@localhost',
41
+ :mailhost => 'localhost:2525',
42
+ :mailsubject => 'TESTSUBJECT',
43
+ :source_files => log_fixture(:rails_1x),
44
+ :format => RequestLogAnalyzer::FileFormat::Rails,
45
+ :no_progress => true
46
+ ).run!
47
+
48
+ Process.wait # Wait for mailer to complete
49
+
50
+ find_string_in_file("Subject: TESTSUBJECT", @log_file).should_not be_nil
35
51
  end
36
52
 
37
53
  it "should send html mail" do
@@ -50,7 +66,7 @@ describe RequestLogAnalyzer, 'running mailer integration' do
50
66
  find_string_in_file("To: <root@localhost>", @log_file).should_not be_nil
51
67
  find_string_in_file("From: Request-log-analyzer reporter <contact@railsdoctors.com>", @log_file).should_not be_nil
52
68
  find_string_in_file('<h1>Request-log-analyzer summary report</h1>', @log_file).should_not be_nil
53
- find_string_in_file('<td class="alt">0.29s</td></tr><tr><td>DashboardController#index.html [GET]</td>', @log_file).should_not be_nil
69
+ find_string_in_file('<td class="alt">0.29s-0.30s</td></tr><tr><td>DashboardController#index.html [GET]</td>', @log_file).should_not be_nil
54
70
  end
55
71
  end
56
72
 
@@ -78,7 +94,7 @@ class Mailtrap
78
94
  @port = port
79
95
  @once = once
80
96
  @msgfile = msgfile
81
-
97
+
82
98
  File.open( @msgfile, "a" ) do |file|
83
99
  file.puts "\n* Mailtrap started at #{@host}:#{port}\n"
84
100
  end
@@ -1,4 +1,4 @@
1
- require File.dirname(__FILE__) + '/../spec_helper.rb'
1
+ require 'spec_helper'
2
2
 
3
3
  describe RequestLogAnalyzer, 'when harvesting like munin-plugins-rails the YAML output' do
4
4
 
@@ -1,4 +1,4 @@
1
- require File.dirname(__FILE__) + '/../spec_helper.rb'
1
+ require 'spec_helper'
2
2
 
3
3
  def capture_stdout_and_stderr_with_warnings_on
4
4
  $stdout, $stderr, warnings, $VERBOSE =
data/spec/lib/helpers.rb CHANGED
@@ -1,4 +1,4 @@
1
- module RequestLogAnalyzer::Spec::Helpers
1
+ module RequestLogAnalyzer::RSpec::Helpers
2
2
 
3
3
  # Create or return a new TestingFormat
4
4
  def testing_format
@@ -53,7 +53,7 @@ module RequestLogAnalyzer::Spec::Helpers
53
53
  # Check if a given string can be found in the given file
54
54
  # Returns the line number if found, nil otherwise
55
55
  def find_string_in_file(string, file, options = {})
56
- return nil unless File::exists?(file)
56
+ return nil unless File.exists?(file)
57
57
 
58
58
  line_counter = 0
59
59
 
data/spec/lib/macros.rb CHANGED
@@ -1,4 +1,4 @@
1
- module RequestLogAnalyzer::Spec::Macros
1
+ module RequestLogAnalyzer::RSpec::Macros
2
2
 
3
3
  def test_databases
4
4
  require 'yaml'
data/spec/lib/matchers.rb CHANGED
@@ -1,4 +1,4 @@
1
- module RequestLogAnalyzer::Spec::Matchers
1
+ module RequestLogAnalyzer::RSpec::Matchers
2
2
 
3
3
  class HasLineDefinition
4
4
 
data/spec/lib/mocks.rb CHANGED
@@ -1,4 +1,4 @@
1
- module RequestLogAnalyzer::Spec::Mocks
1
+ module RequestLogAnalyzer::RSpec::Mocks
2
2
 
3
3
  def mock_source
4
4
  source = mock('RequestLogAnalyzer::Source::Base')
data/spec/spec_helper.rb CHANGED
@@ -1,11 +1,11 @@
1
- $:.reject! { |e| e.include? 'TextMate' }
2
- $: << File.join(File.dirname(__FILE__), '..', 'lib')
3
-
4
1
  require 'rubygems'
5
- require 'spec/autorun'
2
+ require 'bundler/setup'
3
+
4
+ require 'rspec'
6
5
  require 'request_log_analyzer'
6
+ require 'stringio'
7
7
 
8
- module RequestLogAnalyzer::Spec
8
+ module RequestLogAnalyzer::RSpec
9
9
  end
10
10
 
11
11
  # Include all files in the spec_helper directory
@@ -15,10 +15,10 @@ end
15
15
 
16
16
  Dir.mkdir("#{File.dirname(__FILE__)}/../tmp") unless File.exist?("#{File.dirname(__FILE__)}/../tmp")
17
17
 
18
- Spec::Runner.configure do |config|
19
- config.include RequestLogAnalyzer::Spec::Matchers
20
- config.include RequestLogAnalyzer::Spec::Mocks
21
- config.include RequestLogAnalyzer::Spec::Helpers
18
+ RSpec.configure do |config|
19
+ config.include RequestLogAnalyzer::RSpec::Matchers
20
+ config.include RequestLogAnalyzer::RSpec::Mocks
21
+ config.include RequestLogAnalyzer::RSpec::Helpers
22
22
 
23
- config.extend RequestLogAnalyzer::Spec::Macros
23
+ config.extend RequestLogAnalyzer::RSpec::Macros
24
24
  end
@@ -1,4 +1,4 @@
1
- require File.dirname(__FILE__) + '/../../spec_helper'
1
+ require 'spec_helper'
2
2
 
3
3
  describe RequestLogAnalyzer::Aggregator::DatabaseInserter do
4
4
 
@@ -1,4 +1,4 @@
1
- require File.dirname(__FILE__) + '/../../spec_helper'
1
+ require 'spec_helper'
2
2
 
3
3
  describe RequestLogAnalyzer::Aggregator::Summarizer do
4
4
 
@@ -1,4 +1,4 @@
1
- require File.dirname(__FILE__) + '/../../spec_helper'
1
+ require 'spec_helper'
2
2
 
3
3
  describe RequestLogAnalyzer::Controller do
4
4
 
@@ -1,4 +1,4 @@
1
- require File.dirname(__FILE__) + '/../../spec_helper'
1
+ require 'spec_helper'
2
2
 
3
3
  require 'request_log_analyzer/log_processor'
4
4
 
@@ -1,4 +1,4 @@
1
- require File.dirname(__FILE__) + '/../../spec_helper'
1
+ require 'spec_helper'
2
2
 
3
3
  describe RequestLogAnalyzer::Database::Base do
4
4
 
@@ -1,4 +1,4 @@
1
- require File.dirname(__FILE__) + '/../../spec_helper'
1
+ require 'spec_helper'
2
2
 
3
3
  describe RequestLogAnalyzer::Database::Connection do
4
4
  describe '.from_string' do
@@ -1,4 +1,4 @@
1
- require File.dirname(__FILE__) + '/../../spec_helper'
1
+ require 'spec_helper'
2
2
 
3
3
  describe RequestLogAnalyzer::Database do
4
4
 
@@ -1,4 +1,4 @@
1
- require File.dirname(__FILE__) + '/../../spec_helper'
1
+ require 'spec_helper'
2
2
 
3
3
  describe RequestLogAnalyzer::FileFormat::AmazonS3 do
4
4
 
@@ -1,4 +1,4 @@
1
- require File.dirname(__FILE__) + '/../../spec_helper'
1
+ require 'spec_helper'
2
2
 
3
3
  describe RequestLogAnalyzer::FileFormat::Apache do
4
4
 
@@ -28,6 +28,23 @@ describe RequestLogAnalyzer::FileFormat::Apache do
28
28
  @line_definition.captures?(:duration).should be_true
29
29
  end
30
30
  end
31
+
32
+ describe '.access_line_definition' do
33
+ it "should parse values in microseconds when no argument is given to %D" do
34
+ format = RequestLogAnalyzer::FileFormat::Apache.create('%D')
35
+ format.should parse_line('12345').and_capture(:duration => 0.012345)
36
+ end
37
+
38
+ it "should parse values in microseconds when micro is given as argument to %D" do
39
+ format = RequestLogAnalyzer::FileFormat::Apache.create('%{micro}D')
40
+ format.should parse_line('12345').and_capture(:duration => 0.012345)
41
+ end
42
+
43
+ it "should parse values in microseconds when micro is given as argument to %D" do
44
+ format = RequestLogAnalyzer::FileFormat::Apache.create('%{milli}D')
45
+ format.should parse_line('12345').and_capture(:duration => 12.345)
46
+ end
47
+ end
31
48
 
32
49
  describe '.create' do
33
50
 
@@ -1,4 +1,4 @@
1
- require File.dirname(__FILE__) + '/../../spec_helper'
1
+ require 'spec_helper'
2
2
 
3
3
  describe RequestLogAnalyzer::FileFormat::CommonRegularExpressions do
4
4
 
@@ -0,0 +1,60 @@
1
+ require 'spec_helper'
2
+
3
+ describe RequestLogAnalyzer::FileFormat::DelayedJob do
4
+
5
+ it "should be a valid file format" do
6
+ RequestLogAnalyzer::FileFormat.load(:delayed_job).should be_valid
7
+ end
8
+
9
+ describe '#parse_line' do
10
+
11
+ before(:each) do
12
+ @file_format = RequestLogAnalyzer::FileFormat.load(:delayed_job21)
13
+ end
14
+
15
+ it "should parse a :job_lock line correctly" do
16
+ line = "2010-05-17T17:37:34+0000: [Worker(delayed_job host:hostname.co.uk pid:11888)] acquired lock on S3FileJob"
17
+ @file_format.should parse_line(line).as(:job_lock).and_capture(:timestamp => 20100517173734,
18
+ :job => 'S3FileJob', :host => 'hostname.co.uk', :pid => 11888)
19
+ end
20
+
21
+ it "should parse a :job_lock line correctly when the worker is one of many" do
22
+ line = "2010-05-17T17:37:34+0000: [Worker(delayed_job.0 host:hostname.co.uk pid:11888)] acquired lock on S3FileJob"
23
+ @file_format.should parse_line(line).as(:job_lock).and_capture(:timestamp => 20100517173734,
24
+ :job => 'S3FileJob', :host => 'hostname.co.uk', :pid => 11888)
25
+ end
26
+
27
+ it "should parse a :job_completed line correctly" do
28
+ line = '2010-05-17T17:37:35+0000: [Worker(delayed_job host:hostname.co.uk pid:11888)] S3FileJob completed after 1.0676'
29
+ @file_format.should parse_line(line).as(:job_completed).and_capture(:timestamp => 20100517173735,
30
+ :duration => 1.0676, :host => 'hostname.co.uk', :pid => 11888, :job => 'S3FileJob')
31
+ end
32
+ end
33
+
34
+ describe '#parse_io' do
35
+ before(:each) do
36
+ @log_parser = RequestLogAnalyzer::Source::LogParser.new(RequestLogAnalyzer::FileFormat.load(:delayed_job21))
37
+ end
38
+
39
+ it "should parse a batch of completed jobs without warnings" do
40
+ fragment = <<-EOLOG
41
+ 2010-05-17T17:36:44+0000: *** Starting job worker delayed_job host:hostname.co.uk pid:11888
42
+ 2010-05-17T17:37:34+0000: [Worker(delayed_job host:hostname.co.uk pid:11888)] acquired lock on S3FileJob
43
+ 2010-05-17T17:37:35+0000: [Worker(delayed_job host:hostname.co.uk pid:11888)] S3FileJob completed after 1.0676
44
+ 2010-05-17T17:37:35+0000: [Worker(delayed_job host:hostname.co.uk pid:11888)] acquired lock on S3FileJob
45
+ 2010-05-17T17:37:37+0000: [Worker(delayed_job host:hostname.co.uk pid:11888)] S3FileJob completed after 1.4407
46
+ 2010-05-17T17:37:37+0000: [Worker(delayed_job host:hostname.co.uk pid:11888)] acquired lock on S3FileJob
47
+ 2010-05-17T17:37:44+0000: [Worker(delayed_job host:hostname.co.uk pid:11888)] S3FileJob completed after 6.9374
48
+ 2010-05-17T17:37:44+0000: [Worker(delayed_job host:hostname.co.uk pid:11888)] 3 jobs processed at 0.3163 j/s, 0 failed ...
49
+ 2010-05-19T11:47:26+0000: Exiting...
50
+ EOLOG
51
+
52
+ request_counter.should_receive(:hit!).exactly(3).times
53
+ @log_parser.should_not_receive(:warn)
54
+
55
+ @log_parser.parse_io(StringIO.new(fragment)) do |request|
56
+ request_counter.hit! if request.kind_of?(RequestLogAnalyzer::FileFormat::DelayedJob21::Request)
57
+ end
58
+ end
59
+ end
60
+ end
@@ -0,0 +1,60 @@
1
+ require 'spec_helper'
2
+
3
+ describe RequestLogAnalyzer::FileFormat::DelayedJob do
4
+
5
+ it "should be a valid file format" do
6
+ RequestLogAnalyzer::FileFormat.load(:delayed_job).should be_valid
7
+ end
8
+
9
+ describe '#parse_line' do
10
+
11
+ before(:each) do
12
+ @file_format = RequestLogAnalyzer::FileFormat.load(:delayed_job2)
13
+ end
14
+
15
+ it "should parse a :job_lock line correctly" do
16
+ line = "2010-05-17T17:37:34+0000: * [Worker(delayed_job host:hostname.co.uk pid:11888)] acquired lock on S3FileJob"
17
+ @file_format.should parse_line(line).as(:job_lock).and_capture(:timestamp => 20100517173734,
18
+ :job => 'S3FileJob', :host => 'hostname.co.uk', :pid => 11888)
19
+ end
20
+
21
+ it "should parse a :job_lock line correctly when the worker is one of many" do
22
+ line = "2010-05-17T17:37:34+0000: * [Worker(delayed_job.0 host:hostname.co.uk pid:11888)] acquired lock on S3FileJob"
23
+ @file_format.should parse_line(line).as(:job_lock).and_capture(:timestamp => 20100517173734,
24
+ :job => 'S3FileJob', :host => 'hostname.co.uk', :pid => 11888)
25
+ end
26
+
27
+ it "should parse a :job_completed line correctly" do
28
+ line = '2010-05-17T17:37:35+0000: * [JOB] delayed_job host:hostname.co.uk pid:11888 completed after 1.0676'
29
+ @file_format.should parse_line(line).as(:job_completed).and_capture(:timestamp => 20100517173735,
30
+ :duration => 1.0676, :host => 'hostname.co.uk', :pid => 11888)
31
+ end
32
+ end
33
+
34
+ describe '#parse_io' do
35
+ before(:each) do
36
+ @log_parser = RequestLogAnalyzer::Source::LogParser.new(RequestLogAnalyzer::FileFormat.load(:delayed_job2))
37
+ end
38
+
39
+ it "should parse a batch of completed jobs without warnings" do
40
+ fragment = <<-EOLOG
41
+ 2010-05-17T17:36:44+0000: *** Starting job worker delayed_job host:hostname.co.uk pid:11888
42
+ 2010-05-17T17:37:34+0000: * [Worker(delayed_job host:hostname.co.uk pid:11888)] acquired lock on S3FileJob
43
+ 2010-05-17T17:37:35+0000: * [JOB] delayed_job host:hostname.co.uk pid:11888 completed after 1.0676
44
+ 2010-05-17T17:37:35+0000: * [Worker(delayed_job host:hostname.co.uk pid:11888)] acquired lock on S3FileJob
45
+ 2010-05-17T17:37:37+0000: * [JOB] delayed_job host:hostname.co.uk pid:11888 completed after 1.4407
46
+ 2010-05-17T17:37:37+0000: * [Worker(delayed_job host:hostname.co.uk pid:11888)] acquired lock on S3FileJob
47
+ 2010-05-17T17:37:44+0000: * [JOB] delayed_job host:hostname.co.uk pid:11888 completed after 6.9374
48
+ 2010-05-17T17:37:44+0000: 3 jobs processed at 0.3163 j/s, 0 failed ...
49
+ 2010-05-19T11:47:26+0000: Exiting...
50
+ EOLOG
51
+
52
+ request_counter.should_receive(:hit!).exactly(3).times
53
+ @log_parser.should_not_receive(:warn)
54
+
55
+ @log_parser.parse_io(StringIO.new(fragment)) do |request|
56
+ request_counter.hit! if request.kind_of?(RequestLogAnalyzer::FileFormat::DelayedJob2::Request)
57
+ end
58
+ end
59
+ end
60
+ end
@@ -1,4 +1,4 @@
1
- require File.dirname(__FILE__) + '/../../spec_helper'
1
+ require 'spec_helper'
2
2
 
3
3
  describe RequestLogAnalyzer::FileFormat::DelayedJob do
4
4
 
@@ -28,7 +28,7 @@ describe RequestLogAnalyzer::FileFormat::DelayedJob do
28
28
  :failed_job => 'BackgroundJob::ThumbnailSaver', :exception => 'ActiveRecord::RecordNotFound')
29
29
  end
30
30
 
31
- it "" do
31
+ it "should parse a failed job lock line correctly" do
32
32
  line = "* [JOB] failed to acquire exclusive lock for BackgroundJob::ThumbnailSaver"
33
33
  @file_format.should parse_line(line).as(:job_lock_failed).and_capture(:locked_job => 'BackgroundJob::ThumbnailSaver')
34
34
  end
@@ -1,4 +1,4 @@
1
- require File.dirname(__FILE__) + '/../../spec_helper'
1
+ require 'spec_helper'
2
2
 
3
3
  describe RequestLogAnalyzer::FileFormat do
4
4
 
@@ -60,7 +60,7 @@ describe RequestLogAnalyzer::FileFormat do
60
60
  end
61
61
 
62
62
  it "should load a provided format file" do
63
- format_filename = File.dirname(__FILE__) + '/../../lib/testing_format.rb'
63
+ format_filename = File.expand_path('../../lib/testing_format.rb', File.dirname(__FILE__))
64
64
  @file_format = RequestLogAnalyzer::FileFormat.load(format_filename)
65
65
  @file_format.should be_kind_of(TestingFormat)
66
66
  end
@@ -1,36 +1,36 @@
1
- require File.dirname(__FILE__) + '/../../spec_helper'
1
+ require 'spec_helper'
2
2
 
3
3
  describe RequestLogAnalyzer::FileFormat do
4
4
 
5
5
  describe '.autodetect' do
6
6
  it "should autodetect a Merb log" do
7
7
  file_format = RequestLogAnalyzer::FileFormat.autodetect(log_fixture(:merb))
8
- file_format.should be_kind_of(RequestLogAnalyzer::FileFormat::Merb)
8
+ file_format.should be_instance_of(RequestLogAnalyzer::FileFormat::Merb)
9
9
  end
10
10
 
11
11
  it "should autodetect a MySQL slow query log" do
12
12
  file_format = RequestLogAnalyzer::FileFormat.autodetect(log_fixture(:mysql_slow_query))
13
- file_format.should be_kind_of(RequestLogAnalyzer::FileFormat::Mysql)
13
+ file_format.should be_instance_of(RequestLogAnalyzer::FileFormat::Mysql)
14
14
  end
15
15
 
16
16
  it "should autodetect a Rails 1.x log" do
17
17
  file_format = RequestLogAnalyzer::FileFormat.autodetect(log_fixture(:rails_1x))
18
- file_format.should be_kind_of(RequestLogAnalyzer::FileFormat::Rails)
18
+ file_format.should be_instance_of(RequestLogAnalyzer::FileFormat::Rails)
19
19
  end
20
20
 
21
21
  it "should autodetect a Rails 2.x log" do
22
22
  file_format = RequestLogAnalyzer::FileFormat.autodetect(log_fixture(:rails_22))
23
- file_format.should be_kind_of(RequestLogAnalyzer::FileFormat::Rails)
23
+ file_format.should be_instance_of(RequestLogAnalyzer::FileFormat::RailsDevelopment)
24
24
  end
25
25
 
26
26
  it "should autodetect an Apache access log" do
27
27
  file_format = RequestLogAnalyzer::FileFormat.autodetect(log_fixture(:apache_common))
28
- file_format.should be_kind_of(RequestLogAnalyzer::FileFormat::Apache)
28
+ file_format.should be_instance_of(RequestLogAnalyzer::FileFormat::Apache)
29
29
  end
30
30
 
31
31
  it "should autodetect a Rack access log" do
32
32
  file_format = RequestLogAnalyzer::FileFormat.autodetect(log_fixture(:sinatra))
33
- file_format.should be_kind_of(RequestLogAnalyzer::FileFormat::Rack)
33
+ file_format.should be_instance_of(RequestLogAnalyzer::FileFormat::Rack)
34
34
  end
35
35
 
36
36
  it "should not find any file format with a bogus file" do
@@ -0,0 +1,84 @@
1
+ require 'spec_helper'
2
+
3
+ describe RequestLogAnalyzer::FileFormat::Haproxy do
4
+
5
+ before(:each) do
6
+ @file_format = RequestLogAnalyzer::FileFormat.load(:haproxy)
7
+ @log_parser = RequestLogAnalyzer::Source::LogParser.new(@file_format)
8
+ @sample1 = 'Feb 6 12:14:14 localhost haproxy[14389]: 10.0.1.2:33317 [06/Feb/2009:12:14:14.655] http-in static/srv1 10/0/30/69/109 200 2750 - - ---- 1/1/1/1/0 0/0 {1wt.eu} {} "GET /index.html HTTP/1.1"'
9
+ @sample2 = 'haproxy[18113]: 127.0.0.1:34549 [15/Oct/2003:15:19:06.103] px-http px-http/<NOSRV> -1/-1/-1/-1/+50001 408 +2750 - - cR-- 2/2/2/0/+2 0/0 ""'
10
+ end
11
+
12
+ it "should be a valid file format" do
13
+ @file_format.should be_valid
14
+ end
15
+
16
+ it "should parse access lines and capture all of its fields" do
17
+ @file_format.should have_line_definition(:haproxy).capturing(:client_ip, :accept_date, :frontend_name, :backend_name, :server_name, :tq, :tw, :tc, :tr, :tt, :status_code, :bytes_read, :captured_request_cookie, :captured_response_cookie, :termination_event_code, :terminated_session_state, :clientside_persistence_cookie, :serverside_persistence_cookie, :actconn, :feconn, :beconn, :srv_conn, :retries, :srv_queue, :backend_queue, :captured_request_headers, :captured_response_headers, :http_request)
18
+ end
19
+
20
+ it "should match the sample line" do
21
+ @file_format.parse_line(@sample1).should include(:line_definition, :captures)
22
+ end
23
+
24
+ it "should not match a nonsense line" do
25
+ @file_format.parse_line('dsadasdas dsaadsads dsaadsads').should be_nil
26
+ end
27
+
28
+ it "should parse and convert the sample fields correctly" do
29
+ @log_parser.parse_io(StringIO.new(@sample1)) do |request|
30
+ request[:client_ip].should == '10.0.1.2'
31
+ request[:accept_date].should == 20090206121414
32
+ request[:frontend_name].should == 'http-in'
33
+ request[:backend_name].should == 'static'
34
+ request[:server_name].should == 'srv1'
35
+ request[:tq].should == 0.010
36
+ request[:tw].should == 0.000
37
+ request[:tc].should == 0.030
38
+ request[:tr].should == 0.069
39
+ request[:tt].should == 0.109
40
+ request[:status_code].should == 200
41
+ request[:bytes_read].should == 2750
42
+ request[:captured_request_cookie].should == nil
43
+ request[:captured_response_cookie].should == nil
44
+ request[:termination_event_code].should == nil
45
+ request[:terminated_session_state].should == nil
46
+ request[:clientside_persistence_cookie].should == nil
47
+ request[:serverside_persistence_cookie].should == nil
48
+ request[:actconn].should == 1
49
+ request[:feconn].should == 1
50
+ request[:beconn].should == 1
51
+ request[:srv_conn].should == 1
52
+ request[:retries].should == 0
53
+ request[:srv_queue].should == 0
54
+ request[:backend_queue].should == 0
55
+ request[:captured_request_headers].should == '{1wt.eu}'
56
+ request[:captured_response_headers].should == nil
57
+ request[:http_request].should == 'GET /index.html HTTP/1.1'
58
+ end
59
+ end
60
+
61
+ it "should parse and convert edge case sample fields correctly" do
62
+ @log_parser.parse_io(StringIO.new(@sample2)) do |request|
63
+ request[:accept_date].should == 20031015151906
64
+ request[:server_name].should == '<NOSRV>'
65
+ request[:tq].should == nil
66
+ request[:tw].should == nil
67
+ request[:tc].should == nil
68
+ request[:tr].should == nil
69
+ request[:tt].should == 50.001
70
+ request[:bytes_read].should == 2750
71
+ request[:captured_request_cookie].should == nil
72
+ request[:captured_response_cookie].should == nil
73
+ request[:termination_event_code].should == 'c'
74
+ request[:terminated_session_state].should == 'R'
75
+ request[:clientside_persistence_cookie].should == nil
76
+ request[:serverside_persistence_cookie].should == nil
77
+ request[:retries].should == 2
78
+ request[:captured_request_headers].should == nil
79
+ request[:captured_response_headers].should == nil
80
+ request[:http_request].should == nil
81
+ end
82
+ end
83
+
84
+ end
@@ -1,4 +1,4 @@
1
- require File.dirname(__FILE__) + '/../../spec_helper'
1
+ require 'spec_helper'
2
2
 
3
3
  describe RequestLogAnalyzer::LineDefinition do
4
4
 
@@ -1,4 +1,4 @@
1
- require File.dirname(__FILE__) + '/../../spec_helper'
1
+ require 'spec_helper'
2
2
 
3
3
  describe RequestLogAnalyzer::FileFormat::Merb do
4
4
 
@@ -1,4 +1,4 @@
1
- require File.dirname(__FILE__) + '/../../spec_helper'
1
+ require 'spec_helper'
2
2
 
3
3
  describe RequestLogAnalyzer::FileFormat::Mysql do
4
4