logfile_interval 1.1.2 → 1.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: b10f4f6bcfa47f21b82b70bf541ef5403638c1ac
4
- data.tar.gz: a0d4c42cd3f531b96fcbfc3aa8c7a0b4b2b3de1b
3
+ metadata.gz: 4b50ea1942cafe43439265b1d781190b4a60af69
4
+ data.tar.gz: 35fe78e837a6f1377baf2c62b3b4df5d95b814f7
5
5
  SHA512:
6
- metadata.gz: 954c42936dee2d249e2fc71f25cf175455f0350e4d96951bdddcb0c766ed18a53415c43a279746779df8d6e6ea89c83dc6e433272599bb0d5792d6f9b3a3f89e
7
- data.tar.gz: 289e3c63c70eeb4d606bc5a5431d6ea0c69c277b6b31cb717bcdfd4ef28f5a2bce0d660ed3bd8de0d3257eed237d1baf7c8f3b64fd879f204553037ab95c4f46
6
+ metadata.gz: 37ae14fa68272ef51c2127f65046e8068ad1937b00f2d66cbce37377bbde9582deb4a6312130dd630991a404d832d34a497bb60dc0fc8fe2bf303870241bed06
7
+ data.tar.gz: d12a79f9e706e5e02fce96d2ccc8e8b301936245d5819f93376873f837bbe2554ed14c37729895b3f720b48306f71d07193ec65fd9eb4ec797391f6d7ccee736
data/Gemfile.lock CHANGED
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- logfile_interval (1.1.2)
4
+ logfile_interval (1.2.1)
5
5
 
6
6
  GEM
7
7
  remote: https://rubygems.org/
@@ -3,9 +3,10 @@ module LogfileInterval
3
3
  class Base
4
4
  include Enumerable
5
5
 
6
- def initialize
6
+ def initialize(options = {})
7
7
  @val = Util::Counter.new
8
8
  @size = Util::Counter.new
9
+ @options = options
9
10
  end
10
11
 
11
12
  def value(group = nil)
@@ -1,7 +1,5 @@
1
1
  lib_dir = File.expand_path('..', __FILE__)
2
2
 
3
- puts "lib_dir=#{lib_dir}"
4
-
5
3
  require "#{lib_dir}/aggregator/base"
6
4
  require "#{lib_dir}/aggregator/sum"
7
5
  require "#{lib_dir}/aggregator/count"
@@ -11,13 +9,18 @@ require "#{lib_dir}/aggregator/delta"
11
9
 
12
10
  module LogfileInterval
13
11
  module Aggregator
14
- def self.klass(aggregator)
15
- case aggregator
12
+ def self.klass(options)
13
+ case options[:aggregator]
16
14
  when :sum then Sum
17
15
  when :average then Average
18
- when :count then Count
19
- when :group_and_count then GroupAndCount
16
+ when :count
17
+ if options[:group_by] && options[:group_by] != options[:name]
18
+ GroupAndCount
19
+ else
20
+ Count
21
+ end
20
22
  when :delta then Delta
23
+ when :custom then options.fetch(:custom_class)
21
24
  end
22
25
  end
23
26
  end
@@ -16,12 +16,17 @@ module LogfileInterval
16
16
 
17
17
  @data = {}
18
18
  parser.columns.each do |name, options|
19
- next unless agg = options[:aggregator]
20
- @data[name] = agg.new
19
+ next unless agg = options[:aggregator_class]
20
+ if custom_options = options[:custom_options]
21
+ @data[name] = agg.new(custom_options)
22
+ else
23
+ @data[name] = agg.new
24
+ end
21
25
  end
22
26
  end
23
27
 
24
28
  def [](name)
29
+ raise ArgumentError, "#{name} field does not exist" unless @data.has_key?(name)
25
30
  @data[name.to_sym].values
26
31
  end
27
32
 
@@ -10,7 +10,7 @@ module LogfileInterval
10
10
 
11
11
  def each_interval
12
12
  return enum_for(:each_interval) unless block_given?
13
-
13
+
14
14
  secs = (Time.now.to_i / length.to_i) * length.to_i
15
15
  rounded_end_time = Time.at(secs)
16
16
  current_interval = Interval.new(rounded_end_time, length, parser)
@@ -1,8 +1,7 @@
1
1
  module LogfileInterval
2
2
  module LineParser
3
- AGGREGATION_FUNCTIONS = [ :sum, :average, :timestamp, :count, :delta ]
3
+ AGGREGATION_FUNCTIONS = [ :sum, :average, :timestamp, :count, :delta, :custom ]
4
4
 
5
- class InvalidLine < StandardError; end
6
5
  class ConfigurationError < StandardError; end
7
6
 
8
7
  class Base
@@ -20,25 +19,11 @@ module LogfileInterval
20
19
  end
21
20
 
22
21
  def add_column(options)
23
- name = options.fetch(:name)
24
- pos = options.fetch(:pos)
25
- conversion = options.fetch(:conversion, :string)
26
- group_by = options.fetch(:group_by, nil)
27
- aggregator = options.fetch(:aggregator)
28
- unless AGGREGATION_FUNCTIONS.include?(aggregator)
29
- raise ArgumentError, "aggregator must be one of #{AGGREGATION_FUNCTIONS.join(', ')}"
30
- end
31
-
32
- name = name.to_sym
33
- group_by = group_by.to_sym unless group_by.nil?
34
-
35
- if aggregator == :count && group_by && group_by != name
36
- aggregator = :group_and_count
37
- end
22
+ validate_column_options(options)
23
+ options = sanitize_column_options(options)
38
24
 
39
- aggregator = Aggregator.klass(aggregator)
40
- columns[name] = { :pos => pos, :aggregator => aggregator, :conversion => conversion }
41
- columns[name][:group_by] = group_by
25
+ name = options[:name]
26
+ columns[name] = options
42
27
 
43
28
  define_method(name) do
44
29
  @data[name]
@@ -51,7 +36,6 @@ module LogfileInterval
51
36
 
52
37
  match_data = regex.match(line)
53
38
  return nil unless match_data
54
- return nil unless match_data.size >= columns.size+1
55
39
 
56
40
  data = {}
57
41
  columns.each do |name, options|
@@ -67,9 +51,42 @@ module LogfileInterval
67
51
  return nil
68
52
  end
69
53
 
54
+ def set_column_custom_options(column_name, options)
55
+ raise ArgumentError, "Invalid column name: #{column_name}" unless columns.has_key?(column_name)
56
+ raise ArgumentError, "This column is not custom: #{column_name}" unless columns[column_name].has_key?(:custom_class)
57
+ columns[column_name][:custom_options] = options
58
+ end
59
+
70
60
  private
71
61
 
72
62
  def validate_column_options(options)
63
+ validate_option(options, :name)
64
+ validate_option(options, :pos)
65
+ validate_option(options, :aggregator)
66
+ unless AGGREGATION_FUNCTIONS.include?(options[:aggregator])
67
+ raise ConfigurationError, "aggregator must be one of #{AGGREGATION_FUNCTIONS.join(', ')}"
68
+ end
69
+ if options[:aggregator] == :custom
70
+ validate_option(options, :custom_class, ':custom_class must be set for :custom aggregator type')
71
+ end
72
+ end
73
+
74
+ def validate_option(options, key, errmsg = nil)
75
+ raise ConfigurationError, errmsg || "#{key} is a mandatory column option" unless options.has_key?(key)
76
+ end
77
+
78
+ def sanitize_column_options(options)
79
+ options[:name] = options[:name].to_sym
80
+ if options.has_key?(:group_by)
81
+ options[:group_by] = options[:group_by].to_sym
82
+ end
83
+ options[:conversion] = options.fetch(:conversion, :string)
84
+ if options[:aggregator] == :custom
85
+ options[:custom_options] = options.fetch(:custom_options, {})
86
+ end
87
+ options[:aggregator_class] = Aggregator.klass(options)
88
+ options.delete(:aggregator)
89
+ options
73
90
  end
74
91
 
75
92
  def convert(val, conversion)
@@ -1,3 +1,3 @@
1
1
  module LogfileInterval
2
- VERSION = "1.1.2"
2
+ VERSION = "1.2.1"
3
3
  end
@@ -1,14 +1,18 @@
1
1
  require 'spec_helper'
2
2
 
3
3
  module LogfileInterval
4
+
4
5
  module Aggregator
6
+ class CustomAggregator; end
7
+
5
8
  describe Aggregator do
6
9
  it 'finds the aggregator class' do
7
- Aggregator.klass(:sum).should == Sum
8
- Aggregator.klass(:average).should == Average
9
- Aggregator.klass(:count).should == Count
10
- Aggregator.klass(:group_and_count).should == GroupAndCount
11
- Aggregator.klass(:delta).should == Delta
10
+ Aggregator.klass({ :aggregator => :sum}).should == Sum
11
+ Aggregator.klass({ :aggregator => :average}).should == Average
12
+ Aggregator.klass({ :aggregator => :count}).should == Count
13
+ Aggregator.klass({ :aggregator => :count, :group_by => :foo}).should == GroupAndCount
14
+ Aggregator.klass({ :aggregator => :delta}).should == Delta
15
+ Aggregator.klass({ :aggregator => :custom, :custom_class => CustomAggregator}).should == CustomAggregator
12
16
  end
13
17
  end
14
18
 
@@ -0,0 +1,41 @@
1
+ require 'spec_helper'
2
+ require File.join(File.dirname(__FILE__), '..', 'support/lib/custom_timing_log')
3
+
4
+ module LogfileInterval
5
+ describe 'Custom Aggregator' do
6
+ before :each do
7
+ @end_time = Time.new(2013, 12, 01, 16, 00, 00, '-08:00')
8
+ @length = 300
9
+ @line_parser_class = LineParser::CustomTimingLog
10
+ end
11
+
12
+ def fill_interval
13
+ @interval = Interval.new(@end_time, @length, @line_parser_class)
14
+ record1 = @line_parser_class.create_record('1385942400, 192.168.0.5, posts#index, 150, 20000, 53.0')
15
+ @interval.add_record(record1)
16
+ record2 = @line_parser_class.create_record('1385942300, 192.168.0.5, posts#show, 50, 10000, 51.0')
17
+ @interval.add_record(record2)
18
+ record3 = @line_parser_class.create_record('1385942200, 10.10.10.10, posts#show, 70, 12000, 50.0')
19
+ @interval.add_record(record3)
20
+ end
21
+
22
+ it 'gets instantiated with empty data' do
23
+ fill_interval
24
+
25
+ @interval.size.should == 3
26
+ @interval[:num_slow].should == 1
27
+ @interval[:ip].should == 3
28
+ end
29
+
30
+ describe 'set_column_custom_options' do
31
+ it 'overwrites custom aggregator custom options' do
32
+ @line_parser_class.set_column_custom_options(:num_slow, :threshold => 60)
33
+ fill_interval
34
+
35
+ @interval.size.should == 3
36
+ @interval[:num_slow].should == 2
37
+ @interval[:ip].should == 3
38
+ end
39
+ end
40
+ end
41
+ end
@@ -50,6 +50,10 @@ module LogfileInterval
50
50
  set_regex /^([\d\.]+)\s+\S+\s+\S+\s+\[(\d\d.*\d\d)\]\s+"(?:GET|POST|PUT|HEAD|DELETE)\s+(\S+)\s+HTTP\S+"\s+(\d+)\s+(\d+)\s+"([^"]*)"\s+"([^"]+)"$/
51
51
  end
52
52
 
53
+ class MissingCustomClass < Base
54
+ set_regex /(.*)/
55
+ end
56
+
53
57
  before :each do
54
58
  @line = '74.75.19.145 - - [31/Mar/2013:06:54:12 -0700] "GET /ppa/google_chrome HTTP/1.1" 200 7855 "https://www.google.com/" "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.22 (KHTML, like Gecko) Ubuntu Chromium/25.0.1364.160 Chrome/25.0.1364.160 Safari/537.22"'
55
59
  end
@@ -61,6 +65,10 @@ module LogfileInterval
61
65
  it 'must fail unless a column is configured'do
62
66
  lambda { NoColumnLog.new(@line) }.should raise_error ConfigurationError
63
67
  end
68
+
69
+ it 'must fail with custom aggregator but no custom class' do
70
+ lambda { MissingCustomClass.add_column(:name => 'ip', :pos => 1, :aggregator => :custom) }.should raise_error ConfigurationError
71
+ end
64
72
  end
65
73
 
66
74
  describe TimingLog do
@@ -0,0 +1,32 @@
1
+ module LogfileInterval
2
+ module Aggregator
3
+ class CountOverThreshold < Base
4
+ def initialize(options)
5
+ super
6
+ @threshold = options.fetch(:threshold)
7
+ end
8
+
9
+ def add(value, group_by = nil)
10
+ @val.add(key(group_by), 1) if value > @threshold
11
+ end
12
+ end
13
+ end
14
+
15
+ module LineParser
16
+ class CustomTimingLog < Base
17
+ # Line format:
18
+ # timestamp, ip, controller#action, total_time, bytes, rss
19
+
20
+ set_regex /^(\d+),\s*([\d\.]+),\s*(\w+#\w+),\s*(\d+),\s*(\d+),\s*([\d\.]+)$/
21
+
22
+ add_column :name => :timestamp, :pos => 1, :aggregator => :timestamp
23
+ add_column :name => :ip, :pos => 2, :aggregator => :count
24
+ add_column :name => :num_slow, :pos => 4, :aggregator => :custom, :conversion => :integer,
25
+ :custom_class => Aggregator::CountOverThreshold, :custom_options => { :threshold => 100 }
26
+
27
+ def time
28
+ Time.at(self.timestamp.to_i)
29
+ end
30
+ end
31
+ end
32
+ end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logfile_interval
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.1.2
4
+ version: 1.2.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Philippe Le Rohellec
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2014-01-11 00:00:00.000000000 Z
11
+ date: 2014-01-13 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -118,6 +118,7 @@ files:
118
118
  - logfile_interval.gemspec
119
119
  - spec/lib/aggregator_spec.rb
120
120
  - spec/lib/counter_spec.rb
121
+ - spec/lib/custom_aggregator_spec.rb
121
122
  - spec/lib/interval_builder_spec.rb
122
123
  - spec/lib/interval_spec.rb
123
124
  - spec/lib/line_parser/base_spec.rb
@@ -125,6 +126,7 @@ files:
125
126
  - spec/lib/logfile_spec.rb
126
127
  - spec/spec_helper.rb
127
128
  - spec/support/lib/access_log.rb
129
+ - spec/support/lib/custom_timing_log.rb
128
130
  - spec/support/lib/timing_log.rb
129
131
  - spec/support/logfiles/access.log
130
132
  - spec/support/logfiles/access.log.1
@@ -159,6 +161,7 @@ summary: Aggregate logfile data into intervals
159
161
  test_files:
160
162
  - spec/lib/aggregator_spec.rb
161
163
  - spec/lib/counter_spec.rb
164
+ - spec/lib/custom_aggregator_spec.rb
162
165
  - spec/lib/interval_builder_spec.rb
163
166
  - spec/lib/interval_spec.rb
164
167
  - spec/lib/line_parser/base_spec.rb
@@ -166,6 +169,7 @@ test_files:
166
169
  - spec/lib/logfile_spec.rb
167
170
  - spec/spec_helper.rb
168
171
  - spec/support/lib/access_log.rb
172
+ - spec/support/lib/custom_timing_log.rb
169
173
  - spec/support/lib/timing_log.rb
170
174
  - spec/support/logfiles/access.log
171
175
  - spec/support/logfiles/access.log.1