logfile_interval 1.1.2 → 1.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile.lock +1 -1
- data/lib/logfile_interval/aggregator/base.rb +2 -1
- data/lib/logfile_interval/aggregator.rb +9 -6
- data/lib/logfile_interval/interval.rb +7 -2
- data/lib/logfile_interval/interval_builder.rb +1 -1
- data/lib/logfile_interval/line_parser/base.rb +38 -21
- data/lib/logfile_interval/version.rb +1 -1
- data/spec/lib/aggregator_spec.rb +9 -5
- data/spec/lib/custom_aggregator_spec.rb +41 -0
- data/spec/lib/line_parser/base_spec.rb +8 -0
- data/spec/support/lib/custom_timing_log.rb +32 -0
- metadata +6 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 4b50ea1942cafe43439265b1d781190b4a60af69
|
4
|
+
data.tar.gz: 35fe78e837a6f1377baf2c62b3b4df5d95b814f7
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 37ae14fa68272ef51c2127f65046e8068ad1937b00f2d66cbce37377bbde9582deb4a6312130dd630991a404d832d34a497bb60dc0fc8fe2bf303870241bed06
|
7
|
+
data.tar.gz: d12a79f9e706e5e02fce96d2ccc8e8b301936245d5819f93376873f837bbe2554ed14c37729895b3f720b48306f71d07193ec65fd9eb4ec797391f6d7ccee736
|
data/Gemfile.lock
CHANGED
@@ -1,7 +1,5 @@
|
|
1
1
|
lib_dir = File.expand_path('..', __FILE__)
|
2
2
|
|
3
|
-
puts "lib_dir=#{lib_dir}"
|
4
|
-
|
5
3
|
require "#{lib_dir}/aggregator/base"
|
6
4
|
require "#{lib_dir}/aggregator/sum"
|
7
5
|
require "#{lib_dir}/aggregator/count"
|
@@ -11,13 +9,18 @@ require "#{lib_dir}/aggregator/delta"
|
|
11
9
|
|
12
10
|
module LogfileInterval
|
13
11
|
module Aggregator
|
14
|
-
def self.klass(
|
15
|
-
case aggregator
|
12
|
+
def self.klass(options)
|
13
|
+
case options[:aggregator]
|
16
14
|
when :sum then Sum
|
17
15
|
when :average then Average
|
18
|
-
when :count
|
19
|
-
|
16
|
+
when :count
|
17
|
+
if options[:group_by] && options[:group_by] != options[:name]
|
18
|
+
GroupAndCount
|
19
|
+
else
|
20
|
+
Count
|
21
|
+
end
|
20
22
|
when :delta then Delta
|
23
|
+
when :custom then options.fetch(:custom_class)
|
21
24
|
end
|
22
25
|
end
|
23
26
|
end
|
@@ -16,12 +16,17 @@ module LogfileInterval
|
|
16
16
|
|
17
17
|
@data = {}
|
18
18
|
parser.columns.each do |name, options|
|
19
|
-
next unless agg = options[:
|
20
|
-
|
19
|
+
next unless agg = options[:aggregator_class]
|
20
|
+
if custom_options = options[:custom_options]
|
21
|
+
@data[name] = agg.new(custom_options)
|
22
|
+
else
|
23
|
+
@data[name] = agg.new
|
24
|
+
end
|
21
25
|
end
|
22
26
|
end
|
23
27
|
|
24
28
|
def [](name)
|
29
|
+
raise ArgumentError, "#{name} field does not exist" unless @data.has_key?(name)
|
25
30
|
@data[name.to_sym].values
|
26
31
|
end
|
27
32
|
|
@@ -10,7 +10,7 @@ module LogfileInterval
|
|
10
10
|
|
11
11
|
def each_interval
|
12
12
|
return enum_for(:each_interval) unless block_given?
|
13
|
-
|
13
|
+
|
14
14
|
secs = (Time.now.to_i / length.to_i) * length.to_i
|
15
15
|
rounded_end_time = Time.at(secs)
|
16
16
|
current_interval = Interval.new(rounded_end_time, length, parser)
|
@@ -1,8 +1,7 @@
|
|
1
1
|
module LogfileInterval
|
2
2
|
module LineParser
|
3
|
-
AGGREGATION_FUNCTIONS = [ :sum, :average, :timestamp, :count, :delta ]
|
3
|
+
AGGREGATION_FUNCTIONS = [ :sum, :average, :timestamp, :count, :delta, :custom ]
|
4
4
|
|
5
|
-
class InvalidLine < StandardError; end
|
6
5
|
class ConfigurationError < StandardError; end
|
7
6
|
|
8
7
|
class Base
|
@@ -20,25 +19,11 @@ module LogfileInterval
|
|
20
19
|
end
|
21
20
|
|
22
21
|
def add_column(options)
|
23
|
-
|
24
|
-
|
25
|
-
conversion = options.fetch(:conversion, :string)
|
26
|
-
group_by = options.fetch(:group_by, nil)
|
27
|
-
aggregator = options.fetch(:aggregator)
|
28
|
-
unless AGGREGATION_FUNCTIONS.include?(aggregator)
|
29
|
-
raise ArgumentError, "aggregator must be one of #{AGGREGATION_FUNCTIONS.join(', ')}"
|
30
|
-
end
|
31
|
-
|
32
|
-
name = name.to_sym
|
33
|
-
group_by = group_by.to_sym unless group_by.nil?
|
34
|
-
|
35
|
-
if aggregator == :count && group_by && group_by != name
|
36
|
-
aggregator = :group_and_count
|
37
|
-
end
|
22
|
+
validate_column_options(options)
|
23
|
+
options = sanitize_column_options(options)
|
38
24
|
|
39
|
-
|
40
|
-
columns[name] =
|
41
|
-
columns[name][:group_by] = group_by
|
25
|
+
name = options[:name]
|
26
|
+
columns[name] = options
|
42
27
|
|
43
28
|
define_method(name) do
|
44
29
|
@data[name]
|
@@ -51,7 +36,6 @@ module LogfileInterval
|
|
51
36
|
|
52
37
|
match_data = regex.match(line)
|
53
38
|
return nil unless match_data
|
54
|
-
return nil unless match_data.size >= columns.size+1
|
55
39
|
|
56
40
|
data = {}
|
57
41
|
columns.each do |name, options|
|
@@ -67,9 +51,42 @@ module LogfileInterval
|
|
67
51
|
return nil
|
68
52
|
end
|
69
53
|
|
54
|
+
def set_column_custom_options(column_name, options)
|
55
|
+
raise ArgumentError, "Invalid column name: #{column_name}" unless columns.has_key?(column_name)
|
56
|
+
raise ArgumentError, "This column is not custom: #{column_name}" unless columns[column_name].has_key?(:custom_class)
|
57
|
+
columns[column_name][:custom_options] = options
|
58
|
+
end
|
59
|
+
|
70
60
|
private
|
71
61
|
|
72
62
|
def validate_column_options(options)
|
63
|
+
validate_option(options, :name)
|
64
|
+
validate_option(options, :pos)
|
65
|
+
validate_option(options, :aggregator)
|
66
|
+
unless AGGREGATION_FUNCTIONS.include?(options[:aggregator])
|
67
|
+
raise ConfigurationError, "aggregator must be one of #{AGGREGATION_FUNCTIONS.join(', ')}"
|
68
|
+
end
|
69
|
+
if options[:aggregator] == :custom
|
70
|
+
validate_option(options, :custom_class, ':custom_class must be set for :custom aggregator type')
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
def validate_option(options, key, errmsg = nil)
|
75
|
+
raise ConfigurationError, errmsg || "#{key} is a mandatory column option" unless options.has_key?(key)
|
76
|
+
end
|
77
|
+
|
78
|
+
def sanitize_column_options(options)
|
79
|
+
options[:name] = options[:name].to_sym
|
80
|
+
if options.has_key?(:group_by)
|
81
|
+
options[:group_by] = options[:group_by].to_sym
|
82
|
+
end
|
83
|
+
options[:conversion] = options.fetch(:conversion, :string)
|
84
|
+
if options[:aggregator] == :custom
|
85
|
+
options[:custom_options] = options.fetch(:custom_options, {})
|
86
|
+
end
|
87
|
+
options[:aggregator_class] = Aggregator.klass(options)
|
88
|
+
options.delete(:aggregator)
|
89
|
+
options
|
73
90
|
end
|
74
91
|
|
75
92
|
def convert(val, conversion)
|
data/spec/lib/aggregator_spec.rb
CHANGED
@@ -1,14 +1,18 @@
|
|
1
1
|
require 'spec_helper'
|
2
2
|
|
3
3
|
module LogfileInterval
|
4
|
+
|
4
5
|
module Aggregator
|
6
|
+
class CustomAggregator; end
|
7
|
+
|
5
8
|
describe Aggregator do
|
6
9
|
it 'finds the aggregator class' do
|
7
|
-
Aggregator.klass(:sum).should == Sum
|
8
|
-
Aggregator.klass(:average).should == Average
|
9
|
-
Aggregator.klass(:count).should == Count
|
10
|
-
Aggregator.klass(:
|
11
|
-
Aggregator.klass(:delta).should == Delta
|
10
|
+
Aggregator.klass({ :aggregator => :sum}).should == Sum
|
11
|
+
Aggregator.klass({ :aggregator => :average}).should == Average
|
12
|
+
Aggregator.klass({ :aggregator => :count}).should == Count
|
13
|
+
Aggregator.klass({ :aggregator => :count, :group_by => :foo}).should == GroupAndCount
|
14
|
+
Aggregator.klass({ :aggregator => :delta}).should == Delta
|
15
|
+
Aggregator.klass({ :aggregator => :custom, :custom_class => CustomAggregator}).should == CustomAggregator
|
12
16
|
end
|
13
17
|
end
|
14
18
|
|
@@ -0,0 +1,41 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require File.join(File.dirname(__FILE__), '..', 'support/lib/custom_timing_log')
|
3
|
+
|
4
|
+
module LogfileInterval
|
5
|
+
describe 'Custom Aggregator' do
|
6
|
+
before :each do
|
7
|
+
@end_time = Time.new(2013, 12, 01, 16, 00, 00, '-08:00')
|
8
|
+
@length = 300
|
9
|
+
@line_parser_class = LineParser::CustomTimingLog
|
10
|
+
end
|
11
|
+
|
12
|
+
def fill_interval
|
13
|
+
@interval = Interval.new(@end_time, @length, @line_parser_class)
|
14
|
+
record1 = @line_parser_class.create_record('1385942400, 192.168.0.5, posts#index, 150, 20000, 53.0')
|
15
|
+
@interval.add_record(record1)
|
16
|
+
record2 = @line_parser_class.create_record('1385942300, 192.168.0.5, posts#show, 50, 10000, 51.0')
|
17
|
+
@interval.add_record(record2)
|
18
|
+
record3 = @line_parser_class.create_record('1385942200, 10.10.10.10, posts#show, 70, 12000, 50.0')
|
19
|
+
@interval.add_record(record3)
|
20
|
+
end
|
21
|
+
|
22
|
+
it 'gets instantiated with empty data' do
|
23
|
+
fill_interval
|
24
|
+
|
25
|
+
@interval.size.should == 3
|
26
|
+
@interval[:num_slow].should == 1
|
27
|
+
@interval[:ip].should == 3
|
28
|
+
end
|
29
|
+
|
30
|
+
describe 'set_column_custom_options' do
|
31
|
+
it 'overwrites custom aggregator custom options' do
|
32
|
+
@line_parser_class.set_column_custom_options(:num_slow, :threshold => 60)
|
33
|
+
fill_interval
|
34
|
+
|
35
|
+
@interval.size.should == 3
|
36
|
+
@interval[:num_slow].should == 2
|
37
|
+
@interval[:ip].should == 3
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
@@ -50,6 +50,10 @@ module LogfileInterval
|
|
50
50
|
set_regex /^([\d\.]+)\s+\S+\s+\S+\s+\[(\d\d.*\d\d)\]\s+"(?:GET|POST|PUT|HEAD|DELETE)\s+(\S+)\s+HTTP\S+"\s+(\d+)\s+(\d+)\s+"([^"]*)"\s+"([^"]+)"$/
|
51
51
|
end
|
52
52
|
|
53
|
+
class MissingCustomClass < Base
|
54
|
+
set_regex /(.*)/
|
55
|
+
end
|
56
|
+
|
53
57
|
before :each do
|
54
58
|
@line = '74.75.19.145 - - [31/Mar/2013:06:54:12 -0700] "GET /ppa/google_chrome HTTP/1.1" 200 7855 "https://www.google.com/" "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.22 (KHTML, like Gecko) Ubuntu Chromium/25.0.1364.160 Chrome/25.0.1364.160 Safari/537.22"'
|
55
59
|
end
|
@@ -61,6 +65,10 @@ module LogfileInterval
|
|
61
65
|
it 'must fail unless a column is configured'do
|
62
66
|
lambda { NoColumnLog.new(@line) }.should raise_error ConfigurationError
|
63
67
|
end
|
68
|
+
|
69
|
+
it 'must fail with custom aggregator but no custom class' do
|
70
|
+
lambda { MissingCustomClass.add_column(:name => 'ip', :pos => 1, :aggregator => :custom) }.should raise_error ConfigurationError
|
71
|
+
end
|
64
72
|
end
|
65
73
|
|
66
74
|
describe TimingLog do
|
@@ -0,0 +1,32 @@
|
|
1
|
+
module LogfileInterval
|
2
|
+
module Aggregator
|
3
|
+
class CountOverThreshold < Base
|
4
|
+
def initialize(options)
|
5
|
+
super
|
6
|
+
@threshold = options.fetch(:threshold)
|
7
|
+
end
|
8
|
+
|
9
|
+
def add(value, group_by = nil)
|
10
|
+
@val.add(key(group_by), 1) if value > @threshold
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
module LineParser
|
16
|
+
class CustomTimingLog < Base
|
17
|
+
# Line format:
|
18
|
+
# timestamp, ip, controller#action, total_time, bytes, rss
|
19
|
+
|
20
|
+
set_regex /^(\d+),\s*([\d\.]+),\s*(\w+#\w+),\s*(\d+),\s*(\d+),\s*([\d\.]+)$/
|
21
|
+
|
22
|
+
add_column :name => :timestamp, :pos => 1, :aggregator => :timestamp
|
23
|
+
add_column :name => :ip, :pos => 2, :aggregator => :count
|
24
|
+
add_column :name => :num_slow, :pos => 4, :aggregator => :custom, :conversion => :integer,
|
25
|
+
:custom_class => Aggregator::CountOverThreshold, :custom_options => { :threshold => 100 }
|
26
|
+
|
27
|
+
def time
|
28
|
+
Time.at(self.timestamp.to_i)
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logfile_interval
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.1
|
4
|
+
version: 1.2.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Philippe Le Rohellec
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-01-
|
11
|
+
date: 2014-01-13 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -118,6 +118,7 @@ files:
|
|
118
118
|
- logfile_interval.gemspec
|
119
119
|
- spec/lib/aggregator_spec.rb
|
120
120
|
- spec/lib/counter_spec.rb
|
121
|
+
- spec/lib/custom_aggregator_spec.rb
|
121
122
|
- spec/lib/interval_builder_spec.rb
|
122
123
|
- spec/lib/interval_spec.rb
|
123
124
|
- spec/lib/line_parser/base_spec.rb
|
@@ -125,6 +126,7 @@ files:
|
|
125
126
|
- spec/lib/logfile_spec.rb
|
126
127
|
- spec/spec_helper.rb
|
127
128
|
- spec/support/lib/access_log.rb
|
129
|
+
- spec/support/lib/custom_timing_log.rb
|
128
130
|
- spec/support/lib/timing_log.rb
|
129
131
|
- spec/support/logfiles/access.log
|
130
132
|
- spec/support/logfiles/access.log.1
|
@@ -159,6 +161,7 @@ summary: Aggregate logfile data into intervals
|
|
159
161
|
test_files:
|
160
162
|
- spec/lib/aggregator_spec.rb
|
161
163
|
- spec/lib/counter_spec.rb
|
164
|
+
- spec/lib/custom_aggregator_spec.rb
|
162
165
|
- spec/lib/interval_builder_spec.rb
|
163
166
|
- spec/lib/interval_spec.rb
|
164
167
|
- spec/lib/line_parser/base_spec.rb
|
@@ -166,6 +169,7 @@ test_files:
|
|
166
169
|
- spec/lib/logfile_spec.rb
|
167
170
|
- spec/spec_helper.rb
|
168
171
|
- spec/support/lib/access_log.rb
|
172
|
+
- spec/support/lib/custom_timing_log.rb
|
169
173
|
- spec/support/lib/timing_log.rb
|
170
174
|
- spec/support/logfiles/access.log
|
171
175
|
- spec/support/logfiles/access.log.1
|