logfile_interval 1.1.0 → 1.1.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile.lock +1 -1
- data/README.md +91 -15
- data/bin/aggregate_access_log.rb +43 -0
- data/bin/readme.rb +44 -0
- data/lib/logfile_interval.rb +11 -9
- data/lib/logfile_interval/interval.rb +5 -1
- data/lib/logfile_interval/line_parser/aggregator.rb +14 -5
- data/lib/logfile_interval/line_parser/base.rb +7 -3
- data/lib/logfile_interval/line_parser/counter.rb +0 -2
- data/lib/logfile_interval/logfile.rb +6 -1
- data/lib/logfile_interval/version.rb +1 -1
- data/spec/lib/interval_builder_spec.rb +13 -1
- data/spec/lib/line_parser/counter_spec.rb +70 -0
- data/spec/support/lib/access_log.rb +2 -2
- data/spec/support/logfiles/access.log.3 +7 -0
- metadata +11 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 37c52b3b1355cf1589c84a6f2f8e76233222eb80
|
4
|
+
data.tar.gz: 5ec64eb915a57de6c94974756d4a3d8a58042dc6
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 2b6eb7f3ecbfc8b12fa5900c02ca45e6f58ec1730018500c860c3be7e63ee0a83f402ee000e9141720ec13f6d89c7585dc6befbde89ec86929f51707e8fc47f1
|
7
|
+
data.tar.gz: 865b1ae02eb35830c8723f3b9ad5b013a3f0a335eaecc05cc95f077f468db34855af91510a00d77d4a0caf45183f708263daa78c99cb6da9f7b12c574e2bc5be
|
data/Gemfile.lock
CHANGED
data/README.md
CHANGED
@@ -2,27 +2,92 @@
|
|
2
2
|
|
3
3
|
Logfile parser and aggregator.
|
4
4
|
|
5
|
-
It iterates over
|
6
|
-
|
5
|
+
It iterates over 1 or more logfiles, parses each line and aggregates them into time intervals. Each interval object
|
6
|
+
includes aggregated data for each field of the logfile.
|
7
7
|
|
8
|
-
|
8
|
+
Aggregated data can be for example the sum, the average value or the number of occurences of each value.
|
9
9
|
|
10
|
-
|
10
|
+
## Example
|
11
|
+
This example will parse an access.log file and aggregate the data into 5 minute intervals.
|
11
12
|
|
12
|
-
|
13
|
+
In each interval, it counts
|
14
|
+
* the number of requests per IP address
|
15
|
+
* the number of requests for each HTTP status code
|
16
|
+
* the number of requests for each HTTP status code and IP address.
|
13
17
|
|
14
|
-
|
18
|
+
Full script is in [readme.rb](bin/readme.rb).
|
19
|
+
### Code
|
20
|
+
```ruby
|
21
|
+
require 'pp'
|
22
|
+
require 'date'
|
23
|
+
require 'logfile_interval'
|
15
24
|
|
16
|
-
|
25
|
+
class AccessLog < LogfileInterval::LineParser::Base
|
26
|
+
# Example line:
|
27
|
+
# 74.75.19.145 - - [31/Mar/2013:06:54:12 -0700] "GET /ppa/google_chrome HTTP/1.1" 200 7855 "https://www.google.com/" "Mozilla/5.0 Chrome/25.0.1364.160"
|
17
28
|
|
18
|
-
|
29
|
+
set_regex /^([\d\.]+)\s+.*\s+\[(\d\d.*\d\d)\]\s+"(?:GET|POST|PUT|HEAD|DELETE)\s+(\S+)\s+HTTP\S+"\s+(\d+)\s+/
|
19
30
|
|
20
|
-
|
31
|
+
add_column :name => 'ip', :pos => 1, :aggregator => :count, :group_by => 'ip'
|
32
|
+
add_column :name => 'timestamp', :pos => 2, :aggregator => :timestamp
|
33
|
+
add_column :name => 'code', :pos => 4, :aggregator => :count, :group_by => 'code'
|
34
|
+
add_column :name => 'code_by_ip', :pos => 4, :aggregator => :count, :group_by => 'ip'
|
21
35
|
|
22
|
-
|
36
|
+
def time
|
37
|
+
DateTime.strptime(self.timestamp, '%d/%b/%Y:%H:%M:%S %z').to_time
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
path = ENV['ACCESS_LOG_PATH']
|
42
|
+
file = LogfileInterval::Logfile.new(path, AccessLog)
|
43
|
+
unless file.exist?
|
44
|
+
puts "#{path} is not found"
|
45
|
+
exit 1
|
46
|
+
end
|
23
47
|
|
48
|
+
builder = LogfileInterval::IntervalBuilder.new(file, 300)
|
49
|
+
builder.each_interval do |interval|
|
50
|
+
next unless interval.size > 0
|
51
|
+
|
52
|
+
puts
|
53
|
+
puts "start time of interval: #{interval.start_time}"
|
54
|
+
puts "number of seconds in interval: #{interval.length}"
|
55
|
+
puts "number of requests found in interval: #{interval.size}"
|
56
|
+
puts "number of requests per ip address in interval:"
|
57
|
+
pp interval[:ip]
|
58
|
+
puts "number of requests per http code in interval:"
|
59
|
+
pp interval[:code]
|
60
|
+
puts "for each http code, number of requests grouped by ip:"
|
61
|
+
pp interval[:code_by_ip]
|
62
|
+
end
|
63
|
+
```
|
64
|
+
### Output
|
65
|
+
Logfile used for example: [access.log](spec/support/logfiles/access.log.3).
|
66
|
+
```
|
67
|
+
start time of interval: 2012-01-01 16:30:00 -0800
|
68
|
+
number of seconds in interval: 300
|
69
|
+
number of requests found in interval: 4
|
70
|
+
number of requests per ip address in interval:
|
71
|
+
{"78.54.172.146"=>3, "66.249.68.148"=>1}
|
72
|
+
number of requests per http code in interval:
|
73
|
+
{"200"=>3, "302"=>1}
|
74
|
+
for each ip, number of requests grouped by http code:
|
75
|
+
{"200"=>{"78.54.172.146"=>2, "66.249.68.148"=>1}, "302"=>{"78.54.172.146"=>1}}
|
76
|
+
|
77
|
+
start time of interval: 2012-01-01 16:25:00 -0800
|
78
|
+
number of seconds in interval: 300
|
79
|
+
number of requests found in interval: 3
|
80
|
+
number of requests per ip address in interval:
|
81
|
+
{"78.54.172.146"=>1, "173.192.238.51"=>1, "66.249.67.176"=>1}
|
82
|
+
number of requests per http code in interval:
|
83
|
+
{"200"=>1, "301"=>2}
|
84
|
+
for each ip, number of requests grouped by http code:
|
85
|
+
{"200"=>{"78.54.172.146"=>1}, "301"=>{"173.192.238.51"=>1, "66.249.67.176"=>1}}
|
86
|
+
```
|
87
|
+
|
88
|
+
## Usage
|
24
89
|
### Write a LineParser class
|
25
|
-
|
90
|
+
The first step is to define a LineParser class as in the example above. The parser lists the fields that must be parsed, how a timestamp can be extracted from each line and how to aggregate values into intervals.
|
26
91
|
```ruby
|
27
92
|
module LogfileInterval
|
28
93
|
module LineParser
|
@@ -46,8 +111,7 @@ module LogfileInterval
|
|
46
111
|
end
|
47
112
|
end
|
48
113
|
```
|
49
|
-
####
|
50
|
-
The parser must define:
|
114
|
+
#### The parser must define:
|
51
115
|
* A regex that extracts the fields out of each line.
|
52
116
|
* A set of columns that will to be parsed and aggregated in time intervals.
|
53
117
|
* A 'time' method that converts the mandatory timestamp field of a line into a Time object.
|
@@ -57,7 +121,7 @@ The parser must define:
|
|
57
121
|
* pos: the position of the captured field in the regex matched data
|
58
122
|
* aggregator : the aggregation mode for this field
|
59
123
|
* conversion: the parser will convert the field to an integer or a float when building the parsed record
|
60
|
-
* group_by: group_by value is the name of another field.
|
124
|
+
* group_by: group_by value is the name of another field. Values will be aggregated for each 'name', 'group_by' pair.
|
61
125
|
|
62
126
|
#### Aggregator types and options
|
63
127
|
* timestamp: the timestamp field will be used to determine to which interval the line belongs, each line MUST have a timestamp
|
@@ -121,8 +185,20 @@ interval_builder.each_interval do |interval|
|
|
121
185
|
end
|
122
186
|
```
|
123
187
|
|
124
|
-
##
|
188
|
+
## Installation
|
189
|
+
Add this line to your application's Gemfile:
|
190
|
+
|
191
|
+
gem 'logfile_interval'
|
125
192
|
|
193
|
+
And then execute:
|
194
|
+
|
195
|
+
$ bundle install
|
196
|
+
|
197
|
+
Or install it yourself as:
|
198
|
+
|
199
|
+
$ gem install logfile_interval
|
200
|
+
|
201
|
+
## Contributing
|
126
202
|
1. Fork it
|
127
203
|
2. Create your feature branch (`git checkout -b my-new-feature`)
|
128
204
|
3. Commit your changes (`git commit -am 'Add some feature'`)
|
@@ -0,0 +1,43 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'pp'
|
4
|
+
require 'date'
|
5
|
+
require File.join(File.expand_path('../../lib', __FILE__), 'logfile_interval')
|
6
|
+
|
7
|
+
logfile = ARGV[0]
|
8
|
+
unless File.exist?(String(logfile))
|
9
|
+
puts "#{logfile} does not exist."
|
10
|
+
exit 1
|
11
|
+
end
|
12
|
+
|
13
|
+
class AccessLogParser < LogfileInterval::LineParser::Base
|
14
|
+
# Example line:
|
15
|
+
# 74.75.19.145 - - [31/Mar/2013:06:54:12 -0700] "GET /ppa/google_chrome HTTP/1.1" 200 7855 "https://www.google.com/" "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.22 (KHTML, like Gecko) Ubuntu Chromium/25.0.1364.160 Chrome/25.0.1364.160 Safari/537.22"
|
16
|
+
|
17
|
+
set_regex /^([\d\.]+)\s+\S+\s+\S+\s+\[(\d\d.*\d\d)\]\s+"(?:GET|POST|PUT|HEAD|DELETE)\s+(\S+)\s+HTTP\S+"\s+(\d+)\s+(\d+)\s+"([^"]*)"\s+"([^"]+)"$/
|
18
|
+
|
19
|
+
add_column :name => 'ip', :pos => 1, :aggregator => :count, :group_by => 'ip'
|
20
|
+
add_column :name => 'timestamp', :pos => 2, :aggregator => :timestamp
|
21
|
+
add_column :name => 'code_by_ip', :pos => 4, :aggregator => :count, :group_by => 'ip'
|
22
|
+
add_column :name => 'length', :pos => 5, :aggregator => :average, :conversion => :integer
|
23
|
+
add_column :name => 'length_by_ip', :pos => 5, :aggregator => :average, :conversion => :integer, :group_by => 'ip'
|
24
|
+
add_column :name => 'referer', :pos => 6, :aggregator => :count, :group_by => :referer
|
25
|
+
add_column :name => 'referer_by_ip', :pos => 6, :aggregator => :count, :group_by => :ip
|
26
|
+
|
27
|
+
def time
|
28
|
+
DateTime.strptime(self.timestamp, '%d/%b/%Y:%H:%M:%S %z').to_time
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
file = LogfileInterval::Logfile.new(logfile, AccessLogParser)
|
33
|
+
builder = LogfileInterval::IntervalBuilder.new(file, 300)
|
34
|
+
builder.each_interval do |interval|
|
35
|
+
next unless interval.size > 0
|
36
|
+
|
37
|
+
puts
|
38
|
+
puts "start_time=#{interval.start_time} size=#{interval.size}"
|
39
|
+
pp interval[:ip]
|
40
|
+
pp interval[:referer_by_ip]
|
41
|
+
STDIN.gets
|
42
|
+
end
|
43
|
+
|
data/bin/readme.rb
ADDED
@@ -0,0 +1,44 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'pp'
|
4
|
+
require 'date'
|
5
|
+
require File.join(File.expand_path('../../lib', __FILE__), 'logfile_interval')
|
6
|
+
|
7
|
+
class AccessLog < LogfileInterval::LineParser::Base
|
8
|
+
# Example line:
|
9
|
+
# 74.75.19.145 - - [31/Mar/2013:06:54:12 -0700] "GET /ppa/google_chrome HTTP/1.1" 200 7855 "https://www.google.com/" "Mozilla/5.0 Chrome/25.0.1364.160"
|
10
|
+
|
11
|
+
set_regex /^([\d\.]+)\s+.*\s+\[(\d\d.*\d\d)\]\s+"(?:GET|POST|PUT|HEAD|DELETE)\s+(\S+)\s+HTTP\S+"\s+(\d+)\s+/
|
12
|
+
|
13
|
+
add_column :name => 'ip', :pos => 1, :aggregator => :count, :group_by => 'ip'
|
14
|
+
add_column :name => 'timestamp', :pos => 2, :aggregator => :timestamp
|
15
|
+
add_column :name => 'code', :pos => 4, :aggregator => :count, :group_by => 'code'
|
16
|
+
add_column :name => 'code_by_ip', :pos => 4, :aggregator => :count, :group_by => 'ip'
|
17
|
+
|
18
|
+
def time
|
19
|
+
DateTime.strptime(self.timestamp, '%d/%b/%Y:%H:%M:%S %z').to_time
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
path = ENV['ACCESS_LOG_PATH']
|
24
|
+
file = LogfileInterval::Logfile.new(path, AccessLog)
|
25
|
+
unless file.exist?
|
26
|
+
puts "#{path} is not found"
|
27
|
+
exit 1
|
28
|
+
end
|
29
|
+
|
30
|
+
builder = LogfileInterval::IntervalBuilder.new(file, 300)
|
31
|
+
builder.each_interval do |interval|
|
32
|
+
next unless interval.size > 0
|
33
|
+
|
34
|
+
puts
|
35
|
+
puts "start time of interval: #{interval.start_time}"
|
36
|
+
puts "number of seconds in interval: #{interval.length}"
|
37
|
+
puts "number of requests found in interval: #{interval.size}"
|
38
|
+
puts "number of requests per ip address in interval:"
|
39
|
+
pp interval[:ip]
|
40
|
+
puts "number of requests per http code in interval:"
|
41
|
+
pp interval[:code]
|
42
|
+
puts "for each ip, number of requests grouped by http code:"
|
43
|
+
pp interval[:code_by_ip]
|
44
|
+
end
|
data/lib/logfile_interval.rb
CHANGED
@@ -1,12 +1,14 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
require "logfile_interval/
|
4
|
-
require "logfile_interval/
|
5
|
-
require "logfile_interval/
|
6
|
-
require "logfile_interval/
|
7
|
-
require "logfile_interval/
|
8
|
-
require "logfile_interval/
|
9
|
-
require "logfile_interval/line_parser/
|
1
|
+
lib_dir = File.expand_path('..', __FILE__)
|
2
|
+
|
3
|
+
require "#{lib_dir}/logfile_interval/version"
|
4
|
+
require "#{lib_dir}/logfile_interval/file_backward"
|
5
|
+
require "#{lib_dir}/logfile_interval/interval"
|
6
|
+
require "#{lib_dir}/logfile_interval/interval_builder"
|
7
|
+
require "#{lib_dir}/logfile_interval/logfile"
|
8
|
+
require "#{lib_dir}/logfile_interval/logfile_set"
|
9
|
+
require "#{lib_dir}/logfile_interval/line_parser/base"
|
10
|
+
require "#{lib_dir}/logfile_interval/line_parser/aggregator"
|
11
|
+
require "#{lib_dir}/logfile_interval/line_parser/counter"
|
10
12
|
|
11
13
|
module LogfileInterval
|
12
14
|
end
|
@@ -20,7 +20,7 @@ module LogfileInterval
|
|
20
20
|
end
|
21
21
|
|
22
22
|
def value(group = nil)
|
23
|
-
|
23
|
+
val(key(group))
|
24
24
|
end
|
25
25
|
|
26
26
|
def values
|
@@ -47,10 +47,14 @@ module LogfileInterval
|
|
47
47
|
|
48
48
|
def each
|
49
49
|
@val.each_key do |k|
|
50
|
-
yield k,
|
50
|
+
yield k, val(k)
|
51
51
|
end
|
52
52
|
end
|
53
53
|
|
54
|
+
def val(k)
|
55
|
+
@val[k]
|
56
|
+
end
|
57
|
+
|
54
58
|
def average(k)
|
55
59
|
@size[k] > 0 ? @val[k].to_f / @size[k].to_f : 0
|
56
60
|
end
|
@@ -59,7 +63,6 @@ module LogfileInterval
|
|
59
63
|
class Sum < Base
|
60
64
|
def add(value, group_by = nil)
|
61
65
|
@val.add(key(group_by), value)
|
62
|
-
@size.set(key(group_by), 1)
|
63
66
|
end
|
64
67
|
end
|
65
68
|
|
@@ -68,12 +71,15 @@ module LogfileInterval
|
|
68
71
|
@val.add(key(group_by), value)
|
69
72
|
@size.increment(key(group_by))
|
70
73
|
end
|
74
|
+
|
75
|
+
def val(k)
|
76
|
+
average(k)
|
77
|
+
end
|
71
78
|
end
|
72
79
|
|
73
80
|
class Count < Base
|
74
81
|
def add(value, group_by = nil)
|
75
82
|
@val.add(key(group_by), 1)
|
76
|
-
@size.set(key(group_by), 1)
|
77
83
|
end
|
78
84
|
end
|
79
85
|
|
@@ -85,7 +91,6 @@ module LogfileInterval
|
|
85
91
|
def add(value, group_by)
|
86
92
|
raise ArgumentError, 'group_by argument is mandatory for GroupAndCount#add' unless group_by
|
87
93
|
@val.increment_subkey(value, key(group_by))
|
88
|
-
@size.set(key(group_by), 1)
|
89
94
|
end
|
90
95
|
end
|
91
96
|
|
@@ -102,6 +107,10 @@ module LogfileInterval
|
|
102
107
|
end
|
103
108
|
@previous.set(key(group_by), value)
|
104
109
|
end
|
110
|
+
|
111
|
+
def val(k)
|
112
|
+
average(k)
|
113
|
+
end
|
105
114
|
end
|
106
115
|
end
|
107
116
|
end
|
@@ -22,19 +22,23 @@ module LogfileInterval
|
|
22
22
|
def add_column(options)
|
23
23
|
name = options.fetch(:name)
|
24
24
|
pos = options.fetch(:pos)
|
25
|
-
aggregator = options.fetch(:aggregator)
|
26
25
|
conversion = options.fetch(:conversion, :string)
|
26
|
+
group_by = options.fetch(:group_by, nil)
|
27
|
+
aggregator = options.fetch(:aggregator)
|
27
28
|
unless AGGREGATION_FUNCTIONS.include?(aggregator)
|
28
29
|
raise ArgumentError, "aggregator must be one of #{AGGREGATION_FUNCTIONS.join(', ')}"
|
29
30
|
end
|
30
31
|
|
31
|
-
|
32
|
+
name = name.to_sym
|
33
|
+
group_by = group_by.to_sym unless group_by.nil?
|
34
|
+
|
35
|
+
if aggregator == :count && group_by && group_by != name
|
32
36
|
aggregator = :group_and_count
|
33
37
|
end
|
34
38
|
|
35
39
|
aggregator = Aggregator.klass(aggregator)
|
36
40
|
columns[name] = { :pos => pos, :aggregator => aggregator, :conversion => conversion }
|
37
|
-
columns[name][:group_by] =
|
41
|
+
columns[name][:group_by] = group_by
|
38
42
|
|
39
43
|
define_method(name) do
|
40
44
|
@data[name]
|
@@ -7,8 +7,12 @@ module LogfileInterval
|
|
7
7
|
@parser = parser
|
8
8
|
end
|
9
9
|
|
10
|
+
def exist?
|
11
|
+
filename && File.exist?(@filename)
|
12
|
+
end
|
13
|
+
|
10
14
|
def first_timestamp
|
11
|
-
return
|
15
|
+
return unless exist?
|
12
16
|
File.open(@filename) do |f|
|
13
17
|
while line = f.gets
|
14
18
|
if record = parser.create_record(line)
|
@@ -19,6 +23,7 @@ module LogfileInterval
|
|
19
23
|
end
|
20
24
|
|
21
25
|
def each_line
|
26
|
+
return unless exist?
|
22
27
|
f = FileBackward.new(@filename)
|
23
28
|
while(line = f.gets)
|
24
29
|
yield line.chomp
|
@@ -13,7 +13,7 @@ module LogfileInterval
|
|
13
13
|
|
14
14
|
context :each_interval do
|
15
15
|
before :each do
|
16
|
-
Time.stub(:now).and_return(Time.new(2013,12,01,16,0,
|
16
|
+
Time.stub(:now).and_return(Time.new(2013,12,01,16,0,1,'-08:00'))
|
17
17
|
@intervals = []
|
18
18
|
@builder.each_interval do |interval|
|
19
19
|
@intervals << interval
|
@@ -27,6 +27,7 @@ module LogfileInterval
|
|
27
27
|
context 'first interval' do
|
28
28
|
it 'got records from both logfiles' do
|
29
29
|
@intervals.first.size.should == 4
|
30
|
+
@intervals.first.end_time.should == Time.new(2013,12,01,16,0,0,'-08:00')
|
30
31
|
@intervals.first[:total_time].should == 700.0/4
|
31
32
|
@intervals.first[:num_bytes].should == 52000
|
32
33
|
@intervals.first[:rss].round(5).should == 0.60
|
@@ -38,6 +39,7 @@ module LogfileInterval
|
|
38
39
|
context 'second interval' do
|
39
40
|
it 'got records from second logfile only' do
|
40
41
|
@intervals.last.size.should == 2
|
42
|
+
@intervals.last.end_time.should == Time.new(2013,12,01,15,55,0,'-08:00')
|
41
43
|
@intervals.last[:total_time].should == 300
|
42
44
|
@intervals.last[:num_bytes].should == 41000
|
43
45
|
@intervals.last[:rss].round(5).should == 0.20
|
@@ -46,5 +48,15 @@ module LogfileInterval
|
|
46
48
|
end
|
47
49
|
end
|
48
50
|
end
|
51
|
+
|
52
|
+
context :last_interval do
|
53
|
+
it 'returns the most recent interval' do
|
54
|
+
Time.stub(:now).and_return(Time.new(2013,12,01,16,0,1,'-08:00'))
|
55
|
+
interval = @builder.last_interval
|
56
|
+
interval.end_time.should == Time.new(2013,12,01,16,0,0,'-08:00')
|
57
|
+
interval.size.should == 4
|
58
|
+
interval[:num_bytes].should == 52000
|
59
|
+
end
|
60
|
+
end
|
49
61
|
end
|
50
62
|
end
|
@@ -0,0 +1,70 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
module LogfileInterval
|
4
|
+
module LineParser
|
5
|
+
describe Counter do
|
6
|
+
it 'behaves like a hash' do
|
7
|
+
c = Counter.new
|
8
|
+
c[:a] = 1
|
9
|
+
c.keys.should == [:a]
|
10
|
+
c.values.should == [1]
|
11
|
+
c.delete(:a)
|
12
|
+
c.keys.should be_empty
|
13
|
+
|
14
|
+
c[:a] = 1
|
15
|
+
c[:b] = 2
|
16
|
+
c.keys.sort.should == [:a, :b]
|
17
|
+
end
|
18
|
+
|
19
|
+
it 'returns 0 when key does not exist' do
|
20
|
+
c = Counter.new
|
21
|
+
c[:a].should == 0
|
22
|
+
end
|
23
|
+
|
24
|
+
describe :increment do
|
25
|
+
it 'adds 1 to value for key argument' do
|
26
|
+
c = Counter.new
|
27
|
+
c[:a] = 1
|
28
|
+
c.increment(:a)
|
29
|
+
c[:a].should == 2
|
30
|
+
end
|
31
|
+
|
32
|
+
it 'creates new key when missing' do
|
33
|
+
c = Counter.new
|
34
|
+
c.increment(:a)
|
35
|
+
c[:a].should == 1
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
describe :add do
|
40
|
+
it 'adds number to hash value' do
|
41
|
+
c = Counter.new
|
42
|
+
c[:a] = 1
|
43
|
+
c.add(:a, 5)
|
44
|
+
c[:a].should == 6
|
45
|
+
end
|
46
|
+
|
47
|
+
it 'creates a new key when missing' do
|
48
|
+
c = Counter.new
|
49
|
+
c.add(:a, 4)
|
50
|
+
c[:a].should == 4
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
describe :increment_subkey do
|
55
|
+
it 'saves values of type Counter' do
|
56
|
+
c = Counter.new
|
57
|
+
c.increment_subkey(:a, :sub)
|
58
|
+
c[:a].should be_a(Counter)
|
59
|
+
c[:a][:sub].should == 1
|
60
|
+
end
|
61
|
+
|
62
|
+
it 'raises an error when trying to increment an existing integer key' do
|
63
|
+
c = Counter.new
|
64
|
+
c[:a] = 1
|
65
|
+
lambda { c.increment_subkey(:a, :sub) }.should raise_error(RuntimeError)
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|
@@ -9,9 +9,9 @@ module LogfileInterval
|
|
9
9
|
|
10
10
|
set_regex /^([\d\.]+)\s+\S+\s+\S+\s+\[(\d\d.*\d\d)\]\s+"(?:GET|POST|PUT|HEAD|DELETE)\s+(\S+)\s+HTTP\S+"\s+(\d+)\s+(\d+)\s+"([^"]*)"\s+"([^"]+)"$/
|
11
11
|
|
12
|
-
add_column :name => 'ip', :pos => 1, :aggregator => :count, :group_by => '
|
12
|
+
add_column :name => 'ip', :pos => 1, :aggregator => :count, :group_by => 'ip'
|
13
13
|
add_column :name => 'timestamp', :pos => 2, :aggregator => :timestamp
|
14
|
-
add_column :name => 'code', :pos => 4, :aggregator => :count, :group_by => '
|
14
|
+
add_column :name => 'code', :pos => 4, :aggregator => :count, :group_by => 'ip'
|
15
15
|
add_column :name => 'length', :pos => 5, :aggregator => :average, :conversion => :integer
|
16
16
|
add_column :name => 'length_by_ip', :pos => 5, :aggregator => :average, :conversion => :integer, :group_by => 'ip'
|
17
17
|
|
@@ -0,0 +1,7 @@
|
|
1
|
+
66.249.67.176 - - [01/Jan/2012:16:25:47 -0800] "GET /packages/show/1 HTTP/1.1" 301 185 "-" "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"
|
2
|
+
173.192.238.51 - - [01/Jan/2012:16:26:31 -0800] "GET / HTTP/1.1" 301 185 "-" "Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.0.19; aggregator:Spinn3r (Spinn3r 3.1);"
|
3
|
+
78.54.172.146 - - [01/Jan/2012:16:26:51 -0800] "GET /package/core/oneiric/main/base/abrowser HTTP/1.1" 200 6801 "http://www.google.com/url?sa=t&rct=j&q=abrowser" "Mozilla/5.0 (Ubuntu; X11; Linux x86_64; rv:9.0.1) Gecko/20100101 Firefox/9.0.1"
|
4
|
+
78.54.172.146 - - [01/Jan/2012:16:30:51 -0800] "GET /package/core/oneiric/main/base/firefox HTTP/1.1" 200 6801 "http://www.google.com/url?sa=t&rct=j&q=firefox" "Mozilla/5.0 (Ubuntu; X11; Linux x86_64; rv:9.0.1) Gecko/20100101 Firefox/9.0.1"
|
5
|
+
66.249.68.148 - - [01/Jan/2012:16:30:51 -0800] "GET /package/core/hardy/main/updates/libldap2-dev HTTP/1.1" 200 7058 "-" "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"
|
6
|
+
78.54.172.146 - - [01/Jan/2012:16:30:51 -0800] "GET /package/show/2 HTTP/1.1" 302 6801 "http://www.google.com/url?sa=t&rct=j&q=abrowser" "Mozilla/5.0 (Ubuntu; X11; Linux x86_64; rv:9.0.1) Gecko/20100101 Firefox/9.0.1"
|
7
|
+
78.54.172.146 - - [01/Jan/2012:16:31:51 -0800] "GET /package/core/oneiric/main/base/abrowser-6.0 HTTP/1.1" 200 6801 "http://www.google.com/url?sa=t&rct=j&q=abrowser" "Mozilla/5.0 (Ubuntu; X11; Linux x86_64; rv:9.0.1) Gecko/20100101 Firefox/9.0.1"
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logfile_interval
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.1.
|
4
|
+
version: 1.1.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Philippe Le Rohellec
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2014-01-03 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -83,7 +83,9 @@ dependencies:
|
|
83
83
|
description: Logfile parser and aggregator
|
84
84
|
email:
|
85
85
|
- philippe@lerohellec.com
|
86
|
-
executables:
|
86
|
+
executables:
|
87
|
+
- aggregate_access_log.rb
|
88
|
+
- readme.rb
|
87
89
|
extensions: []
|
88
90
|
extra_rdoc_files: []
|
89
91
|
files:
|
@@ -94,6 +96,8 @@ files:
|
|
94
96
|
- LICENSE.txt
|
95
97
|
- README.md
|
96
98
|
- Rakefile
|
99
|
+
- bin/aggregate_access_log.rb
|
100
|
+
- bin/readme.rb
|
97
101
|
- docs/design.rb
|
98
102
|
- docs/design2.rb
|
99
103
|
- docs/design3.rb
|
@@ -113,6 +117,7 @@ files:
|
|
113
117
|
- spec/lib/interval_spec.rb
|
114
118
|
- spec/lib/line_parser/aggregator_spec.rb
|
115
119
|
- spec/lib/line_parser/base_spec.rb
|
120
|
+
- spec/lib/line_parser/counter_spec.rb
|
116
121
|
- spec/lib/logfile_set_spec.rb
|
117
122
|
- spec/lib/logfile_spec.rb
|
118
123
|
- spec/spec_helper.rb
|
@@ -121,6 +126,7 @@ files:
|
|
121
126
|
- spec/support/logfiles/access.log
|
122
127
|
- spec/support/logfiles/access.log.1
|
123
128
|
- spec/support/logfiles/access.log.2
|
129
|
+
- spec/support/logfiles/access.log.3
|
124
130
|
- spec/support/logfiles/timing.log
|
125
131
|
- spec/support/logfiles/timing.log.1
|
126
132
|
homepage: https://github.com/plerohellec/logfile_interval
|
@@ -152,6 +158,7 @@ test_files:
|
|
152
158
|
- spec/lib/interval_spec.rb
|
153
159
|
- spec/lib/line_parser/aggregator_spec.rb
|
154
160
|
- spec/lib/line_parser/base_spec.rb
|
161
|
+
- spec/lib/line_parser/counter_spec.rb
|
155
162
|
- spec/lib/logfile_set_spec.rb
|
156
163
|
- spec/lib/logfile_spec.rb
|
157
164
|
- spec/spec_helper.rb
|
@@ -160,5 +167,6 @@ test_files:
|
|
160
167
|
- spec/support/logfiles/access.log
|
161
168
|
- spec/support/logfiles/access.log.1
|
162
169
|
- spec/support/logfiles/access.log.2
|
170
|
+
- spec/support/logfiles/access.log.3
|
163
171
|
- spec/support/logfiles/timing.log
|
164
172
|
- spec/support/logfiles/timing.log.1
|