logfile_interval 1.1.1 → 1.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/docs/design3.rb DELETED
@@ -1,177 +0,0 @@
1
- module LogfileInterval
2
- module LineParser
3
- class Base
4
- class << self
5
- def set_regex(regex)
6
- end
7
-
8
- def add_column(name, options)
9
- agg = Aggregators.klass(aggregator)
10
- @columns[name] = { :pos => pos, :aggregator => agg, :conversion => conversion }
11
- define_method(name)
12
- end
13
-
14
- def parse(line)
15
- match_data = regex.match(line)
16
- @data = f(match_data)
17
- end
18
-
19
- def create_record(line)
20
- record = new(line)
21
- return record.valid? ? record : nil
22
- end
23
- end
24
-
25
- end
26
-
27
- class AccessLog < Base
28
- set_regex /blah/
29
- add_column :name => :foo, :pos => 1, :conversion => integer, :aggregator => :average
30
-
31
- def initialize(line)
32
- @data = self.class.parse(line)
33
- end
34
- end
35
-
36
- module Aggregator
37
- def self.klass(aggregator)
38
- case aggregator
39
- when :sum then Sum
40
- end
41
- end
42
-
43
- class Sum
44
- def initialize
45
- @val = 0
46
- end
47
-
48
- def add(value)
49
- @val += value
50
- end
51
-
52
- def value
53
- @val
54
- end
55
- end
56
-
57
- class Count
58
- def initialize
59
- @val = Counter.new
60
- end
61
-
62
- def add(value)
63
- @val.increment(value)
64
- end
65
- end
66
- end
67
- end
68
-
69
- class Logfile
70
- def initialize(filename, parser)
71
- end
72
-
73
- def each_line
74
- end
75
-
76
- def each_parsed_line
77
- each_line do |line|
78
- record = parser.create_record(line)
79
- yield record if record
80
- end
81
- end
82
- end
83
-
84
- class LogfileSet
85
- def initialize(filenames_array, parser)
86
- end
87
-
88
- def ordered_filenams
89
- end
90
-
91
- def each_line
92
- end
93
-
94
- def each_parsed_line
95
- end
96
- end
97
-
98
- class IntervalBuilder
99
- def initialize(logfile_set, length)
100
- parser = logfile_set.parser
101
- end
102
-
103
- def each_interval
104
- interval = Interval.new(now, length)
105
- set.each_parsed_line(parser) do |record|
106
- while record.time < interval.start_time do
107
- yield interval
108
- interval = Interval.new(interval.start_time, length)
109
- end
110
- interval.add(record)
111
- end
112
- end
113
- end
114
-
115
- class Counter < Hash
116
- def increment(key)
117
- self[key] = self[key] ? self[key] + 1 : 1
118
- end
119
- end
120
-
121
- class Interval
122
- def initialize(end_time, length, parser)
123
- @data = {}
124
- parser.columns.each do |name, options|
125
- @data[name] = options[:aggregator].new
126
- end
127
- end
128
-
129
- def [](name)
130
- @data[name].value
131
- end
132
-
133
- def add_record(record)
134
- return unless record.valid?
135
- raise ParserMismatch unless record.class == parser
136
-
137
- @size += 1
138
- parser.columns.each do |name, options|
139
- @data[name].add(record[name])
140
- end
141
- end
142
- end
143
- end
144
-
145
- logfiles = [ 'access.log', 'access.log.1', 'access.log.2' ]
146
- logfile = logfiles.first
147
-
148
- parser = LineParser::AccessLog
149
-
150
- logfile_iterator = LogfileInterval::Logfile.new(logfile, parser)
151
- logfile_iterator.each_line do |line|
152
- puts line.class # String
153
- puts line
154
- end
155
-
156
- parser = LineParser::AccessLog
157
- logfile_iterator.each_parsed_line do |record|
158
- puts record.class # LineParser::AccessLog
159
- puts record.ip
160
- puts record.time
161
- end
162
-
163
- set_iterator = LogfileInterval::LogfileSet.new(logfiles, parser)
164
- set_iterator.each_parsed_line do |record|
165
- puts record.class # LineParser::AccessLog
166
- end
167
-
168
- length = 5.minutes
169
- interval_builder = LogfileInterval::IntervalBuilder.new(logfiles, length)
170
- interval_builder.each_interval do |interval|
171
- puts interval.class # LogfileInterval::Interval
172
- puts interval.start_time
173
- puts interval.length
174
- interval[:ip].each do |ip, count|
175
- puts "#{ip}, #{count}"
176
- end
177
- end
@@ -1,49 +0,0 @@
1
- module LogfileInterval
2
- # Based on Perl's File::ReadBackwards module, by Uri Guttman.
3
- class FileBackward
4
- MAX_READ_SIZE = 1 << 10 # 1024
5
-
6
- def initialize( *args )
7
- return unless File.exist?(args[0])
8
- @file = File.new(*args)
9
- @file.seek(0, IO::SEEK_END)
10
-
11
- @current_pos = @file.pos
12
-
13
- @read_size = @file.pos % MAX_READ_SIZE
14
- @read_size = MAX_READ_SIZE if @read_size.zero?
15
-
16
- @line_buffer = Array.new
17
- end
18
-
19
- def gets( sep_string = $/ )
20
- return nil unless @file
21
- return @line_buffer.pop if @line_buffer.size > 2 or @current_pos.zero?
22
-
23
- @current_pos -= @read_size
24
- @file.seek(@current_pos, IO::SEEK_SET)
25
-
26
- @line_buffer[0] = "#{@file.read(@read_size)}#{@line_buffer[0]}"
27
- @read_size = MAX_READ_SIZE # Set a size for the next read.
28
-
29
- @line_buffer[0] =
30
- @line_buffer[0].scan(/.*?#{Regexp.escape(sep_string)}|.+/)
31
- @line_buffer.flatten!
32
-
33
- gets(sep_string)
34
- end
35
-
36
- def close
37
- return unless @file
38
- @file.close()
39
- end
40
- end
41
- end
42
-
43
- # f = FileBackward.new('../log/development.log')
44
- # i = 0
45
- # while(line = f.gets())
46
- # puts line
47
- # i += 1
48
- # break if i>30
49
- # end
@@ -1,47 +0,0 @@
1
- module LogfileInterval
2
- class IntervalLength
3
- MAX_PERIODS = { 5 * 60 => 6 * 3600,
4
- 3600 => 3600 * 24,
5
- 3600 * 24 => 3600 * 24 * 30,
6
- 3600 * 24 * 30 => 365 * 3600 * 24 }
7
- LENGTHS = MAX_PERIODS.keys.sort
8
-
9
- attr_reader :length
10
-
11
- def initialize(l)
12
- raise ArgumentError unless LENGTHS.include?(l)
13
- @length = l
14
- end
15
-
16
- def lower
17
- pos = LENGTHS.index(@length)
18
- return nil if pos==0
19
- IntervalLength.new(LENGTHS[pos-1])
20
- end
21
-
22
- def higher
23
- pos = LENGTHS.index(@length)
24
- return nil if pos==LENGTHS.size-1
25
- IntervalLength.new(LENGTHS[pos+1])
26
- end
27
-
28
- def smallest?
29
- pos = LENGTHS.index(@length)
30
- pos == 0
31
- end
32
-
33
- def start_time(t)
34
- ts = (t.to_i / @length.to_i) * @length.to_i
35
- ts -= @length.to_i if t.to_i % @length.to_i == 0
36
- Time.at(ts)
37
- end
38
-
39
- def end_time(t)
40
- start_time(t) + @length
41
- end
42
-
43
- def to_i
44
- @length.to_i
45
- end
46
- end
47
- end
@@ -1,117 +0,0 @@
1
- module LogfileInterval
2
- module LineParser
3
- module Aggregator
4
- def self.klass(aggregator)
5
- case aggregator
6
- when :sum then Sum
7
- when :average then Average
8
- when :count then Count
9
- when :group_and_count then GroupAndCount
10
- when :delta then Delta
11
- end
12
- end
13
-
14
- class Base
15
- include Enumerable
16
-
17
- def initialize
18
- @val = Counter.new
19
- @size = Counter.new
20
- end
21
-
22
- def value(group = nil)
23
- val(key(group))
24
- end
25
-
26
- def values
27
- if single_value?
28
- value
29
- else
30
- self.inject({}) { |h, v| h[v[0]] = v[1]; h }
31
- end
32
- end
33
-
34
- def add(value, group_by = nil)
35
- raise NotImplementedError
36
- end
37
-
38
- private
39
- def key(group_by = nil)
40
- group_by ? group_by : :all
41
- end
42
-
43
- def single_value?
44
- return true if @val.empty?
45
- @val.keys.count == 1 && @val.keys.first == :all
46
- end
47
-
48
- def each
49
- @val.each_key do |k|
50
- yield k, val(k)
51
- end
52
- end
53
-
54
- def val(k)
55
- @val[k]
56
- end
57
-
58
- def average(k)
59
- @size[k] > 0 ? @val[k].to_f / @size[k].to_f : 0
60
- end
61
- end
62
-
63
- class Sum < Base
64
- def add(value, group_by = nil)
65
- @val.add(key(group_by), value)
66
- end
67
- end
68
-
69
- class Average < Base
70
- def add(value, group_by = nil)
71
- @val.add(key(group_by), value)
72
- @size.increment(key(group_by))
73
- end
74
-
75
- def val(k)
76
- average(k)
77
- end
78
- end
79
-
80
- class Count < Base
81
- def add(value, group_by = nil)
82
- @val.add(key(group_by), 1)
83
- end
84
- end
85
-
86
- class GroupAndCount < Base
87
- def each
88
- @val.each { |k, v| yield k, v }
89
- end
90
-
91
- def add(value, group_by)
92
- raise ArgumentError, 'group_by argument is mandatory for GroupAndCount#add' unless group_by
93
- @val.increment_subkey(value, key(group_by))
94
- end
95
- end
96
-
97
- class Delta < Base
98
- def initialize
99
- @previous = Counter.new
100
- super
101
- end
102
-
103
- def add(value, group_by = nil)
104
- if @previous.has_key?(key(group_by))
105
- @val.add(key(group_by), @previous[key(group_by)] - value)
106
- @size.increment(key(group_by))
107
- end
108
- @previous.set(key(group_by), value)
109
- end
110
-
111
- def val(k)
112
- average(k)
113
- end
114
- end
115
- end
116
- end
117
- end
@@ -1,211 +0,0 @@
1
- require 'spec_helper'
2
-
3
- module LogfileInterval
4
- module LineParser
5
- module Aggregator
6
- describe Aggregator do
7
- it 'finds the aggregator class' do
8
- Aggregator.klass(:sum).should == Sum
9
- Aggregator.klass(:average).should == Average
10
- Aggregator.klass(:count).should == Count
11
- Aggregator.klass(:group_and_count).should == GroupAndCount
12
- Aggregator.klass(:delta).should == Delta
13
- end
14
- end
15
-
16
- shared_examples 'an aggregator' do
17
- let(:aggregator) { described_class.new }
18
-
19
- [ :add, :value, :values ].each do |method|
20
- it "responds to #{method}" do
21
- aggregator.should respond_to(method)
22
- end
23
- end
24
-
25
- context 'values' do
26
- context 'with one group' do
27
- before :each do
28
- aggregator.add(5, :key1)
29
- end
30
-
31
- it 'returns a hash' do
32
- aggregator.values.should be_a(Hash) unless aggregator.is_a?(Delta)
33
- end
34
- end
35
-
36
- context 'with several groups' do
37
- before :each do
38
- aggregator.add(5, :key1)
39
- aggregator.add(3, :key2)
40
- aggregator.add(3, :key1)
41
- end
42
-
43
- it 'returns a hash' do
44
- aggregator.values.should be_a(Hash)
45
- end
46
- end
47
-
48
- context 'with no group' do
49
- before :each do
50
- aggregator.add(5)
51
- aggregator.add(3)
52
- end
53
-
54
- it 'returns a numeric' do
55
- aggregator.values.should be_a(Numeric) unless aggregator.is_a?(Count)
56
- end
57
- end
58
- end
59
- end
60
-
61
- [ Count, Sum, Average, Delta ]. each do |klass|
62
- describe klass do
63
- it_behaves_like 'an aggregator'
64
- end
65
- end
66
-
67
-
68
- describe 'without group_by key' do
69
- describe Sum do
70
- it 'sums up values' do
71
- sum = Sum.new
72
- sum.add(3)
73
- sum.add(5)
74
- sum.value.should == 8
75
- end
76
- end
77
-
78
- describe Average do
79
- it 'averages values' do
80
- avg = Average.new
81
- avg.add(3)
82
- avg.add(5)
83
- avg.value.should == 4
84
- end
85
- end
86
-
87
- describe Delta do
88
- it 'averages delta values' do
89
- d = Delta.new
90
- d.add(1.4)
91
- d.add(1.1)
92
- d.add(1.0)
93
- d.value.round(5).should == 0.2
94
- end
95
- end
96
-
97
- describe Count do
98
- it 'groups values and increment counters' do
99
- g = Count.new
100
- g.add('200')
101
- g.add('500')
102
- g.add('301')
103
- g.add('200')
104
- g.value.should == 4
105
- end
106
- end
107
- end
108
-
109
- describe 'with group_by key' do
110
-
111
- describe Sum do
112
- it 'sums up values by key' do
113
- sum = Sum.new
114
- sum.add(3, :key1)
115
- sum.add(5, :key2)
116
- sum.add(5, :key1)
117
- sum.values.should be_a(Hash)
118
- sum.values.size.should == 2
119
- sum.value(:key1).should == 8
120
- sum.values[:key1].should == 8
121
- sum.value(:key2).should == 5
122
- sum.values[:key2].should == 5
123
- end
124
- end
125
-
126
-
127
- describe Average do
128
- it 'averages values by key' do
129
- avg = Average.new
130
- avg.add(3, :key1)
131
- avg.add(5, :key2)
132
- avg.add(5, :key1)
133
- avg.values.should be_a(Hash)
134
- avg.values.size.should == 2
135
- avg.value(:key1).should == 4
136
- avg.values[:key1].should == 4
137
- avg.value(:key2).should == 5
138
- avg.values[:key2].should == 5
139
- end
140
- end
141
-
142
- describe Count do
143
- it 'groups values and increment counters' do
144
- g = Count.new
145
- g.add('200', '200')
146
- g.add('500', '500')
147
- g.add('301', '301')
148
- g.add('200', '200')
149
- g.values.should be_a(Hash)
150
- g.values.should include({'200' => 2})
151
- g.values.should include({'301' => 1})
152
- g.values.should include({'500' => 1})
153
- end
154
- end
155
-
156
- describe GroupAndCount do
157
- it 'each yields a key and a hash' do
158
- gac = GroupAndCount.new
159
- gac.add :key1, :subkey1
160
- gac.first.should be_an(Array)
161
- gac.first.size.should == 2
162
- gac.first[1].should be_a(Hash)
163
- end
164
-
165
- context :add do
166
- before :each do
167
- @gac = GroupAndCount.new
168
- end
169
-
170
- it 'requires a group_by argument' do
171
- lambda { @gac.add('foo') }.should raise_error ArgumentError
172
- end
173
-
174
- it 'counts number of occurence of subkey for key' do
175
- @gac.add :key1, :subkey1
176
- @gac.add :key1, :subkey2
177
- @gac.add :key2, :subkey1
178
- @gac.add :key2, :subkey1
179
- @gac.add :key2, :subkey3
180
-
181
- @gac.values[:key1][:subkey1].should == 1
182
- @gac.values[:key1][:subkey2].should == 1
183
- @gac.values[:key2][:subkey1].should == 2
184
- @gac.values[:key2][:subkey2].should == 0
185
- @gac.values[:key2][:subkey3].should == 1
186
- end
187
- end
188
- end
189
-
190
- describe Delta do
191
- it 'averages deltas by key' do
192
- d = Delta.new
193
- d.add(9, :key1)
194
- d.add(10, :key2)
195
- d.add(5, :key1)
196
- d.add(8, :key2)
197
- d.add(3, :key1)
198
- d.add(5, :key2)
199
- d.values.should be_a(Hash)
200
- d.values.size.should == 2
201
- d.value(:key1).should == 3
202
- d.values[:key1].should == 3
203
- d.value(:key2).should == 2.5
204
- d.values[:key2].should == 2.5
205
- end
206
- end
207
-
208
- end
209
- end
210
- end
211
- end