logfile_interval 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 304480b4bf3c9525f74bf9b835b3ec7fe72b64be
4
+ data.tar.gz: ca5f34f5a987aca00b22ea838e083c744793c545
5
+ SHA512:
6
+ metadata.gz: e36b8d0c42fbaafed596dcb5c2770450bf7e10ce68d92326f3f04eab197bbb4faa8ea8fa0f7706b55883fd2d65a876435ab689533222148151787c3c58861ba6
7
+ data.tar.gz: 34bd857fd154d29703959693c7b33b0e15b4ab0824f97e84f4362aad1e009a332e8f0b1c96fdcde1321f3462666e3bd862c9f53cb15aab41807f6321346551b0
data/.gitignore ADDED
@@ -0,0 +1,17 @@
1
+ *.gem
2
+ *.rbc
3
+ .bundle
4
+ .config
5
+ .yardoc
6
+ .kateproject
7
+ InstalledFiles
8
+ _yardoc
9
+ coverage
10
+ doc/
11
+ lib/bundler/man
12
+ pkg
13
+ rdoc
14
+ spec/reports
15
+ test/tmp
16
+ test/version_tmp
17
+ tmp
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in logfile_interval.gemspec
4
+ gemspec
data/Gemfile.lock ADDED
@@ -0,0 +1,35 @@
1
+ PATH
2
+ remote: .
3
+ specs:
4
+ logfile_interval (0.0.1)
5
+
6
+ GEM
7
+ remote: https://rubygems.org/
8
+ specs:
9
+ columnize (0.3.6)
10
+ debugger (1.6.2)
11
+ columnize (>= 0.3.1)
12
+ debugger-linecache (~> 1.2.0)
13
+ debugger-ruby_core_source (~> 1.2.3)
14
+ debugger-linecache (1.2.0)
15
+ debugger-ruby_core_source (1.2.4)
16
+ diff-lcs (1.2.5)
17
+ rake (10.1.0)
18
+ rspec (2.14.1)
19
+ rspec-core (~> 2.14.0)
20
+ rspec-expectations (~> 2.14.0)
21
+ rspec-mocks (~> 2.14.0)
22
+ rspec-core (2.14.7)
23
+ rspec-expectations (2.14.4)
24
+ diff-lcs (>= 1.1.3, < 2.0)
25
+ rspec-mocks (2.14.4)
26
+
27
+ PLATFORMS
28
+ ruby
29
+
30
+ DEPENDENCIES
31
+ bundler (~> 1.3)
32
+ debugger
33
+ logfile_interval!
34
+ rake
35
+ rspec (~> 2.14.0)
data/LICENSE.txt ADDED
@@ -0,0 +1,22 @@
1
+ Copyright (c) 2013 Philippe Le Rohellec
2
+
3
+ MIT License
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining
6
+ a copy of this software and associated documentation files (the
7
+ "Software"), to deal in the Software without restriction, including
8
+ without limitation the rights to use, copy, modify, merge, publish,
9
+ distribute, sublicense, and/or sell copies of the Software, and to
10
+ permit persons to whom the Software is furnished to do so, subject to
11
+ the following conditions:
12
+
13
+ The above copyright notice and this permission notice shall be
14
+ included in all copies or substantial portions of the Software.
15
+
16
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
19
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
20
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
21
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
22
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,94 @@
1
+ # LogfileInterval
2
+
3
+ Logfile parser and aggregator
4
+
5
+ ## Installation
6
+
7
+ Add this line to your application's Gemfile:
8
+
9
+ gem 'logfile_interval'
10
+
11
+ And then execute:
12
+
13
+ $ bundle
14
+
15
+ Or install it yourself as:
16
+
17
+ $ gem install logfile_interval
18
+
19
+ ## Usage
20
+
21
+ ### Write a LineParser class
22
+ ```ruby
23
+ module LogfileInterval
24
+ module LineParser
25
+ class AccessLog < Base
26
+ # Example line:
27
+ # 74.75.19.145 - - [31/Mar/2013:06:54:12 -0700] "GET /ppa/google_chrome HTTP/1.1" 200 7855 "https://www.google.com/" "Mozilla/5.0 Chrome/25.0.1364.160"
28
+
29
+ set_regex /^([\d\.]+)\s+\S+\s+\S+\s+\[(\d\d.*\d\d)\]\s+"(?:GET|POST|PUT|HEAD|DELETE)\s+(\S+)\s+HTTP\S+"\s+(\d+)\s+(\d+)\s+"([^"]*)"\s+"([^"]+)"$/
30
+
31
+ add_column :name => 'ip', :pos => 1, :agg_function => :group
32
+ add_column :name => 'timestamp', :pos => 2, :agg_function => :timestamp
33
+ add_column :name => 'code', :pos => 4, :agg_function => :group
34
+ add_column :name => 'length', :pos => 5, :agg_function => :average, :conversion => :integer
35
+
36
+ def time
37
+ Time.strptime(self.timestamp, '%d/%b/%Y:%H:%M:%S %z')
38
+ end
39
+ end
40
+ end
41
+ end
42
+ ```
43
+
44
+ ### Iterate through lines of a single file
45
+ And get a parsed record for each line.
46
+ ```ruby
47
+ logfile = 'access.log'
48
+ parser = LineParser::AccessLog
49
+
50
+ log = LogfileInterval::Logfile.new(logfile, parser)
51
+ log.each_line do |line|
52
+ puts line.class # String
53
+ puts line
54
+ end
55
+
56
+ parser = LineParser::AccessLog
57
+ log.each_parsed_line do |record|
58
+ puts record.class # LineParser::AccessLog
59
+ puts record.ip
60
+ puts record.time
61
+ end
62
+ ```
63
+
64
+ ### Iterate through lines of multiples files
65
+ And get a parsed record for each line.
66
+ ```ruby
67
+ logfiles = [ 'access.log', 'access.log.1', 'access.log.2' ]
68
+ set = LogfileInterval::LogfileSet.new(logfiles, parser)
69
+ set.each_parsed_line do |record|
70
+ puts record.class # LineParser::AccessLog
71
+ end
72
+ ```
73
+ ### Aggregate lines into intervals
74
+ ```ruby
75
+ length = 5.minutes
76
+ interval_builder = LogfileInterval::IntervalBuilder.new(logfiles, length)
77
+ interval_builder.each_interval do |interval|
78
+ puts interval.class # LogfileInterval::Interval
79
+ puts interval.start_time
80
+ puts interval.length
81
+ interval[:ip].each do |ip, count|
82
+ puts "#{ip}, #{count}"
83
+ end
84
+ end
85
+ ```
86
+
87
+
88
+ ## Contributing
89
+
90
+ 1. Fork it
91
+ 2. Create your feature branch (`git checkout -b my-new-feature`)
92
+ 3. Commit your changes (`git commit -am 'Add some feature'`)
93
+ 4. Push to the branch (`git push origin my-new-feature`)
94
+ 5. Create new Pull Request
data/Rakefile ADDED
@@ -0,0 +1 @@
1
+ require "bundler/gem_tasks"
data/docs/design.rb ADDED
@@ -0,0 +1,91 @@
1
+ module LogfileInterval
2
+ class Logfile
3
+ end
4
+
5
+ class LogfileSet
6
+ end
7
+
8
+ class Interval
9
+ end
10
+
11
+ module LineParser
12
+ class Base
13
+ class << self
14
+ def set_regex(regex)
15
+ end
16
+
17
+ def add_column(name, options)
18
+ end
19
+
20
+ def parse(line)
21
+ @data = {}
22
+
23
+ match_data = regex.match(line)
24
+ columns.each do |name, options|
25
+ val = match_data[options[:pos]]
26
+ @data[name] = convert(val, options[:conversion])
27
+ end
28
+ @data
29
+ end
30
+
31
+ def convert(val, conversion)
32
+ case options[:conversion]
33
+ when :integer then val.to_i
34
+ else val
35
+ end
36
+ end
37
+ end
38
+
39
+ end
40
+
41
+ class AccessLog < Base
42
+ set_regex /blah/
43
+ add_column :name => :foo, :pos => 1, :conversion => integer, :agg_function => :average
44
+
45
+ end
46
+ end
47
+
48
+ class Record
49
+ def initialize(parser, line)
50
+ @parser = parser
51
+ @data = parser.parse(line)
52
+ end
53
+
54
+ def valid_columns
55
+ @parser.columns.keys
56
+ end
57
+
58
+ def method_missing(meth, *args)
59
+ if valid_columns.include?(meth) && args.none
60
+ self[meth]
61
+ else
62
+ super
63
+ end
64
+ end
65
+ end
66
+ end
67
+
68
+ logfiles = [ 'access.log', 'access.log.1', 'access.log.2' ]
69
+ logfile = logfiles.first
70
+ parser = LineParser::AccessLog.new
71
+
72
+ logfile_iterator = LogfileInterval::Logfile.new(parser, logfile)
73
+
74
+ logfile_iterator.each_line do |line|
75
+ puts line
76
+ end
77
+
78
+ logfile_iterator.each_parsed_line do |record|
79
+ puts record.ip
80
+ puts record.time
81
+ end
82
+
83
+ interval_builder = LogfileInterval::Interval.new(parser, logfiles)
84
+
85
+ interval_builder.each_interval do |interval|
86
+ puts interval.start_time
87
+ puts interval.length
88
+ interval[:ip].each do |ip, count|
89
+ puts "#{ip}, #{count}"
90
+ end
91
+ end
data/docs/design2.rb ADDED
@@ -0,0 +1,77 @@
1
+ module LogfileInterval
2
+ class Logfile
3
+ end
4
+
5
+ class LogfileSet
6
+ end
7
+
8
+ class Interval
9
+ end
10
+
11
+ class LineParser
12
+ def initialize(regex, columns)
13
+ end
14
+
15
+ def parse(line)
16
+ @data = {}
17
+
18
+ match_data = regex.match(line)
19
+ columns.each do |name, options|
20
+ val = match_data[options[:pos]]
21
+ @data[name] = convert(val, options[:conversion])
22
+ end
23
+ @data
24
+ end
25
+
26
+ def convert(val, conversion)
27
+ case options[:conversion]
28
+ when :integer then val.to_i
29
+ else val
30
+ end
31
+ end
32
+ end
33
+
34
+ class Record
35
+ def initialize(parser, line)
36
+ @parser = parser
37
+ @data = parser.parse(line)
38
+ end
39
+
40
+ def valid_columns
41
+ @parser.columns.keys
42
+ end
43
+
44
+ def method_missing(meth, *args)
45
+ if valid_columns.include?(meth) && args.none
46
+ self[meth]
47
+ else
48
+ super
49
+ end
50
+ end
51
+ end
52
+ end
53
+
54
+ logfiles = [ 'access.log', 'access.log.1', 'access.log.2' ]
55
+ logfile = logfiles.first
56
+ parser = LineParser.new(/blah/, [ { :name=>:ip , :pos => 1 }, ...])
57
+
58
+ logfile_iterator = LogfileInterval::Logfile.new(parser, logfile)
59
+
60
+ logfile_iterator.each_line do |line|
61
+ puts line
62
+ end
63
+
64
+ logfile_iterator.each_parsed_line do |record|
65
+ puts record.ip
66
+ puts record.time
67
+ end
68
+
69
+ interval_builder = LogfileInterval::Interval.new(parser, logfiles)
70
+
71
+ interval_builder.each_interval do |interval|
72
+ puts interval.start_time
73
+ puts interval.length
74
+ interval[:ip].each do |ip, count|
75
+ puts "#{ip}, #{count}"
76
+ end
77
+ end
data/docs/design3.rb ADDED
@@ -0,0 +1,177 @@
1
+ module LogfileInterval
2
+ module LineParser
3
+ class Base
4
+ class << self
5
+ def set_regex(regex)
6
+ end
7
+
8
+ def add_column(name, options)
9
+ aggregator = Aggregators.klass(agg_function)
10
+ @columns[name] = { :pos => pos, :agg_function => aggregator, :conversion => conversion }
11
+ define_method(name)
12
+ end
13
+
14
+ def parse(line)
15
+ match_data = regex.match(line)
16
+ @data = f(match_data)
17
+ end
18
+
19
+ def create_record(line)
20
+ record = new(line)
21
+ return record.valid? ? record : nil
22
+ end
23
+ end
24
+
25
+ end
26
+
27
+ class AccessLog < Base
28
+ set_regex /blah/
29
+ add_column :name => :foo, :pos => 1, :conversion => integer, :agg_function => :average
30
+
31
+ def initialize(line)
32
+ @data = self.class.parse(line)
33
+ end
34
+ end
35
+
36
+ module Aggregator
37
+ def self.klass(agg_function)
38
+ case agg_function
39
+ when :sum then Sum
40
+ end
41
+ end
42
+
43
+ class Sum
44
+ def initialize
45
+ @val = 0
46
+ end
47
+
48
+ def add(value)
49
+ @val += value
50
+ end
51
+
52
+ def value
53
+ @val
54
+ end
55
+ end
56
+
57
+ class Group
58
+ def initialize
59
+ @val = Counter.new
60
+ end
61
+
62
+ def add(value)
63
+ @val.increment(value)
64
+ end
65
+ end
66
+ end
67
+ end
68
+
69
+ class Logfile
70
+ def initialize(filename, parser)
71
+ end
72
+
73
+ def each_line
74
+ end
75
+
76
+ def each_parsed_line
77
+ each_line do |line|
78
+ record = parser.create_record(line)
79
+ yield record if record
80
+ end
81
+ end
82
+ end
83
+
84
+ class LogfileSet
85
+ def initialize(filenames_array, parser)
86
+ end
87
+
88
+ def ordered_filenams
89
+ end
90
+
91
+ def each_line
92
+ end
93
+
94
+ def each_parsed_line
95
+ end
96
+ end
97
+
98
+ class IntervalBuilder
99
+ def initialize(logfile_set, length)
100
+ parser = logfile_set.parser
101
+ end
102
+
103
+ def each_interval
104
+ interval = Interval.new(now, length)
105
+ set.each_parsed_line(parser) do |record|
106
+ while record.time < interval.start_time do
107
+ yield interval
108
+ interval = Interval.new(interval.start_time, length)
109
+ end
110
+ interval.add(record)
111
+ end
112
+ end
113
+ end
114
+
115
+ class Counter < Hash
116
+ def increment(key)
117
+ self[key] = self[key] ? self[key] + 1 : 1
118
+ end
119
+ end
120
+
121
+ class Interval
122
+ def initialize(end_time, length, parser)
123
+ @data = {}
124
+ parser.columns.each do |name, options|
125
+ @data[name] = options[:aggregator].new
126
+ end
127
+ end
128
+
129
+ def [](name)
130
+ @data[name].value
131
+ end
132
+
133
+ def add_record(record)
134
+ return unless record.valid?
135
+ raise ParserMismatch unless record.class == parser
136
+
137
+ @size += 1
138
+ parser.columns.each do |name, options|
139
+ @data[name].add(record[name])
140
+ end
141
+ end
142
+ end
143
+ end
144
+
145
+ logfiles = [ 'access.log', 'access.log.1', 'access.log.2' ]
146
+ logfile = logfiles.first
147
+
148
+ parser = LineParser::AccessLog
149
+
150
+ logfile_iterator = LogfileInterval::Logfile.new(logfile, parser)
151
+ logfile_iterator.each_line do |line|
152
+ puts line.class # String
153
+ puts line
154
+ end
155
+
156
+ parser = LineParser::AccessLog
157
+ logfile_iterator.each_parsed_line do |record|
158
+ puts record.class # LineParser::AccessLog
159
+ puts record.ip
160
+ puts record.time
161
+ end
162
+
163
+ set_iterator = LogfileInterval::LogfileSet.new(logfiles, parser)
164
+ set_iterator.each_parsed_line do |record|
165
+ puts record.class # LineParser::AccessLog
166
+ end
167
+
168
+ length = 5.minutes
169
+ interval_builder = LogfileInterval::IntervalBuilder.new(logfiles, length)
170
+ interval_builder.each_interval do |interval|
171
+ puts interval.class # LogfileInterval::Interval
172
+ puts interval.start_time
173
+ puts interval.length
174
+ interval[:ip].each do |ip, count|
175
+ puts "#{ip}, #{count}"
176
+ end
177
+ end