logfile_interval 1.0.0 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 304480b4bf3c9525f74bf9b835b3ec7fe72b64be
-  data.tar.gz: ca5f34f5a987aca00b22ea838e083c744793c545
+  metadata.gz: e0f1a02f57605b2eb8eacf703a456dbe9038abcd
+  data.tar.gz: f1d68cd7e72dc15023f6c3f8ae72d7e2eb104102
 SHA512:
-  metadata.gz: e36b8d0c42fbaafed596dcb5c2770450bf7e10ce68d92326f3f04eab197bbb4faa8ea8fa0f7706b55883fd2d65a876435ab689533222148151787c3c58861ba6
-  data.tar.gz: 34bd857fd154d29703959693c7b33b0e15b4ab0824f97e84f4362aad1e009a332e8f0b1c96fdcde1321f3462666e3bd862c9f53cb15aab41807f6321346551b0
+  metadata.gz: 3bc11945565915aaa028f61f01b670340a2765640abf44aca2d8b1d36cb6a6b09c8626b7e3d964e3e3a7f0479893240defa0b600feec3f98e1f6c5176c5cda8a
+  data.tar.gz: cb91a1b9aad9bd67aa5fa1ccfa9a9f0d69cab5bbc177ecdfc4ac4f80d08c1edcf39cbf5df250c8251913ca50a2d4e9abf17e92335c41b9b10b17dc32bddf5cc5

data/.travis.yml

@@ -0,0 +1,5 @@
+language: ruby
+rvm:
+  - 1.9.3
+  - 2.0.0
+script: bundle exec rspec spec

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    logfile_interval (0.0.1)
+    logfile_interval (1.1.0)
 
 GEM
   remote: https://rubygems.org/
@@ -14,6 +14,8 @@ GEM
     debugger-linecache (1.2.0)
     debugger-ruby_core_source (1.2.4)
     diff-lcs (1.2.5)
+    docile (1.1.1)
+    multi_json (1.8.2)
     rake (10.1.0)
     rspec (2.14.1)
       rspec-core (~> 2.14.0)
@@ -23,6 +25,11 @@ GEM
     rspec-expectations (2.14.4)
       diff-lcs (>= 1.1.3, < 2.0)
     rspec-mocks (2.14.4)
+    simplecov (0.8.2)
+      docile (~> 1.1.0)
+      multi_json
+      simplecov-html (~> 0.8.0)
+    simplecov-html (0.8.0)
 
 PLATFORMS
   ruby
@@ -33,3 +40,4 @@ DEPENDENCIES
   logfile_interval!
   rake
   rspec (~> 2.14.0)
+  simplecov

data/README.md

@@ -1,6 +1,9 @@
-# LogfileInterval
+# LogfileInterval [![Build Status](https://travis-ci.org/plerohellec/logfile_interval.png?branch=master)](https://travis-ci.org/plerohellec/logfile_interval)
 
-Logfile parser and aggregator
+Logfile parser and aggregator.
+
+It iterates over each line of logfiles, parses each line and aggregates all lines in a time interval into a single
+record made up of the sum, the average, the number of occurences per value or average of the deltas between lines.
 
 ## Installation
 
@@ -19,6 +22,7 @@ Or install it yourself as:
 ## Usage
 
 ### Write a LineParser class
+#### Example
 ```ruby
 module LogfileInterval
   module LineParser
@@ -28,10 +32,12 @@ module LogfileInterval
 
       set_regex /^([\d\.]+)\s+\S+\s+\S+\s+\[(\d\d.*\d\d)\]\s+"(?:GET|POST|PUT|HEAD|DELETE)\s+(\S+)\s+HTTP\S+"\s+(\d+)\s+(\d+)\s+"([^"]*)"\s+"([^"]+)"$/
 
-      add_column :name => 'ip',        :pos => 1, :agg_function => :group
-      add_column :name => 'timestamp', :pos => 2, :agg_function => :timestamp
-      add_column :name => 'code',      :pos => 4, :agg_function => :group
-      add_column :name => 'length',    :pos => 5, :agg_function => :average,   :conversion => :integer
+      add_column :name => 'ip',           :pos => 1, :aggregator => :count,     :group_by => 'ip'
+      add_column :name => 'timestamp',    :pos => 2, :aggregator => :timestamp
+      add_column :name => 'code',         :pos => 4, :aggregator => :count,     :group_by => 'code'
+      add_column :name => 'code_by_ip',   :pos => 4, :aggregator => :count,     :group_by => 'ip'
+      add_column :name => 'length',       :pos => 5, :aggregator => :average,                      :conversion => :integer
+      add_column :name => 'length_by_ip', :pos => 5, :aggregator => :average,   :group_by => 'ip', :conversion => :integer
 
       def time
         Time.strptime(self.timestamp, '%d/%b/%Y:%H:%M:%S %z')
@@ -40,6 +46,29 @@ module LogfileInterval
   end
 end
 ```
+#### Writing a parser class
+The parser must define:
+* A regex that extracts the fields out of each line.
+* A set of columns that will to be parsed and aggregated in time intervals.
+* A 'time' method that converts the mandatory timestamp field of a line into a Time object.
+
+#### Attributes of a column:
+* name: a parsed record will have a method with that name returning the value found at that position
+* pos:  the position of the captured field in the regex matched data
+* aggregator : the aggregation mode for this field
+* conversion: the parser will convert the field to an integer or a float when building the parsed record
+* group_by: group_by value is the name of another field. The aggregator will apply the aggregator to this field for each distinct value found in the other field.
+
+#### Aggregator types and options
+* timestamp: the timestamp field will be used to determine to which interval the line belongs, each line MUST have a timestamp
+* count: the aggregator will count the number of occurence of this field
+  * without the group_by option, it will just count the total number of lines (probably useless)
+  * with a group_by option pointing to the same field as the current one, it will count the number of occurence
+    per distinct value of this column
+  * with a group_by option pointing to another field, it will count the number of occurences of (this field, other field) pairs.
+* average: the aggregator will calculate the average value of this field
+* sum: the aggregator will add up the values of this field
+* delta: the aggregator will caclculate the difference between each line and the next and will average all the deltas
 
 ### Iterate through lines of a single file
 And get a parsed record for each line.
@@ -58,8 +87,11 @@ log.each_parsed_line do |record|
   puts record.class # LineParser::AccessLog
   puts record.ip
   puts record.time
+  puts record.code
+  puts record.length
 end
 ```
+**Note**: the Logfile iterators always start with the last line in the file and works its way backward.
 
 ### Iterate through lines of multiples files
 And get a parsed record for each line.
@@ -70,21 +102,25 @@ set.each_parsed_line do |record|
   puts record.class # LineParser::AccessLog
 end
 ```
+**Note**: the LogfileSet iterators always starts with the most recent file.
+
 ### Aggregate lines into intervals
 ```ruby
 length = 5.minutes
-interval_builder = LogfileInterval::IntervalBuilder.new(logfiles, length)
+interval_builder = LogfileInterval::IntervalBuilder.new(set, length)
 interval_builder.each_interval do |interval|
   puts interval.class  # LogfileInterval::Interval
   puts interval.start_time
-  puts interval.length
+  puts interval[:length]
   interval[:ip].each do |ip, count|
     puts "#{ip}, #{count}"
   end
+  interval[:length_by_ip].each do |ip, avg_length|
+    puts "#{ip}, #{avg_length}"
+  end
 end
 ```
 
-
 ## Contributing
 
 1. Fork it

data/docs/design.rb

@@ -40,7 +40,7 @@ module LogfileInterval
 
     class AccessLog < Base
       set_regex /blah/
-      add_column :name => :foo, :pos => 1, :conversion => integer, :agg_function => :average
+      add_column :name => :foo, :pos => 1, :conversion => integer, :aggregator => :average
 
     end
   end

data/docs/design3.rb

@@ -6,8 +6,8 @@ module LogfileInterval
         end
 
         def add_column(name, options)
-          aggregator = Aggregators.klass(agg_function)
-          @columns[name] = { :pos => pos, :agg_function => aggregator, :conversion => conversion }
+          agg = Aggregators.klass(aggregator)
+          @columns[name] = { :pos => pos, :aggregator => agg, :conversion => conversion }
           define_method(name)
         end
 
@@ -26,7 +26,7 @@ module LogfileInterval
 
     class AccessLog < Base
       set_regex /blah/
-      add_column :name => :foo, :pos => 1, :conversion => integer, :agg_function => :average
+      add_column :name => :foo, :pos => 1, :conversion => integer, :aggregator => :average
 
       def initialize(line)
         @data = self.class.parse(line)
@@ -34,8 +34,8 @@ module LogfileInterval
     end
 
     module Aggregator
-      def self.klass(agg_function)
-        case agg_function
+      def self.klass(aggregator)
+        case aggregator
         when :sum then Sum
         end
       end
@@ -54,7 +54,7 @@ module LogfileInterval
         end
       end
 
-      class Group
+      class Count
         def initialize
           @val = Counter.new
         end

data/lib/logfile_interval/interval.rb

@@ -22,7 +22,7 @@ module LogfileInterval
     end
 
     def [](name)
-      @data[name].value
+      @data[name].values
     end
 
     def add_record(record)
@@ -35,7 +35,8 @@ module LogfileInterval
 
       parser.columns.each do |name, options|
         next unless @data[name]
-        @data[name].add(record[name])
+        group_by_value = record[options[:group_by]] if options[:group_by]
+        @data[name].add(record[name], group_by_value)
       end
     end
   end

data/lib/logfile_interval/line_parser/aggregator.rb

@@ -1,59 +1,106 @@
 module LogfileInterval
   module LineParser
     module Aggregator
-      def self.klass(agg_function)
-        case agg_function
-        when :sum     then Sum
-        when :average then Average
-        when :group   then Group
+      def self.klass(aggregator)
+        case aggregator
+        when :sum               then Sum
+        when :average           then Average
+        when :count             then Count
+        when :group_and_count   then GroupAndCount
+        when :delta             then Delta
         end
       end
 
-      class Sum
+      class Base
+        include Enumerable
+
         def initialize
-          @val = 0
+          @val = Counter.new
+          @size = Counter.new
         end
 
-        def add(value)
-          @val += value
+        def value(group = nil)
+          average(key(group))
         end
 
-        def value
-          @val
+        def values
+          if single_value?
+            value
+          else
+            self.inject({}) { |h, v| h[v[0]] = v[1]; h }
+          end
         end
-      end
 
-      class Average
-        def initialize
-          @val  = 0
-          @size = 0
+        def add(value, group_by = nil)
+          raise NotImplementedError
         end
 
-        def add(value)
-          @val += value
-          @size += 1
+        private
+        def key(group_by = nil)
+          group_by ? group_by : :all
         end
 
-        def value
-          if @size > 0
-            @val.to_f / @size.to_f
-          else
-            0
+        def single_value?
+          return true if @val.empty?
+          @val.keys.count == 1 && @val.keys.first == :all
+        end
+
+        def each
+          @val.each_key do |k|
+            yield k, average(k)
           end
         end
+
+        def average(k)
+          @size[k] > 0 ? @val[k].to_f / @size[k].to_f : 0
+        end
       end
 
-      class Group
-        def initialize
-          @val = Counter.new
+      class Sum < Base
+        def add(value, group_by = nil)
+          @val.add(key(group_by), value)
+          @size.set(key(group_by), 1)
         end
+      end
 
-        def add(value)
-          @val.increment(value)
+      class Average < Base
+        def add(value, group_by = nil)
+          @val.add(key(group_by), value)
+          @size.increment(key(group_by))
         end
+      end
+
+      class Count < Base
+        def add(value, group_by = nil)
+          @val.add(key(group_by), 1)
+          @size.set(key(group_by), 1)
+        end
+      end
 
-        def value
-          @val
+      class GroupAndCount < Base
+        def each
+          @val.each { |k, v| yield k, v }
+        end
+
+        def add(value, group_by)
+          raise ArgumentError, 'group_by argument is mandatory for GroupAndCount#add' unless group_by
+          @val.increment_subkey(value, key(group_by))
+          @size.set(key(group_by), 1)
+        end
+      end
+
+      class Delta < Base
+        def initialize
+          @previous = Counter.new
+          super
+        end
+
+        def add(value, group_by = nil)
+          if @previous.has_key?(key(group_by))
+            @val.add(key(group_by), @previous[key(group_by)] - value)
+            @size.increment(key(group_by))
+          end
+          @previous.set(key(group_by), value)
         end
       end
     end

data/lib/logfile_interval/line_parser/base.rb

@@ -1,11 +1,13 @@
 module LogfileInterval
   module LineParser
-    AGGREGATION_FUNCTIONS = [ :sum, :average, :timestamp, :group ]
+    AGGREGATION_FUNCTIONS = [ :sum, :average, :timestamp, :count, :delta ]
 
     class InvalidLine         < StandardError; end
     class ConfigurationError  < StandardError; end
 
     class Base
+      attr_reader :data
+
       class << self
         attr_reader :regex
 
@@ -20,14 +22,19 @@ module LogfileInterval
         def add_column(options)
           name          = options.fetch(:name)
           pos           = options.fetch(:pos)
-          agg_function  = options.fetch(:agg_function)
+          aggregator  = options.fetch(:aggregator)
           conversion    = options.fetch(:conversion, :string)
-          unless AGGREGATION_FUNCTIONS.include?(agg_function)
-            raise ArgumentError, "agg_function must be one of #{AGGREGATION_FUNCTIONS.join(', ')}"
+          unless AGGREGATION_FUNCTIONS.include?(aggregator)
+            raise ArgumentError, "aggregator must be one of #{AGGREGATION_FUNCTIONS.join(', ')}"
+          end
+
+          if aggregator == :count && options[:group_by] && options[:group_by] != name
+            aggregator = :group_and_count
           end
 
-          aggregator = Aggregator.klass(agg_function)
+          aggregator = Aggregator.klass(aggregator)
           columns[name] = { :pos => pos, :aggregator => aggregator, :conversion => conversion }
+          columns[name][:group_by] = options[:group_by]
 
           define_method(name) do
             @data[name]

data/lib/logfile_interval/line_parser/counter.rb

@@ -1,22 +1,43 @@
 module LogfileInterval
   module LineParser
     class Counter < Hash
-      def increment(val)
-        if self.has_key?(val)
-          self[val] += 1
+      class ValueTypeError < StandardError; end
+
+      def increment(key)
+        if self.has_key?(key)
+          self[key] += 1
         else
-          self[val] = 1
+          self[key] = 1
+        end
+      end
+
+      def increment_subkey(key, subkey)
+        if self.has_key?(key) && !self[key].is_a?(Counter)
+          raise "Value for #{key} is not a Counter"
+        end
+
+        unless self.has_key?(key)
+          self[key] = Counter.new
         end
+        self[key].increment(subkey)
       end
 
-      def add(val, num)
-        if self.has_key?(val)
-          self[val] += num
+      def add(key, num)
+        if self.has_key?(key)
+          self[key] += num
         else
-          self[val] = num
+          self[key] = num
         end
       end
 
+      def set(key, num)
+        self[key] = num
+      end
+
+      def [](key)
+        self.fetch(key, 0)
+      end
+
       def merge(c)
         c.keys.each do |k|
           self.add c[k]

data/lib/logfile_interval/logfile.rb

@@ -10,8 +10,11 @@ module LogfileInterval
     def first_timestamp
       return nil unless File.exist?(@filename)
       File.open(@filename) do |f|
-        line = parser.create_record(f.gets)
-        line.time
+        while line = f.gets
+          if record = parser.create_record(line)
+            return record.time
+          end
+        end
       end
     end
 

data/lib/logfile_interval/version.rb

@@ -1,3 +1,3 @@
 module LogfileInterval
-  VERSION = "1.0.0"
+  VERSION = "1.1.0"
 end

data/logfile_interval.gemspec

@@ -8,9 +8,9 @@ Gem::Specification.new do |spec|
   spec.version       = LogfileInterval::VERSION
   spec.authors       = ["Philippe Le Rohellec"]
   spec.email         = ["philippe@lerohellec.com"]
-  spec.description   = %q{Logfile parser and aggregator}
-  spec.summary       = %q{Aggregate logfile data into intervals}
-  spec.homepage      = ""
+  spec.description   = "Logfile parser and aggregator"
+  spec.summary       = "Aggregate logfile data into intervals"
+  spec.homepage      = "https://github.com/plerohellec/logfile_interval"
   spec.license       = "MIT"
 
   spec.files         = `git ls-files`.split($/)
@@ -19,7 +19,8 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_development_dependency "bundler", "~> 1.3"
-  spec.add_development_dependency(%q<debugger>, [">= 0"])
-  spec.add_development_dependency(%q<rspec>, ["~> 2.14.0"])
+  spec.add_development_dependency "debugger", [">= 0"]
+  spec.add_development_dependency "rspec", ["~> 2.14.0"]
   spec.add_development_dependency "rake"
+  spec.add_development_dependency "simplecov"
 end

data/spec/lib/interval_builder_spec.rb

@@ -29,9 +29,9 @@ module LogfileInterval
           @intervals.first.size.should == 4
           @intervals.first[:total_time].should == 700.0/4
           @intervals.first[:num_bytes].should == 52000
-          @intervals.first[:action]['posts#index'] = 1
-          @intervals.first[:action]['posts#create'] = 1
-          @intervals.first[:action]['posts#show'] = 2
+          @intervals.first[:rss].round(5).should == 0.60
+          @intervals.first[:ip].should == 4
+          @intervals.first[:action].should == 4
         end
       end
 
@@ -40,8 +40,9 @@ module LogfileInterval
           @intervals.last.size.should == 2
           @intervals.last[:total_time].should == 300
           @intervals.last[:num_bytes].should == 41000
-          @intervals.last[:action]['posts#index'] = 1
-          @intervals.last[:action]['posts#show'] = 1
+          @intervals.last[:rss].round(5).should == 0.20
+          @intervals.last[:ip].should == 2
+          @intervals.last[:action].should == 2
         end
       end
     end

data/spec/lib/interval_spec.rb

@@ -11,43 +11,66 @@ module LogfileInterval
       interval.size.should == 0
       interval[:total_time].should == 0
       interval[:num_bytes].should == 0
-      interval[:action].should be_a(Hash)
+      interval[:action].should == 0
+      interval[:ip].should == 0
     end
 
     context :add_record do
       before :each do
         @end_time = Time.new(2013, 12, 01, 16, 00, 00, '-08:00')
         @length = 300
-        @interval = Interval.new(@end_time, @length, LineParser::TimingLog)
       end
 
-      it 'rejects record out of interval' do
-        oor_record = LineParser::TimingLog.create_record('1385942450, posts#index, 100, 20000')
-        lambda { @interval.add_record(oor_record) }.should raise_error(Interval::OutOfRange)
-      end
+      context 'basics' do
+        before :each do
+          @interval = Interval.new(@end_time, @length, LineParser::TimingLog)
+        end
+
+        it 'rejects record out of interval' do
+          oor_record = LineParser::TimingLog.create_record('1385942450, 192.168.0.5, posts#index, 100, 20000, 50.0')
+          lambda { @interval.add_record(oor_record) }.should raise_error(Interval::OutOfRange)
+        end
+
+        it 'rejects record at interval start_time' do
+          oor_record = LineParser::TimingLog.create_record('1385942100, 192.168.0.5, posts#index, 100, 20000, 50.0')
+          lambda { @interval.add_record(oor_record) }.should raise_error(Interval::OutOfRange)
+        end
+
+        it 'adds 1 record to interval' do
+          record1 = LineParser::TimingLog.create_record('1385942400, 192.168.0.5, posts#index, 100, 20000, 50.0')
+          @interval.add_record(record1)
 
-      it 'rejects record at interval start_time' do
-        oor_record = LineParser::TimingLog.create_record('1385942100, posts#index, 100, 20000')
-        lambda { @interval.add_record(oor_record) }.should raise_error(Interval::OutOfRange)
+          @interval.size.should == 1
+          @interval[:total_time].should == 100
+          @interval[:num_bytes].should == 20000
+          @interval[:action].should == 1
+          @interval[:ip].should == 1
+        end
       end
 
-      it 'adds 1 record to interval' do
-        record1 = LineParser::TimingLog.create_record('1385942400, posts#index, 100, 20000')
-        @interval.add_record(record1)
+      context 'with count and group by options' do
+        it 'creates an aggregator of type GroupAndCount' do
+          expect(LineParser::Aggregator::GroupAndCount).to receive(:new)
+          interval = Interval.new(@end_time, @length, LineParser::TimingLogWithGrouping)
+        end
 
-        @interval.size.should == 1
-        @interval[:total_time].should == 100
-        @interval[:num_bytes].should == 20000
-        @interval[:action]['posts#index'].should == 1
+        it 'add_record accepts key and subkey' do
+          interval = Interval.new(@end_time, @length, LineParser::TimingLogWithGrouping)
+          record1 = LineParser::TimingLogWithGrouping.create_record('1385942400, 192.168.0.5, posts#index, 100, 20000, 53.0')
+          interval.add_record(record1)
+          interval.size.should == 1
+        end
       end
 
-      context '3 records' do
+      context 'with 3 records' do
         before :each do
-          record1 = LineParser::TimingLog.create_record('1385942400, posts#index, 100, 20000')
+          @interval = Interval.new(@end_time, @length, LineParser::TimingLog)
+
+          record1 = LineParser::TimingLog.create_record('1385942400, 192.168.0.5, posts#index, 100, 20000, 53.0')
           @interval.add_record(record1)
-          record2 = LineParser::TimingLog.create_record('1385942300, posts#show, 50, 10000')
+          record2 = LineParser::TimingLog.create_record('1385942300, 192.168.0.5, posts#show, 50, 10000, 51.0')
           @interval.add_record(record2)
-          record3 = LineParser::TimingLog.create_record('1385942200, posts#show, 60, 12000')
+          record3 = LineParser::TimingLog.create_record('1385942200, 10.10.10.10, posts#show, 60, 12000, 50.0')
           @interval.add_record(record3)
         end
 
@@ -55,17 +78,72 @@ module LogfileInterval
           @interval.size.should == 3
         end
 
-        it 'averages columns with average agg_function' do
+        it 'averages columns with average aggregator' do
           @interval[:total_time].should == 70
         end
 
-        it 'sums up columns with sum agg_function' do
+        it 'sums up columns with sum aggregator' do
           @interval[:num_bytes].should == 42000
         end
 
-        it 'groups and counts columns with group agg_function' do
-          @interval[:action]['posts#index'].should == 1
-          @interval[:action]['posts#show'].should == 2
+        it 'averages the delta columns with delta aggregator' do
+          @interval[:rss].should == 1.5
+        end
+
+        it 'counts columns with group aggregator' do
+          @interval[:ip].should == 3
+          @interval[:action].should == 3
+        end
+      end
+
+      context 'with group_by key' do
+        before :each do
+          @interval = Interval.new(@end_time, @length, LineParser::TimingLogWithGrouping)
+
+          record1 = LineParser::TimingLogWithGrouping.create_record('1385942400, 192.168.0.5, posts#index, 100, 20000, 53.0')
+          @interval.add_record(record1)
+          record2 = LineParser::TimingLogWithGrouping.create_record('1385942300, 192.168.0.5, posts#show, 50, 10000, 51.0')
+          @interval.add_record(record2)
+          record3 = LineParser::TimingLogWithGrouping.create_record('1385942200, 192.168.0.5, posts#show, 60, 12000, 50.0')
+          @interval.add_record(record3)
+          record4 = LineParser::TimingLogWithGrouping.create_record('1385942180, 10.10.10.10, posts#index, 100, 20000, 48.0')
+          @interval.add_record(record4)
+        end
+
+        it 'counts value column per group column' do
+          @interval[:action].should be_a(Hash)
+          @interval[:action].size.should == 2
+          @interval[:action]['posts#index'].should == 2
+          @interval[:action]['posts#show'].should  == 2
+        end
+
+        it 'counts value and group_by pairs' do
+          @interval[:ip_by_action].should be_a(Hash)
+          @interval[:ip_by_action]['192.168.0.5'].should be_a(Hash)
+          @interval[:ip_by_action]['192.168.0.5']['posts#index'].should == 1
+          @interval[:ip_by_action]['192.168.0.5']['posts#show'].should  == 2
+          @interval[:ip_by_action]['10.10.10.10']['posts#index'].should == 1
+        end
+
+        it 'averages value column per group column' do
+          @interval[:total_time].should be_a(Hash)
+          @interval[:total_time].size.should == 2
+          @interval[:total_time]['posts#index'].should == 100
+          @interval[:total_time]['posts#show'].should  == 55
+        end
+
+        it 'sums up value column per group column' do
+          @interval[:num_bytes].should be_a(Hash)
+          @interval[:num_bytes].size.should == 2
+          @interval[:num_bytes]['posts#index'].should == 40000
+          @interval[:num_bytes]['posts#show'].should  == 22000
+        end
+
+        it 'averages deltas on value column per group column' do
+          @interval[:rss].should be_a(Hash)
+          @interval[:rss].size.should == 2
+          @interval[:rss]['posts#index'].should == 5
+          @interval[:rss]['posts#show'].should  == 1
         end
       end
     end

data/spec/lib/line_parser/aggregator_spec.rb

@@ -7,40 +7,204 @@ module LogfileInterval
         it 'finds the aggregator class' do
           Aggregator.klass(:sum).should == Sum
           Aggregator.klass(:average).should == Average
-          Aggregator.klass(:group).should == Group
+          Aggregator.klass(:count).should == Count
+          Aggregator.klass(:group_and_count).should == GroupAndCount
+          Aggregator.klass(:delta).should == Delta
         end
       end
 
-      describe Sum do
-        it 'sums up values' do
-          sum = Sum.new
-          sum.add(3)
-          sum.add(5)
-          sum.value.should == 8
+      shared_examples 'an aggregator' do
+        let(:aggregator) { described_class.new }
+
+        [ :add, :value, :values ].each do |method|
+          it "responds to #{method}" do
+            aggregator.should respond_to(method)
+          end
+        end
+
+        context 'values' do
+          context 'with one group' do
+            before :each do
+              aggregator.add(5, :key1)
+            end
+
+            it 'returns a hash' do
+              aggregator.values.should be_a(Hash) unless aggregator.is_a?(Delta)
+            end
+          end
+
+          context 'with several groups' do
+            before :each do
+              aggregator.add(5, :key1)
+              aggregator.add(3, :key2)
+              aggregator.add(3, :key1)
+            end
+
+            it 'returns a hash' do
+              aggregator.values.should be_a(Hash)
+            end
+          end
+
+          context 'with no group' do
+            before :each do
+              aggregator.add(5)
+              aggregator.add(3)
+            end
+
+            it 'returns a numeric' do
+              aggregator.values.should be_a(Numeric) unless aggregator.is_a?(Count)
+            end
+          end
         end
       end
 
-      describe Average do
-        it 'averages values' do
-          sum = Average.new
-          sum.add(3)
-          sum.add(5)
-          sum.value.should == 4
+      [ Count, Sum, Average, Delta ]. each do |klass|
+        describe klass do
+          it_behaves_like 'an aggregator'
         end
       end
 
-      describe Group do
-        it 'groups values and increment counters' do
-          g = Group.new
-          g.add('200')
-          g.add('500')
-          g.add('301')
-          g.add('200')
-          g.value.should be_a(Hash)
-          g.value.should include({'200' => 2})
-          g.value.should include({'301' => 1})
-          g.value.should include({'500' => 1})
+
+      describe 'without group_by key' do
+        describe Sum do
+          it 'sums up values' do
+            sum = Sum.new
+            sum.add(3)
+            sum.add(5)
+            sum.value.should == 8
+          end
         end
+
+        describe Average do
+          it 'averages values' do
+            avg = Average.new
+            avg.add(3)
+            avg.add(5)
+            avg.value.should == 4
+          end
+        end
+
+        describe Delta do
+          it 'averages delta values' do
+            d = Delta.new
+            d.add(1.4)
+            d.add(1.1)
+            d.add(1.0)
+            d.value.round(5).should == 0.2
+          end
+        end
+
+        describe Count do
+          it 'groups values and increment counters' do
+            g = Count.new
+            g.add('200')
+            g.add('500')
+            g.add('301')
+            g.add('200')
+            g.value.should == 4
+          end
+        end
+      end
+
+      describe 'with group_by key' do
+
+        describe Sum do
+          it 'sums up values by key' do
+            sum = Sum.new
+            sum.add(3, :key1)
+            sum.add(5, :key2)
+            sum.add(5, :key1)
+            sum.values.should be_a(Hash)
+            sum.values.size.should == 2
+            sum.value(:key1).should == 8
+            sum.values[:key1].should == 8
+            sum.value(:key2).should == 5
+            sum.values[:key2].should == 5
+          end
+        end
+
+
+        describe Average do
+          it 'averages values by key' do
+            avg = Average.new
+            avg.add(3, :key1)
+            avg.add(5, :key2)
+            avg.add(5, :key1)
+            avg.values.should be_a(Hash)
+            avg.values.size.should == 2
+            avg.value(:key1).should == 4
+            avg.values[:key1].should == 4
+            avg.value(:key2).should == 5
+            avg.values[:key2].should == 5
+          end
+        end
+
+        describe Count do
+          it 'groups values and increment counters' do
+            g = Count.new
+            g.add('200', '200')
+            g.add('500', '500')
+            g.add('301', '301')
+            g.add('200', '200')
+            g.values.should be_a(Hash)
+            g.values.should include({'200' => 2})
+            g.values.should include({'301' => 1})
+            g.values.should include({'500' => 1})
+          end
+        end
+
+        describe GroupAndCount do
+          it 'each yields a key and a hash' do
+            gac = GroupAndCount.new
+            gac.add :key1, :subkey1
+            gac.first.should be_an(Array)
+            gac.first.size.should == 2
+            gac.first[1].should be_a(Hash)
+          end
+
+          context :add do
+            before :each do
+              @gac = GroupAndCount.new
+            end
+
+            it 'requires a group_by argument' do
+              lambda { @gac.add('foo') }.should raise_error ArgumentError
+            end
+
+            it 'counts number of occurence of subkey for key' do
+              @gac.add :key1, :subkey1
+              @gac.add :key1, :subkey2
+              @gac.add :key2, :subkey1
+              @gac.add :key2, :subkey1
+              @gac.add :key2, :subkey3
+
+              @gac.values[:key1][:subkey1].should == 1
+              @gac.values[:key1][:subkey2].should == 1
+              @gac.values[:key2][:subkey1].should == 2
+              @gac.values[:key2][:subkey2].should == 0
+              @gac.values[:key2][:subkey3].should == 1
+            end
+          end
+        end
+
+        describe Delta do
+          it 'averages deltas by key' do
+            d = Delta.new
+            d.add(9, :key1)
+            d.add(10, :key2)
+            d.add(5, :key1)
+            d.add(8, :key2)
+            d.add(3, :key1)
+            d.add(5, :key2)
+            d.values.should be_a(Hash)
+            d.values.size.should == 2
+            d.value(:key1).should == 3
+            d.values[:key1].should == 3
+            d.value(:key2).should == 2.5
+            d.values[:key2].should == 2.5
+          end
+        end
+
       end
     end
   end

data/spec/lib/line_parser/base_spec.rb

@@ -43,7 +43,7 @@ module LogfileInterval
 
     describe 'Broken parsers' do
       class NoRegexLog < Base
-        add_column :name => 'ip',        :pos => 1, :agg_function => :group
+        add_column :name => 'ip',        :pos => 1, :aggregator => :count
       end
 
       class NoColumnLog < Base
@@ -66,7 +66,7 @@ module LogfileInterval
     describe TimingLog do
       before :each do
         # 1385942400 = 2013/12/01 16:00:00
-        @line = '1385942400, posts#index, 100, 20000'
+        @line = '1385942400, 192.168.0.5, posts#index, 100, 20000, 50.00'
       end
 
       it 'parses a timing line' do
@@ -76,6 +76,7 @@ module LogfileInterval
         record.action.should == 'posts#index'
         record.total_time.should == 100
         record.num_bytes.should == 20000
+        record.rss.should == 50.0
       end
     end
   end

data/spec/lib/logfile_spec.rb

@@ -32,8 +32,12 @@ module LogfileInterval
 
       records.first.time.should == Time.new(2012, 01, 01, 16, 30, 51, '-08:00')
       records.first.code.should == '200'
+      records.first.length.should == 6801
+      records.first.length_by_ip.should == 6801
       records.last.time.should  == Time.new(2012, 01, 01, 00, 57, 47, '-08:00')
       records.last.code.should == '301'
+      records.last.length.should == 185
+      records.last.length_by_ip.should == 185
     end
   end
 end

data/spec/spec_helper.rb

@@ -1,6 +1,13 @@
 $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
 $LOAD_PATH.unshift(File.dirname(__FILE__))
 
+require 'simplecov'
+
+SimpleCov.start do
+  add_filter '/spec/'
+  add_filter '/config/'
+end
+
 require 'rspec'
 require 'logfile_interval'
 

data/spec/support/lib/access_log.rb

@@ -9,10 +9,11 @@ module LogfileInterval
 
       set_regex /^([\d\.]+)\s+\S+\s+\S+\s+\[(\d\d.*\d\d)\]\s+"(?:GET|POST|PUT|HEAD|DELETE)\s+(\S+)\s+HTTP\S+"\s+(\d+)\s+(\d+)\s+"([^"]*)"\s+"([^"]+)"$/
 
-      add_column :name => 'ip',        :pos => 1, :agg_function => :group
-      add_column :name => 'timestamp', :pos => 2, :agg_function => :timestamp
-      add_column :name => 'code',      :pos => 4, :agg_function => :group
-      add_column :name => 'length',    :pos => 5, :agg_function => :average,   :conversion => :integer
+      add_column :name => 'ip',           :pos => 1, :aggregator => :count,                              :group_by => 'id'
+      add_column :name => 'timestamp',    :pos => 2, :aggregator => :timestamp
+      add_column :name => 'code',         :pos => 4, :aggregator => :count,                              :group_by => 'id'
+      add_column :name => 'length',       :pos => 5, :aggregator => :average,   :conversion => :integer
+      add_column :name => 'length_by_ip', :pos => 5, :aggregator => :average,   :conversion => :integer, :group_by => 'ip'
 
       def time
         Time.strptime(self.timestamp, '%d/%b/%Y:%H:%M:%S %z')

data/spec/support/lib/timing_log.rb

@@ -1,15 +1,35 @@
 module LogfileInterval
   module LineParser
     class TimingLog < Base
+      # Line format:
+      # timestamp, ip, controller#action, total_time, bytes, rss
+
+      set_regex /^(\d+),\s*([\d\.]+),\s*(\w+#\w+),\s*(\d+),\s*(\d+),\s*([\d\.]+)$/
+
+      add_column :name => :timestamp,    :pos => 1, :aggregator => :timestamp
+      add_column :name => :ip,           :pos => 2, :aggregator => :count
+      add_column :name => :action,       :pos => 3, :aggregator => :count
+      add_column :name => :total_time,   :pos => 4, :aggregator => :average,   :conversion => :integer
+      add_column :name => :num_bytes,    :pos => 5, :aggregator => :sum,       :conversion => :integer
+      add_column :name => :rss,          :pos => 6, :aggregator => :delta,     :conversion => :float
+
+      def time
+        Time.at(self.timestamp.to_i)
+      end
+    end
+
+    class TimingLogWithGrouping < Base
       # Line format:
       # timestamp, controller#action, total_time, bytes
 
-      set_regex /^(\d+),\s*(\w+#\w+),\s*(\d+),\s*(\d+)$/
+      set_regex /^(\d+),\s*([\d\.]+),\s*(\w+#\w+),\s*(\d+),\s*(\d+),\s*([\d\.]+)$/
 
-      add_column :name => :timestamp,    :pos => 1, :agg_function => :timestamp
-      add_column :name => :action,       :pos => 2, :agg_function => :group
-      add_column :name => :total_time,   :pos => 3, :agg_function => :average,   :conversion => :integer
-      add_column :name => :num_bytes,    :pos => 4, :agg_function => :sum,       :conversion => :integer
+      add_column :name => :timestamp,    :pos => 1, :aggregator => :timestamp
+      add_column :name => :ip_by_action, :pos => 2, :aggregator => :count,     :group_by => :action
+      add_column :name => :action,       :pos => 3, :aggregator => :count,     :group_by => :action
+      add_column :name => :total_time,   :pos => 4, :aggregator => :average,   :group_by => :action, :conversion => :integer
+      add_column :name => :num_bytes,    :pos => 5, :aggregator => :sum,       :group_by => :action, :conversion => :integer
+      add_column :name => :rss,          :pos => 6, :aggregator => :delta,     :group_by => :action, :conversion => :float
 
       def time
         Time.at(self.timestamp.to_i)

data/spec/support/logfiles/access.log.1

@@ -1,3 +1,4 @@
+# invalid line
 66.249.67.176 - - [23/Jun/2013:16:58:00 -0800] "GET /package/core/raring/universe/proposed/openldap HTTP/1.1" 200 185 "-" "Google"
 12.24.48.96 - - [23/Jun/2013:16:59:00 -0800] "GET /package/core/raring/universe/proposed/openldap HTTP/1.1" 200 4555 "-" "Bing)"
 66.249.67.176 - - [23/Jun/2013:17:00:01 -0800] "GET /package/core/raring/universe/proposed/openldap HTTP/1.1" 200 185 "-" "Google"

data/spec/support/logfiles/timing.log

@@ -1,3 +1,3 @@
-1385942280, posts#index,  100, 20000
-1385942340, posts#create, 200, 5000
-1385942400, posts#show,   100, 15000
+1385942280, 192.168.0.10, posts#index,  100, 20000, 50.20
+1385942340, 192.168.0.5, posts#create, 200, 5000,  50.20
+1385942400, 192.168.0.5, posts#show,   100, 15000, 51.00

data/spec/support/logfiles/timing.log.1

@@ -1,3 +1,3 @@
-1385941980, posts#show,   100, 16000
-1385942040, posts#index,  500, 25000
-1385942160, posts#show,   300, 12000
+1385941980, 192.168.0.5, posts#show,   100, 16000, 48.00
+1385942040, 192.168.0.10, posts#index,  500, 25000, 48.20
+1385942160, 192.168.0.5, posts#show,   300, 12000, 49.20

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logfile_interval
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.1.0
 platform: ruby
 authors:
 - Philippe Le Rohellec
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-12-03 00:00:00.000000000 Z
+date: 2013-12-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -66,6 +66,20 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Logfile parser and aggregator
 email:
 - philippe@lerohellec.com
@@ -74,6 +88,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
+- .travis.yml
 - Gemfile
 - Gemfile.lock
 - LICENSE.txt
@@ -108,7 +123,7 @@ files:
 - spec/support/logfiles/access.log.2
 - spec/support/logfiles/timing.log
 - spec/support/logfiles/timing.log.1
-homepage: ''
+homepage: https://github.com/plerohellec/logfile_interval
 licenses:
 - MIT
 metadata: {}