logfile_interval 1.2.1 → 2.0.0

data/spec/lib/interval_spec.rb

@@ -11,7 +11,7 @@ module LogfileInterval
     end
 
     it 'gets instantiated with empty data' do
-      interval = Interval.new(@end_time, @length, LineParser::TimingLog)
+      interval = Interval.new(@end_time, @length, ParsedLine::TimingLog.columns)
       interval.size.should == 0
       interval[:total_time].should == 0
       interval[:num_bytes].should == 0
@@ -21,21 +21,22 @@ module LogfileInterval
 
     context :to_hash do
       it 'returns a hash' do
-        interval = Interval.new(@end_time, @length, LineParser::TimingLog)
+        interval = Interval.new(@end_time, @length, ParsedLine::TimingLog)
         interval.to_hash.should be_a(Hash)
       end
 
       it 'has a key for all columns' do
-        record   = LineParser::TimingLog.create_record('1385942400, 192.168.0.5, posts#index, 100, 2000, 53.0')
-        interval = Interval.new(@end_time, @length, LineParser::TimingLog)
+        record   = ParsedLine::TimingLog.create_record('1385942400, 192.168.0.5, posts#index, 100, 2000, 53.0')
+        interval = Interval.new(@end_time, @length, ParsedLine::TimingLog.columns)
         interval.add_record(record)
         hinterval = interval.to_hash
         hinterval.keys.should include(:ip, :total_time, :action, :num_bytes, :rss)
       end
 
       it 'with no data, should have keys with 0 values' do
-        interval = Interval.new(@end_time, @length, LineParser::TimingLog)
+        interval = Interval.new(@end_time, @length, ParsedLine::TimingLog.columns)
         hinterval = interval.to_hash
+        hinterval[:num_lines].should == 0
         hinterval[:ip].should == 0
         hinterval[:action].should == 0
         hinterval[:total_time].should == 0
@@ -47,40 +48,46 @@ module LogfileInterval
     context :add_record do
       context 'basics' do
         before :each do
-          @interval = Interval.new(@end_time, @length, LineParser::TimingLog)
+          @interval = Interval.new(@end_time, @length, ParsedLine::TimingLog.columns)
         end
 
         it 'rejects record out of interval' do
-          oor_record = LineParser::TimingLog.create_record('1385942450, 192.168.0.5, posts#index, 100, 20000, 50.0')
+          oor_record = ParsedLine::TimingLog.create_record('1385942450, 192.168.0.5, posts#index, 100, 20000, 50.0')
           lambda { @interval.add_record(oor_record) }.should raise_error(Interval::OutOfRange)
         end
 
         it 'rejects record at interval start_time' do
-          oor_record = LineParser::TimingLog.create_record('1385942100, 192.168.0.5, posts#index, 100, 20000, 50.0')
+          oor_record = ParsedLine::TimingLog.create_record('1385942100, 192.168.0.5, posts#index, 100, 20000, 50.0')
           lambda { @interval.add_record(oor_record) }.should raise_error(Interval::OutOfRange)
         end
 
+        it 'accepts record at interval end_time' do
+          oor_record = ParsedLine::TimingLog.create_record('1385942400, 192.168.0.5, posts#index, 100, 20000, 50.0')
+          lambda { @interval.add_record(oor_record) }.should_not raise_error
+        end
+
         it 'adds 1 record to interval' do
-          record1 = LineParser::TimingLog.create_record('1385942400, 192.168.0.5, posts#index, 100, 20000, 50.0')
+          record1 = ParsedLine::TimingLog.create_record('1385942400, 192.168.0.5, posts#index, 100, 20000, 50.0')
           @interval.add_record(record1)
 
           @interval.size.should == 1
+          @interval[:num_lines].should == 1
           @interval[:total_time].should == 100
           @interval[:num_bytes].should == 20000
-          @interval[:action].should == 1
-          @interval[:ip].should == 1
+          @interval[:action].should == {"posts#index"=>1}
+          @interval[:ip].should == {"192.168.0.5"=>1}
         end
       end
 
       context 'with count and group by options' do
-        it 'creates an aggregator of type GroupAndCount' do
-          expect(Aggregator::GroupAndCount).to receive(:new)
-          interval = Interval.new(@end_time, @length, LineParser::TimingLogWithGrouping)
+        it 'creates an aggregator of type Count' do
+          expect(Aggregator::Count).to receive(:new).twice
+          interval = Interval.new(@end_time, @length, ParsedLine::TimingLogWithGrouping.columns)
         end
 
         it 'add_record accepts key and subkey' do
-          interval = Interval.new(@end_time, @length, LineParser::TimingLogWithGrouping)
-          record1 = LineParser::TimingLogWithGrouping.create_record('1385942400, 192.168.0.5, posts#index, 100, 20000, 53.0')
+          interval = Interval.new(@end_time, @length, ParsedLine::TimingLogWithGrouping.columns)
+          record1 = ParsedLine::TimingLogWithGrouping.create_record('1385942400, 192.168.0.5, posts#index, 100, 20000, 53.0')
           interval.add_record(record1)
           interval.size.should == 1
         end
@@ -88,13 +95,13 @@ module LogfileInterval
 
       context 'with 3 records' do
         before :each do
-          @interval = Interval.new(@end_time, @length, LineParser::TimingLog)
+          @interval = Interval.new(@end_time, @length, ParsedLine::TimingLog.columns)
 
-          record1 = LineParser::TimingLog.create_record('1385942400, 192.168.0.5, posts#index, 100, 20000, 53.0')
+          record1 = ParsedLine::TimingLog.create_record('1385942400, 192.168.0.5, posts#index, 100, 20000, 53.0')
           @interval.add_record(record1)
-          record2 = LineParser::TimingLog.create_record('1385942300, 192.168.0.5, posts#show, 50, 10000, 51.0')
+          record2 = ParsedLine::TimingLog.create_record('1385942300, 192.168.0.5, posts#show, 50, 10000, 51.0')
           @interval.add_record(record2)
-          record3 = LineParser::TimingLog.create_record('1385942200, 10.10.10.10, posts#show, 60, 12000, 50.0')
+          record3 = ParsedLine::TimingLog.create_record('1385942200, 10.10.10.10, posts#show, 60, 12000, 50.0')
           @interval.add_record(record3)
         end
 
@@ -102,6 +109,10 @@ module LogfileInterval
           @interval.size.should == 3
         end
 
+        it 'counts the number of lines with the num_lines aggregator' do
+          @interval[:num_lines].should == 3
+        end
+
         it 'averages columns with average aggregator' do
           @interval[:total_time].should == 70
         end
@@ -111,26 +122,26 @@ module LogfileInterval
         end
 
         it 'averages the delta columns with delta aggregator' do
-          @interval[:rss].should == 1.5
+          @interval[:rss].should == -1.5
         end
 
         it 'counts columns with group aggregator' do
-          @interval[:ip].should == 3
-          @interval[:action].should == 3
+          @interval[:ip].should ==  { '192.168.0.5' => 2, '10.10.10.10' => 1 }
+          @interval[:action].should == { 'posts#index' => 1, 'posts#show' => 2}
         end
       end
 
       context 'with group_by key' do
         before :each do
-          @interval = Interval.new(@end_time, @length, LineParser::TimingLogWithGrouping)
+          @interval = Interval.new(@end_time, @length, ParsedLine::TimingLogWithGrouping.columns)
 
-          record1 = LineParser::TimingLogWithGrouping.create_record('1385942400, 192.168.0.5, posts#index, 100, 20000, 53.0')
+          record1 = ParsedLine::TimingLogWithGrouping.create_record('1385942400, 192.168.0.5, posts#index, 100, 20000, 53.0')
           @interval.add_record(record1)
-          record2 = LineParser::TimingLogWithGrouping.create_record('1385942300, 192.168.0.5, posts#show, 50, 10000, 51.0')
+          record2 = ParsedLine::TimingLogWithGrouping.create_record('1385942300, 192.168.0.5, posts#show, 50, 10000, 51.0')
           @interval.add_record(record2)
-          record3 = LineParser::TimingLogWithGrouping.create_record('1385942200, 192.168.0.5, posts#show, 60, 12000, 50.0')
+          record3 = ParsedLine::TimingLogWithGrouping.create_record('1385942200, 192.168.0.5, posts#show, 60, 12000, 50.0')
           @interval.add_record(record3)
-          record4 = LineParser::TimingLogWithGrouping.create_record('1385942180, 10.10.10.10, posts#index, 100, 20000, 48.0')
+          record4 = ParsedLine::TimingLogWithGrouping.create_record('1385942180, 10.10.10.10, posts#index, 100, 20000, 48.0')
           @interval.add_record(record4)
         end
 
@@ -166,8 +177,8 @@ module LogfileInterval
         it 'averages deltas on value column per group column' do
           @interval[:rss].should be_a(Hash)
           @interval[:rss].size.should == 2
-          @interval[:rss]['posts#index'].should == 5
-          @interval[:rss]['posts#show'].should  == 1
+          @interval[:rss]['posts#index'].should == -5
+          @interval[:rss]['posts#show'].should  == -1
         end
       end
     end

data/spec/lib/line_parser/base_spec.rb

@@ -3,7 +3,7 @@ require 'spec_helper'
 module LogfileInterval
   data_dir = File.join(File.dirname(__FILE__), '..', 'support/logfiles')
 
-  module LineParser
+  module ParsedLine
 
     describe Base do
       before :each do
@@ -25,6 +25,21 @@ module LogfileInterval
         record.valid?.should be_false
       end
 
+      describe 'class' do
+        subject { AccessLog }
+
+        it { should respond_to :each }
+
+        describe '#each' do
+          it 'iterates over columns' do
+            AccessLog.each do |col|
+              col.first.should be_a(Symbol)
+              col.last.should be_a(Hash)
+            end
+          end
+        end
+    end
+
       context :create_record do
 
         it 'instanciates a new AccessLog object' do
@@ -65,10 +80,6 @@ module LogfileInterval
       it 'must fail unless a column is configured'do
         lambda { NoColumnLog.new(@line) }.should raise_error ConfigurationError
       end
-
-      it 'must fail with custom aggregator but no custom class' do
-        lambda { MissingCustomClass.add_column(:name => 'ip', :pos => 1, :aggregator => :custom) }.should raise_error ConfigurationError
-      end
     end
 
     describe TimingLog do

data/spec/lib/logfile_set_spec.rb

@@ -7,7 +7,7 @@ module LogfileInterval
   describe LogfileSet do
     before :each do
       @logfiles = ["#{data_dir}/access.log.2", "#{data_dir}/access.log.1"]
-      @set = LogfileSet.new(@logfiles, LineParser::AccessLog)
+      @set = LogfileSet.new(@logfiles, ParsedLine::AccessLog)
       @first_line = '66.249.67.176 - - [23/Jun/2013:17:00:01 -0800] "GET /package/core/raring/universe/proposed/openldap HTTP/1.1" 200 185 "-" "Google"'
       @second_line = '12.24.48.96 - - [23/Jun/2013:16:59:00 -0800] "GET /package/core/raring/universe/proposed/openldap HTTP/1.1" 200 4555 "-" "Bing)"'
       @last_line  = '12.24.48.96 - - [23/Jun/2013:16:49:00 -0800] "GET /package/core/raring/universe/proposed/bash HTTP/1.1" 200 4555 "-" "Bing)"'
@@ -37,6 +37,28 @@ module LogfileInterval
           e.next.should  == @second_line
         end
       end
+
+      context :order do
+        it 'iterates backward when order is :desc' do
+          lines = []
+          set = LogfileSet.new(@logfiles, ParsedLine::AccessLog, :desc)
+          set.each_line do |line|
+            lines << line
+          end
+          lines.last.should  == @last_line
+          lines.first.should == @first_line
+        end
+
+        it 'iterates upward when order is :asc' do
+          lines = []
+          set = LogfileSet.new(@logfiles, ParsedLine::AccessLog, :asc)
+          set.each_line do |line|
+            lines << line
+          end
+          lines.first.should == @last_line
+          lines.last.should  == @first_line
+        end
+      end
     end
 
     describe :each_parsed_line do

data/spec/lib/logfile_spec.rb

@@ -6,7 +6,7 @@ module LogfileInterval
 
   describe Logfile do
     before :each do
-      @alf = Logfile.new("#{data_dir}/access.log", LineParser::AccessLog)
+      @alf = Logfile.new("#{data_dir}/access.log", ParsedLine::AccessLog)
       @first_line = '78.54.172.146 - - [01/Jan/2012:16:30:51 -0800] "GET /package/core/oneiric/main/base/abrowser-6.0  HTTP/1.1" 200 6801 "http://www.google.com/url?sa=t&rct=j&q=abrowser 6.0&esrc=s&source=web&cd=4&sqi=2&ved=0CDYQFjAD&url=http%3A%2F%2Fwww.ubuntuupdates.org%2Fpackages%2Fshow%2F268762&ei=s-QlT8vJFon1sgb54unBDg&usg=AFQjCNHCHC0bxTf6aXAfUwT6Erjta6WLaQ&sig2=ceCi1odtaB8Vcf6IWg2a3w" "Mozilla/5.0 (Ubuntu; X11; Linux x86_64; rv:9.0.1) Gecko/20100101 Firefox/9.0.1"'
       @second_line = '78.54.172.146 - - [01/Jan/2012:16:30:51 -0800] "GET /package/show/2  HTTP/1.1" 302 6801 "http://www.google.com/url?sa=t&rct=j&q=abrowser 6.0&esrc=s&source=web&cd=4&sqi=2&ved=0CDYQFjAD&url=http%3A%2F%2Fwww.ubuntuupdates.org%2Fpackages%2Fshow%2F268762&ei=s-QlT8vJFon1sgb54unBDg&usg=AFQjCNHCHC0bxTf6aXAfUwT6Erjta6WLaQ&sig2=ceCi1odtaB8Vcf6IWg2a3w" "Mozilla/5.0 (Ubuntu; X11; Linux x86_64; rv:9.0.1) Gecko/20100101 Firefox/9.0.1"'
       @last_line = '66.249.67.176 - - [01/Jan/2012:00:57:47 -0800] "GET /packages/show/1 HTTP/1.1" 301 185 "-" "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"'
@@ -37,6 +37,28 @@ module LogfileInterval
           e.next.should  == @second_line
         end
       end
+
+      context :order do
+        it 'iterates backward when order is :desc' do
+          lines = []
+          lf = Logfile.new("#{data_dir}/access.log", ParsedLine::AccessLog, :desc)
+          lf.each_line do |line|
+            lines << line
+          end
+          lines.last.should  == @last_line
+          lines.first.should == @first_line
+        end
+
+        it 'iterates upward when order is :asc' do
+          lines = []
+          lf = Logfile.new("#{data_dir}/access.log", ParsedLine::AccessLog, :asc)
+          lf.each_line do |line|
+            lines << line
+          end
+          lines.first.should == @last_line
+          lines.last.should  == @first_line
+        end
+      end
     end
 
     describe :each_parsed_line do

data/spec/support/lib/access_log.rb

@@ -1,7 +1,7 @@
 module LogfileInterval
   data_dir = File.join(File.dirname(__FILE__), '..', 'support/logfiles')
 
-  module LineParser
+  module ParsedLine
 
     class AccessLog < Base
       # Example line:

data/spec/support/lib/custom_timing_log.rb

@@ -12,7 +12,7 @@ module LogfileInterval
     end
   end
 
-  module LineParser
+  module ParsedLine
     class CustomTimingLog < Base
       # Line format:
       # timestamp, ip, controller#action, total_time, bytes, rss
@@ -21,8 +21,8 @@ module LogfileInterval
 
       add_column :name => :timestamp,    :pos => 1, :aggregator => :timestamp
       add_column :name => :ip,           :pos => 2, :aggregator => :count
-      add_column :name => :num_slow,     :pos => 4, :aggregator => :custom,    :conversion => :integer,
-                 :custom_class => Aggregator::CountOverThreshold, :custom_options => { :threshold => 100 }
+      add_column :name => :num_slow,     :pos => 4, :aggregator => :count_over_threshold,    :conversion => :integer,
+                 :custom_options => { :threshold => 100 }
 
       def time
         Time.at(self.timestamp.to_i)

data/spec/support/lib/timing_log.rb

@@ -1,5 +1,5 @@
 module LogfileInterval
-  module LineParser
+  module ParsedLine
     class TimingLog < Base
       # Line format:
       # timestamp, ip, controller#action, total_time, bytes, rss
@@ -7,6 +7,7 @@ module LogfileInterval
       set_regex /^(\d+),\s*([\d\.]+),\s*(\w+#\w+),\s*(\d+),\s*(\d+),\s*([\d\.]+)$/
 
       add_column :name => :timestamp,    :pos => 1, :aggregator => :timestamp
+      add_column :name => :num_lines,    :pos => 2, :aggregator => :num_lines
       add_column :name => :ip,           :pos => 2, :aggregator => :count
       add_column :name => :action,       :pos => 3, :aggregator => :count
       add_column :name => :total_time,   :pos => 4, :aggregator => :average,   :conversion => :integer
@@ -25,8 +26,9 @@ module LogfileInterval
       set_regex /^(\d+),\s*([\d\.]+),\s*(\w+#\w+),\s*(\d+),\s*(\d+),\s*([\d\.]+)$/
 
       add_column :name => :timestamp,    :pos => 1, :aggregator => :timestamp
+      add_column :name => :num_lines,    :pos => 2, :aggregator => :num_lines
       add_column :name => :ip_by_action, :pos => 2, :aggregator => :count,     :group_by => :action
-      add_column :name => :action,       :pos => 3, :aggregator => :count,     :group_by => :action
+      add_column :name => :action,       :pos => 3, :aggregator => :count
       add_column :name => :total_time,   :pos => 4, :aggregator => :average,   :group_by => :action, :conversion => :integer
       add_column :name => :num_bytes,    :pos => 5, :aggregator => :sum,       :group_by => :action, :conversion => :integer
       add_column :name => :rss,          :pos => 6, :aggregator => :delta,     :group_by => :action, :conversion => :float

data/spec/support/logfiles/timing.log.2

@@ -0,0 +1 @@
+1385941440, 192.168.0.5, posts#show,   100, 16000, 48.00

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logfile_interval
 version: !ruby/object:Gem::Version
-  version: 1.2.1
+  version: 2.0.0
 platform: ruby
 authors:
 - Philippe Le Rohellec
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-01-13 00:00:00.000000000 Z
+date: 2014-03-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -100,22 +100,27 @@ files:
 - bin/readme.rb
 - docs/design.rb
 - lib/logfile_interval.rb
-- lib/logfile_interval/aggregator.rb
 - lib/logfile_interval/aggregator/average.rb
 - lib/logfile_interval/aggregator/base.rb
 - lib/logfile_interval/aggregator/count.rb
 - lib/logfile_interval/aggregator/delta.rb
-- lib/logfile_interval/aggregator/group_and_count.rb
+- lib/logfile_interval/aggregator/num_lines.rb
+- lib/logfile_interval/aggregator/registrar.rb
 - lib/logfile_interval/aggregator/sum.rb
+- lib/logfile_interval/aggregator_set.rb
 - lib/logfile_interval/interval.rb
 - lib/logfile_interval/interval_builder.rb
-- lib/logfile_interval/line_parser/base.rb
+- lib/logfile_interval/interval_builder/ascending.rb
+- lib/logfile_interval/interval_builder/descending.rb
 - lib/logfile_interval/logfile.rb
 - lib/logfile_interval/logfile_set.rb
+- lib/logfile_interval/parsed_line/base.rb
+- lib/logfile_interval/parsed_line/parser.rb
 - lib/logfile_interval/util/counter.rb
 - lib/logfile_interval/util/file_backward.rb
 - lib/logfile_interval/version.rb
 - logfile_interval.gemspec
+- spec/lib/aggregator_set_spec.rb
 - spec/lib/aggregator_spec.rb
 - spec/lib/counter_spec.rb
 - spec/lib/custom_aggregator_spec.rb
@@ -134,6 +139,7 @@ files:
 - spec/support/logfiles/access.log.3
 - spec/support/logfiles/timing.log
 - spec/support/logfiles/timing.log.1
+- spec/support/logfiles/timing.log.2
 homepage: https://github.com/plerohellec/logfile_interval
 licenses:
 - MIT
@@ -159,6 +165,7 @@ signing_key:
 specification_version: 4
 summary: Aggregate logfile data into intervals
 test_files:
+- spec/lib/aggregator_set_spec.rb
 - spec/lib/aggregator_spec.rb
 - spec/lib/counter_spec.rb
 - spec/lib/custom_aggregator_spec.rb
@@ -177,3 +184,4 @@ test_files:
 - spec/support/logfiles/access.log.3
 - spec/support/logfiles/timing.log
 - spec/support/logfiles/timing.log.1
+- spec/support/logfiles/timing.log.2

data/lib/logfile_interval/aggregator/group_and_count.rb

@@ -1,14 +0,0 @@
-module LogfileInterval
-  module Aggregator
-    class GroupAndCount < Base
-      def each
-        @val.each { |k, v| yield k, v }
-      end
-
-      def add(value, group_by)
-        raise ArgumentError, 'group_by argument is mandatory for GroupAndCount#add' unless group_by
-        @val.increment_subkey(value, key(group_by))
-      end
-    end
-  end
-end

data/lib/logfile_interval/aggregator.rb

@@ -1,27 +0,0 @@
-lib_dir = File.expand_path('..', __FILE__)
-
-require "#{lib_dir}/aggregator/base"
-require "#{lib_dir}/aggregator/sum"
-require "#{lib_dir}/aggregator/count"
-require "#{lib_dir}/aggregator/group_and_count"
-require "#{lib_dir}/aggregator/average"
-require "#{lib_dir}/aggregator/delta"
-
-module LogfileInterval
-  module Aggregator
-    def self.klass(options)
-      case options[:aggregator]
-      when :sum               then Sum
-      when :average           then Average
-      when :count
-        if options[:group_by] && options[:group_by] != options[:name]
-          GroupAndCount
-        else
-          Count
-        end
-      when :delta             then Delta
-      when :custom            then options.fetch(:custom_class)
-      end
-    end
-  end
-end