logfile_interval 1.1.1 → 1.1.2

data/spec/lib/aggregator_spec.rb

@@ -0,0 +1,208 @@
+require 'spec_helper'
+
+module LogfileInterval
+  module Aggregator
+    describe Aggregator do
+      it 'finds the aggregator class' do
+        Aggregator.klass(:sum).should == Sum
+        Aggregator.klass(:average).should == Average
+        Aggregator.klass(:count).should == Count
+        Aggregator.klass(:group_and_count).should == GroupAndCount
+        Aggregator.klass(:delta).should == Delta
+      end
+    end
+
+    shared_examples 'an aggregator' do
+      let(:aggregator) { described_class.new }
+
+      [ :add, :value, :values ].each do |method|
+        it "responds to #{method}" do
+          aggregator.should respond_to(method)
+        end
+      end
+
+      context 'values' do
+        context 'with one group' do
+          before :each do
+            aggregator.add(5, :key1)
+          end
+
+          it 'returns a hash' do
+            aggregator.values.should be_a(Hash) unless aggregator.is_a?(Delta)
+          end
+        end
+
+        context 'with several groups' do
+          before :each do
+            aggregator.add(5, :key1)
+            aggregator.add(3, :key2)
+            aggregator.add(3, :key1)
+          end
+
+          it 'returns a hash' do
+            aggregator.values.should be_a(Hash)
+          end
+        end
+
+        context 'with no group' do
+          before :each do
+            aggregator.add(5)
+            aggregator.add(3)
+          end
+
+          it 'returns a numeric' do
+            aggregator.values.should be_a(Numeric) unless aggregator.is_a?(Count)
+          end
+        end
+      end
+    end
+
+    [ Count, Sum, Average, Delta ]. each do |klass|
+      describe klass do
+        it_behaves_like 'an aggregator'
+      end
+    end
+
+
+    describe 'without group_by key' do
+      describe Sum do
+        it 'sums up values' do
+          sum = Sum.new
+          sum.add(3)
+          sum.add(5)
+          sum.value.should == 8
+        end
+      end
+
+      describe Average do
+        it 'averages values' do
+          avg = Average.new
+          avg.add(3)
+          avg.add(5)
+          avg.value.should == 4
+        end
+      end
+
+      describe Delta do
+        it 'averages delta values' do
+          d = Delta.new
+          d.add(1.4)
+          d.add(1.1)
+          d.add(1.0)
+          d.value.round(5).should == 0.2
+        end
+      end
+
+      describe Count do
+        it 'groups values and increment counters' do
+          g = Count.new
+          g.add('200')
+          g.add('500')
+          g.add('301')
+          g.add('200')
+          g.value.should == 4
+        end
+      end
+    end
+
+    describe 'with group_by key' do
+
+      describe Sum do
+        it 'sums up values by key' do
+          sum = Sum.new
+          sum.add(3, :key1)
+          sum.add(5, :key2)
+          sum.add(5, :key1)
+          sum.values.should be_a(Hash)
+          sum.values.size.should == 2
+          sum.value(:key1).should == 8
+          sum.values[:key1].should == 8
+          sum.value(:key2).should == 5
+          sum.values[:key2].should == 5
+        end
+      end
+
+
+      describe Average do
+        it 'averages values by key' do
+          avg = Average.new
+          avg.add(3, :key1)
+          avg.add(5, :key2)
+          avg.add(5, :key1)
+          avg.values.should be_a(Hash)
+          avg.values.size.should == 2
+          avg.value(:key1).should == 4
+          avg.values[:key1].should == 4
+          avg.value(:key2).should == 5
+          avg.values[:key2].should == 5
+        end
+      end
+
+      describe Count do
+        it 'groups values and increment counters' do
+          g = Count.new
+          g.add('200', '200')
+          g.add('500', '500')
+          g.add('301', '301')
+          g.add('200', '200')
+          g.values.should be_a(Hash)
+          g.values.should include({'200' => 2})
+          g.values.should include({'301' => 1})
+          g.values.should include({'500' => 1})
+        end
+      end
+
+      describe GroupAndCount do
+        it 'each yields a key and a hash' do
+          gac = GroupAndCount.new
+          gac.add :key1, :subkey1
+          gac.first.should be_an(Array)
+          gac.first.size.should == 2
+          gac.first[1].should be_a(Hash)
+        end
+
+        context :add do
+          before :each do
+            @gac = GroupAndCount.new
+          end
+
+          it 'requires a group_by argument' do
+            lambda { @gac.add('foo') }.should raise_error ArgumentError
+          end
+
+          it 'counts number of occurence of subkey for key' do
+            @gac.add :key1, :subkey1
+            @gac.add :key1, :subkey2
+            @gac.add :key2, :subkey1
+            @gac.add :key2, :subkey1
+            @gac.add :key2, :subkey3
+
+            @gac.values[:key1][:subkey1].should == 1
+            @gac.values[:key1][:subkey2].should == 1
+            @gac.values[:key2][:subkey1].should == 2
+            @gac.values[:key2][:subkey2].should == 0
+            @gac.values[:key2][:subkey3].should == 1
+          end
+        end
+      end
+
+      describe Delta do
+        it 'averages deltas by key' do
+          d = Delta.new
+          d.add(9, :key1)
+          d.add(10, :key2)
+          d.add(5, :key1)
+          d.add(8, :key2)
+          d.add(3, :key1)
+          d.add(5, :key2)
+          d.values.should be_a(Hash)
+          d.values.size.should == 2
+          d.value(:key1).should == 3
+          d.values[:key1].should == 3
+          d.value(:key2).should == 2.5
+          d.values[:key2].should == 2.5
+        end
+      end
+    end
+  end
+end

data/spec/lib/{line_parser/counter_spec.rb → counter_spec.rb}

@@ -1,7 +1,7 @@
 require 'spec_helper'
 
 module LogfileInterval
-  module LineParser
+  module Util
     describe Counter do
       it 'behaves like a hash' do
         c = Counter.new

data/spec/lib/interval_builder_spec.rb

@@ -47,6 +47,14 @@ module LogfileInterval
           @intervals.last[:action].should == 2
         end
       end
+
+      context 'without a block' do
+        it 'should return an iterator' do
+          e = @builder.each_interval
+          e.should be_an(Enumerator)
+          e.next.end_time.should == Time.new(2013,12,01,16,0,0,'-08:00')
+        end
+      end
     end
 
     context :last_interval do

data/spec/lib/interval_spec.rb

@@ -5,9 +5,13 @@ module LogfileInterval
   data_dir = File.join(File.dirname(__FILE__), '..', 'support/logfiles')
 
   describe Interval do
+    before :each do
+      @end_time = Time.new(2013, 12, 01, 16, 00, 00, '-08:00')
+      @length = 300
+    end
+
     it 'gets instantiated with empty data' do
-      end_time = Time.new(2013, 12, 01, 16, 00, 00, '-08:00')
-      interval = Interval.new(end_time, 300, LineParser::TimingLog)
+      interval = Interval.new(@end_time, @length, LineParser::TimingLog)
       interval.size.should == 0
       interval[:total_time].should == 0
       interval[:num_bytes].should == 0
@@ -15,12 +19,32 @@ module LogfileInterval
       interval[:ip].should == 0
     end
 
-    context :add_record do
-      before :each do
-        @end_time = Time.new(2013, 12, 01, 16, 00, 00, '-08:00')
-        @length = 300
+    context :to_hash do
+      it 'returns a hash' do
+        interval = Interval.new(@end_time, @length, LineParser::TimingLog)
+        interval.to_hash.should be_a(Hash)
+      end
+
+      it 'has a key for all columns' do
+        record   = LineParser::TimingLog.create_record('1385942400, 192.168.0.5, posts#index, 100, 2000, 53.0')
+        interval = Interval.new(@end_time, @length, LineParser::TimingLog)
+        interval.add_record(record)
+        hinterval = interval.to_hash
+        hinterval.keys.should include(:ip, :total_time, :action, :num_bytes, :rss)
       end
 
+      it 'with no data, should have keys with 0 values' do
+        interval = Interval.new(@end_time, @length, LineParser::TimingLog)
+        hinterval = interval.to_hash
+        hinterval[:ip].should == 0
+        hinterval[:action].should == 0
+        hinterval[:total_time].should == 0
+        hinterval[:num_bytes].should == 0
+        hinterval[:rss].should == 0
+      end
+    end
+
+    context :add_record do
       context 'basics' do
         before :each do
           @interval = Interval.new(@end_time, @length, LineParser::TimingLog)
@@ -50,7 +74,7 @@ module LogfileInterval
 
       context 'with count and group by options' do
         it 'creates an aggregator of type GroupAndCount' do
-          expect(LineParser::Aggregator::GroupAndCount).to receive(:new)
+          expect(Aggregator::GroupAndCount).to receive(:new)
           interval = Interval.new(@end_time, @length, LineParser::TimingLogWithGrouping)
         end
 

data/spec/lib/logfile_set_spec.rb

@@ -8,32 +8,57 @@ module LogfileInterval
     before :each do
       @logfiles = ["#{data_dir}/access.log.2", "#{data_dir}/access.log.1"]
       @set = LogfileSet.new(@logfiles, LineParser::AccessLog)
+      @first_line = '66.249.67.176 - - [23/Jun/2013:17:00:01 -0800] "GET /package/core/raring/universe/proposed/openldap HTTP/1.1" 200 185 "-" "Google"'
+      @second_line = '12.24.48.96 - - [23/Jun/2013:16:59:00 -0800] "GET /package/core/raring/universe/proposed/openldap HTTP/1.1" 200 4555 "-" "Bing)"'
+      @last_line  = '12.24.48.96 - - [23/Jun/2013:16:49:00 -0800] "GET /package/core/raring/universe/proposed/bash HTTP/1.1" 200 4555 "-" "Bing)"'
     end
 
     it 'ordered_filenames should return the most recent file first' do
       @set.ordered_filenames.should == @logfiles.reverse
     end
 
-    it 'each_line should enumerate each line in file backwards' do
-      lines = []
-      @set.each_line do |line|
-        lines << line
+    describe :each_line do
+      it 'should enumerate each line in file backwards' do
+        lines = []
+        @set.each_line do |line|
+          lines << line
+        end
+
+        lines.first.should == @first_line
+        lines.last.should  == @last_line
       end
 
-      lines.first.should == '66.249.67.176 - - [23/Jun/2013:17:00:01 -0800] "GET /package/core/raring/universe/proposed/openldap HTTP/1.1" 200 185 "-" "Google"'
-      lines.last.should == '12.24.48.96 - - [23/Jun/2013:16:49:00 -0800] "GET /package/core/raring/universe/proposed/bash HTTP/1.1" 200 4555 "-" "Bing)"'
+      context 'without a block' do
+        it 'should return an enumerator' do
+          e = @set.each_line
+          e.should be_a(Enumerator)
+          e.first.should == @first_line
+          e.next.should  == @first_line
+          e.next.should  == @second_line
+        end
+      end
     end
 
-    it 'each_parsed_line should enumerate each line backwards' do
-      records = []
-      @set.each_parsed_line do |record|
-        records << record
+    describe :each_parsed_line do
+      it 'should enumerate each line backwards' do
+        records = []
+        @set.each_parsed_line do |record|
+          records << record
+        end
+
+        records.first.time.should == Time.new(2013, 06, 23, 17, 00, 01, '-08:00')
+        records.first.code.should == '200'
+        records.last.time.should  == Time.new(2013, 06, 23, 16, 49, 00, '-08:00')
+        records.last.code.should == '200'
       end
 
-      records.first.time.should == Time.new(2013, 06, 23, 17, 00, 01, '-08:00')
-      records.first.code.should == '200'
-      records.last.time.should  == Time.new(2013, 06, 23, 16, 49, 00, '-08:00')
-      records.last.code.should == '200'
+      context 'without a block' do
+        it 'should return an enumerator' do
+          e = @set.each_parsed_line
+          e.should be_a(Enumerator)
+          e.next.time.should == Time.new(2013, 06, 23, 17, 00, 01, '-08:00')
+        end
+      end
     end
   end
 end

data/spec/lib/logfile_spec.rb

@@ -7,6 +7,9 @@ module LogfileInterval
   describe Logfile do
     before :each do
       @alf = Logfile.new("#{data_dir}/access.log", LineParser::AccessLog)
+      @first_line = '78.54.172.146 - - [01/Jan/2012:16:30:51 -0800] "GET /package/core/oneiric/main/base/abrowser-6.0  HTTP/1.1" 200 6801 "http://www.google.com/url?sa=t&rct=j&q=abrowser 6.0&esrc=s&source=web&cd=4&sqi=2&ved=0CDYQFjAD&url=http%3A%2F%2Fwww.ubuntuupdates.org%2Fpackages%2Fshow%2F268762&ei=s-QlT8vJFon1sgb54unBDg&usg=AFQjCNHCHC0bxTf6aXAfUwT6Erjta6WLaQ&sig2=ceCi1odtaB8Vcf6IWg2a3w" "Mozilla/5.0 (Ubuntu; X11; Linux x86_64; rv:9.0.1) Gecko/20100101 Firefox/9.0.1"'
+      @second_line = '78.54.172.146 - - [01/Jan/2012:16:30:51 -0800] "GET /package/show/2  HTTP/1.1" 302 6801 "http://www.google.com/url?sa=t&rct=j&q=abrowser 6.0&esrc=s&source=web&cd=4&sqi=2&ved=0CDYQFjAD&url=http%3A%2F%2Fwww.ubuntuupdates.org%2Fpackages%2Fshow%2F268762&ei=s-QlT8vJFon1sgb54unBDg&usg=AFQjCNHCHC0bxTf6aXAfUwT6Erjta6WLaQ&sig2=ceCi1odtaB8Vcf6IWg2a3w" "Mozilla/5.0 (Ubuntu; X11; Linux x86_64; rv:9.0.1) Gecko/20100101 Firefox/9.0.1"'
+      @last_line = '66.249.67.176 - - [01/Jan/2012:00:57:47 -0800] "GET /packages/show/1 HTTP/1.1" 301 185 "-" "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"'
     end
 
     it 'first_timestamp returns time of first line in file' do
@@ -14,30 +17,52 @@ module LogfileInterval
       @alf.first_timestamp.should == Time.new(2012, 01, 01, 00, 57, 47, '-08:00')
     end
 
-    it 'each_line should enumerate each line in file backwards' do
-      lines = []
-      @alf.each_line do |line|
-        lines << line
+    describe :each_line do
+      it 'should enumerate each line in file backwards' do
+        lines = []
+        @alf.each_line do |line|
+          lines << line
+        end
+
+        lines.first.should == @first_line
+        lines.last.should  == @last_line
       end
 
-      lines.first.should == '78.54.172.146 - - [01/Jan/2012:16:30:51 -0800] "GET /package/core/oneiric/main/base/abrowser-6.0  HTTP/1.1" 200 6801 "http://www.google.com/url?sa=t&rct=j&q=abrowser 6.0&esrc=s&source=web&cd=4&sqi=2&ved=0CDYQFjAD&url=http%3A%2F%2Fwww.ubuntuupdates.org%2Fpackages%2Fshow%2F268762&ei=s-QlT8vJFon1sgb54unBDg&usg=AFQjCNHCHC0bxTf6aXAfUwT6Erjta6WLaQ&sig2=ceCi1odtaB8Vcf6IWg2a3w" "Mozilla/5.0 (Ubuntu; X11; Linux x86_64; rv:9.0.1) Gecko/20100101 Firefox/9.0.1"'
-      lines.last.should == '66.249.67.176 - - [01/Jan/2012:00:57:47 -0800] "GET /packages/show/1 HTTP/1.1" 301 185 "-" "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"'
+      context 'without a block' do
+        it 'should return an enumerator' do
+          e = @alf.each_line
+          e.should be_a(Enumerator)
+          e.first.should == @first_line
+          e.next.should  == @first_line
+          e.next.should  == @second_line
+        end
+      end
     end
 
-    it 'each_parsed_line should enumerate each line backwards' do
-      records = []
-      @alf.each_parsed_line do |record|
-        records << record
+    describe :each_parsed_line do
+      it 'should enumerate each line backwards' do
+        records = []
+        @alf.each_parsed_line do |record|
+          records << record
+        end
+
+        records.first.time.should == Time.new(2012, 01, 01, 16, 30, 51, '-08:00')
+        records.first.code.should == '200'
+        records.first.length.should == 6801
+        records.first.length_by_ip.should == 6801
+        records.last.time.should  == Time.new(2012, 01, 01, 00, 57, 47, '-08:00')
+        records.last.code.should == '301'
+        records.last.length.should == 185
+        records.last.length_by_ip.should == 185
       end
 
-      records.first.time.should == Time.new(2012, 01, 01, 16, 30, 51, '-08:00')
-      records.first.code.should == '200'
-      records.first.length.should == 6801
-      records.first.length_by_ip.should == 6801
-      records.last.time.should  == Time.new(2012, 01, 01, 00, 57, 47, '-08:00')
-      records.last.code.should == '301'
-      records.last.length.should == 185
-      records.last.length_by_ip.should == 185
+      context 'without a block' do
+        it 'should return an enumerator' do
+          e = @alf.each_parsed_line
+          e.should be_a(Enumerator)
+          e.next.time.should == Time.new(2012, 01, 01, 16, 30, 51, '-08:00')
+        end
+      end
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logfile_interval
 version: !ruby/object:Gem::Version
-  version: 1.1.1
+  version: 1.1.2
 platform: ruby
 authors:
 - Philippe Le Rohellec
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-01-03 00:00:00.000000000 Z
+date: 2014-01-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -99,25 +99,28 @@ files:
 - bin/aggregate_access_log.rb
 - bin/readme.rb
 - docs/design.rb
-- docs/design2.rb
-- docs/design3.rb
 - lib/logfile_interval.rb
-- lib/logfile_interval/file_backward.rb
+- lib/logfile_interval/aggregator.rb
+- lib/logfile_interval/aggregator/average.rb
+- lib/logfile_interval/aggregator/base.rb
+- lib/logfile_interval/aggregator/count.rb
+- lib/logfile_interval/aggregator/delta.rb
+- lib/logfile_interval/aggregator/group_and_count.rb
+- lib/logfile_interval/aggregator/sum.rb
 - lib/logfile_interval/interval.rb
 - lib/logfile_interval/interval_builder.rb
-- lib/logfile_interval/interval_length.rb
-- lib/logfile_interval/line_parser/aggregator.rb
 - lib/logfile_interval/line_parser/base.rb
-- lib/logfile_interval/line_parser/counter.rb
 - lib/logfile_interval/logfile.rb
 - lib/logfile_interval/logfile_set.rb
+- lib/logfile_interval/util/counter.rb
+- lib/logfile_interval/util/file_backward.rb
 - lib/logfile_interval/version.rb
 - logfile_interval.gemspec
+- spec/lib/aggregator_spec.rb
+- spec/lib/counter_spec.rb
 - spec/lib/interval_builder_spec.rb
 - spec/lib/interval_spec.rb
-- spec/lib/line_parser/aggregator_spec.rb
 - spec/lib/line_parser/base_spec.rb
-- spec/lib/line_parser/counter_spec.rb
 - spec/lib/logfile_set_spec.rb
 - spec/lib/logfile_spec.rb
 - spec/spec_helper.rb
@@ -154,11 +157,11 @@ signing_key:
 specification_version: 4
 summary: Aggregate logfile data into intervals
 test_files:
+- spec/lib/aggregator_spec.rb
+- spec/lib/counter_spec.rb
 - spec/lib/interval_builder_spec.rb
 - spec/lib/interval_spec.rb
-- spec/lib/line_parser/aggregator_spec.rb
 - spec/lib/line_parser/base_spec.rb
-- spec/lib/line_parser/counter_spec.rb
 - spec/lib/logfile_set_spec.rb
 - spec/lib/logfile_spec.rb
 - spec/spec_helper.rb

data/docs/design2.rb

@@ -1,77 +0,0 @@
-module LogfileInterval
-  class Logfile
-  end
-
-  class LogfileSet
-  end
-
-  class Interval
-  end
-
-  class LineParser
-    def initialize(regex, columns)
-    end
-
-    def parse(line)
-      @data = {}
-
-      match_data = regex.match(line)
-      columns.each do |name, options|
-        val = match_data[options[:pos]]
-        @data[name] = convert(val, options[:conversion])
-      end
-      @data
-    end
-
-    def convert(val, conversion)
-      case options[:conversion]
-      when :integer then val.to_i
-      else val
-      end
-    end
-  end
-
-  class Record
-    def initialize(parser, line)
-      @parser = parser
-      @data = parser.parse(line)
-    end
-
-    def valid_columns
-      @parser.columns.keys
-    end
-
-    def method_missing(meth, *args)
-      if valid_columns.include?(meth) && args.none
-        self[meth]
-      else
-        super
-      end
-    end
-  end
-end
-
-logfiles = [ 'access.log', 'access.log.1', 'access.log.2' ]
-logfile = logfiles.first
-parser = LineParser.new(/blah/, [ { :name=>:ip , :pos => 1 }, ...])
-
-logfile_iterator = LogfileInterval::Logfile.new(parser, logfile)
-
-logfile_iterator.each_line do |line|
-  puts line
-end
-
-logfile_iterator.each_parsed_line do |record|
-  puts record.ip
-  puts record.time
-end
-
-interval_builder = LogfileInterval::Interval.new(parser, logfiles)
-
-interval_builder.each_interval do |interval|
-  puts interval.start_time
-  puts interval.length
-  interval[:ip].each do |ip, count|
-    puts "#{ip}, #{count}"
-  end
-end