logfile_interval 1.2.1 → 2.0.0

data/lib/logfile_interval/parsed_line/base.rb

@@ -0,0 +1,31 @@
+module LogfileInterval
+  module ParsedLine
+    class ConfigurationError  < StandardError; end
+
+    class Base
+      attr_reader :data
+
+      extend Parser
+
+      def initialize(line)
+        @data = self.class.parse(line)
+        @valid = @data ? true : false
+      end
+
+      def valid?
+        @valid
+      end
+
+      def time
+        raise NotImplemented
+      end
+
+      def [](name)
+        @data[name]
+      end
+    end
+  end
+end
+
+
+

data/lib/logfile_interval/{line_parser/base.rb → parsed_line/parser.rb}

@@ -1,15 +1,9 @@
 module LogfileInterval
-  module LineParser
-    AGGREGATION_FUNCTIONS = [ :sum, :average, :timestamp, :count, :delta, :custom ]
-
+  module ParsedLine
     class ConfigurationError  < StandardError; end
 
-    class Base
-      attr_reader :data
-
-      class << self
+    module Parser
         attr_reader :regex
-
         def columns
           @columns ||= {}
         end
@@ -53,21 +47,22 @@ module LogfileInterval
 
         def set_column_custom_options(column_name, options)
           raise ArgumentError, "Invalid column name: #{column_name}" unless columns.has_key?(column_name)
-          raise ArgumentError, "This column is not custom: #{column_name}" unless columns[column_name].has_key?(:custom_class)
           columns[column_name][:custom_options] = options
         end
 
+
+        def each(&block)
+          columns.each(&block)
+        end
+
         private
 
         def validate_column_options(options)
           validate_option(options, :name)
           validate_option(options, :pos)
           validate_option(options, :aggregator)
-          unless AGGREGATION_FUNCTIONS.include?(options[:aggregator])
-            raise ConfigurationError, "aggregator must be one of #{AGGREGATION_FUNCTIONS.join(', ')}"
-          end
-          if options[:aggregator] == :custom
-            validate_option(options, :custom_class, ':custom_class must be set for :custom aggregator type')
+          unless Aggregator::Base.exist?(options[:aggregator]) || options[:aggregator] == :timestamp
+            raise ConfigurationError, "aggregator must be one of #{Aggregator::Base.all.join(', ')}"
           end
         end
 
@@ -78,13 +73,14 @@ module LogfileInterval
         def sanitize_column_options(options)
           options[:name]       = options[:name].to_sym
           if options.has_key?(:group_by)
-            options[:group_by]   = options[:group_by].to_sym
+            if options[:group_by].to_sym != options[:name]
+              options[:group_by] = options[:group_by].to_sym
+            else
+              options.delete(:group_by)
+            end
           end
           options[:conversion] = options.fetch(:conversion, :string)
-          if options[:aggregator] == :custom
-            options[:custom_options] = options.fetch(:custom_options, {})
-          end
-          options[:aggregator_class] = Aggregator.klass(options)
+          options[:aggregator_class] = Aggregator::Base.klass(options[:aggregator])
           options.delete(:aggregator)
           options
         end
@@ -96,27 +92,6 @@ module LogfileInterval
           else val
           end
         end
-      end
-
-      def initialize(line)
-        @data = self.class.parse(line)
-        @valid = @data ? true : false
-      end
-
-      def valid?
-        @valid
-      end
-
-      def time
-        raise NotImplemented
-      end
-
-      def [](name)
-        @data[name]
-      end
     end
   end
 end
-
-
-

data/lib/logfile_interval/version.rb

@@ -1,3 +1,3 @@
 module LogfileInterval
-  VERSION = "1.2.1"
+  VERSION = "2.0.0"
 end

data/lib/logfile_interval.rb

@@ -1,14 +1,16 @@
 lib_dir = File.expand_path('..', __FILE__)
 
 require "#{lib_dir}/logfile_interval/version"
+require "#{lib_dir}/logfile_interval/aggregator_set"
 require "#{lib_dir}/logfile_interval/interval"
 require "#{lib_dir}/logfile_interval/interval_builder"
 require "#{lib_dir}/logfile_interval/logfile"
 require "#{lib_dir}/logfile_interval/logfile_set"
-require "#{lib_dir}/logfile_interval/line_parser/base"
+require "#{lib_dir}/logfile_interval/parsed_line/parser"
+require "#{lib_dir}/logfile_interval/parsed_line/base"
 require "#{lib_dir}/logfile_interval/util/counter"
 require "#{lib_dir}/logfile_interval/util/file_backward"
-require "#{lib_dir}/logfile_interval/aggregator"
+require "#{lib_dir}/logfile_interval/aggregator/base"
 
 module LogfileInterval
 end

data/spec/lib/aggregator_set_spec.rb

@@ -0,0 +1,15 @@
+require 'spec_helper'
+require File.join(File.dirname(__FILE__), '..', 'support/lib/timing_log')
+
+module LogfileInterval
+  data_dir = File.join(File.dirname(__FILE__), '..', 'support/logfiles')
+
+  describe AggregatorSet, 'with empty columns' do
+    subject { AggregatorSet.new({}) }
+
+    it { should respond_to :add }
+    it { should respond_to :to_hash }
+    it { should respond_to :[] }
+
+  end
+end

data/spec/lib/aggregator_spec.rb

@@ -3,16 +3,21 @@ require 'spec_helper'
 module LogfileInterval
 
   module Aggregator
-    class CustomAggregator; end
+    class CustomAggregator < Base; end
+
+    class BizarroAggregator < Base
+      register_aggregator :weird_add, self
+    end
       
-    describe Aggregator do
+    describe Base do
       it 'finds the aggregator class' do
-        Aggregator.klass({ :aggregator => :sum}).should == Sum
-        Aggregator.klass({ :aggregator => :average}).should == Average
-        Aggregator.klass({ :aggregator => :count}).should == Count
-        Aggregator.klass({ :aggregator => :count, :group_by => :foo}).should == GroupAndCount
-        Aggregator.klass({ :aggregator => :delta}).should == Delta
-        Aggregator.klass({ :aggregator => :custom, :custom_class => CustomAggregator}).should == CustomAggregator
+        Aggregator::Base.klass(:num_lines).should == NumLines
+        Aggregator::Base.klass(:sum).should == Sum
+        Aggregator::Base.klass(:average).should == Average
+        Aggregator::Base.klass(:count).should == Count
+        Aggregator::Base.klass(:delta).should == Delta
+        Aggregator::Base.klass(:custom_aggregator).should == CustomAggregator
+        Aggregator::Base.klass(:weird_add).should == BizarroAggregator
       end
     end
 
@@ -32,7 +37,7 @@ module LogfileInterval
           end
 
           it 'returns a hash' do
-            aggregator.values.should be_a(Hash) unless aggregator.is_a?(Delta)
+            aggregator.values.should be_a(Hash) unless [ Delta, NumLines ].include?(aggregator.class)
           end
         end
 
@@ -61,7 +66,7 @@ module LogfileInterval
       end
     end
 
-    [ Count, Sum, Average, Delta ]. each do |klass|
+    [ NumLines, Count, Sum, Average, Delta ]. each do |klass|
       describe klass do
         it_behaves_like 'an aggregator'
       end
@@ -69,6 +74,16 @@ module LogfileInterval
 
 
     describe 'without group_by key' do
+      describe NumLines do
+        it 'counts total number of lines' do
+          nl = NumLines.new
+          nl.add(55)
+          nl.add(54)
+          nl.add(1008)
+          nl.value.should == 3
+        end
+      end
+
       describe Sum do
         it 'sums up values' do
           sum = Sum.new
@@ -90,9 +105,9 @@ module LogfileInterval
       describe Delta do
         it 'averages delta values' do
           d = Delta.new
-          d.add(1.4)
-          d.add(1.1)
           d.add(1.0)
+          d.add(1.1)
+          d.add(1.4)
           d.value.round(5).should == 0.2
         end
       end
@@ -104,7 +119,8 @@ module LogfileInterval
           g.add('500')
           g.add('301')
           g.add('200')
-          g.value.should == 4
+          g.value.should == 0
+          g.values.size.should == 3
         end
       end
     end
@@ -145,20 +161,18 @@ module LogfileInterval
       describe Count do
         it 'groups values and increment counters' do
           g = Count.new
-          g.add('200', '200')
-          g.add('500', '500')
-          g.add('301', '301')
-          g.add('200', '200')
+          g.add('200')
+          g.add('500')
+          g.add('301')
+          g.add('200')
           g.values.should be_a(Hash)
           g.values.should include({'200' => 2})
           g.values.should include({'301' => 1})
           g.values.should include({'500' => 1})
         end
-      end
 
-      describe GroupAndCount do
         it 'each yields a key and a hash' do
-          gac = GroupAndCount.new
+          gac = Count.new
           gac.add :key1, :subkey1
           gac.first.should be_an(Array)
           gac.first.size.should == 2
@@ -167,11 +181,7 @@ module LogfileInterval
 
         context :add do
           before :each do
-            @gac = GroupAndCount.new
-          end
-
-          it 'requires a group_by argument' do
-            lambda { @gac.add('foo') }.should raise_error ArgumentError
+            @gac = Count.new
           end
 
           it 'counts number of occurence of subkey for key' do
@@ -201,10 +211,10 @@ module LogfileInterval
           d.add(5, :key2)
           d.values.should be_a(Hash)
           d.values.size.should == 2
-          d.value(:key1).should == 3
-          d.values[:key1].should == 3
-          d.value(:key2).should == 2.5
-          d.values[:key2].should == 2.5
+          d.value(:key1).should == -3
+          d.values[:key1].should == -3
+          d.value(:key2).should == -2.5
+          d.values[:key2].should == -2.5
         end
       end
     end

data/spec/lib/custom_aggregator_spec.rb

@@ -6,7 +6,7 @@ module LogfileInterval
     before :each do
       @end_time = Time.new(2013, 12, 01, 16, 00, 00, '-08:00')
       @length = 300
-      @line_parser_class = LineParser::CustomTimingLog
+      @line_parser_class = ParsedLine::CustomTimingLog
     end
 
     def fill_interval
@@ -24,7 +24,7 @@ module LogfileInterval
 
       @interval.size.should == 3
       @interval[:num_slow].should == 1
-      @interval[:ip].should == 3
+      @interval[:ip].should == {"192.168.0.5"=>2, "10.10.10.10"=>1}
     end
 
     describe 'set_column_custom_options' do
@@ -34,7 +34,7 @@ module LogfileInterval
 
         @interval.size.should == 3
         @interval[:num_slow].should == 2
-        @interval[:ip].should == 3
+        @interval[:ip].should == {"192.168.0.5"=>2, "10.10.10.10"=>1}
       end
     end
   end

data/spec/lib/interval_builder_spec.rb

@@ -7,63 +7,210 @@ module LogfileInterval
   describe IntervalBuilder do
     before :each do
       @logfiles = ["#{data_dir}/timing.log", "#{data_dir}/timing.log.1" ]
-      @set = LogfileSet.new(@logfiles, LineParser::TimingLog)
-      @builder = IntervalBuilder.new(@set, 300)
     end
 
-    context :each_interval do
+    describe 'initialization' do
       before :each do
+        @set = LogfileSet.new(@logfiles, ParsedLine::TimingLog)
+        @builder = IntervalBuilder.new(@set.each_parsed_line, ParsedLine::TimingLog, 300)
+      end
+
+      it 'accepts a logfile as the parsed_lines_enum argument' do
+        logfile = Logfile.new("#{data_dir}/timing.log", ParsedLine::TimingLog)
+        builder = IntervalBuilder.new(logfile, ParsedLine::TimingLog, 300)
         Time.stub(:now).and_return(Time.new(2013,12,01,16,0,1,'-08:00'))
-        @intervals = []
-        @builder.each_interval do |interval|
-          @intervals << interval
+        intervals = []
+        builder.each_interval do |interval|
+          intervals << interval
         end
+        intervals.size.should == 1
       end
 
-      it 'finds intervals from all logfiles' do
-        @intervals.size.should == 2
+      it 'accepts a logfile_set as the parsed_lines_enum argument' do
+        builder = IntervalBuilder.new(@set, ParsedLine::TimingLog, 300)
+        Time.stub(:now).and_return(Time.new(2013,12,01,16,0,1,'-08:00'))
+        intervals = []
+        builder.each_interval do |interval|
+          intervals << interval
+        end
+        intervals.size.should == 2
       end
+    end
 
-      context 'first interval' do
-        it 'got records from both logfiles' do
-          @intervals.first.size.should == 4
-          @intervals.first.end_time.should == Time.new(2013,12,01,16,0,0,'-08:00')
-          @intervals.first[:total_time].should == 700.0/4
-          @intervals.first[:num_bytes].should == 52000
-          @intervals.first[:rss].round(5).should == 0.60
-          @intervals.first[:ip].should == 4
-          @intervals.first[:action].should == 4
+    describe :each_interval do
+      context 'without a block' do
+        it 'returns an enumerator' do
+          set = LogfileSet.new(@logfiles, ParsedLine::TimingLog, :desc)
+          builder = IntervalBuilder.new(set, ParsedLine::TimingLog, 300)
+          e = builder.each_interval
+          e.should be_a(Enumerator)
         end
       end
 
-      context 'second interval' do
-        it 'got records from second logfile only' do
-          @intervals.last.size.should == 2
-          @intervals.last.end_time.should == Time.new(2013,12,01,15,55,0,'-08:00')
-          @intervals.last[:total_time].should == 300
-          @intervals.last[:num_bytes].should == 41000
-          @intervals.last[:rss].round(5).should == 0.20
-          @intervals.last[:ip].should == 2
-          @intervals.last[:action].should == 2
+      context 'with empty logfiles' do
+        it 'does not yield any interval' do
+          logfiles = ["#{data_dir}/non_existing_timing.log", "#{data_dir}/non_existing_timing.log.1" ]
+          set = LogfileSet.new(logfiles, ParsedLine::TimingLog)
+          builder = IntervalBuilder.new(set, ParsedLine::TimingLog, 300)
+          intervals = []
+          builder.each_interval do |interval|
+            intervals << interval
+          end
+          intervals.should be_empty
         end
       end
 
-      context 'without a block' do
-        it 'should return an iterator' do
-          e = @builder.each_interval
-          e.should be_an(Enumerator)
-          e.next.end_time.should == Time.new(2013,12,01,16,0,0,'-08:00')
+      context 'in descending order' do
+        before :each do
+          Time.stub(:now).and_return(Time.new(2013,12,01,16,0,1,'-08:00'))
+          @set = LogfileSet.new(@logfiles, ParsedLine::TimingLog, :desc)
+          @builder = IntervalBuilder.new(@set, ParsedLine::TimingLog, 300)
+          @intervals = []
+          @builder.each_interval do |interval|
+            @intervals << interval
+          end
+        end
+
+        it 'finds intervals from all logfiles' do
+          @intervals.size.should == 2
+        end
+
+        context 'first interval' do
+          it 'got records from both logfiles' do
+            @intervals.first.size.should == 4
+            @intervals.first.end_time.should == Time.new(2013,12,01,16,0,0,'-08:00')
+            @intervals.first[:total_time].should == 700.0/4
+            @intervals.first[:num_bytes].should == 52000
+            @intervals.first[:rss].round(5).should == -0.60
+            @intervals.first[:ip].should == {"192.168.0.5"=>3, "192.168.0.10"=>1}
+            @intervals.first[:action].should == {"posts#show"=>2, "posts#create"=>1, "posts#index"=>1}
+          end
+        end
+
+        context 'second interval' do
+          it 'got records from second logfile only' do
+            @intervals.last.size.should == 2
+            @intervals.last.end_time.should == Time.new(2013,12,01,15,55,0,'-08:00')
+            @intervals.last[:total_time].should == 300
+            @intervals.last[:num_bytes].should == 41000
+            @intervals.last[:rss].round(5).should == -0.20
+            @intervals.last[:ip].should == {"192.168.0.10"=>1, "192.168.0.5"=>1}
+            @intervals.last[:action].should == {"posts#index"=>1, "posts#show"=>1}
+          end
+        end
+      end
+
+      context 'in ascending order' do
+        before :each do
+          Time.stub(:now).and_return(Time.new(2013,12,01,16,0,1,'-08:00'))
+          @intervals = []
+          @set = LogfileSet.new(@logfiles, ParsedLine::TimingLog, :asc)
+          @builder = IntervalBuilder.new(@set, ParsedLine::TimingLog, 300)
+          @builder.each_interval do |interval|
+            @intervals << interval
+          end
+        end
+
+        it 'builds first interval older than last interval' do
+          first_time = @intervals.first.start_time
+          last_time  = @intervals.last.start_time
+          first_time.should be < last_time
+        end
+
+        it 'builds first interval with start time at 5 minute boundary below first record' do
+          first_start_time = @intervals.first.start_time
+          first_start_time.should == Time.new(2013,12,01,15,50,0,'-08:00')
+        end
+
+        it 'builds last interval with end time at 5 minute boundary following last record' do
+          last_end_time = @intervals.last.end_time
+          last_end_time.should == Time.new(2013,12,01,16,0,0,'-08:00')
+        end
+
+        it 'puts the right data in the right intervals' do
+          @intervals.first.size.should == 2
+          @intervals.first.end_time.should == Time.new(2013,12,01,15,55,0,'-08:00')
+          @intervals.first[:total_time].should == 300
+          @intervals.first[:num_bytes].should == 41000
+          @intervals.first[:rss].round(5).should == 0.20
+          @intervals.first[:ip].should == {"192.168.0.10"=>1, "192.168.0.5"=>1}
+          @intervals.first[:action].should == {"posts#index"=>1, "posts#show"=>1}
+
+          @intervals.last.size.should == 4
+          @intervals.last.end_time.should == Time.new(2013,12,01,16,0,0,'-08:00')
+          @intervals.last[:total_time].should == 700.0/4
+          @intervals.last[:num_bytes].should == 52000
+          @intervals.last[:rss].round(5).should == 0.60
+          @intervals.last[:ip].should == {"192.168.0.5"=>3, "192.168.0.10"=>1}
+          @intervals.last[:action].should == {"posts#show"=>2, "posts#create"=>1, "posts#index"=>1}
+        end
+      end
+
+      context 'with a gap in the logfiles' do
+        before :each do
+          Time.stub(:now).and_return(Time.new(2013,12,01,16,0,1,'-08:00'))
+          @logfiles = ["#{data_dir}/timing.log", "#{data_dir}/timing.log.1", "#{data_dir}/timing.log.2" ]
+        end
+
+        context 'in descending order' do
+          before :each do
+            @set = LogfileSet.new(@logfiles, ParsedLine::TimingLog, :desc)
+            @builder = IntervalBuilder.new(@set, ParsedLine::TimingLog, 300)
+            @intervals = []
+            @builder.each_interval do |interval|
+              @intervals << interval
+            end
+          end
+
+          it 'creates an empty interval' do
+            @intervals.size.should == 4
+            gap_interval = @intervals[2]
+            gap_interval.size.should == 0
+            gap_interval.end_time.should == Time.new(2013,12,01,15,50,0,'-08:00')
+          end
+        end
+
+        context 'in ascending order' do
+          before :each do
+            @set = LogfileSet.new(@logfiles, ParsedLine::TimingLog, :asc)
+            @builder = IntervalBuilder.new(@set, ParsedLine::TimingLog, 300)
+            @intervals = []
+            @builder.each_interval do |interval|
+              @intervals << interval
+            end
+          end
+
+          it 'creates an empty interval' do
+            @intervals.size.should == 4
+            gap_interval = @intervals[1]
+            gap_interval.size.should == 0
+            gap_interval.end_time.should == Time.new(2013,12,01,15,50,0,'-08:00')
+          end
         end
       end
     end
 
-    context :last_interval do
-      it 'returns the most recent interval' do
-        Time.stub(:now).and_return(Time.new(2013,12,01,16,0,1,'-08:00'))
-        interval = @builder.last_interval
-        interval.end_time.should == Time.new(2013,12,01,16,0,0,'-08:00')
-        interval.size.should == 4
-        interval[:num_bytes].should == 52000
+    describe :first_interval do
+      context 'with parsed_lines_enum in ascending order' do
+        it 'returns the oldest interval' do
+          Time.stub(:now).and_return(Time.new(2013,12,01,16,0,1,'-08:00'))
+          set = LogfileSet.new(@logfiles, ParsedLine::TimingLog, :asc)
+          builder = IntervalBuilder.new(set, ParsedLine::TimingLog, 300)
+          interval = builder.first_interval
+          interval.end_time.should == Time.new(2013,12,01,15,55,0,'-08:00')
+          interval.size.should == 2
+        end
+      end
+
+      context 'with parsed_lines_enum in descending order' do
+        it 'returns the most recent interval' do
+          Time.stub(:now).and_return(Time.new(2013,12,01,16,0,1,'-08:00'))
+          set = LogfileSet.new(@logfiles, ParsedLine::TimingLog, :desc)
+          builder = IntervalBuilder.new(set, ParsedLine::TimingLog, 300)
+          interval = builder.first_interval
+          interval.end_time.should == Time.new(2013,12,01,16,0,0,'-08:00')
+          interval.size.should == 4
+        end
       end
     end
   end