logfile_interval 1.1.2 → 1.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b10f4f6bcfa47f21b82b70bf541ef5403638c1ac
-  data.tar.gz: a0d4c42cd3f531b96fcbfc3aa8c7a0b4b2b3de1b
+  metadata.gz: 4b50ea1942cafe43439265b1d781190b4a60af69
+  data.tar.gz: 35fe78e837a6f1377baf2c62b3b4df5d95b814f7
 SHA512:
-  metadata.gz: 954c42936dee2d249e2fc71f25cf175455f0350e4d96951bdddcb0c766ed18a53415c43a279746779df8d6e6ea89c83dc6e433272599bb0d5792d6f9b3a3f89e
-  data.tar.gz: 289e3c63c70eeb4d606bc5a5431d6ea0c69c277b6b31cb717bcdfd4ef28f5a2bce0d660ed3bd8de0d3257eed237d1baf7c8f3b64fd879f204553037ab95c4f46
+  metadata.gz: 37ae14fa68272ef51c2127f65046e8068ad1937b00f2d66cbce37377bbde9582deb4a6312130dd630991a404d832d34a497bb60dc0fc8fe2bf303870241bed06
+  data.tar.gz: d12a79f9e706e5e02fce96d2ccc8e8b301936245d5819f93376873f837bbe2554ed14c37729895b3f720b48306f71d07193ec65fd9eb4ec797391f6d7ccee736

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    logfile_interval (1.1.2)
+    logfile_interval (1.2.1)
 
 GEM
   remote: https://rubygems.org/

data/lib/logfile_interval/aggregator/base.rb

@@ -3,9 +3,10 @@ module LogfileInterval
     class Base
       include Enumerable
 
-      def initialize
+      def initialize(options = {})
         @val = Util::Counter.new
         @size = Util::Counter.new
+        @options = options
       end
 
       def value(group = nil)

data/lib/logfile_interval/aggregator.rb

@@ -1,7 +1,5 @@
 lib_dir = File.expand_path('..', __FILE__)
 
-puts "lib_dir=#{lib_dir}"
-
 require "#{lib_dir}/aggregator/base"
 require "#{lib_dir}/aggregator/sum"
 require "#{lib_dir}/aggregator/count"
@@ -11,13 +9,18 @@ require "#{lib_dir}/aggregator/delta"
 
 module LogfileInterval
   module Aggregator
-    def self.klass(aggregator)
-      case aggregator
+    def self.klass(options)
+      case options[:aggregator]
       when :sum               then Sum
       when :average           then Average
-      when :count             then Count
-      when :group_and_count   then GroupAndCount
+      when :count
+        if options[:group_by] && options[:group_by] != options[:name]
+          GroupAndCount
+        else
+          Count
+        end
       when :delta             then Delta
+      when :custom            then options.fetch(:custom_class)
       end
     end
   end

data/lib/logfile_interval/interval.rb

@@ -16,12 +16,17 @@ module LogfileInterval
 
       @data = {}
       parser.columns.each do |name, options|
-        next unless agg = options[:aggregator]
-        @data[name] = agg.new
+        next unless agg = options[:aggregator_class]
+        if custom_options = options[:custom_options]
+          @data[name] = agg.new(custom_options)
+        else
+          @data[name] = agg.new
+        end
       end
     end
 
     def [](name)
+      raise ArgumentError, "#{name} field does not exist" unless @data.has_key?(name)
       @data[name.to_sym].values
     end
 

data/lib/logfile_interval/interval_builder.rb

@@ -10,7 +10,7 @@ module LogfileInterval
 
     def each_interval
       return enum_for(:each_interval) unless block_given?
-      
+
       secs = (Time.now.to_i / length.to_i) * length.to_i
       rounded_end_time = Time.at(secs)
       current_interval = Interval.new(rounded_end_time, length, parser)

data/lib/logfile_interval/line_parser/base.rb

@@ -1,8 +1,7 @@
 module LogfileInterval
   module LineParser
-    AGGREGATION_FUNCTIONS = [ :sum, :average, :timestamp, :count, :delta ]
+    AGGREGATION_FUNCTIONS = [ :sum, :average, :timestamp, :count, :delta, :custom ]
 
-    class InvalidLine         < StandardError; end
     class ConfigurationError  < StandardError; end
 
     class Base
@@ -20,25 +19,11 @@ module LogfileInterval
         end
 
         def add_column(options)
-          name          = options.fetch(:name)
-          pos           = options.fetch(:pos)
-          conversion    = options.fetch(:conversion, :string)
-          group_by      = options.fetch(:group_by, nil)
-          aggregator    = options.fetch(:aggregator)
-          unless AGGREGATION_FUNCTIONS.include?(aggregator)
-            raise ArgumentError, "aggregator must be one of #{AGGREGATION_FUNCTIONS.join(', ')}"
-          end
-
-          name      = name.to_sym
-          group_by  = group_by.to_sym unless group_by.nil?
-
-          if aggregator == :count && group_by && group_by != name
-            aggregator = :group_and_count
-          end
+          validate_column_options(options)
+          options = sanitize_column_options(options)
 
-          aggregator = Aggregator.klass(aggregator)
-          columns[name] = { :pos => pos, :aggregator => aggregator, :conversion => conversion }
-          columns[name][:group_by] = group_by
+          name = options[:name]
+          columns[name] = options
 
           define_method(name) do
             @data[name]
@@ -51,7 +36,6 @@ module LogfileInterval
 
           match_data = regex.match(line)
           return nil unless match_data
-          return nil unless match_data.size >= columns.size+1
 
           data = {}
           columns.each do |name, options|
@@ -67,9 +51,42 @@ module LogfileInterval
           return nil
         end
 
+        def set_column_custom_options(column_name, options)
+          raise ArgumentError, "Invalid column name: #{column_name}" unless columns.has_key?(column_name)
+          raise ArgumentError, "This column is not custom: #{column_name}" unless columns[column_name].has_key?(:custom_class)
+          columns[column_name][:custom_options] = options
+        end
+
         private
 
         def validate_column_options(options)
+          validate_option(options, :name)
+          validate_option(options, :pos)
+          validate_option(options, :aggregator)
+          unless AGGREGATION_FUNCTIONS.include?(options[:aggregator])
+            raise ConfigurationError, "aggregator must be one of #{AGGREGATION_FUNCTIONS.join(', ')}"
+          end
+          if options[:aggregator] == :custom
+            validate_option(options, :custom_class, ':custom_class must be set for :custom aggregator type')
+          end
+        end
+
+        def validate_option(options, key, errmsg = nil)
+          raise ConfigurationError, errmsg || "#{key} is a mandatory column option" unless options.has_key?(key)
+        end
+
+        def sanitize_column_options(options)
+          options[:name]       = options[:name].to_sym
+          if options.has_key?(:group_by)
+            options[:group_by]   = options[:group_by].to_sym
+          end
+          options[:conversion] = options.fetch(:conversion, :string)
+          if options[:aggregator] == :custom
+            options[:custom_options] = options.fetch(:custom_options, {})
+          end
+          options[:aggregator_class] = Aggregator.klass(options)
+          options.delete(:aggregator)
+          options
         end
 
         def convert(val, conversion)

data/lib/logfile_interval/version.rb

@@ -1,3 +1,3 @@
 module LogfileInterval
-  VERSION = "1.1.2"
+  VERSION = "1.2.1"
 end

data/spec/lib/aggregator_spec.rb

@@ -1,14 +1,18 @@
 require 'spec_helper'
 
 module LogfileInterval
+
   module Aggregator
+    class CustomAggregator; end
+      
     describe Aggregator do
       it 'finds the aggregator class' do
-        Aggregator.klass(:sum).should == Sum
-        Aggregator.klass(:average).should == Average
-        Aggregator.klass(:count).should == Count
-        Aggregator.klass(:group_and_count).should == GroupAndCount
-        Aggregator.klass(:delta).should == Delta
+        Aggregator.klass({ :aggregator => :sum}).should == Sum
+        Aggregator.klass({ :aggregator => :average}).should == Average
+        Aggregator.klass({ :aggregator => :count}).should == Count
+        Aggregator.klass({ :aggregator => :count, :group_by => :foo}).should == GroupAndCount
+        Aggregator.klass({ :aggregator => :delta}).should == Delta
+        Aggregator.klass({ :aggregator => :custom, :custom_class => CustomAggregator}).should == CustomAggregator
       end
     end
 

data/spec/lib/custom_aggregator_spec.rb

@@ -0,0 +1,41 @@
+require 'spec_helper'
+require File.join(File.dirname(__FILE__), '..', 'support/lib/custom_timing_log')
+
+module LogfileInterval
+  describe 'Custom Aggregator' do
+    before :each do
+      @end_time = Time.new(2013, 12, 01, 16, 00, 00, '-08:00')
+      @length = 300
+      @line_parser_class = LineParser::CustomTimingLog
+    end
+
+    def fill_interval
+      @interval = Interval.new(@end_time, @length, @line_parser_class)
+      record1 = @line_parser_class.create_record('1385942400, 192.168.0.5, posts#index, 150, 20000, 53.0')
+      @interval.add_record(record1)
+      record2 = @line_parser_class.create_record('1385942300, 192.168.0.5, posts#show, 50, 10000, 51.0')
+      @interval.add_record(record2)
+      record3 = @line_parser_class.create_record('1385942200, 10.10.10.10, posts#show, 70, 12000, 50.0')
+      @interval.add_record(record3)
+    end
+
+    it 'gets instantiated with empty data' do
+      fill_interval
+
+      @interval.size.should == 3
+      @interval[:num_slow].should == 1
+      @interval[:ip].should == 3
+    end
+
+    describe 'set_column_custom_options' do
+      it 'overwrites custom aggregator custom options' do
+        @line_parser_class.set_column_custom_options(:num_slow, :threshold => 60)
+        fill_interval
+
+        @interval.size.should == 3
+        @interval[:num_slow].should == 2
+        @interval[:ip].should == 3
+      end
+    end
+  end
+end

data/spec/lib/line_parser/base_spec.rb

@@ -50,6 +50,10 @@ module LogfileInterval
         set_regex /^([\d\.]+)\s+\S+\s+\S+\s+\[(\d\d.*\d\d)\]\s+"(?:GET|POST|PUT|HEAD|DELETE)\s+(\S+)\s+HTTP\S+"\s+(\d+)\s+(\d+)\s+"([^"]*)"\s+"([^"]+)"$/
       end
 
+      class MissingCustomClass < Base
+        set_regex /(.*)/
+      end
+
       before :each do
         @line = '74.75.19.145 - - [31/Mar/2013:06:54:12 -0700] "GET /ppa/google_chrome HTTP/1.1" 200 7855 "https://www.google.com/" "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.22 (KHTML, like Gecko) Ubuntu Chromium/25.0.1364.160 Chrome/25.0.1364.160 Safari/537.22"'
       end
@@ -61,6 +65,10 @@ module LogfileInterval
       it 'must fail unless a column is configured'do
         lambda { NoColumnLog.new(@line) }.should raise_error ConfigurationError
       end
+
+      it 'must fail with custom aggregator but no custom class' do
+        lambda { MissingCustomClass.add_column(:name => 'ip', :pos => 1, :aggregator => :custom) }.should raise_error ConfigurationError
+      end
     end
 
     describe TimingLog do

data/spec/support/lib/custom_timing_log.rb

@@ -0,0 +1,32 @@
+module LogfileInterval
+  module Aggregator
+    class CountOverThreshold < Base
+      def initialize(options)
+        super
+        @threshold = options.fetch(:threshold)
+      end
+
+      def add(value, group_by = nil)
+        @val.add(key(group_by), 1) if value > @threshold
+      end
+    end
+  end
+
+  module LineParser
+    class CustomTimingLog < Base
+      # Line format:
+      # timestamp, ip, controller#action, total_time, bytes, rss
+
+      set_regex /^(\d+),\s*([\d\.]+),\s*(\w+#\w+),\s*(\d+),\s*(\d+),\s*([\d\.]+)$/
+
+      add_column :name => :timestamp,    :pos => 1, :aggregator => :timestamp
+      add_column :name => :ip,           :pos => 2, :aggregator => :count
+      add_column :name => :num_slow,     :pos => 4, :aggregator => :custom,    :conversion => :integer,
+                 :custom_class => Aggregator::CountOverThreshold, :custom_options => { :threshold => 100 }
+
+      def time
+        Time.at(self.timestamp.to_i)
+      end
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logfile_interval
 version: !ruby/object:Gem::Version
-  version: 1.1.2
+  version: 1.2.1
 platform: ruby
 authors:
 - Philippe Le Rohellec
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-01-11 00:00:00.000000000 Z
+date: 2014-01-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -118,6 +118,7 @@ files:
 - logfile_interval.gemspec
 - spec/lib/aggregator_spec.rb
 - spec/lib/counter_spec.rb
+- spec/lib/custom_aggregator_spec.rb
 - spec/lib/interval_builder_spec.rb
 - spec/lib/interval_spec.rb
 - spec/lib/line_parser/base_spec.rb
@@ -125,6 +126,7 @@ files:
 - spec/lib/logfile_spec.rb
 - spec/spec_helper.rb
 - spec/support/lib/access_log.rb
+- spec/support/lib/custom_timing_log.rb
 - spec/support/lib/timing_log.rb
 - spec/support/logfiles/access.log
 - spec/support/logfiles/access.log.1
@@ -159,6 +161,7 @@ summary: Aggregate logfile data into intervals
 test_files:
 - spec/lib/aggregator_spec.rb
 - spec/lib/counter_spec.rb
+- spec/lib/custom_aggregator_spec.rb
 - spec/lib/interval_builder_spec.rb
 - spec/lib/interval_spec.rb
 - spec/lib/line_parser/base_spec.rb
@@ -166,6 +169,7 @@ test_files:
 - spec/lib/logfile_spec.rb
 - spec/spec_helper.rb
 - spec/support/lib/access_log.rb
+- spec/support/lib/custom_timing_log.rb
 - spec/support/lib/timing_log.rb
 - spec/support/logfiles/access.log
 - spec/support/logfiles/access.log.1