logfile_interval 1.2.1 → 2.0.0

data/lib/logfile_interval/aggregator/delta.rb

@@ -1,17 +1,17 @@
 module LogfileInterval
   module Aggregator
     class Delta < Base
-      def initialize
+      def initialize(options = {})
         @previous = Util::Counter.new
         super
       end
 
-      def add(value, group_by = nil)
-        if @previous.has_key?(key(group_by))
-          @val.add(key(group_by), @previous[key(group_by)] - value)
-          @size.increment(key(group_by))
+      def add(value, group_by_value = nil)
+        if @previous.has_key?(key(group_by_value))
+          @val.add(key(group_by_value), value - @previous[key(group_by_value)])
+          @size.increment(key(group_by_value))
         end
-        @previous.set(key(group_by), value)
+        @previous.set(key(group_by_value), value)
       end
 
       def val(k)

data/lib/logfile_interval/aggregator/num_lines.rb

@@ -0,0 +1,13 @@
+module LogfileInterval
+  module Aggregator
+    class NumLines < Base
+      def add(value, group_by_value = nil)
+        if group_by_value
+          @val.increment_subkey(:all, key(group_by_value))
+        else
+          @val.increment(:all)
+        end
+      end
+    end
+  end
+end

data/lib/logfile_interval/aggregator/registrar.rb

@@ -0,0 +1,32 @@
+module LogfileInterval
+  module Aggregator
+    module Registrar
+      def inherited(subclass)
+        name = subclass.to_s
+        name = $1 if name =~ /(\w+)$/
+        name = name.scan(/[A-Z][a-z]*/).join("_").downcase.to_sym
+        aggregator_classes[name] = subclass
+      end
+
+      def aggregator_classes
+        @@aggregator_classes ||= {}
+      end
+
+      def register_aggregator(name, klass)
+        aggregator_classes[name] = klass
+      end
+
+      def klass(name)
+        aggregator_classes[name]
+      end
+
+      def exist?(name)
+        aggregator_classes.include?(name)
+      end
+
+      def all
+        aggregator_classes.keys
+      end
+    end
+  end
+end

data/lib/logfile_interval/aggregator_set.rb

@@ -0,0 +1,34 @@
+module LogfileInterval
+  class AggregatorSet
+    def initialize(parser_columns)
+      @parser_columns = parser_columns
+      @aggregators = {}
+      parser_columns.each do |name, options|
+        next unless klass = options[:aggregator_class]
+        @aggregators[name.to_sym] = klass.new(options.fetch(:custom_options, {}))
+      end
+    end
+
+    def add(record)
+      @parser_columns.each do |name, options|
+        next unless @aggregators[name]
+        group_by_value = record[options[:group_by]] if options[:group_by]
+        @aggregators[name].add(record[name], group_by_value)
+      end
+    end
+
+    def [](name)
+      raise ArgumentError, "#{name} field does not exist" unless @aggregators.has_key?(name)
+      @aggregators[name.to_sym].values
+    end
+
+    def to_hash
+      @aggregators.inject({}) do |h, pair|
+        k = pair[0]
+        v = pair[1]
+        h[k] = v.values
+        h
+      end
+    end
+  end
+end

data/lib/logfile_interval/interval.rb

@@ -4,9 +4,8 @@ module LogfileInterval
     attr_reader   :size
 
     class OutOfRange < StandardError; end
-    class ParserMismatch < StandardError; end
 
-    def initialize(end_time, length, parser)
+    def initialize(end_time, length, parser_columns)
       raise ArgumentError, 'end_time must be round' unless (end_time.to_i % length.to_i == 0)
       @end_time   = end_time
       @start_time = end_time - length
@@ -14,48 +13,23 @@ module LogfileInterval
       @parser     = parser
       @size = 0
 
-      @data = {}
-      parser.columns.each do |name, options|
-        next unless agg = options[:aggregator_class]
-        if custom_options = options[:custom_options]
-          @data[name] = agg.new(custom_options)
-        else
-          @data[name] = agg.new
-        end
-      end
+      @aggregators = AggregatorSet.new(parser_columns)
     end
 
     def [](name)
-      raise ArgumentError, "#{name} field does not exist" unless @data.has_key?(name)
-      @data[name.to_sym].values
-    end
-
-    def each(&block)
-      @data.each(&block)
+      @aggregators[name]
     end
 
     def to_hash
-      @data.inject({}) do |h, pair|
-        k = pair[0]
-        v = pair[1]
-        h[k] = v.values
-        h
-      end
+      @aggregators.to_hash
     end
 
     def add_record(record)
-      return unless record.valid?
-      raise ParserMismatch unless record.class == parser
       raise OutOfRange, 'too recent' if record.time>@end_time
       raise OutOfRange, 'too old'    if record.time<=@start_time
 
       @size += 1
-
-      parser.columns.each do |name, options|
-        next unless @data[name]
-        group_by_value = record[options[:group_by]] if options[:group_by]
-        @data[name].add(record[name], group_by_value)
-      end
+      @aggregators.add(record)
     end
   end
 end

data/lib/logfile_interval/interval_builder/ascending.rb

@@ -0,0 +1,23 @@
+module LogfileInterval
+  class IntervalBuilder
+    module Ascending
+      def create_first_interval
+        first_record = parsed_lines_enum.first
+        interval_end_time = upper_boundary_time(first_record.time)
+        Interval.new(interval_end_time, length, parser_columns)
+      end
+
+      def past_current_interval?(current_interval, record)
+        record.time  > current_interval.end_time
+      end
+
+      def out_of_order_record?(current_interval, record)
+        record.time <= current_interval.start_time
+      end
+
+      def next_interval_end_time(current_interval)
+        current_interval.end_time + length
+      end
+    end
+  end
+end

data/lib/logfile_interval/interval_builder/descending.rb

@@ -0,0 +1,22 @@
+module LogfileInterval
+  class IntervalBuilder
+    module Descending
+      def create_first_interval
+        interval_end_time = lower_boundary_time(Time.now)
+        Interval.new(interval_end_time, length, parser_columns)
+      end
+
+      def past_current_interval?(current_interval, record)
+        record.time <= current_interval.start_time
+      end
+
+      def out_of_order_record?(current_interval, record)
+        record.time  > current_interval.end_time
+      end
+
+      def next_interval_end_time(current_interval)
+        current_interval.end_time - length
+      end
+    end
+  end
+end

data/lib/logfile_interval/interval_builder.rb

@@ -1,36 +1,79 @@
+require File.join(File.expand_path('..', __FILE__), '/interval_builder/ascending')
+require File.join(File.expand_path('..', __FILE__), '/interval_builder/descending')
+
 module LogfileInterval
   class IntervalBuilder
-    attr_reader :logfile_set, :parser, :length
+    attr_reader :parsed_lines_enum, :parser_columns, :length
+
+    def initialize(parsed_lines_enum, parser_columns, length)
+      @parsed_lines_enum = parsed_lines_enum
+      @parser_columns    = parser_columns
+      @length            = length
 
-    def initialize(logfile_set, length)
-      @logfile_set  = logfile_set
-      @parser       = logfile_set.parser
-      @length       = length
+      case order
+      when :asc  then self.extend Ascending
+      when :desc then self.extend Descending
+      when :empty then nil
+      else raise ArgumentError, "Can't determine parsed_lines_enum sort order"
+      end
     end
 
-    def each_interval
+    def each_interval(&block)
       return enum_for(:each_interval) unless block_given?
+      return if order == :empty
 
-      secs = (Time.now.to_i / length.to_i) * length.to_i
-      rounded_end_time = Time.at(secs)
-      current_interval = Interval.new(rounded_end_time, length, parser)
+      current_interval = create_first_interval
 
-      logfile_set.each_parsed_line do |record|
-        next if record.time > current_interval.end_time
-        while record.time <= current_interval.start_time
-          yield current_interval
-          current_interval = Interval.new(current_interval.start_time, length, parser)
-        end
+      parsed_lines_enum.each do |record|
+        next if out_of_order_record?(current_interval, record)
+        current_interval = move_over_empty_intervals(current_interval, record) { |interval| yield interval }
         current_interval.add_record(record)
       end
 
-      yield current_interval if current_interval.size>0
+      yield current_interval if current_interval.size > 0
+    end
+
+    def first_interval
+      each_interval.first
+    end
+
+    private
+
+    def lower_boundary_time(t)
+      secs = (t.to_i / length.to_i) * length.to_i
+      Time.at(secs)
+    end
+
+    def upper_boundary_time(t)
+      secs = (t.to_i / length.to_i + 1) * length.to_i
+      Time.at(secs)
+    end
+
+    def order
+      return @order if @order
+      num_lines = 0
+      previous = nil
+      parsed_lines_enum.each do |pl|
+        num_lines += 1
+        if previous
+          if pl.time > previous.time
+            return @order = :asc
+          elsif pl.time < previous.time
+            return @order = :desc
+          end
+        end
+        previous = pl
+      end
+      return @order = :empty if num_lines == 0
+      return @order = :unknown
     end
 
-    def last_interval
-      each_interval do |interval|
-        return interval
+    def move_over_empty_intervals(current_interval, record)
+      while past_current_interval?(current_interval, record)
+        yield current_interval
+        current_interval = Interval.new(next_interval_end_time(current_interval), length, parser_columns)
       end
+      current_interval
     end
   end
 end

data/lib/logfile_interval/logfile.rb

@@ -2,9 +2,14 @@ module LogfileInterval
   class Logfile
     attr_reader :filename, :parser
 
-    def initialize(filename, parser)
+    ORDER_VALID_VALUES = [ :asc, :desc ]
+
+    def initialize(filename, parser, order = :desc)
       @filename = filename
       @parser   = parser
+      @order    = order
+
+      raise ArgumentError, "invalid order value: #{@order}" unless ORDER_VALID_VALUES.include?(@order.to_sym)
     end
 
     def exist?
@@ -26,11 +31,12 @@ module LogfileInterval
       return unless exist?
       return enum_for(:each_line) unless block_given?
 
-      f = Util::FileBackward.new(@filename)
-      while(line = f.gets)
-        yield line.chomp
+      case @order
+      when :asc
+        each_line_ascending { |l| yield l }
+      when :desc
+        each_line_descending { |l| yield l }
       end
-      f.close
     end
 
     def each_parsed_line
@@ -40,5 +46,28 @@ module LogfileInterval
         yield record if record
       end
     end
+    alias_method :each, :each_parsed_line
+
+    def first_parsed_line
+      each_parsed_line.first
+    end
+    alias_method :first, :first_parsed_line
+
+    private
+    def each_line_descending
+      f = Util::FileBackward.new(@filename)
+      while(line = f.gets)
+        yield line.chomp
+      end
+      f.close
+    end
+
+    def each_line_ascending
+      File.open(@filename) do |f|
+        f.each_line do |line|
+          yield line.chomp
+        end
+      end
+    end
   end
 end

data/lib/logfile_interval/logfile_set.rb

@@ -2,49 +2,63 @@ module LogfileInterval
   class LogfileSet
     attr_reader :parser
 
-    def initialize(filenames, parser)
+    ORDER_VALID_VALUES = [ :asc, :desc ]
+
+    def initialize(filenames, parser, order = :desc)
       @parser    = parser
       @filenames = filenames
-    end
+      @order    = order
 
-    def existing_filenames
-      @existing_filenames ||= @filenames.select { |f| File.exist?(f) }
+      raise ArgumentError, "invalid order value: #{@order}" unless ORDER_VALID_VALUES.include?(@order.to_sym)
     end
 
     def ordered_filenames
-      time_for_file = existing_filenames.inject({}) do |h, filename|
-        file = Logfile.new(filename, parser)
-        h[filename] = file.first_timestamp
-        h
+      time_for_files = time_for_files(existing_filenames)
+      order_filenames_asc = time_for_files.to_a.sort_by { |arr| arr[1] }.map { |arr| arr[0] }
+      case @order
+      when :desc
+        order_filenames_asc.reverse
+      when :asc
+        order_filenames_asc
       end
-      time_for_file.to_a.sort_by { |arr| arr[1] }.map { |arr| arr[0] }.reverse
     end
 
-    def each_parsed_line
-      return enum_for(:each_parsed_line) unless block_given?
+    def each_parsed_line(&block)
+      return enum_for(__method__) unless block_given?
+      each_by_method(__method__, &block)
+    end
+    alias_method :each, :each_parsed_line
 
-      ordered_filenames.each do |filename|
-        tfile = Logfile.new(filename, parser)
-        tfile.each_parsed_line do |record|
-          yield record
-        end
-      end
+    def each_line(&block)
+      return enum_for(__method__) unless block_given?
+      each_by_method(__method__, &block)
+    end
+
+    def first_parsed_line
+      each_parsed_line.first
     end
+    alias_method :first, :first_parsed_line
 
-    def each_line
-      return enum_for(:each_line) unless block_given?
+    private
+
+    def existing_filenames
+      @existing_filenames ||= @filenames.select { |f| File.exist?(f) }
+    end
 
+    def each_by_method(method, &block)
       ordered_filenames.each do |filename|
-        tfile = Logfile.new(filename, parser)
-        tfile.each_line do |line|
+        tfile = Logfile.new(filename, parser, @order)
+        tfile.send(method) do |line|
           yield line
         end
       end
     end
 
-    def last_record
-      each_parsed_line do |record|
-        return record
+    def time_for_files(filenames)
+      filenames.inject({}) do |h, filename|
+        file = Logfile.new(filename, parser)
+        h[filename] = file.first_timestamp
+        h
       end
     end
   end