logfile_interval 1.0.0

data/lib/logfile_interval.rb

@@ -0,0 +1,12 @@
+require "logfile_interval/version"
+require "logfile_interval/file_backward"
+require "logfile_interval/interval"
+require "logfile_interval/interval_builder"
+require "logfile_interval/logfile"
+require "logfile_interval/logfile_set"
+require "logfile_interval/line_parser/base"
+require "logfile_interval/line_parser/aggregator"
+require "logfile_interval/line_parser/counter"
+
+module LogfileInterval
+end

data/lib/logfile_interval/file_backward.rb

@@ -0,0 +1,49 @@
+module LogfileInterval
+  # Based on Perl's File::ReadBackwards module, by Uri Guttman.
+  class FileBackward
+    MAX_READ_SIZE = 1 << 10 # 1024
+
+    def initialize( *args )
+      return unless File.exist?(args[0])
+      @file = File.new(*args)
+      @file.seek(0, IO::SEEK_END)
+
+      @current_pos = @file.pos
+
+      @read_size = @file.pos % MAX_READ_SIZE
+      @read_size = MAX_READ_SIZE if @read_size.zero?
+
+      @line_buffer = Array.new
+    end
+
+    def gets( sep_string = $/ )
+      return nil unless @file
+      return @line_buffer.pop if @line_buffer.size > 2 or @current_pos.zero?
+
+      @current_pos -= @read_size
+      @file.seek(@current_pos, IO::SEEK_SET)
+
+      @line_buffer[0] = "#{@file.read(@read_size)}#{@line_buffer[0]}"
+      @read_size = MAX_READ_SIZE # Set a size for the next read.
+
+      @line_buffer[0] =
+      @line_buffer[0].scan(/.*?#{Regexp.escape(sep_string)}|.+/)
+      @line_buffer.flatten!
+
+      gets(sep_string)
+    end
+
+    def close
+      return unless @file
+      @file.close()
+    end
+  end
+end
+
+# f = FileBackward.new('../log/development.log')
+# i = 0
+# while(line = f.gets())
+#     puts line
+#     i += 1
+#     break if i>30
+# end

data/lib/logfile_interval/interval.rb

@@ -0,0 +1,42 @@
+module LogfileInterval
+  class Interval
+    attr_reader   :start_time, :end_time, :length, :parser
+    attr_reader   :size
+
+    class OutOfRange < StandardError; end
+    class ParserMismatch < StandardError; end
+
+    def initialize(end_time, length, parser)
+      raise ArgumentError, 'end_time must be round' unless (end_time.to_i % length.to_i == 0)
+      @end_time   = end_time
+      @start_time = end_time - length
+      @length     = length
+      @parser     = parser
+      @size = 0
+
+      @data = {}
+      parser.columns.each do |name, options|
+        next unless agg = options[:aggregator]
+        @data[name] = agg.new
+      end
+    end
+
+    def [](name)
+      @data[name].value
+    end
+
+    def add_record(record)
+      return unless record.valid?
+      raise ParserMismatch unless record.class == parser
+      raise OutOfRange, 'too recent' if record.time>@end_time
+      raise OutOfRange, 'too old'    if record.time<=@start_time
+
+      @size += 1
+
+      parser.columns.each do |name, options|
+        next unless @data[name]
+        @data[name].add(record[name])
+      end
+    end
+  end
+end

data/lib/logfile_interval/interval_builder.rb

@@ -0,0 +1,34 @@
+module LogfileInterval
+  class IntervalBuilder
+    attr_reader :logfile_set, :parser, :length
+
+    def initialize(logfile_set, length)
+      @logfile_set  = logfile_set
+      @parser       = logfile_set.parser
+      @length       = length
+    end
+
+    def each_interval
+      secs = (Time.now.to_i / length.to_i) * length.to_i
+      rounded_end_time = Time.at(secs)
+      current_interval = Interval.new(rounded_end_time, length, parser)
+
+      logfile_set.each_parsed_line do |record|
+        next if record.time > current_interval.end_time
+        while record.time <= current_interval.start_time
+          yield current_interval
+          current_interval = Interval.new(current_interval.start_time, length, parser)
+        end
+        current_interval.add_record(record)
+      end
+
+      yield current_interval if current_interval.size>0
+    end
+
+    def last_interval
+      each_interval do |interval|
+        return interval
+      end
+    end
+  end
+end

data/lib/logfile_interval/interval_length.rb

@@ -0,0 +1,47 @@
+module LogfileInterval
+  class IntervalLength
+    MAX_PERIODS = { 5 * 60           => 6 * 3600,
+                    3600             => 3600 * 24,
+                    3600 * 24        => 3600 * 24 * 30,
+                    3600 * 24 * 30   => 365 * 3600 * 24 }
+    LENGTHS = MAX_PERIODS.keys.sort
+
+    attr_reader :length
+
+    def initialize(l)
+      raise ArgumentError unless LENGTHS.include?(l)
+      @length = l
+    end
+
+    def lower
+      pos = LENGTHS.index(@length)
+      return nil if pos==0
+      IntervalLength.new(LENGTHS[pos-1])
+    end
+
+    def higher
+      pos = LENGTHS.index(@length)
+      return nil if pos==LENGTHS.size-1
+      IntervalLength.new(LENGTHS[pos+1])
+    end
+
+    def smallest?
+      pos = LENGTHS.index(@length)
+      pos == 0
+    end
+
+    def start_time(t)
+      ts = (t.to_i / @length.to_i) * @length.to_i
+      ts -= @length.to_i if t.to_i % @length.to_i == 0
+      Time.at(ts)
+    end
+
+    def end_time(t)
+      start_time(t) + @length
+    end
+
+    def to_i
+      @length.to_i
+    end
+  end
+end

data/lib/logfile_interval/line_parser/aggregator.rb

@@ -0,0 +1,61 @@
+module LogfileInterval
+  module LineParser
+    module Aggregator
+      def self.klass(agg_function)
+        case agg_function
+        when :sum     then Sum
+        when :average then Average
+        when :group   then Group
+        end
+      end
+
+      class Sum
+        def initialize
+          @val = 0
+        end
+
+        def add(value)
+          @val += value
+        end
+
+        def value
+          @val
+        end
+      end
+
+      class Average
+        def initialize
+          @val  = 0
+          @size = 0
+        end
+
+        def add(value)
+          @val += value
+          @size += 1
+        end
+
+        def value
+          if @size > 0
+            @val.to_f / @size.to_f
+          else
+            0
+          end
+        end
+      end
+
+      class Group
+        def initialize
+          @val = Counter.new
+        end
+
+        def add(value)
+          @val.increment(value)
+        end
+
+        def value
+          @val
+        end
+      end
+    end
+  end
+end

data/lib/logfile_interval/line_parser/base.rb

@@ -0,0 +1,94 @@
+module LogfileInterval
+  module LineParser
+    AGGREGATION_FUNCTIONS = [ :sum, :average, :timestamp, :group ]
+
+    class InvalidLine         < StandardError; end
+    class ConfigurationError  < StandardError; end
+
+    class Base
+      class << self
+        attr_reader :regex
+
+        def columns
+          @columns ||= {}
+        end
+
+        def set_regex(regex)
+          @regex = regex
+        end
+
+        def add_column(options)
+          name          = options.fetch(:name)
+          pos           = options.fetch(:pos)
+          agg_function  = options.fetch(:agg_function)
+          conversion    = options.fetch(:conversion, :string)
+          unless AGGREGATION_FUNCTIONS.include?(agg_function)
+            raise ArgumentError, "agg_function must be one of #{AGGREGATION_FUNCTIONS.join(', ')}"
+          end
+
+          aggregator = Aggregator.klass(agg_function)
+          columns[name] = { :pos => pos, :aggregator => aggregator, :conversion => conversion }
+
+          define_method(name) do
+            @data[name]
+          end
+        end
+
+        def parse(line)
+          raise ConfigurationError, 'There must be at least 1 configured column' unless columns.any?
+          raise ConfigurationError, 'A regex must be set' unless regex
+
+          match_data = regex.match(line)
+          return nil unless match_data
+          return nil unless match_data.size >= columns.size+1
+
+          data = {}
+          columns.each do |name, options|
+            val = match_data[options[:pos]]
+            data[name] = convert(val, options[:conversion])
+          end
+          data
+        end
+
+        def create_record(line)
+          record = new(line)
+          return record if record.valid?
+          return nil
+        end
+
+        private
+
+        def validate_column_options(options)
+        end
+
+        def convert(val, conversion)
+          case conversion
+          when :integer then val.to_i
+          when :float   then val.to_f
+          else val
+          end
+        end
+      end
+
+      def initialize(line)
+        @data = self.class.parse(line)
+        @valid = @data ? true : false
+      end
+
+      def valid?
+        @valid
+      end
+
+      def time
+        raise NotImplemented
+      end
+
+      def [](name)
+        @data[name]
+      end
+    end
+  end
+end
+
+
+

data/lib/logfile_interval/line_parser/counter.rb

@@ -0,0 +1,27 @@
+module LogfileInterval
+  module LineParser
+    class Counter < Hash
+      def increment(val)
+        if self.has_key?(val)
+          self[val] += 1
+        else
+          self[val] = 1
+        end
+      end
+
+      def add(val, num)
+        if self.has_key?(val)
+          self[val] += num
+        else
+          self[val] = num
+        end
+      end
+
+      def merge(c)
+        c.keys.each do |k|
+          self.add c[k]
+        end
+      end
+    end
+  end
+end

data/lib/logfile_interval/logfile.rb

@@ -0,0 +1,33 @@
+module LogfileInterval
+  class Logfile
+    attr_reader :filename, :parser
+
+    def initialize(filename, parser)
+      @filename = filename
+      @parser   = parser
+    end
+
+    def first_timestamp
+      return nil unless File.exist?(@filename)
+      File.open(@filename) do |f|
+        line = parser.create_record(f.gets)
+        line.time
+      end
+    end
+
+    def each_line
+      f = FileBackward.new(@filename)
+      while(line = f.gets)
+        yield line.chomp
+      end
+      f.close
+    end
+
+    def each_parsed_line
+      each_line do |line|
+        record = parser.create_record(line)
+        yield record if record
+      end
+    end
+  end
+end

data/lib/logfile_interval/logfile_set.rb

@@ -0,0 +1,47 @@
+module LogfileInterval
+  class LogfileSet
+    attr_reader :parser
+
+    def initialize(filenames, parser)
+      @parser    = parser
+      @filenames = filenames
+    end
+
+    def existing_filenames
+      @existing_filenames ||= @filenames.select { |f| File.exist?(f) }
+    end
+
+    def ordered_filenames
+      time_for_file = existing_filenames.inject({}) do |h, filename|
+        file = Logfile.new(filename, parser)
+        h[filename] = file.first_timestamp
+        h
+      end
+      time_for_file.to_a.sort_by { |arr| arr[1] }.map { |arr| arr[0] }.reverse
+    end
+
+    def each_parsed_line
+      ordered_filenames.each do |filename|
+        tfile = Logfile.new(filename, parser)
+        tfile.each_parsed_line do |record|
+          yield record
+        end
+      end
+    end
+
+    def each_line
+      ordered_filenames.each do |filename|
+        tfile = Logfile.new(filename, parser)
+        tfile.each_line do |line|
+          yield line
+        end
+      end
+    end
+
+    def last_record
+      each_parsed_line do |record|
+        return record
+      end
+    end
+  end
+end