logfile_interval 1.2.1 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4b50ea1942cafe43439265b1d781190b4a60af69
-  data.tar.gz: 35fe78e837a6f1377baf2c62b3b4df5d95b814f7
+  metadata.gz: 4eb614cbe0e43877376ccae43e38b666c3b58270
+  data.tar.gz: 723459738582e7da66b6d53e0f5ce43965576a6c
 SHA512:
-  metadata.gz: 37ae14fa68272ef51c2127f65046e8068ad1937b00f2d66cbce37377bbde9582deb4a6312130dd630991a404d832d34a497bb60dc0fc8fe2bf303870241bed06
-  data.tar.gz: d12a79f9e706e5e02fce96d2ccc8e8b301936245d5819f93376873f837bbe2554ed14c37729895b3f720b48306f71d07193ec65fd9eb4ec797391f6d7ccee736
+  metadata.gz: 4d799b181c1d65abaa35910b49c4efa85a31aefc44d009c0ee3c6966979206cec31acd09ffb1cbd97544ec74798a872d7906f43b57758a51e910e1f499935b8b
+  data.tar.gz: 21a2d6737ac8d3e92ed6b40dbba0f18d424ecddfac1678c9c067fafb7ea7db1e1fcde234a5fe66cc09c3d1da1077acc14a354ce0eef42bdaed2fbb6e1b16d3ce

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    logfile_interval (1.2.1)
+    logfile_interval (2.0.0)
 
 GEM
   remote: https://rubygems.org/

data/README.md

@@ -1,4 +1,4 @@
-# LogfileInterval [![Build Status](https://travis-ci.org/plerohellec/logfile_interval.png?branch=master)](https://travis-ci.org/plerohellec/logfile_interval)
+# LogfileInterval [![Build Status](https://travis-ci.org/plerohellec/logfile_interval.png?branch=master)](https://travis-ci.org/plerohellec/logfile_interval) [![Code Climate](https://codeclimate.com/github/plerohellec/logfile_interval.png)](https://codeclimate.com/github/plerohellec/logfile_interval)
 
 Logfile parser and aggregator.
 
@@ -28,9 +28,9 @@ class AccessLog < LogfileInterval::LineParser::Base
 
   set_regex /^([\d\.]+)\s+.*\s+\[(\d\d.*\d\d)\]\s+"(?:GET|POST|PUT|HEAD|DELETE)\s+(\S+)\s+HTTP\S+"\s+(\d+)\s+/
 
-  add_column :name => 'ip',           :pos => 1, :aggregator => :count,     :group_by => 'ip'
+  add_column :name => 'ip',           :pos => 1, :aggregator => :count
   add_column :name => 'timestamp',    :pos => 2, :aggregator => :timestamp
-  add_column :name => 'code',         :pos => 4, :aggregator => :count,     :group_by => 'code'
+  add_column :name => 'code',         :pos => 4, :aggregator => :count
   add_column :name => 'code_by_ip',   :pos => 4, :aggregator => :count,     :group_by => 'ip'
 
   def time
@@ -38,9 +38,11 @@ class AccessLog < LogfileInterval::LineParser::Base
   end
 end
 
-file = LogfileInterval::Logfile.new('path_to_logfile', AccessLog)
+parser = AccessLog
+file = LogfileInterval::Logfile.new('path_to_logfile', parser)
+parsed_line_enum = file.each_parsed_line
 
-builder = LogfileInterval::IntervalBuilder.new(file, 300)
+builder = LogfileInterval::IntervalBuilder.new(parsed_line_enum, parser, 300)
 builder.each_interval do |interval|
   next unless interval.size > 0
 
@@ -92,9 +94,9 @@ module LogfileInterval
 
       set_regex /^([\d\.]+)\s+\S+\s+\S+\s+\[(\d\d.*\d\d)\]\s+"(?:GET|POST|PUT|HEAD|DELETE)\s+(\S+)\s+HTTP\S+"\s+(\d+)\s+(\d+)\s+"([^"]*)"\s+"([^"]+)"$/
 
-      add_column :name => 'ip',           :pos => 1, :aggregator => :count,     :group_by => 'ip'
+      add_column :name => 'ip',           :pos => 1, :aggregator => :count
       add_column :name => 'timestamp',    :pos => 2, :aggregator => :timestamp
-      add_column :name => 'code',         :pos => 4, :aggregator => :count,     :group_by => 'code'
+      add_column :name => 'code',         :pos => 4, :aggregator => :count
       add_column :name => 'code_by_ip',   :pos => 4, :aggregator => :count,     :group_by => 'ip'
       add_column :name => 'length',       :pos => 5, :aggregator => :average,                      :conversion => :integer
       add_column :name => 'length_by_ip', :pos => 5, :aggregator => :average,   :group_by => 'ip', :conversion => :integer
@@ -120,6 +122,7 @@ end
 
 #### Aggregator types and options
 * timestamp: the timestamp field will be used to determine to which interval the line belongs, each line MUST have a timestamp
+* num_lines: the most simple aggregator simply counts the number of lines
 * count: the aggregator will count the number of occurence of this field
   * without the group_by option, it will just count the total number of lines (probably useless)
   * with a group_by option pointing to the same field as the current one, it will count the number of occurence
@@ -127,7 +130,7 @@ end
   * with a group_by option pointing to another field, it will count the number of occurences of (this field, other field) pairs.
 * average: the aggregator will calculate the average value of this field
 * sum: the aggregator will add up the values of this field
-* delta: the aggregator will caclculate the difference between each line and the next and will average all the deltas
+* delta: the aggregator will calculate the difference between each line and the previous one and will average all the deltas
 
 ### Iterate through lines of a single file
 And get a parsed record for each line.
@@ -150,7 +153,7 @@ log.each_parsed_line do |record|
   puts record.length
 end
 ```
-**Note**: the Logfile iterators always start with the last line in the file and works its way backward.
+**Note**: the Logfile iterators by default start with the last line in the file and works its way backward. To iterate in ascending order, pass :asc as the order argument in Logfile or LogfileSet#initialize.
 
 ### Iterate through lines of multiples files
 And get a parsed record for each line.
@@ -161,12 +164,13 @@ set.each_parsed_line do |record|
   puts record.class # LineParser::AccessLog
 end
 ```
-**Note**: the LogfileSet iterators always starts with the most recent file.
+**Note**: the LogfileSet iterators advances in the order of logfiles enumerator. If the enumerator is ascending, the builder will yield intervals in ascending order and vice-versa.
 
 ### Aggregate lines into intervals
 ```ruby
 length = 5.minutes
-interval_builder = LogfileInterval::IntervalBuilder.new(set, length)
+parsed_lines_enum = set.each_parsed_line
+interval_builder = LogfileInterval::IntervalBuilder.new(parsed_lines_enum, parser, length)
 interval_builder.each_interval do |interval|
   puts interval.class  # LogfileInterval::Interval
   puts interval.start_time

data/bin/aggregate_access_log.rb

@@ -10,27 +10,27 @@ unless File.exist?(String(logfile))
   exit 1
 end
 
-class AccessLogParser < LogfileInterval::LineParser::Base
+class AccessLogParsedLine < LogfileInterval::ParsedLine::Base
   # Example line:
   # 74.75.19.145 - - [31/Mar/2013:06:54:12 -0700] "GET /ppa/google_chrome HTTP/1.1" 200 7855 "https://www.google.com/" "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.22 (KHTML, like Gecko) Ubuntu Chromium/25.0.1364.160 Chrome/25.0.1364.160 Safari/537.22"
 
   set_regex /^([\d\.]+)\s+\S+\s+\S+\s+\[(\d\d.*\d\d)\]\s+"(?:GET|POST|PUT|HEAD|DELETE)\s+(\S+)\s+HTTP\S+"\s+(\d+)\s+(\d+)\s+"([^"]*)"\s+"([^"]+)"$/
 
-  add_column :name => 'ip',           :pos => 1, :aggregator => :count,                              :group_by => 'ip'
+  add_column :name => 'ip',           :pos => 1, :aggregator => :count
   add_column :name => 'timestamp',    :pos => 2, :aggregator => :timestamp
-  add_column :name => 'code_by_ip',   :pos => 4, :aggregator => :count,                              :group_by => 'ip'
-  add_column :name => 'length',       :pos => 5, :aggregator => :average,   :conversion => :integer
-  add_column :name => 'length_by_ip', :pos => 5, :aggregator => :average,   :conversion => :integer, :group_by => 'ip'
-  add_column :name => 'referer',      :pos => 6, :aggregator => :count,                              :group_by => :referer
-  add_column :name => 'referer_by_ip', :pos => 6, :aggregator => :count,                             :group_by => :ip
+  add_column :name => 'code_by_ip',   :pos => 4, :aggregator => :count,     :group_by => 'ip'
+  add_column :name => 'length',       :pos => 5, :aggregator => :average,                      :conversion => :integer
+  add_column :name => 'length_by_ip', :pos => 5, :aggregator => :average,   :group_by => 'ip', :conversion => :integer
+  add_column :name => 'referer',      :pos => 6, :aggregator => :count
+  add_column :name => 'referer_by_ip', :pos => 6, :aggregator => :count,    :group_by => :ip
 
   def time
     DateTime.strptime(self.timestamp, '%d/%b/%Y:%H:%M:%S %z').to_time
   end
 end
 
-file = LogfileInterval::Logfile.new(logfile, AccessLogParser)
-builder = LogfileInterval::IntervalBuilder.new(file, 300)
+file = LogfileInterval::Logfile.new(logfile, AccessLogParsedLine)
+builder = LogfileInterval::IntervalBuilder.new(file.each_parsed_line, AccessLogParsedLine, 300)
 builder.each_interval do |interval|
   next unless interval.size > 0
 

data/bin/readme.rb

@@ -4,15 +4,15 @@ require 'pp'
 require 'date'
 require File.join(File.expand_path('../../lib', __FILE__), 'logfile_interval')
 
-class AccessLog < LogfileInterval::LineParser::Base
+class AccessLogParsedLine < LogfileInterval::ParsedLine::Base
   # Example line:
   # 74.75.19.145 - - [31/Mar/2013:06:54:12 -0700] "GET /ppa/google_chrome HTTP/1.1" 200 7855 "https://www.google.com/" "Mozilla/5.0 Chrome/25.0.1364.160"
 
   set_regex /^([\d\.]+)\s+.*\s+\[(\d\d.*\d\d)\]\s+"(?:GET|POST|PUT|HEAD|DELETE)\s+(\S+)\s+HTTP\S+"\s+(\d+)\s+/
 
-  add_column :name => 'ip',           :pos => 1, :aggregator => :count,     :group_by => 'ip'
+  add_column :name => 'ip',           :pos => 1, :aggregator => :count
   add_column :name => 'timestamp',    :pos => 2, :aggregator => :timestamp
-  add_column :name => 'code',         :pos => 4, :aggregator => :count,     :group_by => 'code'
+  add_column :name => 'code',         :pos => 4, :aggregator => :count
   add_column :name => 'code_by_ip',   :pos => 4, :aggregator => :count,     :group_by => 'ip'
 
   def time
@@ -21,13 +21,14 @@ class AccessLog < LogfileInterval::LineParser::Base
 end
 
 path = ENV['ACCESS_LOG_PATH']
-file = LogfileInterval::Logfile.new(path, AccessLog)
+file = LogfileInterval::Logfile.new(path, AccessLogParsedLine)
 unless file.exist?
   puts "#{path} is not found"
   exit 1
 end
+parsed_line_enum = file.each_parsed_line
 
-builder = LogfileInterval::IntervalBuilder.new(file, 300)
+builder = LogfileInterval::IntervalBuilder.new(parsed_line_enum, AccessLogParsedLine, 300)
 builder.each_interval do |interval|
   next unless interval.size > 0
 

data/docs/design.rb

@@ -1,31 +1,33 @@
 module LogfileInterval
-  module LineParser
-    class Base
-      class << self
-        def set_regex(regex)
-        end
+  module ParsedLine
 
-        def add_column(name, options)
-          agg = Aggregators.klass(aggregator)
-          @columns[name] = { :pos => pos, :aggregator => agg, :conversion => conversion }
-          define_method(name)
-        end
+    module Parser
+      def columns
+        @columns ||= {}
+      end
 
-        def parse(line)
-          match_data = regex.match(line)
-          data = f(match_data)
-        end
+      def set_regex
+      end
 
-        def create_record(line)
-          record = new(line)
-          return record.valid? ? record : nil
-        end
+      def add_column
+        agg = Aggregator::Base.klass(aggregator)
+        @columns[name] = { :pos => pos, :aggregator => agg, :conversion => conversion }
+        define_method(name)
+      end
+
+      def parse(line)
+        match_data = regex.match(line)
+        data = {}
+        data = f(match_data)
+      end
+
+      def each(&block)
+        columns.each(&block)
       end
     end
 
-    class AccessLog < Base
-      set_regex /blah/
-      add_column :name => :foo, :pos => 1, :conversion => integer, :aggregator => :average
+    class Base
+      extend Parser
 
       def initialize(line)
         @data = self.class.parse(line)
@@ -33,38 +35,72 @@ module LogfileInterval
     end
   end
 
+  class AggregatorSet
+    def initialize(parser_columns)
+      @aggregators = {}
+      parser_columns.each do |name, options|
+        @aggregators[name] = options[:aggregator].new(options)
+      end
+    end
 
-  class Interval
-    def initialize(end_time, length, parser)
-      @data = {}
-      parser.columns.each do |name, options|
-        @data[name] = options[:aggregator].new
+    def add(record)
+      @aggregators.each do |name, agg|
+        agg.add_record(record)
       end
     end
 
-    def [](name)
-      @data[name].value
+    def each
+      @aggregators.each do |name, agg|
+        yield name, agg
+      end
     end
+  end
 
-    def add_record(record)
-      return unless record.valid?
-      raise ParserMismatch unless record.class == parser
+  class Interval
+    def initialize(end_time, length, parser_columns)
+      @aggregators = AggregatorSet.new(parser_columns)
+    end
 
+    def [](name)
+      @aggregators[name].value
+    end
+
+    def add(record)
       @size += 1
-      parser.columns.each do |name, options|
-        @data[name].add(record[name])
+      @aggregators.add_record(record)
+    end
+  end
+
+  class IntervalBuilder
+    def initialize(parsed_lines_enum, parser_columns, length)
+    end
+
+    def each_interval
+      interval = Interval.new(now, length, parser_columns)
+      parsed_lines_enum.each do |record|
+        while record.time < interval.start_time do
+          yield interval
+          interval = Interval.new(interval.start_time, length, aggregators)
+        end
+        interval.add(record)
       end
     end
   end
 
   module Aggregator
-    def self.klass(aggregator)
-      case aggregator
-      when :sum then Sum
+    class Base
+      def self.register(name, klass)
+        @aggregator_classes[name] = klass
+      end
+
+      def self.klass(name)
+        @aggregator_classes.fetch(name)
       end
     end
 
-    class Sum
+    class Sum < Base
+      register :sum, self
+
       def initialize
         @val = 0
       end
@@ -74,7 +110,7 @@ module LogfileInterval
       end
     end
 
-    class Count
+    class Count < Base
       def initialize
         @val = Counter.new
       end
@@ -90,6 +126,8 @@ module LogfileInterval
     end
 
     def each_line
+      return enum_for(:each_line) unless block_given?
+      ...
     end
 
     def each_parsed_line
@@ -114,23 +152,6 @@ module LogfileInterval
     end
   end
 
-  class IntervalBuilder
-    def initialize(logfile_set, length)
-      parser = logfile_set.parser
-    end
-
-    def each_interval
-      interval = Interval.new(now, length)
-      set.each_parsed_line(parser) do |record|
-        while record.time < interval.start_time do
-          yield interval
-          interval = Interval.new(interval.start_time, length)
-        end
-        interval.add(record)
-      end
-    end
-  end
-
   class Counter < Hash
     def increment(key)
       self[key] = self[key] ? self[key] + 1 : 1
@@ -138,10 +159,16 @@ module LogfileInterval
   end
 end
 
+class AccessLogParsedLine < LogfileInterval::Parse::Base
+  set_regex /blah/
+  add_column :name => :foo, :pos => 1, :conversion => integer, :aggregator => :average
+end
+
+
 logfiles = [ 'access.log', 'access.log.1', 'access.log.2' ]
 logfile = logfiles.first
 
-parser = LineParser::AccessLog
+parser = AccessLogParsedLine
 
 logfile_iterator = LogfileInterval::Logfile.new(logfile, parser)
 logfile_iterator.each_line do |line|
@@ -149,20 +176,19 @@ logfile_iterator.each_line do |line|
   puts line
 end
 
-parser = LineParser::AccessLog
 logfile_iterator.each_parsed_line do |record|
-  puts record.class # LineParser::AccessLog
+  puts record.class # ParsedLine::AccessLog
   puts record.ip
   puts record.time
 end
 
-set_iterator = LogfileInterval::LogfileSet.new(logfiles, parser)
-set_iterator.each_parsed_line do |record|
-  puts record.class # LineParser::AccessLog
+set = LogfileInterval::LogfileSet.new(logfiles, parser)
+set.each_parsed_line do |record|
+  puts record.class # ParsedLine::AccessLog
 end
 
 length = 5.minutes
-interval_builder = LogfileInterval::IntervalBuilder.new(logfiles, length)
+interval_builder = LogfileInterval::IntervalBuilder.new(set.each_parsed_line, parser, length)
 interval_builder.each_interval do |interval|
   puts interval.class  # LogfileInterval::Interval
   puts interval.start_time

data/lib/logfile_interval/aggregator/base.rb

@@ -1,9 +1,16 @@
+require File.join(File.expand_path('..', __FILE__), '/registrar')
+
 module LogfileInterval
   module Aggregator
     class Base
+
+      extend Registrar
       include Enumerable
 
+      attr_reader :name
+
       def initialize(options = {})
+        @name = options[:name]
         @val = Util::Counter.new
         @size = Util::Counter.new
         @options = options
@@ -21,11 +28,12 @@ module LogfileInterval
         end
       end
 
-      def add(value, group_by = nil)
+      def add(value, group_by_value = nil)
         raise NotImplementedError
       end
 
       private
+
       def key(group_by = nil)
         group_by ? group_by : :all
       end
@@ -51,3 +59,9 @@ module LogfileInterval
     end
   end
 end
+
+current_dir = File.expand_path('..', __FILE__)
+agg_files = Dir.glob("#{current_dir}/*.rb").reject { |file| file =~ /base\.rb/ || file =~ /registrar\.rb/ }
+agg_files.each do |agg_file|
+  require agg_file
+end

data/lib/logfile_interval/aggregator/count.rb

@@ -1,8 +1,16 @@
 module LogfileInterval
   module Aggregator
     class Count < Base
-      def add(value, group_by = nil)
-        @val.add(key(group_by), 1)
+      def each
+        @val.each { |k, v| yield k, v }
+      end
+
+      def add(value, group_by_value = nil)
+        if group_by_value
+          @val.increment_subkey(value, key(group_by_value))
+        else
+          @val.increment(value)
+        end
       end
     end
   end