logfile_interval 1.1.1 → 1.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 37c52b3b1355cf1589c84a6f2f8e76233222eb80
-  data.tar.gz: 5ec64eb915a57de6c94974756d4a3d8a58042dc6
+  metadata.gz: b10f4f6bcfa47f21b82b70bf541ef5403638c1ac
+  data.tar.gz: a0d4c42cd3f531b96fcbfc3aa8c7a0b4b2b3de1b
 SHA512:
-  metadata.gz: 2b6eb7f3ecbfc8b12fa5900c02ca45e6f58ec1730018500c860c3be7e63ee0a83f402ee000e9141720ec13f6d89c7585dc6befbde89ec86929f51707e8fc47f1
-  data.tar.gz: 865b1ae02eb35830c8723f3b9ad5b013a3f0a335eaecc05cc95f077f468db34855af91510a00d77d4a0caf45183f708263daa78c99cb6da9f7b12c574e2bc5be
+  metadata.gz: 954c42936dee2d249e2fc71f25cf175455f0350e4d96951bdddcb0c766ed18a53415c43a279746779df8d6e6ea89c83dc6e433272599bb0d5792d6f9b3a3f89e
+  data.tar.gz: 289e3c63c70eeb4d606bc5a5431d6ea0c69c277b6b31cb717bcdfd4ef28f5a2bce0d660ed3bd8de0d3257eed237d1baf7c8f3b64fd879f204553037ab95c4f46

data/.travis.yml

@@ -2,4 +2,5 @@ language: ruby
 rvm:
   - 1.9.3
   - 2.0.0
+  - 2.1.0
 script: bundle exec rspec spec

data/Gemfile.lock

@@ -1,18 +1,18 @@
 PATH
   remote: .
   specs:
-    logfile_interval (1.1.1)
+    logfile_interval (1.1.2)
 
 GEM
   remote: https://rubygems.org/
   specs:
     columnize (0.3.6)
-    debugger (1.6.2)
+    debugger (1.6.5)
       columnize (>= 0.3.1)
       debugger-linecache (~> 1.2.0)
-      debugger-ruby_core_source (~> 1.2.3)
+      debugger-ruby_core_source (~> 1.3.1)
     debugger-linecache (1.2.0)
-    debugger-ruby_core_source (1.2.4)
+    debugger-ruby_core_source (1.3.1)
     diff-lcs (1.2.5)
     docile (1.1.1)
     multi_json (1.8.2)

data/README.md

@@ -38,12 +38,7 @@ class AccessLog < LogfileInterval::LineParser::Base
   end
 end
 
-path = ENV['ACCESS_LOG_PATH']
-file = LogfileInterval::Logfile.new(path, AccessLog)
-unless file.exist?
-  puts "#{path} is not found"
-  exit 1
-end
+file = LogfileInterval::Logfile.new('path_to_logfile', AccessLog)
 
 builder = LogfileInterval::IntervalBuilder.new(file, 300)
 builder.each_interval do |interval|
@@ -184,6 +179,8 @@ interval_builder.each_interval do |interval|
   end
 end
 ```
+## Design document
+Design outline is at [design.rb](docs/design.rb).
 
 ## Installation
 Add this line to your application's Gemfile:

data/docs/design.rb

@@ -1,13 +1,4 @@
 module LogfileInterval
-  class Logfile
-  end
-
-  class LogfileSet
-  end
-
-  class Interval
-  end
-
   module LineParser
     class Base
       class << self
@@ -15,74 +6,165 @@ module LogfileInterval
         end
 
         def add_column(name, options)
+          agg = Aggregators.klass(aggregator)
+          @columns[name] = { :pos => pos, :aggregator => agg, :conversion => conversion }
+          define_method(name)
         end
 
         def parse(line)
-          @data = {}
-
           match_data = regex.match(line)
-          columns.each do |name, options|
-            val = match_data[options[:pos]]
-            @data[name] = convert(val, options[:conversion])
-          end
-          @data
+          data = f(match_data)
         end
 
-        def convert(val, conversion)
-          case options[:conversion]
-          when :integer then val.to_i
-          else val
-          end
+        def create_record(line)
+          record = new(line)
+          return record.valid? ? record : nil
         end
       end
-
     end
 
     class AccessLog < Base
       set_regex /blah/
       add_column :name => :foo, :pos => 1, :conversion => integer, :aggregator => :average
 
+      def initialize(line)
+        @data = self.class.parse(line)
+      end
+    end
+  end
+
+
+  class Interval
+    def initialize(end_time, length, parser)
+      @data = {}
+      parser.columns.each do |name, options|
+        @data[name] = options[:aggregator].new
+      end
+    end
+
+    def [](name)
+      @data[name].value
+    end
+
+    def add_record(record)
+      return unless record.valid?
+      raise ParserMismatch unless record.class == parser
+
+      @size += 1
+      parser.columns.each do |name, options|
+        @data[name].add(record[name])
+      end
+    end
+  end
+
+  module Aggregator
+    def self.klass(aggregator)
+      case aggregator
+      when :sum then Sum
+      end
+    end
+
+    class Sum
+      def initialize
+        @val = 0
+      end
+
+      def add(value)
+        @val += value
+      end
+    end
+
+    class Count
+      def initialize
+        @val = Counter.new
+      end
+
+      def add(value)
+        @val.increment(value)
+      end
+    end
+  end
+
+  class Logfile
+    def initialize(filename, parser)
+    end
+
+    def each_line
+    end
+
+    def each_parsed_line
+      each_line do |line|
+        record = parser.create_record(line)
+        yield record if record
+      end
     end
   end
 
-  class Record
-    def initialize(parser, line)
-      @parser = parser
-      @data = parser.parse(line)
+  class LogfileSet
+    def initialize(filenames_array, parser)
+    end
+
+    def ordered_filenames
+    end
+
+    def each_line
     end
 
-    def valid_columns
-      @parser.columns.keys
+    def each_parsed_line
     end
+  end
 
-    def method_missing(meth, *args)
-      if valid_columns.include?(meth) && args.none
-        self[meth]
-      else
-        super
+  class IntervalBuilder
+    def initialize(logfile_set, length)
+      parser = logfile_set.parser
+    end
+
+    def each_interval
+      interval = Interval.new(now, length)
+      set.each_parsed_line(parser) do |record|
+        while record.time < interval.start_time do
+          yield interval
+          interval = Interval.new(interval.start_time, length)
+        end
+        interval.add(record)
       end
     end
   end
+
+  class Counter < Hash
+    def increment(key)
+      self[key] = self[key] ? self[key] + 1 : 1
+    end
+  end
 end
 
 logfiles = [ 'access.log', 'access.log.1', 'access.log.2' ]
 logfile = logfiles.first
-parser = LineParser::AccessLog.new
 
-logfile_iterator = LogfileInterval::Logfile.new(parser, logfile)
+parser = LineParser::AccessLog
 
+logfile_iterator = LogfileInterval::Logfile.new(logfile, parser)
 logfile_iterator.each_line do |line|
+  puts line.class # String
   puts line
 end
 
+parser = LineParser::AccessLog
 logfile_iterator.each_parsed_line do |record|
+  puts record.class # LineParser::AccessLog
   puts record.ip
   puts record.time
 end
 
-interval_builder = LogfileInterval::Interval.new(parser, logfiles)
+set_iterator = LogfileInterval::LogfileSet.new(logfiles, parser)
+set_iterator.each_parsed_line do |record|
+  puts record.class # LineParser::AccessLog
+end
 
+length = 5.minutes
+interval_builder = LogfileInterval::IntervalBuilder.new(logfiles, length)
 interval_builder.each_interval do |interval|
+  puts interval.class  # LogfileInterval::Interval
   puts interval.start_time
   puts interval.length
   interval[:ip].each do |ip, count|

data/lib/logfile_interval/aggregator/average.rb

@@ -0,0 +1,14 @@
+module LogfileInterval
+  module Aggregator
+    class Average < Base
+      def add(value, group_by = nil)
+        @val.add(key(group_by), value)
+        @size.increment(key(group_by))
+      end
+
+      def val(k)
+        average(k)
+      end
+    end
+  end
+end

data/lib/logfile_interval/aggregator/base.rb

@@ -0,0 +1,52 @@
+module LogfileInterval
+  module Aggregator
+    class Base
+      include Enumerable
+
+      def initialize
+        @val = Util::Counter.new
+        @size = Util::Counter.new
+      end
+
+      def value(group = nil)
+        val(key(group))
+      end
+
+      def values
+        if single_value?
+          value
+        else
+          self.inject({}) { |h, v| h[v[0]] = v[1]; h }
+        end
+      end
+
+      def add(value, group_by = nil)
+        raise NotImplementedError
+      end
+
+      private
+      def key(group_by = nil)
+        group_by ? group_by : :all
+      end
+
+      def single_value?
+        return true if @val.empty?
+        @val.keys.count == 1 && @val.keys.first == :all
+      end
+
+      def each
+        @val.each_key do |k|
+          yield k, val(k)
+        end
+      end
+
+      def val(k)
+        @val[k]
+      end
+
+      def average(k)
+        @size[k] > 0 ? @val[k].to_f / @size[k].to_f : 0
+      end
+    end
+  end
+end

data/lib/logfile_interval/aggregator/count.rb

@@ -0,0 +1,9 @@
+module LogfileInterval
+  module Aggregator
+    class Count < Base
+      def add(value, group_by = nil)
+        @val.add(key(group_by), 1)
+      end
+    end
+  end
+end

data/lib/logfile_interval/aggregator/delta.rb

@@ -0,0 +1,22 @@
+module LogfileInterval
+  module Aggregator
+    class Delta < Base
+      def initialize
+        @previous = Util::Counter.new
+        super
+      end
+
+      def add(value, group_by = nil)
+        if @previous.has_key?(key(group_by))
+          @val.add(key(group_by), @previous[key(group_by)] - value)
+          @size.increment(key(group_by))
+        end
+        @previous.set(key(group_by), value)
+      end
+
+      def val(k)
+        average(k)
+      end
+    end
+  end
+end

data/lib/logfile_interval/aggregator/group_and_count.rb

@@ -0,0 +1,14 @@
+module LogfileInterval
+  module Aggregator
+    class GroupAndCount < Base
+      def each
+        @val.each { |k, v| yield k, v }
+      end
+
+      def add(value, group_by)
+        raise ArgumentError, 'group_by argument is mandatory for GroupAndCount#add' unless group_by
+        @val.increment_subkey(value, key(group_by))
+      end
+    end
+  end
+end

data/lib/logfile_interval/aggregator/sum.rb

@@ -0,0 +1,9 @@
+module LogfileInterval
+  module Aggregator
+    class Sum < Base
+      def add(value, group_by = nil)
+        @val.add(key(group_by), value)
+      end
+    end
+  end
+end

data/lib/logfile_interval/aggregator.rb

@@ -0,0 +1,24 @@
+lib_dir = File.expand_path('..', __FILE__)
+
+puts "lib_dir=#{lib_dir}"
+
+require "#{lib_dir}/aggregator/base"
+require "#{lib_dir}/aggregator/sum"
+require "#{lib_dir}/aggregator/count"
+require "#{lib_dir}/aggregator/group_and_count"
+require "#{lib_dir}/aggregator/average"
+require "#{lib_dir}/aggregator/delta"
+
+module LogfileInterval
+  module Aggregator
+    def self.klass(aggregator)
+      case aggregator
+      when :sum               then Sum
+      when :average           then Average
+      when :count             then Count
+      when :group_and_count   then GroupAndCount
+      when :delta             then Delta
+      end
+    end
+  end
+end

data/lib/logfile_interval/interval.rb

@@ -29,6 +29,15 @@ module LogfileInterval
       @data.each(&block)
     end
 
+    def to_hash
+      @data.inject({}) do |h, pair|
+        k = pair[0]
+        v = pair[1]
+        h[k] = v.values
+        h
+      end
+    end
+
     def add_record(record)
       return unless record.valid?
       raise ParserMismatch unless record.class == parser

data/lib/logfile_interval/interval_builder.rb

@@ -9,6 +9,8 @@ module LogfileInterval
     end
 
     def each_interval
+      return enum_for(:each_interval) unless block_given?
+      
       secs = (Time.now.to_i / length.to_i) * length.to_i
       rounded_end_time = Time.at(secs)
       current_interval = Interval.new(rounded_end_time, length, parser)

data/lib/logfile_interval/logfile.rb

@@ -24,7 +24,9 @@ module LogfileInterval
 
     def each_line
       return unless exist?
-      f = FileBackward.new(@filename)
+      return enum_for(:each_line) unless block_given?
+
+      f = Util::FileBackward.new(@filename)
       while(line = f.gets)
         yield line.chomp
       end
@@ -32,6 +34,7 @@ module LogfileInterval
     end
 
     def each_parsed_line
+      return enum_for(:each_parsed_line) unless block_given?
       each_line do |line|
         record = parser.create_record(line)
         yield record if record

data/lib/logfile_interval/logfile_set.rb

@@ -21,6 +21,8 @@ module LogfileInterval
     end
 
     def each_parsed_line
+      return enum_for(:each_parsed_line) unless block_given?
+
       ordered_filenames.each do |filename|
         tfile = Logfile.new(filename, parser)
         tfile.each_parsed_line do |record|
@@ -30,6 +32,8 @@ module LogfileInterval
     end
 
     def each_line
+      return enum_for(:each_line) unless block_given?
+
       ordered_filenames.each do |filename|
         tfile = Logfile.new(filename, parser)
         tfile.each_line do |line|

data/lib/logfile_interval/{line_parser → util}/counter.rb

@@ -1,5 +1,5 @@
 module LogfileInterval
-  module LineParser
+  module Util
     class Counter < Hash
       def increment(key)
         if self.has_key?(key)

data/lib/logfile_interval/util/file_backward.rb

@@ -0,0 +1,51 @@
+module LogfileInterval
+  module Util
+    # Based on Perl's File::ReadBackwards module, by Uri Guttman.
+    class FileBackward
+      MAX_READ_SIZE = 1 << 10 # 1024
+
+      def initialize( *args )
+        return unless File.exist?(args[0])
+        @file = File.new(*args)
+        @file.seek(0, IO::SEEK_END)
+
+        @current_pos = @file.pos
+
+        @read_size = @file.pos % MAX_READ_SIZE
+        @read_size = MAX_READ_SIZE if @read_size.zero?
+
+        @line_buffer = Array.new
+      end
+
+      def gets( sep_string = $/ )
+        return nil unless @file
+        return @line_buffer.pop if @line_buffer.size > 2 or @current_pos.zero?
+
+        @current_pos -= @read_size
+        @file.seek(@current_pos, IO::SEEK_SET)
+
+        @line_buffer[0] = "#{@file.read(@read_size)}#{@line_buffer[0]}"
+        @read_size = MAX_READ_SIZE # Set a size for the next read.
+
+        @line_buffer[0] =
+        @line_buffer[0].scan(/.*?#{Regexp.escape(sep_string)}|.+/)
+        @line_buffer.flatten!
+
+        gets(sep_string)
+      end
+
+      def close
+        return unless @file
+        @file.close()
+      end
+    end
+  end
+end
+
+# f = FileBackward.new('../log/development.log')
+# i = 0
+# while(line = f.gets())
+#     puts line
+#     i += 1
+#     break if i>30
+# end

data/lib/logfile_interval/version.rb

@@ -1,3 +1,3 @@
 module LogfileInterval
-  VERSION = "1.1.1"
+  VERSION = "1.1.2"
 end

data/lib/logfile_interval.rb

@@ -1,14 +1,14 @@
 lib_dir = File.expand_path('..', __FILE__)
 
 require "#{lib_dir}/logfile_interval/version"
-require "#{lib_dir}/logfile_interval/file_backward"
 require "#{lib_dir}/logfile_interval/interval"
 require "#{lib_dir}/logfile_interval/interval_builder"
 require "#{lib_dir}/logfile_interval/logfile"
 require "#{lib_dir}/logfile_interval/logfile_set"
 require "#{lib_dir}/logfile_interval/line_parser/base"
-require "#{lib_dir}/logfile_interval/line_parser/aggregator"
-require "#{lib_dir}/logfile_interval/line_parser/counter"
+require "#{lib_dir}/logfile_interval/util/counter"
+require "#{lib_dir}/logfile_interval/util/file_backward"
+require "#{lib_dir}/logfile_interval/aggregator"
 
 module LogfileInterval
 end