logfile_interval 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 304480b4bf3c9525f74bf9b835b3ec7fe72b64be
+  data.tar.gz: ca5f34f5a987aca00b22ea838e083c744793c545
+SHA512:
+  metadata.gz: e36b8d0c42fbaafed596dcb5c2770450bf7e10ce68d92326f3f04eab197bbb4faa8ea8fa0f7706b55883fd2d65a876435ab689533222148151787c3c58861ba6
+  data.tar.gz: 34bd857fd154d29703959693c7b33b0e15b4ab0824f97e84f4362aad1e009a332e8f0b1c96fdcde1321f3462666e3bd862c9f53cb15aab41807f6321346551b0

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+.kateproject
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in logfile_interval.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,35 @@
+PATH
+  remote: .
+  specs:
+    logfile_interval (0.0.1)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    columnize (0.3.6)
+    debugger (1.6.2)
+      columnize (>= 0.3.1)
+      debugger-linecache (~> 1.2.0)
+      debugger-ruby_core_source (~> 1.2.3)
+    debugger-linecache (1.2.0)
+    debugger-ruby_core_source (1.2.4)
+    diff-lcs (1.2.5)
+    rake (10.1.0)
+    rspec (2.14.1)
+      rspec-core (~> 2.14.0)
+      rspec-expectations (~> 2.14.0)
+      rspec-mocks (~> 2.14.0)
+    rspec-core (2.14.7)
+    rspec-expectations (2.14.4)
+      diff-lcs (>= 1.1.3, < 2.0)
+    rspec-mocks (2.14.4)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.3)
+  debugger
+  logfile_interval!
+  rake
+  rspec (~> 2.14.0)

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Philippe Le Rohellec
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,94 @@
+# LogfileInterval
+
+Logfile parser and aggregator
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'logfile_interval'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install logfile_interval
+
+## Usage
+
+### Write a LineParser class
+```ruby
+module LogfileInterval
+  module LineParser
+    class AccessLog < Base
+      # Example line:
+      # 74.75.19.145 - - [31/Mar/2013:06:54:12 -0700] "GET /ppa/google_chrome HTTP/1.1" 200 7855 "https://www.google.com/" "Mozilla/5.0 Chrome/25.0.1364.160"
+
+      set_regex /^([\d\.]+)\s+\S+\s+\S+\s+\[(\d\d.*\d\d)\]\s+"(?:GET|POST|PUT|HEAD|DELETE)\s+(\S+)\s+HTTP\S+"\s+(\d+)\s+(\d+)\s+"([^"]*)"\s+"([^"]+)"$/
+
+      add_column :name => 'ip',        :pos => 1, :agg_function => :group
+      add_column :name => 'timestamp', :pos => 2, :agg_function => :timestamp
+      add_column :name => 'code',      :pos => 4, :agg_function => :group
+      add_column :name => 'length',    :pos => 5, :agg_function => :average,   :conversion => :integer
+
+      def time
+        Time.strptime(self.timestamp, '%d/%b/%Y:%H:%M:%S %z')
+      end
+    end
+  end
+end
+```
+
+### Iterate through lines of a single file
+And get a parsed record for each line.
+```ruby
+logfile = 'access.log'
+parser = LineParser::AccessLog
+
+log = LogfileInterval::Logfile.new(logfile, parser)
+log.each_line do |line|
+  puts line.class # String
+  puts line
+end
+
+parser = LineParser::AccessLog
+log.each_parsed_line do |record|
+  puts record.class # LineParser::AccessLog
+  puts record.ip
+  puts record.time
+end
+```
+
+### Iterate through lines of multiples files
+And get a parsed record for each line.
+```ruby
+logfiles = [ 'access.log', 'access.log.1', 'access.log.2' ]
+set = LogfileInterval::LogfileSet.new(logfiles, parser)
+set.each_parsed_line do |record|
+  puts record.class # LineParser::AccessLog
+end
+```
+### Aggregate lines into intervals
+```ruby
+length = 5.minutes
+interval_builder = LogfileInterval::IntervalBuilder.new(logfiles, length)
+interval_builder.each_interval do |interval|
+  puts interval.class  # LogfileInterval::Interval
+  puts interval.start_time
+  puts interval.length
+  interval[:ip].each do |ip, count|
+    puts "#{ip}, #{count}"
+  end
+end
+```
+
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/docs/design.rb

@@ -0,0 +1,91 @@
+module LogfileInterval
+  class Logfile
+  end
+
+  class LogfileSet
+  end
+
+  class Interval
+  end
+
+  module LineParser
+    class Base
+      class << self
+        def set_regex(regex)
+        end
+
+        def add_column(name, options)
+        end
+
+        def parse(line)
+          @data = {}
+
+          match_data = regex.match(line)
+          columns.each do |name, options|
+            val = match_data[options[:pos]]
+            @data[name] = convert(val, options[:conversion])
+          end
+          @data
+        end
+
+        def convert(val, conversion)
+          case options[:conversion]
+          when :integer then val.to_i
+          else val
+          end
+        end
+      end
+
+    end
+
+    class AccessLog < Base
+      set_regex /blah/
+      add_column :name => :foo, :pos => 1, :conversion => integer, :agg_function => :average
+
+    end
+  end
+
+  class Record
+    def initialize(parser, line)
+      @parser = parser
+      @data = parser.parse(line)
+    end
+
+    def valid_columns
+      @parser.columns.keys
+    end
+
+    def method_missing(meth, *args)
+      if valid_columns.include?(meth) && args.none
+        self[meth]
+      else
+        super
+      end
+    end
+  end
+end
+
+logfiles = [ 'access.log', 'access.log.1', 'access.log.2' ]
+logfile = logfiles.first
+parser = LineParser::AccessLog.new
+
+logfile_iterator = LogfileInterval::Logfile.new(parser, logfile)
+
+logfile_iterator.each_line do |line|
+  puts line
+end
+
+logfile_iterator.each_parsed_line do |record|
+  puts record.ip
+  puts record.time
+end
+
+interval_builder = LogfileInterval::Interval.new(parser, logfiles)
+
+interval_builder.each_interval do |interval|
+  puts interval.start_time
+  puts interval.length
+  interval[:ip].each do |ip, count|
+    puts "#{ip}, #{count}"
+  end
+end

data/docs/design2.rb

@@ -0,0 +1,77 @@
+module LogfileInterval
+  class Logfile
+  end
+
+  class LogfileSet
+  end
+
+  class Interval
+  end
+
+  class LineParser
+    def initialize(regex, columns)
+    end
+
+    def parse(line)
+      @data = {}
+
+      match_data = regex.match(line)
+      columns.each do |name, options|
+        val = match_data[options[:pos]]
+        @data[name] = convert(val, options[:conversion])
+      end
+      @data
+    end
+
+    def convert(val, conversion)
+      case options[:conversion]
+      when :integer then val.to_i
+      else val
+      end
+    end
+  end
+
+  class Record
+    def initialize(parser, line)
+      @parser = parser
+      @data = parser.parse(line)
+    end
+
+    def valid_columns
+      @parser.columns.keys
+    end
+
+    def method_missing(meth, *args)
+      if valid_columns.include?(meth) && args.none
+        self[meth]
+      else
+        super
+      end
+    end
+  end
+end
+
+logfiles = [ 'access.log', 'access.log.1', 'access.log.2' ]
+logfile = logfiles.first
+parser = LineParser.new(/blah/, [ { :name=>:ip , :pos => 1 }, ...])
+
+logfile_iterator = LogfileInterval::Logfile.new(parser, logfile)
+
+logfile_iterator.each_line do |line|
+  puts line
+end
+
+logfile_iterator.each_parsed_line do |record|
+  puts record.ip
+  puts record.time
+end
+
+interval_builder = LogfileInterval::Interval.new(parser, logfiles)
+
+interval_builder.each_interval do |interval|
+  puts interval.start_time
+  puts interval.length
+  interval[:ip].each do |ip, count|
+    puts "#{ip}, #{count}"
+  end
+end

data/docs/design3.rb

@@ -0,0 +1,177 @@
+module LogfileInterval
+  module LineParser
+    class Base
+      class << self
+        def set_regex(regex)
+        end
+
+        def add_column(name, options)
+          aggregator = Aggregators.klass(agg_function)
+          @columns[name] = { :pos => pos, :agg_function => aggregator, :conversion => conversion }
+          define_method(name)
+        end
+
+        def parse(line)
+          match_data = regex.match(line)
+          @data = f(match_data)
+        end
+
+        def create_record(line)
+          record = new(line)
+          return record.valid? ? record : nil
+        end
+      end
+
+    end
+
+    class AccessLog < Base
+      set_regex /blah/
+      add_column :name => :foo, :pos => 1, :conversion => integer, :agg_function => :average
+
+      def initialize(line)
+        @data = self.class.parse(line)
+      end
+    end
+
+    module Aggregator
+      def self.klass(agg_function)
+        case agg_function
+        when :sum then Sum
+        end
+      end
+
+      class Sum
+        def initialize
+          @val = 0
+        end
+
+        def add(value)
+          @val += value
+        end
+
+        def value
+          @val
+        end
+      end
+
+      class Group
+        def initialize
+          @val = Counter.new
+        end
+
+        def add(value)
+          @val.increment(value)
+        end
+      end
+    end
+  end
+
+  class Logfile
+    def initialize(filename, parser)
+    end
+
+    def each_line
+    end
+
+    def each_parsed_line
+      each_line do |line|
+        record = parser.create_record(line)
+        yield record if record
+      end
+    end
+  end
+
+  class LogfileSet
+    def initialize(filenames_array, parser)
+    end
+
+    def ordered_filenams
+    end
+
+    def each_line
+    end
+
+    def each_parsed_line
+    end
+  end
+
+  class IntervalBuilder
+    def initialize(logfile_set, length)
+      parser = logfile_set.parser
+    end
+
+    def each_interval
+      interval = Interval.new(now, length)
+      set.each_parsed_line(parser) do |record|
+        while record.time < interval.start_time do
+          yield interval
+          interval = Interval.new(interval.start_time, length)
+        end
+        interval.add(record)
+      end
+    end
+  end
+
+  class Counter < Hash
+    def increment(key)
+      self[key] = self[key] ? self[key] + 1 : 1
+    end
+  end
+
+  class Interval
+    def initialize(end_time, length, parser)
+      @data = {}
+      parser.columns.each do |name, options|
+        @data[name] = options[:aggregator].new
+      end
+    end
+
+    def [](name)
+      @data[name].value
+    end
+
+    def add_record(record)
+      return unless record.valid?
+      raise ParserMismatch unless record.class == parser
+
+      @size += 1
+      parser.columns.each do |name, options|
+        @data[name].add(record[name])
+      end
+    end
+  end
+end
+
+logfiles = [ 'access.log', 'access.log.1', 'access.log.2' ]
+logfile = logfiles.first
+
+parser = LineParser::AccessLog
+
+logfile_iterator = LogfileInterval::Logfile.new(logfile, parser)
+logfile_iterator.each_line do |line|
+  puts line.class # String
+  puts line
+end
+
+parser = LineParser::AccessLog
+logfile_iterator.each_parsed_line do |record|
+  puts record.class # LineParser::AccessLog
+  puts record.ip
+  puts record.time
+end
+
+set_iterator = LogfileInterval::LogfileSet.new(logfiles, parser)
+set_iterator.each_parsed_line do |record|
+  puts record.class # LineParser::AccessLog
+end
+
+length = 5.minutes
+interval_builder = LogfileInterval::IntervalBuilder.new(logfiles, length)
+interval_builder.each_interval do |interval|
+  puts interval.class  # LogfileInterval::Interval
+  puts interval.start_time
+  puts interval.length
+  interval[:ip].each do |ip, count|
+    puts "#{ip}, #{count}"
+  end
+end