logfile_interval 1.1.1 → 1.1.2

data/docs/design3.rb

@@ -1,177 +0,0 @@
-module LogfileInterval
-  module LineParser
-    class Base
-      class << self
-        def set_regex(regex)
-        end
-
-        def add_column(name, options)
-          agg = Aggregators.klass(aggregator)
-          @columns[name] = { :pos => pos, :aggregator => agg, :conversion => conversion }
-          define_method(name)
-        end
-
-        def parse(line)
-          match_data = regex.match(line)
-          @data = f(match_data)
-        end
-
-        def create_record(line)
-          record = new(line)
-          return record.valid? ? record : nil
-        end
-      end
-
-    end
-
-    class AccessLog < Base
-      set_regex /blah/
-      add_column :name => :foo, :pos => 1, :conversion => integer, :aggregator => :average
-
-      def initialize(line)
-        @data = self.class.parse(line)
-      end
-    end
-
-    module Aggregator
-      def self.klass(aggregator)
-        case aggregator
-        when :sum then Sum
-        end
-      end
-
-      class Sum
-        def initialize
-          @val = 0
-        end
-
-        def add(value)
-          @val += value
-        end
-
-        def value
-          @val
-        end
-      end
-
-      class Count
-        def initialize
-          @val = Counter.new
-        end
-
-        def add(value)
-          @val.increment(value)
-        end
-      end
-    end
-  end
-
-  class Logfile
-    def initialize(filename, parser)
-    end
-
-    def each_line
-    end
-
-    def each_parsed_line
-      each_line do |line|
-        record = parser.create_record(line)
-        yield record if record
-      end
-    end
-  end
-
-  class LogfileSet
-    def initialize(filenames_array, parser)
-    end
-
-    def ordered_filenams
-    end
-
-    def each_line
-    end
-
-    def each_parsed_line
-    end
-  end
-
-  class IntervalBuilder
-    def initialize(logfile_set, length)
-      parser = logfile_set.parser
-    end
-
-    def each_interval
-      interval = Interval.new(now, length)
-      set.each_parsed_line(parser) do |record|
-        while record.time < interval.start_time do
-          yield interval
-          interval = Interval.new(interval.start_time, length)
-        end
-        interval.add(record)
-      end
-    end
-  end
-
-  class Counter < Hash
-    def increment(key)
-      self[key] = self[key] ? self[key] + 1 : 1
-    end
-  end
-
-  class Interval
-    def initialize(end_time, length, parser)
-      @data = {}
-      parser.columns.each do |name, options|
-        @data[name] = options[:aggregator].new
-      end
-    end
-
-    def [](name)
-      @data[name].value
-    end
-
-    def add_record(record)
-      return unless record.valid?
-      raise ParserMismatch unless record.class == parser
-
-      @size += 1
-      parser.columns.each do |name, options|
-        @data[name].add(record[name])
-      end
-    end
-  end
-end
-
-logfiles = [ 'access.log', 'access.log.1', 'access.log.2' ]
-logfile = logfiles.first
-
-parser = LineParser::AccessLog
-
-logfile_iterator = LogfileInterval::Logfile.new(logfile, parser)
-logfile_iterator.each_line do |line|
-  puts line.class # String
-  puts line
-end
-
-parser = LineParser::AccessLog
-logfile_iterator.each_parsed_line do |record|
-  puts record.class # LineParser::AccessLog
-  puts record.ip
-  puts record.time
-end
-
-set_iterator = LogfileInterval::LogfileSet.new(logfiles, parser)
-set_iterator.each_parsed_line do |record|
-  puts record.class # LineParser::AccessLog
-end
-
-length = 5.minutes
-interval_builder = LogfileInterval::IntervalBuilder.new(logfiles, length)
-interval_builder.each_interval do |interval|
-  puts interval.class  # LogfileInterval::Interval
-  puts interval.start_time
-  puts interval.length
-  interval[:ip].each do |ip, count|
-    puts "#{ip}, #{count}"
-  end
-end

data/lib/logfile_interval/file_backward.rb

@@ -1,49 +0,0 @@
-module LogfileInterval
-  # Based on Perl's File::ReadBackwards module, by Uri Guttman.
-  class FileBackward
-    MAX_READ_SIZE = 1 << 10 # 1024
-
-    def initialize( *args )
-      return unless File.exist?(args[0])
-      @file = File.new(*args)
-      @file.seek(0, IO::SEEK_END)
-
-      @current_pos = @file.pos
-
-      @read_size = @file.pos % MAX_READ_SIZE
-      @read_size = MAX_READ_SIZE if @read_size.zero?
-
-      @line_buffer = Array.new
-    end
-
-    def gets( sep_string = $/ )
-      return nil unless @file
-      return @line_buffer.pop if @line_buffer.size > 2 or @current_pos.zero?
-
-      @current_pos -= @read_size
-      @file.seek(@current_pos, IO::SEEK_SET)
-
-      @line_buffer[0] = "#{@file.read(@read_size)}#{@line_buffer[0]}"
-      @read_size = MAX_READ_SIZE # Set a size for the next read.
-
-      @line_buffer[0] =
-      @line_buffer[0].scan(/.*?#{Regexp.escape(sep_string)}|.+/)
-      @line_buffer.flatten!
-
-      gets(sep_string)
-    end
-
-    def close
-      return unless @file
-      @file.close()
-    end
-  end
-end
-
-# f = FileBackward.new('../log/development.log')
-# i = 0
-# while(line = f.gets())
-#     puts line
-#     i += 1
-#     break if i>30
-# end

data/lib/logfile_interval/interval_length.rb

@@ -1,47 +0,0 @@
-module LogfileInterval
-  class IntervalLength
-    MAX_PERIODS = { 5 * 60           => 6 * 3600,
-                    3600             => 3600 * 24,
-                    3600 * 24        => 3600 * 24 * 30,
-                    3600 * 24 * 30   => 365 * 3600 * 24 }
-    LENGTHS = MAX_PERIODS.keys.sort
-
-    attr_reader :length
-
-    def initialize(l)
-      raise ArgumentError unless LENGTHS.include?(l)
-      @length = l
-    end
-
-    def lower
-      pos = LENGTHS.index(@length)
-      return nil if pos==0
-      IntervalLength.new(LENGTHS[pos-1])
-    end
-
-    def higher
-      pos = LENGTHS.index(@length)
-      return nil if pos==LENGTHS.size-1
-      IntervalLength.new(LENGTHS[pos+1])
-    end
-
-    def smallest?
-      pos = LENGTHS.index(@length)
-      pos == 0
-    end
-
-    def start_time(t)
-      ts = (t.to_i / @length.to_i) * @length.to_i
-      ts -= @length.to_i if t.to_i % @length.to_i == 0
-      Time.at(ts)
-    end
-
-    def end_time(t)
-      start_time(t) + @length
-    end
-
-    def to_i
-      @length.to_i
-    end
-  end
-end

data/lib/logfile_interval/line_parser/aggregator.rb

@@ -1,117 +0,0 @@
-module LogfileInterval
-  module LineParser
-    module Aggregator
-      def self.klass(aggregator)
-        case aggregator
-        when :sum               then Sum
-        when :average           then Average
-        when :count             then Count
-        when :group_and_count   then GroupAndCount
-        when :delta             then Delta
-        end
-      end
-
-      class Base
-        include Enumerable
-
-        def initialize
-          @val = Counter.new
-          @size = Counter.new
-        end
-
-        def value(group = nil)
-          val(key(group))
-        end
-
-        def values
-          if single_value?
-            value
-          else
-            self.inject({}) { |h, v| h[v[0]] = v[1]; h }
-          end
-        end
-
-        def add(value, group_by = nil)
-          raise NotImplementedError
-        end
-
-        private
-        def key(group_by = nil)
-          group_by ? group_by : :all
-        end
-
-        def single_value?
-          return true if @val.empty?
-          @val.keys.count == 1 && @val.keys.first == :all
-        end
-
-        def each
-          @val.each_key do |k|
-            yield k, val(k)
-          end
-        end
-
-        def val(k)
-          @val[k]
-        end
-
-        def average(k)
-          @size[k] > 0 ? @val[k].to_f / @size[k].to_f : 0
-        end
-      end
-
-      class Sum < Base
-        def add(value, group_by = nil)
-          @val.add(key(group_by), value)
-        end
-      end
-
-      class Average < Base
-        def add(value, group_by = nil)
-          @val.add(key(group_by), value)
-          @size.increment(key(group_by))
-        end
-
-        def val(k)
-          average(k)
-        end
-      end
-
-      class Count < Base
-        def add(value, group_by = nil)
-          @val.add(key(group_by), 1)
-        end
-      end
-
-      class GroupAndCount < Base
-        def each
-          @val.each { |k, v| yield k, v }
-        end
-
-        def add(value, group_by)
-          raise ArgumentError, 'group_by argument is mandatory for GroupAndCount#add' unless group_by
-          @val.increment_subkey(value, key(group_by))
-        end
-      end
-
-      class Delta < Base
-        def initialize
-          @previous = Counter.new
-          super
-        end
-
-        def add(value, group_by = nil)
-          if @previous.has_key?(key(group_by))
-            @val.add(key(group_by), @previous[key(group_by)] - value)
-            @size.increment(key(group_by))
-          end
-          @previous.set(key(group_by), value)
-        end
-
-        def val(k)
-          average(k)
-        end
-      end
-    end
-  end
-end

data/spec/lib/line_parser/aggregator_spec.rb

@@ -1,211 +0,0 @@
-require 'spec_helper'
-
-module LogfileInterval
-  module LineParser
-    module Aggregator
-      describe Aggregator do
-        it 'finds the aggregator class' do
-          Aggregator.klass(:sum).should == Sum
-          Aggregator.klass(:average).should == Average
-          Aggregator.klass(:count).should == Count
-          Aggregator.klass(:group_and_count).should == GroupAndCount
-          Aggregator.klass(:delta).should == Delta
-        end
-      end
-
-      shared_examples 'an aggregator' do
-        let(:aggregator) { described_class.new }
-
-        [ :add, :value, :values ].each do |method|
-          it "responds to #{method}" do
-            aggregator.should respond_to(method)
-          end
-        end
-
-        context 'values' do
-          context 'with one group' do
-            before :each do
-              aggregator.add(5, :key1)
-            end
-
-            it 'returns a hash' do
-              aggregator.values.should be_a(Hash) unless aggregator.is_a?(Delta)
-            end
-          end
-
-          context 'with several groups' do
-            before :each do
-              aggregator.add(5, :key1)
-              aggregator.add(3, :key2)
-              aggregator.add(3, :key1)
-            end
-
-            it 'returns a hash' do
-              aggregator.values.should be_a(Hash)
-            end
-          end
-
-          context 'with no group' do
-            before :each do
-              aggregator.add(5)
-              aggregator.add(3)
-            end
-
-            it 'returns a numeric' do
-              aggregator.values.should be_a(Numeric) unless aggregator.is_a?(Count)
-            end
-          end
-        end
-      end
-
-      [ Count, Sum, Average, Delta ]. each do |klass|
-        describe klass do
-          it_behaves_like 'an aggregator'
-        end
-      end
-
-
-      describe 'without group_by key' do
-        describe Sum do
-          it 'sums up values' do
-            sum = Sum.new
-            sum.add(3)
-            sum.add(5)
-            sum.value.should == 8
-          end
-        end
-
-        describe Average do
-          it 'averages values' do
-            avg = Average.new
-            avg.add(3)
-            avg.add(5)
-            avg.value.should == 4
-          end
-        end
-
-        describe Delta do
-          it 'averages delta values' do
-            d = Delta.new
-            d.add(1.4)
-            d.add(1.1)
-            d.add(1.0)
-            d.value.round(5).should == 0.2
-          end
-        end
-
-        describe Count do
-          it 'groups values and increment counters' do
-            g = Count.new
-            g.add('200')
-            g.add('500')
-            g.add('301')
-            g.add('200')
-            g.value.should == 4
-          end
-        end
-      end
-
-      describe 'with group_by key' do
-
-        describe Sum do
-          it 'sums up values by key' do
-            sum = Sum.new
-            sum.add(3, :key1)
-            sum.add(5, :key2)
-            sum.add(5, :key1)
-            sum.values.should be_a(Hash)
-            sum.values.size.should == 2
-            sum.value(:key1).should == 8
-            sum.values[:key1].should == 8
-            sum.value(:key2).should == 5
-            sum.values[:key2].should == 5
-          end
-        end
-
-
-        describe Average do
-          it 'averages values by key' do
-            avg = Average.new
-            avg.add(3, :key1)
-            avg.add(5, :key2)
-            avg.add(5, :key1)
-            avg.values.should be_a(Hash)
-            avg.values.size.should == 2
-            avg.value(:key1).should == 4
-            avg.values[:key1].should == 4
-            avg.value(:key2).should == 5
-            avg.values[:key2].should == 5
-          end
-        end
-
-        describe Count do
-          it 'groups values and increment counters' do
-            g = Count.new
-            g.add('200', '200')
-            g.add('500', '500')
-            g.add('301', '301')
-            g.add('200', '200')
-            g.values.should be_a(Hash)
-            g.values.should include({'200' => 2})
-            g.values.should include({'301' => 1})
-            g.values.should include({'500' => 1})
-          end
-        end
-
-        describe GroupAndCount do
-          it 'each yields a key and a hash' do
-            gac = GroupAndCount.new
-            gac.add :key1, :subkey1
-            gac.first.should be_an(Array)
-            gac.first.size.should == 2
-            gac.first[1].should be_a(Hash)
-          end
-
-          context :add do
-            before :each do
-              @gac = GroupAndCount.new
-            end
-
-            it 'requires a group_by argument' do
-              lambda { @gac.add('foo') }.should raise_error ArgumentError
-            end
-
-            it 'counts number of occurence of subkey for key' do
-              @gac.add :key1, :subkey1
-              @gac.add :key1, :subkey2
-              @gac.add :key2, :subkey1
-              @gac.add :key2, :subkey1
-              @gac.add :key2, :subkey3
-
-              @gac.values[:key1][:subkey1].should == 1
-              @gac.values[:key1][:subkey2].should == 1
-              @gac.values[:key2][:subkey1].should == 2
-              @gac.values[:key2][:subkey2].should == 0
-              @gac.values[:key2][:subkey3].should == 1
-            end
-          end
-        end
-
-        describe Delta do
-          it 'averages deltas by key' do
-            d = Delta.new
-            d.add(9, :key1)
-            d.add(10, :key2)
-            d.add(5, :key1)
-            d.add(8, :key2)
-            d.add(3, :key1)
-            d.add(5, :key2)
-            d.values.should be_a(Hash)
-            d.values.size.should == 2
-            d.value(:key1).should == 3
-            d.values[:key1].should == 3
-            d.value(:key2).should == 2.5
-            d.values[:key2].should == 2.5
-          end
-        end
-
-      end
-    end
-  end
-end