logfile_interval 1.0.0

data/lib/logfile_interval/version.rb

@@ -0,0 +1,3 @@
+module LogfileInterval
+  VERSION = "1.0.0"
+end

data/logfile_interval.gemspec

@@ -0,0 +1,25 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'logfile_interval/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "logfile_interval"
+  spec.version       = LogfileInterval::VERSION
+  spec.authors       = ["Philippe Le Rohellec"]
+  spec.email         = ["philippe@lerohellec.com"]
+  spec.description   = %q{Logfile parser and aggregator}
+  spec.summary       = %q{Aggregate logfile data into intervals}
+  spec.homepage      = ""
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^spec/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency(%q<debugger>, [">= 0"])
+  spec.add_development_dependency(%q<rspec>, ["~> 2.14.0"])
+  spec.add_development_dependency "rake"
+end

data/spec/lib/interval_builder_spec.rb

@@ -0,0 +1,49 @@
+require 'spec_helper'
+require File.join(File.dirname(__FILE__), '..', 'support/lib/timing_log')
+
+module LogfileInterval
+  data_dir = File.join(File.dirname(__FILE__), '..', 'support/logfiles')
+
+  describe IntervalBuilder do
+    before :each do
+      @logfiles = ["#{data_dir}/timing.log", "#{data_dir}/timing.log.1" ]
+      @set = LogfileSet.new(@logfiles, LineParser::TimingLog)
+      @builder = IntervalBuilder.new(@set, 300)
+    end
+
+    context :each_interval do
+      before :each do
+        Time.stub(:now).and_return(Time.new(2013,12,01,16,0,0,'-08:00'))
+        @intervals = []
+        @builder.each_interval do |interval|
+          @intervals << interval
+        end
+      end
+
+      it 'finds intervals from all logfiles' do
+        @intervals.size.should == 2
+      end
+
+      context 'first interval' do
+        it 'got records from both logfiles' do
+          @intervals.first.size.should == 4
+          @intervals.first[:total_time].should == 700.0/4
+          @intervals.first[:num_bytes].should == 52000
+          @intervals.first[:action]['posts#index'] = 1
+          @intervals.first[:action]['posts#create'] = 1
+          @intervals.first[:action]['posts#show'] = 2
+        end
+      end
+
+      context 'second interval' do
+        it 'got records from second logfile only' do
+          @intervals.last.size.should == 2
+          @intervals.last[:total_time].should == 300
+          @intervals.last[:num_bytes].should == 41000
+          @intervals.last[:action]['posts#index'] = 1
+          @intervals.last[:action]['posts#show'] = 1
+        end
+      end
+    end
+  end
+end

data/spec/lib/interval_spec.rb

@@ -0,0 +1,73 @@
+require 'spec_helper'
+require File.join(File.dirname(__FILE__), '..', 'support/lib/timing_log')
+
+module LogfileInterval
+  data_dir = File.join(File.dirname(__FILE__), '..', 'support/logfiles')
+
+  describe Interval do
+    it 'gets instantiated with empty data' do
+      end_time = Time.new(2013, 12, 01, 16, 00, 00, '-08:00')
+      interval = Interval.new(end_time, 300, LineParser::TimingLog)
+      interval.size.should == 0
+      interval[:total_time].should == 0
+      interval[:num_bytes].should == 0
+      interval[:action].should be_a(Hash)
+    end
+
+    context :add_record do
+      before :each do
+        @end_time = Time.new(2013, 12, 01, 16, 00, 00, '-08:00')
+        @length = 300
+        @interval = Interval.new(@end_time, @length, LineParser::TimingLog)
+      end
+
+      it 'rejects record out of interval' do
+        oor_record = LineParser::TimingLog.create_record('1385942450, posts#index, 100, 20000')
+        lambda { @interval.add_record(oor_record) }.should raise_error(Interval::OutOfRange)
+      end
+
+      it 'rejects record at interval start_time' do
+        oor_record = LineParser::TimingLog.create_record('1385942100, posts#index, 100, 20000')
+        lambda { @interval.add_record(oor_record) }.should raise_error(Interval::OutOfRange)
+      end
+
+      it 'adds 1 record to interval' do
+        record1 = LineParser::TimingLog.create_record('1385942400, posts#index, 100, 20000')
+        @interval.add_record(record1)
+
+        @interval.size.should == 1
+        @interval[:total_time].should == 100
+        @interval[:num_bytes].should == 20000
+        @interval[:action]['posts#index'].should == 1
+      end
+
+      context '3 records' do
+        before :each do
+          record1 = LineParser::TimingLog.create_record('1385942400, posts#index, 100, 20000')
+          @interval.add_record(record1)
+          record2 = LineParser::TimingLog.create_record('1385942300, posts#show, 50, 10000')
+          @interval.add_record(record2)
+          record3 = LineParser::TimingLog.create_record('1385942200, posts#show, 60, 12000')
+          @interval.add_record(record3)
+        end
+
+        it 'increments size' do
+          @interval.size.should == 3
+        end
+
+        it 'averages columns with average agg_function' do
+          @interval[:total_time].should == 70
+        end
+
+        it 'sums up columns with sum agg_function' do
+          @interval[:num_bytes].should == 42000
+        end
+
+        it 'groups and counts columns with group agg_function' do
+          @interval[:action]['posts#index'].should == 1
+          @interval[:action]['posts#show'].should == 2
+        end
+      end
+    end
+  end
+end

data/spec/lib/line_parser/aggregator_spec.rb

@@ -0,0 +1,47 @@
+require 'spec_helper'
+
+module LogfileInterval
+  module LineParser
+    module Aggregator
+      describe Aggregator do
+        it 'finds the aggregator class' do
+          Aggregator.klass(:sum).should == Sum
+          Aggregator.klass(:average).should == Average
+          Aggregator.klass(:group).should == Group
+        end
+      end
+
+      describe Sum do
+        it 'sums up values' do
+          sum = Sum.new
+          sum.add(3)
+          sum.add(5)
+          sum.value.should == 8
+        end
+      end
+
+      describe Average do
+        it 'averages values' do
+          sum = Average.new
+          sum.add(3)
+          sum.add(5)
+          sum.value.should == 4
+        end
+      end
+
+      describe Group do
+        it 'groups values and increment counters' do
+          g = Group.new
+          g.add('200')
+          g.add('500')
+          g.add('301')
+          g.add('200')
+          g.value.should be_a(Hash)
+          g.value.should include({'200' => 2})
+          g.value.should include({'301' => 1})
+          g.value.should include({'500' => 1})
+        end
+      end
+    end
+  end
+end

data/spec/lib/line_parser/base_spec.rb

@@ -0,0 +1,84 @@
+require 'spec_helper'
+
+module LogfileInterval
+  data_dir = File.join(File.dirname(__FILE__), '..', 'support/logfiles')
+
+  module LineParser
+
+    describe Base do
+      before :each do
+        @line = '74.75.19.145 - - [31/Mar/2013:06:54:12 -0700] "GET /ppa/google_chrome HTTP/1.1" 200 7855 "https://www.google.com/" "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.22 (KHTML, like Gecko) Ubuntu Chromium/25.0.1364.160 Chrome/25.0.1364.160 Safari/537.22"'
+      end
+
+      it 'parses an access.log line' do
+        parsed_line = AccessLog.new(@line)
+        parsed_line.ip.should == '74.75.19.145'
+        parsed_line.length.should == 7855
+        parsed_line.timestamp.should == '31/Mar/2013:06:54:12 -0700'
+        parsed_line.time.should == Time.strptime('31/Mar/2013:06:54:12 -0700', '%d/%b/%Y:%H:%M:%S %z')
+      end
+
+      it 'returns an invalid record if line is malformed' do
+        line = 'abcdef'
+        record = 'unset'
+        lambda { record = AccessLog.new(line) }.should_not raise_error
+        record.valid?.should be_false
+      end
+
+      context :create_record do
+
+        it 'instanciates a new AccessLog object' do
+          record = AccessLog.create_record(@line)
+          record.should be_a(AccessLog)
+          record.ip.should == '74.75.19.145'
+        end
+
+        it 'returns nil if line is malformed' do
+          line = 'abcdef'
+          record = AccessLog.create_record(line)
+          record.should be_nil
+        end
+      end
+    end
+
+    describe 'Broken parsers' do
+      class NoRegexLog < Base
+        add_column :name => 'ip',        :pos => 1, :agg_function => :group
+      end
+
+      class NoColumnLog < Base
+        set_regex /^([\d\.]+)\s+\S+\s+\S+\s+\[(\d\d.*\d\d)\]\s+"(?:GET|POST|PUT|HEAD|DELETE)\s+(\S+)\s+HTTP\S+"\s+(\d+)\s+(\d+)\s+"([^"]*)"\s+"([^"]+)"$/
+      end
+
+      before :each do
+        @line = '74.75.19.145 - - [31/Mar/2013:06:54:12 -0700] "GET /ppa/google_chrome HTTP/1.1" 200 7855 "https://www.google.com/" "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.22 (KHTML, like Gecko) Ubuntu Chromium/25.0.1364.160 Chrome/25.0.1364.160 Safari/537.22"'
+      end
+
+      it 'must fail unless a regex is set' do
+        lambda { NoRegexLog.new(@line) }.should raise_error ConfigurationError
+      end
+
+      it 'must fail unless a column is configured'do
+        lambda { NoColumnLog.new(@line) }.should raise_error ConfigurationError
+      end
+    end
+
+    describe TimingLog do
+      before :each do
+        # 1385942400 = 2013/12/01 16:00:00
+        @line = '1385942400, posts#index, 100, 20000'
+      end
+
+      it 'parses a timing line' do
+        record = TimingLog.create_record(@line)
+        record.should_not be_nil
+        record.time.should == Time.new(2013, 12, 01, 16, 00, 00, '-08:00')
+        record.action.should == 'posts#index'
+        record.total_time.should == 100
+        record.num_bytes.should == 20000
+      end
+    end
+  end
+end
+
+

data/spec/lib/logfile_set_spec.rb

@@ -0,0 +1,39 @@
+require 'spec_helper'
+require File.join(File.dirname(__FILE__), '..', 'support/lib/access_log')
+
+module LogfileInterval
+  data_dir = File.join(File.dirname(__FILE__), '..', 'support/logfiles')
+
+  describe LogfileSet do
+    before :each do
+      @logfiles = ["#{data_dir}/access.log.2", "#{data_dir}/access.log.1"]
+      @set = LogfileSet.new(@logfiles, LineParser::AccessLog)
+    end
+
+    it 'ordered_filenames should return the most recent file first' do
+      @set.ordered_filenames.should == @logfiles.reverse
+    end
+
+    it 'each_line should enumerate each line in file backwards' do
+      lines = []
+      @set.each_line do |line|
+        lines << line
+      end
+
+      lines.first.should == '66.249.67.176 - - [23/Jun/2013:17:00:01 -0800] "GET /package/core/raring/universe/proposed/openldap HTTP/1.1" 200 185 "-" "Google"'
+      lines.last.should == '12.24.48.96 - - [23/Jun/2013:16:49:00 -0800] "GET /package/core/raring/universe/proposed/bash HTTP/1.1" 200 4555 "-" "Bing)"'
+    end
+
+    it 'each_parsed_line should enumerate each line backwards' do
+      records = []
+      @set.each_parsed_line do |record|
+        records << record
+      end
+
+      records.first.time.should == Time.new(2013, 06, 23, 17, 00, 01, '-08:00')
+      records.first.code.should == '200'
+      records.last.time.should  == Time.new(2013, 06, 23, 16, 49, 00, '-08:00')
+      records.last.code.should == '200'
+    end
+  end
+end

data/spec/lib/logfile_spec.rb

@@ -0,0 +1,39 @@
+require 'spec_helper'
+require File.join(File.dirname(__FILE__), '..', 'support/lib/access_log')
+
+module LogfileInterval
+  data_dir = File.join(File.dirname(__FILE__), '..', 'support/logfiles')
+
+  describe Logfile do
+    before :each do
+      @alf = Logfile.new("#{data_dir}/access.log", LineParser::AccessLog)
+    end
+
+    it 'first_timestamp returns time of first line in file' do
+      #01/Jan/2012:00:57:47 -0800
+      @alf.first_timestamp.should == Time.new(2012, 01, 01, 00, 57, 47, '-08:00')
+    end
+
+    it 'each_line should enumerate each line in file backwards' do
+      lines = []
+      @alf.each_line do |line|
+        lines << line
+      end
+
+      lines.first.should == '78.54.172.146 - - [01/Jan/2012:16:30:51 -0800] "GET /package/core/oneiric/main/base/abrowser-6.0  HTTP/1.1" 200 6801 "http://www.google.com/url?sa=t&rct=j&q=abrowser 6.0&esrc=s&source=web&cd=4&sqi=2&ved=0CDYQFjAD&url=http%3A%2F%2Fwww.ubuntuupdates.org%2Fpackages%2Fshow%2F268762&ei=s-QlT8vJFon1sgb54unBDg&usg=AFQjCNHCHC0bxTf6aXAfUwT6Erjta6WLaQ&sig2=ceCi1odtaB8Vcf6IWg2a3w" "Mozilla/5.0 (Ubuntu; X11; Linux x86_64; rv:9.0.1) Gecko/20100101 Firefox/9.0.1"'
+      lines.last.should == '66.249.67.176 - - [01/Jan/2012:00:57:47 -0800] "GET /packages/show/1 HTTP/1.1" 301 185 "-" "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"'
+    end
+
+    it 'each_parsed_line should enumerate each line backwards' do
+      records = []
+      @alf.each_parsed_line do |record|
+        records << record
+      end
+
+      records.first.time.should == Time.new(2012, 01, 01, 16, 30, 51, '-08:00')
+      records.first.code.should == '200'
+      records.last.time.should  == Time.new(2012, 01, 01, 00, 57, 47, '-08:00')
+      records.last.code.should == '301'
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,9 @@
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+
+require 'rspec'
+require 'logfile_interval'
+
+# Requires supporting ruby files with custom matchers and macros, etc,
+# in spec/support/ and its subdirectories.
+Dir["#{File.dirname(__FILE__)}/support/**/*.rb"].each {|f| require f}

data/spec/support/lib/access_log.rb

@@ -0,0 +1,22 @@
+module LogfileInterval
+  data_dir = File.join(File.dirname(__FILE__), '..', 'support/logfiles')
+
+  module LineParser
+
+    class AccessLog < Base
+      # Example line:
+      # 74.75.19.145 - - [31/Mar/2013:06:54:12 -0700] "GET /ppa/google_chrome HTTP/1.1" 200 7855 "https://www.google.com/" "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.22 (KHTML, like Gecko) Ubuntu Chromium/25.0.1364.160 Chrome/25.0.1364.160 Safari/537.22"
+
+      set_regex /^([\d\.]+)\s+\S+\s+\S+\s+\[(\d\d.*\d\d)\]\s+"(?:GET|POST|PUT|HEAD|DELETE)\s+(\S+)\s+HTTP\S+"\s+(\d+)\s+(\d+)\s+"([^"]*)"\s+"([^"]+)"$/
+
+      add_column :name => 'ip',        :pos => 1, :agg_function => :group
+      add_column :name => 'timestamp', :pos => 2, :agg_function => :timestamp
+      add_column :name => 'code',      :pos => 4, :agg_function => :group
+      add_column :name => 'length',    :pos => 5, :agg_function => :average,   :conversion => :integer
+
+      def time
+        Time.strptime(self.timestamp, '%d/%b/%Y:%H:%M:%S %z')
+      end
+    end
+  end
+end

data/spec/support/lib/timing_log.rb

@@ -0,0 +1,19 @@
+module LogfileInterval
+  module LineParser
+    class TimingLog < Base
+      # Line format:
+      # timestamp, controller#action, total_time, bytes
+
+      set_regex /^(\d+),\s*(\w+#\w+),\s*(\d+),\s*(\d+)$/
+
+      add_column :name => :timestamp,    :pos => 1, :agg_function => :timestamp
+      add_column :name => :action,       :pos => 2, :agg_function => :group
+      add_column :name => :total_time,   :pos => 3, :agg_function => :average,   :conversion => :integer
+      add_column :name => :num_bytes,    :pos => 4, :agg_function => :sum,       :conversion => :integer
+
+      def time
+        Time.at(self.timestamp.to_i)
+      end
+    end
+  end
+end

data/spec/support/logfiles/access.log

@@ -0,0 +1,7 @@
+66.249.67.176 - - [01/Jan/2012:00:57:47 -0800] "GET /packages/show/1 HTTP/1.1" 301 185 "-" "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"
+173.192.238.51 - - [01/Jan/2012:00:59:31 -0800] "GET /?dist=all+releases&noppa=2&format=atom HTTP/1.1" 301 185 "-" "Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.0.19; aggregator:Spinn3r (Spinn3r 3.1);"
+66.249.68.148 - - [01/Jan/2012:00:04:04 -0800] "GET /package/core/hardy/main/updates/libldap2-dev HTTP/1.1" 200 7058 "-" "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"
+78.54.172.146 - - [01/Jan/2012:16:30:51 -0800] "GET /package/core/oneiric/main/base/abrowser HTTP/1.1" 200 6801 "http://www.google.com/url?sa=t&rct=j&q=abrowser&esrc=s&source=web&cd=4&sqi=2&ved=0CDYQFjAD&url=http%3A%2F%2Fwww.ubuntuupdates.org%2Fpackages%2Fshow%2F268762&ei=s-QlT8vJFon1sgb54unBDg&usg=AFQjCNHCHC0bxTf6aXAfUwT6Erjta6WLaQ&sig2=ceCi1odtaB8Vcf6IWg2a3w" "Mozilla/5.0 (Ubuntu; X11; Linux x86_64; rv:9.0.1) Gecko/20100101 Firefox/9.0.1"
+78.54.172.146 - - [01/Jan/2012:16:30:51 -0800] "GET /package/core/oneiric/main/base/firefox  HTTP/1.1" 200 6801 "http://www.google.com/url?sa=t&rct=j&q=firefox&esrc=s&source=web&cd=4&sqi=2&ved=0CDYQFjAD&url=http%3A%2F%2Fwww.ubuntuupdates.org%2Fpackages%2Fshow%2F268762&ei=s-QlT8vJFon1sgb54unBDg&usg=AFQjCNHCHC0bxTf6aXAfUwT6Erjta6WLaQ&sig2=ceCi1odtaB8Vcf6IWg2a3w" "Mozilla/5.0 (Ubuntu; X11; Linux x86_64; rv:9.0.1) Gecko/20100101 Firefox/9.0.1"
+78.54.172.146 - - [01/Jan/2012:16:30:51 -0800] "GET /package/show/2  HTTP/1.1" 302 6801 "http://www.google.com/url?sa=t&rct=j&q=abrowser 6.0&esrc=s&source=web&cd=4&sqi=2&ved=0CDYQFjAD&url=http%3A%2F%2Fwww.ubuntuupdates.org%2Fpackages%2Fshow%2F268762&ei=s-QlT8vJFon1sgb54unBDg&usg=AFQjCNHCHC0bxTf6aXAfUwT6Erjta6WLaQ&sig2=ceCi1odtaB8Vcf6IWg2a3w" "Mozilla/5.0 (Ubuntu; X11; Linux x86_64; rv:9.0.1) Gecko/20100101 Firefox/9.0.1"
+78.54.172.146 - - [01/Jan/2012:16:30:51 -0800] "GET /package/core/oneiric/main/base/abrowser-6.0  HTTP/1.1" 200 6801 "http://www.google.com/url?sa=t&rct=j&q=abrowser 6.0&esrc=s&source=web&cd=4&sqi=2&ved=0CDYQFjAD&url=http%3A%2F%2Fwww.ubuntuupdates.org%2Fpackages%2Fshow%2F268762&ei=s-QlT8vJFon1sgb54unBDg&usg=AFQjCNHCHC0bxTf6aXAfUwT6Erjta6WLaQ&sig2=ceCi1odtaB8Vcf6IWg2a3w" "Mozilla/5.0 (Ubuntu; X11; Linux x86_64; rv:9.0.1) Gecko/20100101 Firefox/9.0.1"

data/spec/support/logfiles/access.log.1

@@ -0,0 +1,3 @@
+66.249.67.176 - - [23/Jun/2013:16:58:00 -0800] "GET /package/core/raring/universe/proposed/openldap HTTP/1.1" 200 185 "-" "Google"
+12.24.48.96 - - [23/Jun/2013:16:59:00 -0800] "GET /package/core/raring/universe/proposed/openldap HTTP/1.1" 200 4555 "-" "Bing)"
+66.249.67.176 - - [23/Jun/2013:17:00:01 -0800] "GET /package/core/raring/universe/proposed/openldap HTTP/1.1" 200 185 "-" "Google"

data/spec/support/logfiles/access.log.2

@@ -0,0 +1,3 @@
+12.24.48.96 - - [23/Jun/2013:16:49:00 -0800] "GET /package/core/raring/universe/proposed/bash HTTP/1.1" 200 4555 "-" "Bing)"
+66.249.67.176 - - [23/Jun/2013:16:56:00 -0800] "GET /package/core/raring/universe/proposed/openldap HTTP/1.1" 200 185 "-" "Google"
+