logparser 0.1.0

data/Rakefile

@@ -0,0 +1,46 @@
+require 'rake'
+require 'rake/testtask'
+require "rake/rdoctask"
+
+desc 'Default: run unit tests.'
+task :default => :test
+
+desc 'Test the library.'
+Rake::TestTask.new(:test) do |t|
+  t.pattern = 'test/**/*_test.rb'
+  t.verbose = true
+end
+
+begin
+  require "rake/gempackagetask"
+  require File.dirname(__FILE__)+"/lib/logparser/version"
+
+  spec = Gem::Specification.new do |s|
+    # Change these as appropriate
+    s.name              = "logparser"
+    s.version           = LogParser::VERSION::STRING
+    s.summary           = "Parse log files using a simple syntax."
+    s.author            = "Paul Battley"
+    s.email             = "pbattley@gmail.com"
+    s.homepage          = "http://github.com/threedaymonk/logparser"
+
+    s.has_rdoc          = false
+
+    # Add any extra files to include in the gem (like your README)
+    s.files             = %w(Rakefile) + Dir.glob("{test,lib}/**/*")
+
+    s.require_paths     = ["lib"]
+
+    # If you want to depend on other gems, add them here, along with any
+    # relevant versions
+    # s.add_dependency("some_other_gem", "~> 0.1.0")
+
+    # If your tests use any gems, include them here
+    # s.add_development_dependency("mocha")
+  end
+
+  Rake::GemPackageTask.new(spec) do |pkg|
+    pkg.gem_spec = spec
+  end
+rescue LoadError
+end

data/lib/logparser.rb

@@ -0,0 +1,3 @@
+require "logparser/line"
+require "logparser/template"
+require "logparser/version"

data/lib/logparser/line.rb

@@ -0,0 +1,69 @@
+require 'date'
+
+module LogParser
+  class Line
+    def initialize(template, raw)
+      @template = template
+      @raw = raw.strip
+    end
+
+    def raw_fields
+      @raw_fields ||= @template.apply(@raw)
+    end
+
+    def host
+      @host ||= decode_string(raw_fields[:host])
+    end
+
+    def domain
+      @domain ||= decode_string(raw_fields[:domain])
+    end
+
+    def timestamp
+      @timestamp ||= DateTime.strptime(raw_fields[:timestamp], '%d/%b/%Y:%H:%M:%S %Z')
+    end
+
+    def verb
+      @verb ||= raw_fields[:verb].downcase.to_sym
+    end
+
+    def path
+      @path ||= raw_fields[:path]
+    end
+
+    def protocol
+      @protocol ||= raw_fields[:protocol]
+    end
+
+    def status
+      @status ||= raw_fields[:status].to_i
+    end
+
+    def bytes
+      @bytes ||= raw_fields[:bytes].to_i
+    end
+
+    def referrer
+      @referrer ||= decode_string(raw_fields[:referrer])
+    end
+
+    def user_agent
+      @user_agent ||= decode_string(raw_fields[:user_agent])
+    end
+
+    def time_taken
+      @time_taken ||= raw_fields[:time_taken] ? raw_fields[:time_taken].to_f : nil
+    end
+
+  private
+
+    def decode_string(str)
+      case str
+      when '-', ''
+        return nil
+      else
+        return str
+      end
+    end
+  end
+end

data/lib/logparser/template.rb

@@ -0,0 +1,29 @@
+module LogParser
+  class Template
+    attr_reader :fields, :regexp
+
+    def initialize(pattern)
+      build_regexp(pattern)
+    end
+
+    def build_regexp(pattern)
+      @fields = []
+      @regexp = Regexp.new("\\A" << Regexp.escape(pattern).gsub(/:([a-z_][a-z0-9_]*)/){
+        @fields << $1.to_sym
+        "(.*?)"
+      } << "\\Z" )
+    end
+
+    def apply(str)
+      if matches = str.match(@regexp)
+        hash = {}
+        @fields.each_with_index do |key, i|
+          hash[key] = matches[i+1]
+        end
+        return hash
+      else
+        return nil
+      end
+    end
+  end
+end

data/lib/logparser/version.rb

@@ -0,0 +1,9 @@
+module LogParser
+  module VERSION #:nodoc:
+    MAJOR = 0
+    MINOR = 1
+    TINY  = 0
+
+    STRING = [MAJOR, MINOR, TINY].join('.')
+  end
+end

data/test/parsing_test.rb

@@ -0,0 +1,104 @@
+$:.unshift(File.join(File.dirname(__FILE__), "..", "lib"))
+require 'test/unit'
+require 'logparser'
+
+class GeneralParsingTest < Test::Unit::TestCase
+  include LogParser
+
+  def test_should_extract_host
+    line = Line.new(Template.new(':host'), '83.148.169.161')
+    assert_equal '83.148.169.161', line.host
+  end
+
+  def test_should_extract_blank_data
+    line = Line.new(Template.new('":host"'), '""')
+    assert_nil line.host
+  end
+
+  def test_should_extract_domain
+    line = Line.new(Template.new(':domain'), 'www.reevoo.com')
+    assert_equal 'www.reevoo.com', line.domain
+  end
+
+  def test_should_extract_timestamp
+    line = Line.new(Template.new('[:timestamp]'), '[02/Nov/2006:13:41:41 +0000]')
+    assert_equal DateTime.parse('2006-11-02T13:41:41Z'), line.timestamp
+  end
+
+  def test_should_extract_verb
+    line = Line.new(Template.new('":verb :path :protocol"'), '"GET /javascripts/effects.js?1161276768 HTTP/1.0"')
+    assert_equal :get, line.verb
+  end
+
+  def test_should_extract_path
+    line = Line.new(Template.new('":verb :path :protocol"'), '"GET /javascripts/effects.js?1161276768 HTTP/1.0"')
+    assert_equal '/javascripts/effects.js?1161276768', line.path
+  end
+
+  def test_should_extract_protocol
+    line = Line.new(Template.new('":verb :path :protocol"'), '"GET /javascripts/effects.js?1161276768 HTTP/1.0"')
+    assert_equal 'HTTP/1.0', line.protocol
+  end
+
+  def test_should_extract_status
+    line = Line.new(Template.new(':status'), '200')
+    assert_equal 200, line.status
+  end
+
+  def test_should_extract_bytes
+    line = Line.new(Template.new(':bytes'), '32871')
+    assert_equal 32871, line.bytes
+  end
+
+  def test_should_extract_referrer
+    line = Line.new(Template.new('":referrer"'), '"http://www.reevoo.com/reviews/mpn/hotpoint/fdw60p"')
+    assert_equal 'http://www.reevoo.com/reviews/mpn/hotpoint/fdw60p', line.referrer
+  end
+
+  def test_should_extract_user_agent
+    line = Line.new(Template.new('":user_agent"'), '"Mozilla/4.0 (compatible; MSIE 6.0; Windows 98)"')
+    assert_equal 'Mozilla/4.0 (compatible; MSIE 6.0; Windows 98)', line.user_agent
+  end
+
+  def test_should_have_nil_referrer_when_log_contains_hyphen_placeholder
+    line = Line.new(Template.new('":referrer"'), '"-"')
+    assert_nil line.referrer
+  end
+
+  def test_should_extract_time_taken
+    line = Line.new(Template.new(':time_taken msec'), '0.004 msec')
+    assert_in_delta 0.004, line.time_taken, 0.00001
+  end
+
+  def test_should_give_time_taken_as_nil_when_not_given
+    line = Line.new(Template.new(''), '')
+    assert_nil line.time_taken
+  end
+
+  def test_should_extract_all_fields_from_sample_line
+    sample =  '83.148.169.161 www.reevoo.com - [02/Nov/2006:13:41:41 +0000] '+
+              '"GET /javascripts/effects.js?1161276768 HTTP/1.0" 200 32871 '+
+              '"http://www.reevoo.com/reviews/mpn/hotpoint/fdw60p" '+
+              '"Mozilla/4.0 (compatible; MSIE 6.0; Windows 98)"'
+    line = Line.new(
+      Template.new(':host :domain :unknown [:timestamp] ":verb :path :protocol" :status :bytes ":referrer" ":user_agent"'),
+      sample
+    )
+    assert_equal '83.148.169.161', line.host
+    assert_equal 'www.reevoo.com', line.domain
+    assert_equal DateTime.parse('2006-11-02T13:41:41Z'), line.timestamp
+    assert_equal :get, line.verb
+    assert_equal '/javascripts/effects.js?1161276768', line.path
+    assert_equal '/javascripts/effects.js?1161276768', line.path
+    assert_equal 'HTTP/1.0', line.protocol
+    assert_equal 200, line.status
+    assert_equal 32871, line.bytes
+    assert_equal 'http://www.reevoo.com/reviews/mpn/hotpoint/fdw60p', line.referrer
+    assert_equal 'Mozilla/4.0 (compatible; MSIE 6.0; Windows 98)', line.user_agent
+  end
+
+  def test_should_ignore_trailing_newline
+    line = Line.new(Template.new('":referrer"'), %{"foo"\n})
+    assert_equal 'foo', line.referrer
+  end
+end

data/test/template_test.rb

@@ -0,0 +1,20 @@
+$:.unshift(File.join(File.dirname(__FILE__), "..", "lib"))
+require 'test/unit'
+require 'logparser/template'
+
+class TemplateTest < Test::Unit::TestCase
+  def test_should_extract_fields_from_pattern
+    template = LogParser::Template.new(":a :b :c")
+    assert_equal [:a, :b, :c], template.fields
+  end
+
+  def test_should_escape_regexp_properly
+    template = LogParser::Template.new("[:a]")
+    assert_equal(/\A\[(.*?)\]\Z/, template.regexp)
+  end
+
+  def test_should_extract_data_according_to_pattern
+    template = LogParser::Template.new(":foo :bar [:baz]")
+    assert_equal({:foo => 'Foo', :bar => 'Bar', :baz => 'Baz'}, template.apply('Foo Bar [Baz]'))
+  end
+end

metadata

@@ -0,0 +1,61 @@
+--- !ruby/object:Gem::Specification 
+name: logparser
+version: !ruby/object:Gem::Version 
+  version: 0.1.0
+platform: ruby
+authors: 
+- Paul Battley
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-10-11 00:00:00 +01:00
+default_executable: 
+dependencies: []
+
+description: 
+email: pbattley@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- Rakefile
+- test/template_test.rb
+- test/parsing_test.rb
+- lib/logparser.rb
+- lib/logparser/line.rb
+- lib/logparser/template.rb
+- lib/logparser/version.rb
+has_rdoc: true
+homepage: http://github.com/threedaymonk/logparser
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.5
+signing_key: 
+specification_version: 3
+summary: Parse log files using a simple syntax.
+test_files: []
+