ncsa-parser 0.0.2

data/.gitignore

@@ -0,0 +1,12 @@
+*.orig
+.*.swp
+.*.swo
+*.tmp
+*.patch
+*.kpf
+*~
+.DS_Store
+Thumbs.db
+/doc
+/pkg
+Gemfile.lock

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2012 J Smith <dark.panda@#gmail.com>
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,23 @@
+
+= NCSA Parser
+
+Here's a quick little library for reading NCSA-style web server logs. Quick
+usage:
+
+  NCSAParser.each_line(File.open('/var/log/httpd/access_log'), :pattern => %w{
+    host ident username datetime request
+    status bytes referer ua
+    outstream instream ratio
+  }) do |b|
+    puts b.inspect
+  end
+
+  parser = NCSAParser::Parser.new
+  parsed = parser.parse_line('...')
+
+There are more examples available in the tests.
+
+== License
+
+This gem is licensed under an MIT-style license. See the +MIT-LICENSE+ file for
+details.

data/Rakefile

@@ -0,0 +1,37 @@
+# -*- ruby -*-
+
+require 'rubygems'
+
+gem 'rdoc', '~> 3.12'
+
+require 'rubygems/package_task'
+require 'rake/testtask'
+require 'rdoc/task'
+require 'bundler/gem_tasks'
+
+if RUBY_VERSION >= '1.9'
+  begin
+    gem 'psych'
+  rescue Exception => e
+    # it's okay, fall back on the bundled psych
+  end
+end
+
+$:.push 'lib'
+
+version = NCSAParser::VERSION
+
+desc 'Test NCSA parser library'
+Rake::TestTask.new(:test) do |t|
+  t.test_files = FileList['test/**/*_tests.rb']
+  t.verbose = !!ENV['VERBOSE_TESTS']
+  t.warning = !!ENV['WARNINGS']
+end
+
+desc 'Build docs'
+Rake::RDocTask.new do |t|
+  t.title = "NCSA Parser #{version}"
+  t.main = 'README.rdoc'
+  t.rdoc_dir = 'doc'
+  t.rdoc_files.include('README.rdoc', 'MIT-LICENSE', 'lib/**/*.rb')
+end

data/lib/ncsa-parser.rb

@@ -0,0 +1,22 @@
+
+require 'ncsa-parser/version'
+require 'ncsa-parser/parser'
+require 'ncsa-parser/helper'
+require 'ncsa-parser/parsed_line'
+require 'ncsa-parser/log'
+
+module NCSAParser
+  class << self
+    # Opens a log file and iterates through the lines.
+    def each_line(log, options = {}, &block)
+      self.open(log, options).each(&block)
+    end
+    alias :foreach :each_line
+
+    # Opens a log file for parsing. This is a convenience method that proxies
+    # to NCSAParser::Log.open.
+    def open(log, options = {})
+      Log.open(log, options)
+    end
+  end
+end

data/lib/ncsa-parser/helper.rb

@@ -0,0 +1,20 @@
+
+module NCSAParser
+  module Helper
+    def self.clean_uri(uri)
+      uri.
+        gsub(/ /, '+').
+        gsub(/\\"/, '%22').
+        gsub(/,/, '%2C')
+    end
+
+    def self.deep_symbolize_keys(hash)
+      hash.inject({}) do |memo, (key, value)|
+        key = key.to_sym if key.respond_to?(:to_sym) rescue :nil
+        value = NCSAHelper.deep_symbolize_keys(value) if value.is_a?(Hash)
+        memo[key] = value
+        memo
+      end
+    end
+  end
+end

data/lib/ncsa-parser/log.rb

@@ -0,0 +1,41 @@
+
+module NCSAParser
+  class Log
+    include Enumerable
+
+    attr_reader :log, :parser
+
+    def initialize(log, options = {})
+      @log = log
+      @parser = Parser.new(options)
+    end
+
+    def self.open(file, options = {})
+      file = if file.is_a?(String)
+        File.open(file)
+      else
+        file
+      end
+
+      self.new(file, options)
+    end
+
+    def each
+      if block_given?
+        self.log.each do |l|
+          yield self.parser.parse_line(l)
+        end
+      else
+        self.log.collect do |l|
+          self.parser.parse_line(l)
+        end
+      end
+    end
+
+    def next_line
+      self.parser.parse_line(self.log.gets).tap { |parsed|
+        yield parsed if block_given?
+      }
+    end
+  end
+end

data/lib/ncsa-parser/parsed_line.rb

@@ -0,0 +1,152 @@
+
+module NCSAParser
+  # NCSAParser::ParsedLine handles some token conversions and the like on the
+  # fly after a successful line parse. You can add your own token conversions
+  # or override existing ones by passing along a +:token_conversions+ option
+  # that contains converters in the same manner as those found in
+  # NCSAParser::ParsedLine::TOKEN_CONVERSIONS.
+  #
+  # To access a parsed value without any sort of token conversion, use the
+  # +attributes+ method. The +[]+ method will perform the token conversion
+  # on the fly for you.
+  #
+  # For token converters that handle URIs, the Symbol :bad_uri will be returned
+  # if the URI parser fails for whatever reason.
+  class ParsedLine
+    TOKEN_CONVERSIONS = {
+      :datetime => proc { |match, options|
+        DateTime.strptime(match.attributes[:datetime], options[:datetime_format])
+      },
+
+      :request_uri => proc { |match, options|
+        begin
+          request = match.attributes[:request].scan(/^"[A-Z]+ (.+) HTTP\/\d+\.\d+"$/).flatten[0]
+          URI.parse("http://#{options[:domain]}#{request}")
+        rescue
+          :bad_uri
+        end if match.attributes[:request]
+      },
+
+      :request_path => proc { |match, options|
+        match.attributes[:request].scan(/^"[A-Z]+ ([^?]+)/).flatten[0] rescue nil if match.attributes[:request]
+      },
+
+      :http_method => proc { |match, options|
+        match.attributes[:request].scan(/^"([A-Z]+)/).flatten[0] rescue nil if match.attributes[:request]
+      },
+
+      :http_version => proc { |match, options|
+        match.attributes[:request].scan(/(\d+\.\d+)"$/).flatten[0] rescue nil if match.attributes[:request]
+      },
+
+      :query_string => proc { |match, options|
+        if match[:request_uri]
+          if match[:request_uri] && match[:request_uri].query
+            CGI.parse(match[:request_uri].query)
+          else
+            Hash.new
+          end
+        end
+      },
+
+      :referer_uri => proc { |match, options|
+        if match[:referer]
+          if match[:referer] != '"-"'
+            referer = match[:referer].sub(/^"(.+)"$/, '\1')
+            NCSAParser::Helper.clean_uri(referer)
+
+            begin
+              URI.parse(referer)
+            rescue
+              :bad_uri
+            end
+          else
+            '-'
+          end
+        end
+      },
+
+      :browscap => proc { |match, options|
+        options[:browscap].query(match[:ua].sub(/^"(.+)"$/, '\1')) if options[:browscap]
+      },
+
+      :ratio => proc { |match, options|
+        match.attributes[:ratio].to_f / 100 rescue nil if match.attributes[:ratio]
+      },
+
+      :host => proc { |match, options|
+        if match.attributes[:host]
+          match.attributes[:host]
+        elsif match.attributes[:host_proxy]
+          match.attributes[:host_proxy].split(',')[0].strip
+        end
+      }
+    }
+
+    %w{ status instream outstream bytes }.each do |field|
+      class_eval(<<-EOF, __FILE__, __LINE__ + 1)
+        TOKEN_CONVERSIONS[:#{field}] = proc { |match, options|
+          match.attributes[:#{field}].to_i rescue nil if match.attributes[:#{field}]
+        }
+      EOF
+    end
+
+    attr_reader :attributes
+
+    def initialize(attributes, options = {})
+      @attributes, @options = attributes, options
+      @parsed_attributes = {}
+
+      if options[:browscap] && !options[:browscap].respond_to?(:query)
+        raise ArgumentError.new("The :browscap object should respond to the #query method.")
+      end
+    end
+
+    # Accesses either an attribute or an attribute that has been passed
+    # through a token converter. You can access the raw, unconverted attributes
+    # via the +attributes+ method. If a converter fails for whatever reason,
+    # a value of +:bad_conversion+ is returned.
+    def [](key)
+      key = key.to_sym unless key.is_a?(Symbol)
+
+      if @parsed_attributes.has_key?(key)
+        @parsed_attributes[key]
+      elsif @options[:token_conversions] && @options[:token_conversions][key]
+        @parsed_attributes[key] = @options[:token_conversions][key].call(self, @options)
+      elsif TOKEN_CONVERSIONS[key]
+        @parsed_attributes[key] = (TOKEN_CONVERSIONS[key].call(self, @options) rescue :bad_conversion)
+      else
+        @attributes[key]
+      end
+    end
+
+    # Gathers up the requested attributes and spits them out into a Hash.
+    # The +values+ argument determines what gets inserted into the Hash:
+    #
+    # * +:all+ - both attributes and parsed attributes. In cases where
+    #   the values share the same names, the parsed attribute wins out.
+    # * +:attributes+ - unparsed attributes only.
+    # * +:parsed+ - parsed attributes only.
+    #
+    # The default value is +:all+. Any +nil+ values are automatically stripped
+    # from the Hash.
+    def to_hash(values = :all)
+      retval = {}
+
+      if values == :all || values == :attributes
+        retval.merge!(@attributes)
+      end
+
+      if values == :all || values == :parsed
+        TOKEN_CONVERSIONS.each { |t, v| self[t] }
+        retval.merge!(@parsed_attributes)
+      end
+
+      retval.reject! { |k, v|
+        v.nil?
+      }
+
+      retval
+    end
+  end
+end

data/lib/ncsa-parser/parser.rb

@@ -0,0 +1,111 @@
+
+require 'uri'
+require 'date'
+require 'cgi'
+
+module NCSAParser
+  class BadLogLine < Exception
+    def initialize(line, pattern)
+      super("Bad log line. Pattern: |#{pattern.join(' ')}| Line: |#{line}|")
+    end
+  end
+
+  # A line parser for a log file. Lines are parsed via Regexps. You can
+  # inject new tokens or override existing ones by modifying the passing along
+  # a +:tokens+ option and adding the keys to the +:pattern+ option
+  # accordingly.
+  class Parser
+    IP_ADDRESS = '\d+\.\d+\.\d+\.\d+|unknown'
+
+    TOKENS = {
+      :host => "(?:#{IP_ADDRESS}|-|::1)",
+      :host_proxy => "(?:#{IP_ADDRESS})(?:,\\s+#{IP_ADDRESS})*|-",
+      :ident => '[^\s]+',
+      :username => '[^\s]+',
+      :datetime => '\[[^\]]+\]',
+      :request => '".+"',
+      :status => '\d+',
+      :bytes => '\d+|-',
+      :referer => '".*"',
+      :ua => '".*"',
+      :usertrack => "(?:#{IP_ADDRESS})[^ ]+|-",
+      :outstream => '\d+|-',
+      :instream => '\d+|-',
+      :ratio => '\d+%|-%'
+    }
+
+    LOG_FORMAT_COMMON = %w{
+      host ident username datetime request status bytes
+    }
+
+    LOG_FORMAT_COMBINED = %w{
+      host ident username datetime request status bytes referer ua
+    }
+
+    attr_reader :pattern, :matcher, :re
+
+    # Creates a new Parser object.
+    #
+    # == Options
+    #
+    # * +:domain+ - when parsing query strings, use this domain as the URL's
+    #   domain. The default is +"www.example.com"+.
+    # * +:datetime_format+ - sets the datetime format for when tokens are
+    #   converted in NCSAParser::ParsedLine. The default is +"[%d/%b/%Y:%H:%M:%S %Z]"+.
+    # * +:pattern+ - the default log line format to use. The default is
+    #   +LOG_FORMAT_COMBINED+, which matches the "combined" log format in
+    #   Apache. The value for +:pattern+ can be either a space-delimited
+    #   String of token names or an Array of token names.
+    # * +:browscap+ - a browser capabilities object to use when sniffing out
+    #   user agents. This object should be able to respond to the +query+
+    #   method. Several browscap extensions are available for Ruby, and the
+    #   the author of this extension's version is called Browscapper and is
+    #   available at https://github.com/dark-panda/browscapper .
+    # * +:token_conversions+ - converters to pass along to the line parser.
+    #   See NCSAParser::ParsedLine for details.
+    # * +:tokens+ - tokens to add to the generated Regexp.
+    def initialize(options = {})
+      options = {
+        :domain => 'www.example.com',
+        :datetime_format => '[%d/%b/%Y:%H:%M:%S %Z]',
+        :pattern => LOG_FORMAT_COMBINED
+      }.merge(options)
+
+      @options = options
+      @pattern = if options[:pattern].is_a?(Array)
+        options[:pattern]
+      else
+        options[:pattern].to_s.split(/\s+/)
+      end
+
+      @re = '^' + @pattern.collect { |tk|
+        tk = tk.to_sym
+        token = if options[:tokens] && options[:tokens][tk]
+          options[:tokens][tk]
+        elsif TOKENS[tk]
+          TOKENS[tk]
+        else
+          raise ArgumentError.new("Token :#{tk} not found!")
+        end
+
+        "(#{token})"
+      }.join(' ') + '$'
+      @matcher = Regexp.new(@re)
+    end
+
+    # Parses a single line and returns an NCSAParser::ParsedLine object.
+    def parse_line(line)
+      match = Hash.new
+      if md = @matcher.match(line)
+        @pattern.each_with_index do |k, j|
+          match[k.to_sym] = md[j + 1]
+        end
+        match[:original] = line.strip
+      else
+        raise BadLogLine.new(line, @options[:pattern])
+      end
+      ParsedLine.new(match, @options)
+    end
+    alias :parse :parse_line
+  end
+end

data/lib/ncsa-parser/version.rb

@@ -0,0 +1,5 @@
+
+module NCSAParser
+  VERSION = '0.0.2'
+end
+

data/ncsa-parser.gemspec

@@ -0,0 +1,27 @@
+# -*- encoding: utf-8 -*-
+
+require File.expand_path('../lib/ncsa-parser/version', __FILE__)
+
+Gem::Specification.new do |s|
+  s.name = "ncsa-parser"
+  s.version = NCSAParser::VERSION
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["J Smith"]
+  s.description = "A simple NCSA-style log file parser."
+  s.summary = s.description
+  s.email = "dark.panda@gmail.com"
+  s.extra_rdoc_files = [
+    "README.rdoc"
+  ]
+  s.files = `git ls-files`.split($\)
+  s.executables = s.files.grep(%r{^bin/}).map { |f| File.basename(f) }
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+  s.homepage = "http://github.com/dark-panda/ncsa-parser"
+  s.require_paths = ["lib"]
+
+  s.add_dependency("rdoc")
+  s.add_dependency("rake", ["~> 0.9"])
+  s.add_dependency("minitest")
+  s.add_dependency("turn")
+end

data/test/ncsa_parser_tests.rb

@@ -0,0 +1,175 @@
+
+$: << File.dirname(__FILE__)
+require 'test_helper'
+
+class NCSAParserTests < MiniTest::Unit::TestCase
+  include TestHelper
+
+  def test_format_default
+    parser = NCSAParser::Parser.new
+    parsed = parser.parse_line(LOG_COMBINED)
+
+    assert_equal({
+      :host => %{123.123.123.123},
+      :ident => %{-},
+      :username => %{-},
+      :datetime => %{[08/Oct/2012:14:36:07 -0400]},
+      :request => %{"GET /path/to/something?foo=bar&hello=world HTTP/1.1"},
+      :status => %{200},
+      :bytes => %{923},
+      :referer => %{"http://www.example.com/referer"},
+      :ua => %{"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/22.0.1229.79 Safari/537.4"},
+      :original => LOG_COMBINED,
+    }, parsed.attributes)
+  end
+
+  def test_format_common
+    parser = NCSAParser::Parser.new(:pattern => NCSAParser::Parser::LOG_FORMAT_COMMON)
+
+    parsed = parser.parse_line(LOG_COMMON)
+    assert_equal({
+      :host => %{123.123.123.123},
+      :ident => %{-},
+      :username => %{-},
+      :datetime => %{[08/Oct/2012:14:36:07 -0400]},
+      :request => %{"GET /path/to/something?foo=bar&hello=world HTTP/1.1"},
+      :status => %{200},
+      :bytes => %{923},
+      :original => LOG_COMMON,
+    }, parsed.attributes)
+  end
+
+  def test_format_usertrack
+    parser = NCSAParser::Parser.new(:pattern => %w{
+      host ident username datetime request status bytes referer ua usertrack
+    })
+
+    parsed = parser.parse_line(LOG_USERTRACK)
+
+    assert_equal({
+      :host => %{123.123.123.123},
+      :ident => %{-},
+      :username => %{-},
+      :datetime => %{[08/Oct/2012:14:36:07 -0400]},
+      :request => %{"GET /path/to/something?foo=bar&hello=world HTTP/1.1"},
+      :status => %{200},
+      :bytes => %{923},
+      :referer => %{"http://www.example.com/referer"},
+      :ua => %{"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/22.0.1229.79 Safari/537.4"},
+      :usertrack => %{123.123.123.123.1349718542489266},
+      :original => LOG_USERTRACK
+    }, parsed.attributes)
+  end
+
+  def test_format_deflate
+    parser = NCSAParser::Parser.new(:pattern => %w{
+      host ident username datetime request status bytes referer ua instream outstream ratio
+    })
+
+    parsed = parser.parse_line(LOG_DEFLATE)
+
+    assert_equal({
+      :host => %{123.123.123.123},
+      :ident => %{-},
+      :username => %{-},
+      :datetime => %{[08/Oct/2012:14:36:07 -0400]},
+      :request => %{"GET /path/to/something?foo=bar&hello=world HTTP/1.1"},
+      :status => %{200},
+      :bytes => %{923},
+      :referer => %{"http://www.example.com/referer"},
+      :ua => %{"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/22.0.1229.79 Safari/537.4"},
+      :instream => %{905},
+      :outstream => %{1976},
+      :ratio => %{45%},
+      :original => LOG_DEFLATE
+    }, parsed.attributes)
+  end
+
+  def test_format_bad
+    parser = NCSAParser::Parser.new
+
+    assert_raises(NCSAParser::BadLogLine) do
+      parser.parse_line('what happen')
+    end
+  end
+
+  def test_open_file
+    log = NCSAParser.open('./test/resources/access_log', :pattern => %w{
+      host ident username datetime request status bytes referer ua usertrack instream outstream ratio
+    })
+
+    expect = {
+      :host => %{123.123.123.123},
+      :ident => %{-},
+      :username => %{-},
+      :datetime => %{[08/Oct/2012:14:36:07 -0400]},
+      :request => %{"GET /path/to/something?foo=bar&hello=world HTTP/1.1"},
+      :status => %{200},
+      :bytes => %{923},
+      :referer => %{"http://www.example.com/referer"},
+      :ua => %{"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/22.0.1229.79 Safari/537.4"},
+      :usertrack => %{123.123.123.123.1349718542489266},
+      :instream => %{905},
+      :outstream => %{1976},
+      :ratio => %{45%},
+      :original => %{123.123.123.123 - - [08/Oct/2012:14:36:07 -0400] "GET /path/to/something?foo=bar&hello=world HTTP/1.1" 200 923 "http://www.example.com/referer" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/22.0.1229.79 Safari/537.4" 123.123.123.123.1349718542489266 905 1976 45%}
+    }
+
+    log.each do |parsed|
+      assert_equal(expect, parsed.attributes)
+    end
+  end
+
+  def test_token_conversions
+    line = %{[08/Oct/2012:14:36:07 -0400] "GET /path/to/something?foo=bar&hello=world HTTP/1.1" "http://www.example.com/referer" 1000 100 10% 123.123.123.123 200 110}
+
+    parser = NCSAParser::Parser.new(:pattern => %w{
+      datetime request referer instream outstream ratio host_proxy status bytes
+    })
+
+    parsed = parser.parse_line(line)
+
+    assert_equal(DateTime.strptime('[08/Oct/2012:14:36:07 -0400]', '[%d/%b/%Y:%H:%M:%S %Z]'), parsed[:datetime])
+    assert_equal(URI.parse('http://www.example.com/path/to/something?foo=bar&hello=world'), parsed[:request_uri])
+    assert_equal('/path/to/something', parsed[:request_path])
+    assert_equal('GET', parsed[:http_method])
+    assert_equal('1.1', parsed[:http_version])
+    assert_equal(URI.parse('http://www.example.com/referer'), parsed[:referer_uri])
+    assert_equal(0.1, parsed[:ratio])
+    assert_equal('123.123.123.123', parsed[:host])
+    assert_equal(200, parsed[:status])
+    assert_equal(1000, parsed[:instream])
+    assert_equal(100, parsed[:outstream])
+    assert_equal(110, parsed[:bytes])
+  end
+
+  def test_custom_tokens_and_conversion
+    parser = NCSAParser::Parser.new(
+      :pattern => 'email',
+
+      :tokens => {
+        :email => '[^@]+@[^@]+'
+      },
+
+      :token_conversions => {
+        :email => proc { |match, options|
+          URI.parse(match.attributes[:email])
+        }
+      }
+    )
+
+    parsed = parser.parse_line('test@example.com')
+    assert_equal(URI.parse('test@example.com'), parsed[:email])
+  end
+
+  def test_to_hash
+    parser = NCSAParser::Parser.new(:pattern => NCSAParser::Parser::LOG_FORMAT_COMMON)
+    parsed = parser.parse_line(LOG_COMMON)
+
+    assert_equal([
+      :host, :ident, :username, :datetime, :request, :status, :bytes,
+      :original, :request_uri, :request_path, :http_method,
+      :http_version, :query_string
+    ].sort_by(&:to_s), parsed.to_hash.keys.sort_by(&:to_s))
+  end
+end

data/test/resources/access_log

@@ -0,0 +1,7 @@
+123.123.123.123 - - [08/Oct/2012:14:36:07 -0400] "GET /path/to/something?foo=bar&hello=world HTTP/1.1" 200 923 "http://www.example.com/referer" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/22.0.1229.79 Safari/537.4" 123.123.123.123.1349718542489266 905 1976 45%
+123.123.123.123 - - [08/Oct/2012:14:36:07 -0400] "GET /path/to/something?foo=bar&hello=world HTTP/1.1" 200 923 "http://www.example.com/referer" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/22.0.1229.79 Safari/537.4" 123.123.123.123.1349718542489266 905 1976 45%
+123.123.123.123 - - [08/Oct/2012:14:36:07 -0400] "GET /path/to/something?foo=bar&hello=world HTTP/1.1" 200 923 "http://www.example.com/referer" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/22.0.1229.79 Safari/537.4" 123.123.123.123.1349718542489266 905 1976 45%
+123.123.123.123 - - [08/Oct/2012:14:36:07 -0400] "GET /path/to/something?foo=bar&hello=world HTTP/1.1" 200 923 "http://www.example.com/referer" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/22.0.1229.79 Safari/537.4" 123.123.123.123.1349718542489266 905 1976 45%
+123.123.123.123 - - [08/Oct/2012:14:36:07 -0400] "GET /path/to/something?foo=bar&hello=world HTTP/1.1" 200 923 "http://www.example.com/referer" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/22.0.1229.79 Safari/537.4" 123.123.123.123.1349718542489266 905 1976 45%
+123.123.123.123 - - [08/Oct/2012:14:36:07 -0400] "GET /path/to/something?foo=bar&hello=world HTTP/1.1" 200 923 "http://www.example.com/referer" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/22.0.1229.79 Safari/537.4" 123.123.123.123.1349718542489266 905 1976 45%
+123.123.123.123 - - [08/Oct/2012:14:36:07 -0400] "GET /path/to/something?foo=bar&hello=world HTTP/1.1" 200 923 "http://www.example.com/referer" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/22.0.1229.79 Safari/537.4" 123.123.123.123.1349718542489266 905 1976 45%

data/test/test_helper.rb

@@ -0,0 +1,25 @@
+
+require 'rubygems'
+require 'minitest/autorun'
+require 'turn/autorun'
+require File.join(File.dirname(__FILE__), %w{ .. lib ncsa-parser })
+
+puts "NCSAParser version #{NCSAParser::VERSION}"
+
+module TestHelper
+  LOG_COMMON = %{123.123.123.123 - - [08/Oct/2012:14:36:07 -0400] "GET /path/to/something?foo=bar&hello=world HTTP/1.1" 200 923}
+
+  LOG_COMBINED = %{123.123.123.123 - - [08/Oct/2012:14:36:07 -0400] "GET /path/to/something?foo=bar&hello=world HTTP/1.1" 200 923 "http://www.example.com/referer" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/22.0.1229.79 Safari/537.4"}
+
+  LOG_USERTRACK = %{123.123.123.123 - - [08/Oct/2012:14:36:07 -0400] "GET /path/to/something?foo=bar&hello=world HTTP/1.1" 200 923 "http://www.example.com/referer" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/22.0.1229.79 Safari/537.4" 123.123.123.123.1349718542489266}
+
+  LOG_DEFLATE = %{123.123.123.123 - - [08/Oct/2012:14:36:07 -0400] "GET /path/to/something?foo=bar&hello=world HTTP/1.1" 200 923 "http://www.example.com/referer" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/22.0.1229.79 Safari/537.4" 905 1976 45%}
+end
+
+if ENV['autotest']
+  module Turn::Colorize
+    def self.color_supported?
+      true
+    end
+  end
+end

metadata

@@ -0,0 +1,127 @@
+--- !ruby/object:Gem::Specification
+name: ncsa-parser
+version: !ruby/object:Gem::Version
+  version: 0.0.2
+  prerelease: 
+platform: ruby
+authors:
+- J Smith
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-10-08 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rdoc
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.9'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.9'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: turn
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: A simple NCSA-style log file parser.
+email: dark.panda@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files:
+- README.rdoc
+files:
+- .gitignore
+- Gemfile
+- MIT-LICENSE
+- README.rdoc
+- Rakefile
+- lib/ncsa-parser.rb
+- lib/ncsa-parser/helper.rb
+- lib/ncsa-parser/log.rb
+- lib/ncsa-parser/parsed_line.rb
+- lib/ncsa-parser/parser.rb
+- lib/ncsa-parser/version.rb
+- ncsa-parser.gemspec
+- test/ncsa_parser_tests.rb
+- test/resources/access_log
+- test/test_helper.rb
+homepage: http://github.com/dark-panda/ncsa-parser
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: A simple NCSA-style log file parser.
+test_files:
+- test/ncsa_parser_tests.rb
+- test/resources/access_log
+- test/test_helper.rb