http-log-parser 0.0.1

data/README.rdoc

@@ -0,0 +1,30 @@
+= LogParser
+
+== Introduction
+
+This gem provides an easy to use parser to various HTTP log formats.
+
+== Installation
+
+Just run:
+
+    gem install http-log-parser
+
+That should take care of it.
+
+== Usage
+
+    require 'http_log_parser'
+    
+    parser = HttpLogParser.new
+
+    File.open('/path/to.log', 'r') do |file|
+      while(line = file.gets)
+        parsed_data = parser.parse_line(line)
+        p parsed_data
+      end
+    end
+
+== License
+
+This code is made availablie under the MIT license. It is based on based on code from Jan Wikholm.

data/Rakefile

@@ -0,0 +1,36 @@
+require 'rubygems'
+require 'rake/gempackagetask'
+require 'rake/rdoctask'
+
+spec = Gem::Specification.new do |s|
+    s.name      =   "http-log-parser"
+    s.version   =   "0.0.1"
+    s.author    =   "Torsten Curdt"
+    s.email     =   "tcurdt at vafer.org"
+    s.homepage  =   "http://github.com/tcurdt/http-log-parser"
+    s.description = "HTTP log file parser"
+    s.summary   =   "A package for parsing web server logs."
+
+    s.platform  =   Gem::Platform::RUBY
+    s.has_rdoc  =   true
+    s.extra_rdoc_files  =   ["README.rdoc"]
+
+    s.require_path  =   "lib"
+    s.files     =   %w(README.rdoc Rakefile) + Dir.glob("lib/**/*")
+end
+
+Rake::GemPackageTask.new(spec) do |pkg|
+    pkg.need_tar = true
+end
+
+Rake::RDocTask.new(:rdoc) do |rdoc|
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title    = 'HttpLogParser'
+  rdoc.options << '--line-numbers' << '--inline-source'
+  rdoc.rdoc_files.include('README')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+
+task :default => "pkg/#{spec.name}-#{spec.version}.gem" do
+    puts "generated latest version"
+end

data/lib/http/parser.rb

@@ -0,0 +1,96 @@
+class HttpLogFormat
+  attr_reader :name, :format, :format_symbols, :format_regex
+
+  DIRECTIVES = {
+    'h' => [:ip, /\d+\.\d+\.\d+\.\d+/],
+    'l' => [:auth, /.*?/],
+    'u' => [:username, /.*?/],
+    't' => [:datetime, /\[.*?\]/],
+    'r' => [:request, /.*?/],
+    's' => [:status, /\d+/],
+    'b' => [:bytecount, /-|\d+/],
+    'v' => [:domain, /.*?/],
+    'i' => [:header_lines, /.*?/],
+    'e' => [:errorlevel, /\[.*?\]/],
+  }
+
+  def initialize(name, format)
+    @name, @format = name, format
+    parse_format(format)
+  end
+
+  def parse_format(format)
+    format_directive = /%(.*?)(\{.*?\})?([#{[DIRECTIVES.keys.join('|')]}])([\s\\"]*)/
+
+    log_format_symbols = []
+    format_regex = ""
+    format.scan(format_directive) do |condition, subdirective, directive_char, ignored|
+      log_format, match_regex = process_directive(directive_char, subdirective, condition)
+      ignored.gsub!(/\s/, '\\s') unless ignored.nil?
+      log_format_symbols << log_format
+      format_regex << "(#{match_regex})#{ignored}"
+    end
+    @format_symbols = log_format_symbols
+    @format_regex =  /^#{format_regex}/
+  end
+
+  def process_directive(directive_char, subdirective, condition)
+    directive = DIRECTIVES[directive_char]
+    case directive_char
+    when 'i'
+      log_format = subdirective[1...-1].downcase.tr('-', '_').to_sym
+      [log_format, directive[1].source]
+    else
+      [directive[0], directive[1].source]
+    end
+  end
+end
+
+class HttpLogParser
+
+  LOG_FORMATS = {
+    :common => '%h %l %u %t \"%r\" %>s %b',
+    :common_with_virtual => '%v %h %l %u %t \"%r\" %>s %b',
+    :combined => '%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\"',
+    :combined_with_virtual => '%v %h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\"',
+    :combined_with_cookies => '%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\" \"%{Cookies}i\"'
+  }
+
+  attr_reader :known_formats
+
+  def initialize
+    @log_format = []
+    initialize_known_formats
+  end
+
+  def initialize_known_formats
+    @known_formats = {}
+    LOG_FORMATS.each do |name, format|
+      @known_formats[name] = HttpLogFormat.new(name, format)
+    end
+  end
+
+  def check_format(line)
+    @known_formats.sort_by { |key, log_format| log_format.format_regex.source.size }.reverse.each { |key, log_format|
+      return key if line.match(log_format.format_regex)
+    }
+    return :unknown
+  end
+
+  def parse_line(line)
+    @format = check_format(line)
+    log_format = @known_formats[@format]
+    raise ArgumentError if log_format.nil? or line !~ log_format.format_regex
+    data = line.scan(log_format.format_regex).flatten
+    parsed_data = {}
+    log_format.format_symbols.size.times do |i|
+      parsed_data[log_format.format_symbols[i]] = data[i]
+    end
+
+    parsed_data[:datetime] = parsed_data[:datetime][1...-1] if parsed_data[:datetime]
+    parsed_data[:domain] = parsed_data[:ip] unless parsed_data[:domain]
+    parsed_data[:format] = @format
+
+    parsed_data
+  end
+end

data/lib/http_log_parser.rb

@@ -0,0 +1,4 @@
+# encoding: utf-8
+# module HttpLogParser # :doc:
+  require 'http/parser'
+# end

metadata

@@ -0,0 +1,65 @@
+--- !ruby/object:Gem::Specification 
+name: http-log-parser
+version: !ruby/object:Gem::Version 
+  prerelease: false
+  segments: 
+  - 0
+  - 0
+  - 1
+  version: 0.0.1
+platform: ruby
+authors: 
+- Torsten Curdt
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-04-22 00:00:00 +02:00
+default_executable: 
+dependencies: []
+
+description: HTTP log file parser
+email: tcurdt at vafer.org
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- README.rdoc
+files: 
+- README.rdoc
+- Rakefile
+- lib/http/parser.rb
+- lib/http_log_parser.rb
+has_rdoc: true
+homepage: http://github.com/tcurdt/http-log-parser
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.6
+signing_key: 
+specification_version: 3
+summary: A package for parsing web server logs.
+test_files: []
+