log2counter 0.0.3

data/lib/log2counter/vendor/log_parser.rb

@@ -0,0 +1,220 @@
+# This program parses weblogs in the NCSA Common (access log) format or
+# NCSA Combined log format
+#
+# One line consists of
+#  host rfc931 username date:time request statuscode bytes
+# For example
+#  1.2.3.4 - dsmith [10/Oct/1999:21:15:05 +0500] "GET /index.html HTTP/1.0" 200 12
+#                   [dd/MMM/yyyy:hh:mm:ss +-hhmm]
+# Where
+#  dd is the day of the month
+#  MMM is the month
+#  yyy is the year
+#  :hh is the hour
+#  :mm is the minute
+#  :ss is the seconds
+#  +-hhmm is the time zone
+#
+# In practice, the day is typically logged in two-digit format even for
+# single-digit days.
+# For example, the second day of the month would be represented as 02.
+# However, some HTTP servers do log a single digit day as a single digit.
+# When parsing log records, you should be aware of both possible day
+# representations.
+#
+# Author:: Jan Wikholm [jw@jw.fi]
+# License:: MIT
+#
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+#
+# Modified by Jens Wille <jens.wille@uni-koeln.de>; see log_parser.rb.orig for the
+# original file.
+
+require 'logger'
+
+class LogFormat
+
+  attr_reader :name, :format, :format_symbols, :format_regex
+
+  # Add more format directives here.
+  DIRECTIVES = {
+    # format string char => [:symbol to use, /regex to use when matching against log/]
+    'h' => [:ip,           /(?:\d+\.\d+\.\d+\.\d+)|(?:[\w.-]+)/],
+    'l' => [:auth,         /.*?/],
+    'u' => [:username,     /.*?/],
+    't' => [:datetime,     /\[.*?\]/],
+    'r' => [:request,      /.*?/],
+    'R' => [:request,      /.*?(:?\"|\z)/],
+    's' => [:status,       /\d+/],
+    'b' => [:bytecount,    /-|\d+/],
+    'v' => [:domain,       /.*?/],
+    'i' => [:header_lines, /.*?/],
+  }
+
+  def initialize(name, format)
+    @name, @format = name, format
+    parse_format(format)
+  end
+
+  # The symbols are used to map the log to the env variables.
+  # The regex is used when checking what format the log is and to extract data.
+  def parse_format(format)
+    format_directive = /%(.*?)(\{.*?\})?([#{[DIRECTIVES.keys.join('|')]}])([\s\\"]*)/
+
+    log_format_symbols = []
+    format_regex       = ''
+
+    format.scan(format_directive) { |condition, subdirective, directive_char, ignored|
+      log_format, match_regex = process_directive(directive_char, subdirective, condition)
+
+      ignored.gsub!(/\s/, '\\s') if ignored
+
+      log_format_symbols << log_format
+      format_regex       << "(#{match_regex})#{ignored}"
+    }
+
+    @format_symbols = log_format_symbols
+    @format_regex   = /\A#{format_regex}/
+  end
+
+  def process_directive(directive_char, subdirective, condition)
+    directive = DIRECTIVES[directive_char]
+
+    case directive_char
+      when 'i'
+        log_format = subdirective[1...-1].downcase.tr('-', '_').to_sym
+        [log_format, directive[1].source]
+      else
+        [directive[0], directive[1].source]
+    end
+  end
+
+end
+
+class LogParser
+
+  LOG_FORMATS = {
+    :common   => '%h %l %u %t \"%r\" %>s %b',
+    :combined => '%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\"',
+    :minimal  => '%h %l %u %t \"%R'
+  }
+
+  # Add any values that you may return here.
+  STAT_ENV_MAP = {
+    :ip         => 'REMOTE_ADDR',
+    :page       => 'PATH_INFO',
+    :datetime   => 'DATETIME',
+    :status     => 'STATUS',
+    :domain     => 'HTTP_HOST',
+    :referer    => 'HTTP_REFERER',
+    :user_agent => 'HTTP_USER_AGENT'
+  }
+
+  attr_reader :constraint, :known_formats, :log_format
+
+  def initialize(format = nil, constraint = nil)
+    @format     = format
+    @constraint = constraint
+
+    initialize_known_formats
+
+    @log_format = known_formats[@format] if @format
+  end
+
+  # Processes the format string into symbols and test regex
+  # and saves using LogFormat class.
+  def initialize_known_formats
+    @known_formats = {}
+
+    LOG_FORMATS.each { |name, format|
+      @known_formats[name] = LogFormat.new(name, format)
+    }
+  end
+
+  # Checks which standard the log file (well one line) is.
+  # Automatically checks for most complex (longest) regex first.
+  def check_format(line)
+    @known_formats.sort_by { |key, log_format|
+      log_format.format_regex.source.size
+    }.reverse.each { |key, log_format|
+      return key if line.match(log_format.format_regex)
+    }
+
+    return :unknown
+  end
+
+  # This is where the magic happens.
+  # This is the end-to-end business logic of the class.
+  #
+  # Call with a block that will be called with each line, as a hash.
+  def parse_io_stream(stream)
+    stats = []
+    lines_parsed = 0
+
+    stream.each { |line|
+      line.chomp!
+      lines_parsed += 1
+      warn "##{lines_parsed}" if (lines_parsed % 10000).zero?
+
+      next if constraint && line !~ constraint
+
+      begin
+        parsed_data = parse_line(line)
+        yield generate_stats(parsed_data)
+      rescue FormatError
+        warn "Corrupt line [#{lines_parsed}]: #{line.inspect}"
+      rescue => err
+        raise err.class, "#{err.class} [#{lines_parsed}]: #{line.inspect}\n\n" <<
+                         "#{parsed_data.inspect}\n\n#{err}"
+      end
+    }
+  end
+
+  # Populate a stats hash one line at a time.
+  # Add extra fields into the STAT_ENV_MAP hash at the top of this file.
+  def generate_stats(parsed_data)
+    stats = { 'PATH_INFO' => get_page(parsed_data[:request]) }
+
+    STAT_ENV_MAP.each { |stat_name, env_name|
+      stats[env_name] = parsed_data[stat_name] if parsed_data.has_key?(stat_name)
+    }
+
+    stats
+  end
+
+  def get_page(request)
+    (request[/\/.*?(?:\s|\z)/] || request).strip
+  end
+
+  def parse_line(line)
+    unless @format && @log_format
+      @format     = check_format(line)
+      @log_format = known_formats[@format]
+
+      unless log_format && line =~ log_format.format_regex
+        raise FormatError, line
+      end
+    end
+
+    data = line.scan(log_format.format_regex).flatten
+
+    parsed_data = {}
+    log_format.format_symbols.each_with_index { |format_symbol, index|
+      parsed_data[format_symbol] = data[index]
+    }
+
+    # Remove [] from time.
+    parsed_data[:datetime] &&= parsed_data[:datetime][1...-1]
+
+    # Add IP as domain if we don't have a domain (virtual host).
+    # Assumes we always have an IP.
+    parsed_data[:domain] ||= parsed_data[:ip]
+
+    parsed_data[:format] = @format
+
+    parsed_data
+  end
+
+  class FormatError < ArgumentError; end
+
+end

data/lib/log2counter/vendor/log_parser.rb.orig

@@ -0,0 +1,188 @@
+#!/usr/bin/ruby
+# This program parses weblogs in the NCSA Common (access log) format or
+# NCSA Combined log format
+#
+# One line consists of
+#  host rfc931 username date:time request statuscode bytes
+# For example
+#  1.2.3.4 - dsmith [10/Oct/1999:21:15:05 +0500] "GET /index.html HTTP/1.0" 200 12
+#                   [dd/MMM/yyyy:hh:mm:ss +-hhmm]
+# Where
+#  dd is the day of the month
+#  MMM is the month
+#  yyy is the year
+#  :hh is the hour
+#  :mm is the minute
+#  :ss is the seconds
+#  +-hhmm is the time zone
+#
+# In practice, the day is typically logged in two-digit format even for 
+# single-digit days. 
+# For example, the second day of the month would be represented as 02. 
+# However, some HTTP servers do log a single digit day as a single digit. 
+# When parsing log records, you should be aware of both possible day 
+# representations.
+#
+# Author:: Jan Wikholm [jw@jw.fi]
+# License:: MIT
+
+require 'logger'
+
+class LogFormat
+  attr_reader :name, :format, :format_symbols, :format_regex
+
+  # add more format directives here..
+  DIRECTIVES = {
+    # format string char => [:symbol to use, /regex to use when matching against log/]
+    'h' => [:ip, /\d+\.\d+\.\d+\.\d+/],
+    'l' => [:auth, /.*?/],
+    'u' => [:username, /.*?/],
+    't' => [:datetime, /\[.*?\]/],
+    'r' => [:request, /.*?/],
+    's' => [:status, /\d+/],
+    'b' => [:bytecount, /-|\d+/],
+    'v' => [:domain, /.*?/],
+    'i' => [:header_lines, /.*?/], 
+  }
+
+  def initialize(name, format)
+    @name, @format = name, format
+    parse_format(format)
+  end
+  
+  # The symbols are used to map the log to the env variables
+  # The regex is used when checking what format the log is and to extract data
+  def parse_format(format)
+    format_directive = /%(.*?)(\{.*?\})?([#{[DIRECTIVES.keys.join('|')]}])([\s\\"]*)/
+
+    log_format_symbols = []
+    format_regex = ""
+    format.scan(format_directive) do |condition, subdirective, directive_char, ignored|
+      log_format, match_regex = process_directive(directive_char, subdirective, condition)
+      ignored.gsub!(/\s/, '\\s') unless ignored.nil?
+      log_format_symbols << log_format
+      format_regex << "(#{match_regex})#{ignored}"
+    end
+    @format_symbols = log_format_symbols
+    @format_regex =  /^#{format_regex}/
+  end
+
+  def process_directive(directive_char, subdirective, condition)
+    directive = DIRECTIVES[directive_char]
+    case directive_char 
+    when 'i'
+      log_format = subdirective[1...-1].downcase.tr('-', '_').to_sym
+      [log_format, directive[1].source]
+    else
+      [directive[0], directive[1].source]
+    end
+  end
+end
+
+class LogParser
+
+  LOG_FORMATS = {
+    :common => '%h %l %u %t \"%r\" %>s %b',
+    :common_with_virtual => '%v %h %l %u %t \"%r\" %>s %b',
+    :combined => '%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\"',
+    :combined_with_virtual => '%v %h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\"',
+    :combined_with_cookies => '%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\" \"%{Cookies}i\"',
+  }
+
+  # add any values that you may return here
+  STAT_ENV_MAP = {
+    :referer => "HTTP_REFERER",
+    :user_agent => "HTTP_USER_AGENT",
+    :ip => "REMOTE_ADDR",
+    :page => "PATH_INFO",
+    :domain => "HTTP_HOST",
+    :datetime => 'DATETIME',
+    :status => 'STATUS'
+  }
+  
+  attr_reader :known_formats
+
+  @@log = ActiveRecord::Base.logger
+  
+  def initialize()
+    @log_format = []
+    initialize_known_formats
+  end
+
+  # processes the format string into symbols and test regex
+  # and saves using LogFormat class
+  def initialize_known_formats
+    @known_formats = {}
+    LOG_FORMATS.each do |name, format|
+      @known_formats[name] = LogFormat.new(name, format)
+    end
+  end
+
+
+  # Checks which standard the log file (well one line) is
+  # Automatigally checks for most complex (longest) regex first..
+  def check_format(line)
+    @known_formats.sort_by { |key, log_format| log_format.format_regex.source.size }.reverse.each { |key, log_format|
+      #@@log.debug "check format: #{key}"
+      return key if line.match(log_format.format_regex)
+    }
+    return :unknown
+  end
+  
+  # This is where the magic happens
+  # This is the end-to-end business logic of the class
+  #
+  # Call with a block that will be called with each line, as a hash
+  def parse_io_stream(stream)
+    stats = []
+    lines_parsed = 0
+    begin
+      stream.each do |line|
+        lines_parsed += 1
+        #@@log.debug("parse_io_stream() line: #{line.to_s}")        
+        raw_data = parse_line(line)
+        #@@log.debug(raw_data.inspect)        
+        #@@log.debug("parse_io_stream() lines parsed: #{lines_parsed}")
+        yield generate_stats(raw_data)
+      end
+    end
+  end
+
+  # Generate_stats will populate a stats hash one line at a time
+  # Add extra fields into the STAT_ENV_MAP hash at the top of this file.
+  def generate_stats(raw_data)
+    stats = { "PATH_INFO" => get_page(raw_data[:request]) }
+    STAT_ENV_MAP.each do |stat_name, env_name|
+      stats[env_name] = raw_data[stat_name] if raw_data.has_key? stat_name
+    end
+    #@@log.debug("stats: " + stats.inspect)        
+    stats
+  end
+
+  def get_page(request)
+    @@log.debug "get_page: #{request}"
+    request[/\/.*?\s/].rstrip
+  end  
+
+  def parse_line(line)
+    @format = check_format(line)
+    log_format = @known_formats[@format]
+    raise ArgumentError if log_format.nil? or line !~ log_format.format_regex
+    data = line.scan(log_format.format_regex).flatten
+    #@@log.debug "parse_line() scanned data: #{data.inspect}"
+    parsed_data = {}
+    log_format.format_symbols.size.times do |i|
+      #@@log.debug "setting #{log_format.format_symbols[i]} to #{data[i]}"
+      parsed_data[log_format.format_symbols[i]] = data[i]
+    end
+
+    #remove [] from time if present
+    parsed_data[:datetime] = parsed_data[:datetime][1...-1] if parsed_data[:datetime]
+    # Add ip as domain if we don't have a domain (virtual host)
+    # Assumes we always have an ip
+    parsed_data[:domain] = parsed_data[:ip] unless parsed_data[:domain]
+    parsed_data[:format] = @format
+    #@@log.debug "parse_line() parsed data: #{parsed_data.inspect}"
+    parsed_data
+  end
+end

data/lib/log2counter/version.rb

@@ -0,0 +1,27 @@
+module Log2COUNTER
+
+  module Version
+
+    MAJOR = 0
+    MINOR = 0
+    TINY  = 3
+
+    class << self
+
+      # Returns array representation.
+      def to_a
+        [MAJOR, MINOR, TINY]
+      end
+
+      # Short-cut for version string.
+      def to_s
+        to_a.join('.')
+      end
+
+    end
+
+  end
+
+  VERSION = Version.to_s
+
+end

data/lib/log2counter.rb

@@ -0,0 +1,53 @@
+#--
+###############################################################################
+#                                                                             #
+# log2counter -- Convert (analyse) Apache log files to COUNTER CSV.           #
+#                                                                             #
+# Copyright (C) 2007-2009 University of Cologne,                              #
+#                         Albertus-Magnus-Platz,                              #
+#                         50932 Cologne, Germany                              #
+#                                                                             #
+# Authors:                                                                    #
+#     Jens Wille <jens.wille@uni-koeln.de>                                    #
+#                                                                             #
+# log2counter is free software; you can redistribute it and/or modify it      #
+# under the terms of the GNU General Public License as published by the Free  #
+# Software Foundation; either version 3 of the License, or (at your option)   #
+# any later version.                                                          #
+#                                                                             #
+# log2counter is distributed in the hope that it will be useful, but WITHOUT  #
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or       #
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for    #
+# more details.                                                               #
+#                                                                             #
+# You should have received a copy of the GNU General Public License along     #
+# with log2counter. If not, see <http://www.gnu.org/licenses/>.               #
+#                                                                             #
+###############################################################################
+#++
+
+module Log2COUNTER
+end
+
+require 'log2counter/parser'
+require 'log2counter/printer'
+
+require 'log2counter/version'
+
+module Log2COUNTER
+
+  extend self
+
+  def load(*args)
+    Parser.load(*args)
+  end
+
+  def parse(*args)
+    Parser.new(*args).parse
+  end
+
+  def print(stats, *args)
+    Printer.new(*args).print(stats)
+  end
+
+end

data/sample/licensees.yaml

@@ -0,0 +1,8 @@
+---
+#"Somewhere, Inst.":
+#  :name: "Inst_Somewhere"
+#  :ip:
+#    - "12.34.56"
+#  :id:
+#    - "inst.somewhere"
+#  :export: true

metadata

@@ -0,0 +1,87 @@
+--- !ruby/object:Gem::Specification 
+name: log2counter
+version: !ruby/object:Gem::Version 
+  version: 0.0.3
+platform: ruby
+authors: 
+- Jens Wille
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-07-23 00:00:00 +02:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: fastercsv
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.2.3
+    version: 
+description: Convert (analyse) Apache log files to COUNTER CSV.
+email: jens.wille@uni-koeln.de
+executables: 
+- log2counter
+extensions: []
+
+extra_rdoc_files: 
+- COPYING
+- ChangeLog
+- README
+files: 
+- lib/log2counter.rb
+- lib/log2counter/vendor/log_parser.rb
+- lib/log2counter/printer.rb
+- lib/log2counter/core_ext/compare_strings_and_fixnums.rb
+- lib/log2counter/core_ext/sort_by_ip_or_host.rb
+- lib/log2counter/version.rb
+- lib/log2counter/parser.rb
+- bin/log2counter
+- Rakefile
+- COPYING
+- ChangeLog
+- README
+- sample/licensees.yaml
+- lib/log2counter/vendor/log_parser.rb.orig
+has_rdoc: true
+homepage: http://prometheus.rubyforge.org/log2counter
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --charset
+- UTF-8
+- --main
+- README
+- --line-numbers
+- --inline-source
+- --all
+- --title
+- log2counter Application documentation
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: prometheus
+rubygems_version: 1.3.5
+signing_key: 
+specification_version: 3
+summary: Convert (analyse) Apache log files to COUNTER CSV.
+test_files: []
+