log2counter 0.0.3

data/README

@@ -0,0 +1,33 @@
+= log2counter - Convert (analyse) Apache log files to COUNTER CSV.
+
+== VERSION
+
+This documentation refers to log2counter version 0.0.3
+
+
+== DESCRIPTION
+
+Convert (analyse) Apache log files to COUNTER[http://www.projectcounter.org/] CSV.
+
+
+== AUTHORS
+
+* Jens Wille <mailto:jens.wille@uni-koeln.de>
+
+
+== LICENSE AND COPYRIGHT
+
+Copyright (C) 2007-2009 University of Cologne,
+Albertus-Magnus-Platz, 50932 Cologne, Germany
+
+log2counter is free software: you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free Software
+Foundation, either version 3 of the License, or (at your option) any later
+version.
+
+log2counter is distributed in the hope that it will be useful, but WITHOUT
+ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along with
+log2counter. If not, see <http://www.gnu.org/licenses/>.

data/Rakefile

@@ -0,0 +1,24 @@
+require %q{lib/log2counter/version}
+
+begin
+  require 'hen'
+
+  Hen.lay! {{
+    :rubyforge => {
+      :project => %q{prometheus},
+      :package => %q{log2counter}
+    },
+
+    :gem => {
+      :version      => Log2COUNTER::VERSION,
+      :summary      => %q{Convert (analyse) Apache log files to COUNTER CSV.},
+      :files        => FileList['lib/**/*.rb', 'bin/*'].to_a,
+      :extra_files  => FileList['[A-Z]*', 'sample/*', 'lib/**/vendor/*'].to_a,
+      :dependencies => [['fastercsv', '>= 1.2.3']]
+    }
+  }}
+rescue LoadError
+  abort "Please install the 'hen' gem first."
+end
+
+### Place your custom Rake tasks here.

data/bin/log2counter

@@ -0,0 +1,143 @@
+#! /usr/bin/ruby
+
+#--
+###############################################################################
+#                                                                             #
+# log2counter -- Convert (analyse) Apache log files to COUNTER CSV.           #
+#                                                                             #
+# Copyright (C) 2007-2009 University of Cologne,                              #
+#                         Albertus-Magnus-Platz,                              #
+#                         50932 Cologne, Germany                              #
+#                                                                             #
+# Authors:                                                                    #
+#     Jens Wille <jens.wille@uni-koeln.de>                                    #
+#                                                                             #
+# log2counter is free software; you can redistribute it and/or modify it      #
+# under the terms of the GNU General Public License as published by the Free  #
+# Software Foundation; either version 3 of the License, or (at your option)   #
+# any later version.                                                          #
+#                                                                             #
+# log2counter is distributed in the hope that it will be useful, but WITHOUT  #
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or       #
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for    #
+# more details.                                                               #
+#                                                                             #
+# You should have received a copy of the GNU General Public License along     #
+# with log2counter. If not, see <http://www.gnu.org/licenses/>.               #
+#                                                                             #
+###############################################################################
+#++
+
+require 'optparse'
+require 'yaml'
+require 'zlib'
+
+$: << File.join(File.dirname(__FILE__), '..', 'lib')
+
+require 'log2counter'
+
+USAGE = "Usage: #{$0} [-h|--help] [options]"
+abort USAGE if ARGV.empty?
+
+def open_file(file, mode = 'r')
+  if file =~ /\.gz/i
+    case mode
+      when 'r' then Zlib::GzipReader.open(file)
+      when 'w' then Zlib::GzipWriter.open(file)
+      else          raise ArgumentError, "invalid mode #{mode}"
+    end
+  else
+    File.open(file, mode)
+  end
+end
+
+options = {
+  :licensees      => 'licensees.yaml',
+  :log_file       => STDIN,
+  :csv_file       => STDOUT,
+  :months         => nil,
+  :regexp         => nil,
+  :summarize      => false,
+  :summarize_file => nil
+}
+
+OptionParser.new { |opts|
+  opts.banner = USAGE
+
+  opts.separator ' '
+  opts.separator 'Options:'
+
+  opts.on('-i', '--log-file LOG',  'Input log file  [Default: STDIN]') { |f|
+    abort "Can't find log file #{f}" unless File.readable?(f)
+    options[:log_file] = open_file(f)
+  }
+
+  opts.on('-o', '--csv-file CSV',  'Output CSV file [Default: STDOUT]') { |f|
+    options[:csv_file] = open_file(f, 'w')
+  }
+
+  opts.separator ' '
+
+  opts.on('-l', '--licensees YAML', "Licensees list  [Default: #{options[:licensees]}]") { |f|
+    abort "Can't find licensee file #{f}" unless File.readable?(f)
+    options[:licensees] = f
+  }
+
+  opts.separator ' '
+
+  opts.on('-m', '--months LIST', 'Comma-separated list of month(s) [YYYY_MM]') { |m|
+    options[:months] = m.split(',')
+  }
+
+  opts.on('-r', '--regexp REGEXP', 'Comma-separated list of regular expressions', '[id=...,login=...,search=...,download=...]') { |r|
+    r.split(',').each { |str|
+      key, pat = str.split('=', 2)
+      (options[:regexp] ||= {})[key.to_sym] = Regexp.new(pat)
+    }
+  }
+
+  opts.separator ' '
+
+  opts.on('-s', '--summarize', "Summarize stats per licensee per month; don't", 'print client details') {
+    options[:summarize] = true
+  }
+
+  opts.on('-S', '--summarize-file CSV', 'Summarize existing CSV file (in full format);', "use '-' for STDIN") { |f|
+    options[:summarize] = true
+
+    if f == '-'
+      options[:summarize_file] = STDIN
+    else
+      abort "Can't find CSV file #{f}" unless File.readable?(f)
+      options[:summarize_file] = open_file(f)
+    end
+  }
+
+  opts.separator ' '
+  opts.separator 'Generic options:'
+
+  opts.on('-h', '--help', 'Print this help message and exit') {
+    puts opts
+    exit
+  }
+
+  opts.on('--version', 'Print program version and exit') {
+    puts "#{File.basename($0)} v#{Log2COUNTER::VERSION}"
+    exit
+  }
+
+  opts.separator ' '
+  opts.separator "If LOG or CSV ends in '.gz', it's treated as a Gzipped file."
+}.parse!
+
+begin
+  stats = if options[:summarize_file]
+    Log2COUNTER.load(options[:summarize_file])
+  else
+    Log2COUNTER.parse(options[:log_file], YAML.load_file(options[:licensees]), options[:months], options[:regexp])
+  end
+
+  Log2COUNTER.print(stats, options[:csv_file], options[:summarize])
+ensure
+   options.each { |k, v| v.close if v.respond_to?(:close) }
+end

data/lib/log2counter/core_ext/compare_strings_and_fixnums.rb

@@ -0,0 +1,56 @@
+#--
+###############################################################################
+#                                                                             #
+# A component of log2counter, the Apache log to COUNTER CSV converter.        #
+#                                                                             #
+# Copyright (C) 2007-2009 University of Cologne,                              #
+#                         Albertus-Magnus-Platz,                              #
+#                         50932 Cologne, Germany                              #
+#                                                                             #
+# Authors:                                                                    #
+#     Jens Wille <jens.wille@uni-koeln.de>                                    #
+#                                                                             #
+# log2counter is free software; you can redistribute it and/or modify it      #
+# under the terms of the GNU General Public License as published by the Free  #
+# Software Foundation; either version 3 of the License, or (at your option)   #
+# any later version.                                                          #
+#                                                                             #
+# log2counter is distributed in the hope that it will be useful, but WITHOUT  #
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or       #
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for    #
+# more details.                                                               #
+#                                                                             #
+# You should have received a copy of the GNU General Public License along     #
+# with log2counter. If not, see <http://www.gnu.org/licenses/>.               #
+#                                                                             #
+###############################################################################
+#++
+
+# Allow to compare strings and fixnums; the latter sorting before the other.
+
+class String
+
+  alias_method :_log2counter_original_cmp, :<=>
+
+  def <=>(other)
+    case other
+      when Fixnum then 1  # Fixnums always sort before us.
+      else             _log2counter_original_cmp(other)
+    end
+  end
+
+end
+
+
+class Fixnum
+
+  alias_method :_log2counter_original_cmp, :<=>
+
+  def <=>(other)
+    case other
+      when String then -1  # Strings always sort after us.
+      else             _log2counter_original_cmp(other)
+    end
+  end
+
+end

data/lib/log2counter/core_ext/sort_by_ip_or_host.rb

@@ -0,0 +1,43 @@
+#--
+###############################################################################
+#                                                                             #
+# A component of log2counter, the Apache log to COUNTER CSV converter.        #
+#                                                                             #
+# Copyright (C) 2007-2009 University of Cologne,                              #
+#                         Albertus-Magnus-Platz,                              #
+#                         50932 Cologne, Germany                              #
+#                                                                             #
+# Authors:                                                                    #
+#     Jens Wille <jens.wille@uni-koeln.de>                                    #
+#                                                                             #
+# log2counter is free software; you can redistribute it and/or modify it      #
+# under the terms of the GNU General Public License as published by the Free  #
+# Software Foundation; either version 3 of the License, or (at your option)   #
+# any later version.                                                          #
+#                                                                             #
+# log2counter is distributed in the hope that it will be useful, but WITHOUT  #
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or       #
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for    #
+# more details.                                                               #
+#                                                                             #
+# You should have received a copy of the GNU General Public License along     #
+# with log2counter. If not, see <http://www.gnu.org/licenses/>.               #
+#                                                                             #
+###############################################################################
+#++
+
+# We need strings and fixnums to be comparable.
+require 'log2counter/core_ext/compare_strings_and_fixnums'
+
+class Hash
+
+  # Sort IP addresses numerically by net part, and host names just as usual.
+  def sort_by_ip_or_host
+    sort_by { |key, _|
+      key ? key.split('.').map { |part|
+        part =~ /\A\d+\z/ ? part.to_i : part
+      } : []
+    }
+  end
+
+end

data/lib/log2counter/parser.rb

@@ -0,0 +1,213 @@
+#--
+###############################################################################
+#                                                                             #
+# A component of log2counter, the Apache log to COUNTER CSV converter.        #
+#                                                                             #
+# Copyright (C) 2007-2009 University of Cologne,                              #
+#                         Albertus-Magnus-Platz,                              #
+#                         50932 Cologne, Germany                              #
+#                                                                             #
+# Authors:                                                                    #
+#     Jens Wille <jens.wille@uni-koeln.de>                                    #
+#                                                                             #
+# log2counter is free software; you can redistribute it and/or modify it      #
+# under the terms of the GNU General Public License as published by the Free  #
+# Software Foundation; either version 3 of the License, or (at your option)   #
+# any later version.                                                          #
+#                                                                             #
+# log2counter is distributed in the hope that it will be useful, but WITHOUT  #
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or       #
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for    #
+# more details.                                                               #
+#                                                                             #
+# You should have received a copy of the GNU General Public License along     #
+# with log2counter. If not, see <http://www.gnu.org/licenses/>.               #
+#                                                                             #
+###############################################################################
+#++
+
+require 'resolv'
+require 'uri'
+
+# Get us some help for the hard part... -- um, it's not that hard, is it? ;-)
+require 'log2counter/vendor/log_parser'
+
+class Log2COUNTER::Parser
+
+  # Map month abbreviations to their two-digit number equivalent.
+  ABBR2MONTH = {}
+  Date::ABBR_MONTHNAMES.each_with_index { |abbr, index|
+    ABBR2MONTH[abbr] = '%02d' % index
+  }
+
+  year = Time.now.year
+
+  # By default we will consider the current year.
+  DEFAULT_MONTHS = (1..12).map { |month| '%d_%02d' % [year, month] }
+
+  # This is what we start with -- all zero.
+  DEFAULT_STATS = {
+    :sessions  => 0,
+    :searches  => 0,
+    :downloads => 0
+  }
+
+  # NOTE: <tt>:id</tt> should contain capture group for ID
+  DEFAULT_REGEXP = {
+    :id       => //,
+    :login    => //,
+    :search   => //,
+    :download => //
+  }
+
+  class << self
+
+    def load(csv_file)
+      FasterCSV.new(csv_file, :headers => true).inject({}) { |stats, row|
+        month, licensee, name, address, sessions, searches, downloads = row.fields
+
+        (((stats[month] ||= {})[licensee] ||= {})[name] ||= {})[address] = {
+          :sessions  => sessions.to_i,
+          :searches  => searches.to_i,
+          :downloads => downloads.to_i
+        }
+
+        stats
+      }
+    end
+
+  end
+
+  attr_reader :log_file, :licensees, :months, :licensees_by_ip, :licensees_by_id, :regexp, :constraint
+
+  def initialize(log_file, licensees, months = nil, regexp = nil)
+    @log_file = log_file
+
+    @months = months || DEFAULT_MONTHS
+    raise ArgumentError, "illegal format for month; must be YYYY_MM" if @months.any? { |month|
+      month !~ /\A\d\d\d\d_\d\d\z/
+    }
+
+    @regexp = DEFAULT_REGEXP.merge(regexp || {})
+    @constraint = Regexp.union(*@regexp.values)
+
+    @licensees = licensees.reject { |_, hash| !hash[:export] }
+    initialize_licensees
+  end
+
+  # Now here's the method you want to call. Returns a hash:
+  #
+  #   stats = {
+  #     '2007_06' => {
+  #       'Somewhere, Inst.' => {
+  #         '12.34.56.78' => {
+  #           :sessions  => 12,
+  #           :searches  => 34,
+  #           :downloads => 56
+  #         },
+  #         ...
+  #       },
+  #       ...
+  #     },
+  #     ...
+  #   }
+  def parse
+    # Cache resolved host names.
+    addr2addr = Hash.new { |hash, addr|
+      hash[addr] = begin
+        Resolv.getaddress(addr)
+      rescue Resolv::ResolvError
+        addr
+      end
+    }
+
+    # Cache licensees.
+    addr2lcee = Hash.new { |hash, addr|
+      hash[addr] = licensees_by_ip.get(addr)
+    }
+
+    # Our result hash
+    stats = {}
+
+    # Create a new LogParser and send our log file. Yields a hash per line.
+    LogParser.new(:minimal, constraint).parse_io_stream(log_file) { |stat|
+      path = stat['PATH_INFO']
+
+      # Skip lines that don't have any useful information for us anyway.
+      next unless path =~ constraint
+
+      # Maybe we already captured the licensee ID? (see DEFAULT_REGEXP above)
+      id = $1
+
+      m, y  = stat['DATETIME'][/\/(.*?):/, 1].split('/')  # Extract month and year
+      month = [y, ABBR2MONTH[m]].join('_')                # Target format is 'YYYY_MM'
+
+      # Skip lines that fall out of the range we're interested in.
+      next unless months.include?(month)
+
+      address  = addr2addr[stat['REMOTE_ADDR']]
+      licensee = addr2lcee[address] || licensees_by_id[
+        URI.decode(id || path[regexp[:id], 1] || '')
+      ]
+
+      # Couldn't find a matching licensee? Skip it!
+      next unless licensee
+
+      name     = licensee[:name]
+      licensee = licensee[:licensee]
+
+      (((stats[month] ||= {})[licensee] ||= {})[name] ||= {})[address] ||= DEFAULT_STATS.dup
+      _address = stats[month][licensee][name][address]
+
+      # Increment our counts, since that's what we're here for...
+      _address[:sessions]  += 1 if path =~ regexp[:login]
+      _address[:searches]  += 1 if path =~ regexp[:search]
+      _address[:downloads] += 1 if path =~ regexp[:download]
+    }
+
+    # Now we need to fill in any months and licensees we didn't come across before.
+    months.each { |month|
+      stats[month] ||= {}
+
+      licensees.each { |licensee, hash|
+        stats[month][licensee] ||= {}
+        addresses = stats[month][licensee][hash[:name]]
+
+        if addresses
+          # Drop entries with zero sessions -- how come they occur, anyway?
+          addresses.delete_if { |_, stat| stat[:sessions].zero? }
+        end
+
+        # Add a default "empty" entry for completeness' sake.
+        if addresses.nil? || addresses.empty?
+          stats[month][licensee][hash[:name]] = { nil => DEFAULT_STATS }
+        end
+      }
+    }
+
+    # That's it, return what we've got.
+    stats
+  end
+
+  protected
+
+  # Create additional mappings for our licensees.
+  def initialize_licensees
+    @licensees_by_ip = {}
+    @licensees_by_id = {}
+
+    licensees.each { |licensee, hash|
+      _hash = { :licensee => licensee, :name => hash[:name]}
+
+      hash[:ip].each { |ip| licensees_by_ip[ip] = _hash }
+      hash[:id].each { |id| licensees_by_id[id] = _hash }
+    }
+
+    # Convenience method to get a licensee from an address. Note that
+    # +licensees_by_ip+ usually has subnets instead of full IPs.
+    def licensees_by_ip.get(ip)
+      find(lambda { [] }) { |key, _| ip[0, key.length] == key }.last
+    end
+  end
+
+end

data/lib/log2counter/printer.rb

@@ -0,0 +1,100 @@
+#--
+###############################################################################
+#                                                                             #
+# A component of log2counter, the Apache log to COUNTER CSV converter.        #
+#                                                                             #
+# Copyright (C) 2007-2009 University of Cologne,                              #
+#                         Albertus-Magnus-Platz,                              #
+#                         50932 Cologne, Germany                              #
+#                                                                             #
+# Authors:                                                                    #
+#     Jens Wille <jens.wille@uni-koeln.de>                                    #
+#                                                                             #
+# log2counter is free software; you can redistribute it and/or modify it      #
+# under the terms of the GNU General Public License as published by the Free  #
+# Software Foundation; either version 3 of the License, or (at your option)   #
+# any later version.                                                          #
+#                                                                             #
+# log2counter is distributed in the hope that it will be useful, but WITHOUT  #
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or       #
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for    #
+# more details.                                                               #
+#                                                                             #
+# You should have received a copy of the GNU General Public License along     #
+# with log2counter. If not, see <http://www.gnu.org/licenses/>.               #
+#                                                                             #
+###############################################################################
+#++
+
+require 'rubygems'
+require 'fastercsv'  # >= 1.2.3 for Gzip support
+
+# We'd like to sort hashes by keys of IPs and host names.
+require 'log2counter/core_ext/sort_by_ip_or_host'
+
+class Log2COUNTER::Printer
+
+  COLUMNS = %w[
+    Jahr_Monat
+    Konsortial-Mitglied
+    Nutzer
+    Einzelkunden-Subidentifier
+    Sessions
+    Searches
+    Downloads
+  ]
+
+  attr_reader :csv, :summarize
+
+  def initialize(csv_file, summarize = false)
+    @csv = FasterCSV.new(csv_file)
+
+    @summarize = summarize
+  end
+
+  def print(stats)
+    # Output CSV header.
+    csv << COLUMNS
+
+    stats.sort.each { |month, licensees|
+      licensees.sort.each { |licensee, names|
+        names.sort.each { |name, addresses|
+          if summarize
+            total = Log2COUNTER::Parser::DEFAULT_STATS.dup
+
+            addresses.each { |address, stat|
+              total[:sessions]  += stat[:sessions]
+              total[:searches]  += stat[:searches]
+              total[:downloads] += stat[:downloads]
+            }
+
+            csv << [
+              month,
+              licensee,
+              name,
+              nil,
+              total[:sessions],
+              total[:searches],
+              total[:downloads]
+            ]
+          else
+            addresses.sort_by_ip_or_host.each { |address, stat|
+              csv << [
+                month,
+                licensee,
+                name,
+                address,
+                stat[:sessions],
+                stat[:searches],
+                stat[:downloads]
+              ]
+            }
+          end
+        }
+      }
+    }
+
+    csv.flush
+  end
+
+end