Aplo 0.1

data/example/comb.rb

@@ -0,0 +1,49 @@
+#
+# Comb - an Apache log analysis script.
+#
+# By Tim Fletcher <twoggle@gmail.com>
+#
+# Licensed under a Creative Commons Attribution-ShareAlike 2.5 License
+# http://creativecommons.org/licenses/by-sa/2.5/
+#
+# Requirements:
+#
+#   GeoIP <http://www.maxmind.com/download/geoip/api/c/>
+#   Lazy.rb <http://moonbase.rydia.net/software/lazy.rb/>
+#
+# Usage:
+#
+#   ruby comb.rb relative/glob/of/access_log*
+#
+#
+__DIR__ = File.dirname(__FILE__)
+
+$:.unshift File.join(__DIR__, '..', 'lib')
+
+require 'rubygems' rescue nil
+require 'aplo'
+require 'lazy'
+require 'comb/geoip'
+require 'comb/ascii_chart'
+require 'comb/itself'
+
+class Numeric
+  def MB
+    self / 1000000
+  end
+  def round_to_nearest(n)
+    r = self % n; self - r + (r < (n / 2.0) ? 0 : n)
+  end
+end
+
+def mean(enum)
+  enum.inject { |sum, time| sum + time } / enum.size
+end
+
+if __FILE__ == $0 then
+  Comb(
+    :logs => Dir[ARGV[0] || '../logs/access_log*'],
+    :cache => File.join(__DIR__, 'comb', 'geoip.cache'),
+    :top => 10 # countries
+  )
+end

data/example/comb/ascii_chart.rb

@@ -0,0 +1,46 @@
+class AsciiPercentageChart
+
+  def initialize(data, options = {})
+    @data, @options = data, options
+  end
+
+  # c.f. MenTaLguY's rjust_lines example [ruby-talk 164045]
+
+  def to_s
+    resulting = promise do
+      @data.inject({ :output => [], :label_length => 0 }) do |line, (label, value)|
+
+        line[:label_length] = label.length if label.length > line[:label_length]
+        
+        percentage = value.to_f * 100 / max
+
+        bar_length = percentage.round_to_nearest(granularity) / granularity
+
+        end_label = (bar_length == 0 ? '' : '  ') + ("%d, %.1f%%" % [ value, percentage ])
+      
+        line[:output] << promise {
+          "#{label.rjust(resulting[:label_length] + 2)} : " + (symbol * bar_length) + end_label
+        }
+
+        next line
+      end
+    end
+    return resulting[:output] * "\n"
+  end
+
+
+  private
+
+  def max
+    @options[:max]
+  end
+  
+  def granularity
+    @options[:granularity] ||= 2
+  end
+  
+  def symbol
+    @options[:symbol] ||= '*'
+  end
+
+end

data/example/comb/geoip.rb

@@ -0,0 +1,40 @@
+module GeoIP
+  class Cache
+    
+    # c.f. http://eigenclass.org/hiki.rb?Hash+default+block
+  
+    def initialize(filename)
+      @filename = filename
+      update!
+    end
+
+    def [](ip)
+      data[ip]
+    end
+
+    def save
+      data.default = nil
+      File.open(@filename, 'w+') { |f| Marshal.dump(data, f) }
+    end
+
+
+    private
+
+    def data
+      @data ||= Hash.new { |hash, ip| hash[ip] = lookup(ip) }
+    end
+
+    def lookup(ip)
+      convert `geoiplookup #{ip}`
+    end
+    
+    def convert(str)
+      str =~ /--, N\/A/ ? nil : str.sub(/^GeoIP Country Edition: /, '').sub(/[A-Z]+,/, '').strip
+    end
+
+    def update!
+      data.update Marshal.load(File.read(@filename)) rescue {}
+    end
+    
+  end
+end

data/example/comb/itself.rb

@@ -0,0 +1,37 @@
+def Comb(options)
+  logs, geoip_cache = options[:logs], GeoIP::Cache.new(options[:cache])
+
+  ip_addresses, hits, hits_per_country = {}, 0, Hash.new { |h, k| h[k] = 0 }
+  
+  parse_times, start_time = [], Time.now
+
+  logs.each do |filename|
+    time = Time.now
+    log = Apache::Log.parse(File.read(filename))
+    parse_times << (Time.now - time)
+    log.each do |entry|
+      hits += 1
+      ip_addresses[entry.remote_host] = true
+      hits_per_country[ geoip_cache[entry.remote_host] ] += 1
+    end
+  end
+  
+  time_taken = Time.now - start_time
+
+  top_countries = hits_per_country.delete_if { |label, _| label.nil? }.
+    sort_by { |_, n| n }.reverse[0, options[:top]]
+  
+  throughput = logs.inject(0) { |t, f| t + File.size(f) }
+  
+  geoip_cache.save
+  
+  puts
+  puts "> Processed #{logs.size} log files (%.1f Mb) in %.1f seconds" % [ throughput.MB, time_taken ]
+  puts "> Average log parse time of %.3f seconds" % mean(parse_times)
+  puts "> Total hits: %d" % hits
+  puts "> Unique IPs: %d" % ip_addresses.size
+  puts "> Top %d countries..." % options[:top]
+  puts
+  puts AsciiPercentageChart.new(top_countries, :max => hits, :symbol => '|')
+  puts
+end

data/lib/aplo.rb

@@ -0,0 +1,45 @@
+# Aplo - A Ruby wrapper for Apache log files.
+#
+# Copyright (c) 2006 Tim Fletcher <twoggle@gmail.com>
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+# of the Software, and to permit persons to whom the Software is furnished to
+# do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS
+# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR
+# IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+require 'aplo/directive'
+require 'aplo/directives'
+require 'aplo/entry'
+require 'aplo/format'
+
+module Apache
+
+  CommonLogFormat = Log::Format.new("%h %l %u %t \"%r\" %>s %b")
+
+  CombinedLogFormat = Log::Format.new("%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\"")
+
+  RefererLogFormat = Log::Format.new("%{Referer}i -> %U")
+  
+  AgentLogFormat = Log::Format.new("%{User-agent}i")
+
+  CommonTimeFormat = '%d/%b/%Y:%H:%M:%S %z'
+
+  module Log
+    def self.parse(lines, format = CombinedLogFormat)
+      lines.to_s.split(/\n/).map { |line| format.parse(line) }
+    end
+  end
+
+end

data/lib/aplo/directive.rb

@@ -0,0 +1,64 @@
+module Apache
+  module Log
+    class Directive
+
+      class <<self
+        def dir(char, name, pattern = nil)
+          directives[char] = [name, pattern]
+        end
+        def convert(directive, conversion = nil, &converter)
+          converters[directive] = conversion || converter
+        end
+      end
+      
+      attr_reader :name, :key
+      
+      alias format key
+    
+      def initialize(str)
+        @name, @pattern, @conversion = @@directives[str[-1, 1]]
+
+        if /\{(.+)\}/ =~ str
+          @key = $1
+        end
+      end
+
+      def pattern
+        @pattern ||= '(.+?)'
+      end
+      
+      def keyed?
+        !@key.nil?
+      end
+      
+      def formatted?
+        @name == :time || @name == :process_id
+      end
+
+      def process(str)
+        str == '-' ? nil : convert(str)
+      end
+
+
+      private
+
+      class <<self
+        def directives
+          @@directives ||= {}
+        end
+        def converters
+          @@converters ||= {}
+        end
+      end
+
+      def convert(str)
+        if converter = self.class.converters[name]
+          converter.respond_to?(:call) ? converter.call(str, self) : str.send(converter)
+        else
+          return str
+        end
+      end
+
+    end
+  end
+end

data/lib/aplo/directives.rb

@@ -0,0 +1,45 @@
+module Apache
+  module Log
+    class Directive
+
+      # c.f. http://httpd.apache.org/docs/2.2/mod/mod_log_config.html
+    
+      dir 'a', :remote_ip
+      dir 'A', :local_ip
+      dir 'B', :bytes_sent
+      dir 'b', :bytes_sent # CLF format, i.e. '-' instead of '0'
+      dir 'O', :bytes_sent_including_headers
+      dir 'I', :bytes_received
+      dir 'C', :cookies
+      dir 'e', :env
+      dir 'f', :filename
+      dir 'h', :remote_host
+      dir 'm', :request_method
+      dir 'H', :request_protocol
+      dir 'i', :request_headers, '(.*?)'
+      dir 'l', :remote_logname
+      dir 'n', :notes
+      dir 'o', :reply_headers
+      dir 'p', :port
+      dir 'P', :process_id
+      dir 'q', :query_string, '(.*?)'
+      dir 'r', :request
+      dir 's', :status
+      dir 'T', :time_taken
+      dir 'D', :time_taken # microseconds
+      dir 'u', :remote_user
+      dir 'U', :url_path
+      dir 'v', :canonical_server_name
+      dir 'V', :server_name
+      dir 'X', :connection_status
+      dir 't', :time, '\[(.+?)\]'
+
+      convert :bytes_sent, :to_i
+      convert :bytes_sent_including_headers, :to_i
+      convert :bytes_received, :to_i
+      convert :status, :to_i
+      convert :time_taken, :to_i
+
+    end
+  end
+end

data/lib/aplo/entry.rb

@@ -0,0 +1,45 @@
+module Apache
+  module Log
+    class Entry
+
+      include Enumerable
+      
+      def initialize(attributes)
+        @attributes = attributes.to_hash
+      end
+
+      def [](key)
+        @attributes[key]
+      end
+
+      def each
+        @attributes.each { |key, value| yield key, value }
+      end
+      
+      def inspect
+        @@inspect ||= "#<Apache::Log::Entry>"
+      end
+
+      def to_s
+        inject([]) do |lines, (directive, value)|
+          if value.is_a? Hash
+            lines << "#{directive}:"
+            value.each { |(k, v)| lines << "  #{k}: #{v}" }
+          else
+            lines << "#{directive}: #{value}"
+          end
+          next lines
+        end * "\n"
+      end
+
+
+      private
+
+      def method_missing(name, *args, &block)
+        return @attributes[name] if @attributes.has_key?(name)
+        super name, *args, &block
+      end
+
+    end
+  end
+end

data/lib/aplo/format.rb

@@ -0,0 +1,50 @@
+module Apache
+  module Log
+    class Format
+
+      def initialize(format_string)
+        @format_string = format_string.to_s
+      end
+
+      def to_s
+        @format_string
+      end
+
+      def parse(str)
+        if matchdata = pattern.match(str)
+          data = directives.zip(matchdata.captures).inject({}) do |data, (directive, capture)|
+            if directive.keyed? && !directive.formatted?
+              data[directive.name] ||= {}
+              data[directive.name][directive.key] = directive.process(capture)
+            else
+              data[directive.name] = directive.process(capture)
+            end
+            next data
+          end
+          return Entry.new(data)
+        else
+          raise "failed to match: %p" % str
+        end
+      end          
+
+
+      private
+
+      def pattern
+        @pattern ||= /^#{compile(@format_string)}$/
+      end
+
+      def compile(str)
+        str.gsub(/%(\S+)?([a-zA-Z])/) do |str|
+          directives << Directive.new(str)
+          directives.last.pattern
+        end
+      end
+
+      def directives
+        @directives ||= []
+      end
+      
+    end
+  end
+end

data/lib/aplo/strptime.rb

@@ -0,0 +1,50 @@
+require 'date'
+require 'inline'
+
+
+module Kernel
+  
+  # About 20 times faster than DateTime._strptime
+  
+  inline do |builder|
+    builder.include '<time.h>'
+    builder.c_raw %{
+      static VALUE strptime_c(int argc, VALUE *argv, VALUE self){
+        struct tm t;
+        char *string, *format;
+      
+        string = StringValuePtr(argv[0]);
+        format = StringValuePtr(argv[1]);
+
+        if(strptime(string, format, &t) == NULL)
+          /* TODO: handle error */;
+
+        t.tm_isdst = -1;
+
+        return rb_time_new(timegm(&t), 0); 
+      }
+    }
+  end
+
+  def strptime(str, fmt)
+    time = strptime_c(str, fmt)
+
+    # Most C strptime implementations don't handle timezones.
+    if fmt =~ / %z$/ && str =~ / ([-+:a-z0-9]+(?:\s+dst\b)?)$/io
+      time + Date.zone_to_diff($1)
+    else
+      time
+    end
+  end
+
+end
+
+module Apache
+  module Log
+    class Directive
+      convert(:time) do |str, dir|
+        strptime(str, dir.format || CommonTimeFormat)
+      end
+    end
+  end
+end

data/readme.txt

@@ -0,0 +1,46 @@
+= Aplo
+
+A Ruby wrapper for Apache log files.
+
+== Basic Usage
+
+  log_contents = File.read('/path/to/log/file')
+
+  Apache::Log.parse(log_contents).each do |entry|
+    ...
+    entry.url_path
+    entry.bytes_sent
+    entry.remote_host
+    entry.request_headers['Referer']
+    entry.request_headers['User-agent']
+    ...
+  end
+
+== Formats
+
+The `parse` method takes a second (optional) argument that specifies the format
+of the log. This defaults to `Apache::CombinedLogFormat`; `Apache::CommonLogFormat`,
+`RefererLogFormat`, and `AgentLogFormat` are also defined. Other formats can be
+created like so:
+
+  custom_format = Apache::Log::Format.new(format_string)
+
+== Default Type-casting
+
+Integers are converted using `to_i`, and `-` is converted to `nil`.
+
+== Shortcuts
+
+Adding methods to the `Entry` class is the easiest way to enhance the wrapper,
+for example:
+
+  require 'ipaddr'
+  
+  class Apache::Log::Entry
+    def ip
+      IPAddr.new(@attributes[:remote_host])
+    end
+  end
+
+A C/strptime/RubyInline extension is also included to help convert times;
+require 'aplo/strptime' to use it.

metadata

@@ -0,0 +1,55 @@
+--- !ruby/object:Gem::Specification 
+rubygems_version: 0.8.11
+specification_version: 1
+name: Aplo
+version: !ruby/object:Gem::Version 
+  version: "0.1"
+date: 2006-05-29 00:00:00 +01:00
+summary: A Ruby wrapper for Apache log files.
+require_paths: 
+- lib
+email: twoggle@gmail.com
+homepage: http://aplo.rubyforge.org/
+rubyforge_project: aplo
+description: 
+autorequire: aplo
+default_executable: 
+bindir: bin
+has_rdoc: false
+required_ruby_version: !ruby/object:Gem::Version::Requirement 
+  requirements: 
+  - - ">"
+    - !ruby/object:Gem::Version 
+      version: 0.0.0
+  version: 
+platform: ruby
+signing_key: 
+cert_chain: 
+authors: 
+- Tim Fletcher
+files: 
+- lib/aplo.rb
+- lib/aplo/directive.rb
+- lib/aplo/directives.rb
+- lib/aplo/entry.rb
+- lib/aplo/format.rb
+- lib/aplo/strptime.rb
+- example/comb.rb
+- example/comb/ascii_chart.rb
+- example/comb/geoip.rb
+- example/comb/itself.rb
+- readme.txt
+test_files: []
+
+rdoc_options: []
+
+extra_rdoc_files: []
+
+executables: []
+
+extensions: []
+
+requirements: []
+
+dependencies: []
+