logmerge 1.0.0

data/LICENSE

@@ -0,0 +1,28 @@
+All original code copyright 2005 Eric Hodel, The Robot Co-op.  All rights
+reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+   notice, this list of conditions and the following disclaimer in the
+   documentation and/or other materials provided with the distribution.
+3. Neither the names of the authors nor the names of their contributors
+   may be used to endorse or promote products derived from this software
+   without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS
+OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
+OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
+OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+

data/Manifest.txt

@@ -0,0 +1,14 @@
+LICENSE
+Manifest.txt
+README
+Rakefile
+bin/ip2name
+bin/logmerge
+lib/logmerge.rb
+lib/logmerge/merger.rb
+lib/logmerge/resolv.rb
+lib/logmerge/resolver.rb
+resolv_test
+test/access.log
+test/test_merger.rb
+test/test_resolver.rb

data/README

@@ -0,0 +1,35 @@
+= logmerge
+
+Rubyforge Project:
+
+http://rubyforge.org/projects/rctools/
+
+== About
+
+Logmerge contains two utilities logmerge and ip2name.  logmerge merges Apache
+access logs into one log ordered by date.  ip2name performs DNS lookups on
+Apache access logs using multiple threads and Ruby's DNS resolver library to
+speed through log files.
+
+== Using logmerge
+
+Simply pass in all the logs you want to merge on the command line.  logmerge
+gives you the merged log on STDOUT.
+
+  logmerge hal/access.log nestor/access.log herbie/access.log > merged.log
+
+Note that logmerge expects the input log files to be ordered by date.
+
+== Using ip2name
+
+Simply pass in the log files you want to perform DNS lookups on the command
+line or via STDIN.  ip2name gives you the looked-up log lines on STDOUT.
+
+  ip2name < merged.log > resolved.log
+
+  ip2name merged.log > resolved.log
+
+In order to speed DNS lookups, ip2name creates a .name_cache file in the
+current directory.  ip2name uses double the DNS record TTL value so
+rapidly-changing names may not be correctly resolved.
+

data/Rakefile

@@ -0,0 +1,63 @@
+require 'rake'
+require 'rake/testtask'
+require 'rake/rdoctask'
+require 'rake/gempackagetask'
+
+$VERBOSE = nil
+
+spec = Gem::Specification.new do |s|
+  s.name = 'logmerge'
+  s.version = '1.0.0'
+  s.summary = 'Resolves IP addresses and merges Apache access logs.'
+  s.author = 'Eric Hodel'
+  s.email = 'eric@robotcoop.com'
+
+  s.files = File.read('Manifest.txt').split($/)
+  s.require_path = 'lib'
+
+  s.executables = %w[ip2name logmerge]
+end
+
+desc 'Run tests'
+task :default => [ :test ]
+
+Rake::TestTask.new('test') do |t|
+  t.libs << 'test'
+  t.verbose = true
+end
+
+desc 'Update Manifest.txt'
+task :update_manifest do
+  sh "find . -type f | sed -e 's%./%%' | egrep -v 'svn|swp|~' | egrep -v '^(doc|pkg)/' | sort > Manifest.txt"
+end
+
+desc 'Generate RDoc'
+Rake::RDocTask.new :rdoc do |rd|
+  rd.rdoc_dir = 'doc'
+  rd.rdoc_files.add 'lib', 'README', 'LICENSE'
+  rd.rdoc_files.exclude '*/resolv.rb' # WTF doesn't this work?
+  rd.main = 'README'
+  rd.options << '-d' if `which dot` =~ /\/dot/
+end
+
+desc 'Generate RDoc for dev.robotcoop.com'
+Rake::RDocTask.new :dev_rdoc do |rd|
+  rd.rdoc_dir = '../../../www/trunk/dev/html/Tools/logmerge'
+  rd.rdoc_files.add 'lib', 'README', 'LICENSE'
+  rd.main = 'README'
+  rd.options << '-d' if `which dot` =~ /\/dot/
+end
+
+desc 'Build Gem'
+Rake::GemPackageTask.new spec do |pkg|
+  pkg.need_tar = true
+end
+
+desc 'Clean up'
+task :clean => [ :clobber_rdoc, :clobber_package ]
+
+desc 'Clean up'
+task :clobber => [ :clean ]
+
+# vim: syntax=Ruby
+

data/bin/ip2name

@@ -0,0 +1,8 @@
+#!/usr/local/bin/ruby -w
+
+require 'logmerge'
+
+Thread.abort_on_exception = true
+
+LogMerge::IPResolver.resolve
+

data/bin/logmerge

@@ -0,0 +1,6 @@
+#!/usr/local/bin/ruby -w
+
+require 'logmerge'
+
+LogMerge::Merger.merge
+

data/lib/logmerge/merger.rb

@@ -0,0 +1,74 @@
+$TESTING = defined? $TESTING
+
+require 'logmerge'
+
+##
+# Merges multiple Apache log files by date.
+
+class LogMerge::Merger
+
+  ##
+  # Merges +files+ into +output+.  If +files+ is nil, ARGV is used.
+
+  def self.merge(output = STDOUT, *files)
+    files = ARGV if files.empty?
+    merger = LogMerge::Merger.new files.map { |file| File.open file }
+    merger.merge output
+  end
+
+  ##
+  # Creates a new Merger that will operate on an array of IO-like objects
+  # +streams+.
+
+  def initialize(streams)
+    @streams = streams
+    @buf = []
+
+    @streams.each_index { |slot| buf_fill slot }
+
+    @all_closed = @streams.empty?
+  end
+
+  ##
+  # Are all the streams closed?
+
+  def all_closed?
+    @all_closed
+  end
+
+  ##
+  # Adds a line to the buffer from input strem +slot+.
+
+  def buf_fill(slot)
+    line = @streams[slot].gets
+
+    if line.nil? and @streams[slot].eof? then
+      @streams.delete_at slot
+      @buf.delete_at slot
+    elsif line.nil? then
+      raise "WTF? line.nil? and not eof?"
+    else
+      line =~ / \[(.+?)\] /
+      time = Time.parse($1.sub(':', ' ').gsub('/', '-')).to_i
+      @buf[slot] = [time, line]
+    end
+  end
+
+  ##
+  # Merges streams into +out_stream+.
+
+  def merge(out_stream)
+    index, min, line = nil
+
+    until @streams.empty? do
+      min = @buf.min { |a,b| a.first <=> b.first}
+      slot = @buf.index min
+      buf_fill slot
+      out_stream.puts min.last
+    end
+
+    @all_closed = true
+  end
+
+end
+