distributed_logreader 0.11.0

data/.gitignore

@@ -0,0 +1,7 @@
+*.sw?
+.DS_Store
+coverage
+rdoc
+pkg
+*log
+temp_backup_dir/*

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2009 Gary Tsang
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,8 @@
+= Distributed Log Reader
+
+is a log reader that allows for parallel processing.  It is highly configurable
+allowing you to choose different parallel distribution and archiving strategies.
+
+== Copyright
+
+Copyright (c) 2009 Gary Tsang. See LICENSE for details.

data/Rakefile

@@ -0,0 +1,48 @@
+require 'rubygems'
+require 'rake'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gem|
+    gem.name = "distributed_logreader"
+    gem.summary = %Q{incomplete distribute log reader. planned to use pandemic to distribute work}
+    gem.email = "gary@garru.com"
+    gem.homepage = "http://github.com/garru/distributed_logreader"
+    gem.authors = ["Gary Tsang"]
+    # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
+  end
+
+rescue LoadError
+  puts "Jeweler (or a dependency) not available. Install it with: sudo gem install jeweler"
+end
+
+require 'spec/rake/spectask'
+Spec::Rake::SpecTask.new(:spec) do |spec|
+  spec.libs << 'lib' << 'spec'
+  spec.spec_files = FileList['spec/**/*_spec.rb']
+end
+
+Spec::Rake::SpecTask.new(:rcov) do |spec|
+  spec.libs << 'lib' << 'spec'
+  spec.pattern = 'spec/**/*_spec.rb'
+  spec.rcov = true
+end
+
+
+task :default => :spec
+
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  if File.exist?('VERSION.yml')
+    config = YAML.load(File.read('VERSION.yml'))
+    version = "#{config[:major]}.#{config[:minor]}.#{config[:patch]}"
+  else
+    version = ""
+  end
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "distributed_logreader #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+

data/VERSION

@@ -0,0 +1 @@
+0.11.0

data/distributed_logreader.gemspec

@@ -0,0 +1,90 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE
+# Instead, edit Jeweler::Tasks in Rakefile, and run `rake gemspec`
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{distributed_logreader}
+  s.version = "0.11.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Gary Tsang"]
+  s.date = %q{2009-10-27}
+  s.email = %q{gary@garru.com}
+  s.extra_rdoc_files = [
+    "LICENSE",
+     "README.rdoc"
+  ]
+  s.files = [
+    ".gitignore",
+     "LICENSE",
+     "README.rdoc",
+     "Rakefile",
+     "VERSION",
+     "distributed_logreader.gemspec",
+     "lib/distributed_logreader.rb",
+     "lib/distributed_logreader/achiver.rb",
+     "lib/distributed_logreader/archiver/date_dir.rb",
+     "lib/distributed_logreader/distributed_log_reader.rb",
+     "lib/distributed_logreader/distributed_log_reader/rotater_reader.rb",
+     "lib/distributed_logreader/distributed_log_reader/scribe_reader.rb",
+     "lib/distributed_logreader/distributer.rb",
+     "lib/distributed_logreader/distributer/mutex_counter.rb",
+     "lib/distributed_logreader/distributer/pandemic_processor.rb",
+     "lib/distributed_logreader/distributer/simple_forked_process.rb",
+     "lib/distributed_logreader/distributer/simple_thread_pool.rb",
+     "lib/distributed_logreader/log_reader.rb",
+     "lib/distributed_logreader/selector.rb",
+     "lib/distributed_logreader/selector/rotating_log.rb",
+     "lib/distributed_logreader/util.rb",
+     "spec/archiver/date_dir_spec.rb",
+     "spec/archiver_spec.rb",
+     "spec/distributed_log_reader/rotater_reader_spec.rb",
+     "spec/distributed_log_reader/scribe_reader_spec.rb",
+     "spec/distributed_log_reader_spec.rb",
+     "spec/distributer/simple_thread_pool_spec.rb",
+     "spec/distributer_spec.rb",
+     "spec/fixtures/copytruncate/test",
+     "spec/fixtures/copytruncate/test.1",
+     "spec/fixtures/copytruncate/test_current",
+     "spec/fixtures/logrotate/test-20090101",
+     "spec/fixtures/logrotate/test-20090102",
+     "spec/fixtures/symlink/test",
+     "spec/fixtures/symlink/test_older_sym",
+     "spec/fixtures/test_file",
+     "spec/fixtures/virality_metrics/test",
+     "spec/fixtures/virality_metrics/virality_metrics_current",
+     "spec/log_reader_spec.rb",
+     "spec/selector/rotating_log_spec.rb",
+     "spec/selector_spec.rb",
+     "spec/spec_helper.rb"
+  ]
+  s.homepage = %q{http://github.com/garru/distributed_logreader}
+  s.rdoc_options = ["--charset=UTF-8"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.3.5}
+  s.summary = %q{incomplete distributed log reader. plan to use pandemic to distribute}
+  s.test_files = [
+    "spec/archiver/date_dir_spec.rb",
+     "spec/archiver_spec.rb",
+     "spec/distributed_log_reader/rotater_reader_spec.rb",
+     "spec/distributed_log_reader/scribe_reader_spec.rb",
+     "spec/distributed_log_reader_spec.rb",
+     "spec/distributer/simple_thread_pool_spec.rb",
+     "spec/distributer_spec.rb",
+     "spec/log_reader_spec.rb",
+     "spec/selector/rotating_log_spec.rb",
+     "spec/selector_spec.rb",
+     "spec/spec_helper.rb"
+  ]
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+    else
+    end
+  else
+  end
+end

data/lib/distributed_logreader.rb

@@ -0,0 +1,11 @@
+require 'distributed_logreader/selector.rb'
+require 'distributed_logreader/achiver.rb'
+require 'distributed_logreader/util.rb'
+require 'distributed_logreader/distributed_log_reader'
+require 'distributed_logreader/distributed_log_reader/rotater_reader'
+require 'distributed_logreader/distributed_log_reader/scribe_reader'
+require 'logger'
+
+$dlog_logger = Logger.new("/var/log/distributed_logreader.log")
+$dlog_logger.level = Logger::DEBUG
+$dlog_logger.datetime_format = "%Y-%m-%d %H:%M:%S "

data/lib/distributed_logreader/achiver.rb

@@ -0,0 +1,10 @@
+# This abstract class defines the interface that handles log file archiving
+
+module DLogReader
+  class Archiver
+    # archive file as you see fit
+    def archive(filename)
+      raise NotImplementedError.new("archive not implemented. Are you sure you created a conrete class?")
+    end
+  end
+end

data/lib/distributed_logreader/archiver/date_dir.rb

@@ -0,0 +1,36 @@
+require 'fileutils'
+module DLogReader
+  class DateDir < Archiver
+    include FileUtils
+    attr_accessor :base_backup_dir
+  
+    def initialize(backup_dir)
+      self.base_backup_dir = backup_dir
+    end
+  
+    def archive(file)
+      unless base_backup_dir.nil?
+        backup_filename = backup_dir
+        mv(file, backup_dir)
+        `bzip2 #{File.join(backup_dir, file.split('/').last)}`
+      end
+    end
+
+  protected
+
+    def backup_dir
+      time = Time.now
+      directory_structure = [base_backup_dir, time.year.to_s, time.month.to_s, time.day.to_s]
+      temp_dir = []
+      directory_structure.each do |x|
+        temp_dir << x
+        temp_file = File.join(temp_dir)
+        unless File.exist?(temp_file)
+          mkdir(temp_file) 
+        end
+      end
+      File.join(temp_dir)
+    end
+  
+  end
+end

data/lib/distributed_logreader/distributed_log_reader.rb

@@ -0,0 +1,46 @@
+require File.join(File.dirname(__FILE__), 'distributer', 'simple_thread_pool')
+require File.join(File.dirname(__FILE__), 'distributer', 'simple_forked_process')
+require File.join(File.dirname(__FILE__), 'selector', 'rotating_log')
+require File.join(File.dirname(__FILE__), 'archiver', 'date_dir')
+require File.join(File.dirname(__FILE__), 'log_reader')
+
+module DLogReader
+  class DistributedLogReader
+    attr_accessor :distributer, :filename
+    attr_reader :log_reader
+    def initialize(filename, worker, num_threads = 100)
+      self.filename = filename
+      # self.distributer = SimpleForked.new(worker, 5, num_threads)
+      self.distributer = SimpleThreadPool.new(worker, num_threads)
+    end
+    
+    # selector/archiver seem to be strongly connected.  it's possible it
+    # needs to be moved into LogReader
+    def process
+      pre_process
+
+      $dlog_logger.info("Started  #{log_file}:")
+        lines_processed = 0
+      @log_reader = LogReader.new(log_file) do |line|
+        self.distributer.process(line)
+        lines_processed += 1
+      end
+      @log_reader.run
+      self.distributer.join
+      $dlog_logger.info("Finished #{log_file}: Processed (#{lines_processed}) lines")
+      post_process
+    end
+    
+    def log_file
+      self.filename
+    end
+    
+    #predefined hooks
+    def pre_process
+    end
+    
+    #predefined hooks
+    def post_process
+    end
+  end
+end

data/lib/distributed_logreader/distributed_log_reader/rotater_reader.rb

@@ -0,0 +1,21 @@
+module DLogReader
+  class RotaterReader < DistributedLogReader
+    attr_accessor :selector, :archiver
+    attr_reader :log_reader
+    def initialize(filename, backupdir, worker, num_threads = 10)
+      super(filename, worker, num_threads)
+      self.selector = RotatingLog.new
+      self.archiver = DateDir.new(backupdir)
+    end
+        
+    def log_file
+      @log_file ||= begin
+        selector.file_to_process(filename)
+      end
+    end
+    
+    def post_process
+      self.archiver.archive(log_file)
+    end
+  end
+end

data/lib/distributed_logreader/distributed_log_reader/scribe_reader.rb

@@ -0,0 +1,53 @@
+module DLogReader
+  class ScribeReader
+    attr_accessor :selector, :archiver, :filename
+    attr_reader :log_reader
+    def initialize(filename, backupdir, worker, num_threads = 10)
+      self.filename = filename
+      self.selector = RotatingLog.new
+      self.selector.ignore_conditions << lambda{|x| !x.match(/scribe_stats/).nil?}
+      @worker = worker
+      self.archiver = DateDir.new(backupdir)
+    end
+    
+    def process
+      $dlog_logger.info("Started  #{log_file}:")
+      lines_processed = 0
+      @log_reader = LogReader.new(log_file) do |line|
+        begin
+          @worker.call(line)
+        rescue Exception => e
+          $dlog_logger.warn("Exception thrown in worker #{e.message}")
+        end
+        lines_processed += 1
+      end
+      @log_reader.run
+      $dlog_logger.info("Finished #{log_file}: Processed (#{lines_processed}) lines")
+      post_process
+    end
+    
+    def log_file
+      self.filename
+    end
+    
+    def log_file
+      @log_file ||= begin
+        selector.file_to_process(filename)
+      end
+    end
+    
+    def post_process
+      unless current?
+        self.archiver.archive(log_file)
+        File.delete(@log_reader.statefile) rescue nil
+      end
+    end
+    
+    def current?
+      directory = File.dirname(log_file)
+      basename = File.basename(directory)
+      current_file = Dir[File.join(directory, "*")].detect{|x| x.match(/current/)}
+      !current_file.nil? && File.exists?(current_file) && File.identical?(current_file, log_file) 
+    end
+  end
+end

data/lib/distributed_logreader/distributer.rb

@@ -0,0 +1,13 @@
+module DLogReader
+  class Distributer
+    attr_accessor :worker
+
+    def initialize(worker)
+      self.worker = worker
+    end
+    
+    def process(line)
+      worker.call(line)
+    end
+  end
+end

data/lib/distributed_logreader/distributer/mutex_counter.rb

@@ -0,0 +1,48 @@
+class MutexCounter
+  MAX = (2 ** 30) - 1
+  def initialize(max = MAX)
+    @mutex = Mutex.new
+    @counter = 0
+    @resets = 0
+    @max = max
+  end
+
+  def real_total
+    @mutex.synchronize { (@resets * @max) + @counter }
+  end
+  alias_method :to_i, :real_total
+  
+  def value
+    @mutex.synchronize { @counter }
+  end
+
+  def inc
+    @mutex.synchronize do
+      if @counter >= @max
+        @counter = 0  # to avoid Bignum, it's about 4x slower
+        @resets += 1
+      end
+      @counter += 1
+    end
+  end
+  alias_method :next, :inc
+  alias_method :succ, :inc
+  
+  # decr only to zero
+  def decr
+    @mutex.synchronize do
+      if @counter > 0
+        @counter -= 1
+      else
+        if @resets > 1
+          @resets -= 1 
+          @counter = @max
+        end
+      end
+      @counter
+    end
+  end
+  alias_method :pred, :decr
+  alias_method :prev, :decr
+  
+end

data/lib/distributed_logreader/distributer/pandemic_processor.rb

@@ -0,0 +1,149 @@
+require 'rubygems'
+#stolen from pandemic
+module DLogReader
+  class Processor
+    def initialize(handler, num_threads = 10)
+      read_from_parent, write_to_child = IO.pipe
+      read_from_child, write_to_parent = IO.pipe
+  
+      @child_process_id = fork
+      if @child_process_id
+        # I'm the parent
+        write_to_parent.close
+        read_from_parent.close
+        @out = write_to_child
+        @in = read_from_child
+        @max_queue_size = 100
+        @counter = MutexCounter.new
+        @job_mutex = Mutex.new
+        wait_for_responses
+      else
+        $dlog_logger.debug("Forked")
+        # I'm the child
+        write_to_child.close
+        read_from_child.close
+        @out = write_to_parent
+        @in = read_from_parent
+        @handler = handler
+        @job_queue = Queue.new
+        @response_queue = Queue.new
+        wait_for_job_completion
+        num_threads.times do
+          create_thread
+        end
+        wait_for_jobs
+      end
+    end
+  
+    def num_jobs
+      if parent?
+        @counter.real_total
+      end
+    end
+  
+    def process(body)
+      if parent?
+        while(@counter.real_total > @max_queue_size)
+          $dlog_logger.debug("Max process queue size: #{@counter.real_total}")
+          sleep(0.01)
+        end
+        body = (body.chomp + "\n")
+        @job_mutex.synchronize do
+          @out.write(body)
+          in_queue = @counter.inc
+          $dlog_logger.debug("Parent: writing #{body.inspect} - #{in_queue}")
+        end
+      else
+        $dlog_logger.debug("Child Processing #{body}")
+        return @handler.call(body)
+      end
+    end
+
+    def close(status = 0)
+      if parent? && child_alive?
+        Process.detach(@child_process_id)
+        @out.puts(status.to_s)
+        @out.close
+        @in.close
+      else
+        Process.exit!(status)
+      end
+    end
+
+    def closed?
+      !child_alive?
+    end
+
+    private
+
+    def create_thread
+      Thread.new do
+        loop do
+          line = @job_queue.pop
+          @job_mutex.synchronize do
+            process(line)
+            @response_queue << :a
+            $dlog_logger.debug("Child: Finished #{line.inspect}")
+          end
+        end
+      end
+    end
+
+    def wait_for_responses
+      Thread.new do
+        loop do
+          ready, = IO.select([@in], nil, nil)
+          if ready
+            @in.readchar
+            in_queue = @counter.decr
+            $dlog_logger.debug("Parent: Reading Response #{in_queue}")
+          end
+        end
+      end
+    end
+
+    #child process
+    def wait_for_jobs
+      if child?
+        while true
+          $dlog_logger.debug("Child waiting")  
+          ready, = IO.select([@in], nil, nil)
+          if ready && !@in.eof?
+            line = @in.gets
+            $dlog_logger.debug("Child: #{line.inspect}")
+            @job_queue << line
+          else
+            self.close(line.to_i)
+            break  
+          end
+        end
+      end
+    end
+
+    #child process
+    def wait_for_job_completion
+      if child?
+        Thread.new do
+          while true
+            @response_queue.pop
+            $dlog_logger.debug("Child: Writing To Parent")
+            @out.write("|")
+          end
+        end
+      end
+    end
+
+
+    def parent?
+      !!@child_process_id
+    end
+
+    def child?
+      !parent?
+    end
+
+    def child_alive?
+      parent? && !@in.closed?
+    end
+  end
+end

data/lib/distributed_logreader/distributer/simple_forked_process.rb

@@ -0,0 +1,47 @@
+require 'thread'
+require File.join(File.dirname(__FILE__), 'pandemic_processor')
+require File.join(File.dirname(__FILE__), 'mutex_counter')
+module DLogReader
+  class SimpleForked
+    attr_accessor :num_threads_per_process, :worker, :thread_pool, :queue, :processors
+    def initialize(worker, num_processes = 3, num_threads_per_process = 10)
+      self.worker = worker
+      self.num_threads_per_process = (num_threads_per_process || 10)
+      self.queue = Queue.new
+      self.processors = []
+      num_processes.times do |x|
+        $dlog_logger.debug("Forking #{x} process")
+        self.processors << create_process
+      end
+    end
+    
+    def process(line)
+      self.queue << line
+    end
+    
+    def join
+      num_jobs_outstanding = self.processors.inject(0){|a,b| a + b.num_jobs}
+      while(queue.size > 0 || num_jobs_outstanding > 0)
+        sleep 0.1
+        num_jobs_outstanding = self.processors.inject(0){|a,b| a + b.num_jobs}
+        $dlog_logger.debug("Shutting down #{num_jobs_outstanding} left")
+      end
+    end
+    
+protected    
+    def create_process
+      processor = Processor.new(self.worker, self.num_threads_per_process)
+      Thread.new do
+        loop do
+          line = self.queue.pop
+          begin
+            processor.process(line)
+          rescue Exception => e
+            $dlog_logger.warn("Exception in processing thread #{line} -- #{e.message}")
+          end
+        end
+      end
+      processor
+    end
+  end
+end

data/lib/distributed_logreader/distributer/simple_thread_pool.rb

@@ -0,0 +1,43 @@
+require 'thread'
+
+module DLogReader
+  class SimpleThreadPool
+    attr_accessor :num_threads, :worker, :thread_pool, :queue, :max_queue_size
+    def initialize(worker, num_threads = 5)
+      self.worker = worker
+      self.num_threads = num_threads
+      self.queue = Queue.new
+      self.max_queue_size = 100
+      num_threads.times do 
+        create_thread
+      end
+    end
+    
+    def process(line)
+      while(queue.size > self.max_queue_size)
+        sleep(0.01)
+      end
+      self.queue << line
+    end
+    
+    def join
+      while(queue.size > 0)
+        sleep 0.1
+      end
+    end
+    
+protected    
+    def create_thread
+      Thread.new do
+        loop do
+          line = self.queue.pop
+          begin
+            self.worker.call(line)
+          rescue Exception => e
+            $dlog_logger.warn("Exception in processing thread #{line} -- #{e.message}")
+          end
+        end
+      end
+    end
+  end
+end

data/lib/distributed_logreader/log_reader.rb

@@ -0,0 +1,67 @@
+require 'digest/md5'
+module DLogReader
+  class LogReader
+    attr_accessor :filename
+    attr_writer :statefile
+    
+    def initialize(filename, &b)
+      self.filename = filename
+      @b = b
+    end
+  
+    def run
+      # raise IOError.new("no file given") if filename.nil?
+      raise IOError.new("File not readable") unless File.readable?(filename)
+      f = File.open(filename, "r+")
+      load_saved_state(f)
+      # raise IOError.new("File is locked") unless f.flock(File::LOCK_EX | File::LOCK_NB)
+      unless f.eof?
+        last_report = Time.now
+        line_count = 0
+        f.each_line do |line|
+          @b.call(line)
+          line_count += 1
+          if (line_count % 100 == 0)
+            time_passed = Time.now - last_report
+            $dlog_logger.info( "#{Time.now.to_s} #{filename}: Processed (#{line_count}) lines [#{(100.0 / time_passed.to_f).to_i} lines/s]")
+            last_report = Time.now
+            save_state(f) 
+          end
+        end
+        save_state(f)
+      end
+      # f.flock(File::LOCK_UN)
+    end
+    
+    def statefile
+      @statefile ||= begin
+        log_basename = File.basename(filename)
+        File.join("/tmp", "log_state_#{log_basename}")
+      end
+    end
+
+  protected
+    
+    def load_saved_state(log_filehandle)
+      return unless File.exists?(statefile) && !(state = File.read(statefile)).nil?
+      pos, l_digest = Marshal.load(state)
+      return if File.size(log_filehandle) < pos
+      log_filehandle.pos = pos if digest(log_filehandle, pos) == l_digest
+    end
+
+    def save_state(log_filehandle)
+      File.open(statefile, "w") do |f|
+        f.write(Marshal.dump([log_filehandle.pos, digest(log_filehandle, log_filehandle.pos)]))
+      end
+    end
+
+    def digest(log_filehandle, position)
+      log_filehandle.pos = 0
+      read_length = [position, 50].min
+      l = log_filehandle.read(read_length)
+      f_digest = Digest::MD5.hexdigest(l)
+      log_filehandle.pos = position
+      f_digest
+    end
+  end
+end

data/lib/distributed_logreader/selector.rb

@@ -0,0 +1,12 @@
+module DLogReader
+  # This abstract class defines the interface to decide which log 
+  # file to read. The identity strategy is the simplist, to return the file 
+  # inputed. However, to handle rotating log files, we'll need some more complex 
+  # strategies.
+  class Selector
+    # determines the file to process from file path input
+    def file_to_process(file_or_dir)
+      raise NotImplementedError.new, "file_to_process not implemented. Are you sure you created a conrete class?"
+    end
+  end
+end

data/lib/distributed_logreader/selector/rotating_log.rb

@@ -0,0 +1,43 @@
+module DLogReader
+  # This class chooses the oldest log file in the directory that matches the
+  # input filename.  This should work with a variety of log rotating schemes:
+  # including copytruncate and date suffix.
+  class RotatingLog < Selector
+    
+    attr_accessor :ignore_conditions
+    
+    def initialize
+      self.ignore_conditions = []
+      self.ignore_conditions << lambda{|x| symlink_file_in_dir?(x)}
+      # self.ignore_conditions << lambda{|x| true}
+    end
+  
+    def file_to_process(file_or_dir)
+      if File.directory?(file_or_dir)
+        directory = file_or_dir
+        basename = '/'
+      else
+        directory = File.dirname(file_or_dir)
+        basename = File.basename(file_or_dir)
+      end    
+      oldest_logfile(directory, basename)
+    end
+
+protected
+
+    def oldest_logfile(directory, basename)
+      file_list = Dir[File.join(directory, "#{basename}*")]
+      file_list.reject!{|x| reject?(x)}
+      file = file_list.size > 0 ? file_list.sort_by{|a| File.new(a).mtime}.first : nil
+    end
+    
+    def reject?(filename)
+      self.ignore_conditions.inject(false){|candidate, condition| candidate || condition.call(filename)}
+    end
+    
+    # returns true if filename is a symlink and its referring to a file already inside the current directory
+    def symlink_file_in_dir?(filename)
+      File.symlink?(filename) && File.dirname(File.readlink(filename)) == File.dirname(filename)
+    end
+  end
+end

data/lib/distributed_logreader/util.rb

@@ -0,0 +1,7 @@
+module DLogReader
+  module Util
+    def logger
+      $dlog_logger
+    end
+  end
+end

data/spec/archiver/date_dir_spec.rb

@@ -0,0 +1,25 @@
+require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
+require 'distributed_logreader/archiver/date_dir'
+require 'fileutils'
+
+describe "DLogReader::DateDir" do
+  before(:all) do
+    FileUtils.cp(File.join(File.dirname(__FILE__), '..', 'fixtures', 'test_file'), File.join(File.dirname(__FILE__), '..', 'fixtures', 'test_file2'))
+    @file_path = File.join(File.dirname(__FILE__), '..', 'fixtures', 'test_file2')
+    @base_backup_dir = File.join(File.dirname(__FILE__), '..', 'fixtures', 'temp_backup_dir')
+    time = Time.now
+    @backup_dir = File.join([@base_backup_dir, time.year.to_s, time.month.to_s, time.day.to_s])
+    @archiver = DLogReader::DateDir.new(@base_backup_dir)
+  end
+    
+  describe "backup" do
+    it "should move file into Y/M/D backup directory" do
+      @archiver.archive(@file_path)
+      File.exist?(@backup_dir).should == true
+    end
+  end
+  
+  after(:all) do
+    FileUtils.rm_r(@base_backup_dir)
+  end
+end

data/spec/archiver_spec.rb

@@ -0,0 +1,13 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+
+describe "DLogReader::LogArchiver" do
+  before(:all) do
+    @archiver = DLogReader::Archiver.new
+  end
+
+  describe "archive" do
+    it "should raise NotImplementedError" do
+      lambda{ @archiver.archive('dummy_file') }.should raise_error(NotImplementedError)
+    end
+  end
+end

data/spec/distributed_log_reader/rotater_reader_spec.rb

@@ -0,0 +1,26 @@
+require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
+require 'fileutils'
+
+describe "DLogReader::RotaterLogreader" do
+  before(:all) do
+    FileUtils.cp(File.join(File.dirname(__FILE__), '..', 'fixtures', 'test_file'), File.join(File.dirname(__FILE__), '..', 'fixtures', 'test_file2'))
+    @file_path = File.join(File.dirname(__FILE__), '..', 'fixtures', 'test_file2')
+    @base_backup_dir = File.join(File.dirname(__FILE__), '..', 'fixtures', 'temp_backup_dir')
+    time = Time.now
+    @backup_dir = File.join([@base_backup_dir, time.year.to_s, time.month.to_s, time.day.to_s])
+    @logreader = DLogReader::RotaterReader.new(@file_path, @base_backup_dir, lambda{|x| puts x})
+  end
+
+  describe "process" do
+    it 'should' do 
+      @logreader.process
+      File.exist?(@backup_dir).should == true
+      File.exist?(@file_path).should == false
+    end
+  end
+  
+  after(:all) do
+    FileUtils.rm_r(@base_backup_dir)
+    FileUtils.rm_r(@logreader.log_reader.statefile)
+  end
+end

data/spec/distributed_log_reader/scribe_reader_spec.rb

@@ -0,0 +1,15 @@
+require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
+require 'fileutils'
+
+describe "DLogReader::RotaterLogreader" do
+  before(:all) do
+    @file_path = File.join(File.dirname(__FILE__), '..', 'fixtures', 'virality_metrics')
+    @logreader = DLogReader::ScribeReader.new(@file_path, 'tmp', lambda{|x| puts x})
+  end
+
+  describe "current" do
+    it 'should say that the log_file is the currently rotated one' do 
+      @logreader.current?.should == true
+    end
+  end  
+end

data/spec/distributed_log_reader_spec.rb

@@ -0,0 +1,20 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+require 'fileutils'
+
+describe "DLogReader::DistributedLogreader" do
+  before(:all) do
+    FileUtils.cp(File.join(File.dirname(__FILE__), 'fixtures', 'test_file'), File.join(File.dirname(__FILE__), 'fixtures', 'test_file2'))
+    @file_path = File.join(File.dirname(__FILE__), 'fixtures', 'test_file2')
+    @logreader = DLogReader::DistributedLogReader.new(@file_path, lambda{|x| puts x})
+  end
+
+  describe "process" do
+    it 'should' do 
+      @logreader.process
+    end
+  end
+  
+  after(:all) do
+    FileUtils.rm_r(@logreader.log_reader.statefile)
+  end
+end

data/spec/distributer/simple_thread_pool_spec.rb

@@ -0,0 +1,17 @@
+require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
+require 'distributed_logreader/distributer/simple_thread_pool'
+
+describe "DLogReader::SimpleThreadPool" do
+  before(:all) do
+    @thread_pool = DLogReader::SimpleThreadPool.new(lambda{|x| x}, 10)
+  end
+  
+  describe "process" do
+    it 'should process with lots of threads' do
+      100.times do |x|
+        @thread_pool.process(x.to_s)
+      end
+      @thread_pool.join
+    end
+  end
+end

data/spec/distributer_spec.rb

@@ -0,0 +1,13 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+require 'distributed_logreader/distributer'
+
+describe "DLogReader::Distributer" do
+  before(:all) do
+    @distributer = DLogReader::Distributer.new(lambda{|x| x})
+  end
+  
+  describe "process" do
+    it "should raise NotImplementedError" do
+    end
+  end
+end

data/spec/fixtures/test_file

@@ -0,0 +1,4 @@
+This is a test log file.
+With many lines.
+Lots and Lots of lines.
+Just kidding this is the last line.

data/spec/fixtures/virality_metrics/test

@@ -0,0 +1,7 @@
+asdf
+asdf
+asdf
+asdf
+asdfa
+sdfa
+sdfas

data/spec/fixtures/virality_metrics/virality_metrics_current

@@ -0,0 +1,7 @@
+asdf
+asdf
+asdf
+asdf
+asdfa
+sdfa
+sdfas

data/spec/log_reader_spec.rb

@@ -0,0 +1,57 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+require 'distributed_logreader/log_reader'
+require 'fileutils'
+
+describe "DLogReader::LogReader" do
+  before(:all) do
+    test_file = File.join(File.dirname(__FILE__), 'fixtures', 'test_file')
+    FileUtils.mkdir(File.join(File.dirname(__FILE__), 'fixtures', 'logreading')) rescue nil
+    @test_cp = File.join(File.dirname(__FILE__), 'fixtures', 'logreading', 'test')
+    FileUtils.cp(test_file, @test_cp)
+    test_fh = File.open(test_file)
+    
+    @reader = DLogReader::LogReader.new(@test_cp) do |line|
+      unless test_fh.readline == line
+        raise RuntimeError.new, 'you messed up bud'
+      end
+    end
+    @test_line = "this is an added line.  this should be read first\n"
+    @state_writer = DLogReader::LogReader.new(@test_cp){|line| line;}
+    @state_reader = DLogReader::LogReader.new(@test_cp) do |line|
+      unless line == @test_line
+        raise RuntimeError.new, 'you messed up worse'
+      end
+    end
+  end
+  
+  describe "run" do
+    it 'should read log files' do
+      lambda{@reader.run}.should_not raise_error
+    end
+    
+    it 'should resume from last access' do
+      #lets read to the end of file and write state
+      lambda{@state_writer.run}.should_not raise_error
+      fh = File.open(@test_cp, 'a')
+      fh.write(@test_line)
+      fh.close
+      lambda{@state_reader.run}.should_not raise_error
+    end
+    
+    it 'should detect if log is different from last and to start from beg of file' do
+      lambda{@state_writer.run}.should_not raise_error
+      fh = File.open(@test_cp, 'w')
+      fh.write(@test_line)
+      fh.close
+      lambda{@state_reader.run}.should_not raise_error
+      fh = File.open(@test_cp, 'w')
+      fh.write('')
+      fh.close
+      lambda{@state_reader.run}.should_not raise_error
+    end
+  end
+  
+  after(:all) do
+    FileUtils.rm_r(File.dirname(@test_cp))
+  end 
+end

data/spec/selector/rotating_log_spec.rb

@@ -0,0 +1,24 @@
+require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
+require 'distributed_logreader/selector/rotating_log'
+require 'fileutils'
+
+describe "DLogReader::RotatingLog" do
+  before(:all) do
+    @chooser = DLogReader::RotatingLog.new
+  end
+    
+  describe "file_to_process" do
+    it "should pick the oldest file for logs in copytruncate format (file)" do
+      @chooser.file_to_process(File.join(File.dirname(__FILE__), '..', 'fixtures', 'copytruncate', 'test')).should == File.join(File.dirname(__FILE__), '..', 'fixtures', 'copytruncate', 'test.1')      
+    end
+    
+    it "should pick the oldest file in timestamp suffix log format (dirname)" do
+      @chooser.file_to_process(File.join(File.dirname(__FILE__), '..', 'fixtures', 'logrotate')).should == File.join(File.dirname(__FILE__), '..', 'fixtures', 'logrotate', 'test-20090101')
+    end
+    
+    it "should pick the oldest file ignoring symlinks pointing to files already in dir" do
+      @chooser.file_to_process(File.join(File.dirname(__FILE__), '..', 'fixtures', 'symlink')).should == File.join(File.dirname(__FILE__), '..', 'fixtures', 'symlink', 'test')
+    end
+  end
+
+end

data/spec/selector_spec.rb

@@ -0,0 +1,13 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+
+describe "DLogReader::Selector" do
+  before(:all) do
+    @chooser = DLogReader::Selector.new
+  end
+
+  describe "file_to_process" do
+    it "should raise NotImplementedError" do
+      lambda{ @chooser.file_to_process('dummy_file') }.should raise_error(NotImplementedError)
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,9 @@
+require 'spec'
+
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+require 'distributed_logreader'
+
+Spec::Runner.configure do |config|
+
+end

metadata

@@ -0,0 +1,107 @@
+--- !ruby/object:Gem::Specification 
+name: distributed_logreader
+version: !ruby/object:Gem::Version 
+  version: 0.11.0
+platform: ruby
+authors: 
+- Gary Tsang
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-10-27 00:00:00 -07:00
+default_executable: 
+dependencies: []
+
+description: 
+email: gary@garru.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- LICENSE
+- README.rdoc
+files: 
+- .gitignore
+- LICENSE
+- README.rdoc
+- Rakefile
+- VERSION
+- distributed_logreader.gemspec
+- lib/distributed_logreader.rb
+- lib/distributed_logreader/achiver.rb
+- lib/distributed_logreader/archiver/date_dir.rb
+- lib/distributed_logreader/distributed_log_reader.rb
+- lib/distributed_logreader/distributed_log_reader/rotater_reader.rb
+- lib/distributed_logreader/distributed_log_reader/scribe_reader.rb
+- lib/distributed_logreader/distributer.rb
+- lib/distributed_logreader/distributer/mutex_counter.rb
+- lib/distributed_logreader/distributer/pandemic_processor.rb
+- lib/distributed_logreader/distributer/simple_forked_process.rb
+- lib/distributed_logreader/distributer/simple_thread_pool.rb
+- lib/distributed_logreader/log_reader.rb
+- lib/distributed_logreader/selector.rb
+- lib/distributed_logreader/selector/rotating_log.rb
+- lib/distributed_logreader/util.rb
+- spec/archiver/date_dir_spec.rb
+- spec/archiver_spec.rb
+- spec/distributed_log_reader/rotater_reader_spec.rb
+- spec/distributed_log_reader/scribe_reader_spec.rb
+- spec/distributed_log_reader_spec.rb
+- spec/distributer/simple_thread_pool_spec.rb
+- spec/distributer_spec.rb
+- spec/fixtures/copytruncate/test
+- spec/fixtures/copytruncate/test.1
+- spec/fixtures/copytruncate/test_current
+- spec/fixtures/logrotate/test-20090101
+- spec/fixtures/logrotate/test-20090102
+- spec/fixtures/symlink/test
+- spec/fixtures/symlink/test_older_sym
+- spec/fixtures/test_file
+- spec/fixtures/virality_metrics/test
+- spec/fixtures/virality_metrics/virality_metrics_current
+- spec/log_reader_spec.rb
+- spec/selector/rotating_log_spec.rb
+- spec/selector_spec.rb
+- spec/spec_helper.rb
+has_rdoc: true
+homepage: http://github.com/garru/distributed_logreader
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.5
+signing_key: 
+specification_version: 3
+summary: incomplete distributed log reader. plan to use pandemic to distribute
+test_files: 
+- spec/archiver/date_dir_spec.rb
+- spec/archiver_spec.rb
+- spec/distributed_log_reader/rotater_reader_spec.rb
+- spec/distributed_log_reader/scribe_reader_spec.rb
+- spec/distributed_log_reader_spec.rb
+- spec/distributer/simple_thread_pool_spec.rb
+- spec/distributer_spec.rb
+- spec/log_reader_spec.rb
+- spec/selector/rotating_log_spec.rb
+- spec/selector_spec.rb
+- spec/spec_helper.rb