garru-distributed_logreader 0.2.0 → 0.5.0

data/README.rdoc

@@ -1,6 +1,7 @@
-= distributed_logreader
+= Distributed Log Reader
 
-Description goes here.
+is a log reader that allows for parallel processing.  It is highly configurable
+allowing you to choose different parallel distribution and archiving strategies.
 
 == Copyright
 

data/VERSION

@@ -1 +1 @@
-0.2.0
+0.5.0

data/distributed_logreader.gemspec

@@ -2,11 +2,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{distributed_logreader}
-  s.version = "0.2.0"
+  s.version = "0.5.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Gary Tsang"]
-  s.date = %q{2009-07-30}
+  s.date = %q{2009-08-05}
   s.email = %q{gary@garru.com}
   s.extra_rdoc_files = [
     "LICENSE",
@@ -24,6 +24,7 @@ Gem::Specification.new do |s|
      "lib/distributed_logreader/archiver/date_dir.rb",
      "lib/distributed_logreader/distributed_log_reader.rb",
      "lib/distributed_logreader/distributed_log_reader/rotater_reader.rb",
+     "lib/distributed_logreader/distributed_log_reader/scribe_reader.rb",
      "lib/distributed_logreader/distributer.rb",
      "lib/distributed_logreader/distributer/simple_thread_pool.rb",
      "lib/distributed_logreader/log_reader.rb",
@@ -33,6 +34,7 @@ Gem::Specification.new do |s|
      "spec/archiver/date_dir_spec.rb",
      "spec/archiver_spec.rb",
      "spec/distributed_log_reader/rotater_reader_spec.rb",
+     "spec/distributed_log_reader/scribe_reader_spec.rb",
      "spec/distributed_log_reader_spec.rb",
      "spec/distributer/simple_thread_pool_spec.rb",
      "spec/distributer_spec.rb",
@@ -44,6 +46,8 @@ Gem::Specification.new do |s|
      "spec/fixtures/symlink/test",
      "spec/fixtures/symlink/test_older_sym",
      "spec/fixtures/test_file",
+     "spec/fixtures/virality_metrics/test",
+     "spec/fixtures/virality_metrics/virality_metrics_current",
      "spec/log_reader_spec.rb",
      "spec/selector/rotating_log_spec.rb",
      "spec/selector_spec.rb",
@@ -59,6 +63,7 @@ Gem::Specification.new do |s|
     "spec/archiver/date_dir_spec.rb",
      "spec/archiver_spec.rb",
      "spec/distributed_log_reader/rotater_reader_spec.rb",
+     "spec/distributed_log_reader/scribe_reader_spec.rb",
      "spec/distributed_log_reader_spec.rb",
      "spec/distributer/simple_thread_pool_spec.rb",
      "spec/distributer_spec.rb",

data/lib/distributed_logreader.rb

@@ -4,8 +4,9 @@ require 'distributed_logreader/achiver.rb'
 require 'distributed_logreader/util.rb'
 require 'distributed_logreader/distributed_log_reader'
 require 'distributed_logreader/distributed_log_reader/rotater_reader'
+require 'distributed_logreader/distributed_log_reader/scribe_reader'
 require 'logger'
 
-$dlog_logger = Logger.new("distributed_logreader.log")
+$dlog_logger = Logger.new("/var/log/distributed_logreader.log")
 $dlog_logger.level = Logger::INFO
 $dlog_logger.datetime_format = "%Y-%m-%d %H:%M:%S "

data/lib/distributed_logreader/archiver/date_dir.rb

@@ -8,7 +8,7 @@ module DLogReader
       self.base_backup_dir = backup_dir
     end
   
-    def backup(file)
+    def archive(file)
       mv(file, backup_dir) unless base_backup_dir.nil?
         
     end

data/lib/distributed_logreader/distributed_log_reader.rb

@@ -16,11 +16,20 @@ module DLogReader
     # needs to be moved into LogReader
     def process
       pre_process
+
+      $dlog_logger.info("Started #{log_file}:")
+        lines_processed = 0
       @log_reader = LogReader.new(log_file) do |line|
-        self.distributer.process(line)
+        begin
+          self.distributer.process(line)
+        rescue Exception => e
+          $dlog_logger.warn("Exception in processing thread #{log_file}:#{log_file.pos} -- #{line} -- #{e.message}")
+        end
+        lines_processed += 1
       end
       @log_reader.run
       self.distributer.join
+      $dlog_logger.info("Finished #{log_file}: Processed (#{lines_processed}) lines")
       post_process
     end
     

data/lib/distributed_logreader/distributed_log_reader/rotater_reader.rb

@@ -15,7 +15,7 @@ module DLogReader
     end
     
     def post_process
-      self.archiver.backup(log_file)
+      self.archiver.archive(log_file)
     end
   end
 end

data/lib/distributed_logreader/distributed_log_reader/scribe_reader.rb

@@ -0,0 +1,29 @@
+module DLogReader
+  class ScribeReader < DistributedLogReader
+    attr_accessor :selector, :archiver
+    attr_reader :log_reader
+    def initialize(filename, backupdir, worker, num_threads = 10)
+      super(filename, worker, num_threads)
+      self.selector = RotatingLog.new
+      self.selector.ignore_conditions << lambda{|x| x.match(/scribe_stats/)}
+      self.archiver = DateDir.new(backupdir)
+    end
+        
+    def log_file
+      @log_file ||= begin
+        selector.file_to_process(filename)
+      end
+    end
+    
+    def post_process
+      self.archiver.archive(log_file) unless current?
+    end
+    
+    def current?
+      directory = File.dirname(log_file)
+      basename = File.basename(directory)
+      current_file = File.join(directory, "#{basename}_current")
+      File.identical?(current_file, log_file)
+    end
+  end
+end

data/lib/distributed_logreader/selector/rotating_log.rb

@@ -3,6 +3,14 @@ module DLogReader
   # input filename.  This should work with a variety of log rotating schemes:
   # including copytruncate and date suffix.
   class RotatingLog < Selector
+    
+    attr_accessor :ignore_conditions
+    
+    def initialize
+      self.ignore_conditions = []
+      self.ignore_conditions << lambda{|x| symlink_file_in_dir?(x)}
+      # self.ignore_conditions << lambda{|x| true}
+    end
   
     def file_to_process(file_or_dir)
       if File.directory?(file_or_dir)
@@ -15,14 +23,18 @@ module DLogReader
       oldest_logfile(directory, basename)
     end
 
-  protected
+protected
 
     def oldest_logfile(directory, basename)
       file_list = Dir[File.join(directory, "#{basename}*")]
-      file_list.reject!{|x| symlink_file_in_dir?(x)}
+      file_list.reject!{|x| reject?(x)}
       file = file_list.size > 0 ? file_list.sort_by{|a| File.new(a).mtime}.first : nil
     end
-
+    
+    def reject?(filename)
+      self.ignore_conditions.inject(false){|candidate, condition| candidate || condition.call(filename)}
+    end
+    
     # returns true if filename is a symlink and its referring to a file already inside the current directory
     def symlink_file_in_dir?(filename)
       File.symlink?(filename) && File.dirname(File.readlink(filename)) == File.dirname(filename)

data/spec/archiver/date_dir_spec.rb

@@ -14,7 +14,7 @@ describe "DLogReader::DateDir" do
     
   describe "backup" do
     it "should move file into Y/M/D backup directory" do
-      @archiver.backup(@file_path)
+      @archiver.archive(@file_path)
       File.exist?(@backup_dir).should == true
     end
   end

data/spec/distributed_log_reader/scribe_reader_spec.rb

@@ -0,0 +1,15 @@
+require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
+require 'fileutils'
+
+describe "DLogReader::RotaterLogreader" do
+  before(:all) do
+    @file_path = File.join(File.dirname(__FILE__), '..', 'fixtures', 'virality_metrics')
+    @logreader = DLogReader::ScribeReader.new(@file_path, 'tmp', lambda{|x| puts x})
+  end
+
+  describe "current" do
+    it 'should say that the log_file is the currently rotated one' do 
+      @logreader.current?.should == true
+    end
+  end  
+end

data/spec/fixtures/virality_metrics/test

@@ -0,0 +1,7 @@
+asdf
+asdf
+asdf
+asdf
+asdfa
+sdfa
+sdfas

data/spec/fixtures/virality_metrics/virality_metrics_current

@@ -0,0 +1,7 @@
+asdf
+asdf
+asdf
+asdf
+asdfa
+sdfa
+sdfas

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: garru-distributed_logreader
 version: !ruby/object:Gem::Version 
-  version: 0.2.0
+  version: 0.5.0
 platform: ruby
 authors: 
 - Gary Tsang
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-07-30 00:00:00 -07:00
+date: 2009-08-05 00:00:00 -07:00
 default_executable: 
 dependencies: []
 
@@ -34,6 +34,7 @@ files:
 - lib/distributed_logreader/archiver/date_dir.rb
 - lib/distributed_logreader/distributed_log_reader.rb
 - lib/distributed_logreader/distributed_log_reader/rotater_reader.rb
+- lib/distributed_logreader/distributed_log_reader/scribe_reader.rb
 - lib/distributed_logreader/distributer.rb
 - lib/distributed_logreader/distributer/simple_thread_pool.rb
 - lib/distributed_logreader/log_reader.rb
@@ -43,6 +44,7 @@ files:
 - spec/archiver/date_dir_spec.rb
 - spec/archiver_spec.rb
 - spec/distributed_log_reader/rotater_reader_spec.rb
+- spec/distributed_log_reader/scribe_reader_spec.rb
 - spec/distributed_log_reader_spec.rb
 - spec/distributer/simple_thread_pool_spec.rb
 - spec/distributer_spec.rb
@@ -54,6 +56,8 @@ files:
 - spec/fixtures/symlink/test
 - spec/fixtures/symlink/test_older_sym
 - spec/fixtures/test_file
+- spec/fixtures/virality_metrics/test
+- spec/fixtures/virality_metrics/virality_metrics_current
 - spec/log_reader_spec.rb
 - spec/selector/rotating_log_spec.rb
 - spec/selector_spec.rb
@@ -89,6 +93,7 @@ test_files:
 - spec/archiver/date_dir_spec.rb
 - spec/archiver_spec.rb
 - spec/distributed_log_reader/rotater_reader_spec.rb
+- spec/distributed_log_reader/scribe_reader_spec.rb
 - spec/distributed_log_reader_spec.rb
 - spec/distributer/simple_thread_pool_spec.rb
 - spec/distributer_spec.rb