garru-distributed_logreader 0.8.0 → 0.10.0

data/VERSION

@@ -1 +1 @@
-0.8.0
+0.10.0

data/distributed_logreader.gemspec

@@ -1,12 +1,15 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE
+# Instead, edit Jeweler::Tasks in Rakefile, and run `rake gemspec`
 # -*- encoding: utf-8 -*-
 
 Gem::Specification.new do |s|
   s.name = %q{distributed_logreader}
-  s.version = "0.8.0"
+  s.version = "0.10.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Gary Tsang"]
-  s.date = %q{2009-08-12}
+  s.date = %q{2009-09-14}
   s.email = %q{gary@garru.com}
   s.extra_rdoc_files = [
     "LICENSE",

data/lib/distributed_logreader.rb

@@ -1,4 +1,3 @@
-$LOAD_PATH.unshift(File.dirname(__FILE__))
 require 'distributed_logreader/selector.rb'
 require 'distributed_logreader/achiver.rb'
 require 'distributed_logreader/util.rb'
@@ -9,4 +8,4 @@ require 'logger'
 
 $dlog_logger = Logger.new("/var/log/distributed_logreader.log")
 $dlog_logger.level = Logger::DEBUG
-$dlog_logger.datetime_format = "%Y-%m-%d %H:%M:%S "
+$dlog_logger.datetime_format = "%Y-%m-%d %H:%M:%S "

data/lib/distributed_logreader/archiver/date_dir.rb

@@ -9,8 +9,11 @@ module DLogReader
     end
   
     def archive(file)
-      mv(file, backup_dir) unless base_backup_dir.nil?
-        
+      unless base_backup_dir.nil?
+        backup_filename = backup_dir
+        mv(file, backup_dir)
+        `bzip2 #{File.join(backup_dir, file.split('/').last)}`
+      end
     end
 
   protected

data/lib/distributed_logreader/distributed_log_reader/scribe_reader.rb

@@ -14,7 +14,11 @@ module DLogReader
       $dlog_logger.info("Started  #{log_file}:")
       lines_processed = 0
       @log_reader = LogReader.new(log_file) do |line|
-        @worker.call(line)
+        begin
+          @worker.call(line)
+        rescue Exception => e
+          $dlog_logger.warn("Exception thrown in worker #{e.message}")
+        end
         lines_processed += 1
       end
       @log_reader.run
@@ -34,13 +38,14 @@ module DLogReader
     
     def post_process
       self.archiver.archive(log_file) unless current?
+      File.delete(@log_reader.statefile) rescue nil
     end
     
     def current?
       directory = File.dirname(log_file)
       basename = File.basename(directory)
       current_file = Dir[File.join(directory, "*")].detect{|x| x.match(/current/)}
-      File.exists?(current_file) && File.identical?(current_file, log_file) 
+      !current_file.nil? && File.exists?(current_file) && File.identical?(current_file, log_file) 
     end
   end
 end

data/lib/distributed_logreader/log_reader.rb

@@ -16,12 +16,15 @@ module DLogReader
       load_saved_state(f)
       # raise IOError.new("File is locked") unless f.flock(File::LOCK_EX | File::LOCK_NB)
       unless f.eof?
+        last_report = Time.now
         line_count = 0
         f.each_line do |line|
           @b.call(line)
           line_count += 1
           if (line_count % 100 == 0)
-            $dlog_logger.info( "#{Time.now.to_s} #{filename}: Processed (#{line_count}) lines")
+            time_passed = Time.now - last_report
+            $dlog_logger.info( "#{Time.now.to_s} #{filename}: Processed (#{line_count}) lines in #{time_passed}s [#{(line_count.to_f / time_passed.to_f).to_i} lines/s]")
+            last_report = Time.now
             save_state(f) 
           end
         end

data/spec/log_reader_spec.rb

@@ -5,7 +5,7 @@ require 'fileutils'
 describe "DLogReader::LogReader" do
   before(:all) do
     test_file = File.join(File.dirname(__FILE__), 'fixtures', 'test_file')
-    FileUtils.mkdir(File.join(File.dirname(__FILE__), 'fixtures', 'logreading'))
+    FileUtils.mkdir(File.join(File.dirname(__FILE__), 'fixtures', 'logreading')) rescue nil
     @test_cp = File.join(File.dirname(__FILE__), 'fixtures', 'logreading', 'test')
     FileUtils.cp(test_file, @test_cp)
     test_fh = File.open(test_file)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: garru-distributed_logreader
 version: !ruby/object:Gem::Version 
-  version: 0.8.0
+  version: 0.10.0
 platform: ruby
 authors: 
 - Gary Tsang
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-08-12 00:00:00 -07:00
+date: 2009-09-14 00:00:00 -07:00
 default_executable: 
 dependencies: []