garru-distributed_logreader 0.5.0 → 0.6.0

data/VERSION

@@ -1 +1 @@
-0.5.0
+0.6.0

data/distributed_logreader.gemspec

@@ -2,11 +2,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{distributed_logreader}
-  s.version = "0.5.0"
+  s.version = "0.6.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Gary Tsang"]
-  s.date = %q{2009-08-05}
+  s.date = %q{2009-08-06}
   s.email = %q{gary@garru.com}
   s.extra_rdoc_files = [
     "LICENSE",

data/lib/distributed_logreader/distributed_log_reader/scribe_reader.rb

@@ -5,7 +5,7 @@ module DLogReader
     def initialize(filename, backupdir, worker, num_threads = 10)
       super(filename, worker, num_threads)
       self.selector = RotatingLog.new
-      self.selector.ignore_conditions << lambda{|x| x.match(/scribe_stats/)}
+      self.selector.ignore_conditions << lambda{|x| !x.match(/scribe_stats/).nil?}
       self.archiver = DateDir.new(backupdir)
     end
         

data/lib/distributed_logreader/distributed_log_reader.rb

@@ -17,14 +17,10 @@ module DLogReader
     def process
       pre_process
 
-      $dlog_logger.info("Started #{log_file}:")
+      $dlog_logger.info("Started  #{log_file}:")
         lines_processed = 0
       @log_reader = LogReader.new(log_file) do |line|
-        begin
-          self.distributer.process(line)
-        rescue Exception => e
-          $dlog_logger.warn("Exception in processing thread #{log_file}:#{log_file.pos} -- #{line} -- #{e.message}")
-        end
+        self.distributer.process(line)
         lines_processed += 1
       end
       @log_reader.run

data/lib/distributed_logreader/distributer/simple_thread_pool.rb

@@ -2,18 +2,21 @@ require 'thread'
 
 module DLogReader
   class SimpleThreadPool
-    attr_accessor :num_threads, :worker, :thread_pool, :queue
-    
-    def initialize(worker, num_threads = 10)
+    attr_accessor :num_threads, :worker, :thread_pool, :queue, :max_queue_size
+    def initialize(worker, num_threads = 5)
       self.worker = worker
       self.num_threads = num_threads
       self.queue = Queue.new
+      self.max_queue_size = 100
       num_threads.times do 
         create_thread
       end
     end
     
     def process(line)
+      while(queue.size > self.max_queue_size)
+        sleep(0.1)
+      end
       self.queue << line
     end
     
@@ -28,7 +31,11 @@ protected
       Thread.new do
         loop do
           line = self.queue.pop
-          self.worker.call(line)
+          begin
+            self.worker.call(line)
+          rescue Exception => e
+            $dlog_logger.warn("Exception in processing thread #{line} -- #{e.message}")
+          end
         end
       end
     end

data/lib/distributed_logreader/log_reader.rb

@@ -15,10 +15,18 @@ module DLogReader
       f = File.open(filename, "r+")
       load_saved_state(f)
       raise IOError.new("File is locked") unless f.flock(File::LOCK_EX | File::LOCK_NB)
-      f.each_line do |line|
-        @b.call(line)
+      unless f.eof?
+        line_count = 0
+        f.each_line do |line|
+          @b.call(line)
+          line_count += 1
+          if (line_count % 100 == 0)
+            $dlog_logger.info( "#{Time.now.to_s} #{filename}: Processed (#{line_count}) lines")
+            save_state(f) 
+          end
+        end
+        save_state(f)
       end
-      save_state(f)
       f.flock(File::LOCK_UN)
     end
     

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: garru-distributed_logreader
 version: !ruby/object:Gem::Version 
-  version: 0.5.0
+  version: 0.6.0
 platform: ruby
 authors: 
 - Gary Tsang
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-08-05 00:00:00 -07:00
+date: 2009-08-06 00:00:00 -07:00
 default_executable: 
 dependencies: []