garru-distributed_logreader 0.7.1 → 0.8.0

data/VERSION

@@ -1 +1 @@
-0.7.1
+0.8.0

data/distributed_logreader.gemspec

@@ -2,11 +2,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{distributed_logreader}
-  s.version = "0.7.1"
+  s.version = "0.8.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Gary Tsang"]
-  s.date = %q{2009-08-10}
+  s.date = %q{2009-08-12}
   s.email = %q{gary@garru.com}
   s.extra_rdoc_files = [
     "LICENSE",

data/lib/distributed_logreader.rb

@@ -8,5 +8,5 @@ require 'distributed_logreader/distributed_log_reader/scribe_reader'
 require 'logger'
 
 $dlog_logger = Logger.new("/var/log/distributed_logreader.log")
-$dlog_logger.level = Logger::INFO
+$dlog_logger.level = Logger::DEBUG
 $dlog_logger.datetime_format = "%Y-%m-%d %H:%M:%S "

data/lib/distributed_logreader/distributed_log_reader.rb

@@ -11,7 +11,7 @@ module DLogReader
     def initialize(filename, worker, num_threads = 100)
       self.filename = filename
       # self.distributer = SimpleForked.new(worker, 5, num_threads)
-      self.distributer = SimpleThreadPool.new(worker, 500)
+      self.distributer = SimpleThreadPool.new(worker, num_threads)
     end
     
     # selector/archiver seem to be strongly connected.  it's possible it

data/lib/distributed_logreader/distributed_log_reader/scribe_reader.rb

@@ -1,14 +1,31 @@
 module DLogReader
-  class ScribeReader < DistributedLogReader
-    attr_accessor :selector, :archiver
+  class ScribeReader
+    attr_accessor :selector, :archiver, :filename
     attr_reader :log_reader
     def initialize(filename, backupdir, worker, num_threads = 10)
-      super(filename, worker, num_threads)
+      self.filename = filename
       self.selector = RotatingLog.new
       self.selector.ignore_conditions << lambda{|x| !x.match(/scribe_stats/).nil?}
+      @worker = worker
       self.archiver = DateDir.new(backupdir)
     end
-        
+    
+    def process
+      $dlog_logger.info("Started  #{log_file}:")
+      lines_processed = 0
+      @log_reader = LogReader.new(log_file) do |line|
+        @worker.call(line)
+        lines_processed += 1
+      end
+      @log_reader.run
+      $dlog_logger.info("Finished #{log_file}: Processed (#{lines_processed}) lines")
+      post_process
+    end
+    
+    def log_file
+      self.filename
+    end
+    
     def log_file
       @log_file ||= begin
         selector.file_to_process(filename)
@@ -22,7 +39,7 @@ module DLogReader
     def current?
       directory = File.dirname(log_file)
       basename = File.basename(directory)
-      current_file = File.join(directory, "#{basename}_current")
+      current_file = Dir[File.join(directory, "*")].detect{|x| x.match(/current/)}
       File.exists?(current_file) && File.identical?(current_file, log_file) 
     end
   end

data/lib/distributed_logreader/distributer/pandemic_processor.rb

@@ -15,6 +15,7 @@ module DLogReader
         @in = read_from_child
         @max_queue_size = 100
         @counter = MutexCounter.new
+        @job_mutex = Mutex.new
         wait_for_responses
       else
         $dlog_logger.debug("Forked")
@@ -43,12 +44,15 @@ module DLogReader
     def process(body)
       if parent?
         while(@counter.real_total > @max_queue_size)
+          $dlog_logger.debug("Max process queue size: #{@counter.real_total}")
           sleep(0.01)
         end
         body = (body.chomp + "\n")
-        $dlog_logger.debug("Parent: writing #{body.inspect}")
-        @out.write(body)
-        @counter.inc
+        @job_mutex.synchronize do
+          @out.write(body)
+          in_queue = @counter.inc
+          $dlog_logger.debug("Parent: writing #{body.inspect} - #{in_queue}")
+        end
       else
         $dlog_logger.debug("Child Processing #{body}")
         return @handler.call(body)
@@ -76,9 +80,11 @@ module DLogReader
       Thread.new do
         loop do
           line = @job_queue.pop
-          process(line)
-          $dlog_logger.debug("Child: Finished #{line.inspect}")
-          @response_queue << :a
+          @job_mutex.synchronize do
+            process(line)
+            @response_queue << :a
+            $dlog_logger.debug("Child: Finished #{line.inspect}")
+          end
         end
       end
     end
@@ -88,9 +94,9 @@ module DLogReader
         loop do
           ready, = IO.select([@in], nil, nil)
           if ready
-            $dlog_logger.debug("Parent: Reading Response")  
             @in.readchar
-            @counter.decr
+            in_queue = @counter.decr
+            $dlog_logger.debug("Parent: Reading Response #{in_queue}")
           end
         end
       end

data/lib/distributed_logreader/distributer/simple_forked_process.rb

@@ -3,12 +3,11 @@ require File.join(File.dirname(__FILE__), 'pandemic_processor')
 require File.join(File.dirname(__FILE__), 'mutex_counter')
 module DLogReader
   class SimpleForked
-    attr_accessor :num_threads_per_process, :worker, :thread_pool, :queue, :max_queue_size, :processors
+    attr_accessor :num_threads_per_process, :worker, :thread_pool, :queue, :processors
     def initialize(worker, num_processes = 3, num_threads_per_process = 10)
       self.worker = worker
-      self.num_threads_per_process = num_threads_per_process
+      self.num_threads_per_process = (num_threads_per_process || 10)
       self.queue = Queue.new
-      self.max_queue_size = 100
       self.processors = []
       num_processes.times do |x|
         $dlog_logger.debug("Forking #{x} process")
@@ -17,9 +16,6 @@ module DLogReader
     end
     
     def process(line)
-      while(queue.size > self.max_queue_size)
-        sleep(0.1)
-      end
       self.queue << line
     end
     

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: garru-distributed_logreader
 version: !ruby/object:Gem::Version 
-  version: 0.7.1
+  version: 0.8.0
 platform: ruby
 authors: 
 - Gary Tsang
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-08-10 00:00:00 -07:00
+date: 2009-08-12 00:00:00 -07:00
 default_executable: 
 dependencies: []