garru-distributed_logreader 0.6.0 → 0.7.0

data/VERSION

@@ -1 +1 @@
-0.6.0
+0.7.0

data/distributed_logreader.gemspec

@@ -2,11 +2,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{distributed_logreader}
-  s.version = "0.6.0"
+  s.version = "0.7.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Gary Tsang"]
-  s.date = %q{2009-08-06}
+  s.date = %q{2009-08-10}
   s.email = %q{gary@garru.com}
   s.extra_rdoc_files = [
     "LICENSE",
@@ -26,6 +26,9 @@ Gem::Specification.new do |s|
      "lib/distributed_logreader/distributed_log_reader/rotater_reader.rb",
      "lib/distributed_logreader/distributed_log_reader/scribe_reader.rb",
      "lib/distributed_logreader/distributer.rb",
+     "lib/distributed_logreader/distributer/mutex_counter.rb",
+     "lib/distributed_logreader/distributer/pandemic_processor.rb",
+     "lib/distributed_logreader/distributer/simple_forked_process.rb",
      "lib/distributed_logreader/distributer/simple_thread_pool.rb",
      "lib/distributed_logreader/log_reader.rb",
      "lib/distributed_logreader/selector.rb",

data/lib/distributed_logreader/distributed_log_reader.rb

@@ -1,4 +1,5 @@
 require File.join(File.dirname(__FILE__), 'distributer', 'simple_thread_pool')
+require File.join(File.dirname(__FILE__), 'distributer', 'simple_forked_process')
 require File.join(File.dirname(__FILE__), 'selector', 'rotating_log')
 require File.join(File.dirname(__FILE__), 'archiver', 'date_dir')
 require File.join(File.dirname(__FILE__), 'log_reader')
@@ -7,9 +8,10 @@ module DLogReader
   class DistributedLogReader
     attr_accessor :distributer, :filename
     attr_reader :log_reader
-    def initialize(filename, worker, num_threads = 10)
+    def initialize(filename, worker, num_threads = 100)
       self.filename = filename
-      self.distributer = SimpleThreadPool.new(worker, num_threads)
+      # self.distributer = SimpleForked.new(worker, 5, num_threads)
+      self.distributer = SimpleThreadPool.new(worker, 500)
     end
     
     # selector/archiver seem to be strongly connected.  it's possible it
@@ -34,7 +36,7 @@ module DLogReader
     end
     
     #predefined hooks
-    def pre_process  
+    def pre_process
     end
     
     #predefined hooks

data/lib/distributed_logreader/distributer/mutex_counter.rb

@@ -0,0 +1,48 @@
+class MutexCounter
+  MAX = (2 ** 30) - 1
+  def initialize(max = MAX)
+    @mutex = Mutex.new
+    @counter = 0
+    @resets = 0
+    @max = max
+  end
+
+  def real_total
+    @mutex.synchronize { (@resets * @max) + @counter }
+  end
+  alias_method :to_i, :real_total
+  
+  def value
+    @mutex.synchronize { @counter }
+  end
+
+  def inc
+    @mutex.synchronize do
+      if @counter >= @max
+        @counter = 0  # to avoid Bignum, it's about 4x slower
+        @resets += 1
+      end
+      @counter += 1
+    end
+  end
+  alias_method :next, :inc
+  alias_method :succ, :inc
+  
+  # decr only to zero
+  def decr
+    @mutex.synchronize do
+      if @counter > 0
+        @counter -= 1
+      else
+        if @resets > 1
+          @resets -= 1 
+          @counter = @max
+        end
+      end
+      @counter
+    end
+  end
+  alias_method :pred, :decr
+  alias_method :prev, :decr
+  
+end

data/lib/distributed_logreader/distributer/pandemic_processor.rb

@@ -0,0 +1,143 @@
+require 'rubygems'
+#stolen from pandemic
+module DLogReader
+  class Processor
+    def initialize(handler, num_threads = 10)
+      read_from_parent, write_to_child = IO.pipe
+      read_from_child, write_to_parent = IO.pipe
+  
+      @child_process_id = fork
+      if @child_process_id
+        # I'm the parent
+        write_to_parent.close
+        read_from_parent.close
+        @out = write_to_child
+        @in = read_from_child
+        @max_queue_size = 100
+        @counter = MutexCounter.new
+        wait_for_responses
+      else
+        $dlog_logger.debug("Forked")
+        # I'm the child
+        write_to_child.close
+        read_from_child.close
+        @out = write_to_parent
+        @in = read_from_parent
+        @handler = handler
+        @job_queue = Queue.new
+        @response_queue = Queue.new
+        wait_for_job_completion
+        num_threads.times do
+          create_thread
+        end
+        wait_for_jobs
+      end
+    end
+  
+    def num_jobs
+      if parent?
+        @counter.real_total
+      end
+    end
+  
+    def process(body)
+      if parent?
+        while(@counter.real_total > @max_queue_size)
+          sleep(0.01)
+        end
+        body = (body.chomp + "\n")
+        $dlog_logger.debug("Parent: writing #{body.inspect}")
+        @out.write(body)
+        @counter.inc
+      else
+        $dlog_logger.debug("Child Processing #{body}")
+        return @handler.call(body)
+      end
+    end
+
+    def close(status = 0)
+      if parent? && child_alive?
+        Process.detach(@child_process_id)
+        @out.puts(status.to_s)
+        @out.close
+        @in.close
+      else
+        Process.exit!(status)
+      end
+    end
+
+    def closed?
+      !child_alive?
+    end
+
+    private
+
+    def create_thread
+      Thread.new do
+        loop do
+          line = @job_queue.pop
+          process(line)
+          $dlog_logger.debug("Child: Finished #{line.inspect}")
+          @response_queue << :a
+        end
+      end
+    end
+
+    def wait_for_responses
+      Thread.new do
+        loop do
+          ready, = IO.select([@in], nil, nil)
+          if ready
+            $dlog_logger.debug("Parent: Reading Response")  
+            @in.readchar
+            @counter.decr
+          end
+        end
+      end
+    end
+
+    #child process
+    def wait_for_jobs
+      if child?
+        while true
+          $dlog_logger.debug("Child waiting")  
+          ready, = IO.select([@in], nil, nil)
+          if ready && !@in.eof?
+            line = @in.gets
+            $dlog_logger.debug("Child: #{line.inspect}")
+            @job_queue << line
+          else
+            self.close(line.to_i)
+            break  
+          end
+        end
+      end
+    end
+
+    #child process
+    def wait_for_job_completion
+      if child?
+        Thread.new do
+          while true
+            @response_queue.pop
+            $dlog_logger.debug("Child: Writing To Parent")
+            @out.write("|")
+          end
+        end
+      end
+    end
+
+
+    def parent?
+      !!@child_process_id
+    end
+
+    def child?
+      !parent?
+    end
+
+    def child_alive?
+      parent? && !@in.closed?
+    end
+  end
+end

data/lib/distributed_logreader/distributer/simple_forked_process.rb

@@ -0,0 +1,51 @@
+require 'thread'
+require File.join(File.dirname(__FILE__), 'pandemic_processor')
+require File.join(File.dirname(__FILE__), 'mutex_counter')
+module DLogReader
+  class SimpleForked
+    attr_accessor :num_threads_per_process, :worker, :thread_pool, :queue, :max_queue_size, :processors
+    def initialize(worker, num_processes = 3, num_threads_per_process = 10)
+      self.worker = worker
+      self.num_threads_per_process = num_threads_per_process
+      self.queue = Queue.new
+      self.max_queue_size = 100
+      self.processors = []
+      num_processes.times do |x|
+        $dlog_logger.debug("Forking #{x} process")
+        self.processors << create_process
+      end
+    end
+    
+    def process(line)
+      while(queue.size > self.max_queue_size)
+        sleep(0.1)
+      end
+      self.queue << line
+    end
+    
+    def join
+      num_jobs_outstanding = self.processors.inject(0){|a,b| a + b.num_jobs}
+      while(queue.size > 0 || num_jobs_outstanding > 0)
+        sleep 0.1
+        num_jobs_outstanding = self.processors.inject(0){|a,b| a + b.num_jobs}
+        $dlog_logger.debug("Shutting down #{num_jobs_outstanding} left")
+      end
+    end
+    
+protected    
+    def create_process
+      processor = Processor.new(self.worker, self.num_threads_per_process)
+      Thread.new do
+        loop do
+          line = self.queue.pop
+          begin
+            processor.process(line)
+          rescue Exception => e
+            $dlog_logger.warn("Exception in processing thread #{line} -- #{e.message}")
+          end
+        end
+      end
+      processor
+    end
+  end
+end

data/lib/distributed_logreader/distributer/simple_thread_pool.rb

@@ -15,7 +15,7 @@ module DLogReader
     
     def process(line)
       while(queue.size > self.max_queue_size)
-        sleep(0.1)
+        sleep(0.01)
       end
       self.queue << line
     end

data/lib/distributed_logreader/log_reader.rb

@@ -10,11 +10,11 @@ module DLogReader
     end
   
     def run
-      raise IOError.new("no file given") if filename.nil?
+      # raise IOError.new("no file given") if filename.nil?
       raise IOError.new("File not readable") unless File.readable?(filename)
       f = File.open(filename, "r+")
       load_saved_state(f)
-      raise IOError.new("File is locked") unless f.flock(File::LOCK_EX | File::LOCK_NB)
+      # raise IOError.new("File is locked") unless f.flock(File::LOCK_EX | File::LOCK_NB)
       unless f.eof?
         line_count = 0
         f.each_line do |line|
@@ -27,7 +27,7 @@ module DLogReader
         end
         save_state(f)
       end
-      f.flock(File::LOCK_UN)
+      # f.flock(File::LOCK_UN)
     end
     
     def statefile

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: garru-distributed_logreader
 version: !ruby/object:Gem::Version 
-  version: 0.6.0
+  version: 0.7.0
 platform: ruby
 authors: 
 - Gary Tsang
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-08-06 00:00:00 -07:00
+date: 2009-08-10 00:00:00 -07:00
 default_executable: 
 dependencies: []
 
@@ -36,6 +36,9 @@ files:
 - lib/distributed_logreader/distributed_log_reader/rotater_reader.rb
 - lib/distributed_logreader/distributed_log_reader/scribe_reader.rb
 - lib/distributed_logreader/distributer.rb
+- lib/distributed_logreader/distributer/mutex_counter.rb
+- lib/distributed_logreader/distributer/pandemic_processor.rb
+- lib/distributed_logreader/distributer/simple_forked_process.rb
 - lib/distributed_logreader/distributer/simple_thread_pool.rb
 - lib/distributed_logreader/log_reader.rb
 - lib/distributed_logreader/selector.rb