resque-pool 0.0.2

data/lib/resque/pool/tasks.rb

@@ -0,0 +1,17 @@
+# -*- encoding: utf-8 -*-
+
+# require 'resque/pool/tasks'
+# and configure "resque:setup" task to start up the environment, initialize
+# RESQUE_POOL_CONFIG, and setup other resque hooks
+
+namespace :resque do
+  task :setup
+
+  desc "Launch a pool of resque workers (set RESQUE_POOL_CONFIG)"
+  task :pool => :setup do
+    GC.respond_to?(:copy_on_write_friendly=) && GC.copy_on_write_friendly = true
+    require 'resque/pool'
+    Resque::Pool.new(RESQUE_POOL_CONFIG).start.join
+  end
+
+end

data/lib/resque/pool.rb

@@ -0,0 +1,203 @@
+# -*- encoding: utf-8 -*-
+require 'resque'
+require 'fcntl'
+
+module Resque
+  class Pool
+    attr_reader :pool_config
+    attr_reader :workers
+
+    SELF_PIPE = []
+    SIG_QUEUE = []
+    QUEUE_SIGS = [ :QUIT, :INT, :TERM, :USR1, :USR2, :HUP, ]
+    CHUNK_SIZE=(16 * 1024)
+
+    def initialize(config)
+      @pool_config = config.dup
+      @workers = pool_config.keys.inject({}) {|h,k| h[k] = {}; h}
+      procline "(initialized)"
+    end
+
+
+    # The `after_prefork` hook will be run in workers if you are using the
+    # preforking master worker to save memory. Use this hook to reload
+    # database connections and so forth to ensure that they're not shared
+    # among workers.
+    #
+    # Call with a block to set the hook.
+    # Call with no arguments to return the hook.
+    def self.after_prefork(&block)
+      block ? (@after_prefork = block) : @after_prefork
+    end
+
+    # Set the after_prefork proc.
+    def self.after_prefork=(after_prefork)
+      @after_prefork = after_prefork
+    end
+
+    def call_after_prefork!
+      self.class.after_prefork && self.class.after_prefork.call
+    end
+
+    # Given a string, sets the procline ($0)
+    # Procline is always in the format of:
+    #   resque-pool-master: STRING
+    def procline(string)
+      $0 = "resque-pool-master: #{string}"
+    end
+
+    # TODO: make this use an actual logger
+    def log(message)
+      puts message
+    end
+
+    def init_self_pipe!
+      SELF_PIPE.each { |io| io.close rescue nil }
+      SELF_PIPE.replace(IO.pipe)
+      SELF_PIPE.each { |io| io.fcntl(Fcntl::F_SETFD, Fcntl::FD_CLOEXEC) }
+    end
+
+    def init_sig_handlers!
+      QUEUE_SIGS.each { |sig| trap_deferred(sig) }
+      trap(:CHLD) { |_| awaken_master }
+    end
+
+    # defer a signal for later processing in #join (master process)
+    def trap_deferred(signal)
+      trap(signal) do |sig_nr|
+        if SIG_QUEUE.size < 5
+          SIG_QUEUE << signal
+          awaken_master
+        else
+          log "ignoring SIG#{signal}, queue=#{SIG_QUEUE.inspect}"
+        end
+      end
+    end
+
+    def start
+      procline("(starting)")
+      init_self_pipe!
+      init_sig_handlers!
+      maintain_worker_count
+      procline("(started)")
+      log "**** done in pool master #initialize"
+      log "**** Pool contains PIDs: #{all_pids.inspect}"
+      self
+    end
+
+    def join
+      loop do
+        reap_all_workers
+        break if handle_sig_queue! == :break
+        maintain_worker_count if SIG_QUEUE.empty?
+        procline("managing #{all_pids.inspect}")
+      end
+      procline("(shutting down)")
+      #stop # gracefully shutdown all workers on our way out
+      #unlink_pid_safe(pid) if pid
+    end
+
+    def handle_sig_queue!
+      case SIG_QUEUE.shift
+      when nil
+        master_sleep
+      when :QUIT # graceful shutdown
+        :break
+      when :TERM, :INT # immediate shutdown
+        #stop(false)
+        :break
+      end
+    end
+
+    def reap_all_workers
+      begin
+        loop do
+          wpid, status = Process.waitpid2(-1, Process::WNOHANG)
+          wpid or break
+          #if reexec_pid == wpid
+            #log "reaped #{status.inspect} exec()-ed"
+            #self.reexec_pid = 0
+            #self.pid = pid.chomp('.oldbin') if pid
+            #proc_name 'master'
+          #else
+            worker = delete_worker(wpid) #and worker.tmp.close rescue nil
+            log "reaped #{status.inspect} " \
+                        "worker=#{worker.nr rescue 'unknown'}"
+          #end
+        end
+      rescue Errno::ECHILD
+      end
+    end
+
+    def delete_worker(pid)
+      workers.each do |queues, pid_to_worker|
+        pid_to_worker.delete(pid)
+      end
+    end
+
+    def master_sleep
+      begin
+        ready = IO.select([SELF_PIPE.first], nil, nil, 1) or return
+        ready.first && ready.first.first or return
+        loop { SELF_PIPE.first.read_nonblock(CHUNK_SIZE) }
+      rescue Errno::EAGAIN, Errno::EINTR
+      end
+    end
+
+    def awaken_master
+      begin
+        SELF_PIPE.last.write_nonblock('.') # wakeup master process from select
+      rescue Errno::EAGAIN, Errno::EINTR
+        # pipe is full, master should wake up anyways
+        retry
+      end
+    end
+
+    def maintain_worker_count
+      pool_config.each do |queues, count|
+        next if (delta = worker_delta_for(queues)) == 0
+        spawn_missing_workers_for(queues) if delta > 0
+        #TODO: quit_excess_workers_for(queues)
+      end
+    end
+
+    def all_pids
+      workers.map {|q,workers| workers.keys }.flatten
+    end
+
+    ##
+    # all methods below operate on a single grouping of queues
+    # perhaps this means a class is waiting to be extracted
+
+    def spawn_missing_workers_for(queues)
+      worker_delta_for(queues).times do |nr|
+        spawn_worker!(queues)
+      end
+    end
+
+    def worker_delta_for(queues)
+      pool_config[queues] - workers[queues].size
+    end
+
+    def spawn_worker!(queues)
+      worker = create_worker(queues)
+      pid = fork do
+        log "*** Starting worker #{worker}"
+        call_after_prefork!
+        worker.work(ENV['INTERVAL'] || 5) # interval, will block
+      end
+      workers[queues][pid] = worker
+    end
+
+    def create_worker(queues)
+      queues = queues.to_s.split(',')
+      worker = Resque::Worker.new(*queues)
+      worker.verbose = ENV['LOGGING'] || ENV['VERBOSE']
+      worker.very_verbose = ENV['VVERBOSE']
+      worker
+    rescue Resque::NoQueueError
+      abort "set QUEUE env var, e.g. $ QUEUE=critical,high rake resque:work"
+    end
+
+  end
+end

metadata

@@ -0,0 +1,101 @@
+--- !ruby/object:Gem::Specification 
+name: resque-pool
+version: !ruby/object:Gem::Version 
+  hash: 27
+  prerelease: false
+  segments: 
+  - 0
+  - 0
+  - 2
+  version: 0.0.2
+platform: ruby
+authors: 
+- nicholas a. evans
+- Unicorn hackers
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-06-15 00:00:00 -04:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: rspec
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: resque
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 49
+        segments: 
+        - 1
+        - 9
+        - 1
+        version: 1.9.1
+  type: :runtime
+  version_requirements: *id002
+description: "    quickly and easily fork a pool of resque workers,\n    saving memory (w/REE) and monitoring their uptime\n"
+email: 
+- nick@ekenosen.net
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- lib/resque/pool.rb
+- lib/resque/pool/tasks.rb
+has_rdoc: true
+homepage: http://github.com/nevans/resque-pool
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 23
+      segments: 
+      - 1
+      - 3
+      - 6
+      version: 1.3.6
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: quickly and easily fork a pool of resque workers
+test_files: []
+