chore-core 1.10.0 → 4.0.0

data/lib/chore/strategies/worker/helpers/work_distributor.rb

@@ -0,0 +1,65 @@
+require 'chore/strategies/worker/helpers/ipc'
+
+module Chore
+  module Strategy
+    class WorkDistributor #:nodoc:
+      class << self
+        include Ipc
+
+        def fetch_and_assign_jobs(workers, manager)
+          jobs = manager.fetch_work(workers.size)
+          raise "DW: jobs needs to be a list got #{jobs.class}" unless jobs.is_a?(Array)
+          if jobs.empty?
+            # This conditon is due to the internal consumer queue being empty.
+            # Assuming that the the consumer has to fetch from an external queue,
+            # if we return here, we would create a tight loop that would use up
+            # a lot the CPU's time. In order to prevent that, we wait for the
+            # consumer queue to be populated, by sleeping. 
+            sleep(0.1)
+            return
+          end
+          jobs_to_return = assign_jobs(jobs, workers)
+          manager.return_work(jobs_to_return)
+        end
+
+        private
+
+        def assign_jobs(jobs, workers)
+          raise 'DW: assign_jobs got 0 workers' if workers.empty?
+          jobs_to_return = []
+          jobs.each_with_index do |job, i|
+            raise 'DW: More Jobs than Sockets' if workers[i].nil?
+            unless push_job_to_worker(job, workers[i])
+              jobs_to_return << job
+            end
+          end
+
+          jobs_to_return
+        end
+
+        def push_job_to_worker(job, worker)
+          Chore.run_hooks_for(:before_send_to_worker, job)
+          clear_ready(worker.socket)
+          send_msg(worker.socket, job)
+          true
+        rescue => e
+          Chore.logger.error "DW: Could not assign job #{job.inspect} (worker: #{worker.pid})\nException #{e.message} #{e.backtrace * "\n"}"
+          
+          # We generally shouldn't get into this situations since we've already
+          # tested that we can read/write to the Worker's socket.  However,
+          # the Worker could still fail between that check and pushing the
+          # job, so we need to allow the work to be re-assigned to handle that
+          # edge case.
+          false
+        end
+
+        private
+
+        # Used for unit tests
+        def sleep(n)
+          Kernel.sleep(n)
+        end
+      end
+    end
+  end
+end

data/lib/chore/strategies/worker/helpers/worker_info.rb

@@ -0,0 +1,13 @@
+module Chore
+  module Strategy
+    class WorkerInfo
+      # Holds meta information about the worker: pid, and connection socket
+      attr_accessor :pid, :socket
+
+      def initialize(pid)
+        @pid = pid
+        @socket = nil
+      end
+    end
+  end
+end

data/lib/chore/strategies/worker/helpers/worker_killer.rb

@@ -0,0 +1,40 @@
+require 'get_process_mem'
+
+module Chore
+  module Strategy
+    class WorkerKiller  #:nodoc:
+      def initialize
+        @memory_limit = Chore.config.memory_limit_bytes
+        @request_limit = Chore.config.request_limit
+        @check_cycle = Chore.config.worker_check_cycle || 16
+        @check_count = 0
+        @current_requests = 0
+      end
+
+      def check_memory
+        return if @memory_limit.nil? || (@memory_limit == 0)
+        @check_count += 1
+
+        if @check_count == @check_cycle
+          rss = GetProcessMem.new.bytes.to_i
+          if rss > @memory_limit
+            Chore.logger.info "WK: (pid: #{Process.pid}) exceeded memory limit (#{rss.to_i} bytes > #{@memory_limit} bytes)"
+            Chore.run_hooks_for(:worker_mem_kill)
+            exit(true)
+          end
+          @check_count = 0
+        end
+      end
+
+      def check_requests
+        return if @request_limit.nil? || (@request_limit == 0)
+
+        if (@current_requests += 1) >= @request_limit
+          Chore.logger.info "WK: (pid: #{Process.pid}) exceeded max number of requests (limit: #{@request_limit})"
+          Chore.run_hooks_for(:worker_req_kill)
+          exit(true)
+        end
+      end
+    end
+  end
+end

data/lib/chore/strategies/worker/helpers/worker_manager.rb

@@ -0,0 +1,183 @@
+require 'chore/strategies/worker/helpers/ipc'
+
+module Chore
+  module Strategy
+    class WorkerManager #:nodoc:
+      include Ipc
+
+      def initialize(master_socket)
+        @master_socket = master_socket
+        @pid_to_worker = {}
+        @socket_to_worker = {}
+      end
+
+      # Create num of missing workers and sockets and attach them for the
+      # master
+      def create_and_attach_workers
+        create_workers do |num_workers|
+          attach_workers(num_workers)
+        end
+      end
+
+      # Reap dead workers and create new ones to replace them
+      def respawn_terminated_workers!
+        Chore.logger.info 'WM: Respawning terminated workers'
+        reap_workers
+        create_and_attach_workers
+      end
+
+      # Stop children with the given kill signal and wait for them to die
+      def stop_workers(sig)
+        @pid_to_worker.each do |pid, worker|
+          begin
+            Chore.logger.info { "WM: Sending #{sig} to: #{pid}" }
+            Process.kill(sig, pid)
+          rescue Errno::ESRCH => e
+            Chore.logger.error "WM: Signal to children error: #{e}"
+          end
+        end
+        # TODO: Sleep for the shutdown timeout and kill any remaining workers
+        reap_workers
+      end
+
+      # Return all the worker sockets
+      def worker_sockets
+        @socket_to_worker.keys
+      end
+
+      # Return the workers associated with a given array of sockets.
+      # +block+:: A block can be provided to perform tasks on the workers
+      # associated with the sockets given
+      def ready_workers(sockets = [], &block)
+        workers = @socket_to_worker.values_at(*sockets)
+        yield workers if block_given?
+        workers
+      end
+
+      private
+
+      # Creates worker processes until we have the number of workers defined
+      # by the configuration. Initializes and starts a worker instance in each
+      # of the new processes.
+      # +block+:: Block can be provided to run tasks on the number of newly
+      # created worker processes.
+      def create_workers(&block)
+        num_created_workers = 0
+
+        while @pid_to_worker.size < Chore.config.num_workers
+          pid = fork do
+            run_worker_instance
+          end
+
+          Chore.logger.info "WM: created_worker #{pid}"
+          # Keep track of the new worker process
+          @pid_to_worker[pid] = WorkerInfo.new(pid)
+          num_created_workers += 1
+        end
+
+        raise 'WM: Not enough workers' if inconsistent_worker_number
+        Chore.logger.info "WM: created #{num_created_workers} workers"
+        yield num_created_workers if block_given?
+        num_created_workers
+      end
+
+      # Check that number of workers registered in master match the config
+      def inconsistent_worker_number
+        Chore.config.num_workers != @pid_to_worker.size
+      end
+
+      # Initialize and start a new worker instance
+      def run_worker_instance
+        PreforkedWorker.new.start_worker(@master_socket)
+      ensure
+        exit(true)
+      end
+
+      # Creates individual sockets for each worker to use and attaches them to
+      # the correct worker
+      def attach_workers(num)
+        Chore.logger.info "WM: Started attaching #{num} workers"
+
+        create_worker_sockets(num).each do |socket|
+          begin
+            readable, _, _ = select_sockets(socket, nil, 2)
+
+            if readable.nil?
+              Chore.logger.info "WM: #{socket} timeout waiting for a worker"
+              socket.close
+              next
+            end
+
+            r_socket = readable.first
+            reported_pid = read_msg(r_socket)
+
+            assigned_worker = @pid_to_worker[reported_pid]
+            assigned_worker.socket = socket
+            @socket_to_worker[socket] = assigned_worker
+
+            Chore.logger.info "WM: Connected #{reported_pid} with #{r_socket}"
+          rescue Errno::ECONNRESET
+            Chore.logger.info "WM: A worker failed to connect to #{socket}"
+            socket.close
+            next
+          end
+        end
+
+        # If the connection from a worker times out, we are unable to associate
+        # the process with a connection and so we kill the worker process
+        kill_unattached_workers
+        Chore.logger.info 'WM: Finished attaching workers'
+      end
+
+      # Create num amount of sockets that are available for worker connections
+      def create_worker_sockets(num)
+        Array.new(num) do
+          add_worker_socket
+        end
+      end
+
+      # Kill workers that failed to connect to the master
+      def kill_unattached_workers
+        @pid_to_worker.each do |pid, worker|
+          next unless worker.socket.nil?
+          Chore.logger.info "WM: kill_unattached_workers #{pid}"
+          Process.kill('KILL', pid)
+        end
+      end
+
+      # Wait for terminated workers to die and remove their references from
+      # master
+      def reap_workers
+        Chore.logger.info "WM: reaping workers.."
+        dead_workers = @pid_to_worker.select do |pid, worker|
+          reap_process(pid)
+        end
+
+        dead_workers.each do |pid, worker|
+          dead_worker = @pid_to_worker.delete(pid)
+          dead_worker.socket.close
+          @socket_to_worker.delete(dead_worker.socket)
+          Chore.logger.info "WM: Removed preforked worker:#{worker.pid} - #{worker.socket}"
+        end
+      end
+
+      # Non-blocking wait for process to die. Returns whether it stopped
+      def reap_process(pid)
+        status = Process.wait(pid, Process::WNOHANG)
+        case status
+        when nil # Process is still running
+          return false
+        when pid # Collected status of this pid
+          return true
+        end
+      rescue Errno::ECHILD
+        # Child process has already terminated
+        true
+      end
+
+      def fork(&block)
+        Kernel.fork(&block)
+      end
+    end
+  end
+end

data/lib/chore/strategies/worker/preforked_worker_strategy.rb

@@ -0,0 +1,150 @@
+require 'chore/signal'
+require 'socket'
+require 'chore/strategies/worker/helpers/ipc'
+require 'chore/strategies/worker/helpers/preforked_worker'
+require 'chore/strategies/worker/helpers/worker_manager'
+require 'chore/strategies/worker/helpers/work_distributor'
+
+module Chore
+  module Strategy
+    class PreForkedWorkerStrategy #:nodoc:
+      include Ipc
+
+      NUM_TO_SIGNAL = {  '1' => :CHLD,
+                         '2' => :INT,
+                         '3' => :QUIT,
+                         '4' => :TERM,
+                         '5' => :USR1 }.freeze
+
+      def initialize(manager, opts = {})
+        @options = opts
+        @manager = manager
+        @self_read, @self_write = IO.pipe
+        trap_signals(NUM_TO_SIGNAL, @self_write)
+        @worker_manager = WorkerManager.new(create_master_socket)
+        at_exit { delete_socket_file }
+        @running = true
+      end
+
+      def start
+        Chore.logger.info "PWS: Starting up worker strategy: #{self.class.name}"
+        Chore.run_hooks_for(:before_first_fork)
+        @worker_manager.create_and_attach_workers
+        worker_assignment_thread
+      end
+
+      def stop!
+        Chore.logger.info "PWS: Stopping worker strategy: #{self.class.name}"
+        @running = false
+      end
+
+      private
+
+      def worker_assignment_thread
+        Thread.new do
+          begin
+            worker_assignment_loop
+          rescue Chore::TerribleMistake => e
+            Chore.logger.error 'PWS: Terrible mistake, shutting down Chore'
+            Chore.logger.error e.message
+            Chore.logger.error e.backtrace
+            @manager.shutdown!
+          ensure
+            Chore.logger.info 'PWS: worker_assignment_thread ending'
+            # WorkerAssignment thread is independent of the main thread.
+            # The main thread is waiting on the consumer threads to join,
+            # Due to some weird SQS behaviour, its possible that these threads
+            # maynot join, and the assigment thread always exits, since it's
+            # nonblocking. This will ensure that the master process exits.
+            Process.exit(true)
+          end
+        end
+      end
+
+      def worker_assignment_loop
+        while running?
+          w_sockets = @worker_manager.worker_sockets
+
+          # select_sockets returns a list of readable sockets
+          # This would include worker connections and the read end
+          # of the self-pipe.
+          # 
+          # Note this not only returns sockets from live workers
+          # that are readable, but it also returns sockets from
+          # *dead* workers.  If the worker hasn't already been reaped,
+          # then we might get a socket for a dead worker than will
+          # fail on write.
+          readables, = select_sockets(w_sockets, @self_read)
+
+          # If select timed out, retry
+          if readables.nil?
+            Chore.logger.debug 'PWS: All sockets busy.. retry'
+            next
+          end
+
+          # Handle the signal from the self-pipe
+          if readables.include?(@self_read)
+            handle_signal
+            next
+          end
+
+          # Confirm they're actually alive!  A socket will be readable even
+          # if the worker has died but not yet been reaped by the master.  We
+          # need to confirm that the "Ready" flag has actually been written by
+          # the worker and readable by the master.
+          readables.reject! {|readable| readable.eof?}
+
+          # Check again to see if there are still sockets available
+          if readables.empty?
+            Chore.logger.debug 'PWS: All sockets busy.. retry'
+            next
+          end
+
+          # Fetch and assign work for the readable worker connections
+          @worker_manager.ready_workers(readables) do |workers|
+            WorkDistributor.fetch_and_assign_jobs(workers, @manager)
+          end
+        end
+        Chore.logger.info 'PWS: worker_assignment_loop ending'
+      end
+
+      # Wrapper need around running to help writing specs for worker_assignment_loop
+      def running?
+        @running
+      end
+
+      def handle_signal
+        signal = NUM_TO_SIGNAL[@self_read.read_nonblock(1)]
+        Chore.logger.info "PWS: recv #{signal}"
+
+        case signal
+        when :CHLD
+          @worker_manager.respawn_terminated_workers!
+        when :INT, :QUIT, :TERM
+          Signal.reset
+          @worker_manager.stop_workers(signal)
+          @manager.shutdown!
+        when :USR1
+          Chore.reopen_logs
+          Chore.logger.info 'PWS: Master process reopened log'
+        end
+      end
+
+      # Wrapper around fork for specs.
+      def fork(&block)
+        Kernel.fork(&block)
+      end
+
+      # In the event of a trapped signal, write to the self-pipe
+      def trap_signals(signal_hash, write_end)
+        Signal.reset
+
+        signal_hash.each do |sig_num, signal|
+          Signal.trap(signal) do
+            write_end.write(sig_num)
+          end
+        end
+      end
+    end
+  end
+end