chore-core 1.8.2 → 3.2.3

data/lib/chore/strategies/worker/preforked_worker_strategy.rb

@@ -0,0 +1,150 @@
+require 'chore/signal'
+require 'socket'
+require 'chore/strategies/worker/helpers/ipc'
+require 'chore/strategies/worker/helpers/preforked_worker'
+require 'chore/strategies/worker/helpers/worker_manager'
+require 'chore/strategies/worker/helpers/work_distributor'
+
+module Chore
+  module Strategy
+    class PreForkedWorkerStrategy #:nodoc:
+      include Ipc
+
+      NUM_TO_SIGNAL = {  '1' => :CHLD,
+                         '2' => :INT,
+                         '3' => :QUIT,
+                         '4' => :TERM,
+                         '5' => :USR1 }.freeze
+
+      def initialize(manager, opts = {})
+        @options = opts
+        @manager = manager
+        @self_read, @self_write = IO.pipe
+        trap_signals(NUM_TO_SIGNAL, @self_write)
+        @worker_manager = WorkerManager.new(create_master_socket)
+        at_exit { delete_socket_file }
+        @running = true
+      end
+
+      def start
+        Chore.logger.info "PWS: Starting up worker strategy: #{self.class.name}"
+        Chore.run_hooks_for(:before_first_fork)
+        @worker_manager.create_and_attach_workers
+        worker_assignment_thread
+      end
+
+      def stop!
+        Chore.logger.info "PWS: Stopping worker strategy: #{self.class.name}"
+        @running = false
+      end
+
+      private
+
+      def worker_assignment_thread
+        Thread.new do
+          begin
+            worker_assignment_loop
+          rescue Chore::TerribleMistake => e
+            Chore.logger.error 'PWS: Terrible mistake, shutting down Chore'
+            Chore.logger.error e.message
+            Chore.logger.error e.backtrace
+            @manager.shutdown!
+          ensure
+            Chore.logger.info 'PWS: worker_assignment_thread ending'
+            # WorkerAssignment thread is independent of the main thread.
+            # The main thread is waiting on the consumer threads to join,
+            # Due to some weird SQS behaviour, its possible that these threads
+            # maynot join, and the assigment thread always exits, since it's
+            # nonblocking. This will ensure that the master process exits.
+            Process.exit(true)
+          end
+        end
+      end
+
+      def worker_assignment_loop
+        while running?
+          w_sockets = @worker_manager.worker_sockets
+
+          # select_sockets returns a list of readable sockets
+          # This would include worker connections and the read end
+          # of the self-pipe.
+          # 
+          # Note this not only returns sockets from live workers
+          # that are readable, but it also returns sockets from
+          # *dead* workers.  If the worker hasn't already been reaped,
+          # then we might get a socket for a dead worker than will
+          # fail on write.
+          readables, = select_sockets(w_sockets, @self_read)
+
+          # If select timed out, retry
+          if readables.nil?
+            Chore.logger.debug 'PWS: All sockets busy.. retry'
+            next
+          end
+
+          # Handle the signal from the self-pipe
+          if readables.include?(@self_read)
+            handle_signal
+            next
+          end
+
+          # Confirm they're actually alive!  A socket will be readable even
+          # if the worker has died but not yet been reaped by the master.  We
+          # need to confirm that the "Ready" flag has actually been written by
+          # the worker and readable by the master.
+          readables.reject! {|readable| readable.eof?}
+
+          # Check again to see if there are still sockets available
+          if readables.empty?
+            Chore.logger.debug 'PWS: All sockets busy.. retry'
+            next
+          end
+
+          # Fetch and assign work for the readable worker connections
+          @worker_manager.ready_workers(readables) do |workers|
+            WorkDistributor.fetch_and_assign_jobs(workers, @manager)
+          end
+        end
+        Chore.logger.info 'PWS: worker_assignment_loop ending'
+      end
+
+      # Wrapper need around running to help writing specs for worker_assignment_loop
+      def running?
+        @running
+      end
+
+      def handle_signal
+        signal = NUM_TO_SIGNAL[@self_read.read_nonblock(1)]
+        Chore.logger.info "PWS: recv #{signal}"
+
+        case signal
+        when :CHLD
+          @worker_manager.respawn_terminated_workers!
+        when :INT, :QUIT, :TERM
+          Signal.reset
+          @worker_manager.stop_workers(signal)
+          @manager.shutdown!
+        when :USR1
+          Chore.reopen_logs
+          Chore.logger.info 'PWS: Master process reopened log'
+        end
+      end
+
+      # Wrapper around fork for specs.
+      def fork(&block)
+        Kernel.fork(&block)
+      end
+
+      # In the event of a trapped signal, write to the self-pipe
+      def trap_signals(signal_hash, write_end)
+        Signal.reset
+
+        signal_hash.each do |sig_num, signal|
+          Signal.trap(signal) do
+            write_end.write(sig_num)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/chore/strategies/worker/single_worker_strategy.rb

@@ -10,7 +10,10 @@ module Chore
       def initialize(manager, opts={})
         @options = opts
         @manager = manager
+        @stopped = false
         @worker = nil
+        @queue = Queue.new
+        @queue << :worker
       end
 
       # Starts the <tt>SingleWorkerStrategy</tt>. Currently a noop
@@ -18,6 +21,11 @@ module Chore
 
       # Stops the <tt>SingleWorkerStrategy</tt> if there is a worker to stop
       def stop!
+        return if @stopped
+
+        @stopped = true
+        Chore.logger.info { "Manager #{Process.pid} stopping" }
+
         worker.stop! if worker
       end
 
@@ -25,16 +33,14 @@ module Chore
       # single worker strategy, this should never be called if the worker is in
       # progress.
       def assign(work)
-        if workers_available?
-          begin
-            @worker = worker_klass.new(work, @options)
-            @worker.start
-            true
-          ensure
-            @worker = nil
-          end
-        else
-          Chore.logger.error { "#{self.class}#assign: single worker is unavailable, but assign has been re-entered: #{caller * "\n"}" }
+        return unless acquire_worker
+
+        begin
+          @worker = worker_klass.new(work, @options)
+          @worker.start
+          true
+        ensure
+          release_worker
         end
       end
 
@@ -42,9 +48,25 @@ module Chore
         Worker
       end
 
-      # Returns true if there is currently no worker
-      def workers_available?
-        @worker.nil?
+      private
+
+      # Attempts to essentially acquire a lock on a worker.  If no workers are
+      # available, then this will block until one is.
+      def acquire_worker
+        result = @queue.pop
+
+        if @stopped
+          # Strategy has stopped since the worker was acquired
+          release_worker
+          nil
+        else
+          result
+        end
+      end
+
+      # Releases the lock on a worker so that another thread can pick it up.
+      def release_worker
+        @queue << :worker
       end
     end
   end

data/lib/chore/unit_of_work.rb

@@ -9,6 +9,14 @@ module Chore
   # * +:consumer+ The consumer instance used to fetch this message. Most queue implementations won't need access to this, but some (RabbitMQ) will. So we
   # make sure to pass it along with each message. This instance will be used by the Worker for things like <tt>complete</tt> and </tt>reject</tt>.
   class UnitOfWork < Struct.new(:id,:queue_name,:queue_timeout,:message,:previous_attempts,:consumer,:decoded_message, :klass)
+    # The time at which this unit of work was created
+    attr_accessor :created_at
+
+    def initialize(*) #:nodoc:
+      super
+      @created_at = Time.now
+    end
+
     # The current attempt number for the worker processing this message.
     def current_attempt
       previous_attempts + 1

data/lib/chore/util.rb

@@ -2,7 +2,7 @@ module Chore
 
   # Collection of utilities and helpers used by Chore internally
   module Util
-    
+
     # To avoid bringing in all of active_support, we implemented constantize here
     def constantize(camel_cased_word)
       names = camel_cased_word.split('::')
@@ -14,5 +14,9 @@ module Chore
       end
       constant
     end
+
+    def procline(str)
+      $0 = str
+    end
   end
 end

data/lib/chore/version.rb

@@ -1,8 +1,8 @@
 module Chore
   module Version #:nodoc:
-    MAJOR = 1
-    MINOR = 8
-    PATCH = 2
+    MAJOR = 3
+    MINOR = 2
+    PATCH = 3
 
     STRING = [ MAJOR, MINOR, PATCH ].join('.')
   end

data/lib/chore/worker.rb

@@ -42,6 +42,28 @@ module Chore
       @started_at + total_timeout
     end
 
+    def duplicate_work?(item)
+      # if we've got a duplicate, remove the message from the queue by not actually running and also not reporting any errors
+      payload = options[:payload_handler].payload(item.decoded_message)
+
+       # if we're hitting the custom dedupe key, we want to remove this message from the queue
+      if item.klass.has_dedupe_lambda?
+        dedupe_key = item.klass.dedupe_key(*payload)
+        if dedupe_key.nil? || dedupe_key.strip.empty? # if the dedupe key is nil, don't continue with the rest of the dedupe lambda logic
+          Chore.logger.info { "#{item.klass} dedupe key nil, skipping memcached lookup." }
+          return false
+        end
+
+        if item.consumer.duplicate_message?(dedupe_key, item.klass, item.queue_timeout)
+          Chore.logger.info { "Found and deleted duplicate job #{item.klass}"}
+          item.consumer.complete(item.id)
+          return true
+        end
+      end
+
+      return false
+    end
+
     # The workhorse. Do the work, all of it. This will block for an entirely unspecified amount
     # of time based on the work to be performed. This will:
     # * Decode each message.
@@ -58,6 +80,10 @@ module Chore
         begin
           item.decoded_message = options[:payload_handler].decode(item.message)
           item.klass = options[:payload_handler].payload_class(item.decoded_message)
+
+          next if duplicate_work?(item)
+
+          Chore.run_hooks_for(:worker_to_start, item)
           start_item(item)
         rescue => e
           Chore.logger.error { "Failed to run job for #{item.message} with error: #{e.message} #{e.backtrace * "\n"}" }
@@ -66,6 +92,7 @@ module Chore
             item.consumer.complete(item.id)
           else
             Chore.run_hooks_for(:on_failure,item.message,e)
+            item.consumer.reject(item.id)
           end
         end
       end
@@ -88,6 +115,7 @@ module Chore
         item.consumer.complete(item.id)
         Chore.logger.info { "Finished job #{klass} with params #{message}"}
         klass.run_hooks_for(:after_perform, message)
+        Chore.run_hooks_for(:worker_ended, item)
       rescue Job::RejectMessageException
         item.consumer.reject(item.id)
         Chore.logger.error { "Failed to run job for #{item.message}  with error: Job raised a RejectMessageException" }
@@ -116,6 +144,7 @@ module Chore
         item.consumer.complete(item.id)
       else
         klass.run_hooks_for(:on_failure, message, e)
+        item.consumer.reject(item.id)
       end
     end
 

data/spec/chore/cli_spec.rb

@@ -201,8 +201,8 @@ describe Chore::CLI do
 
       context 'given no value' do
         let(:command) { [] }
-        it 'is the default value, nil' do
-          subject.should == nil
+        it 'is the default value, 1' do
+          subject.should == 1
         end
       end
     end

data/spec/chore/consumer_spec.rb

@@ -22,10 +22,6 @@ describe Chore::Consumer do
     Chore::Consumer.should respond_to :reset_connection!
   end
 
-  it 'should have a class level cleanup method' do
-    Chore::Consumer.should respond_to :cleanup
-  end
-
   it 'should not have an implemented consume method' do
     expect { consumer.consume }.to raise_error(NotImplementedError)
   end

data/spec/chore/duplicate_detector_spec.rb

@@ -2,7 +2,21 @@ require 'spec_helper'
 require 'securerandom'
 
 describe Chore::DuplicateDetector do
-  let(:memcache) { double("memcache") }
+  class FakeDalli
+    def initialize
+      @store = {}
+    end
+    def add(id, val, ttl=0)
+      if @store[id] && @store[id][:inserted] + @store[id][:ttl] > Time.now.to_i
+        return false
+      else
+        @store[id] = {:val => val, :ttl => ttl, :inserted => Time.now.to_i}
+        return true
+      end
+    end
+  end
+
+  let(:memcache) { FakeDalli.new }
   let(:dupe_on_cache_failure) { false }
   let(:dedupe_params)  { { :memcached_client => memcache, :dupe_on_cache_failure => dupe_on_cache_failure } }
   let(:dedupe) { Chore::DuplicateDetector.new(dedupe_params)}
@@ -15,12 +29,11 @@ describe Chore::DuplicateDetector do
 
   describe "#found_duplicate" do
     it 'should not return true if the message has not already been seen' do
-      expect(memcache).to receive(:add).and_return(true)
       expect(dedupe.found_duplicate?(message_data)).to_not be true
     end
 
     it 'should return true if the message has already been seen' do
-      expect(memcache).to receive(:add).and_return(false)
+      memcache.add(message_data[:id], 1, message_data[:visibility_timeout])
       expect(dedupe.found_duplicate?(message_data)).to be true
     end
 
@@ -34,13 +47,12 @@ describe Chore::DuplicateDetector do
     end
 
     it "should set the timeout to be the queue's " do
-      expect(memcache).to receive(:add).with(id,"1",timeout).and_return(true)
+      expect(memcache).to receive(:add).with(id,"1",timeout).and_call_original
       expect(dedupe.found_duplicate?(message_data)).to be false
     end
 
     it "should call #visibility_timeout once and only once" do
       expect(queue).to receive(:visibility_timeout).once
-      expect(memcache).to receive(:add).at_least(3).times.and_return(true)
       3.times { dedupe.found_duplicate?(message_data) }
     end
 

data/spec/chore/fetcher_spec.rb

@@ -35,15 +35,4 @@ describe Chore::Fetcher do
       fetcher.start
     end
   end
-
-  describe "cleaning up" do
-    before(:each) do
-      manager.stub(:assign)
-    end
-    
-    it "should run cleanup on each queue" do
-      consumer.should_receive(:cleanup).with('test')
-      fetcher.start
-    end
-  end
 end

data/spec/chore/manager_spec.rb

@@ -35,6 +35,13 @@ describe Chore::Manager do
         manager.assign(work)
       end
     end
+
+    describe 'returning work' do
+      it 'should return work to the fetcher' do
+        expect(fetcher).to receive(:return_work).with([work])
+        manager.return_work([work])
+      end
+    end
   end
 
 end

data/spec/chore/queues/filesystem/filesystem_consumer_spec.rb

@@ -8,10 +8,16 @@ describe Chore::Queues::Filesystem::Consumer do
   let(:publisher) { Chore::Queues::Filesystem::Publisher.new }
   let(:test_queues_dir) { "test-queues" }
   let(:test_queue) { "test-queue" }
+  let(:default_timeout) { 60 }
+  let(:timeout) { nil }
 
   before do
     Chore.config.fs_queue_root = test_queues_dir
-    expect(Chore.config).to receive(:default_queue_timeout).and_return(60)
+    if timeout
+      File.open("#{config_dir}/timeout", "w") {|f| f << timeout.to_s}
+    else
+      expect(Chore.config).to receive(:default_queue_timeout).and_return(default_timeout)
+    end
     allow(consumer).to receive(:sleep)
   end
 
@@ -22,35 +28,79 @@ describe Chore::Queues::Filesystem::Consumer do
   let(:test_job_hash) {{:class => "TestClass", :args => "test-args"}}
   let(:new_dir) { described_class.new_dir(test_queue) }
   let(:in_progress_dir) { described_class.in_progress_dir(test_queue) }
+  let(:config_dir) { described_class.config_dir(test_queue) }
 
   describe ".cleanup" do
-    it "should move in_progress jobs to new dir" do
+    it "should move expired in_progress jobs to new dir" do
+      timestamp = Time.now.to_i - 1
+
+      FileUtils.touch("#{in_progress_dir}/foo.1.#{timestamp}.job")
+      described_class.cleanup(Time.now.to_i, new_dir, in_progress_dir)
+      expect(File.exist?("#{new_dir}/foo.2.job")).to eq(true)
+    end
+
+    it "should move non-timestamped jobs from in_progress_dir to new dir" do
       FileUtils.touch("#{in_progress_dir}/foo.1.job")
-      described_class.cleanup(test_queue)
+      described_class.cleanup(Time.now.to_i, new_dir, in_progress_dir)
       expect(File.exist?("#{new_dir}/foo.2.job")).to eq(true)
     end
+
+    it "should not affect non-expired jobs" do
+      timestamp = Time.now.to_i - 1
+
+      FileUtils.touch("#{in_progress_dir}/foo.1.#{timestamp}.job")
+      described_class.cleanup(Time.now.to_i - 2, new_dir, in_progress_dir)
+      expect(File.exist?("#{new_dir}/foo.2.job")).to eq(false)
+    end
   end
 
   describe ".make_in_progress" do
-    it "should move job to in_progress dir" do
+    it "should move non-empty job to in_progress dir" do
+      now = Time.now
+
+      Timecop.freeze(now) do
+        File.open("#{new_dir}/foo.1.job", "w") {|f| f << "{}"}
+        described_class.make_in_progress("foo.1.job", new_dir, in_progress_dir, default_timeout)
+        expect(File.exist?("#{in_progress_dir}/foo.1.#{now.to_i}.job")).to eq(true)
+      end
+    end
+
+    it "should not move empty jobs to in_progress dir" do
+      now = Time.now
+
+      Timecop.freeze(now) do
+        FileUtils.touch("#{new_dir}/foo.1.job")
+        described_class.make_in_progress("foo.1.job", new_dir, in_progress_dir, default_timeout)
+        expect(File.exist?("#{new_dir}/foo.1.job")).to eq(true)
+        expect(File.exist?("#{in_progress_dir}/foo.1.#{now.to_i}.job")).to eq(false)
+      end
+    end
+
+    it "should delete expired empty jobs" do
       FileUtils.touch("#{new_dir}/foo.1.job")
-      described_class.make_in_progress("foo.1.job", new_dir, in_progress_dir)
-      expect(File.exist?("#{in_progress_dir}/foo.1.job")).to eq(true)
+
+      now = Time.now + default_timeout
+      Timecop.freeze(now) do
+        described_class.make_in_progress("foo.1.job", new_dir, in_progress_dir, default_timeout)
+        expect(File.exist?("#{new_dir}/foo.1.job")).to eq(false)
+        expect(File.exist?("#{in_progress_dir}/foo.1.#{now.to_i}.job")).to eq(false)
+      end
     end
   end
 
   describe ".make_new_again" do
     it "should move job to new dir" do
-      FileUtils.touch("#{in_progress_dir}/foo.1.job")
-      described_class.make_new_again("foo.1.job", new_dir, in_progress_dir)
+      timestamp = Time.now.to_i
+      FileUtils.touch("#{in_progress_dir}/foo.1.#{timestamp}.job")
+      described_class.make_new_again("foo.1.#{timestamp}.job", new_dir, in_progress_dir)
       expect(File.exist?("#{new_dir}/foo.2.job")).to eq(true)
     end
   end
 
-  describe ".job_files" do
+  describe ".each_file" do
     it "should list jobs in dir" do
       FileUtils.touch("#{new_dir}/foo.1.job")
-      expect(described_class.job_files(new_dir)).to eq(["foo.1.job"])
+      expect {|b| described_class.each_file(new_dir, &b) }.to yield_with_args("foo.1.job")
     end
   end
 
@@ -85,7 +135,9 @@ describe Chore::Queues::Filesystem::Consumer do
           end
           expect(rejected).to be true
 
-          expect { |b| consumer.consume(&b) }.to yield_with_args(anything, 'test-queue', 60, test_job_hash.to_json, 1)
+          Timecop.freeze(Time.now + 61) do
+            expect { |b| consumer.consume(&b) }.to yield_with_args(anything, 'test-queue', 60, test_job_hash.to_json, 1)
+          end
         end
       end
 
@@ -103,6 +155,14 @@ describe Chore::Queues::Filesystem::Consumer do
           expect { |b| consumer.consume(&b) }.to_not yield_control
         end
       end
+
+      context "with queue-specific timeout config" do
+        let(:timeout) { 30 }
+
+        it "should consume a published job and yield the job to the handler block" do
+          expect { |b| consumer.consume(&b) }.to yield_with_args(anything, 'test-queue', 30, test_job_hash.to_json, 0)
+        end
+      end
     end
 
     context "not finding a published job" do