chore-core 1.8.2 → 3.2.3

data/spec/chore/strategies/worker/preforked_worker_strategy_spec.rb

@@ -0,0 +1,183 @@
+require 'spec_helper'
+
+describe Chore::Strategy::PreForkedWorkerStrategy do
+  let(:manager) { double('manager') }
+  let(:socket)  { double('socket') }
+  let(:pipe)    { double('pipe') }
+  let(:worker)  { double('worker') }
+  let(:worker_manager)   { Chore::Strategy::WorkerManager.new(socket) }
+  let(:strategy)         { Chore::Strategy::PreForkedWorkerStrategy.new(manager) }
+  let(:work_distributor) { Chore::Strategy::WorkDistributor }
+
+  before(:each) do
+    allow_any_instance_of(Chore::Strategy::PreForkedWorkerStrategy).to receive(:trap_signals).and_return(true)
+    strategy.instance_variable_set(:@worker_manager, worker_manager)
+  end
+
+  context '#start' do
+    it 'should create, attach workers and start the worker manager' do
+      allow(worker_manager).to receive(:create_and_attach_workers)
+      allow(strategy).to receive(:worker_assignment_thread).and_return(true)
+
+      expect(worker_manager).to receive(:create_and_attach_workers)
+      expect(strategy).to receive(:worker_assignment_thread)
+      strategy.start
+    end
+  end
+
+  context '#stop!' do
+    it 'should set the system\'s running state to false' do
+      strategy.instance_variable_set(:@running, true)
+      strategy.stop!
+      expect(strategy.instance_variable_get(:@running)).to eq false
+    end
+  end
+
+  context '#worker_assignment_thread' do
+    before(:each) do
+      allow(Thread).to receive(:new).and_return(true)
+    end
+
+    it 'create a new thread with the worker_assignment_loop' do
+      allow(strategy).to receive(:worker_assignment_loop).and_return(true)
+      expect(Thread).to receive(:new).once.and_yield
+      expect(strategy).to receive(:worker_assignment_loop)
+      expect(Process).to receive(:exit)
+      strategy.send(:worker_assignment_thread)
+    end
+
+    it 'rescues a \'TerribleMistake\' exception and performs a shutdown of chore' do
+      allow(strategy).to receive(:worker_assignment_loop).and_raise(Chore::TerribleMistake)
+      allow(manager).to receive(:shutdown!)
+      allow(Thread).to receive(:new).and_yield
+      expect(Process).to receive(:exit)
+      strategy.send(:worker_assignment_thread)
+    end
+  end
+
+  context '#worker_assignment_loop' do
+    before(:each) do
+      allow(socket).to receive(:eof?).and_return(false)
+
+      allow(strategy).to receive(:running?).and_return(true, false)
+      strategy.instance_variable_set(:@self_read, pipe)
+      allow(strategy).to receive(:select_sockets).and_return([[socket], nil, nil])
+      allow(strategy).to receive(:handle_self_pipe_signal).and_return(true)
+      allow(strategy).to receive(:fetch_and_assign_jobs).and_return(true)
+
+      allow(worker_manager).to receive(:worker_sockets).and_return([socket])
+      allow(worker_manager).to receive(:ready_workers).and_yield([worker])
+      allow(worker_manager).to receive(:destroy_expired!)
+
+      allow(work_distributor).to receive(:fetch_and_assign_jobs)
+    end
+
+    it 'should terminate when @running is set to false' do
+      allow(strategy).to receive(:running?).and_return(false)
+      expect(strategy).to receive(:select_sockets).exactly(0).times
+      strategy.send(:worker_assignment_loop)
+    end
+
+    it 'should get the worker_sockets' do
+      expect(worker_manager).to receive(:worker_sockets)
+      strategy.send(:worker_assignment_loop)
+    end
+
+    it 'should handle no sockets being ready' do
+      allow(strategy).to receive(:select_sockets).and_return(nil)
+      expect(strategy).to receive(:select_sockets).once
+      strategy.send(:worker_assignment_loop)
+    end
+
+    it 'should handle signals if alerted on a self pipe' do
+      allow(strategy).to receive(:select_sockets).and_return([[pipe], nil, nil])
+      expect(strategy).to receive(:handle_signal).once
+      strategy.send(:worker_assignment_loop)
+    end
+
+    it 'should check if sockets have hit EOF' do
+      allow(strategy).to receive(:handle_signal)
+      expect(socket).to receive(:eof?).once.and_return(false)
+      strategy.send(:worker_assignment_loop)
+    end
+
+    it 'should not assign jobs if sockets have hit EOF' do
+      allow(strategy).to receive(:handle_signal)
+      allow(socket).to receive(:eof?).and_return(true)
+      expect(work_distributor).to_not receive(:fetch_and_assign_jobs)
+      strategy.send(:worker_assignment_loop)
+    end
+
+    it 'should handle fetch and assign jobs when workers are ready' do
+      expect(work_distributor).to receive(:fetch_and_assign_jobs).with([worker], manager).once
+      strategy.send(:worker_assignment_loop)
+    end
+  end
+
+  context '#handle_signal' do
+    before(:each) do
+      strategy.instance_variable_set(:@self_read, pipe)
+      allow(pipe).to receive(:read_nonblock).and_return(nil)
+
+      allow(worker_manager).to receive(:respawn_terminated_workers!).and_return(true)
+      allow(worker_manager).to receive(:stop_workers).and_return(true)
+      allow(manager).to receive(:shutdown!).and_return(true)
+    end
+
+    it 'should respawn terminated workers in the event of a SIGCHLD' do
+      allow(pipe).to receive(:read_nonblock).and_return('1')
+      expect(worker_manager).to receive(:respawn_terminated_workers!).once
+
+      strategy.send(:handle_signal)
+    end
+
+    it 'should signal its children, and shutdown in the event of one of INT, QUIT or TERM signals' do
+      allow(pipe).to receive(:read_nonblock).and_return('2', '3', '4')
+      expect(worker_manager).to receive(:stop_workers).exactly(3).times
+      expect(manager).to receive(:shutdown!).exactly(3).times
+      3.times do
+        strategy.send(:handle_signal)
+      end
+    end
+
+    it 'should propagte the signal it receives to its children' do
+      allow(pipe).to receive(:read_nonblock).and_return('3')
+      expect(worker_manager).to receive(:stop_workers).with(:QUIT)
+      strategy.send(:handle_signal)
+    end
+
+    it 'should reset logs on a USR1 signal' do
+      allow(pipe).to receive(:read_nonblock).and_return('5')
+      expect(Chore).to receive(:reopen_logs)
+      strategy.send(:handle_signal)
+    end
+
+    it 'should not preform any task when an unhandled signal is called' do
+      allow(pipe).to receive(:read_nonblock).and_return('9')
+      expect(worker_manager).to receive(:respawn_terminated_workers!).exactly(0).times
+      expect(worker_manager).to receive(:stop_workers).exactly(0).times
+      expect(manager).to receive(:shutdown!).exactly(0).times
+      strategy.send(:handle_signal)
+    end
+  end
+
+  context '#trap_signals' do
+    before(:each) do
+      allow_any_instance_of(Chore::Strategy::PreForkedWorkerStrategy).to receive(:trap_signals).and_call_original
+    end
+
+    let(:signals) { { '1' => 'QUIT' } }
+
+    it 'should reset signals' do
+      allow(Chore::Signal).to receive(:reset)
+      expect(Chore::Signal).to receive(:reset)
+      strategy.send(:trap_signals, {}, pipe)
+    end
+
+    it 'should trap the signals passed to it' do
+      allow(Chore::Signal).to receive(:reset)
+      expect(Chore::Signal).to receive(:trap).with('QUIT').once
+      strategy.send(:trap_signals, signals, pipe)
+    end
+  end
+end

data/spec/chore/strategies/worker/single_worker_strategy_spec.rb

@@ -2,6 +2,8 @@ require 'spec_helper'
 
 describe Chore::Strategy::SingleWorkerStrategy do
   let(:manager) { double('Manager') }
+  let(:job_timeout) { 60 }
+  let(:job) { Chore::UnitOfWork.new(SecureRandom.uuid, 'test', job_timeout, Chore::Encoder::JsonEncoder.encode(TestJob.job_hash([1,2,"3"])), 0) }
   subject       { described_class.new(manager) }
 
   describe '#stop!' do
@@ -31,4 +33,27 @@ describe Chore::Strategy::SingleWorkerStrategy do
     end
   end
 
+  describe '#assign' do
+    let(:worker) { double('Worker', start: nil) }
+
+    it 'starts a new worker' do
+      expect(Chore::Worker).to receive(:new).with(job, {}).and_return(worker)
+      subject.assign(job)
+    end
+
+    it 'can be called multiple times' do
+      expect(Chore::Worker).to receive(:new).twice.with(job, {}).and_return(worker)
+      2.times { subject.assign(job) }
+    end
+
+    it 'should release the worker if an exception occurs' do
+      allow_any_instance_of(Chore::Worker).to receive(:start).and_raise(ArgumentError)
+      expect(subject).to receive(:release_worker)
+
+      begin
+        subject.assign(job)
+      rescue ArgumentError
+      end
+    end
+  end
 end

data/spec/chore/worker_spec.rb

@@ -2,6 +2,10 @@ require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
 
 describe Chore::Worker do
 
+  before(:each) do
+    allow(consumer).to receive(:duplicate_message?).and_return(false)
+  end
+
   class SimpleJob
     include Chore::Job
     queue_options :name => 'test',
@@ -13,7 +17,31 @@ describe Chore::Worker do
     end
   end
 
-  let(:consumer) { double('consumer', :complete => nil) }
+  class SimpleDedupeJob
+    include Chore::Job
+    queue_options :name => 'dedupe_test',
+      :publisher => FakePublisher,
+      :max_attempts => 100,
+      :dedupe_lambda => lambda { |first, second, third| first }
+    
+    def perform(first, second, third)
+      return second
+    end
+  end
+
+  class InvalidDedupeJob
+    include Chore::Job
+    queue_options :name => 'invalid_dedupe_test',
+      :publisher => FakePublisher,
+      :max_attempts => 100,
+      :dedupe_lambda => lambda { |first, second, third| first }
+    
+    def perform(first, second)
+      return second
+    end
+  end
+
+  let(:consumer) { double('consumer', :complete => nil, :reject => nil) }
   let(:job_args) { [1,2,'3'] }
   let(:job) { SimpleJob.job_hash(job_args) }
 
@@ -40,6 +68,33 @@ describe Chore::Worker do
       consumer.should_receive(:complete).exactly(10).times
       Chore::Worker.start(work, {:payload_handler => payload_handler})
     end
+
+    context 'when the job has a dedupe_lambda defined' do
+      context 'when the value being deduped on is unique' do
+        let(:job_args) { [rand,2,'3'] }
+        let(:encoded_job) { Chore::Encoder::JsonEncoder.encode(job) }
+        let(:job) { SimpleDedupeJob.job_hash(job_args) }
+        it 'should call complete for each unique value' do
+          allow(consumer).to receive(:duplicate_message?).and_return(false)
+          work = []
+          work << Chore::UnitOfWork.new(1, 'dedupe_test', 60, Chore::Encoder::JsonEncoder.encode(SimpleDedupeJob.job_hash([rand,2,'3'])), 0, consumer)
+          SimpleDedupeJob.should_receive(:perform).exactly(1).times
+          consumer.should_receive(:complete).exactly(1).times
+          Chore::Worker.start(work, {:payload_handler => payload_handler})
+        end
+      end
+
+      context 'when the dedupe lambda does not take the same number of arguments as perform' do
+        it 'should raise an error and not complete the job' do
+          work = []
+          work << Chore::UnitOfWork.new(1, 'invalid_dedupe_test', 60, Chore::Encoder::JsonEncoder.encode(InvalidDedupeJob.job_hash([rand,2,'3'])), 0, consumer)
+          work << Chore::UnitOfWork.new(2, 'invalid_dedupe_test', 60, Chore::Encoder::JsonEncoder.encode(InvalidDedupeJob.job_hash([rand,2,'3'])), 0, consumer)
+          work << Chore::UnitOfWork.new(1, 'invalid_dedupe_test', 60, Chore::Encoder::JsonEncoder.encode(InvalidDedupeJob.job_hash([rand,2,'3'])), 0, consumer)
+          consumer.should_not_receive(:complete)
+          Chore::Worker.start(work, {:payload_handler => payload_handler})
+        end
+      end
+    end
   end
 
   describe "with default payload handler" do
@@ -91,6 +146,12 @@ describe Chore::Worker do
         Chore::Worker.start(work)
       end
 
+      it 'should reject job' do
+        work = Chore::UnitOfWork.new(2,'test',60,job,0,consumer)
+        consumer.should_receive(:reject).with(2)
+        Chore::Worker.start(work)
+      end
+
       context 'more than the maximum allowed times' do
         before(:each) do
           Chore.config.stub(:max_attempts).and_return(10)
@@ -127,6 +188,13 @@ describe Chore::Worker do
         Chore::Worker.start(work)
       end
 
+      it 'should reject job' do
+        work = Chore::UnitOfWork.new(2,'test',60,encoded_job,0,consumer)
+        consumer.should_receive(:reject).with(2)
+
+        Chore::Worker.start(work)
+      end
+
       context 'more than the maximum allowed times' do
         it 'should permanently fail' do
           work = Chore::UnitOfWork.new(2,'test',60,encoded_job,999,consumer)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: chore-core
 version: !ruby/object:Gem::Version
-  version: 1.8.2
+  version: 3.2.3
 platform: ruby
 authors:
 - Tapjoy
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-02-02 00:00:00.000000000 Z
+date: 2020-09-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: json
@@ -58,6 +58,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.1.3
+- !ruby/object:Gem::Dependency
+  name: get_process_mem
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.0
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -137,7 +151,15 @@ files:
 - lib/chore/strategies/consumer/batcher.rb
 - lib/chore/strategies/consumer/single_consumer_strategy.rb
 - lib/chore/strategies/consumer/threaded_consumer_strategy.rb
+- lib/chore/strategies/consumer/throttled_consumer_strategy.rb
 - lib/chore/strategies/worker/forked_worker_strategy.rb
+- lib/chore/strategies/worker/helpers/ipc.rb
+- lib/chore/strategies/worker/helpers/preforked_worker.rb
+- lib/chore/strategies/worker/helpers/work_distributor.rb
+- lib/chore/strategies/worker/helpers/worker_info.rb
+- lib/chore/strategies/worker/helpers/worker_killer.rb
+- lib/chore/strategies/worker/helpers/worker_manager.rb
+- lib/chore/strategies/worker/preforked_worker_strategy.rb
 - lib/chore/strategies/worker/single_worker_strategy.rb
 - lib/chore/tasks/queues.task
 - lib/chore/unit_of_work.rb
@@ -160,7 +182,15 @@ files:
 - spec/chore/strategies/consumer/batcher_spec.rb
 - spec/chore/strategies/consumer/single_consumer_strategy_spec.rb
 - spec/chore/strategies/consumer/threaded_consumer_strategy_spec.rb
+- spec/chore/strategies/consumer/throttled_consumer_strategy_spec.rb
 - spec/chore/strategies/worker/forked_worker_strategy_spec.rb
+- spec/chore/strategies/worker/helpers/ipc_spec.rb
+- spec/chore/strategies/worker/helpers/preforked_worker_spec.rb
+- spec/chore/strategies/worker/helpers/work_distributor_spec.rb
+- spec/chore/strategies/worker/helpers/worker_info_spec.rb
+- spec/chore/strategies/worker/helpers/worker_killer_spec.rb
+- spec/chore/strategies/worker/helpers/worker_manager_spec.rb
+- spec/chore/strategies/worker/preforked_worker_strategy_spec.rb
 - spec/chore/strategies/worker/single_worker_strategy_spec.rb
 - spec/chore/worker_spec.rb
 - spec/chore_spec.rb
@@ -185,10 +215,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.4.5
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: Job processing... for the future!
 test_files: []
-has_rdoc: