chore-core 1.10.0 → 4.0.0

data/spec/chore/strategies/worker/helpers/worker_info_spec.rb

@@ -0,0 +1,14 @@
+require 'spec_helper'
+
+describe Chore::Strategy::WorkerInfo do
+  let(:pid)         { 1 }
+  let(:worker_info) { Chore::Strategy::WorkerInfo.new(pid) }
+
+  context '#initialize' do
+    it 'should initialize the WorkerInfo with a pid' do
+      wi = Chore::Strategy::WorkerInfo.new(pid)
+      expect(wi.pid).to equal(pid)
+      expect(wi.socket).to equal(nil)
+    end
+  end
+end

data/spec/chore/strategies/worker/helpers/worker_killer_spec.rb

@@ -0,0 +1,97 @@
+require 'spec_helper'
+require 'get_process_mem'
+
+describe Chore::Strategy::WorkerKiller do
+  let(:memory_limit)    { 1024 }
+  let(:request_limit)   { 100 }
+  let(:check_cycle)     { 16 }
+  let(:worker_killer)   { Chore::Strategy::WorkerKiller.new }
+  let(:process_mem_obj) { double('process_mem_obj', bytes: 10) }
+
+  context '#initialize' do
+    it 'should initialize the WorkerKiller correctly' do
+      allow(Chore.config).to receive(:memory_limit_bytes).and_return(memory_limit)
+      allow(Chore.config).to receive(:request_limit).and_return(request_limit)
+      allow(Chore.config).to receive(:check_cycle).and_return(check_cycle)
+
+      wk = Chore::Strategy::WorkerKiller.new
+      expect(wk.instance_variable_get(:@memory_limit)).to equal(memory_limit)
+      expect(wk.instance_variable_get(:@request_limit)).to equal(request_limit)
+      expect(wk.instance_variable_get(:@check_cycle)).to equal(check_cycle)
+      expect(wk.instance_variable_get(:@check_count)).to equal(0)
+      expect(wk.instance_variable_get(:@current_requests)).to equal(0)
+    end
+  end
+
+  context '#check_memory' do
+    before(:each) do
+      allow(GetProcessMem).to receive(:new).and_return(process_mem_obj)
+      worker_killer.instance_variable_set(:@memory_limit, memory_limit)
+      worker_killer.instance_variable_set(:@check_cycle, check_cycle)
+    end
+
+    it 'should return nil when memory_limit is nil' do
+      worker_killer.instance_variable_set(:@memory_limit, nil)
+      expect(worker_killer.check_memory).to eq(nil)
+    end
+
+    it 'should increment the check count by 1' do
+      worker_killer.instance_variable_set(:@check_count, 1)
+      worker_killer.check_memory
+      expect(worker_killer.instance_variable_get(:@check_count)).to eq(2)
+    end
+
+    context 'check_count equals check_cycle' do
+      before(:each) do
+        worker_killer.instance_variable_set(:@check_count, 15)
+      end
+
+      it 'should check memory' do
+        expect(process_mem_obj).to receive(:bytes)
+        worker_killer.check_memory
+      end
+
+      it 'should reset the check_count to zero' do
+        allow(process_mem_obj).to receive(:bytes).and_return(0)
+        worker_killer.check_memory
+        expect(worker_killer.instance_variable_get(:@check_count)).to equal(0)
+      end
+
+      it 'should exit if the process mem exceeds the memory_limit' do
+        allow(process_mem_obj).to receive(:bytes).and_return(2048)
+        begin
+          worker_killer.check_memory
+        rescue SystemExit=>e
+          expect(e.status).to eq(0)
+        end
+      end
+    end
+  end
+
+  context '#check_requests' do
+    before(:each) do
+      worker_killer.instance_variable_set(:@request_limit, request_limit)
+      worker_killer.instance_variable_set(:@current_requests, 0)
+    end
+
+    it 'should return nil when request_limit is nil' do
+      worker_killer.instance_variable_set(:@request_limit, nil)
+      expect(worker_killer.check_requests).to eq(nil)
+    end
+
+    it 'should increment current requests' do
+      worker_killer.check_requests
+      expect(worker_killer.instance_variable_get(:@current_requests)).to eq(1)
+    end
+
+    it 'should exit when current_requests exceeds request_limit' do
+      worker_killer.instance_variable_set(:@current_requests, request_limit - 1)
+
+      begin
+        worker_killer.check_requests
+      rescue SystemExit=>e
+        expect(e.status).to eq(0)
+      end
+    end
+  end
+end

data/spec/chore/strategies/worker/helpers/worker_manager_spec.rb

@@ -0,0 +1,304 @@
+require 'spec_helper'
+require 'chore/signal'
+require 'chore/strategies/worker/helpers/worker_manager'
+
+describe Chore::Strategy::WorkerManager do
+  let(:master_socket) { double('master_socket') }
+
+  let(:worker_manager) { Chore::Strategy::WorkerManager.new(master_socket) }
+
+
+  let(:worker_pid_1) { 1 }
+  let(:socket_1) { double('socket_1') }
+
+  let(:worker_pid_2) { 2 }
+  let(:socket_2) { double('socket_2') }
+
+  let(:worker_pid_3) { 3 }
+
+  let(:pid_sock_hash) { { worker_pid_1 => socket_1,
+                          worker_pid_2 => socket_2 } }
+  let(:pid_sock_hash_2) { { worker_pid_1 => socket_1 } }
+
+  let(:pid_sock_hash_3) { { worker_pid_1 => socket_1,
+                          worker_pid_2 => socket_2,
+                          worker_pid_3 => nil } }
+
+  let(:config) { double("config") }
+  let(:worker) { double("worker") }
+
+  let(:worker_info_1) { double('worker_info_1') }
+  let(:worker_info_2) { double('worker_info_2') }
+
+  let(:pid_to_worker) { {
+                          worker_pid_1 => worker_info_1,
+                          worker_pid_2 => worker_info_2
+                      } }
+  let(:socket_to_worker) { {
+                          socket_1 => worker_info_1,
+                          socket_2 => worker_info_2
+                      } }
+
+  context '#include_ipc' do
+    it 'should include the Ipc module' do
+      expect(worker_manager.ipc_help).to eq(:available)
+    end
+  end
+
+  context '#create_and_attach_workers' do
+    before(:each) do
+      allow(worker_manager).to receive(:create_workers).and_yield(2)
+      allow(worker_manager).to receive(:attach_workers).and_return(true)
+    end
+
+    it 'should call to create replacement workers' do
+      expect(worker_manager).to receive(:create_workers)
+      worker_manager.create_and_attach_workers
+    end
+
+    it 'should create a map between new workers to new sockets' do
+      expect(worker_manager).to receive(:attach_workers).with(2)
+      worker_manager.create_and_attach_workers
+    end
+  end
+
+  context '#respawn_terminated_workers!' do
+    before(:each) do
+      allow(worker_manager).to receive(:create_and_attach_workers).and_return(true)
+      allow(worker_manager).to receive(:reap_workers).and_return(true)
+    end
+
+    it 'should reap all the terminated worker processes' do
+      expect(worker_manager).to receive(:reap_workers)
+      worker_manager.respawn_terminated_workers!
+    end
+
+    it 'should re-create and attach all the workers that died' do
+      expect(worker_manager).to receive(:create_and_attach_workers)
+      worker_manager.respawn_terminated_workers!
+    end
+  end
+
+  context '#stop_workers' do
+    let(:signal) { 'TERM' }
+    before(:each) do
+      allow(Process).to receive(:kill).and_return(nil)
+      allow(worker_manager).to receive(:reap_workers)
+      worker_manager.instance_variable_set(:@pid_to_worker, pid_sock_hash)
+    end
+
+    it 'should forward the signal received to each of the child processes' do
+      pid_sock_hash.each do |pid, sock|
+        expect(Process).to receive(:kill).with(signal, pid)
+      end
+      worker_manager.stop_workers(signal)
+    end
+
+    it 'should reap all terminated child processes' do
+      expect(worker_manager).to receive(:reap_workers)
+      worker_manager.stop_workers(signal)
+    end
+  end
+
+  context 'worker_sockets' do
+    it 'should return a list of socket assoicated with workers' do
+      worker_manager.instance_variable_set(:@socket_to_worker, socket_to_worker)
+      res = worker_manager.worker_sockets
+      expect(res).to eq([socket_1, socket_2])
+    end
+  end
+
+  context '#ready_workers' do
+    it 'should return a list of workers assoicated with given sockets' do
+      worker_manager.instance_variable_set(:@socket_to_worker, socket_to_worker)
+      allow(worker_info_1).to receive(:reset_start_time!)
+      res = worker_manager.ready_workers([socket_1])
+      expect(res).to eq([worker_info_1])
+    end
+
+    it 'should yield when a block is passed to it' do
+      worker_manager.instance_variable_set(:@socket_to_worker, socket_to_worker)
+      allow(worker_info_1).to receive(:reset_start_time!)
+      expect{ |b| worker_manager.ready_workers([socket_1], &b) }.to yield_control
+    end
+  end
+
+  context '#create_workers' do
+    before(:each) do
+      allow(worker_manager).to receive(:fork).and_yield
+      allow(worker_manager).to receive(:run_worker_instance)
+      allow(Chore).to receive(:config).and_return(config)
+    end
+
+    it 'should fork it running process till we have the right optimized number of workers and return the number of workers it created' do
+      allow(config).to receive(:num_workers).and_return(1)
+      expect(worker_manager).to receive(:fork).once
+      expect(worker_manager).to receive(:run_worker_instance).once
+      worker_manager.send(:create_workers)
+    end
+
+    it 'should raise an exception if an inconsistent number of workers are created' do
+      allow(config).to receive(:num_workers).and_return(0)
+      allow(worker_manager).to receive(:inconsistent_worker_number).and_return(true)
+      expect(worker_manager).to receive(:inconsistent_worker_number)
+      expect { worker_manager.send(:create_workers) }.to raise_error(RuntimeError)
+    end
+
+    it 'should call the block passed to it with the number of workers it created' do
+      allow(config).to receive(:num_workers).and_return(0)
+      allow(worker_manager).to receive(:inconsistent_worker_number).and_return(false)
+      expect { |b| worker_manager.send(:create_workers, &b) }.to yield_control.once
+    end
+  end
+
+  context '#inconsistent_worker_number' do
+    it 'should check if the worker numbers match the number configured' do
+      allow(Chore).to receive(:config).and_return(config)
+      allow(config).to receive(:num_workers).and_return(2)
+      worker_manager.instance_variable_set(:@pid_to_worker, pid_to_worker)
+      res_false = worker_manager.send(:inconsistent_worker_number)
+      allow(config).to receive(:num_workers).and_return(4)
+      res_true = worker_manager.send(:inconsistent_worker_number)
+      expect(res_true).to be(true)
+      expect(res_false).to be(false)
+    end
+  end
+
+  context '#run_worker_instance' do
+    before(:each) do
+      allow(Chore::Strategy::PreforkedWorker).to receive(:new).and_return(worker)
+      allow(worker).to receive(:start_worker).and_return(true)
+      allow(worker_manager).to receive(:exit).and_return(true)
+    end
+
+    it 'should create a PreforkedWorker object and start it' do
+      expect(Chore::Strategy::PreforkedWorker).to receive(:new).and_return(worker)
+      expect(worker).to receive(:start_worker)
+      worker_manager.send(:run_worker_instance)
+    end
+
+    it 'should ensure that the process exits when the PreforkedWorker completes' do
+      expect(worker_manager).to receive(:exit).with(true)
+      worker_manager.send(:run_worker_instance)
+    end
+  end
+
+  context '#attach_workers' do
+    before(:each) do
+      allow(worker_manager).to receive(:create_worker_sockets).and_return([socket_1, socket_2])
+      allow(worker_manager).to receive(:read_msg).and_return(worker_pid_1, worker_pid_2)
+      allow(worker_manager).to receive(:kill_unattached_workers).and_return(true)
+      worker_manager.instance_variable_set(:@pid_to_worker, pid_to_worker)
+      worker_manager.instance_variable_set(:@socket_to_worker, {})
+      allow(worker_info_1).to receive(:socket=)
+      allow(worker_info_2).to receive(:socket=)
+      allow(worker_manager).to receive(:select_sockets).and_return([[socket_1], [], []], [[socket_2], [], []])
+    end
+
+    it 'should add as many sockets as the number passed to it as a param' do
+      expect(worker_manager).to receive(:read_msg).twice
+      worker_manager.send(:attach_workers, 2)
+    end
+
+    it 'should select on each socket to make sure its readable' do
+      expect(worker_manager).to receive(:select_sockets).twice
+      worker_manager.send(:attach_workers, 2)
+    end
+
+    it 'should get the PID from each socket it creates and map that to the worker that it is connected to' do
+      worker_manager.send(:attach_workers, 2)
+      expect(worker_manager.instance_variable_get(:@socket_to_worker)).to eq(socket_to_worker)
+    end
+
+    it 'should kill any unattached workers' do
+      expect(worker_manager).to receive(:kill_unattached_workers)
+      worker_manager.send(:attach_workers, 2)
+    end
+
+    it 'should close sockets that failed to get a connection by timing out' do
+      allow(worker_manager).to receive(:select_sockets).and_return([[socket_1], [], []], [nil, nil, nil])
+      expect(socket_1).not_to receive(:close)
+      expect(socket_2).to receive(:close)
+      worker_manager.send(:attach_workers, 2)
+    end
+
+    it 'should close sockets that failed to get a connection by econnreset' do
+      allow(worker_manager).to receive(:select_sockets).with(socket_1, nil, 2).and_return([[socket_1], [], []])
+      allow(worker_manager).to receive(:select_sockets).with(socket_2, nil, 2).and_raise(Errno::ECONNRESET)
+      expect(socket_1).not_to receive(:close)
+      expect(socket_2).to receive(:close)
+      worker_manager.send(:attach_workers, 2)
+    end
+  end
+
+  context "#create_worker_sockets" do
+    it 'should return an array of sockets equal to the number passed to it' do
+      allow(worker_manager).to receive(:add_worker_socket).and_return(socket_1, socket_2, socket_2, socket_1)
+      num = 3
+      res = worker_manager.send(:create_worker_sockets, num)
+      expect(res.size).to eq(num)
+    end
+  end
+
+  context "#kill_unattached_workers" do
+    it 'should send a kill -9 to PIDs that do not have a socket attached to them' do
+      worker_manager.instance_variable_set(:@pid_to_worker, pid_to_worker)
+      allow(worker_info_1).to receive(:socket).and_return(socket_1)
+      allow(worker_info_2).to receive(:socket).and_return(nil)
+      allow(Process).to receive(:kill).and_return(true)
+      expect(Process).to receive(:kill).with('KILL', worker_pid_2).once
+      worker_manager.send(:kill_unattached_workers)
+    end
+  end
+
+  context "#reap_workers" do
+    before(:each) do
+      worker_manager.instance_variable_set(:@pid_to_worker, pid_to_worker)
+      allow(worker_info_1).to receive(:socket).and_return(socket_1)
+      allow(worker_info_1).to receive(:pid).and_return(worker_pid_1)
+      allow(worker_info_2).to receive(:socket).and_return(socket_2)
+      allow(worker_info_2).to receive(:pid).and_return(worker_pid_2)
+    end
+
+    it 'should run through each worker-pid map and delete the worker pids from the hash, if they were dead' do
+      allow(worker_manager).to receive(:reap_process).and_return(false, true)
+      allow(socket_2).to receive(:close)
+      expect(worker_manager.instance_variable_get(:@pid_to_worker).size).to eq(2)
+      worker_manager.send(:reap_workers)
+      expect(worker_manager.instance_variable_get(:@pid_to_worker).size).to eq(1)
+    end
+
+    it 'closes the socket associated with the reaped worker' do
+      allow(worker_manager).to receive(:reap_process).and_return(false, true)
+      expect(socket_2).to receive(:close)
+      worker_manager.send(:reap_workers)
+    end
+
+    it 'should not change the worker pids map, if all the childern are running' do
+      allow(worker_manager).to receive(:reap_process).and_return(false, false)
+      expect(worker_manager.instance_variable_get(:@pid_to_worker).size).to eq(2)
+      worker_manager.send(:reap_workers)
+      expect(worker_manager.instance_variable_get(:@pid_to_worker).size).to eq(2)
+    end
+  end
+
+  context "#reap_worker" do
+    it 'should return true if the pid was dead' do
+      allow(Process).to receive(:wait).and_return(worker_pid_1)
+      res = worker_manager.send(:reap_process, worker_pid_1)
+      expect(res).to eq(true)
+    end
+
+    it 'should return false if the pid was running' do
+      allow(Process).to receive(:wait).and_return(nil)
+      res = worker_manager.send(:reap_process, worker_pid_1)
+      expect(res).to eq(false)
+    end
+
+    it 'should return true if the pid was dead' do
+     allow(Process).to receive(:wait).and_raise(Errno::ECHILD)
+     res = worker_manager.send(:reap_process, worker_pid_1)
+     expect(res).to eq(true)
+   end
+ end
+end

data/spec/chore/strategies/worker/preforked_worker_strategy_spec.rb

@@ -0,0 +1,183 @@
+require 'spec_helper'
+
+describe Chore::Strategy::PreForkedWorkerStrategy do
+  let(:manager) { double('manager') }
+  let(:socket)  { double('socket') }
+  let(:pipe)    { double('pipe') }
+  let(:worker)  { double('worker') }
+  let(:worker_manager)   { Chore::Strategy::WorkerManager.new(socket) }
+  let(:strategy)         { Chore::Strategy::PreForkedWorkerStrategy.new(manager) }
+  let(:work_distributor) { Chore::Strategy::WorkDistributor }
+
+  before(:each) do
+    allow_any_instance_of(Chore::Strategy::PreForkedWorkerStrategy).to receive(:trap_signals).and_return(true)
+    strategy.instance_variable_set(:@worker_manager, worker_manager)
+  end
+
+  context '#start' do
+    it 'should create, attach workers and start the worker manager' do
+      allow(worker_manager).to receive(:create_and_attach_workers)
+      allow(strategy).to receive(:worker_assignment_thread).and_return(true)
+
+      expect(worker_manager).to receive(:create_and_attach_workers)
+      expect(strategy).to receive(:worker_assignment_thread)
+      strategy.start
+    end
+  end
+
+  context '#stop!' do
+    it 'should set the system\'s running state to false' do
+      strategy.instance_variable_set(:@running, true)
+      strategy.stop!
+      expect(strategy.instance_variable_get(:@running)).to eq false
+    end
+  end
+
+  context '#worker_assignment_thread' do
+    before(:each) do
+      allow(Thread).to receive(:new).and_return(true)
+    end
+
+    it 'create a new thread with the worker_assignment_loop' do
+      allow(strategy).to receive(:worker_assignment_loop).and_return(true)
+      expect(Thread).to receive(:new).once.and_yield
+      expect(strategy).to receive(:worker_assignment_loop)
+      expect(Process).to receive(:exit)
+      strategy.send(:worker_assignment_thread)
+    end
+
+    it 'rescues a \'TerribleMistake\' exception and performs a shutdown of chore' do
+      allow(strategy).to receive(:worker_assignment_loop).and_raise(Chore::TerribleMistake)
+      allow(manager).to receive(:shutdown!)
+      allow(Thread).to receive(:new).and_yield
+      expect(Process).to receive(:exit)
+      strategy.send(:worker_assignment_thread)
+    end
+  end
+
+  context '#worker_assignment_loop' do
+    before(:each) do
+      allow(socket).to receive(:eof?).and_return(false)
+
+      allow(strategy).to receive(:running?).and_return(true, false)
+      strategy.instance_variable_set(:@self_read, pipe)
+      allow(strategy).to receive(:select_sockets).and_return([[socket], nil, nil])
+      allow(strategy).to receive(:handle_self_pipe_signal).and_return(true)
+      allow(strategy).to receive(:fetch_and_assign_jobs).and_return(true)
+
+      allow(worker_manager).to receive(:worker_sockets).and_return([socket])
+      allow(worker_manager).to receive(:ready_workers).and_yield([worker])
+      allow(worker_manager).to receive(:destroy_expired!)
+
+      allow(work_distributor).to receive(:fetch_and_assign_jobs)
+    end
+
+    it 'should terminate when @running is set to false' do
+      allow(strategy).to receive(:running?).and_return(false)
+      expect(strategy).to receive(:select_sockets).exactly(0).times
+      strategy.send(:worker_assignment_loop)
+    end
+
+    it 'should get the worker_sockets' do
+      expect(worker_manager).to receive(:worker_sockets)
+      strategy.send(:worker_assignment_loop)
+    end
+
+    it 'should handle no sockets being ready' do
+      allow(strategy).to receive(:select_sockets).and_return(nil)
+      expect(strategy).to receive(:select_sockets).once
+      strategy.send(:worker_assignment_loop)
+    end
+
+    it 'should handle signals if alerted on a self pipe' do
+      allow(strategy).to receive(:select_sockets).and_return([[pipe], nil, nil])
+      expect(strategy).to receive(:handle_signal).once
+      strategy.send(:worker_assignment_loop)
+    end
+
+    it 'should check if sockets have hit EOF' do
+      allow(strategy).to receive(:handle_signal)
+      expect(socket).to receive(:eof?).once.and_return(false)
+      strategy.send(:worker_assignment_loop)
+    end
+
+    it 'should not assign jobs if sockets have hit EOF' do
+      allow(strategy).to receive(:handle_signal)
+      allow(socket).to receive(:eof?).and_return(true)
+      expect(work_distributor).to_not receive(:fetch_and_assign_jobs)
+      strategy.send(:worker_assignment_loop)
+    end
+
+    it 'should handle fetch and assign jobs when workers are ready' do
+      expect(work_distributor).to receive(:fetch_and_assign_jobs).with([worker], manager).once
+      strategy.send(:worker_assignment_loop)
+    end
+  end
+
+  context '#handle_signal' do
+    before(:each) do
+      strategy.instance_variable_set(:@self_read, pipe)
+      allow(pipe).to receive(:read_nonblock).and_return(nil)
+
+      allow(worker_manager).to receive(:respawn_terminated_workers!).and_return(true)
+      allow(worker_manager).to receive(:stop_workers).and_return(true)
+      allow(manager).to receive(:shutdown!).and_return(true)
+    end
+
+    it 'should respawn terminated workers in the event of a SIGCHLD' do
+      allow(pipe).to receive(:read_nonblock).and_return('1')
+      expect(worker_manager).to receive(:respawn_terminated_workers!).once
+
+      strategy.send(:handle_signal)
+    end
+
+    it 'should signal its children, and shutdown in the event of one of INT, QUIT or TERM signals' do
+      allow(pipe).to receive(:read_nonblock).and_return('2', '3', '4')
+      expect(worker_manager).to receive(:stop_workers).exactly(3).times
+      expect(manager).to receive(:shutdown!).exactly(3).times
+      3.times do
+        strategy.send(:handle_signal)
+      end
+    end
+
+    it 'should propagte the signal it receives to its children' do
+      allow(pipe).to receive(:read_nonblock).and_return('3')
+      expect(worker_manager).to receive(:stop_workers).with(:QUIT)
+      strategy.send(:handle_signal)
+    end
+
+    it 'should reset logs on a USR1 signal' do
+      allow(pipe).to receive(:read_nonblock).and_return('5')
+      expect(Chore).to receive(:reopen_logs)
+      strategy.send(:handle_signal)
+    end
+
+    it 'should not preform any task when an unhandled signal is called' do
+      allow(pipe).to receive(:read_nonblock).and_return('9')
+      expect(worker_manager).to receive(:respawn_terminated_workers!).exactly(0).times
+      expect(worker_manager).to receive(:stop_workers).exactly(0).times
+      expect(manager).to receive(:shutdown!).exactly(0).times
+      strategy.send(:handle_signal)
+    end
+  end
+
+  context '#trap_signals' do
+    before(:each) do
+      allow_any_instance_of(Chore::Strategy::PreForkedWorkerStrategy).to receive(:trap_signals).and_call_original
+    end
+
+    let(:signals) { { '1' => 'QUIT' } }
+
+    it 'should reset signals' do
+      allow(Chore::Signal).to receive(:reset)
+      expect(Chore::Signal).to receive(:reset)
+      strategy.send(:trap_signals, {}, pipe)
+    end
+
+    it 'should trap the signals passed to it' do
+      allow(Chore::Signal).to receive(:reset)
+      expect(Chore::Signal).to receive(:trap).with('QUIT').once
+      strategy.send(:trap_signals, signals, pipe)
+    end
+  end
+end