chore-core 1.8.2 → 4.0.0

data/spec/chore/strategies/worker/helpers/preforked_worker_spec.rb

@@ -0,0 +1,236 @@
+require 'spec_helper'
+
+describe Chore::Strategy::PreforkedWorker do
+  before(:each) do
+    allow_any_instance_of(Chore::Strategy::PreforkedWorker).to receive(:post_fork_setup)
+  end
+
+  let(:preforkedworker) { Chore::Strategy::PreforkedWorker.new }
+  let(:socket)          { double("socket") }
+  let(:work)            { double("work") }
+  let(:consumer)        { double("consumer") }
+  let(:worker)          { double("worker") }
+  let(:config)          { double("config") }
+  let(:queues)          { double("queues") }
+  let(:consumer_object) { double("consumer_object") }
+
+  context '#start_worker' do
+    it 'should connect to the master to signal that it is ready, and process messages with the worker' do
+      allow(preforkedworker).to receive(:connect_to_master).and_return(socket)
+      expect(preforkedworker).to receive(:connect_to_master).with(socket)
+      allow(preforkedworker).to receive(:worker).and_return(true)
+      expect(preforkedworker).to receive(:worker).with(socket)
+      preforkedworker.start_worker(socket)
+    end
+  end
+
+  describe '#worker' do
+    before(:each) do
+      preforkedworker.instance_variable_set(:@self_read, socket)
+      allow(preforkedworker).to receive(:select_sockets).and_return([[socket],nil,nil])
+      allow(preforkedworker).to receive(:read_msg).and_return(nil)
+      allow(preforkedworker).to receive(:is_orphan?).and_return(false)
+      allow(preforkedworker).to receive(:process_work).and_return(true)
+      allow(preforkedworker).to receive(:signal_ready).and_return(true)
+      allow(work).to receive(:queue_timeout).and_return(20*60)
+    end
+
+
+    it 'should not run while @running is false' do
+      allow(preforkedworker).to receive(:running?).and_return(false)
+      expect(preforkedworker).to receive(:select_sockets).exactly(0).times
+      begin
+        preforkedworker.send(:worker, nil)
+      rescue SystemExit=>e
+        expect(e.status).to eq(0)
+      end
+    end
+
+    it 'should be able to handle timeouts on readable sockets' do    
+       allow(preforkedworker).to receive(:running?).and_return(true, true, false)   
+       allow(preforkedworker).to receive(:select_sockets).and_return(nil)   
+       expect(preforkedworker).to receive(:select_sockets).exactly(2).times   
+       begin    
+         preforkedworker.send(:worker, nil)   
+       rescue SystemExit=>e   
+         expect(e.status).to eq(0)    
+       end    
+     end    
+
+    it 'should read a message if the connection is readable' do
+      allow(preforkedworker).to receive(:running?).and_return(true, false)
+      expect(preforkedworker).to receive(:read_msg).once
+      begin
+        preforkedworker.send(:worker, nil)
+      rescue SystemExit=>e
+        expect(e.status).to eq(0)
+      end
+    end
+
+    it 'should check if the master is alive, and if not, it should end' do
+      allow(preforkedworker).to receive(:running).and_return(true)
+      allow(preforkedworker).to receive(:select_sockets).and_return([[socket],nil,nil])
+      allow(preforkedworker).to receive(:read_msg).and_return(nil)
+      allow(preforkedworker).to receive(:is_orphan?).and_return(true)
+      begin
+        preforkedworker.send(:worker, socket)
+      rescue SystemExit=>e
+        expect(e.status).to eq(0)
+      end
+    end
+
+    it 'should process work if it is read from the master and signal ready' do
+      allow(preforkedworker).to receive(:running?).and_return(true, false)
+      allow(preforkedworker).to receive(:read_msg).and_return(work)
+      expect(preforkedworker).to receive(:process_work).once
+      expect(preforkedworker).to receive(:signal_ready).once
+      begin
+        preforkedworker.send(:worker, socket)
+      rescue SystemExit=>e
+        expect(e.status).to eq(0)
+      end
+    end
+
+    it 'should exit the process if the connection to master is closed' do
+      allow(preforkedworker).to receive(:running?).and_return(true)
+      allow(preforkedworker).to receive(:read_msg).and_raise(Errno::ECONNRESET)
+      begin
+        preforkedworker.send(:worker, socket)
+      rescue SystemExit=>e
+        expect(e.status).to eq(0)
+      end
+    end
+  end
+
+  context '#connect_to_master' do
+    it 'should create a connection to the master, and send it its PID and a ready message' do
+      allow(preforkedworker).to receive(:child_connection).and_return(socket)
+      allow(preforkedworker).to receive(:send_msg).and_return(true)
+      allow(preforkedworker).to receive(:signal_ready).and_return(true)
+
+      expect(preforkedworker).to receive(:child_connection).once
+      expect(preforkedworker).to receive(:send_msg).once
+      expect(preforkedworker).to receive(:signal_ready).once
+
+      res = preforkedworker.send(:connect_to_master,socket)
+
+      expect(res).to eq(socket)
+    end
+  end
+
+  context '#post_fork_setup' do
+    before(:each) do
+      allow(preforkedworker).to receive(:procline).and_return(true)
+      allow(preforkedworker).to receive(:trap_signals)
+      allow_any_instance_of(Chore::Strategy::PreforkedWorker).to receive(:post_fork_setup).and_call_original
+    end
+
+    it 'should change the process name' do
+      expect(preforkedworker).to receive(:procline)
+      preforkedworker.send(:post_fork_setup)
+    end
+
+    it 'should trap new relevant signals' do
+      expect(preforkedworker).to receive(:trap_signals)
+      preforkedworker.send(:post_fork_setup)
+    end
+
+    it "should run after_fork hooks" do
+      hook_called = false
+      Chore.add_hook(:after_fork) { hook_called = true }
+      preforkedworker.send(:post_fork_setup)
+      expect(hook_called).to be true
+    end
+  end
+
+  context '#process_work' do
+    before(:each) do
+      allow(preforkedworker).to receive(:consumer).and_return(consumer)
+      allow(work).to receive(:queue_name).and_return("test_queue")
+      allow(work).to receive(:consumer=)
+      allow(work).to receive(:queue_timeout).and_return(10)
+      allow(Chore::Worker).to receive(:new).and_return(worker)
+      allow(worker).to receive(:start)
+    end
+
+    it 'should fetch the consumer object associated with the queue' do
+      expect(preforkedworker).to receive(:consumer)
+      preforkedworker.send(:process_work, [work])
+    end
+
+    it 'should create and start a worker object with the job sent to it' do
+      expect(worker).to receive(:start)
+      preforkedworker.send(:process_work, [work])
+    end
+
+    it 'should timeout if the work runs for longer than the queue timeout' do
+      allow(work).to receive(:queue_timeout).and_return(1)
+      allow(worker).to receive(:start) { sleep 5 }
+      begin
+        preforkedworker.send(:process_work, [work])
+      rescue => e
+        expect(e.class).to eq(Timeout::Error)
+      end
+    end
+  end
+
+  context '#consumer' do
+    before(:each) do
+      preforkedworker.instance_variable_set(:@consumer_cache, {key_1: :value_1})
+      allow(Chore).to receive(:config).and_return(config)
+      allow(config).to receive(:queues).and_return(queues)
+      allow(queues).to receive(:size).and_return(2)
+      allow(config).to receive(:consumer).and_return(consumer)
+      allow(consumer).to receive(:new).and_return(:value_2)
+    end
+
+    it 'should fetch a consumer object if it was created previously for this queue' do
+      preforkedworker.instance_variable_set(:@consumer_cache, {key_1: :value_1})
+      res = preforkedworker.send(:consumer,:key_1)
+      expect(res).to eq(:value_1)
+    end
+
+    it 'should create and return a new consumer object if one does not exist for this queue' do
+      preforkedworker.instance_variable_set(:@consumer_cache, {})
+      expect(consumer).to receive(:new)
+      res = preforkedworker.send(:consumer,:key_2)
+      expect(res).to eq(:value_2)
+    end
+  end
+
+  context '#trap_signals' do
+    it 'should reset signals' do
+      allow(Chore::Signal).to receive(:reset)
+      allow(Chore::Signal).to receive(:trap)
+      expect(Chore::Signal).to receive(:reset)
+      preforkedworker.send(:trap_signals)
+    end
+
+    it 'should trap the signals passed to it' do
+      allow(Chore::Signal).to receive(:reset)
+      allow(Chore::Signal).to receive(:trap)
+      expect(Chore::Signal).to receive(:trap).with(:INT).once
+      expect(Chore::Signal).to receive(:trap).with(:QUIT).once
+      expect(Chore::Signal).to receive(:trap).with(:TERM).once
+      expect(Chore::Signal).to receive(:trap).with(:USR1).once
+      preforkedworker.send(:trap_signals)
+    end
+  end
+
+  context '#is_orphan?' do
+    it 'should return true if the parent is dead' do
+      allow(Process).to receive(:ppid).and_return(10)
+      preforkedworker.instance_variable_set(:@manager_pid, 9)
+      res = preforkedworker.send(:is_orphan?)
+      expect(res).to eq(true)
+    end
+
+    it 'should return false if the parent is alive' do
+      allow(Process).to receive(:ppid).and_return(10)
+      preforkedworker.instance_variable_set(:@manager_pid, 10)
+      res = preforkedworker.send(:is_orphan?)
+      expect(res).to eq(false)
+    end
+  end
+end
+

data/spec/chore/strategies/worker/helpers/work_distributor_spec.rb

@@ -0,0 +1,131 @@
+require 'spec_helper'
+require 'chore/strategies/worker/helpers/work_distributor'
+
+describe Chore::Strategy::WorkDistributor do
+  let(:timestamp) { Time.now }
+  let(:manager) { double('manager') }
+  let(:worker) { Chore::Strategy::WorkerInfo.new(1) }
+  let(:consumer) { double('consumer') }
+  let(:job) do
+    Chore::UnitOfWork.new(
+      SecureRandom.uuid,
+      'test',
+      60,
+      Chore::Encoder::JsonEncoder.encode(TestJob.job_hash([1,2,"3"])),
+      0
+    )
+  end
+  let(:socket) { double('socket') }
+
+  context '#include_ipc' do
+    it 'should include the Ipc module' do
+      expect(described_class.ipc_help).to eq(:available)
+    end
+  end
+
+  context '#fetch_and_assign_jobs' do
+    it 'should fetch jobs from the consumer' do
+      allow(described_class).to receive(:assign_jobs).and_return(true)
+      allow(manager).to receive(:fetch_work).with(1).and_return([job])
+      allow(manager).to receive(:return_work)
+      expect(manager).to receive(:fetch_work).with(1)
+      described_class.fetch_and_assign_jobs([worker], manager)
+    end
+
+    it 'should assign the fetched jobs to the workers' do
+      allow(manager).to receive(:fetch_work).with(1).and_return([job])
+      allow(manager).to receive(:return_work)
+      expect(described_class).to receive(:assign_jobs).with([job], [worker])
+      described_class.fetch_and_assign_jobs([worker], manager)
+    end
+
+    it 'should not return any work' do
+      allow(described_class).to receive(:assign_jobs).and_return([])
+      allow(manager).to receive(:fetch_work).with(1).and_return([job])
+      expect(manager).to receive(:return_work).with([])
+      described_class.fetch_and_assign_jobs([worker], manager)
+    end
+
+    it 'should raise and exception if it does not get an array from the manager' do
+      allow(manager).to receive(:fetch_work).with(1).and_return(nil)
+      allow(manager).to receive(:return_work)
+      expect { described_class.fetch_and_assign_jobs([worker], manager) }.to raise_error("DW: jobs needs to be a list got NilClass")
+    end
+
+    it 'should sleep if no jobs are available' do
+      expect(described_class).to receive(:sleep).with(0.1)
+      allow(manager).to receive(:fetch_work).with(1).and_return([])
+      allow(manager).to receive(:return_work)
+      described_class.fetch_and_assign_jobs([worker], manager)
+    end
+  end
+
+  context '#assign_jobs' do
+    it 'should raise an exception if we have no free workers' do
+      expect { described_class.send(:assign_jobs, [job], []) }.to raise_error('DW: assign_jobs got 0 workers')
+    end
+
+    it 'should remove the consumer object from the job object' do
+      allow(described_class).to receive(:push_job_to_worker).and_return(true)
+
+      described_class.send(:assign_jobs, [job], [worker])
+    end
+
+    it 'should raise an exception if more jobs than workers are provided to it' do
+      allow(described_class).to receive(:push_job_to_worker).and_return(true)
+
+      expect { described_class.send(:assign_jobs, [job, job], [worker]) }.to raise_error('DW: More Jobs than Sockets')
+    end
+
+    it 'should send the job object to the free worker' do
+      allow(described_class).to receive(:push_job_to_worker).and_return(true)
+
+      expect(described_class).to receive(:push_job_to_worker).with(job, worker)
+      described_class.send(:assign_jobs, [job], [worker])
+    end
+
+    it 'should return jobs that failed to be assigned' do
+      job2 = Chore::UnitOfWork.new(
+        SecureRandom.uuid,
+        'test',
+        60,
+        Chore::Encoder::JsonEncoder.encode(TestJob.job_hash([1,2,"3"])),
+        0
+      )
+      worker2 = Chore::Strategy::WorkerInfo.new(2)
+
+      allow(described_class).to receive(:push_job_to_worker).and_return(true, false)
+
+      expect(described_class).to receive(:push_job_to_worker).with(job, worker)
+      expect(described_class).to receive(:push_job_to_worker).with(job2, worker2)
+      unassigned_jobs = described_class.send(:assign_jobs, [job, job2], [worker, worker2])
+      expect(unassigned_jobs).to eq([job2])
+    end
+  end
+
+  context '#push_job_to_worker' do
+    before(:each) do
+      allow(described_class).to receive(:clear_ready).with(worker.socket).and_return(true)
+      allow(described_class).to receive(:send_msg).with(worker.socket, job).and_return(true)
+    end
+
+    it 'should clear all signals from the worker' do
+      expect(described_class).to receive(:clear_ready).with(worker.socket)
+      described_class.send(:push_job_to_worker, job, worker)
+    end
+
+    it 'should send the job as a message on the worker' do
+      expect(described_class).to receive(:send_msg).with(worker.socket, job)
+      described_class.send(:push_job_to_worker, job, worker)
+    end
+
+    it 'should return true if job successfully sent' do
+      expect(described_class.send(:push_job_to_worker, job, worker)).to eq(true)
+    end
+
+    it 'should return false if job failed to send' do
+      expect(described_class).to receive(:send_msg).and_raise(StandardError)
+      expect(described_class.send(:push_job_to_worker, job, worker)).to eq(false)
+    end
+  end
+end

data/spec/chore/strategies/worker/helpers/worker_info_spec.rb

@@ -0,0 +1,14 @@
+require 'spec_helper'
+
+describe Chore::Strategy::WorkerInfo do
+  let(:pid)         { 1 }
+  let(:worker_info) { Chore::Strategy::WorkerInfo.new(pid) }
+
+  context '#initialize' do
+    it 'should initialize the WorkerInfo with a pid' do
+      wi = Chore::Strategy::WorkerInfo.new(pid)
+      expect(wi.pid).to equal(pid)
+      expect(wi.socket).to equal(nil)
+    end
+  end
+end

data/spec/chore/strategies/worker/helpers/worker_killer_spec.rb

@@ -0,0 +1,97 @@
+require 'spec_helper'
+require 'get_process_mem'
+
+describe Chore::Strategy::WorkerKiller do
+  let(:memory_limit)    { 1024 }
+  let(:request_limit)   { 100 }
+  let(:check_cycle)     { 16 }
+  let(:worker_killer)   { Chore::Strategy::WorkerKiller.new }
+  let(:process_mem_obj) { double('process_mem_obj', bytes: 10) }
+
+  context '#initialize' do
+    it 'should initialize the WorkerKiller correctly' do
+      allow(Chore.config).to receive(:memory_limit_bytes).and_return(memory_limit)
+      allow(Chore.config).to receive(:request_limit).and_return(request_limit)
+      allow(Chore.config).to receive(:check_cycle).and_return(check_cycle)
+
+      wk = Chore::Strategy::WorkerKiller.new
+      expect(wk.instance_variable_get(:@memory_limit)).to equal(memory_limit)
+      expect(wk.instance_variable_get(:@request_limit)).to equal(request_limit)
+      expect(wk.instance_variable_get(:@check_cycle)).to equal(check_cycle)
+      expect(wk.instance_variable_get(:@check_count)).to equal(0)
+      expect(wk.instance_variable_get(:@current_requests)).to equal(0)
+    end
+  end
+
+  context '#check_memory' do
+    before(:each) do
+      allow(GetProcessMem).to receive(:new).and_return(process_mem_obj)
+      worker_killer.instance_variable_set(:@memory_limit, memory_limit)
+      worker_killer.instance_variable_set(:@check_cycle, check_cycle)
+    end
+
+    it 'should return nil when memory_limit is nil' do
+      worker_killer.instance_variable_set(:@memory_limit, nil)
+      expect(worker_killer.check_memory).to eq(nil)
+    end
+
+    it 'should increment the check count by 1' do
+      worker_killer.instance_variable_set(:@check_count, 1)
+      worker_killer.check_memory
+      expect(worker_killer.instance_variable_get(:@check_count)).to eq(2)
+    end
+
+    context 'check_count equals check_cycle' do
+      before(:each) do
+        worker_killer.instance_variable_set(:@check_count, 15)
+      end
+
+      it 'should check memory' do
+        expect(process_mem_obj).to receive(:bytes)
+        worker_killer.check_memory
+      end
+
+      it 'should reset the check_count to zero' do
+        allow(process_mem_obj).to receive(:bytes).and_return(0)
+        worker_killer.check_memory
+        expect(worker_killer.instance_variable_get(:@check_count)).to equal(0)
+      end
+
+      it 'should exit if the process mem exceeds the memory_limit' do
+        allow(process_mem_obj).to receive(:bytes).and_return(2048)
+        begin
+          worker_killer.check_memory
+        rescue SystemExit=>e
+          expect(e.status).to eq(0)
+        end
+      end
+    end
+  end
+
+  context '#check_requests' do
+    before(:each) do
+      worker_killer.instance_variable_set(:@request_limit, request_limit)
+      worker_killer.instance_variable_set(:@current_requests, 0)
+    end
+
+    it 'should return nil when request_limit is nil' do
+      worker_killer.instance_variable_set(:@request_limit, nil)
+      expect(worker_killer.check_requests).to eq(nil)
+    end
+
+    it 'should increment current requests' do
+      worker_killer.check_requests
+      expect(worker_killer.instance_variable_get(:@current_requests)).to eq(1)
+    end
+
+    it 'should exit when current_requests exceeds request_limit' do
+      worker_killer.instance_variable_set(:@current_requests, request_limit - 1)
+
+      begin
+        worker_killer.check_requests
+      rescue SystemExit=>e
+        expect(e.status).to eq(0)
+      end
+    end
+  end
+end