chore-core 1.8.2 → 3.2.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. checksums.yaml +5 -5
  2. data/README.md +6 -0
  3. data/chore-core.gemspec +1 -0
  4. data/lib/chore.rb +11 -5
  5. data/lib/chore/cli.rb +21 -2
  6. data/lib/chore/consumer.rb +15 -5
  7. data/lib/chore/fetcher.rb +12 -7
  8. data/lib/chore/hooks.rb +2 -1
  9. data/lib/chore/job.rb +17 -0
  10. data/lib/chore/manager.rb +18 -2
  11. data/lib/chore/queues/filesystem/consumer.rb +116 -59
  12. data/lib/chore/queues/filesystem/filesystem_queue.rb +19 -0
  13. data/lib/chore/queues/filesystem/publisher.rb +12 -18
  14. data/lib/chore/queues/sqs/consumer.rb +6 -21
  15. data/lib/chore/strategies/consumer/batcher.rb +8 -9
  16. data/lib/chore/strategies/consumer/threaded_consumer_strategy.rb +3 -1
  17. data/lib/chore/strategies/consumer/throttled_consumer_strategy.rb +121 -0
  18. data/lib/chore/strategies/worker/forked_worker_strategy.rb +5 -6
  19. data/lib/chore/strategies/worker/helpers/ipc.rb +88 -0
  20. data/lib/chore/strategies/worker/helpers/preforked_worker.rb +163 -0
  21. data/lib/chore/strategies/worker/helpers/work_distributor.rb +65 -0
  22. data/lib/chore/strategies/worker/helpers/worker_info.rb +13 -0
  23. data/lib/chore/strategies/worker/helpers/worker_killer.rb +40 -0
  24. data/lib/chore/strategies/worker/helpers/worker_manager.rb +183 -0
  25. data/lib/chore/strategies/worker/preforked_worker_strategy.rb +150 -0
  26. data/lib/chore/strategies/worker/single_worker_strategy.rb +35 -13
  27. data/lib/chore/unit_of_work.rb +8 -0
  28. data/lib/chore/util.rb +5 -1
  29. data/lib/chore/version.rb +3 -3
  30. data/lib/chore/worker.rb +29 -0
  31. data/spec/chore/cli_spec.rb +2 -2
  32. data/spec/chore/consumer_spec.rb +0 -4
  33. data/spec/chore/duplicate_detector_spec.rb +17 -5
  34. data/spec/chore/fetcher_spec.rb +0 -11
  35. data/spec/chore/manager_spec.rb +7 -0
  36. data/spec/chore/queues/filesystem/filesystem_consumer_spec.rb +71 -11
  37. data/spec/chore/queues/sqs/consumer_spec.rb +1 -3
  38. data/spec/chore/strategies/consumer/batcher_spec.rb +50 -0
  39. data/spec/chore/strategies/consumer/threaded_consumer_strategy_spec.rb +1 -0
  40. data/spec/chore/strategies/consumer/throttled_consumer_strategy_spec.rb +165 -0
  41. data/spec/chore/strategies/worker/forked_worker_strategy_spec.rb +16 -1
  42. data/spec/chore/strategies/worker/helpers/ipc_spec.rb +127 -0
  43. data/spec/chore/strategies/worker/helpers/preforked_worker_spec.rb +236 -0
  44. data/spec/chore/strategies/worker/helpers/work_distributor_spec.rb +131 -0
  45. data/spec/chore/strategies/worker/helpers/worker_info_spec.rb +14 -0
  46. data/spec/chore/strategies/worker/helpers/worker_killer_spec.rb +97 -0
  47. data/spec/chore/strategies/worker/helpers/worker_manager_spec.rb +304 -0
  48. data/spec/chore/strategies/worker/preforked_worker_strategy_spec.rb +183 -0
  49. data/spec/chore/strategies/worker/single_worker_strategy_spec.rb +25 -0
  50. data/spec/chore/worker_spec.rb +69 -1
  51. metadata +33 -5
@@ -0,0 +1,183 @@
1
+ require 'spec_helper'
2
+
3
+ describe Chore::Strategy::PreForkedWorkerStrategy do
4
+ let(:manager) { double('manager') }
5
+ let(:socket) { double('socket') }
6
+ let(:pipe) { double('pipe') }
7
+ let(:worker) { double('worker') }
8
+ let(:worker_manager) { Chore::Strategy::WorkerManager.new(socket) }
9
+ let(:strategy) { Chore::Strategy::PreForkedWorkerStrategy.new(manager) }
10
+ let(:work_distributor) { Chore::Strategy::WorkDistributor }
11
+
12
+ before(:each) do
13
+ allow_any_instance_of(Chore::Strategy::PreForkedWorkerStrategy).to receive(:trap_signals).and_return(true)
14
+ strategy.instance_variable_set(:@worker_manager, worker_manager)
15
+ end
16
+
17
+ context '#start' do
18
+ it 'should create, attach workers and start the worker manager' do
19
+ allow(worker_manager).to receive(:create_and_attach_workers)
20
+ allow(strategy).to receive(:worker_assignment_thread).and_return(true)
21
+
22
+ expect(worker_manager).to receive(:create_and_attach_workers)
23
+ expect(strategy).to receive(:worker_assignment_thread)
24
+ strategy.start
25
+ end
26
+ end
27
+
28
+ context '#stop!' do
29
+ it 'should set the system\'s running state to false' do
30
+ strategy.instance_variable_set(:@running, true)
31
+ strategy.stop!
32
+ expect(strategy.instance_variable_get(:@running)).to eq false
33
+ end
34
+ end
35
+
36
+ context '#worker_assignment_thread' do
37
+ before(:each) do
38
+ allow(Thread).to receive(:new).and_return(true)
39
+ end
40
+
41
+ it 'create a new thread with the worker_assignment_loop' do
42
+ allow(strategy).to receive(:worker_assignment_loop).and_return(true)
43
+ expect(Thread).to receive(:new).once.and_yield
44
+ expect(strategy).to receive(:worker_assignment_loop)
45
+ expect(Process).to receive(:exit)
46
+ strategy.send(:worker_assignment_thread)
47
+ end
48
+
49
+ it 'rescues a \'TerribleMistake\' exception and performs a shutdown of chore' do
50
+ allow(strategy).to receive(:worker_assignment_loop).and_raise(Chore::TerribleMistake)
51
+ allow(manager).to receive(:shutdown!)
52
+ allow(Thread).to receive(:new).and_yield
53
+ expect(Process).to receive(:exit)
54
+ strategy.send(:worker_assignment_thread)
55
+ end
56
+ end
57
+
58
+ context '#worker_assignment_loop' do
59
+ before(:each) do
60
+ allow(socket).to receive(:eof?).and_return(false)
61
+
62
+ allow(strategy).to receive(:running?).and_return(true, false)
63
+ strategy.instance_variable_set(:@self_read, pipe)
64
+ allow(strategy).to receive(:select_sockets).and_return([[socket], nil, nil])
65
+ allow(strategy).to receive(:handle_self_pipe_signal).and_return(true)
66
+ allow(strategy).to receive(:fetch_and_assign_jobs).and_return(true)
67
+
68
+ allow(worker_manager).to receive(:worker_sockets).and_return([socket])
69
+ allow(worker_manager).to receive(:ready_workers).and_yield([worker])
70
+ allow(worker_manager).to receive(:destroy_expired!)
71
+
72
+ allow(work_distributor).to receive(:fetch_and_assign_jobs)
73
+ end
74
+
75
+ it 'should terminate when @running is set to false' do
76
+ allow(strategy).to receive(:running?).and_return(false)
77
+ expect(strategy).to receive(:select_sockets).exactly(0).times
78
+ strategy.send(:worker_assignment_loop)
79
+ end
80
+
81
+ it 'should get the worker_sockets' do
82
+ expect(worker_manager).to receive(:worker_sockets)
83
+ strategy.send(:worker_assignment_loop)
84
+ end
85
+
86
+ it 'should handle no sockets being ready' do
87
+ allow(strategy).to receive(:select_sockets).and_return(nil)
88
+ expect(strategy).to receive(:select_sockets).once
89
+ strategy.send(:worker_assignment_loop)
90
+ end
91
+
92
+ it 'should handle signals if alerted on a self pipe' do
93
+ allow(strategy).to receive(:select_sockets).and_return([[pipe], nil, nil])
94
+ expect(strategy).to receive(:handle_signal).once
95
+ strategy.send(:worker_assignment_loop)
96
+ end
97
+
98
+ it 'should check if sockets have hit EOF' do
99
+ allow(strategy).to receive(:handle_signal)
100
+ expect(socket).to receive(:eof?).once.and_return(false)
101
+ strategy.send(:worker_assignment_loop)
102
+ end
103
+
104
+ it 'should not assign jobs if sockets have hit EOF' do
105
+ allow(strategy).to receive(:handle_signal)
106
+ allow(socket).to receive(:eof?).and_return(true)
107
+ expect(work_distributor).to_not receive(:fetch_and_assign_jobs)
108
+ strategy.send(:worker_assignment_loop)
109
+ end
110
+
111
+ it 'should handle fetch and assign jobs when workers are ready' do
112
+ expect(work_distributor).to receive(:fetch_and_assign_jobs).with([worker], manager).once
113
+ strategy.send(:worker_assignment_loop)
114
+ end
115
+ end
116
+
117
+ context '#handle_signal' do
118
+ before(:each) do
119
+ strategy.instance_variable_set(:@self_read, pipe)
120
+ allow(pipe).to receive(:read_nonblock).and_return(nil)
121
+
122
+ allow(worker_manager).to receive(:respawn_terminated_workers!).and_return(true)
123
+ allow(worker_manager).to receive(:stop_workers).and_return(true)
124
+ allow(manager).to receive(:shutdown!).and_return(true)
125
+ end
126
+
127
+ it 'should respawn terminated workers in the event of a SIGCHLD' do
128
+ allow(pipe).to receive(:read_nonblock).and_return('1')
129
+ expect(worker_manager).to receive(:respawn_terminated_workers!).once
130
+
131
+ strategy.send(:handle_signal)
132
+ end
133
+
134
+ it 'should signal its children, and shutdown in the event of one of INT, QUIT or TERM signals' do
135
+ allow(pipe).to receive(:read_nonblock).and_return('2', '3', '4')
136
+ expect(worker_manager).to receive(:stop_workers).exactly(3).times
137
+ expect(manager).to receive(:shutdown!).exactly(3).times
138
+ 3.times do
139
+ strategy.send(:handle_signal)
140
+ end
141
+ end
142
+
143
+ it 'should propagte the signal it receives to its children' do
144
+ allow(pipe).to receive(:read_nonblock).and_return('3')
145
+ expect(worker_manager).to receive(:stop_workers).with(:QUIT)
146
+ strategy.send(:handle_signal)
147
+ end
148
+
149
+ it 'should reset logs on a USR1 signal' do
150
+ allow(pipe).to receive(:read_nonblock).and_return('5')
151
+ expect(Chore).to receive(:reopen_logs)
152
+ strategy.send(:handle_signal)
153
+ end
154
+
155
+ it 'should not preform any task when an unhandled signal is called' do
156
+ allow(pipe).to receive(:read_nonblock).and_return('9')
157
+ expect(worker_manager).to receive(:respawn_terminated_workers!).exactly(0).times
158
+ expect(worker_manager).to receive(:stop_workers).exactly(0).times
159
+ expect(manager).to receive(:shutdown!).exactly(0).times
160
+ strategy.send(:handle_signal)
161
+ end
162
+ end
163
+
164
+ context '#trap_signals' do
165
+ before(:each) do
166
+ allow_any_instance_of(Chore::Strategy::PreForkedWorkerStrategy).to receive(:trap_signals).and_call_original
167
+ end
168
+
169
+ let(:signals) { { '1' => 'QUIT' } }
170
+
171
+ it 'should reset signals' do
172
+ allow(Chore::Signal).to receive(:reset)
173
+ expect(Chore::Signal).to receive(:reset)
174
+ strategy.send(:trap_signals, {}, pipe)
175
+ end
176
+
177
+ it 'should trap the signals passed to it' do
178
+ allow(Chore::Signal).to receive(:reset)
179
+ expect(Chore::Signal).to receive(:trap).with('QUIT').once
180
+ strategy.send(:trap_signals, signals, pipe)
181
+ end
182
+ end
183
+ end
@@ -2,6 +2,8 @@ require 'spec_helper'
2
2
 
3
3
  describe Chore::Strategy::SingleWorkerStrategy do
4
4
  let(:manager) { double('Manager') }
5
+ let(:job_timeout) { 60 }
6
+ let(:job) { Chore::UnitOfWork.new(SecureRandom.uuid, 'test', job_timeout, Chore::Encoder::JsonEncoder.encode(TestJob.job_hash([1,2,"3"])), 0) }
5
7
  subject { described_class.new(manager) }
6
8
 
7
9
  describe '#stop!' do
@@ -31,4 +33,27 @@ describe Chore::Strategy::SingleWorkerStrategy do
31
33
  end
32
34
  end
33
35
 
36
+ describe '#assign' do
37
+ let(:worker) { double('Worker', start: nil) }
38
+
39
+ it 'starts a new worker' do
40
+ expect(Chore::Worker).to receive(:new).with(job, {}).and_return(worker)
41
+ subject.assign(job)
42
+ end
43
+
44
+ it 'can be called multiple times' do
45
+ expect(Chore::Worker).to receive(:new).twice.with(job, {}).and_return(worker)
46
+ 2.times { subject.assign(job) }
47
+ end
48
+
49
+ it 'should release the worker if an exception occurs' do
50
+ allow_any_instance_of(Chore::Worker).to receive(:start).and_raise(ArgumentError)
51
+ expect(subject).to receive(:release_worker)
52
+
53
+ begin
54
+ subject.assign(job)
55
+ rescue ArgumentError
56
+ end
57
+ end
58
+ end
34
59
  end
@@ -2,6 +2,10 @@ require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
2
2
 
3
3
  describe Chore::Worker do
4
4
 
5
+ before(:each) do
6
+ allow(consumer).to receive(:duplicate_message?).and_return(false)
7
+ end
8
+
5
9
  class SimpleJob
6
10
  include Chore::Job
7
11
  queue_options :name => 'test',
@@ -13,7 +17,31 @@ describe Chore::Worker do
13
17
  end
14
18
  end
15
19
 
16
- let(:consumer) { double('consumer', :complete => nil) }
20
+ class SimpleDedupeJob
21
+ include Chore::Job
22
+ queue_options :name => 'dedupe_test',
23
+ :publisher => FakePublisher,
24
+ :max_attempts => 100,
25
+ :dedupe_lambda => lambda { |first, second, third| first }
26
+
27
+ def perform(first, second, third)
28
+ return second
29
+ end
30
+ end
31
+
32
+ class InvalidDedupeJob
33
+ include Chore::Job
34
+ queue_options :name => 'invalid_dedupe_test',
35
+ :publisher => FakePublisher,
36
+ :max_attempts => 100,
37
+ :dedupe_lambda => lambda { |first, second, third| first }
38
+
39
+ def perform(first, second)
40
+ return second
41
+ end
42
+ end
43
+
44
+ let(:consumer) { double('consumer', :complete => nil, :reject => nil) }
17
45
  let(:job_args) { [1,2,'3'] }
18
46
  let(:job) { SimpleJob.job_hash(job_args) }
19
47
 
@@ -40,6 +68,33 @@ describe Chore::Worker do
40
68
  consumer.should_receive(:complete).exactly(10).times
41
69
  Chore::Worker.start(work, {:payload_handler => payload_handler})
42
70
  end
71
+
72
+ context 'when the job has a dedupe_lambda defined' do
73
+ context 'when the value being deduped on is unique' do
74
+ let(:job_args) { [rand,2,'3'] }
75
+ let(:encoded_job) { Chore::Encoder::JsonEncoder.encode(job) }
76
+ let(:job) { SimpleDedupeJob.job_hash(job_args) }
77
+ it 'should call complete for each unique value' do
78
+ allow(consumer).to receive(:duplicate_message?).and_return(false)
79
+ work = []
80
+ work << Chore::UnitOfWork.new(1, 'dedupe_test', 60, Chore::Encoder::JsonEncoder.encode(SimpleDedupeJob.job_hash([rand,2,'3'])), 0, consumer)
81
+ SimpleDedupeJob.should_receive(:perform).exactly(1).times
82
+ consumer.should_receive(:complete).exactly(1).times
83
+ Chore::Worker.start(work, {:payload_handler => payload_handler})
84
+ end
85
+ end
86
+
87
+ context 'when the dedupe lambda does not take the same number of arguments as perform' do
88
+ it 'should raise an error and not complete the job' do
89
+ work = []
90
+ work << Chore::UnitOfWork.new(1, 'invalid_dedupe_test', 60, Chore::Encoder::JsonEncoder.encode(InvalidDedupeJob.job_hash([rand,2,'3'])), 0, consumer)
91
+ work << Chore::UnitOfWork.new(2, 'invalid_dedupe_test', 60, Chore::Encoder::JsonEncoder.encode(InvalidDedupeJob.job_hash([rand,2,'3'])), 0, consumer)
92
+ work << Chore::UnitOfWork.new(1, 'invalid_dedupe_test', 60, Chore::Encoder::JsonEncoder.encode(InvalidDedupeJob.job_hash([rand,2,'3'])), 0, consumer)
93
+ consumer.should_not_receive(:complete)
94
+ Chore::Worker.start(work, {:payload_handler => payload_handler})
95
+ end
96
+ end
97
+ end
43
98
  end
44
99
 
45
100
  describe "with default payload handler" do
@@ -91,6 +146,12 @@ describe Chore::Worker do
91
146
  Chore::Worker.start(work)
92
147
  end
93
148
 
149
+ it 'should reject job' do
150
+ work = Chore::UnitOfWork.new(2,'test',60,job,0,consumer)
151
+ consumer.should_receive(:reject).with(2)
152
+ Chore::Worker.start(work)
153
+ end
154
+
94
155
  context 'more than the maximum allowed times' do
95
156
  before(:each) do
96
157
  Chore.config.stub(:max_attempts).and_return(10)
@@ -127,6 +188,13 @@ describe Chore::Worker do
127
188
  Chore::Worker.start(work)
128
189
  end
129
190
 
191
+ it 'should reject job' do
192
+ work = Chore::UnitOfWork.new(2,'test',60,encoded_job,0,consumer)
193
+ consumer.should_receive(:reject).with(2)
194
+
195
+ Chore::Worker.start(work)
196
+ end
197
+
130
198
  context 'more than the maximum allowed times' do
131
199
  it 'should permanently fail' do
132
200
  work = Chore::UnitOfWork.new(2,'test',60,encoded_job,999,consumer)
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: chore-core
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.8.2
4
+ version: 3.2.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Tapjoy
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-02-02 00:00:00.000000000 Z
11
+ date: 2020-09-14 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: json
@@ -58,6 +58,20 @@ dependencies:
58
58
  - - "~>"
59
59
  - !ruby/object:Gem::Version
60
60
  version: 0.1.3
61
+ - !ruby/object:Gem::Dependency
62
+ name: get_process_mem
63
+ requirement: !ruby/object:Gem::Requirement
64
+ requirements:
65
+ - - "~>"
66
+ - !ruby/object:Gem::Version
67
+ version: 0.2.0
68
+ type: :runtime
69
+ prerelease: false
70
+ version_requirements: !ruby/object:Gem::Requirement
71
+ requirements:
72
+ - - "~>"
73
+ - !ruby/object:Gem::Version
74
+ version: 0.2.0
61
75
  - !ruby/object:Gem::Dependency
62
76
  name: rspec
63
77
  requirement: !ruby/object:Gem::Requirement
@@ -137,7 +151,15 @@ files:
137
151
  - lib/chore/strategies/consumer/batcher.rb
138
152
  - lib/chore/strategies/consumer/single_consumer_strategy.rb
139
153
  - lib/chore/strategies/consumer/threaded_consumer_strategy.rb
154
+ - lib/chore/strategies/consumer/throttled_consumer_strategy.rb
140
155
  - lib/chore/strategies/worker/forked_worker_strategy.rb
156
+ - lib/chore/strategies/worker/helpers/ipc.rb
157
+ - lib/chore/strategies/worker/helpers/preforked_worker.rb
158
+ - lib/chore/strategies/worker/helpers/work_distributor.rb
159
+ - lib/chore/strategies/worker/helpers/worker_info.rb
160
+ - lib/chore/strategies/worker/helpers/worker_killer.rb
161
+ - lib/chore/strategies/worker/helpers/worker_manager.rb
162
+ - lib/chore/strategies/worker/preforked_worker_strategy.rb
141
163
  - lib/chore/strategies/worker/single_worker_strategy.rb
142
164
  - lib/chore/tasks/queues.task
143
165
  - lib/chore/unit_of_work.rb
@@ -160,7 +182,15 @@ files:
160
182
  - spec/chore/strategies/consumer/batcher_spec.rb
161
183
  - spec/chore/strategies/consumer/single_consumer_strategy_spec.rb
162
184
  - spec/chore/strategies/consumer/threaded_consumer_strategy_spec.rb
185
+ - spec/chore/strategies/consumer/throttled_consumer_strategy_spec.rb
163
186
  - spec/chore/strategies/worker/forked_worker_strategy_spec.rb
187
+ - spec/chore/strategies/worker/helpers/ipc_spec.rb
188
+ - spec/chore/strategies/worker/helpers/preforked_worker_spec.rb
189
+ - spec/chore/strategies/worker/helpers/work_distributor_spec.rb
190
+ - spec/chore/strategies/worker/helpers/worker_info_spec.rb
191
+ - spec/chore/strategies/worker/helpers/worker_killer_spec.rb
192
+ - spec/chore/strategies/worker/helpers/worker_manager_spec.rb
193
+ - spec/chore/strategies/worker/preforked_worker_strategy_spec.rb
164
194
  - spec/chore/strategies/worker/single_worker_strategy_spec.rb
165
195
  - spec/chore/worker_spec.rb
166
196
  - spec/chore_spec.rb
@@ -185,10 +215,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
185
215
  - !ruby/object:Gem::Version
186
216
  version: '0'
187
217
  requirements: []
188
- rubyforge_project:
189
- rubygems_version: 2.4.5
218
+ rubygems_version: 3.1.2
190
219
  signing_key:
191
220
  specification_version: 4
192
221
  summary: Job processing... for the future!
193
222
  test_files: []
194
- has_rdoc: