employer 0.0.1 → 0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +1 -0
- data/.pryrc +2 -0
- data/.rspec +1 -0
- data/README.md +133 -2
- data/bin/employer +7 -0
- data/employer.gemspec +7 -1
- data/lib/employer.rb +6 -4
- data/lib/employer/boss.rb +110 -0
- data/lib/employer/cli.rb +70 -0
- data/lib/employer/employees.rb +3 -0
- data/lib/employer/employees/abstract_employee.rb +55 -0
- data/lib/employer/employees/forking_employee.rb +52 -0
- data/lib/employer/employees/threading_employee.rb +38 -0
- data/lib/employer/errors.rb +5 -0
- data/lib/employer/errors/employee_busy.rb +6 -0
- data/lib/employer/errors/error.rb +6 -0
- data/lib/employer/errors/job_class_mismatch.rb +6 -0
- data/lib/employer/errors/no_employee_free.rb +6 -0
- data/lib/employer/errors/pipeline_backend_required.rb +6 -0
- data/lib/employer/job.rb +55 -0
- data/lib/employer/pipeline.rb +55 -0
- data/lib/employer/version.rb +1 -1
- data/lib/employer/workshop.rb +66 -0
- data/spec/employer/boss_spec.rb +254 -0
- data/spec/employer/employees/forking_employee_spec.rb +6 -0
- data/spec/employer/employees/threading_employee_spec.rb +6 -0
- data/spec/employer/job_spec.rb +89 -0
- data/spec/employer/pipeline_spec.rb +92 -0
- data/spec/employer/workshop_spec.rb +105 -0
- data/spec/spec_helper.rb +2 -0
- data/spec/support/shared_examples/employee.rb +132 -0
- metadata +91 -7
@@ -0,0 +1,38 @@
|
|
1
|
+
require_relative "abstract_employee"
|
2
|
+
|
3
|
+
module Employer
|
4
|
+
module Employees
|
5
|
+
class ThreadingEmployee < AbstractEmployee
|
6
|
+
def work(job)
|
7
|
+
super
|
8
|
+
@work_state = :busy
|
9
|
+
|
10
|
+
@thread = Thread.new do
|
11
|
+
begin
|
12
|
+
perform_job
|
13
|
+
@work_state = :complete
|
14
|
+
rescue => exception
|
15
|
+
ensure
|
16
|
+
@work_state = :failed if @work_state == :busy
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
def free
|
22
|
+
super
|
23
|
+
@thread = nil
|
24
|
+
end
|
25
|
+
|
26
|
+
def work_state(wait = false)
|
27
|
+
@thread.join if wait && @thread
|
28
|
+
return @work_state
|
29
|
+
end
|
30
|
+
|
31
|
+
def force_work_stop
|
32
|
+
return if free?
|
33
|
+
@thread.kill
|
34
|
+
@work_state = :failed
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
data/lib/employer/job.rb
ADDED
@@ -0,0 +1,55 @@
|
|
1
|
+
require_relative "errors"
|
2
|
+
|
3
|
+
module Employer
|
4
|
+
module Job
|
5
|
+
attr_accessor :id
|
6
|
+
|
7
|
+
module ClassMethods
|
8
|
+
def attribute(name)
|
9
|
+
name = name.to_sym
|
10
|
+
unless attribute_names.include?(name)
|
11
|
+
attribute_names << name
|
12
|
+
attr_accessor name
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
def attribute_names
|
17
|
+
@attribute_names ||= []
|
18
|
+
end
|
19
|
+
|
20
|
+
def deserialize(serialized_job)
|
21
|
+
raise Employer::Errors::JobClassMismatch unless serialized_job[:class] == self.name
|
22
|
+
job = new
|
23
|
+
job.id = serialized_job[:id]
|
24
|
+
serialized_job[:attributes].each_pair do |attribute, value|
|
25
|
+
job.public_send("#{attribute}=", value)
|
26
|
+
end
|
27
|
+
job
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
def self.included(base)
|
32
|
+
base.extend(ClassMethods)
|
33
|
+
end
|
34
|
+
|
35
|
+
def attribute_names
|
36
|
+
self.class.attribute_names
|
37
|
+
end
|
38
|
+
|
39
|
+
def try_again?
|
40
|
+
false
|
41
|
+
end
|
42
|
+
|
43
|
+
def serialize
|
44
|
+
{
|
45
|
+
id: id,
|
46
|
+
class: self.class.name,
|
47
|
+
attributes: Hash[
|
48
|
+
attribute_names.
|
49
|
+
reject { |name| self.send(name).nil? }.
|
50
|
+
map { |name| [name, self.send(name)] }
|
51
|
+
]
|
52
|
+
}.reject { |key, value| value.nil? }
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
@@ -0,0 +1,55 @@
|
|
1
|
+
require_relative "errors"
|
2
|
+
|
3
|
+
module Employer
|
4
|
+
class Pipeline
|
5
|
+
def backend=(backend)
|
6
|
+
@backend = backend
|
7
|
+
end
|
8
|
+
|
9
|
+
def backend
|
10
|
+
@backend
|
11
|
+
end
|
12
|
+
|
13
|
+
def enqueue(job)
|
14
|
+
raise Employer::Errors::PipelineBackendRequired if backend.nil?
|
15
|
+
serialized_job = job.serialize
|
16
|
+
backend.enqueue(serialized_job)
|
17
|
+
end
|
18
|
+
|
19
|
+
def dequeue
|
20
|
+
raise Employer::Errors::PipelineBackendRequired if backend.nil?
|
21
|
+
if serialized_job = backend.dequeue
|
22
|
+
job_class = constantize(serialized_job[:class])
|
23
|
+
job_class.deserialize(serialized_job)
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
def complete(job)
|
28
|
+
raise Employer::Errors::PipelineBackendRequired if backend.nil?
|
29
|
+
backend.complete(job)
|
30
|
+
end
|
31
|
+
|
32
|
+
def reset(job)
|
33
|
+
raise Employer::Errors::PipelineBackendRequired if backend.nil?
|
34
|
+
backend.reset(job)
|
35
|
+
end
|
36
|
+
|
37
|
+
def fail(job)
|
38
|
+
raise Employer::Errors::PipelineBackendRequired if backend.nil?
|
39
|
+
backend.fail(job)
|
40
|
+
end
|
41
|
+
|
42
|
+
private
|
43
|
+
|
44
|
+
def constantize(camel_cased_word)
|
45
|
+
names = camel_cased_word.split('::')
|
46
|
+
names.shift if names.empty? || names.first.empty?
|
47
|
+
|
48
|
+
constant = Object
|
49
|
+
names.each do |name|
|
50
|
+
constant = constant.const_defined?(name, false) ? constant.const_get(name) : constant.const_missing(name)
|
51
|
+
end
|
52
|
+
constant
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
data/lib/employer/version.rb
CHANGED
@@ -0,0 +1,66 @@
|
|
1
|
+
require_relative "errors"
|
2
|
+
|
3
|
+
module Employer
|
4
|
+
class Workshop
|
5
|
+
def self.setup(config_code, skip_employees = false)
|
6
|
+
boss = Employer::Boss.new
|
7
|
+
pipeline = Employer::Pipeline.new
|
8
|
+
boss.pipeline = pipeline
|
9
|
+
workshop = new(boss, config_code, skip_employees)
|
10
|
+
end
|
11
|
+
|
12
|
+
def self.pipeline(filename)
|
13
|
+
workshop = setup(File.read(filename), true)
|
14
|
+
workshop.pipeline
|
15
|
+
end
|
16
|
+
|
17
|
+
def run
|
18
|
+
@boss.manage
|
19
|
+
end
|
20
|
+
|
21
|
+
def stop
|
22
|
+
@boss.stop_managing
|
23
|
+
end
|
24
|
+
|
25
|
+
def stop_now
|
26
|
+
@boss.stop_managing
|
27
|
+
@boss.stop_employees
|
28
|
+
end
|
29
|
+
|
30
|
+
def pipeline
|
31
|
+
@boss.pipeline
|
32
|
+
end
|
33
|
+
|
34
|
+
private
|
35
|
+
|
36
|
+
def forking_employees(number)
|
37
|
+
@forking_employees = number
|
38
|
+
end
|
39
|
+
|
40
|
+
def threading_employees(number)
|
41
|
+
@threading_employees = number
|
42
|
+
end
|
43
|
+
|
44
|
+
def pipeline_backend(backend)
|
45
|
+
@boss.pipeline.backend = backend
|
46
|
+
end
|
47
|
+
|
48
|
+
def initialize(boss, config_code, skip_employees)
|
49
|
+
@boss = boss
|
50
|
+
@forking_employees = 0
|
51
|
+
@threading_employees = 0
|
52
|
+
|
53
|
+
instance_eval(config_code)
|
54
|
+
|
55
|
+
unless skip_employees
|
56
|
+
@forking_employees.times do
|
57
|
+
@boss.allocate_employee(Employer::Employees::ForkingEmployee.new)
|
58
|
+
end
|
59
|
+
|
60
|
+
@threading_employees.times do
|
61
|
+
@boss.allocate_employee(Employer::Employees::ThreadingEmployee.new)
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
66
|
+
end
|
@@ -0,0 +1,254 @@
|
|
1
|
+
require "employer/boss"
|
2
|
+
|
3
|
+
describe Employer::Boss do
|
4
|
+
let(:pipeline) { double("Pipeline") }
|
5
|
+
let(:employee) { double("Employee") }
|
6
|
+
let(:free_employee) { double("Free employee", free?: true) }
|
7
|
+
let(:busy_employee) { double("Busy employee", free?: false) }
|
8
|
+
let(:boss) { Employer::Boss.new }
|
9
|
+
|
10
|
+
it "can be given a pipeline" do
|
11
|
+
boss.pipeline = pipeline
|
12
|
+
boss.pipeline.should eq(pipeline)
|
13
|
+
end
|
14
|
+
|
15
|
+
it "can be given employees" do
|
16
|
+
john = double
|
17
|
+
jane = double
|
18
|
+
boss.allocate_employee(john)
|
19
|
+
boss.allocate_employee(jane)
|
20
|
+
boss.employees.should eq([john, jane])
|
21
|
+
end
|
22
|
+
|
23
|
+
describe "#manage" do
|
24
|
+
it "delegates work and collects progress updates until stopped, and then wait on employees" do
|
25
|
+
boss.should_receive(:keep_going).and_return(true, true, false)
|
26
|
+
boss.should_receive(:delegate_work).twice
|
27
|
+
boss.should_receive(:progress_update).twice
|
28
|
+
boss.should_receive(:wait_on_employees)
|
29
|
+
boss.manage
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
describe "#stop_managing" do
|
34
|
+
it "sets keep_going to false" do
|
35
|
+
boss.keep_going.should eq(nil)
|
36
|
+
boss.stop_managing
|
37
|
+
boss.keep_going.should eq(false)
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
describe "#delegate_work" do
|
42
|
+
let(:job1) { double("Job 1") }
|
43
|
+
let(:job2) { double("Job 2") }
|
44
|
+
let(:employee1) { double("Employee 1") }
|
45
|
+
let(:employee2) { double("Employee 2") }
|
46
|
+
let(:employee3) { double("Employee 3") }
|
47
|
+
|
48
|
+
before(:each) do
|
49
|
+
boss.stub(:get_work).and_return(job1, job2, nil, nil)
|
50
|
+
boss.pipeline = pipeline
|
51
|
+
end
|
52
|
+
|
53
|
+
it "puts free employees to work while work is available" do
|
54
|
+
employee1_free = true
|
55
|
+
employee2_free = true
|
56
|
+
employee1.stub(:free?) { employee1_free }
|
57
|
+
employee2.stub(:free?) { employee2_free }
|
58
|
+
employee3.stub(:free?).and_return(false)
|
59
|
+
employee1.should_receive(:work).with(job1) { employee1_free = false }
|
60
|
+
employee2.should_receive(:work).with(job2) { employee2_free = false }
|
61
|
+
employee3.should_receive(:work).never
|
62
|
+
boss.allocate_employee(employee1)
|
63
|
+
boss.allocate_employee(employee2)
|
64
|
+
boss.allocate_employee(employee3)
|
65
|
+
boss.delegate_work
|
66
|
+
end
|
67
|
+
|
68
|
+
it "puts free employees to work" do
|
69
|
+
employee1_free = true
|
70
|
+
employee1.stub(:free?) { employee1_free }
|
71
|
+
employee1.should_receive(:work).with(job1) { employee1_free = false }
|
72
|
+
boss.allocate_employee(employee1)
|
73
|
+
boss.delegate_work
|
74
|
+
|
75
|
+
employee1_free = true
|
76
|
+
employee1.should_receive(:work).with(job2) { employee1_free = false }
|
77
|
+
boss.delegate_work
|
78
|
+
|
79
|
+
employee1_free = true
|
80
|
+
employee1.should_receive(:work).never
|
81
|
+
boss.delegate_work
|
82
|
+
end
|
83
|
+
end
|
84
|
+
|
85
|
+
describe "#get_work" do
|
86
|
+
before(:each) do
|
87
|
+
pipeline.stub(:dequeue).and_return(nil)
|
88
|
+
boss.pipeline = pipeline
|
89
|
+
end
|
90
|
+
|
91
|
+
it "increases sleep time each time it gets no job" do
|
92
|
+
boss.should_receive(:sleep).with(0.5).ordered
|
93
|
+
boss.get_work.should be_nil
|
94
|
+
boss.should_receive(:sleep).with(1).ordered
|
95
|
+
boss.get_work.should be_nil
|
96
|
+
boss.should_receive(:sleep).with(2.5).ordered
|
97
|
+
boss.get_work.should be_nil
|
98
|
+
boss.should_receive(:sleep).with(5).ordered
|
99
|
+
boss.get_work.should be_nil
|
100
|
+
boss.should_receive(:sleep).with(5).ordered
|
101
|
+
boss.get_work.should be_nil
|
102
|
+
end
|
103
|
+
|
104
|
+
it "increases resets sleep time when it gets a job" do
|
105
|
+
job = double("Job")
|
106
|
+
pipeline.stub(:dequeue).and_return(nil, nil, job, job)
|
107
|
+
boss.should_receive(:sleep).with(0.5).ordered
|
108
|
+
boss.get_work.should be_nil
|
109
|
+
boss.should_receive(:sleep).with(1).ordered
|
110
|
+
boss.get_work.should be_nil
|
111
|
+
boss.should_receive(:sleep).with(0.1).ordered
|
112
|
+
boss.get_work.should eq(job)
|
113
|
+
boss.should_receive(:sleep).with(0.1).ordered
|
114
|
+
boss.get_work.should eq(job)
|
115
|
+
end
|
116
|
+
end
|
117
|
+
|
118
|
+
describe "#update_job_status" do
|
119
|
+
before(:each) do
|
120
|
+
boss.pipeline = pipeline
|
121
|
+
end
|
122
|
+
|
123
|
+
let(:job) { double("Job") }
|
124
|
+
let(:employee) { employee = double("Employee", job: job) }
|
125
|
+
|
126
|
+
it "completes the job and frees the employee if employee reports job completed" do
|
127
|
+
employee.should_receive(:work_in_progress?).and_return(false)
|
128
|
+
employee.should_receive(:work_completed?).and_return(true)
|
129
|
+
employee.should_receive(:free)
|
130
|
+
pipeline.should_receive(:complete).with(job)
|
131
|
+
boss.update_job_status(employee)
|
132
|
+
end
|
133
|
+
|
134
|
+
it "fails the job and frees the employee if employee reports job failed, and the job may not be retried" do
|
135
|
+
employee.should_receive(:work_in_progress?).and_return(false)
|
136
|
+
employee.should_receive(:work_completed?).and_return(false)
|
137
|
+
employee.should_receive(:work_failed?).and_return(true)
|
138
|
+
employee.should_receive(:free)
|
139
|
+
job.should_receive(:try_again?).and_return(false)
|
140
|
+
pipeline.should_receive(:fail).with(job)
|
141
|
+
boss.update_job_status(employee)
|
142
|
+
end
|
143
|
+
|
144
|
+
it "resets the job and frees the employee if employee reports job failed, and the job may be retried" do
|
145
|
+
employee.should_receive(:work_in_progress?).and_return(false)
|
146
|
+
employee.should_receive(:work_completed?).and_return(false)
|
147
|
+
employee.should_receive(:work_failed?).and_return(true)
|
148
|
+
employee.should_receive(:free)
|
149
|
+
job.should_receive(:try_again?).and_return(true)
|
150
|
+
pipeline.should_receive(:reset).with(job)
|
151
|
+
boss.update_job_status(employee)
|
152
|
+
end
|
153
|
+
|
154
|
+
it "does nothing when the work is still in progress" do
|
155
|
+
employee.should_receive(:work_in_progress?).and_return(true)
|
156
|
+
boss.update_job_status(employee)
|
157
|
+
end
|
158
|
+
end
|
159
|
+
|
160
|
+
describe "#progess_update" do
|
161
|
+
it "gets progress updates from busy employees with #update_job_status" do
|
162
|
+
busy_employee = double("Busy employee", free?: false)
|
163
|
+
free_employee = double("Free employee", free?: true)
|
164
|
+
boss.allocate_employee(free_employee)
|
165
|
+
boss.allocate_employee(busy_employee)
|
166
|
+
boss.should_receive(:update_job_status).with(busy_employee)
|
167
|
+
boss.should_receive(:update_job_status).with(free_employee).never
|
168
|
+
boss.progress_update
|
169
|
+
end
|
170
|
+
end
|
171
|
+
|
172
|
+
describe "#wait_on_employees" do
|
173
|
+
it "will wait for all busy employees to complete their work, then perform a progress update" do
|
174
|
+
busy_employee = double("Busy employee", free?: false)
|
175
|
+
free_employee = double("Free employee", free?: true)
|
176
|
+
boss.allocate_employee(free_employee)
|
177
|
+
boss.allocate_employee(busy_employee)
|
178
|
+
busy_employee.should_receive(:wait_for_completion)
|
179
|
+
free_employee.should_receive(:wait_for_completion).never
|
180
|
+
boss.should_receive(:update_job_status).with(busy_employee)
|
181
|
+
boss.wait_on_employees
|
182
|
+
end
|
183
|
+
end
|
184
|
+
|
185
|
+
describe "#stop_employees" do
|
186
|
+
it "will force all employees to stop their work" do
|
187
|
+
busy_employee = double("Busy employee", free?: false).as_null_object
|
188
|
+
free_employee = double("Free employee", free?: true)
|
189
|
+
boss.allocate_employee(free_employee)
|
190
|
+
boss.allocate_employee(busy_employee)
|
191
|
+
busy_employee.should_receive(:stop_working)
|
192
|
+
busy_employee.should_receive(:free)
|
193
|
+
free_employee.should_receive(:stop_working).never
|
194
|
+
free_employee.should_receive(:free).never
|
195
|
+
boss.should_receive(:update_job_status).with(busy_employee)
|
196
|
+
boss.stop_employees
|
197
|
+
end
|
198
|
+
end
|
199
|
+
|
200
|
+
describe "#delegate_job" do
|
201
|
+
let(:job) { double("Job") }
|
202
|
+
|
203
|
+
it "will put a free employee to work on a job" do
|
204
|
+
employee.should_receive(:free?).and_return(true)
|
205
|
+
boss.allocate_employee(employee)
|
206
|
+
employee.should_receive(:work).with(job)
|
207
|
+
boss.delegate_job(job)
|
208
|
+
end
|
209
|
+
|
210
|
+
it "will raise when there is no free employee" do
|
211
|
+
expect { boss.delegate_job(job) }.to raise_error(Employer::Errors::NoEmployeeFree)
|
212
|
+
end
|
213
|
+
end
|
214
|
+
|
215
|
+
describe "#free_employee?" do
|
216
|
+
it "true if there is atleast one employee free" do
|
217
|
+
employee.should_receive(:free?).and_return(true)
|
218
|
+
boss.allocate_employee(employee)
|
219
|
+
boss.free_employee?.should be_true
|
220
|
+
end
|
221
|
+
|
222
|
+
it "false if there is no free employee" do
|
223
|
+
employee.should_receive(:free?).and_return(false)
|
224
|
+
boss.allocate_employee(employee)
|
225
|
+
boss.free_employee?.should be_false
|
226
|
+
end
|
227
|
+
end
|
228
|
+
|
229
|
+
describe "#busy_employees" do
|
230
|
+
it "returns the busy employees" do
|
231
|
+
boss.allocate_employee(busy_employee)
|
232
|
+
boss.allocate_employee(free_employee)
|
233
|
+
boss.busy_employees.should eq([busy_employee])
|
234
|
+
end
|
235
|
+
|
236
|
+
it "returns [] when there is are no busy employees" do
|
237
|
+
boss.allocate_employee(free_employee)
|
238
|
+
boss.busy_employees.should eq([])
|
239
|
+
end
|
240
|
+
end
|
241
|
+
|
242
|
+
describe "#free_employee" do
|
243
|
+
it "returns the first free employee" do
|
244
|
+
boss.allocate_employee(busy_employee)
|
245
|
+
boss.allocate_employee(free_employee)
|
246
|
+
boss.free_employee.should eq(free_employee)
|
247
|
+
end
|
248
|
+
|
249
|
+
it "returns nil when there is no free employee" do
|
250
|
+
boss.allocate_employee(busy_employee)
|
251
|
+
boss.free_employee.should be_nil
|
252
|
+
end
|
253
|
+
end
|
254
|
+
end
|