taskinator 0.2.0 → 0.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +10 -0
- data/Gemfile +17 -2
- data/Gemfile.lock +57 -18
- data/README.md +20 -16
- data/lib/taskinator/definition.rb +2 -2
- data/lib/taskinator/instrumentation.rb +77 -0
- data/lib/taskinator/persistence.rb +72 -61
- data/lib/taskinator/process.rb +118 -99
- data/lib/taskinator/queues/delayed_job.rb +0 -14
- data/lib/taskinator/queues/resque.rb +0 -18
- data/lib/taskinator/queues/sidekiq.rb +0 -14
- data/lib/taskinator/queues.rb +0 -5
- data/lib/taskinator/task.rb +113 -70
- data/lib/taskinator/version.rb +1 -1
- data/lib/taskinator/visitor.rb +6 -0
- data/lib/taskinator/workflow.rb +36 -0
- data/lib/taskinator.rb +3 -2
- data/spec/examples/process_examples.rb +6 -9
- data/spec/examples/queue_adapter_examples.rb +2 -12
- data/spec/examples/task_examples.rb +5 -8
- data/spec/support/process_methods.rb +25 -0
- data/spec/support/task_methods.rb +13 -0
- data/spec/support/test_flows.rb +1 -3
- data/spec/support/test_instrumenter.rb +39 -0
- data/spec/support/test_queue.rb +0 -12
- data/spec/taskinator/definition_spec.rb +3 -5
- data/spec/taskinator/instrumentation_spec.rb +98 -0
- data/spec/taskinator/persistence_spec.rb +3 -41
- data/spec/taskinator/process_spec.rb +36 -34
- data/spec/taskinator/queues/delayed_job_spec.rb +0 -41
- data/spec/taskinator/queues/resque_spec.rb +0 -51
- data/spec/taskinator/queues/sidekiq_spec.rb +0 -50
- data/spec/taskinator/queues_spec.rb +1 -1
- data/spec/taskinator/task_spec.rb +96 -64
- data/spec/taskinator/test_flows_spec.rb +266 -1
- data/taskinator.gemspec +0 -21
- metadata +12 -173
- data/lib/taskinator/job_worker.rb +0 -17
- data/spec/taskinator/job_worker_spec.rb +0 -62
data/lib/taskinator/task.rb
CHANGED
@@ -1,7 +1,10 @@
|
|
1
1
|
module Taskinator
|
2
2
|
class Task
|
3
3
|
include ::Comparable
|
4
|
-
|
4
|
+
|
5
|
+
include Workflow
|
6
|
+
include Persistence
|
7
|
+
include Instrumentation
|
5
8
|
|
6
9
|
class << self
|
7
10
|
def define_step_task(process, method, args, options={})
|
@@ -25,6 +28,8 @@ module Taskinator
|
|
25
28
|
attr_reader :uuid
|
26
29
|
attr_reader :options
|
27
30
|
attr_reader :queue
|
31
|
+
attr_reader :created_at
|
32
|
+
attr_reader :updated_at
|
28
33
|
|
29
34
|
# the next task in the sequence
|
30
35
|
attr_accessor :next
|
@@ -36,6 +41,8 @@ module Taskinator
|
|
36
41
|
@process = process
|
37
42
|
@options = options
|
38
43
|
@queue = options.delete(:queue)
|
44
|
+
@created_at = Time.now.utc
|
45
|
+
@updated_at = created_at
|
39
46
|
end
|
40
47
|
|
41
48
|
def accept(visitor)
|
@@ -44,6 +51,8 @@ module Taskinator
|
|
44
51
|
visitor.visit_task_reference(:next)
|
45
52
|
visitor.visit_args(:options)
|
46
53
|
visitor.visit_attribute(:queue)
|
54
|
+
visitor.visit_attribute_time(:created_at)
|
55
|
+
visitor.visit_attribute_time(:updated_at)
|
47
56
|
end
|
48
57
|
|
49
58
|
def <=>(other)
|
@@ -54,73 +63,98 @@ module Taskinator
|
|
54
63
|
"#<#{self.class.name}:#{uuid}>"
|
55
64
|
end
|
56
65
|
|
57
|
-
|
58
|
-
|
59
|
-
event :enqueue, :transitions_to => :enqueued
|
60
|
-
event :start, :transitions_to => :processing
|
61
|
-
event :complete, :transitions_to => :completed # specific to a SubProcess which has no tasks
|
62
|
-
event :fail, :transitions_to => :failed
|
63
|
-
end
|
64
|
-
|
65
|
-
state :enqueued do
|
66
|
-
event :start, :transitions_to => :processing
|
67
|
-
event :complete, :transitions_to => :completed
|
68
|
-
event :fail, :transitions_to => :failed
|
69
|
-
end
|
66
|
+
def enqueue!
|
67
|
+
return if paused? || cancelled?
|
70
68
|
|
71
|
-
|
72
|
-
|
73
|
-
|
69
|
+
transition(:enqueued) do
|
70
|
+
instrument('taskinator.task.enqueued', enqueued_payload) do
|
71
|
+
enqueue
|
72
|
+
end
|
74
73
|
end
|
74
|
+
end
|
75
75
|
|
76
|
-
|
77
|
-
|
76
|
+
def start!
|
77
|
+
return if paused? || cancelled?
|
78
|
+
self.incr_processing if incr_count?
|
78
79
|
|
79
|
-
|
80
|
-
|
80
|
+
transition(:processing) do
|
81
|
+
instrument('taskinator.task.processing', processing_payload) do
|
82
|
+
start
|
83
|
+
end
|
81
84
|
end
|
82
|
-
|
83
85
|
end
|
84
86
|
|
85
|
-
#
|
86
|
-
#
|
87
|
-
#
|
88
|
-
|
87
|
+
#
|
88
|
+
# NOTE: a task can't be paused (it's too difficult to implement)
|
89
|
+
# so rather, the parent process is paused, and the task checks it
|
90
|
+
#
|
89
91
|
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
92
|
+
# helper method
|
93
|
+
def paused?
|
94
|
+
super || process.paused?
|
95
|
+
end
|
96
|
+
|
97
|
+
def complete!
|
98
|
+
transition(:completed) do
|
99
|
+
self.incr_completed if incr_count?
|
100
|
+
instrument('taskinator.task.completed', completed_payload) do
|
101
|
+
complete if respond_to?(:complete)
|
102
|
+
# notify the process that this task has completed
|
103
|
+
process.task_completed(self)
|
104
|
+
end
|
95
105
|
end
|
96
106
|
end
|
97
107
|
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
108
|
+
def cancel!
|
109
|
+
transition(:cancelled) do
|
110
|
+
self.incr_cancelled if incr_count?
|
111
|
+
instrument('taskinator.task.cancelled', cancelled_payload) do
|
112
|
+
cancel if respond_to?(:cancel)
|
113
|
+
end
|
104
114
|
end
|
105
115
|
end
|
106
116
|
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
117
|
+
def cancelled?
|
118
|
+
super || process.cancelled?
|
119
|
+
end
|
120
|
+
|
121
|
+
def fail!(error)
|
122
|
+
transition(:failed) do
|
123
|
+
self.incr_failed if incr_count?
|
124
|
+
instrument('taskinator.task.failed', failed_payload(error)) do
|
125
|
+
fail(error) if respond_to?(:fail)
|
126
|
+
# notify the process that this task has failed
|
127
|
+
process.task_failed(self, error)
|
128
|
+
end
|
111
129
|
end
|
112
130
|
end
|
113
131
|
|
114
|
-
|
115
|
-
|
116
|
-
process.paused?
|
132
|
+
def incr_count?
|
133
|
+
true
|
117
134
|
end
|
118
135
|
|
119
|
-
|
120
|
-
|
121
|
-
|
136
|
+
#--------------------------------------------------
|
137
|
+
# subclasses must implement the following methods
|
138
|
+
#--------------------------------------------------
|
139
|
+
|
140
|
+
def enqueue
|
141
|
+
raise NotImplementedError
|
142
|
+
end
|
143
|
+
|
144
|
+
def start
|
145
|
+
raise NotImplementedError
|
122
146
|
end
|
123
147
|
|
148
|
+
#--------------------------------------------------
|
149
|
+
# and optionally, provide methods:
|
150
|
+
#--------------------------------------------------
|
151
|
+
#
|
152
|
+
# * cancel
|
153
|
+
# * complete
|
154
|
+
# * fail(error)
|
155
|
+
#
|
156
|
+
#--------------------------------------------------
|
157
|
+
|
124
158
|
# a task which invokes the specified method on the definition
|
125
159
|
# the args must be intrinsic types, since they are serialized to YAML
|
126
160
|
class Step < Task
|
@@ -140,15 +174,11 @@ module Taskinator
|
|
140
174
|
end
|
141
175
|
|
142
176
|
def enqueue
|
143
|
-
Taskinator.
|
144
|
-
Taskinator.queue.enqueue_task(self)
|
145
|
-
end
|
177
|
+
Taskinator.queue.enqueue_task(self)
|
146
178
|
end
|
147
179
|
|
148
180
|
def start
|
149
|
-
|
150
|
-
executor.send(method, *args)
|
151
|
-
end
|
181
|
+
executor.send(method, *args)
|
152
182
|
# ASSUMPTION: when the method returns, the task is considered to be complete
|
153
183
|
complete!
|
154
184
|
|
@@ -171,10 +201,12 @@ module Taskinator
|
|
171
201
|
end
|
172
202
|
|
173
203
|
def inspect
|
174
|
-
%(#<#{self.class.name}:0x#{self.__id__.to_s(16)} uuid="#{uuid}", method=:#{method}, args=#{args},
|
204
|
+
%(#<#{self.class.name}:0x#{self.__id__.to_s(16)} uuid="#{uuid}", method=:#{method}, args=#{args}, current_state=:#{current_state}>)
|
175
205
|
end
|
176
206
|
end
|
177
207
|
|
208
|
+
#--------------------------------------------------
|
209
|
+
|
178
210
|
# a task which invokes the specified background job
|
179
211
|
# the args must be intrinsic types, since they are serialized to YAML
|
180
212
|
class Job < Task
|
@@ -194,17 +226,22 @@ module Taskinator
|
|
194
226
|
end
|
195
227
|
|
196
228
|
def enqueue
|
197
|
-
Taskinator.
|
198
|
-
Taskinator.queue.enqueue_job(self)
|
199
|
-
end
|
229
|
+
Taskinator.queue.enqueue_task(self)
|
200
230
|
end
|
201
231
|
|
202
|
-
|
203
|
-
|
204
|
-
|
205
|
-
|
232
|
+
def start
|
233
|
+
# NNB: if other job types are required, may need to implement how they get invoked here!
|
234
|
+
# FIXME: possible implement using ActiveJob instead, so it doesn't matter how the worker is implemented
|
235
|
+
|
236
|
+
if job.instance_of?(Module)
|
237
|
+
# resque
|
238
|
+
job.perform(args)
|
239
|
+
else
|
240
|
+
# delayedjob and sidekiq
|
241
|
+
job.new.perform(args)
|
206
242
|
end
|
207
|
-
|
243
|
+
|
244
|
+
# ASSUMPTION: when the job returns, the task is considered to be complete
|
208
245
|
complete!
|
209
246
|
|
210
247
|
rescue => e
|
@@ -222,14 +259,18 @@ module Taskinator
|
|
222
259
|
end
|
223
260
|
|
224
261
|
def inspect
|
225
|
-
%(#<#{self.class.name}:0x#{self.__id__.to_s(16)} uuid="#{uuid}", job=#{job}, args=#{args},
|
262
|
+
%(#<#{self.class.name}:0x#{self.__id__.to_s(16)} uuid="#{uuid}", job=#{job}, args=#{args}, current_state=:#{current_state}>)
|
226
263
|
end
|
227
264
|
end
|
228
265
|
|
266
|
+
#--------------------------------------------------
|
267
|
+
|
229
268
|
# a task which delegates to another process
|
230
269
|
class SubProcess < Task
|
231
270
|
attr_reader :sub_process
|
232
271
|
|
272
|
+
# NOTE: also wraps sequential and concurrent processes
|
273
|
+
|
233
274
|
def initialize(process, sub_process, options={})
|
234
275
|
super(process, options)
|
235
276
|
raise ArgumentError, 'sub_process' if sub_process.nil? || !sub_process.is_a?(Process)
|
@@ -239,15 +280,11 @@ module Taskinator
|
|
239
280
|
end
|
240
281
|
|
241
282
|
def enqueue
|
242
|
-
|
243
|
-
sub_process.enqueue!
|
244
|
-
end
|
283
|
+
sub_process.enqueue!
|
245
284
|
end
|
246
285
|
|
247
286
|
def start
|
248
|
-
|
249
|
-
sub_process.start!
|
250
|
-
end
|
287
|
+
sub_process.start!
|
251
288
|
|
252
289
|
rescue => e
|
253
290
|
Taskinator.logger.error(e)
|
@@ -256,13 +293,19 @@ module Taskinator
|
|
256
293
|
raise e
|
257
294
|
end
|
258
295
|
|
296
|
+
def incr_count?
|
297
|
+
# subprocess tasks aren't included in the total count of tasks
|
298
|
+
# since they simply delegate to the tasks of the respective subprocess definition
|
299
|
+
false
|
300
|
+
end
|
301
|
+
|
259
302
|
def accept(visitor)
|
260
303
|
super
|
261
304
|
visitor.visit_process(:sub_process)
|
262
305
|
end
|
263
306
|
|
264
307
|
def inspect
|
265
|
-
%(#<#{self.class.name}:0x#{self.__id__.to_s(16)} uuid="#{uuid}", sub_process=#{sub_process.inspect},
|
308
|
+
%(#<#{self.class.name}:0x#{self.__id__.to_s(16)} uuid="#{uuid}", sub_process=#{sub_process.inspect}, current_state=:#{current_state}>)
|
266
309
|
end
|
267
310
|
end
|
268
311
|
end
|
data/lib/taskinator/version.rb
CHANGED
data/lib/taskinator/visitor.rb
CHANGED
@@ -0,0 +1,36 @@
|
|
1
|
+
module Taskinator
|
2
|
+
module Workflow
|
3
|
+
|
4
|
+
def current_state
|
5
|
+
@current_state ||= load_workflow_state
|
6
|
+
end
|
7
|
+
|
8
|
+
def current_state=(new_state)
|
9
|
+
@current_state = persist_workflow_state(new_state)
|
10
|
+
end
|
11
|
+
|
12
|
+
def transition(new_state)
|
13
|
+
self.current_state = new_state
|
14
|
+
yield if block_given?
|
15
|
+
current_state
|
16
|
+
end
|
17
|
+
|
18
|
+
%i(
|
19
|
+
initial
|
20
|
+
enqueued
|
21
|
+
processing
|
22
|
+
paused
|
23
|
+
resumed
|
24
|
+
completed
|
25
|
+
cancelled
|
26
|
+
failed
|
27
|
+
).each do |state|
|
28
|
+
|
29
|
+
define_method :"#{state}?" do
|
30
|
+
@current_state == state
|
31
|
+
end
|
32
|
+
|
33
|
+
end
|
34
|
+
|
35
|
+
end
|
36
|
+
end
|
data/lib/taskinator.rb
CHANGED
@@ -1,7 +1,6 @@
|
|
1
1
|
require 'json'
|
2
2
|
require 'yaml'
|
3
3
|
require 'securerandom'
|
4
|
-
require 'workflow'
|
5
4
|
|
6
5
|
require 'taskinator/version'
|
7
6
|
|
@@ -11,15 +10,17 @@ require 'taskinator/logger'
|
|
11
10
|
|
12
11
|
require 'taskinator/definition'
|
13
12
|
|
13
|
+
require 'taskinator/workflow'
|
14
|
+
|
14
15
|
require 'taskinator/visitor'
|
15
16
|
require 'taskinator/persistence'
|
17
|
+
require 'taskinator/instrumentation'
|
16
18
|
|
17
19
|
require 'taskinator/task'
|
18
20
|
require 'taskinator/tasks'
|
19
21
|
require 'taskinator/process'
|
20
22
|
|
21
23
|
require 'taskinator/task_worker'
|
22
|
-
require 'taskinator/job_worker'
|
23
24
|
require 'taskinator/create_process_worker'
|
24
25
|
|
25
26
|
require 'taskinator/executor'
|
@@ -2,15 +2,12 @@ require 'spec_helper'
|
|
2
2
|
|
3
3
|
shared_examples_for "a process" do |process_type|
|
4
4
|
|
5
|
-
# NOTE: definition and
|
5
|
+
# NOTE: definition and subject must be defined by callee
|
6
6
|
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
it { expect(
|
11
|
-
it { expect(
|
12
|
-
it { expect(process.to_s).to match(/#{process.uuid}/) }
|
13
|
-
it { expect(process.options).to_not be_nil }
|
14
|
-
it { expect(process.tasks).to_not be_nil }
|
7
|
+
it { expect(subject.definition).to eq(definition) }
|
8
|
+
it { expect(subject.uuid).to_not be_nil }
|
9
|
+
it { expect(subject.to_s).to match(/#{subject.uuid}/) }
|
10
|
+
it { expect(subject.options).to_not be_nil }
|
11
|
+
it { expect(subject.tasks).to_not be_nil }
|
15
12
|
|
16
13
|
end
|
@@ -16,7 +16,7 @@ shared_examples_for "a queue adapter" do |adapter_name, adapter_type|
|
|
16
16
|
it "should enqueue a create process" do
|
17
17
|
expect {
|
18
18
|
subject.enqueue_create_process(double('definition', :name => 'definition', :queue => nil), 'xx-xx-xx-xx', :foo => :bar)
|
19
|
-
}.to_not raise_error
|
19
|
+
}.to_not raise_error(StandardError)
|
20
20
|
end
|
21
21
|
end
|
22
22
|
|
@@ -26,17 +26,7 @@ shared_examples_for "a queue adapter" do |adapter_name, adapter_type|
|
|
26
26
|
it "should enqueue a task" do
|
27
27
|
expect {
|
28
28
|
subject.enqueue_task(double('task', :uuid => 'xx-xx-xx-xx', :queue => nil))
|
29
|
-
}.to_not raise_error
|
30
|
-
end
|
31
|
-
end
|
32
|
-
|
33
|
-
describe "#enqueue_job" do
|
34
|
-
it { expect(subject).to respond_to(:enqueue_job) }
|
35
|
-
|
36
|
-
it "should enqueue a job" do
|
37
|
-
expect {
|
38
|
-
subject.enqueue_job(double('job', :uuid => 'xx-xx-xx-xx', :job => job, :queue => nil))
|
39
|
-
}.to_not raise_error
|
29
|
+
}.to_not raise_error(StandardError)
|
40
30
|
end
|
41
31
|
end
|
42
32
|
|
@@ -2,14 +2,11 @@ require 'spec_helper'
|
|
2
2
|
|
3
3
|
shared_examples_for "a task" do |task_type|
|
4
4
|
|
5
|
-
# NOTE: process and
|
5
|
+
# NOTE: process and subject must be defined by callee
|
6
6
|
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
it { expect(
|
11
|
-
it { expect(task.uuid).to_not be_nil }
|
12
|
-
it { expect(task.to_s).to match(/#{task.uuid}/) }
|
13
|
-
it { expect(task.options).to_not be_nil }
|
7
|
+
it { expect(subject.process).to eq(process) }
|
8
|
+
it { expect(subject.uuid).to_not be_nil }
|
9
|
+
it { expect(subject.to_s).to match(/#{subject.uuid}/) }
|
10
|
+
it { expect(subject.options).to_not be_nil }
|
14
11
|
|
15
12
|
end
|
data/spec/support/test_flows.rb
CHANGED
@@ -0,0 +1,39 @@
|
|
1
|
+
class TestInstrumenter
|
2
|
+
|
3
|
+
class << self
|
4
|
+
|
5
|
+
def subscribe(callback, filter=nil, &block)
|
6
|
+
|
7
|
+
# create test instrumenter instance
|
8
|
+
instrumenter = TestInstrumenter.new do |name, payload|
|
9
|
+
if filter
|
10
|
+
callback.call(name, payload) if name =~ filter
|
11
|
+
else
|
12
|
+
callback.call(name, payload)
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
# hook up this instrumenter in the context of the spec
|
17
|
+
# (assuming called from RSpec binding)
|
18
|
+
spec_binding = block.binding.eval('self')
|
19
|
+
spec_binding.instance_exec do
|
20
|
+
allow(Taskinator).to receive(:instrumenter).and_return(instrumenter)
|
21
|
+
end
|
22
|
+
|
23
|
+
yield
|
24
|
+
end
|
25
|
+
|
26
|
+
end
|
27
|
+
|
28
|
+
attr_reader :callback
|
29
|
+
|
30
|
+
def initialize(&block)
|
31
|
+
@callback = block
|
32
|
+
end
|
33
|
+
|
34
|
+
def instrument(event, payload={})
|
35
|
+
@callback.call(event, payload)
|
36
|
+
yield(payload) if block_given?
|
37
|
+
end
|
38
|
+
|
39
|
+
end
|
data/spec/support/test_queue.rb
CHANGED
@@ -13,7 +13,6 @@ module Taskinator
|
|
13
13
|
|
14
14
|
attr_reader :creates
|
15
15
|
attr_reader :tasks
|
16
|
-
attr_reader :jobs
|
17
16
|
|
18
17
|
def initialize
|
19
18
|
clear
|
@@ -33,10 +32,6 @@ module Taskinator
|
|
33
32
|
@tasks << task
|
34
33
|
end
|
35
34
|
|
36
|
-
def enqueue_job(job)
|
37
|
-
@jobs << job
|
38
|
-
end
|
39
|
-
|
40
35
|
def empty?
|
41
36
|
@creates.empty? && @tasks.empty? && @jobs.empty?
|
42
37
|
end
|
@@ -62,13 +57,6 @@ module Taskinator
|
|
62
57
|
end
|
63
58
|
end
|
64
59
|
|
65
|
-
def enqueue_job(job)
|
66
|
-
super
|
67
|
-
invoke do
|
68
|
-
Taskinator::JobWorker.new(job.uuid).perform
|
69
|
-
end
|
70
|
-
end
|
71
|
-
|
72
60
|
def invoke(&block)
|
73
61
|
block.call
|
74
62
|
end
|
@@ -136,7 +136,7 @@ describe Taskinator::Definition do
|
|
136
136
|
# if an error is raised, then the context was incorrect
|
137
137
|
expect {
|
138
138
|
subject.create_process
|
139
|
-
}.to_not raise_error
|
139
|
+
}.to_not raise_error(StandardError)
|
140
140
|
end
|
141
141
|
|
142
142
|
context "is instrumented" do
|
@@ -148,8 +148,7 @@ describe Taskinator::Definition do
|
|
148
148
|
expect(args.first).to eq('taskinator.process.created')
|
149
149
|
end
|
150
150
|
|
151
|
-
|
152
|
-
ActiveSupport::Notifications.subscribed(instrumentation_block, /taskinator.process.created/) do
|
151
|
+
TestInstrumenter.subscribe(instrumentation_block, /taskinator.process.created/) do
|
153
152
|
subject.create_process :foo
|
154
153
|
end
|
155
154
|
end
|
@@ -160,8 +159,7 @@ describe Taskinator::Definition do
|
|
160
159
|
expect(args.first).to eq('taskinator.process.saved')
|
161
160
|
end
|
162
161
|
|
163
|
-
|
164
|
-
ActiveSupport::Notifications.subscribed(instrumentation_block, /taskinator.process.saved/) do
|
162
|
+
TestInstrumenter.subscribe(instrumentation_block, /taskinator.process.saved/) do
|
165
163
|
subject.create_process :foo
|
166
164
|
end
|
167
165
|
end
|