dwf 0.1.7 → 0.1.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/CHANGELOG.md +137 -0
- data/README.md +155 -33
- data/dwf.gemspec +2 -1
- data/lib/dwf/callback.rb +13 -10
- data/lib/dwf/client.rb +76 -1
- data/lib/dwf/concerns/checkable.rb +29 -0
- data/lib/dwf/configuration.rb +1 -1
- data/lib/dwf/errors.rb +5 -0
- data/lib/dwf/item.rb +71 -37
- data/lib/dwf/utils.rb +6 -0
- data/lib/dwf/version.rb +1 -1
- data/lib/dwf/worker.rb +1 -1
- data/lib/dwf/workflow.rb +145 -34
- data/spec/dwf/client_spec.rb +100 -3
- data/spec/dwf/item_spec.rb +81 -6
- data/spec/dwf/utils_spec.rb +9 -0
- data/spec/dwf/worker_spec.rb +32 -0
- data/spec/dwf/workflow_spec.rb +349 -0
- data/spec/spec_helper.rb +3 -0
- metadata +32 -12
data/lib/dwf/item.rb
CHANGED
@@ -1,35 +1,36 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
require_relative 'client'
|
4
|
+
require_relative 'concerns/checkable'
|
4
5
|
|
5
6
|
module Dwf
|
6
7
|
class Item
|
8
|
+
include Concerns::Checkable
|
9
|
+
|
7
10
|
attr_reader :workflow_id, :id, :params, :queue, :klass, :started_at,
|
8
|
-
|
9
|
-
|
11
|
+
:enqueued_at, :finished_at, :failed_at, :output_payload
|
12
|
+
attr_writer :payloads
|
13
|
+
attr_accessor :incoming, :outgoing, :callback_type
|
10
14
|
|
11
15
|
def initialize(options = {})
|
12
|
-
|
13
|
-
@id = options[:id]
|
14
|
-
@params = options[:params]
|
15
|
-
@queue = options[:queue]
|
16
|
-
@incoming = options[:incoming] || []
|
17
|
-
@outgoing = options[:outgoing] || []
|
18
|
-
@klass = options[:klass] || self.class
|
19
|
-
@failed_at = options[:failed_at]
|
20
|
-
@finished_at = options[:finished_at]
|
21
|
-
@enqueued_at = options[:enqueued_at]
|
22
|
-
@started_at = options[:started_at]
|
23
|
-
@callback_type = options[:callback_type]
|
16
|
+
assign_attributes(options)
|
24
17
|
end
|
25
18
|
|
26
19
|
def self.from_hash(hash)
|
27
20
|
Module.const_get(hash[:klass]).new(hash)
|
28
21
|
end
|
29
22
|
|
23
|
+
def start_initial!
|
24
|
+
cb_build_in? ? persist_and_perform_async! : start_batch!
|
25
|
+
end
|
26
|
+
|
27
|
+
def start_batch!
|
28
|
+
enqueue_and_persist!
|
29
|
+
Dwf::Callback.new.start(self)
|
30
|
+
end
|
31
|
+
|
30
32
|
def persist_and_perform_async!
|
31
|
-
|
32
|
-
persist!
|
33
|
+
enqueue_and_persist!
|
33
34
|
perform_async
|
34
35
|
end
|
35
36
|
|
@@ -39,23 +40,34 @@ module Dwf
|
|
39
40
|
callback_type == Dwf::Workflow::BUILD_IN
|
40
41
|
end
|
41
42
|
|
43
|
+
def reload
|
44
|
+
item = client.find_job(workflow_id, name)
|
45
|
+
assign_attributes(item.to_hash)
|
46
|
+
end
|
47
|
+
|
42
48
|
def perform_async
|
43
49
|
Dwf::Worker.set(queue: queue || client.config.namespace)
|
44
|
-
|
50
|
+
.perform_async(workflow_id, name)
|
45
51
|
end
|
46
52
|
|
47
53
|
def name
|
48
54
|
@name ||= "#{klass}|#{id}"
|
49
55
|
end
|
50
56
|
|
57
|
+
def output(data)
|
58
|
+
@output_payload = data
|
59
|
+
end
|
60
|
+
|
51
61
|
def no_dependencies?
|
52
62
|
incoming.empty?
|
53
63
|
end
|
54
64
|
|
55
65
|
def parents_succeeded?
|
56
|
-
incoming.all?
|
57
|
-
|
58
|
-
|
66
|
+
incoming.all? { |name| client.find_node(name, workflow_id).succeeded? }
|
67
|
+
end
|
68
|
+
|
69
|
+
def payloads
|
70
|
+
@payloads ||= build_payloads
|
59
71
|
end
|
60
72
|
|
61
73
|
def enqueue!
|
@@ -100,22 +112,6 @@ module Dwf
|
|
100
112
|
!failed_at.nil?
|
101
113
|
end
|
102
114
|
|
103
|
-
def succeeded?
|
104
|
-
finished? && !failed?
|
105
|
-
end
|
106
|
-
|
107
|
-
def started?
|
108
|
-
!started_at.nil?
|
109
|
-
end
|
110
|
-
|
111
|
-
def running?
|
112
|
-
started? && !finished?
|
113
|
-
end
|
114
|
-
|
115
|
-
def ready_to_start?
|
116
|
-
!running? && !enqueued? && !finished? && !failed? && parents_succeeded?
|
117
|
-
end
|
118
|
-
|
119
115
|
def current_timestamp
|
120
116
|
Time.now.to_i
|
121
117
|
end
|
@@ -142,7 +138,9 @@ module Dwf
|
|
142
138
|
failed_at: failed_at,
|
143
139
|
params: params,
|
144
140
|
workflow_id: workflow_id,
|
145
|
-
callback_type: callback_type
|
141
|
+
callback_type: callback_type,
|
142
|
+
output_payload: output_payload,
|
143
|
+
payloads: @payloads
|
146
144
|
}
|
147
145
|
end
|
148
146
|
|
@@ -156,8 +154,44 @@ module Dwf
|
|
156
154
|
|
157
155
|
private
|
158
156
|
|
157
|
+
def enqueue_and_persist!
|
158
|
+
enqueue!
|
159
|
+
persist!
|
160
|
+
end
|
161
|
+
|
159
162
|
def client
|
160
163
|
@client ||= Dwf::Client.new
|
161
164
|
end
|
165
|
+
|
166
|
+
def assign_attributes(options)
|
167
|
+
@workflow_id = options[:workflow_id]
|
168
|
+
@id = options[:id]
|
169
|
+
@params = options[:params]
|
170
|
+
@queue = options[:queue]
|
171
|
+
@incoming = options[:incoming] || []
|
172
|
+
@outgoing = options[:outgoing] || []
|
173
|
+
@klass = options[:klass] || self.class
|
174
|
+
@failed_at = options[:failed_at]
|
175
|
+
@finished_at = options[:finished_at]
|
176
|
+
@enqueued_at = options[:enqueued_at]
|
177
|
+
@started_at = options[:started_at]
|
178
|
+
@callback_type = options[:callback_type]
|
179
|
+
@output_payload = options[:output_payload]
|
180
|
+
@payloads = options[:payloads]
|
181
|
+
end
|
182
|
+
|
183
|
+
def build_payloads
|
184
|
+
data = incoming.map do |job_name|
|
185
|
+
node = client.find_node(job_name, workflow_id)
|
186
|
+
next if node.output_payload.nil?
|
187
|
+
|
188
|
+
{
|
189
|
+
id: node.name,
|
190
|
+
class: node.klass.to_s,
|
191
|
+
output: node.output_payload
|
192
|
+
}
|
193
|
+
end.compact
|
194
|
+
data.empty? ? nil : data
|
195
|
+
end
|
162
196
|
end
|
163
197
|
end
|
data/lib/dwf/utils.rb
CHANGED
data/lib/dwf/version.rb
CHANGED
data/lib/dwf/worker.rb
CHANGED
data/lib/dwf/workflow.rb
CHANGED
@@ -2,15 +2,19 @@
|
|
2
2
|
|
3
3
|
require_relative 'client'
|
4
4
|
require_relative 'worker'
|
5
|
+
require_relative 'concerns/checkable'
|
5
6
|
require_relative 'callback'
|
6
7
|
|
7
8
|
module Dwf
|
8
9
|
class Workflow
|
10
|
+
include Concerns::Checkable
|
11
|
+
|
9
12
|
CALLBACK_TYPES = [
|
10
13
|
BUILD_IN = 'build-in',
|
11
14
|
SK_BATCH = 'sk-batch'
|
12
15
|
].freeze
|
13
|
-
|
16
|
+
attr_accessor :jobs, :stopped, :id, :incoming, :outgoing, :parent_id
|
17
|
+
attr_reader :dependencies, :started_at, :finished_at, :persisted, :arguments, :klass,
|
14
18
|
:callback_type
|
15
19
|
|
16
20
|
class << self
|
@@ -19,51 +23,91 @@ module Dwf
|
|
19
23
|
flow.save
|
20
24
|
flow
|
21
25
|
end
|
26
|
+
|
27
|
+
def find(id)
|
28
|
+
Dwf::Client.new.find_workflow(id)
|
29
|
+
end
|
22
30
|
end
|
23
31
|
|
24
|
-
def initialize(
|
32
|
+
def initialize(*args)
|
25
33
|
@dependencies = []
|
26
|
-
@id =
|
34
|
+
@id = build_id
|
27
35
|
@jobs = []
|
28
36
|
@persisted = false
|
29
37
|
@stopped = false
|
30
|
-
@
|
38
|
+
@arguments = *args
|
39
|
+
@parent_id = nil
|
40
|
+
@klass = self.class
|
41
|
+
@callback_type = BUILD_IN
|
42
|
+
@incoming = []
|
43
|
+
@outgoing = []
|
31
44
|
|
32
45
|
setup
|
33
46
|
end
|
34
47
|
|
48
|
+
def persist!
|
49
|
+
client.persist_workflow(self)
|
50
|
+
jobs.each(&:persist!)
|
51
|
+
mark_as_persisted
|
52
|
+
true
|
53
|
+
end
|
54
|
+
|
55
|
+
def name
|
56
|
+
"#{self.class.name}|#{id}"
|
57
|
+
end
|
58
|
+
|
59
|
+
def sub_workflow?
|
60
|
+
!parent_id.nil?
|
61
|
+
end
|
62
|
+
|
63
|
+
def callback_type=(type = BUILD_IN)
|
64
|
+
@callback_type = type
|
65
|
+
jobs.each { |job| job.callback_type = type }
|
66
|
+
end
|
67
|
+
|
68
|
+
alias save persist!
|
69
|
+
|
35
70
|
def start!
|
71
|
+
raise UnsupportCallback, 'Sub workflow only works with Sidekiq batch callback' if invalid_callback?
|
72
|
+
|
73
|
+
mark_as_started
|
74
|
+
persist!
|
36
75
|
initial_jobs.each do |job|
|
37
|
-
|
76
|
+
job.payloads = payloads if sub_workflow?
|
77
|
+
job.start_initial!
|
38
78
|
end
|
39
79
|
end
|
40
80
|
|
41
|
-
def
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
81
|
+
def payloads
|
82
|
+
@payloads ||= build_payloads
|
83
|
+
end
|
84
|
+
|
85
|
+
def start_initial!
|
86
|
+
cb_build_in? ? start! : Callback.new.start(self)
|
87
|
+
end
|
88
|
+
|
89
|
+
alias persist_and_perform_async! start!
|
90
|
+
|
91
|
+
def reload
|
92
|
+
flow = self.class.find(id)
|
93
|
+
self.stopped = flow.stopped
|
94
|
+
self.jobs = flow.jobs
|
95
|
+
|
96
|
+
self
|
46
97
|
end
|
47
98
|
|
48
99
|
def cb_build_in?
|
49
100
|
callback_type == BUILD_IN
|
50
101
|
end
|
51
102
|
|
52
|
-
def
|
53
|
-
|
103
|
+
def build_id
|
104
|
+
client.build_workflow_id
|
54
105
|
end
|
55
106
|
|
56
|
-
def configure; end
|
107
|
+
def configure(*arguments); end
|
57
108
|
|
58
109
|
def run(klass, options = {})
|
59
|
-
node = klass
|
60
|
-
workflow_id: id,
|
61
|
-
id: client.build_job_id(id, klass.to_s),
|
62
|
-
params: options.fetch(:params, {}),
|
63
|
-
queue: options[:queue],
|
64
|
-
callback_type: callback_type
|
65
|
-
)
|
66
|
-
|
110
|
+
node = build_node(klass, options)
|
67
111
|
jobs << node
|
68
112
|
|
69
113
|
build_dependencies_structure(node, options)
|
@@ -95,7 +139,10 @@ module Dwf
|
|
95
139
|
stopped: stopped,
|
96
140
|
started_at: started_at,
|
97
141
|
finished_at: finished_at,
|
98
|
-
callback_type: callback_type
|
142
|
+
callback_type: callback_type,
|
143
|
+
incoming: incoming,
|
144
|
+
outgoing: outgoing,
|
145
|
+
parent_id: parent_id
|
99
146
|
}
|
100
147
|
end
|
101
148
|
|
@@ -107,13 +154,7 @@ module Dwf
|
|
107
154
|
jobs.all?(&:finished?)
|
108
155
|
end
|
109
156
|
|
110
|
-
|
111
|
-
!!started_at
|
112
|
-
end
|
113
|
-
|
114
|
-
def running?
|
115
|
-
started? && !finished?
|
116
|
-
end
|
157
|
+
alias enqueued? started?
|
117
158
|
|
118
159
|
def failed?
|
119
160
|
jobs.any?(&:failed?)
|
@@ -123,6 +164,10 @@ module Dwf
|
|
123
164
|
stopped
|
124
165
|
end
|
125
166
|
|
167
|
+
def parents_succeeded?
|
168
|
+
incoming.all? { |name| client.find_node(name, parent_id).succeeded? }
|
169
|
+
end
|
170
|
+
|
126
171
|
def status
|
127
172
|
return :failed if failed?
|
128
173
|
return :running if running?
|
@@ -140,28 +185,94 @@ module Dwf
|
|
140
185
|
@stopped = false
|
141
186
|
end
|
142
187
|
|
188
|
+
def leaf_nodes
|
189
|
+
jobs.select(&:leaf?)
|
190
|
+
end
|
191
|
+
|
192
|
+
def output_payload
|
193
|
+
leaf_nodes.map do |node|
|
194
|
+
data = node.output_payload
|
195
|
+
next if data.nil?
|
196
|
+
|
197
|
+
data
|
198
|
+
end.compact
|
199
|
+
end
|
143
200
|
|
144
201
|
private
|
145
202
|
|
203
|
+
def build_node(klass, options)
|
204
|
+
if klass < Dwf::Workflow
|
205
|
+
node = options[:params].nil? ? klass.new : klass.new(options[:params])
|
206
|
+
node.parent_id = id
|
207
|
+
node.callback_type = SK_BATCH
|
208
|
+
node.save
|
209
|
+
node
|
210
|
+
else
|
211
|
+
klass.new(
|
212
|
+
workflow_id: id,
|
213
|
+
id: client.build_job_id(id, klass.to_s),
|
214
|
+
params: options.fetch(:params, {}),
|
215
|
+
queue: options[:queue],
|
216
|
+
callback_type: callback_type
|
217
|
+
)
|
218
|
+
end
|
219
|
+
end
|
220
|
+
|
146
221
|
def initial_jobs
|
147
222
|
jobs.select(&:no_dependencies?)
|
148
223
|
end
|
149
224
|
|
150
225
|
def setup
|
151
|
-
configure
|
226
|
+
configure(*arguments)
|
152
227
|
resolve_dependencies
|
153
228
|
end
|
154
229
|
|
230
|
+
def find_node(node_name)
|
231
|
+
if Utils.workflow_name?(node_name)
|
232
|
+
find_subworkflow(node_name)
|
233
|
+
else
|
234
|
+
find_job(node_name)
|
235
|
+
end
|
236
|
+
end
|
237
|
+
|
238
|
+
def find_subworkflow(node_name)
|
239
|
+
fname, = node_name.split('|')
|
240
|
+
jobs.find { |j| j.klass.name == fname }
|
241
|
+
end
|
242
|
+
|
155
243
|
def resolve_dependencies
|
156
244
|
@dependencies.each do |dependency|
|
157
|
-
from =
|
158
|
-
to =
|
245
|
+
from = find_node(dependency[:from])
|
246
|
+
to = find_node(dependency[:to])
|
159
247
|
|
160
|
-
to.incoming <<
|
161
|
-
from.outgoing <<
|
248
|
+
to.incoming << from.name
|
249
|
+
from.outgoing << to.name
|
162
250
|
end
|
163
251
|
end
|
164
252
|
|
253
|
+
def invalid_callback?
|
254
|
+
cb_build_in? && jobs.any? { |job| job.class < Workflow }
|
255
|
+
end
|
256
|
+
|
257
|
+
def build_payloads
|
258
|
+
return unless sub_workflow?
|
259
|
+
|
260
|
+
data = incoming.map do |job_name|
|
261
|
+
next if Utils.workflow_name?(job_name)
|
262
|
+
|
263
|
+
node = client.find_node(job_name, parent_id)
|
264
|
+
next if node.output_payload.nil?
|
265
|
+
|
266
|
+
{
|
267
|
+
id: node.name,
|
268
|
+
class: node.klass.to_s,
|
269
|
+
output: node.output_payload
|
270
|
+
}
|
271
|
+
end.compact
|
272
|
+
|
273
|
+
data.empty? ? nil : data
|
274
|
+
end
|
275
|
+
|
165
276
|
def build_dependencies_structure(node, options)
|
166
277
|
deps_after = [*options[:after]]
|
167
278
|
|
data/spec/dwf/client_spec.rb
CHANGED
@@ -2,6 +2,8 @@
|
|
2
2
|
|
3
3
|
require 'spec_helper'
|
4
4
|
require 'mock_redis'
|
5
|
+
FirstWorkflow = Class.new(Dwf::Workflow)
|
6
|
+
SecondWorkflow = Class.new(Dwf::Workflow)
|
5
7
|
|
6
8
|
describe Dwf::Client, client: true do
|
7
9
|
let(:client) { described_class.new }
|
@@ -30,12 +32,76 @@ describe Dwf::Client, client: true do
|
|
30
32
|
end
|
31
33
|
|
32
34
|
context 'find by item name' do
|
33
|
-
it
|
35
|
+
it do
|
34
36
|
item = client.find_job(workflow_id, job.name)
|
35
37
|
expect(item.workflow_id).to eq workflow_id
|
36
38
|
expect(item.id).to eq id
|
37
39
|
expect(item.name).to eq job.name
|
38
|
-
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
describe '#find_workflow' do
|
45
|
+
before do
|
46
|
+
wf = Dwf::Workflow.new
|
47
|
+
wf.id = workflow_id
|
48
|
+
wf.save
|
49
|
+
j = Dwf::Item.new(id: id, workflow_id: workflow_id)
|
50
|
+
j.persist!
|
51
|
+
end
|
52
|
+
|
53
|
+
it do
|
54
|
+
wf = client.find_workflow(workflow_id)
|
55
|
+
|
56
|
+
expect(wf).not_to be_nil
|
57
|
+
expect(wf.jobs.first).to be_kind_of(Dwf::Item)
|
58
|
+
end
|
59
|
+
|
60
|
+
it do
|
61
|
+
expect do
|
62
|
+
client.find_workflow(SecureRandom.uuid)
|
63
|
+
end.to raise_error Dwf::WorkflowNotFound
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
describe '#find_node' do
|
68
|
+
context 'find job' do
|
69
|
+
let!(:job) do
|
70
|
+
j = Dwf::Item.new(workflow_id: workflow_id, id: id)
|
71
|
+
j.persist!
|
72
|
+
j
|
73
|
+
end
|
74
|
+
|
75
|
+
it do
|
76
|
+
item = client.find_node(Dwf::Item.name, workflow_id)
|
77
|
+
expect(item.workflow_id).to eq workflow_id
|
78
|
+
expect(item.id).to eq id
|
79
|
+
expect(item.name).to eq job.name
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
83
|
+
context 'find_workflow' do
|
84
|
+
let!(:wf1) { FirstWorkflow.create }
|
85
|
+
let!(:wf2) do
|
86
|
+
wf = SecondWorkflow.new
|
87
|
+
wf.parent_id = wf1.id
|
88
|
+
wf.save
|
89
|
+
wf
|
90
|
+
end
|
91
|
+
|
92
|
+
context 'find with class name and parent id' do
|
93
|
+
it do
|
94
|
+
wf = client.find_node(wf2.class.name, wf1.id)
|
95
|
+
expect(wf).to be_kind_of(SecondWorkflow)
|
96
|
+
end
|
97
|
+
end
|
98
|
+
|
99
|
+
context 'find with name and parent id' do
|
100
|
+
it do
|
101
|
+
wf = client.find_node(wf2.name, wf1.id)
|
102
|
+
expect(wf).to be_kind_of(SecondWorkflow)
|
103
|
+
end
|
104
|
+
end
|
39
105
|
end
|
40
106
|
end
|
41
107
|
|
@@ -53,13 +119,44 @@ describe Dwf::Client, client: true do
|
|
53
119
|
end
|
54
120
|
end
|
55
121
|
|
122
|
+
describe '#find_sub_workflow' do
|
123
|
+
let!(:wf1) { FirstWorkflow.create }
|
124
|
+
let!(:wf2) do
|
125
|
+
wf = SecondWorkflow.new
|
126
|
+
wf.parent_id = wf1.id
|
127
|
+
wf.save
|
128
|
+
wf
|
129
|
+
end
|
130
|
+
|
131
|
+
it do
|
132
|
+
wf = client.find_sub_workflow(wf2.class.name, wf1.id)
|
133
|
+
expect(wf).to be_kind_of(SecondWorkflow)
|
134
|
+
end
|
135
|
+
end
|
136
|
+
|
137
|
+
describe '#sub_workflows' do
|
138
|
+
let!(:wf1) { FirstWorkflow.create }
|
139
|
+
let!(:wf2) do
|
140
|
+
wf = SecondWorkflow.new
|
141
|
+
wf.parent_id = wf1.id
|
142
|
+
wf.save
|
143
|
+
wf
|
144
|
+
end
|
145
|
+
|
146
|
+
it do
|
147
|
+
wfs = client.sub_workflows(wf1.id)
|
148
|
+
expect(wfs).not_to be_empty
|
149
|
+
expect(wfs.first).to be_kind_of(SecondWorkflow)
|
150
|
+
end
|
151
|
+
end
|
152
|
+
|
56
153
|
describe '#persist_workflow' do
|
57
154
|
let(:workflow) { Dwf::Workflow.new }
|
58
155
|
|
59
156
|
it do
|
60
157
|
expect(redis.exists?("dwf.workflows.#{workflow.id}")).to be_falsy
|
61
158
|
client.persist_workflow(workflow)
|
62
|
-
expect(redis.
|
159
|
+
expect(redis.keys("dwf.workflows.#{workflow.id}*").any?).to be_truthy
|
63
160
|
end
|
64
161
|
end
|
65
162
|
|