dwf 0.1.7 → 0.1.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/lib/dwf/item.rb CHANGED
@@ -1,35 +1,36 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require_relative 'client'
4
+ require_relative 'concerns/checkable'
4
5
 
5
6
  module Dwf
6
7
  class Item
8
+ include Concerns::Checkable
9
+
7
10
  attr_reader :workflow_id, :id, :params, :queue, :klass, :started_at,
8
- :enqueued_at, :finished_at, :failed_at, :callback_type
9
- attr_accessor :incoming, :outgoing
11
+ :enqueued_at, :finished_at, :failed_at, :output_payload
12
+ attr_writer :payloads
13
+ attr_accessor :incoming, :outgoing, :callback_type
10
14
 
11
15
  def initialize(options = {})
12
- @workflow_id = options[:workflow_id]
13
- @id = options[:id]
14
- @params = options[:params]
15
- @queue = options[:queue]
16
- @incoming = options[:incoming] || []
17
- @outgoing = options[:outgoing] || []
18
- @klass = options[:klass] || self.class
19
- @failed_at = options[:failed_at]
20
- @finished_at = options[:finished_at]
21
- @enqueued_at = options[:enqueued_at]
22
- @started_at = options[:started_at]
23
- @callback_type = options[:callback_type]
16
+ assign_attributes(options)
24
17
  end
25
18
 
26
19
  def self.from_hash(hash)
27
20
  Module.const_get(hash[:klass]).new(hash)
28
21
  end
29
22
 
23
+ def start_initial!
24
+ cb_build_in? ? persist_and_perform_async! : start_batch!
25
+ end
26
+
27
+ def start_batch!
28
+ enqueue_and_persist!
29
+ Dwf::Callback.new.start(self)
30
+ end
31
+
30
32
  def persist_and_perform_async!
31
- enqueue!
32
- persist!
33
+ enqueue_and_persist!
33
34
  perform_async
34
35
  end
35
36
 
@@ -39,23 +40,34 @@ module Dwf
39
40
  callback_type == Dwf::Workflow::BUILD_IN
40
41
  end
41
42
 
43
+ def reload
44
+ item = client.find_job(workflow_id, name)
45
+ assign_attributes(item.to_hash)
46
+ end
47
+
42
48
  def perform_async
43
49
  Dwf::Worker.set(queue: queue || client.config.namespace)
44
- .perform_async(workflow_id, name)
50
+ .perform_async(workflow_id, name)
45
51
  end
46
52
 
47
53
  def name
48
54
  @name ||= "#{klass}|#{id}"
49
55
  end
50
56
 
57
+ def output(data)
58
+ @output_payload = data
59
+ end
60
+
51
61
  def no_dependencies?
52
62
  incoming.empty?
53
63
  end
54
64
 
55
65
  def parents_succeeded?
56
- incoming.all? do |name|
57
- client.find_job(workflow_id, name).succeeded?
58
- end
66
+ incoming.all? { |name| client.find_node(name, workflow_id).succeeded? }
67
+ end
68
+
69
+ def payloads
70
+ @payloads ||= build_payloads
59
71
  end
60
72
 
61
73
  def enqueue!
@@ -100,22 +112,6 @@ module Dwf
100
112
  !failed_at.nil?
101
113
  end
102
114
 
103
- def succeeded?
104
- finished? && !failed?
105
- end
106
-
107
- def started?
108
- !started_at.nil?
109
- end
110
-
111
- def running?
112
- started? && !finished?
113
- end
114
-
115
- def ready_to_start?
116
- !running? && !enqueued? && !finished? && !failed? && parents_succeeded?
117
- end
118
-
119
115
  def current_timestamp
120
116
  Time.now.to_i
121
117
  end
@@ -142,7 +138,9 @@ module Dwf
142
138
  failed_at: failed_at,
143
139
  params: params,
144
140
  workflow_id: workflow_id,
145
- callback_type: callback_type
141
+ callback_type: callback_type,
142
+ output_payload: output_payload,
143
+ payloads: @payloads
146
144
  }
147
145
  end
148
146
 
@@ -156,8 +154,44 @@ module Dwf
156
154
 
157
155
  private
158
156
 
157
+ def enqueue_and_persist!
158
+ enqueue!
159
+ persist!
160
+ end
161
+
159
162
  def client
160
163
  @client ||= Dwf::Client.new
161
164
  end
165
+
166
+ def assign_attributes(options)
167
+ @workflow_id = options[:workflow_id]
168
+ @id = options[:id]
169
+ @params = options[:params]
170
+ @queue = options[:queue]
171
+ @incoming = options[:incoming] || []
172
+ @outgoing = options[:outgoing] || []
173
+ @klass = options[:klass] || self.class
174
+ @failed_at = options[:failed_at]
175
+ @finished_at = options[:finished_at]
176
+ @enqueued_at = options[:enqueued_at]
177
+ @started_at = options[:started_at]
178
+ @callback_type = options[:callback_type]
179
+ @output_payload = options[:output_payload]
180
+ @payloads = options[:payloads]
181
+ end
182
+
183
+ def build_payloads
184
+ data = incoming.map do |job_name|
185
+ node = client.find_node(job_name, workflow_id)
186
+ next if node.output_payload.nil?
187
+
188
+ {
189
+ id: node.name,
190
+ class: node.klass.to_s,
191
+ output: node.output_payload
192
+ }
193
+ end.compact
194
+ data.empty? ? nil : data
195
+ end
162
196
  end
163
197
  end
data/lib/dwf/utils.rb CHANGED
@@ -34,5 +34,11 @@ module Dwf
34
34
  obj
35
35
  end
36
36
  end
37
+
38
+ def self.workflow_name?(name)
39
+ node_name = name.include?('|') ? name.split('|').first : name
40
+
41
+ Module.const_get(node_name) <= Workflow
42
+ end
37
43
  end
38
44
  end
data/lib/dwf/version.rb CHANGED
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Dwf
4
- VERSION = '0.1.7'
4
+ VERSION = '0.1.11'
5
5
  end
data/lib/dwf/worker.rb CHANGED
@@ -9,7 +9,7 @@ module Dwf
9
9
 
10
10
  def perform(workflow_id, job_name)
11
11
  job = client.find_job(workflow_id, job_name)
12
- return job.enqueue_outgoing_jobs if job. succeeded?
12
+ return job.enqueue_outgoing_jobs if job.succeeded?
13
13
 
14
14
  job.mark_as_started
15
15
  job.perform
data/lib/dwf/workflow.rb CHANGED
@@ -2,15 +2,19 @@
2
2
 
3
3
  require_relative 'client'
4
4
  require_relative 'worker'
5
+ require_relative 'concerns/checkable'
5
6
  require_relative 'callback'
6
7
 
7
8
  module Dwf
8
9
  class Workflow
10
+ include Concerns::Checkable
11
+
9
12
  CALLBACK_TYPES = [
10
13
  BUILD_IN = 'build-in',
11
14
  SK_BATCH = 'sk-batch'
12
15
  ].freeze
13
- attr_reader :dependencies, :jobs, :started_at, :finished_at, :persisted, :stopped,
16
+ attr_accessor :jobs, :stopped, :id, :incoming, :outgoing, :parent_id
17
+ attr_reader :dependencies, :started_at, :finished_at, :persisted, :arguments, :klass,
14
18
  :callback_type
15
19
 
16
20
  class << self
@@ -19,51 +23,91 @@ module Dwf
19
23
  flow.save
20
24
  flow
21
25
  end
26
+
27
+ def find(id)
28
+ Dwf::Client.new.find_workflow(id)
29
+ end
22
30
  end
23
31
 
24
- def initialize(options = {})
32
+ def initialize(*args)
25
33
  @dependencies = []
26
- @id = id
34
+ @id = build_id
27
35
  @jobs = []
28
36
  @persisted = false
29
37
  @stopped = false
30
- @callback_type = options[:callback_type] || BUILD_IN
38
+ @arguments = *args
39
+ @parent_id = nil
40
+ @klass = self.class
41
+ @callback_type = BUILD_IN
42
+ @incoming = []
43
+ @outgoing = []
31
44
 
32
45
  setup
33
46
  end
34
47
 
48
+ def persist!
49
+ client.persist_workflow(self)
50
+ jobs.each(&:persist!)
51
+ mark_as_persisted
52
+ true
53
+ end
54
+
55
+ def name
56
+ "#{self.class.name}|#{id}"
57
+ end
58
+
59
+ def sub_workflow?
60
+ !parent_id.nil?
61
+ end
62
+
63
+ def callback_type=(type = BUILD_IN)
64
+ @callback_type = type
65
+ jobs.each { |job| job.callback_type = type }
66
+ end
67
+
68
+ alias save persist!
69
+
35
70
  def start!
71
+ raise UnsupportCallback, 'Sub workflow only works with Sidekiq batch callback' if invalid_callback?
72
+
73
+ mark_as_started
74
+ persist!
36
75
  initial_jobs.each do |job|
37
- cb_build_in? ? job.persist_and_perform_async! : Dwf::Callback.new.start(job)
76
+ job.payloads = payloads if sub_workflow?
77
+ job.start_initial!
38
78
  end
39
79
  end
40
80
 
41
- def save
42
- client.persist_workflow(self)
43
- jobs.each(&:persist!)
44
- mark_as_persisted
45
- true
81
+ def payloads
82
+ @payloads ||= build_payloads
83
+ end
84
+
85
+ def start_initial!
86
+ cb_build_in? ? start! : Callback.new.start(self)
87
+ end
88
+
89
+ alias persist_and_perform_async! start!
90
+
91
+ def reload
92
+ flow = self.class.find(id)
93
+ self.stopped = flow.stopped
94
+ self.jobs = flow.jobs
95
+
96
+ self
46
97
  end
47
98
 
48
99
  def cb_build_in?
49
100
  callback_type == BUILD_IN
50
101
  end
51
102
 
52
- def id
53
- @id ||= client.build_workflow_id
103
+ def build_id
104
+ client.build_workflow_id
54
105
  end
55
106
 
56
- def configure; end
107
+ def configure(*arguments); end
57
108
 
58
109
  def run(klass, options = {})
59
- node = klass.new(
60
- workflow_id: id,
61
- id: client.build_job_id(id, klass.to_s),
62
- params: options.fetch(:params, {}),
63
- queue: options[:queue],
64
- callback_type: callback_type
65
- )
66
-
110
+ node = build_node(klass, options)
67
111
  jobs << node
68
112
 
69
113
  build_dependencies_structure(node, options)
@@ -95,7 +139,10 @@ module Dwf
95
139
  stopped: stopped,
96
140
  started_at: started_at,
97
141
  finished_at: finished_at,
98
- callback_type: callback_type
142
+ callback_type: callback_type,
143
+ incoming: incoming,
144
+ outgoing: outgoing,
145
+ parent_id: parent_id
99
146
  }
100
147
  end
101
148
 
@@ -107,13 +154,7 @@ module Dwf
107
154
  jobs.all?(&:finished?)
108
155
  end
109
156
 
110
- def started?
111
- !!started_at
112
- end
113
-
114
- def running?
115
- started? && !finished?
116
- end
157
+ alias enqueued? started?
117
158
 
118
159
  def failed?
119
160
  jobs.any?(&:failed?)
@@ -123,6 +164,10 @@ module Dwf
123
164
  stopped
124
165
  end
125
166
 
167
+ def parents_succeeded?
168
+ incoming.all? { |name| client.find_node(name, parent_id).succeeded? }
169
+ end
170
+
126
171
  def status
127
172
  return :failed if failed?
128
173
  return :running if running?
@@ -140,28 +185,94 @@ module Dwf
140
185
  @stopped = false
141
186
  end
142
187
 
188
+ def leaf_nodes
189
+ jobs.select(&:leaf?)
190
+ end
191
+
192
+ def output_payload
193
+ leaf_nodes.map do |node|
194
+ data = node.output_payload
195
+ next if data.nil?
196
+
197
+ data
198
+ end.compact
199
+ end
143
200
 
144
201
  private
145
202
 
203
+ def build_node(klass, options)
204
+ if klass < Dwf::Workflow
205
+ node = options[:params].nil? ? klass.new : klass.new(options[:params])
206
+ node.parent_id = id
207
+ node.callback_type = SK_BATCH
208
+ node.save
209
+ node
210
+ else
211
+ klass.new(
212
+ workflow_id: id,
213
+ id: client.build_job_id(id, klass.to_s),
214
+ params: options.fetch(:params, {}),
215
+ queue: options[:queue],
216
+ callback_type: callback_type
217
+ )
218
+ end
219
+ end
220
+
146
221
  def initial_jobs
147
222
  jobs.select(&:no_dependencies?)
148
223
  end
149
224
 
150
225
  def setup
151
- configure
226
+ configure(*arguments)
152
227
  resolve_dependencies
153
228
  end
154
229
 
230
+ def find_node(node_name)
231
+ if Utils.workflow_name?(node_name)
232
+ find_subworkflow(node_name)
233
+ else
234
+ find_job(node_name)
235
+ end
236
+ end
237
+
238
+ def find_subworkflow(node_name)
239
+ fname, = node_name.split('|')
240
+ jobs.find { |j| j.klass.name == fname }
241
+ end
242
+
155
243
  def resolve_dependencies
156
244
  @dependencies.each do |dependency|
157
- from = find_job(dependency[:from])
158
- to = find_job(dependency[:to])
245
+ from = find_node(dependency[:from])
246
+ to = find_node(dependency[:to])
159
247
 
160
- to.incoming << dependency[:from]
161
- from.outgoing << dependency[:to]
248
+ to.incoming << from.name
249
+ from.outgoing << to.name
162
250
  end
163
251
  end
164
252
 
253
+ def invalid_callback?
254
+ cb_build_in? && jobs.any? { |job| job.class < Workflow }
255
+ end
256
+
257
+ def build_payloads
258
+ return unless sub_workflow?
259
+
260
+ data = incoming.map do |job_name|
261
+ next if Utils.workflow_name?(job_name)
262
+
263
+ node = client.find_node(job_name, parent_id)
264
+ next if node.output_payload.nil?
265
+
266
+ {
267
+ id: node.name,
268
+ class: node.klass.to_s,
269
+ output: node.output_payload
270
+ }
271
+ end.compact
272
+
273
+ data.empty? ? nil : data
274
+ end
275
+
165
276
  def build_dependencies_structure(node, options)
166
277
  deps_after = [*options[:after]]
167
278
 
@@ -2,6 +2,8 @@
2
2
 
3
3
  require 'spec_helper'
4
4
  require 'mock_redis'
5
+ FirstWorkflow = Class.new(Dwf::Workflow)
6
+ SecondWorkflow = Class.new(Dwf::Workflow)
5
7
 
6
8
  describe Dwf::Client, client: true do
7
9
  let(:client) { described_class.new }
@@ -30,12 +32,76 @@ describe Dwf::Client, client: true do
30
32
  end
31
33
 
32
34
  context 'find by item name' do
33
- it {
35
+ it do
34
36
  item = client.find_job(workflow_id, job.name)
35
37
  expect(item.workflow_id).to eq workflow_id
36
38
  expect(item.id).to eq id
37
39
  expect(item.name).to eq job.name
38
- }
40
+ end
41
+ end
42
+ end
43
+
44
+ describe '#find_workflow' do
45
+ before do
46
+ wf = Dwf::Workflow.new
47
+ wf.id = workflow_id
48
+ wf.save
49
+ j = Dwf::Item.new(id: id, workflow_id: workflow_id)
50
+ j.persist!
51
+ end
52
+
53
+ it do
54
+ wf = client.find_workflow(workflow_id)
55
+
56
+ expect(wf).not_to be_nil
57
+ expect(wf.jobs.first).to be_kind_of(Dwf::Item)
58
+ end
59
+
60
+ it do
61
+ expect do
62
+ client.find_workflow(SecureRandom.uuid)
63
+ end.to raise_error Dwf::WorkflowNotFound
64
+ end
65
+ end
66
+
67
+ describe '#find_node' do
68
+ context 'find job' do
69
+ let!(:job) do
70
+ j = Dwf::Item.new(workflow_id: workflow_id, id: id)
71
+ j.persist!
72
+ j
73
+ end
74
+
75
+ it do
76
+ item = client.find_node(Dwf::Item.name, workflow_id)
77
+ expect(item.workflow_id).to eq workflow_id
78
+ expect(item.id).to eq id
79
+ expect(item.name).to eq job.name
80
+ end
81
+ end
82
+
83
+ context 'find_workflow' do
84
+ let!(:wf1) { FirstWorkflow.create }
85
+ let!(:wf2) do
86
+ wf = SecondWorkflow.new
87
+ wf.parent_id = wf1.id
88
+ wf.save
89
+ wf
90
+ end
91
+
92
+ context 'find with class name and parent id' do
93
+ it do
94
+ wf = client.find_node(wf2.class.name, wf1.id)
95
+ expect(wf).to be_kind_of(SecondWorkflow)
96
+ end
97
+ end
98
+
99
+ context 'find with name and parent id' do
100
+ it do
101
+ wf = client.find_node(wf2.name, wf1.id)
102
+ expect(wf).to be_kind_of(SecondWorkflow)
103
+ end
104
+ end
39
105
  end
40
106
  end
41
107
 
@@ -53,13 +119,44 @@ describe Dwf::Client, client: true do
53
119
  end
54
120
  end
55
121
 
122
+ describe '#find_sub_workflow' do
123
+ let!(:wf1) { FirstWorkflow.create }
124
+ let!(:wf2) do
125
+ wf = SecondWorkflow.new
126
+ wf.parent_id = wf1.id
127
+ wf.save
128
+ wf
129
+ end
130
+
131
+ it do
132
+ wf = client.find_sub_workflow(wf2.class.name, wf1.id)
133
+ expect(wf).to be_kind_of(SecondWorkflow)
134
+ end
135
+ end
136
+
137
+ describe '#sub_workflows' do
138
+ let!(:wf1) { FirstWorkflow.create }
139
+ let!(:wf2) do
140
+ wf = SecondWorkflow.new
141
+ wf.parent_id = wf1.id
142
+ wf.save
143
+ wf
144
+ end
145
+
146
+ it do
147
+ wfs = client.sub_workflows(wf1.id)
148
+ expect(wfs).not_to be_empty
149
+ expect(wfs.first).to be_kind_of(SecondWorkflow)
150
+ end
151
+ end
152
+
56
153
  describe '#persist_workflow' do
57
154
  let(:workflow) { Dwf::Workflow.new }
58
155
 
59
156
  it do
60
157
  expect(redis.exists?("dwf.workflows.#{workflow.id}")).to be_falsy
61
158
  client.persist_workflow(workflow)
62
- expect(redis.exists?("dwf.workflows.#{workflow.id}")).to be_truthy
159
+ expect(redis.keys("dwf.workflows.#{workflow.id}*").any?).to be_truthy
63
160
  end
64
161
  end
65
162