dwf 0.1.11 → 0.1.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/build_gem.yaml +0 -4
- data/README.md +5 -9
- data/dwf.gemspec +4 -4
- data/lib/dwf/errors.rb +0 -2
- data/lib/dwf/item.rb +7 -1
- data/lib/dwf/version.rb +1 -1
- data/lib/dwf/workflow.rb +12 -4
- data/spec/dwf/item_spec.rb +41 -23
- data/spec/dwf/workflow_spec.rb +34 -1
- metadata +3 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 4d6ec140cc691fd53ceb5d495add6e3a76ce6a7b6daf2bfbde982e6e40a19987
|
4
|
+
data.tar.gz: 66911fc7ec1a54e38aaaca160ee455b2463f1888ecbbf981c64a17e9d790c1a2
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 240a6603023b3edc79b69d1793e8805e86ec298f2927df7bd3a1342ad241fb3a0b619a6aa50266348003c20f751fdd58dc91a68dead319916ca351a09bee5f0f
|
7
|
+
data.tar.gz: 4af1937daa29fab5f2fd818f22a64bea90d35074a0ca70109e4a36cbaa754fae199799167385f78579a854152563c1e8074c6ff91ad931e544b20c67bf7de054
|
data/README.md
CHANGED
@@ -1,10 +1,10 @@
|
|
1
|
-
#
|
2
|
-
[Gush](https://github.com/chaps-io/gush)
|
1
|
+
# DWF
|
2
|
+
Distributed workflow runner following [Gush](https://github.com/chaps-io/gush) interface using [Sidekiq](https://github.com/mperham/sidekiq) and [Redis](https://redis.io/). This project is for researching DSL purpose
|
3
3
|
|
4
4
|
# Installation
|
5
5
|
## 1. Add `dwf` to Gemfile
|
6
6
|
```ruby
|
7
|
-
gem 'dwf', '~> 0.1.
|
7
|
+
gem 'dwf', '~> 0.1.12'
|
8
8
|
```
|
9
9
|
## 2. Execute flow example
|
10
10
|
### Declare jobs
|
@@ -138,15 +138,11 @@ end
|
|
138
138
|
}
|
139
139
|
]
|
140
140
|
```
|
141
|
-
##
|
141
|
+
## Sub workflow
|
142
142
|
There might be a case when you want to reuse a workflow in another workflow
|
143
143
|
|
144
144
|
As an example, let's write a workflow which contain another workflow, expected that the SubWorkflow workflow execute after `SecondItem` and the `ThirdItem` execute after `SubWorkflow`
|
145
145
|
|
146
|
-
```ruby
|
147
|
-
gem 'dwf', '~> 0.1.11'
|
148
|
-
```
|
149
|
-
|
150
146
|
### Setup
|
151
147
|
```ruby
|
152
148
|
class FirstItem < Dwf::Item
|
@@ -211,7 +207,7 @@ Main flow: ThirtItem finish
|
|
211
207
|
- [x] Redis configurable
|
212
208
|
- [x] Pipelining
|
213
209
|
- [X] Test
|
214
|
-
- [
|
210
|
+
- [x] Sub workflow
|
215
211
|
- [ ] Support [Resque](https://github.com/resque/resque)
|
216
212
|
- [ ] Key value store plugable
|
217
213
|
- [ ] research https://github.com/moneta-rb/moneta
|
data/dwf.gemspec
CHANGED
@@ -6,12 +6,12 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
6
6
|
require_relative 'lib/dwf/version'
|
7
7
|
|
8
8
|
Gem::Specification.new do |spec|
|
9
|
-
spec.name =
|
9
|
+
spec.name = 'dwf'
|
10
10
|
spec.version = Dwf::VERSION
|
11
|
-
spec.authors = [
|
12
|
-
spec.email = [
|
11
|
+
spec.authors = ['dthtien']
|
12
|
+
spec.email = ['tiendt2311@gmail.com']
|
13
13
|
|
14
|
-
spec.summary = '
|
14
|
+
spec.summary = 'Distributed workflow runner following Gush interface using Sidekiq and Redis'
|
15
15
|
spec.description = 'Workflow'
|
16
16
|
spec.homepage = 'https://github.com/dthtien/wf'
|
17
17
|
spec.license = "MIT"
|
data/lib/dwf/errors.rb
CHANGED
data/lib/dwf/item.rb
CHANGED
@@ -40,6 +40,10 @@ module Dwf
|
|
40
40
|
callback_type == Dwf::Workflow::BUILD_IN
|
41
41
|
end
|
42
42
|
|
43
|
+
def workflow
|
44
|
+
@workflow ||= client.find_workflow(workflow_id)
|
45
|
+
end
|
46
|
+
|
43
47
|
def reload
|
44
48
|
item = client.find_job(workflow_id, name)
|
45
49
|
assign_attributes(item.to_hash)
|
@@ -117,9 +121,11 @@ module Dwf
|
|
117
121
|
end
|
118
122
|
|
119
123
|
def enqueue_outgoing_jobs
|
124
|
+
return workflow.enqueue_outgoing_jobs if leaf?
|
125
|
+
|
120
126
|
outgoing.each do |job_name|
|
121
127
|
client.check_or_lock(workflow_id, job_name)
|
122
|
-
out = client.
|
128
|
+
out = client.find_node(job_name, workflow_id)
|
123
129
|
out.persist_and_perform_async! if out.ready_to_start?
|
124
130
|
client.release_lock(workflow_id, job_name)
|
125
131
|
end
|
data/lib/dwf/version.rb
CHANGED
data/lib/dwf/workflow.rb
CHANGED
@@ -3,7 +3,6 @@
|
|
3
3
|
require_relative 'client'
|
4
4
|
require_relative 'worker'
|
5
5
|
require_relative 'concerns/checkable'
|
6
|
-
require_relative 'callback'
|
7
6
|
|
8
7
|
module Dwf
|
9
8
|
class Workflow
|
@@ -68,8 +67,6 @@ module Dwf
|
|
68
67
|
alias save persist!
|
69
68
|
|
70
69
|
def start!
|
71
|
-
raise UnsupportCallback, 'Sub workflow only works with Sidekiq batch callback' if invalid_callback?
|
72
|
-
|
73
70
|
mark_as_started
|
74
71
|
persist!
|
75
72
|
initial_jobs.each do |job|
|
@@ -177,6 +174,17 @@ module Dwf
|
|
177
174
|
:running
|
178
175
|
end
|
179
176
|
|
177
|
+
def enqueue_outgoing_jobs
|
178
|
+
return unless sub_workflow?
|
179
|
+
|
180
|
+
outgoing.each do |job_name|
|
181
|
+
client.check_or_lock(parent_id, job_name)
|
182
|
+
node = client.find_node(job_name, parent_id)
|
183
|
+
node.persist_and_perform_async! if node.ready_to_start?
|
184
|
+
client.release_lock(parent_id, job_name)
|
185
|
+
end
|
186
|
+
end
|
187
|
+
|
180
188
|
def mark_as_persisted
|
181
189
|
@persisted = true
|
182
190
|
end
|
@@ -204,7 +212,7 @@ module Dwf
|
|
204
212
|
if klass < Dwf::Workflow
|
205
213
|
node = options[:params].nil? ? klass.new : klass.new(options[:params])
|
206
214
|
node.parent_id = id
|
207
|
-
node.callback_type =
|
215
|
+
node.callback_type = callback_type
|
208
216
|
node.save
|
209
217
|
node
|
210
218
|
else
|
data/spec/dwf/item_spec.rb
CHANGED
@@ -149,37 +149,55 @@ describe Dwf::Item, item: true do
|
|
149
149
|
end
|
150
150
|
|
151
151
|
describe '#enqueue_outgoing_jobs' do
|
152
|
-
let(:outgoing) { ["A|#{SecureRandom.uuid}"] }
|
153
152
|
let(:client_double) do
|
154
153
|
double(
|
155
|
-
|
154
|
+
find_node: nil,
|
156
155
|
check_or_lock: nil,
|
157
|
-
release_lock: nil
|
156
|
+
release_lock: nil,
|
157
|
+
build_workflow_id: SecureRandom.uuid
|
158
158
|
)
|
159
159
|
end
|
160
|
-
let(:a_item) do
|
161
|
-
described_class.new(
|
162
|
-
workflow_id: SecureRandom.uuid,
|
163
|
-
id: SecureRandom.uuid,
|
164
|
-
started_at: started_at
|
165
|
-
)
|
166
|
-
end
|
167
|
-
before do
|
168
|
-
allow(Dwf::Client).to receive(:new).and_return client_double
|
169
|
-
allow(a_item).to receive(:persist_and_perform_async!)
|
170
|
-
allow(client_double)
|
171
|
-
.to receive(:find_job).and_return a_item
|
172
|
-
item.enqueue_outgoing_jobs
|
173
|
-
end
|
174
160
|
|
175
|
-
context '
|
176
|
-
let(:
|
177
|
-
|
161
|
+
context 'when item is not a leaf' do
|
162
|
+
let(:outgoing) { ["A|#{SecureRandom.uuid}"] }
|
163
|
+
let(:a_item) do
|
164
|
+
described_class.new(
|
165
|
+
workflow_id: SecureRandom.uuid,
|
166
|
+
id: SecureRandom.uuid,
|
167
|
+
started_at: started_at
|
168
|
+
)
|
169
|
+
end
|
170
|
+
before do
|
171
|
+
allow(Dwf::Client).to receive(:new).and_return client_double
|
172
|
+
allow(a_item).to receive(:persist_and_perform_async!)
|
173
|
+
allow(client_double)
|
174
|
+
.to receive(:find_node).and_return a_item
|
175
|
+
item.enqueue_outgoing_jobs
|
176
|
+
end
|
177
|
+
|
178
|
+
context 'outgoing jobs ready to start' do
|
179
|
+
let(:started_at) { nil }
|
180
|
+
it { expect(a_item).to have_received(:persist_and_perform_async!) }
|
181
|
+
end
|
182
|
+
|
183
|
+
context 'outgoing jobs havent ready to start' do
|
184
|
+
let(:started_at) { Time.now.to_i }
|
185
|
+
it { expect(a_item).not_to have_received(:persist_and_perform_async!) }
|
186
|
+
end
|
178
187
|
end
|
179
188
|
|
180
|
-
context '
|
181
|
-
let(:
|
182
|
-
|
189
|
+
context 'when item is a leaf' do
|
190
|
+
let(:leaf) { true }
|
191
|
+
let(:workflow) { Dwf::Workflow.new }
|
192
|
+
before do
|
193
|
+
allow(Dwf::Client).to receive(:new).and_return client_double
|
194
|
+
allow(client_double).to receive(:find_workflow).and_return workflow
|
195
|
+
allow(workflow).to receive(:enqueue_outgoing_jobs)
|
196
|
+
item.enqueue_outgoing_jobs
|
197
|
+
end
|
198
|
+
|
199
|
+
let(:started_at) { nil }
|
200
|
+
it { expect(workflow).to have_received(:enqueue_outgoing_jobs) }
|
183
201
|
end
|
184
202
|
end
|
185
203
|
|
data/spec/dwf/workflow_spec.rb
CHANGED
@@ -18,7 +18,9 @@ describe Dwf::Workflow, workflow: true do
|
|
18
18
|
build_workflow_id: workflow_id,
|
19
19
|
build_job_id: item_id,
|
20
20
|
find_workflow: nil,
|
21
|
-
find_node: item
|
21
|
+
find_node: item,
|
22
|
+
check_or_lock: nil,
|
23
|
+
release_lock: nil
|
22
24
|
)
|
23
25
|
end
|
24
26
|
before do
|
@@ -346,4 +348,35 @@ describe Dwf::Workflow, workflow: true do
|
|
346
348
|
expect(job_callback_types).to eq [described_class::SK_BATCH]
|
347
349
|
end
|
348
350
|
end
|
351
|
+
|
352
|
+
describe '#enqueue_outgoing_jobs' do
|
353
|
+
let(:outgoing) { ["A|#{SecureRandom.uuid}"] }
|
354
|
+
let!(:workflow) do
|
355
|
+
flow = described_class.new
|
356
|
+
flow.parent_id = SecureRandom.uuid
|
357
|
+
flow.outgoing = outgoing
|
358
|
+
flow
|
359
|
+
end
|
360
|
+
let(:item) do
|
361
|
+
Dwf::Item.new(
|
362
|
+
workflow_id: SecureRandom.uuid,
|
363
|
+
id: SecureRandom.uuid,
|
364
|
+
started_at: started_at
|
365
|
+
)
|
366
|
+
end
|
367
|
+
before do
|
368
|
+
allow(item).to receive(:persist_and_perform_async!)
|
369
|
+
workflow.enqueue_outgoing_jobs
|
370
|
+
end
|
371
|
+
|
372
|
+
context 'outgoing jobs ready to start' do
|
373
|
+
let(:started_at) { nil }
|
374
|
+
it { expect(item).to have_received(:persist_and_perform_async!) }
|
375
|
+
end
|
376
|
+
|
377
|
+
context 'outgoing jobs havent ready to start' do
|
378
|
+
let(:started_at) { Time.now.to_i }
|
379
|
+
it { expect(item).not_to have_received(:persist_and_perform_async!) }
|
380
|
+
end
|
381
|
+
end
|
349
382
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: dwf
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.12
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- dthtien
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2021-10-
|
11
|
+
date: 2021-10-10 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: byebug
|
@@ -151,8 +151,7 @@ requirements: []
|
|
151
151
|
rubygems_version: 3.2.3
|
152
152
|
signing_key:
|
153
153
|
specification_version: 4
|
154
|
-
summary:
|
155
|
-
DSL purpose
|
154
|
+
summary: Distributed workflow runner following Gush interface using Sidekiq and Redis
|
156
155
|
test_files:
|
157
156
|
- spec/dwf/client_spec.rb
|
158
157
|
- spec/dwf/configuration_spec.rb
|