gush 0.3 → 0.3.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +2 -2
- data/gush.gemspec +1 -1
- data/lib/gush/cli.rb +0 -2
- data/lib/gush/client.rb +25 -8
- data/lib/gush/graph.rb +6 -6
- data/lib/gush/job.rb +13 -2
- data/lib/gush/json.rb +1 -2
- data/lib/gush/worker.rb +10 -13
- data/lib/gush/workflow.rb +30 -8
- data/spec/features/integration_spec.rb +48 -6
- data/spec/gush/cli_spec.rb +8 -0
- data/spec/{lib/gush → gush}/client_spec.rb +5 -3
- data/spec/{lib/gush → gush}/configuration_spec.rb +0 -0
- data/spec/gush/graph_spec.rb +43 -0
- data/spec/{lib/gush → gush}/job_spec.rb +1 -0
- data/spec/gush/json_spec.rb +21 -0
- data/spec/{lib/gush → gush}/worker_spec.rb +6 -2
- data/spec/{lib/gush → gush}/workflow_spec.rb +68 -45
- data/spec/{lib/gush_spec.rb → gush_spec.rb} +1 -1
- data/spec/spec_helper.rb +9 -0
- metadata +21 -16
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 5c0f45a09eb1260f6bcb00f1503181ddb4705735
|
4
|
+
data.tar.gz: 13eaec20c3b1165127b5cb7761c99cdd5abc1f41
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 1d1060e1575b133245e78280f96818a83c599fbd7f9b08a93d14c54c212cf3d1f4ae0f75a81883fcb49758b059a1bc51063e46c9121405408280570c124906ef
|
7
|
+
data.tar.gz: 7bc51f1ad03adc625ebe497757c86fc588514d8a98d93e278ab961d1817653c21cf706858ad080d4e4fbc29e3d792b0a8d33598f67f126eccd0910d39f222843
|
data/README.md
CHANGED
@@ -210,13 +210,13 @@ flow.status
|
|
210
210
|
- of a specific workflow:
|
211
211
|
|
212
212
|
```
|
213
|
-
bundle gush show <workflow_id>
|
213
|
+
bundle exec gush show <workflow_id>
|
214
214
|
```
|
215
215
|
|
216
216
|
- of all created workflows:
|
217
217
|
|
218
218
|
```
|
219
|
-
bundle gush list
|
219
|
+
bundle exec gush list
|
220
220
|
```
|
221
221
|
|
222
222
|
|
data/gush.gemspec
CHANGED
@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
4
4
|
|
5
5
|
Gem::Specification.new do |spec|
|
6
6
|
spec.name = "gush"
|
7
|
-
spec.version = "0.3"
|
7
|
+
spec.version = "0.3.1"
|
8
8
|
spec.authors = ["Piotrek Okoński"]
|
9
9
|
spec.email = ["piotrek@okonski.org"]
|
10
10
|
spec.summary = "Fast and distributed workflow runner using only Sidekiq and Redis"
|
data/lib/gush/cli.rb
CHANGED
data/lib/gush/client.rb
CHANGED
@@ -16,12 +16,10 @@ module Gush
|
|
16
16
|
|
17
17
|
def create_workflow(name)
|
18
18
|
begin
|
19
|
-
|
20
|
-
flow.save
|
19
|
+
name.constantize.create
|
21
20
|
rescue NameError
|
22
21
|
raise WorkflowNotFound.new("Workflow with given name doesn't exist")
|
23
22
|
end
|
24
|
-
|
25
23
|
flow
|
26
24
|
end
|
27
25
|
|
@@ -46,7 +44,18 @@ module Gush
|
|
46
44
|
persist_workflow(workflow)
|
47
45
|
end
|
48
46
|
|
49
|
-
def
|
47
|
+
def next_free_job_id(workflow_id,job_klass)
|
48
|
+
job_identifier = nil
|
49
|
+
loop do
|
50
|
+
id = SecureRandom.uuid
|
51
|
+
job_identifier = "#{job_klass}-#{id}"
|
52
|
+
break if !redis.exists("gush.jobs.#{workflow_id}.#{job_identifier}")
|
53
|
+
end
|
54
|
+
|
55
|
+
job_identifier
|
56
|
+
end
|
57
|
+
|
58
|
+
def next_free_workflow_id
|
50
59
|
id = nil
|
51
60
|
loop do
|
52
61
|
id = SecureRandom.uuid
|
@@ -83,13 +92,20 @@ module Gush
|
|
83
92
|
end
|
84
93
|
|
85
94
|
def persist_job(workflow_id, job)
|
86
|
-
redis.set("gush.jobs.#{workflow_id}.#{job.
|
95
|
+
redis.set("gush.jobs.#{workflow_id}.#{job.name}", job.to_json)
|
87
96
|
end
|
88
97
|
|
89
98
|
def load_job(workflow_id, job_id)
|
90
99
|
workflow = find_workflow(workflow_id)
|
91
|
-
|
100
|
+
job_name_match = /(?<klass>\w*[^-])-(?<identifier>.*)/.match(job_id)
|
101
|
+
hypen = '-' if job_name_match.nil?
|
102
|
+
|
103
|
+
keys = redis.keys("gush.jobs.#{workflow_id}.#{job_id}#{hypen}*")
|
104
|
+
return nil if keys.nil?
|
105
|
+
|
106
|
+
data = redis.get(keys.first)
|
92
107
|
return nil if data.nil?
|
108
|
+
|
93
109
|
data = Gush::JSON.decode(data, symbolize_keys: true)
|
94
110
|
Gush::Job.from_hash(workflow, data)
|
95
111
|
end
|
@@ -100,7 +116,7 @@ module Gush
|
|
100
116
|
end
|
101
117
|
|
102
118
|
def destroy_job(workflow_id, job)
|
103
|
-
redis.del("gush.jobs.#{workflow_id}.#{job.
|
119
|
+
redis.del("gush.jobs.#{workflow_id}.#{job.name}")
|
104
120
|
end
|
105
121
|
|
106
122
|
def worker_report(message)
|
@@ -118,7 +134,7 @@ module Gush
|
|
118
134
|
sidekiq.push(
|
119
135
|
'class' => Gush::Worker,
|
120
136
|
'queue' => configuration.namespace,
|
121
|
-
'args' => [workflow_id, job.
|
137
|
+
'args' => [workflow_id, job.name]
|
122
138
|
)
|
123
139
|
end
|
124
140
|
|
@@ -128,6 +144,7 @@ module Gush
|
|
128
144
|
|
129
145
|
def workflow_from_hash(hash, nodes = nil)
|
130
146
|
flow = hash[:klass].constantize.new
|
147
|
+
flow.jobs = []
|
131
148
|
flow.stopped = hash.fetch(:stopped, false)
|
132
149
|
flow.id = hash[:id]
|
133
150
|
|
data/lib/gush/graph.rb
CHANGED
@@ -2,7 +2,7 @@ module Gush
|
|
2
2
|
class Graph
|
3
3
|
attr_reader :workflow, :filename, :path, :start, :end_node
|
4
4
|
|
5
|
-
def initialize(workflow, options
|
5
|
+
def initialize(workflow, options = {})
|
6
6
|
@workflow = workflow
|
7
7
|
@filename = options.fetch(:filename, "graph.png")
|
8
8
|
@path = options.fetch(:path, Pathname.new(Dir.tmpdir).join(filename))
|
@@ -31,17 +31,17 @@ module Gush
|
|
31
31
|
private
|
32
32
|
def add_job(graph, job)
|
33
33
|
name = job.class.to_s
|
34
|
-
graph.add_nodes(name)
|
34
|
+
graph.add_nodes(job.name, label: name)
|
35
35
|
|
36
36
|
if job.incoming.empty?
|
37
|
-
graph.add_edges(start, name)
|
37
|
+
graph.add_edges(start, job.name)
|
38
38
|
end
|
39
39
|
|
40
40
|
if job.outgoing.empty?
|
41
|
-
graph.add_edges(name, end_node)
|
41
|
+
graph.add_edges(job.name, end_node)
|
42
42
|
else
|
43
|
-
job.outgoing.each do |
|
44
|
-
graph.add_edges(name,
|
43
|
+
job.outgoing.each do |id|
|
44
|
+
graph.add_edges(job.name, id)
|
45
45
|
end
|
46
46
|
end
|
47
47
|
end
|
data/lib/gush/job.rb
CHANGED
@@ -1,8 +1,8 @@
|
|
1
1
|
module Gush
|
2
2
|
class Job
|
3
3
|
attr_accessor :workflow_id, :incoming, :outgoing, :params,
|
4
|
-
:finished_at, :failed_at, :started_at, :enqueued_at, :
|
5
|
-
attr_reader :name, :output_payload, :params
|
4
|
+
:finished_at, :failed_at, :started_at, :enqueued_at, :payloads_hash, :klass
|
5
|
+
attr_reader :name, :output_payload, :params, :payloads
|
6
6
|
|
7
7
|
def initialize(workflow, opts = {})
|
8
8
|
@workflow = workflow
|
@@ -37,6 +37,12 @@ module Gush
|
|
37
37
|
@output_payload = data
|
38
38
|
end
|
39
39
|
|
40
|
+
def payloads
|
41
|
+
payload_h = {}
|
42
|
+
payloads_hash.each {|k,val| payload_h[k.to_s] = val.map {|h| h[:payload] }}
|
43
|
+
payload_h
|
44
|
+
end
|
45
|
+
|
40
46
|
def work
|
41
47
|
end
|
42
48
|
|
@@ -98,6 +104,10 @@ module Gush
|
|
98
104
|
end
|
99
105
|
|
100
106
|
private
|
107
|
+
def logger
|
108
|
+
Sidekiq.logger
|
109
|
+
end
|
110
|
+
|
101
111
|
def current_timestamp
|
102
112
|
Time.now.to_i
|
103
113
|
end
|
@@ -111,6 +121,7 @@ module Gush
|
|
111
121
|
@started_at = opts[:started_at]
|
112
122
|
@enqueued_at = opts[:enqueued_at]
|
113
123
|
@params = opts[:params] || {}
|
124
|
+
@klass = opts[:klass]
|
114
125
|
@output_payload = opts[:output_payload]
|
115
126
|
end
|
116
127
|
end
|
data/lib/gush/json.rb
CHANGED
data/lib/gush/worker.rb
CHANGED
@@ -9,7 +9,7 @@ module Gush
|
|
9
9
|
def perform(workflow_id, job_id)
|
10
10
|
setup_job(workflow_id, job_id)
|
11
11
|
|
12
|
-
job.
|
12
|
+
job.payloads_hash = incoming_payloads
|
13
13
|
|
14
14
|
start = Time.now
|
15
15
|
report(:started, start)
|
@@ -20,19 +20,15 @@ module Gush
|
|
20
20
|
mark_as_started
|
21
21
|
begin
|
22
22
|
job.work
|
23
|
-
rescue Exception =>
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
unless failed
|
29
|
-
report(:finished, start)
|
23
|
+
rescue Exception => error
|
24
|
+
mark_as_failed
|
25
|
+
report(:failed, start, error.message)
|
26
|
+
raise error
|
27
|
+
else
|
30
28
|
mark_as_finished
|
29
|
+
report(:finished, start)
|
31
30
|
|
32
31
|
enqueue_outgoing_jobs
|
33
|
-
else
|
34
|
-
mark_as_failed
|
35
|
-
report(:failed, start, error.message)
|
36
32
|
end
|
37
33
|
end
|
38
34
|
|
@@ -51,9 +47,10 @@ module Gush
|
|
51
47
|
def incoming_payloads
|
52
48
|
payloads = {}
|
53
49
|
job.incoming.each do |job_name|
|
54
|
-
|
50
|
+
job = client.load_job(workflow.id, job_name)
|
51
|
+
payloads[job.klass.to_s] ||= []
|
52
|
+
payloads[job.klass.to_s] << {:id => job.name, :payload => job.output_payload}
|
55
53
|
end
|
56
|
-
|
57
54
|
payloads
|
58
55
|
end
|
59
56
|
|
data/lib/gush/workflow.rb
CHANGED
@@ -11,6 +11,8 @@ module Gush
|
|
11
11
|
@persisted = false
|
12
12
|
@stopped = false
|
13
13
|
@arguments = args
|
14
|
+
|
15
|
+
setup
|
14
16
|
end
|
15
17
|
|
16
18
|
def self.find(id)
|
@@ -23,10 +25,17 @@ module Gush
|
|
23
25
|
flow
|
24
26
|
end
|
25
27
|
|
28
|
+
def continue
|
29
|
+
client = Gush::Client.new
|
30
|
+
failed_jobs = jobs.select(&:failed?)
|
31
|
+
|
32
|
+
failed_jobs.each do |job|
|
33
|
+
client.enqueue_job(id, job)
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
26
37
|
def save
|
27
|
-
|
28
|
-
resolve_dependencies
|
29
|
-
client.persist_workflow(self)
|
38
|
+
persist!
|
30
39
|
end
|
31
40
|
|
32
41
|
def configure(*args)
|
@@ -63,7 +72,13 @@ module Gush
|
|
63
72
|
end
|
64
73
|
|
65
74
|
def find_job(name)
|
66
|
-
|
75
|
+
match_data = /(?<klass>\w*[^-])-(?<identifier>.*)/.match(name.to_s)
|
76
|
+
if match_data.nil?
|
77
|
+
job = jobs.find { |node| node.class.to_s == name.to_s }
|
78
|
+
else
|
79
|
+
job = jobs.find { |node| node.name.to_s == name.to_s }
|
80
|
+
end
|
81
|
+
job
|
67
82
|
end
|
68
83
|
|
69
84
|
def finished?
|
@@ -90,7 +105,7 @@ module Gush
|
|
90
105
|
options =
|
91
106
|
|
92
107
|
node = klass.new(self, {
|
93
|
-
name: klass.to_s,
|
108
|
+
name: client.next_free_job_id(id,klass.to_s),
|
94
109
|
params: opts.fetch(:params, {})
|
95
110
|
})
|
96
111
|
|
@@ -98,13 +113,15 @@ module Gush
|
|
98
113
|
|
99
114
|
deps_after = [*opts[:after]]
|
100
115
|
deps_after.each do |dep|
|
101
|
-
@dependencies << {from: dep.to_s, to:
|
116
|
+
@dependencies << {from: dep.to_s, to: node.name.to_s }
|
102
117
|
end
|
103
118
|
|
104
119
|
deps_before = [*opts[:before]]
|
105
120
|
deps_before.each do |dep|
|
106
|
-
@dependencies << {from:
|
121
|
+
@dependencies << {from: node.name.to_s, to: dep.to_s }
|
107
122
|
end
|
123
|
+
|
124
|
+
node.name
|
108
125
|
end
|
109
126
|
|
110
127
|
def reload
|
@@ -164,11 +181,16 @@ module Gush
|
|
164
181
|
end
|
165
182
|
|
166
183
|
def id
|
167
|
-
@id ||= client.
|
184
|
+
@id ||= client.next_free_workflow_id
|
168
185
|
end
|
169
186
|
|
170
187
|
private
|
171
188
|
|
189
|
+
def setup
|
190
|
+
configure(*@arguments)
|
191
|
+
resolve_dependencies
|
192
|
+
end
|
193
|
+
|
172
194
|
def client
|
173
195
|
@client ||= Client.new
|
174
196
|
end
|
@@ -18,19 +18,19 @@ describe "Workflows" do
|
|
18
18
|
flow = TestWorkflow.create
|
19
19
|
flow.start!
|
20
20
|
|
21
|
-
expect(Gush::Worker).to have_jobs(flow.id, [
|
21
|
+
expect(Gush::Worker).to have_jobs(flow.id, jobs_with_id(['Prepare']))
|
22
22
|
|
23
23
|
Gush::Worker.perform_one
|
24
|
-
expect(Gush::Worker).to have_jobs(flow.id, ["FetchFirstJob", "FetchSecondJob"])
|
24
|
+
expect(Gush::Worker).to have_jobs(flow.id, jobs_with_id(["FetchFirstJob", "FetchSecondJob"]))
|
25
25
|
|
26
26
|
Gush::Worker.perform_one
|
27
|
-
expect(Gush::Worker).to have_jobs(flow.id, ["FetchSecondJob", "PersistFirstJob"])
|
27
|
+
expect(Gush::Worker).to have_jobs(flow.id, jobs_with_id(["FetchSecondJob", "PersistFirstJob"]))
|
28
28
|
|
29
29
|
Gush::Worker.perform_one
|
30
|
-
expect(Gush::Worker).to have_jobs(flow.id, ["PersistFirstJob"])
|
30
|
+
expect(Gush::Worker).to have_jobs(flow.id, jobs_with_id(["PersistFirstJob"]))
|
31
31
|
|
32
32
|
Gush::Worker.perform_one
|
33
|
-
expect(Gush::Worker).to have_jobs(flow.id, ["NormalizeJob"])
|
33
|
+
expect(Gush::Worker).to have_jobs(flow.id, jobs_with_id(["NormalizeJob"]))
|
34
34
|
|
35
35
|
Gush::Worker.perform_one
|
36
36
|
|
@@ -52,7 +52,7 @@ describe "Workflows" do
|
|
52
52
|
|
53
53
|
class PrependJob < Gush::Job
|
54
54
|
def work
|
55
|
-
string = "#{payloads[
|
55
|
+
string = "#{payloads['PrefixJob'].first}: #{payloads['UpcaseJob'].first}"
|
56
56
|
output string
|
57
57
|
end
|
58
58
|
end
|
@@ -76,5 +76,47 @@ describe "Workflows" do
|
|
76
76
|
|
77
77
|
Gush::Worker.perform_one
|
78
78
|
expect(flow.reload.find_job("PrependJob").output_payload).to eq("A prefix: SOME TEXT")
|
79
|
+
|
80
|
+
|
81
|
+
end
|
82
|
+
|
83
|
+
it "passes payloads from workflow that runs multiple same class jobs with nameized payloads" do
|
84
|
+
class RepetitiveJob < Gush::Job
|
85
|
+
def work
|
86
|
+
output params[:input]
|
87
|
+
end
|
88
|
+
end
|
89
|
+
|
90
|
+
class SummaryJob < Gush::Job
|
91
|
+
def work
|
92
|
+
output payloads['RepetitiveJob']
|
93
|
+
end
|
94
|
+
end
|
95
|
+
|
96
|
+
class PayloadWorkflow < Gush::Workflow
|
97
|
+
def configure
|
98
|
+
jobs = []
|
99
|
+
jobs << run(RepetitiveJob, params: {input: 'first'})
|
100
|
+
jobs << run(RepetitiveJob, params: {input: 'second'})
|
101
|
+
jobs << run(RepetitiveJob, params: {input: 'third'})
|
102
|
+
run SummaryJob, after: jobs
|
103
|
+
end
|
104
|
+
end
|
105
|
+
|
106
|
+
flow = PayloadWorkflow.create
|
107
|
+
flow.start!
|
108
|
+
|
109
|
+
Gush::Worker.perform_one
|
110
|
+
expect(flow.reload.find_job(flow.jobs[0].name).output_payload).to eq('first')
|
111
|
+
|
112
|
+
Gush::Worker.perform_one
|
113
|
+
expect(flow.reload.find_job(flow.jobs[1].name).output_payload).to eq('second')
|
114
|
+
|
115
|
+
Gush::Worker.perform_one
|
116
|
+
expect(flow.reload.find_job(flow.jobs[2].name).output_payload).to eq('third')
|
117
|
+
|
118
|
+
Gush::Worker.perform_one
|
119
|
+
expect(flow.reload.find_job(flow.jobs[3].name).output_payload).to eq(%w(first second third))
|
120
|
+
|
79
121
|
end
|
80
122
|
end
|
@@ -34,9 +34,9 @@ describe Gush::Client do
|
|
34
34
|
end
|
35
35
|
|
36
36
|
it "removes stopped flag when the workflow is started" do
|
37
|
-
workflow = TestWorkflow.
|
37
|
+
workflow = TestWorkflow.create
|
38
38
|
workflow.mark_as_stopped
|
39
|
-
workflow.
|
39
|
+
workflow.persist!
|
40
40
|
expect {
|
41
41
|
client.start_workflow(workflow)
|
42
42
|
}.to change{client.find_workflow(workflow.id).stopped?}.from(true).to(false)
|
@@ -85,7 +85,9 @@ describe Gush::Client do
|
|
85
85
|
|
86
86
|
describe "#persist_job" do
|
87
87
|
it "persists JSON dump of the job in Redis" do
|
88
|
-
|
88
|
+
|
89
|
+
job = BobJob.new(name: 'bob')
|
90
|
+
|
89
91
|
client.persist_job('deadbeef', job)
|
90
92
|
expect(redis.keys("gush.jobs.deadbeef.*").length).to eq(1)
|
91
93
|
end
|
File without changes
|
@@ -0,0 +1,43 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Gush::Graph do
|
4
|
+
subject { described_class.new(TestWorkflow.create) }
|
5
|
+
let(:filename) { "graph.png" }
|
6
|
+
|
7
|
+
describe "#viz" do
|
8
|
+
it "runs GraphViz to render graph" do
|
9
|
+
node = double("node", :[]= => true)
|
10
|
+
edge = double("edge", :[]= => true)
|
11
|
+
graph = double("graph", node: node, edge: edge)
|
12
|
+
path = Pathname.new(Dir.tmpdir).join(filename)
|
13
|
+
expect(graph).to receive(:start).with(shape: 'diamond', fillcolor: '#CFF09E')
|
14
|
+
expect(graph).to receive(:end).with(shape: 'diamond', fillcolor: '#F56991')
|
15
|
+
|
16
|
+
expect(graph).to receive(:output).with(png: path.to_s)
|
17
|
+
|
18
|
+
expect(graph).to receive(:add_nodes).with(/Prepare/, label: "Prepare")
|
19
|
+
expect(graph).to receive(:add_nodes).with(/FetchFirstJob/, label: "FetchFirstJob")
|
20
|
+
expect(graph).to receive(:add_nodes).with(/FetchSecondJob/, label: "FetchSecondJob")
|
21
|
+
expect(graph).to receive(:add_nodes).with(/NormalizeJob/, label: "NormalizeJob")
|
22
|
+
expect(graph).to receive(:add_nodes).with(/PersistFirstJob/, label: "PersistFirstJob")
|
23
|
+
|
24
|
+
expect(graph).to receive(:add_edges).with(nil, /Prepare/)
|
25
|
+
expect(graph).to receive(:add_edges).with(/Prepare/, /FetchFirstJob/)
|
26
|
+
expect(graph).to receive(:add_edges).with(/Prepare/, /FetchSecondJob/)
|
27
|
+
expect(graph).to receive(:add_edges).with(/FetchFirstJob/, /PersistFirstJob/)
|
28
|
+
expect(graph).to receive(:add_edges).with(/FetchSecondJob/, /NormalizeJob/)
|
29
|
+
expect(graph).to receive(:add_edges).with(/PersistFirstJob/, /NormalizeJob/)
|
30
|
+
expect(graph).to receive(:add_edges).with(/NormalizeJob/, nil)
|
31
|
+
|
32
|
+
expect(GraphViz).to receive(:new).and_yield(graph)
|
33
|
+
|
34
|
+
subject.viz
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
describe "#path" do
|
39
|
+
it "returns string path to the rendered graph" do
|
40
|
+
expect(subject.path).to eq(Pathname.new(Dir.tmpdir).join(filename).to_s)
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Gush::JSON do
|
4
|
+
subject { described_class }
|
5
|
+
|
6
|
+
describe ".encode" do
|
7
|
+
it "encodes data to JSON" do
|
8
|
+
expect(subject.encode({a: 123})).to eq("{\"a\":123}")
|
9
|
+
end
|
10
|
+
end
|
11
|
+
|
12
|
+
describe ".decode" do
|
13
|
+
it "decodes JSON to data" do
|
14
|
+
expect(subject.decode("{\"a\":123}")).to eq({"a" => 123})
|
15
|
+
end
|
16
|
+
|
17
|
+
it "passes options to the internal parser" do
|
18
|
+
expect(subject.decode("{\"a\":123}", symbolize_keys: true)).to eq({a: 123})
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
@@ -23,7 +23,9 @@ describe Gush::Worker do
|
|
23
23
|
allow(job).to receive(:work).and_raise(StandardError)
|
24
24
|
expect(client).to receive(:worker_report).with(hash_including(status: :failed)).ordered
|
25
25
|
|
26
|
-
|
26
|
+
expect do
|
27
|
+
subject.perform(workflow.id, "Prepare")
|
28
|
+
end.to raise_error(StandardError)
|
27
29
|
expect(workflow.find_job("Prepare")).to be_failed
|
28
30
|
end
|
29
31
|
|
@@ -31,7 +33,9 @@ describe Gush::Worker do
|
|
31
33
|
allow(job).to receive(:work).and_raise(StandardError)
|
32
34
|
expect(client).to receive(:worker_report).with(hash_including(status: :failed)).ordered
|
33
35
|
|
34
|
-
|
36
|
+
expect do
|
37
|
+
subject.perform(workflow.id, "Prepare")
|
38
|
+
end.to raise_error(StandardError)
|
35
39
|
end
|
36
40
|
end
|
37
41
|
|
@@ -4,9 +4,6 @@ describe Gush::Workflow do
|
|
4
4
|
subject { TestWorkflow.create }
|
5
5
|
|
6
6
|
describe "#initialize" do
|
7
|
-
end
|
8
|
-
|
9
|
-
describe "#save" do
|
10
7
|
it "passes constructor arguments to the method" do
|
11
8
|
klass = Class.new(Gush::Workflow) do
|
12
9
|
def configure(*args)
|
@@ -15,12 +12,24 @@ describe Gush::Workflow do
|
|
15
12
|
end
|
16
13
|
end
|
17
14
|
|
15
|
+
expect_any_instance_of(klass).to receive(:configure).with("arg1", "arg2")
|
18
16
|
flow = klass.new("arg1", "arg2")
|
19
17
|
|
20
|
-
expect(flow).to receive(:configure).with("arg1", "arg2")
|
21
|
-
flow.save
|
22
18
|
end
|
19
|
+
end
|
20
|
+
|
21
|
+
describe "#status" do
|
22
|
+
context "when failed" do
|
23
|
+
it "returns :failed" do
|
24
|
+
flow = TestWorkflow.create
|
25
|
+
flow.find_job("Prepare").fail!
|
26
|
+
flow.persist!
|
27
|
+
expect(flow.reload.status).to eq(:failed)
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
23
31
|
|
32
|
+
describe "#save" do
|
24
33
|
context "workflow not persisted" do
|
25
34
|
it "sets persisted to true" do
|
26
35
|
flow = TestWorkflow.new
|
@@ -46,6 +55,20 @@ describe Gush::Workflow do
|
|
46
55
|
end
|
47
56
|
end
|
48
57
|
|
58
|
+
describe "#continue" do
|
59
|
+
it "enqueues failed jobs" do
|
60
|
+
flow = TestWorkflow.create
|
61
|
+
flow.find_job('Prepare').fail!
|
62
|
+
|
63
|
+
expect(flow.jobs.select(&:failed?)).not_to be_empty
|
64
|
+
|
65
|
+
flow.continue
|
66
|
+
|
67
|
+
expect(flow.jobs.select(&:failed?)).to be_empty
|
68
|
+
expect(flow.find_job('Prepare').failed_at).to be_nil
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
49
72
|
describe "#mark_as_stopped" do
|
50
73
|
it "marks workflow as stopped" do
|
51
74
|
expect{ subject.mark_as_stopped }.to change{subject.stopped?}.from(false).to(true)
|
@@ -70,42 +93,42 @@ describe Gush::Workflow do
|
|
70
93
|
|
71
94
|
result = JSON.parse(klass.create("arg1", "arg2").to_json)
|
72
95
|
expected = {
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
96
|
+
"id" => an_instance_of(String),
|
97
|
+
"name" => klass.to_s,
|
98
|
+
"klass" => klass.to_s,
|
99
|
+
"status" => "running",
|
100
|
+
"total" => 2,
|
101
|
+
"finished" => 0,
|
102
|
+
"started_at" => nil,
|
103
|
+
"finished_at" => nil,
|
104
|
+
"stopped" => false,
|
105
|
+
"arguments" => ["arg1", "arg2"],
|
106
|
+
"jobs" => [
|
107
|
+
{
|
108
|
+
"name"=>a_string_starting_with('FetchFirstJob'),
|
109
|
+
"klass"=>"FetchFirstJob",
|
110
|
+
"incoming"=>[],
|
111
|
+
"outgoing"=>[a_string_starting_with('PersistFirstJob')],
|
112
|
+
"finished_at"=>nil,
|
113
|
+
"started_at"=>nil,
|
114
|
+
"enqueued_at"=>nil,
|
115
|
+
"failed_at"=>nil,
|
116
|
+
"params" => {},
|
117
|
+
"output_payload" => nil
|
118
|
+
},
|
119
|
+
{
|
120
|
+
"name"=>a_string_starting_with('PersistFirstJob'),
|
121
|
+
"klass"=>"PersistFirstJob",
|
122
|
+
"incoming"=>["FetchFirstJob"],
|
123
|
+
"outgoing"=>[],
|
124
|
+
"finished_at"=>nil,
|
125
|
+
"started_at"=>nil,
|
126
|
+
"enqueued_at"=>nil,
|
127
|
+
"failed_at"=>nil,
|
128
|
+
"params" => {},
|
129
|
+
"output_payload" => nil
|
130
|
+
}
|
131
|
+
]
|
109
132
|
}
|
110
133
|
expect(result).to match(expected)
|
111
134
|
end
|
@@ -145,7 +168,7 @@ describe Gush::Workflow do
|
|
145
168
|
|
146
169
|
tree.resolve_dependencies
|
147
170
|
|
148
|
-
expect(tree.jobs.first.outgoing).to match_array([klass2.to_s])
|
171
|
+
expect(tree.jobs.first.outgoing).to match_array(jobs_with_id([klass2.to_s]))
|
149
172
|
end
|
150
173
|
|
151
174
|
it "allows `before` to accept an array of jobs" do
|
@@ -159,7 +182,7 @@ describe Gush::Workflow do
|
|
159
182
|
|
160
183
|
tree.resolve_dependencies
|
161
184
|
|
162
|
-
expect(tree.jobs.first.incoming).to match_array([klass2.to_s])
|
185
|
+
expect(tree.jobs.first.incoming).to match_array(jobs_with_id([klass2.to_s]))
|
163
186
|
end
|
164
187
|
|
165
188
|
it "attaches job as a child of the job in `after` key" do
|
@@ -170,7 +193,7 @@ describe Gush::Workflow do
|
|
170
193
|
tree.run(klass2, after: klass1)
|
171
194
|
tree.resolve_dependencies
|
172
195
|
job = tree.jobs.first
|
173
|
-
expect(job.outgoing).to match_array([klass2.to_s])
|
196
|
+
expect(job.outgoing).to match_array(jobs_with_id([klass2.to_s]))
|
174
197
|
end
|
175
198
|
|
176
199
|
it "attaches job as a parent of the job in `before` key" do
|
@@ -181,7 +204,7 @@ describe Gush::Workflow do
|
|
181
204
|
tree.run(klass2, before: klass1)
|
182
205
|
tree.resolve_dependencies
|
183
206
|
job = tree.jobs.first
|
184
|
-
expect(job.incoming).to match_array([klass2.to_s])
|
207
|
+
expect(job.incoming).to match_array(jobs_with_id([klass2.to_s]))
|
185
208
|
end
|
186
209
|
end
|
187
210
|
|
data/spec/spec_helper.rb
CHANGED
@@ -11,6 +11,7 @@ class FetchSecondJob < Gush::Job; end
|
|
11
11
|
class PersistFirstJob < Gush::Job; end
|
12
12
|
class PersistSecondJob < Gush::Job; end
|
13
13
|
class NormalizeJob < Gush::Job; end
|
14
|
+
class BobJob < Gush::Job; end
|
14
15
|
|
15
16
|
GUSHFILE = Pathname.new(__FILE__).parent.join("Gushfile.rb")
|
16
17
|
|
@@ -39,6 +40,14 @@ module GushHelpers
|
|
39
40
|
def redis
|
40
41
|
@redis ||= Redis.new(url: REDIS_URL)
|
41
42
|
end
|
43
|
+
|
44
|
+
def jobs_with_id(jobs_array)
|
45
|
+
jobs_array.map {|job_name| job_with_id(job_name) }
|
46
|
+
end
|
47
|
+
|
48
|
+
def job_with_id(job_name)
|
49
|
+
/#{job_name}-(?<identifier>.*)/
|
50
|
+
end
|
42
51
|
end
|
43
52
|
|
44
53
|
RSpec::Matchers.define :have_jobs do |flow, jobs|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: gush
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 0.3.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Piotrek Okoński
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2016-01-20 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: sidekiq
|
@@ -209,12 +209,15 @@ files:
|
|
209
209
|
- lib/gush/workflow.rb
|
210
210
|
- spec/Gushfile.rb
|
211
211
|
- spec/features/integration_spec.rb
|
212
|
-
- spec/
|
213
|
-
- spec/
|
214
|
-
- spec/
|
215
|
-
- spec/
|
216
|
-
- spec/
|
217
|
-
- spec/
|
212
|
+
- spec/gush/cli_spec.rb
|
213
|
+
- spec/gush/client_spec.rb
|
214
|
+
- spec/gush/configuration_spec.rb
|
215
|
+
- spec/gush/graph_spec.rb
|
216
|
+
- spec/gush/job_spec.rb
|
217
|
+
- spec/gush/json_spec.rb
|
218
|
+
- spec/gush/worker_spec.rb
|
219
|
+
- spec/gush/workflow_spec.rb
|
220
|
+
- spec/gush_spec.rb
|
218
221
|
- spec/spec_helper.rb
|
219
222
|
homepage: https://github.com/pokonski/gush
|
220
223
|
licenses:
|
@@ -236,18 +239,20 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
236
239
|
version: '0'
|
237
240
|
requirements: []
|
238
241
|
rubyforge_project:
|
239
|
-
rubygems_version: 2.4.
|
242
|
+
rubygems_version: 2.4.8
|
240
243
|
signing_key:
|
241
244
|
specification_version: 4
|
242
245
|
summary: Fast and distributed workflow runner using only Sidekiq and Redis
|
243
246
|
test_files:
|
244
247
|
- spec/Gushfile.rb
|
245
248
|
- spec/features/integration_spec.rb
|
246
|
-
- spec/
|
247
|
-
- spec/
|
248
|
-
- spec/
|
249
|
-
- spec/
|
250
|
-
- spec/
|
251
|
-
- spec/
|
249
|
+
- spec/gush/cli_spec.rb
|
250
|
+
- spec/gush/client_spec.rb
|
251
|
+
- spec/gush/configuration_spec.rb
|
252
|
+
- spec/gush/graph_spec.rb
|
253
|
+
- spec/gush/job_spec.rb
|
254
|
+
- spec/gush/json_spec.rb
|
255
|
+
- spec/gush/worker_spec.rb
|
256
|
+
- spec/gush/workflow_spec.rb
|
257
|
+
- spec/gush_spec.rb
|
252
258
|
- spec/spec_helper.rb
|
253
|
-
has_rdoc:
|