gush 3.0.0 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/ruby.yml +4 -0
- data/.rubocop.yml +232 -0
- data/CHANGELOG.md +1 -77
- data/README.md +107 -6
- data/gush.gemspec +4 -2
- data/lib/gush/cli/overview.rb +2 -0
- data/lib/gush/cli.rb +27 -4
- data/lib/gush/client.rb +116 -24
- data/lib/gush/graph.rb +2 -2
- data/lib/gush/job.rb +13 -4
- data/lib/gush/migrate/1_create_gush_workflows_created.rb +21 -0
- data/lib/gush/migration.rb +37 -0
- data/lib/gush/version.rb +1 -1
- data/lib/gush/worker.rb +1 -1
- data/lib/gush/workflow.rb +27 -15
- data/lib/gush.rb +1 -0
- data/spec/features/integration_spec.rb +3 -3
- data/spec/gush/client_spec.rb +303 -6
- data/spec/gush/job_spec.rb +48 -2
- data/spec/gush/migrate/1_create_gush_workflows_created_spec.rb +42 -0
- data/spec/gush/migration_spec.rb +23 -0
- data/spec/gush/workflow_spec.rb +114 -13
- data/spec/spec_helper.rb +7 -7
- metadata +51 -2
data/lib/gush/client.rb
CHANGED
@@ -72,7 +72,7 @@ module Gush
|
|
72
72
|
id = nil
|
73
73
|
loop do
|
74
74
|
id = SecureRandom.uuid
|
75
|
-
available = !redis.exists?("gush.
|
75
|
+
available = !redis.exists?("gush.workflows.#{id}")
|
76
76
|
|
77
77
|
break if available
|
78
78
|
end
|
@@ -80,6 +80,41 @@ module Gush
|
|
80
80
|
id
|
81
81
|
end
|
82
82
|
|
83
|
+
# Returns the specified range of workflow ids, sorted by created timestamp.
|
84
|
+
#
|
85
|
+
# @param start, stop [Integer] see https://redis.io/docs/latest/commands/zrange/#index-ranges
|
86
|
+
# for details on the start and stop parameters.
|
87
|
+
# @param by_ts [Boolean] if true, start and stop are treated as timestamps
|
88
|
+
# rather than as element indexes, which allows the workflows to be indexed
|
89
|
+
# by created timestamp
|
90
|
+
# @param order [Symbol] if :asc, finds ids in ascending created timestamp;
|
91
|
+
# if :desc, finds ids in descending created timestamp
|
92
|
+
# @returns [Array<String>] array of workflow ids
|
93
|
+
def workflow_ids(start=nil, stop=nil, by_ts: false, order: :asc)
|
94
|
+
start ||= 0
|
95
|
+
stop ||= 99
|
96
|
+
|
97
|
+
redis.zrange(
|
98
|
+
"gush.idx.workflows.created_at",
|
99
|
+
start,
|
100
|
+
stop,
|
101
|
+
by_score: by_ts,
|
102
|
+
rev: order&.to_sym == :desc
|
103
|
+
)
|
104
|
+
end
|
105
|
+
|
106
|
+
def workflows(start=nil, stop=nil, **kwargs)
|
107
|
+
workflow_ids(start, stop, **kwargs).map { |id| find_workflow(id) }
|
108
|
+
end
|
109
|
+
|
110
|
+
def workflows_count
|
111
|
+
redis.zcard('gush.idx.workflows.created_at')
|
112
|
+
end
|
113
|
+
|
114
|
+
# Deprecated.
|
115
|
+
#
|
116
|
+
# This method is not performant when there are a large number of workflows
|
117
|
+
# or when the redis keyspace is large. Use workflows instead with pagination.
|
83
118
|
def all_workflows
|
84
119
|
redis.scan_each(match: "gush.workflows.*").map do |key|
|
85
120
|
id = key.sub("gush.workflows.", "")
|
@@ -92,10 +127,16 @@ module Gush
|
|
92
127
|
|
93
128
|
unless data.nil?
|
94
129
|
hash = Gush::JSON.decode(data, symbolize_keys: true)
|
95
|
-
|
130
|
+
|
131
|
+
if hash[:job_klasses]
|
132
|
+
keys = hash[:job_klasses].map { |klass| "gush.jobs.#{id}.#{klass}" }
|
133
|
+
else
|
134
|
+
# For backwards compatibility, get job keys via a full keyspace scan
|
135
|
+
keys = redis.scan_each(match: "gush.jobs.#{id}.*")
|
136
|
+
end
|
96
137
|
|
97
138
|
nodes = keys.each_with_object([]) do |key, array|
|
98
|
-
array.concat
|
139
|
+
array.concat(redis.hvals(key).map { |json| Gush::JSON.decode(json, symbolize_keys: true) })
|
99
140
|
end
|
100
141
|
|
101
142
|
workflow_from_hash(hash, nodes)
|
@@ -105,15 +146,25 @@ module Gush
|
|
105
146
|
end
|
106
147
|
|
107
148
|
def persist_workflow(workflow)
|
149
|
+
created_at = Time.now.to_f
|
150
|
+
added = redis.zadd("gush.idx.workflows.created_at", created_at, workflow.id, nx: true)
|
151
|
+
|
152
|
+
if added && configuration.ttl&.positive?
|
153
|
+
expires_at = created_at + configuration.ttl
|
154
|
+
redis.zadd("gush.idx.workflows.expires_at", expires_at, workflow.id, nx: true)
|
155
|
+
end
|
156
|
+
|
108
157
|
redis.set("gush.workflows.#{workflow.id}", workflow.to_json)
|
109
158
|
|
110
|
-
workflow.jobs.each {|job| persist_job(workflow.id, job) }
|
159
|
+
workflow.jobs.each {|job| persist_job(workflow.id, job, expires_at: expires_at) }
|
111
160
|
workflow.mark_as_persisted
|
112
161
|
|
113
162
|
true
|
114
163
|
end
|
115
164
|
|
116
|
-
def persist_job(workflow_id, job)
|
165
|
+
def persist_job(workflow_id, job, expires_at: nil)
|
166
|
+
redis.zadd("gush.idx.jobs.expires_at", expires_at, "#{workflow_id}.#{job.klass}", nx: true) if expires_at
|
167
|
+
|
117
168
|
redis.hset("gush.jobs.#{workflow_id}.#{job.klass}", job.id, job.to_json)
|
118
169
|
end
|
119
170
|
|
@@ -134,35 +185,67 @@ module Gush
|
|
134
185
|
|
135
186
|
def destroy_workflow(workflow)
|
136
187
|
redis.del("gush.workflows.#{workflow.id}")
|
188
|
+
redis.zrem("gush.idx.workflows.created_at", workflow.id)
|
189
|
+
redis.zrem("gush.idx.workflows.expires_at", workflow.id)
|
137
190
|
workflow.jobs.each {|job| destroy_job(workflow.id, job) }
|
138
191
|
end
|
139
192
|
|
140
193
|
def destroy_job(workflow_id, job)
|
141
194
|
redis.del("gush.jobs.#{workflow_id}.#{job.klass}")
|
195
|
+
redis.zrem("gush.idx.jobs.expires_at", "#{workflow_id}.#{job.klass}")
|
196
|
+
end
|
197
|
+
|
198
|
+
def expire_workflows(expires_at=nil)
|
199
|
+
expires_at ||= Time.now.to_f
|
200
|
+
|
201
|
+
ids = redis.zrange("gush.idx.workflows.expires_at", "-inf", expires_at, by_score: true)
|
202
|
+
return if ids.empty?
|
203
|
+
|
204
|
+
redis.del(ids.map { |id| "gush.workflows.#{id}" })
|
205
|
+
redis.zrem("gush.idx.workflows.created_at", ids)
|
206
|
+
redis.zrem("gush.idx.workflows.expires_at", ids)
|
207
|
+
|
208
|
+
expire_jobs(expires_at)
|
209
|
+
end
|
210
|
+
|
211
|
+
def expire_jobs(expires_at=nil)
|
212
|
+
expires_at ||= Time.now.to_f
|
213
|
+
|
214
|
+
keys = redis.zrange("gush.idx.jobs.expires_at", "-inf", expires_at, by_score: true)
|
215
|
+
return if keys.empty?
|
216
|
+
|
217
|
+
redis.del(keys.map { |key| "gush.jobs.#{key}" })
|
218
|
+
redis.zrem("gush.idx.jobs.expires_at", keys)
|
142
219
|
end
|
143
220
|
|
144
221
|
def expire_workflow(workflow, ttl=nil)
|
145
|
-
ttl
|
146
|
-
|
222
|
+
ttl ||= configuration.ttl
|
223
|
+
|
224
|
+
if ttl&.positive?
|
225
|
+
redis.zadd("gush.idx.workflows.expires_at", Time.now.to_f + ttl, workflow.id)
|
226
|
+
else
|
227
|
+
redis.zrem("gush.idx.workflows.expires_at", workflow.id)
|
228
|
+
end
|
229
|
+
|
147
230
|
workflow.jobs.each {|job| expire_job(workflow.id, job, ttl) }
|
148
231
|
end
|
149
232
|
|
150
233
|
def expire_job(workflow_id, job, ttl=nil)
|
151
|
-
ttl
|
152
|
-
|
234
|
+
ttl ||= configuration.ttl
|
235
|
+
|
236
|
+
if ttl&.positive?
|
237
|
+
redis.zadd("gush.idx.jobs.expires_at", Time.now.to_f + ttl, "#{workflow_id}.#{job.klass}")
|
238
|
+
else
|
239
|
+
redis.zrem("gush.idx.jobs.expires_at", "#{workflow_id}.#{job.klass}")
|
240
|
+
end
|
153
241
|
end
|
154
242
|
|
155
243
|
def enqueue_job(workflow_id, job)
|
156
244
|
job.enqueue!
|
157
245
|
persist_job(workflow_id, job)
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
if wait.present?
|
162
|
-
Gush::Worker.set(queue: queue, wait: wait).perform_later(*[workflow_id, job.name])
|
163
|
-
else
|
164
|
-
Gush::Worker.set(queue: queue).perform_later(*[workflow_id, job.name])
|
165
|
-
end
|
246
|
+
|
247
|
+
options = { queue: configuration.namespace }.merge(job.worker_options)
|
248
|
+
job.enqueue_worker!(options)
|
166
249
|
end
|
167
250
|
|
168
251
|
private
|
@@ -183,16 +266,25 @@ module Gush
|
|
183
266
|
end
|
184
267
|
|
185
268
|
def workflow_from_hash(hash, nodes = [])
|
186
|
-
|
187
|
-
flow.jobs = []
|
188
|
-
flow.stopped = hash.fetch(:stopped, false)
|
189
|
-
flow.id = hash[:id]
|
190
|
-
|
191
|
-
flow.jobs = nodes.map do |node|
|
269
|
+
jobs = nodes.map do |node|
|
192
270
|
Gush::Job.from_hash(node)
|
193
271
|
end
|
194
272
|
|
195
|
-
|
273
|
+
internal_state = {
|
274
|
+
persisted: true,
|
275
|
+
jobs: jobs,
|
276
|
+
# For backwards compatibility, setup can only be skipped for a persisted
|
277
|
+
# workflow if there is no data missing from the persistence.
|
278
|
+
# 2024-07-23: dependencies added to persistence
|
279
|
+
skip_setup: !hash[:dependencies].nil?
|
280
|
+
}.merge(hash)
|
281
|
+
|
282
|
+
hash[:klass].constantize.new(
|
283
|
+
*hash[:arguments],
|
284
|
+
**hash[:kwargs],
|
285
|
+
globals: hash[:globals],
|
286
|
+
internal_state: internal_state
|
287
|
+
)
|
196
288
|
end
|
197
289
|
|
198
290
|
def redis
|
data/lib/gush/graph.rb
CHANGED
@@ -4,7 +4,7 @@ require 'tmpdir'
|
|
4
4
|
|
5
5
|
module Gush
|
6
6
|
class Graph
|
7
|
-
attr_reader :workflow, :filename, :
|
7
|
+
attr_reader :workflow, :filename, :start_node, :end_node
|
8
8
|
|
9
9
|
def initialize(workflow, options = {})
|
10
10
|
@workflow = workflow
|
@@ -32,7 +32,7 @@ module Gush
|
|
32
32
|
file_format = path.split('.')[-1]
|
33
33
|
format = file_format if file_format.length == 3
|
34
34
|
|
35
|
-
Graphviz
|
35
|
+
Graphviz.output(@graph, path: path, format: format)
|
36
36
|
end
|
37
37
|
|
38
38
|
def path
|
data/lib/gush/job.rb
CHANGED
@@ -1,9 +1,9 @@
|
|
1
1
|
module Gush
|
2
2
|
class Job
|
3
3
|
attr_accessor :workflow_id, :incoming, :outgoing, :params,
|
4
|
-
:finished_at, :failed_at, :started_at, :enqueued_at, :payloads,
|
5
|
-
:klass, :queue, :wait
|
6
|
-
attr_reader :id, :
|
4
|
+
:finished_at, :failed_at, :started_at, :enqueued_at, :payloads,
|
5
|
+
:klass, :queue, :wait
|
6
|
+
attr_reader :id, :output_payload
|
7
7
|
|
8
8
|
def initialize(opts = {})
|
9
9
|
options = opts.dup
|
@@ -23,7 +23,8 @@ module Gush
|
|
23
23
|
failed_at: failed_at,
|
24
24
|
params: params,
|
25
25
|
workflow_id: workflow_id,
|
26
|
-
output_payload: output_payload
|
26
|
+
output_payload: output_payload,
|
27
|
+
wait: wait
|
27
28
|
}
|
28
29
|
end
|
29
30
|
|
@@ -58,6 +59,14 @@ module Gush
|
|
58
59
|
@failed_at = nil
|
59
60
|
end
|
60
61
|
|
62
|
+
def enqueue_worker!(options = {})
|
63
|
+
Gush::Worker.set(options).perform_later(workflow_id, name)
|
64
|
+
end
|
65
|
+
|
66
|
+
def worker_options
|
67
|
+
{ queue: queue, wait: wait }.compact
|
68
|
+
end
|
69
|
+
|
61
70
|
def finish!
|
62
71
|
@finished_at = current_timestamp
|
63
72
|
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
module Gush
|
2
|
+
class IndexWorkflowsByCreatedAtAndExpiresAt < Gush::Migration
|
3
|
+
def self.version
|
4
|
+
1
|
5
|
+
end
|
6
|
+
|
7
|
+
def up
|
8
|
+
redis.scan_each(match: "gush.workflows.*").map do |key|
|
9
|
+
id = key.sub("gush.workflows.", "")
|
10
|
+
workflow = client.find_workflow(id)
|
11
|
+
|
12
|
+
ttl = redis.ttl(key)
|
13
|
+
redis.persist(key)
|
14
|
+
workflow.jobs.each { |job| redis.persist("gush.jobs.#{id}.#{job.klass}") }
|
15
|
+
|
16
|
+
client.persist_workflow(workflow)
|
17
|
+
client.expire_workflow(workflow, ttl.positive? ? ttl : -1)
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
@@ -0,0 +1,37 @@
|
|
1
|
+
module Gush
|
2
|
+
class Migration
|
3
|
+
def migrate
|
4
|
+
return if migrated?
|
5
|
+
|
6
|
+
up
|
7
|
+
migrated!
|
8
|
+
end
|
9
|
+
|
10
|
+
def up
|
11
|
+
# subclass responsibility
|
12
|
+
raise NotImplementedError
|
13
|
+
end
|
14
|
+
|
15
|
+
def version
|
16
|
+
self.class.version
|
17
|
+
end
|
18
|
+
|
19
|
+
def migrated?
|
20
|
+
redis.sismember("gush.migration.schema_migrations", version)
|
21
|
+
end
|
22
|
+
|
23
|
+
private
|
24
|
+
|
25
|
+
def migrated!
|
26
|
+
redis.sadd("gush.migration.schema_migrations", version)
|
27
|
+
end
|
28
|
+
|
29
|
+
def client
|
30
|
+
@client ||= Client.new
|
31
|
+
end
|
32
|
+
|
33
|
+
def redis
|
34
|
+
Gush::Client.redis_connection(client.configuration)
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
data/lib/gush/version.rb
CHANGED
data/lib/gush/worker.rb
CHANGED
data/lib/gush/workflow.rb
CHANGED
@@ -2,25 +2,33 @@ require 'securerandom'
|
|
2
2
|
|
3
3
|
module Gush
|
4
4
|
class Workflow
|
5
|
-
attr_accessor :
|
5
|
+
attr_accessor :jobs, :dependencies, :stopped, :persisted, :arguments, :kwargs, :globals
|
6
|
+
attr_writer :id
|
6
7
|
|
7
|
-
def initialize(*args)
|
8
|
-
@id = id
|
9
|
-
@jobs = []
|
10
|
-
@dependencies = []
|
11
|
-
@persisted = false
|
12
|
-
@stopped = false
|
8
|
+
def initialize(*args, globals: nil, internal_state: {}, **kwargs)
|
13
9
|
@arguments = args
|
10
|
+
@kwargs = kwargs
|
11
|
+
@globals = globals || {}
|
12
|
+
|
13
|
+
@id = internal_state[:id] || id
|
14
|
+
@jobs = internal_state[:jobs] || []
|
15
|
+
@dependencies = internal_state[:dependencies] || []
|
16
|
+
@persisted = internal_state[:persisted] || false
|
17
|
+
@stopped = internal_state[:stopped] || false
|
14
18
|
|
15
|
-
setup
|
19
|
+
setup unless internal_state[:skip_setup]
|
16
20
|
end
|
17
21
|
|
18
22
|
def self.find(id)
|
19
23
|
Gush::Client.new.find_workflow(id)
|
20
24
|
end
|
21
25
|
|
22
|
-
def self.
|
23
|
-
|
26
|
+
def self.page(start=0, stop=99, order: :asc)
|
27
|
+
Gush::Client.new.workflows(start, stop, order: order)
|
28
|
+
end
|
29
|
+
|
30
|
+
def self.create(*args, **kwargs)
|
31
|
+
flow = new(*args, **kwargs)
|
24
32
|
flow.save
|
25
33
|
flow
|
26
34
|
end
|
@@ -38,7 +46,7 @@ module Gush
|
|
38
46
|
persist!
|
39
47
|
end
|
40
48
|
|
41
|
-
def configure(*args)
|
49
|
+
def configure(*args, **kwargs)
|
42
50
|
end
|
43
51
|
|
44
52
|
def mark_as_stopped
|
@@ -53,7 +61,7 @@ module Gush
|
|
53
61
|
client.persist_workflow(self)
|
54
62
|
end
|
55
63
|
|
56
|
-
def expire!
|
64
|
+
def expire!(ttl=nil)
|
57
65
|
client.expire_workflow(self, ttl)
|
58
66
|
end
|
59
67
|
|
@@ -111,7 +119,7 @@ module Gush
|
|
111
119
|
node = klass.new({
|
112
120
|
workflow_id: id,
|
113
121
|
id: client.next_free_job_id(id, klass.to_s),
|
114
|
-
params: opts.fetch(:params, {}),
|
122
|
+
params: (@globals || {}).merge(opts.fetch(:params, {})),
|
115
123
|
queue: opts[:queue],
|
116
124
|
wait: opts[:wait]
|
117
125
|
})
|
@@ -157,7 +165,7 @@ module Gush
|
|
157
165
|
when stopped?
|
158
166
|
:stopped
|
159
167
|
else
|
160
|
-
:
|
168
|
+
:pending
|
161
169
|
end
|
162
170
|
end
|
163
171
|
|
@@ -175,9 +183,13 @@ module Gush
|
|
175
183
|
name: name,
|
176
184
|
id: id,
|
177
185
|
arguments: @arguments,
|
186
|
+
kwargs: @kwargs,
|
187
|
+
globals: @globals,
|
188
|
+
dependencies: @dependencies,
|
178
189
|
total: jobs.count,
|
179
190
|
finished: jobs.count(&:finished?),
|
180
191
|
klass: name,
|
192
|
+
job_klasses: jobs.map(&:class).map(&:to_s).uniq,
|
181
193
|
status: status,
|
182
194
|
stopped: stopped,
|
183
195
|
started_at: started_at,
|
@@ -200,7 +212,7 @@ module Gush
|
|
200
212
|
private
|
201
213
|
|
202
214
|
def setup
|
203
|
-
configure(*@arguments)
|
215
|
+
configure(*@arguments, **@kwargs)
|
204
216
|
resolve_dependencies
|
205
217
|
end
|
206
218
|
|
data/lib/gush.rb
CHANGED
@@ -136,7 +136,7 @@ describe "Workflows" do
|
|
136
136
|
|
137
137
|
class SummaryJob < Gush::Job
|
138
138
|
def perform
|
139
|
-
output
|
139
|
+
output(payloads.map { |payload| payload[:output] })
|
140
140
|
end
|
141
141
|
end
|
142
142
|
|
@@ -169,8 +169,8 @@ describe "Workflows" do
|
|
169
169
|
INTERNAL_CONFIGURE_SPY = double('configure spy')
|
170
170
|
expect(INTERNAL_SPY).to receive(:some_method).exactly(110).times
|
171
171
|
|
172
|
-
# One time when persisting
|
173
|
-
expect(INTERNAL_CONFIGURE_SPY).to receive(:some_method).exactly(
|
172
|
+
# One time when persisting; reloading does not call configure again
|
173
|
+
expect(INTERNAL_CONFIGURE_SPY).to receive(:some_method).exactly(1).time
|
174
174
|
|
175
175
|
class SimpleJob < Gush::Job
|
176
176
|
def perform
|