gush 3.0.0 → 4.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/lib/gush/client.rb CHANGED
@@ -72,7 +72,7 @@ module Gush
72
72
  id = nil
73
73
  loop do
74
74
  id = SecureRandom.uuid
75
- available = !redis.exists?("gush.workflow.#{id}")
75
+ available = !redis.exists?("gush.workflows.#{id}")
76
76
 
77
77
  break if available
78
78
  end
@@ -80,6 +80,41 @@ module Gush
80
80
  id
81
81
  end
82
82
 
83
+ # Returns the specified range of workflow ids, sorted by created timestamp.
84
+ #
85
+ # @param start, stop [Integer] see https://redis.io/docs/latest/commands/zrange/#index-ranges
86
+ # for details on the start and stop parameters.
87
+ # @param by_ts [Boolean] if true, start and stop are treated as timestamps
88
+ # rather than as element indexes, which allows the workflows to be indexed
89
+ # by created timestamp
90
+ # @param order [Symbol] if :asc, finds ids in ascending created timestamp;
91
+ # if :desc, finds ids in descending created timestamp
92
+ # @returns [Array<String>] array of workflow ids
93
+ def workflow_ids(start=nil, stop=nil, by_ts: false, order: :asc)
94
+ start ||= 0
95
+ stop ||= 99
96
+
97
+ redis.zrange(
98
+ "gush.idx.workflows.created_at",
99
+ start,
100
+ stop,
101
+ by_score: by_ts,
102
+ rev: order&.to_sym == :desc
103
+ )
104
+ end
105
+
106
+ def workflows(start=nil, stop=nil, **kwargs)
107
+ workflow_ids(start, stop, **kwargs).map { |id| find_workflow(id) }
108
+ end
109
+
110
+ def workflows_count
111
+ redis.zcard('gush.idx.workflows.created_at')
112
+ end
113
+
114
+ # Deprecated.
115
+ #
116
+ # This method is not performant when there are a large number of workflows
117
+ # or when the redis keyspace is large. Use workflows instead with pagination.
83
118
  def all_workflows
84
119
  redis.scan_each(match: "gush.workflows.*").map do |key|
85
120
  id = key.sub("gush.workflows.", "")
@@ -92,10 +127,16 @@ module Gush
92
127
 
93
128
  unless data.nil?
94
129
  hash = Gush::JSON.decode(data, symbolize_keys: true)
95
- keys = redis.scan_each(match: "gush.jobs.#{id}.*")
130
+
131
+ if hash[:job_klasses]
132
+ keys = hash[:job_klasses].map { |klass| "gush.jobs.#{id}.#{klass}" }
133
+ else
134
+ # For backwards compatibility, get job keys via a full keyspace scan
135
+ keys = redis.scan_each(match: "gush.jobs.#{id}.*")
136
+ end
96
137
 
97
138
  nodes = keys.each_with_object([]) do |key, array|
98
- array.concat redis.hvals(key).map { |json| Gush::JSON.decode(json, symbolize_keys: true) }
139
+ array.concat(redis.hvals(key).map { |json| Gush::JSON.decode(json, symbolize_keys: true) })
99
140
  end
100
141
 
101
142
  workflow_from_hash(hash, nodes)
@@ -105,15 +146,25 @@ module Gush
105
146
  end
106
147
 
107
148
  def persist_workflow(workflow)
149
+ created_at = Time.now.to_f
150
+ added = redis.zadd("gush.idx.workflows.created_at", created_at, workflow.id, nx: true)
151
+
152
+ if added && configuration.ttl&.positive?
153
+ expires_at = created_at + configuration.ttl
154
+ redis.zadd("gush.idx.workflows.expires_at", expires_at, workflow.id, nx: true)
155
+ end
156
+
108
157
  redis.set("gush.workflows.#{workflow.id}", workflow.to_json)
109
158
 
110
- workflow.jobs.each {|job| persist_job(workflow.id, job) }
159
+ workflow.jobs.each {|job| persist_job(workflow.id, job, expires_at: expires_at) }
111
160
  workflow.mark_as_persisted
112
161
 
113
162
  true
114
163
  end
115
164
 
116
- def persist_job(workflow_id, job)
165
+ def persist_job(workflow_id, job, expires_at: nil)
166
+ redis.zadd("gush.idx.jobs.expires_at", expires_at, "#{workflow_id}.#{job.klass}", nx: true) if expires_at
167
+
117
168
  redis.hset("gush.jobs.#{workflow_id}.#{job.klass}", job.id, job.to_json)
118
169
  end
119
170
 
@@ -134,35 +185,67 @@ module Gush
134
185
 
135
186
  def destroy_workflow(workflow)
136
187
  redis.del("gush.workflows.#{workflow.id}")
188
+ redis.zrem("gush.idx.workflows.created_at", workflow.id)
189
+ redis.zrem("gush.idx.workflows.expires_at", workflow.id)
137
190
  workflow.jobs.each {|job| destroy_job(workflow.id, job) }
138
191
  end
139
192
 
140
193
  def destroy_job(workflow_id, job)
141
194
  redis.del("gush.jobs.#{workflow_id}.#{job.klass}")
195
+ redis.zrem("gush.idx.jobs.expires_at", "#{workflow_id}.#{job.klass}")
196
+ end
197
+
198
+ def expire_workflows(expires_at=nil)
199
+ expires_at ||= Time.now.to_f
200
+
201
+ ids = redis.zrange("gush.idx.workflows.expires_at", "-inf", expires_at, by_score: true)
202
+ return if ids.empty?
203
+
204
+ redis.del(ids.map { |id| "gush.workflows.#{id}" })
205
+ redis.zrem("gush.idx.workflows.created_at", ids)
206
+ redis.zrem("gush.idx.workflows.expires_at", ids)
207
+
208
+ expire_jobs(expires_at)
209
+ end
210
+
211
+ def expire_jobs(expires_at=nil)
212
+ expires_at ||= Time.now.to_f
213
+
214
+ keys = redis.zrange("gush.idx.jobs.expires_at", "-inf", expires_at, by_score: true)
215
+ return if keys.empty?
216
+
217
+ redis.del(keys.map { |key| "gush.jobs.#{key}" })
218
+ redis.zrem("gush.idx.jobs.expires_at", keys)
142
219
  end
143
220
 
144
221
  def expire_workflow(workflow, ttl=nil)
145
- ttl = ttl || configuration.ttl
146
- redis.expire("gush.workflows.#{workflow.id}", ttl)
222
+ ttl ||= configuration.ttl
223
+
224
+ if ttl&.positive?
225
+ redis.zadd("gush.idx.workflows.expires_at", Time.now.to_f + ttl, workflow.id)
226
+ else
227
+ redis.zrem("gush.idx.workflows.expires_at", workflow.id)
228
+ end
229
+
147
230
  workflow.jobs.each {|job| expire_job(workflow.id, job, ttl) }
148
231
  end
149
232
 
150
233
  def expire_job(workflow_id, job, ttl=nil)
151
- ttl = ttl || configuration.ttl
152
- redis.expire("gush.jobs.#{workflow_id}.#{job.klass}", ttl)
234
+ ttl ||= configuration.ttl
235
+
236
+ if ttl&.positive?
237
+ redis.zadd("gush.idx.jobs.expires_at", Time.now.to_f + ttl, "#{workflow_id}.#{job.klass}")
238
+ else
239
+ redis.zrem("gush.idx.jobs.expires_at", "#{workflow_id}.#{job.klass}")
240
+ end
153
241
  end
154
242
 
155
243
  def enqueue_job(workflow_id, job)
156
244
  job.enqueue!
157
245
  persist_job(workflow_id, job)
158
- queue = job.queue || configuration.namespace
159
- wait = job.wait
160
-
161
- if wait.present?
162
- Gush::Worker.set(queue: queue, wait: wait).perform_later(*[workflow_id, job.name])
163
- else
164
- Gush::Worker.set(queue: queue).perform_later(*[workflow_id, job.name])
165
- end
246
+
247
+ options = { queue: configuration.namespace }.merge(job.worker_options)
248
+ job.enqueue_worker!(options)
166
249
  end
167
250
 
168
251
  private
@@ -183,16 +266,25 @@ module Gush
183
266
  end
184
267
 
185
268
  def workflow_from_hash(hash, nodes = [])
186
- flow = hash[:klass].constantize.new(*hash[:arguments])
187
- flow.jobs = []
188
- flow.stopped = hash.fetch(:stopped, false)
189
- flow.id = hash[:id]
190
-
191
- flow.jobs = nodes.map do |node|
269
+ jobs = nodes.map do |node|
192
270
  Gush::Job.from_hash(node)
193
271
  end
194
272
 
195
- flow
273
+ internal_state = {
274
+ persisted: true,
275
+ jobs: jobs,
276
+ # For backwards compatibility, setup can only be skipped for a persisted
277
+ # workflow if there is no data missing from the persistence.
278
+ # 2024-07-23: dependencies added to persistence
279
+ skip_setup: !hash[:dependencies].nil?
280
+ }.merge(hash)
281
+
282
+ hash[:klass].constantize.new(
283
+ *hash[:arguments],
284
+ **hash[:kwargs],
285
+ globals: hash[:globals],
286
+ internal_state: internal_state
287
+ )
196
288
  end
197
289
 
198
290
  def redis
data/lib/gush/graph.rb CHANGED
@@ -4,7 +4,7 @@ require 'tmpdir'
4
4
 
5
5
  module Gush
6
6
  class Graph
7
- attr_reader :workflow, :filename, :path, :start_node, :end_node
7
+ attr_reader :workflow, :filename, :start_node, :end_node
8
8
 
9
9
  def initialize(workflow, options = {})
10
10
  @workflow = workflow
@@ -32,7 +32,7 @@ module Gush
32
32
  file_format = path.split('.')[-1]
33
33
  format = file_format if file_format.length == 3
34
34
 
35
- Graphviz::output(@graph, path: path, format: format)
35
+ Graphviz.output(@graph, path: path, format: format)
36
36
  end
37
37
 
38
38
  def path
data/lib/gush/job.rb CHANGED
@@ -1,9 +1,9 @@
1
1
  module Gush
2
2
  class Job
3
3
  attr_accessor :workflow_id, :incoming, :outgoing, :params,
4
- :finished_at, :failed_at, :started_at, :enqueued_at, :payloads,
5
- :klass, :queue, :wait
6
- attr_reader :id, :klass, :output_payload, :params
4
+ :finished_at, :failed_at, :started_at, :enqueued_at, :payloads,
5
+ :klass, :queue, :wait
6
+ attr_reader :id, :output_payload
7
7
 
8
8
  def initialize(opts = {})
9
9
  options = opts.dup
@@ -23,7 +23,8 @@ module Gush
23
23
  failed_at: failed_at,
24
24
  params: params,
25
25
  workflow_id: workflow_id,
26
- output_payload: output_payload
26
+ output_payload: output_payload,
27
+ wait: wait
27
28
  }
28
29
  end
29
30
 
@@ -58,6 +59,14 @@ module Gush
58
59
  @failed_at = nil
59
60
  end
60
61
 
62
+ def enqueue_worker!(options = {})
63
+ Gush::Worker.set(options).perform_later(workflow_id, name)
64
+ end
65
+
66
+ def worker_options
67
+ { queue: queue, wait: wait }.compact
68
+ end
69
+
61
70
  def finish!
62
71
  @finished_at = current_timestamp
63
72
  end
@@ -0,0 +1,21 @@
1
+ module Gush
2
+ class IndexWorkflowsByCreatedAtAndExpiresAt < Gush::Migration
3
+ def self.version
4
+ 1
5
+ end
6
+
7
+ def up
8
+ redis.scan_each(match: "gush.workflows.*").map do |key|
9
+ id = key.sub("gush.workflows.", "")
10
+ workflow = client.find_workflow(id)
11
+
12
+ ttl = redis.ttl(key)
13
+ redis.persist(key)
14
+ workflow.jobs.each { |job| redis.persist("gush.jobs.#{id}.#{job.klass}") }
15
+
16
+ client.persist_workflow(workflow)
17
+ client.expire_workflow(workflow, ttl.positive? ? ttl : -1)
18
+ end
19
+ end
20
+ end
21
+ end
@@ -0,0 +1,37 @@
1
+ module Gush
2
+ class Migration
3
+ def migrate
4
+ return if migrated?
5
+
6
+ up
7
+ migrated!
8
+ end
9
+
10
+ def up
11
+ # subclass responsibility
12
+ raise NotImplementedError
13
+ end
14
+
15
+ def version
16
+ self.class.version
17
+ end
18
+
19
+ def migrated?
20
+ redis.sismember("gush.migration.schema_migrations", version)
21
+ end
22
+
23
+ private
24
+
25
+ def migrated!
26
+ redis.sadd("gush.migration.schema_migrations", version)
27
+ end
28
+
29
+ def client
30
+ @client ||= Client.new
31
+ end
32
+
33
+ def redis
34
+ Gush::Client.redis_connection(client.configuration)
35
+ end
36
+ end
37
+ end
data/lib/gush/version.rb CHANGED
@@ -1,3 +1,3 @@
1
1
  module Gush
2
- VERSION = '3.0.0'
2
+ VERSION = '4.0.0'.freeze
3
3
  end
data/lib/gush/worker.rb CHANGED
@@ -30,7 +30,7 @@ module Gush
30
30
 
31
31
  private
32
32
 
33
- attr_reader :client, :workflow_id, :job, :configuration
33
+ attr_reader :workflow_id, :job
34
34
 
35
35
  def client
36
36
  @client ||= Gush::Client.new(Gush.configuration)
data/lib/gush/workflow.rb CHANGED
@@ -2,25 +2,33 @@ require 'securerandom'
2
2
 
3
3
  module Gush
4
4
  class Workflow
5
- attr_accessor :id, :jobs, :stopped, :persisted, :arguments
5
+ attr_accessor :jobs, :dependencies, :stopped, :persisted, :arguments, :kwargs, :globals
6
+ attr_writer :id
6
7
 
7
- def initialize(*args)
8
- @id = id
9
- @jobs = []
10
- @dependencies = []
11
- @persisted = false
12
- @stopped = false
8
+ def initialize(*args, globals: nil, internal_state: {}, **kwargs)
13
9
  @arguments = args
10
+ @kwargs = kwargs
11
+ @globals = globals || {}
12
+
13
+ @id = internal_state[:id] || id
14
+ @jobs = internal_state[:jobs] || []
15
+ @dependencies = internal_state[:dependencies] || []
16
+ @persisted = internal_state[:persisted] || false
17
+ @stopped = internal_state[:stopped] || false
14
18
 
15
- setup
19
+ setup unless internal_state[:skip_setup]
16
20
  end
17
21
 
18
22
  def self.find(id)
19
23
  Gush::Client.new.find_workflow(id)
20
24
  end
21
25
 
22
- def self.create(*args)
23
- flow = new(*args)
26
+ def self.page(start=0, stop=99, order: :asc)
27
+ Gush::Client.new.workflows(start, stop, order: order)
28
+ end
29
+
30
+ def self.create(*args, **kwargs)
31
+ flow = new(*args, **kwargs)
24
32
  flow.save
25
33
  flow
26
34
  end
@@ -38,7 +46,7 @@ module Gush
38
46
  persist!
39
47
  end
40
48
 
41
- def configure(*args)
49
+ def configure(*args, **kwargs)
42
50
  end
43
51
 
44
52
  def mark_as_stopped
@@ -53,7 +61,7 @@ module Gush
53
61
  client.persist_workflow(self)
54
62
  end
55
63
 
56
- def expire! (ttl=nil)
64
+ def expire!(ttl=nil)
57
65
  client.expire_workflow(self, ttl)
58
66
  end
59
67
 
@@ -111,7 +119,7 @@ module Gush
111
119
  node = klass.new({
112
120
  workflow_id: id,
113
121
  id: client.next_free_job_id(id, klass.to_s),
114
- params: opts.fetch(:params, {}),
122
+ params: (@globals || {}).merge(opts.fetch(:params, {})),
115
123
  queue: opts[:queue],
116
124
  wait: opts[:wait]
117
125
  })
@@ -157,7 +165,7 @@ module Gush
157
165
  when stopped?
158
166
  :stopped
159
167
  else
160
- :running
168
+ :pending
161
169
  end
162
170
  end
163
171
 
@@ -175,9 +183,13 @@ module Gush
175
183
  name: name,
176
184
  id: id,
177
185
  arguments: @arguments,
186
+ kwargs: @kwargs,
187
+ globals: @globals,
188
+ dependencies: @dependencies,
178
189
  total: jobs.count,
179
190
  finished: jobs.count(&:finished?),
180
191
  klass: name,
192
+ job_klasses: jobs.map(&:class).map(&:to_s).uniq,
181
193
  status: status,
182
194
  stopped: stopped,
183
195
  started_at: started_at,
@@ -200,7 +212,7 @@ module Gush
200
212
  private
201
213
 
202
214
  def setup
203
- configure(*@arguments)
215
+ configure(*@arguments, **@kwargs)
204
216
  resolve_dependencies
205
217
  end
206
218
 
data/lib/gush.rb CHANGED
@@ -15,6 +15,7 @@ require "gush/client"
15
15
  require "gush/configuration"
16
16
  require "gush/errors"
17
17
  require "gush/job"
18
+ require "gush/migration"
18
19
  require "gush/worker"
19
20
  require "gush/workflow"
20
21
 
@@ -136,7 +136,7 @@ describe "Workflows" do
136
136
 
137
137
  class SummaryJob < Gush::Job
138
138
  def perform
139
- output payloads.map { |payload| payload[:output] }
139
+ output(payloads.map { |payload| payload[:output] })
140
140
  end
141
141
  end
142
142
 
@@ -169,8 +169,8 @@ describe "Workflows" do
169
169
  INTERNAL_CONFIGURE_SPY = double('configure spy')
170
170
  expect(INTERNAL_SPY).to receive(:some_method).exactly(110).times
171
171
 
172
- # One time when persisting, second time when reloading in the spec
173
- expect(INTERNAL_CONFIGURE_SPY).to receive(:some_method).exactly(2).times
172
+ # One time when persisting; reloading does not call configure again
173
+ expect(INTERNAL_CONFIGURE_SPY).to receive(:some_method).exactly(1).time
174
174
 
175
175
  class SimpleJob < Gush::Job
176
176
  def perform