gush 2.1.0 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/ruby.yml +3 -28
- data/README.md +49 -4
- data/gush.gemspec +9 -6
- data/lib/gush/client.rb +7 -2
- data/lib/gush/job.rb +3 -1
- data/lib/gush/version.rb +3 -0
- data/lib/gush/workflow.rb +2 -1
- data/spec/features/integration_spec.rb +4 -3
- data/spec/gush/client_spec.rb +13 -1
- data/spec/gush/workflow_spec.rb +7 -0
- data/spec/spec_helper.rb +22 -0
- metadata +15 -12
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: e588202fe13ded7c99f192bda5bc33d3d6e8994deb3fb17f5a823d4882c4552f
|
4
|
+
data.tar.gz: 759593f344caf579cce496b1da9c64c9431d6f2cbb92cbcced181d49421fdefb
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 5d6068f23d178beb5dbfaa9cf317ae086542dedf33aeab82f82c8cab5a6ea569d932b2dc0a787978629e892a39f25270827c55fa4a6b0a6bb349f9ab87fca1db
|
7
|
+
data.tar.gz: 69ddc27452586d4b188969ece9ab73aefd90377fd93503f3e83c9fd074722d808a09cc620f98953756d7c684bab808db14550c15bada3e93111815613e52b78c
|
data/.github/workflows/ruby.yml
CHANGED
@@ -26,35 +26,10 @@ jobs:
|
|
26
26
|
runs-on: ubuntu-latest
|
27
27
|
strategy:
|
28
28
|
matrix:
|
29
|
-
rails_version: ['
|
30
|
-
ruby-version: ['
|
31
|
-
exclude:
|
32
|
-
- ruby-version: '3.0'
|
33
|
-
rails_version: '4.2.7'
|
34
|
-
- ruby-version: '3.1'
|
35
|
-
rails_version: '4.2.7'
|
36
|
-
- ruby-version: '3.0'
|
37
|
-
rails_version: '5.0'
|
38
|
-
- ruby-version: '3.1'
|
39
|
-
rails_version: '5.0'
|
40
|
-
- ruby-version: '3.0'
|
41
|
-
rails_version: '5.1'
|
42
|
-
- ruby-version: '3.1'
|
43
|
-
rails_version: '5.1'
|
44
|
-
- ruby-version: '3.0'
|
45
|
-
rails_version: '5.2'
|
46
|
-
- ruby-version: '3.1'
|
47
|
-
rails_version: '5.2'
|
48
|
-
- ruby-version: '3.0'
|
49
|
-
rails_version: '6.0'
|
50
|
-
- ruby-version: '3.1'
|
51
|
-
rails_version: '6.0'
|
52
|
-
- ruby-version: '3.1'
|
53
|
-
rails_version: '6.1'
|
54
|
-
- ruby-version: '2.6'
|
55
|
-
rails_version: '7.0'
|
29
|
+
rails_version: ['6.1.0', '7.0', '7.1.0']
|
30
|
+
ruby-version: ['3.0', '3.1', '3.2', '3.3']
|
56
31
|
steps:
|
57
|
-
- uses: actions/checkout@
|
32
|
+
- uses: actions/checkout@v4
|
58
33
|
- name: Set up Ruby
|
59
34
|
uses: ruby/setup-ruby@v1
|
60
35
|
with:
|
data/README.md
CHANGED
@@ -1,5 +1,9 @@
|
|
1
1
|
# Gush
|
2
2
|
|
3
|
+

|
4
|
+

|
5
|
+
|
6
|
+
|
3
7
|
Gush is a parallel workflow runner using only Redis as storage and [ActiveJob](http://guides.rubyonrails.org/v4.2/active_job_basics.html#introduction) for scheduling and executing jobs.
|
4
8
|
|
5
9
|
## Theory
|
@@ -15,7 +19,7 @@ This README is about the latest `master` code, which might differ from what is r
|
|
15
19
|
### 1. Add `gush` to Gemfile
|
16
20
|
|
17
21
|
```ruby
|
18
|
-
gem 'gush', '~>
|
22
|
+
gem 'gush', '~> 3.0'
|
19
23
|
```
|
20
24
|
|
21
25
|
### 2. Create `Gushfile`
|
@@ -74,7 +78,17 @@ end
|
|
74
78
|
|
75
79
|
and this is how the graph will look like:
|
76
80
|
|
77
|
-
|
81
|
+
```mermaid
|
82
|
+
graph TD
|
83
|
+
A{Start} --> B[FetchJob1]
|
84
|
+
A --> C[FetchJob2]
|
85
|
+
B --> D[PersistJob1]
|
86
|
+
C --> E[PersistJob2]
|
87
|
+
D --> F[NormalizeJob]
|
88
|
+
E --> F
|
89
|
+
F --> G[IndexJob]
|
90
|
+
G --> H{Finish}
|
91
|
+
```
|
78
92
|
|
79
93
|
|
80
94
|
## Defining workflows
|
@@ -211,7 +225,7 @@ For example, in case of Sidekiq this would be:
|
|
211
225
|
bundle exec sidekiq -q gush
|
212
226
|
```
|
213
227
|
|
214
|
-
**[Click here to see backends section in official ActiveJob documentation about configuring backends](http://guides.rubyonrails.org/
|
228
|
+
**[Click here to see backends section in official ActiveJob documentation about configuring backends](http://guides.rubyonrails.org/active_job_basics.html#backends)**
|
215
229
|
|
216
230
|
**Hint**: gush uses `gush` queue name by default. Keep that in mind, because some backends (like Sidekiq) will only run jobs from explicitly stated queues.
|
217
231
|
|
@@ -319,7 +333,21 @@ flow = NotifyWorkflow.create([54, 21, 24, 154, 65]) # 5 user ids as an argument
|
|
319
333
|
|
320
334
|
it will generate a workflow with 5 `NotificationJob`s and one `AdminNotificationJob` which will depend on all of them:
|
321
335
|
|
322
|
-
|
336
|
+
|
337
|
+
```mermaid
|
338
|
+
graph TD
|
339
|
+
A{Start} --> B[NotificationJob]
|
340
|
+
A --> C[NotificationJob]
|
341
|
+
A --> D[NotificationJob]
|
342
|
+
A --> E[NotificationJob]
|
343
|
+
A --> F[NotificationJob]
|
344
|
+
B --> G[AdminNotificationJob]
|
345
|
+
C --> G
|
346
|
+
D --> G
|
347
|
+
E --> G
|
348
|
+
F --> G
|
349
|
+
G --> H{Finish}
|
350
|
+
```
|
323
351
|
|
324
352
|
### Dynamic queue for jobs
|
325
353
|
|
@@ -338,6 +366,23 @@ class NotifyWorkflow < Gush::Workflow
|
|
338
366
|
end
|
339
367
|
```
|
340
368
|
|
369
|
+
### Dynamic waitable time for jobs
|
370
|
+
|
371
|
+
There might be a case you want to configure a job to be executed after a time. Based on above example, we want to configure `AdminNotificationJob` to be executed after 5 seconds.
|
372
|
+
|
373
|
+
```ruby
|
374
|
+
|
375
|
+
class NotifyWorkflow < Gush::Workflow
|
376
|
+
def configure(user_ids)
|
377
|
+
notification_jobs = user_ids.map do |user_id|
|
378
|
+
run NotificationJob, params: {user_id: user_id}, queue: 'user'
|
379
|
+
end
|
380
|
+
|
381
|
+
run AdminNotificationJob, after: notification_jobs, queue: 'admin', wait: 5.seconds
|
382
|
+
end
|
383
|
+
end
|
384
|
+
```
|
385
|
+
|
341
386
|
## Command line interface (CLI)
|
342
387
|
|
343
388
|
### Checking status
|
data/gush.gemspec
CHANGED
@@ -2,11 +2,13 @@
|
|
2
2
|
lib = File.expand_path('../lib', __FILE__)
|
3
3
|
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
4
|
|
5
|
+
require_relative 'lib/gush/version'
|
6
|
+
|
5
7
|
Gem::Specification.new do |spec|
|
6
8
|
spec.name = "gush"
|
7
|
-
spec.version =
|
8
|
-
spec.authors = ["Piotrek Okoński"]
|
9
|
-
spec.email = ["piotrek@okonski.org"]
|
9
|
+
spec.version = Gush::VERSION
|
10
|
+
spec.authors = ["Piotrek Okoński", "Michał Krzyżanowski"]
|
11
|
+
spec.email = ["piotrek@okonski.org", "michal.krzyzanowski+github@gmail.com"]
|
10
12
|
spec.summary = "Fast and distributed workflow runner based on ActiveJob and Redis"
|
11
13
|
spec.description = "Gush is a parallel workflow runner using Redis as storage and ActiveJob for executing jobs."
|
12
14
|
spec.homepage = "https://github.com/chaps-io/gush"
|
@@ -16,11 +18,12 @@ Gem::Specification.new do |spec|
|
|
16
18
|
spec.executables = "gush"
|
17
19
|
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
18
20
|
spec.require_paths = ["lib"]
|
21
|
+
spec.required_ruby_version = '>= 3.0.0'
|
19
22
|
|
20
|
-
spec.add_dependency "activejob", ">=
|
23
|
+
spec.add_dependency "activejob", ">= 6.1.0", "< 7.2"
|
21
24
|
spec.add_dependency "concurrent-ruby", "~> 1.0"
|
22
25
|
spec.add_dependency "multi_json", "~> 1.11"
|
23
|
-
spec.add_dependency "redis", ">= 3.2", "<
|
26
|
+
spec.add_dependency "redis", ">= 3.2", "< 6"
|
24
27
|
spec.add_dependency "redis-mutex", "~> 4.0.1"
|
25
28
|
spec.add_dependency "hiredis", "~> 0.6"
|
26
29
|
spec.add_dependency "graphviz", "~> 1.2"
|
@@ -29,7 +32,7 @@ Gem::Specification.new do |spec|
|
|
29
32
|
spec.add_dependency "thor", ">= 0.19", "< 1.3"
|
30
33
|
spec.add_dependency "launchy", "~> 2.4"
|
31
34
|
spec.add_development_dependency "bundler"
|
32
|
-
spec.add_development_dependency "rake", "~>
|
35
|
+
spec.add_development_dependency "rake", "~> 12"
|
33
36
|
spec.add_development_dependency "rspec", '~> 3.0'
|
34
37
|
spec.add_development_dependency "pry", '~> 0.10'
|
35
38
|
end
|
data/lib/gush/client.rb
CHANGED
@@ -156,8 +156,13 @@ module Gush
|
|
156
156
|
job.enqueue!
|
157
157
|
persist_job(workflow_id, job)
|
158
158
|
queue = job.queue || configuration.namespace
|
159
|
-
|
160
|
-
|
159
|
+
wait = job.wait
|
160
|
+
|
161
|
+
if wait.present?
|
162
|
+
Gush::Worker.set(queue: queue, wait: wait).perform_later(*[workflow_id, job.name])
|
163
|
+
else
|
164
|
+
Gush::Worker.set(queue: queue).perform_later(*[workflow_id, job.name])
|
165
|
+
end
|
161
166
|
end
|
162
167
|
|
163
168
|
private
|
data/lib/gush/job.rb
CHANGED
@@ -1,7 +1,8 @@
|
|
1
1
|
module Gush
|
2
2
|
class Job
|
3
3
|
attr_accessor :workflow_id, :incoming, :outgoing, :params,
|
4
|
-
:finished_at, :failed_at, :started_at, :enqueued_at, :payloads,
|
4
|
+
:finished_at, :failed_at, :started_at, :enqueued_at, :payloads,
|
5
|
+
:klass, :queue, :wait
|
5
6
|
attr_reader :id, :klass, :output_payload, :params
|
6
7
|
|
7
8
|
def initialize(opts = {})
|
@@ -126,6 +127,7 @@ module Gush
|
|
126
127
|
@output_payload = opts[:output_payload]
|
127
128
|
@workflow_id = opts[:workflow_id]
|
128
129
|
@queue = opts[:queue]
|
130
|
+
@wait = opts[:wait]
|
129
131
|
end
|
130
132
|
end
|
131
133
|
end
|
data/lib/gush/version.rb
ADDED
data/lib/gush/workflow.rb
CHANGED
@@ -5,9 +5,10 @@ describe "Workflows" do
|
|
5
5
|
context "when all jobs finish successfuly" do
|
6
6
|
it "marks workflow as completed" do
|
7
7
|
flow = TestWorkflow.create
|
8
|
-
|
9
|
-
|
10
|
-
|
8
|
+
|
9
|
+
ActiveJob::Base.queue_adapter.perform_enqueued_jobs = true
|
10
|
+
flow.start!
|
11
|
+
ActiveJob::Base.queue_adapter.perform_enqueued_jobs = false
|
11
12
|
|
12
13
|
flow = flow.reload
|
13
14
|
expect(flow).to be_finished
|
data/spec/gush/client_spec.rb
CHANGED
@@ -37,6 +37,18 @@ describe Gush::Client do
|
|
37
37
|
end
|
38
38
|
|
39
39
|
describe "#start_workflow" do
|
40
|
+
context "when there is wait parameter configured" do
|
41
|
+
let(:freeze_time) { Time.utc(2023, 01, 21, 14, 36, 0) }
|
42
|
+
|
43
|
+
it "schedules job execution" do
|
44
|
+
travel_to freeze_time do
|
45
|
+
workflow = WaitableTestWorkflow.create
|
46
|
+
client.start_workflow(workflow)
|
47
|
+
expect(Gush::Worker).to have_a_job_enqueued_at(workflow.id, job_with_id("Prepare"), 5.minutes)
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
40
52
|
it "enqueues next jobs from the workflow" do
|
41
53
|
workflow = TestWorkflow.create
|
42
54
|
expect {
|
@@ -113,7 +125,7 @@ describe Gush::Client do
|
|
113
125
|
describe "#persist_job" do
|
114
126
|
it "persists JSON dump of the job in Redis" do
|
115
127
|
|
116
|
-
job = BobJob.new(name: 'bob')
|
128
|
+
job = BobJob.new(name: 'bob', id: 'abcd123')
|
117
129
|
|
118
130
|
client.persist_job('deadbeef', job)
|
119
131
|
expect(redis.keys("gush.jobs.deadbeef.*").length).to eq(1)
|
data/spec/gush/workflow_spec.rb
CHANGED
@@ -121,6 +121,13 @@ describe Gush::Workflow do
|
|
121
121
|
expect(flow.jobs.first.params).to eq ({ something: 1 })
|
122
122
|
end
|
123
123
|
|
124
|
+
it "allows passing wait param to the job" do
|
125
|
+
flow = Gush::Workflow.new
|
126
|
+
flow.run(Gush::Job, wait: 5.seconds)
|
127
|
+
flow.save
|
128
|
+
expect(flow.jobs.first.wait).to eq (5.seconds)
|
129
|
+
end
|
130
|
+
|
124
131
|
context "when graph is empty" do
|
125
132
|
it "adds new job with the given class as a node" do
|
126
133
|
flow = Gush::Workflow.new
|
data/spec/spec_helper.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
require 'active_support'
|
2
|
+
require 'active_support/testing/time_helpers'
|
1
3
|
require 'gush'
|
2
4
|
require 'json'
|
3
5
|
require 'pry'
|
@@ -34,6 +36,11 @@ class ParameterTestWorkflow < Gush::Workflow
|
|
34
36
|
end
|
35
37
|
end
|
36
38
|
|
39
|
+
class WaitableTestWorkflow < Gush::Workflow
|
40
|
+
def configure
|
41
|
+
run Prepare, wait: 5.minutes
|
42
|
+
end
|
43
|
+
end
|
37
44
|
|
38
45
|
REDIS_URL = ENV["REDIS_URL"] || "redis://localhost:6379/12"
|
39
46
|
|
@@ -86,7 +93,22 @@ RSpec::Matchers.define :have_no_jobs do |flow, jobs|
|
|
86
93
|
end
|
87
94
|
end
|
88
95
|
|
96
|
+
RSpec::Matchers.define :have_a_job_enqueued_at do |flow, job, at|
|
97
|
+
expected_execution_timestamp = (Time.current.utc + at).to_i
|
98
|
+
|
99
|
+
match do |actual|
|
100
|
+
expected = hash_including(args: include(flow, job), at: expected_execution_timestamp)
|
101
|
+
|
102
|
+
expect(ActiveJob::Base.queue_adapter.enqueued_jobs).to match_array(expected)
|
103
|
+
end
|
104
|
+
|
105
|
+
failure_message do |actual|
|
106
|
+
"expected to have enqueued job #{job} to be executed at #{Time.current.utc + at}, but instead has: #{Time.at(enqueued_jobs.first[:at]).to_datetime.utc}"
|
107
|
+
end
|
108
|
+
end
|
109
|
+
|
89
110
|
RSpec.configure do |config|
|
111
|
+
config.include ActiveSupport::Testing::TimeHelpers
|
90
112
|
config.include ActiveJob::TestHelper
|
91
113
|
config.include GushHelpers
|
92
114
|
|
metadata
CHANGED
@@ -1,14 +1,15 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: gush
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 3.0.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Piotrek Okoński
|
8
|
+
- Michał Krzyżanowski
|
8
9
|
autorequire:
|
9
10
|
bindir: bin
|
10
11
|
cert_chain: []
|
11
|
-
date:
|
12
|
+
date: 2024-02-29 00:00:00.000000000 Z
|
12
13
|
dependencies:
|
13
14
|
- !ruby/object:Gem::Dependency
|
14
15
|
name: activejob
|
@@ -16,20 +17,20 @@ dependencies:
|
|
16
17
|
requirements:
|
17
18
|
- - ">="
|
18
19
|
- !ruby/object:Gem::Version
|
19
|
-
version:
|
20
|
+
version: 6.1.0
|
20
21
|
- - "<"
|
21
22
|
- !ruby/object:Gem::Version
|
22
|
-
version: '7.
|
23
|
+
version: '7.2'
|
23
24
|
type: :runtime
|
24
25
|
prerelease: false
|
25
26
|
version_requirements: !ruby/object:Gem::Requirement
|
26
27
|
requirements:
|
27
28
|
- - ">="
|
28
29
|
- !ruby/object:Gem::Version
|
29
|
-
version:
|
30
|
+
version: 6.1.0
|
30
31
|
- - "<"
|
31
32
|
- !ruby/object:Gem::Version
|
32
|
-
version: '7.
|
33
|
+
version: '7.2'
|
33
34
|
- !ruby/object:Gem::Dependency
|
34
35
|
name: concurrent-ruby
|
35
36
|
requirement: !ruby/object:Gem::Requirement
|
@@ -67,7 +68,7 @@ dependencies:
|
|
67
68
|
version: '3.2'
|
68
69
|
- - "<"
|
69
70
|
- !ruby/object:Gem::Version
|
70
|
-
version: '
|
71
|
+
version: '6'
|
71
72
|
type: :runtime
|
72
73
|
prerelease: false
|
73
74
|
version_requirements: !ruby/object:Gem::Requirement
|
@@ -77,7 +78,7 @@ dependencies:
|
|
77
78
|
version: '3.2'
|
78
79
|
- - "<"
|
79
80
|
- !ruby/object:Gem::Version
|
80
|
-
version: '
|
81
|
+
version: '6'
|
81
82
|
- !ruby/object:Gem::Dependency
|
82
83
|
name: redis-mutex
|
83
84
|
requirement: !ruby/object:Gem::Requirement
|
@@ -208,14 +209,14 @@ dependencies:
|
|
208
209
|
requirements:
|
209
210
|
- - "~>"
|
210
211
|
- !ruby/object:Gem::Version
|
211
|
-
version: '
|
212
|
+
version: '12'
|
212
213
|
type: :development
|
213
214
|
prerelease: false
|
214
215
|
version_requirements: !ruby/object:Gem::Requirement
|
215
216
|
requirements:
|
216
217
|
- - "~>"
|
217
218
|
- !ruby/object:Gem::Version
|
218
|
-
version: '
|
219
|
+
version: '12'
|
219
220
|
- !ruby/object:Gem::Dependency
|
220
221
|
name: rspec
|
221
222
|
requirement: !ruby/object:Gem::Requirement
|
@@ -248,6 +249,7 @@ description: Gush is a parallel workflow runner using Redis as storage and Activ
|
|
248
249
|
for executing jobs.
|
249
250
|
email:
|
250
251
|
- piotrek@okonski.org
|
252
|
+
- michal.krzyzanowski+github@gmail.com
|
251
253
|
executables:
|
252
254
|
- gush
|
253
255
|
extensions: []
|
@@ -272,6 +274,7 @@ files:
|
|
272
274
|
- lib/gush/graph.rb
|
273
275
|
- lib/gush/job.rb
|
274
276
|
- lib/gush/json.rb
|
277
|
+
- lib/gush/version.rb
|
275
278
|
- lib/gush/worker.rb
|
276
279
|
- lib/gush/workflow.rb
|
277
280
|
- spec/Gushfile
|
@@ -298,14 +301,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
298
301
|
requirements:
|
299
302
|
- - ">="
|
300
303
|
- !ruby/object:Gem::Version
|
301
|
-
version:
|
304
|
+
version: 3.0.0
|
302
305
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
303
306
|
requirements:
|
304
307
|
- - ">="
|
305
308
|
- !ruby/object:Gem::Version
|
306
309
|
version: '0'
|
307
310
|
requirements: []
|
308
|
-
rubygems_version: 3.
|
311
|
+
rubygems_version: 3.4.22
|
309
312
|
signing_key:
|
310
313
|
specification_version: 4
|
311
314
|
summary: Fast and distributed workflow runner based on ActiveJob and Redis
|