gush 2.0.2 → 3.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/ruby.yml +46 -0
- data/CHANGELOG.md +12 -5
- data/Gemfile +4 -0
- data/README.md +85 -9
- data/bin/gush +2 -2
- data/gush.gemspec +13 -11
- data/lib/gush/cli/overview.rb +13 -13
- data/lib/gush/cli.rb +51 -20
- data/lib/gush/client.rb +8 -3
- data/lib/gush/configuration.rb +14 -10
- data/lib/gush/graph.rb +43 -32
- data/lib/gush/job.rb +3 -1
- data/lib/gush/version.rb +3 -0
- data/lib/gush/worker.rb +10 -2
- data/lib/gush/workflow.rb +2 -1
- data/spec/features/integration_spec.rb +9 -10
- data/spec/gush/client_spec.rb +13 -1
- data/spec/gush/configuration_spec.rb +6 -0
- data/spec/gush/graph_spec.rb +37 -20
- data/spec/gush/worker_spec.rb +8 -0
- data/spec/gush/workflow_spec.rb +7 -0
- data/spec/spec_helper.rb +25 -7
- metadata +39 -38
- data/.travis.yml +0 -16
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: e588202fe13ded7c99f192bda5bc33d3d6e8994deb3fb17f5a823d4882c4552f
|
4
|
+
data.tar.gz: 759593f344caf579cce496b1da9c64c9431d6f2cbb92cbcced181d49421fdefb
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 5d6068f23d178beb5dbfaa9cf317ae086542dedf33aeab82f82c8cab5a6ea569d932b2dc0a787978629e892a39f25270827c55fa4a6b0a6bb349f9ab87fca1db
|
7
|
+
data.tar.gz: 69ddc27452586d4b188969ece9ab73aefd90377fd93503f3e83c9fd074722d808a09cc620f98953756d7c684bab808db14550c15bada3e93111815613e52b78c
|
@@ -0,0 +1,46 @@
|
|
1
|
+
# This workflow uses actions that are not certified by GitHub.
|
2
|
+
# They are provided by a third-party and are governed by
|
3
|
+
# separate terms of service, privacy policy, and support
|
4
|
+
# documentation.
|
5
|
+
# This workflow will download a prebuilt Ruby version, install dependencies and run tests with Rake
|
6
|
+
# For more information see: https://github.com/marketplace/actions/setup-ruby-jruby-and-truffleruby
|
7
|
+
|
8
|
+
name: Ruby
|
9
|
+
|
10
|
+
on:
|
11
|
+
pull_request:
|
12
|
+
paths-ignore:
|
13
|
+
- 'README.md'
|
14
|
+
push:
|
15
|
+
paths-ignore:
|
16
|
+
- 'README.md'
|
17
|
+
|
18
|
+
jobs:
|
19
|
+
test:
|
20
|
+
services:
|
21
|
+
redis:
|
22
|
+
image: redis:alpine
|
23
|
+
ports: ["6379:6379"]
|
24
|
+
options: --entrypoint redis-server
|
25
|
+
|
26
|
+
runs-on: ubuntu-latest
|
27
|
+
strategy:
|
28
|
+
matrix:
|
29
|
+
rails_version: ['6.1.0', '7.0', '7.1.0']
|
30
|
+
ruby-version: ['3.0', '3.1', '3.2', '3.3']
|
31
|
+
steps:
|
32
|
+
- uses: actions/checkout@v4
|
33
|
+
- name: Set up Ruby
|
34
|
+
uses: ruby/setup-ruby@v1
|
35
|
+
with:
|
36
|
+
ruby-version: ${{ matrix.ruby-version }}
|
37
|
+
bundler-cache: true # runs 'bundle install' and caches installed gems automatically
|
38
|
+
env:
|
39
|
+
RAILS_VERSION: "${{ matrix.rails_version }}"
|
40
|
+
- name: Install Graphviz
|
41
|
+
run: sudo apt-get install graphviz
|
42
|
+
- name: Run tests
|
43
|
+
run: bundle exec rspec
|
44
|
+
env:
|
45
|
+
REDIS_URL: redis://localhost:6379/1
|
46
|
+
RAILS_VERSION: "${{ matrix.rails_version }}"
|
data/CHANGELOG.md
CHANGED
@@ -5,6 +5,13 @@ All notable changes to this project will be documented in this file.
|
|
5
5
|
The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
|
6
6
|
and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
|
7
7
|
|
8
|
+
## 2.1.0
|
9
|
+
|
10
|
+
### Added
|
11
|
+
|
12
|
+
- Allow RedisMutex’s locking duration and polling interval to be customizable, thanks to @thukim! [See pull request](https://github.com/chaps-io/gush/pull/74)
|
13
|
+
- Support for Rails 7.0 and Ruby 3.0-3.1, thanks to @joshRpowell and @kzkn!
|
14
|
+
|
8
15
|
## 2.0.1
|
9
16
|
|
10
17
|
### Fixed
|
@@ -13,31 +20,31 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
|
|
13
20
|
|
14
21
|
## 2.0.0
|
15
22
|
|
16
|
-
|
23
|
+
### Changed
|
17
24
|
|
18
25
|
- *[BREAKING]* Store gush jobs on redis hash instead of plain keys - this improves performance when retrieving keys (Thanks to @Saicheg! [See pull request](https://github.com/chaps-io/gush/pull/56))
|
19
26
|
|
20
27
|
|
21
|
-
|
28
|
+
### Added
|
22
29
|
|
23
30
|
- Allow setting queue for each job via `:queue` option in `run` method (Thanks to @devilankur18! [See pull request](https://github.com/chaps-io/gush/pull/58))
|
24
31
|
|
25
32
|
|
26
33
|
## 1.1.1 - 2018-06-09
|
27
34
|
|
28
|
-
|
35
|
+
### Changed
|
29
36
|
|
30
37
|
- Relax dependency on ActiveSupport to work with 4.2 up to 5.X (Thanks to @iacobus! [See pull request](https://github.com/chaps-io/gush/pull/54))
|
31
38
|
|
32
39
|
|
33
40
|
## 1.1.0 - 2018-02-05
|
34
41
|
|
35
|
-
|
42
|
+
### Added
|
36
43
|
|
37
44
|
- Added ability to specify TTL for Redis keys and manually expire whole workflows (Thanks to @dmitrypol! [See pull request](https://github.com/chaps-io/gush/pull/48))
|
38
45
|
- Loosened dependency on redis-rb library to >= 3.2 and < 5.0 (Thanks to @mofumofu3n! [See pull request](https://github.com/chaps-io/gush/pull/52))
|
39
46
|
|
40
|
-
|
47
|
+
### Fixed
|
41
48
|
|
42
49
|
- Improved performance of (de)serializing workflows by not storing job array inside workflow JSON and other smaller improvements ([See pull request](https://github.com/chaps-io/gush/pull/53))
|
43
50
|
|
data/Gemfile
CHANGED
data/README.md
CHANGED
@@ -1,4 +1,8 @@
|
|
1
|
-
# Gush
|
1
|
+
# Gush
|
2
|
+
|
3
|
+
![Gem Version](https://img.shields.io/gem/v/gush)
|
4
|
+
![GitHub Actions Workflow Status](https://img.shields.io/github/actions/workflow/status/chaps-io/gush/ruby.yml)
|
5
|
+
|
2
6
|
|
3
7
|
Gush is a parallel workflow runner using only Redis as storage and [ActiveJob](http://guides.rubyonrails.org/v4.2/active_job_basics.html#introduction) for scheduling and executing jobs.
|
4
8
|
|
@@ -8,14 +12,14 @@ Gush relies on directed acyclic graphs to store dependencies, see [Parallelizing
|
|
8
12
|
|
9
13
|
## **WARNING - version notice**
|
10
14
|
|
11
|
-
This README is about the `
|
15
|
+
This README is about the latest `master` code, which might differ from what is released on RubyGems. See tags to browse previous READMEs.
|
12
16
|
|
13
17
|
## Installation
|
14
18
|
|
15
19
|
### 1. Add `gush` to Gemfile
|
16
20
|
|
17
21
|
```ruby
|
18
|
-
gem 'gush', '~>
|
22
|
+
gem 'gush', '~> 3.0'
|
19
23
|
```
|
20
24
|
|
21
25
|
### 2. Create `Gushfile`
|
@@ -74,7 +78,17 @@ end
|
|
74
78
|
|
75
79
|
and this is how the graph will look like:
|
76
80
|
|
77
|
-
|
81
|
+
```mermaid
|
82
|
+
graph TD
|
83
|
+
A{Start} --> B[FetchJob1]
|
84
|
+
A --> C[FetchJob2]
|
85
|
+
B --> D[PersistJob1]
|
86
|
+
C --> E[PersistJob2]
|
87
|
+
D --> F[NormalizeJob]
|
88
|
+
E --> F
|
89
|
+
F --> G[IndexJob]
|
90
|
+
G --> H{Finish}
|
91
|
+
```
|
78
92
|
|
79
93
|
|
80
94
|
## Defining workflows
|
@@ -211,7 +225,7 @@ For example, in case of Sidekiq this would be:
|
|
211
225
|
bundle exec sidekiq -q gush
|
212
226
|
```
|
213
227
|
|
214
|
-
**[Click here to see backends section in official ActiveJob documentation about configuring backends](http://guides.rubyonrails.org/
|
228
|
+
**[Click here to see backends section in official ActiveJob documentation about configuring backends](http://guides.rubyonrails.org/active_job_basics.html#backends)**
|
215
229
|
|
216
230
|
**Hint**: gush uses `gush` queue name by default. Keep that in mind, because some backends (like Sidekiq) will only run jobs from explicitly stated queues.
|
217
231
|
|
@@ -274,7 +288,7 @@ class EncodeVideo < Gush::Job
|
|
274
288
|
end
|
275
289
|
```
|
276
290
|
|
277
|
-
`payloads` is an array containing outputs from all ancestor jobs. So for our `
|
291
|
+
`payloads` is an array containing outputs from all ancestor jobs. So for our `EncodeVideo` job from above, the array will look like:
|
278
292
|
|
279
293
|
|
280
294
|
```ruby
|
@@ -319,7 +333,55 @@ flow = NotifyWorkflow.create([54, 21, 24, 154, 65]) # 5 user ids as an argument
|
|
319
333
|
|
320
334
|
it will generate a workflow with 5 `NotificationJob`s and one `AdminNotificationJob` which will depend on all of them:
|
321
335
|
|
322
|
-
|
336
|
+
|
337
|
+
```mermaid
|
338
|
+
graph TD
|
339
|
+
A{Start} --> B[NotificationJob]
|
340
|
+
A --> C[NotificationJob]
|
341
|
+
A --> D[NotificationJob]
|
342
|
+
A --> E[NotificationJob]
|
343
|
+
A --> F[NotificationJob]
|
344
|
+
B --> G[AdminNotificationJob]
|
345
|
+
C --> G
|
346
|
+
D --> G
|
347
|
+
E --> G
|
348
|
+
F --> G
|
349
|
+
G --> H{Finish}
|
350
|
+
```
|
351
|
+
|
352
|
+
### Dynamic queue for jobs
|
353
|
+
|
354
|
+
There might be a case you want to configure different jobs in the workflow using different queues. Based on the above the example, we want to config `AdminNotificationJob` to use queue `admin` and `NotificationJob` use queue `user`.
|
355
|
+
|
356
|
+
```ruby
|
357
|
+
|
358
|
+
class NotifyWorkflow < Gush::Workflow
|
359
|
+
def configure(user_ids)
|
360
|
+
notification_jobs = user_ids.map do |user_id|
|
361
|
+
run NotificationJob, params: {user_id: user_id}, queue: 'user'
|
362
|
+
end
|
363
|
+
|
364
|
+
run AdminNotificationJob, after: notification_jobs, queue: 'admin'
|
365
|
+
end
|
366
|
+
end
|
367
|
+
```
|
368
|
+
|
369
|
+
### Dynamic waitable time for jobs
|
370
|
+
|
371
|
+
There might be a case you want to configure a job to be executed after a time. Based on above example, we want to configure `AdminNotificationJob` to be executed after 5 seconds.
|
372
|
+
|
373
|
+
```ruby
|
374
|
+
|
375
|
+
class NotifyWorkflow < Gush::Workflow
|
376
|
+
def configure(user_ids)
|
377
|
+
notification_jobs = user_ids.map do |user_id|
|
378
|
+
run NotificationJob, params: {user_id: user_id}, queue: 'user'
|
379
|
+
end
|
380
|
+
|
381
|
+
run AdminNotificationJob, after: notification_jobs, queue: 'admin', wait: 5.seconds
|
382
|
+
end
|
383
|
+
end
|
384
|
+
```
|
323
385
|
|
324
386
|
## Command line interface (CLI)
|
325
387
|
|
@@ -346,9 +408,23 @@ This requires that you have imagemagick installed on your computer:
|
|
346
408
|
bundle exec gush viz <NameOfTheWorkflow>
|
347
409
|
```
|
348
410
|
|
411
|
+
### Customizing locking options
|
412
|
+
|
413
|
+
In order to prevent getting the RedisMutex::LockError error when having a large number of jobs, you can customize these 2 fields `locking_duration` and `polling_interval` as below
|
414
|
+
|
415
|
+
```ruby
|
416
|
+
# config/initializers/gush.rb
|
417
|
+
Gush.configure do |config|
|
418
|
+
config.redis_url = "redis://localhost:6379"
|
419
|
+
config.concurrency = 5
|
420
|
+
config.locking_duration = 2 # how long you want to wait for the lock to be released, in seconds
|
421
|
+
config.polling_interval = 0.3 # how long the polling interval should be, in seconds
|
422
|
+
end
|
423
|
+
```
|
424
|
+
|
349
425
|
### Cleaning up afterwards
|
350
426
|
|
351
|
-
Running `NotifyWorkflow.create` inserts multiple keys into Redis every time it is ran. This data might be useful for analysis but at a certain point it can be purged via Redis TTL. By default gush and Redis will keep keys forever. To configure expiration you need to 2 things. Create initializer (specify config.ttl in seconds, be different per environment).
|
427
|
+
Running `NotifyWorkflow.create` inserts multiple keys into Redis every time it is ran. This data might be useful for analysis but at a certain point it can be purged via Redis TTL. By default gush and Redis will keep keys forever. To configure expiration you need to 2 things. Create initializer (specify config.ttl in seconds, be different per environment).
|
352
428
|
|
353
429
|
```ruby
|
354
430
|
# config/initializers/gush.rb
|
@@ -359,7 +435,7 @@ Gush.configure do |config|
|
|
359
435
|
end
|
360
436
|
```
|
361
437
|
|
362
|
-
And you need to call `flow.expire!` (optionally passing custom TTL value overriding `config.ttl`). This gives you control whether to expire data for specific workflow. Best NOT to set TTL to be too short (like minutes) but about a week in length. And you can run `Client.expire_workflow` and `Client.expire_job` passing appropriate IDs and TTL (pass -1 to NOT expire) values.
|
438
|
+
And you need to call `flow.expire!` (optionally passing custom TTL value overriding `config.ttl`). This gives you control whether to expire data for specific workflow. Best NOT to set TTL to be too short (like minutes) but about a week in length. And you can run `Client.expire_workflow` and `Client.expire_job` passing appropriate IDs and TTL (pass -1 to NOT expire) values.
|
363
439
|
|
364
440
|
### Avoid overlapping workflows
|
365
441
|
|
data/bin/gush
CHANGED
@@ -12,7 +12,7 @@ require 'gush'
|
|
12
12
|
begin
|
13
13
|
Gush::CLI.start(ARGV)
|
14
14
|
rescue Gush::WorkflowNotFound
|
15
|
-
puts "Workflow not found"
|
15
|
+
puts Paint["Workflow not found", :red]
|
16
16
|
rescue Gush::DependencyLevelTooDeep
|
17
|
-
puts "Dependency level too deep. Perhaps you have a dependency cycle?"
|
17
|
+
puts Paint["Dependency level too deep. Perhaps you have a dependency cycle?", :red]
|
18
18
|
end
|
data/gush.gemspec
CHANGED
@@ -2,11 +2,13 @@
|
|
2
2
|
lib = File.expand_path('../lib', __FILE__)
|
3
3
|
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
4
|
|
5
|
+
require_relative 'lib/gush/version'
|
6
|
+
|
5
7
|
Gem::Specification.new do |spec|
|
6
8
|
spec.name = "gush"
|
7
|
-
spec.version =
|
8
|
-
spec.authors = ["Piotrek Okoński"]
|
9
|
-
spec.email = ["piotrek@okonski.org"]
|
9
|
+
spec.version = Gush::VERSION
|
10
|
+
spec.authors = ["Piotrek Okoński", "Michał Krzyżanowski"]
|
11
|
+
spec.email = ["piotrek@okonski.org", "michal.krzyzanowski+github@gmail.com"]
|
10
12
|
spec.summary = "Fast and distributed workflow runner based on ActiveJob and Redis"
|
11
13
|
spec.description = "Gush is a parallel workflow runner using Redis as storage and ActiveJob for executing jobs."
|
12
14
|
spec.homepage = "https://github.com/chaps-io/gush"
|
@@ -16,21 +18,21 @@ Gem::Specification.new do |spec|
|
|
16
18
|
spec.executables = "gush"
|
17
19
|
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
18
20
|
spec.require_paths = ["lib"]
|
21
|
+
spec.required_ruby_version = '>= 3.0.0'
|
19
22
|
|
20
|
-
spec.add_dependency "activejob", ">=
|
23
|
+
spec.add_dependency "activejob", ">= 6.1.0", "< 7.2"
|
21
24
|
spec.add_dependency "concurrent-ruby", "~> 1.0"
|
22
25
|
spec.add_dependency "multi_json", "~> 1.11"
|
23
|
-
spec.add_dependency "redis", ">= 3.2", "<
|
26
|
+
spec.add_dependency "redis", ">= 3.2", "< 6"
|
24
27
|
spec.add_dependency "redis-mutex", "~> 4.0.1"
|
25
28
|
spec.add_dependency "hiredis", "~> 0.6"
|
26
|
-
spec.add_dependency "
|
27
|
-
spec.add_dependency "terminal-table", "
|
28
|
-
spec.add_dependency "
|
29
|
-
spec.add_dependency "thor", "
|
29
|
+
spec.add_dependency "graphviz", "~> 1.2"
|
30
|
+
spec.add_dependency "terminal-table", ">= 1.4", "< 3.1"
|
31
|
+
spec.add_dependency "paint", "~> 2.2"
|
32
|
+
spec.add_dependency "thor", ">= 0.19", "< 1.3"
|
30
33
|
spec.add_dependency "launchy", "~> 2.4"
|
31
34
|
spec.add_development_dependency "bundler"
|
32
|
-
spec.add_development_dependency "rake", "~>
|
35
|
+
spec.add_development_dependency "rake", "~> 12"
|
33
36
|
spec.add_development_dependency "rspec", '~> 3.0'
|
34
37
|
spec.add_development_dependency "pry", '~> 0.10'
|
35
|
-
spec.add_development_dependency 'fakeredis', '~> 0.5'
|
36
38
|
end
|
data/lib/gush/cli/overview.rb
CHANGED
@@ -17,11 +17,11 @@ module Gush
|
|
17
17
|
elsif workflow.running?
|
18
18
|
running_status
|
19
19
|
elsif workflow.finished?
|
20
|
-
"done"
|
20
|
+
Paint["done", :green]
|
21
21
|
elsif workflow.stopped?
|
22
|
-
"stopped"
|
22
|
+
Paint["stopped", :red]
|
23
23
|
else
|
24
|
-
"ready to start"
|
24
|
+
Paint["ready to start", :blue]
|
25
25
|
end
|
26
26
|
end
|
27
27
|
|
@@ -48,10 +48,10 @@ module Gush
|
|
48
48
|
"ID" => workflow.id,
|
49
49
|
"Name" => workflow.class.to_s,
|
50
50
|
"Jobs" => workflow.jobs.count,
|
51
|
-
"Failed jobs" => failed_jobs_count
|
52
|
-
"Succeeded jobs" => succeeded_jobs_count
|
53
|
-
"Enqueued jobs" => enqueued_jobs_count
|
54
|
-
"Running jobs" => running_jobs_count
|
51
|
+
"Failed jobs" => Paint[failed_jobs_count, :red],
|
52
|
+
"Succeeded jobs" => Paint[succeeded_jobs_count, :green],
|
53
|
+
"Enqueued jobs" => Paint[enqueued_jobs_count, :yellow],
|
54
|
+
"Running jobs" => Paint[running_jobs_count, :blue],
|
55
55
|
"Remaining jobs" => remaining_jobs_count,
|
56
56
|
"Started at" => started_at,
|
57
57
|
"Status" => status
|
@@ -60,7 +60,7 @@ module Gush
|
|
60
60
|
|
61
61
|
def running_status
|
62
62
|
finished = succeeded_jobs_count.to_i
|
63
|
-
status = "running"
|
63
|
+
status = Paint["running", :yellow]
|
64
64
|
status += "\n#{finished}/#{total_jobs_count} [#{(finished*100)/total_jobs_count}%]"
|
65
65
|
end
|
66
66
|
|
@@ -69,7 +69,7 @@ module Gush
|
|
69
69
|
end
|
70
70
|
|
71
71
|
def failed_status
|
72
|
-
status = "failed"
|
72
|
+
status = Paint["failed", :red]
|
73
73
|
status += "\n#{failed_job} failed"
|
74
74
|
end
|
75
75
|
|
@@ -77,13 +77,13 @@ module Gush
|
|
77
77
|
name = job.name
|
78
78
|
case
|
79
79
|
when job.failed?
|
80
|
-
"[✗] #{name
|
80
|
+
"[✗] #{Paint[name, :red]} \n"
|
81
81
|
when job.finished?
|
82
|
-
"[✓] #{name
|
82
|
+
"[✓] #{Paint[name, :green]} \n"
|
83
83
|
when job.enqueued?
|
84
|
-
"[•] #{name
|
84
|
+
"[•] #{Paint[name, :yellow]} \n"
|
85
85
|
when job.running?
|
86
|
-
"[•] #{name
|
86
|
+
"[•] #{Paint[name, :blue]} \n"
|
87
87
|
else
|
88
88
|
"[ ] #{name} \n"
|
89
89
|
end
|
data/lib/gush/cli.rb
CHANGED
@@ -1,5 +1,7 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'terminal-table'
|
2
|
-
require '
|
4
|
+
require 'paint'
|
3
5
|
require 'thor'
|
4
6
|
require 'launchy'
|
5
7
|
|
@@ -12,43 +14,45 @@ module Gush
|
|
12
14
|
def initialize(*)
|
13
15
|
super
|
14
16
|
Gush.configure do |config|
|
15
|
-
config.gushfile
|
16
|
-
config.concurrency
|
17
|
-
config.redis_url
|
18
|
-
config.namespace
|
19
|
-
config.ttl
|
17
|
+
config.gushfile = options.fetch("gushfile", config.gushfile)
|
18
|
+
config.concurrency = options.fetch("concurrency", config.concurrency)
|
19
|
+
config.redis_url = options.fetch("redis", config.redis_url)
|
20
|
+
config.namespace = options.fetch("namespace", config.namespace)
|
21
|
+
config.ttl = options.fetch("ttl", config.ttl)
|
22
|
+
config.locking_duration = options.fetch("locking_duration", config.locking_duration)
|
23
|
+
config.polling_interval = options.fetch("polling_interval", config.polling_interval)
|
20
24
|
end
|
21
25
|
load_gushfile
|
22
26
|
end
|
23
27
|
|
24
|
-
desc "create
|
28
|
+
desc "create WORKFLOW_CLASS", "Registers new workflow"
|
25
29
|
def create(name)
|
26
30
|
workflow = client.create_workflow(name)
|
27
31
|
puts "Workflow created with id: #{workflow.id}"
|
28
32
|
puts "Start it with command: gush start #{workflow.id}"
|
29
33
|
end
|
30
34
|
|
31
|
-
desc "start [
|
35
|
+
desc "start WORKFLOW_ID [ARG ...]", "Starts Workflow with given ID"
|
32
36
|
def start(*args)
|
33
37
|
id = args.shift
|
34
38
|
workflow = client.find_workflow(id)
|
35
39
|
client.start_workflow(workflow, args)
|
36
40
|
end
|
37
41
|
|
38
|
-
desc "create_and_start [
|
42
|
+
desc "create_and_start WORKFLOW_CLASS [ARG ...]", "Create and instantly start the new workflow"
|
39
43
|
def create_and_start(name, *args)
|
40
44
|
workflow = client.create_workflow(name)
|
41
45
|
client.start_workflow(workflow.id, args)
|
42
46
|
puts "Created and started workflow with id: #{workflow.id}"
|
43
47
|
end
|
44
48
|
|
45
|
-
desc "stop
|
49
|
+
desc "stop WORKFLOW_ID", "Stops Workflow with given ID"
|
46
50
|
def stop(*args)
|
47
51
|
id = args.shift
|
48
52
|
client.stop_workflow(id)
|
49
53
|
end
|
50
54
|
|
51
|
-
desc "show
|
55
|
+
desc "show WORKFLOW_ID", "Shows details about workflow with given ID"
|
52
56
|
option :skip_overview, type: :boolean
|
53
57
|
option :skip_jobs, type: :boolean
|
54
58
|
option :jobs, default: :all
|
@@ -60,7 +64,7 @@ module Gush
|
|
60
64
|
display_jobs_list_for(workflow, options[:jobs]) unless options[:skip_jobs]
|
61
65
|
end
|
62
66
|
|
63
|
-
desc "rm
|
67
|
+
desc "rm WORKFLOW_ID", "Delete workflow with given ID"
|
64
68
|
def rm(workflow_id)
|
65
69
|
workflow = client.find_workflow(workflow_id)
|
66
70
|
client.destroy_workflow(workflow)
|
@@ -81,13 +85,39 @@ module Gush
|
|
81
85
|
puts Terminal::Table.new(headings: headers, rows: rows)
|
82
86
|
end
|
83
87
|
|
84
|
-
desc "viz
|
85
|
-
|
88
|
+
desc "viz {WORKFLOW_CLASS|WORKFLOW_ID}", "Displays graph, visualising job dependencies"
|
89
|
+
option :filename, type: :string, default: nil
|
90
|
+
option :open, type: :boolean, default: nil
|
91
|
+
def viz(class_or_id)
|
86
92
|
client
|
87
|
-
|
88
|
-
|
93
|
+
|
94
|
+
begin
|
95
|
+
workflow = client.find_workflow(class_or_id)
|
96
|
+
rescue WorkflowNotFound
|
97
|
+
workflow = nil
|
98
|
+
end
|
99
|
+
|
100
|
+
unless workflow
|
101
|
+
begin
|
102
|
+
workflow = class_or_id.constantize.new
|
103
|
+
rescue NameError => e
|
104
|
+
STDERR.puts Paint["'#{class_or_id}' is not a valid workflow class or id", :red]
|
105
|
+
exit 1
|
106
|
+
end
|
107
|
+
end
|
108
|
+
|
109
|
+
opts = {}
|
110
|
+
|
111
|
+
if options[:filename]
|
112
|
+
opts[:filename], opts[:path] = File.split(options[:filename])
|
113
|
+
end
|
114
|
+
|
115
|
+
graph = Graph.new(workflow, **opts)
|
89
116
|
graph.viz
|
90
|
-
|
117
|
+
|
118
|
+
if (options[:open].nil? && !options[:filename]) || options[:open]
|
119
|
+
Launchy.open Pathname.new(graph.path).realpath.to_s
|
120
|
+
end
|
91
121
|
end
|
92
122
|
|
93
123
|
private
|
@@ -118,13 +148,14 @@ module Gush
|
|
118
148
|
|
119
149
|
def load_gushfile
|
120
150
|
file = client.configuration.gushfile
|
121
|
-
|
122
|
-
|
151
|
+
|
152
|
+
unless gushfile.exist?
|
153
|
+
raise Thor::Error, Paint["#{file} not found, please add it to your project", :red]
|
123
154
|
end
|
124
155
|
|
125
156
|
load file.to_s
|
126
157
|
rescue LoadError
|
127
|
-
raise Thor::Error, "failed to require #{file}"
|
158
|
+
raise Thor::Error, Paint["failed to require #{file}", :red]
|
128
159
|
end
|
129
160
|
end
|
130
161
|
end
|
data/lib/gush/client.rb
CHANGED
@@ -72,7 +72,7 @@ module Gush
|
|
72
72
|
id = nil
|
73
73
|
loop do
|
74
74
|
id = SecureRandom.uuid
|
75
|
-
available = !redis.exists("gush.workflow.#{id}")
|
75
|
+
available = !redis.exists?("gush.workflow.#{id}")
|
76
76
|
|
77
77
|
break if available
|
78
78
|
end
|
@@ -156,8 +156,13 @@ module Gush
|
|
156
156
|
job.enqueue!
|
157
157
|
persist_job(workflow_id, job)
|
158
158
|
queue = job.queue || configuration.namespace
|
159
|
-
|
160
|
-
|
159
|
+
wait = job.wait
|
160
|
+
|
161
|
+
if wait.present?
|
162
|
+
Gush::Worker.set(queue: queue, wait: wait).perform_later(*[workflow_id, job.name])
|
163
|
+
else
|
164
|
+
Gush::Worker.set(queue: queue).perform_later(*[workflow_id, job.name])
|
165
|
+
end
|
161
166
|
end
|
162
167
|
|
163
168
|
private
|
data/lib/gush/configuration.rb
CHANGED
@@ -1,17 +1,19 @@
|
|
1
1
|
module Gush
|
2
2
|
class Configuration
|
3
|
-
attr_accessor :concurrency, :namespace, :redis_url, :ttl
|
3
|
+
attr_accessor :concurrency, :namespace, :redis_url, :ttl, :locking_duration, :polling_interval
|
4
4
|
|
5
5
|
def self.from_json(json)
|
6
6
|
new(Gush::JSON.decode(json, symbolize_keys: true))
|
7
7
|
end
|
8
8
|
|
9
9
|
def initialize(hash = {})
|
10
|
-
self.concurrency
|
11
|
-
self.namespace
|
12
|
-
self.redis_url
|
13
|
-
self.gushfile
|
14
|
-
self.ttl
|
10
|
+
self.concurrency = hash.fetch(:concurrency, 5)
|
11
|
+
self.namespace = hash.fetch(:namespace, 'gush')
|
12
|
+
self.redis_url = hash.fetch(:redis_url, 'redis://localhost:6379')
|
13
|
+
self.gushfile = hash.fetch(:gushfile, 'Gushfile')
|
14
|
+
self.ttl = hash.fetch(:ttl, -1)
|
15
|
+
self.locking_duration = hash.fetch(:locking_duration, 2) # how long you want to wait for the lock to be released, in seconds
|
16
|
+
self.polling_interval = hash.fetch(:polling_internal, 0.3) # how long the polling interval should be, in seconds
|
15
17
|
end
|
16
18
|
|
17
19
|
def gushfile=(path)
|
@@ -24,10 +26,12 @@ module Gush
|
|
24
26
|
|
25
27
|
def to_hash
|
26
28
|
{
|
27
|
-
concurrency:
|
28
|
-
namespace:
|
29
|
-
redis_url:
|
30
|
-
ttl:
|
29
|
+
concurrency: concurrency,
|
30
|
+
namespace: namespace,
|
31
|
+
redis_url: redis_url,
|
32
|
+
ttl: ttl,
|
33
|
+
locking_duration: locking_duration,
|
34
|
+
polling_interval: polling_interval
|
31
35
|
}
|
32
36
|
end
|
33
37
|
|
data/lib/gush/graph.rb
CHANGED
@@ -1,6 +1,10 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'tmpdir'
|
4
|
+
|
1
5
|
module Gush
|
2
6
|
class Graph
|
3
|
-
attr_reader :workflow, :filename, :path, :
|
7
|
+
attr_reader :workflow, :filename, :path, :start_node, :end_node
|
4
8
|
|
5
9
|
def initialize(workflow, options = {})
|
6
10
|
@workflow = workflow
|
@@ -9,19 +13,26 @@ module Gush
|
|
9
13
|
end
|
10
14
|
|
11
15
|
def viz
|
12
|
-
|
13
|
-
|
14
|
-
|
16
|
+
@graph = Graphviz::Graph.new(**graph_options)
|
17
|
+
@start_node = add_node('start', shape: 'diamond', fillcolor: '#CFF09E')
|
18
|
+
@end_node = add_node('end', shape: 'diamond', fillcolor: '#F56991')
|
15
19
|
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
end
|
20
|
+
# First, create nodes for all jobs
|
21
|
+
@job_name_to_node_map = {}
|
22
|
+
workflow.jobs.each do |job|
|
23
|
+
add_job_node(job)
|
24
|
+
end
|
22
25
|
|
23
|
-
|
26
|
+
# Next, link up the jobs with edges
|
27
|
+
workflow.jobs.each do |job|
|
28
|
+
link_job_edges(job)
|
24
29
|
end
|
30
|
+
|
31
|
+
format = 'png'
|
32
|
+
file_format = path.split('.')[-1]
|
33
|
+
format = file_format if file_format.length == 3
|
34
|
+
|
35
|
+
Graphviz::output(@graph, path: path, format: format)
|
25
36
|
end
|
26
37
|
|
27
38
|
def path
|
@@ -29,43 +40,43 @@ module Gush
|
|
29
40
|
end
|
30
41
|
|
31
42
|
private
|
32
|
-
|
33
|
-
|
34
|
-
graph.
|
43
|
+
|
44
|
+
def add_node(name, **specific_options)
|
45
|
+
@graph.add_node(name, **node_options.merge(specific_options))
|
46
|
+
end
|
47
|
+
|
48
|
+
def add_job_node(job)
|
49
|
+
@job_name_to_node_map[job.name] = add_node(job.name, label: node_label_for_job(job))
|
50
|
+
end
|
51
|
+
|
52
|
+
def link_job_edges(job)
|
53
|
+
job_node = @job_name_to_node_map[job.name]
|
35
54
|
|
36
55
|
if job.incoming.empty?
|
37
|
-
|
56
|
+
@start_node.connect(job_node, **edge_options)
|
38
57
|
end
|
39
58
|
|
40
59
|
if job.outgoing.empty?
|
41
|
-
|
60
|
+
job_node.connect(@end_node, **edge_options)
|
42
61
|
else
|
43
62
|
job.outgoing.each do |id|
|
44
63
|
outgoing_job = workflow.find_job(id)
|
45
|
-
|
64
|
+
job_node.connect(@job_name_to_node_map[outgoing_job.name], **edge_options)
|
46
65
|
end
|
47
66
|
end
|
48
67
|
end
|
49
68
|
|
50
|
-
def
|
51
|
-
|
52
|
-
graph.node[key] = value
|
53
|
-
end
|
54
|
-
end
|
55
|
-
|
56
|
-
def set_edge_options!(graph)
|
57
|
-
edge_options.each do |key, value|
|
58
|
-
graph.edge[key] = value
|
59
|
-
end
|
69
|
+
def node_label_for_job(job)
|
70
|
+
job.class.to_s
|
60
71
|
end
|
61
72
|
|
62
73
|
def graph_options
|
63
74
|
{
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
75
|
+
dpi: 200,
|
76
|
+
compound: true,
|
77
|
+
rankdir: "LR",
|
78
|
+
center: true,
|
79
|
+
format: 'png'
|
69
80
|
}
|
70
81
|
end
|
71
82
|
|
data/lib/gush/job.rb
CHANGED
@@ -1,7 +1,8 @@
|
|
1
1
|
module Gush
|
2
2
|
class Job
|
3
3
|
attr_accessor :workflow_id, :incoming, :outgoing, :params,
|
4
|
-
:finished_at, :failed_at, :started_at, :enqueued_at, :payloads,
|
4
|
+
:finished_at, :failed_at, :started_at, :enqueued_at, :payloads,
|
5
|
+
:klass, :queue, :wait
|
5
6
|
attr_reader :id, :klass, :output_payload, :params
|
6
7
|
|
7
8
|
def initialize(opts = {})
|
@@ -126,6 +127,7 @@ module Gush
|
|
126
127
|
@output_payload = opts[:output_payload]
|
127
128
|
@workflow_id = opts[:workflow_id]
|
128
129
|
@queue = opts[:queue]
|
130
|
+
@wait = opts[:wait]
|
129
131
|
end
|
130
132
|
end
|
131
133
|
end
|
data/lib/gush/version.rb
ADDED
data/lib/gush/worker.rb
CHANGED
@@ -30,12 +30,16 @@ module Gush
|
|
30
30
|
|
31
31
|
private
|
32
32
|
|
33
|
-
attr_reader :client, :workflow_id, :job
|
33
|
+
attr_reader :client, :workflow_id, :job, :configuration
|
34
34
|
|
35
35
|
def client
|
36
36
|
@client ||= Gush::Client.new(Gush.configuration)
|
37
37
|
end
|
38
38
|
|
39
|
+
def configuration
|
40
|
+
@configuration ||= client.configuration
|
41
|
+
end
|
42
|
+
|
39
43
|
def setup_job(workflow_id, job_id)
|
40
44
|
@workflow_id = workflow_id
|
41
45
|
@job ||= client.find_job(workflow_id, job_id)
|
@@ -73,7 +77,11 @@ module Gush
|
|
73
77
|
|
74
78
|
def enqueue_outgoing_jobs
|
75
79
|
job.outgoing.each do |job_name|
|
76
|
-
RedisMutex.with_lock(
|
80
|
+
RedisMutex.with_lock(
|
81
|
+
"gush_enqueue_outgoing_jobs_#{workflow_id}-#{job_name}",
|
82
|
+
sleep: configuration.polling_interval,
|
83
|
+
block: configuration.locking_duration
|
84
|
+
) do
|
77
85
|
out = client.find_job(workflow_id, job_name)
|
78
86
|
|
79
87
|
if out.ready_to_start?
|
data/lib/gush/workflow.rb
CHANGED
@@ -5,9 +5,10 @@ describe "Workflows" do
|
|
5
5
|
context "when all jobs finish successfuly" do
|
6
6
|
it "marks workflow as completed" do
|
7
7
|
flow = TestWorkflow.create
|
8
|
-
|
9
|
-
|
10
|
-
|
8
|
+
|
9
|
+
ActiveJob::Base.queue_adapter.perform_enqueued_jobs = true
|
10
|
+
flow.start!
|
11
|
+
ActiveJob::Base.queue_adapter.perform_enqueued_jobs = false
|
11
12
|
|
12
13
|
flow = flow.reload
|
13
14
|
expect(flow).to be_finished
|
@@ -152,17 +153,15 @@ describe "Workflows" do
|
|
152
153
|
flow = PayloadWorkflow.create
|
153
154
|
flow.start!
|
154
155
|
|
155
|
-
perform_one
|
156
|
-
expect(flow.reload.find_job(flow.jobs[0].name).output_payload).to eq('first')
|
156
|
+
3.times { perform_one }
|
157
157
|
|
158
|
-
|
159
|
-
expect(
|
158
|
+
outputs = flow.reload.jobs.select { |j| j.klass == 'RepetitiveJob' }.map { |j| j.output_payload }
|
159
|
+
expect(outputs).to match_array(['first', 'second', 'third'])
|
160
160
|
|
161
161
|
perform_one
|
162
|
-
expect(flow.reload.find_job(flow.jobs[2].name).output_payload).to eq('third')
|
163
162
|
|
164
|
-
|
165
|
-
expect(
|
163
|
+
summary_job = flow.reload.jobs.find { |j| j.klass == 'SummaryJob' }
|
164
|
+
expect(summary_job.output_payload).to eq(%w(first second third))
|
166
165
|
end
|
167
166
|
|
168
167
|
it "does not execute `configure` on each job for huge workflows" do
|
data/spec/gush/client_spec.rb
CHANGED
@@ -37,6 +37,18 @@ describe Gush::Client do
|
|
37
37
|
end
|
38
38
|
|
39
39
|
describe "#start_workflow" do
|
40
|
+
context "when there is wait parameter configured" do
|
41
|
+
let(:freeze_time) { Time.utc(2023, 01, 21, 14, 36, 0) }
|
42
|
+
|
43
|
+
it "schedules job execution" do
|
44
|
+
travel_to freeze_time do
|
45
|
+
workflow = WaitableTestWorkflow.create
|
46
|
+
client.start_workflow(workflow)
|
47
|
+
expect(Gush::Worker).to have_a_job_enqueued_at(workflow.id, job_with_id("Prepare"), 5.minutes)
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
40
52
|
it "enqueues next jobs from the workflow" do
|
41
53
|
workflow = TestWorkflow.create
|
42
54
|
expect {
|
@@ -113,7 +125,7 @@ describe Gush::Client do
|
|
113
125
|
describe "#persist_job" do
|
114
126
|
it "persists JSON dump of the job in Redis" do
|
115
127
|
|
116
|
-
job = BobJob.new(name: 'bob')
|
128
|
+
job = BobJob.new(name: 'bob', id: 'abcd123')
|
117
129
|
|
118
130
|
client.persist_job('deadbeef', job)
|
119
131
|
expect(redis.keys("gush.jobs.deadbeef.*").length).to eq(1)
|
@@ -8,6 +8,8 @@ describe Gush::Configuration do
|
|
8
8
|
expect(subject.concurrency).to eq(5)
|
9
9
|
expect(subject.namespace).to eq('gush')
|
10
10
|
expect(subject.gushfile).to eq(GUSHFILE.realpath)
|
11
|
+
expect(subject.locking_duration).to eq(2)
|
12
|
+
expect(subject.polling_interval).to eq(0.3)
|
11
13
|
end
|
12
14
|
|
13
15
|
describe "#configure" do
|
@@ -15,10 +17,14 @@ describe Gush::Configuration do
|
|
15
17
|
Gush.configure do |config|
|
16
18
|
config.redis_url = "redis://localhost"
|
17
19
|
config.concurrency = 25
|
20
|
+
config.locking_duration = 5
|
21
|
+
config.polling_interval = 0.5
|
18
22
|
end
|
19
23
|
|
20
24
|
expect(Gush.configuration.redis_url).to eq("redis://localhost")
|
21
25
|
expect(Gush.configuration.concurrency).to eq(25)
|
26
|
+
expect(Gush.configuration.locking_duration).to eq(5)
|
27
|
+
expect(Gush.configuration.polling_interval).to eq(0.5)
|
22
28
|
end
|
23
29
|
end
|
24
30
|
end
|
data/spec/gush/graph_spec.rb
CHANGED
@@ -10,26 +10,43 @@ describe Gush::Graph do
|
|
10
10
|
edge = double("edge", :[]= => true)
|
11
11
|
graph = double("graph", node: node, edge: edge)
|
12
12
|
path = Pathname.new(Dir.tmpdir).join(filename)
|
13
|
-
|
14
|
-
expect(
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
expect(graph).to receive(:
|
25
|
-
expect(graph).to receive(:
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
expect(graph).to receive(:
|
30
|
-
expect(graph).to receive(:
|
31
|
-
|
32
|
-
expect(
|
13
|
+
|
14
|
+
expect(Graphviz::Graph).to receive(:new).and_return(graph)
|
15
|
+
|
16
|
+
node_start = double('start')
|
17
|
+
node_end = double('end')
|
18
|
+
node_prepare = double('Prepare')
|
19
|
+
node_fetch_first_job = double('FetchFirstJob')
|
20
|
+
node_fetch_second_job = double('FetchSecondJob')
|
21
|
+
node_normalize_job = double('NormalizeJob')
|
22
|
+
node_persist_first_job = double('PersistFirstJob')
|
23
|
+
|
24
|
+
expect(graph).to receive(:add_node).with('start', {shape: 'diamond', fillcolor: '#CFF09E', color: "#555555", style: 'filled'}).and_return(node_start)
|
25
|
+
expect(graph).to receive(:add_node).with('end', {shape: 'diamond', fillcolor: '#F56991', color: "#555555", style: 'filled'}).and_return(node_end)
|
26
|
+
|
27
|
+
standard_options = {:color=>"#555555", :fillcolor=>"white", :label=>"Prepare", :shape=>"ellipse", :style=>"filled"}
|
28
|
+
|
29
|
+
expect(graph).to receive(:add_node).with(/Prepare/, standard_options.merge(label: "Prepare")).and_return(node_prepare)
|
30
|
+
expect(graph).to receive(:add_node).with(/FetchFirstJob/, standard_options.merge(label: "FetchFirstJob")).and_return(node_fetch_first_job)
|
31
|
+
expect(graph).to receive(:add_node).with(/FetchSecondJob/, standard_options.merge(label: "FetchSecondJob")).and_return(node_fetch_second_job)
|
32
|
+
expect(graph).to receive(:add_node).with(/NormalizeJob/, standard_options.merge(label: "NormalizeJob")).and_return(node_normalize_job)
|
33
|
+
expect(graph).to receive(:add_node).with(/PersistFirstJob/, standard_options.merge(label: "PersistFirstJob")).and_return(node_persist_first_job)
|
34
|
+
|
35
|
+
edge_options = {
|
36
|
+
dir: "forward",
|
37
|
+
penwidth: 1,
|
38
|
+
color: "#555555"
|
39
|
+
}
|
40
|
+
|
41
|
+
expect(node_start).to receive(:connect).with(node_prepare, **edge_options)
|
42
|
+
expect(node_prepare).to receive(:connect).with(node_fetch_first_job, **edge_options)
|
43
|
+
expect(node_prepare).to receive(:connect).with(node_fetch_second_job, **edge_options)
|
44
|
+
expect(node_fetch_first_job).to receive(:connect).with(node_persist_first_job, **edge_options)
|
45
|
+
expect(node_fetch_second_job).to receive(:connect).with(node_normalize_job, **edge_options)
|
46
|
+
expect(node_persist_first_job).to receive(:connect).with(node_normalize_job, **edge_options)
|
47
|
+
expect(node_normalize_job).to receive(:connect).with(node_end, **edge_options)
|
48
|
+
|
49
|
+
expect(graph).to receive(:dump_graph).and_return(nil)
|
33
50
|
|
34
51
|
subject.viz
|
35
52
|
end
|
data/spec/gush/worker_spec.rb
CHANGED
@@ -4,6 +4,8 @@ describe Gush::Worker do
|
|
4
4
|
subject { described_class.new }
|
5
5
|
|
6
6
|
let!(:workflow) { TestWorkflow.create }
|
7
|
+
let(:locking_duration) { 5 }
|
8
|
+
let(:polling_interval) { 0.5 }
|
7
9
|
let!(:job) { client.find_job(workflow.id, "Prepare") }
|
8
10
|
let(:config) { Gush.configuration.to_json }
|
9
11
|
let!(:client) { Gush::Client.new }
|
@@ -71,5 +73,11 @@ describe Gush::Worker do
|
|
71
73
|
|
72
74
|
subject.perform(workflow.id, 'OkayJob')
|
73
75
|
end
|
76
|
+
|
77
|
+
it 'calls RedisMutex.with_lock with customizable locking_duration and polling_interval' do
|
78
|
+
expect(RedisMutex).to receive(:with_lock)
|
79
|
+
.with(anything, block: 5, sleep: 0.5).twice
|
80
|
+
subject.perform(workflow.id, 'Prepare')
|
81
|
+
end
|
74
82
|
end
|
75
83
|
end
|
data/spec/gush/workflow_spec.rb
CHANGED
@@ -121,6 +121,13 @@ describe Gush::Workflow do
|
|
121
121
|
expect(flow.jobs.first.params).to eq ({ something: 1 })
|
122
122
|
end
|
123
123
|
|
124
|
+
it "allows passing wait param to the job" do
|
125
|
+
flow = Gush::Workflow.new
|
126
|
+
flow.run(Gush::Job, wait: 5.seconds)
|
127
|
+
flow.save
|
128
|
+
expect(flow.jobs.first.wait).to eq (5.seconds)
|
129
|
+
end
|
130
|
+
|
124
131
|
context "when graph is empty" do
|
125
132
|
it "adds new job with the given class as a node" do
|
126
133
|
flow = Gush::Workflow.new
|
data/spec/spec_helper.rb
CHANGED
@@ -1,5 +1,6 @@
|
|
1
|
+
require 'active_support'
|
2
|
+
require 'active_support/testing/time_helpers'
|
1
3
|
require 'gush'
|
2
|
-
require 'fakeredis'
|
3
4
|
require 'json'
|
4
5
|
require 'pry'
|
5
6
|
|
@@ -35,12 +36,13 @@ class ParameterTestWorkflow < Gush::Workflow
|
|
35
36
|
end
|
36
37
|
end
|
37
38
|
|
38
|
-
class
|
39
|
-
def
|
39
|
+
class WaitableTestWorkflow < Gush::Workflow
|
40
|
+
def configure
|
41
|
+
run Prepare, wait: 5.minutes
|
40
42
|
end
|
41
43
|
end
|
42
44
|
|
43
|
-
REDIS_URL = "redis://localhost:6379/12"
|
45
|
+
REDIS_URL = ENV["REDIS_URL"] || "redis://localhost:6379/12"
|
44
46
|
|
45
47
|
module GushHelpers
|
46
48
|
def redis
|
@@ -91,7 +93,22 @@ RSpec::Matchers.define :have_no_jobs do |flow, jobs|
|
|
91
93
|
end
|
92
94
|
end
|
93
95
|
|
96
|
+
RSpec::Matchers.define :have_a_job_enqueued_at do |flow, job, at|
|
97
|
+
expected_execution_timestamp = (Time.current.utc + at).to_i
|
98
|
+
|
99
|
+
match do |actual|
|
100
|
+
expected = hash_including(args: include(flow, job), at: expected_execution_timestamp)
|
101
|
+
|
102
|
+
expect(ActiveJob::Base.queue_adapter.enqueued_jobs).to match_array(expected)
|
103
|
+
end
|
104
|
+
|
105
|
+
failure_message do |actual|
|
106
|
+
"expected to have enqueued job #{job} to be executed at #{Time.current.utc + at}, but instead has: #{Time.at(enqueued_jobs.first[:at]).to_datetime.utc}"
|
107
|
+
end
|
108
|
+
end
|
109
|
+
|
94
110
|
RSpec.configure do |config|
|
111
|
+
config.include ActiveSupport::Testing::TimeHelpers
|
95
112
|
config.include ActiveJob::TestHelper
|
96
113
|
config.include GushHelpers
|
97
114
|
|
@@ -104,12 +121,13 @@ RSpec.configure do |config|
|
|
104
121
|
clear_performed_jobs
|
105
122
|
|
106
123
|
Gush.configure do |config|
|
107
|
-
config.redis_url
|
108
|
-
config.gushfile
|
124
|
+
config.redis_url = REDIS_URL
|
125
|
+
config.gushfile = GUSHFILE
|
126
|
+
config.locking_duration = defined?(locking_duration) ? locking_duration : 2
|
127
|
+
config.polling_interval = defined?(polling_interval) ? polling_interval : 0.3
|
109
128
|
end
|
110
129
|
end
|
111
130
|
|
112
|
-
|
113
131
|
config.after(:each) do
|
114
132
|
clear_enqueued_jobs
|
115
133
|
clear_performed_jobs
|
metadata
CHANGED
@@ -1,14 +1,15 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: gush
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 3.0.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Piotrek Okoński
|
8
|
-
|
8
|
+
- Michał Krzyżanowski
|
9
|
+
autorequire:
|
9
10
|
bindir: bin
|
10
11
|
cert_chain: []
|
11
|
-
date:
|
12
|
+
date: 2024-02-29 00:00:00.000000000 Z
|
12
13
|
dependencies:
|
13
14
|
- !ruby/object:Gem::Dependency
|
14
15
|
name: activejob
|
@@ -16,20 +17,20 @@ dependencies:
|
|
16
17
|
requirements:
|
17
18
|
- - ">="
|
18
19
|
- !ruby/object:Gem::Version
|
19
|
-
version:
|
20
|
+
version: 6.1.0
|
20
21
|
- - "<"
|
21
22
|
- !ruby/object:Gem::Version
|
22
|
-
version: '7.
|
23
|
+
version: '7.2'
|
23
24
|
type: :runtime
|
24
25
|
prerelease: false
|
25
26
|
version_requirements: !ruby/object:Gem::Requirement
|
26
27
|
requirements:
|
27
28
|
- - ">="
|
28
29
|
- !ruby/object:Gem::Version
|
29
|
-
version:
|
30
|
+
version: 6.1.0
|
30
31
|
- - "<"
|
31
32
|
- !ruby/object:Gem::Version
|
32
|
-
version: '7.
|
33
|
+
version: '7.2'
|
33
34
|
- !ruby/object:Gem::Dependency
|
34
35
|
name: concurrent-ruby
|
35
36
|
requirement: !ruby/object:Gem::Requirement
|
@@ -67,7 +68,7 @@ dependencies:
|
|
67
68
|
version: '3.2'
|
68
69
|
- - "<"
|
69
70
|
- !ruby/object:Gem::Version
|
70
|
-
version: '
|
71
|
+
version: '6'
|
71
72
|
type: :runtime
|
72
73
|
prerelease: false
|
73
74
|
version_requirements: !ruby/object:Gem::Requirement
|
@@ -77,7 +78,7 @@ dependencies:
|
|
77
78
|
version: '3.2'
|
78
79
|
- - "<"
|
79
80
|
- !ruby/object:Gem::Version
|
80
|
-
version: '
|
81
|
+
version: '6'
|
81
82
|
- !ruby/object:Gem::Dependency
|
82
83
|
name: redis-mutex
|
83
84
|
requirement: !ruby/object:Gem::Requirement
|
@@ -107,7 +108,7 @@ dependencies:
|
|
107
108
|
- !ruby/object:Gem::Version
|
108
109
|
version: '0.6'
|
109
110
|
- !ruby/object:Gem::Dependency
|
110
|
-
name:
|
111
|
+
name: graphviz
|
111
112
|
requirement: !ruby/object:Gem::Requirement
|
112
113
|
requirements:
|
113
114
|
- - "~>"
|
@@ -124,44 +125,56 @@ dependencies:
|
|
124
125
|
name: terminal-table
|
125
126
|
requirement: !ruby/object:Gem::Requirement
|
126
127
|
requirements:
|
127
|
-
- - "
|
128
|
+
- - ">="
|
128
129
|
- !ruby/object:Gem::Version
|
129
130
|
version: '1.4'
|
131
|
+
- - "<"
|
132
|
+
- !ruby/object:Gem::Version
|
133
|
+
version: '3.1'
|
130
134
|
type: :runtime
|
131
135
|
prerelease: false
|
132
136
|
version_requirements: !ruby/object:Gem::Requirement
|
133
137
|
requirements:
|
134
|
-
- - "
|
138
|
+
- - ">="
|
135
139
|
- !ruby/object:Gem::Version
|
136
140
|
version: '1.4'
|
141
|
+
- - "<"
|
142
|
+
- !ruby/object:Gem::Version
|
143
|
+
version: '3.1'
|
137
144
|
- !ruby/object:Gem::Dependency
|
138
|
-
name:
|
145
|
+
name: paint
|
139
146
|
requirement: !ruby/object:Gem::Requirement
|
140
147
|
requirements:
|
141
148
|
- - "~>"
|
142
149
|
- !ruby/object:Gem::Version
|
143
|
-
version: '
|
150
|
+
version: '2.2'
|
144
151
|
type: :runtime
|
145
152
|
prerelease: false
|
146
153
|
version_requirements: !ruby/object:Gem::Requirement
|
147
154
|
requirements:
|
148
155
|
- - "~>"
|
149
156
|
- !ruby/object:Gem::Version
|
150
|
-
version: '
|
157
|
+
version: '2.2'
|
151
158
|
- !ruby/object:Gem::Dependency
|
152
159
|
name: thor
|
153
160
|
requirement: !ruby/object:Gem::Requirement
|
154
161
|
requirements:
|
155
|
-
- - "
|
162
|
+
- - ">="
|
156
163
|
- !ruby/object:Gem::Version
|
157
164
|
version: '0.19'
|
165
|
+
- - "<"
|
166
|
+
- !ruby/object:Gem::Version
|
167
|
+
version: '1.3'
|
158
168
|
type: :runtime
|
159
169
|
prerelease: false
|
160
170
|
version_requirements: !ruby/object:Gem::Requirement
|
161
171
|
requirements:
|
162
|
-
- - "
|
172
|
+
- - ">="
|
163
173
|
- !ruby/object:Gem::Version
|
164
174
|
version: '0.19'
|
175
|
+
- - "<"
|
176
|
+
- !ruby/object:Gem::Version
|
177
|
+
version: '1.3'
|
165
178
|
- !ruby/object:Gem::Dependency
|
166
179
|
name: launchy
|
167
180
|
requirement: !ruby/object:Gem::Requirement
|
@@ -196,14 +209,14 @@ dependencies:
|
|
196
209
|
requirements:
|
197
210
|
- - "~>"
|
198
211
|
- !ruby/object:Gem::Version
|
199
|
-
version: '
|
212
|
+
version: '12'
|
200
213
|
type: :development
|
201
214
|
prerelease: false
|
202
215
|
version_requirements: !ruby/object:Gem::Requirement
|
203
216
|
requirements:
|
204
217
|
- - "~>"
|
205
218
|
- !ruby/object:Gem::Version
|
206
|
-
version: '
|
219
|
+
version: '12'
|
207
220
|
- !ruby/object:Gem::Dependency
|
208
221
|
name: rspec
|
209
222
|
requirement: !ruby/object:Gem::Requirement
|
@@ -232,32 +245,19 @@ dependencies:
|
|
232
245
|
- - "~>"
|
233
246
|
- !ruby/object:Gem::Version
|
234
247
|
version: '0.10'
|
235
|
-
- !ruby/object:Gem::Dependency
|
236
|
-
name: fakeredis
|
237
|
-
requirement: !ruby/object:Gem::Requirement
|
238
|
-
requirements:
|
239
|
-
- - "~>"
|
240
|
-
- !ruby/object:Gem::Version
|
241
|
-
version: '0.5'
|
242
|
-
type: :development
|
243
|
-
prerelease: false
|
244
|
-
version_requirements: !ruby/object:Gem::Requirement
|
245
|
-
requirements:
|
246
|
-
- - "~>"
|
247
|
-
- !ruby/object:Gem::Version
|
248
|
-
version: '0.5'
|
249
248
|
description: Gush is a parallel workflow runner using Redis as storage and ActiveJob
|
250
249
|
for executing jobs.
|
251
250
|
email:
|
252
251
|
- piotrek@okonski.org
|
252
|
+
- michal.krzyzanowski+github@gmail.com
|
253
253
|
executables:
|
254
254
|
- gush
|
255
255
|
extensions: []
|
256
256
|
extra_rdoc_files: []
|
257
257
|
files:
|
258
|
+
- ".github/workflows/ruby.yml"
|
258
259
|
- ".gitignore"
|
259
260
|
- ".rspec"
|
260
|
-
- ".travis.yml"
|
261
261
|
- CHANGELOG.md
|
262
262
|
- Gemfile
|
263
263
|
- LICENSE.txt
|
@@ -274,6 +274,7 @@ files:
|
|
274
274
|
- lib/gush/graph.rb
|
275
275
|
- lib/gush/job.rb
|
276
276
|
- lib/gush/json.rb
|
277
|
+
- lib/gush/version.rb
|
277
278
|
- lib/gush/worker.rb
|
278
279
|
- lib/gush/workflow.rb
|
279
280
|
- spec/Gushfile
|
@@ -292,7 +293,7 @@ homepage: https://github.com/chaps-io/gush
|
|
292
293
|
licenses:
|
293
294
|
- MIT
|
294
295
|
metadata: {}
|
295
|
-
post_install_message:
|
296
|
+
post_install_message:
|
296
297
|
rdoc_options: []
|
297
298
|
require_paths:
|
298
299
|
- lib
|
@@ -300,15 +301,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
300
301
|
requirements:
|
301
302
|
- - ">="
|
302
303
|
- !ruby/object:Gem::Version
|
303
|
-
version:
|
304
|
+
version: 3.0.0
|
304
305
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
305
306
|
requirements:
|
306
307
|
- - ">="
|
307
308
|
- !ruby/object:Gem::Version
|
308
309
|
version: '0'
|
309
310
|
requirements: []
|
310
|
-
rubygems_version: 3.
|
311
|
-
signing_key:
|
311
|
+
rubygems_version: 3.4.22
|
312
|
+
signing_key:
|
312
313
|
specification_version: 4
|
313
314
|
summary: Fast and distributed workflow runner based on ActiveJob and Redis
|
314
315
|
test_files:
|
data/.travis.yml
DELETED