gush 1.0.0 → 1.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/CHANGELOG.md +12 -0
- data/README.md +31 -0
- data/gush.gemspec +2 -2
- data/lib/gush/cli.rb +1 -0
- data/lib/gush/client.rb +21 -6
- data/lib/gush/configuration.rb +4 -2
- data/lib/gush/job.rb +2 -2
- data/lib/gush/workflow.rb +4 -1
- data/spec/gush/client_spec.rb +10 -0
- data/spec/gush/workflow_spec.rb +2 -31
- metadata +11 -5
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: ef37f8a54b14ecb74031350dafb2a72525d2a67434bb50dee29e37b60d97ec08
|
4
|
+
data.tar.gz: 4117a8ffd532a337f3da58eea840734c2a2dc2f7bda8706867f6b1056921ad28
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: df491287ba0d78f87373c11e30979a66035e42006e6e177b8d47372e6079772367df97429ef4c12c7f68d117c31b736965e0b060199fdc5ebfef525821b559bf
|
7
|
+
data.tar.gz: ac0cb6ddbc8705712683ccfa5098f02fab45cef70c64c9de80e22631fff297a55c76c4db5c0a5b9e38fb0466917130ad28f35aa4b065c4500b8383046c2b3c2f
|
data/CHANGELOG.md
CHANGED
@@ -5,6 +5,18 @@ All notable changes to this project will be documented in this file.
|
|
5
5
|
The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
|
6
6
|
and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
|
7
7
|
|
8
|
+
## 1.1.0 - 2018-02-05
|
9
|
+
|
10
|
+
## Added
|
11
|
+
|
12
|
+
- Added ability to specify TTL for Redis keys and manually expire whole workflows (Thanks to @dmitrypol! [See pull request](https://github.com/chaps-io/gush/pull/48))
|
13
|
+
- Loosened dependency on redis-rb library to >= 3.2 and < 5.0 (Thanks to @mofumofu3n! [See pull request](https://github.com/chaps-io/gush/pull/52))
|
14
|
+
|
15
|
+
## Fixed
|
16
|
+
|
17
|
+
- Improved performance of (de)serializing workflows by not storing job array inside workflow JSON and other smaller improvements ([See pull request](https://github.com/chaps-io/gush/pull/53))
|
18
|
+
|
19
|
+
|
8
20
|
## 1.0.0 - 2017-10-02
|
9
21
|
|
10
22
|
### Added
|
data/README.md
CHANGED
@@ -348,6 +348,37 @@ This requires that you have imagemagick installed on your computer:
|
|
348
348
|
bundle exec gush viz <NameOfTheWorkflow>
|
349
349
|
```
|
350
350
|
|
351
|
+
### Cleaning up afterwards
|
352
|
+
|
353
|
+
Running `NotifyWorkflow.create` inserts multiple keys into Redis every time it is ran. This data might be useful for analysis but at a certain point it can be purged via Redis TTL. By default gush and Redis will keep keys forever. To configure expiration you need to 2 things. Create initializer (specify config.ttl in seconds, be different per environment).
|
354
|
+
|
355
|
+
```ruby
|
356
|
+
# config/initializers/gush.rb
|
357
|
+
Gush.configure do |config|
|
358
|
+
config.redis_url = "redis://localhost:6379"
|
359
|
+
config.concurrency = 5
|
360
|
+
config.ttl = 3600*24*7
|
361
|
+
end
|
362
|
+
```
|
363
|
+
|
364
|
+
And you need to call `flow.expire!` (optionally passing custom TTL value overriding `config.ttl`). This gives you control whether to expire data for specific workflow. Best NOT to set TTL to be too short (like minutes) but about a week in length. And you can run `Client.expire_workflow` and `Client.expire_job` passing appropriate IDs and TTL (pass -1 to NOT expire) values.
|
365
|
+
|
366
|
+
### Avoid overlapping workflows
|
367
|
+
|
368
|
+
Since we do not know how long our workflow execution will take we might want to avoid starting the next scheduled workflow iteration while the current one with same class is still running. Long term this could be moved into core library, perhaps `Workflow.find_by_class(klass)`
|
369
|
+
|
370
|
+
```ruby
|
371
|
+
# config/initializers/gush.rb
|
372
|
+
GUSH_CLIENT = Gush::Client.new
|
373
|
+
# call this method before NotifyWorkflow.create
|
374
|
+
def find_by_class klass
|
375
|
+
GUSH_CLIENT.all_workflows.each do |flow|
|
376
|
+
return true if flow.to_hash[:name] == klass && flow.running?
|
377
|
+
end
|
378
|
+
return false
|
379
|
+
end
|
380
|
+
```
|
381
|
+
|
351
382
|
## Contributors
|
352
383
|
|
353
384
|
- [Mateusz Lenik](https://github.com/mlen)
|
data/gush.gemspec
CHANGED
@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
4
4
|
|
5
5
|
Gem::Specification.new do |spec|
|
6
6
|
spec.name = "gush"
|
7
|
-
spec.version = "1.
|
7
|
+
spec.version = "1.1.0"
|
8
8
|
spec.authors = ["Piotrek Okoński"]
|
9
9
|
spec.email = ["piotrek@okonski.org"]
|
10
10
|
spec.summary = "Fast and distributed workflow runner based on ActiveJob and Redis"
|
@@ -20,7 +20,7 @@ Gem::Specification.new do |spec|
|
|
20
20
|
spec.add_dependency "activejob", "~> 5.0"
|
21
21
|
spec.add_dependency "connection_pool", "~> 2.2.1"
|
22
22
|
spec.add_dependency "multi_json", "~> 1.11"
|
23
|
-
spec.add_dependency "redis", "
|
23
|
+
spec.add_dependency "redis", ">= 3.2", "< 5"
|
24
24
|
spec.add_dependency "hiredis", "~> 0.6"
|
25
25
|
spec.add_dependency "ruby-graphviz", "~> 1.2"
|
26
26
|
spec.add_dependency "terminal-table", "~> 1.4"
|
data/lib/gush/cli.rb
CHANGED
@@ -16,6 +16,7 @@ module Gush
|
|
16
16
|
config.concurrency = options.fetch("concurrency", config.concurrency)
|
17
17
|
config.redis_url = options.fetch("redis", config.redis_url)
|
18
18
|
config.namespace = options.fetch("namespace", config.namespace)
|
19
|
+
config.ttl = options.fetch("ttl", config.ttl)
|
19
20
|
end
|
20
21
|
load_gushfile
|
21
22
|
end
|
data/lib/gush/client.rb
CHANGED
@@ -73,7 +73,7 @@ module Gush
|
|
73
73
|
|
74
74
|
def all_workflows
|
75
75
|
connection_pool.with do |redis|
|
76
|
-
redis.
|
76
|
+
redis.scan_each(match: "gush.workflows.*").map do |key|
|
77
77
|
id = key.sub("gush.workflows.", "")
|
78
78
|
find_workflow(id)
|
79
79
|
end
|
@@ -86,7 +86,7 @@ module Gush
|
|
86
86
|
|
87
87
|
unless data.nil?
|
88
88
|
hash = Gush::JSON.decode(data, symbolize_keys: true)
|
89
|
-
keys = redis.
|
89
|
+
keys = redis.scan_each(match: "gush.jobs.#{id}.*")
|
90
90
|
nodes = redis.mget(*keys).map { |json| Gush::JSON.decode(json, symbolize_keys: true) }
|
91
91
|
workflow_from_hash(hash, nodes)
|
92
92
|
else
|
@@ -116,7 +116,7 @@ module Gush
|
|
116
116
|
hypen = '-' if job_name_match.nil?
|
117
117
|
|
118
118
|
keys = connection_pool.with do |redis|
|
119
|
-
redis.
|
119
|
+
redis.scan_each(match: "gush.jobs.#{workflow_id}.#{job_id}#{hypen}*").to_a
|
120
120
|
end
|
121
121
|
|
122
122
|
return nil if keys.nil?
|
@@ -144,6 +144,21 @@ module Gush
|
|
144
144
|
end
|
145
145
|
end
|
146
146
|
|
147
|
+
def expire_workflow(workflow, ttl=nil)
|
148
|
+
ttl = ttl || configuration.ttl
|
149
|
+
connection_pool.with do |redis|
|
150
|
+
redis.expire("gush.workflows.#{workflow.id}", ttl)
|
151
|
+
end
|
152
|
+
workflow.jobs.each {|job| expire_job(workflow.id, job, ttl) }
|
153
|
+
end
|
154
|
+
|
155
|
+
def expire_job(workflow_id, job, ttl=nil)
|
156
|
+
ttl = ttl || configuration.ttl
|
157
|
+
connection_pool.with do |redis|
|
158
|
+
redis.expire("gush.jobs.#{workflow_id}.#{job.name}", ttl)
|
159
|
+
end
|
160
|
+
end
|
161
|
+
|
147
162
|
def enqueue_job(workflow_id, job)
|
148
163
|
job.enqueue!
|
149
164
|
persist_job(workflow_id, job)
|
@@ -153,14 +168,14 @@ module Gush
|
|
153
168
|
|
154
169
|
private
|
155
170
|
|
156
|
-
def workflow_from_hash(hash, nodes =
|
171
|
+
def workflow_from_hash(hash, nodes = [])
|
157
172
|
flow = hash[:klass].constantize.new(*hash[:arguments])
|
158
173
|
flow.jobs = []
|
159
174
|
flow.stopped = hash.fetch(:stopped, false)
|
160
175
|
flow.id = hash[:id]
|
161
176
|
|
162
|
-
|
163
|
-
|
177
|
+
flow.jobs = nodes.map do |node|
|
178
|
+
Gush::Job.from_hash(node)
|
164
179
|
end
|
165
180
|
|
166
181
|
flow
|
data/lib/gush/configuration.rb
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
module Gush
|
2
2
|
class Configuration
|
3
|
-
attr_accessor :concurrency, :namespace, :redis_url
|
3
|
+
attr_accessor :concurrency, :namespace, :redis_url, :ttl
|
4
4
|
|
5
5
|
def self.from_json(json)
|
6
6
|
new(Gush::JSON.decode(json, symbolize_keys: true))
|
@@ -11,6 +11,7 @@ module Gush
|
|
11
11
|
self.namespace = hash.fetch(:namespace, 'gush')
|
12
12
|
self.redis_url = hash.fetch(:redis_url, 'redis://localhost:6379')
|
13
13
|
self.gushfile = hash.fetch(:gushfile, 'Gushfile')
|
14
|
+
self.ttl = hash.fetch(:ttl, -1)
|
14
15
|
end
|
15
16
|
|
16
17
|
def gushfile=(path)
|
@@ -25,7 +26,8 @@ module Gush
|
|
25
26
|
{
|
26
27
|
concurrency: concurrency,
|
27
28
|
namespace: namespace,
|
28
|
-
redis_url: redis_url
|
29
|
+
redis_url: redis_url,
|
30
|
+
ttl: ttl
|
29
31
|
}
|
30
32
|
end
|
31
33
|
|
data/lib/gush/job.rb
CHANGED
data/lib/gush/workflow.rb
CHANGED
@@ -53,6 +53,10 @@ module Gush
|
|
53
53
|
client.persist_workflow(self)
|
54
54
|
end
|
55
55
|
|
56
|
+
def expire! (ttl=nil)
|
57
|
+
client.expire_workflow(self, ttl)
|
58
|
+
end
|
59
|
+
|
56
60
|
def mark_as_persisted
|
57
61
|
@persisted = true
|
58
62
|
end
|
@@ -168,7 +172,6 @@ module Gush
|
|
168
172
|
total: jobs.count,
|
169
173
|
finished: jobs.count(&:finished?),
|
170
174
|
klass: name,
|
171
|
-
jobs: jobs.map(&:as_json),
|
172
175
|
status: status,
|
173
176
|
stopped: stopped,
|
174
177
|
started_at: started_at,
|
data/spec/gush/client_spec.rb
CHANGED
@@ -94,6 +94,16 @@ describe Gush::Client do
|
|
94
94
|
end
|
95
95
|
end
|
96
96
|
|
97
|
+
describe "#expire_workflow" do
|
98
|
+
it "sets TTL for all Redis keys related to the workflow" do
|
99
|
+
workflow = TestWorkflow.create
|
100
|
+
|
101
|
+
client.expire_workflow(workflow, -1)
|
102
|
+
|
103
|
+
# => TODO - I believe fakeredis does not handle TTL the same.
|
104
|
+
end
|
105
|
+
end
|
106
|
+
|
97
107
|
describe "#persist_job" do
|
98
108
|
it "persists JSON dump of the job in Redis" do
|
99
109
|
|
data/spec/gush/workflow_spec.rb
CHANGED
@@ -13,8 +13,7 @@ describe Gush::Workflow do
|
|
13
13
|
end
|
14
14
|
|
15
15
|
expect_any_instance_of(klass).to receive(:configure).with("arg1", "arg2")
|
16
|
-
|
17
|
-
|
16
|
+
klass.new("arg1", "arg2")
|
18
17
|
end
|
19
18
|
end
|
20
19
|
|
@@ -102,35 +101,7 @@ describe Gush::Workflow do
|
|
102
101
|
"started_at" => nil,
|
103
102
|
"finished_at" => nil,
|
104
103
|
"stopped" => false,
|
105
|
-
"arguments" => ["arg1", "arg2"]
|
106
|
-
"jobs" => [
|
107
|
-
{
|
108
|
-
"name"=>a_string_starting_with('FetchFirstJob'),
|
109
|
-
"klass"=>"FetchFirstJob",
|
110
|
-
"incoming"=>[],
|
111
|
-
"outgoing"=>[a_string_starting_with('PersistFirstJob')],
|
112
|
-
"finished_at"=>nil,
|
113
|
-
"started_at"=>nil,
|
114
|
-
"enqueued_at"=>nil,
|
115
|
-
"failed_at"=>nil,
|
116
|
-
"params" => {},
|
117
|
-
"output_payload" => nil,
|
118
|
-
"workflow_id" => an_instance_of(String)
|
119
|
-
},
|
120
|
-
{
|
121
|
-
"name"=>a_string_starting_with('PersistFirstJob'),
|
122
|
-
"klass"=>"PersistFirstJob",
|
123
|
-
"incoming"=>["FetchFirstJob"],
|
124
|
-
"outgoing"=>[],
|
125
|
-
"finished_at"=>nil,
|
126
|
-
"started_at"=>nil,
|
127
|
-
"enqueued_at"=>nil,
|
128
|
-
"failed_at"=>nil,
|
129
|
-
"params" => {},
|
130
|
-
"output_payload" => nil,
|
131
|
-
"workflow_id" => an_instance_of(String)
|
132
|
-
}
|
133
|
-
]
|
104
|
+
"arguments" => ["arg1", "arg2"]
|
134
105
|
}
|
135
106
|
expect(result).to match(expected)
|
136
107
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: gush
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Piotrek Okoński
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2018-02-05 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activejob
|
@@ -56,16 +56,22 @@ dependencies:
|
|
56
56
|
name: redis
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|
58
58
|
requirements:
|
59
|
-
- - "
|
59
|
+
- - ">="
|
60
60
|
- !ruby/object:Gem::Version
|
61
61
|
version: '3.2'
|
62
|
+
- - "<"
|
63
|
+
- !ruby/object:Gem::Version
|
64
|
+
version: '5'
|
62
65
|
type: :runtime
|
63
66
|
prerelease: false
|
64
67
|
version_requirements: !ruby/object:Gem::Requirement
|
65
68
|
requirements:
|
66
|
-
- - "
|
69
|
+
- - ">="
|
67
70
|
- !ruby/object:Gem::Version
|
68
71
|
version: '3.2'
|
72
|
+
- - "<"
|
73
|
+
- !ruby/object:Gem::Version
|
74
|
+
version: '5'
|
69
75
|
- !ruby/object:Gem::Dependency
|
70
76
|
name: hiredis
|
71
77
|
requirement: !ruby/object:Gem::Requirement
|
@@ -282,7 +288,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
282
288
|
version: '0'
|
283
289
|
requirements: []
|
284
290
|
rubyforge_project:
|
285
|
-
rubygems_version: 2.
|
291
|
+
rubygems_version: 2.7.4
|
286
292
|
signing_key:
|
287
293
|
specification_version: 4
|
288
294
|
summary: Fast and distributed workflow runner based on ActiveJob and Redis
|