gush 1.0.0 → 1.1.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
- SHA1:
3
- metadata.gz: 5ac3b43abd8eeb7cc6403a0be5064b0dd0ba6f2a
4
- data.tar.gz: 8038bf272a8562a632cd67ae9dc8adfdc59e50a6
2
+ SHA256:
3
+ metadata.gz: ef37f8a54b14ecb74031350dafb2a72525d2a67434bb50dee29e37b60d97ec08
4
+ data.tar.gz: 4117a8ffd532a337f3da58eea840734c2a2dc2f7bda8706867f6b1056921ad28
5
5
  SHA512:
6
- metadata.gz: 155e88cd3026703aac1a00ebbd02d64380402982eeabd7f384f09aec654f93f48bb934861c6d7208e45eba860c8dadbae0f7e0f19c280cb68ed26941d674dd72
7
- data.tar.gz: 5214ae1bc690bf3d84afaf27e3748be2526b63112c56f203e292fb73604606942172e94a7722028f3f809012b84f708978320ce868ca075d6427c391112cb9c3
6
+ metadata.gz: df491287ba0d78f87373c11e30979a66035e42006e6e177b8d47372e6079772367df97429ef4c12c7f68d117c31b736965e0b060199fdc5ebfef525821b559bf
7
+ data.tar.gz: ac0cb6ddbc8705712683ccfa5098f02fab45cef70c64c9de80e22631fff297a55c76c4db5c0a5b9e38fb0466917130ad28f35aa4b065c4500b8383046c2b3c2f
@@ -5,6 +5,18 @@ All notable changes to this project will be documented in this file.
5
5
  The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
6
6
  and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
7
7
 
8
+ ## 1.1.0 - 2018-02-05
9
+
10
+ ## Added
11
+
12
+ - Added ability to specify TTL for Redis keys and manually expire whole workflows (Thanks to @dmitrypol! [See pull request](https://github.com/chaps-io/gush/pull/48))
13
+ - Loosened dependency on redis-rb library to >= 3.2 and < 5.0 (Thanks to @mofumofu3n! [See pull request](https://github.com/chaps-io/gush/pull/52))
14
+
15
+ ## Fixed
16
+
17
+ - Improved performance of (de)serializing workflows by not storing job array inside workflow JSON and other smaller improvements ([See pull request](https://github.com/chaps-io/gush/pull/53))
18
+
19
+
8
20
  ## 1.0.0 - 2017-10-02
9
21
 
10
22
  ### Added
data/README.md CHANGED
@@ -348,6 +348,37 @@ This requires that you have imagemagick installed on your computer:
348
348
  bundle exec gush viz <NameOfTheWorkflow>
349
349
  ```
350
350
 
351
+ ### Cleaning up afterwards
352
+
353
+ Running `NotifyWorkflow.create` inserts multiple keys into Redis every time it is ran. This data might be useful for analysis but at a certain point it can be purged via Redis TTL. By default gush and Redis will keep keys forever. To configure expiration you need to 2 things. Create initializer (specify config.ttl in seconds, be different per environment).
354
+
355
+ ```ruby
356
+ # config/initializers/gush.rb
357
+ Gush.configure do |config|
358
+ config.redis_url = "redis://localhost:6379"
359
+ config.concurrency = 5
360
+ config.ttl = 3600*24*7
361
+ end
362
+ ```
363
+
364
+ And you need to call `flow.expire!` (optionally passing custom TTL value overriding `config.ttl`). This gives you control whether to expire data for specific workflow. Best NOT to set TTL to be too short (like minutes) but about a week in length. And you can run `Client.expire_workflow` and `Client.expire_job` passing appropriate IDs and TTL (pass -1 to NOT expire) values.
365
+
366
+ ### Avoid overlapping workflows
367
+
368
+ Since we do not know how long our workflow execution will take we might want to avoid starting the next scheduled workflow iteration while the current one with same class is still running. Long term this could be moved into core library, perhaps `Workflow.find_by_class(klass)`
369
+
370
+ ```ruby
371
+ # config/initializers/gush.rb
372
+ GUSH_CLIENT = Gush::Client.new
373
+ # call this method before NotifyWorkflow.create
374
+ def find_by_class klass
375
+ GUSH_CLIENT.all_workflows.each do |flow|
376
+ return true if flow.to_hash[:name] == klass && flow.running?
377
+ end
378
+ return false
379
+ end
380
+ ```
381
+
351
382
  ## Contributors
352
383
 
353
384
  - [Mateusz Lenik](https://github.com/mlen)
@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
4
 
5
5
  Gem::Specification.new do |spec|
6
6
  spec.name = "gush"
7
- spec.version = "1.0.0"
7
+ spec.version = "1.1.0"
8
8
  spec.authors = ["Piotrek Okoński"]
9
9
  spec.email = ["piotrek@okonski.org"]
10
10
  spec.summary = "Fast and distributed workflow runner based on ActiveJob and Redis"
@@ -20,7 +20,7 @@ Gem::Specification.new do |spec|
20
20
  spec.add_dependency "activejob", "~> 5.0"
21
21
  spec.add_dependency "connection_pool", "~> 2.2.1"
22
22
  spec.add_dependency "multi_json", "~> 1.11"
23
- spec.add_dependency "redis", "~> 3.2"
23
+ spec.add_dependency "redis", ">= 3.2", "< 5"
24
24
  spec.add_dependency "hiredis", "~> 0.6"
25
25
  spec.add_dependency "ruby-graphviz", "~> 1.2"
26
26
  spec.add_dependency "terminal-table", "~> 1.4"
@@ -16,6 +16,7 @@ module Gush
16
16
  config.concurrency = options.fetch("concurrency", config.concurrency)
17
17
  config.redis_url = options.fetch("redis", config.redis_url)
18
18
  config.namespace = options.fetch("namespace", config.namespace)
19
+ config.ttl = options.fetch("ttl", config.ttl)
19
20
  end
20
21
  load_gushfile
21
22
  end
@@ -73,7 +73,7 @@ module Gush
73
73
 
74
74
  def all_workflows
75
75
  connection_pool.with do |redis|
76
- redis.keys("gush.workflows.*").map do |key|
76
+ redis.scan_each(match: "gush.workflows.*").map do |key|
77
77
  id = key.sub("gush.workflows.", "")
78
78
  find_workflow(id)
79
79
  end
@@ -86,7 +86,7 @@ module Gush
86
86
 
87
87
  unless data.nil?
88
88
  hash = Gush::JSON.decode(data, symbolize_keys: true)
89
- keys = redis.keys("gush.jobs.#{id}.*")
89
+ keys = redis.scan_each(match: "gush.jobs.#{id}.*")
90
90
  nodes = redis.mget(*keys).map { |json| Gush::JSON.decode(json, symbolize_keys: true) }
91
91
  workflow_from_hash(hash, nodes)
92
92
  else
@@ -116,7 +116,7 @@ module Gush
116
116
  hypen = '-' if job_name_match.nil?
117
117
 
118
118
  keys = connection_pool.with do |redis|
119
- redis.keys("gush.jobs.#{workflow_id}.#{job_id}#{hypen}*")
119
+ redis.scan_each(match: "gush.jobs.#{workflow_id}.#{job_id}#{hypen}*").to_a
120
120
  end
121
121
 
122
122
  return nil if keys.nil?
@@ -144,6 +144,21 @@ module Gush
144
144
  end
145
145
  end
146
146
 
147
+ def expire_workflow(workflow, ttl=nil)
148
+ ttl = ttl || configuration.ttl
149
+ connection_pool.with do |redis|
150
+ redis.expire("gush.workflows.#{workflow.id}", ttl)
151
+ end
152
+ workflow.jobs.each {|job| expire_job(workflow.id, job, ttl) }
153
+ end
154
+
155
+ def expire_job(workflow_id, job, ttl=nil)
156
+ ttl = ttl || configuration.ttl
157
+ connection_pool.with do |redis|
158
+ redis.expire("gush.jobs.#{workflow_id}.#{job.name}", ttl)
159
+ end
160
+ end
161
+
147
162
  def enqueue_job(workflow_id, job)
148
163
  job.enqueue!
149
164
  persist_job(workflow_id, job)
@@ -153,14 +168,14 @@ module Gush
153
168
 
154
169
  private
155
170
 
156
- def workflow_from_hash(hash, nodes = nil)
171
+ def workflow_from_hash(hash, nodes = [])
157
172
  flow = hash[:klass].constantize.new(*hash[:arguments])
158
173
  flow.jobs = []
159
174
  flow.stopped = hash.fetch(:stopped, false)
160
175
  flow.id = hash[:id]
161
176
 
162
- (nodes || hash[:nodes]).each do |node|
163
- flow.jobs << Gush::Job.from_hash(node)
177
+ flow.jobs = nodes.map do |node|
178
+ Gush::Job.from_hash(node)
164
179
  end
165
180
 
166
181
  flow
@@ -1,6 +1,6 @@
1
1
  module Gush
2
2
  class Configuration
3
- attr_accessor :concurrency, :namespace, :redis_url
3
+ attr_accessor :concurrency, :namespace, :redis_url, :ttl
4
4
 
5
5
  def self.from_json(json)
6
6
  new(Gush::JSON.decode(json, symbolize_keys: true))
@@ -11,6 +11,7 @@ module Gush
11
11
  self.namespace = hash.fetch(:namespace, 'gush')
12
12
  self.redis_url = hash.fetch(:redis_url, 'redis://localhost:6379')
13
13
  self.gushfile = hash.fetch(:gushfile, 'Gushfile')
14
+ self.ttl = hash.fetch(:ttl, -1)
14
15
  end
15
16
 
16
17
  def gushfile=(path)
@@ -25,7 +26,8 @@ module Gush
25
26
  {
26
27
  concurrency: concurrency,
27
28
  namespace: namespace,
28
- redis_url: redis_url
29
+ redis_url: redis_url,
30
+ ttl: ttl
29
31
  }
30
32
  end
31
33
 
@@ -88,8 +88,8 @@ module Gush
88
88
  end
89
89
 
90
90
  def parents_succeeded?
91
- incoming.all? do |name|
92
- client.find_job(workflow_id, name).succeeded?
91
+ !incoming.any? do |name|
92
+ !client.find_job(workflow_id, name).succeeded?
93
93
  end
94
94
  end
95
95
 
@@ -53,6 +53,10 @@ module Gush
53
53
  client.persist_workflow(self)
54
54
  end
55
55
 
56
+ def expire! (ttl=nil)
57
+ client.expire_workflow(self, ttl)
58
+ end
59
+
56
60
  def mark_as_persisted
57
61
  @persisted = true
58
62
  end
@@ -168,7 +172,6 @@ module Gush
168
172
  total: jobs.count,
169
173
  finished: jobs.count(&:finished?),
170
174
  klass: name,
171
- jobs: jobs.map(&:as_json),
172
175
  status: status,
173
176
  stopped: stopped,
174
177
  started_at: started_at,
@@ -94,6 +94,16 @@ describe Gush::Client do
94
94
  end
95
95
  end
96
96
 
97
+ describe "#expire_workflow" do
98
+ it "sets TTL for all Redis keys related to the workflow" do
99
+ workflow = TestWorkflow.create
100
+
101
+ client.expire_workflow(workflow, -1)
102
+
103
+ # => TODO - I believe fakeredis does not handle TTL the same.
104
+ end
105
+ end
106
+
97
107
  describe "#persist_job" do
98
108
  it "persists JSON dump of the job in Redis" do
99
109
 
@@ -13,8 +13,7 @@ describe Gush::Workflow do
13
13
  end
14
14
 
15
15
  expect_any_instance_of(klass).to receive(:configure).with("arg1", "arg2")
16
- flow = klass.new("arg1", "arg2")
17
-
16
+ klass.new("arg1", "arg2")
18
17
  end
19
18
  end
20
19
 
@@ -102,35 +101,7 @@ describe Gush::Workflow do
102
101
  "started_at" => nil,
103
102
  "finished_at" => nil,
104
103
  "stopped" => false,
105
- "arguments" => ["arg1", "arg2"],
106
- "jobs" => [
107
- {
108
- "name"=>a_string_starting_with('FetchFirstJob'),
109
- "klass"=>"FetchFirstJob",
110
- "incoming"=>[],
111
- "outgoing"=>[a_string_starting_with('PersistFirstJob')],
112
- "finished_at"=>nil,
113
- "started_at"=>nil,
114
- "enqueued_at"=>nil,
115
- "failed_at"=>nil,
116
- "params" => {},
117
- "output_payload" => nil,
118
- "workflow_id" => an_instance_of(String)
119
- },
120
- {
121
- "name"=>a_string_starting_with('PersistFirstJob'),
122
- "klass"=>"PersistFirstJob",
123
- "incoming"=>["FetchFirstJob"],
124
- "outgoing"=>[],
125
- "finished_at"=>nil,
126
- "started_at"=>nil,
127
- "enqueued_at"=>nil,
128
- "failed_at"=>nil,
129
- "params" => {},
130
- "output_payload" => nil,
131
- "workflow_id" => an_instance_of(String)
132
- }
133
- ]
104
+ "arguments" => ["arg1", "arg2"]
134
105
  }
135
106
  expect(result).to match(expected)
136
107
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: gush
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.0.0
4
+ version: 1.1.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Piotrek Okoński
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2017-10-02 00:00:00.000000000 Z
11
+ date: 2018-02-05 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: activejob
@@ -56,16 +56,22 @@ dependencies:
56
56
  name: redis
57
57
  requirement: !ruby/object:Gem::Requirement
58
58
  requirements:
59
- - - "~>"
59
+ - - ">="
60
60
  - !ruby/object:Gem::Version
61
61
  version: '3.2'
62
+ - - "<"
63
+ - !ruby/object:Gem::Version
64
+ version: '5'
62
65
  type: :runtime
63
66
  prerelease: false
64
67
  version_requirements: !ruby/object:Gem::Requirement
65
68
  requirements:
66
- - - "~>"
69
+ - - ">="
67
70
  - !ruby/object:Gem::Version
68
71
  version: '3.2'
72
+ - - "<"
73
+ - !ruby/object:Gem::Version
74
+ version: '5'
69
75
  - !ruby/object:Gem::Dependency
70
76
  name: hiredis
71
77
  requirement: !ruby/object:Gem::Requirement
@@ -282,7 +288,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
282
288
  version: '0'
283
289
  requirements: []
284
290
  rubyforge_project:
285
- rubygems_version: 2.5.2
291
+ rubygems_version: 2.7.4
286
292
  signing_key:
287
293
  specification_version: 4
288
294
  summary: Fast and distributed workflow runner based on ActiveJob and Redis