gush 2.0.1 → 2.0.2

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: ffcdce12d2f530e27afdd048328d8942e60efcddfbe20a18426b8d245589c759
4
- data.tar.gz: 3c95194d44f446a694af83b06b06eac55a3e5ed1f8555fe86958fa110b357647
3
+ metadata.gz: 12fdb9a62d33353f827194c198c011ff42d7491669d0ab9717be0920a8066313
4
+ data.tar.gz: 7f3d4ada23215e818f0cb801c92f4752e80b21b344f3b48d9ab4b83787c029ec
5
5
  SHA512:
6
- metadata.gz: e75e7f0ae5c5c83302d0507bb502a30a860f276e0706edd5a4119bb9c50bf88c73c9839ac1f141f4569017301ebb61fe943317d214c0c9a10f49309e674bd183
7
- data.tar.gz: 38f3baaa43a4d512d5d5ec6c476d21965d27116e7dc8bc1ae3cc3fa4a4fac13e6d18272ed236fd413aba0c304f0e79aee59f3c06cdad19db7fa02e41305fe491
6
+ metadata.gz: bec4bcb3e251bdb1a2e184b6b62ae896fd40af4a0cd4bad2ec7ee4925ab356e36a3388d2cb78ed21582c0f6cfdd429c304f0591a5ee3760c73255d03a2b1d80e
7
+ data.tar.gz: 67fba0b65b575449114233a4ff9cfaf1cb6ac3ff61633f94b7c595988165b3315fe7ae5d0744792315d9329795da0e42ae840789e3d4f72c84ef8956081c5fbd
data/.gitignore CHANGED
@@ -19,3 +19,5 @@ tmp
19
19
  test.rb
20
20
  /Gushfile
21
21
  dump.rdb
22
+ .ruby-version
23
+ .ruby-gemset
data/.travis.yml CHANGED
@@ -4,6 +4,9 @@ rvm:
4
4
  - 2.2.2
5
5
  - 2.3.4
6
6
  - 2.4.1
7
+ - 2.5
8
+ - 2.6
9
+ - 2.7
7
10
  services:
8
11
  - redis-server
9
12
  email:
data/README.md CHANGED
@@ -1,7 +1,5 @@
1
1
  # Gush [![Build Status](https://travis-ci.org/chaps-io/gush.svg?branch=master)](https://travis-ci.org/chaps-io/gush)
2
2
 
3
- ## [![](http://i.imgur.com/ya8Wnyl.png)](https://chaps.io) proudly made by [Chaps](https://chaps.io)
4
-
5
3
  Gush is a parallel workflow runner using only Redis as storage and [ActiveJob](http://guides.rubyonrails.org/v4.2/active_job_basics.html#introduction) for scheduling and executing jobs.
6
4
 
7
5
  ## Theory
data/gush.gemspec CHANGED
@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
4
 
5
5
  Gem::Specification.new do |spec|
6
6
  spec.name = "gush"
7
- spec.version = "2.0.1"
7
+ spec.version = "2.0.2"
8
8
  spec.authors = ["Piotrek Okoński"]
9
9
  spec.email = ["piotrek@okonski.org"]
10
10
  spec.summary = "Fast and distributed workflow runner based on ActiveJob and Redis"
@@ -17,8 +17,8 @@ Gem::Specification.new do |spec|
17
17
  spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
18
18
  spec.require_paths = ["lib"]
19
19
 
20
- spec.add_dependency "activejob", ">= 4.2.7", "< 6.0"
21
- spec.add_dependency "connection_pool", "~> 2.2.1"
20
+ spec.add_dependency "activejob", ">= 4.2.7", "< 7.0"
21
+ spec.add_dependency "concurrent-ruby", "~> 1.0"
22
22
  spec.add_dependency "multi_json", "~> 1.11"
23
23
  spec.add_dependency "redis", ">= 3.2", "< 5"
24
24
  spec.add_dependency "redis-mutex", "~> 4.0.1"
@@ -28,7 +28,7 @@ Gem::Specification.new do |spec|
28
28
  spec.add_dependency "colorize", "~> 0.7"
29
29
  spec.add_dependency "thor", "~> 0.19"
30
30
  spec.add_dependency "launchy", "~> 2.4"
31
- spec.add_development_dependency "bundler", "~> 1.5"
31
+ spec.add_development_dependency "bundler"
32
32
  spec.add_development_dependency "rake", "~> 10.4"
33
33
  spec.add_development_dependency "rspec", '~> 3.0'
34
34
  spec.add_development_dependency "pry", '~> 0.10'
data/lib/gush/client.rb CHANGED
@@ -1,9 +1,22 @@
1
- require 'connection_pool'
1
+ require 'redis'
2
+ require 'concurrent-ruby'
2
3
 
3
4
  module Gush
4
5
  class Client
5
6
  attr_reader :configuration
6
7
 
8
+ @@redis_connection = Concurrent::ThreadLocalVar.new(nil)
9
+
10
+ def self.redis_connection(config)
11
+ cached = (@@redis_connection.value ||= { url: config.redis_url, connection: nil })
12
+ return cached[:connection] if !cached[:connection].nil? && config.redis_url == cached[:url]
13
+
14
+ Redis.new(url: config.redis_url).tap do |instance|
15
+ RedisClassy.redis = instance
16
+ @@redis_connection.value = { url: config.redis_url, connection: instance }
17
+ end
18
+ end
19
+
7
20
  def initialize(config = Gush.configuration)
8
21
  @configuration = config
9
22
  end
@@ -47,9 +60,7 @@ module Gush
47
60
 
48
61
  loop do
49
62
  job_id = SecureRandom.uuid
50
- available = connection_pool.with do |redis|
51
- !redis.hexists("gush.jobs.#{workflow_id}.#{job_klass}", job_id)
52
- end
63
+ available = !redis.hexists("gush.jobs.#{workflow_id}.#{job_klass}", job_id)
53
64
 
54
65
  break if available
55
66
  end
@@ -61,9 +72,7 @@ module Gush
61
72
  id = nil
62
73
  loop do
63
74
  id = SecureRandom.uuid
64
- available = connection_pool.with do |redis|
65
- !redis.exists("gush.workflow.#{id}")
66
- end
75
+ available = !redis.exists("gush.workflow.#{id}")
67
76
 
68
77
  break if available
69
78
  end
@@ -72,37 +81,31 @@ module Gush
72
81
  end
73
82
 
74
83
  def all_workflows
75
- connection_pool.with do |redis|
76
- redis.scan_each(match: "gush.workflows.*").map do |key|
77
- id = key.sub("gush.workflows.", "")
78
- find_workflow(id)
79
- end
84
+ redis.scan_each(match: "gush.workflows.*").map do |key|
85
+ id = key.sub("gush.workflows.", "")
86
+ find_workflow(id)
80
87
  end
81
88
  end
82
89
 
83
90
  def find_workflow(id)
84
- connection_pool.with do |redis|
85
- data = redis.get("gush.workflows.#{id}")
91
+ data = redis.get("gush.workflows.#{id}")
86
92
 
87
- unless data.nil?
88
- hash = Gush::JSON.decode(data, symbolize_keys: true)
89
- keys = redis.scan_each(match: "gush.jobs.#{id}.*")
93
+ unless data.nil?
94
+ hash = Gush::JSON.decode(data, symbolize_keys: true)
95
+ keys = redis.scan_each(match: "gush.jobs.#{id}.*")
90
96
 
91
- nodes = keys.each_with_object([]) do |key, array|
92
- array.concat redis.hvals(key).map { |json| Gush::JSON.decode(json, symbolize_keys: true) }
93
- end
94
-
95
- workflow_from_hash(hash, nodes)
96
- else
97
- raise WorkflowNotFound.new("Workflow with given id doesn't exist")
97
+ nodes = keys.each_with_object([]) do |key, array|
98
+ array.concat redis.hvals(key).map { |json| Gush::JSON.decode(json, symbolize_keys: true) }
98
99
  end
100
+
101
+ workflow_from_hash(hash, nodes)
102
+ else
103
+ raise WorkflowNotFound.new("Workflow with given id doesn't exist")
99
104
  end
100
105
  end
101
106
 
102
107
  def persist_workflow(workflow)
103
- connection_pool.with do |redis|
104
- redis.set("gush.workflows.#{workflow.id}", workflow.to_json)
105
- end
108
+ redis.set("gush.workflows.#{workflow.id}", workflow.to_json)
106
109
 
107
110
  workflow.jobs.each {|job| persist_job(workflow.id, job) }
108
111
  workflow.mark_as_persisted
@@ -111,9 +114,7 @@ module Gush
111
114
  end
112
115
 
113
116
  def persist_job(workflow_id, job)
114
- connection_pool.with do |redis|
115
- redis.hset("gush.jobs.#{workflow_id}.#{job.klass}", job.id, job.to_json)
116
- end
117
+ redis.hset("gush.jobs.#{workflow_id}.#{job.klass}", job.id, job.to_json)
117
118
  end
118
119
 
119
120
  def find_job(workflow_id, job_name)
@@ -132,31 +133,23 @@ module Gush
132
133
  end
133
134
 
134
135
  def destroy_workflow(workflow)
135
- connection_pool.with do |redis|
136
- redis.del("gush.workflows.#{workflow.id}")
137
- end
136
+ redis.del("gush.workflows.#{workflow.id}")
138
137
  workflow.jobs.each {|job| destroy_job(workflow.id, job) }
139
138
  end
140
139
 
141
140
  def destroy_job(workflow_id, job)
142
- connection_pool.with do |redis|
143
- redis.del("gush.jobs.#{workflow_id}.#{job.klass}")
144
- end
141
+ redis.del("gush.jobs.#{workflow_id}.#{job.klass}")
145
142
  end
146
143
 
147
144
  def expire_workflow(workflow, ttl=nil)
148
145
  ttl = ttl || configuration.ttl
149
- connection_pool.with do |redis|
150
- redis.expire("gush.workflows.#{workflow.id}", ttl)
151
- end
146
+ redis.expire("gush.workflows.#{workflow.id}", ttl)
152
147
  workflow.jobs.each {|job| expire_job(workflow.id, job, ttl) }
153
148
  end
154
149
 
155
150
  def expire_job(workflow_id, job, ttl=nil)
156
151
  ttl = ttl || configuration.ttl
157
- connection_pool.with do |redis|
158
- redis.expire("gush.jobs.#{workflow_id}.#{job.name}", ttl)
159
- end
152
+ redis.expire("gush.jobs.#{workflow_id}.#{job.klass}", ttl)
160
153
  end
161
154
 
162
155
  def enqueue_job(workflow_id, job)
@@ -172,16 +165,11 @@ module Gush
172
165
  def find_job_by_klass_and_id(workflow_id, job_name)
173
166
  job_klass, job_id = job_name.split('|')
174
167
 
175
- connection_pool.with do |redis|
176
- redis.hget("gush.jobs.#{workflow_id}.#{job_klass}", job_id)
177
- end
168
+ redis.hget("gush.jobs.#{workflow_id}.#{job_klass}", job_id)
178
169
  end
179
170
 
180
171
  def find_job_by_klass(workflow_id, job_name)
181
- new_cursor, result = connection_pool.with do |redis|
182
- redis.hscan("gush.jobs.#{workflow_id}.#{job_name}", 0, count: 1)
183
- end
184
-
172
+ new_cursor, result = redis.hscan("gush.jobs.#{workflow_id}.#{job_name}", 0, count: 1)
185
173
  return nil if result.empty?
186
174
 
187
175
  job_id, job = *result[0]
@@ -202,14 +190,8 @@ module Gush
202
190
  flow
203
191
  end
204
192
 
205
- def build_redis
206
- Redis.new(url: configuration.redis_url).tap do |instance|
207
- RedisClassy.redis = instance
208
- end
209
- end
210
-
211
- def connection_pool
212
- @connection_pool ||= ConnectionPool.new(size: configuration.concurrency, timeout: 1) { build_redis }
193
+ def redis
194
+ self.class.redis_connection(configuration)
213
195
  end
214
196
  end
215
197
  end
data/lib/gush/worker.rb CHANGED
@@ -6,6 +6,12 @@ module Gush
6
6
  def perform(workflow_id, job_id)
7
7
  setup_job(workflow_id, job_id)
8
8
 
9
+ if job.succeeded?
10
+ # Try to enqueue outgoing jobs again because the last job has redis mutex lock error
11
+ enqueue_outgoing_jobs
12
+ return
13
+ end
14
+
9
15
  job.payloads = incoming_payloads
10
16
 
11
17
  error = nil
@@ -75,6 +81,8 @@ module Gush
75
81
  end
76
82
  end
77
83
  end
84
+ rescue RedisMutex::LockError
85
+ Worker.set(wait: 2.seconds).perform_later(workflow_id, job.name)
78
86
  end
79
87
  end
80
88
  end
@@ -95,12 +95,18 @@ describe Gush::Client do
95
95
  end
96
96
 
97
97
  describe "#expire_workflow" do
98
+ let(:ttl) { 2000 }
99
+
98
100
  it "sets TTL for all Redis keys related to the workflow" do
99
101
  workflow = TestWorkflow.create
100
102
 
101
- client.expire_workflow(workflow, -1)
103
+ client.expire_workflow(workflow, ttl)
104
+
105
+ expect(redis.ttl("gush.workflows.#{workflow.id}")).to eq(ttl)
102
106
 
103
- # => TODO - I believe fakeredis does not handle TTL the same.
107
+ workflow.jobs.each do |job|
108
+ expect(redis.ttl("gush.jobs.#{workflow.id}.#{job.klass}")).to eq(ttl)
109
+ end
104
110
  end
105
111
  end
106
112
 
@@ -39,6 +39,18 @@ describe Gush::Worker do
39
39
  end
40
40
  end
41
41
 
42
+ context 'when job failed to enqueue outgoing jobs' do
43
+ it 'enqeues another job to handling enqueue_outgoing_jobs' do
44
+ allow(RedisMutex).to receive(:with_lock).and_raise(RedisMutex::LockError)
45
+ subject.perform(workflow.id, 'Prepare')
46
+ expect(Gush::Worker).to have_no_jobs(workflow.id, jobs_with_id(["FetchFirstJob", "FetchSecondJob"]))
47
+
48
+ allow(RedisMutex).to receive(:with_lock).and_call_original
49
+ perform_one
50
+ expect(Gush::Worker).to have_jobs(workflow.id, jobs_with_id(["FetchFirstJob", "FetchSecondJob"]))
51
+ end
52
+ end
53
+
42
54
  it "calls job.perform method" do
43
55
  SPY = double()
44
56
  expect(SPY).to receive(:some_method)
data/spec/spec_helper.rb CHANGED
@@ -78,6 +78,19 @@ RSpec::Matchers.define :have_jobs do |flow, jobs|
78
78
  end
79
79
  end
80
80
 
81
+ RSpec::Matchers.define :have_no_jobs do |flow, jobs|
82
+ match do |actual|
83
+ expected = jobs.map do |job|
84
+ hash_including(args: include(flow, job))
85
+ end
86
+ expect(ActiveJob::Base.queue_adapter.enqueued_jobs).not_to match_array(expected)
87
+ end
88
+
89
+ failure_message do |actual|
90
+ "expected queue to have no #{jobs}, but instead has: #{ActiveJob::Base.queue_adapter.enqueued_jobs.map{ |j| j[:args][1]}}"
91
+ end
92
+ end
93
+
81
94
  RSpec.configure do |config|
82
95
  config.include ActiveJob::TestHelper
83
96
  config.include GushHelpers
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: gush
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.0.1
4
+ version: 2.0.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - Piotrek Okoński
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2019-03-12 00:00:00.000000000 Z
11
+ date: 2022-03-10 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: activejob
@@ -19,7 +19,7 @@ dependencies:
19
19
  version: 4.2.7
20
20
  - - "<"
21
21
  - !ruby/object:Gem::Version
22
- version: '6.0'
22
+ version: '7.0'
23
23
  type: :runtime
24
24
  prerelease: false
25
25
  version_requirements: !ruby/object:Gem::Requirement
@@ -29,21 +29,21 @@ dependencies:
29
29
  version: 4.2.7
30
30
  - - "<"
31
31
  - !ruby/object:Gem::Version
32
- version: '6.0'
32
+ version: '7.0'
33
33
  - !ruby/object:Gem::Dependency
34
- name: connection_pool
34
+ name: concurrent-ruby
35
35
  requirement: !ruby/object:Gem::Requirement
36
36
  requirements:
37
37
  - - "~>"
38
38
  - !ruby/object:Gem::Version
39
- version: 2.2.1
39
+ version: '1.0'
40
40
  type: :runtime
41
41
  prerelease: false
42
42
  version_requirements: !ruby/object:Gem::Requirement
43
43
  requirements:
44
44
  - - "~>"
45
45
  - !ruby/object:Gem::Version
46
- version: 2.2.1
46
+ version: '1.0'
47
47
  - !ruby/object:Gem::Dependency
48
48
  name: multi_json
49
49
  requirement: !ruby/object:Gem::Requirement
@@ -180,16 +180,16 @@ dependencies:
180
180
  name: bundler
181
181
  requirement: !ruby/object:Gem::Requirement
182
182
  requirements:
183
- - - "~>"
183
+ - - ">="
184
184
  - !ruby/object:Gem::Version
185
- version: '1.5'
185
+ version: '0'
186
186
  type: :development
187
187
  prerelease: false
188
188
  version_requirements: !ruby/object:Gem::Requirement
189
189
  requirements:
190
- - - "~>"
190
+ - - ">="
191
191
  - !ruby/object:Gem::Version
192
- version: '1.5'
192
+ version: '0'
193
193
  - !ruby/object:Gem::Dependency
194
194
  name: rake
195
195
  requirement: !ruby/object:Gem::Requirement
@@ -307,8 +307,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
307
307
  - !ruby/object:Gem::Version
308
308
  version: '0'
309
309
  requirements: []
310
- rubyforge_project:
311
- rubygems_version: 2.7.6
310
+ rubygems_version: 3.1.4
312
311
  signing_key:
313
312
  specification_version: 4
314
313
  summary: Fast and distributed workflow runner based on ActiveJob and Redis