gush 0.3.2 → 0.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +3 -0
- data/Gemfile +3 -5
- data/gush.gemspec +4 -2
- data/lib/gush.rb +1 -0
- data/lib/gush/client.rb +52 -26
- data/lib/gush/configuration.rb +0 -2
- data/lib/gush/json.rb +2 -2
- data/lib/gush/worker.rb +0 -1
- data/spec/spec_helper.rb +1 -1
- metadata +34 -6
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: d26693f019dfd49b57cb7828274bec009c96dfb4
|
4
|
+
data.tar.gz: 1666a4d1edc877d4ac597428ed56f73a5f625fa9
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: b1f55d88bbe1719395103c6eeeda6a9d140d9cd7b6d35296f1ab97b1d381a7f69bbbe83e21faa660a05fe6183d7d5443c006244cbbcc99ebbc58edf554d2adbf
|
7
|
+
data.tar.gz: c2d6e5aa2a0613dae3e893ca368e527cac28fe1b9d6df66793de83bfe73dcad7940ee9d0cfd0663e8fd1fcc30eb313c0e465f98da1a350f5958d82beef9faa61
|
data/CHANGELOG.md
ADDED
data/Gemfile
CHANGED
data/gush.gemspec
CHANGED
@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
4
4
|
|
5
5
|
Gem::Specification.new do |spec|
|
6
6
|
spec.name = "gush"
|
7
|
-
spec.version = "0.3.
|
7
|
+
spec.version = "0.3.3"
|
8
8
|
spec.authors = ["Piotrek Okoński"]
|
9
9
|
spec.email = ["piotrek@okonski.org"]
|
10
10
|
spec.summary = "Fast and distributed workflow runner using only Sidekiq and Redis"
|
@@ -18,7 +18,7 @@ Gem::Specification.new do |spec|
|
|
18
18
|
spec.require_paths = ["lib"]
|
19
19
|
|
20
20
|
spec.add_dependency "sidekiq", "~> 4.0"
|
21
|
-
spec.add_dependency "
|
21
|
+
spec.add_dependency "multi_json", "~> 1.11"
|
22
22
|
spec.add_dependency "redis", "~> 3.2"
|
23
23
|
spec.add_dependency "hiredis", "~> 0.6"
|
24
24
|
spec.add_dependency "ruby-graphviz", "~> 1.2"
|
@@ -29,4 +29,6 @@ Gem::Specification.new do |spec|
|
|
29
29
|
spec.add_development_dependency "bundler", "~> 1.5"
|
30
30
|
spec.add_development_dependency "rake", "~> 10.4"
|
31
31
|
spec.add_development_dependency "rspec", '~> 3.0'
|
32
|
+
spec.add_development_dependency "pry", '~> 0.10'
|
33
|
+
spec.add_development_dependency 'fakeredis', '~> 0.5'
|
32
34
|
end
|
data/lib/gush.rb
CHANGED
data/lib/gush/client.rb
CHANGED
@@ -1,17 +1,15 @@
|
|
1
1
|
module Gush
|
2
2
|
class Client
|
3
|
-
attr_reader :configuration
|
3
|
+
attr_reader :configuration, :sidekiq
|
4
4
|
|
5
5
|
def initialize(config = Gush.configuration)
|
6
6
|
@configuration = config
|
7
7
|
@sidekiq = build_sidekiq
|
8
|
-
@redis = build_redis
|
9
8
|
end
|
10
9
|
|
11
10
|
def configure
|
12
11
|
yield configuration
|
13
12
|
@sidekiq = build_sidekiq
|
14
|
-
@redis = build_redis
|
15
13
|
end
|
16
14
|
|
17
15
|
def create_workflow(name)
|
@@ -49,7 +47,11 @@ module Gush
|
|
49
47
|
loop do
|
50
48
|
id = SecureRandom.uuid
|
51
49
|
job_identifier = "#{job_klass}-#{id}"
|
52
|
-
|
50
|
+
available = connection_pool.with do |redis|
|
51
|
+
!redis.exists("gush.jobs.#{workflow_id}.#{job_identifier}")
|
52
|
+
end
|
53
|
+
|
54
|
+
break if available
|
53
55
|
end
|
54
56
|
|
55
57
|
job_identifier
|
@@ -59,40 +61,54 @@ module Gush
|
|
59
61
|
id = nil
|
60
62
|
loop do
|
61
63
|
id = SecureRandom.uuid
|
62
|
-
|
64
|
+
available = connection_pool.with do |redis|
|
65
|
+
!redis.exists("gush.workflow.#{id}")
|
66
|
+
end
|
67
|
+
|
68
|
+
break if available
|
63
69
|
end
|
64
70
|
|
65
71
|
id
|
66
72
|
end
|
67
73
|
|
68
74
|
def all_workflows
|
69
|
-
|
70
|
-
|
71
|
-
|
75
|
+
connection_pool.with do |redis|
|
76
|
+
redis.keys("gush.workflows.*").map do |key|
|
77
|
+
id = key.sub("gush.workflows.", "")
|
78
|
+
find_workflow(id)
|
79
|
+
end
|
72
80
|
end
|
73
81
|
end
|
74
82
|
|
75
83
|
def find_workflow(id)
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
+
connection_pool.with do |redis|
|
85
|
+
data = redis.get("gush.workflows.#{id}")
|
86
|
+
|
87
|
+
unless data.nil?
|
88
|
+
hash = Gush::JSON.decode(data, symbolize_keys: true)
|
89
|
+
keys = redis.keys("gush.jobs.#{id}.*")
|
90
|
+
nodes = redis.mget(*keys).map { |json| Gush::JSON.decode(json, symbolize_keys: true) }
|
91
|
+
workflow_from_hash(hash, nodes)
|
92
|
+
else
|
93
|
+
raise WorkflowNotFound.new("Workflow with given id doesn't exist")
|
94
|
+
end
|
84
95
|
end
|
85
96
|
end
|
86
97
|
|
87
98
|
def persist_workflow(workflow)
|
88
|
-
|
99
|
+
connection_pool.with do |redis|
|
100
|
+
redis.set("gush.workflows.#{workflow.id}", workflow.to_json)
|
101
|
+
end
|
102
|
+
|
89
103
|
workflow.jobs.each {|job| persist_job(workflow.id, job) }
|
90
104
|
workflow.mark_as_persisted
|
91
105
|
true
|
92
106
|
end
|
93
107
|
|
94
108
|
def persist_job(workflow_id, job)
|
95
|
-
|
109
|
+
connection_pool.with do |redis|
|
110
|
+
redis.set("gush.jobs.#{workflow_id}.#{job.name}", job.to_json)
|
111
|
+
end
|
96
112
|
end
|
97
113
|
|
98
114
|
def load_job(workflow_id, job_id)
|
@@ -100,10 +116,16 @@ module Gush
|
|
100
116
|
job_name_match = /(?<klass>\w*[^-])-(?<identifier>.*)/.match(job_id)
|
101
117
|
hypen = '-' if job_name_match.nil?
|
102
118
|
|
103
|
-
keys = redis
|
119
|
+
keys = connection_pool.with do |redis|
|
120
|
+
redis.keys("gush.jobs.#{workflow_id}.#{job_id}#{hypen}*")
|
121
|
+
end
|
122
|
+
|
104
123
|
return nil if keys.nil?
|
105
124
|
|
106
|
-
data = redis
|
125
|
+
data = connection_pool.with do |redis|
|
126
|
+
redis.get(keys.first)
|
127
|
+
end
|
128
|
+
|
107
129
|
return nil if data.nil?
|
108
130
|
|
109
131
|
data = Gush::JSON.decode(data, symbolize_keys: true)
|
@@ -111,12 +133,16 @@ module Gush
|
|
111
133
|
end
|
112
134
|
|
113
135
|
def destroy_workflow(workflow)
|
114
|
-
redis
|
136
|
+
connection_pool.with do |redis|
|
137
|
+
redis.del("gush.workflows.#{workflow.id}")
|
138
|
+
end
|
115
139
|
workflow.jobs.each {|job| destroy_job(workflow.id, job) }
|
116
140
|
end
|
117
141
|
|
118
142
|
def destroy_job(workflow_id, job)
|
119
|
-
redis
|
143
|
+
connection_pool.with do |redis|
|
144
|
+
redis.del("gush.jobs.#{workflow_id}.#{job.name}")
|
145
|
+
end
|
120
146
|
end
|
121
147
|
|
122
148
|
def worker_report(message)
|
@@ -140,8 +166,6 @@ module Gush
|
|
140
166
|
|
141
167
|
private
|
142
168
|
|
143
|
-
attr_reader :sidekiq, :redis
|
144
|
-
|
145
169
|
def workflow_from_hash(hash, nodes = nil)
|
146
170
|
flow = hash[:klass].constantize.new *hash[:arguments]
|
147
171
|
flow.jobs = []
|
@@ -156,7 +180,9 @@ module Gush
|
|
156
180
|
end
|
157
181
|
|
158
182
|
def report(key, message)
|
159
|
-
|
183
|
+
connection_pool.with do |redis|
|
184
|
+
redis.publish(key, Gush::JSON.encode(message))
|
185
|
+
end
|
160
186
|
end
|
161
187
|
|
162
188
|
|
@@ -169,7 +195,7 @@ module Gush
|
|
169
195
|
end
|
170
196
|
|
171
197
|
def connection_pool
|
172
|
-
ConnectionPool.new(size: configuration.concurrency, timeout: 1) { build_redis }
|
198
|
+
@connection_pool ||= ConnectionPool.new(size: configuration.concurrency, timeout: 1) { build_redis }
|
173
199
|
end
|
174
200
|
end
|
175
201
|
end
|
data/lib/gush/configuration.rb
CHANGED
data/lib/gush/json.rb
CHANGED
data/lib/gush/worker.rb
CHANGED
data/spec/spec_helper.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: gush
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.3.
|
4
|
+
version: 0.3.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Piotrek Okoński
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-
|
11
|
+
date: 2016-08-23 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: sidekiq
|
@@ -25,19 +25,19 @@ dependencies:
|
|
25
25
|
- !ruby/object:Gem::Version
|
26
26
|
version: '4.0'
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
|
-
name:
|
28
|
+
name: multi_json
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
30
30
|
requirements:
|
31
31
|
- - "~>"
|
32
32
|
- !ruby/object:Gem::Version
|
33
|
-
version: '1.
|
33
|
+
version: '1.11'
|
34
34
|
type: :runtime
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
38
|
- - "~>"
|
39
39
|
- !ruby/object:Gem::Version
|
40
|
-
version: '1.
|
40
|
+
version: '1.11'
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
42
|
name: redis
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
@@ -178,6 +178,34 @@ dependencies:
|
|
178
178
|
- - "~>"
|
179
179
|
- !ruby/object:Gem::Version
|
180
180
|
version: '3.0'
|
181
|
+
- !ruby/object:Gem::Dependency
|
182
|
+
name: pry
|
183
|
+
requirement: !ruby/object:Gem::Requirement
|
184
|
+
requirements:
|
185
|
+
- - "~>"
|
186
|
+
- !ruby/object:Gem::Version
|
187
|
+
version: '0.10'
|
188
|
+
type: :development
|
189
|
+
prerelease: false
|
190
|
+
version_requirements: !ruby/object:Gem::Requirement
|
191
|
+
requirements:
|
192
|
+
- - "~>"
|
193
|
+
- !ruby/object:Gem::Version
|
194
|
+
version: '0.10'
|
195
|
+
- !ruby/object:Gem::Dependency
|
196
|
+
name: fakeredis
|
197
|
+
requirement: !ruby/object:Gem::Requirement
|
198
|
+
requirements:
|
199
|
+
- - "~>"
|
200
|
+
- !ruby/object:Gem::Version
|
201
|
+
version: '0.5'
|
202
|
+
type: :development
|
203
|
+
prerelease: false
|
204
|
+
version_requirements: !ruby/object:Gem::Requirement
|
205
|
+
requirements:
|
206
|
+
- - "~>"
|
207
|
+
- !ruby/object:Gem::Version
|
208
|
+
version: '0.5'
|
181
209
|
description: Gush is a parallel workflow runner using only Redis as its message broker
|
182
210
|
and Sidekiq for workers.
|
183
211
|
email:
|
@@ -190,6 +218,7 @@ files:
|
|
190
218
|
- ".gitignore"
|
191
219
|
- ".rspec"
|
192
220
|
- ".travis.yml"
|
221
|
+
- CHANGELOG.md
|
193
222
|
- Gemfile
|
194
223
|
- LICENSE.txt
|
195
224
|
- README.md
|
@@ -256,4 +285,3 @@ test_files:
|
|
256
285
|
- spec/gush/workflow_spec.rb
|
257
286
|
- spec/gush_spec.rb
|
258
287
|
- spec/spec_helper.rb
|
259
|
-
has_rdoc:
|