gush 0.3.2 → 0.3.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +3 -0
- data/Gemfile +3 -5
- data/gush.gemspec +4 -2
- data/lib/gush.rb +1 -0
- data/lib/gush/client.rb +52 -26
- data/lib/gush/configuration.rb +0 -2
- data/lib/gush/json.rb +2 -2
- data/lib/gush/worker.rb +0 -1
- data/spec/spec_helper.rb +1 -1
- metadata +34 -6
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: d26693f019dfd49b57cb7828274bec009c96dfb4
|
4
|
+
data.tar.gz: 1666a4d1edc877d4ac597428ed56f73a5f625fa9
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: b1f55d88bbe1719395103c6eeeda6a9d140d9cd7b6d35296f1ab97b1d381a7f69bbbe83e21faa660a05fe6183d7d5443c006244cbbcc99ebbc58edf554d2adbf
|
7
|
+
data.tar.gz: c2d6e5aa2a0613dae3e893ca368e527cac28fe1b9d6df66793de83bfe73dcad7940ee9d0cfd0663e8fd1fcc30eb313c0e465f98da1a350f5958d82beef9faa61
|
data/CHANGELOG.md
ADDED
data/Gemfile
CHANGED
data/gush.gemspec
CHANGED
@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
4
4
|
|
5
5
|
Gem::Specification.new do |spec|
|
6
6
|
spec.name = "gush"
|
7
|
-
spec.version = "0.3.
|
7
|
+
spec.version = "0.3.3"
|
8
8
|
spec.authors = ["Piotrek Okoński"]
|
9
9
|
spec.email = ["piotrek@okonski.org"]
|
10
10
|
spec.summary = "Fast and distributed workflow runner using only Sidekiq and Redis"
|
@@ -18,7 +18,7 @@ Gem::Specification.new do |spec|
|
|
18
18
|
spec.require_paths = ["lib"]
|
19
19
|
|
20
20
|
spec.add_dependency "sidekiq", "~> 4.0"
|
21
|
-
spec.add_dependency "
|
21
|
+
spec.add_dependency "multi_json", "~> 1.11"
|
22
22
|
spec.add_dependency "redis", "~> 3.2"
|
23
23
|
spec.add_dependency "hiredis", "~> 0.6"
|
24
24
|
spec.add_dependency "ruby-graphviz", "~> 1.2"
|
@@ -29,4 +29,6 @@ Gem::Specification.new do |spec|
|
|
29
29
|
spec.add_development_dependency "bundler", "~> 1.5"
|
30
30
|
spec.add_development_dependency "rake", "~> 10.4"
|
31
31
|
spec.add_development_dependency "rspec", '~> 3.0'
|
32
|
+
spec.add_development_dependency "pry", '~> 0.10'
|
33
|
+
spec.add_development_dependency 'fakeredis', '~> 0.5'
|
32
34
|
end
|
data/lib/gush.rb
CHANGED
data/lib/gush/client.rb
CHANGED
@@ -1,17 +1,15 @@
|
|
1
1
|
module Gush
|
2
2
|
class Client
|
3
|
-
attr_reader :configuration
|
3
|
+
attr_reader :configuration, :sidekiq
|
4
4
|
|
5
5
|
def initialize(config = Gush.configuration)
|
6
6
|
@configuration = config
|
7
7
|
@sidekiq = build_sidekiq
|
8
|
-
@redis = build_redis
|
9
8
|
end
|
10
9
|
|
11
10
|
def configure
|
12
11
|
yield configuration
|
13
12
|
@sidekiq = build_sidekiq
|
14
|
-
@redis = build_redis
|
15
13
|
end
|
16
14
|
|
17
15
|
def create_workflow(name)
|
@@ -49,7 +47,11 @@ module Gush
|
|
49
47
|
loop do
|
50
48
|
id = SecureRandom.uuid
|
51
49
|
job_identifier = "#{job_klass}-#{id}"
|
52
|
-
|
50
|
+
available = connection_pool.with do |redis|
|
51
|
+
!redis.exists("gush.jobs.#{workflow_id}.#{job_identifier}")
|
52
|
+
end
|
53
|
+
|
54
|
+
break if available
|
53
55
|
end
|
54
56
|
|
55
57
|
job_identifier
|
@@ -59,40 +61,54 @@ module Gush
|
|
59
61
|
id = nil
|
60
62
|
loop do
|
61
63
|
id = SecureRandom.uuid
|
62
|
-
|
64
|
+
available = connection_pool.with do |redis|
|
65
|
+
!redis.exists("gush.workflow.#{id}")
|
66
|
+
end
|
67
|
+
|
68
|
+
break if available
|
63
69
|
end
|
64
70
|
|
65
71
|
id
|
66
72
|
end
|
67
73
|
|
68
74
|
def all_workflows
|
69
|
-
|
70
|
-
|
71
|
-
|
75
|
+
connection_pool.with do |redis|
|
76
|
+
redis.keys("gush.workflows.*").map do |key|
|
77
|
+
id = key.sub("gush.workflows.", "")
|
78
|
+
find_workflow(id)
|
79
|
+
end
|
72
80
|
end
|
73
81
|
end
|
74
82
|
|
75
83
|
def find_workflow(id)
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
+
connection_pool.with do |redis|
|
85
|
+
data = redis.get("gush.workflows.#{id}")
|
86
|
+
|
87
|
+
unless data.nil?
|
88
|
+
hash = Gush::JSON.decode(data, symbolize_keys: true)
|
89
|
+
keys = redis.keys("gush.jobs.#{id}.*")
|
90
|
+
nodes = redis.mget(*keys).map { |json| Gush::JSON.decode(json, symbolize_keys: true) }
|
91
|
+
workflow_from_hash(hash, nodes)
|
92
|
+
else
|
93
|
+
raise WorkflowNotFound.new("Workflow with given id doesn't exist")
|
94
|
+
end
|
84
95
|
end
|
85
96
|
end
|
86
97
|
|
87
98
|
def persist_workflow(workflow)
|
88
|
-
|
99
|
+
connection_pool.with do |redis|
|
100
|
+
redis.set("gush.workflows.#{workflow.id}", workflow.to_json)
|
101
|
+
end
|
102
|
+
|
89
103
|
workflow.jobs.each {|job| persist_job(workflow.id, job) }
|
90
104
|
workflow.mark_as_persisted
|
91
105
|
true
|
92
106
|
end
|
93
107
|
|
94
108
|
def persist_job(workflow_id, job)
|
95
|
-
|
109
|
+
connection_pool.with do |redis|
|
110
|
+
redis.set("gush.jobs.#{workflow_id}.#{job.name}", job.to_json)
|
111
|
+
end
|
96
112
|
end
|
97
113
|
|
98
114
|
def load_job(workflow_id, job_id)
|
@@ -100,10 +116,16 @@ module Gush
|
|
100
116
|
job_name_match = /(?<klass>\w*[^-])-(?<identifier>.*)/.match(job_id)
|
101
117
|
hypen = '-' if job_name_match.nil?
|
102
118
|
|
103
|
-
keys = redis
|
119
|
+
keys = connection_pool.with do |redis|
|
120
|
+
redis.keys("gush.jobs.#{workflow_id}.#{job_id}#{hypen}*")
|
121
|
+
end
|
122
|
+
|
104
123
|
return nil if keys.nil?
|
105
124
|
|
106
|
-
data = redis
|
125
|
+
data = connection_pool.with do |redis|
|
126
|
+
redis.get(keys.first)
|
127
|
+
end
|
128
|
+
|
107
129
|
return nil if data.nil?
|
108
130
|
|
109
131
|
data = Gush::JSON.decode(data, symbolize_keys: true)
|
@@ -111,12 +133,16 @@ module Gush
|
|
111
133
|
end
|
112
134
|
|
113
135
|
def destroy_workflow(workflow)
|
114
|
-
redis
|
136
|
+
connection_pool.with do |redis|
|
137
|
+
redis.del("gush.workflows.#{workflow.id}")
|
138
|
+
end
|
115
139
|
workflow.jobs.each {|job| destroy_job(workflow.id, job) }
|
116
140
|
end
|
117
141
|
|
118
142
|
def destroy_job(workflow_id, job)
|
119
|
-
redis
|
143
|
+
connection_pool.with do |redis|
|
144
|
+
redis.del("gush.jobs.#{workflow_id}.#{job.name}")
|
145
|
+
end
|
120
146
|
end
|
121
147
|
|
122
148
|
def worker_report(message)
|
@@ -140,8 +166,6 @@ module Gush
|
|
140
166
|
|
141
167
|
private
|
142
168
|
|
143
|
-
attr_reader :sidekiq, :redis
|
144
|
-
|
145
169
|
def workflow_from_hash(hash, nodes = nil)
|
146
170
|
flow = hash[:klass].constantize.new *hash[:arguments]
|
147
171
|
flow.jobs = []
|
@@ -156,7 +180,9 @@ module Gush
|
|
156
180
|
end
|
157
181
|
|
158
182
|
def report(key, message)
|
159
|
-
|
183
|
+
connection_pool.with do |redis|
|
184
|
+
redis.publish(key, Gush::JSON.encode(message))
|
185
|
+
end
|
160
186
|
end
|
161
187
|
|
162
188
|
|
@@ -169,7 +195,7 @@ module Gush
|
|
169
195
|
end
|
170
196
|
|
171
197
|
def connection_pool
|
172
|
-
ConnectionPool.new(size: configuration.concurrency, timeout: 1) { build_redis }
|
198
|
+
@connection_pool ||= ConnectionPool.new(size: configuration.concurrency, timeout: 1) { build_redis }
|
173
199
|
end
|
174
200
|
end
|
175
201
|
end
|
data/lib/gush/configuration.rb
CHANGED
data/lib/gush/json.rb
CHANGED
data/lib/gush/worker.rb
CHANGED
data/spec/spec_helper.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: gush
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.3.
|
4
|
+
version: 0.3.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Piotrek Okoński
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-
|
11
|
+
date: 2016-08-23 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: sidekiq
|
@@ -25,19 +25,19 @@ dependencies:
|
|
25
25
|
- !ruby/object:Gem::Version
|
26
26
|
version: '4.0'
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
|
-
name:
|
28
|
+
name: multi_json
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
30
30
|
requirements:
|
31
31
|
- - "~>"
|
32
32
|
- !ruby/object:Gem::Version
|
33
|
-
version: '1.
|
33
|
+
version: '1.11'
|
34
34
|
type: :runtime
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
38
|
- - "~>"
|
39
39
|
- !ruby/object:Gem::Version
|
40
|
-
version: '1.
|
40
|
+
version: '1.11'
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
42
|
name: redis
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
@@ -178,6 +178,34 @@ dependencies:
|
|
178
178
|
- - "~>"
|
179
179
|
- !ruby/object:Gem::Version
|
180
180
|
version: '3.0'
|
181
|
+
- !ruby/object:Gem::Dependency
|
182
|
+
name: pry
|
183
|
+
requirement: !ruby/object:Gem::Requirement
|
184
|
+
requirements:
|
185
|
+
- - "~>"
|
186
|
+
- !ruby/object:Gem::Version
|
187
|
+
version: '0.10'
|
188
|
+
type: :development
|
189
|
+
prerelease: false
|
190
|
+
version_requirements: !ruby/object:Gem::Requirement
|
191
|
+
requirements:
|
192
|
+
- - "~>"
|
193
|
+
- !ruby/object:Gem::Version
|
194
|
+
version: '0.10'
|
195
|
+
- !ruby/object:Gem::Dependency
|
196
|
+
name: fakeredis
|
197
|
+
requirement: !ruby/object:Gem::Requirement
|
198
|
+
requirements:
|
199
|
+
- - "~>"
|
200
|
+
- !ruby/object:Gem::Version
|
201
|
+
version: '0.5'
|
202
|
+
type: :development
|
203
|
+
prerelease: false
|
204
|
+
version_requirements: !ruby/object:Gem::Requirement
|
205
|
+
requirements:
|
206
|
+
- - "~>"
|
207
|
+
- !ruby/object:Gem::Version
|
208
|
+
version: '0.5'
|
181
209
|
description: Gush is a parallel workflow runner using only Redis as its message broker
|
182
210
|
and Sidekiq for workers.
|
183
211
|
email:
|
@@ -190,6 +218,7 @@ files:
|
|
190
218
|
- ".gitignore"
|
191
219
|
- ".rspec"
|
192
220
|
- ".travis.yml"
|
221
|
+
- CHANGELOG.md
|
193
222
|
- Gemfile
|
194
223
|
- LICENSE.txt
|
195
224
|
- README.md
|
@@ -256,4 +285,3 @@ test_files:
|
|
256
285
|
- spec/gush/workflow_spec.rb
|
257
286
|
- spec/gush_spec.rb
|
258
287
|
- spec/spec_helper.rb
|
259
|
-
has_rdoc:
|