gush 0.3.2 → 0.3.3

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 530121f8210176d74b4ee481c929d123c227c247
4
- data.tar.gz: 6c69a21fdba018faae42f33de3a506ac01c7b881
3
+ metadata.gz: d26693f019dfd49b57cb7828274bec009c96dfb4
4
+ data.tar.gz: 1666a4d1edc877d4ac597428ed56f73a5f625fa9
5
5
  SHA512:
6
- metadata.gz: 353836d226553352646b590ca924df5f6b64f1dcfb1c07e1a677fbf93bd3875fe9f569db3c4f6b9a3817aff4907eaa827178fcdefcaca234668585682b8f36e7
7
- data.tar.gz: 6d8613db3477f98f9e66ff72f873f126fec5fee0a5262aaa37adc163a9bebc785698967fc17764b3ce0393a5ab98d4c4aa6117d689bc4496233ae846d050d0c0
6
+ metadata.gz: b1f55d88bbe1719395103c6eeeda6a9d140d9cd7b6d35296f1ab97b1d381a7f69bbbe83e21faa660a05fe6183d7d5443c006244cbbcc99ebbc58edf554d2adbf
7
+ data.tar.gz: c2d6e5aa2a0613dae3e893ca368e527cac28fe1b9d6df66793de83bfe73dcad7940ee9d0cfd0663e8fd1fcc30eb313c0e465f98da1a350f5958d82beef9faa61
@@ -0,0 +1,3 @@
1
+ # 0.4
2
+
3
+ - remove hard dependency on Yajl, so Gush can work with non-MRI Rubies ([#31](https://github.com/chaps-io/gush/pull/31) by [Nick Rakochy](https://github.com/chaps-io/gush/pull/31))
data/Gemfile CHANGED
@@ -1,8 +1,6 @@
1
1
  source 'https://rubygems.org'
2
-
3
- # Specify your gem's dependencies in gush.gemspec
4
2
  gemspec
5
3
 
6
- gem 'pry'
7
- gem 'yajl-ruby'
8
- gem "fakeredis", require: false
4
+ platforms :mri, :ruby do
5
+ gem 'yajl-ruby'
6
+ end
@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
4
 
5
5
  Gem::Specification.new do |spec|
6
6
  spec.name = "gush"
7
- spec.version = "0.3.2"
7
+ spec.version = "0.3.3"
8
8
  spec.authors = ["Piotrek Okoński"]
9
9
  spec.email = ["piotrek@okonski.org"]
10
10
  spec.summary = "Fast and distributed workflow runner using only Sidekiq and Redis"
@@ -18,7 +18,7 @@ Gem::Specification.new do |spec|
18
18
  spec.require_paths = ["lib"]
19
19
 
20
20
  spec.add_dependency "sidekiq", "~> 4.0"
21
- spec.add_dependency "yajl-ruby", "~> 1.2"
21
+ spec.add_dependency "multi_json", "~> 1.11"
22
22
  spec.add_dependency "redis", "~> 3.2"
23
23
  spec.add_dependency "hiredis", "~> 0.6"
24
24
  spec.add_dependency "ruby-graphviz", "~> 1.2"
@@ -29,4 +29,6 @@ Gem::Specification.new do |spec|
29
29
  spec.add_development_dependency "bundler", "~> 1.5"
30
30
  spec.add_development_dependency "rake", "~> 10.4"
31
31
  spec.add_development_dependency "rspec", '~> 3.0'
32
+ spec.add_development_dependency "pry", '~> 0.10'
33
+ spec.add_development_dependency 'fakeredis', '~> 0.5'
32
34
  end
@@ -6,6 +6,7 @@ require "pathname"
6
6
  require "redis"
7
7
  require "securerandom"
8
8
  require "sidekiq"
9
+ require "multi_json"
9
10
 
10
11
  require "gush/json"
11
12
  require "gush/cli"
@@ -1,17 +1,15 @@
1
1
  module Gush
2
2
  class Client
3
- attr_reader :configuration
3
+ attr_reader :configuration, :sidekiq
4
4
 
5
5
  def initialize(config = Gush.configuration)
6
6
  @configuration = config
7
7
  @sidekiq = build_sidekiq
8
- @redis = build_redis
9
8
  end
10
9
 
11
10
  def configure
12
11
  yield configuration
13
12
  @sidekiq = build_sidekiq
14
- @redis = build_redis
15
13
  end
16
14
 
17
15
  def create_workflow(name)
@@ -49,7 +47,11 @@ module Gush
49
47
  loop do
50
48
  id = SecureRandom.uuid
51
49
  job_identifier = "#{job_klass}-#{id}"
52
- break if !redis.exists("gush.jobs.#{workflow_id}.#{job_identifier}")
50
+ available = connection_pool.with do |redis|
51
+ !redis.exists("gush.jobs.#{workflow_id}.#{job_identifier}")
52
+ end
53
+
54
+ break if available
53
55
  end
54
56
 
55
57
  job_identifier
@@ -59,40 +61,54 @@ module Gush
59
61
  id = nil
60
62
  loop do
61
63
  id = SecureRandom.uuid
62
- break if !redis.exists("gush.workflow.#{id}")
64
+ available = connection_pool.with do |redis|
65
+ !redis.exists("gush.workflow.#{id}")
66
+ end
67
+
68
+ break if available
63
69
  end
64
70
 
65
71
  id
66
72
  end
67
73
 
68
74
  def all_workflows
69
- redis.keys("gush.workflows.*").map do |key|
70
- id = key.sub("gush.workflows.", "")
71
- find_workflow(id)
75
+ connection_pool.with do |redis|
76
+ redis.keys("gush.workflows.*").map do |key|
77
+ id = key.sub("gush.workflows.", "")
78
+ find_workflow(id)
79
+ end
72
80
  end
73
81
  end
74
82
 
75
83
  def find_workflow(id)
76
- data = redis.get("gush.workflows.#{id}")
77
- unless data.nil?
78
- hash = Gush::JSON.decode(data, symbolize_keys: true)
79
- keys = redis.keys("gush.jobs.#{id}.*")
80
- nodes = redis.mget(*keys).map { |json| Gush::JSON.decode(json, symbolize_keys: true) }
81
- workflow_from_hash(hash, nodes)
82
- else
83
- raise WorkflowNotFound.new("Workflow with given id doesn't exist")
84
+ connection_pool.with do |redis|
85
+ data = redis.get("gush.workflows.#{id}")
86
+
87
+ unless data.nil?
88
+ hash = Gush::JSON.decode(data, symbolize_keys: true)
89
+ keys = redis.keys("gush.jobs.#{id}.*")
90
+ nodes = redis.mget(*keys).map { |json| Gush::JSON.decode(json, symbolize_keys: true) }
91
+ workflow_from_hash(hash, nodes)
92
+ else
93
+ raise WorkflowNotFound.new("Workflow with given id doesn't exist")
94
+ end
84
95
  end
85
96
  end
86
97
 
87
98
  def persist_workflow(workflow)
88
- redis.set("gush.workflows.#{workflow.id}", workflow.to_json)
99
+ connection_pool.with do |redis|
100
+ redis.set("gush.workflows.#{workflow.id}", workflow.to_json)
101
+ end
102
+
89
103
  workflow.jobs.each {|job| persist_job(workflow.id, job) }
90
104
  workflow.mark_as_persisted
91
105
  true
92
106
  end
93
107
 
94
108
  def persist_job(workflow_id, job)
95
- redis.set("gush.jobs.#{workflow_id}.#{job.name}", job.to_json)
109
+ connection_pool.with do |redis|
110
+ redis.set("gush.jobs.#{workflow_id}.#{job.name}", job.to_json)
111
+ end
96
112
  end
97
113
 
98
114
  def load_job(workflow_id, job_id)
@@ -100,10 +116,16 @@ module Gush
100
116
  job_name_match = /(?<klass>\w*[^-])-(?<identifier>.*)/.match(job_id)
101
117
  hypen = '-' if job_name_match.nil?
102
118
 
103
- keys = redis.keys("gush.jobs.#{workflow_id}.#{job_id}#{hypen}*")
119
+ keys = connection_pool.with do |redis|
120
+ redis.keys("gush.jobs.#{workflow_id}.#{job_id}#{hypen}*")
121
+ end
122
+
104
123
  return nil if keys.nil?
105
124
 
106
- data = redis.get(keys.first)
125
+ data = connection_pool.with do |redis|
126
+ redis.get(keys.first)
127
+ end
128
+
107
129
  return nil if data.nil?
108
130
 
109
131
  data = Gush::JSON.decode(data, symbolize_keys: true)
@@ -111,12 +133,16 @@ module Gush
111
133
  end
112
134
 
113
135
  def destroy_workflow(workflow)
114
- redis.del("gush.workflows.#{workflow.id}")
136
+ connection_pool.with do |redis|
137
+ redis.del("gush.workflows.#{workflow.id}")
138
+ end
115
139
  workflow.jobs.each {|job| destroy_job(workflow.id, job) }
116
140
  end
117
141
 
118
142
  def destroy_job(workflow_id, job)
119
- redis.del("gush.jobs.#{workflow_id}.#{job.name}")
143
+ connection_pool.with do |redis|
144
+ redis.del("gush.jobs.#{workflow_id}.#{job.name}")
145
+ end
120
146
  end
121
147
 
122
148
  def worker_report(message)
@@ -140,8 +166,6 @@ module Gush
140
166
 
141
167
  private
142
168
 
143
- attr_reader :sidekiq, :redis
144
-
145
169
  def workflow_from_hash(hash, nodes = nil)
146
170
  flow = hash[:klass].constantize.new *hash[:arguments]
147
171
  flow.jobs = []
@@ -156,7 +180,9 @@ module Gush
156
180
  end
157
181
 
158
182
  def report(key, message)
159
- redis.publish(key, Gush::JSON.encode(message))
183
+ connection_pool.with do |redis|
184
+ redis.publish(key, Gush::JSON.encode(message))
185
+ end
160
186
  end
161
187
 
162
188
 
@@ -169,7 +195,7 @@ module Gush
169
195
  end
170
196
 
171
197
  def connection_pool
172
- ConnectionPool.new(size: configuration.concurrency, timeout: 1) { build_redis }
198
+ @connection_pool ||= ConnectionPool.new(size: configuration.concurrency, timeout: 1) { build_redis }
173
199
  end
174
200
  end
175
201
  end
@@ -1,5 +1,3 @@
1
- require 'yajl'
2
-
3
1
  module Gush
4
2
  class Configuration
5
3
  attr_accessor :concurrency, :namespace, :redis_url, :environment
@@ -1,11 +1,11 @@
1
1
  module Gush
2
2
  class JSON
3
3
  def self.encode(data)
4
- Yajl::Encoder.new.encode(data)
4
+ MultiJson.dump(data)
5
5
  end
6
6
 
7
7
  def self.decode(data, options = {})
8
- Yajl::Parser.parse(data, options)
8
+ MultiJson.load(data, options)
9
9
  end
10
10
  end
11
11
  end
@@ -1,5 +1,4 @@
1
1
  require 'sidekiq'
2
- require 'yajl'
3
2
 
4
3
  module Gush
5
4
  class Worker
@@ -1,5 +1,5 @@
1
1
  require 'gush'
2
- require 'pry'
2
+ require 'fakeredis'
3
3
  require 'sidekiq/testing'
4
4
 
5
5
  Sidekiq::Testing.fake!
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: gush
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.2
4
+ version: 0.3.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Piotrek Okoński
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-02-18 00:00:00.000000000 Z
11
+ date: 2016-08-23 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: sidekiq
@@ -25,19 +25,19 @@ dependencies:
25
25
  - !ruby/object:Gem::Version
26
26
  version: '4.0'
27
27
  - !ruby/object:Gem::Dependency
28
- name: yajl-ruby
28
+ name: multi_json
29
29
  requirement: !ruby/object:Gem::Requirement
30
30
  requirements:
31
31
  - - "~>"
32
32
  - !ruby/object:Gem::Version
33
- version: '1.2'
33
+ version: '1.11'
34
34
  type: :runtime
35
35
  prerelease: false
36
36
  version_requirements: !ruby/object:Gem::Requirement
37
37
  requirements:
38
38
  - - "~>"
39
39
  - !ruby/object:Gem::Version
40
- version: '1.2'
40
+ version: '1.11'
41
41
  - !ruby/object:Gem::Dependency
42
42
  name: redis
43
43
  requirement: !ruby/object:Gem::Requirement
@@ -178,6 +178,34 @@ dependencies:
178
178
  - - "~>"
179
179
  - !ruby/object:Gem::Version
180
180
  version: '3.0'
181
+ - !ruby/object:Gem::Dependency
182
+ name: pry
183
+ requirement: !ruby/object:Gem::Requirement
184
+ requirements:
185
+ - - "~>"
186
+ - !ruby/object:Gem::Version
187
+ version: '0.10'
188
+ type: :development
189
+ prerelease: false
190
+ version_requirements: !ruby/object:Gem::Requirement
191
+ requirements:
192
+ - - "~>"
193
+ - !ruby/object:Gem::Version
194
+ version: '0.10'
195
+ - !ruby/object:Gem::Dependency
196
+ name: fakeredis
197
+ requirement: !ruby/object:Gem::Requirement
198
+ requirements:
199
+ - - "~>"
200
+ - !ruby/object:Gem::Version
201
+ version: '0.5'
202
+ type: :development
203
+ prerelease: false
204
+ version_requirements: !ruby/object:Gem::Requirement
205
+ requirements:
206
+ - - "~>"
207
+ - !ruby/object:Gem::Version
208
+ version: '0.5'
181
209
  description: Gush is a parallel workflow runner using only Redis as its message broker
182
210
  and Sidekiq for workers.
183
211
  email:
@@ -190,6 +218,7 @@ files:
190
218
  - ".gitignore"
191
219
  - ".rspec"
192
220
  - ".travis.yml"
221
+ - CHANGELOG.md
193
222
  - Gemfile
194
223
  - LICENSE.txt
195
224
  - README.md
@@ -256,4 +285,3 @@ test_files:
256
285
  - spec/gush/workflow_spec.rb
257
286
  - spec/gush_spec.rb
258
287
  - spec/spec_helper.rb
259
- has_rdoc: