gush 0.3.2 → 0.3.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 530121f8210176d74b4ee481c929d123c227c247
4
- data.tar.gz: 6c69a21fdba018faae42f33de3a506ac01c7b881
3
+ metadata.gz: d26693f019dfd49b57cb7828274bec009c96dfb4
4
+ data.tar.gz: 1666a4d1edc877d4ac597428ed56f73a5f625fa9
5
5
  SHA512:
6
- metadata.gz: 353836d226553352646b590ca924df5f6b64f1dcfb1c07e1a677fbf93bd3875fe9f569db3c4f6b9a3817aff4907eaa827178fcdefcaca234668585682b8f36e7
7
- data.tar.gz: 6d8613db3477f98f9e66ff72f873f126fec5fee0a5262aaa37adc163a9bebc785698967fc17764b3ce0393a5ab98d4c4aa6117d689bc4496233ae846d050d0c0
6
+ metadata.gz: b1f55d88bbe1719395103c6eeeda6a9d140d9cd7b6d35296f1ab97b1d381a7f69bbbe83e21faa660a05fe6183d7d5443c006244cbbcc99ebbc58edf554d2adbf
7
+ data.tar.gz: c2d6e5aa2a0613dae3e893ca368e527cac28fe1b9d6df66793de83bfe73dcad7940ee9d0cfd0663e8fd1fcc30eb313c0e465f98da1a350f5958d82beef9faa61
@@ -0,0 +1,3 @@
1
+ # 0.4
2
+
3
+ - remove hard dependency on Yajl, so Gush can work with non-MRI Rubies ([#31](https://github.com/chaps-io/gush/pull/31) by [Nick Rakochy](https://github.com/chaps-io/gush/pull/31))
data/Gemfile CHANGED
@@ -1,8 +1,6 @@
1
1
  source 'https://rubygems.org'
2
-
3
- # Specify your gem's dependencies in gush.gemspec
4
2
  gemspec
5
3
 
6
- gem 'pry'
7
- gem 'yajl-ruby'
8
- gem "fakeredis", require: false
4
+ platforms :mri, :ruby do
5
+ gem 'yajl-ruby'
6
+ end
@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
4
 
5
5
  Gem::Specification.new do |spec|
6
6
  spec.name = "gush"
7
- spec.version = "0.3.2"
7
+ spec.version = "0.3.3"
8
8
  spec.authors = ["Piotrek Okoński"]
9
9
  spec.email = ["piotrek@okonski.org"]
10
10
  spec.summary = "Fast and distributed workflow runner using only Sidekiq and Redis"
@@ -18,7 +18,7 @@ Gem::Specification.new do |spec|
18
18
  spec.require_paths = ["lib"]
19
19
 
20
20
  spec.add_dependency "sidekiq", "~> 4.0"
21
- spec.add_dependency "yajl-ruby", "~> 1.2"
21
+ spec.add_dependency "multi_json", "~> 1.11"
22
22
  spec.add_dependency "redis", "~> 3.2"
23
23
  spec.add_dependency "hiredis", "~> 0.6"
24
24
  spec.add_dependency "ruby-graphviz", "~> 1.2"
@@ -29,4 +29,6 @@ Gem::Specification.new do |spec|
29
29
  spec.add_development_dependency "bundler", "~> 1.5"
30
30
  spec.add_development_dependency "rake", "~> 10.4"
31
31
  spec.add_development_dependency "rspec", '~> 3.0'
32
+ spec.add_development_dependency "pry", '~> 0.10'
33
+ spec.add_development_dependency 'fakeredis', '~> 0.5'
32
34
  end
@@ -6,6 +6,7 @@ require "pathname"
6
6
  require "redis"
7
7
  require "securerandom"
8
8
  require "sidekiq"
9
+ require "multi_json"
9
10
 
10
11
  require "gush/json"
11
12
  require "gush/cli"
@@ -1,17 +1,15 @@
1
1
  module Gush
2
2
  class Client
3
- attr_reader :configuration
3
+ attr_reader :configuration, :sidekiq
4
4
 
5
5
  def initialize(config = Gush.configuration)
6
6
  @configuration = config
7
7
  @sidekiq = build_sidekiq
8
- @redis = build_redis
9
8
  end
10
9
 
11
10
  def configure
12
11
  yield configuration
13
12
  @sidekiq = build_sidekiq
14
- @redis = build_redis
15
13
  end
16
14
 
17
15
  def create_workflow(name)
@@ -49,7 +47,11 @@ module Gush
49
47
  loop do
50
48
  id = SecureRandom.uuid
51
49
  job_identifier = "#{job_klass}-#{id}"
52
- break if !redis.exists("gush.jobs.#{workflow_id}.#{job_identifier}")
50
+ available = connection_pool.with do |redis|
51
+ !redis.exists("gush.jobs.#{workflow_id}.#{job_identifier}")
52
+ end
53
+
54
+ break if available
53
55
  end
54
56
 
55
57
  job_identifier
@@ -59,40 +61,54 @@ module Gush
59
61
  id = nil
60
62
  loop do
61
63
  id = SecureRandom.uuid
62
- break if !redis.exists("gush.workflow.#{id}")
64
+ available = connection_pool.with do |redis|
65
+ !redis.exists("gush.workflow.#{id}")
66
+ end
67
+
68
+ break if available
63
69
  end
64
70
 
65
71
  id
66
72
  end
67
73
 
68
74
  def all_workflows
69
- redis.keys("gush.workflows.*").map do |key|
70
- id = key.sub("gush.workflows.", "")
71
- find_workflow(id)
75
+ connection_pool.with do |redis|
76
+ redis.keys("gush.workflows.*").map do |key|
77
+ id = key.sub("gush.workflows.", "")
78
+ find_workflow(id)
79
+ end
72
80
  end
73
81
  end
74
82
 
75
83
  def find_workflow(id)
76
- data = redis.get("gush.workflows.#{id}")
77
- unless data.nil?
78
- hash = Gush::JSON.decode(data, symbolize_keys: true)
79
- keys = redis.keys("gush.jobs.#{id}.*")
80
- nodes = redis.mget(*keys).map { |json| Gush::JSON.decode(json, symbolize_keys: true) }
81
- workflow_from_hash(hash, nodes)
82
- else
83
- raise WorkflowNotFound.new("Workflow with given id doesn't exist")
84
+ connection_pool.with do |redis|
85
+ data = redis.get("gush.workflows.#{id}")
86
+
87
+ unless data.nil?
88
+ hash = Gush::JSON.decode(data, symbolize_keys: true)
89
+ keys = redis.keys("gush.jobs.#{id}.*")
90
+ nodes = redis.mget(*keys).map { |json| Gush::JSON.decode(json, symbolize_keys: true) }
91
+ workflow_from_hash(hash, nodes)
92
+ else
93
+ raise WorkflowNotFound.new("Workflow with given id doesn't exist")
94
+ end
84
95
  end
85
96
  end
86
97
 
87
98
  def persist_workflow(workflow)
88
- redis.set("gush.workflows.#{workflow.id}", workflow.to_json)
99
+ connection_pool.with do |redis|
100
+ redis.set("gush.workflows.#{workflow.id}", workflow.to_json)
101
+ end
102
+
89
103
  workflow.jobs.each {|job| persist_job(workflow.id, job) }
90
104
  workflow.mark_as_persisted
91
105
  true
92
106
  end
93
107
 
94
108
  def persist_job(workflow_id, job)
95
- redis.set("gush.jobs.#{workflow_id}.#{job.name}", job.to_json)
109
+ connection_pool.with do |redis|
110
+ redis.set("gush.jobs.#{workflow_id}.#{job.name}", job.to_json)
111
+ end
96
112
  end
97
113
 
98
114
  def load_job(workflow_id, job_id)
@@ -100,10 +116,16 @@ module Gush
100
116
  job_name_match = /(?<klass>\w*[^-])-(?<identifier>.*)/.match(job_id)
101
117
  hypen = '-' if job_name_match.nil?
102
118
 
103
- keys = redis.keys("gush.jobs.#{workflow_id}.#{job_id}#{hypen}*")
119
+ keys = connection_pool.with do |redis|
120
+ redis.keys("gush.jobs.#{workflow_id}.#{job_id}#{hypen}*")
121
+ end
122
+
104
123
  return nil if keys.nil?
105
124
 
106
- data = redis.get(keys.first)
125
+ data = connection_pool.with do |redis|
126
+ redis.get(keys.first)
127
+ end
128
+
107
129
  return nil if data.nil?
108
130
 
109
131
  data = Gush::JSON.decode(data, symbolize_keys: true)
@@ -111,12 +133,16 @@ module Gush
111
133
  end
112
134
 
113
135
  def destroy_workflow(workflow)
114
- redis.del("gush.workflows.#{workflow.id}")
136
+ connection_pool.with do |redis|
137
+ redis.del("gush.workflows.#{workflow.id}")
138
+ end
115
139
  workflow.jobs.each {|job| destroy_job(workflow.id, job) }
116
140
  end
117
141
 
118
142
  def destroy_job(workflow_id, job)
119
- redis.del("gush.jobs.#{workflow_id}.#{job.name}")
143
+ connection_pool.with do |redis|
144
+ redis.del("gush.jobs.#{workflow_id}.#{job.name}")
145
+ end
120
146
  end
121
147
 
122
148
  def worker_report(message)
@@ -140,8 +166,6 @@ module Gush
140
166
 
141
167
  private
142
168
 
143
- attr_reader :sidekiq, :redis
144
-
145
169
  def workflow_from_hash(hash, nodes = nil)
146
170
  flow = hash[:klass].constantize.new *hash[:arguments]
147
171
  flow.jobs = []
@@ -156,7 +180,9 @@ module Gush
156
180
  end
157
181
 
158
182
  def report(key, message)
159
- redis.publish(key, Gush::JSON.encode(message))
183
+ connection_pool.with do |redis|
184
+ redis.publish(key, Gush::JSON.encode(message))
185
+ end
160
186
  end
161
187
 
162
188
 
@@ -169,7 +195,7 @@ module Gush
169
195
  end
170
196
 
171
197
  def connection_pool
172
- ConnectionPool.new(size: configuration.concurrency, timeout: 1) { build_redis }
198
+ @connection_pool ||= ConnectionPool.new(size: configuration.concurrency, timeout: 1) { build_redis }
173
199
  end
174
200
  end
175
201
  end
@@ -1,5 +1,3 @@
1
- require 'yajl'
2
-
3
1
  module Gush
4
2
  class Configuration
5
3
  attr_accessor :concurrency, :namespace, :redis_url, :environment
@@ -1,11 +1,11 @@
1
1
  module Gush
2
2
  class JSON
3
3
  def self.encode(data)
4
- Yajl::Encoder.new.encode(data)
4
+ MultiJson.dump(data)
5
5
  end
6
6
 
7
7
  def self.decode(data, options = {})
8
- Yajl::Parser.parse(data, options)
8
+ MultiJson.load(data, options)
9
9
  end
10
10
  end
11
11
  end
@@ -1,5 +1,4 @@
1
1
  require 'sidekiq'
2
- require 'yajl'
3
2
 
4
3
  module Gush
5
4
  class Worker
@@ -1,5 +1,5 @@
1
1
  require 'gush'
2
- require 'pry'
2
+ require 'fakeredis'
3
3
  require 'sidekiq/testing'
4
4
 
5
5
  Sidekiq::Testing.fake!
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: gush
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.2
4
+ version: 0.3.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Piotrek Okoński
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-02-18 00:00:00.000000000 Z
11
+ date: 2016-08-23 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: sidekiq
@@ -25,19 +25,19 @@ dependencies:
25
25
  - !ruby/object:Gem::Version
26
26
  version: '4.0'
27
27
  - !ruby/object:Gem::Dependency
28
- name: yajl-ruby
28
+ name: multi_json
29
29
  requirement: !ruby/object:Gem::Requirement
30
30
  requirements:
31
31
  - - "~>"
32
32
  - !ruby/object:Gem::Version
33
- version: '1.2'
33
+ version: '1.11'
34
34
  type: :runtime
35
35
  prerelease: false
36
36
  version_requirements: !ruby/object:Gem::Requirement
37
37
  requirements:
38
38
  - - "~>"
39
39
  - !ruby/object:Gem::Version
40
- version: '1.2'
40
+ version: '1.11'
41
41
  - !ruby/object:Gem::Dependency
42
42
  name: redis
43
43
  requirement: !ruby/object:Gem::Requirement
@@ -178,6 +178,34 @@ dependencies:
178
178
  - - "~>"
179
179
  - !ruby/object:Gem::Version
180
180
  version: '3.0'
181
+ - !ruby/object:Gem::Dependency
182
+ name: pry
183
+ requirement: !ruby/object:Gem::Requirement
184
+ requirements:
185
+ - - "~>"
186
+ - !ruby/object:Gem::Version
187
+ version: '0.10'
188
+ type: :development
189
+ prerelease: false
190
+ version_requirements: !ruby/object:Gem::Requirement
191
+ requirements:
192
+ - - "~>"
193
+ - !ruby/object:Gem::Version
194
+ version: '0.10'
195
+ - !ruby/object:Gem::Dependency
196
+ name: fakeredis
197
+ requirement: !ruby/object:Gem::Requirement
198
+ requirements:
199
+ - - "~>"
200
+ - !ruby/object:Gem::Version
201
+ version: '0.5'
202
+ type: :development
203
+ prerelease: false
204
+ version_requirements: !ruby/object:Gem::Requirement
205
+ requirements:
206
+ - - "~>"
207
+ - !ruby/object:Gem::Version
208
+ version: '0.5'
181
209
  description: Gush is a parallel workflow runner using only Redis as its message broker
182
210
  and Sidekiq for workers.
183
211
  email:
@@ -190,6 +218,7 @@ files:
190
218
  - ".gitignore"
191
219
  - ".rspec"
192
220
  - ".travis.yml"
221
+ - CHANGELOG.md
193
222
  - Gemfile
194
223
  - LICENSE.txt
195
224
  - README.md
@@ -256,4 +285,3 @@ test_files:
256
285
  - spec/gush/workflow_spec.rb
257
286
  - spec/gush_spec.rb
258
287
  - spec/spec_helper.rb
259
- has_rdoc: