parallel_workforce 1.0.0 → 4.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/.gitignore +1 -0
- data/.rubocop_parallel_workforce.yml +3 -3
- data/.ruby-version +1 -1
- data/CHANGELOG.md +10 -0
- data/Gemfile.lock +3 -1
- data/README.md +2 -2
- data/lib/parallel_workforce/configuration.rb +1 -1
- data/lib/parallel_workforce/executor.rb +2 -2
- data/lib/parallel_workforce/job/active_job.rb +6 -3
- data/lib/parallel_workforce/job/active_job_rails.rb +4 -5
- data/lib/parallel_workforce/job/sidekiq.rb +4 -3
- data/lib/parallel_workforce/job/sidekiq_rails.rb +4 -5
- data/lib/parallel_workforce/job/util/job_helper.rb +62 -0
- data/lib/parallel_workforce/requires.rb +2 -0
- data/lib/parallel_workforce/serializer/json_marshal.rb +17 -0
- data/lib/parallel_workforce/version.rb +1 -1
- data/lib/parallel_workforce.rb +9 -1
- data/parallel_workforce.gemspec +1 -0
- metadata +22 -7
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 00bb452ed8179ae83eb70606d23616f2bc8b0eb5b0ead9d04a5895e622149cbe
|
4
|
+
data.tar.gz: d9e9876a7f91010ee421921a0ff01aa97ab18ce7703900eb4371347803d137cb
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: bdb174cf36984a1028e9ab6faffabe400be1969f49ce5f9b75e3c5271f6821b7f5c9164a2f50930fc3baf44ec6af8080fd9cc3436c3dba13ca65127447e82b08
|
7
|
+
data.tar.gz: 86bbd5676b72aa9b8905d58ddd31a9fdae313d024e9433b62df3a8fac174188ef3f25e3f7c239e6496b2113b8e1593d3e8910eafea6d417dcb7f2b969a439918
|
data/.gitignore
CHANGED
@@ -42,6 +42,9 @@ RSpec/DescribedClass:
|
|
42
42
|
# RSpec/VerifiedDoubles:
|
43
43
|
# Enabled: true
|
44
44
|
|
45
|
+
Layout/FirstParameterIndentation:
|
46
|
+
Enabled: false
|
47
|
+
|
45
48
|
Layout/AlignHash:
|
46
49
|
Enabled: false
|
47
50
|
|
@@ -135,9 +138,6 @@ Style/GuardClause:
|
|
135
138
|
Style/IfUnlessModifier:
|
136
139
|
Enabled: false
|
137
140
|
|
138
|
-
Layout/IndentArray:
|
139
|
-
EnforcedStyle: consistent
|
140
|
-
|
141
141
|
Layout/MultilineOperationIndentation:
|
142
142
|
Enabled: true
|
143
143
|
EnforcedStyle: indented
|
data/.ruby-version
CHANGED
@@ -1 +1 @@
|
|
1
|
-
ruby-2.
|
1
|
+
ruby-2.6.6
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,13 @@
|
|
1
|
+
## 2.0.0
|
2
|
+
|
3
|
+
* Ruby version tested is 2.6.6.
|
4
|
+
* Include result_values as part of TimeoutError to allow access to partial results that did not timeout.
|
5
|
+
* Default to using JsonMarshal serialization that serializes as a JSON string for Sidekiq compataibility.
|
6
|
+
|
7
|
+
## 1.0.0
|
8
|
+
|
9
|
+
* RubyGems version 1.
|
10
|
+
|
1
11
|
## 0.2.0
|
2
12
|
|
3
13
|
* Add optional execution block to `ParallelWorkforce.perform_all` that is evaluated in calling thread after jobs enqueued,
|
data/Gemfile.lock
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
parallel_workforce (0.
|
4
|
+
parallel_workforce (3.0.0)
|
5
5
|
|
6
6
|
GEM
|
7
7
|
remote: https://rubygems.org/
|
@@ -146,6 +146,7 @@ GEM
|
|
146
146
|
sprockets (>= 3.0.0)
|
147
147
|
thor (0.20.3)
|
148
148
|
thread_safe (0.3.6)
|
149
|
+
timecop (0.9.4)
|
149
150
|
tzinfo (1.2.5)
|
150
151
|
thread_safe (~> 0.1)
|
151
152
|
unicode-display_width (1.4.1)
|
@@ -164,6 +165,7 @@ DEPENDENCIES
|
|
164
165
|
rubocop (~> 0.65)
|
165
166
|
rubocop-rspec (~> 1.32)
|
166
167
|
sidekiq (~> 4.0)
|
168
|
+
timecop (~> 0.9)
|
167
169
|
|
168
170
|
BUNDLED WITH
|
169
171
|
1.17.3
|
data/README.md
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
|
3
3
|
Easily parallelize functionality by partitioning work in either a Sidekiq (preferred) or ActiveJob worker pool.
|
4
4
|
|
5
|
-
See more info at the EnjoyTech blog.
|
5
|
+
See more info on [why Enjoy created this Gem](https://medium.com/enjoy-engineering/ruby-and-rails-can-be-as-fast-as-you-need-if-you-execute-in-parallel-632f2dff5429) at the [EnjoyTech blog](https://medium.com/enjoy-engineering).
|
6
6
|
|
7
7
|
## Installation
|
8
8
|
|
@@ -92,7 +92,7 @@ end
|
|
92
92
|
* `logger` - Defaults to `Rails.logger` if `Rails` is loaded.
|
93
93
|
* `revision_builder` - Used to determine if the calling thread and worker have a different revision of code. Defaults to a class that builds a hash from the contents of all `.rb` files contained within the current working directory.
|
94
94
|
* `serial_mode_checker` - An object with an `execute_serially?` method that allows forcing Actors to execute in calling thread instead of in workers. This can be used to turn off parallel execution globally. Default is `nil`.
|
95
|
-
* `serializer` - An object with a `serialize(object)` method that returns a `String` and `deserialize(string)` method that returns an `Object`. Default imlementation uses `Marshal.dump` and `Marshal.load
|
95
|
+
* `serializer` - An object with a `serialize(object)` method that returns a `String` and `deserialize(string)` method that returns an `Object`. Default imlementation uses `Marshal.dump` and `Marshal.load` and serializes as a JSON string for Sidekiq compatibility.
|
96
96
|
* `redis_connector` - An object with a `with` method that yields a Redis connection. Defaults to `ParallelWorkforce::RedisConnector::RedisPool`. If using Sidekiq, it's best to use `ParallelWorkforce::RedisConnector::SidekiqRedisPool`.
|
97
97
|
* `job_timeout` - Time allowed to execte a job before timing out. Default is `10` seconds.
|
98
98
|
* `job_key_expiration` - Time allowed for result key in Redis to remain before it expires. This should be larger than `job_timeout`. Default is `20` seconds.
|
@@ -18,7 +18,7 @@ module ParallelWorkforce
|
|
18
18
|
@logger = defined?(Rails) && Rails.logger
|
19
19
|
@revision_builder = ParallelWorkforce::RevisionBuilder::FilesHash.new
|
20
20
|
@serial_mode_checker = nil
|
21
|
-
@serializer = ParallelWorkforce::Serializer::
|
21
|
+
@serializer = ParallelWorkforce::Serializer::JsonMarshal.new
|
22
22
|
@redis_connector = ParallelWorkforce::RedisConnector::RedisPool.new
|
23
23
|
@job_timeout = 10
|
24
24
|
@job_key_expiration = 20
|
@@ -180,7 +180,7 @@ module ParallelWorkforce
|
|
180
180
|
Timeout.timeout(configuration.job_timeout) do
|
181
181
|
until result_count == num_results
|
182
182
|
_key, response = redis.blpop(result_key, configuration.job_timeout)
|
183
|
-
raise ParallelWorkforce::TimeoutError.new("Timeout waiting for Redis#blpop") if response.nil?
|
183
|
+
raise ParallelWorkforce::TimeoutError.new("Timeout waiting for Redis#blpop", result_values) if response.nil?
|
184
184
|
|
185
185
|
result_count += 1
|
186
186
|
|
@@ -190,7 +190,7 @@ module ParallelWorkforce
|
|
190
190
|
end
|
191
191
|
end
|
192
192
|
rescue Timeout::Error
|
193
|
-
raise ParallelWorkforce::TimeoutError.new("Timeout from ParallelWorkforce job")
|
193
|
+
raise ParallelWorkforce::TimeoutError.new("Timeout from ParallelWorkforce job", result_values)
|
194
194
|
end
|
195
195
|
end
|
196
196
|
|
@@ -1,11 +1,14 @@
|
|
1
1
|
module ParallelWorkforce
|
2
2
|
module Job
|
3
|
-
class ActiveJob < ActiveJob::Base
|
3
|
+
class ActiveJob < ::ActiveJob::Base
|
4
|
+
include ParallelWorkforce::Job::Util::JobHelper
|
5
|
+
|
4
6
|
discard_on Exception if defined?(discard_on)
|
5
7
|
|
6
8
|
class << self
|
7
9
|
def enqueue_actor(actor_class_name:, result_key:, index:, server_revision:, serialized_actor_args:)
|
8
|
-
|
10
|
+
enqueue_actor_job(
|
11
|
+
:perform_later,
|
9
12
|
actor_class_name: actor_class_name,
|
10
13
|
result_key: result_key,
|
11
14
|
index: index,
|
@@ -16,7 +19,7 @@ module ParallelWorkforce
|
|
16
19
|
end
|
17
20
|
|
18
21
|
def perform(args)
|
19
|
-
|
22
|
+
invoke_performer(args)
|
20
23
|
end
|
21
24
|
end
|
22
25
|
end
|
@@ -3,22 +3,21 @@ module ParallelWorkforce
|
|
3
3
|
class ActiveJobRails < ParallelWorkforce::Job::ActiveJob
|
4
4
|
class << self
|
5
5
|
def enqueue_actor(actor_class_name:, result_key:, index:, server_revision:, serialized_actor_args:)
|
6
|
-
|
6
|
+
enqueue_actor_job(
|
7
|
+
:perform_later,
|
7
8
|
actor_class_name: actor_class_name,
|
8
9
|
result_key: result_key,
|
9
10
|
index: index,
|
10
11
|
server_revision: server_revision,
|
11
12
|
serialized_actor_args: serialized_actor_args,
|
12
13
|
time_zone_name: Time.zone.name,
|
14
|
+
locale: I18n.locale&.to_s,
|
13
15
|
)
|
14
16
|
end
|
15
17
|
end
|
16
18
|
|
17
19
|
def perform(args)
|
18
|
-
args
|
19
|
-
Time.use_zone(args.delete(:time_zone_name)) do
|
20
|
-
ParallelWorkforce::Job::Util::Performer.new(**args).perform
|
21
|
-
end
|
20
|
+
invoke_performer_with_time_zone_name_and_locale(args)
|
22
21
|
end
|
23
22
|
end
|
24
23
|
end
|
@@ -2,12 +2,14 @@ module ParallelWorkforce
|
|
2
2
|
module Job
|
3
3
|
class Sidekiq
|
4
4
|
include ::Sidekiq::Worker
|
5
|
+
include ParallelWorkforce::Job::Util::JobHelper
|
5
6
|
|
6
7
|
sidekiq_options retry: false
|
7
8
|
|
8
9
|
class << self
|
9
10
|
def enqueue_actor(actor_class_name:, result_key:, index:, server_revision:, serialized_actor_args:)
|
10
|
-
|
11
|
+
enqueue_actor_job(
|
12
|
+
:perform_async,
|
11
13
|
actor_class_name: actor_class_name,
|
12
14
|
result_key: result_key,
|
13
15
|
index: index,
|
@@ -18,8 +20,7 @@ module ParallelWorkforce
|
|
18
20
|
end
|
19
21
|
|
20
22
|
def perform(args)
|
21
|
-
args
|
22
|
-
ParallelWorkforce::Job::Util::Performer.new(**args).perform
|
23
|
+
invoke_performer(args)
|
23
24
|
end
|
24
25
|
end
|
25
26
|
end
|
@@ -3,22 +3,21 @@ module ParallelWorkforce
|
|
3
3
|
class SidekiqRails < ParallelWorkforce::Job::Sidekiq
|
4
4
|
class << self
|
5
5
|
def enqueue_actor(actor_class_name:, result_key:, index:, server_revision:, serialized_actor_args:)
|
6
|
-
|
6
|
+
enqueue_actor_job(
|
7
|
+
:perform_async,
|
7
8
|
actor_class_name: actor_class_name,
|
8
9
|
result_key: result_key,
|
9
10
|
index: index,
|
10
11
|
server_revision: server_revision,
|
11
12
|
serialized_actor_args: serialized_actor_args,
|
12
13
|
time_zone_name: Time.zone.name,
|
14
|
+
locale: I18n.locale&.to_s,
|
13
15
|
)
|
14
16
|
end
|
15
17
|
end
|
16
18
|
|
17
19
|
def perform(args)
|
18
|
-
args
|
19
|
-
Time.use_zone(args.delete(:time_zone_name)) do
|
20
|
-
ParallelWorkforce::Job::Util::Performer.new(**args).perform
|
21
|
-
end
|
20
|
+
invoke_performer_with_time_zone_name_and_locale(args)
|
22
21
|
end
|
23
22
|
end
|
24
23
|
end
|
@@ -0,0 +1,62 @@
|
|
1
|
+
module ParallelWorkforce
|
2
|
+
module Job
|
3
|
+
module Util
|
4
|
+
module JobHelper
|
5
|
+
def self.included(klass)
|
6
|
+
klass.extend(ClassMethods)
|
7
|
+
end
|
8
|
+
|
9
|
+
module ClassMethods
|
10
|
+
def build_serialized_actor_args_key(result_key, index)
|
11
|
+
"#{result_key}:#{index}:serialized-actor-args"
|
12
|
+
end
|
13
|
+
|
14
|
+
def enqueue_actor_job(enqueue_method, **kwargs)
|
15
|
+
serialized_actor_args = kwargs.delete(:serialized_actor_args)
|
16
|
+
|
17
|
+
::ParallelWorkforce.configuration.redis_connector.with do |redis|
|
18
|
+
redis.setex(
|
19
|
+
build_serialized_actor_args_key(kwargs[:result_key], kwargs[:index]),
|
20
|
+
::ParallelWorkforce.configuration.job_key_expiration,
|
21
|
+
serialized_actor_args,
|
22
|
+
)
|
23
|
+
end
|
24
|
+
|
25
|
+
send(
|
26
|
+
enqueue_method,
|
27
|
+
**kwargs,
|
28
|
+
)
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
def invoke_performer(args)
|
33
|
+
args.transform_keys!(&:to_sym)
|
34
|
+
|
35
|
+
serialized_actor_args = ParallelWorkforce.configuration.redis_connector.with do |redis|
|
36
|
+
serialized_actor_args_key = self.class.build_serialized_actor_args_key(args[:result_key], args[:index])
|
37
|
+
|
38
|
+
redis.getset(serialized_actor_args_key, nil).tap do
|
39
|
+
redis.del(serialized_actor_args_key)
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
raise "Unable to locate serialized data required for Performer" if serialized_actor_args.nil?
|
44
|
+
|
45
|
+
args[:serialized_actor_args] = serialized_actor_args
|
46
|
+
|
47
|
+
ParallelWorkforce::Job::Util::Performer.new(**args).perform
|
48
|
+
end
|
49
|
+
|
50
|
+
def invoke_performer_with_time_zone_name_and_locale(args)
|
51
|
+
args.transform_keys!(&:to_sym)
|
52
|
+
|
53
|
+
Time.use_zone(args.delete(:time_zone_name)) do
|
54
|
+
I18n.with_locale(args.delete(:locale)) do
|
55
|
+
invoke_performer(args)
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
62
|
+
end
|
@@ -2,9 +2,11 @@ require_relative "version"
|
|
2
2
|
require_relative "executor"
|
3
3
|
require_relative "configuration"
|
4
4
|
require_relative "job/util/performer"
|
5
|
+
require_relative "job/util/job_helper"
|
5
6
|
require_relative "redis_connector/redis_pool"
|
6
7
|
require_relative "revision_builder/files_hash"
|
7
8
|
require_relative "serializer/marshal"
|
9
|
+
require_relative "serializer/json_marshal"
|
8
10
|
|
9
11
|
# rubocop:disable Lint/HandleExceptions
|
10
12
|
begin
|
@@ -0,0 +1,17 @@
|
|
1
|
+
module ParallelWorkforce
|
2
|
+
module Serializer
|
3
|
+
class JsonMarshal < Marshal
|
4
|
+
def serialize(object)
|
5
|
+
JSON.dump(value: super(object)) # super always returns a String
|
6
|
+
end
|
7
|
+
|
8
|
+
def deserialize(string)
|
9
|
+
super(JSON.parse(string)['value'])
|
10
|
+
rescue JSON::ParserError => e
|
11
|
+
ParallelWorkforce.log(:warn, "#{self.class}: Unable to deserialize string: #{e}", e, *e.backtrace)
|
12
|
+
|
13
|
+
raise ParallelWorkforce::SerializerError.new("Unable to deserialize string: #{e}")
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
data/lib/parallel_workforce.rb
CHANGED
@@ -4,7 +4,15 @@ module ParallelWorkforce
|
|
4
4
|
Error = Class.new(StandardError)
|
5
5
|
ActorPerformError = Class.new(Error)
|
6
6
|
ActorNotFoundError = Class.new(Error)
|
7
|
-
TimeoutError = Class.new(Error)
|
7
|
+
TimeoutError = Class.new(Error) do
|
8
|
+
attr_reader :result_values
|
9
|
+
|
10
|
+
def initialize(message, result_values=nil)
|
11
|
+
@result_values = result_values
|
12
|
+
|
13
|
+
super(message)
|
14
|
+
end
|
15
|
+
end
|
8
16
|
SerializerError = Class.new(Error)
|
9
17
|
|
10
18
|
class << self
|
data/parallel_workforce.gemspec
CHANGED
@@ -44,5 +44,6 @@ Gem::Specification.new do |spec|
|
|
44
44
|
spec.add_development_dependency "rubocop", "~> 0.65"
|
45
45
|
spec.add_development_dependency "rubocop-rspec", "~> 1.32"
|
46
46
|
spec.add_development_dependency "sidekiq", "~> 4.0"
|
47
|
+
spec.add_development_dependency "timecop", "~> 0.9"
|
47
48
|
end
|
48
49
|
# rubocop:enable Metrics/BlockLength
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: parallel_workforce
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 4.0.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Michael Pearce
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2022-03-29 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -136,6 +136,20 @@ dependencies:
|
|
136
136
|
- - "~>"
|
137
137
|
- !ruby/object:Gem::Version
|
138
138
|
version: '4.0'
|
139
|
+
- !ruby/object:Gem::Dependency
|
140
|
+
name: timecop
|
141
|
+
requirement: !ruby/object:Gem::Requirement
|
142
|
+
requirements:
|
143
|
+
- - "~>"
|
144
|
+
- !ruby/object:Gem::Version
|
145
|
+
version: '0.9'
|
146
|
+
type: :development
|
147
|
+
prerelease: false
|
148
|
+
version_requirements: !ruby/object:Gem::Requirement
|
149
|
+
requirements:
|
150
|
+
- - "~>"
|
151
|
+
- !ruby/object:Gem::Version
|
152
|
+
version: '0.9'
|
139
153
|
description: Simplify parallel code execution into workers.
|
140
154
|
email:
|
141
155
|
- michael.p@enjoy.com
|
@@ -163,11 +177,13 @@ files:
|
|
163
177
|
- lib/parallel_workforce/job/active_job_rails.rb
|
164
178
|
- lib/parallel_workforce/job/sidekiq.rb
|
165
179
|
- lib/parallel_workforce/job/sidekiq_rails.rb
|
180
|
+
- lib/parallel_workforce/job/util/job_helper.rb
|
166
181
|
- lib/parallel_workforce/job/util/performer.rb
|
167
182
|
- lib/parallel_workforce/redis_connector/redis_pool.rb
|
168
183
|
- lib/parallel_workforce/redis_connector/sidekiq_redis_pool.rb
|
169
184
|
- lib/parallel_workforce/requires.rb
|
170
185
|
- lib/parallel_workforce/revision_builder/files_hash.rb
|
186
|
+
- lib/parallel_workforce/serializer/json_marshal.rb
|
171
187
|
- lib/parallel_workforce/serializer/marshal.rb
|
172
188
|
- lib/parallel_workforce/version.rb
|
173
189
|
- parallel_workforce.gemspec
|
@@ -178,7 +194,7 @@ metadata:
|
|
178
194
|
homepage_uri: https://github.com/EnjoyTech/parallel_workforce
|
179
195
|
source_code_uri: https://github.com/EnjoyTech/parallel_workforce
|
180
196
|
changelog_uri: https://github.com/EnjoyTech/parallel_workforce/CHANGELOG.md
|
181
|
-
post_install_message:
|
197
|
+
post_install_message:
|
182
198
|
rdoc_options: []
|
183
199
|
require_paths:
|
184
200
|
- lib
|
@@ -193,9 +209,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
193
209
|
- !ruby/object:Gem::Version
|
194
210
|
version: '0'
|
195
211
|
requirements: []
|
196
|
-
|
197
|
-
|
198
|
-
signing_key:
|
212
|
+
rubygems_version: 3.0.9
|
213
|
+
signing_key:
|
199
214
|
specification_version: 4
|
200
215
|
summary: Simplify parallel code execution into workers
|
201
216
|
test_files: []
|