parallel_workforce 1.0.0 → 4.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 2faae1c8d652df656ef1a625fb048207f340ef86
-  data.tar.gz: 197ca9d26aca9c1db4ced4d7266974a870978a9f
+SHA256:
+  metadata.gz: 00bb452ed8179ae83eb70606d23616f2bc8b0eb5b0ead9d04a5895e622149cbe
+  data.tar.gz: d9e9876a7f91010ee421921a0ff01aa97ab18ce7703900eb4371347803d137cb
 SHA512:
-  metadata.gz: 86399e51585659405aae9774eac587c493f1f4acd5779077235f89c9eaf41f71a3cd1f7894d3addb97a8afb9f9b9ccc9aeafda0f15f73d6f2e345fda26042ad6
-  data.tar.gz: 10e835470259ae517cf383e19b517e318249ec542bdf1e7e701e197f8706b329069f243877542e7e39cf54559bc2f31ecb38bfa8bd0464f5400f20c21cb6ee83
+  metadata.gz: bdb174cf36984a1028e9ab6faffabe400be1969f49ce5f9b75e3c5271f6821b7f5c9164a2f50930fc3baf44ec6af8080fd9cc3436c3dba13ca65127447e82b08
+  data.tar.gz: 86bbd5676b72aa9b8905d58ddd31a9fdae313d024e9433b62df3a8fac174188ef3f25e3f7c239e6496b2113b8e1593d3e8910eafea6d417dcb7f2b969a439918

data/.gitignore

@@ -6,4 +6,5 @@
 /pkg/
 /spec/reports/
 /tmp/
+/parallel_workforce*.gem
 **/tags

data/.rubocop_parallel_workforce.yml

@@ -42,6 +42,9 @@ RSpec/DescribedClass:
 # RSpec/VerifiedDoubles:
 #   Enabled: true
 
+Layout/FirstParameterIndentation:
+  Enabled: false
+
 Layout/AlignHash:
   Enabled: false
 
@@ -135,9 +138,6 @@ Style/GuardClause:
 Style/IfUnlessModifier:
   Enabled: false
 
-Layout/IndentArray:
-  EnforcedStyle: consistent
-
 Layout/MultilineOperationIndentation:
   Enabled: true
   EnforcedStyle: indented

data/.ruby-version

@@ -1 +1 @@
-ruby-2.3.5
+ruby-2.6.6

data/CHANGELOG.md

@@ -1,3 +1,13 @@
+## 2.0.0
+
+* Ruby version tested is 2.6.6.
+* Include result_values as part of TimeoutError to allow access to partial results that did not timeout.
+* Default to using JsonMarshal serialization that serializes as a JSON string for Sidekiq compataibility.
+
+## 1.0.0
+
+* RubyGems version 1.
+
 ## 0.2.0
 
 * Add optional execution block to `ParallelWorkforce.perform_all` that is evaluated in calling thread after jobs enqueued,

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    parallel_workforce (0.2.0)
+    parallel_workforce (3.0.0)
 
 GEM
   remote: https://rubygems.org/
@@ -146,6 +146,7 @@ GEM
       sprockets (>= 3.0.0)
     thor (0.20.3)
     thread_safe (0.3.6)
+    timecop (0.9.4)
     tzinfo (1.2.5)
       thread_safe (~> 0.1)
     unicode-display_width (1.4.1)
@@ -164,6 +165,7 @@ DEPENDENCIES
   rubocop (~> 0.65)
   rubocop-rspec (~> 1.32)
   sidekiq (~> 4.0)
+  timecop (~> 0.9)
 
 BUNDLED WITH
    1.17.3

data/README.md

@@ -2,7 +2,7 @@
 
 Easily parallelize functionality by partitioning work in either a Sidekiq (preferred) or ActiveJob worker pool.
 
-See more info at the EnjoyTech blog.
+See more info on [why Enjoy created this Gem](https://medium.com/enjoy-engineering/ruby-and-rails-can-be-as-fast-as-you-need-if-you-execute-in-parallel-632f2dff5429) at the [EnjoyTech blog](https://medium.com/enjoy-engineering).
 
 ## Installation
 
@@ -92,7 +92,7 @@ end
 * `logger` - Defaults to `Rails.logger` if `Rails` is loaded.
 * `revision_builder` - Used to determine if the calling thread and worker have a different revision of code. Defaults to a class that builds a hash from the contents of all `.rb` files contained within the current working directory.
 * `serial_mode_checker` - An object with an `execute_serially?` method that allows forcing Actors to execute in calling thread instead of in workers. This can be used to turn off parallel execution globally. Default is `nil`.
-* `serializer` - An object with a `serialize(object)` method that returns a `String` and `deserialize(string)` method that returns an `Object`. Default imlementation uses `Marshal.dump` and `Marshal.load`.
+* `serializer` - An object with a `serialize(object)` method that returns a `String` and `deserialize(string)` method that returns an `Object`. Default imlementation uses `Marshal.dump` and `Marshal.load` and serializes as a JSON string for Sidekiq compatibility.
 * `redis_connector` - An object with a `with` method that yields a Redis connection. Defaults to `ParallelWorkforce::RedisConnector::RedisPool`. If using Sidekiq, it's best to use `ParallelWorkforce::RedisConnector::SidekiqRedisPool`.
 * `job_timeout` - Time allowed to execte a job before timing out. Default is `10` seconds.
 * `job_key_expiration` - Time allowed for result key in Redis to remain before it expires. This should be larger than `job_timeout`. Default is `20` seconds.

data/lib/parallel_workforce/configuration.rb

@@ -18,7 +18,7 @@ module ParallelWorkforce
       @logger = defined?(Rails) && Rails.logger
       @revision_builder = ParallelWorkforce::RevisionBuilder::FilesHash.new
       @serial_mode_checker = nil
-      @serializer = ParallelWorkforce::Serializer::Marshal.new
+      @serializer = ParallelWorkforce::Serializer::JsonMarshal.new
       @redis_connector = ParallelWorkforce::RedisConnector::RedisPool.new
       @job_timeout = 10
       @job_key_expiration = 20

data/lib/parallel_workforce/executor.rb

@@ -180,7 +180,7 @@ module ParallelWorkforce
           Timeout.timeout(configuration.job_timeout) do
             until result_count == num_results
               _key, response = redis.blpop(result_key, configuration.job_timeout)
-              raise ParallelWorkforce::TimeoutError.new("Timeout waiting for Redis#blpop") if response.nil?
+              raise ParallelWorkforce::TimeoutError.new("Timeout waiting for Redis#blpop", result_values) if response.nil?
 
               result_count += 1
 
@@ -190,7 +190,7 @@ module ParallelWorkforce
             end
           end
         rescue Timeout::Error
-          raise ParallelWorkforce::TimeoutError.new("Timeout from ParallelWorkforce job")
+          raise ParallelWorkforce::TimeoutError.new("Timeout from ParallelWorkforce job", result_values)
         end
       end
 

data/lib/parallel_workforce/job/active_job.rb

@@ -1,11 +1,14 @@
 module ParallelWorkforce
   module Job
-    class ActiveJob < ActiveJob::Base
+    class ActiveJob < ::ActiveJob::Base
+      include ParallelWorkforce::Job::Util::JobHelper
+
       discard_on Exception if defined?(discard_on)
 
       class << self
         def enqueue_actor(actor_class_name:, result_key:, index:, server_revision:, serialized_actor_args:)
-          perform_later(
+          enqueue_actor_job(
+            :perform_later,
             actor_class_name: actor_class_name,
             result_key: result_key,
             index: index,
@@ -16,7 +19,7 @@ module ParallelWorkforce
       end
 
       def perform(args)
-        ParallelWorkforce::Job::Util::Performer.new(**args.symbolize_keys).perform
+        invoke_performer(args)
       end
     end
   end

data/lib/parallel_workforce/job/active_job_rails.rb

@@ -3,22 +3,21 @@ module ParallelWorkforce
     class ActiveJobRails < ParallelWorkforce::Job::ActiveJob
       class << self
         def enqueue_actor(actor_class_name:, result_key:, index:, server_revision:, serialized_actor_args:)
-          perform_later(
+          enqueue_actor_job(
+            :perform_later,
             actor_class_name: actor_class_name,
             result_key: result_key,
             index: index,
             server_revision: server_revision,
             serialized_actor_args: serialized_actor_args,
             time_zone_name: Time.zone.name,
+            locale: I18n.locale&.to_s,
           )
         end
       end
 
       def perform(args)
-        args.symbolize_keys!
-        Time.use_zone(args.delete(:time_zone_name)) do
-          ParallelWorkforce::Job::Util::Performer.new(**args).perform
-        end
+        invoke_performer_with_time_zone_name_and_locale(args)
       end
     end
   end

data/lib/parallel_workforce/job/sidekiq.rb

@@ -2,12 +2,14 @@ module ParallelWorkforce
   module Job
     class Sidekiq
       include ::Sidekiq::Worker
+      include ParallelWorkforce::Job::Util::JobHelper
 
       sidekiq_options retry: false
 
       class << self
         def enqueue_actor(actor_class_name:, result_key:, index:, server_revision:, serialized_actor_args:)
-          perform_async(
+          enqueue_actor_job(
+            :perform_async,
             actor_class_name: actor_class_name,
             result_key: result_key,
             index: index,
@@ -18,8 +20,7 @@ module ParallelWorkforce
       end
 
       def perform(args)
-        args = args.each_with_object({}) { |(k, v), result| result[k.to_sym] = v }
-        ParallelWorkforce::Job::Util::Performer.new(**args).perform
+        invoke_performer(args)
       end
     end
   end

data/lib/parallel_workforce/job/sidekiq_rails.rb

@@ -3,22 +3,21 @@ module ParallelWorkforce
     class SidekiqRails < ParallelWorkforce::Job::Sidekiq
       class << self
         def enqueue_actor(actor_class_name:, result_key:, index:, server_revision:, serialized_actor_args:)
-          perform_async(
+          enqueue_actor_job(
+            :perform_async,
             actor_class_name: actor_class_name,
             result_key: result_key,
             index: index,
             server_revision: server_revision,
             serialized_actor_args: serialized_actor_args,
             time_zone_name: Time.zone.name,
+            locale: I18n.locale&.to_s,
           )
         end
       end
 
       def perform(args)
-        args.symbolize_keys!
-        Time.use_zone(args.delete(:time_zone_name)) do
-          ParallelWorkforce::Job::Util::Performer.new(**args).perform
-        end
+        invoke_performer_with_time_zone_name_and_locale(args)
       end
     end
   end

data/lib/parallel_workforce/job/util/job_helper.rb

@@ -0,0 +1,62 @@
+module ParallelWorkforce
+  module Job
+    module Util
+      module JobHelper
+        def self.included(klass)
+          klass.extend(ClassMethods)
+        end
+
+        module ClassMethods
+          def build_serialized_actor_args_key(result_key, index)
+            "#{result_key}:#{index}:serialized-actor-args"
+          end
+
+          def enqueue_actor_job(enqueue_method, **kwargs)
+            serialized_actor_args = kwargs.delete(:serialized_actor_args)
+
+            ::ParallelWorkforce.configuration.redis_connector.with do |redis|
+              redis.setex(
+                build_serialized_actor_args_key(kwargs[:result_key], kwargs[:index]),
+                ::ParallelWorkforce.configuration.job_key_expiration,
+                serialized_actor_args,
+              )
+            end
+
+            send(
+              enqueue_method,
+              **kwargs,
+            )
+          end
+        end
+
+        def invoke_performer(args)
+          args.transform_keys!(&:to_sym)
+
+          serialized_actor_args = ParallelWorkforce.configuration.redis_connector.with do |redis|
+            serialized_actor_args_key = self.class.build_serialized_actor_args_key(args[:result_key], args[:index])
+
+            redis.getset(serialized_actor_args_key, nil).tap do
+              redis.del(serialized_actor_args_key)
+            end
+          end
+
+          raise "Unable to locate serialized data required for Performer" if serialized_actor_args.nil?
+
+          args[:serialized_actor_args] = serialized_actor_args
+
+          ParallelWorkforce::Job::Util::Performer.new(**args).perform
+        end
+
+        def invoke_performer_with_time_zone_name_and_locale(args)
+          args.transform_keys!(&:to_sym)
+
+          Time.use_zone(args.delete(:time_zone_name)) do
+            I18n.with_locale(args.delete(:locale)) do
+              invoke_performer(args)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/parallel_workforce/requires.rb

@@ -2,9 +2,11 @@ require_relative "version"
 require_relative "executor"
 require_relative "configuration"
 require_relative "job/util/performer"
+require_relative "job/util/job_helper"
 require_relative "redis_connector/redis_pool"
 require_relative "revision_builder/files_hash"
 require_relative "serializer/marshal"
+require_relative "serializer/json_marshal"
 
 # rubocop:disable Lint/HandleExceptions
 begin

data/lib/parallel_workforce/serializer/json_marshal.rb

@@ -0,0 +1,17 @@
+module ParallelWorkforce
+  module Serializer
+    class JsonMarshal < Marshal
+      def serialize(object)
+        JSON.dump(value: super(object)) # super always returns a String
+      end
+
+      def deserialize(string)
+        super(JSON.parse(string)['value'])
+      rescue JSON::ParserError => e
+        ParallelWorkforce.log(:warn, "#{self.class}: Unable to deserialize string: #{e}", e, *e.backtrace)
+
+        raise ParallelWorkforce::SerializerError.new("Unable to deserialize string: #{e}")
+      end
+    end
+  end
+end

data/lib/parallel_workforce/version.rb

@@ -1,3 +1,3 @@
 module ParallelWorkforce
-  VERSION = "1.0.0".freeze
+  VERSION = "4.0.0".freeze
 end

data/lib/parallel_workforce.rb

@@ -4,7 +4,15 @@ module ParallelWorkforce
   Error = Class.new(StandardError)
   ActorPerformError = Class.new(Error)
   ActorNotFoundError = Class.new(Error)
-  TimeoutError = Class.new(Error)
+  TimeoutError = Class.new(Error) do
+    attr_reader :result_values
+
+    def initialize(message, result_values=nil)
+      @result_values = result_values
+
+      super(message)
+    end
+  end
   SerializerError = Class.new(Error)
 
   class << self

data/parallel_workforce.gemspec

@@ -44,5 +44,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rubocop", "~> 0.65"
   spec.add_development_dependency "rubocop-rspec", "~> 1.32"
   spec.add_development_dependency "sidekiq", "~> 4.0"
+  spec.add_development_dependency "timecop", "~> 0.9"
 end
 # rubocop:enable Metrics/BlockLength

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: parallel_workforce
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 4.0.0
 platform: ruby
 authors:
 - Michael Pearce
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2019-05-21 00:00:00.000000000 Z
+date: 2022-03-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -136,6 +136,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '4.0'
+- !ruby/object:Gem::Dependency
+  name: timecop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
 description: Simplify parallel code execution into workers.
 email:
 - michael.p@enjoy.com
@@ -163,11 +177,13 @@ files:
 - lib/parallel_workforce/job/active_job_rails.rb
 - lib/parallel_workforce/job/sidekiq.rb
 - lib/parallel_workforce/job/sidekiq_rails.rb
+- lib/parallel_workforce/job/util/job_helper.rb
 - lib/parallel_workforce/job/util/performer.rb
 - lib/parallel_workforce/redis_connector/redis_pool.rb
 - lib/parallel_workforce/redis_connector/sidekiq_redis_pool.rb
 - lib/parallel_workforce/requires.rb
 - lib/parallel_workforce/revision_builder/files_hash.rb
+- lib/parallel_workforce/serializer/json_marshal.rb
 - lib/parallel_workforce/serializer/marshal.rb
 - lib/parallel_workforce/version.rb
 - parallel_workforce.gemspec
@@ -178,7 +194,7 @@ metadata:
   homepage_uri: https://github.com/EnjoyTech/parallel_workforce
   source_code_uri: https://github.com/EnjoyTech/parallel_workforce
   changelog_uri: https://github.com/EnjoyTech/parallel_workforce/CHANGELOG.md
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -193,9 +209,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.5.2.1
-signing_key: 
+rubygems_version: 3.0.9
+signing_key:
 specification_version: 4
 summary: Simplify parallel code execution into workers
 test_files: []