parallel_workforce 1.0.0

data/lib/parallel_workforce/job/sidekiq_rails.rb

@@ -0,0 +1,25 @@
+module ParallelWorkforce
+  module Job
+    class SidekiqRails < ParallelWorkforce::Job::Sidekiq
+      class << self
+        def enqueue_actor(actor_class_name:, result_key:, index:, server_revision:, serialized_actor_args:)
+          perform_async(
+            actor_class_name: actor_class_name,
+            result_key: result_key,
+            index: index,
+            server_revision: server_revision,
+            serialized_actor_args: serialized_actor_args,
+            time_zone_name: Time.zone.name,
+          )
+        end
+      end
+
+      def perform(args)
+        args.symbolize_keys!
+        Time.use_zone(args.delete(:time_zone_name)) do
+          ParallelWorkforce::Job::Util::Performer.new(**args).perform
+        end
+      end
+    end
+  end
+end

data/lib/parallel_workforce/job/util/performer.rb

@@ -0,0 +1,120 @@
+module ParallelWorkforce
+  module Job
+    module Util
+      class Performer
+        attr_reader(
+          :actor_class_name,
+          :result_key,
+          :index,
+          :server_revision,
+          :serialized_actor_args,
+        )
+
+        class << self
+          def perform(actor_class, actor_args)
+            actor_args = actor_args.each_with_object({}) { |(k, v), result| result[k.to_sym] = v }
+
+            (actor_args.empty? ? actor_class.new : actor_class.new(**actor_args)).perform
+          end
+
+          def parallel_workforce_thread?
+            !!Thread.current[:parallel_workforce_thread]
+          end
+        end
+
+        def initialize(actor_class_name:, result_key:, index:, server_revision:, serialized_actor_args:)
+          @actor_class_name = actor_class_name
+          @result_key = result_key
+          @index = index
+          @server_revision = server_revision
+          @serialized_actor_args = serialized_actor_args
+        end
+
+        # rubocop:disable Metrics/MethodLength
+        # rubocop:disable Metrics/AbcSize
+        def perform
+          result = {}
+
+          if server_revision == worker_revision
+            begin
+              result[:serialized_value] = serialize(perform_actor)
+            rescue ParallelWorkforce::Error => e
+              warn("#{self.class}: Actor revision error: #{e}")
+              result[:error_revision] = worker_revision
+            rescue => e
+              warn("#{self.class}: Error in actor perform: #{e}", *e.backtrace)
+              result[:error] = "Error in actor perform. #{e.class} #{e.message}"
+            end
+          else
+            warn("#{self.class}: Revision mismatch from caller")
+            result[:error_revision] = worker_revision
+          end
+
+          result
+        rescue Exception => exception
+          result = handle_exception(exception)
+        ensure
+          ParallelWorkforce.configuration.redis_connector.with do |redis|
+            # always publish a message result to avoid a Timeout in subscriber
+            # NOTE: always using Ruby marshaling to store in Redis, not serializer
+            redis.rpush(result_key, Marshal.dump(result.merge(index: index)))
+          end
+        end
+        # rubocop:enable Metrics/AbcSize
+        # rubocop:enable Metrics/MethodLength
+
+        private
+
+        def perform_actor
+          actor_class = begin
+            Object.const_get(actor_class_name)
+          rescue NameError
+            raise ParallelWorkforce::SerializerError.new("Unable to locate actor class: #{actor_class_name}")
+          end
+
+          in_parallel_workforce_thread do
+            self.class.perform(actor_class, deserialize(serialized_actor_args))
+          end
+        end
+
+        def in_parallel_workforce_thread(&block)
+          original_parallel_workforce_thread = Thread.current[:parallel_workforce_thread]
+
+          Thread.current[:parallel_workforce_thread] = true
+          yield
+        ensure
+          Thread.current[:parallel_workforce_thread] = original_parallel_workforce_thread
+        end
+
+        def serialize(object)
+          ParallelWorkforce.configuration.serializer.serialize(object)
+        end
+
+        def deserialize(string)
+          ParallelWorkforce.configuration.serializer.deserialize(string)
+        end
+
+        def worker_revision
+          ParallelWorkforce.configuration.revision_builder&.revision
+        end
+
+        def warn(*messages)
+          ParallelWorkforce.log(:warn, *messages)
+        end
+
+        def error(*messages)
+          ParallelWorkforce.log(:error, *messages)
+        end
+
+        def handle_exception(exception)
+          # swallow exceptions, no need to retry
+          error("#{self.class}: #{exception}", *exception.backtrace)
+
+          {
+            error: "Unhandled exception. #{exception.class} #{exception.message}",
+          }
+        end
+      end
+    end
+  end
+end

data/lib/parallel_workforce/redis_connector/redis_pool.rb

@@ -0,0 +1,11 @@
+module ParallelWorkforce
+  module RedisConnector
+    class RedisPool
+      def with(&block)
+        redis = (Thread.current["#{self.class.name}:redis_connection"] ||= Redis.new)
+
+        yield(redis)
+      end
+    end
+  end
+end

data/lib/parallel_workforce/redis_connector/sidekiq_redis_pool.rb

@@ -0,0 +1,11 @@
+module ParallelWorkforce
+  module RedisConnector
+    class SidekiqRedisPool
+      def with(&block)
+        Sidekiq.redis_pool.with do |redis|
+          yield(redis)
+        end
+      end
+    end
+  end
+end

data/lib/parallel_workforce/requires.rb

@@ -0,0 +1,37 @@
+require_relative "version"
+require_relative "executor"
+require_relative "configuration"
+require_relative "job/util/performer"
+require_relative "redis_connector/redis_pool"
+require_relative "revision_builder/files_hash"
+require_relative "serializer/marshal"
+
+# rubocop:disable Lint/HandleExceptions
+begin
+  require 'active_job'
+rescue LoadError
+end
+begin
+  require 'rails'
+rescue LoadError
+end
+begin
+  require 'sidekiq'
+rescue LoadError
+end
+# rubocop:enable Lint/HandleExceptions
+
+if defined?(::ActiveJob)
+  require_relative 'job/active_job'
+  if defined?(::Rails)
+    require_relative 'job/active_job_rails'
+  end
+end
+
+if defined?(::Sidekiq)
+  require_relative 'redis_connector/sidekiq_redis_pool'
+  require_relative 'job/sidekiq'
+  if defined?(::Rails)
+    require_relative 'job/sidekiq_rails'
+  end
+end

data/lib/parallel_workforce/revision_builder/files_hash.rb

@@ -0,0 +1,19 @@
+module ParallelWorkforce
+  module RevisionBuilder
+    class FilesHash
+      attr_reader :revision
+
+      def initialize(files=Dir["#{File.expand_path('.')}/**/*.rb"])
+        @revision = build_revision(files).freeze
+      end
+
+      protected
+
+      def build_revision(files)
+        Digest::MD5.new.tap do |digest|
+          files.sort.each { |file| digest << File.read(file) }
+        end.hexdigest
+      end
+    end
+  end
+end

data/lib/parallel_workforce/serializer/marshal.rb

@@ -0,0 +1,21 @@
+module ParallelWorkforce
+  module Serializer
+    class Marshal
+      def serialize(object)
+        ::Marshal.dump(object)
+      rescue TypeError => e
+        ParallelWorkforce.log(:error, "#{self.class}: Unable to serialize object: #{e}", e, *e.backtrace)
+
+        raise ParallelWorkforce::SerializerError.new("Unable to serialize object: #{e}")
+      end
+
+      def deserialize(string)
+        ::Marshal.load(string)
+      rescue TypeError => e
+        ParallelWorkforce.log(:warn, "#{self.class}: Unable to deserialize string: #{e}", e, *e.backtrace)
+
+        raise ParallelWorkforce::SerializerError.new("Unable to deserialize string: #{e}")
+      end
+    end
+  end
+end

data/lib/parallel_workforce/version.rb

@@ -0,0 +1,3 @@
+module ParallelWorkforce
+  VERSION = "1.0.0".freeze
+end

data/lib/parallel_workforce.rb

@@ -0,0 +1,46 @@
+require_relative 'parallel_workforce/requires'
+
+module ParallelWorkforce
+  Error = Class.new(StandardError)
+  ActorPerformError = Class.new(Error)
+  ActorNotFoundError = Class.new(Error)
+  TimeoutError = Class.new(Error)
+  SerializerError = Class.new(Error)
+
+  class << self
+    # +actor_classes+: a single class or array of classes that have a perform method and can be initialized with no args.
+    # If an array is passed, the array size must batch the action_args_array size.
+    # Return results array with element from each action in the order of the job_args_array
+    # rubocop:disable Metrics/ParameterLists
+    def perform_all(actor_classes:, actor_args_array:,
+        serial_execution_indexes: nil, execute_serially: nil, job_class: nil, &execution_block)
+      ParallelWorkforce::Executor.new(
+        actor_classes: actor_classes,
+        actor_args_array: actor_args_array,
+        execute_serially: execute_serially,
+        serial_execution_indexes: serial_execution_indexes,
+        job_class: job_class,
+        execution_block: execution_block,
+      ).perform_all
+    end
+    # rubocop:enable Metrics/ParameterLists
+
+    def configuration
+      @configuration ||= Configuration.new
+    end
+
+    def configure(&block)
+      yield(configuration)
+    end
+
+    def log(level, *messages)
+      return if configuration.logger.nil?
+
+      messages.each do |message|
+        configuration.logger.send(level, message)
+      end
+
+      nil
+    end
+  end
+end

data/parallel_workforce.gemspec

@@ -0,0 +1,48 @@
+lib = File.expand_path('lib', __dir__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "parallel_workforce/version"
+
+# rubocop:disable Metrics/BlockLength
+Gem::Specification.new do |spec|
+  spec.name = "parallel_workforce"
+  spec.version = ParallelWorkforce::VERSION
+  spec.authors = ["Michael Pearce"]
+  spec.email = ["michael.p@enjoy.com"]
+
+  spec.summary = %q{Simplify parallel code execution into workers}
+  spec.description = %q{Simplify parallel code execution into workers.}
+  spec.homepage = "https://github.com/EnjoyTech/parallel_workforce"
+
+  # Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
+  # to allow pushing to a single host or delete this section to allow pushing to any host.
+  if spec.respond_to?(:metadata)
+    spec.metadata["allowed_push_host"] = "https://rubygems.org"
+
+    spec.metadata["homepage_uri"] = spec.homepage
+    spec.metadata["source_code_uri"] = spec.homepage
+    spec.metadata["changelog_uri"] = "https://github.com/EnjoyTech/parallel_workforce/CHANGELOG.md"
+  else
+    raise "RubyGems 2.0 or newer is required to protect against " \
+      "public gem pushes."
+  end
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files = Dir.chdir(File.expand_path('.', __dir__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+  spec.bindir = "exe"
+  spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.17"
+  spec.add_development_dependency "fakeredis", "~> 0.7"
+  spec.add_development_dependency "pry", "~> 0.12"
+  spec.add_development_dependency "rails", "~> 4.2"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3.8"
+  spec.add_development_dependency "rubocop", "~> 0.65"
+  spec.add_development_dependency "rubocop-rspec", "~> 1.32"
+  spec.add_development_dependency "sidekiq", "~> 4.0"
+end
+# rubocop:enable Metrics/BlockLength

metadata

@@ -0,0 +1,201 @@
+--- !ruby/object:Gem::Specification
+name: parallel_workforce
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Michael Pearce
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2019-05-21 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.17'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.17'
+- !ruby/object:Gem::Dependency
+  name: fakeredis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.7'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.12'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.12'
+- !ruby/object:Gem::Dependency
+  name: rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.2'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.8'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.8'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.65'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.65'
+- !ruby/object:Gem::Dependency
+  name: rubocop-rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.32'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.32'
+- !ruby/object:Gem::Dependency
+  name: sidekiq
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+description: Simplify parallel code execution into workers.
+email:
+- michael.p@enjoy.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".rubocop.yml"
+- ".rubocop_parallel_workforce.yml"
+- ".ruby-version"
+- CHANGELOG.md
+- Gemfile
+- Gemfile.lock
+- LICENSE
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- lib/parallel_workforce.rb
+- lib/parallel_workforce/configuration.rb
+- lib/parallel_workforce/executor.rb
+- lib/parallel_workforce/job/active_job.rb
+- lib/parallel_workforce/job/active_job_rails.rb
+- lib/parallel_workforce/job/sidekiq.rb
+- lib/parallel_workforce/job/sidekiq_rails.rb
+- lib/parallel_workforce/job/util/performer.rb
+- lib/parallel_workforce/redis_connector/redis_pool.rb
+- lib/parallel_workforce/redis_connector/sidekiq_redis_pool.rb
+- lib/parallel_workforce/requires.rb
+- lib/parallel_workforce/revision_builder/files_hash.rb
+- lib/parallel_workforce/serializer/marshal.rb
+- lib/parallel_workforce/version.rb
+- parallel_workforce.gemspec
+homepage: https://github.com/EnjoyTech/parallel_workforce
+licenses: []
+metadata:
+  allowed_push_host: https://rubygems.org
+  homepage_uri: https://github.com/EnjoyTech/parallel_workforce
+  source_code_uri: https://github.com/EnjoyTech/parallel_workforce
+  changelog_uri: https://github.com/EnjoyTech/parallel_workforce/CHANGELOG.md
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.2.1
+signing_key: 
+specification_version: 4
+summary: Simplify parallel code execution into workers
+test_files: []