dipa 0.1.0.pre.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 6403004fa28c6c0ae9f9cc40f9da8f5b812ed5ea4feb94bd00210785dd4bb950
+  data.tar.gz: 5d485e3096f6b835520844bf6564354eb07640159f4fc7579566df78c8c7c999
+SHA512:
+  metadata.gz: 4903b379f5772adf30f2791c74e6355a3a753bb8808a3d8bf655790e7d601e925806d110ec3561993979b3f3aee9a19c5fade0a7d2b17afc7d856e3b5709fecf
+  data.tar.gz: 4022abee161ffd77df8a8a11f9712285689287b369d994334c0bc82ca0f6615f087d1f9aeafa184536b09e399a74921e26d15e670274e98ae836d7aff033c1fd

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2021 Merten Falk
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,126 @@
+![status-badge](https://ci.codeberg.org/api/badges/empunkt/dipa/status.svg)
+
+# Dipa
+
+This gem provides an API for parallel processing like the [parallel
+gem](https://github.com/grosser/parallel) but distributed and scalable over
+different machines. All this with minimum configuration and minimum dependencies
+to specific technologies and using the rails ecosystem.
+
+Dipa provides a rails engine which depends on
+[ActiveJob](https://guides.rubyonrails.org/active_job_basics.html) and
+[ActiveStorage](https://guides.rubyonrails.org/active_storage_overview.html).
+You can use whatever backend you like for any of this components and configure
+them for your specific usecase.
+
+The purpose of this gem is to distribute load heavy and long running processing
+of large datasets over multiple processes or machines using
+[ActiveJob](https://guides.rubyonrails.org/active_job_basics.html).
+
+## Installation
+
+Before you install Dipa make sure
+[ActiveJob](https://guides.rubyonrails.org/active_job_basics.html) and
+[ActiveStorage](https://guides.rubyonrails.org/active_storage_overview.html) are
+installed and configured properly.
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'dipa'
+```
+
+And then execute:
+```bash
+$ bundle install
+```
+
+Or install it yourself as:
+```bash
+$ gem install dipa
+```
+
+Install Dipa migrations
+```bash
+bundle exec rake app:dipa:install:migrations
+bundle exec rake db:migrate
+```
+
+## Configuration
+
+Dipa can be configured in the application config.
+
+```ruby
+config.dipa.agent_queue = :default_queue_for_dipa_agent_jobs
+config.dipa.coordinator_queue = :default_queue_for_coordinator_queue_jobs
+```
+
+If not configured `config.active_job.default_queue_name` or `:default` will be
+used.
+
+## Usage
+
+Minimum example:
+```ruby
+Dipa.map(1..100).with('Integer', :sqrt)
+```
+
+More realistic examples:
+```ruby
+Dipa.map(large_dataset, options).with('ProcessorClassName', :processor_class_method)
+Dipa.each(large_dataset, options).with('ProcessorClassName', :processor_class_method)
+```
+
+`Dipa.map` returns an `Array` of the processed items. The result is in the same order as the input (`large_dataset`).
+
+`Dipa.each` returns `large_dataset.to_a`.
+
+`large_dataset` must be an `Enumerable`.
+
+`options` is a hash. Following keys are allowed:
+
+- `agent_queue:` [Symbol] Defaults to `config.dipa.agent_queue`.
+- `coordinator_queue:` [Symbol] Defaults to `config.dipa.coordinator_queue`.
+- `async:` [true|false] Defaults to `false`. Usually no need to change, but
+  could be useful for `Dipa.each` if you have a alternative way to monitor your
+  jobs.
+- `keep_data:` [true|false] Defaults to `false`. Useful for debugging. After
+  processing all `Dipa::Agent` and `Dipa::Coordinator` records and the
+  associated ActiveStorage data will be removed. If you don't want that to
+  happen, set this to `true`.
+
+`ProcessorClassName` must be a `Class` or a `String`. Defines the class which
+provides the processor method.
+
+`:processor_class_method` must be a `Symbol` or a `String`. Defines the method
+which is used to process each single element of `large_dataset`. MUST be a class
+method. MUST except just one element as argument.
+
+## TODO
+
+[TODO.md](TODO.md)
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run
+`bundle exec rspec` to run the tests. You can also run `bin/console` for an
+interactive prompt that will allow you to experiment.
+
+## Contributing
+
+Bug reports and pull requests are welcome on Codeberg at
+https://codeberg.org/empunkt/dipa. This project is intended to be a safe,
+welcoming space for collaboration, and contributors are expected to adhere to
+the [code of
+conduct](https://codeberg.org/empunkt/dipa/src/branch/main/CODE_OF_CONDUCT.md).
+
+## License
+
+The gem is available as open source under the terms of the
+[MIT License](https://opensource.org/licenses/MIT).
+
+## Code of Conduct
+
+Everyone interacting in the Dipa project's codebases, issue trackers, chat rooms
+and mailing lists is expected to follow the
+[code of conduct](https://codeberg.org/empunkt/dipa/src/branch/main/CODE_OF_CONDUCT.md).

data/Rakefile

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+require 'bundler/setup'
+
+APP_RAKEFILE = File.expand_path('spec/dummy/Rakefile', __dir__)
+load 'rails/tasks/engine.rake'
+
+load 'rails/tasks/statistics.rake'
+
+require 'bundler/gem_tasks'

data/app/jobs/dipa/application_job.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+module Dipa
+  class ApplicationJob < ActiveJob::Base
+  end
+end

data/app/jobs/dipa/service_job.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Dipa
+  class ServiceJob < ApplicationJob
+    def perform(service_class_name:, args: [], kwargs: {})
+      service_class_name.constantize.call(*args, **kwargs)
+    end
+  end
+end

data/app/models/concerns/models/dipa/dumpable.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Models
+  module Dipa
+    module Dumpable
+      extend ActiveSupport::Concern
+
+      def dump_to_file(data:, attacher:)
+        io = StringIO.new(Marshal.dump(data), 'rb')
+        filename = "#{attacher}.dat"
+
+        public_send(attacher).attach(io: io, filename: filename)
+      end
+    end
+  end
+end

data/app/models/concerns/models/dipa/loadable.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Models
+  module Dipa
+    module Loadable
+      extend ActiveSupport::Concern
+
+      def load_from_file(attacher:)
+        return unless public_send(attacher).attached?
+
+        Marshal.load( # rubocop:disable Security/MarshalLoad
+          public_send(attacher).download
+        )
+      end
+    end
+  end
+end

data/app/models/concerns/models/dipa/state_attribute_handling.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Models
+  module Dipa
+    module StateAttributeHandling
+      extend ActiveSupport::Concern
+
+      include Modules::Models::Dipa::StatusConstants
+
+      included do
+        attribute :state, :string, default: INITIALIZED_STATE
+
+        validates :state, presence: true, inclusion: { in: STATES }
+        if Rails.version >= '7'
+          validates :state, comparison: { equal_to: INITIALIZED_STATE },
+                            on: :create
+        else
+          validates :state, inclusion: { in: [INITIALIZED_STATE] },
+                            on: :create
+        end
+
+        STATES.each do |state_value|
+          define_method("#{state_value}?".to_sym) do
+            state == state_value
+          end
+        end
+      end
+
+      def started!
+        update!(started_at: Time.zone.now, state: PROCESSING_STATE)
+      end
+
+      def finished!
+        update!(finished_at: Time.zone.now, state: PROCESSED_STATE)
+      end
+    end
+  end
+end

data/app/models/dipa/agent.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+# == Schema Information
+#
+# Table name: dipa_agents
+#
+#  id                  :bigint           not null, primary key
+#  finished_at         :datetime
+#  index               :integer          not null
+#  started_at          :datetime
+#  state               :string(255)      not null
+#  created_at          :datetime         not null
+#  updated_at          :datetime         not null
+#  dipa_coordinator_id :bigint           not null
+#
+# Indexes
+#
+#  index_dipa_agents_on_dipa_coordinator_id  (dipa_coordinator_id)
+#
+# Foreign Keys
+#
+#  fk_rails_...  (dipa_coordinator_id => dipa_coordinators.id)
+#
+
+module Dipa
+  class Agent < ApplicationRecord
+    include Models::Dipa::Dumpable
+    include Models::Dipa::Loadable
+    include Models::Dipa::StateAttributeHandling
+
+    # validation and default for `state` attribute is included by
+    # Models::Dipa::StateAttributeHandling
+
+    validates :index, numericality: { only_integer: true }
+
+    validates :started_at, 'dipa/date' => true, allow_nil: true
+    validates :finished_at, 'dipa/date' => true, allow_nil: true
+
+    belongs_to :coordinator, inverse_of: :agents,
+                             foreign_key: :dipa_coordinator_id
+
+    has_one_attached :source_dump
+    has_one_attached :result_dump
+
+    def result
+      return unless processed?
+
+      load_from_file(attacher: :result_dump)
+    end
+
+    def source
+      load_from_file(attacher: :source_dump)
+    end
+
+    def process!
+      processor_result = coordinator.processor_class_name.constantize
+                                    .public_send(
+                                      coordinator.processor_method_name,
+                                      source
+                                    )
+
+      dump_to_file(data: processor_result, attacher: :result_dump)
+
+      finished!
+    end
+  end
+end

data/app/models/dipa/application_record.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Dipa
+  class ApplicationRecord < ActiveRecord::Base
+    self.abstract_class = true
+  end
+end

data/app/models/dipa/coordinator.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+# == Schema Information
+#
+# Table name: dipa_coordinators
+#
+#  id                    :bigint           not null, primary key
+#  agent_queue           :string(255)      not null
+#  coordinator_queue     :string(255)      not null
+#  finished_at           :datetime
+#  keep_data             :boolean          default(FALSE), not null
+#  processor_class_name  :string(255)      not null
+#  processor_method_name :string(255)      not null
+#  size                  :integer          not null
+#  started_at            :datetime
+#  state                 :string(255)      not null
+#  want_result           :boolean          default(TRUE), not null
+#  created_at            :datetime         not null
+#  updated_at            :datetime         not null
+#
+
+module Dipa
+  class Coordinator < ApplicationRecord
+    include Models::Dipa::Dumpable
+    include Models::Dipa::Loadable
+    include Models::Dipa::StateAttributeHandling
+
+    # validation and default for `state` attribute is included by
+    # Models::Dipa::StateAttributeHandling
+
+    attribute :keep_data, :boolean, default: false
+    attribute :want_result, :boolean, default: true
+
+    validates :agent_queue, presence: true
+    validates :coordinator_queue, presence: true
+
+    validates :size, numericality: { only_integer: true }
+
+    validates :started_at, 'dipa/date' => true, allow_nil: true
+    validates :finished_at, 'dipa/date' => true, allow_nil: true
+
+    validates :processor_class_name, presence: true
+    validates :processor_method_name, presence: true
+
+    validates :keep_data, inclusion: [true, false]
+    validates :want_result, inclusion: [true, false]
+
+    has_many :agents, dependent: :destroy, inverse_of: :coordinator,
+                      foreign_key: :dipa_coordinator_id
+
+    has_one_attached :source_dump
+
+    def result
+      return unless processed?
+
+      _result_from_agents
+    end
+
+    def source
+      load_from_file(attacher: :source_dump)
+    end
+
+    def all_agents_created_and_processed?
+      _all_agents_created? && _all_agents_processed?
+    end
+
+    private
+
+    def _result_from_agents
+      agents.with_attached_result_dump.order(:index).map(&:result)
+    end
+
+    def _all_agents_processed?
+      agents.all?(&:processed?)
+    end
+
+    def _all_agents_created?
+      agents.length == size
+    end
+  end
+end

data/app/models/modules/models/dipa/status_constants.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Modules
+  module Models
+    module Dipa
+      module StatusConstants
+        ABORTED_STATE = 'aborted'
+        INITIALIZED_STATE = 'initialized'
+        PROCESSED_STATE = 'processed'
+        PROCESSING_STATE = 'processing'
+        PROCESSING_FAILED_STATE = 'processing_failed'
+        TIMEOUT_STATE = 'timed_out'
+        STATES = [
+          ABORTED_STATE,
+          INITIALIZED_STATE,
+          PROCESSED_STATE,
+          PROCESSING_STATE,
+          PROCESSING_FAILED_STATE,
+          TIMEOUT_STATE
+        ].freeze
+      end
+    end
+  end
+end

data/app/services/dipa/agent_services/coordinator_state_service.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Dipa
+  module AgentServices
+    class CoordinatorStateService < ApplicationService
+      def call(agent:)
+        return if agent.coordinator.processed?
+        return unless agent.coordinator.all_agents_created_and_processed?
+
+        agent.coordinator.finished!
+      end
+    end
+  end
+end

data/app/services/dipa/agent_services/post_processing_service.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Dipa
+  module AgentServices
+    class PostProcessingService < ApplicationService
+      def call(agent:)
+        Dipa::ServiceJob.set(queue_as: agent.coordinator.agent_queue)
+                        .perform_later(
+                          service_class_name:
+                            'Dipa::AgentServices::CoordinatorStateService',
+                          kwargs: { agent: agent }
+                        )
+      end
+    end
+  end
+end

data/app/services/dipa/agent_services/processing_service.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Dipa
+  module AgentServices
+    class ProcessingService < ApplicationService
+      def call(agent:)
+        agent.process!
+
+        Dipa::AgentServices::PostProcessingService.call(agent: agent)
+      end
+    end
+  end
+end

data/app/services/dipa/agent_services/start_processing_service.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Dipa
+  module AgentServices
+    class StartProcessingService < ApplicationService
+      def call(agent:)
+        agent.started!
+
+        Dipa::ServiceJob.set(queue_as: agent.coordinator.agent_queue)
+                        .perform_later(
+                          service_class_name:
+                            'Dipa::AgentServices::ProcessingService',
+                          kwargs: { agent: agent }
+                        )
+      end
+    end
+  end
+end

data/app/services/dipa/application_service.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Dipa
+  class ApplicationService
+    def self.call(...)
+      new.call(...)
+    end
+  end
+end

data/app/services/dipa/coordinator_services/create_agents_service.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Dipa
+  module CoordinatorServices
+    class CreateAgentsService < ApplicationService
+      def call(coordinator:)
+        coordinator.source.each_with_index do |item, i|
+          _create_agent(coordinator: coordinator, item: item, index: i)
+        end
+      end
+
+      private
+
+      def _create_agent(coordinator:, item:, index:)
+        agent = coordinator.agents.create!(index: index)
+
+        agent.dump_to_file(data: item, attacher: :source_dump)
+
+        Dipa::AgentServices::StartProcessingService.call(agent: agent)
+      end
+    end
+  end
+end

data/app/services/dipa/coordinator_services/start_processing_service.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Dipa
+  module CoordinatorServices
+    class StartProcessingService < ApplicationService
+      def call(coordinator:)
+        coordinator.started!
+
+        ServiceJob.set(queue_as: coordinator.coordinator_queue)
+                  .perform_later(
+                    service_class_name:
+                      'Dipa::CoordinatorServices::CreateAgentsService',
+                    kwargs: { coordinator: coordinator }
+                  )
+      end
+    end
+  end
+end

data/app/validators/dipa/date_validator.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Dipa
+  class DateValidator < ActiveModel::EachValidator
+    def validate_each(record, attribute, value)
+      return if self.class.valid?(value: value)
+
+      record.errors.add(
+        attribute,
+        :invalid,
+        message: (options[:message] || 'is not valid Date')
+      )
+    end
+
+    def self.valid?(value:)
+      return false if value.blank?
+
+      begin
+        Date.parse(value.to_s)
+        true
+      rescue ArgumentError
+        false
+      end
+    end
+  end
+end

data/db/migrate/20220102132652_create_dipa_coordinators.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+class CreateDipaCoordinators < ActiveRecord::Migration[6.0]
+  def change # rubocop:disable Metrics/MethodLength
+    create_table :dipa_coordinators do |t|
+      t.boolean :keep_data, default: false, null: false
+      t.boolean :want_result, default: true, null: false
+      t.datetime :finished_at
+      t.datetime :started_at
+      t.integer :size, null: false
+      t.string :agent_queue, null: false
+      t.string :coordinator_queue, null: false
+      t.string :processor_class_name, null: false
+      t.string :processor_method_name, null: false
+      t.string :state, null: false
+
+      t.timestamps
+    end
+  end
+end

data/db/migrate/20220106183616_create_dipa_agents.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+class CreateDipaAgents < ActiveRecord::Migration[6.0]
+  def change # rubocop:disable Metrics/MethodLength
+    create_table :dipa_agents do |t|
+      t.datetime :finished_at
+      t.datetime :started_at
+      t.integer :index, null: false
+      t.string :state, null: false
+
+      t.timestamps
+    end
+
+    add_belongs_to(
+      :dipa_agents, :dipa_coordinator,
+      foreign_key: true, index: true, null: false
+    )
+  end
+end

data/lib/dipa/engine.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+require 'socket'
+require 'rails'
+require 'active_job/railtie'
+require 'active_record/railtie'
+require 'active_storage/engine'
+
+module Dipa
+  class Engine < ::Rails::Engine
+    isolate_namespace Dipa
+
+    config.eager_load_namespaces << Dipa
+
+    config.generators do |g|
+      g.test_framework :rspec
+      g.api_only = true
+    end
+
+    config.dipa = ActiveSupport::OrderedOptions.new
+
+    initializer 'dipa.queue_names' do
+      config.after_initialize do |app|
+        Dipa.agent_queue = (
+          app.config.dipa.agent_queue ||
+          app.config.active_job.default_queue_name ||
+          :default
+        ).to_sym
+        Dipa.coordinator_queue = (
+          app.config.dipa.coordinator_queue ||
+          app.config.active_job.default_queue_name ||
+          :default
+        ).to_sym
+      end
+    end
+
+    initializer 'dipa.timeouts' do
+      config.after_initialize do |app|
+        Dipa.agent_timeout = (
+          app.config.dipa.agent_timeout || Dipa::DEFAULT_AGENT_TIMEOUT
+        ).to_i
+        Dipa.coordinator_timeout = (
+          app.config.dipa.coordinator_timeout ||
+          Dipa::DEFAULT_COORDINATOR_TIMEOUT
+        ).to_i
+      end
+    end
+  end
+end

data/lib/dipa/errors.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module Dipa
+  class Error < StandardError; end
+
+  class AbortedError < Error; end
+  class ProcessingFailedError < Error; end
+  class TimeoutError < Error; end
+  class UnknownProcessingStateError < Error; end
+  class UnknownProcessorClassError < Error; end
+  class UnknownProcessorMethodError < Error; end
+end

data/lib/dipa/processor/base.rb

@@ -0,0 +1,141 @@
+# frozen_string_literal: true
+
+module Dipa
+  module Processor
+    class Base
+      SYNC_MODE_WAIT_CYCLE_SECONDS = 2
+
+      DEFAULT_OPTIONS = {
+        # queue names
+        agent_queue: Dipa.agent_queue,
+        coordinator_queue: Dipa.coordinator_queue,
+        # timeouts
+        agent_timeout: Dipa.agent_timeout,
+        coordinator_timeout: Dipa.coordinator_timeout,
+        # misc
+        async: false,
+        keep_data: false,
+        want_result: true
+      }.freeze
+      OVERRIDE_OPTIONS = {}.freeze
+
+      def with(processor_class, processor_method)
+        _validate_processor_arguments(processor_class: processor_class.to_s,
+                                      processor_method: processor_method.to_s)
+
+        _prepare_coordinator(processor_class: processor_class.to_s,
+                             processor_method: processor_method.to_s)
+
+        _start_process
+
+        return if _async?
+
+        _wait_for_it
+      end
+
+      private
+
+      attr_reader :_source, :_raw_opts, :_coordinator
+
+      def initialize(source, options: {})
+        options.assert_valid_keys(*DEFAULT_OPTIONS.keys)
+
+        @_source = source
+        @_raw_opts = options
+      end
+
+      def _validate_processor_arguments(processor_class:, processor_method:)
+        return if processor_class.constantize.respond_to?(processor_method)
+
+        raise Dipa::UnknownProcessorMethodError,
+              "Method .#{processor_method} does not exist on processor class " \
+              "#{processor_class}"
+      rescue NameError => e
+        raise Dipa::UnknownProcessorClassError, e.original_message
+      end
+
+      def _wait_for_it
+        sleep(SYNC_MODE_WAIT_CYCLE_SECONDS) while _wait_for_it?
+
+        return _result if _coordinator.processed?
+
+        # must be an error then
+        _raise_error
+      end
+
+      def _wait_for_it?
+        _coordinator.reload
+
+        _coordinator.initialized? || _coordinator.processing?
+      end
+
+      def _raise_error
+        raise Dipa::AbortedError if _coordinator.aborted?
+        raise Dipa::ProcessingFailedError if _coordinator.processing_failed?
+        raise Dipa::TimeoutError if _coordinator.timed_out?
+
+        raise Dipa::UnknownProcessingStateError
+      end
+
+      def _result
+        result = _fetch_result
+
+        _maybe_cleanup
+
+        result
+      end
+
+      def _maybe_cleanup
+        _coordinator.destroy! unless _keep_data?
+      end
+
+      def _fetch_result
+        _want_result? ? _coordinator.result : _coordinator.source
+      end
+
+      def _start_process
+        Dipa::CoordinatorServices::StartProcessingService.call(
+          coordinator: _coordinator
+        )
+      end
+
+      def _prepare_coordinator(processor_class:, processor_method:)
+        @_coordinator = Dipa::Coordinator.create!(
+          agent_queue: _agent_queue,
+          coordinator_queue: _coordinator_queue,
+          keep_data: _keep_data?,
+          processor_class_name: processor_class,
+          processor_method_name: processor_method,
+          size: _source.to_a.size,
+          want_result: _want_result?
+        )
+
+        _coordinator.dump_to_file(data: _source.to_a, attacher: :source_dump)
+      end
+
+      def _agent_queue
+        _option(option: :agent_queue)
+      end
+
+      def _coordinator_queue
+        _option(option: :coordinator_queue)
+      end
+
+      def _async?
+        _option(option: :async)
+      end
+
+      def _keep_data?
+        _option(option: :keep_data)
+      end
+
+      def _want_result?
+        _option(option: :want_result)
+      end
+
+      def _option(option:)
+        OVERRIDE_OPTIONS[option] || _raw_opts[option] || DEFAULT_OPTIONS[option]
+      end
+    end
+  end
+end

data/lib/dipa/processor/each.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Dipa
+  module Processor
+    class Each < Base
+      OVERRIDE_OPTIONS = {
+        want_result: false
+      }.freeze
+    end
+  end
+end

data/lib/dipa/processor/map.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module Dipa
+  module Processor
+    class Map < Base
+    end
+  end
+end

data/lib/dipa/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Dipa
+  VERSION = '0.1.0.pre.1'
+end

data/lib/dipa.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require 'active_support'
+
+require 'dipa/version'
+require 'dipa/engine'
+require 'dipa/errors'
+
+module Dipa
+  extend ActiveSupport::Autoload
+
+  DEFAULT_AGENT_TIMEOUT = 0
+  DEFAULT_COORDINATOR_TIMEOUT = 0
+
+  # rubocop:disable ThreadSafety/ClassAndModuleAttributes
+  mattr_accessor :agent_queue
+  mattr_accessor :agent_timeout, default: DEFAULT_AGENT_TIMEOUT
+  mattr_accessor :coordinator_queue
+  mattr_accessor :coordinator_timeout, default: DEFAULT_COORDINATOR_TIMEOUT
+  # rubocop:enable ThreadSafety/ClassAndModuleAttributes
+
+  def self.map(source, options: {})
+    Dipa::Processor::Map.new(source, options: options)
+  end
+
+  def self.each(source, options: {})
+    Dipa::Processor::Each.new(source, options: options)
+  end
+
+  module Processor
+    extend ActiveSupport::Autoload
+
+    autoload :Base
+    autoload :Each
+    autoload :Map
+  end
+end

data/lib/tasks/auto_annotate_models.rake

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+# NOTE: only doing this in development as some production environments (Heroku)
+# NOTE: are sensitive to local FS writes, and besides -- it's just not proper
+# NOTE: to have a dev-mode tool do its thing in production.
+if Rails.env.development?
+  require 'annotate'
+  desc 'annotate'
+  task set_annotation_options: :environment do # rubocop:disable Metrics/BlockLength
+    # You can override any of these by setting an environment variable of the
+    # same name.
+    Annotate.set_defaults(
+      'active_admin' => 'false',
+      'additional_file_patterns' => [],
+      'routes' => 'false',
+      'models' => 'true',
+      'position_in_routes' => 'before',
+      'position_in_class' => 'before',
+      'position_in_test' => 'before',
+      'position_in_fixture' => 'before',
+      'position_in_factory' => 'before',
+      'position_in_serializer' => 'before',
+      'show_foreign_keys' => 'true',
+      'show_complete_foreign_keys' => 'false',
+      'show_indexes' => 'true',
+      'simple_indexes' => 'false',
+      'model_dir' => 'app/models',
+      'root_dir' => '',
+      'include_version' => 'false',
+      'require' => '',
+      'exclude_tests' => 'true',
+      'exclude_fixtures' => 'true',
+      'exclude_factories' => 'false',
+      'exclude_serializers' => 'false',
+      'exclude_scaffolds' => 'true',
+      'exclude_controllers' => 'true',
+      'exclude_helpers' => 'true',
+      'exclude_sti_subclasses' => 'false',
+      'ignore_model_sub_dir' => 'false',
+      'ignore_columns' => nil,
+      'ignore_routes' => nil,
+      'ignore_unknown_models' => 'false',
+      'hide_limit_column_types' => 'integer,bigint,boolean',
+      'hide_default_column_types' => 'json,jsonb,hstore',
+      'skip_on_db_migrate' => 'false',
+      'format_bare' => 'true',
+      'format_rdoc' => 'false',
+      'format_yard' => 'false',
+      'format_markdown' => 'false',
+      'sort' => 'true',
+      'force' => 'false',
+      'frozen' => 'false',
+      'classified_sort' => 'true',
+      'trace' => 'false',
+      'wrapper_open' => nil,
+      'wrapper_close' => nil,
+      'with_comment' => 'true'
+    )
+  end
+
+  Annotate.load_tasks
+end

data/lib/tasks/dipa_tasks.rake

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+# desc "Explaining what the task does"
+# task :dipa do
+#   # Task goes here
+# end

metadata

@@ -0,0 +1,184 @@
+--- !ruby/object:Gem::Specification
+name: dipa
+version: !ruby/object:Gem::Version
+  version: 0.1.0.pre.1
+platform: ruby
+authors:
+- Merten Falk
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2022-03-05 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activejob
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">"
+      - !ruby/object:Gem::Version
+        version: 6.0.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 8.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">"
+      - !ruby/object:Gem::Version
+        version: 6.0.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 8.0.0
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">"
+      - !ruby/object:Gem::Version
+        version: 6.0.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 8.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">"
+      - !ruby/object:Gem::Version
+        version: 6.0.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 8.0.0
+- !ruby/object:Gem::Dependency
+  name: activestorage
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">"
+      - !ruby/object:Gem::Version
+        version: 6.0.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 8.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">"
+      - !ruby/object:Gem::Version
+        version: 6.0.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 8.0.0
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">"
+      - !ruby/object:Gem::Version
+        version: 6.0.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 8.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">"
+      - !ruby/object:Gem::Version
+        version: 6.0.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 8.0.0
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+description: |
+  This gem provides an API for parallel processing like the [parallel
+  gem](https://github.com/grosser/parallel) but distributed and scalable over
+  different machines. All this with minimum configuration and minimum
+  dependencies to specific technologies and using the rails ecosystem.
+
+  Dipa provides a rails engine which depends on
+  [ActiveJob](https://guides.rubyonrails.org/active_job_basics.html) and
+  [ActiveStorage](https://guides.rubyonrails.org/active_storage_overview.html).
+  You can use whatever backend you like for any of this components and
+  configure them for your specific usecase.
+
+  The purpose of this gem is to distribute load heavy and long running
+  processing of large datasets over multiple processes or machines using
+  [ActiveJob](https://guides.rubyonrails.org/active_job_basics.html).
+email:
+- empunkt@mailbox.org
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE.txt
+- README.md
+- Rakefile
+- app/jobs/dipa/application_job.rb
+- app/jobs/dipa/service_job.rb
+- app/models/concerns/models/dipa/dumpable.rb
+- app/models/concerns/models/dipa/loadable.rb
+- app/models/concerns/models/dipa/state_attribute_handling.rb
+- app/models/dipa/agent.rb
+- app/models/dipa/application_record.rb
+- app/models/dipa/coordinator.rb
+- app/models/modules/models/dipa/status_constants.rb
+- app/services/dipa/agent_services/coordinator_state_service.rb
+- app/services/dipa/agent_services/post_processing_service.rb
+- app/services/dipa/agent_services/processing_service.rb
+- app/services/dipa/agent_services/start_processing_service.rb
+- app/services/dipa/application_service.rb
+- app/services/dipa/coordinator_services/create_agents_service.rb
+- app/services/dipa/coordinator_services/start_processing_service.rb
+- app/validators/dipa/date_validator.rb
+- db/migrate/20220102132652_create_dipa_coordinators.rb
+- db/migrate/20220106183616_create_dipa_agents.rb
+- lib/dipa.rb
+- lib/dipa/engine.rb
+- lib/dipa/errors.rb
+- lib/dipa/processor/base.rb
+- lib/dipa/processor/each.rb
+- lib/dipa/processor/map.rb
+- lib/dipa/version.rb
+- lib/tasks/auto_annotate_models.rake
+- lib/tasks/dipa_tasks.rake
+homepage: https://codeberg.org/empunkt/dipa
+licenses:
+- MIT
+metadata:
+  rubygems_mfa_required: 'true'
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.7.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">"
+    - !ruby/object:Gem::Version
+      version: 1.3.1
+requirements: []
+rubygems_version: 3.3.7
+signing_key:
+specification_version: 4
+summary: Rails engine that provides an API to execute code in parallel and distributed
+  using the rails ecosystem.
+test_files: []