dipa 0.1.0.pre.1 → 0.1.0.pre.3

data/lib/dipa/processor/base.rb

@@ -3,94 +3,46 @@
 module Dipa
   module Processor
     class Base
-      SYNC_MODE_WAIT_CYCLE_SECONDS = 2
-
-      DEFAULT_OPTIONS = {
-        # queue names
-        agent_queue: Dipa.agent_queue,
-        coordinator_queue: Dipa.coordinator_queue,
-        # timeouts
-        agent_timeout: Dipa.agent_timeout,
-        coordinator_timeout: Dipa.coordinator_timeout,
-        # misc
-        async: false,
-        keep_data: false,
-        want_result: true
-      }.freeze
       OVERRIDE_OPTIONS = {}.freeze
 
+      include Dipa::Processor::Concerns::Coordinator
+      include Dipa::Processor::Concerns::Options
+      include Dipa::Processor::Concerns::Source
+      include Dipa::Processor::Concerns::State
+      include Dipa::Processor::Concerns::Wait
+
       def with(processor_class, processor_method)
-        _validate_processor_arguments(processor_class: processor_class.to_s,
-                                      processor_method: processor_method.to_s)
+        _check_state
 
         _prepare_coordinator(processor_class: processor_class.to_s,
                              processor_method: processor_method.to_s)
 
         _start_process
 
-        return if _async?
-
-        _wait_for_it
+        _result_and_cleanup
+      ensure
+        _reset_state
       end
 
       private
 
-      attr_reader :_source, :_raw_opts, :_coordinator
-
       def initialize(source, options: {})
-        options.assert_valid_keys(*DEFAULT_OPTIONS.keys)
+        _validate_options(options: options)
+        _write_options(options: options)
 
         @_source = source
-        @_raw_opts = options
-      end
-
-      def _validate_processor_arguments(processor_class:, processor_method:)
-        return if processor_class.constantize.respond_to?(processor_method)
-
-        raise Dipa::UnknownProcessorMethodError,
-              "Method .#{processor_method} does not exist on processor class " \
-              "#{processor_class}"
-      rescue NameError => e
-        raise Dipa::UnknownProcessorClassError, e.original_message
-      end
-
-      def _wait_for_it
-        sleep(SYNC_MODE_WAIT_CYCLE_SECONDS) while _wait_for_it?
-
-        return _result if _coordinator.processed?
-
-        # must be an error then
-        _raise_error
-      end
-
-      def _wait_for_it?
-        _coordinator.reload
-
-        _coordinator.initialized? || _coordinator.processing?
-      end
-
-      def _raise_error
-        raise Dipa::AbortedError if _coordinator.aborted?
-        raise Dipa::ProcessingFailedError if _coordinator.processing_failed?
-        raise Dipa::TimeoutError if _coordinator.timed_out?
-
-        raise Dipa::UnknownProcessingStateError
       end
 
-      def _result
-        result = _fetch_result
-
-        _maybe_cleanup
-
-        result
+      def _result_and_cleanup
+        _wait_for_it_with_timeout
+      ensure
+        _after_result_action
       end
 
-      def _maybe_cleanup
-        _coordinator.destroy! unless _keep_data?
-      end
-
-      def _fetch_result
-        _want_result? ? _coordinator.result : _coordinator.source
+      def _after_result_action
+        Dipa::CoordinatorServices::MaybeCleanupService.call(
+          coordinator: _coordinator
+        )
       end
 
       def _start_process
@@ -98,44 +50,6 @@ module Dipa
           coordinator: _coordinator
         )
       end
-
-      def _prepare_coordinator(processor_class:, processor_method:)
-        @_coordinator = Dipa::Coordinator.create!(
-          agent_queue: _agent_queue,
-          coordinator_queue: _coordinator_queue,
-          keep_data: _keep_data?,
-          processor_class_name: processor_class,
-          processor_method_name: processor_method,
-          size: _source.to_a.size,
-          want_result: _want_result?
-        )
-
-        _coordinator.dump_to_file(data: _source.to_a, attacher: :source_dump)
-      end
-
-      def _agent_queue
-        _option(option: :agent_queue)
-      end
-
-      def _coordinator_queue
-        _option(option: :coordinator_queue)
-      end
-
-      def _async?
-        _option(option: :async)
-      end
-
-      def _keep_data?
-        _option(option: :keep_data)
-      end
-
-      def _want_result?
-        _option(option: :want_result)
-      end
-
-      def _option(option:)
-        OVERRIDE_OPTIONS[option] || _raw_opts[option] || DEFAULT_OPTIONS[option]
-      end
     end
   end
 end

data/lib/dipa/processor/concerns/coordinator.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+module Dipa
+  module Processor
+    module Concerns
+      module Coordinator
+        extend ActiveSupport::Concern
+
+        include Dipa::Processor::Concerns::Options
+        include Dipa::Processor::Concerns::Source
+
+        private
+
+        included do
+          attr_reader :_coordinator
+        end
+
+        def _prepare_coordinator(processor_class:, processor_method:)
+          _validate_processor_arguments(processor_class: processor_class,
+                                        processor_method: processor_method)
+
+          _create_coordinator(
+            processor_class_name: processor_class,
+            processor_method_name: processor_method
+          )
+        end
+
+        def _validate_processor_arguments(processor_class:, processor_method:)
+          return if processor_class.constantize.respond_to?(processor_method)
+
+          raise Dipa::UnknownProcessorMethodError,
+                "Method .#{processor_method} does not exist on processor " \
+                "class #{processor_class}"
+        rescue NameError => e
+          raise Dipa::UnknownProcessorClassError, e.original_message
+        end
+
+        def _create_coordinator(processor_class_name:, processor_method_name:)
+          @_coordinator = Dipa::Coordinator.create!(
+            _coordinator_create_params.merge(
+              {
+                processor_class_name: processor_class_name,
+                processor_method_name: processor_method_name
+              }
+            )
+          )
+
+          _coordinator.dump_to_file(data: _source.to_a, attacher: :source_dump)
+        end
+
+        def _coordinator_create_params
+          {
+            agent_queue: _agent_queue,
+            coordinator_queue: _coordinator_queue,
+            size: _source.to_a.size,
+            coordinator_options_record_attributes:
+              _coordinator_options_record_create_params
+          }
+        end
+
+        def _coordinator_options_record_create_params
+          {
+            keep_data: _keep_data?,
+            want_result: _want_result?,
+            agent_timeout: _agent_timeout,
+            agent_processing_timeout: _agent_processing_timeout,
+            coordinator_timeout: _coordinator_timeout,
+            coordinator_processing_timeout: _coordinator_processing_timeout
+          }
+        end
+
+        def _coordinator_waiting_state?
+          _coordinator.reload
+
+          !_coordinator.finished_state?
+        end
+
+        def _result
+          _coordinator.reload
+
+          _coordinator.want_result? ? _coordinator.result : _coordinator.source
+        end
+      end
+    end
+  end
+end

data/lib/dipa/processor/concerns/options.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+module Dipa
+  module Processor
+    module Concerns
+      module Options
+        extend ActiveSupport::Concern
+
+        include ActiveSupport::Configurable
+
+        DEFAULT_ASYNC = false
+        DEFAULT_KEEP_DATA = false
+        DEFAULT_WANT_RESULT = true
+
+        OPTIONS_MAPPING = {
+          # queue names
+          agent_queue: { accessor: :_agent_queue, default: Dipa.agent_queue },
+          coordinator_queue: { accessor: :_coordinator_queue, default: Dipa.coordinator_queue },
+          # timeouts
+          agent_timeout: { accessor: :_agent_timeout, default: Dipa.agent_timeout },
+          agent_processing_timeout: { accessor: :_agent_processing_timeout, default: Dipa.agent_processing_timeout },
+          coordinator_timeout: { accessor: :_coordinator_timeout, default: Dipa.coordinator_timeout },
+          coordinator_processing_timeout: { accessor: :_coordinator_processing_timeout,
+                                            default: Dipa.coordinator_processing_timeout },
+          # misc
+          keep_data: { accessor: :_keep_data, default: DEFAULT_KEEP_DATA },
+          want_result: { accessor: :_want_result, default: DEFAULT_WANT_RESULT }
+        }.freeze
+
+        private
+
+        included do
+          OPTIONS_MAPPING.each_value do |args|
+            # Since ActiveSupport < 7 does not implement `default:`` option
+            # for config_accessor, defaults are set explicitly with
+            # `#_write_unprovided_options`
+            config_accessor(args[:accessor])
+          end
+
+          alias_method :_keep_data?, :_keep_data
+          alias_method :_want_result?, :_want_result
+        end
+
+        def _validate_options(options:)
+          options.assert_valid_keys(*OPTIONS_MAPPING.keys)
+        end
+
+        def _write_options(options:)
+          _write_provided_options(options: options)
+          _write_unprovided_options(options: options)
+        end
+
+        def _write_provided_options(options:)
+          options.each { |option, value| _write_option(option, value) }
+        end
+
+        def _write_unprovided_options(options:)
+          unprovided_options = OPTIONS_MAPPING.except(*options.keys)
+
+          unprovided_options.each do |option, args|
+            _write_option(option, args[:default])
+          end
+        end
+
+        def _write_option(option, value)
+          new_value = _value_with_override_check(option, value)
+          __send__("#{OPTIONS_MAPPING[option][:accessor]}=", new_value)
+        end
+
+        def _value_with_override_check(option, value)
+          if self.class::OVERRIDE_OPTIONS.key?(option)
+            self.class::OVERRIDE_OPTIONS[option]
+          else
+            value
+          end
+        end
+      end
+    end
+  end
+end

data/lib/dipa/processor/concerns/source.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Dipa
+  module Processor
+    module Concerns
+      module Source
+        extend ActiveSupport::Concern
+
+        included do
+          private
+
+          attr_reader :_source
+        end
+      end
+    end
+  end
+end

data/lib/dipa/processor/concerns/state.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Dipa
+  module Processor
+    module Concerns
+      module State
+        extend ActiveSupport::Concern
+
+        private
+
+        def _processing_state?
+          @_processing_state ? true : false
+        end
+
+        def _check_state
+          raise Dipa::AlreadyProcessingError if _processing_state?
+
+          @_processing_state = true
+        end
+
+        def _reset_state
+          @_processing_state = false
+        end
+      end
+    end
+  end
+end

data/lib/dipa/processor/concerns/wait.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module Dipa
+  module Processor
+    module Concerns
+      module Wait
+        extend ActiveSupport::Concern
+
+        include Dipa::Processor::Concerns::Coordinator
+        include Dipa::Processor::Concerns::Options
+
+        SYNC_MODE_SLEEP_START_DURATION = 2
+        SYNC_MODE_MAX_SLEEP_DURATION = 120
+        SYNC_MODE_SLEEP_DURATION_INCREASE_FACTOR = 1.1
+
+        private
+
+        def _wait_for_it_with_timeout
+          _init_timeout_and_sleep_variables
+
+          _wait_for_it
+        end
+
+        def _init_timeout_and_sleep_variables
+          @_start = Time.current
+          @_sleep_duration = SYNC_MODE_SLEEP_START_DURATION
+        end
+
+        def _wait_for_it
+          _sleep while _coordinator_waiting_state?
+
+          return _result if _coordinator.finished?
+
+          # must be an error then
+          _raise_error
+        end
+
+        def _sleep
+          sleep(@_sleep_duration)
+
+          _adjust_sleep_duration
+        end
+
+        def _adjust_sleep_duration
+          return if @_sleep_duration > SYNC_MODE_MAX_SLEEP_DURATION
+
+          @_sleep_duration *= SYNC_MODE_SLEEP_DURATION_INCREASE_FACTOR
+        end
+
+        def _raise_error
+          if _coordinator.failed_state?
+            raise(
+              "Dipa::Coordinator#{_coordinator.state.to_s.camelize}Error".constantize
+            )
+          end
+
+          raise Dipa::UnknownProcessingStateError
+        end
+      end
+    end
+  end
+end

data/lib/dipa/processor/map.rb

@@ -2,7 +2,6 @@
 
 module Dipa
   module Processor
-    class Map < Base
-    end
+    class Map < Base; end
   end
 end

data/lib/dipa/version.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+# :nocov:
 module Dipa
-  VERSION = '0.1.0.pre.1'
+  VERSION = '0.1.0.pre.3'
 end
+# :nocov:

data/lib/dipa.rb

@@ -9,14 +9,22 @@ require 'dipa/errors'
 module Dipa
   extend ActiveSupport::Autoload
 
+  # DEFAULT_TIMEOUT = 0
   DEFAULT_AGENT_TIMEOUT = 0
+  DEFAULT_AGENT_PROCESSING_TIMEOUT = 0
   DEFAULT_COORDINATOR_TIMEOUT = 0
+  DEFAULT_COORDINATOR_PROCESSING_TIMEOUT = 0
+
+  DATETIME_PRECISION = 6
 
   # rubocop:disable ThreadSafety/ClassAndModuleAttributes
   mattr_accessor :agent_queue
   mattr_accessor :agent_timeout, default: DEFAULT_AGENT_TIMEOUT
+  mattr_accessor :agent_processing_timeout, default: DEFAULT_AGENT_PROCESSING_TIMEOUT
   mattr_accessor :coordinator_queue
   mattr_accessor :coordinator_timeout, default: DEFAULT_COORDINATOR_TIMEOUT
+  mattr_accessor :coordinator_processing_timeout, default: DEFAULT_COORDINATOR_PROCESSING_TIMEOUT
+  mattr_accessor :datetime_precision, default: DATETIME_PRECISION
   # rubocop:enable ThreadSafety/ClassAndModuleAttributes
 
   def self.map(source, options: {})
@@ -30,6 +38,16 @@ module Dipa
   module Processor
     extend ActiveSupport::Autoload
 
+    module Concerns
+      extend ActiveSupport::Autoload
+
+      autoload :Coordinator
+      autoload :Options
+      autoload :Source
+      autoload :State
+      autoload :Wait
+    end
+
     autoload :Base
     autoload :Each
     autoload :Map

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dipa
 version: !ruby/object:Gem::Version
-  version: 0.1.0.pre.1
+  version: 0.1.0.pre.3
 platform: ruby
 authors:
 - Merten Falk
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-03-05 00:00:00.000000000 Z
+date: 2023-09-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activejob
@@ -16,7 +16,7 @@ dependencies:
     requirements:
     - - ">"
       - !ruby/object:Gem::Version
-        version: 6.0.0
+        version: 6.1.0
     - - "<"
       - !ruby/object:Gem::Version
         version: 8.0.0
@@ -26,7 +26,7 @@ dependencies:
     requirements:
     - - ">"
       - !ruby/object:Gem::Version
-        version: 6.0.0
+        version: 6.1.0
     - - "<"
       - !ruby/object:Gem::Version
         version: 8.0.0
@@ -36,7 +36,7 @@ dependencies:
     requirements:
     - - ">"
       - !ruby/object:Gem::Version
-        version: 6.0.0
+        version: 6.1.0
     - - "<"
       - !ruby/object:Gem::Version
         version: 8.0.0
@@ -46,7 +46,7 @@ dependencies:
     requirements:
     - - ">"
       - !ruby/object:Gem::Version
-        version: 6.0.0
+        version: 6.1.0
     - - "<"
       - !ruby/object:Gem::Version
         version: 8.0.0
@@ -56,7 +56,7 @@ dependencies:
     requirements:
     - - ">"
       - !ruby/object:Gem::Version
-        version: 6.0.0
+        version: 6.1.0
     - - "<"
       - !ruby/object:Gem::Version
         version: 8.0.0
@@ -66,7 +66,7 @@ dependencies:
     requirements:
     - - ">"
       - !ruby/object:Gem::Version
-        version: 6.0.0
+        version: 6.1.0
     - - "<"
       - !ruby/object:Gem::Version
         version: 8.0.0
@@ -76,7 +76,7 @@ dependencies:
     requirements:
     - - ">"
       - !ruby/object:Gem::Version
-        version: 6.0.0
+        version: 6.1.0
     - - "<"
       - !ruby/object:Gem::Version
         version: 8.0.0
@@ -86,7 +86,7 @@ dependencies:
     requirements:
     - - ">"
       - !ruby/object:Gem::Version
-        version: 6.0.0
+        version: 6.1.0
     - - "<"
       - !ruby/object:Gem::Version
         version: 8.0.0
@@ -104,21 +104,33 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '13.0'
+- !ruby/object:Gem::Dependency
+  name: validates_timeliness
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: |
-  This gem provides an API for parallel processing like the [parallel
-  gem](https://github.com/grosser/parallel) but distributed and scalable over
-  different machines. All this with minimum configuration and minimum
-  dependencies to specific technologies and using the rails ecosystem.
+  This gem provides an API for parallel processing like the parallel gem but
+  distributed and scalable over different machines. All this with minimum
+  configuration and minimum dependencies to specific technologies and using
+  the rails ecosystem.
 
-  Dipa provides a rails engine which depends on
-  [ActiveJob](https://guides.rubyonrails.org/active_job_basics.html) and
-  [ActiveStorage](https://guides.rubyonrails.org/active_storage_overview.html).
+  Dipa provides a rails engine which depends on ActiveJob and ActiveStorage.
   You can use whatever backend you like for any of this components and
   configure them for your specific usecase.
 
   The purpose of this gem is to distribute load heavy and long running
   processing of large datasets over multiple processes or machines using
-  [ActiveJob](https://guides.rubyonrails.org/active_job_basics.html).
+  ActiveJob.
 email:
 - empunkt@mailbox.org
 executables: []
@@ -130,36 +142,53 @@ files:
 - Rakefile
 - app/jobs/dipa/application_job.rb
 - app/jobs/dipa/service_job.rb
-- app/models/concerns/models/dipa/dumpable.rb
-- app/models/concerns/models/dipa/loadable.rb
-- app/models/concerns/models/dipa/state_attribute_handling.rb
+- app/models/concerns/dipa/models/agent_state_attributes.rb
+- app/models/concerns/dipa/models/coordinator_state_attributes.rb
+- app/models/concerns/dipa/models/dumpable.rb
+- app/models/concerns/dipa/models/loadable.rb
+- app/models/concerns/dipa/models/setting_constants.rb
+- app/models/concerns/dipa/models/state_attribute_handling.rb
 - app/models/dipa/agent.rb
 - app/models/dipa/application_record.rb
 - app/models/dipa/coordinator.rb
-- app/models/modules/models/dipa/status_constants.rb
+- app/models/dipa/coordinator_options_record.rb
+- app/services/concerns/dipa/services/process_failure_handling.rb
+- app/services/dipa/agent_services/base_service.rb
 - app/services/dipa/agent_services/coordinator_state_service.rb
 - app/services/dipa/agent_services/post_processing_service.rb
 - app/services/dipa/agent_services/processing_service.rb
 - app/services/dipa/agent_services/start_processing_service.rb
 - app/services/dipa/application_service.rb
+- app/services/dipa/coordinator_services/base_service.rb
 - app/services/dipa/coordinator_services/create_agents_service.rb
+- app/services/dipa/coordinator_services/destroy_service.rb
+- app/services/dipa/coordinator_services/maybe_cleanup_service.rb
 - app/services/dipa/coordinator_services/start_processing_service.rb
 - app/validators/dipa/date_validator.rb
 - db/migrate/20220102132652_create_dipa_coordinators.rb
 - db/migrate/20220106183616_create_dipa_agents.rb
+- db/migrate/20230624053724_create_dipa_coordinator_options_record.rb
 - lib/dipa.rb
 - lib/dipa/engine.rb
 - lib/dipa/errors.rb
 - lib/dipa/processor/base.rb
+- lib/dipa/processor/concerns/coordinator.rb
+- lib/dipa/processor/concerns/options.rb
+- lib/dipa/processor/concerns/source.rb
+- lib/dipa/processor/concerns/state.rb
+- lib/dipa/processor/concerns/wait.rb
 - lib/dipa/processor/each.rb
 - lib/dipa/processor/map.rb
 - lib/dipa/version.rb
-- lib/tasks/auto_annotate_models.rake
 - lib/tasks/dipa_tasks.rake
 homepage: https://codeberg.org/empunkt/dipa
 licenses:
 - MIT
 metadata:
+  allowed_push_host: https://rubygems.org
+  homepage_uri: https://codeberg.org/empunkt/dipa
+  source_code_uri: https://codeberg.org/empunkt/dipa
+  changelog_uri: https://codeberg.org/empunkt/dipa/src/branch/main/CHANGELOG.md
   rubygems_mfa_required: 'true'
 post_install_message:
 rdoc_options: []
@@ -169,14 +198,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.7.0
+      version: 3.0.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">"
     - !ruby/object:Gem::Version
       version: 1.3.1
 requirements: []
-rubygems_version: 3.3.7
+rubygems_version: 3.4.14
 signing_key:
 specification_version: 4
 summary: Rails engine that provides an API to execute code in parallel and distributed