dipa 0.1.0.pre.1 → 0.1.0.pre.3

data/app/services/concerns/dipa/services/process_failure_handling.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+module Dipa
+  module Services
+    module ProcessFailureHandling
+      extend ActiveSupport::Concern
+
+      # Order of the checks is important here
+      STATES_TO_CHECK_FOR = %i[
+        aborted_through_coordinator
+        aborted_through_coordinator_timed_out
+        aborted_through_coordinator_processing_timed_out
+        timed_out
+        processing_timed_out
+      ].freeze
+
+      private
+
+      def _stop_processing?(agent:)
+        STATES_TO_CHECK_FOR.any? do |state|
+          __send__("_#{state}_state_desired?".to_sym, agent: agent)
+        end
+      end
+
+      def _stop_processing!(agent:, thread:)
+        agent_state = _determine_agent_state(agent: agent)
+
+        error_class_name = _determine_error_class_name(state: agent_state)
+
+        _end_it_all!(agent: agent, thread: thread, state: agent_state, error_class_name: error_class_name)
+      end
+
+      def _determine_error_class_name(state:)
+        "Dipa::Agent#{state.to_s.camelize}Error"
+      end
+
+      def _determine_agent_state(agent:)
+        _determine_agent_current_state(agent: agent) || _determine_agent_desired_state(agent: agent)
+      end
+
+      def _determine_agent_current_state(agent:)
+        return unless agent.reload.failed_state?
+
+        agent.state.to_sym
+      end
+
+      def _determine_agent_desired_state(agent:)
+        STATES_TO_CHECK_FOR.each do |state|
+          return state if __send__("_#{state}_state_desired?", agent: agent)
+        end
+      end
+
+      def _aborted_through_coordinator_state_desired?(agent:)
+        agent.reload.coordinator.failed_state?
+      end
+
+      def _aborted_through_coordinator_timed_out_state_desired?(agent:)
+        agent.reload.coordinator.timeout_reached?
+      end
+
+      def _aborted_through_coordinator_processing_timed_out_state_desired?(agent:)
+        agent.reload.coordinator.processing_timeout_reached?
+      end
+
+      def _timed_out_state_desired?(agent:)
+        agent.reload.timeout_reached?
+      end
+
+      def _processing_timed_out_state_desired?(agent:)
+        agent.reload.processing_timeout_reached?
+      end
+
+      def _end_it_all!(agent:, thread:, state:, error_class_name:)
+        agent.__send__("#{state}!") unless agent.reload.__send__("#{state}?")
+
+        thread.raise error_class_name.constantize
+      end
+    end
+  end
+end

data/app/services/dipa/agent_services/base_service.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Dipa
+  module AgentServices
+    class BaseService < ApplicationService
+      def call(agent:)
+        @_agent = agent
+      end
+
+      private
+
+      attr_reader :_agent
+
+      def initialize
+        super
+
+        @_agent = nil
+      end
+    end
+  end
+end

data/app/services/dipa/agent_services/coordinator_state_service.rb

@@ -2,12 +2,66 @@
 
 module Dipa
   module AgentServices
-    class CoordinatorStateService < ApplicationService
+    class CoordinatorStateService < Dipa::AgentServices::BaseService
+      AGENT_STATES_TO_CHECK_FOR = %i[
+        timed_out
+        processing_timed_out
+        processing_failed
+      ].freeze
+
+      COORDINATOR_STATES_TO_CHECK_FOR = %i[
+        aborted_through_coordinator_timed_out
+        aborted_through_coordinator_processing_timed_out
+      ].freeze
+
       def call(agent:)
-        return if agent.coordinator.processed?
-        return unless agent.coordinator.all_agents_created_and_processed?
+        super
+
+        return if _agent.reload.coordinator.finished_state?
+
+        _handle_agent_states
+      end
+
+      private
+
+      def _handle_agent_states
+        if _agent.reload.failed_state?
+          _handle_agent_failed_states
+        else
+          _handle_agent_success_states
+        end
+      end
+
+      def _determine_coordinator_failed_state
+        _determine_aborted_through_coordinator_state || _determine_aborted_through_agent_state || :aborted_through_agent
+      end
+
+      def _determine_aborted_through_agent_state
+        state = _agent.reload.state
+
+        return unless AGENT_STATES_TO_CHECK_FOR.include?(state.to_sym)
+
+        "aborted_through_agent_#{state}".to_sym
+      end
+
+      def _determine_aborted_through_coordinator_state
+        state = _agent.reload.state
+
+        return unless COORDINATOR_STATES_TO_CHECK_FOR.include?(state.to_sym)
+
+        state.delete_prefix('aborted_through_coordinator_').to_sym
+      end
+
+      def _handle_agent_failed_states
+        state = _determine_coordinator_failed_state
+
+        _agent.coordinator.__send__("#{state}!".to_sym)
+      end
+
+      def _handle_agent_success_states
+        return unless _agent.reload.coordinator.all_agents_created_and_finished?
 
-        agent.coordinator.finished!
+        _agent.coordinator.finished!
       end
     end
   end

data/app/services/dipa/agent_services/post_processing_service.rb

@@ -2,13 +2,15 @@
 
 module Dipa
   module AgentServices
-    class PostProcessingService < ApplicationService
+    class PostProcessingService < Dipa::AgentServices::BaseService
       def call(agent:)
-        Dipa::ServiceJob.set(queue_as: agent.coordinator.agent_queue)
+        super
+
+        Dipa::ServiceJob.set(queue: _agent.coordinator.agent_queue)
                         .perform_later(
                           service_class_name:
                             'Dipa::AgentServices::CoordinatorStateService',
-                          kwargs: { agent: agent }
+                          kwargs: { agent: _agent }
                         )
       end
     end

data/app/services/dipa/agent_services/processing_service.rb

@@ -2,11 +2,92 @@
 
 module Dipa
   module AgentServices
-    class ProcessingService < ApplicationService
+    class ProcessingService < Dipa::AgentServices::BaseService
+      include Dipa::Services::ProcessFailureHandling
+
+      TIMER_TASK_SLEEP_START_DURATION = 2
+      TIMER_TASK_MAX_SLEEP_DURATION = 120
+      TIMER_TASK_SLEEP_DURATION_INCREASE_FACTOR = 1.1
+      TIMER_TASK_ARGS = {
+        execution_interval: TIMER_TASK_SLEEP_START_DURATION
+      }.freeze
+
       def call(agent:)
-        agent.process!
+        super
+
+        _process!
+      ensure
+        _post_processing
+      end
+
+      private
+
+      attr_reader :_processor_thread
+
+      def initialize
+        super
+
+        @_processor_thread = Thread.current
+      end
+
+      def _process!
+        _start_monitoring_task
+
+        processor_result = _processor_result
+
+        _agent.dump_to_file(data: processor_result, attacher: :result_dump)
+        _agent.finished!
+      rescue StandardError => e
+        _agent.processing_failed! unless e.is_a?(Dipa::AgentError)
+      end
+
+      def _processor_result
+        _agent.coordinator.processor_class_name.constantize
+              .__send__(
+                _agent.coordinator.processor_method_name,
+                _agent.source
+              )
+      end
+
+      def _monitoring_task
+        @_monitoring_task ||= Concurrent::TimerTask.new(TIMER_TASK_ARGS, &_monitoring_task_iteration_block)
+      end
+
+      def _monitoring_task_iteration_block
+        proc { |task| _monitoring_task_iteration(task: task) }
+      end
+
+      def _monitoring_task_iteration(task:)
+        execution_context = Rails.application.executor.run!
+
+        _handle_execution_interval(task: task)
+
+        return unless _stop_processing?(agent: _agent)
+
+        _stop_processing!(agent: _agent, thread: _processor_thread)
+
+        task.shutdown
+      ensure
+        execution_context&.complete!
+      end
+
+      def _handle_execution_interval(task:)
+        return if _max_execution_interval_reached?(task: task)
+
+        task.execution_interval *= TIMER_TASK_SLEEP_DURATION_INCREASE_FACTOR
+      end
+
+      def _max_execution_interval_reached?(task:)
+        task.execution_interval > TIMER_TASK_MAX_SLEEP_DURATION
+      end
+
+      def _start_monitoring_task
+        _monitoring_task.execute
+      end
 
-        Dipa::AgentServices::PostProcessingService.call(agent: agent)
+      def _post_processing
+        _monitoring_task.shutdown if _monitoring_task.running?
+        Dipa::AgentServices::PostProcessingService.call(agent: _agent)
       end
     end
   end

data/app/services/dipa/agent_services/start_processing_service.rb

@@ -2,15 +2,17 @@
 
 module Dipa
   module AgentServices
-    class StartProcessingService < ApplicationService
+    class StartProcessingService < Dipa::AgentServices::BaseService
       def call(agent:)
-        agent.started!
+        super
 
-        Dipa::ServiceJob.set(queue_as: agent.coordinator.agent_queue)
+        _agent.processing!
+
+        Dipa::ServiceJob.set(queue: _agent.coordinator.agent_queue)
                         .perform_later(
                           service_class_name:
                             'Dipa::AgentServices::ProcessingService',
-                          kwargs: { agent: agent }
+                          kwargs: { agent: _agent }
                         )
       end
     end

data/app/services/dipa/coordinator_services/base_service.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Dipa
+  module CoordinatorServices
+    class BaseService < ApplicationService
+      def call(coordinator:)
+        @_coordinator = coordinator
+      end
+
+      private
+
+      attr_reader :_coordinator
+
+      def initialize
+        super
+
+        @_coordinator = nil
+      end
+    end
+  end
+end

data/app/services/dipa/coordinator_services/create_agents_service.rb

@@ -2,17 +2,19 @@
 
 module Dipa
   module CoordinatorServices
-    class CreateAgentsService < ApplicationService
+    class CreateAgentsService < Dipa::CoordinatorServices::BaseService
       def call(coordinator:)
-        coordinator.source.each_with_index do |item, i|
-          _create_agent(coordinator: coordinator, item: item, index: i)
+        super
+
+        _coordinator.source.each_with_index do |item, i|
+          _create_agent(item: item, index: i)
         end
       end
 
       private
 
-      def _create_agent(coordinator:, item:, index:)
-        agent = coordinator.agents.create!(index: index)
+      def _create_agent(item:, index:)
+        agent = _coordinator.agents.create!(index: index)
 
         agent.dump_to_file(data: item, attacher: :source_dump)
 

data/app/services/dipa/coordinator_services/destroy_service.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Dipa
+  module CoordinatorServices
+    class DestroyService < Dipa::CoordinatorServices::BaseService
+      def call(coordinator:)
+        super
+
+        _coordinator&.destroy!
+      end
+    end
+  end
+end

data/app/services/dipa/coordinator_services/maybe_cleanup_service.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Dipa
+  module CoordinatorServices
+    class MaybeCleanupService < Dipa::CoordinatorServices::BaseService
+      def call(coordinator:)
+        super
+
+        return if _coordinator.keep_data?
+
+        Dipa::ServiceJob.set(queue: _coordinator.coordinator_queue)
+                        .perform_later(
+                          service_class_name:
+                            'Dipa::CoordinatorServices::DestroyService',
+                          kwargs: { coordinator: _coordinator }
+                        )
+      end
+    end
+  end
+end

data/app/services/dipa/coordinator_services/start_processing_service.rb

@@ -2,15 +2,17 @@
 
 module Dipa
   module CoordinatorServices
-    class StartProcessingService < ApplicationService
+    class StartProcessingService < Dipa::CoordinatorServices::BaseService
       def call(coordinator:)
-        coordinator.started!
+        super
 
-        ServiceJob.set(queue_as: coordinator.coordinator_queue)
+        _coordinator.processing!
+
+        ServiceJob.set(queue: _coordinator.coordinator_queue)
                   .perform_later(
                     service_class_name:
                       'Dipa::CoordinatorServices::CreateAgentsService',
-                    kwargs: { coordinator: coordinator }
+                    kwargs: { coordinator: _coordinator }
                   )
       end
     end

data/db/migrate/20220102132652_create_dipa_coordinators.rb

@@ -3,10 +3,8 @@
 class CreateDipaCoordinators < ActiveRecord::Migration[6.0]
   def change # rubocop:disable Metrics/MethodLength
     create_table :dipa_coordinators do |t|
-      t.boolean :keep_data, default: false, null: false
-      t.boolean :want_result, default: true, null: false
-      t.datetime :finished_at
-      t.datetime :started_at
+      t.datetime :finished_at, precision: Dipa.datetime_precision
+      t.datetime :started_at, precision: Dipa.datetime_precision
       t.integer :size, null: false
       t.string :agent_queue, null: false
       t.string :coordinator_queue, null: false
@@ -14,7 +12,7 @@ class CreateDipaCoordinators < ActiveRecord::Migration[6.0]
       t.string :processor_method_name, null: false
       t.string :state, null: false
 
-      t.timestamps
+      t.timestamps precision: Dipa.datetime_precision
     end
   end
 end

data/db/migrate/20220106183616_create_dipa_agents.rb

@@ -3,12 +3,12 @@
 class CreateDipaAgents < ActiveRecord::Migration[6.0]
   def change # rubocop:disable Metrics/MethodLength
     create_table :dipa_agents do |t|
-      t.datetime :finished_at
-      t.datetime :started_at
+      t.datetime :finished_at, precision: Dipa.datetime_precision
+      t.datetime :started_at, precision: Dipa.datetime_precision
       t.integer :index, null: false
       t.string :state, null: false
 
-      t.timestamps
+      t.timestamps precision: Dipa.datetime_precision
     end
 
     add_belongs_to(

data/db/migrate/20230624053724_create_dipa_coordinator_options_record.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+class CreateDipaCoordinatorOptionsRecord < ActiveRecord::Migration[6.0]
+  def change # rubocop:disable Metrics/MethodLength
+    create_table :dipa_coordinator_options_records do |t|
+      t.boolean :keep_data, default: false, null: false
+      t.boolean :want_result, default: true, null: false
+      t.integer :agent_timeout, null: false
+      t.integer :agent_processing_timeout, null: false
+      t.integer :coordinator_timeout, null: false
+      t.integer :coordinator_processing_timeout, null: false
+
+      t.timestamps precision: Dipa.datetime_precision
+    end
+
+    add_belongs_to(
+      :dipa_coordinator_options_records, :dipa_coordinator,
+      foreign_key: true, index: true, null: false
+    )
+  end
+end

data/lib/dipa/engine.rb

@@ -22,14 +22,10 @@ module Dipa
     initializer 'dipa.queue_names' do
       config.after_initialize do |app|
         Dipa.agent_queue = (
-          app.config.dipa.agent_queue ||
-          app.config.active_job.default_queue_name ||
-          :default
+          app.config.dipa.agent_queue || app.config.active_job.default_queue_name || :default
         ).to_sym
         Dipa.coordinator_queue = (
-          app.config.dipa.coordinator_queue ||
-          app.config.active_job.default_queue_name ||
-          :default
+          app.config.dipa.coordinator_queue || app.config.active_job.default_queue_name || :default
         ).to_sym
       end
     end
@@ -39,11 +35,22 @@ module Dipa
         Dipa.agent_timeout = (
           app.config.dipa.agent_timeout || Dipa::DEFAULT_AGENT_TIMEOUT
         ).to_i
+        Dipa.agent_processing_timeout = (
+          app.config.dipa.agent_processing_timeout || Dipa::DEFAULT_AGENT_PROCESSING_TIMEOUT
+        ).to_i
         Dipa.coordinator_timeout = (
-          app.config.dipa.coordinator_timeout ||
-          Dipa::DEFAULT_COORDINATOR_TIMEOUT
+          app.config.dipa.coordinator_timeout || Dipa::DEFAULT_COORDINATOR_TIMEOUT
+        ).to_i
+        Dipa.coordinator_processing_timeout = (
+          app.config.dipa.coordinator_processing_timeout || Dipa::DEFAULT_COORDINATOR_PROCESSING_TIMEOUT
         ).to_i
       end
     end
+
+    initializer 'dipa.datetime_precision' do
+      config.after_initialize do
+        Dipa.datetime_precision = Dipa::DATETIME_PRECISION
+      end
+    end
   end
 end

data/lib/dipa/errors.rb

@@ -3,10 +3,27 @@
 module Dipa
   class Error < StandardError; end
 
-  class AbortedError < Error; end
-  class ProcessingFailedError < Error; end
-  class TimeoutError < Error; end
+  class AlreadyProcessingError < Error; end
   class UnknownProcessingStateError < Error; end
+
+  class AgentError < Error; end
+  class AgentAbortedThroughCoordinatorError < AgentError; end
+  class AgentAbortedThroughCoordinatorTimedOutError < AgentAbortedThroughCoordinatorError; end
+  class AgentAbortedThroughCoordinatorProcessingTimedOutError < AgentAbortedThroughCoordinatorError; end
+  class AgentProcessingTimedOutError < AgentError; end
+  class AgentTimedOutError < AgentError; end
+
+  class CoordinatorError < Error; end
+  class CoordinatorAbortedError < CoordinatorError; end
+  class CoordinatorAbortedThroughAgentError < CoordinatorError; end
+  class CoordinatorAbortedThroughAgentTimedOutError < CoordinatorAbortedThroughAgentError; end
+  class CoordinatorAbortedThroughAgentProcessingTimedOutError < CoordinatorAbortedThroughAgentError; end
+  class CoordinatorAbortedThroughAgentProcessingFailedError < CoordinatorAbortedThroughAgentError; end
+  class CoordinatorAbortedThroughAgentProcessingFailureError < CoordinatorAbortedThroughAgentError; end
+  class CoordinatorProcessingFailedError < CoordinatorError; end
+  class CoordinatorProcessingTimedOutError < CoordinatorError; end
+  class CoordinatorTimedOutError < CoordinatorError; end
+
   class UnknownProcessorClassError < Error; end
   class UnknownProcessorMethodError < Error; end
 end