acidic_job 1.0.0.rc2 → 1.0.0.rc4

data/lib/acidic_job/context.rb

@@ -8,14 +8,31 @@ module AcidicJob
 
     def set(hash)
       AcidicJob.instrument(:set_context, **hash) do
-        AcidicJob::Value.upsert_all(
-          hash.map do |key, value|
-            { execution_id: @execution.id,
-              key: key,
-              value: value }
-          end,
-          unique_by: %i[execution_id key]
-        )
+        records = hash.map do |key, value|
+          {
+            execution_id: @execution.id,
+            key: key,
+            value: value,
+          }
+        end
+
+        case AcidicJob::Value.connection.adapter_name.downcase.to_sym
+        when :postgresql, :sqlite
+          AcidicJob::Value.upsert_all(records, unique_by: [:execution_id, :key])
+        when :mysql2, :mysql, :trilogy
+          AcidicJob::Value.upsert_all(records)
+        else
+          # Fallback for other adapters - try with unique_by first, fall back without
+          begin
+            AcidicJob::Value.upsert_all(records, unique_by: [:execution_id, :key])
+          rescue ArgumentError => e
+            if e.message.include?('does not support :unique_by')
+              AcidicJob::Value.upsert_all(records)
+            else
+              raise
+            end
+          end
+        end
       end
     end
 
@@ -25,22 +42,21 @@ module AcidicJob
       end
     end
 
-    # TODO: deprecate these methods
+    def fetch(key, default = nil)
+      result = get(key).first
+      return result if result
+
+      fallback = default || yield(key)
+      set(key => fallback)
+      fallback
+    end
+
     def []=(key, value)
-      AcidicJob.instrument(:set_context, key: key, value: value) do
-        AcidicJob::Value.upsert(
-          { execution_id: @execution.id,
-            key: key,
-            value: value },
-          unique_by: %i[execution_id key]
-        )
-      end
+      set(key => value)
     end
 
     def [](key)
-      AcidicJob.instrument(:get_context, key: key) do
-        @execution.values.select(:value).find_by(key: key)&.value
-      end
+      get(key).first
     end
   end
 end

data/lib/acidic_job/errors.rb

@@ -28,7 +28,6 @@ module AcidicJob
     end
   end
 
-  # rubocop:disable Lint/MissingSuper
   class ArgumentMismatchError < Error
     def initialize(expected, existing)
       @expected = expected
@@ -98,5 +97,27 @@ module AcidicJob
       "step method cannot expect arguments: #{@step.inspect}"
     end
   end
+
+  class DoublePluginCallError < Error
+    def initialize(plugin, step)
+      @plugin_name = (Module === plugin) ? plugin.name : plugin.class.name
+      @step = step
+    end
+
+    def message
+      "plugin `#{@plugin_name}` attempted to call step multiple times: #{@step.inspect}"
+    end
+  end
+
+  class MissingPluginCallError < Error
+    def initialize(plugin, step)
+      @plugin_name = (Module === plugin) ? plugin.name : plugin.class.name
+      @step = step
+    end
+
+    def message
+      "plugin `#{@plugin_name}` failed to call step: #{@step.inspect}"
+    end
+  end
   # rubocop:enable Lint/MissingSuper
 end

data/lib/acidic_job/log_subscriber.rb

@@ -5,45 +5,43 @@ require "active_support/log_subscriber"
 module AcidicJob
   class LogSubscriber < ActiveSupport::LogSubscriber
     def define_workflow(event)
-      debug formatted_event(event, action: "Define workflow", **event.payload.slice("job_class", "job_id"))
+      debug formatted_event(event, title: "Define workflow", **event.payload.slice(:job_class, :job_id))
     end
 
     def initialize_workflow(event)
-      debug formatted_event(event, action: "Initialize workflow", **event.payload.slice("steps"))
+      debug formatted_event(event, title: "Initialize workflow", **event.payload.slice(:steps))
     end
 
     def process_workflow(event)
-      debug formatted_event(event, action: "Process workflow", **event.payload["execution"].slice("id", "recover_to"))
+      debug formatted_event(event, title: "Process workflow", **event.payload[:execution].slice(:id, :recover_to))
     end
 
     def process_step(event)
-      debug formatted_event(event, action: "Process step", **event.payload)
+      debug formatted_event(event, title: "Process step", **event.payload)
     end
 
     def perform_step(event)
-      debug formatted_event(event, action: "Perform step", **event.payload)
+      debug formatted_event(event, title: "Perform step", **event.payload)
     end
 
     def record_entry(event)
-      debug formatted_event(event, action: "Record entry", **event.payload.slice(:step, :action, :timestamp))
+      debug formatted_event(event, title: "Record entry", **event.payload.slice(:step, :action, :timestamp))
     end
 
-    private
-
-    def formatted_event(event, action:, **attributes)
-      "AcidicJob-#{AcidicJob::VERSION} #{action} (#{event.duration.round(1)}ms)  #{formatted_attributes(**attributes)}"
+    private def formatted_event(event, title:, **attributes)
+      "AcidicJob-#{AcidicJob::VERSION} #{title} (#{event.duration.round(1)}ms)  #{formatted_attributes(**attributes)}"
     end
 
-    def formatted_attributes(**attributes)
+    private def formatted_attributes(**attributes)
       attributes.map { |attr, value| "#{attr}: #{value.inspect}" }.join(", ")
     end
 
-    def formatted_error(error)
+    private def formatted_error(error)
       [error.class, error.message].compact.join(" ")
     end
 
     # Use the logger configured for AcidicJob
-    def logger
+    private def logger
       AcidicJob.logger
     end
   end

data/lib/acidic_job/plugin_context.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+module AcidicJob
+  class PluginContext
+    PLUGIN_INACTIVE = :__ACIDIC_JOB_PLUGIN_INACTIVE__
+
+    def initialize(plugin, job, execution, context, step_definition)
+      @plugin = plugin
+      @job = job
+      @execution = execution
+      @context = context
+      @step_definition = step_definition
+    end
+
+    def set(hash)
+      @context.set(hash)
+    end
+
+    def get(*keys)
+      @context.get(*keys)
+    end
+
+    def definition
+      @step_definition.fetch(@plugin.keyword.to_s, PLUGIN_INACTIVE)
+    end
+
+    def current_step
+      @step_definition["does"]
+    end
+
+    def inactive?
+      definition == PLUGIN_INACTIVE
+    end
+
+    def entries_for_action(action)
+      @execution.entries.for_action(plugin_action(action))
+    end
+
+    def record!(step:, action:, timestamp:, **kwargs)
+      @execution.record!(
+        step: step,
+        action: plugin_action(action),
+        timestamp: timestamp,
+        **kwargs
+      )
+    end
+
+    def enqueue_job(...)
+      @job.enqueue(...)
+    end
+
+    def halt_workflow!
+      @job.halt_workflow!
+    end
+
+    def repeat_step!
+      @job.repeat_step!
+    end
+
+    def resolve_method(method_name)
+      begin
+        method_obj = @job.method(method_name)
+      rescue NameError
+        raise UndefinedMethodError.new(method_name)
+      end
+
+      method_obj
+    end
+
+    def plugin_action(action)
+      "#{@plugin.keyword}/#{action}"
+    end
+  end
+end

data/lib/acidic_job/plugins/transactional_step.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module AcidicJob
+  module Plugins
+    module TransactionalStep
+      extend self
+
+      def keyword
+        :transactional
+      end
+
+      # transactional: true
+      # transactional: false
+      # transactional: { on: Model }
+      def validate(input)
+        return input if input in true | false
+
+        raise ArgumentError.new("argument must be boolean or hash") unless input in Hash
+        raise ArgumentError.new("argument hash must have `on` key") unless input in Hash[on:]
+        raise ArgumentError.new("`on` key must have module value") unless input in Hash[on: Module]
+
+        input
+      end
+
+      def around_step(context, &block)
+        return yield if context.definition == false
+
+        model = if context.definition == true
+          AcidicJob::Execution
+        else
+          context.definition["on"].constantize
+        end
+
+        model.transaction(&block)
+      end
+    end
+  end
+end

data/lib/acidic_job/serializer.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require "json"
+require_relative "arguments"
+
+module AcidicJob
+  # Used for `serialize` method in ActiveRecord
+  module Serializer
+    extend self
+
+    def load(json)
+      return if json.nil? || json.empty?
+
+      data = JSON.parse json
+
+      Arguments.__send__ :deserialize_argument, data
+    end
+
+    def dump(obj)
+      data = Arguments.__send__ :serialize_argument, obj
+
+      JSON.generate data, strict: true
+    rescue ActiveJob::SerializationError => e
+      e.message << " (`#{obj.inspect}`)"
+      raise e
+    end
+  end
+end

data/lib/acidic_job/serializers/job_serializer.rb

@@ -15,7 +15,7 @@ module AcidicJob
 
       def deserialize(hash)
         job = ::ActiveJob::Base.deserialize(hash)
-        job.send(:deserialize_arguments_if_needed)
+        job.__send__(:deserialize_arguments_if_needed)
         job
       end
 

data/lib/acidic_job/serializers/range_serializer.rb

@@ -17,9 +17,7 @@ module AcidicJob
         klass.new(*Arguments.deserialize(hash.values_at(*KEYS)))
       end
 
-      private
-
-      def klass
+      private def klass
         ::Range
       end
     end

data/lib/acidic_job/testing.rb

@@ -22,25 +22,23 @@ module AcidicJob
       ::DatabaseCleaner.cleaners = @original_cleaners
     end
 
-    private
-
     # Ensure that the system's original DatabaseCleaner configuration is maintained, options included,
     # except that any `transaction` strategies for any ORMs are replaced with a `deletion` strategy.
-    def transaction_free_cleaners_for(original_cleaners)
+    private def transaction_free_cleaners_for(original_cleaners)
       non_transaction_cleaners = original_cleaners.dup.to_h do |(orm, opts), cleaner|
         [[orm, opts], ensure_no_transaction_strategies_for(cleaner)]
       end
       ::DatabaseCleaner::Cleaners.new(non_transaction_cleaners)
     end
 
-    def ensure_no_transaction_strategies_for(cleaner)
+    private def ensure_no_transaction_strategies_for(cleaner)
       return cleaner unless strategy_name_for(cleaner) == "transaction"
 
       cleaner.strategy = deletion_strategy_for(cleaner)
       cleaner
     end
 
-    def strategy_name_for(cleaner)
+    private def strategy_name_for(cleaner)
       cleaner               # <DatabaseCleaner::Cleaner>
         .strategy           # <DatabaseCleaner::ActiveRecord::Truncation>
         .class              # DatabaseCleaner::ActiveRecord::Truncation
@@ -50,19 +48,19 @@ module AcidicJob
         .downcase           # "truncation"
     end
 
-    def deletion_strategy_for(cleaner)
+    private def deletion_strategy_for(cleaner)
       strategy = cleaner.strategy
-      strategy_namespace = strategy                    # <DatabaseCleaner::ActiveRecord::Truncation>
-                           .class                      # DatabaseCleaner::ActiveRecord::Truncation
-                           .name                       # "DatabaseCleaner::ActiveRecord::Truncation"
-                           .rpartition("::")           # ["DatabaseCleaner::ActiveRecord", "::", "Truncation"]
-                           .first                      # "DatabaseCleaner::ActiveRecord"
+      strategy_namespace = strategy # <DatabaseCleaner::ActiveRecord::Truncation>
+        .class                      # DatabaseCleaner::ActiveRecord::Truncation
+        .name                       # "DatabaseCleaner::ActiveRecord::Truncation"
+        .rpartition("::")           # ["DatabaseCleaner::ActiveRecord", "::", "Truncation"]
+        .first                      # "DatabaseCleaner::ActiveRecord"
       deletion_strategy_class_name = [strategy_namespace, "::", "Deletion"].join
       deletion_strategy_class = deletion_strategy_class_name.constantize
       instance_variable_hash = strategy.instance_variables.to_h do |var|
         [
           var.to_s.remove("@"),
-          strategy.instance_variable_get(var)
+          strategy.instance_variable_get(var),
         ]
       end
       options = instance_variable_hash.except("db", "connection_class")

data/lib/acidic_job/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module AcidicJob
-  VERSION = "1.0.0.rc2"
+  VERSION = "1.0.0.rc4"
 end

data/lib/acidic_job/workflow.rb

@@ -4,43 +4,41 @@ require "active_job"
 
 module AcidicJob
   module Workflow
-    NO_OP_WRAPPER = proc { |&block| block.call }
-    REPEAT_STEP = :REPEAT_STEP
-    HALT_STEP = :HALT_STEP
-    private_constant :NO_OP_WRAPPER, :REPEAT_STEP, :HALT_STEP
+    REPEAT_STEP = :__ACIDIC_JOB_REPEAT_STEP_SIGNAL__
+    HALT_STEP = :__ACIDIC_JOB_HALT_STEP_SIGNAL__
+    private_constant :REPEAT_STEP, :HALT_STEP
 
-    attr_reader :execution, :ctx
-
-    def execute_workflow(unique_by:, &block)
+    def execute_workflow(unique_by:, with: AcidicJob.plugins, &block)
+      @__acidic_job_plugins__ = with
       serialized_job = serialize
 
       workflow_definition = AcidicJob.instrument(:define_workflow, **serialized_job) do
-        raise RedefiningWorkflowError if defined? @_builder
+        raise RedefiningWorkflowError if defined? @__acidic_job_builder__
 
-        @_builder = Builder.new
+        @__acidic_job_builder__ = Builder.new(@__acidic_job_plugins__)
 
         raise UndefinedWorkflowBlockError unless block_given?
         raise InvalidWorkflowBlockError if block.arity != 1
 
-        block.call @_builder
+        block.call @__acidic_job_builder__
 
-        raise MissingStepsError if @_builder.steps.empty?
+        raise MissingStepsError if @__acidic_job_builder__.steps.empty?
 
         # convert the array of steps into a hash of recovery_points and next steps
-        @_builder.define_workflow
+        @__acidic_job_builder__.define_workflow
       end
 
-      AcidicJob.instrument(:initialize_workflow, "definition" => workflow_definition) do
+      AcidicJob.instrument(:initialize_workflow, definition: workflow_definition) do
         transaction_args = case ::ActiveRecord::Base.connection.adapter_name.downcase.to_sym
-                           # SQLite doesn't support `serializable` transactions
-                           when :sqlite
-                             {}
-                           else
-                             { isolation: :serializable }
-                           end
-        idempotency_key = Digest::SHA256.hexdigest(JSON.fast_generate([self.class.name, unique_by], strict: true))
-
-        @execution = ::ActiveRecord::Base.transaction(**transaction_args) do
+          # SQLite doesn't support `serializable` transactions
+          when :sqlite
+            {}
+          else
+            { isolation: :serializable }
+        end
+        idempotency_key = Digest::SHA256.hexdigest(JSON.generate([self.class.name, unique_by], strict: true))
+
+        @__acidic_job_execution__ = ::ActiveRecord::Base.transaction(**transaction_args) do
           record = Execution.find_by(idempotency_key: idempotency_key)
 
           if record.present?
@@ -62,41 +60,51 @@ module AcidicJob
               last_run_at: Time.current
             )
           else
+            starting_point = if workflow_definition.key?("steps")
+              workflow_definition["steps"].keys.first
+            else
+              # TODO: add deprecation warning
+              workflow_definition.keys.first
+            end
+
             record = Execution.create!(
               idempotency_key: idempotency_key,
               serialized_job: serialized_job,
               definition: workflow_definition,
-              recover_to: workflow_definition.keys.first
+              recover_to: starting_point
             )
           end
 
           record
         end
       end
-      @ctx ||= Context.new(@execution)
+      @__acidic_job_context__ ||= Context.new(@__acidic_job_execution__)
 
-      AcidicJob.instrument(:process_workflow, execution: @execution.attributes) do
+      AcidicJob.instrument(:process_workflow, execution: @__acidic_job_execution__.attributes) do
         # if the workflow record is already marked as finished, immediately return its result
-        return true if @execution.finished?
+        return true if @__acidic_job_execution__.finished?
 
         loop do
-          break if @execution.finished?
+          break if @__acidic_job_execution__.finished?
 
-          current_step = @execution.recover_to
+          current_step = @__acidic_job_execution__.recover_to
 
-          if not @execution.definition.key?(current_step) # rubocop:disable Style/Not
+          if not @__acidic_job_execution__.defined?(current_step)
             raise UndefinedStepError.new(current_step)
           end
 
-          step_definition = @execution.definition[current_step]
+          step_definition = @__acidic_job_execution__.definition_for(current_step)
           AcidicJob.instrument(:process_step, **step_definition) do
             recover_to = catch(:halt) { take_step(step_definition) }
             case recover_to
             when HALT_STEP
-              @execution.record!(step: step_definition.fetch("does"), action: :halted, timestamp: Time.now)
+              @__acidic_job_execution__.record!(
+                step: step_definition.fetch("does"),
+                action: :halted,
+              )
               return true
             else
-              @execution.update!(recover_to: recover_to)
+              @__acidic_job_execution__.update_column(:recover_to, recover_to)
             end
           end
         end
@@ -107,31 +115,42 @@ module AcidicJob
       throw :repeat, REPEAT_STEP
     end
 
-    def halt_step!
+    def halt_workflow!
       throw :halt, HALT_STEP
     end
 
+    def halt_step!
+      # TODO add deprecation warning
+      halt_workflow!
+    end
+
     def step_retrying?
       step_name = caller_locations.first.label
 
-      if not @execution.definition.key?(step_name) # rubocop:disable Style/IfUnlessModifier, Style/Not
+      if not @__acidic_job_execution__.defined?(step_name)
         raise UndefinedStepError.new(step_name)
       end
 
-      @execution.entries.where(step: step_name, action: "started").count > 1
+      @__acidic_job_execution__.entries.where(step: step_name, action: "started").count > 1
     end
 
-    private
+    def execution
+      @__acidic_job_execution__
+    end
+
+    def ctx
+      @__acidic_job_context__
+    end
 
-    def take_step(step_definition)
+    private def take_step(step_definition)
       curr_step = step_definition.fetch("does")
       next_step = step_definition.fetch("then")
 
-      return next_step if @execution.entries.exists?(step: curr_step, action: :succeeded)
+      return next_step if @__acidic_job_execution__.entries.exists?(step: curr_step, action: :succeeded)
 
       rescued_error = nil
       begin
-        @execution.record!(step: curr_step, action: :started, timestamp: Time.now)
+        @__acidic_job_execution__.record!(step: curr_step, action: :started)
         result = AcidicJob.instrument(:perform_step, **step_definition) do
           perform_step_for(step_definition)
         end
@@ -139,44 +158,77 @@ module AcidicJob
         when REPEAT_STEP
           curr_step
         else
-          @execution.record!(step: curr_step, action: :succeeded, timestamp: Time.now, result: result)
+          @__acidic_job_execution__.record!(
+            step: curr_step,
+            action: :succeeded,
+            ignored: {
+              result: result,
+            }
+          )
           next_step
         end
-      rescue StandardError => e
+      rescue => e
         rescued_error = e
         raise e
       ensure
         if rescued_error
           begin
-            @execution.record!(
+            @__acidic_job_execution__.record!(
               step: curr_step,
               action: :errored,
-              timestamp: Time.now,
               exception_class: rescued_error.class.name,
               message: rescued_error.message
             )
-          rescue StandardError => e
+          rescue => e
             # We're already inside an error condition, so swallow any additional
             # errors from here and just send them to logs.
             logger.error(
-              "Failed to store exception at step #{curr_step} for execution ##{@execution.id} because of #{e}."
+              "Failed to store exception at step #{curr_step} for execution ##{@__acidic_job_execution__.id}: #{e}."
             )
           end
         end
       end
     end
 
-    def perform_step_for(step_definition)
+    private def perform_step_for(step_definition)
       step_name = step_definition.fetch("does")
-      step_method = method(step_name)
+      begin
+        step_method = method(step_name)
+      rescue NameError
+        raise UndefinedMethodError.new(step_name)
+      end
+
+      # raise InvalidMethodError.new(step_name) unless step_method.arity.zero?
+
+      plugin_pipeline_callable = @__acidic_job_plugins__.reverse.reduce(step_method) do |callable, plugin|
+        context = PluginContext.new(plugin, self, @__acidic_job_execution__, @__acidic_job_context__, step_definition)
+
+        if context.inactive?
+          callable
+        else
+          proc do
+            called = false
+
+            result = plugin.around_step(context) do |*args, **kwargs|
+              raise DoublePluginCallError.new(plugin, step_name) if called
 
-      raise InvalidMethodError.new(step_name) unless step_method.arity.zero?
+              called = true
 
-      wrapper = step_definition["transactional"] ? @execution.method(:with_lock) : NO_OP_WRAPPER
+              if callable.arity.zero?
+                callable.call
+              else
+                callable.call(*args, **kwargs)
+              end
+            end
+
+            # raise MissingPluginCallError.new(plugin, step_name) unless called
+
+            result
+          end
+        end
+      end
 
-      catch(:repeat) { wrapper.call { step_method.call } }
-    rescue NameError
-      raise UndefinedMethodError.new(step_name)
+      catch(:repeat) { plugin_pipeline_callable.call }
     end
   end
 end