acidic_job 1.0.0.pre29 → 1.0.0.rc2

data/lib/acidic_job/serializers/job_serializer.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require "active_job/serializers/object_serializer"
+
+module AcidicJob
+  module Serializers
+    class JobSerializer < ::ActiveJob::Serializers::ObjectSerializer
+      def serialize(job)
+        # don't serialize the `enqueued_at` value, as ActiveRecord will check if the Run record has changed
+        # by comparing the deserialized database value with a temporary in-memory generated value.
+        # That temporary in-memory generated value can sometimes have an `enqueued_at` value that is 1 second off
+        # from the original. In this case, ActiveRecord will think the record has unsaved changes and block the lock.
+        super(job.serialize.except("enqueued_at"))
+      end
+
+      def deserialize(hash)
+        job = ::ActiveJob::Base.deserialize(hash)
+        job.send(:deserialize_arguments_if_needed)
+        job
+      end
+
+      def serialize?(argument)
+        defined?(::ActiveJob::Base) && argument.class < ::ActiveJob::Base
+      end
+    end
+  end
+end

data/lib/acidic_job/serializers/new_record_serializer.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+require "active_job/serializers/object_serializer"
+
+module AcidicJob
+  module Serializers
+    class NewRecordSerializer < ::ActiveJob::Serializers::ObjectSerializer
+      def serialize(new_record)
+        super(
+          "class" => new_record.class.name,
+          "attributes" => new_record.attributes
+        )
+      end
+
+      def deserialize(hash)
+        new_record_class = hash["class"].constantize
+        new_record_class.new(hash["attributes"])
+      end
+
+      def serialize?(argument)
+        defined?(::ActiveRecord) && argument.respond_to?(:new_record?) && argument.new_record?
+      end
+    end
+  end
+end

data/lib/acidic_job/serializers/range_serializer.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require "active_job/serializers/object_serializer"
+
+# :nocov:
+module AcidicJob
+  module Serializers
+    class RangeSerializer < ::ActiveJob::Serializers::ObjectSerializer
+      KEYS = %w[begin end exclude_end].freeze
+
+      def serialize(range)
+        args = Arguments.serialize([range.begin, range.end, range.exclude_end?])
+        super(KEYS.zip(args).to_h)
+      end
+
+      def deserialize(hash)
+        klass.new(*Arguments.deserialize(hash.values_at(*KEYS)))
+      end
+
+      private
+
+      def klass
+        ::Range
+      end
+    end
+  end
+end
+# :nocov:

data/lib/acidic_job/testing.rb

@@ -1,9 +1,5 @@
 # frozen_string_literal: true
 
-require "active_job/queue_adapters"
-require "active_job/base"
-require "active_job/test_helper"
-require "active_job/test_case"
 require "database_cleaner"
 
 module AcidicJob
@@ -13,17 +9,17 @@ module AcidicJob
     end
 
     def before_setup
-      @connection = ActiveRecord::Base.connection
-      @original_cleaners = DatabaseCleaner.cleaners
-      DatabaseCleaner.cleaners = transaction_free_cleaners_for(@original_cleaners)
+      @connection = ::ActiveRecord::Base.connection
+      @original_cleaners = ::DatabaseCleaner.cleaners
+      ::DatabaseCleaner.cleaners = transaction_free_cleaners_for(@original_cleaners)
       super
-      DatabaseCleaner.start
+      ::DatabaseCleaner.start
     end
 
     def after_teardown
-      DatabaseCleaner.clean
+      ::DatabaseCleaner.clean
       super
-      DatabaseCleaner.cleaners = @original_cleaners
+      ::DatabaseCleaner.cleaners = @original_cleaners
     end
 
     private
@@ -34,7 +30,7 @@ module AcidicJob
       non_transaction_cleaners = original_cleaners.dup.to_h do |(orm, opts), cleaner|
         [[orm, opts], ensure_no_transaction_strategies_for(cleaner)]
       end
-      DatabaseCleaner::Cleaners.new(non_transaction_cleaners)
+      ::DatabaseCleaner::Cleaners.new(non_transaction_cleaners)
     end
 
     def ensure_no_transaction_strategies_for(cleaner)
@@ -56,7 +52,7 @@ module AcidicJob
 
     def deletion_strategy_for(cleaner)
       strategy = cleaner.strategy
-      strategy_namespace = strategy # <DatabaseCleaner::ActiveRecord::Truncation>
+      strategy_namespace = strategy                    # <DatabaseCleaner::ActiveRecord::Truncation>
                            .class                      # DatabaseCleaner::ActiveRecord::Truncation
                            .name                       # "DatabaseCleaner::ActiveRecord::Truncation"
                            .rpartition("::")           # ["DatabaseCleaner::ActiveRecord", "::", "Truncation"]

data/lib/acidic_job/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module AcidicJob
-  VERSION = "1.0.0.pre29"
+  VERSION = "1.0.0.rc2"
 end

data/lib/acidic_job/workflow.rb

@@ -0,0 +1,182 @@
+# frozen_string_literal: true
+
+require "active_job"
+
+module AcidicJob
+  module Workflow
+    NO_OP_WRAPPER = proc { |&block| block.call }
+    REPEAT_STEP = :REPEAT_STEP
+    HALT_STEP = :HALT_STEP
+    private_constant :NO_OP_WRAPPER, :REPEAT_STEP, :HALT_STEP
+
+    attr_reader :execution, :ctx
+
+    def execute_workflow(unique_by:, &block)
+      serialized_job = serialize
+
+      workflow_definition = AcidicJob.instrument(:define_workflow, **serialized_job) do
+        raise RedefiningWorkflowError if defined? @_builder
+
+        @_builder = Builder.new
+
+        raise UndefinedWorkflowBlockError unless block_given?
+        raise InvalidWorkflowBlockError if block.arity != 1
+
+        block.call @_builder
+
+        raise MissingStepsError if @_builder.steps.empty?
+
+        # convert the array of steps into a hash of recovery_points and next steps
+        @_builder.define_workflow
+      end
+
+      AcidicJob.instrument(:initialize_workflow, "definition" => workflow_definition) do
+        transaction_args = case ::ActiveRecord::Base.connection.adapter_name.downcase.to_sym
+                           # SQLite doesn't support `serializable` transactions
+                           when :sqlite
+                             {}
+                           else
+                             { isolation: :serializable }
+                           end
+        idempotency_key = Digest::SHA256.hexdigest(JSON.fast_generate([self.class.name, unique_by], strict: true))
+
+        @execution = ::ActiveRecord::Base.transaction(**transaction_args) do
+          record = Execution.find_by(idempotency_key: idempotency_key)
+
+          if record.present?
+            # Programs enqueuing multiple jobs with different parameters but the
+            # same idempotency key is a bug.
+            if record.raw_arguments != serialized_job["arguments"]
+              raise ArgumentMismatchError.new(serialized_job["arguments"], record.raw_arguments)
+            end
+
+            if record.definition != workflow_definition
+              raise DefinitionMismatchError.new(workflow_definition, record.definition)
+            end
+
+            # Only acquire a lock if the key is unlocked or its lock has expired
+            # because the original job was long enough ago.
+            # raise "LockedIdempotencyKey" if record.locked_at > Time.current - 2.seconds
+
+            record.update!(
+              last_run_at: Time.current
+            )
+          else
+            record = Execution.create!(
+              idempotency_key: idempotency_key,
+              serialized_job: serialized_job,
+              definition: workflow_definition,
+              recover_to: workflow_definition.keys.first
+            )
+          end
+
+          record
+        end
+      end
+      @ctx ||= Context.new(@execution)
+
+      AcidicJob.instrument(:process_workflow, execution: @execution.attributes) do
+        # if the workflow record is already marked as finished, immediately return its result
+        return true if @execution.finished?
+
+        loop do
+          break if @execution.finished?
+
+          current_step = @execution.recover_to
+
+          if not @execution.definition.key?(current_step) # rubocop:disable Style/Not
+            raise UndefinedStepError.new(current_step)
+          end
+
+          step_definition = @execution.definition[current_step]
+          AcidicJob.instrument(:process_step, **step_definition) do
+            recover_to = catch(:halt) { take_step(step_definition) }
+            case recover_to
+            when HALT_STEP
+              @execution.record!(step: step_definition.fetch("does"), action: :halted, timestamp: Time.now)
+              return true
+            else
+              @execution.update!(recover_to: recover_to)
+            end
+          end
+        end
+      end
+    end
+
+    def repeat_step!
+      throw :repeat, REPEAT_STEP
+    end
+
+    def halt_step!
+      throw :halt, HALT_STEP
+    end
+
+    def step_retrying?
+      step_name = caller_locations.first.label
+
+      if not @execution.definition.key?(step_name) # rubocop:disable Style/IfUnlessModifier, Style/Not
+        raise UndefinedStepError.new(step_name)
+      end
+
+      @execution.entries.where(step: step_name, action: "started").count > 1
+    end
+
+    private
+
+    def take_step(step_definition)
+      curr_step = step_definition.fetch("does")
+      next_step = step_definition.fetch("then")
+
+      return next_step if @execution.entries.exists?(step: curr_step, action: :succeeded)
+
+      rescued_error = nil
+      begin
+        @execution.record!(step: curr_step, action: :started, timestamp: Time.now)
+        result = AcidicJob.instrument(:perform_step, **step_definition) do
+          perform_step_for(step_definition)
+        end
+        case result
+        when REPEAT_STEP
+          curr_step
+        else
+          @execution.record!(step: curr_step, action: :succeeded, timestamp: Time.now, result: result)
+          next_step
+        end
+      rescue StandardError => e
+        rescued_error = e
+        raise e
+      ensure
+        if rescued_error
+          begin
+            @execution.record!(
+              step: curr_step,
+              action: :errored,
+              timestamp: Time.now,
+              exception_class: rescued_error.class.name,
+              message: rescued_error.message
+            )
+          rescue StandardError => e
+            # We're already inside an error condition, so swallow any additional
+            # errors from here and just send them to logs.
+            logger.error(
+              "Failed to store exception at step #{curr_step} for execution ##{@execution.id} because of #{e}."
+            )
+          end
+        end
+      end
+    end
+
+    def perform_step_for(step_definition)
+      step_name = step_definition.fetch("does")
+      step_method = method(step_name)
+
+      raise InvalidMethodError.new(step_name) unless step_method.arity.zero?
+
+      wrapper = step_definition["transactional"] ? @execution.method(:with_lock) : NO_OP_WRAPPER
+
+      catch(:repeat) { wrapper.call { step_method.call } }
+    rescue NameError
+      raise UndefinedMethodError.new(step_name)
+    end
+  end
+end

data/lib/acidic_job.rb

@@ -1,297 +1,28 @@
 # frozen_string_literal: true
 
 require_relative "acidic_job/version"
+require_relative "acidic_job/engine"
 require_relative "acidic_job/errors"
-require_relative "acidic_job/recovery_point"
-require_relative "acidic_job/finished_point"
-require_relative "acidic_job/run"
-require_relative "acidic_job/step"
-require_relative "acidic_job/staging"
-require_relative "acidic_job/awaiting"
-require_relative "acidic_job/perform_wrapper"
-require_relative "acidic_job/idempotency_key"
-require_relative "acidic_job/extensions/sidekiq"
-require_relative "acidic_job/extensions/action_mailer"
-require_relative "acidic_job/extensions/active_job"
-require_relative "acidic_job/extensions/noticed"
-require_relative "acidic_job/upgrade_service"
+require_relative "acidic_job/builder"
+require_relative "acidic_job/context"
+require_relative "acidic_job/arguments"
+require_relative "acidic_job/log_subscriber"
+require_relative "acidic_job/workflow"
 
-require "active_support/concern"
-require "active_job/queue_adapters"
-require "active_job/base"
+require "active_support"
 
 module AcidicJob
-  extend ActiveSupport::Concern
+  extend self
 
-  IDEMPOTENCY_KEY_LOCK_TIMEOUT = 90
+  DEFAULT_LOGGER = ActiveSupport::Logger.new($stdout)
+  FINISHED_RECOVERY_POINT = "FINISHED"
 
-  def self.wire_everything_up(klass)
-    # Ensure our `perform` method always runs first to gather parameters
-    klass.prepend PerformWrapper
+  mattr_accessor :logger, default: DEFAULT_LOGGER
+  mattr_accessor :connects_to
 
-    klass.include Staging
-    klass.include Awaiting
-
-    # Add `deliver_acidicly` to ActionMailer
-    ActionMailer::Parameterized::MessageDelivery.include Extensions::ActionMailer if defined?(ActionMailer)
-    # Add `deliver_acidicly` to Noticed
-    Noticed::Base.include Extensions::Noticed if defined?(Noticed)
-
-    if defined?(ActiveJob) && klass < ActiveJob::Base
-      klass.send(:include, Extensions::ActiveJob)
-    elsif defined?(Sidekiq) && klass.include?(Sidekiq::Worker)
-      klass.send(:include, Extensions::Sidekiq)
-      klass.include ActiveSupport::Callbacks
-      klass.define_callbacks :perform
-    else
-      raise UnknownJobAdapter
-    end
-
-    # TODO: write test for a staged job that uses awaits
-    klass.set_callback :perform, :after, :reenqueue_awaited_by_job,
-                       if: -> { was_awaited_job? && !was_workflow_job? }
-    klass.set_callback :perform, :after, :finish_staged_job, if: -> { was_staged_job? && !was_workflow_job? }
-    klass.define_callbacks :finish
-    klass.set_callback :finish, :after, :reenqueue_awaited_by_job,
-                       if: -> { was_workflow_job? && was_awaited_job? }
-
-    klass.instance_variable_set(:@acidic_identifier, :job_id)
-    klass.define_singleton_method(:acidic_by_job_id) { @acidic_identifier = :job_id }
-    klass.define_singleton_method(:acidic_by_job_args) { @acidic_identifier = :job_args }
-    klass.define_singleton_method(:acidic_by) { |proc| @acidic_identifier = proc }
-    klass.attr_reader(:acidic_job_run)
-  end
-
-  included do
-    AcidicJob.wire_everything_up(self)
-  end
-
-  class_methods do
-    def inherited(subclass)
-      AcidicJob.wire_everything_up(subclass)
-      super
-    end
-
-    def with(*args, **kwargs)
-      new(*args, **kwargs)
-    end
-
-    def acidic_identifier
-      @acidic_identifier
-    end
-  end
-
-  def initialize(*args, **kwargs)
-    # ensure this instance variable is always defined
-    @__acidic_job_steps = []
-    @__acidic_job_args = args
-    @__acidic_job_kwargs = kwargs
-
-    super(*args, **kwargs)
-  rescue ArgumentError => e
-    raise e unless e.message.include?("wrong number of arguments")
-
-    super()
-  end
-
-  def with_acidity(providing: {})
-    # execute the block to gather the info on what steps are defined for this job workflow
-    yield
-
-    # check that the block actually defined at least one step
-    # TODO: WRITE TESTS FOR FAULTY BLOCK VALUES
-    raise NoDefinedSteps if @__acidic_job_steps.nil? || @__acidic_job_steps.empty?
-
-    # convert the array of steps into a hash of recovery_points and next steps
-    workflow = define_workflow(@__acidic_job_steps)
-
-    @acidic_job_run = ensure_run_record(workflow, providing)
-
-    # begin the workflow
-    process_run(@acidic_job_run)
-  end
-
-  # DEPRECATED
-  def idempotently(with: {}, &blk)
-    ActiveSupport::Deprecation.new("1.0", "AcidicJob").deprecation_warning(:idempotently)
-    with_acidity(providing: with, &blk)
-  end
-
-  def safely_finish_acidic_job
-    # Short circuits execution by sending execution right to 'finished'.
-    # So, ends the job "successfully"
-    FinishedPoint.new
-  end
-
-  # rubocop:disable Naming/MemoizedInstanceVariableName
-  def idempotency_key
-    if defined?(@__acidic_job_idempotency_key) && !@__acidic_job_idempotency_key.nil?
-      return @__acidic_job_idempotency_key
-    end
-
-    acidic_identifier = self.class.acidic_identifier
-    @__acidic_job_idempotency_key ||= IdempotencyKey.new(acidic_identifier)
-                                                    .value_for(self, *@__acidic_job_args, **@__acidic_job_kwargs)
-  end
-  # rubocop:enable Naming/MemoizedInstanceVariableName
-
-  private
-
-  def finish_staged_job
-    FinishedPoint.new.call(run: staged_job_run)
-  end
-
-  def was_workflow_job?
-    defined?(@acidic_job_run) && @acidic_job_run.present?
-  end
-
-  def process_run(run)
-    # if the run record is already marked as finished, immediately return its result
-    return run.succeeded? if run.finished?
-
-    # otherwise, we will enter a loop to process each step of the workflow
-    loop do
-      recovery_point = run.recovery_point.to_s
-      current_step = run.workflow[recovery_point]
-
-      # if any step calls `safely_finish_acidic_job` or the workflow has simply completed,
-      # be sure to break out of the loop
-      if recovery_point.to_s == Run::FINISHED_RECOVERY_POINT.to_s # rubocop:disable Style/GuardClause
-        break
-      elsif current_step.nil?
-        raise UnknownRecoveryPoint, "Defined workflow does not reference this step: #{recovery_point}"
-      elsif !Array(jobs = current_step.fetch("awaits", []) || []).compact.empty?
-        step = Step.new(current_step, run, self)
-        # Only execute the current step, without yet progressing the recovery_point to the next step.
-        # This ensures that any failures in parallel jobs will have this step retried in the main workflow
-        step_result = step.execute
-        # We allow the `#step_done` method to manage progressing the recovery_point to the next step,
-        # and then calling `process_run` to restart the main workflow on the next step.
-        # We pass the `step_result` so that the async callback called after the step-parallel-jobs complete
-        # can move on to the appropriate next stage in the workflow.
-        enqueue_step_parallel_jobs(jobs, run, step_result)
-        # after processing the current step, break the processing loop
-        # and stop this method from blocking in the primary worker
-        # as it will continue once the background workers all succeed
-        # so we want to keep the primary worker queue free to process new work
-        # this CANNOT ever be `break` as that wouldn't exit the parent job,
-        # only this step in the workflow, blocking as it awaits the next step
-        return true
-      else
-        step = Step.new(current_step, run, self)
-        step.execute
-        # As this step does not await any parallel jobs, we can immediately progress to the next step
-        step.progress
-      end
-    end
-
-    # the loop will break once the job is finished, so simply report the status
-    run.succeeded?
-  end
-
-  def step(method_name, awaits: [], for_each: nil)
-    @__acidic_job_steps ||= []
-
-    @__acidic_job_steps << {
-      "does" => method_name.to_s,
-      "awaits" => awaits,
-      "for_each" => for_each
-    }
-
-    @__acidic_job_steps
-  end
-
-  def define_workflow(steps)
-    # [ { does: "step 1", awaits: [] }, { does: "step 2", awaits: [] }, ... ]
-    steps << { "does" => Run::FINISHED_RECOVERY_POINT }
-
-    {}.tap do |workflow|
-      steps.each_cons(2).map do |enter_step, exit_step|
-        enter_name = enter_step["does"]
-        workflow[enter_name] = enter_step.merge("then" => exit_step["does"])
-      end
-    end
-    # { "step 1": { does: "step 1", awaits: [], then: "step 2" }, ...  }
-  end
-
-  def ensure_run_record(workflow, accessors)
-    isolation_level = case ActiveRecord::Base.connection.adapter_name.downcase.to_sym
-                      when :sqlite
-                        :read_uncommitted
-                      else
-                        :serializable
-                      end
-
-    ActiveRecord::Base.transaction(isolation: isolation_level) do
-      run = Run.find_by(idempotency_key: idempotency_key)
-      serialized_job = serialize_job(*@__acidic_job_args, **@__acidic_job_kwargs)
-
-      if run.present?
-        # Programs enqueuing multiple jobs with different parameters but the
-        # same idempotency key is a bug.
-        if run.serialized_job.slice("args", "arguments") != serialized_job.slice("args", "arguments")
-          raise MismatchedIdempotencyKeyAndJobArguments
-        end
-
-        # Only acquire a lock if the key is unlocked or its lock has expired
-        # because the original job was long enough ago.
-        raise LockedIdempotencyKey if run.locked_at && run.locked_at > Time.current - IDEMPOTENCY_KEY_LOCK_TIMEOUT
-
-        # Lock the run and update latest run unless the job is already finished.
-        unless run.finished?
-          run.update!(
-            last_run_at: Time.current,
-            locked_at: Time.current,
-            workflow: workflow,
-            recovery_point: run.recovery_point || workflow.first.first
-          )
-        end
-      else
-        run = Run.create!(
-          staged: false,
-          idempotency_key: idempotency_key,
-          job_class: self.class.name,
-          locked_at: Time.current,
-          last_run_at: Time.current,
-          recovery_point: workflow.first.first,
-          workflow: workflow,
-          serialized_job: serialized_job
-        )
-      end
-
-      # set accessors for each argument passed in to ensure they are available
-      # to the step methods the job will have written
-      define_accessors_for_passed_arguments(accessors, run)
-
-      # NOTE: we must return the `key` object from this transaction block
-      # so that it can be returned from this method to the caller
-      run
-    end
+  def instrument(channel, **options, &block)
+    ActiveSupport::Notifications.instrument("#{channel}.acidic_job", **options, &block)
   end
 
-  def define_accessors_for_passed_arguments(passed_arguments, run)
-    # first, get the current state of all accessors for both previously persisted and initialized values
-    current_accessors = passed_arguments.stringify_keys.merge(run.attr_accessors)
-
-    # next, ensure that `Run#attr_accessors` is populated with initial values
-    run.update_column(:attr_accessors, current_accessors)
-
-    current_accessors.each do |accessor, value|
-      # the reader method may already be defined
-      self.class.attr_reader accessor unless respond_to?(accessor)
-      # but we should always update the value to match the current value
-      instance_variable_set("@#{accessor}", value)
-      # and we overwrite the setter to ensure any updates to an accessor update the `Key` stored value
-      # Note: we must define the singleton method on the instance to avoid overwriting setters on other
-      # instances of the same class
-      define_singleton_method("#{accessor}=") do |current_value|
-        instance_variable_set("@#{accessor}", current_value)
-        run.attr_accessors[accessor] = current_value
-        run.save!(validate: false)
-        current_value
-      end
-    end
-
-    true
-  end
+  ActiveSupport.run_load_hooks(:acidic_job, self)
 end

data/lib/generators/acidic_job/install_generator.rb

@@ -8,12 +8,12 @@ module AcidicJob
       include ActiveRecord::Generators::Migration
       source_root File.expand_path("templates", __dir__)
 
-      desc "Generates a migration for the AcidicJob::Run table."
+      desc "Generates a migration for the AcidicJob tables."
 
       # Copies the migration template to db/migrate.
       def copy_acidic_job_runs_migration_files
-        migration_template "create_acidic_job_runs_migration.rb.erb",
-                           "db/migrate/create_acidic_job_runs.rb",
+        migration_template "create_acidic_job_tables_migration.rb.erb",
+                           "db/migrate/create_acidic_job_tables.rb",
                            migration_version: migration_version
       end
 

data/lib/generators/acidic_job/templates/create_acidic_job_tables_migration.rb.erb

@@ -0,0 +1,33 @@
+class <%= migration_class_name %> < ActiveRecord::Migration<%= migration_version %>
+  def change
+    create_table :acidic_job_executions, force: true do |t|
+      t.string      :idempotency_key, null: false,  index: { unique: true }
+      t.json        :serialized_job, 	null: false,  default: "{}"
+      t.datetime    :last_run_at, 		null: true
+      t.datetime    :locked_at, 			null: true
+      t.string      :recover_to, 	    null: true
+      t.json        :definition, 			null: true,   default: "{}"
+      t.timestamps
+    end
+
+    create_table :acidic_job_entries do |t|
+      t.references :execution, null: false, foreign_key: { to_table: :acidic_job_executions }
+      t.string :step, null: false
+      t.string :action, null: false
+      t.datetime :timestamp, null: false
+      t.json :data
+
+      t.timestamps
+    end
+    add_index :acidic_job_entries, [:execution_id, :step]
+
+    create_table :acidic_job_values do |t|
+      t.references :execution, null: false, foreign_key: { to_table: :acidic_job_executions }
+      t.string :key, null: false
+      t.json :value, null: false,   default: "{}"
+
+      t.timestamps
+    end
+    add_index :acidic_job_values, [:execution_id, :key], unique: true
+  end
+end