acidic_job 0.7.6 → 1.0.0.beta.2

data/lib/acidic_job/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module AcidicJob
-  VERSION = "0.7.6"
+  VERSION = "1.0.0.beta.2"
 end

data/lib/acidic_job/workflow.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+module AcidicJob
+  class Workflow
+    # { "step 1": { does: "step 1", awaits: [], then: "step 2" }, ...  }
+    def initialize(run, job, step_result = nil)
+      @run = run
+      @job = job
+      @step_result = step_result
+    end
+
+    def execute_current_step
+      rescued_error = false
+
+      begin
+        run_current_step
+      rescue StandardError => e
+        rescued_error = e
+        raise e
+      ensure
+        if rescued_error
+          begin
+            @run.store_error!(rescued_error)
+          rescue StandardError => e
+            # We're already inside an error condition, so swallow any additional
+            # errors from here and just send them to logs.
+            AcidicJob.logger.error("Failed to unlock AcidicJob::Run #{@run.id} because of #{e}.")
+          end
+        end
+      end
+
+      # be sure to return the `step_result`
+      # which is set by `run_current_step`
+      # which runs the (wrapped) current step method
+      @step_result
+    end
+
+    def progress_to_next_step
+      return if @run.current_step_finished?
+      return run_step_result unless @run.next_step_finishes?
+
+      @job.run_callbacks :finish do
+        run_step_result
+      end
+    end
+
+    private
+
+    def run_current_step
+      wrapped_method = WorkflowStep.new(run: @run, job: @job).wrapped
+      current_step = @run.current_step_name
+
+      # can't reproduce yet, but saw a bug in production where
+      # nested awaits workflows had an unsaved `workflow` attribute
+      @run.save! if @run.has_changes_to_save?
+
+      AcidicJob.logger.log_run_event("Executing #{current_step}...", @job, @run)
+      @run.with_lock do
+        @step_result = wrapped_method.call(@run)
+      end
+      AcidicJob.logger.log_run_event("Executed #{current_step}.", @job, @run)
+    end
+
+    def run_step_result
+      next_step = @run.next_step_name
+
+      # can't reproduce yet, but saw a bug in production where
+      # nested awaits workflows had an unsaved `workflow` attribute
+      @run.save! if @run.has_changes_to_save?
+
+      AcidicJob.logger.log_run_event("Progressing to #{next_step}...", @job, @run)
+      @run.with_lock do
+        @step_result.call(run: @run)
+      end
+      AcidicJob.logger.log_run_event("Progressed to #{next_step}.", @job, @run)
+    end
+  end
+end

data/lib/acidic_job/workflow_builder.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module AcidicJob
+  class WorkflowBuilder
+    attr_reader :steps
+
+    def initialize
+      @steps = []
+    end
+
+    def step(method_name, awaits: [], for_each: nil)
+      @steps << {
+        "does" => method_name.to_s,
+        "awaits" => awaits,
+        "for_each" => for_each
+      }
+
+      @steps
+    end
+    alias_method "✅", :step
+
+    def define_workflow
+      # [ { does: "step 1", awaits: [] }, { does: "step 2", awaits: [] }, ... ]
+      @steps << { "does" => Run::FINISHED_RECOVERY_POINT.to_s }
+
+      {}.tap do |workflow|
+        @steps.each_cons(2).map do |enter_step, exit_step|
+          enter_name = enter_step["does"]
+          workflow[enter_name] = enter_step.merge("then" => exit_step["does"])
+        end
+      end
+      # { "step 1": { does: "step 1", awaits: [], then: "step 2" }, ...  }
+    end
+  end
+end

data/lib/acidic_job/workflow_step.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+module AcidicJob
+  class WorkflowStep
+    def initialize(run:, job:)
+      @run = run
+      @job = job
+    end
+
+    def wrapped
+      # return a callable Proc with a consistent interface for the execution phase
+      proc do |_run|
+        current_step_result = process_current_step
+
+        if current_step_result.is_a?(FinishedPoint)
+          current_step_result
+        elsif next_item.present?
+          @run.attr_accessors[iterated_key] = prev_iterateds + [next_item]
+          @run.save!(validate: false)
+          RecoveryPoint.new(current_step_name)
+        elsif @run.next_step_finishes?
+          FinishedPoint.new
+        else
+          RecoveryPoint.new(@run.next_step_name)
+        end
+      end
+    end
+
+    private
+
+    def process_current_step
+      result = nil
+
+      if iterable_key.present? && next_item.present? # have an item to iterate over, so pass it to the step method
+        result = current_callable.call(next_item)
+      elsif iterable_key.present? && next_item.nil? # have iterated over all items
+        result = true
+      elsif current_callable.arity.zero?
+        result = current_callable.call
+      else
+        raise TooManyParametersForStepMethod
+      end
+
+      result
+    end
+
+    def current_callable
+      return @job.method(current_step_name) if @job.respond_to?(current_step_name, _include_private = true)
+      # jobs can have no-op steps, especially so that they can use only the async/await mechanism for that step
+      return proc {} if @run.current_step_hash["awaits"].present?
+
+      raise UndefinedStepMethod
+    end
+
+    def iterable_key
+      # the `iterable_key` represents the name of the collection accessor
+      # that must be present in `@run.attr_accessors`; that is,
+      # it must have been passed to `persisting` when calling `with_acidic_workflow`
+      for_each = @run.current_step_hash["for_each"]
+
+      return unless for_each.present?
+
+      return for_each if @run.attr_accessors.key?(for_each)
+
+      raise UnknownForEachCollection
+    end
+
+    def iterated_key
+      # in order to ensure we don't iterate over successfully iterated values in previous runs,
+      # we need to store the collection of already processed values.
+      # we store this collection under a key bound to the current step to ensure multiple steps
+      # can iterate over the same collection.
+      "processed_#{current_step_name}_#{iterable_key}"
+    end
+
+    def prev_iterables
+      # The collection of values to iterate over
+      iterables = @run.attr_accessors.fetch(iterable_key, [])
+
+      return Array(iterables) if iterables.is_a?(Enumerable)
+
+      raise UniterableForEachCollection
+    end
+
+    def prev_iterateds
+      # The collection of values already iterated over
+      iterateds = @run.attr_accessors.fetch(iterated_key, [])
+
+      Array(iterateds)
+    end
+
+    def next_item
+      # The collection of values to iterate over now
+      curr_iterables = prev_iterables - prev_iterateds
+
+      curr_iterables.first
+    end
+
+    def current_step_name
+      @run.current_step_name
+    end
+  end
+end

data/lib/acidic_job.rb

@@ -2,342 +2,39 @@
 
 require_relative "acidic_job/version"
 require_relative "acidic_job/errors"
-require_relative "acidic_job/no_op"
+require_relative "acidic_job/logger"
+require_relative "acidic_job/arguments"
+require_relative "acidic_job/serializer"
+require_relative "acidic_job/workflow_builder"
+require_relative "acidic_job/idempotency_key"
 require_relative "acidic_job/recovery_point"
-require_relative "acidic_job/response"
-require_relative "acidic_job/key"
-require_relative "acidic_job/staged"
+require_relative "acidic_job/finished_point"
+require_relative "acidic_job/run"
+require_relative "acidic_job/workflow_step"
+require_relative "acidic_job/workflow"
+require_relative "acidic_job/processor"
 require_relative "acidic_job/perform_wrapper"
-require_relative "acidic_job/perform_transactionally_extension"
-require_relative "acidic_job/deliver_transactionally_extension"
-require_relative "acidic_job/sidekiq_callbacks"
-require "active_support/concern"
+require_relative "acidic_job/mixin"
+require_relative "acidic_job/base"
+require_relative "acidic_job/active_kiq"
+require_relative "acidic_job/extensions/action_mailer"
+require_relative "acidic_job/extensions/noticed"
+
+require_relative "acidic_job/serializers/exception_serializer"
+require_relative "acidic_job/serializers/finished_point_serializer"
+require_relative "acidic_job/serializers/job_serializer"
+require_relative "acidic_job/serializers/range_serializer"
+require_relative "acidic_job/serializers/recovery_point_serializer"
+require_relative "acidic_job/serializers/worker_serializer"
+require "active_job/serializers"
 
-# rubocop:disable Metrics/ModuleLength, Metrics/AbcSize, Metrics/MethodLength
 module AcidicJob
-  extend ActiveSupport::Concern
-
-  def self.wire_everything_up(klass)
-    klass.attr_reader :key
-    klass.attr_reader :staged_job_gid
-    klass.attr_reader :arguments_for_perform
-
-    # Extend ActiveJob with `perform_transactionally` class method
-    klass.include PerformTransactionallyExtension
-
-    ActionMailer::Parameterized::MessageDelivery.include DeliverTransactionallyExtension if defined?(ActionMailer)
-
-    # Ensure our `perform` method always runs first to gather parameters
-    klass.prepend PerformWrapper
-
-    klass.prepend SidekiqCallbacks unless klass.respond_to?(:after_perform)
-
-    klass.after_perform :delete_staged_job_record, if: :staged_job_gid
-  end
-
-  included do
-    AcidicJob.wire_everything_up(self)
-  end
-
-  class_methods do
-    def inherited(subclass)
-      AcidicJob.wire_everything_up(subclass)
-      super
-    end
-
-    def initiate(*args)
-      operation = Sidekiq::Batch.new
-      operation.on(:success, self, *args)
-      operation.jobs do
-        perform_async
-      end
-    end
-  end
-
-  # Number of seconds passed which we consider a held idempotency key lock to be
-  # defunct and eligible to be locked again by a different job run. We try to
-  # unlock keys on our various failure conditions, but software is buggy, and
-  # this might not happen 100% of the time, so this is a hedge against it.
-  IDEMPOTENCY_KEY_LOCK_TIMEOUT = 90
-
-  # takes a block
-  def with_acidity(given:)
-    # execute the block to gather the info on what steps are defined for this job workflow
-    steps = yield || []
-
-    raise NoDefinedSteps if steps.empty?
-
-    # convert the array of steps into a hash of recovery_points and next steps
-    workflow = define_workflow(steps)
-
-    # find or create a Key record (our idempotency key) to store all information about this job
-    #
-    # A key concept here is that if two requests try to insert or update within
-    # close proximity, one of the two will be aborted by Postgres because we're
-    # using a transaction with SERIALIZABLE isolation level. It may not look
-    # it, but this code is safe from races.
-    key = ensure_idempotency_key_record(idempotency_key_value, workflow, given)
-
-    # begin the workflow
-    process_key(key)
-  end
-  
-  # DEPRECATED
-  def idempotently(with:)
-    with_acidity(given: with)
-  end 
-
-  def process_key(key)
-    @key = key
-
-    # if the key record is already marked as finished, immediately return its result
-    return @key.succeeded? if @key.finished?
-
-    # otherwise, we will enter a loop to process each step of the workflow
-    @key.workflow.size.times do
-      recovery_point = @key.recovery_point.to_s
-      current_step = @key.workflow[recovery_point]
-
-      if recovery_point == Key::RECOVERY_POINT_FINISHED.to_s # rubocop:disable Style/GuardClause
-        break
-      elsif current_step.nil?
-        raise UnknownRecoveryPoint, "Defined workflow does not reference this step: #{recovery_point}"
-      elsif (jobs = current_step.fetch("awaits", [])).any?
-        acidic_step @key, current_step
-        # THIS MUST BE DONE AFTER THE KEY RECOVERY POINT HAS BEEN UPDATED
-        enqueue_step_parallel_jobs(jobs)
-        # after processing the current step, break the processing loop
-        # and stop this method from blocking in the primary worker
-        # as it will continue once the background workers all succeed
-        # so we want to keep the primary worker queue free to process new work
-        # this CANNOT ever be `break` as that wouldn't exit the parent job,
-        # only this step in the workflow, blocking as it awaits the next step
-        return true
-      else
-        acidic_step @key, current_step
-      end
-    end
-
-    # the loop will break once the job is finished, so simply report the status
-    @key.succeeded?
-  end
-
-  def step(method_name, awaits: [])
-    @_steps ||= []
-
-    @_steps << {
-      "does" => method_name.to_s,
-      "awaits" => awaits
-    }
-
-    @_steps
-  end
-
-  def safely_finish_acidic_job
-    # Short circuits execution by sending execution right to 'finished'.
-    # So, ends the job "successfully"
-    AcidicJob::Response.new
-  end
-
-  private
-
-  def delete_staged_job_record
-    return unless staged_job_gid
-
-    staged_job = GlobalID::Locator.locate(staged_job_gid)
-    staged_job.delete
-    true
-  rescue ActiveRecord::RecordNotFound
-    true
-  end
-
-  def define_workflow(steps)
-    steps << { "does" => Key::RECOVERY_POINT_FINISHED }
-
-    {}.tap do |workflow|
-      steps.each_cons(2).map do |enter_step, exit_step|
-        enter_name = enter_step["does"]
-        workflow[enter_name] = {
-          "then" => exit_step["does"]
-        }.merge(enter_step)
-      end
-    end
-  end
-
-  def ensure_idempotency_key_record(key_val, workflow, accessors)
-    isolation_level = case ActiveRecord::Base.connection.adapter_name.downcase.to_sym
-                      when :sqlite
-                        :read_uncommitted
-                      else
-                        :serializable
-                      end
-
-    ActiveRecord::Base.transaction(isolation: isolation_level) do
-      key = Key.find_by(idempotency_key: key_val)
-
-      if key.present?
-        # Programs enqueuing multiple jobs with different parameters but the
-        # same idempotency key is a bug.
-        raise MismatchedIdempotencyKeyAndJobArguments if key.job_args != @arguments_for_perform
-
-        # Only acquire a lock if the key is unlocked or its lock has expired
-        # because the original job was long enough ago.
-        raise LockedIdempotencyKey if key.locked_at && key.locked_at > Time.current - IDEMPOTENCY_KEY_LOCK_TIMEOUT
-
-        # Lock the key and update latest run unless the job is already finished.
-        key.update!(last_run_at: Time.current, locked_at: Time.current, workflow: workflow) unless key.finished?
-      else
-        key = Key.create!(
-          idempotency_key: key_val,
-          locked_at: Time.current,
-          last_run_at: Time.current,
-          recovery_point: workflow.first.first,
-          job_name: self.class.name,
-          job_args: @arguments_for_perform,
-          workflow: workflow
-        )
-      end
-
-      # set accessors for each argument passed in to ensure they are available
-      # to the step methods the job will have written
-      define_accessors_for_passed_arguments(accessors, key)
-
-      # NOTE: we must return the `key` object from this transaction block
-      # so that it can be returned from this method to the caller
-      key
-    end
-  end
-
-  def acidic_step(key, step)
-    rescued_error = false
-    step_callable = wrap_step_as_acidic_callable step
-
-    begin
-      key.with_lock do
-        step_result = step_callable.call(key)
-
-        step_result.call(key: key)
-      end
-    # QUESTION: Can an error not inherit from StandardError
-    rescue StandardError => e
-      rescued_error = e
-      raise e
-    ensure
-      if rescued_error
-        # If we're leaving under an error condition, try to unlock the idempotency
-        # key right away so that another request can try again.3
-        begin
-          key.update_columns(locked_at: nil, error_object: rescued_error)
-        rescue StandardError => e
-          # We're already inside an error condition, so swallow any additional
-          # errors from here and just send them to logs.
-          puts "Failed to unlock key #{key.id} because of #{e}."
-        end
-      end
-    end
-  end
-
-  def define_accessors_for_passed_arguments(passed_arguments, key)
-    # first, get the current state of all accessors for both previously persisted and initialized values
-    current_accessors = passed_arguments.stringify_keys.merge(key.attr_accessors)
-
-    # next, ensure that `Key#attr_accessors` is populated with initial values
-    key.update_column(:attr_accessors, current_accessors)
-
-    current_accessors.each do |accessor, value|
-      # the reader method may already be defined
-      self.class.attr_reader accessor unless respond_to?(accessor)
-      # but we should always update the value to match the current value
-      instance_variable_set("@#{accessor}", value)
-      # and we overwrite the setter to ensure any updates to an accessor update the `Key` stored value
-      # Note: we must define the singleton method on the instance to avoid overwriting setters on other
-      # instances of the same class
-      define_singleton_method("#{accessor}=") do |current_value|
-        instance_variable_set("@#{accessor}", current_value)
-        key.attr_accessors[accessor] = current_value
-        key.save!(validate: false)
-        current_value
-      end
-    end
-
-    true
-  end
-
-  # rubocop:disable Metrics/PerceivedComplexity
-  def wrap_step_as_acidic_callable(step)
-    # {:then=>:next_step, :does=>:enqueue_step, :awaits=>[WorkerWithEnqueueStep::FirstWorker]}
-    current_step = step["does"]
-    next_step = step["then"]
-
-    callable = if respond_to? current_step, _include_private = true
-                 method(current_step)
-               else
-                 proc {} # no-op
-               end
-
-    proc do |key|
-      result = if callable.arity.zero?
-                 callable.call
-               elsif callable.arity == 1
-                 callable.call(key)
-               else
-                 raise TooManyParametersForStepMethod
-               end
-
-      if result.is_a?(Response)
-        result
-      elsif next_step.to_s == Key::RECOVERY_POINT_FINISHED
-        Response.new
-      else
-        RecoveryPoint.new(next_step)
-      end
-    end
-  end
-  # rubocop:enable Metrics/PerceivedComplexity
-
-  def enqueue_step_parallel_jobs(jobs)
-    # TODO: GIVE PROPER ERROR
-    # `batch` is available from Sidekiq::Pro
-    raise SidekiqBatchRequired unless defined?(Sidekiq::Batch)
-
-    batch.jobs do
-      step_batch = Sidekiq::Batch.new
-      # step_batch.description = "AcidicJob::Workflow Step: #{step}"
-      step_batch.on(
-        :success,
-        "#{self.class.name}#step_done",
-        # NOTE: options are marshalled through JSON so use only basic types.
-        { "key_id" => @key.id }
-      )
-      # NOTE: The jobs method is atomic.
-      # All jobs created in the block are actually pushed atomically at the end of the block.
-      # If an error is raised, none of the jobs will go to Redis.
-      step_batch.jobs do
-        jobs.each do |worker_name|
-          worker = worker_name.is_a?(String) ? worker_name.constantize : worker_name
-          if worker.instance_method(:perform).arity.zero?
-            worker.perform_async
-          elsif worker.instance_method(:perform).arity == 1
-            worker.perform_async(key.id)
-          else
-            raise TooManyParametersForParallelJob
-          end
-        end
-      end
-    end
-  end
-
-  def idempotency_key_value
-    return job_id if defined?(job_id) && !job_id.nil?
-    return jid if defined?(jid) && !jid.nil?
-
-    Digest::SHA1.hexdigest [self.class.name, arguments_for_perform].flatten.join
-  end
-
-  def step_done(_status, options)
-    key = Key.find(options["key_id"])
-    # when a batch of jobs for a step succeeds, we begin the key processing again
-    process_key(key)
-  end
+  ::ActiveJob::Serializers.add_serializers(
+    Serializers::ExceptionSerializer,
+    Serializers::FinishedPointSerializer,
+    Serializers::JobSerializer,
+    Serializers::RangeSerializer,
+    Serializers::RecoveryPointSerializer,
+    Serializers::WorkerSerializer
+  )
 end
-# rubocop:enable Metrics/ModuleLength, Metrics/AbcSize, Metrics/MethodLength

data/lib/generators/acidic_job/drop_tables_generator.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require "rails/generators/active_record"
+
+module AcidicJob
+  module Generators
+    class DropTablesGenerator < ::Rails::Generators::Base
+      include ActiveRecord::Generators::Migration
+      source_root File.expand_path("templates", __dir__)
+
+      desc "Drops the pre-1.0 tables for the AcidicJob::Key and AcidicJob::Staged models."
+
+      def copy_migration
+        migration_template "drop_acidic_job_keys_migration.rb.erb",
+                           "db/migrate/drop_old_acidic_job_tables.rb",
+                           migration_version: migration_version
+      end
+
+      protected
+
+      def migration_version
+        "[#{ActiveRecord::VERSION::MAJOR}.#{ActiveRecord::VERSION::MINOR}]"
+      end
+    end
+  end
+end

data/lib/generators/acidic_job/install_generator.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require "rails/generators/active_record"
+
+module AcidicJob
+  module Generators
+    class InstallGenerator < ::Rails::Generators::Base
+      include ActiveRecord::Generators::Migration
+      source_root File.expand_path("templates", __dir__)
+
+      desc "Generates a migration for the AcidicJob::Run table."
+
+      # Copies the migration template to db/migrate.
+      def copy_acidic_job_runs_migration_files
+        migration_template "create_acidic_job_runs_migration.rb.erb",
+                           "db/migrate/create_acidic_job_runs.rb",
+                           migration_version: migration_version
+      end
+
+      protected
+
+      def migration_version
+        "[#{ActiveRecord::VERSION::MAJOR}.#{ActiveRecord::VERSION::MINOR}]"
+      end
+    end
+  end
+end

data/lib/generators/acidic_job/templates/create_acidic_job_runs_migration.rb.erb

@@ -0,0 +1,19 @@
+class <%= migration_class_name %> < ActiveRecord::Migration<%= migration_version %>
+  def change
+    create_table :acidic_job_runs do |t|
+      t.boolean     :staged, 					null: false,  default: -> { false }
+      t.string      :idempotency_key, null: false,  index: { unique: true }
+      t.text        :serialized_job, 	null: false
+      t.string      :job_class, 			null: false
+      t.datetime    :last_run_at, 		null: true,   default: -> { "CURRENT_TIMESTAMP" }
+      t.datetime    :locked_at, 			null: true
+      t.string      :recovery_point, 	null: true
+      t.text        :error_object, 		null: true
+      t.text        :attr_accessors, 	null: true
+      t.text        :workflow, 				null: true
+      t.references  :awaited_by,      null: true,   index: true
+      t.text        :returning_to,    null: true
+      t.timestamps
+    end
+  end
+end