acidic_job 1.0.0.pre29 → 1.0.0.rc2

data/lib/acidic_job/idempotency_key.rb

@@ -1,82 +0,0 @@
-# frozen_string_literal: true
-
-module AcidicJob
-  class IdempotencyKey
-    def initialize(identifier = :job_id)
-      @identifier = identifier
-    end
-
-    def value_for(hash_or_job, *args, **kwargs)
-      value = case @identifier
-              when Proc
-                value_from_proc(hash_or_job, *args, **kwargs)
-              when :job_args
-                value_from_job_args(hash_or_job, *args, **kwargs)
-              else
-                if hash_or_job.is_a?(Hash)
-                  value_from_job_id_for_hash(hash_or_job)
-                else
-                  value_from_job_id_for_obj(hash_or_job)
-                end
-              end
-
-      result = value || value_from_job_args(hash_or_job, *args, **kwargs)
-
-      if result.start_with?("STG__")
-        # "STG__#{idempotency_key}__#{encoded_global_id}"
-        _prefix, idempotency_key, _encoded_global_id = result.split("__")
-        idempotency_key
-      else
-        result
-      end
-    end
-
-    private
-
-    def value_from_job_id_for_hash(hash)
-      if hash.key?("job_id")
-        return if hash["job_id"].nil?
-        return if hash["job_id"].empty?
-
-        hash["job_id"]
-      elsif hash.key?("jid")
-        return if hash["jid"].nil?
-        return if hash["jid"].empty?
-
-        hash["jid"]
-      end
-    end
-
-    def value_from_job_id_for_obj(obj)
-      if obj.respond_to?(:job_id)
-        return if obj.job_id.nil?
-        return if obj.job_id.empty?
-
-        obj.job_id
-      elsif obj.respond_to?(:jid)
-        return if obj.jid.nil?
-        return if obj.jid.empty?
-
-        obj.jid
-      end
-    end
-
-    def value_from_job_args(hash_or_job, *args, **kwargs)
-      worker_class = case hash_or_job
-                     when Hash
-                       hash_or_job["worker"] || hash_or_job["job_class"]
-                     else
-                       hash_or_job.class.name
-                     end
-
-      Digest::SHA1.hexdigest [worker_class, args, kwargs].flatten.join
-    end
-
-    def value_from_proc(_hash_or_job, *args, **kwargs)
-      return if args.empty? && kwargs.empty?
-
-      idempotency_args = Array(@identifier.call(*args, **kwargs))
-      Digest::SHA1.hexdigest idempotency_args.flatten.join
-    end
-  end
-end

data/lib/acidic_job/perform_wrapper.rb

@@ -1,22 +0,0 @@
-# frozen_string_literal: true
-
-module AcidicJob
-  # NOTE: it is essential that this be a bare module and not an ActiveSupport::Concern
-  module PerformWrapper
-    def perform(*args, **kwargs)
-      @__acidic_job_args = args
-      @__acidic_job_kwargs = kwargs
-
-      # we don't want to run the `perform` callbacks twice, since ActiveJob already handles that for us
-      if defined?(ActiveJob) && self.class < ActiveJob::Base
-        super(*args, **kwargs)
-      elsif defined?(Sidekiq) && self.class.include?(Sidekiq::Worker)
-        run_callbacks :perform do
-          super(*args, **kwargs)
-        end
-      else
-        raise UnknownJobAdapter
-      end
-    end
-  end
-end

data/lib/acidic_job/recovery_point.rb

@@ -1,18 +0,0 @@
-# frozen_string_literal: true
-
-# Represents an action to set a new recovery point. One possible option for a
-# return from an #atomic_phase block.
-module AcidicJob
-  class RecoveryPoint
-    attr_accessor :name
-
-    def initialize(name)
-      @name = name
-    end
-
-    def call(run:)
-      # Skip AR callbacks as there are none on the model
-      run.update_column(:recovery_point, @name)
-    end
-  end
-end

data/lib/acidic_job/rspec_configuration.rb

@@ -1,31 +0,0 @@
-# frozen_string_literal: true
-
-require "rspec"
-require "database_cleaner/active_record"
-
-# see https://github.com/DatabaseCleaner/database_cleaner#how-to-use
-RSpec.configure do |config|
-  config.use_transactional_fixtures = false
-
-  config.before(:suite) do
-    DatabaseCleaner.clean_with :truncation
-
-    # Here we are defaulting to :transaction but swapping to deletion for some specs;
-    # if your spec or its code-under-test uses
-    # nested transactions then specify :transactional e.g.:
-    #   describe "SomeWorker", :transactional do
-    #
-    DatabaseCleaner.strategy = :transaction
-
-    config.before(:context, transactional: true) { DatabaseCleaner.strategy = :deletion }
-    config.after(:context, transactional: true) { DatabaseCleaner.strategy = :transaction }
-    config.before(:context, type: :system) { DatabaseCleaner.strategy = :deletion }
-    config.after(:context, type: :system) { DatabaseCleaner.strategy = :transaction }
-  end
-
-  config.around(:each) do |example|
-    DatabaseCleaner.cleaning do
-      example.run
-    end
-  end
-end

data/lib/acidic_job/run.rb

@@ -1,100 +0,0 @@
-# frozen_string_literal: true
-
-require "active_record"
-require "global_id"
-require "active_support/core_ext/object/with_options"
-require_relative "./serializer"
-
-module AcidicJob
-  class Run < ActiveRecord::Base
-    include GlobalID::Identification
-
-    FINISHED_RECOVERY_POINT = "FINISHED"
-
-    self.table_name = "acidic_job_runs"
-
-    belongs_to :awaited_by, class_name: "AcidicJob::Run", optional: true
-    has_many :batched_runs, class_name: "AcidicJob::Run", foreign_key: "awaited_by_id"
-
-    after_create_commit :enqueue_staged_job, if: :staged?
-
-    serialize :serialized_job, JSON
-    serialize :error_object, Serializer
-    serialize :workflow, Serializer
-    serialize :returning_to, Serializer
-    store :attr_accessors, coder: Serializer
-
-    validates :staged, inclusion: { in: [true, false] } # uses database default
-    validates :serialized_job, presence: true
-    validates :idempotency_key, presence: true, uniqueness: true
-    validates :job_class, presence: true
-
-    scope :staged, -> { where(staged: true) }
-    scope :unstaged, -> { where(staged: false) }
-    scope :finished, -> { where(recovery_point: FINISHED_RECOVERY_POINT) }
-    scope :outstanding, lambda {
-      where.not(recovery_point: FINISHED_RECOVERY_POINT).or(where(recovery_point: [nil, ""]))
-    }
-
-    with_options unless: :staged? do
-      validates :last_run_at, presence: true
-      validates :recovery_point, presence: true
-      validates :workflow, presence: true
-    end
-
-    def self.purge
-      successfully_completed = where(
-        recovery_point: FINISHED_RECOVERY_POINT,
-        error_object: nil
-      )
-      count = successfully_completed.count
-
-      return 0 if count.zero?
-
-      Rails.logger.info("Deleting #{count} successfully completed AcidicJob runs")
-      successfully_completed.delete_all
-    end
-
-    def finished?
-      recovery_point == FINISHED_RECOVERY_POINT
-    end
-
-    def succeeded?
-      finished? && !failed?
-    end
-
-    def failed?
-      error_object.present?
-    end
-
-    def staged_job_id
-      # encode the identifier for this record in the job ID
-      # base64 encoding for minimal security
-      global_id = to_global_id.to_s.remove("gid://")
-      encoded_global_id = Base64.encode64(global_id).strip
-
-      "STG__#{idempotency_key}__#{encoded_global_id}"
-    end
-
-    private
-
-    def enqueue_staged_job
-      return unless staged?
-
-      serialized_staged_job = if serialized_job.key?("jid")
-                                serialized_job.merge("jid" => staged_job_id)
-                              elsif serialized_job.key?("job_id")
-                                serialized_job.merge("job_id" => staged_job_id)
-                              else
-                                raise UnknownSerializedJobIdentifier
-                              end
-
-      job = job_class.constantize.deserialize(serialized_staged_job)
-
-      job.enqueue
-
-      # NOTE: record will be deleted after the job has successfully been performed
-      true
-    end
-  end
-end

data/lib/acidic_job/serializer.rb

@@ -1,163 +0,0 @@
-# frozen_string_literal: true
-
-require "active_job/serializers"
-require "active_job/arguments"
-require "json"
-
-class WorkerSerializer < ActiveJob::Serializers::ObjectSerializer
-  def serialize(worker)
-    super(
-      "class" => worker.class.name,
-      "args" => worker.instance_variable_get(:@__acidic_job_args),
-      "kwargs" => worker.instance_variable_get(:@__acidic_job_kwargs)
-    )
-  end
-
-  def deserialize(hash)
-    worker_class = hash["class"].constantize
-    worker_class.new(*hash["args"], **hash["kwargs"])
-  end
-
-  def serialize?(argument)
-    defined?(::Sidekiq) && argument.class.include?(::Sidekiq::Worker)
-  end
-end
-
-class JobSerializer < ActiveJob::Serializers::ObjectSerializer
-  def serialize(job)
-    super(job.serialize)
-  end
-
-  def deserialize(hash)
-    job = ActiveJob::Base.deserialize(hash)
-    job.send(:deserialize_arguments_if_needed)
-    if job.arguments.last.is_a?(Hash)
-      *args, kwargs = job.arguments
-    else
-      args = job.arguments
-      kwargs = {}
-    end
-    job.instance_variable_set(:@__acidic_job_args, args)
-    job.instance_variable_set(:@__acidic_job_kwargs, kwargs)
-
-    job
-  end
-
-  def serialize?(argument)
-    defined?(::ActiveJob::Base) && argument.class < ::ActiveJob::Base
-  end
-end
-
-class ExceptionSerializer < ActiveJob::Serializers::ObjectSerializer
-  def serialize(exception)
-    hash = {
-      "class" => exception.class.name,
-      "message" => exception.message,
-      "cause" => exception.cause,
-      "backtrace" => {}
-    }
-
-    exception.backtrace.map do |trace|
-      path, _, location = trace.rpartition("/")
-
-      next if hash["backtrace"].key?(path)
-
-      hash["backtrace"][path] = location
-    end
-
-    super(hash)
-  end
-
-  def deserialize(hash)
-    exception_class = hash["class"].constantize
-    exception = exception_class.new(hash["message"])
-    exception.set_backtrace(hash["backtrace"].map do |path, location|
-      [path, location].join("/")
-    end)
-    exception
-  end
-
-  def serialize?(argument)
-    defined?(Exception) && argument.is_a?(Exception)
-  end
-end
-
-class FinishedPointSerializer < ActiveJob::Serializers::ObjectSerializer
-  def serialize(finished_point)
-    super(
-      "class" => finished_point.class.name
-    )
-  end
-
-  def deserialize(hash)
-    finished_point_class = hash["class"].constantize
-    finished_point_class.new
-  end
-
-  def serialize?(argument)
-    defined?(::AcidicJob::FinishedPoint) && argument.is_a?(::AcidicJob::FinishedPoint)
-  end
-end
-
-class RecoveryPointSerializer < ActiveJob::Serializers::ObjectSerializer
-  def serialize(recovery_point)
-    super(
-      "class" => recovery_point.class.name,
-      "name" => recovery_point.name
-    )
-  end
-
-  def deserialize(hash)
-    recovery_point_class = hash["class"].constantize
-    recovery_point_class.new(hash["name"])
-  end
-
-  def serialize?(argument)
-    defined?(::AcidicJob::RecoveryPoint) && argument.is_a?(::AcidicJob::RecoveryPoint)
-  end
-end
-
-ActiveJob::Serializers.add_serializers(
-  WorkerSerializer,
-  JobSerializer,
-  ExceptionSerializer,
-  FinishedPointSerializer,
-  RecoveryPointSerializer
-)
-
-# ...
-module AcidicJob
-  module Arguments
-    include ActiveJob::Arguments
-    extend self # rubocop:disable Style/ModuleFunction
-
-    # `ActiveJob` will throw an error if it tries to deserialize a GlobalID record.
-    # However, this isn't the behavior that we want for our custom `ActiveRecord` serializer.
-    # Since `ActiveRecord` does _not_ reset instance record state to its pre-transactional state
-    # on a transaction ROLLBACK, we can have GlobalID entries in a serialized column that point to
-    # non-persisted records. This is ok. We should simply return `nil` for that portion of the
-    # serialized field.
-    def deserialize_global_id(hash)
-      GlobalID::Locator.locate hash[GLOBALID_KEY]
-    rescue ActiveRecord::RecordNotFound
-      nil
-    end
-  end
-
-  class Serializer
-    # Used for `serialize` method in ActiveRecord
-    class << self
-      def load(json)
-        return if json.nil? || json.empty?
-
-        data = JSON.parse(json)
-        Arguments.deserialize(data).first
-      end
-
-      def dump(obj)
-        data = Arguments.serialize [obj]
-        data.to_json
-      end
-    end
-  end
-end

data/lib/acidic_job/staging.rb

@@ -1,38 +0,0 @@
-# frozen_string_literal: true
-
-require "active_support/concern"
-require "global_id/locator"
-
-module AcidicJob
-  module Staging
-    extend ActiveSupport::Concern
-
-    private
-
-    def was_staged_job?
-      identifier.start_with? "STG__"
-    end
-
-    def staged_job_run
-      # "STG_#{idempotency_key}__#{encoded_global_id}"
-      encoded_global_id = identifier.split("__").last
-      staged_job_gid = "gid://#{Base64.decode64(encoded_global_id)}"
-
-      GlobalID::Locator.locate(staged_job_gid)
-    rescue ActiveRecord::RecordNotFound
-      nil
-    end
-
-    def identifier
-      return jid if defined?(jid) && !jid.nil?
-      return job_id if defined?(job_id) && !job_id.nil?
-
-      # might be defined already in `with_acidity` method
-      acidic_identifier = self.class.acidic_identifier
-      @__acidic_job_idempotency_key ||= IdempotencyKey.new(acidic_identifier)
-                                                      .value_for(self, *@__acidic_job_args, **@__acidic_job_kwargs)
-
-      @__acidic_job_idempotency_key
-    end
-  end
-end

data/lib/acidic_job/step.rb

@@ -1,104 +0,0 @@
-# frozen_string_literal: true
-
-module AcidicJob
-  # Each AcidicJob::Step requires two phases: [1] execution and [2] progression
-  class Step
-    def initialize(step, run, job, step_result = nil)
-      @step = step
-      @run = run
-      @job = job
-      @step_result = step_result
-    end
-
-    # The execution phase performs the work of the defined step
-    def execute
-      rescued_error = false
-      step_callable = wrap_step_as_acidic_callable @step
-
-      begin
-        @run.with_lock do
-          @step_result = step_callable.call(@run)
-        end
-      # QUESTION: Can an error not inherit from StandardError
-      rescue StandardError => e
-        rescued_error = e
-        raise e
-      ensure
-        if rescued_error
-          # If we're leaving under an error condition, try to unlock the job
-          # run right away so that another request can try again.
-          begin
-            @run.update_columns(locked_at: nil, error_object: rescued_error)
-          rescue StandardError => e
-            # We're already inside an error condition, so swallow any additional
-            # errors from here and just send them to logs.
-            # TODO: implement and use a logger here
-            puts "Failed to unlock AcidicJob::Run #{@run.id} because of #{e}."
-          end
-        end
-      end
-    end
-
-    # The progression phase advances the job run state machine onto the next step
-    def progress
-      @run.with_lock do
-        if @step_result.is_a?(FinishedPoint)
-          @job.run_callbacks :finish do
-            @step_result.call(run: @run)
-          end
-        else
-          @step_result.call(run: @run)
-        end
-      end
-    end
-
-    private
-
-    def wrap_step_as_acidic_callable(step)
-      # {"does" => :enqueue_step, "then" => :next_step, "awaits" => [WorkerWithEnqueueStep::FirstWorker]}
-      current_step = step["does"]
-      next_step = step["then"]
-      # to support iteration within steps
-      iterable_key = step["for_each"]
-      iterated_key = "processed_#{current_step}_#{iterable_key}"
-      iterables = @run.attr_accessors.fetch(iterable_key, []) || []
-      iterateds = @run.attr_accessors.fetch(iterated_key, []) || []
-      next_item = iterables.reject { |item| iterateds.include? item }.first
-
-      # jobs can have no-op steps, especially so that they can use only the async/await mechanism for that step
-      callable = if @job.respond_to?(current_step, _include_private = true)
-                   @job.method(current_step)
-                 else
-                   proc {}
-                 end
-
-      # return a callable Proc with a consistent interface for the execution phase
-      proc do |run|
-        result = if iterable_key.present? && next_item.present?
-                   callable.call(next_item)
-                 elsif iterable_key.present? && next_item.nil?
-                   true
-                 elsif callable.arity.zero?
-                   callable.call
-                 elsif callable.arity == 1
-                   callable.call(run)
-                 else
-                   raise TooManyParametersForStepMethod
-                 end
-
-        if result.is_a?(FinishedPoint)
-          result
-        elsif next_item.present?
-          iterateds << next_item
-          @run.attr_accessors[iterated_key] = iterateds
-          @run.save!(validate: false)
-          RecoveryPoint.new(current_step)
-        elsif next_step.to_s == Run::FINISHED_RECOVERY_POINT
-          FinishedPoint.new
-        else
-          RecoveryPoint.new(next_step)
-        end
-      end
-    end
-  end
-end

data/lib/acidic_job/test_case.rb

@@ -1,9 +0,0 @@
-# frozen_string_literal: true
-
-require_relative "./testing"
-
-module AcidicJob
-  class TestCase < ActiveJob::TestCase
-    include AcidicJob::Testing
-  end
-end

data/lib/acidic_job/upgrade_service.rb

@@ -1,118 +0,0 @@
-# frozen_string_literal: true
-
-require "active_support/concern"
-
-module AcidicJob
-  # recreate the original `Key` model
-  class Key < ::ActiveRecord::Base
-    RECOVERY_POINT_FINISHED = "FINISHED"
-
-    self.table_name = "acidic_job_keys"
-
-    serialize :error_object
-    serialize :job_args
-    serialize :workflow
-    store :attr_accessors
-  end
-
-  # recreate the original `Staged` model
-  class Staged < ActiveRecord::Base
-    self.table_name = "staged_acidic_jobs"
-
-    serialize :job_args
-
-    after_create_commit :enqueue_job
-
-    private
-
-    def enqueue_job
-      gid = { "staged_job_gid" => to_global_id.to_s }
-
-      if job_args.is_a?(Hash) && job_args.key?("arguments")
-        job_args["arguments"].concat([gid])
-      else
-        job_args.concat([gid])
-      end
-
-      case adapter
-      when "activejob"
-        ::ActiveJob::Base.deserialize(job_args).enqueue
-      when "sidekiq"
-        job_name.constantize.perform_async(*job_args)
-      else
-        raise UnknownJobAdapter.new(adapter: adapter)
-      end
-
-      # NOTE: record will be deleted after the job has successfully been performed
-      true
-    end
-  end
-
-  module UpgradeService
-    def self.execute
-      # prepare an array to hold the attribute hashes to be passed to `insert_all`
-      run_attributes = []
-      # prepare an array to hold any `Key` records that we couldn't successfully map to `Run` records
-      errored_keys = []
-
-      # iterate over all `AcidicJob::Key` records in batches,
-      # preparing a `Run` attribute hash to be passed to `insert_all`
-      ::AcidicJob::Key.find_each do |key|
-        # map all of the simple attributes directly
-        attributes = {
-          id: key.id,
-          staged: false,
-          idempotency_key: key.idempotency_key,
-          job_class: key.job_name,
-          last_run_at: key.last_run_at,
-          locked_at: key.locked_at,
-          recovery_point: key.recovery_point,
-          error_object: key.error_object,
-          attr_accessors: key.attr_accessors,
-          workflow: key.workflow,
-          created_at: key.created_at,
-          updated_at: key.updated_at
-        }
-
-        # prepare the more complicated `job_args` -> `serialized_job` translation
-        job_class = key.job_name.constantize
-        if defined?(::Sidekiq) && job_class.include?(::Sidekiq::Worker)
-          unless job_class.include?(::AcidicJob::Extensions::Sidekiq)
-            job_class.include(::AcidicJob::Extensions::Sidekiq)
-          end
-          job_instance = job_class.new
-          serialized_job = job_instance.serialize_job(*key.job_args)
-        elsif defined?(::ActiveJob) && job_class < ::ActiveJob::Base
-          unless job_class.include?(::AcidicJob::Extensions::ActiveJob)
-            job_class.include(::AcidicJob::Extensions::ActiveJob)
-          end
-          job_args = begin
-            ::ActiveJob::Arguments.deserialize(key.job_args)
-          rescue ::ActiveJob::DeserializationError
-            key.job_args
-          end
-          job_instance = job_class.new(*job_args)
-          serialized_job = job_instance.serialize_job
-        end
-
-        attributes[:serialized_job] = serialized_job
-        run_attributes << attributes
-      rescue StandardError => e
-        errored_keys << [e, key]
-      end
-
-      # insert all of the `Run` records
-      ::AcidicJob::Run.insert_all(run_attributes)
-
-      # delete all successfully migrated `Key` record
-      ::AcidicJob::Key.where(id: ::AcidicJob::Run.select(:id)).delete_all
-
-      # return a report of the upgrade migration
-      {
-        run_records: ::AcidicJob::Run.count,
-        key_records: ::AcidicJob::Key.count,
-        errored_keys: errored_keys
-      }
-    end
-  end
-end