acidic_job 0.7.7 → 1.0.0.pre3

data/lib/acidic_job/extensions/noticed.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module AcidicJob
+  module Extensions
+    module Noticed
+      extend ActiveSupport::Concern
+
+      class_methods do
+        def deliver_acidicly(recipients)
+          new.deliver_acidicly(recipients)
+        end
+      end
+
+      def deliver_acidicly(recipients)
+        # THIS IS A HACK THAT COPIES AND PASTES KEY PARTS OF THE `Noticed::Base` CODE
+        # IN ORDER TO ALLOW US TO TRANSACTIONALLY DELIVER NOTIFICATIONS
+        # THIS IS THUS LIABLE TO BREAK WHENEVER THAT GEM IS UPDATED
+        delivery_methods = self.class.delivery_methods.dup
+
+        Array.wrap(recipients).uniq.each do |recipient|
+          if (index = delivery_methods.find_index { |m| m[:name] == :database })
+            database_delivery_method = delivery_methods.delete_at(index)
+            self.record = run_delivery_method(database_delivery_method,
+                                              recipient: recipient,
+                                              enqueue: false,
+                                              record: nil)
+          end
+
+          delivery_methods.map do |delivery_method|
+            job_class = delivery_method_for(delivery_method[:name], delivery_method[:options])
+            args = {
+              notification_class: self.class.name,
+              options: delivery_method[:options],
+              params: params,
+              recipient: recipient,
+              record: record
+            }
+            serialized_job = job_class.send(:job_or_instantiate, args).serialize
+
+            AcidicJob::Run.create!(
+              staged: true,
+              job_class: job_class.name,
+              serialized_job: serialized_job,
+              idempotency_key: IdempotencyKey.value_for(serialized_job)
+            )
+          end
+        end
+      end
+      alias deliver_transactionally deliver_acidicly
+    end
+  end
+end

data/lib/acidic_job/extensions/sidekiq.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+require "active_support/concern"
+require "active_support/callbacks"
+require "active_support/core_ext/module/concerning"
+
+module AcidicJob
+  module Extensions
+    module Sidekiq
+      extend ActiveSupport::Concern
+
+      concerning :Serialization do
+        class_methods do
+          # called only from `AcidicJob::Run#enqueue_staged_job`
+          def deserialize(serialized_job_hash)
+            klass = serialized_job_hash["class"].constantize
+            worker = klass.new
+            worker.jid = serialized_job_hash["jid"]
+            worker.instance_variable_set(:@args, serialized_job_hash["args"])
+
+            worker
+          end
+
+          # called only from `AcidicJob::PerformAcidicly#perform_acidicly`
+          # and `AcidicJob::DeliverAcidicly#deliver_acidicly`
+          def serialize_with_arguments(args = [], _kwargs = nil)
+            # THIS IS A HACK THAT ESSENTIALLY COPIES THE CODE FROM THE SIDEKIQ CODEBASE TO MIMIC THE BEHAVIOR
+            args = Array[args]
+            normalized_args = ::Sidekiq.load_json(::Sidekiq.dump_json(args))
+            item = { "class" => self, "args" => normalized_args }
+            dummy_sidekiq_client = ::Sidekiq::Client.new
+            normed = dummy_sidekiq_client.send :normalize_item, item
+            dummy_sidekiq_client.send :process_single, item["class"], normed
+          end
+        end
+
+        def serialize_job(args = [], _kwargs = nil)
+          # `@args` is only set via `deserialize`; it is not a standard Sidekiq thing
+          arguments = args || @args
+          normalized_args = ::Sidekiq.load_json(::Sidekiq.dump_json(arguments))
+          item = { "class" => self.class, "args" => normalized_args, "jid" => jid }
+          sidekiq_options = sidekiq_options_hash || {}
+
+          sidekiq_options.merge(item)
+        end
+
+        # called only from `AcidicJob::Run#enqueue_staged_job`
+        def enqueue
+          ::Sidekiq::Client.push(
+            "class" => self.class,
+            "args" => @args,
+            "jid" => @jid
+          )
+        end
+      end
+
+      concerning :PerformAcidicly do
+        class_methods do
+          def perform_acidicly(*args, **kwargs)
+            serialized_job = serialize_with_arguments(*args, **kwargs)
+
+            AcidicJob::Run.create!(
+              staged: true,
+              job_class: name,
+              serialized_job: serialized_job,
+              idempotency_key: IdempotencyKey.value_for(serialized_job)
+            )
+          end
+          alias_method :perform_transactionally, :perform_acidicly
+        end
+      end
+
+      # to balance `perform_async` class method
+      concerning :PerformSync do
+        class_methods do
+          def perform_sync(*args, **kwargs)
+            new.perform(*args, **kwargs)
+          end
+        end
+      end
+
+      # Following approach used by ActiveJob
+      # https://github.com/rails/rails/blob/93c9534c9871d4adad4bc33b5edc355672b59c61/activejob/lib/active_job/callbacks.rb
+      concerning :Callbacks do
+        class_methods do
+          def around_perform(*filters, &blk)
+            set_callback(:perform, :around, *filters, &blk)
+          end
+
+          def before_perform(*filters, &blk)
+            set_callback(:perform, :before, *filters, &blk)
+          end
+
+          def after_perform(*filters, &blk)
+            set_callback(:perform, :after, *filters, &blk)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/acidic_job/{response.rb → finished_point.rb}

@@ -4,12 +4,12 @@
 # idempotency key). One  possible option for a return from an #atomic_phase
 # block.
 module AcidicJob
-  class Response
-    def call(key:)
+  class FinishedPoint
+    def call(run:)
       # Skip AR callbacks as there are none on the model
-      key.update_columns(
+      run.update_columns(
         locked_at: nil,
-        recovery_point: Key::RECOVERY_POINT_FINISHED
+        recovery_point: Run::FINISHED_RECOVERY_POINT
       )
     end
   end

data/lib/acidic_job/idempotency_key.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module AcidicJob
+  class IdempotencyKey
+    def self.value_for(hash_or_job, *args, **kwargs)
+      return hash_or_job.job_id if hash_or_job.respond_to?(:job_id) && !hash_or_job.job_id.nil?
+      return hash_or_job.jid if hash_or_job.respond_to?(:jid) && !hash_or_job.jid.nil?
+
+      if hash_or_job.is_a?(Hash) && hash_or_job.key?("job_id") && !hash_or_job["job_id"].nil?
+        return hash_or_job["job_id"]
+      end
+      return hash_or_job["jid"] if hash_or_job.is_a?(Hash) && hash_or_job.key?("jid") && !hash_or_job["jid"].nil?
+
+      worker_class = case hash_or_job
+                     when Hash
+                       hash_or_job["worker"] || hash_or_job["job_class"]
+                     else
+                       hash_or_job.class.name
+                     end
+
+      Digest::SHA1.hexdigest [worker_class, args, kwargs].flatten.join
+    end
+  end
+end

data/lib/acidic_job/perform_wrapper.rb

@@ -1,35 +1,49 @@
 # frozen_string_literal: true
 
 module AcidicJob
+  # NOTE: it is essential that this be a bare module and not an ActiveSupport::Concern
   module PerformWrapper
     def perform(*args, **kwargs)
-      # extract the `staged_job_gid` if present
-      # so that we can later delete the record in an `after_perform` callback
-      final_arg = args.last
-      if final_arg.is_a?(Hash) && final_arg.key?("staged_job_gid")
-        args = args[0..-2]
-        @staged_job_gid = final_arg["staged_job_gid"]
+      super_method = method(:perform).super_method
+
+      # we don't want to run the `perform` callbacks twice, since ActiveJob already handles that for us
+      if aj_job?
+        __acidic_job_perform_for_aj(super_method, *args, **kwargs)
+      elsif sk_job?
+        __acidic_job_perform_for_sk(super_method, *args, **kwargs)
+      else
+        raise UnknownJobAdapter
       end
+    end
 
-      set_arguments_for_perform(*args, **kwargs)
+    def sk_job?
+      defined?(Sidekiq) && self.class.include?(Sidekiq::Worker)
+    end
 
-      super(*args, **kwargs)
+    def aj_job?
+      defined?(ActiveJob) && self.class < ActiveJob::Base
     end
 
     private
 
-    def set_arguments_for_perform(*args, **kwargs)
-      # store arguments passed into `perform` so that we can later persist
-      # them to `AcidicJob::Key#job_args` for both ActiveJob and Sidekiq::Worker
-      @arguments_for_perform = if args.any? && kwargs.any?
-                                 args + [kwargs]
-                               elsif args.any? && kwargs.none?
-                                 args
-                               elsif args.none? && kwargs.any?
-                                 [kwargs]
-                               else
-                                 []
-                               end
+    # don't run `perform` callbacks, as ActiveJob already does this
+    def __acidic_job_perform_for_aj(super_method, *args, **kwargs)
+      __acidic_job_perform_base(super_method, *args, **kwargs)
+    end
+
+    # ensure to run `perform` callbacks
+    def __acidic_job_perform_for_sk(super_method, *args, **kwargs)
+      run_callbacks :perform do
+        __acidic_job_perform_base(super_method, *args, **kwargs)
+      end
+    end
+
+    # capture arguments passed to `perform` to be used by AcidicJob later
+    def __acidic_job_perform_base(super_method, *args, **kwargs)
+      @__acidic_job_args = args
+      @__acidic_job_kwargs = kwargs
+
+      super_method.call(*args, **kwargs)
     end
   end
 end

data/lib/acidic_job/recovery_point.rb

@@ -7,12 +7,12 @@ module AcidicJob
     attr_accessor :name
 
     def initialize(name)
-      self.name = name
+      @name = name
     end
 
-    def call(key:)
+    def call(run:)
       # Skip AR callbacks as there are none on the model
-      key.update_column(:recovery_point, name)
+      run.update_column(:recovery_point, @name)
     end
   end
 end

data/lib/acidic_job/run.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+require "active_record"
+require "global_id"
+require "active_support/core_ext/object/with_options"
+
+module AcidicJob
+  class Run < ActiveRecord::Base
+    include GlobalID::Identification
+
+    FINISHED_RECOVERY_POINT = "FINISHED"
+
+    self.table_name = "acidic_job_runs"
+
+    after_create_commit :enqueue_staged_job, if: :staged?
+
+    serialize :error_object
+    serialize :serialized_job
+    serialize :workflow
+    store :attr_accessors
+
+    validates :staged, inclusion: { in: [true, false] } # uses database default
+    validates :serialized_job, presence: true
+    validates :idempotency_key, presence: true, uniqueness: true
+    validates :job_class, presence: true
+
+    scope :staged, -> { where(staged: true) }
+    scope :unstaged, -> { where(staged: false) }
+    scope :finished, -> { where(recovery_point: FINISHED_RECOVERY_POINT) }
+    scope :running, -> { where.not(recovery_point: FINISHED_RECOVERY_POINT) }
+
+    with_options unless: :staged? do
+      validates :last_run_at, presence: true
+      validates :recovery_point, presence: true
+      validates :workflow, presence: true
+    end
+
+    def finished?
+      recovery_point == FINISHED_RECOVERY_POINT
+    end
+
+    def succeeded?
+      finished? && !failed?
+    end
+
+    def failed?
+      error_object.present?
+    end
+
+    private
+
+    def enqueue_staged_job
+      return unless staged?
+
+      # encode the identifier for this record in the job ID
+      # base64 encoding for minimal security
+      global_id = to_global_id.to_s.remove("gid://")
+      encoded_global_id = Base64.encode64(global_id).strip
+      staged_job_id = "STG_#{idempotency_key}__#{encoded_global_id}"
+
+      serialized_staged_job = if serialized_job.key?("jid")
+                                serialized_job.merge("jid" => staged_job_id)
+                              elsif serialized_job.key?("job_id")
+                                serialized_job.merge("job_id" => staged_job_id)
+                              else
+                                raise UnknownSerializedJobIdentifier
+                              end
+
+      job = job_class.constantize.deserialize(serialized_staged_job)
+
+      job.enqueue
+
+      # NOTE: record will be deleted after the job has successfully been performed
+      true
+    end
+  end
+end

data/lib/acidic_job/staging.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require "active_support/concern"
+
+module AcidicJob
+  module Staging
+    extend ActiveSupport::Concern
+
+    def delete_staged_job_record
+      return unless was_staged_job?
+
+      staged_job_run.delete
+      true
+    rescue ActiveRecord::RecordNotFound
+      true
+    end
+
+    def was_staged_job?
+      identifier.start_with? "STG_"
+    end
+
+    def staged_job_run
+      # "STG_#{idempotency_key}__#{encoded_global_id}"
+      encoded_global_id = identifier.split("__").last
+      staged_job_gid = "gid://#{Base64.decode64(encoded_global_id)}"
+
+      GlobalID::Locator.locate(staged_job_gid)
+    end
+  end
+end

data/lib/acidic_job/step.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+module AcidicJob
+  # Each AcidicJob::Step requires two phases: [1] execution and [2] progression
+  class Step
+    def initialize(step, run, job, step_result = nil)
+      @step = step
+      @run = run
+      @job = job
+      @step_result = step_result
+    end
+
+    # The execution phase performs the work of the defined step
+    def execute
+      rescued_error = false
+      step_callable = wrap_step_as_acidic_callable @step
+
+      begin
+        @run.with_lock do
+          @step_result = step_callable.call(@run)
+        end
+      # QUESTION: Can an error not inherit from StandardError
+      rescue StandardError => e
+        rescued_error = e
+        raise e
+      ensure
+        if rescued_error
+          # If we're leaving under an error condition, try to unlock the job
+          # run right away so that another request can try again.
+          begin
+            @run.update_columns(locked_at: nil, error_object: rescued_error)
+          rescue StandardError => e
+            # We're already inside an error condition, so swallow any additional
+            # errors from here and just send them to logs.
+            # TODO: implement and use a logger here
+            puts "Failed to unlock AcidicJob::Run #{@run.id} because of #{e}."
+          end
+        end
+      end
+    end
+
+    # The progression phase advances the job run state machine onto the next step
+    def progress
+      @run.with_lock do
+        @step_result.call(run: @run)
+      end
+    end
+
+    private
+
+    def wrap_step_as_acidic_callable(step)
+      # {"does" => :enqueue_step, "then" => :next_step, "awaits" => [WorkerWithEnqueueStep::FirstWorker]}
+      current_step = step["does"]
+      next_step = step["then"]
+
+      # jobs can have no-op steps, especially so that they can use only the async/await mechanism for that step
+      callable = if @job.respond_to?(current_step, _include_private = true)
+                   @job.method(current_step)
+                 else
+                   proc {}
+                 end
+
+      # return a callable Proc with a consistent interface for the execution phase
+      proc do |run|
+        result = if callable.arity.zero?
+                   callable.call
+                 elsif callable.arity == 1
+                   callable.call(run)
+                 else
+                   raise TooManyParametersForStepMethod
+                 end
+
+        if result.is_a?(FinishedPoint)
+          result
+        elsif next_step.to_s == Run::FINISHED_RECOVERY_POINT
+          FinishedPoint.new
+        else
+          RecoveryPoint.new(next_step)
+        end
+      end
+    end
+  end
+end

data/lib/acidic_job/upgrade_service.rb

@@ -0,0 +1,115 @@
+# frozen_string_literal: true
+
+require "active_support/concern"
+
+module AcidicJob
+	# recreate the original `Key` model
+	class Key < ::ActiveRecord::Base
+		RECOVERY_POINT_FINISHED = "FINISHED"
+
+		self.table_name = "acidic_job_keys"
+
+		serialize :error_object
+		serialize :job_args
+		serialize :workflow
+		store :attr_accessors
+	end
+
+	# recreate the original `Staged` model
+	class Staged < ActiveRecord::Base
+		self.table_name = "staged_acidic_jobs"
+	
+		serialize :job_args
+
+		after_create_commit :enqueue_job
+
+		private
+	
+		# rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+		def enqueue_job
+			gid = { "staged_job_gid" => to_global_id.to_s }
+	
+			if job_args.is_a?(Hash) && job_args.key?("arguments")
+				job_args["arguments"].concat([gid])
+			else
+				job_args.concat([gid])
+			end
+	
+			case adapter
+			when "activejob"
+				::ActiveJob::Base.deserialize(job_args).enqueue
+			when "sidekiq"
+				job_name.constantize.perform_async(*job_args)
+			else
+				raise UnknownJobAdapter.new(adapter: adapter)
+			end
+	
+			# NOTE: record will be deleted after the job has successfully been performed
+			true
+		end
+		# rubocop:enable Metrics/AbcSize, Metrics/MethodLength
+	end
+
+	module UpgradeService
+		def self.execute()
+			# prepare an array to hold the attribute hashes to be passed to `insert_all`
+			run_attributes = []
+			# prepare an array to hold any `Key` records that we couldn't successfully map to `Run` records
+			errored_keys = []
+
+			# iterate over all `AcidicJob::Key` records in batches, preparing a `Run` attribute hash to be passed to `insert_all`
+			::AcidicJob::Key.find_each do |key|
+				# map all of the simple attributes directly
+				attributes = {
+					id: key.id, 
+					staged: false, 
+					idempotency_key: key.idempotency_key, 
+					job_class: key.job_name, 
+					last_run_at: key.last_run_at, 
+					locked_at: key.locked_at, 
+					recovery_point: key.recovery_point, 
+					error_object: key.error_object, 
+					attr_accessors: key.attr_accessors, 
+					workflow: key.workflow, 
+					created_at: key.created_at, 
+					updated_at: key.updated_at 
+				}
+
+				# prepare the more complicated `job_args` -> `serialized_job` translation
+				job_class = key.job_name.constantize
+				if defined?(::Sidekiq) && job_class.include?(::Sidekiq::Worker)
+					job_class.include(::AcidicJob::Extensions::Sidekiq) unless job_class.include?(::AcidicJob::Extensions::Sidekiq)
+					job_instance = job_class.new
+					serialized_job = job_instance.serialize_job(*key.job_args)
+				elsif defined?(::ActiveJob) && job_class < ::ActiveJob::Base
+					job_class.include(::AcidicJob::Extensions::ActiveJob) unless job_class.include?(::AcidicJob::Extensions::ActiveJob)
+					job_args = begin
+						::ActiveJob::Arguments.deserialize(key.job_args)
+					rescue ::ActiveJob::DeserializationError
+						key.job_args
+					end
+					job_instance = job_class.new(*job_args)
+					serialized_job = job_instance.serialize_job()
+				end
+
+				attributes[:serialized_job] = serialized_job
+				run_attributes << attributes
+			rescue StandardError => exception
+				errored_keys << [exception, key]
+			end
+
+			# insert all of the `Run` records
+			::AcidicJob::Run.insert_all(run_attributes)
+
+			# delete all successfully migrated `Key` record
+			::AcidicJob::Key.where(id: ::AcidicJob::Run.select(:id)).delete_all
+
+			# return a report of the upgrade migration
+			{
+				run_records: ::AcidicJob::Run.count,
+				key_records: ::AcidicJob::Key.count,
+				errored_keys: errored_keys
+			}
+		end
+	end
+end

data/lib/acidic_job/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module AcidicJob
-  VERSION = "0.7.7"
+  VERSION = "1.0.0.pre3"
 end