acidic_job 1.0.0.pre20 → 1.0.0.pre23

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e608eefef4baffbfb2c95dcbef942a53c9ca242c4c593ddbbe0d677ef7d318da
-  data.tar.gz: 65bed32739c1a1a860102e59dac69598c4357512034d8d3e87ed772469b9bac0
+  metadata.gz: 5fb0d14a9e26e9a79f4976dc74318e97c096c695dacfed5a6546dff058ca4b1b
+  data.tar.gz: abe13e27abaf05a1200a1bc245e649e00da1948847576df3ca1774101ca03d28
 SHA512:
-  metadata.gz: 5a5f6155e20f929c631392f00480f57928455778e13117609457be9b8624e7f16c41661df96ae34a4045582f957b044cbdf811bfa7e187dd627b8b72b80edb90
-  data.tar.gz: 6a663d288bd1ef3ba5cfac8e137fd496ebbca76e08573e50d706ed2570c98b6fe3148b8c9335b55313a091ccd0cc226e3b9023878dd418d400af33fd0280c1b4
+  metadata.gz: 1f7abb59cb78202482f82e4c9057afabf3dc687b1c723d10c2b60e8bcea3a7bb6bd4c9c6b8d835f8498076f2a75fa124baba1571e4f16093d80c6fc0228c8658
+  data.tar.gz: cea09574f0843c71b52c542eaeef20cd1ee330fd11a1f7c37977126d2c321ea3e055e8073b4c4ae525af17b133c3d98b50f1fa9d2ead2b93026b45897dc5945e

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    acidic_job (1.0.0.pre20)
+    acidic_job (1.0.0.pre23)
       activerecord
       activesupport
       database_cleaner

data/README.md

@@ -269,6 +269,56 @@ class RideCreateJob < ActiveJob::Base
 end
 ```
 
+If you need to await a job that takes arguments, you can prepare that job along with its arguments using the `with` class method that `acidic_job` will add to your jobs:
+
+```ruby
+class RideCreateJob < ActiveJob::Base
+  include AcidicJob
+
+  def perform(user_id, ride_params)
+    @user = User.find(user_id)
+    @params = ride_params
+
+    with_acidity providing: { ride: nil } do
+      step :create_ride_and_audit_record, awaits: awaits: [SomeJob.with('argument_1', keyword: 'value')]
+      step :create_stripe_charge, args: [1, 2, 3], kwargs: { some: 'thing' }
+      step :send_receipt
+    end
+  end
+end
+```
+
+You can also await a batch of jobs by simply passing multiple jobs to the `awaits` array (e.g. `awaits: [SomeJob, AnotherJob.with('argument_1', keyword: 'value')]`). Your top level workflow job will only continue to the next step once all of the jobs in your `awaits` array have successfully finished.
+
+In some cases, you may need to _dynamically_ determine the collection of jobs that the step should wait for; in these cases, you can pass the name of a method to the `awaits` option:
+
+```ruby
+class RideCreateJob < ActiveJob::Base
+  include AcidicJob
+  set_callback :finish, :after, :delete_run_record
+
+  def perform(user_id, ride_params)
+    @user = User.find(user_id)
+    @params = ride_params
+
+    with_acidity providing: { ride: nil } do
+      step :create_ride_and_audit_record, awaits: :dynamic_awaits
+      step :create_stripe_charge, args: [1, 2, 3], kwargs: { some: 'thing' }
+      step :send_receipt
+    end
+  end
+  
+  def dynamic_awaits
+    if @params["key"].present?
+      [SomeJob.with('argument_1', keyword: 'value')]
+    else
+      [AnotherJob]
+    end
+  end
+end
+```
+
+
 ### Run Finished Callbacks
 
 When working with workflow jobs that make use of the `awaits` feature for a step, it is important to remember that the `after_perform` callback will be called _as soon as the first `awaits` step has enqueued job_, and **not** when the entire job run has finished. `acidic_job` allows the `perform` method to finish so that the queue for the workflow job is cleared to pick up new work while the `awaits` jobs are running. `acidic_job` will automatically re-enqueue the workflow job and progress to the next step when all of the `awaits` jobs have successfully finished. However, this means that `after_perform` **is not necessarily** the same as `after_finish`. In order to provide the opportunity for you to execute callback logic _if and only if_ a job run has finished, we provide callback hooks for the `finish` event.
@@ -285,7 +335,7 @@ class RideCreateJob < ActiveJob::Base
     @params = ride_params
 
     with_acidity providing: { ride: nil } do
-      step :create_ride_and_audit_record, awaits: [SomeJob]
+      step :create_ride_and_audit_record, awaits: [SomeJob.with('argument_1', keyword: 'value')]
       step :create_stripe_charge, args: [1, 2, 3], kwargs: { some: 'thing' }
       step :send_receipt
     end

data/lib/acidic_job/awaiting.rb

@@ -6,37 +6,61 @@ module AcidicJob
   module Awaiting
     extend ActiveSupport::Concern
 
-    def enqueue_step_parallel_jobs(jobs, run, step_result)
-      # `batch` is available from Sidekiq::Pro
-      raise SidekiqBatchRequired unless defined?(Sidekiq::Batch)
-
-      step_batch = Sidekiq::Batch.new
-      # step_batch.description = "AcidicJob::Workflow Step: #{step}"
-      step_batch.on(
-        :success,
-        "#{self.class.name}#step_done",
-        # NOTE: options are marshalled through JSON so use only basic types.
-        { "run_id" => run.id,
-          "step_result_yaml" => step_result.to_yaml.strip,
-          "parent_worker" => self.class.name,
-          "job_names" => jobs.map(&:to_s) }
-      )
-
-      # NOTE: The jobs method is atomic.
-      # All jobs created in the block are actually pushed atomically at the end of the block.
-      # If an error is raised, none of the jobs will go to Redis.
-      step_batch.jobs do
-        jobs.each do |worker_name|
-          # TODO: handle Symbols as well
-          worker = worker_name.is_a?(String) ? worker_name.constantize : worker_name
-
-          if worker.instance_method(:perform).arity.presence_in [0, -1]
-            worker.perform_async
-          elsif worker.instance_method(:perform).arity == 1
-            worker.perform_async(run.id)
-          else
-            raise TooManyParametersForParallelJob
-          end
+    private
+
+    def was_awaited_job?
+      (acidic_job_run.present? && acidic_job_run.awaited_by.present?) ||
+        (staged_job_run.present? && staged_job_run.awaited_by.present?)
+    end
+
+    def reenqueue_awaited_by_job
+      run = staged_job_run&.awaited_by || acidic_job_run&.awaited_by
+
+      return unless run
+
+      current_step = run.workflow[run.recovery_point.to_s]
+      step_result = run.returning_to
+
+      job = run.job_class.constantize.deserialize(run.serialized_job)
+      # this needs to be explicitly set so that `was_workflow_job?` appropriately returns `true`
+      # which is what the `after_finish :reenqueue_awaited_by_job` check needs
+      job.instance_variable_set(:@acidic_job_run, run)
+
+      step = Step.new(current_step, run, job, step_result)
+      # TODO: WRITE REGRESSION TESTS FOR PARALLEL JOB FAILING AND RETRYING THE ORIGINAL STEP
+      step.progress
+
+      return if run.finished?
+
+      # job = job_class.constantize.deserialize(serialized_staged_job)
+      # job.enqueue
+
+      # when a batch of jobs for a step succeeds, we begin processing the `AcidicJob::Run` record again
+      process_run(run)
+    end
+
+    def enqueue_step_parallel_jobs(jobs_or_jobs_getter, run, step_result)
+      awaited_jobs = case jobs_or_jobs_getter
+                     when Array
+                       jobs_or_jobs_getter
+                     when Symbol, String
+                       method(jobs_or_jobs_getter).call
+                     end
+
+      AcidicJob::Run.transaction do
+        awaited_jobs.each do |awaited_job|
+          worker_class, args, kwargs = job_args_and_kwargs(awaited_job)
+
+          job = worker_class.new(*args, **kwargs)
+
+          AcidicJob::Run.create!(
+            staged: true,
+            awaited_by: run,
+            job_class: worker_class,
+            serialized_job: job.serialize_job(*args, **kwargs),
+            idempotency_key: job.idempotency_key
+          )
+          run.update(returning_to: step_result)
         end
       end
     end
@@ -53,5 +77,22 @@ module AcidicJob
       # when a batch of jobs for a step succeeds, we begin processing the `AcidicJob::Run` record again
       process_run(run)
     end
+
+    def job_args_and_kwargs(job)
+      case job
+      when Class
+        [job, [], {}]
+      when String
+        [job.constantize, [], {}]
+      when Symbol
+        [job.to_s.constantize, [], {}]
+      else
+        [
+          job.class,
+          job.instance_variable_get(:@__acidic_job_args),
+          job.instance_variable_get(:@__acidic_job_kwargs)
+        ]
+      end
+    end
   end
 end

data/lib/acidic_job/extensions/sidekiq.rb

@@ -13,7 +13,11 @@ module AcidicJob
         class_methods do
           # called only from `AcidicJob::Run#enqueue_staged_job`
           def deserialize(serialized_job_hash)
-            klass = serialized_job_hash["class"].constantize
+            klass = if serialized_job_hash["class"].is_a?(Class)
+                      serialized_job_hash["class"]
+                    else
+                      serialized_job_hash["class"].constantize
+                    end
             worker = klass.new
             worker.jid = serialized_job_hash["jid"]
             worker.instance_variable_set(:@args, serialized_job_hash["args"])

data/lib/acidic_job/idempotency_key.rb

@@ -20,7 +20,15 @@ module AcidicJob
                 end
               end
 
-      value || value_from_job_args(hash_or_job, *args, **kwargs)
+      result = value || value_from_job_args(hash_or_job, *args, **kwargs)
+
+      if result.start_with?("STG__")
+        # "STG__#{idempotency_key}__#{encoded_global_id}"
+        _prefix, idempotency_key, _encoded_global_id = result.split("__")
+        idempotency_key
+      else
+        result
+      end
     end
 
     private

data/lib/acidic_job/run.rb

@@ -12,11 +12,15 @@ module AcidicJob
 
     self.table_name = "acidic_job_runs"
 
+    belongs_to :awaited_by, class_name: "AcidicJob::Run", optional: true
+    has_many :batched_runs, class_name: "AcidicJob::Run", foreign_key: "awaited_by_id"
+
     after_create_commit :enqueue_staged_job, if: :staged?
 
     serialize :error_object
     serialize :serialized_job
     serialize :workflow
+    serialize :returning_to
     store :attr_accessors
 
     validates :staged, inclusion: { in: [true, false] } # uses database default
@@ -69,7 +73,7 @@ module AcidicJob
       # base64 encoding for minimal security
       global_id = to_global_id.to_s.remove("gid://")
       encoded_global_id = Base64.encode64(global_id).strip
-      staged_job_id = "STG_#{idempotency_key}__#{encoded_global_id}"
+      staged_job_id = "STG__#{idempotency_key}__#{encoded_global_id}"
 
       serialized_staged_job = if serialized_job.key?("jid")
                                 serialized_job.merge("jid" => staged_job_id)

data/lib/acidic_job/staging.rb

@@ -7,6 +7,8 @@ module AcidicJob
   module Staging
     extend ActiveSupport::Concern
 
+    private
+
     def delete_staged_job_record
       return unless was_staged_job?
 
@@ -17,7 +19,7 @@ module AcidicJob
     end
 
     def was_staged_job?
-      identifier.start_with? "STG_"
+      identifier.start_with? "STG__"
     end
 
     def staged_job_run
@@ -26,6 +28,8 @@ module AcidicJob
       staged_job_gid = "gid://#{Base64.decode64(encoded_global_id)}"
 
       GlobalID::Locator.locate(staged_job_gid)
+    rescue ActiveRecord::RecordNotFound
+      nil
     end
 
     def identifier
@@ -33,7 +37,9 @@ module AcidicJob
       return job_id if defined?(job_id) && !job_id.nil?
 
       # might be defined already in `with_acidity` method
-      @__acidic_job_idempotency_key ||= IdempotencyKey.value_for(self, @__acidic_job_args, @__acidic_job_kwargs)
+      acidic_identifier = self.class.acidic_identifier
+      @__acidic_job_idempotency_key ||= IdempotencyKey.new(acidic_identifier)
+                                                      .value_for(self, *@__acidic_job_args, **@__acidic_job_kwargs)
 
       @__acidic_job_idempotency_key
     end

data/lib/acidic_job/step.rb

@@ -42,7 +42,13 @@ module AcidicJob
     # The progression phase advances the job run state machine onto the next step
     def progress
       @run.with_lock do
-        @step_result.call(run: @run)
+        if @step_result.is_a?(FinishedPoint)
+          @job.run_callbacks :finish do
+            @step_result.call(run: @run)
+          end
+        else
+          @step_result.call(run: @run)
+        end
       end
     end
 

data/lib/acidic_job/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module AcidicJob
-  VERSION = "1.0.0.pre20"
+  VERSION = "1.0.0.pre23"
 end

data/lib/acidic_job.rb

@@ -48,8 +48,15 @@ module AcidicJob
     end
 
     # TODO: write test for a staged job that uses awaits
-    klass.set_callback :perform, :after, :delete_staged_job_record, if: :was_staged_job?
+    klass.set_callback :perform, :after, :reenqueue_awaited_by_job,
+                       if: -> { was_awaited_job? && !was_workflow_job? }
+    klass.set_callback :perform, :after, :delete_staged_job_record,
+                       if: -> { was_staged_job? && !was_awaited_job? }
     klass.define_callbacks :finish
+    klass.set_callback :finish, :after, :reenqueue_awaited_by_job,
+                       if: -> { was_workflow_job? && was_awaited_job? }
+    klass.set_callback :finish, :after, :delete_staged_job_record,
+                       if: -> { was_workflow_job? && was_staged_job? && !was_awaited_job? }
 
     klass.instance_variable_set(:@acidic_identifier, :job_id)
     klass.define_singleton_method(:acidic_by_job_id) { @acidic_identifier = :job_id }
@@ -68,6 +75,10 @@ module AcidicJob
       super
     end
 
+    def with(*args, **kwargs)
+      new(*args, **kwargs)
+    end
+
     def acidic_identifier
       @acidic_identifier
     end
@@ -79,11 +90,11 @@ module AcidicJob
     @__acidic_job_args = args
     @__acidic_job_kwargs = kwargs
 
-    if method(__method__).super_method.arity.zero?
-      super()
-    else
-      super(*args, **kwargs)
-    end
+    super(*args, **kwargs)
+  rescue ArgumentError => e
+    raise e unless e.message.include?("wrong number of arguments")
+
+    super()
   end
 
   def with_acidity(providing: {})
@@ -129,9 +140,13 @@ module AcidicJob
 
   private
 
+  def was_workflow_job?
+    defined?(@acidic_job_run) && @acidic_job_run.present?
+  end
+
   def process_run(run)
     # if the run record is already marked as finished, immediately return its result
-    return finish_run(run) if run.finished?
+    return run.succeeded? if run.finished?
 
     # otherwise, we will enter a loop to process each step of the workflow
     loop do
@@ -144,7 +159,7 @@ module AcidicJob
         break
       elsif current_step.nil?
         raise UnknownRecoveryPoint, "Defined workflow does not reference this step: #{recovery_point}"
-      elsif (jobs = current_step.fetch("awaits", [])).any?
+      elsif !(jobs = current_step.fetch("awaits", []) || []).empty?
         step = Step.new(current_step, run, self)
         # Only execute the current step, without yet progressing the recovery_point to the next step.
         # This ensures that any failures in parallel jobs will have this step retried in the main workflow
@@ -170,13 +185,7 @@ module AcidicJob
     end
 
     # the loop will break once the job is finished, so simply report the status
-    finish_run(run)
-  end
-
-  def finish_run(run)
-    run_callbacks :finish do
-      run.succeeded?
-    end
+    run.succeeded?
   end
 
   def step(method_name, awaits: [], for_each: nil)
@@ -219,9 +228,7 @@ module AcidicJob
       if run.present?
         # Programs enqueuing multiple jobs with different parameters but the
         # same idempotency key is a bug.
-        # NOTE: WOULD THE ENQUEUED_AT OR CREATED_AT FIELD BE NECESSARILY DIFFERENT?
-        if run.serialized_job.except("jid", "job_id",
-                                     "enqueued_at") != serialized_job.except("jid", "job_id", "enqueued_at")
+        if run.serialized_job.slice("args", "arguments") != serialized_job.slice("args", "arguments")
           raise MismatchedIdempotencyKeyAndJobArguments
         end
 
@@ -230,7 +237,14 @@ module AcidicJob
         raise LockedIdempotencyKey if run.locked_at && run.locked_at > Time.current - IDEMPOTENCY_KEY_LOCK_TIMEOUT
 
         # Lock the run and update latest run unless the job is already finished.
-        run.update!(last_run_at: Time.current, locked_at: Time.current, workflow: workflow) unless run.finished?
+        unless run.finished?
+          run.update!(
+            last_run_at: Time.current,
+            locked_at: Time.current,
+            workflow: workflow,
+            recovery_point: run.recovery_point || workflow.first.first
+          )
+        end
       else
         run = Run.create!(
           staged: false,

data/lib/generators/acidic_job/templates/create_acidic_job_runs_migration.rb.erb

@@ -1,19 +1,19 @@
 class <%= migration_class_name %> < ActiveRecord::Migration<%= migration_version %>
   def change
     create_table :acidic_job_runs do |t|
-      t.boolean 	:staged, 					null: false, 	default: -> { false }
-      t.string 		:idempotency_key, null: false
-      t.text 			:serialized_job, 	null: false
-      t.string 		:job_class, 			null: false
-      t.datetime 	:last_run_at, 		null: true, 	default: -> { "CURRENT_TIMESTAMP" }
-      t.datetime 	:locked_at, 			null: true
-      t.string 		:recovery_point, 	null: true
-      t.text 			:error_object, 		null: true
-      t.text 			:attr_accessors, 	null: true
-      t.text 			:workflow, 				null: true
+      t.boolean     :staged, 					null: false,  default: -> { false }
+      t.string      :idempotency_key, null: false,  index: { unique: true }
+      t.text        :serialized_job, 	null: false
+      t.string      :job_class, 			null: false
+      t.datetime    :last_run_at, 		null: true,   default: -> { "CURRENT_TIMESTAMP" }
+      t.datetime    :locked_at, 			null: true
+      t.string      :recovery_point, 	null: true
+      t.text        :error_object, 		null: true
+      t.text        :attr_accessors, 	null: true
+      t.text        :workflow, 				null: true
+      t.references  :awaited_by,      null: true,   index: true
+      t.text        :returning_to,    null: true
       t.timestamps
-    
-      t.index :idempotency_key, unique: true
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: acidic_job
 version: !ruby/object:Gem::Version
-  version: 1.0.0.pre20
+  version: 1.0.0.pre23
 platform: ruby
 authors:
 - fractaledmind
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-05-16 00:00:00.000000000 Z
+date: 2022-05-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord