acidic_job 1.0.0.pre19 → 1.0.0.pre22

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cb1a8e3e2658bf86b44012171e33458c6e25803e41c255401575bc772cf033db
-  data.tar.gz: a2672fb1755af7ada91b7e8b156cffe951f22d530cbf7d94aee794622f9ad5c8
+  metadata.gz: 892c6a2598d63440a7e749484e790b3f47380954b615bd7b2c4fc4c8de541759
+  data.tar.gz: bf7383000fa8e0fb1c545e5ccb014c7123884eb14da77178dd088bb9b12e73e1
 SHA512:
-  metadata.gz: d6fc3314e1eb512ef9eab52cada5b7705891563715f1d549912b45b13b85bd0bf778a8950a5657037ddf75e7e81819cf8c85a240c191cb71d615e98ca22c7c3a
-  data.tar.gz: 011f57cb97e93bd6e09c2c07d164d06679c8311b8e76a1b1cf9438376e65da5d4d76851cd7e1b6eb4d2f1ffba7064ff722c8cec75db08e78a1e96b7353672d13
+  metadata.gz: 87610f9f920ddb6e34c727a6b4c9e8d98d9ec556bff1dea221e3f588694b4e5f77b482985ba906cc62c5a3593e398e906e8dd6aa815e9c6d3031e410b71abfeb
+  data.tar.gz: 7ca46ec550598f545d3baaafccf09cf082b6eaad01a1d6bb4b8f5a7298d9023b0b33c824183e3c5b30b3a1fbb2a9200286761b4e671adf39569a62b6ac392740

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    acidic_job (1.0.0.pre19)
+    acidic_job (1.0.0.pre22)
       activerecord
       activesupport
       database_cleaner

data/README.md

@@ -68,8 +68,10 @@ It provides a suite of functionality that empowers you to create complex, robust
 * Persisted Attributes — when retrying jobs at later steps, we need to ensure that data created in previous steps is still available to later steps on retry.
 * Transactionally Staged Jobs — enqueue additional jobs within the acidic transaction safely
 * Custom Idempotency Keys — use something other than the job ID for the idempotency key of the job run
+* Iterable Steps — define steps that iterate over some collection fully until moving on to the next step
 * Sidekiq Callbacks — bring ActiveJob-like callbacks into your pure Sidekiq Workers
 * Sidekiq Batches — leverage the power of Sidekiq Pro's `batch` functionality without the hassle
+* Run Finished Callbacks — set callbacks for when a job run finishes fully
 
 ### Transactional Steps
 
@@ -80,9 +82,10 @@ class RideCreateJob < ActiveJob::Base
   include AcidicJob
 
   def perform(user_id, ride_params)
-    user = User.find(user_id)
+    @user = User.find(user_id)
+    @params = ride_params
 
-    with_acidity providing: { user: user, params: ride_params, ride: nil } do
+    with_acidity providing: { ride: nil } do
       step :create_ride_and_audit_record
       step :create_stripe_charge
       step :send_receipt
@@ -128,7 +131,7 @@ class RideCreateJob < ActiveJob::Base
   end
 
   def create_stripe_charge
-    Stripe::Charge.create(amount: 20_00, customer: self.ride.user)
+    Stripe::Charge.create(amount: 20_00, customer: @ride.user)
   end
 
   # ...
@@ -139,6 +142,8 @@ end
 
 **Note:** You will note the use of `self.ride = ...` in the code sample above. In order to call the attribute setter method that will sync with the database record, you _must_ use this style. `@ride = ...` and/or `ride = ...` will both fail to sync the value with the database record.
 
+The default pattern you should follow when defining your `perform` method is to make any values that your `step` methods need access to, but are present at the start of the `perform` method simply instance variables. You only need to `provide` attributes that will be set _during a step_. This means, the initial value will almost always be `nil`.
+
 ### Transactionally Staged Jobs
 
 A standard problem when inside of database transactions is enqueuing other jobs. On the one hand, you could enqueue a job inside of a transaction that then rollbacks, which would leave that job to fail and retry and fail. On the other hand, you could enqueue a job that is picked up before the transaction commits, which would mean the records are not yet available to this job.
@@ -150,9 +155,10 @@ class RideCreateJob < ActiveJob::Base
   include AcidicJob
 
   def perform(user_id, ride_params)
-    user = User.find(user_id)
+    @user = User.find(user_id)
+    @params = ride_params
     
-    with_acidity providing: { user: user, params: ride_params, ride: nil } do
+    with_acidity providing: { ride: nil } do
       step :create_ride_and_audit_record
       step :create_stripe_charge
       step :send_receipt
@@ -211,6 +217,28 @@ end
 
 > **Note:** The signature of the `acidic_by` proc _needs to match the signature_ of the job's `perform` method.
 
+### Iterable Steps
+
+Sometimes our workflows have steps that need to iterate over a collection and perform an action for each item in the collection before moving on to the next step in the workflow. In these cases, we can use the `for_each` option when defining our step to specific the collection, and `acidic_job` will pass each item into your step method for processing, keeping the same transactional guarantees as for any step. This means that if your step encounters an error in processing any item in the collection, when your job is retried, the job will jump right back to that step and right back to that item in the collection to try again.
+
+```ruby
+class ExampleJob < ActiveJob::Base
+  include AcidicJob
+
+  def perform(record:)
+    with_acidity providing: { collection: [1, 2, 3, 4, 5] } do
+      step :process_item, for_each: :collection
+      step :next_step
+    end
+  end
+  
+  def process_item(item)
+    # do whatever work needs to be done with this individual item
+  end
+end
+```
+
+**Note:** The same restrictions apply here as for any persisted attribute — you can only use objects that can be serialized by ActiveRecord.
 
 ### Sidekiq Callbacks
 
@@ -229,9 +257,10 @@ class RideCreateJob < ActiveJob::Base
   include AcidicJob
 
   def perform(user_id, ride_params)
-    user = User.find(user_id)
+    @user = User.find(user_id)
+    @params = ride_params
 
-    with_acidity providing: { user: user, params: ride_params, ride: nil } do
+    with_acidity providing: { ride: nil } do
       step :create_ride_and_audit_record, awaits: [SomeJob]
       step :create_stripe_charge, args: [1, 2, 3], kwargs: { some: 'thing' }
       step :send_receipt
@@ -240,6 +269,86 @@ class RideCreateJob < ActiveJob::Base
 end
 ```
 
+If you need to await a job that takes arguments, you can prepare that job along with its arguments using the `with` class method that `acidic_job` will add to your jobs:
+
+```ruby
+class RideCreateJob < ActiveJob::Base
+  include AcidicJob
+
+  def perform(user_id, ride_params)
+    @user = User.find(user_id)
+    @params = ride_params
+
+    with_acidity providing: { ride: nil } do
+      step :create_ride_and_audit_record, awaits: awaits: [SomeJob.with('argument_1', keyword: 'value')]
+      step :create_stripe_charge, args: [1, 2, 3], kwargs: { some: 'thing' }
+      step :send_receipt
+    end
+  end
+end
+```
+
+You can also await a batch of jobs by simply passing multiple jobs to the `awaits` array (e.g. `awaits: [SomeJob, AnotherJob.with('argument_1', keyword: 'value')]`). Your top level workflow job will only continue to the next step once all of the jobs in your `awaits` array have successfully finished.
+
+In some cases, you may need to _dynamically_ determine the collection of jobs that the step should wait for; in these cases, you can pass the name of a method to the `awaits` option:
+
+```ruby
+class RideCreateJob < ActiveJob::Base
+  include AcidicJob
+  set_callback :finish, :after, :delete_run_record
+
+  def perform(user_id, ride_params)
+    @user = User.find(user_id)
+    @params = ride_params
+
+    with_acidity providing: { ride: nil } do
+      step :create_ride_and_audit_record, awaits: :dynamic_awaits
+      step :create_stripe_charge, args: [1, 2, 3], kwargs: { some: 'thing' }
+      step :send_receipt
+    end
+  end
+  
+  def dynamic_awaits
+    if @params["key"].present?
+      [SomeJob.with('argument_1', keyword: 'value')]
+    else
+      [AnotherJob]
+    end
+  end
+end
+```
+
+
+### Run Finished Callbacks
+
+When working with workflow jobs that make use of the `awaits` feature for a step, it is important to remember that the `after_perform` callback will be called _as soon as the first `awaits` step has enqueued job_, and **not** when the entire job run has finished. `acidic_job` allows the `perform` method to finish so that the queue for the workflow job is cleared to pick up new work while the `awaits` jobs are running. `acidic_job` will automatically re-enqueue the workflow job and progress to the next step when all of the `awaits` jobs have successfully finished. However, this means that `after_perform` **is not necessarily** the same as `after_finish`. In order to provide the opportunity for you to execute callback logic _if and only if_ a job run has finished, we provide callback hooks for the `finish` event.
+
+For example, you could use this hook to immediately clean up the `AcidicJob::Run` database record whenever the workflow job finishes successfully like so:
+
+```ruby
+class RideCreateJob < ActiveJob::Base
+  include AcidicJob
+  set_callback :finish, :after, :delete_run_record
+
+  def perform(user_id, ride_params)
+    @user = User.find(user_id)
+    @params = ride_params
+
+    with_acidity providing: { ride: nil } do
+      step :create_ride_and_audit_record, awaits: [SomeJob.with('argument_1', keyword: 'value')]
+      step :create_stripe_charge, args: [1, 2, 3], kwargs: { some: 'thing' }
+      step :send_receipt
+    end
+  end
+  
+  def delete_run_record
+    return unless acidic_job_run.succeeded?
+
+    acidic_job_run.destroy!
+  end
+end
+```
+
 ## Testing
 
 When testing acidic jobs, you are likely to run into `ActiveRecord::TransactionIsolationError`s:

data/lib/acidic_job/awaiting.rb

@@ -6,37 +6,41 @@ module AcidicJob
   module Awaiting
     extend ActiveSupport::Concern
 
-    def enqueue_step_parallel_jobs(jobs, run, step_result)
+    private
+
+    def enqueue_step_parallel_jobs(jobs_or_jobs_getter, run, step_result)
       # `batch` is available from Sidekiq::Pro
       raise SidekiqBatchRequired unless defined?(Sidekiq::Batch)
 
+      jobs = case jobs_or_jobs_getter
+             when Array
+               jobs_or_jobs_getter
+             when Symbol, String
+               method(jobs_or_jobs_getter).call
+             end
+
       step_batch = Sidekiq::Batch.new
       # step_batch.description = "AcidicJob::Workflow Step: #{step}"
       step_batch.on(
         :success,
         "#{self.class.name}#step_done",
         # NOTE: options are marshalled through JSON so use only basic types.
-        { "run_id" => run.id,
+        {
+          "run_id" => run.id,
           "step_result_yaml" => step_result.to_yaml.strip,
           "parent_worker" => self.class.name,
-          "job_names" => jobs.map(&:to_s) }
+          "job_names" => jobs.map { |job| job_name(job) }
+        }
       )
 
       # NOTE: The jobs method is atomic.
       # All jobs created in the block are actually pushed atomically at the end of the block.
       # If an error is raised, none of the jobs will go to Redis.
       step_batch.jobs do
-        jobs.each do |worker_name|
-          # TODO: handle Symbols as well
-          worker = worker_name.is_a?(String) ? worker_name.constantize : worker_name
-
-          if worker.instance_method(:perform).arity.presence_in [0, -1]
-            worker.perform_async
-          elsif worker.instance_method(:perform).arity == 1
-            worker.perform_async(run.id)
-          else
-            raise TooManyParametersForParallelJob
-          end
+        jobs.each do |job|
+          worker, args, kwargs = job_args_and_kwargs(job)
+
+          worker.perform_async(*args, **kwargs)
         end
       end
     end
@@ -53,5 +57,33 @@ module AcidicJob
       # when a batch of jobs for a step succeeds, we begin processing the `AcidicJob::Run` record again
       process_run(run)
     end
+
+    def job_name(job)
+      case job
+      when Class, Symbol
+        job.to_s
+      when String
+        job
+      else
+        job.class.name
+      end
+    end
+
+    def job_args_and_kwargs(job)
+      case job
+      when Class
+        [job, [], {}]
+      when String
+        [job.constantize, [], {}]
+      when Symbol
+        [job.to_s.constantize, [], {}]
+      else
+        [
+          job.class,
+          job.instance_variable_get(:@__acidic_job_args),
+          job.instance_variable_get(:@__acidic_job_kwargs)
+        ]
+      end
+    end
   end
 end

data/lib/acidic_job/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module AcidicJob
-  VERSION = "1.0.0.pre19"
+  VERSION = "1.0.0.pre22"
 end

data/lib/acidic_job.rb

@@ -55,6 +55,7 @@ module AcidicJob
     klass.define_singleton_method(:acidic_by_job_id) { @acidic_identifier = :job_id }
     klass.define_singleton_method(:acidic_by_job_args) { @acidic_identifier = :job_args }
     klass.define_singleton_method(:acidic_by) { |proc| @acidic_identifier = proc }
+    klass.attr_reader(:acidic_job_run)
   end
 
   included do
@@ -67,6 +68,10 @@ module AcidicJob
       super
     end
 
+    def with(*args, **kwargs)
+      new(*args, **kwargs)
+    end
+
     def acidic_identifier
       @acidic_identifier
     end
@@ -78,11 +83,11 @@ module AcidicJob
     @__acidic_job_args = args
     @__acidic_job_kwargs = kwargs
 
-    if method(__method__).super_method.arity.zero?
-      super()
-    else
-      super(*args, **kwargs)
-    end
+    super(*args, **kwargs)
+  rescue ArgumentError => e
+    raise e unless e.message.include?("wrong number of arguments")
+
+    super()
   end
 
   def with_acidity(providing: {})
@@ -96,10 +101,10 @@ module AcidicJob
     # convert the array of steps into a hash of recovery_points and next steps
     workflow = define_workflow(@__acidic_job_steps)
 
-    @run = ensure_run_record(workflow, providing)
+    @acidic_job_run = ensure_run_record(workflow, providing)
 
     # begin the workflow
-    process_run(@run)
+    process_run(@acidic_job_run)
   end
 
   # DEPRECATED
@@ -143,7 +148,7 @@ module AcidicJob
         break
       elsif current_step.nil?
         raise UnknownRecoveryPoint, "Defined workflow does not reference this step: #{recovery_point}"
-      elsif (jobs = current_step.fetch("awaits", [])).any?
+      elsif !(jobs = current_step.fetch("awaits", []) || []).empty?
         step = Step.new(current_step, run, self)
         # Only execute the current step, without yet progressing the recovery_point to the next step.
         # This ensures that any failures in parallel jobs will have this step retried in the main workflow

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: acidic_job
 version: !ruby/object:Gem::Version
-  version: 1.0.0.pre19
+  version: 1.0.0.pre22
 platform: ruby
 authors:
 - fractaledmind
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-05-16 00:00:00.000000000 Z
+date: 2022-05-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord