acidic_job 1.0.0.pre17 → 1.0.0.pre20

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c458faa1225d1b5e0fe7590bdd00371169de92b638dfbc74e88958b2430c5c7e
-  data.tar.gz: 8657eea5fd2a3929d049be9a7da1a252413cab1de42b991178673328202f4ee1
+  metadata.gz: e608eefef4baffbfb2c95dcbef942a53c9ca242c4c593ddbbe0d677ef7d318da
+  data.tar.gz: 65bed32739c1a1a860102e59dac69598c4357512034d8d3e87ed772469b9bac0
 SHA512:
-  metadata.gz: 4ae6d08046b2bc2e3388544fa61f308cf57c9dab5a4f67a923d1400ca7743fb48966a9aa0ce504a2d16d5c7e80fc98d8fd5308583236ff305ffb0d1b2eb17c70
-  data.tar.gz: afee2f7723b316410d1144cf71c42cbace68ff304842cc1c29cd0a358c297ae47f9ae1166e3cd1daef8e677dbdd38b11fb64e771d2d1975afc399cafb8329129
+  metadata.gz: 5a5f6155e20f929c631392f00480f57928455778e13117609457be9b8624e7f16c41661df96ae34a4045582f957b044cbdf811bfa7e187dd627b8b72b80edb90
+  data.tar.gz: 6a663d288bd1ef3ba5cfac8e137fd496ebbca76e08573e50d706ed2570c98b6fe3148b8c9335b55313a091ccd0cc226e3b9023878dd418d400af33fd0280c1b4

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    acidic_job (1.0.0.pre17)
+    acidic_job (1.0.0.pre20)
       activerecord
       activesupport
       database_cleaner
@@ -9,67 +9,67 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    actioncable (7.0.2.3)
-      actionpack (= 7.0.2.3)
-      activesupport (= 7.0.2.3)
+    actioncable (7.0.3)
+      actionpack (= 7.0.3)
+      activesupport (= 7.0.3)
       nio4r (~> 2.0)
       websocket-driver (>= 0.6.1)
-    actionmailbox (7.0.2.3)
-      actionpack (= 7.0.2.3)
-      activejob (= 7.0.2.3)
-      activerecord (= 7.0.2.3)
-      activestorage (= 7.0.2.3)
-      activesupport (= 7.0.2.3)
+    actionmailbox (7.0.3)
+      actionpack (= 7.0.3)
+      activejob (= 7.0.3)
+      activerecord (= 7.0.3)
+      activestorage (= 7.0.3)
+      activesupport (= 7.0.3)
       mail (>= 2.7.1)
       net-imap
       net-pop
       net-smtp
-    actionmailer (7.0.2.3)
-      actionpack (= 7.0.2.3)
-      actionview (= 7.0.2.3)
-      activejob (= 7.0.2.3)
-      activesupport (= 7.0.2.3)
+    actionmailer (7.0.3)
+      actionpack (= 7.0.3)
+      actionview (= 7.0.3)
+      activejob (= 7.0.3)
+      activesupport (= 7.0.3)
       mail (~> 2.5, >= 2.5.4)
       net-imap
       net-pop
       net-smtp
       rails-dom-testing (~> 2.0)
-    actionpack (7.0.2.3)
-      actionview (= 7.0.2.3)
-      activesupport (= 7.0.2.3)
+    actionpack (7.0.3)
+      actionview (= 7.0.3)
+      activesupport (= 7.0.3)
       rack (~> 2.0, >= 2.2.0)
       rack-test (>= 0.6.3)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.0, >= 1.2.0)
-    actiontext (7.0.2.3)
-      actionpack (= 7.0.2.3)
-      activerecord (= 7.0.2.3)
-      activestorage (= 7.0.2.3)
-      activesupport (= 7.0.2.3)
+    actiontext (7.0.3)
+      actionpack (= 7.0.3)
+      activerecord (= 7.0.3)
+      activestorage (= 7.0.3)
+      activesupport (= 7.0.3)
       globalid (>= 0.6.0)
       nokogiri (>= 1.8.5)
-    actionview (7.0.2.3)
-      activesupport (= 7.0.2.3)
+    actionview (7.0.3)
+      activesupport (= 7.0.3)
       builder (~> 3.1)
       erubi (~> 1.4)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.1, >= 1.2.0)
-    activejob (7.0.2.3)
-      activesupport (= 7.0.2.3)
+    activejob (7.0.3)
+      activesupport (= 7.0.3)
       globalid (>= 0.3.6)
-    activemodel (7.0.2.3)
-      activesupport (= 7.0.2.3)
-    activerecord (7.0.2.3)
-      activemodel (= 7.0.2.3)
-      activesupport (= 7.0.2.3)
-    activestorage (7.0.2.3)
-      actionpack (= 7.0.2.3)
-      activejob (= 7.0.2.3)
-      activerecord (= 7.0.2.3)
-      activesupport (= 7.0.2.3)
+    activemodel (7.0.3)
+      activesupport (= 7.0.3)
+    activerecord (7.0.3)
+      activemodel (= 7.0.3)
+      activesupport (= 7.0.3)
+    activestorage (7.0.3)
+      actionpack (= 7.0.3)
+      activejob (= 7.0.3)
+      activerecord (= 7.0.3)
+      activesupport (= 7.0.3)
       marcel (~> 1.0)
       mini_mime (>= 1.1.0)
-    activesupport (7.0.2.3)
+    activesupport (7.0.3)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
@@ -82,7 +82,7 @@ GEM
       activesupport (>= 3.0.0)
       railties (>= 3.0.0)
       thor (>= 0.14.6)
-    concurrent-ruby (1.1.9)
+    concurrent-ruby (1.1.10)
     connection_pool (2.2.5)
     crass (1.0.6)
     database_cleaner (2.0.1)
@@ -112,11 +112,10 @@ GEM
     http-form_data (2.3.0)
     i18n (1.10.0)
       concurrent-ruby (~> 1.0)
-    io-wait (0.2.1)
     llhttp-ffi (0.4.0)
       ffi-compiler (~> 1.0)
       rake (~> 13.0)
-    loofah (2.15.0)
+    loofah (2.18.0)
       crass (~> 1.0.2)
       nokogiri (>= 1.5.9)
     mail (2.7.1)
@@ -134,52 +133,51 @@ GEM
       digest
       net-protocol
       timeout
-    net-protocol (0.1.2)
-      io-wait
+    net-protocol (0.1.3)
       timeout
     net-smtp (0.3.1)
       digest
       net-protocol
       timeout
     nio4r (2.5.8)
-    nokogiri (1.13.3)
+    nokogiri (1.13.6)
       mini_portile2 (~> 2.8.0)
       racc (~> 1.4)
-    nokogiri (1.13.3-x86_64-darwin)
+    nokogiri (1.13.6-x86_64-darwin)
       racc (~> 1.4)
     noticed (1.5.9)
       http (>= 4.0.0)
       rails (>= 5.2.0)
-    parallel (1.21.0)
-    parser (3.1.1.0)
+    parallel (1.22.1)
+    parser (3.1.2.0)
       ast (~> 2.4.1)
-    public_suffix (4.0.6)
+    public_suffix (4.0.7)
     racc (1.6.0)
     rack (2.2.3)
     rack-test (1.1.0)
       rack (>= 1.0, < 3)
-    rails (7.0.2.3)
-      actioncable (= 7.0.2.3)
-      actionmailbox (= 7.0.2.3)
-      actionmailer (= 7.0.2.3)
-      actionpack (= 7.0.2.3)
-      actiontext (= 7.0.2.3)
-      actionview (= 7.0.2.3)
-      activejob (= 7.0.2.3)
-      activemodel (= 7.0.2.3)
-      activerecord (= 7.0.2.3)
-      activestorage (= 7.0.2.3)
-      activesupport (= 7.0.2.3)
+    rails (7.0.3)
+      actioncable (= 7.0.3)
+      actionmailbox (= 7.0.3)
+      actionmailer (= 7.0.3)
+      actionpack (= 7.0.3)
+      actiontext (= 7.0.3)
+      actionview (= 7.0.3)
+      activejob (= 7.0.3)
+      activemodel (= 7.0.3)
+      activerecord (= 7.0.3)
+      activestorage (= 7.0.3)
+      activesupport (= 7.0.3)
       bundler (>= 1.15.0)
-      railties (= 7.0.2.3)
+      railties (= 7.0.3)
     rails-dom-testing (2.0.3)
       activesupport (>= 4.2.0)
       nokogiri (>= 1.6)
     rails-html-sanitizer (1.4.2)
       loofah (~> 2.3)
-    railties (7.0.2.3)
-      actionpack (= 7.0.2.3)
-      activesupport (= 7.0.2.3)
+    railties (7.0.3)
+      actionpack (= 7.0.3)
+      activesupport (= 7.0.3)
       method_source
       rake (>= 12.2)
       thor (~> 1.0)
@@ -187,20 +185,20 @@ GEM
     rainbow (3.1.1)
     rake (13.0.6)
     redis (4.6.0)
-    regexp_parser (2.2.1)
+    regexp_parser (2.4.0)
     rexml (3.2.5)
-    rubocop (1.26.0)
+    rubocop (1.29.1)
       parallel (~> 1.10)
       parser (>= 3.1.0.0)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 1.8, < 3.0)
-      rexml
-      rubocop-ast (>= 1.16.0, < 2.0)
+      rexml (>= 3.2.5, < 4.0)
+      rubocop-ast (>= 1.17.0, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 1.4.0, < 3.0)
-    rubocop-ast (1.16.0)
+    rubocop-ast (1.18.0)
       parser (>= 3.1.1.0)
-    rubocop-minitest (0.18.0)
+    rubocop-minitest (0.19.1)
       rubocop (>= 0.90, < 2.0)
     rubocop-rake (0.6.0)
       rubocop (~> 1.0)
@@ -216,7 +214,7 @@ GEM
     simplecov-html (0.12.3)
     simplecov_json_formatter (0.1.4)
     sqlite3 (1.4.2)
-    strscan (3.0.1)
+    strscan (3.0.3)
     thor (1.2.1)
     timeout (0.2.0)
     tzinfo (2.0.4)

data/README.md

@@ -68,8 +68,10 @@ It provides a suite of functionality that empowers you to create complex, robust
 * Persisted Attributes — when retrying jobs at later steps, we need to ensure that data created in previous steps is still available to later steps on retry.
 * Transactionally Staged Jobs — enqueue additional jobs within the acidic transaction safely
 * Custom Idempotency Keys — use something other than the job ID for the idempotency key of the job run
+* Iterable Steps — define steps that iterate over some collection fully until moving on to the next step
 * Sidekiq Callbacks — bring ActiveJob-like callbacks into your pure Sidekiq Workers
 * Sidekiq Batches — leverage the power of Sidekiq Pro's `batch` functionality without the hassle
+* Run Finished Callbacks — set callbacks for when a job run finishes fully
 
 ### Transactional Steps
 
@@ -80,9 +82,10 @@ class RideCreateJob < ActiveJob::Base
   include AcidicJob
 
   def perform(user_id, ride_params)
-    user = User.find(user_id)
+    @user = User.find(user_id)
+    @params = ride_params
 
-    with_acidity providing: { user: user, params: ride_params, ride: nil } do
+    with_acidity providing: { ride: nil } do
       step :create_ride_and_audit_record
       step :create_stripe_charge
       step :send_receipt
@@ -128,7 +131,7 @@ class RideCreateJob < ActiveJob::Base
   end
 
   def create_stripe_charge
-    Stripe::Charge.create(amount: 20_00, customer: self.ride.user)
+    Stripe::Charge.create(amount: 20_00, customer: @ride.user)
   end
 
   # ...
@@ -139,6 +142,8 @@ end
 
 **Note:** You will note the use of `self.ride = ...` in the code sample above. In order to call the attribute setter method that will sync with the database record, you _must_ use this style. `@ride = ...` and/or `ride = ...` will both fail to sync the value with the database record.
 
+The default pattern you should follow when defining your `perform` method is to make any values that your `step` methods need access to, but are present at the start of the `perform` method simply instance variables. You only need to `provide` attributes that will be set _during a step_. This means, the initial value will almost always be `nil`.
+
 ### Transactionally Staged Jobs
 
 A standard problem when inside of database transactions is enqueuing other jobs. On the one hand, you could enqueue a job inside of a transaction that then rollbacks, which would leave that job to fail and retry and fail. On the other hand, you could enqueue a job that is picked up before the transaction commits, which would mean the records are not yet available to this job.
@@ -150,9 +155,10 @@ class RideCreateJob < ActiveJob::Base
   include AcidicJob
 
   def perform(user_id, ride_params)
-    user = User.find(user_id)
+    @user = User.find(user_id)
+    @params = ride_params
     
-    with_acidity providing: { user: user, params: ride_params, ride: nil } do
+    with_acidity providing: { ride: nil } do
       step :create_ride_and_audit_record
       step :create_stripe_charge
       step :send_receipt
@@ -211,6 +217,28 @@ end
 
 > **Note:** The signature of the `acidic_by` proc _needs to match the signature_ of the job's `perform` method.
 
+### Iterable Steps
+
+Sometimes our workflows have steps that need to iterate over a collection and perform an action for each item in the collection before moving on to the next step in the workflow. In these cases, we can use the `for_each` option when defining our step to specific the collection, and `acidic_job` will pass each item into your step method for processing, keeping the same transactional guarantees as for any step. This means that if your step encounters an error in processing any item in the collection, when your job is retried, the job will jump right back to that step and right back to that item in the collection to try again.
+
+```ruby
+class ExampleJob < ActiveJob::Base
+  include AcidicJob
+
+  def perform(record:)
+    with_acidity providing: { collection: [1, 2, 3, 4, 5] } do
+      step :process_item, for_each: :collection
+      step :next_step
+    end
+  end
+  
+  def process_item(item)
+    # do whatever work needs to be done with this individual item
+  end
+end
+```
+
+**Note:** The same restrictions apply here as for any persisted attribute — you can only use objects that can be serialized by ActiveRecord.
 
 ### Sidekiq Callbacks
 
@@ -229,9 +257,10 @@ class RideCreateJob < ActiveJob::Base
   include AcidicJob
 
   def perform(user_id, ride_params)
-    user = User.find(user_id)
+    @user = User.find(user_id)
+    @params = ride_params
 
-    with_acidity providing: { user: user, params: ride_params, ride: nil } do
+    with_acidity providing: { ride: nil } do
       step :create_ride_and_audit_record, awaits: [SomeJob]
       step :create_stripe_charge, args: [1, 2, 3], kwargs: { some: 'thing' }
       step :send_receipt
@@ -240,6 +269,36 @@ class RideCreateJob < ActiveJob::Base
 end
 ```
 
+### Run Finished Callbacks
+
+When working with workflow jobs that make use of the `awaits` feature for a step, it is important to remember that the `after_perform` callback will be called _as soon as the first `awaits` step has enqueued job_, and **not** when the entire job run has finished. `acidic_job` allows the `perform` method to finish so that the queue for the workflow job is cleared to pick up new work while the `awaits` jobs are running. `acidic_job` will automatically re-enqueue the workflow job and progress to the next step when all of the `awaits` jobs have successfully finished. However, this means that `after_perform` **is not necessarily** the same as `after_finish`. In order to provide the opportunity for you to execute callback logic _if and only if_ a job run has finished, we provide callback hooks for the `finish` event.
+
+For example, you could use this hook to immediately clean up the `AcidicJob::Run` database record whenever the workflow job finishes successfully like so:
+
+```ruby
+class RideCreateJob < ActiveJob::Base
+  include AcidicJob
+  set_callback :finish, :after, :delete_run_record
+
+  def perform(user_id, ride_params)
+    @user = User.find(user_id)
+    @params = ride_params
+
+    with_acidity providing: { ride: nil } do
+      step :create_ride_and_audit_record, awaits: [SomeJob]
+      step :create_stripe_charge, args: [1, 2, 3], kwargs: { some: 'thing' }
+      step :send_receipt
+    end
+  end
+  
+  def delete_run_record
+    return unless acidic_job_run.succeeded?
+
+    acidic_job_run.destroy!
+  end
+end
+```
+
 ## Testing
 
 When testing acidic jobs, you are likely to run into `ActiveRecord::TransactionIsolationError`s:

data/lib/acidic_job/awaiting.rb

@@ -17,8 +17,11 @@ module AcidicJob
         "#{self.class.name}#step_done",
         # NOTE: options are marshalled through JSON so use only basic types.
         { "run_id" => run.id,
-          "step_result_yaml" => step_result.to_yaml.strip }
+          "step_result_yaml" => step_result.to_yaml.strip,
+          "parent_worker" => self.class.name,
+          "job_names" => jobs.map(&:to_s) }
       )
+
       # NOTE: The jobs method is atomic.
       # All jobs created in the block are actually pushed atomically at the end of the block.
       # If an error is raised, none of the jobs will go to Redis.
@@ -26,7 +29,8 @@ module AcidicJob
         jobs.each do |worker_name|
           # TODO: handle Symbols as well
           worker = worker_name.is_a?(String) ? worker_name.constantize : worker_name
-          if worker.instance_method(:perform).arity.zero?
+
+          if worker.instance_method(:perform).arity.presence_in [0, -1]
             worker.perform_async
           elsif worker.instance_method(:perform).arity == 1
             worker.perform_async(run.id)

data/lib/acidic_job/step.rb

@@ -54,9 +54,9 @@ module AcidicJob
       next_step = step["then"]
       # to support iteration within steps
       iterable_key = step["for_each"]
-      iterated_key = "processed_#{iterable_key}"
-      iterables = @run.attr_accessors.fetch(iterable_key, [])
-      iterateds = @run.attr_accessors.fetch(iterated_key, [])
+      iterated_key = "processed_#{current_step}_#{iterable_key}"
+      iterables = @run.attr_accessors.fetch(iterable_key, []) || []
+      iterateds = @run.attr_accessors.fetch(iterated_key, []) || []
       next_item = iterables.reject { |item| iterateds.include? item }.first
 
       # jobs can have no-op steps, especially so that they can use only the async/await mechanism for that step

data/lib/acidic_job/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module AcidicJob
-  VERSION = "1.0.0.pre17"
+  VERSION = "1.0.0.pre20"
 end

data/lib/acidic_job.rb

@@ -47,12 +47,15 @@ module AcidicJob
       raise UnknownJobAdapter
     end
 
+    # TODO: write test for a staged job that uses awaits
     klass.set_callback :perform, :after, :delete_staged_job_record, if: :was_staged_job?
+    klass.define_callbacks :finish
 
     klass.instance_variable_set(:@acidic_identifier, :job_id)
     klass.define_singleton_method(:acidic_by_job_id) { @acidic_identifier = :job_id }
     klass.define_singleton_method(:acidic_by_job_args) { @acidic_identifier = :job_args }
     klass.define_singleton_method(:acidic_by) { |proc| @acidic_identifier = proc }
+    klass.attr_reader(:acidic_job_run)
   end
 
   included do
@@ -94,10 +97,10 @@ module AcidicJob
     # convert the array of steps into a hash of recovery_points and next steps
     workflow = define_workflow(@__acidic_job_steps)
 
-    @run = ensure_run_record(workflow, providing)
+    @acidic_job_run = ensure_run_record(workflow, providing)
 
     # begin the workflow
-    process_run(@run)
+    process_run(@acidic_job_run)
   end
 
   # DEPRECATED
@@ -128,7 +131,7 @@ module AcidicJob
 
   def process_run(run)
     # if the run record is already marked as finished, immediately return its result
-    return run.succeeded? if run.finished?
+    return finish_run(run) if run.finished?
 
     # otherwise, we will enter a loop to process each step of the workflow
     loop do
@@ -167,7 +170,13 @@ module AcidicJob
     end
 
     # the loop will break once the job is finished, so simply report the status
-    run.succeeded?
+    finish_run(run)
+  end
+
+  def finish_run(run)
+    run_callbacks :finish do
+      run.succeeded?
+    end
   end
 
   def step(method_name, awaits: [], for_each: nil)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: acidic_job
 version: !ruby/object:Gem::Version
-  version: 1.0.0.pre17
+  version: 1.0.0.pre20
 platform: ruby
 authors:
 - fractaledmind
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-05-14 00:00:00.000000000 Z
+date: 2022-05-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord