acidic_job 1.0.0.pre24 → 1.0.0.pre27

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 223aa8cad5d48e2a4fc04a9e33f82bdf7ef54cb933c2e14f3dc0aa6d46986932
-  data.tar.gz: a8a646da8f9e5987267f1448e44056bdd37c2d4f8cbfbc84650659555567b635
+  metadata.gz: 2b41d1bacc6dc699e4e70e38c535795039f7dbfd5ed4b7ea3671809d7658fc5d
+  data.tar.gz: 53641ee78275b54defdae3ab1c4ca0087d6e193009c0d3df758114019a5a3efb
 SHA512:
-  metadata.gz: 857647ff5cce856d41a8e30014005ef035cefcca383a719e6e679a4fc7bdc254c76f65e2d312c508ad56fd45b1e33ed533aecf2541e13b0480c579e0127ec338
-  data.tar.gz: 4208fa26057e5a1fb0908c554b8e2ef74c6a5018a4d782815caa1fbd7716f18b33598ff6877e3c0c583f449099438d316c506dee6fa84afea34d54aa42f085b8
+  metadata.gz: d383cde62b69a013979df927ff64d1c99ff591261a103b3694733f65f0fa0d44c1216bf3b7e8638b746a59754efbc32afd410c9039bda9f88a89926e552ee1d9
+  data.tar.gz: ced115b0b7607c60b48a51d2fd4e2db6442fd0daa44b2a9b648a1ced7598ca3e0f05dd8894359ed163c27287146769d37aebe9e464c989e46ebabfa6ba07d41d

data/.gitignore

@@ -12,3 +12,5 @@ slides.md
 /test/dummy
 /test/log
 /database.sqlite
+/test/combustion/log/test.log
+/test/combustion/database.sqlite

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    acidic_job (1.0.0.pre24)
+    acidic_job (1.0.0.pre27)
       activerecord
       activesupport
       database_cleaner
@@ -9,67 +9,67 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    actioncable (7.0.3)
-      actionpack (= 7.0.3)
-      activesupport (= 7.0.3)
+    actioncable (7.0.3.1)
+      actionpack (= 7.0.3.1)
+      activesupport (= 7.0.3.1)
       nio4r (~> 2.0)
       websocket-driver (>= 0.6.1)
-    actionmailbox (7.0.3)
-      actionpack (= 7.0.3)
-      activejob (= 7.0.3)
-      activerecord (= 7.0.3)
-      activestorage (= 7.0.3)
-      activesupport (= 7.0.3)
+    actionmailbox (7.0.3.1)
+      actionpack (= 7.0.3.1)
+      activejob (= 7.0.3.1)
+      activerecord (= 7.0.3.1)
+      activestorage (= 7.0.3.1)
+      activesupport (= 7.0.3.1)
       mail (>= 2.7.1)
       net-imap
       net-pop
       net-smtp
-    actionmailer (7.0.3)
-      actionpack (= 7.0.3)
-      actionview (= 7.0.3)
-      activejob (= 7.0.3)
-      activesupport (= 7.0.3)
+    actionmailer (7.0.3.1)
+      actionpack (= 7.0.3.1)
+      actionview (= 7.0.3.1)
+      activejob (= 7.0.3.1)
+      activesupport (= 7.0.3.1)
       mail (~> 2.5, >= 2.5.4)
       net-imap
       net-pop
       net-smtp
       rails-dom-testing (~> 2.0)
-    actionpack (7.0.3)
-      actionview (= 7.0.3)
-      activesupport (= 7.0.3)
+    actionpack (7.0.3.1)
+      actionview (= 7.0.3.1)
+      activesupport (= 7.0.3.1)
       rack (~> 2.0, >= 2.2.0)
       rack-test (>= 0.6.3)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.0, >= 1.2.0)
-    actiontext (7.0.3)
-      actionpack (= 7.0.3)
-      activerecord (= 7.0.3)
-      activestorage (= 7.0.3)
-      activesupport (= 7.0.3)
+    actiontext (7.0.3.1)
+      actionpack (= 7.0.3.1)
+      activerecord (= 7.0.3.1)
+      activestorage (= 7.0.3.1)
+      activesupport (= 7.0.3.1)
       globalid (>= 0.6.0)
       nokogiri (>= 1.8.5)
-    actionview (7.0.3)
-      activesupport (= 7.0.3)
+    actionview (7.0.3.1)
+      activesupport (= 7.0.3.1)
       builder (~> 3.1)
       erubi (~> 1.4)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.1, >= 1.2.0)
-    activejob (7.0.3)
-      activesupport (= 7.0.3)
+    activejob (7.0.3.1)
+      activesupport (= 7.0.3.1)
       globalid (>= 0.3.6)
-    activemodel (7.0.3)
-      activesupport (= 7.0.3)
-    activerecord (7.0.3)
-      activemodel (= 7.0.3)
-      activesupport (= 7.0.3)
-    activestorage (7.0.3)
-      actionpack (= 7.0.3)
-      activejob (= 7.0.3)
-      activerecord (= 7.0.3)
-      activesupport (= 7.0.3)
+    activemodel (7.0.3.1)
+      activesupport (= 7.0.3.1)
+    activerecord (7.0.3.1)
+      activemodel (= 7.0.3.1)
+      activesupport (= 7.0.3.1)
+    activestorage (7.0.3.1)
+      actionpack (= 7.0.3.1)
+      activejob (= 7.0.3.1)
+      activerecord (= 7.0.3.1)
+      activesupport (= 7.0.3.1)
       marcel (~> 1.0)
       mini_mime (>= 1.1.0)
-    activesupport (7.0.3)
+    activesupport (7.0.3.1)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
@@ -78,7 +78,7 @@ GEM
       public_suffix (>= 2.0.2, < 5.0)
     ast (2.4.2)
     builder (3.2.4)
-    combustion (1.3.5)
+    combustion (1.3.7)
       activesupport (>= 3.0.0)
       railties (>= 3.0.0)
       thor (>= 0.14.6)
@@ -102,16 +102,17 @@ GEM
       rake
     globalid (1.0.0)
       activesupport (>= 5.0)
-    http (5.0.4)
+    http (5.1.0)
       addressable (~> 2.8)
       http-cookie (~> 1.0)
       http-form_data (~> 2.2)
       llhttp-ffi (~> 0.4.0)
-    http-cookie (1.0.4)
+    http-cookie (1.0.5)
       domain_name (~> 0.5)
     http-form_data (2.3.0)
-    i18n (1.10.0)
+    i18n (1.12.0)
       concurrent-ruby (~> 1.0)
+    json (2.6.2)
     llhttp-ffi (0.4.0)
       ffi-compiler (~> 1.0)
       rake (~> 13.0)
@@ -124,7 +125,7 @@ GEM
     method_source (1.0.0)
     mini_mime (1.1.2)
     mini_portile2 (2.8.0)
-    minitest (5.15.0)
+    minitest (5.16.2)
     net-imap (0.2.3)
       digest
       net-protocol
@@ -140,70 +141,71 @@ GEM
       net-protocol
       timeout
     nio4r (2.5.8)
-    nokogiri (1.13.6)
+    nokogiri (1.13.7)
       mini_portile2 (~> 2.8.0)
       racc (~> 1.4)
-    nokogiri (1.13.6-x86_64-darwin)
-      racc (~> 1.4)
     noticed (1.5.9)
       http (>= 4.0.0)
       rails (>= 5.2.0)
     parallel (1.22.1)
     parser (3.1.2.0)
       ast (~> 2.4.1)
+    psych (4.0.4)
+      stringio
     public_suffix (4.0.7)
     racc (1.6.0)
-    rack (2.2.3)
-    rack-test (1.1.0)
-      rack (>= 1.0, < 3)
-    rails (7.0.3)
-      actioncable (= 7.0.3)
-      actionmailbox (= 7.0.3)
-      actionmailer (= 7.0.3)
-      actionpack (= 7.0.3)
-      actiontext (= 7.0.3)
-      actionview (= 7.0.3)
-      activejob (= 7.0.3)
-      activemodel (= 7.0.3)
-      activerecord (= 7.0.3)
-      activestorage (= 7.0.3)
-      activesupport (= 7.0.3)
+    rack (2.2.4)
+    rack-test (2.0.2)
+      rack (>= 1.3)
+    rails (7.0.3.1)
+      actioncable (= 7.0.3.1)
+      actionmailbox (= 7.0.3.1)
+      actionmailer (= 7.0.3.1)
+      actionpack (= 7.0.3.1)
+      actiontext (= 7.0.3.1)
+      actionview (= 7.0.3.1)
+      activejob (= 7.0.3.1)
+      activemodel (= 7.0.3.1)
+      activerecord (= 7.0.3.1)
+      activestorage (= 7.0.3.1)
+      activesupport (= 7.0.3.1)
       bundler (>= 1.15.0)
-      railties (= 7.0.3)
+      railties (= 7.0.3.1)
     rails-dom-testing (2.0.3)
       activesupport (>= 4.2.0)
       nokogiri (>= 1.6)
-    rails-html-sanitizer (1.4.2)
+    rails-html-sanitizer (1.4.3)
       loofah (~> 2.3)
-    railties (7.0.3)
-      actionpack (= 7.0.3)
-      activesupport (= 7.0.3)
+    railties (7.0.3.1)
+      actionpack (= 7.0.3.1)
+      activesupport (= 7.0.3.1)
       method_source
       rake (>= 12.2)
       thor (~> 1.0)
       zeitwerk (~> 2.5)
     rainbow (3.1.1)
     rake (13.0.6)
-    redis (4.6.0)
-    regexp_parser (2.4.0)
+    redis (4.7.1)
+    regexp_parser (2.5.0)
     rexml (3.2.5)
-    rubocop (1.29.1)
+    rubocop (1.31.2)
+      json (~> 2.3)
       parallel (~> 1.10)
       parser (>= 3.1.0.0)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 1.8, < 3.0)
       rexml (>= 3.2.5, < 4.0)
-      rubocop-ast (>= 1.17.0, < 2.0)
+      rubocop-ast (>= 1.18.0, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 1.4.0, < 3.0)
-    rubocop-ast (1.18.0)
+    rubocop-ast (1.19.1)
       parser (>= 3.1.1.0)
-    rubocop-minitest (0.19.1)
+    rubocop-minitest (0.20.1)
       rubocop (>= 0.90, < 2.0)
     rubocop-rake (0.6.0)
       rubocop (~> 1.0)
     ruby-progressbar (1.11.0)
-    sidekiq (6.4.2)
+    sidekiq (6.5.1)
       connection_pool (>= 2.2.2)
       rack (~> 2.0)
       redis (>= 4.2.0)
@@ -213,21 +215,22 @@ GEM
       simplecov_json_formatter (~> 0.1)
     simplecov-html (0.12.3)
     simplecov_json_formatter (0.1.4)
-    sqlite3 (1.4.2)
+    sqlite3 (1.4.4)
+    stringio (3.0.2)
     strscan (3.0.3)
     thor (1.2.1)
-    timeout (0.2.0)
+    timeout (0.3.0)
     tzinfo (2.0.4)
       concurrent-ruby (~> 1.0)
     unf (0.1.4)
       unf_ext
-    unf_ext (0.0.8.1)
-    unicode-display_width (2.1.0)
-    warning (1.2.1)
+    unf_ext (0.0.8.2)
+    unicode-display_width (2.2.0)
+    warning (1.3.0)
     websocket-driver (0.7.5)
       websocket-extensions (>= 0.1.0)
     websocket-extensions (0.1.5)
-    zeitwerk (2.5.4)
+    zeitwerk (2.6.0)
 
 PLATFORMS
   ruby
@@ -240,6 +243,7 @@ DEPENDENCIES
   minitest
   net-smtp
   noticed
+  psych (> 4.0)
   railties
   rake
   rubocop

data/README.md

@@ -64,14 +64,23 @@ It provides a suite of functionality that empowers you to create complex, robust
 
 #### Key Features
 
-* Transactional Steps — break your job into a series of steps, each of which will be run within an acidic database transaction, allowing retries to jump back to the last "recovery point".
-* Persisted Attributes — when retrying jobs at later steps, we need to ensure that data created in previous steps is still available to later steps on retry.
-* Transactionally Staged Jobs — enqueue additional jobs within the acidic transaction safely
-* Custom Idempotency Keys — use something other than the job ID for the idempotency key of the job run
-* Iterable Steps — define steps that iterate over some collection fully until moving on to the next step
-* Sidekiq Callbacks — bring ActiveJob-like callbacks into your pure Sidekiq Workers
-* Sidekiq Batches — leverage the power of Sidekiq Pro's `batch` functionality without the hassle
-* Run Finished Callbacks — set callbacks for when a job run finishes fully
+* **Transactional Steps**  
+    break your job into a series of steps, each of which will be run within an acidic database transaction, allowing retries to jump back to the last "recovery point".
+* **Steps that Await Jobs**  
+    have workflow steps await other jobs, which will be enqueued and processed independently, and only when they all have finished will the parent job be re-enqueued to continue the workflow
+* **Iterable Steps**  
+    define steps that iterate over some collection fully until moving on to the next step
+* **Persisted Attributes**  
+    when retrying jobs at later steps, we need to ensure that data created in previous steps is still available to later steps on retry.
+* **Transactionally Staged Jobs**  
+    enqueue additional jobs within the acidic transaction safely
+* **Custom Idempotency Keys**  
+    use something other than the job ID for the idempotency key of the job run
+* **Sidekiq Callbacks**  
+    bring ActiveJob-like callbacks into your pure Sidekiq Workers
+* **Run Finished Callbacks**  
+    set callbacks for when a job run finishes fully
+
 
 ### Transactional Steps
 
@@ -110,15 +119,114 @@ end
 
 Now, each execution of this job will find or create an `AcidicJob::Run` record, which we leverage to wrap every step in a database transaction. Moreover, this database record allows `acidic_job` to ensure that if your job fails on step 3, when it retries, it will simply jump right back to trying to execute the method defined for the 3rd step, and won't even execute the first two step methods. This means your step methods only need to be idempotent on failure, not on success, since they will never be run again if they succeed.
 
+
+### Steps that Await Jobs
+
+By simply adding the `awaits` option to your step declarations, you can attach any number of additional, asynchronous jobs to your step. This is profoundly powerful, as it means that you can define a workflow where step 2 is started _if and only if_ step 1 succeeds, but step 1 can have 3 different jobs enqueued on 3 different queues, each running in parallel. Once all 3 jobs succeed, `acidic_job` will re-enqueue the parent job and it will move on to step 2. That's right, you can have workers that are _executed in parallel_, **on separate queues**, and _asynchronously_, but are still **blocking**—as a group—the next step in your workflow! This unlocks incredible power and flexibility for defining and structuring complex workflows and operations.
+
+```ruby
+class RideCreateJob < ActiveJob::Base
+  include AcidicJob
+
+  def perform(user_id, ride_params)
+    @user = User.find(user_id)
+    @params = ride_params
+
+    with_acidity providing: { ride: nil } do
+      step :create_ride_and_audit_record, awaits: [SomeJob, AnotherJob]
+      step :create_stripe_charge
+      step :send_receipt
+    end
+  end
+end
+```
+
+If you need to await a job that takes arguments, you can prepare that job along with its arguments using the `with` class method that `acidic_job` will add to your jobs:
+
+```ruby
+class RideCreateJob < ActiveJob::Base
+  include AcidicJob
+
+  def perform(user_id, ride_params)
+    @user = User.find(user_id)
+    @params = ride_params
+
+    with_acidity providing: { ride: nil } do
+      step :create_ride_and_audit_record, awaits: awaits: [SomeJob.with('argument_1', keyword: 'value'), AnotherJob.with(1, 2, 3, some: 'thing')]
+      step :create_stripe_charge
+      step :send_receipt
+    end
+  end
+end
+```
+
+If your step awaits multiple jobs (e.g. `awaits: [SomeJob, AnotherJob.with('argument_1', keyword: 'value')]`), your top level workflow job will only continue to the next step once **all** of the jobs in your `awaits` array have finished.
+
+In some cases, you may need to _dynamically_ determine the collection of jobs that the step should wait for; in these cases, you can pass the name of a method to the `awaits` option:
+
+```ruby
+class RideCreateJob < ActiveJob::Base
+  include AcidicJob
+  set_callback :finish, :after, :delete_run_record
+
+  def perform(user_id, ride_params)
+    @user = User.find(user_id)
+    @params = ride_params
+
+    with_acidity providing: { ride: nil } do
+      step :create_ride_and_audit_record, awaits: :dynamic_awaits
+      step :create_stripe_charge
+      step :send_receipt
+    end
+  end
+  
+  def dynamic_awaits
+    if @params["key"].present?
+      [SomeJob.with('argument_1', keyword: 'value')]
+    else
+      [AnotherJob.with(1, 2, 3, some: 'thing')]
+    end
+  end
+end
+```
+
+
+### Iterable Steps
+
+Sometimes our workflows have steps that need to iterate over a collection and perform an action for each item in the collection before moving on to the next step in the workflow. In these cases, we can use the `for_each` option when defining our step to specific the collection, and `acidic_job` will pass each item into your step method for processing, keeping the same transactional guarantees as for any step. This means that if your step encounters an error in processing any item in the collection, when your job is retried, the job will jump right back to that step and right back to that item in the collection to try again.
+
+```ruby
+class ExampleJob < ActiveJob::Base
+  include AcidicJob
+
+  def perform(record:)
+    with_acidity providing: { collection: [1, 2, 3, 4, 5] } do
+      step :process_item, for_each: :collection
+      step :next_step
+    end
+  end
+  
+  def process_item(item)
+    # do whatever work needs to be done with this individual item
+  end
+end
+```
+
+**Note:** This feature relies on the "Persisted Attributes" feature detailed below. This means that you can only iterate over collections that ActiveRecord can serialize.
+
+
 ### Persisted Attributes
 
-Any objects passed to the `providing` option on the `with_acidity` method are not just made available to each of your step methods, they are made available across retries. This means that you can set an attribute in step 1, access it in step 2, have step 2 fail, have the job retry, jump directly back to step 2 on retry, and have that object still accessible. This is done by serializing all objects to a field on the `AcidicJob::Run` and manually providing getters and setters that sync with the database record.
+Any objects passed to the `providing` option on the `with_acidity` method are made available across retries. This means that you can set an attribute in step 1, access it in step 2, have step 2 fail, have the job retry, jump directly back to step 2 on retry, and have that object still accessible. This is done by serializing all objects to a field on the `AcidicJob::Run` and manually providing getters and setters that sync with the database record.
 
 ```ruby
 class RideCreateJob < ActiveJob::Base
   include AcidicJob
 
-  def perform(ride_params)
+  def perform(user_id, ride_params)
+    @user = User.find(user_id)
+    @params = ride_params
+
     with_acidity providing: { ride: nil } do
       step :create_ride_and_audit_record
       step :create_stripe_charge
@@ -144,6 +252,7 @@ end
 
 The default pattern you should follow when defining your `perform` method is to make any values that your `step` methods need access to, but are present at the start of the `perform` method simply instance variables. You only need to `provide` attributes that will be set _during a step_. This means, the initial value will almost always be `nil`.
 
+
 ### Transactionally Staged Jobs
 
 A standard problem when inside of database transactions is enqueuing other jobs. On the one hand, you could enqueue a job inside of a transaction that then rollbacks, which would leave that job to fail and retry and fail. On the other hand, you could enqueue a job that is picked up before the transaction commits, which would mean the records are not yet available to this job.
@@ -173,6 +282,7 @@ class RideCreateJob < ActiveJob::Base
 end
 ```
 
+
 ### Custom Idempotency Keys
 
 By default, `AcidicJob` uses the job identifier provided by the queueing system (ActiveJob or Sidekiq) as the idempotency key for the job run. The idempotency key is what is used to guarantee that no two runs of the same job occur. However, sometimes we need particular jobs to be idempotent based on some other criteria. In these cases, `AcidicJob` provides a collection of tools to allow you to ensure the idempotency of your jobs.
@@ -217,28 +327,6 @@ end
 
 > **Note:** The signature of the `acidic_by` proc _needs to match the signature_ of the job's `perform` method.
 
-### Iterable Steps
-
-Sometimes our workflows have steps that need to iterate over a collection and perform an action for each item in the collection before moving on to the next step in the workflow. In these cases, we can use the `for_each` option when defining our step to specific the collection, and `acidic_job` will pass each item into your step method for processing, keeping the same transactional guarantees as for any step. This means that if your step encounters an error in processing any item in the collection, when your job is retried, the job will jump right back to that step and right back to that item in the collection to try again.
-
-```ruby
-class ExampleJob < ActiveJob::Base
-  include AcidicJob
-
-  def perform(record:)
-    with_acidity providing: { collection: [1, 2, 3, 4, 5] } do
-      step :process_item, for_each: :collection
-      step :next_step
-    end
-  end
-  
-  def process_item(item)
-    # do whatever work needs to be done with this individual item
-  end
-end
-```
-
-**Note:** The same restrictions apply here as for any persisted attribute — you can only use objects that can be serialized by ActiveRecord.
 
 ### Sidekiq Callbacks
 
@@ -246,78 +334,6 @@ In order to ensure that `AcidicJob::Staged` records are only destroyed once the
 
 This allows `acidic_job` to use an `after_perform` callback to delete the `AcidicJob::Staged` record, whether you are using the gem with ActiveJob or pure Sidekiq Workers. Of course, this means that you can add your own callbacks to any jobs or workers that include the `AcidicJob` module as well.
 
-### Sidekiq Batches
-
-One final feature for those of you using Sidekiq Pro: an integrated DSL for Sidekiq Batches. By simply adding the `awaits` option to your step declarations, you can attach any number of additional, asynchronous workers to your step. This is profoundly powerful, as it means that you can define a workflow where step 2 is started _if and only if_ step 1 succeeds, but step 1 can have 3 different workers enqueued on 3 different queues, each running in parallel. Once all 3 workers succeed, `acidic_job` will move on to step 2. That's right, by leveraging the power of Sidekiq Batches, you can have workers that are _executed in parallel_, **on separate queues**, and _asynchronously_, but are still **blocking**—as a group—the next step in your workflow! This unlocks incredible power and flexibility for defining and structuring complex workflows and operations, and in my mind is the number one selling point for Sidekiq Pro.
-
-In my opinion, any commercial software using Sidekiq should get Sidekiq Pro; it is _absolutely_ worth the money. If, however, you are using `acidic_job` in a non-commercial application, you could use the open-source dropin replacement for this functionality: https://github.com/breamware/sidekiq-batch
-
-```ruby
-class RideCreateJob < ActiveJob::Base
-  include AcidicJob
-
-  def perform(user_id, ride_params)
-    @user = User.find(user_id)
-    @params = ride_params
-
-    with_acidity providing: { ride: nil } do
-      step :create_ride_and_audit_record, awaits: [SomeJob]
-      step :create_stripe_charge, args: [1, 2, 3], kwargs: { some: 'thing' }
-      step :send_receipt
-    end
-  end
-end
-```
-
-If you need to await a job that takes arguments, you can prepare that job along with its arguments using the `with` class method that `acidic_job` will add to your jobs:
-
-```ruby
-class RideCreateJob < ActiveJob::Base
-  include AcidicJob
-
-  def perform(user_id, ride_params)
-    @user = User.find(user_id)
-    @params = ride_params
-
-    with_acidity providing: { ride: nil } do
-      step :create_ride_and_audit_record, awaits: awaits: [SomeJob.with('argument_1', keyword: 'value')]
-      step :create_stripe_charge, args: [1, 2, 3], kwargs: { some: 'thing' }
-      step :send_receipt
-    end
-  end
-end
-```
-
-You can also await a batch of jobs by simply passing multiple jobs to the `awaits` array (e.g. `awaits: [SomeJob, AnotherJob.with('argument_1', keyword: 'value')]`). Your top level workflow job will only continue to the next step once all of the jobs in your `awaits` array have successfully finished.
-
-In some cases, you may need to _dynamically_ determine the collection of jobs that the step should wait for; in these cases, you can pass the name of a method to the `awaits` option:
-
-```ruby
-class RideCreateJob < ActiveJob::Base
-  include AcidicJob
-  set_callback :finish, :after, :delete_run_record
-
-  def perform(user_id, ride_params)
-    @user = User.find(user_id)
-    @params = ride_params
-
-    with_acidity providing: { ride: nil } do
-      step :create_ride_and_audit_record, awaits: :dynamic_awaits
-      step :create_stripe_charge, args: [1, 2, 3], kwargs: { some: 'thing' }
-      step :send_receipt
-    end
-  end
-  
-  def dynamic_awaits
-    if @params["key"].present?
-      [SomeJob.with('argument_1', keyword: 'value')]
-    else
-      [AnotherJob]
-    end
-  end
-end
-```
-
 
 ### Run Finished Callbacks
 
@@ -349,6 +365,7 @@ class RideCreateJob < ActiveJob::Base
 end
 ```
 
+
 ## Testing
 
 When testing acidic jobs, you are likely to run into `ActiveRecord::TransactionIsolationError`s:

data/acidic_job.gemspec

@@ -35,6 +35,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "minitest"
   spec.add_development_dependency "net-smtp"
   spec.add_development_dependency "noticed"
+  spec.add_development_dependency "psych", "> 4.0"
   spec.add_development_dependency "railties"
   spec.add_development_dependency "rake"
   spec.add_development_dependency "rubocop"

data/lib/acidic_job/awaiting.rb

@@ -17,6 +17,7 @@ module AcidicJob
       run = staged_job_run&.awaited_by || acidic_job_run&.awaited_by
 
       return unless run
+      return if run.batched_runs.outstanding.any?
 
       current_step = run.workflow[run.recovery_point.to_s]
       step_result = run.returning_to
@@ -47,7 +48,7 @@ module AcidicJob
                      end
 
       AcidicJob::Run.transaction do
-        awaited_jobs.each do |awaited_job|
+        awaited_jobs.compact.each do |awaited_job|
           worker_class, args, kwargs = job_args_and_kwargs(awaited_job)
 
           job = worker_class.new(*args, **kwargs)

data/lib/acidic_job/extensions/sidekiq.rb

@@ -47,7 +47,7 @@ module AcidicJob
           arguments = args || @args
           arguments += [kwargs] unless kwargs.empty?
           normalized_args = ::Sidekiq.load_json(::Sidekiq.dump_json(arguments))
-          item = { "class" => self.class, "args" => normalized_args, "jid" => jid }
+          item = { "class" => self.class.name, "args" => normalized_args, "jid" => jid }
           sidekiq_options = sidekiq_options_hash || {}
 
           sidekiq_options.merge(item)

data/lib/acidic_job/run.rb

@@ -3,6 +3,7 @@
 require "active_record"
 require "global_id"
 require "active_support/core_ext/object/with_options"
+require_relative "./serializer"
 
 module AcidicJob
   class Run < ActiveRecord::Base
@@ -17,11 +18,11 @@ module AcidicJob
 
     after_create_commit :enqueue_staged_job, if: :staged?
 
-    serialize :error_object
-    serialize :serialized_job
-    serialize :workflow
-    serialize :returning_to
-    store :attr_accessors
+    serialize :serialized_job, JSON
+    serialize :error_object, Serializer
+    serialize :workflow, Serializer
+    serialize :returning_to, Serializer
+    store :attr_accessors, coder: Serializer
 
     validates :staged, inclusion: { in: [true, false] } # uses database default
     validates :serialized_job, presence: true
@@ -31,7 +32,9 @@ module AcidicJob
     scope :staged, -> { where(staged: true) }
     scope :unstaged, -> { where(staged: false) }
     scope :finished, -> { where(recovery_point: FINISHED_RECOVERY_POINT) }
-    scope :running, -> { where.not(recovery_point: FINISHED_RECOVERY_POINT) }
+    scope :outstanding, lambda {
+      where.not(recovery_point: FINISHED_RECOVERY_POINT).or(where(recovery_point: [nil, ""]))
+    }
 
     with_options unless: :staged? do
       validates :last_run_at, presence: true

data/lib/acidic_job/serializer.rb

@@ -0,0 +1,134 @@
+# frozen_string_literal: true
+
+require "active_job/serializers"
+require "active_job/arguments"
+require "json"
+
+class WorkerSerializer < ActiveJob::Serializers::ObjectSerializer
+  def serialize(worker)
+    # {"_aj_serialized"=>"WorkerSerializer", "class"=>"SuccessfulArgWorker", "args"=>[123], "kwargs"=>{}}]
+    super(
+      "class" => worker.class.name,
+      "args" => worker.instance_variable_get(:@__acidic_job_args),
+      "kwargs" => worker.instance_variable_get(:@__acidic_job_kwargs)
+    )
+  end
+
+  def deserialize(hash)
+    worker_class = hash["class"].constantize
+    worker_class.new(*hash["args"], **hash["kwargs"])
+  end
+
+  def serialize?(argument)
+    defined?(Sidekiq) && argument.class.include?(Sidekiq::Worker)
+  end
+end
+
+class ExceptionSerializer < ActiveJob::Serializers::ObjectSerializer
+  def serialize(exception)
+    hash = {
+      "class" => exception.class.name,
+      "message" => exception.message,
+      "cause" => exception.cause,
+      "backtrace" => {}
+    }
+
+    exception.backtrace.map do |trace|
+      path, _, location = trace.rpartition("/")
+
+      next if hash["backtrace"].key?(path)
+
+      hash["backtrace"][path] = location
+    end
+
+    super(hash)
+  end
+
+  def deserialize(hash)
+    exception_class = hash["class"].constantize
+    exception = exception_class.new(hash["message"])
+    exception.set_backtrace(hash["backtrace"].map do |path, location|
+      [path, location].join("/")
+    end)
+    exception
+  end
+
+  def serialize?(argument)
+    defined?(Exception) && argument.is_a?(Exception)
+  end
+end
+
+class FinishedPointSerializer < ActiveJob::Serializers::ObjectSerializer
+  def serialize(finished_point)
+    super(
+      "class" => finished_point.class.name
+    )
+  end
+
+  def deserialize(hash)
+    finished_point_class = hash["class"].constantize
+    finished_point_class.new
+  end
+
+  def serialize?(argument)
+    defined?(::AcidicJob::FinishedPoint) && argument.is_a?(::AcidicJob::FinishedPoint)
+  end
+end
+
+class RecoveryPointSerializer < ActiveJob::Serializers::ObjectSerializer
+  def serialize(recovery_point)
+    super(
+      "class" => recovery_point.class.name,
+      "name" => recovery_point.name
+    )
+  end
+
+  def deserialize(hash)
+    recovery_point_class = hash["class"].constantize
+    recovery_point_class.new(hash["name"])
+  end
+
+  def serialize?(argument)
+    defined?(::AcidicJob::RecoveryPoint) && argument.is_a?(::AcidicJob::RecoveryPoint)
+  end
+end
+
+ActiveJob::Serializers.add_serializers WorkerSerializer, ExceptionSerializer, FinishedPointSerializer,
+                                       RecoveryPointSerializer
+
+# ...
+module AcidicJob
+  module Arguments
+    include ActiveJob::Arguments
+    extend self # rubocop:disable Style/ModuleFunction
+
+    # `ActiveJob` will throw an error if it tries to deserialize a GlobalID record.
+    # However, this isn't the behavior that we want for our custom `ActiveRecord` serializer.
+    # Since `ActiveRecord` does _not_ reset instance record state to its pre-transactional state
+    # on a transaction ROLLBACK, we can have GlobalID entries in a serialized column that point to
+    # non-persisted records. This is ok. We should simply return `nil` for that portion of the
+    # serialized field.
+    def deserialize_global_id(hash)
+      GlobalID::Locator.locate hash[GLOBALID_KEY]
+    rescue ActiveRecord::RecordNotFound
+      nil
+    end
+  end
+
+  class Serializer
+    # Used for `serialize` method in ActiveRecord
+    class << self
+      def load(json)
+        return if json.nil? || json.empty?
+
+        data = JSON.parse(json)
+        Arguments.deserialize(data).first
+      end
+
+      def dump(obj)
+        data = Arguments.serialize [obj]
+        data.to_json
+      end
+    end
+  end
+end

data/lib/acidic_job/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module AcidicJob
-  VERSION = "1.0.0.pre24"
+  VERSION = "1.0.0.pre27"
 end

data/lib/acidic_job.rb

@@ -50,6 +50,7 @@ module AcidicJob
     # TODO: write test for a staged job that uses awaits
     klass.set_callback :perform, :after, :reenqueue_awaited_by_job,
                        if: -> { was_awaited_job? && !was_workflow_job? }
+    klass.set_callback :perform, :after, :finish_staged_job, if: -> { was_staged_job? && !was_workflow_job? }
     klass.define_callbacks :finish
     klass.set_callback :finish, :after, :reenqueue_awaited_by_job,
                        if: -> { was_workflow_job? && was_awaited_job? }
@@ -136,6 +137,10 @@ module AcidicJob
 
   private
 
+  def finish_staged_job
+    FinishedPoint.new.call(run: staged_job_run)
+  end
+
   def was_workflow_job?
     defined?(@acidic_job_run) && @acidic_job_run.present?
   end
@@ -155,7 +160,7 @@ module AcidicJob
         break
       elsif current_step.nil?
         raise UnknownRecoveryPoint, "Defined workflow does not reference this step: #{recovery_point}"
-      elsif !(jobs = current_step.fetch("awaits", []) || []).empty?
+      elsif !Array(jobs = current_step.fetch("awaits", []) || []).compact.empty?
         step = Step.new(current_step, run, self)
         # Only execute the current step, without yet progressing the recovery_point to the next step.
         # This ensures that any failures in parallel jobs will have this step retried in the main workflow

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: acidic_job
 version: !ruby/object:Gem::Version
-  version: 1.0.0.pre24
+  version: 1.0.0.pre27
 platform: ruby
 authors:
 - fractaledmind
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-05-26 00:00:00.000000000 Z
+date: 2022-07-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -122,6 +122,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: psych
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">"
+      - !ruby/object:Gem::Version
+        version: '4.0'
 - !ruby/object:Gem::Dependency
   name: railties
   requirement: !ruby/object:Gem::Requirement
@@ -270,6 +284,7 @@ files:
 - bin/console
 - bin/setup
 - blog_post.md
+- combustion/log/test.log
 - gemfiles/rails_6.1.gemfile
 - gemfiles/rails_7.0.gemfile
 - lib/acidic_job.rb
@@ -285,6 +300,7 @@ files:
 - lib/acidic_job/recovery_point.rb
 - lib/acidic_job/rspec_configuration.rb
 - lib/acidic_job/run.rb
+- lib/acidic_job/serializer.rb
 - lib/acidic_job/staging.rb
 - lib/acidic_job/step.rb
 - lib/acidic_job/test_case.rb