acidic_job 1.0.0.pre18 → 1.0.0.pre21

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6a350dbda056af9bf8965e6a68ece409285d0ba18c03c264509e7249e1635883
-  data.tar.gz: 4361b820ca0efb3f30fbd726692ffbe815746d5adcf1582710e4abcee3246ca6
+  metadata.gz: f47391c0a8bdb8c62c967d716d0190eeddc5ebdd28af81309ffea17ae58f52b2
+  data.tar.gz: 575abae7eb9f70a81537b3260fa8a5ac76d64c231936f0f8e4d96b9e4be116d7
 SHA512:
-  metadata.gz: e97d5c3dc34f74249d9aed1a43ac80a9db776cc7f9eb7e19894bbbbce9ccf4c74395f988fb9cf14d918bf8107642b948e018dfc407528523cd056ff596a86bb7
-  data.tar.gz: ac977d26c7dd272ea9c7503846beeed90b909d52abd425b96c354ac0d21658dbe0849752d005e4b847749667bca6298557ae1edcd75a37278fb9db904875e23d
+  metadata.gz: 6d325fad1fb3240a6cf56d5b8d683b4bf4a0ab8b44c68173141d96fc7ca23a4c8d9f9fcf36d3410ed74f6dec9c16ef4739a642bb09e106050f1a21bc473e3a82
+  data.tar.gz: 355057c6f50d2b37a1b5837247d0d54b9e40e9fedc5793115a607bf9c03353e9ae5f847624608778ee54bdc1bfb950888c33b5ff6db96cb42f83001582571377

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    acidic_job (1.0.0.pre18)
+    acidic_job (1.0.0.pre21)
       activerecord
       activesupport
       database_cleaner
@@ -9,67 +9,67 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    actioncable (7.0.2.3)
-      actionpack (= 7.0.2.3)
-      activesupport (= 7.0.2.3)
+    actioncable (7.0.3)
+      actionpack (= 7.0.3)
+      activesupport (= 7.0.3)
       nio4r (~> 2.0)
       websocket-driver (>= 0.6.1)
-    actionmailbox (7.0.2.3)
-      actionpack (= 7.0.2.3)
-      activejob (= 7.0.2.3)
-      activerecord (= 7.0.2.3)
-      activestorage (= 7.0.2.3)
-      activesupport (= 7.0.2.3)
+    actionmailbox (7.0.3)
+      actionpack (= 7.0.3)
+      activejob (= 7.0.3)
+      activerecord (= 7.0.3)
+      activestorage (= 7.0.3)
+      activesupport (= 7.0.3)
       mail (>= 2.7.1)
       net-imap
       net-pop
       net-smtp
-    actionmailer (7.0.2.3)
-      actionpack (= 7.0.2.3)
-      actionview (= 7.0.2.3)
-      activejob (= 7.0.2.3)
-      activesupport (= 7.0.2.3)
+    actionmailer (7.0.3)
+      actionpack (= 7.0.3)
+      actionview (= 7.0.3)
+      activejob (= 7.0.3)
+      activesupport (= 7.0.3)
       mail (~> 2.5, >= 2.5.4)
       net-imap
       net-pop
       net-smtp
       rails-dom-testing (~> 2.0)
-    actionpack (7.0.2.3)
-      actionview (= 7.0.2.3)
-      activesupport (= 7.0.2.3)
+    actionpack (7.0.3)
+      actionview (= 7.0.3)
+      activesupport (= 7.0.3)
       rack (~> 2.0, >= 2.2.0)
       rack-test (>= 0.6.3)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.0, >= 1.2.0)
-    actiontext (7.0.2.3)
-      actionpack (= 7.0.2.3)
-      activerecord (= 7.0.2.3)
-      activestorage (= 7.0.2.3)
-      activesupport (= 7.0.2.3)
+    actiontext (7.0.3)
+      actionpack (= 7.0.3)
+      activerecord (= 7.0.3)
+      activestorage (= 7.0.3)
+      activesupport (= 7.0.3)
       globalid (>= 0.6.0)
       nokogiri (>= 1.8.5)
-    actionview (7.0.2.3)
-      activesupport (= 7.0.2.3)
+    actionview (7.0.3)
+      activesupport (= 7.0.3)
       builder (~> 3.1)
       erubi (~> 1.4)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.1, >= 1.2.0)
-    activejob (7.0.2.3)
-      activesupport (= 7.0.2.3)
+    activejob (7.0.3)
+      activesupport (= 7.0.3)
       globalid (>= 0.3.6)
-    activemodel (7.0.2.3)
-      activesupport (= 7.0.2.3)
-    activerecord (7.0.2.3)
-      activemodel (= 7.0.2.3)
-      activesupport (= 7.0.2.3)
-    activestorage (7.0.2.3)
-      actionpack (= 7.0.2.3)
-      activejob (= 7.0.2.3)
-      activerecord (= 7.0.2.3)
-      activesupport (= 7.0.2.3)
+    activemodel (7.0.3)
+      activesupport (= 7.0.3)
+    activerecord (7.0.3)
+      activemodel (= 7.0.3)
+      activesupport (= 7.0.3)
+    activestorage (7.0.3)
+      actionpack (= 7.0.3)
+      activejob (= 7.0.3)
+      activerecord (= 7.0.3)
+      activesupport (= 7.0.3)
       marcel (~> 1.0)
       mini_mime (>= 1.1.0)
-    activesupport (7.0.2.3)
+    activesupport (7.0.3)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
@@ -82,7 +82,7 @@ GEM
       activesupport (>= 3.0.0)
       railties (>= 3.0.0)
       thor (>= 0.14.6)
-    concurrent-ruby (1.1.9)
+    concurrent-ruby (1.1.10)
     connection_pool (2.2.5)
     crass (1.0.6)
     database_cleaner (2.0.1)
@@ -112,11 +112,10 @@ GEM
     http-form_data (2.3.0)
     i18n (1.10.0)
       concurrent-ruby (~> 1.0)
-    io-wait (0.2.1)
     llhttp-ffi (0.4.0)
       ffi-compiler (~> 1.0)
       rake (~> 13.0)
-    loofah (2.15.0)
+    loofah (2.18.0)
       crass (~> 1.0.2)
       nokogiri (>= 1.5.9)
     mail (2.7.1)
@@ -134,52 +133,51 @@ GEM
       digest
       net-protocol
       timeout
-    net-protocol (0.1.2)
-      io-wait
+    net-protocol (0.1.3)
       timeout
     net-smtp (0.3.1)
       digest
       net-protocol
       timeout
     nio4r (2.5.8)
-    nokogiri (1.13.3)
+    nokogiri (1.13.6)
       mini_portile2 (~> 2.8.0)
       racc (~> 1.4)
-    nokogiri (1.13.3-x86_64-darwin)
+    nokogiri (1.13.6-x86_64-darwin)
       racc (~> 1.4)
     noticed (1.5.9)
       http (>= 4.0.0)
       rails (>= 5.2.0)
-    parallel (1.21.0)
-    parser (3.1.1.0)
+    parallel (1.22.1)
+    parser (3.1.2.0)
       ast (~> 2.4.1)
-    public_suffix (4.0.6)
+    public_suffix (4.0.7)
     racc (1.6.0)
     rack (2.2.3)
     rack-test (1.1.0)
       rack (>= 1.0, < 3)
-    rails (7.0.2.3)
-      actioncable (= 7.0.2.3)
-      actionmailbox (= 7.0.2.3)
-      actionmailer (= 7.0.2.3)
-      actionpack (= 7.0.2.3)
-      actiontext (= 7.0.2.3)
-      actionview (= 7.0.2.3)
-      activejob (= 7.0.2.3)
-      activemodel (= 7.0.2.3)
-      activerecord (= 7.0.2.3)
-      activestorage (= 7.0.2.3)
-      activesupport (= 7.0.2.3)
+    rails (7.0.3)
+      actioncable (= 7.0.3)
+      actionmailbox (= 7.0.3)
+      actionmailer (= 7.0.3)
+      actionpack (= 7.0.3)
+      actiontext (= 7.0.3)
+      actionview (= 7.0.3)
+      activejob (= 7.0.3)
+      activemodel (= 7.0.3)
+      activerecord (= 7.0.3)
+      activestorage (= 7.0.3)
+      activesupport (= 7.0.3)
       bundler (>= 1.15.0)
-      railties (= 7.0.2.3)
+      railties (= 7.0.3)
     rails-dom-testing (2.0.3)
       activesupport (>= 4.2.0)
       nokogiri (>= 1.6)
     rails-html-sanitizer (1.4.2)
       loofah (~> 2.3)
-    railties (7.0.2.3)
-      actionpack (= 7.0.2.3)
-      activesupport (= 7.0.2.3)
+    railties (7.0.3)
+      actionpack (= 7.0.3)
+      activesupport (= 7.0.3)
       method_source
       rake (>= 12.2)
       thor (~> 1.0)
@@ -187,20 +185,20 @@ GEM
     rainbow (3.1.1)
     rake (13.0.6)
     redis (4.6.0)
-    regexp_parser (2.2.1)
+    regexp_parser (2.4.0)
     rexml (3.2.5)
-    rubocop (1.26.0)
+    rubocop (1.29.1)
       parallel (~> 1.10)
       parser (>= 3.1.0.0)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 1.8, < 3.0)
-      rexml
-      rubocop-ast (>= 1.16.0, < 2.0)
+      rexml (>= 3.2.5, < 4.0)
+      rubocop-ast (>= 1.17.0, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 1.4.0, < 3.0)
-    rubocop-ast (1.16.0)
+    rubocop-ast (1.18.0)
       parser (>= 3.1.1.0)
-    rubocop-minitest (0.18.0)
+    rubocop-minitest (0.19.1)
       rubocop (>= 0.90, < 2.0)
     rubocop-rake (0.6.0)
       rubocop (~> 1.0)
@@ -216,7 +214,7 @@ GEM
     simplecov-html (0.12.3)
     simplecov_json_formatter (0.1.4)
     sqlite3 (1.4.2)
-    strscan (3.0.1)
+    strscan (3.0.3)
     thor (1.2.1)
     timeout (0.2.0)
     tzinfo (2.0.4)

data/README.md

@@ -68,8 +68,10 @@ It provides a suite of functionality that empowers you to create complex, robust
 * Persisted Attributes — when retrying jobs at later steps, we need to ensure that data created in previous steps is still available to later steps on retry.
 * Transactionally Staged Jobs — enqueue additional jobs within the acidic transaction safely
 * Custom Idempotency Keys — use something other than the job ID for the idempotency key of the job run
+* Iterable Steps — define steps that iterate over some collection fully until moving on to the next step
 * Sidekiq Callbacks — bring ActiveJob-like callbacks into your pure Sidekiq Workers
 * Sidekiq Batches — leverage the power of Sidekiq Pro's `batch` functionality without the hassle
+* Run Finished Callbacks — set callbacks for when a job run finishes fully
 
 ### Transactional Steps
 
@@ -80,9 +82,10 @@ class RideCreateJob < ActiveJob::Base
   include AcidicJob
 
   def perform(user_id, ride_params)
-    user = User.find(user_id)
+    @user = User.find(user_id)
+    @params = ride_params
 
-    with_acidity providing: { user: user, params: ride_params, ride: nil } do
+    with_acidity providing: { ride: nil } do
       step :create_ride_and_audit_record
       step :create_stripe_charge
       step :send_receipt
@@ -128,7 +131,7 @@ class RideCreateJob < ActiveJob::Base
   end
 
   def create_stripe_charge
-    Stripe::Charge.create(amount: 20_00, customer: self.ride.user)
+    Stripe::Charge.create(amount: 20_00, customer: @ride.user)
   end
 
   # ...
@@ -139,6 +142,8 @@ end
 
 **Note:** You will note the use of `self.ride = ...` in the code sample above. In order to call the attribute setter method that will sync with the database record, you _must_ use this style. `@ride = ...` and/or `ride = ...` will both fail to sync the value with the database record.
 
+The default pattern you should follow when defining your `perform` method is to make any values that your `step` methods need access to, but are present at the start of the `perform` method simply instance variables. You only need to `provide` attributes that will be set _during a step_. This means, the initial value will almost always be `nil`.
+
 ### Transactionally Staged Jobs
 
 A standard problem when inside of database transactions is enqueuing other jobs. On the one hand, you could enqueue a job inside of a transaction that then rollbacks, which would leave that job to fail and retry and fail. On the other hand, you could enqueue a job that is picked up before the transaction commits, which would mean the records are not yet available to this job.
@@ -150,9 +155,10 @@ class RideCreateJob < ActiveJob::Base
   include AcidicJob
 
   def perform(user_id, ride_params)
-    user = User.find(user_id)
+    @user = User.find(user_id)
+    @params = ride_params
     
-    with_acidity providing: { user: user, params: ride_params, ride: nil } do
+    with_acidity providing: { ride: nil } do
       step :create_ride_and_audit_record
       step :create_stripe_charge
       step :send_receipt
@@ -211,6 +217,28 @@ end
 
 > **Note:** The signature of the `acidic_by` proc _needs to match the signature_ of the job's `perform` method.
 
+### Iterable Steps
+
+Sometimes our workflows have steps that need to iterate over a collection and perform an action for each item in the collection before moving on to the next step in the workflow. In these cases, we can use the `for_each` option when defining our step to specific the collection, and `acidic_job` will pass each item into your step method for processing, keeping the same transactional guarantees as for any step. This means that if your step encounters an error in processing any item in the collection, when your job is retried, the job will jump right back to that step and right back to that item in the collection to try again.
+
+```ruby
+class ExampleJob < ActiveJob::Base
+  include AcidicJob
+
+  def perform(record:)
+    with_acidity providing: { collection: [1, 2, 3, 4, 5] } do
+      step :process_item, for_each: :collection
+      step :next_step
+    end
+  end
+  
+  def process_item(item)
+    # do whatever work needs to be done with this individual item
+  end
+end
+```
+
+**Note:** The same restrictions apply here as for any persisted attribute — you can only use objects that can be serialized by ActiveRecord.
 
 ### Sidekiq Callbacks
 
@@ -229,9 +257,10 @@ class RideCreateJob < ActiveJob::Base
   include AcidicJob
 
   def perform(user_id, ride_params)
-    user = User.find(user_id)
+    @user = User.find(user_id)
+    @params = ride_params
 
-    with_acidity providing: { user: user, params: ride_params, ride: nil } do
+    with_acidity providing: { ride: nil } do
       step :create_ride_and_audit_record, awaits: [SomeJob]
       step :create_stripe_charge, args: [1, 2, 3], kwargs: { some: 'thing' }
       step :send_receipt
@@ -240,6 +269,57 @@ class RideCreateJob < ActiveJob::Base
 end
 ```
 
+If you need to await a job that takes arguments, you can prepare that job along with its arguments using the `with` class method that `acidic_job` will add to your jobs:
+
+```ruby
+class RideCreateJob < ActiveJob::Base
+  include AcidicJob
+
+  def perform(user_id, ride_params)
+    @user = User.find(user_id)
+    @params = ride_params
+
+    with_acidity providing: { ride: nil } do
+      step :create_ride_and_audit_record, awaits: [SomeJob]
+      step :create_stripe_charge, args: [1, 2, 3], kwargs: { some: 'thing' }
+      step :send_receipt
+    end
+  end
+end
+```
+
+### Run Finished Callbacks
+
+When working with workflow jobs that make use of the `awaits` feature for a step, it is important to remember that the `after_perform` callback will be called _as soon as the first `awaits` step has enqueued job_, and **not** when the entire job run has finished. `acidic_job` allows the `perform` method to finish so that the queue for the workflow job is cleared to pick up new work while the `awaits` jobs are running. `acidic_job` will automatically re-enqueue the workflow job and progress to the next step when all of the `awaits` jobs have successfully finished. However, this means that `after_perform` **is not necessarily** the same as `after_finish`. In order to provide the opportunity for you to execute callback logic _if and only if_ a job run has finished, we provide callback hooks for the `finish` event.
+
+For example, you could use this hook to immediately clean up the `AcidicJob::Run` database record whenever the workflow job finishes successfully like so:
+
+```ruby
+class RideCreateJob < ActiveJob::Base
+  include AcidicJob
+  set_callback :finish, :after, :delete_run_record
+
+  def perform(user_id, ride_params)
+    @user = User.find(user_id)
+    @params = ride_params
+
+    with_acidity providing: { ride: nil } do
+      step :create_ride_and_audit_record, awaits: [SomeJob.with('argument_1', keyword: 'value')]
+      step :create_stripe_charge, args: [1, 2, 3], kwargs: { some: 'thing' }
+      step :send_receipt
+    end
+  end
+  
+  def delete_run_record
+    return unless acidic_job_run.succeeded?
+
+    acidic_job_run.destroy!
+  end
+end
+```
+
+You can also await a batch of jobs by simply passing multiple jobs to the `awaits` array (e.g. `awaits: [SomeJob, AnotherJob.with('argument_1', keyword: 'value')]`). Your top level workflow job will only continue to the next step once all of the jobs in your `awaits` array have successfully finished.
+
 ## Testing
 
 When testing acidic jobs, you are likely to run into `ActiveRecord::TransactionIsolationError`s:

data/lib/acidic_job/awaiting.rb

@@ -6,6 +6,8 @@ module AcidicJob
   module Awaiting
     extend ActiveSupport::Concern
 
+    private
+
     def enqueue_step_parallel_jobs(jobs, run, step_result)
       # `batch` is available from Sidekiq::Pro
       raise SidekiqBatchRequired unless defined?(Sidekiq::Batch)
@@ -16,23 +18,22 @@ module AcidicJob
         :success,
         "#{self.class.name}#step_done",
         # NOTE: options are marshalled through JSON so use only basic types.
-        { "run_id" => run.id,
-          "step_result_yaml" => step_result.to_yaml.strip }
+        {
+          "run_id" => run.id,
+          "step_result_yaml" => step_result.to_yaml.strip,
+          "parent_worker" => self.class.name,
+          "job_names" => jobs.map { |job| job_name(job) }
+        }
       )
+
       # NOTE: The jobs method is atomic.
       # All jobs created in the block are actually pushed atomically at the end of the block.
       # If an error is raised, none of the jobs will go to Redis.
       step_batch.jobs do
-        jobs.each do |worker_name|
-          # TODO: handle Symbols as well
-          worker = worker_name.is_a?(String) ? worker_name.constantize : worker_name
-          if worker.instance_method(:perform).arity.zero?
-            worker.perform_async
-          elsif worker.instance_method(:perform).arity == 1
-            worker.perform_async(run.id)
-          else
-            raise TooManyParametersForParallelJob
-          end
+        jobs.each do |job|
+          worker, args, kwargs = job_args_and_kwargs(job)
+
+          worker.perform_async(*args, **kwargs)
         end
       end
     end
@@ -49,5 +50,33 @@ module AcidicJob
       # when a batch of jobs for a step succeeds, we begin processing the `AcidicJob::Run` record again
       process_run(run)
     end
+
+    def job_name(job)
+      case job
+      when Class, Symbol
+        job.to_s
+      when String
+        job
+      else
+        job.class.name
+      end
+    end
+
+    def job_args_and_kwargs(job)
+      case job
+      when Class
+        [job, [], {}]
+      when String
+        [job.constantize, [], {}]
+      when Symbol
+        [job.to_s.constantize, [], {}]
+      else
+        [
+          job.class,
+          job.instance_variable_get(:@__acidic_job_args),
+          job.instance_variable_get(:@__acidic_job_kwargs)
+        ]
+      end
+    end
   end
 end

data/lib/acidic_job/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module AcidicJob
-  VERSION = "1.0.0.pre18"
+  VERSION = "1.0.0.pre21"
 end

data/lib/acidic_job.rb

@@ -47,12 +47,15 @@ module AcidicJob
       raise UnknownJobAdapter
     end
 
+    # TODO: write test for a staged job that uses awaits
     klass.set_callback :perform, :after, :delete_staged_job_record, if: :was_staged_job?
+    klass.define_callbacks :finish
 
     klass.instance_variable_set(:@acidic_identifier, :job_id)
     klass.define_singleton_method(:acidic_by_job_id) { @acidic_identifier = :job_id }
     klass.define_singleton_method(:acidic_by_job_args) { @acidic_identifier = :job_args }
     klass.define_singleton_method(:acidic_by) { |proc| @acidic_identifier = proc }
+    klass.attr_reader(:acidic_job_run)
   end
 
   included do
@@ -65,6 +68,10 @@ module AcidicJob
       super
     end
 
+    def with(*args, **kwargs)
+      new(*args, **kwargs)
+    end
+
     def acidic_identifier
       @acidic_identifier
     end
@@ -76,11 +83,11 @@ module AcidicJob
     @__acidic_job_args = args
     @__acidic_job_kwargs = kwargs
 
-    if method(__method__).super_method.arity.zero?
-      super()
-    else
-      super(*args, **kwargs)
-    end
+    super(*args, **kwargs)
+  rescue ArgumentError => e
+    raise e unless e.message.include?("wrong number of arguments")
+
+    super()
   end
 
   def with_acidity(providing: {})
@@ -94,10 +101,10 @@ module AcidicJob
     # convert the array of steps into a hash of recovery_points and next steps
     workflow = define_workflow(@__acidic_job_steps)
 
-    @run = ensure_run_record(workflow, providing)
+    @acidic_job_run = ensure_run_record(workflow, providing)
 
     # begin the workflow
-    process_run(@run)
+    process_run(@acidic_job_run)
   end
 
   # DEPRECATED
@@ -128,7 +135,7 @@ module AcidicJob
 
   def process_run(run)
     # if the run record is already marked as finished, immediately return its result
-    return run.succeeded? if run.finished?
+    return finish_run(run) if run.finished?
 
     # otherwise, we will enter a loop to process each step of the workflow
     loop do
@@ -167,7 +174,13 @@ module AcidicJob
     end
 
     # the loop will break once the job is finished, so simply report the status
-    run.succeeded?
+    finish_run(run)
+  end
+
+  def finish_run(run)
+    run_callbacks :finish do
+      run.succeeded?
+    end
   end
 
   def step(method_name, awaits: [], for_each: nil)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: acidic_job
 version: !ruby/object:Gem::Version
-  version: 1.0.0.pre18
+  version: 1.0.0.pre21
 platform: ruby
 authors:
 - fractaledmind
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-05-15 00:00:00.000000000 Z
+date: 2022-05-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord