acidic_job 1.0.0.rc2 → 1.0.0.rc4

data/.rubocop.yml

@@ -1,83 +1,11 @@
-require:
- - rubocop-minitest
- - rubocop-rake
-
 AllCops:
-  Exclude:
-    - inline.rb
-    - test/simulation.rb
-    - vendor/bundle/**/*
   TargetRubyVersion: 2.7
   NewCops: enable
+  DisabledByDefault: true
+  SuggestExtensions: false
 
-Style/StringLiterals:
-  Enabled: true
-  EnforcedStyle: double_quotes
-
-Style/StringLiteralsInInterpolation:
-  Enabled: true
-  EnforcedStyle: double_quotes
-
-Layout/LineLength:
-  Max: 120
-
-Gemspec/DevelopmentDependencies:
-  Enabled: false
-
-Style/Documentation:
-  Enabled: false
-
-Metrics/ModuleLength:
-  Enabled: false
-
-Metrics/AbcSize:
-  Enabled: false
-
-Metrics/MethodLength:
-  Enabled: false
-
-Metrics/BlockLength:
-  Enabled: false
-
-Metrics/CyclomaticComplexity:
-  Enabled: false
-
-Metrics/PerceivedComplexity:
-  Enabled: false
-
-Metrics/ClassLength:
-  Enabled: false
-
-Minitest/MultipleAssertions:
-  Enabled: false
-
-Style/ModuleFunction:
-  Enabled: false
-
-Style/RaiseArgs:
-  EnforcedStyle: compact
-
-Style/NegatedIf:
-  Enabled: false
-
-Style/ClassAndModuleChildren:
-  Exclude:
-    - test/**/*_test.rb
-
-Naming/VariableNumber:
-  Exclude:
-    - test/**/*_test.rb
-
-Style/SingleLineMethods:
-  Exclude:
-    - test/**/*_test.rb
+inherit_from: "https://www.goodcop.style/base.yml"
 
 Lint/ConstantDefinitionInBlock:
   Exclude:
-    - test/**/*_test.rb
-
-Minitest/AssertTruthy:
-  Enabled: false
-
-Minitest/RefuteFalse:
-  Enabled: false
+    - 'test/**/*'

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    acidic_job (1.0.0.rc2)
+    acidic_job (1.0.0.rc4)
       activejob (>= 7.1)
       activerecord (>= 7.1)
       activesupport (>= 7.1)
@@ -11,41 +11,41 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    actionmailer (7.2.1.1)
-      actionpack (= 7.2.1.1)
-      actionview (= 7.2.1.1)
-      activejob (= 7.2.1.1)
-      activesupport (= 7.2.1.1)
+    actionmailer (8.0.2)
+      actionpack (= 8.0.2)
+      actionview (= 8.0.2)
+      activejob (= 8.0.2)
+      activesupport (= 8.0.2)
       mail (>= 2.8.0)
       rails-dom-testing (~> 2.2)
-    actionpack (7.2.1.1)
-      actionview (= 7.2.1.1)
-      activesupport (= 7.2.1.1)
+    actionpack (8.0.2)
+      actionview (= 8.0.2)
+      activesupport (= 8.0.2)
       nokogiri (>= 1.8.5)
-      racc
-      rack (>= 2.2.4, < 3.2)
+      rack (>= 2.2.4)
       rack-session (>= 1.0.1)
       rack-test (>= 0.6.3)
       rails-dom-testing (~> 2.2)
       rails-html-sanitizer (~> 1.6)
       useragent (~> 0.16)
-    actionview (7.2.1.1)
-      activesupport (= 7.2.1.1)
+    actionview (8.0.2)
+      activesupport (= 8.0.2)
       builder (~> 3.1)
       erubi (~> 1.11)
       rails-dom-testing (~> 2.2)
       rails-html-sanitizer (~> 1.6)
-    activejob (7.2.1.1)
-      activesupport (= 7.2.1.1)
+    activejob (8.0.2)
+      activesupport (= 8.0.2)
       globalid (>= 0.3.6)
-    activemodel (7.2.1.1)
-      activesupport (= 7.2.1.1)
-    activerecord (7.2.1.1)
-      activemodel (= 7.2.1.1)
-      activesupport (= 7.2.1.1)
+    activemodel (8.0.2)
+      activesupport (= 8.0.2)
+    activerecord (8.0.2)
+      activemodel (= 8.0.2)
+      activesupport (= 8.0.2)
       timeout (>= 0.4.0)
-    activesupport (7.2.1.1)
+    activesupport (8.0.2)
       base64
+      benchmark (>= 0.3)
       bigdecimal
       concurrent-ruby (~> 1.0, >= 1.3.1)
       connection_pool (>= 2.2.5)
@@ -55,33 +55,38 @@ GEM
       minitest (>= 5.1)
       securerandom (>= 0.3)
       tzinfo (~> 2.0, >= 2.0.5)
-    ast (2.4.2)
+      uri (>= 0.13.1)
+    ast (2.4.3)
     base64 (0.2.0)
-    bigdecimal (3.1.8)
+    benchmark (0.4.0)
+    bigdecimal (3.1.9)
     builder (3.3.0)
     chaotic_job (0.3.0)
-    combustion (1.3.7)
+    combustion (1.5.0)
       activesupport (>= 3.0.0)
       railties (>= 3.0.0)
       thor (>= 0.14.6)
-    concurrent-ruby (1.3.4)
-    connection_pool (2.4.0)
+    concurrent-ruby (1.3.5)
+    connection_pool (2.5.3)
     crass (1.0.6)
     date (3.4.1)
-    docile (1.4.0)
+    docile (1.4.1)
     drb (2.2.1)
-    erubi (1.13.0)
+    erubi (1.13.1)
     globalid (1.2.1)
       activesupport (>= 6.1)
-    i18n (1.14.6)
+    i18n (1.14.7)
       concurrent-ruby (~> 1.0)
-    io-console (0.7.2)
-    irb (1.14.1)
+    io-console (0.8.0)
+    irb (1.15.2)
+      pp (>= 0.6.0)
       rdoc (>= 4.0.0)
       reline (>= 0.4.2)
-    json (2.10.2)
-    logger (1.6.1)
-    loofah (2.22.0)
+    json (2.12.0)
+    language_server-protocol (3.17.0.5)
+    lint_roller (1.1.0)
+    logger (1.7.0)
+    loofah (2.24.1)
       crass (~> 1.0.2)
       nokogiri (>= 1.12.0)
     mail (2.8.1)
@@ -90,46 +95,52 @@ GEM
       net-pop
       net-smtp
     mini_mime (1.1.5)
-    mini_portile2 (2.8.7)
-    minitest (5.25.1)
-    net-imap (0.5.1)
+    mini_portile2 (2.8.9)
+    minitest (5.25.5)
+    net-imap (0.5.8)
       date
       net-protocol
     net-pop (0.1.2)
       net-protocol
     net-protocol (0.2.2)
       timeout
-    net-smtp (0.5.0)
+    net-smtp (0.5.1)
       net-protocol
-    nokogiri (1.16.7)
+    nokogiri (1.18.8)
       mini_portile2 (~> 2.8.2)
       racc (~> 1.4)
-    nokogiri (1.16.7-x86_64-darwin)
+    nokogiri (1.18.8-x86_64-darwin)
       racc (~> 1.4)
-    parallel (1.22.1)
-    parser (3.2.2.0)
+    parallel (1.27.0)
+    parser (3.3.8.0)
       ast (~> 2.4.1)
-    psych (5.1.2)
+      racc
+    pp (0.6.2)
+      prettyprint
+    prettyprint (0.2.0)
+    prism (1.4.0)
+    psych (5.2.6)
+      date
       stringio
     racc (1.8.1)
-    rack (3.1.8)
-    rack-session (2.0.0)
+    rack (3.1.15)
+    rack-session (2.1.1)
+      base64 (>= 0.1.0)
       rack (>= 3.0.0)
-    rack-test (2.1.0)
+    rack-test (2.2.0)
       rack (>= 1.3)
-    rackup (2.1.0)
+    rackup (2.2.1)
       rack (>= 3)
-      webrick (~> 1.8)
     rails-dom-testing (2.2.0)
       activesupport (>= 5.0.0)
       minitest
       nokogiri (>= 1.6)
-    rails-html-sanitizer (1.6.0)
+    rails-html-sanitizer (1.6.2)
       loofah (~> 2.21)
-      nokogiri (~> 1.14)
-    railties (7.2.1.1)
-      actionpack (= 7.2.1.1)
-      activesupport (= 7.2.1.1)
+      nokogiri (>= 1.15.7, != 1.16.7, != 1.16.6, != 1.16.5, != 1.16.4, != 1.16.3, != 1.16.2, != 1.16.1, != 1.16.0.rc1, != 1.16.0)
+    railties (8.0.2)
+      actionpack (= 8.0.2)
+      activesupport (= 8.0.2)
       irb (~> 1.13)
       rackup (>= 1.0.0)
       rake (>= 12.2)
@@ -137,48 +148,47 @@ GEM
       zeitwerk (~> 2.6)
     rainbow (3.1.1)
     rake (13.2.1)
-    rdoc (6.7.0)
+    rdoc (6.13.1)
       psych (>= 4.0.0)
-    regexp_parser (2.8.0)
-    reline (0.5.10)
+    regexp_parser (2.10.0)
+    reline (0.6.1)
       io-console (~> 0.5)
-    rexml (3.2.5)
-    rubocop (1.50.2)
+    rubocop (1.75.6)
       json (~> 2.3)
+      language_server-protocol (~> 3.17.0.2)
+      lint_roller (~> 1.1.0)
       parallel (~> 1.10)
-      parser (>= 3.2.0.0)
+      parser (>= 3.3.0.2)
       rainbow (>= 2.2.2, < 4.0)
-      regexp_parser (>= 1.8, < 3.0)
-      rexml (>= 3.2.5, < 4.0)
-      rubocop-ast (>= 1.28.0, < 2.0)
+      regexp_parser (>= 2.9.3, < 3.0)
+      rubocop-ast (>= 1.44.0, < 2.0)
       ruby-progressbar (~> 1.7)
-      unicode-display_width (>= 2.4.0, < 3.0)
-    rubocop-ast (1.28.0)
-      parser (>= 3.2.1.0)
-    rubocop-minitest (0.30.0)
-      rubocop (>= 1.39, < 2.0)
-    rubocop-rake (0.6.0)
-      rubocop (~> 1.0)
+      unicode-display_width (>= 2.4.0, < 4.0)
+    rubocop-ast (1.44.1)
+      parser (>= 3.3.7.2)
+      prism (~> 1.4)
     ruby-progressbar (1.13.0)
-    securerandom (0.3.1)
+    securerandom (0.4.1)
     simplecov (0.22.0)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
       simplecov_json_formatter (~> 0.1)
-    simplecov-html (0.12.3)
+    simplecov-html (0.13.1)
     simplecov_json_formatter (0.1.4)
-    sqlite3 (1.6.2)
+    sqlite3 (2.6.0)
       mini_portile2 (~> 2.8.0)
-    sqlite3 (1.6.2-x86_64-darwin)
-    stringio (3.1.1)
+    sqlite3 (2.6.0-x86_64-darwin)
+    stringio (3.1.7)
     thor (1.3.2)
-    timeout (0.4.1)
+    timeout (0.4.3)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
-    unicode-display_width (2.4.2)
-    useragent (0.16.10)
-    webrick (1.8.2)
-    zeitwerk (2.7.1)
+    unicode-display_width (3.1.4)
+      unicode-emoji (~> 4.0, >= 4.0.4)
+    unicode-emoji (4.0.4)
+    uri (1.0.3)
+    useragent (0.16.11)
+    zeitwerk (2.7.2)
 
 PLATFORMS
   ruby
@@ -192,8 +202,6 @@ DEPENDENCIES
   minitest
   rake
   rubocop
-  rubocop-minitest
-  rubocop-rake
   simplecov
   sqlite3
 

data/README.md

@@ -24,7 +24,7 @@ With AcidicJob, you can write reliable and repeatable multi-step distributed ope
 Install the gem and add to the application's Gemfile by executing:
 
 ```sh
-bundle add acidic_job --version "1.0.0.rc1"
+bundle add acidic_job --version "1.0.0.rc3"
 ```
 
 If `bundler` is not being used to manage dependencies, install the gem by executing:
@@ -109,7 +109,7 @@ The block passed to `execute_workflow` is where you define the steps of the work
 The `step` method is the only method available on the yielded workflow builder object, and it simply takes the name of a method available in the job.
 
 > [!IMPORTANT]
-> In order to craft resilient workflows, you need to ensure that each step method wraps a single unit of IO-bound work. You **must not** have a step method that performs multiple IO-bound operations, like writing to your database and calling an external API. Steps should be as granular and self-contained as possible. This allows your own logic to be more durable in case of failures in third-party APIs, network errors, and so on. So, the rule of thumb is to have only one _state mutation_ per step. And this rule of thumb graduates to a hard and fast rule for _foreign state mutations_. You **must** only have **one** foreign state mutation per step, where a foreign state mutation is any operation that writes to a system beyond your own boundaries. This might be creating a charge on Stripe, adding a DNS record, or sending an email.[^1]
+> In order to craft resilient workflows, you need to ensure that each step method wraps a single unit of IO-bound work. You **should not** have a step method that performs multiple IO-bound operations, like writing to your database and calling an external API. Steps should be as granular and self-contained as possible. This allows your own logic to be more durable in case of failures in third-party APIs, network errors, and so on. So, the rule of thumb is to have only one _state mutation_ per step. And this rule of thumb graduates to a hard and fast rule for _foreign state mutations_. You **must** only have **one** foreign state mutation per step, where a foreign state mutation is any operation that writes to a system beyond your own boundaries. This might be creating a charge on Stripe, adding a DNS record, or sending an email.[^1]
 
 [^1]: I first learned this rule from [Brandur Leach](https://twitter.com/brandur) reminds in his post on [Implementing Stripe-like Idempotency Keys in Postgres](https://brandur.org/idempotency-keys).
 
@@ -188,7 +188,7 @@ class Job < ActiveJob::Base
 
 ### Orchestrating steps
 
-In addition to the workflow definition setup, `AcidicJob` also provides a couple of methods to precisely control the workflow step execution. From within any step method, you can call either `repeat_step!` or `halt_step!`.
+In addition to the workflow definition setup, `AcidicJob` also provides a couple of methods to precisely control the workflow step execution. From within any step method, you can call either `repeat_step!` or `halt_workflow!`.
 
 `repeat_step!` will cause the current step to be re-executed on the next iteration of the workflow. This is useful when you need to traverse a collection of items and perform the same operation on each item. For example, if you need to send an email to each user in a collection, you could do something like this:
 
@@ -218,7 +218,7 @@ end
 
 This example demonstrates how you can leverage the basic building blocks provided by `AcidicJob` to orchestrate complex workflows. In this case, the `notify_users` step sends an email to each user in the collection, one at a time, and resiliently handles errors by storing a cursor in the `ctx` object to keep track of the current user being processed. If any error occurs while traversing the `@users` collection, the job will be retried, and the `notify_users` step will be re-executed from the last successful cursor position.
 
-The `halt_step!` method, on the other hand, stops not just the execution of the current step but the job as a whole. This is useful when you either need to conditionally stop the workflow based on some criteria or need to delay the job for some amount of time before being restarted. For example, if you need to send a follow-up email to a user 14 days after they sign up, you could do something like this:
+The `halt_workflow!` method, on the other hand, stops not just the execution of the current step but the job as a whole. This is useful when you either need to conditionally stop the workflow based on some criteria or need to delay the job for some amount of time before being restarted. For example, if you need to send a follow-up email to a user 14 days after they sign up, you could do something like this:
 
 ```ruby
 class Job < ActiveJob::Base
@@ -240,7 +240,7 @@ class Job < ActiveJob::Base
   def send_welcome_email
     if ctx[:halt]
       ctx[:halt] = false
-      halt_step!
+      halt_workflow!
     end
     UserMailer.with(user: @user).welcome_email.deliver_later
   end
@@ -252,7 +252,7 @@ In this example, the `delay` step creates a new instance of the job and enqueues
 
 ### Overview
 
-`AcidicJob` is a library that provides a small yet powerful set of tools to build cohesive and resilient workflows in your Active Jobs. All of the tools are made available by `include`ing the `AcidicJob::Workflow` module. The primary and most important tool is the `execute_workflow` method, which you call within your `perform` method. Then, if you need to store any contextual data, you use the `ctx` objects setters and getters. Finally, within any step methods, you can call `repeat_step!` or `halt_step!` to control the execution of the workflow. If you need, you can also access the `execution` Active Record object to get information about the current execution of the workflow. With these lightweight tools, you can build complex workflows that are resilient to failures and can handle a wide range of use cases.
+`AcidicJob` is a library that provides a small yet powerful set of tools to build cohesive and resilient workflows in your Active Jobs. All of the tools are made available by `include`ing the `AcidicJob::Workflow` module. The primary and most important tool is the `execute_workflow` method, which you call within your `perform` method. Then, if you need to store any contextual data, you use the `ctx` objects setters and getters. Finally, within any step methods, you can call `repeat_step!` or `halt_workflow!` to control the execution of the workflow. If you need, you can also access the `execution` Active Record object to get information about the current execution of the workflow. With these lightweight tools, you can build complex workflows that are resilient to failures and can handle a wide range of use cases.
 
 
 ## Testing

data/acidic_job.gemspec

@@ -41,8 +41,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "minitest"
   spec.add_development_dependency "rake"
   spec.add_development_dependency "rubocop"
-  spec.add_development_dependency "rubocop-minitest"
-  spec.add_development_dependency "rubocop-rake"
   spec.add_development_dependency "simplecov"
   spec.add_development_dependency "sqlite3"
 

data/app/models/acidic_job/entry.rb

@@ -4,16 +4,18 @@ module AcidicJob
   class Entry < Record
     belongs_to :execution, class_name: "AcidicJob::Execution"
 
-    def started?
-      action == "started"
-    end
+    serialize :data, coder: AcidicJob::Serializer
+
+    scope :for_step, -> (step) { where(step: step) }
+    scope :for_action, -> (action) { where(action: action) }
+    scope :ordered, -> { order(timestamp: :asc) }
 
-    def succeeded?
-      action == "succeeded"
+    def self.most_recent
+      order(created_at: :desc).first
     end
 
-    def errored?
-      action == "errored"
+    def action?(check)
+      action == check
     end
   end
 end

data/app/models/acidic_job/execution.rb

@@ -2,25 +2,40 @@
 
 module AcidicJob
   class Execution < Record
-    has_many :entries, class_name: "AcidicJob::Entry"
-    has_many :values, class_name: "AcidicJob::Value"
+    has_many :entries, class_name: "AcidicJob::Entry", dependent: :destroy
+    has_many :values, class_name: "AcidicJob::Value", dependent: :destroy
+
+    serialize :definition, coder: AcidicJob::Serializer
 
     validates :idempotency_key, presence: true # uniqueness constraint is enforced at the database level
     validates :serialized_job, presence: true
 
-    scope :finished, -> { where(recover_to: FINISHED_RECOVERY_POINT) }
-    scope :outstanding, lambda {
-                          where.not(recover_to: FINISHED_RECOVERY_POINT).or(where(recover_to: [nil, ""]))
-                        }
+    scope :finished, -> {
+      where(recover_to: FINISHED_RECOVERY_POINT)
+    }
+    scope :outstanding, -> {
+      where.not(recover_to: FINISHED_RECOVERY_POINT).or(where(recover_to: [nil, ""]))
+    }
+    scope :clearable, -> (finished_before: AcidicJob.clear_finished_executions_after.ago) {
+      finished.where(last_run_at: ...finished_before)
+    }
 
-    def record!(step:, action:, timestamp:, **kwargs)
+    def self.clear_finished_in_batches(batch_size: 500, finished_before: AcidicJob.clear_finished_executions_after.ago, sleep_between_batches: 0)
+      loop do
+        records_deleted = clearable(finished_before: finished_before).limit(batch_size).delete_all
+        sleep(sleep_between_batches) if sleep_between_batches > 0
+        break if records_deleted == 0
+      end
+    end
+
+    def record!(step:, action:, timestamp: Time.current, **kwargs)
       AcidicJob.instrument(:record_entry, step: step, action: action, timestamp: timestamp, data: kwargs) do
-        entries.create!(
+        entries.insert!({
           step: step,
           action: action,
           timestamp: timestamp,
-          data: kwargs.stringify_keys!
-        )
+          data: kwargs.except(:ignored),
+        })
       end
     end
 
@@ -29,7 +44,26 @@ module AcidicJob
     end
 
     def finished?
-      recover_to.to_s == FINISHED_RECOVERY_POINT
+      recover_to.to_s == FINISHED_RECOVERY_POINT ||
+        recover_to.to_s == "FINISHED" # old value pre-1.0, remove at v1.0
+    end
+
+    def defined?(step)
+      if definition.key?("steps")
+        definition["steps"].key?(step)
+      else
+        # TODO: add deprecation warning
+        definition.key?(step)
+      end
+    end
+
+    def definition_for(step)
+      if definition.key?("steps")
+        definition["steps"].fetch(step)
+      else
+        # TODO: add deprecation warning
+        definition.fetch(step)
+      end
     end
 
     def deserialized_job

data/app/models/acidic_job/value.rb

@@ -3,5 +3,7 @@
 module AcidicJob
   class Value < Record
     belongs_to :execution, class_name: "AcidicJob::Execution"
+
+    serialize :value, coder: AcidicJob::Serializer
   end
 end

data/bin/console

@@ -2,8 +2,6 @@
 # frozen_string_literal: true
 
 require "bundler/setup"
-require "rails"
-require "acidic_job"
 
 # You can add fixtures and/or initialization code here to make experimenting
 # with your gem easier. You can also use a different console, if you like.
@@ -11,7 +9,9 @@ require "acidic_job"
 require "combustion"
 require "sqlite3"
 Combustion.path = "test/combustion"
-Combustion.initialize! :active_record
+Combustion.initialize! :active_record, :active_job
+
+require "acidic_job"
 
 # (If you use this, don't forget to add pry to your Gemfile!)
 # require "pry"

data/lib/acidic_job/builder.rb

@@ -4,12 +4,21 @@ module AcidicJob
   class Builder
     attr_reader :steps
 
-    def initialize
+    def initialize(plugins)
+      @plugins = plugins
       @steps = []
     end
 
-    def step(method_name, transactional: false)
-      @steps << { "does" => method_name.to_s, "transactional" => transactional }
+    def step(method_name, **kwargs)
+      step = { "does" => method_name.to_s }
+
+      @plugins.each do |plugin|
+        next unless kwargs.key?(plugin.keyword)
+
+        step[plugin.keyword.to_s] = plugin.validate(kwargs[plugin.keyword])
+      end
+
+      @steps << step
       @steps
     end
 
@@ -17,13 +26,20 @@ module AcidicJob
       # [ { does: "step 1", transactional: true }, { does: "step 2", transactional: false }, ... ]
       @steps << { "does" => FINISHED_RECOVERY_POINT }
 
-      {}.tap do |workflow|
+      definition = {
+        "meta" => {
+          "version" => VERSION,
+        },
+        "steps" => {},
+      }
+
+      definition.tap do |workflow|
         @steps.each_cons(2).map do |enter_step, exit_step|
           enter_name = enter_step["does"]
-          workflow[enter_name] = enter_step.merge("then" => exit_step["does"])
+          workflow["steps"][enter_name] = enter_step.merge("then" => exit_step["does"])
         end
       end
-      # { "step 1": { does: "step 1", transactional: true, then: "step 2" }, ...  }
+      # { meta: { ... }, steps: { "step 1": { does: "step 1", transactional: true, then: "step 2" }, ...  } }
     end
   end
 end