acidic_job 0.1.0 → 0.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 34c65a77db46e2a7e38aaee61428a0739027820a7a796edf42b91e864818af22
-  data.tar.gz: cd0cfaa3efe9244fc82ea32f197099c29dff0326534a44411e73569158396759
+  metadata.gz: 1c51b3ed20bebb6322ab050f122c83d863fe16436e302b61f91eea84fd9cc117
+  data.tar.gz: 7f65a1d6c99d23930c4730557388833b3ebd277117394b18d6665978bf35119d
 SHA512:
-  metadata.gz: 20158e851d244eaa05881e0ca3723397f4e48cc5b17537956f114bf5507831e9e2ddb17ffb9fa35496a50c067539dc3aa5fb4a9051dd61da71dab0e9e04c1833
-  data.tar.gz: 52272d8d404339ecd6442118e5a5c6718febc9688905dece63bda85d9837d43edbbdfe7386284abcd773660c6b95b98b8caeb7ae6af83267ed2f6778ed952d21
+  metadata.gz: 3a1910297ba003ca354ede4dd5312d12af4f6662d8cb0eca451f4f01e51ba54d80f3f27632785d2ba7c69d5504da042861d122c7a81356d19ca04046097928cd
+  data.tar.gz: 6d57ce3ec53d5ed22bfd91f284611c505400ad7f524f0fbfe0bec4fbf335e94fd4c91781248980342d127186c5973dfed9cd32011c2e6b2e1c6ea72531247f87

data/.github/workflows/main.yml

@@ -0,0 +1,18 @@
+name: Ruby
+
+on: [push,pull_request]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: 2.7.1
+    - name: Run the default task
+      run: |
+        gem install bundler -v 2.2.5
+        bundle install
+        bundle exec rake

data/.gitignore

@@ -6,6 +6,5 @@
 /pkg/
 /spec/reports/
 /tmp/
-
-# rspec failure tracking
-.rspec_status
+.DS_Store
+/test/database.sqlite

data/.rubocop.yml

@@ -1,5 +1,6 @@
 AllCops:
-  TargetRubyVersion: 2.4
+  TargetRubyVersion: 2.7
+  NewCops: enable
 
 Style/StringLiterals:
   Enabled: true

data/.ruby_version

@@ -0,0 +1 @@
+2.7.1

data/Gemfile

@@ -7,6 +7,20 @@ gemspec
 
 gem "rake", "~> 13.0"
 
-gem "rspec", "~> 3.0"
+gem "minitest", "~> 5.0"
 
 gem "rubocop", "~> 1.7"
+
+gem "rubocop-minitest"
+
+gem "rubocop-rake"
+
+gem "activerecord", "~> 6.1.3.2"
+
+gem "activejob", "~> 6.1.3.2"
+
+gem "sqlite3"
+
+gem "database_cleaner"
+
+gem "simplecov"

data/Gemfile.lock

@@ -0,0 +1,91 @@
+PATH
+  remote: .
+  specs:
+    acidic_job (0.1.5)
+      activesupport
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activejob (6.1.3.2)
+      activesupport (= 6.1.3.2)
+      globalid (>= 0.3.6)
+    activemodel (6.1.3.2)
+      activesupport (= 6.1.3.2)
+    activerecord (6.1.3.2)
+      activemodel (= 6.1.3.2)
+      activesupport (= 6.1.3.2)
+    activesupport (6.1.3.2)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 1.6, < 2)
+      minitest (>= 5.1)
+      tzinfo (~> 2.0)
+      zeitwerk (~> 2.3)
+    ast (2.4.2)
+    concurrent-ruby (1.1.9)
+    database_cleaner (2.0.1)
+      database_cleaner-active_record (~> 2.0.0)
+    database_cleaner-active_record (2.0.1)
+      activerecord (>= 5.a)
+      database_cleaner-core (~> 2.0.0)
+    database_cleaner-core (2.0.1)
+    docile (1.4.0)
+    globalid (0.4.2)
+      activesupport (>= 4.2.0)
+    i18n (1.8.10)
+      concurrent-ruby (~> 1.0)
+    minitest (5.14.4)
+    parallel (1.20.1)
+    parser (3.0.1.1)
+      ast (~> 2.4.1)
+    rainbow (3.0.0)
+    rake (13.0.4)
+    regexp_parser (2.1.1)
+    rexml (3.2.5)
+    rubocop (1.18.3)
+      parallel (~> 1.10)
+      parser (>= 3.0.0.0)
+      rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 1.8, < 3.0)
+      rexml
+      rubocop-ast (>= 1.7.0, < 2.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 1.4.0, < 3.0)
+    rubocop-ast (1.7.0)
+      parser (>= 3.0.1.1)
+    rubocop-minitest (0.14.0)
+      rubocop (>= 0.90, < 2.0)
+    rubocop-rake (0.6.0)
+      rubocop (~> 1.0)
+    ruby-progressbar (1.11.0)
+    simplecov (0.21.2)
+      docile (~> 1.1)
+      simplecov-html (~> 0.11)
+      simplecov_json_formatter (~> 0.1)
+    simplecov-html (0.12.3)
+    simplecov_json_formatter (0.1.3)
+    sqlite3 (1.4.2)
+    tzinfo (2.0.4)
+      concurrent-ruby (~> 1.0)
+    unicode-display_width (2.0.0)
+    zeitwerk (2.4.2)
+
+PLATFORMS
+  ruby
+  x86_64-darwin-17
+
+DEPENDENCIES
+  acidic_job!
+  activejob (~> 6.1.3.2)
+  activerecord (~> 6.1.3.2)
+  database_cleaner
+  minitest (~> 5.0)
+  rake (~> 13.0)
+  rubocop (~> 1.7)
+  rubocop-minitest
+  rubocop-rake
+  simplecov
+  sqlite3
+
+BUNDLED WITH
+   2.2.5

data/README.md

@@ -1,8 +1,25 @@
 # AcidicJob
 
-Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/acidic_job`. To experiment with that code, run `bin/console` for an interactive prompt.
+### Idempotent operations for Rails apps, built on top of ActiveJob.
 
-TODO: Delete this and the text above, and describe your gem
+At the conceptual heart of basically any software are "operations"—the discrete actions the software performs. Rails provides a powerful abstraction layer for building operations in the form of `ActiveJob`. With `ActiveJob`, we can easily trigger from other Ruby code throughout our Rails application (controller actions, model methods, model callbacks, etc.); we can run operations both synchronously (blocking execution and then returning its response to the caller) and asychronously (non-blocking and the caller doesn't know its response); and we can also retry a specific operation if needed seemlessly.
+
+However, in order to ensure that our operational jobs are _robust_, we need to ensure that they are properly [idempotent and transactional](https://github.com/mperham/sidekiq/wiki/Best-Practices#2-make-your-job-idempotent-and-transactional). As stated in the [GitLab Sidekiq Style Guide](https://docs.gitlab.com/ee/development/sidekiq_style_guide.html#idempotent-jobs):
+
+>As a general rule, a worker can be considered idempotent if:
+>  * It can safely run multiple times with the same arguments.
+>  * Application side-effects are expected to happen only once (or side-effects of a second run do not have an effect).
+
+This is, of course, far easier said than done. Thus, `AcidicJob`.
+
+`AcidicJob` provides a framework to help you make your operational jobs atomic ⚛️, consistent 🤖, isolated 🕴🏼, and durable ⛰️. Its conceptual framework is directly inspired by a truly wonderful loosely collected series of articles written by Brandur Leach, which together lay out core techniques and principles required to make an HTTP API properly ACIDic:
+
+1. https://brandur.org/acid
+2. https://brandur.org/http-transactions
+3. https://brandur.org/job-drain
+4. https://brandur.org/idempotency-keys
+
+`AcididJob` brings these techniques and principles into the world of a standard Rails application.
 
 ## Installation
 
@@ -20,13 +37,49 @@ Or install it yourself as:
 
     $ gem install acidic_job
 
+Then, use the following command to copy over the AcidicJobKey migration.
+
+```
+rails generate acidic_job:key
+```
+
 ## Usage
 
-TODO: Write usage instructions here
+`AcidicJob` is a concern that you `include` into your operation jobs which provides two public methods to help you make your jobs idempotent and robust—`idempotently` and `step`. You can see them "in action" in the example job below:
+
+```ruby
+class RideCreateJob < ActiveJob::Base
+  include AcidicJob
+
+  def perform(ride_params)
+    idempotently with: { user: current_user, params: ride_params, ride: nil } do
+      step :create_ride_and_audit_record
+      step :create_stripe_charge
+      step :send_receipt
+    end
+  end
+
+  def create_ride_and_audit_record
+    # ...
+  end
+
+  def create_stripe_charge
+    # ...
+  end
+
+  def send_receipt
+    # ...
+  end
+end
+```
+
+`idempotently` takes only the `with:` named parameter and a block where you define the steps of this operation. `step` simply takes the name of a method available in the job. That's all!
+
+So, how does `AcidicJob` make this operation idempotent and robust then? In simplest form, `AcidicJob` creates an "idempotency key" record for each job run, where it stores information about that job run, like the parameters passed in and the step the job is on. It then wraps each of your step methods in a database transaction to ensure that each step in the operation is transactionally secure. Finally, it handles a variety of edge-cases and error conditions for you as well. But, basically, by explicitly breaking your operation into steps and storing a record of each job run and updating its current step as it runs, we level up the `ActiveJob` retry mechanism to ensure that we don't retry already finished steps if something goes wrong and the job has to retry. Then, by wrapping each step in a transaction, we ensure each individual step is ACIDic. Taken together, these two strategies help us to ensure that our operational jobs are both idempotent and ACIDic.
 
 ## Development
 
-After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
 
 To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
 

data/Rakefile

@@ -1,12 +1,16 @@
 # frozen_string_literal: true
 
 require "bundler/gem_tasks"
-require "rspec/core/rake_task"
+require "rake/testtask"
 
-RSpec::Core::RakeTask.new(:spec)
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList["test/**/*_test.rb"]
+end
 
 require "rubocop/rake_task"
 
 RuboCop::RakeTask.new
 
-task default: %i[spec rubocop]
+task default: %i[test rubocop]

data/acidic_job.gemspec

@@ -12,7 +12,7 @@ Gem::Specification.new do |spec|
   spec.description   = "Idempotent operations for Rails apps, built on top of ActiveJob."
   spec.homepage      = "https://github.com/fractaledmind/acidic_job"
   spec.license       = "MIT"
-  spec.required_ruby_version = Gem::Requirement.new(">= 2.4.0")
+  spec.required_ruby_version = Gem::Requirement.new(">= 2.7.0")
 
   spec.metadata["homepage_uri"] = spec.homepage
   spec.metadata["source_code_uri"] = "https://github.com/fractaledmind/acidic_job"
@@ -27,8 +27,7 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  # Uncomment to register a new dependency of your gem
-  # spec.add_dependency "example-gem", "~> 1.0"
+  spec.add_dependency "activesupport"
 
   # For more information and examples about making a new gem, checkout our
   # guide at: https://bundler.io/guides/creating_gem.html

data/bin/console

@@ -3,6 +3,7 @@
 
 require "bundler/setup"
 require "acidic_job"
+require_relative "../test/setup"
 
 # You can add fixtures and/or initialization code here to make experimenting
 # with your gem easier. You can also use a different console, if you like.

data/lib/acidic_job.rb

@@ -1,8 +1,190 @@
 # frozen_string_literal: true
 
 require_relative "acidic_job/version"
+require_relative "acidic_job/no_op"
+require_relative "acidic_job/recovery_point"
+require_relative "acidic_job/response"
+require "active_support/concern"
 
+# rubocop:disable Metrics/ModuleLength, Style/Documentation, Metrics/AbcSize, Metrics/MethodLength
 module AcidicJob
-  class Error < StandardError; end
-  # Your code goes here...
+  class IdempotencyKeyRequired < StandardError; end
+
+  class MissingRequiredAttribute < StandardError; end
+
+  class IdempotencyKeyTooShort < StandardError; end
+
+  class MismatchedIdempotencyKeyAndJobArguments < StandardError; end
+
+  class LockedIdempotencyKey < StandardError; end
+
+  class UnknownRecoveryPoint < StandardError; end
+
+  class UnknownAtomicPhaseType < StandardError; end
+
+  class SerializedTransactionConflict < StandardError; end
+
+  extend ActiveSupport::Concern
+
+  included do
+    attr_reader :key
+
+    # discard_on MismatchedIdempotencyKeyAndJobArguments
+    # discard_on UnknownRecoveryPoint
+    # discard_on UnknownAtomicPhaseType
+    # discard_on MissingRequiredAttribute
+    # retry_on LockedIdempotencyKey
+    # retry_on ActiveRecord::SerializationFailure
+  end
+
+  # Number of seconds passed which we consider a held idempotency key lock to be
+  # defunct and eligible to be locked again by a different API call. We try to
+  # unlock keys on our various failure conditions, but software is buggy, and
+  # this might not happen 100% of the time, so this is a hedge against it.
+  IDEMPOTENCY_KEY_LOCK_TIMEOUT = 90
+
+  # To try and enforce some level of required randomness in an idempotency key,
+  # we require a minimum length. This of course is a poor approximate, and in
+  # real life you might want to consider trying to measure actual entropy with
+  # something like the Shannon entropy equation.
+  IDEMPOTENCY_KEY_MIN_LENGTH = 20
+
+  # &block
+  def idempotently(with:)
+    # set accessors for each argument passed in to ensure they are available
+    # to the step methods the job will have written
+    define_accessors_for_passed_arguments(with)
+
+    # execute the block to gather the info on what phases are defined for this job
+    defined_steps = yield
+
+    # convert the array of steps into a hash of recovery_points and callable actions
+    phases = define_atomic_phases(defined_steps)
+
+    # find or create an AcidicJobKey record to store all information about this job
+    # side-effect: will set the @key instance variable
+    ensure_idempotency_key_record(job_id, with, defined_steps.first)
+
+    # if the key record is already marked as finished, immediately return its result
+    return @key.succeeded? if @key.finished?
+
+    # otherwise, we will enter a loop to process each required step of the job
+    100.times do
+      # our `phases` hash uses Symbols for keys
+      recovery_point = @key.recovery_point.to_sym
+
+      case recovery_point
+      when :FINISHED
+        break
+      else
+        raise UnknownRecoveryPoint unless phases.key? recovery_point
+
+        atomic_phase @key, phases[recovery_point]
+      end
+    end
+
+    # the loop will break once the job is finished, so simply report the status
+    @key.succeeded?
+  end
+
+  def step(method_name)
+    @_steps ||= []
+    @_steps << method_name
+    @_steps
+  end
+
+  private
+
+  def atomic_phase(key, proc = nil, &block)
+    error = false
+    phase_callable = (proc || block)
+
+    begin
+      key.with_lock do
+        phase_result = phase_callable.call
+
+        phase_result.call(key: key)
+      end
+    rescue StandardError => e
+      error = e
+      raise e
+    ensure
+      # If we're leaving under an error condition, try to unlock the idempotency
+      # key right away so that another request can try again.
+      begin
+        key.update_columns(locked_at: nil, error_object: error) if error.present?
+      rescue StandardError => e
+        # We're already inside an error condition, so swallow any additional
+        # errors from here and just send them to logs.
+        puts "Failed to unlock key #{key.id} because of #{e}."
+      end
+    end
+  end
+
+  def ensure_idempotency_key_record(key_val, job_args, first_step)
+    # isolation_level = case ActiveRecord::Base.connection.adapter_name.downcase.to_sym
+    #                   when :sqlite
+    #                     :read_uncommitted
+    #                   else # :nocov:
+    #                     :serializable # :nocov:
+    #                   end
+    isolation_level = :read_uncommitted
+
+    ActiveRecord::Base.transaction(isolation: isolation_level) do
+      @key = AcidicJobKey.find_by(idempotency_key: key_val)
+
+      if @key
+        # Programs enqueuing multiple jobs with different parameters but the
+        # same idempotency key is a bug.
+        raise MismatchedIdempotencyKeyAndJobArguments if @key.job_args != job_args.as_json
+
+        # Only acquire a lock if the key is unlocked or its lock has expired
+        # because the original job was long enough ago.
+        raise LockedIdempotencyKey if @key.locked_at && @key.locked_at > Time.current - IDEMPOTENCY_KEY_LOCK_TIMEOUT
+
+        # Lock the key and update latest run unless the job is already
+        # finished.
+        @key.update!(last_run_at: Time.current, locked_at: Time.current) unless @key.finished?
+      else
+        @key = AcidicJobKey.create!(
+          idempotency_key: key_val,
+          locked_at: Time.current,
+          last_run_at: Time.current,
+          recovery_point: first_step,
+          job_name: self.class.name,
+          job_args: job_args.as_json
+        )
+      end
+    end
+  end
+
+  def define_accessors_for_passed_arguments(passed_arguments)
+    passed_arguments.each do |accessor, value|
+      # the reader method may already be defined
+      self.class.attr_reader accessor unless respond_to?(accessor)
+      # but we should always update the value to match the current value
+      instance_variable_set("@#{accessor}", value)
+    end
+
+    true
+  end
+
+  def define_atomic_phases(defined_steps)
+    defined_steps << :FINISHED
+
+    {}.tap do |phases|
+      defined_steps.each_cons(2).map do |enter_method, exit_method|
+        phases[enter_method] = lambda do
+          method(enter_method).call
+
+          if exit_method == :FINISHED
+            Response.new
+          else
+            RecoveryPoint.new(exit_method)
+          end
+        end
+      end
+    end
+  end
 end
+# rubocop:enable Metrics/ModuleLength, Style/Documentation, Metrics/AbcSize, Metrics/MethodLength

data/lib/acidic_job/no_op.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+# Represents an action to perform a no-op. One possible option for a return
+# from an #atomic_phase block.
+class NoOp
+  def call(_key)
+    # no-op
+  end
+end

data/lib/acidic_job/recovery_point.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+# Represents an action to set a new recovery point. One possible option for a
+# return from an #atomic_phase block.
+class RecoveryPoint
+  attr_accessor :name
+
+  def initialize(name)
+    self.name = name
+  end
+
+  def call(key:)
+    key.update(recovery_point: name)
+  end
+end

data/lib/acidic_job/response.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+# Represents an action to set a new API response (which will be stored onto an
+# idempotency key). One  possible option for a return from an #atomic_phase
+# block.
+class Response
+  def call(key:)
+    key.update!(
+      locked_at: nil,
+      recovery_point: :FINISHED
+    )
+  end
+end

data/lib/acidic_job/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module AcidicJob
-  VERSION = "0.1.0"
+  VERSION = "0.1.5"
 end

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: acidic_job
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.5
 platform: ruby
 authors:
 - fractaledmind
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-06-20 00:00:00.000000000 Z
-dependencies: []
+date: 2021-07-09 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Idempotent operations for Rails apps, built on top of ActiveJob.
 email:
 - stephen.margheim@gmail.com
@@ -17,11 +31,12 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- ".circleci/config.yml"
+- ".github/workflows/main.yml"
 - ".gitignore"
-- ".rspec"
 - ".rubocop.yml"
+- ".ruby_version"
 - Gemfile
+- Gemfile.lock
 - LICENSE
 - README.md
 - Rakefile
@@ -29,6 +44,9 @@ files:
 - bin/console
 - bin/setup
 - lib/acidic_job.rb
+- lib/acidic_job/no_op.rb
+- lib/acidic_job/recovery_point.rb
+- lib/acidic_job/response.rb
 - lib/acidic_job/version.rb
 homepage: https://github.com/fractaledmind/acidic_job
 licenses:
@@ -45,7 +63,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.4.0
+      version: 2.7.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="

data/.circleci/config.yml

@@ -1,13 +0,0 @@
-version: 2.1
-jobs:
-  build:
-    docker:
-      - image: ruby:2.7.1
-    steps:
-      - checkout
-      - run:
-          name: Run the default task
-          command: |
-            gem install bundler -v 2.2.5
-            bundle install
-            bundle exec rake

data/.rspec

@@ -1,3 +0,0 @@
---format documentation
---color
---require spec_helper