idempo 1.4.0 → 1.5.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 45bef4736adac3732c5b4d29013ca996f3801f3cb66bbb55d1dc554faeef2f99
-  data.tar.gz: 61e7a98fd4593390dc446d2f1656dde637a5e88612f6439104bdc3a5de3f6828
+  metadata.gz: 5fb87c2193d9db479455638fc1f1bf79b609aa7aa8a841f2c0b81d5a088bcb9f
+  data.tar.gz: a2007aed0a26af541eeca24e1ab88cf2ef091c2f2ebccace840303ac2bad3254
 SHA512:
-  metadata.gz: 5e594aaf41686081d0bb007513d4dcbe436df1d814e4d14106d2b709e766dcad4c13264aba3c6478d80a362a5e7f87e38a1890224e140f8e9d9b9765180e3f01
-  data.tar.gz: 8e7557b7aaf0312edf59b86a536fda9cc703adc639f3ffb8e49e18e73b2dec1e451564cafdd720482cb7b66cf6a02031afb4eada933c78f11344ec1db24518e3
+  metadata.gz: 1ab67dcce139c705b486e15cc17c3a7ed9be838dd77b6bdfb635594167611afad682aa9f48ffcc6bb7d2fe1c3dcbb29388422bc3ac1c20873f795b400b0e3bfa
+  data.tar.gz: 5b292a1329d775ee223ee42c7b5cd2f707d810afd388c4323b7a4f8af51de28cd8101aaa75e302f1274874f4ccd36b29b20b43bfef13c46bea4728b53796cbc8

data/.github/workflows/ci.yml

@@ -7,15 +7,44 @@ on:
 env:
   BUNDLE_PATH: vendor/bundle
 
+# The suite has three independent dimensions - Ruby version, Rack major version, and which
+# datastore a backend talks to - and they do not interact. Only `idempo_spec.rb` touches the
+# Rack API at all; every backend spec is Rack-agnostic, and the datastore adapters care about
+# neither Ruby nor Rack. So these are covered separately rather than as a cross product, and
+# exactly one job needs database containers.
 jobs:
   lint:
     name: Code Style
     runs-on: ubuntu-22.04
     if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Setup Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          # The style target comes from .standard.yml, so the Ruby here does not matter
+          ruby-version: "3.4"
+          bundler-cache: true
+      - name: Standard (Lint)
+        run: bundle exec rake standard
+
+  standalone:
+    name: "Ruby ${{ matrix.ruby }}, Rack ${{ matrix.rack }}.x"
+    runs-on: ubuntu-22.04
+    if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository
     strategy:
+      fail-fast: false
       matrix:
-        ruby:
-          - "2.7"
+        # The oldest Ruby the gemspec allows and the newest released one. Bundler resolves
+        # the newest Rails that runs on each, so this also covers ActiveRecord 7.1 and 8.1
+        # through the SQLite backend without naming a single Rails version anywhere.
+        ruby: ["2.7", "4.0"]
+        rack: ["2", "3"]
+    env:
+      RACK_VERSION: "~> ${{ matrix.rack }}.0"
+      # No database server is involved, so the native clients are not even installed
+      BUNDLE_WITHOUT: db_servers
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -24,54 +53,53 @@ jobs:
         with:
           ruby-version: ${{ matrix.ruby }}
           bundler-cache: true
-      - name: Rubocop Cache
-        uses: actions/cache@v3
-        with:
-          path: ~/.cache/rubocop_cache
-          key: ${{ runner.os }}-rubocop-${{ hashFiles('.rubocop.yml') }}
-          restore-keys: |
-            ${{ runner.os }}-rubocop-
-      - name: Standard (Lint)
-        run: bundle exec rake standard
-  test:
-    name: "Specs (Rack 2 and 3)"
+          # Both Rack variants share a Gemfile, so keep their bundles in separate caches
+          cache-version: rack-${{ matrix.rack }}
+      - name: RSpec (middleware, memory and SQLite backends, install generator)
+        run: bundle exec rake spec:standalone
+
+  servers:
+    name: "Database backend (${{ matrix.backend }})"
     runs-on: ubuntu-22.04
     if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository
+    # The only jobs that need containers, and the only ones that run these specs. The
+    # adapters are independent of the Ruby and Rack dimensions covered above, so newest
+    # of everything is enough here. The backends do not depend on each other either, so
+    # each runs in its own job - a service whose image is blank does not get started.
     strategy:
+      fail-fast: false
       matrix:
-        ruby:
-          - "2.7"
+        backend: [mysql, postgres, redis]
     services:
       mysql:
-        image: mysql:5.7
+        image: ${{ matrix.backend == 'mysql' && 'mysql:5.7' || '' }}
         env:
           MYSQL_ALLOW_EMPTY_PASSWORD: yes
         ports:
           - 3306:3306
         options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3
       postgres:
-        image: postgres
+        image: ${{ matrix.backend == 'postgres' && 'postgres' || '' }}
         env:
           POSTGRES_PASSWORD: postgres
         options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
         ports:
           - 5432:5432
       redis:
-        image: redis
+        image: ${{ matrix.backend == 'redis' && 'redis' || '' }}
         options: --health-cmd "redis-cli ping" --health-interval 10s --health-timeout 5s --health-retries 5
         ports:
           - 6379:6379
     steps:
       - name: Checkout
         uses: actions/checkout@v4
-
       - name: Setup Ruby
         uses: ruby/setup-ruby@v1
         with:
-          ruby-version: ${{ matrix.ruby }}
+          ruby-version: "4.0"
           bundler-cache: true
-      - name: RSpec via Appraisal
-        run: "bundle exec appraisal install && bundle exec appraisal rspec"
+      - name: RSpec (${{ matrix.backend }})
+        run: bundle exec rake spec:${{ matrix.backend }}
         env:
           MYSQL_HOST: 127.0.0.1
           MYSQL_PORT: 3306

data/CHANGELOG.md

@@ -1,3 +1,63 @@
+## Unreleased
+
+## 1.5.1
+
+- Raise `required_ruby_version` from 2.4 to 2.7. Idempo has needed Ruby 2.6 for a while (it uses an endless
+  range, and `Struct.new(..., keyword_init: true)` which arrived in 2.5), so installing it on 2.4 or 2.5 succeeded
+  and then failed to load. 2.7 is the oldest Ruby CI runs against, so it is the oldest version we actually know works.
+- Fix an off-by-one in `ResponseStore#prune` (used by `MemoryBackend`): the expiry handle at the pruning
+  boundary was deleted but also kept, so one already-expired handle stayed in the store and was pruned
+  again on every subsequent call. It also no longer leaks a response whose `expire_at` equals the prune time:
+  its expiry handle was removed but the response itself was kept, and stayed in memory until its key was
+  looked up or overwritten.
+- Add SQLite support to `ActiveRecordBackend`. SQLite has no advisory locks, so the backend now ships a
+  `TokenLock` - a lease lock built on an atomic `INSERT ... ON CONFLICT DO UPDATE ... WHERE` into a new
+  `idempo_locks` table, with a random fencing token and an expiry. It is the same scheme the `RedisBackend`
+  uses, and it recovers by itself from a process killed while holding a lock. MySQL and PostgreSQL keep
+  using advisory locks and are unaffected.
+- Add a `rails g idempo:install` generator. It detects whether `idempo_responses` already exists and
+  generates a migration for only the tables you are missing, so existing installations get a migration
+  which adds `idempo_locks` alone.
+- `ActiveRecordBackend.create_table` is unchanged and still creates only `idempo_responses` - migrations
+  already committed against earlier versions of Idempo keep behaving exactly as before. The table
+  definitions are now also available individually as `create_responses_table` and `create_locks_table`.
+- `ActiveRecordBackend#prune!` also deletes abandoned lock rows.
+- `ActiveRecordBackend.new` accepts a `lock:` argument to override the lock implementation. Passing
+  `TokenLock` makes the backend usable through a transaction-pooling proxy such as PGBouncer, where
+  connection-bound advisory locks are not.
+- A request rejected with `ConcurrentRequest` no longer frees the in-memory lock of the request it
+  collided with, so a third request with the same key is rejected as well instead of running concurrently.
+- `TokenLock` keeps its lease tokens per request key, so the one instance passed via `lock:` serves
+  concurrent requests with different keys - on Puma threads or Falcon fibers - without one request's
+  lease being lost, or its lock row left behind, when another request takes a lock in the meantime.
+- A response is no longer written when the lock lease expired while the request was being served.
+- On PostgreSQL `ActiveRecordBackend` takes the time from the database instead of from the Ruby process, so
+  app hosts with clocks that disagree - all talking to one database through PGBouncer - agree on when a
+  `TokenLock` lease and a stored response expire. `prune!` uses the same clock. SQLite keeps using the Ruby
+  time, since it runs inside the process and would read the very same clock anyway.
+- Test against the latest released Rails instead of pinning old versions. Every Rails in the
+  previous matrix (6.1 and 7.1) had been end of life for a year or more, while the current
+  release was not covered at all. No Rails version is named anywhere now - Bundler resolves
+  the newest one that runs on the Ruby under test.
+- CI runs on Ruby 2.7 (the oldest the gemspec allows) and Ruby 4.0. On 2.7 Bundler resolves the
+  newest Rails that still runs there (7.1), on 4.0 it resolves the current release.
+- Drop the `appraisal` development dependency along with the `Appraisals` file and the
+  generated `gemfiles/` directory. Rack is pinned through a `RACK_VERSION` environment
+  variable read by the Gemfile instead. The checked-in appraisal lockfiles also pinned Bundler
+  to the version that generated them (2.1.4), which cannot run on Ruby 3.x or 4.x.
+- Split the suite into `rake spec:standalone` (needs no database server) and
+  `rake spec:servers` (needs MySQL, PostgreSQL or Redis), so the specs which need containers
+  run once rather than once per Ruby and Rack combination. `spec:mysql`, `spec:postgres` and
+  `spec:redis` run one server's specs each, and CI runs those in parallel, every job starting
+  only the container it needs.
+- Pin `standard` to 1.37.0, the last version which installs on Ruby 2.7, so every Ruby in CI lints with the same
+  rules. Remove the unused `.rubocop.yml`, which inherited from `wetransfer_style` - a gem Idempo does not depend on.
+- Add a `CONTRIBUTING.md`.
+
+Note that ActiveRecord is not a runtime dependency of Idempo - it is required lazily by
+`ActiveRecordBackend`. Nothing in this release changes which databases or Rails versions the
+backend can work with, only which combinations are verified.
+
 ## 1.4.0
 
 - `RequestFingerprint` is now a class instead of a module, with an overridable `extract_user_identity` method.

data/CONTRIBUTING.md

@@ -0,0 +1,80 @@
+# Contributing to Idempo
+
+Contributions are welcome! Please take a moment to review this document so that
+contributing stays easy and effective for everyone involved.
+
+I'd especially love to get:
+
+* Reproductions of races and double execution - ideally as a deterministic spec
+* Fixes from people running a backend in production (PGBouncer, MySQL proxies, SQLite under load)
+* Documentation fixes
+
+## What do I need to know to help?
+
+If you would like to learn more about the technologies Idempo builds on, check out the (non-exhaustive) list below. You can also get in touch with me via an issue or email.
+
+ - [ruby](https://ruby-doc.org) and the [Rack specification](https://github.com/rack/rack/blob/main/SPEC.rdoc)
+ - [Designing robust and predictable APIs with idempotency](https://stripe.com/blog/idempotency) by Brandur Leach - why idempotency keys exist at all
+ - [Implementing Stripe-like Idempotency Keys in Postgres](https://brandur.org/idempotency-keys) and [Using Atomic Transactions to Power an Idempotent API](https://brandur.org/http-transactions), also by Brandur - how to build them on a database
+ - [The Idempotency-Key HTTP Header Field](https://datatracker.ietf.org/doc/draft-ietf-httpapi-idempotency-key-header/), the IETF draft Idempo follows for the header (the draft has since expired)
+ - Advisory locks in [PostgreSQL](https://www.postgresql.org/docs/current/explicit-locking.html#ADVISORY-LOCKS) and [MySQL](https://dev.mysql.com/doc/refman/8.4/en/locking-functions.html), and [UPSERT in SQLite](https://www.sqlite.org/lang_upsert.html), which the `TokenLock` is built on
+ - [rspec](https://rspec.info) for testing
+
+## Issues and pull requests
+
+* Opening an issue first is often the fastest way to get something in. I can shape it into something that fits the library.
+* The most useful bug report for a race is the exact order of events across requests, or better yet a spec which forces that order (with `Queue`s or fibers) instead of hoping for it with `sleep`.
+* I may close a PR and implement the idea differently. That's not a rejection of the idea, and you'll be credited (`Co-authored-by`).
+* Feel free to propose opinionated changes, but be prepared that your opinion won't be shared. It is a give-and-take.
+
+## Code changes
+
+Please adhere to the coding conventions used throughout the project (indentation,
+accurate and terse comments, etc.) and any other requirements (such as test coverage).
+Idempo supports Ruby 2.7 and newer, and Rack 2 and 3 - please don't use syntax or APIs
+which rule either of those out.
+
+Run `bundle exec rake standard` to make sure the code is properly formatted. The tests come in two parts:
+
+* `bundle exec rake spec:standalone` needs no database server - the middleware, the memory and SQLite backends and the install generator. Set `BUNDLE_WITHOUT=db_servers` to skip installing the database clients, and `RACK_VERSION="~> 2.0"` to test against Rack 2.
+* `bundle exec rake spec:mysql`, `spec:postgres` and `spec:redis` need the respective server running locally. `spec:servers` runs all three.
+
+## Basic expectations
+
+This project is something created with love, and I - as an author - expect you, as a contributor, to return the favor. The ground rules:
+
+* Do your best to be kind.
+* Be polite and direct. No abrasiveness, no passive-aggression.
+* [Low-context](https://www.ebsco.com/research-starters/communication-and-mass-media/high-context-and-low-context-cultures) communication is preferred and permitted
+* No ad-hominems, also in code comments
+* No sexualized or aggressive language
+
+As a maintainer, I reserve the right to deny any contribution, discussion post, comment or pull request I find inappropriate.
+
+Similarly, I reserve the right to not accept any contribution that doesn't fit the goals or spirit of the project (or my own), without explaining why it has been refused.
+
+## Decisions are final
+
+You're welcome to make your case once, clearly. After that, please don't reopen closed issues, start meta-threads about the decision, or argue about the wording of this document. This is a list of expectations, not a contract to be lawyered. If something isn't covered, the maintainer's judgment applies.
+
+## What happens elsewhere stays elsewhere
+
+Contributions are judged on what they are, not on who sent them. What anyone says or does outside this project (on social media, at their job, in other repos, in politics) is not a topic here and will not be considered, either for or against them. Please don't bring it in: no call-outs, no "are you aware this person…" threads, no requests to ban someone over things that didn't happen here.
+
+## AI-generated issues and code
+
+This project uses LLMs to produce code and artifacts, and it's fine if you do the same when contributing. LLM use in this project is settled: threads about whether it should be used (utility, harm, societal implications etc.) will be closed.
+
+Please disclose LLM use in every issue and PR: say which model or tool you used and what for (e.g. "Claude wrote the fix, I wrote the test"). This is for transparency, not for judgment. Disclosed LLM use is treated the same as no LLM use.
+
+However, the following is on you:
+
+* Make your LLM output _terse._ Nobody likes reading slop, even my agents. Make it get to the point.
+* Make sure what you submit actually ran. Don't file issues about behavior or APIs your LLM imagined.
+* Do not add external dependencies (or have your LLM do it) unless absolutely necessary
+* If you file a PR, make at least one pass with "Make this at least 50% less code"
+* Code generated by your LLM will be considered as written by you, with the same requirements
+
+## License
+
+By submitting a patch, you agree to allow the project owner to license your work under the same license as that used by the project, which you can see [here](LICENSE.txt).

data/Gemfile

@@ -3,3 +3,16 @@
 source "https://rubygems.org"
 
 gemspec
+
+# Rack is the only runtime dependency whose major version we actively test against.
+# CI pins this to exercise both Rack 2 and Rack 3; left unset it resolves to the newest.
+gem "rack", ENV["RACK_VERSION"] if ENV["RACK_VERSION"]
+
+# Native client libraries for the database servers Idempo supports. These are kept out of
+# the gemspec development dependencies so that CI jobs which need no database server can
+# skip them entirely.
+group :db_servers do
+  gem "mysql2"
+  gem "pg"
+  gem "redis", "~> 4"
+end

data/README.md

@@ -7,7 +7,7 @@ instead of calling your application.
 
 ## Usage
 
-Idempo supports a number of backends, we recommend using Redis if you have multiple application servers / dynos and MemoryBackend if you are only using one single Puma worker. To initialize with Redis as backend pass the `backend:` parameter when adding the middleware:
+Idempo supports a number of backends, we recommend using Redis if you have multiple application servers / dynos, the ActiveRecordBackend if you would rather not run Redis just for this (it works with MySQL, PostgreSQL and SQLite), and MemoryBackend if you are only using one single Puma worker. To initialize with Redis as backend pass the `backend:` parameter when adding the middleware:
 
 ```ruby
 be = Idempo::RedisBackend.new(Rails.application.config.redis_connection_pool)
@@ -47,22 +47,46 @@ Needless to say, if your server terminates or restarts all the data disappears w
 
 ## Using your database for idempotency keys (via ActiveRecord)
 
-The relational database you already have is a perfectly fine place to store idempotency key locks and responses. A requirement for that is that your database supports some form of advisory locking - both PostgreSQL and MySQL do. First you will need to create a table for the records. The table is going to be called `idempo_responses`, and you need to add a migration in your Rails project for it:
+The relational database you already have is a perfectly fine place to store idempotency key locks and responses. Idempo supports MySQL, PostgreSQL and SQLite. Idempo stores its data in two tables - `idempo_responses` and `idempo_locks` - and ships a generator which creates the migration for them:
 
 ```bash
-$ rails g migration add_idempo_responses
+$ rails g idempo:install
+$ rails db:migrate
 ```
 
-and then add a migration like this:
+If you are already using Idempo you will have the `idempo_responses` table from an earlier version. The generator detects this and writes a migration which only adds the table you are missing:
+
+```
+$ rails g idempo:install
+      idempo  idempo_responses is already present, creating idempo_locks only
+      create  db/migrate/20240115120000_add_idempo_locks.rb
+```
+
+Detection queries your database, and falls back to reading `db/schema.rb` (or `db/structure.sql`) when there is no database to connect to. You can always override it with `--locks-only` or `--no-locks-only`.
+
+If you would rather write the migration by hand, the two table definitions are available separately. For a new installation:
 
 ```ruby
-class AddIdempoResponses < ActiveRecord::Migration[7.0]
+class InstallIdempo < ActiveRecord::Migration[7.0]
   def change
-    Idempo::ActiveRecordBackend.create_table(self)
+    Idempo::ActiveRecordBackend.create_responses_table(self)
+    Idempo::ActiveRecordBackend.create_locks_table(self)
   end
 end
 ```
 
+and for an installation which already has `idempo_responses`:
+
+```ruby
+class AddIdempoLocks < ActiveRecord::Migration[7.0]
+  def change
+    Idempo::ActiveRecordBackend.create_locks_table(self)
+  end
+end
+```
+
+`Idempo::ActiveRecordBackend.create_table` still creates just `idempo_responses`, exactly as it did in earlier versions - migrations you have already committed keep working and keep meaning the same thing.
+
 Then configure Idempo to use the backend (in your `application.rb`):
 
 ```ruby
@@ -70,13 +94,52 @@ be = Idempo::ActiveRecordBackend.new
 config.middleware.insert Idempo, backend: be
 ```
 
-In your regular tasks (cron or Rake) you will want to add a call to delete old Idempo responses (there is an index on `expire_at`):
+In your regular tasks (cron or Rake) you will want to add a call to delete old Idempo responses and abandoned locks (both tables have an index on `expire_at`):
 
 ```ruby
 Idempo::ActiveRecordBackend.new.prune!
 ```
 
-If you need to use Idempo with PGBouncer you will need to write your own locking implementation based on fencing tokens or similar.
+### How locking works per database
+
+Idempo picks the locking strategy from your database adapter:
+
+* **MySQL** uses `GET_LOCK` / `RELEASE_LOCK` advisory locks
+* **PostgreSQL** uses `pg_try_advisory_lock` / `pg_advisory_unlock` advisory locks
+* **SQLite** has no advisory locks, so it uses the `TokenLock` - see below
+
+Advisory locks are held by the database connection, so they are released the moment the connection goes away - which makes them ideal, but also makes them unusable through a transaction-pooling proxy such as PGBouncer. If that is your setup, pass the `TokenLock` explicitly:
+
+```ruby
+be = Idempo::ActiveRecordBackend.new(lock: Idempo::ActiveRecordBackend::TokenLock.new)
+```
+
+### Using SQLite
+
+SQLite has no advisory locks and only one writer at a time, but it does apply a single `INSERT ... ON CONFLICT DO UPDATE ... WHERE` statement atomically - and that is all a lease lock needs. The `TokenLock` inserts a row with a random fencing token into `idempo_locks`; the unique index on the request key is what makes the acquisition mutually exclusive. It is the same scheme the `RedisBackend` uses (`SET NX PX` plus a token), only expressed in SQL:
+
+* Acquiring the lock is one statement which inserts when no lock row exists, overwrites the row when the previous lease has expired, and does nothing at all when somebody else holds the lock. Idempo then raises `ConcurrentRequest`, exactly like it does for a contended advisory lock.
+* The lock row carries an expiry (`Idempo::ActiveRecordBackend::LOCK_TTL_SECONDS`, 5 minutes by default), so a lock left behind by a killed process gets taken over by a later request instead of wedging that idempotency key forever.
+* Releasing deletes only a row still carrying our own token, so we never release a lock somebody else has taken over.
+* If the lease does expire while your `app.call` is still running, the response is not written at all - another request may have generated a different response under the same key in the meantime, and overwriting it would be worse than not caching.
+
+Nothing beyond the migration is required, but two settings in your `database.yml` are worth having:
+
+```yaml
+production:
+  adapter: sqlite3
+  database: storage/production.sqlite3
+  timeout: 5000  # Wait up to 5s for the write lock instead of raising SQLITE_BUSY
+```
+
+and WAL mode, so that reading a cached response does not block on the process currently writing one:
+
+```ruby
+# config/initializers/sqlite.rb, if your Rails version does not do this already
+ActiveRecord::Base.connection.execute("PRAGMA journal_mode=WAL") if ActiveRecord::Base.connection.adapter_name.match?(/sqlite/i)
+```
+
+Rails 7.1 and newer set `journal_mode=WAL` for new applications by default, and starting with Rails 7.1 the SQLite adapter also opens write transactions with `BEGIN IMMEDIATE`. Idempo does not depend on either - it orders the statements in its write transaction so that the write lock is taken by the first statement - but WAL will make a meaningful difference to your throughput.
 
 ## Using Redis for idempotency keys
 

data/Rakefile

@@ -3,11 +3,28 @@
 require "bundler/gem_tasks"
 require "rspec/core/rake_task"
 require "standard/rake"
-require "appraisal"
 
-RSpec::Core::RakeTask.new(:spec)
-task default: :spec
+SERVER_BACKENDS = %w[mysql postgres redis]
+SERVER_BACKED_SPECS = "spec/**/*{#{SERVER_BACKENDS.join(",")}}*_spec.rb"
+
+# Everything that needs no database server: the middleware itself, the memory backend,
+# the SQLite backend and the install generator. This is the bulk of the suite.
+RSpec::Core::RakeTask.new("spec:standalone") do |t|
+  t.rspec_opts = %(--exclude-pattern "#{SERVER_BACKED_SPECS.sub("spec/", "")}")
+end
+
+# Only the specs which need MySQL, PostgreSQL or Redis to be running.
+RSpec::Core::RakeTask.new("spec:servers") do |t|
+  t.pattern = SERVER_BACKED_SPECS
+end
 
-if !ENV["APPRAISAL_INITIALIZED"]
-  task default: :appraisal
+# One per server, so that CI can run each in its own job with just that one container
+SERVER_BACKENDS.each do |backend|
+  RSpec::Core::RakeTask.new("spec:#{backend}") do |t|
+    t.pattern = "spec/**/*#{backend}*_spec.rb"
+  end
 end
+
+RSpec::Core::RakeTask.new(:spec)
+
+task default: :spec

data/idempo.gemspec

@@ -12,7 +12,7 @@ Gem::Specification.new do |spec|
   spec.description = "Provides idempotency keys for Rack applications."
   spec.homepage = "https://github.com/julik/idempo"
   spec.license = "MIT"
-  spec.required_ruby_version = Gem::Requirement.new(">= 2.4.0")
+  spec.required_ruby_version = Gem::Requirement.new(">= 2.7.0")
 
   # Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
   # to allow pushing to a single host or delete this section to allow pushing to any host.
@@ -29,7 +29,7 @@ Gem::Specification.new do |spec|
   # Specify which files should be added to the gem when it is released.
   # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
   spec.files = Dir.chdir(File.expand_path(__dir__)) do
-    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{\A(?:test|spec|features|gemfiles)/}) }
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{\A(?:test|spec|features)/}) }
   end
   spec.bindir = "exe"
   spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
@@ -41,13 +41,11 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "rspec", "~> 3.0"
-  spec.add_development_dependency "redis", "~> 4"
   spec.add_development_dependency "rack-test"
   spec.add_development_dependency "activerecord"
-  spec.add_development_dependency "mysql2"
-  spec.add_development_dependency "pg"
-  spec.add_development_dependency "standard"
-  spec.add_development_dependency "appraisal"
+  spec.add_development_dependency "sqlite3"
+  spec.add_development_dependency "railties"
+  spec.add_development_dependency "standard", "1.37.0" # The last version which installs on Ruby 2.7
 
   # For more information and examples about making a new gem, checkout our
   # guide at: https://bundler.io/guides/creating_gem.html

data/lib/generators/idempo/install/install_generator.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+require "rails/generators"
+require "rails/generators/active_record"
+
+# Note this is `class Idempo` and not `module Idempo` - Idempo is a class, and reopening
+# it as a module raises a TypeError once lib/idempo.rb has been loaded
+class Idempo
+  module Generators
+    # Generates the migration for the Idempo tables. Existing installations already have
+    # `idempo_responses` from an earlier version of Idempo and only need `idempo_locks`,
+    # so the generator detects which of the two situations it is in.
+    class InstallGenerator < Rails::Generators::Base
+      include ActiveRecord::Generators::Migration
+
+      source_root File.expand_path("templates", __dir__)
+
+      desc "Creates a migration for the Idempo tables (idempo_responses and idempo_locks)"
+
+      class_option :locks_only, type: :boolean, default: nil,
+        desc: "Only create idempo_locks (default: detect whether idempo_responses already exists)"
+
+      def create_migration_file
+        say_status :idempo, migration_explanation
+        migration_template "install_migration.rb.tt", File.join(db_migrate_path, "#{migration_name}.rb")
+      end
+
+      private
+
+      def migration_name
+        locks_only? ? "add_idempo_locks" : "install_idempo"
+      end
+
+      def migration_class_name
+        migration_name.camelize
+      end
+
+      def migration_explanation
+        if !options[:locks_only].nil?
+          locks_only? ? "creating idempo_locks only (--locks-only)" : "creating both Idempo tables (--no-locks-only)"
+        elsif locks_only?
+          "idempo_responses is already present, creating idempo_locks only"
+        else
+          "creating both Idempo tables"
+        end
+      end
+
+      def locks_only?
+        return options[:locks_only] unless options[:locks_only].nil?
+        @locks_only = responses_table_already_present? if !defined?(@locks_only)
+        @locks_only
+      end
+
+      # An existing installation may not have a database we can connect to at the moment
+      # the generator runs (a fresh checkout, or a CI box), so fall back to looking at the
+      # schema dump before assuming this is a fresh install.
+      def responses_table_already_present?
+        connection_says_responses_table_exists?
+      rescue => e
+        say_status :idempo, "could not query the database (#{e.class}), reading the schema dump instead", :yellow
+        schema_dump_mentions_responses_table?
+      end
+
+      def connection_says_responses_table_exists?
+        ActiveRecord::Base.connection.table_exists?("idempo_responses")
+      end
+
+      def schema_dump_mentions_responses_table?
+        %w[db/schema.rb db/structure.sql].any? do |relative_path|
+          path = File.expand_path(relative_path, destination_root)
+          File.exist?(path) && File.read(path).include?("idempo_responses")
+        end
+      end
+
+      def migration_version
+        "[%d.%d]" % [ActiveRecord::VERSION::MAJOR, ActiveRecord::VERSION::MINOR]
+      end
+    end
+  end
+end

data/lib/generators/idempo/install/templates/install_migration.rb.tt

@@ -0,0 +1,8 @@
+class <%= migration_class_name %> < ActiveRecord::Migration<%= migration_version %>
+  def change
+<% unless locks_only? -%>
+    Idempo::ActiveRecordBackend.create_responses_table(self)
+<% end -%>
+    Idempo::ActiveRecordBackend.create_locks_table(self)
+  end
+end

data/lib/idempo/active_record_backend.rb

@@ -1,6 +1,19 @@
-# This backend currently only works with mysql2 since it uses advisory locks
+# This backend uses advisory locks where the database provides them (MySQL and PostgreSQL)
+# and a lock table with fencing tokens where it does not (SQLite)
 class Idempo::ActiveRecordBackend
+  # The TTL for a lease taken by the TokenLock. It only comes into play if the process
+  # holding the lock crashes or gets killed without releasing it - the lease is what
+  # allows a subsequent request to take the lock over.
+  LOCK_TTL_SECONDS = 5 * 60
+
+  # Kept as-is for backwards compatibility: migrations generated by earlier versions of
+  # Idempo call this, and they must keep creating exactly the one table they always did.
+  # A fresh install gets both tables via `rails g idempo:install`.
   def self.create_table(via_migration)
+    create_responses_table(via_migration)
+  end
+
+  def self.create_responses_table(via_migration)
     via_migration.create_table "idempo_responses", charset: "utf8mb4", collation: "utf8mb4_unicode_ci" do |t|
       t.string :idempotent_request_key, index: {unique: true}, null: false
       t.datetime :expire_at, index: true, null: false # Needs an index for cleanup
@@ -9,16 +22,64 @@ class Idempo::ActiveRecordBackend
     end
   end
 
-  class Store < Struct.new(:key, :model)
+  # Only databases without advisory locks (SQLite) actually use this table, but having it
+  # on MySQL and PostgreSQL costs nothing - it just stays empty, and it means switching
+  # databases later needs no schema change.
+  def self.create_locks_table(via_migration)
+    via_migration.create_table "idempo_locks", charset: "utf8mb4", collation: "utf8mb4_unicode_ci" do |t|
+      t.string :idempotent_request_key, index: {unique: true}, null: false
+      t.string :lock_token, null: false
+      t.datetime :expire_at, index: true, null: false # Needs an index for cleanup
+    end
+  end
+
+  # Several app hosts may share one PostgreSQL (behind PGBouncer, say) and their clocks need
+  # not agree, so there we let the database tell the time. clock_timestamp() and not now(),
+  # because now() stays put for the duration of a transaction. SQLite runs inside our own
+  # process and would read the same clock as Time.now does, so it gets the Ruby time.
+  def self.now_sql(connection, plus_seconds: 0)
+    if postgres?(connection)
+      "((clock_timestamp() AT TIME ZONE 'UTC') + make_interval(secs => %s))" % connection.quote(plus_seconds.to_f)
+    else
+      connection.quote(Time.now.utc + plus_seconds)
+    end
+  end
+
+  # For the few places where ActiveRecord wants a Time and not an SQL expression. The epoch
+  # comes back as a plain number, so there is no timezone for anybody to reinterpret
+  def self.now(connection)
+    return Time.now.utc unless postgres?(connection)
+    Time.at(connection.select_value("SELECT EXTRACT(EPOCH FROM clock_timestamp())").to_r).utc
+  end
+
+  def self.postgres?(connection)
+    /^postgres/i.match?(connection.adapter_name)
+  end
+
+  class Store < Struct.new(:key, :model, :verify_lock_held)
     def lookup
-      model.where(idempotent_request_key: key).where("expire_at > ?", Time.now).first&.idempotent_response_payload
+      model.where(idempotent_request_key: key)
+        .where("expire_at > #{Idempo::ActiveRecordBackend.now_sql(model.connection)}")
+        .first&.idempotent_response_payload
     end
 
     def store(data:, ttl:)
       # MySQL does not support datetime with subsecont precision, so ceil() it is
-      expire_at = Time.now.utc + ttl.ceil
+      expire_at = Idempo::ActiveRecordBackend.now(model.connection) + ttl.ceil
       model.transaction do
+        # The DELETE comes first on purpose. On SQLite it takes the database write lock
+        # at the very start of the transaction, so the read done by verify_lock_held
+        # below is not a read-then-write upgrade - those cannot be resolved by waiting
+        # and fail with SQLITE_BUSY_SNAPSHOT instead.
         model.where(idempotent_request_key: key).delete_all
+
+        # With a lease lock our lock can expire while the request is still running - if
+        # app.call takes longer than the lease, another request may already have taken
+        # over generating this response. We then have no guarantee that our response was
+        # generated exclusively, so the only sensible thing to do is to skip the write
+        # entirely - including the DELETE, which is why this rolls back.
+        raise ActiveRecord::Rollback if verify_lock_held && !verify_lock_held.call
+
         model.create(idempotent_request_key: key, idempotent_response_payload: data, expire_at: expire_at)
       end
       true
@@ -58,8 +119,84 @@ class Idempo::ActiveRecordBackend
     end
   end
 
-  def initialize
+  # SQLite has no advisory locks - and barely any write concurrency - but it does give us
+  # atomic single-statement upserts, and that is all a lease lock needs. We insert a row
+  # carrying a random fencing token into `idempo_locks`; the unique index on the key is
+  # what makes the acquisition mutually exclusive. The row carries an expiry so a lock left
+  # behind by a crashed process can be taken over later, and the token ensures we only ever
+  # release - or write under - a lock we are still holding ourselves.
+  #
+  # This is the same scheme the RedisBackend uses (SET NX PX + token, delete by token),
+  # just expressed as one INSERT ... ON CONFLICT DO UPDATE ... WHERE, which the database
+  # applies atomically. Unlike an advisory lock this one is not bound to a connection,
+  # which also makes it usable through a transaction-pooling proxy such as PGBouncer.
+  #
+  # One instance serves every request of the process when passed via `lock:`, and requests
+  # with different keys run concurrently - so the tokens are kept per key.
+  class TokenLock
+    def initialize(ttl_seconds: LOCK_TTL_SECONDS)
+      @ttl_seconds = ttl_seconds
+      @tokens = {}
+      @tokens_mutex = Mutex.new
+    end
+
+    def acquire(connection, based_on_str)
+      token = SecureRandom.hex(16)
+      key = derive_lock_key(based_on_str)
+
+      # A single statement, so the database applies it atomically. It inserts when no lock
+      # row exists, overwrites the row when the previous lease has expired, and does
+      # nothing at all when the lock is currently held by somebody else.
+      sql = <<~SQL
+        INSERT INTO idempo_locks (idempotent_request_key, lock_token, expire_at)
+        VALUES (#{connection.quote(key)}, #{connection.quote(token)}, #{Idempo::ActiveRecordBackend.now_sql(connection, plus_seconds: @ttl_seconds)})
+        ON CONFLICT (idempotent_request_key) DO UPDATE SET
+          lock_token = excluded.lock_token,
+          expire_at = excluded.expire_at
+        WHERE idempo_locks.expire_at <= #{Idempo::ActiveRecordBackend.now_sql(connection)}
+      SQL
+      did_acquire = connection.exec_update(sql, "Idempo lock acquire") == 1
+      @tokens_mutex.synchronize { @tokens[key] = token } if did_acquire
+      did_acquire
+    end
+
+    def release(connection, based_on_str)
+      key = derive_lock_key(based_on_str)
+      token = @tokens_mutex.synchronize { @tokens.delete(key) } or return
+      connection.exec_delete(<<~SQL, "Idempo lock release")
+        DELETE FROM idempo_locks
+        WHERE idempotent_request_key = #{connection.quote(key)}
+          AND lock_token = #{connection.quote(token)}
+      SQL
+    end
+
+    # Tells whether the lease taken in `acquire` is still ours and has not expired yet
+    def held?(connection, based_on_str)
+      key = derive_lock_key(based_on_str)
+      token = @tokens_mutex.synchronize { @tokens[key] } or return false
+      row = connection.select_value(<<~SQL, "Idempo lock check")
+        SELECT 1 FROM idempo_locks
+        WHERE idempotent_request_key = #{connection.quote(key)}
+          AND lock_token = #{connection.quote(token)}
+          AND expire_at > #{Idempo::ActiveRecordBackend.now_sql(connection)}
+      SQL
+      !row.nil?
+    end
+
+    def derive_lock_key(from_str)
+      Digest::SHA1.base64digest(from_str)
+    end
+  end
+
+  # Pass `lock:` to override the lock implementation Idempo picks for your database - for
+  # example to use the TokenLock on PostgreSQL behind PGBouncer, where advisory locks are
+  # not usable. The object needs to respond to `acquire(connection, key)` and
+  # `release(connection, key)`, and may respond to `held?(connection, key)`. The same
+  # object serves every request, so any state it keeps must be safe to share between them.
+  def initialize(lock: nil)
     require "active_record"
+    require "securerandom"
+    @lock = lock
     @memory_lock = Idempo::MemoryLock.new
   end
 
@@ -70,6 +207,12 @@ class Idempo::ActiveRecordBackend
     end
   end
 
+  def lock_model
+    @lock_model_class ||= Class.new(ActiveRecord::Base) do
+      self.table_name = "idempo_locks"
+    end
+  end
+
   def with_idempotency_key(request_key)
     # We need to use an in-memory lock because database advisory locks are
     # reentrant. Both Postgres and MySQL allow multiple acquisitions of the
@@ -83,11 +226,14 @@ class Idempo::ActiveRecordBackend
     @memory_lock.with(request_key) do
       db_safe_key = Digest::SHA1.base64digest(request_key)
       ActiveRecord::Base.connection_pool.with_connection do |connection|
-        database_lock = lock_implementation_for_connection(connection)
+        database_lock = @lock || lock_implementation_for_connection(connection)
         raise Idempo::ConcurrentRequest unless database_lock.acquire(connection, request_key)
 
         begin
-          yield(Store.new(db_safe_key, model))
+          # Only lease-based locks can be lost while the request is running - an advisory
+          # lock is held by the connection for as long as we keep it
+          verify_lock_held = -> { database_lock.held?(connection, request_key) } if database_lock.respond_to?(:held?)
+          yield(Store.new(db_safe_key, model, verify_lock_held))
         ensure
           database_lock.release(connection, request_key)
         end
@@ -97,7 +243,12 @@ class Idempo::ActiveRecordBackend
 
   # Deletes expired cached Idempo responses from the database, in batches
   def prune!
-    model.where("expire_at < ?", Time.now).in_batches.delete_all
+    model.where("expire_at < #{self.class.now_sql(model.connection)}").in_batches.delete_all
+    # Locks for keys which are never requested again would linger otherwise. Guarded
+    # because the table may not exist yet on an installation which predates it.
+    if lock_model.table_exists?
+      lock_model.where("expire_at < #{self.class.now_sql(lock_model.connection)}").in_batches.delete_all
+    end
   end
 
   private
@@ -107,8 +258,10 @@ class Idempo::ActiveRecordBackend
       MysqlLock.new
     elsif /^postgres/i.match?(connection.adapter_name)
       PostgresLock.new
+    elsif /^sqlite/i.match?(connection.adapter_name)
+      TokenLock.new
     else
-      raise "Unsupported database driver #{model.connection.adapter_name.downcase} - we don't know whether it supports advisory locks"
+      raise "Unsupported database driver #{connection.adapter_name.downcase} - we don't know whether it supports advisory locks"
     end
   end
 end

data/lib/idempo/memory_lock.rb

@@ -14,8 +14,13 @@ class Idempo::MemoryLock
         @in_progress << request_key
       end
     end
-    yield
-  ensure
-    @requests_in_flight_mutex.synchronize { @in_progress.delete(request_key) }
+
+    # Only the request which got the key in may remove it again - a rejected request
+    # must leave the key of the request still in flight alone
+    begin
+      yield
+    ensure
+      @requests_in_flight_mutex.synchronize { @in_progress.delete(request_key) }
+    end
   end
 end

data/lib/idempo/response_store.rb

@@ -29,26 +29,26 @@ class Idempo::ResponseStore
     now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
     items_to_delete = remove_lower_than(@expiries, now, &:expire_at)
     items_to_delete.each do |expiry_handle|
-      @values.delete(expiry_handle.key) if @values[expiry_handle.key] && @values[expiry_handle.key].expire_at < now
+      @values.delete(expiry_handle.key) if @values[expiry_handle.key] && @values[expiry_handle.key].expire_at <= now
     end
   end
 
   private
 
-  def binary_insert(array, item, &property_getter)
+  def binary_insert(array, item)
     at_i = array.bsearch_index do |stored_item|
       yield(stored_item) <= yield(item)
     end
     at_i ? array.insert(at_i, item) : array.push(item)
   end
 
-  def remove_lower_than(array, threshold_value, &property_getter)
+  def remove_lower_than(array, threshold_value)
     at_i = array.bsearch_index do |stored_item|
       yield(stored_item) <= threshold_value
     end
     if at_i
       array[at_i..array.length].tap do |_deleted_items|
-        array.replace(array[0..at_i])
+        array.replace(array[0...at_i])
       end
     else
       []

data/lib/idempo/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 class Idempo
-  VERSION = "1.4.0"
+  VERSION = "1.5.1"
 end

metadata

@@ -1,15 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: idempo
 version: !ruby/object:Gem::Version
-  version: 1.4.0
+  version: 1.5.1
 platform: ruby
 authors:
 - Julik Tarkhanov
 - Pablo Crivella
-autorequire:
 bindir: exe
 cert_chain: []
-date: 2026-04-06 00:00:00.000000000 Z
+date: 2026-10-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
@@ -81,20 +80,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
-- !ruby/object:Gem::Dependency
-  name: redis
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '4'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '4'
 - !ruby/object:Gem::Dependency
   name: rack-test
   requirement: !ruby/object:Gem::Requirement
@@ -124,7 +109,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: mysql2
+  name: sqlite3
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -138,7 +123,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: pg
+  name: railties
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -155,30 +140,16 @@ dependencies:
   name: standard
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: appraisal
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.37.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.37.0
 description: Provides idempotency keys for Rack applications.
 email:
 - me@julik.nl
@@ -189,10 +160,9 @@ extra_rdoc_files: []
 files:
 - ".github/workflows/ci.yml"
 - ".gitignore"
-- ".rubocop.yml"
 - ".standard.yml"
-- Appraisals
 - CHANGELOG.md
+- CONTRIBUTING.md
 - Gemfile
 - LICENSE.txt
 - README.md
@@ -202,6 +172,8 @@ files:
 - examples/custom_locking.rb
 - examples/jwt_iss_fingerprint.rb
 - idempo.gemspec
+- lib/generators/idempo/install/install_generator.rb
+- lib/generators/idempo/install/templates/install_migration.rb.tt
 - lib/idempo.rb
 - lib/idempo/active_record_backend.rb
 - lib/idempo/concurrent_request_error_app.rb
@@ -220,7 +192,6 @@ metadata:
   homepage_uri: https://github.com/julik/idempo
   source_code_uri: https://github.com/julik/idempo
   changelog_uri: https://github.com/julik/idempo/blob/main/CHANGELOG.md
-post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -228,15 +199,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.4.0
+      version: 2.7.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.6
-signing_key:
+rubygems_version: 3.6.2
 specification_version: 4
 summary: Idempotency keys for all.
 test_files: []

data/.rubocop.yml

@@ -1,5 +0,0 @@
-inherit_gem:
-  wetransfer_style: ruby/default.yml
-
-AllCops:
-  TargetRubyVersion: 2.7

data/Appraisals

@@ -1,9 +0,0 @@
-appraise "rack-3" do
-  gem "rack", ">= 3.0"
-  gem "activerecord", "~> 7.0", "< 9.0"
-end
-
-appraise "rack-2" do
-  gem "rack", ">= 2.0", "< 3.0"
-  gem "activerecord", "~> 6", "< 7.0"
-end