idempo 1.3.1 → 1.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 93f7d90c5ffda4a513a27f7bc55ead7fd02ca35767135d726a4eaf42f9e71045
4
- data.tar.gz: 9ac0f66203734a0744d553eec1760342f89ee8f436f84297edbea002298ef7db
3
+ metadata.gz: 5fb87c2193d9db479455638fc1f1bf79b609aa7aa8a841f2c0b81d5a088bcb9f
4
+ data.tar.gz: a2007aed0a26af541eeca24e1ab88cf2ef091c2f2ebccace840303ac2bad3254
5
5
  SHA512:
6
- metadata.gz: f3f42ab5bc2c256ad672555fe0e1768421de19b8ee46af1d3d25548b467804227197e159cba8cfbb8397d6103b62aea49d5e187b856bd438603f85a9eeadc17c
7
- data.tar.gz: 2e701bfd31e53607820bbaae2bef63633e9c903db7be25406a8a84d9565ebf49d2903dc7afcb6c27e614b5625444e9071f2e9f1be894ddfac0a59b2933ceeee5
6
+ metadata.gz: 1ab67dcce139c705b486e15cc17c3a7ed9be838dd77b6bdfb635594167611afad682aa9f48ffcc6bb7d2fe1c3dcbb29388422bc3ac1c20873f795b400b0e3bfa
7
+ data.tar.gz: 5b292a1329d775ee223ee42c7b5cd2f707d810afd388c4323b7a4f8af51de28cd8101aaa75e302f1274874f4ccd36b29b20b43bfef13c46bea4728b53796cbc8
@@ -7,15 +7,44 @@ on:
7
7
  env:
8
8
  BUNDLE_PATH: vendor/bundle
9
9
 
10
+ # The suite has three independent dimensions - Ruby version, Rack major version, and which
11
+ # datastore a backend talks to - and they do not interact. Only `idempo_spec.rb` touches the
12
+ # Rack API at all; every backend spec is Rack-agnostic, and the datastore adapters care about
13
+ # neither Ruby nor Rack. So these are covered separately rather than as a cross product, and
14
+ # exactly one job needs database containers.
10
15
  jobs:
11
16
  lint:
12
17
  name: Code Style
13
18
  runs-on: ubuntu-22.04
14
19
  if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository
20
+ steps:
21
+ - name: Checkout
22
+ uses: actions/checkout@v4
23
+ - name: Setup Ruby
24
+ uses: ruby/setup-ruby@v1
25
+ with:
26
+ # The style target comes from .standard.yml, so the Ruby here does not matter
27
+ ruby-version: "3.4"
28
+ bundler-cache: true
29
+ - name: Standard (Lint)
30
+ run: bundle exec rake standard
31
+
32
+ standalone:
33
+ name: "Ruby ${{ matrix.ruby }}, Rack ${{ matrix.rack }}.x"
34
+ runs-on: ubuntu-22.04
35
+ if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository
15
36
  strategy:
37
+ fail-fast: false
16
38
  matrix:
17
- ruby:
18
- - "2.7"
39
+ # The oldest Ruby the gemspec allows and the newest released one. Bundler resolves
40
+ # the newest Rails that runs on each, so this also covers ActiveRecord 7.1 and 8.1
41
+ # through the SQLite backend without naming a single Rails version anywhere.
42
+ ruby: ["2.7", "4.0"]
43
+ rack: ["2", "3"]
44
+ env:
45
+ RACK_VERSION: "~> ${{ matrix.rack }}.0"
46
+ # No database server is involved, so the native clients are not even installed
47
+ BUNDLE_WITHOUT: db_servers
19
48
  steps:
20
49
  - name: Checkout
21
50
  uses: actions/checkout@v4
@@ -24,54 +53,53 @@ jobs:
24
53
  with:
25
54
  ruby-version: ${{ matrix.ruby }}
26
55
  bundler-cache: true
27
- - name: Rubocop Cache
28
- uses: actions/cache@v3
29
- with:
30
- path: ~/.cache/rubocop_cache
31
- key: ${{ runner.os }}-rubocop-${{ hashFiles('.rubocop.yml') }}
32
- restore-keys: |
33
- ${{ runner.os }}-rubocop-
34
- - name: Standard (Lint)
35
- run: bundle exec rake standard
36
- test:
37
- name: "Specs (Rack 2 and 3)"
56
+ # Both Rack variants share a Gemfile, so keep their bundles in separate caches
57
+ cache-version: rack-${{ matrix.rack }}
58
+ - name: RSpec (middleware, memory and SQLite backends, install generator)
59
+ run: bundle exec rake spec:standalone
60
+
61
+ servers:
62
+ name: "Database backend (${{ matrix.backend }})"
38
63
  runs-on: ubuntu-22.04
39
64
  if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository
65
+ # The only jobs that need containers, and the only ones that run these specs. The
66
+ # adapters are independent of the Ruby and Rack dimensions covered above, so newest
67
+ # of everything is enough here. The backends do not depend on each other either, so
68
+ # each runs in its own job - a service whose image is blank does not get started.
40
69
  strategy:
70
+ fail-fast: false
41
71
  matrix:
42
- ruby:
43
- - "2.7"
72
+ backend: [mysql, postgres, redis]
44
73
  services:
45
74
  mysql:
46
- image: mysql:5.7
75
+ image: ${{ matrix.backend == 'mysql' && 'mysql:5.7' || '' }}
47
76
  env:
48
77
  MYSQL_ALLOW_EMPTY_PASSWORD: yes
49
78
  ports:
50
79
  - 3306:3306
51
80
  options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3
52
81
  postgres:
53
- image: postgres
82
+ image: ${{ matrix.backend == 'postgres' && 'postgres' || '' }}
54
83
  env:
55
84
  POSTGRES_PASSWORD: postgres
56
85
  options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
57
86
  ports:
58
87
  - 5432:5432
59
88
  redis:
60
- image: redis
89
+ image: ${{ matrix.backend == 'redis' && 'redis' || '' }}
61
90
  options: --health-cmd "redis-cli ping" --health-interval 10s --health-timeout 5s --health-retries 5
62
91
  ports:
63
92
  - 6379:6379
64
93
  steps:
65
94
  - name: Checkout
66
95
  uses: actions/checkout@v4
67
-
68
96
  - name: Setup Ruby
69
97
  uses: ruby/setup-ruby@v1
70
98
  with:
71
- ruby-version: ${{ matrix.ruby }}
99
+ ruby-version: "4.0"
72
100
  bundler-cache: true
73
- - name: RSpec via Appraisal
74
- run: "bundle exec appraisal install && bundle exec appraisal rspec"
101
+ - name: RSpec (${{ matrix.backend }})
102
+ run: bundle exec rake spec:${{ matrix.backend }}
75
103
  env:
76
104
  MYSQL_HOST: 127.0.0.1
77
105
  MYSQL_PORT: 3306
data/CHANGELOG.md CHANGED
@@ -1,3 +1,71 @@
1
+ ## Unreleased
2
+
3
+ ## 1.5.1
4
+
5
+ - Raise `required_ruby_version` from 2.4 to 2.7. Idempo has needed Ruby 2.6 for a while (it uses an endless
6
+ range, and `Struct.new(..., keyword_init: true)` which arrived in 2.5), so installing it on 2.4 or 2.5 succeeded
7
+ and then failed to load. 2.7 is the oldest Ruby CI runs against, so it is the oldest version we actually know works.
8
+ - Fix an off-by-one in `ResponseStore#prune` (used by `MemoryBackend`): the expiry handle at the pruning
9
+ boundary was deleted but also kept, so one already-expired handle stayed in the store and was pruned
10
+ again on every subsequent call. It also no longer leaks a response whose `expire_at` equals the prune time:
11
+ its expiry handle was removed but the response itself was kept, and stayed in memory until its key was
12
+ looked up or overwritten.
13
+ - Add SQLite support to `ActiveRecordBackend`. SQLite has no advisory locks, so the backend now ships a
14
+ `TokenLock` - a lease lock built on an atomic `INSERT ... ON CONFLICT DO UPDATE ... WHERE` into a new
15
+ `idempo_locks` table, with a random fencing token and an expiry. It is the same scheme the `RedisBackend`
16
+ uses, and it recovers by itself from a process killed while holding a lock. MySQL and PostgreSQL keep
17
+ using advisory locks and are unaffected.
18
+ - Add a `rails g idempo:install` generator. It detects whether `idempo_responses` already exists and
19
+ generates a migration for only the tables you are missing, so existing installations get a migration
20
+ which adds `idempo_locks` alone.
21
+ - `ActiveRecordBackend.create_table` is unchanged and still creates only `idempo_responses` - migrations
22
+ already committed against earlier versions of Idempo keep behaving exactly as before. The table
23
+ definitions are now also available individually as `create_responses_table` and `create_locks_table`.
24
+ - `ActiveRecordBackend#prune!` also deletes abandoned lock rows.
25
+ - `ActiveRecordBackend.new` accepts a `lock:` argument to override the lock implementation. Passing
26
+ `TokenLock` makes the backend usable through a transaction-pooling proxy such as PGBouncer, where
27
+ connection-bound advisory locks are not.
28
+ - A request rejected with `ConcurrentRequest` no longer frees the in-memory lock of the request it
29
+ collided with, so a third request with the same key is rejected as well instead of running concurrently.
30
+ - `TokenLock` keeps its lease tokens per request key, so the one instance passed via `lock:` serves
31
+ concurrent requests with different keys - on Puma threads or Falcon fibers - without one request's
32
+ lease being lost, or its lock row left behind, when another request takes a lock in the meantime.
33
+ - A response is no longer written when the lock lease expired while the request was being served.
34
+ - On PostgreSQL `ActiveRecordBackend` takes the time from the database instead of from the Ruby process, so
35
+ app hosts with clocks that disagree - all talking to one database through PGBouncer - agree on when a
36
+ `TokenLock` lease and a stored response expire. `prune!` uses the same clock. SQLite keeps using the Ruby
37
+ time, since it runs inside the process and would read the very same clock anyway.
38
+ - Test against the latest released Rails instead of pinning old versions. Every Rails in the
39
+ previous matrix (6.1 and 7.1) had been end of life for a year or more, while the current
40
+ release was not covered at all. No Rails version is named anywhere now - Bundler resolves
41
+ the newest one that runs on the Ruby under test.
42
+ - CI runs on Ruby 2.7 (the oldest the gemspec allows) and Ruby 4.0. On 2.7 Bundler resolves the
43
+ newest Rails that still runs there (7.1), on 4.0 it resolves the current release.
44
+ - Drop the `appraisal` development dependency along with the `Appraisals` file and the
45
+ generated `gemfiles/` directory. Rack is pinned through a `RACK_VERSION` environment
46
+ variable read by the Gemfile instead. The checked-in appraisal lockfiles also pinned Bundler
47
+ to the version that generated them (2.1.4), which cannot run on Ruby 3.x or 4.x.
48
+ - Split the suite into `rake spec:standalone` (needs no database server) and
49
+ `rake spec:servers` (needs MySQL, PostgreSQL or Redis), so the specs which need containers
50
+ run once rather than once per Ruby and Rack combination. `spec:mysql`, `spec:postgres` and
51
+ `spec:redis` run one server's specs each, and CI runs those in parallel, every job starting
52
+ only the container it needs.
53
+ - Pin `standard` to 1.37.0, the last version which installs on Ruby 2.7, so every Ruby in CI lints with the same
54
+ rules. Remove the unused `.rubocop.yml`, which inherited from `wetransfer_style` - a gem Idempo does not depend on.
55
+ - Add a `CONTRIBUTING.md`.
56
+
57
+ Note that ActiveRecord is not a runtime dependency of Idempo - it is required lazily by
58
+ `ActiveRecordBackend`. Nothing in this release changes which databases or Rails versions the
59
+ backend can work with, only which combinations are verified.
60
+
61
+ ## 1.4.0
62
+
63
+ - `RequestFingerprint` is now a class instead of a module, with an overridable `extract_user_identity` method.
64
+ The default implementation uses the `Authorization` header when present, and falls back to the Rails session
65
+ cookie (`_<appname>_session`) when it is not. This prevents cross-user response leakage for apps using
66
+ cookie-based authentication. For custom auth mechanisms, subclass `RequestFingerprint` and override
67
+ `extract_user_identity`. Backward compatible — the class still works as the default `compute_fingerprint_via:` value.
68
+
1
69
  ## 1.3.1
2
70
 
3
71
  - Instead of retaining the ActiveRecord connection in Idempo operations, check it out temporarily from the AR pool.
data/CONTRIBUTING.md ADDED
@@ -0,0 +1,80 @@
1
+ # Contributing to Idempo
2
+
3
+ Contributions are welcome! Please take a moment to review this document so that
4
+ contributing stays easy and effective for everyone involved.
5
+
6
+ I'd especially love to get:
7
+
8
+ * Reproductions of races and double execution - ideally as a deterministic spec
9
+ * Fixes from people running a backend in production (PGBouncer, MySQL proxies, SQLite under load)
10
+ * Documentation fixes
11
+
12
+ ## What do I need to know to help?
13
+
14
+ If you would like to learn more about the technologies Idempo builds on, check out the (non-exhaustive) list below. You can also get in touch with me via an issue or email.
15
+
16
+ - [ruby](https://ruby-doc.org) and the [Rack specification](https://github.com/rack/rack/blob/main/SPEC.rdoc)
17
+ - [Designing robust and predictable APIs with idempotency](https://stripe.com/blog/idempotency) by Brandur Leach - why idempotency keys exist at all
18
+ - [Implementing Stripe-like Idempotency Keys in Postgres](https://brandur.org/idempotency-keys) and [Using Atomic Transactions to Power an Idempotent API](https://brandur.org/http-transactions), also by Brandur - how to build them on a database
19
+ - [The Idempotency-Key HTTP Header Field](https://datatracker.ietf.org/doc/draft-ietf-httpapi-idempotency-key-header/), the IETF draft Idempo follows for the header (the draft has since expired)
20
+ - Advisory locks in [PostgreSQL](https://www.postgresql.org/docs/current/explicit-locking.html#ADVISORY-LOCKS) and [MySQL](https://dev.mysql.com/doc/refman/8.4/en/locking-functions.html), and [UPSERT in SQLite](https://www.sqlite.org/lang_upsert.html), which the `TokenLock` is built on
21
+ - [rspec](https://rspec.info) for testing
22
+
23
+ ## Issues and pull requests
24
+
25
+ * Opening an issue first is often the fastest way to get something in. I can shape it into something that fits the library.
26
+ * The most useful bug report for a race is the exact order of events across requests, or better yet a spec which forces that order (with `Queue`s or fibers) instead of hoping for it with `sleep`.
27
+ * I may close a PR and implement the idea differently. That's not a rejection of the idea, and you'll be credited (`Co-authored-by`).
28
+ * Feel free to propose opinionated changes, but be prepared that your opinion won't be shared. It is a give-and-take.
29
+
30
+ ## Code changes
31
+
32
+ Please adhere to the coding conventions used throughout the project (indentation,
33
+ accurate and terse comments, etc.) and any other requirements (such as test coverage).
34
+ Idempo supports Ruby 2.7 and newer, and Rack 2 and 3 - please don't use syntax or APIs
35
+ which rule either of those out.
36
+
37
+ Run `bundle exec rake standard` to make sure the code is properly formatted. The tests come in two parts:
38
+
39
+ * `bundle exec rake spec:standalone` needs no database server - the middleware, the memory and SQLite backends and the install generator. Set `BUNDLE_WITHOUT=db_servers` to skip installing the database clients, and `RACK_VERSION="~> 2.0"` to test against Rack 2.
40
+ * `bundle exec rake spec:mysql`, `spec:postgres` and `spec:redis` need the respective server running locally. `spec:servers` runs all three.
41
+
42
+ ## Basic expectations
43
+
44
+ This project is something created with love, and I - as an author - expect you, as a contributor, to return the favor. The ground rules:
45
+
46
+ * Do your best to be kind.
47
+ * Be polite and direct. No abrasiveness, no passive-aggression.
48
+ * [Low-context](https://www.ebsco.com/research-starters/communication-and-mass-media/high-context-and-low-context-cultures) communication is preferred and permitted
49
+ * No ad-hominems, also in code comments
50
+ * No sexualized or aggressive language
51
+
52
+ As a maintainer, I reserve the right to deny any contribution, discussion post, comment or pull request I find inappropriate.
53
+
54
+ Similarly, I reserve the right to not accept any contribution that doesn't fit the goals or spirit of the project (or my own), without explaining why it has been refused.
55
+
56
+ ## Decisions are final
57
+
58
+ You're welcome to make your case once, clearly. After that, please don't reopen closed issues, start meta-threads about the decision, or argue about the wording of this document. This is a list of expectations, not a contract to be lawyered. If something isn't covered, the maintainer's judgment applies.
59
+
60
+ ## What happens elsewhere stays elsewhere
61
+
62
+ Contributions are judged on what they are, not on who sent them. What anyone says or does outside this project (on social media, at their job, in other repos, in politics) is not a topic here and will not be considered, either for or against them. Please don't bring it in: no call-outs, no "are you aware this person…" threads, no requests to ban someone over things that didn't happen here.
63
+
64
+ ## AI-generated issues and code
65
+
66
+ This project uses LLMs to produce code and artifacts, and it's fine if you do the same when contributing. LLM use in this project is settled: threads about whether it should be used (utility, harm, societal implications etc.) will be closed.
67
+
68
+ Please disclose LLM use in every issue and PR: say which model or tool you used and what for (e.g. "Claude wrote the fix, I wrote the test"). This is for transparency, not for judgment. Disclosed LLM use is treated the same as no LLM use.
69
+
70
+ However, the following is on you:
71
+
72
+ * Make your LLM output _terse._ Nobody likes reading slop, even my agents. Make it get to the point.
73
+ * Make sure what you submit actually ran. Don't file issues about behavior or APIs your LLM imagined.
74
+ * Do not add external dependencies (or have your LLM do it) unless absolutely necessary
75
+ * If you file a PR, make at least one pass with "Make this at least 50% less code"
76
+ * Code generated by your LLM will be considered as written by you, with the same requirements
77
+
78
+ ## License
79
+
80
+ By submitting a patch, you agree to allow the project owner to license your work under the same license as that used by the project, which you can see [here](LICENSE.txt).
data/Gemfile CHANGED
@@ -3,3 +3,16 @@
3
3
  source "https://rubygems.org"
4
4
 
5
5
  gemspec
6
+
7
+ # Rack is the only runtime dependency whose major version we actively test against.
8
+ # CI pins this to exercise both Rack 2 and Rack 3; left unset it resolves to the newest.
9
+ gem "rack", ENV["RACK_VERSION"] if ENV["RACK_VERSION"]
10
+
11
+ # Native client libraries for the database servers Idempo supports. These are kept out of
12
+ # the gemspec development dependencies so that CI jobs which need no database server can
13
+ # skip them entirely.
14
+ group :db_servers do
15
+ gem "mysql2"
16
+ gem "pg"
17
+ gem "redis", "~> 4"
18
+ end
data/README.md CHANGED
@@ -7,7 +7,7 @@ instead of calling your application.
7
7
 
8
8
  ## Usage
9
9
 
10
- Idempo supports a number of backends, we recommend using Redis if you have multiple application servers / dynos and MemoryBackend if you are only using one single Puma worker. To initialize with Redis as backend pass the `backend:` parameter when adding the middleware:
10
+ Idempo supports a number of backends, we recommend using Redis if you have multiple application servers / dynos, the ActiveRecordBackend if you would rather not run Redis just for this (it works with MySQL, PostgreSQL and SQLite), and MemoryBackend if you are only using one single Puma worker. To initialize with Redis as backend pass the `backend:` parameter when adding the middleware:
11
11
 
12
12
  ```ruby
13
13
  be = Idempo::RedisBackend.new(Rails.application.config.redis_connection_pool)
@@ -47,22 +47,46 @@ Needless to say, if your server terminates or restarts all the data disappears w
47
47
 
48
48
  ## Using your database for idempotency keys (via ActiveRecord)
49
49
 
50
- The relational database you already have is a perfectly fine place to store idempotency key locks and responses. A requirement for that is that your database supports some form of advisory locking - both PostgreSQL and MySQL do. First you will need to create a table for the records. The table is going to be called `idempo_responses`, and you need to add a migration in your Rails project for it:
50
+ The relational database you already have is a perfectly fine place to store idempotency key locks and responses. Idempo supports MySQL, PostgreSQL and SQLite. Idempo stores its data in two tables - `idempo_responses` and `idempo_locks` - and ships a generator which creates the migration for them:
51
51
 
52
52
  ```bash
53
- $ rails g migration add_idempo_responses
53
+ $ rails g idempo:install
54
+ $ rails db:migrate
54
55
  ```
55
56
 
56
- and then add a migration like this:
57
+ If you are already using Idempo you will have the `idempo_responses` table from an earlier version. The generator detects this and writes a migration which only adds the table you are missing:
58
+
59
+ ```
60
+ $ rails g idempo:install
61
+ idempo idempo_responses is already present, creating idempo_locks only
62
+ create db/migrate/20240115120000_add_idempo_locks.rb
63
+ ```
64
+
65
+ Detection queries your database, and falls back to reading `db/schema.rb` (or `db/structure.sql`) when there is no database to connect to. You can always override it with `--locks-only` or `--no-locks-only`.
66
+
67
+ If you would rather write the migration by hand, the two table definitions are available separately. For a new installation:
57
68
 
58
69
  ```ruby
59
- class AddIdempoResponses < ActiveRecord::Migration[7.0]
70
+ class InstallIdempo < ActiveRecord::Migration[7.0]
60
71
  def change
61
- Idempo::ActiveRecordBackend.create_table(self)
72
+ Idempo::ActiveRecordBackend.create_responses_table(self)
73
+ Idempo::ActiveRecordBackend.create_locks_table(self)
62
74
  end
63
75
  end
64
76
  ```
65
77
 
78
+ and for an installation which already has `idempo_responses`:
79
+
80
+ ```ruby
81
+ class AddIdempoLocks < ActiveRecord::Migration[7.0]
82
+ def change
83
+ Idempo::ActiveRecordBackend.create_locks_table(self)
84
+ end
85
+ end
86
+ ```
87
+
88
+ `Idempo::ActiveRecordBackend.create_table` still creates just `idempo_responses`, exactly as it did in earlier versions - migrations you have already committed keep working and keep meaning the same thing.
89
+
66
90
  Then configure Idempo to use the backend (in your `application.rb`):
67
91
 
68
92
  ```ruby
@@ -70,13 +94,52 @@ be = Idempo::ActiveRecordBackend.new
70
94
  config.middleware.insert Idempo, backend: be
71
95
  ```
72
96
 
73
- In your regular tasks (cron or Rake) you will want to add a call to delete old Idempo responses (there is an index on `expire_at`):
97
+ In your regular tasks (cron or Rake) you will want to add a call to delete old Idempo responses and abandoned locks (both tables have an index on `expire_at`):
74
98
 
75
99
  ```ruby
76
100
  Idempo::ActiveRecordBackend.new.prune!
77
101
  ```
78
102
 
79
- If you need to use Idempo with PGBouncer you will need to write your own locking implementation based on fencing tokens or similar.
103
+ ### How locking works per database
104
+
105
+ Idempo picks the locking strategy from your database adapter:
106
+
107
+ * **MySQL** uses `GET_LOCK` / `RELEASE_LOCK` advisory locks
108
+ * **PostgreSQL** uses `pg_try_advisory_lock` / `pg_advisory_unlock` advisory locks
109
+ * **SQLite** has no advisory locks, so it uses the `TokenLock` - see below
110
+
111
+ Advisory locks are held by the database connection, so they are released the moment the connection goes away - which makes them ideal, but also makes them unusable through a transaction-pooling proxy such as PGBouncer. If that is your setup, pass the `TokenLock` explicitly:
112
+
113
+ ```ruby
114
+ be = Idempo::ActiveRecordBackend.new(lock: Idempo::ActiveRecordBackend::TokenLock.new)
115
+ ```
116
+
117
+ ### Using SQLite
118
+
119
+ SQLite has no advisory locks and only one writer at a time, but it does apply a single `INSERT ... ON CONFLICT DO UPDATE ... WHERE` statement atomically - and that is all a lease lock needs. The `TokenLock` inserts a row with a random fencing token into `idempo_locks`; the unique index on the request key is what makes the acquisition mutually exclusive. It is the same scheme the `RedisBackend` uses (`SET NX PX` plus a token), only expressed in SQL:
120
+
121
+ * Acquiring the lock is one statement which inserts when no lock row exists, overwrites the row when the previous lease has expired, and does nothing at all when somebody else holds the lock. Idempo then raises `ConcurrentRequest`, exactly like it does for a contended advisory lock.
122
+ * The lock row carries an expiry (`Idempo::ActiveRecordBackend::LOCK_TTL_SECONDS`, 5 minutes by default), so a lock left behind by a killed process gets taken over by a later request instead of wedging that idempotency key forever.
123
+ * Releasing deletes only a row still carrying our own token, so we never release a lock somebody else has taken over.
124
+ * If the lease does expire while your `app.call` is still running, the response is not written at all - another request may have generated a different response under the same key in the meantime, and overwriting it would be worse than not caching.
125
+
126
+ Nothing beyond the migration is required, but two settings in your `database.yml` are worth having:
127
+
128
+ ```yaml
129
+ production:
130
+ adapter: sqlite3
131
+ database: storage/production.sqlite3
132
+ timeout: 5000 # Wait up to 5s for the write lock instead of raising SQLITE_BUSY
133
+ ```
134
+
135
+ and WAL mode, so that reading a cached response does not block on the process currently writing one:
136
+
137
+ ```ruby
138
+ # config/initializers/sqlite.rb, if your Rails version does not do this already
139
+ ActiveRecord::Base.connection.execute("PRAGMA journal_mode=WAL") if ActiveRecord::Base.connection.adapter_name.match?(/sqlite/i)
140
+ ```
141
+
142
+ Rails 7.1 and newer set `journal_mode=WAL` for new applications by default, and starting with Rails 7.1 the SQLite adapter also opens write transactions with `BEGIN IMMEDIATE`. Idempo does not depend on either - it orders the statements in its write transaction so that the write lock is taken by the first statement - but WAL will make a meaningful difference to your throughput.
80
143
 
81
144
  ## Using Redis for idempotency keys
82
145
 
data/Rakefile CHANGED
@@ -3,11 +3,28 @@
3
3
  require "bundler/gem_tasks"
4
4
  require "rspec/core/rake_task"
5
5
  require "standard/rake"
6
- require "appraisal"
7
6
 
8
- RSpec::Core::RakeTask.new(:spec)
9
- task default: :spec
7
+ SERVER_BACKENDS = %w[mysql postgres redis]
8
+ SERVER_BACKED_SPECS = "spec/**/*{#{SERVER_BACKENDS.join(",")}}*_spec.rb"
9
+
10
+ # Everything that needs no database server: the middleware itself, the memory backend,
11
+ # the SQLite backend and the install generator. This is the bulk of the suite.
12
+ RSpec::Core::RakeTask.new("spec:standalone") do |t|
13
+ t.rspec_opts = %(--exclude-pattern "#{SERVER_BACKED_SPECS.sub("spec/", "")}")
14
+ end
15
+
16
+ # Only the specs which need MySQL, PostgreSQL or Redis to be running.
17
+ RSpec::Core::RakeTask.new("spec:servers") do |t|
18
+ t.pattern = SERVER_BACKED_SPECS
19
+ end
10
20
 
11
- if !ENV["APPRAISAL_INITIALIZED"]
12
- task default: :appraisal
21
+ # One per server, so that CI can run each in its own job with just that one container
22
+ SERVER_BACKENDS.each do |backend|
23
+ RSpec::Core::RakeTask.new("spec:#{backend}") do |t|
24
+ t.pattern = "spec/**/*#{backend}*_spec.rb"
25
+ end
13
26
  end
27
+
28
+ RSpec::Core::RakeTask.new(:spec)
29
+
30
+ task default: :spec
@@ -2,26 +2,15 @@
2
2
  # which includes some form of expiration. This means that every time a request is made, the `Authorization`
3
3
  # HTTP header may have a different value, and thus the request fingerprint could change every time,
4
4
  # even though the idempotency key is the same.
5
- # For this case, a custom fingerprinting function can be used. For example, if the bearer token is
6
- # generated in JWT format by the client, it may include the `iss` (issuer) claim, identifying the
7
- # specific device. This identifier can then be used instead of the entire Authorization header.
5
+ # For this case, you can subclass Idempo::RequestFingerprint and override `extract_user_identity`.
6
+ # For example, if the bearer token is generated in JWT format by the client, it may include the
7
+ # `iss` (issuer) claim, identifying the specific device. This identifier can then be used instead
8
+ # of the entire Authorization header.
8
9
 
9
- module FingerprinterWithIssuerClaim
10
- def self.call(idempotency_key, rack_request)
11
- d = Digest::SHA256.new
12
- d << idempotency_key << "\n"
13
- d << rack_request.url << "\n"
14
- d << rack_request.request_method << "\n"
15
- d << extract_jwt_iss_claim(rack_request) << "\n"
16
- while (chunk = rack_request.env["rack.input"].read(1024 * 65))
17
- d << chunk
18
- end
19
- Base64.strict_encode64(d.digest)
20
- ensure
21
- rack_request.env["rack.input"].rewind
22
- end
10
+ class JwtIssFingerprint < Idempo::RequestFingerprint
11
+ private
23
12
 
24
- def self.extract_jwt_iss_claim(rack_request)
13
+ def extract_user_identity(rack_request)
25
14
  header_value = rack_request.get_header("HTTP_AUTHORIZATION").to_s
26
15
  return header_value unless header_value.start_with?("Bearer ")
27
16
 
@@ -39,3 +28,5 @@ module FingerprinterWithIssuerClaim
39
28
  SecureRandom.bytes(32)
40
29
  end
41
30
  end
31
+
32
+ use Idempo, compute_fingerprint_via: JwtIssFingerprint.new
data/idempo.gemspec CHANGED
@@ -12,7 +12,7 @@ Gem::Specification.new do |spec|
12
12
  spec.description = "Provides idempotency keys for Rack applications."
13
13
  spec.homepage = "https://github.com/julik/idempo"
14
14
  spec.license = "MIT"
15
- spec.required_ruby_version = Gem::Requirement.new(">= 2.4.0")
15
+ spec.required_ruby_version = Gem::Requirement.new(">= 2.7.0")
16
16
 
17
17
  # Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
18
18
  # to allow pushing to a single host or delete this section to allow pushing to any host.
@@ -29,7 +29,7 @@ Gem::Specification.new do |spec|
29
29
  # Specify which files should be added to the gem when it is released.
30
30
  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
31
31
  spec.files = Dir.chdir(File.expand_path(__dir__)) do
32
- `git ls-files -z`.split("\x0").reject { |f| f.match(%r{\A(?:test|spec|features|gemfiles)/}) }
32
+ `git ls-files -z`.split("\x0").reject { |f| f.match(%r{\A(?:test|spec|features)/}) }
33
33
  end
34
34
  spec.bindir = "exe"
35
35
  spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
@@ -41,13 +41,11 @@ Gem::Specification.new do |spec|
41
41
 
42
42
  spec.add_development_dependency "rake", "~> 13.0"
43
43
  spec.add_development_dependency "rspec", "~> 3.0"
44
- spec.add_development_dependency "redis", "~> 4"
45
44
  spec.add_development_dependency "rack-test"
46
45
  spec.add_development_dependency "activerecord"
47
- spec.add_development_dependency "mysql2"
48
- spec.add_development_dependency "pg"
49
- spec.add_development_dependency "standard"
50
- spec.add_development_dependency "appraisal"
46
+ spec.add_development_dependency "sqlite3"
47
+ spec.add_development_dependency "railties"
48
+ spec.add_development_dependency "standard", "1.37.0" # The last version which installs on Ruby 2.7
51
49
 
52
50
  # For more information and examples about making a new gem, checkout our
53
51
  # guide at: https://bundler.io/guides/creating_gem.html
@@ -0,0 +1,80 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "rails/generators"
4
+ require "rails/generators/active_record"
5
+
6
+ # Note this is `class Idempo` and not `module Idempo` - Idempo is a class, and reopening
7
+ # it as a module raises a TypeError once lib/idempo.rb has been loaded
8
+ class Idempo
9
+ module Generators
10
+ # Generates the migration for the Idempo tables. Existing installations already have
11
+ # `idempo_responses` from an earlier version of Idempo and only need `idempo_locks`,
12
+ # so the generator detects which of the two situations it is in.
13
+ class InstallGenerator < Rails::Generators::Base
14
+ include ActiveRecord::Generators::Migration
15
+
16
+ source_root File.expand_path("templates", __dir__)
17
+
18
+ desc "Creates a migration for the Idempo tables (idempo_responses and idempo_locks)"
19
+
20
+ class_option :locks_only, type: :boolean, default: nil,
21
+ desc: "Only create idempo_locks (default: detect whether idempo_responses already exists)"
22
+
23
+ def create_migration_file
24
+ say_status :idempo, migration_explanation
25
+ migration_template "install_migration.rb.tt", File.join(db_migrate_path, "#{migration_name}.rb")
26
+ end
27
+
28
+ private
29
+
30
+ def migration_name
31
+ locks_only? ? "add_idempo_locks" : "install_idempo"
32
+ end
33
+
34
+ def migration_class_name
35
+ migration_name.camelize
36
+ end
37
+
38
+ def migration_explanation
39
+ if !options[:locks_only].nil?
40
+ locks_only? ? "creating idempo_locks only (--locks-only)" : "creating both Idempo tables (--no-locks-only)"
41
+ elsif locks_only?
42
+ "idempo_responses is already present, creating idempo_locks only"
43
+ else
44
+ "creating both Idempo tables"
45
+ end
46
+ end
47
+
48
+ def locks_only?
49
+ return options[:locks_only] unless options[:locks_only].nil?
50
+ @locks_only = responses_table_already_present? if !defined?(@locks_only)
51
+ @locks_only
52
+ end
53
+
54
+ # An existing installation may not have a database we can connect to at the moment
55
+ # the generator runs (a fresh checkout, or a CI box), so fall back to looking at the
56
+ # schema dump before assuming this is a fresh install.
57
+ def responses_table_already_present?
58
+ connection_says_responses_table_exists?
59
+ rescue => e
60
+ say_status :idempo, "could not query the database (#{e.class}), reading the schema dump instead", :yellow
61
+ schema_dump_mentions_responses_table?
62
+ end
63
+
64
+ def connection_says_responses_table_exists?
65
+ ActiveRecord::Base.connection.table_exists?("idempo_responses")
66
+ end
67
+
68
+ def schema_dump_mentions_responses_table?
69
+ %w[db/schema.rb db/structure.sql].any? do |relative_path|
70
+ path = File.expand_path(relative_path, destination_root)
71
+ File.exist?(path) && File.read(path).include?("idempo_responses")
72
+ end
73
+ end
74
+
75
+ def migration_version
76
+ "[%d.%d]" % [ActiveRecord::VERSION::MAJOR, ActiveRecord::VERSION::MINOR]
77
+ end
78
+ end
79
+ end
80
+ end
@@ -0,0 +1,8 @@
1
+ class <%= migration_class_name %> < ActiveRecord::Migration<%= migration_version %>
2
+ def change
3
+ <% unless locks_only? -%>
4
+ Idempo::ActiveRecordBackend.create_responses_table(self)
5
+ <% end -%>
6
+ Idempo::ActiveRecordBackend.create_locks_table(self)
7
+ end
8
+ end
@@ -1,6 +1,19 @@
1
- # This backend currently only works with mysql2 since it uses advisory locks
1
+ # This backend uses advisory locks where the database provides them (MySQL and PostgreSQL)
2
+ # and a lock table with fencing tokens where it does not (SQLite)
2
3
  class Idempo::ActiveRecordBackend
4
+ # The TTL for a lease taken by the TokenLock. It only comes into play if the process
5
+ # holding the lock crashes or gets killed without releasing it - the lease is what
6
+ # allows a subsequent request to take the lock over.
7
+ LOCK_TTL_SECONDS = 5 * 60
8
+
9
+ # Kept as-is for backwards compatibility: migrations generated by earlier versions of
10
+ # Idempo call this, and they must keep creating exactly the one table they always did.
11
+ # A fresh install gets both tables via `rails g idempo:install`.
3
12
  def self.create_table(via_migration)
13
+ create_responses_table(via_migration)
14
+ end
15
+
16
+ def self.create_responses_table(via_migration)
4
17
  via_migration.create_table "idempo_responses", charset: "utf8mb4", collation: "utf8mb4_unicode_ci" do |t|
5
18
  t.string :idempotent_request_key, index: {unique: true}, null: false
6
19
  t.datetime :expire_at, index: true, null: false # Needs an index for cleanup
@@ -9,16 +22,64 @@ class Idempo::ActiveRecordBackend
9
22
  end
10
23
  end
11
24
 
12
- class Store < Struct.new(:key, :model)
25
+ # Only databases without advisory locks (SQLite) actually use this table, but having it
26
+ # on MySQL and PostgreSQL costs nothing - it just stays empty, and it means switching
27
+ # databases later needs no schema change.
28
+ def self.create_locks_table(via_migration)
29
+ via_migration.create_table "idempo_locks", charset: "utf8mb4", collation: "utf8mb4_unicode_ci" do |t|
30
+ t.string :idempotent_request_key, index: {unique: true}, null: false
31
+ t.string :lock_token, null: false
32
+ t.datetime :expire_at, index: true, null: false # Needs an index for cleanup
33
+ end
34
+ end
35
+
36
+ # Several app hosts may share one PostgreSQL (behind PGBouncer, say) and their clocks need
37
+ # not agree, so there we let the database tell the time. clock_timestamp() and not now(),
38
+ # because now() stays put for the duration of a transaction. SQLite runs inside our own
39
+ # process and would read the same clock as Time.now does, so it gets the Ruby time.
40
+ def self.now_sql(connection, plus_seconds: 0)
41
+ if postgres?(connection)
42
+ "((clock_timestamp() AT TIME ZONE 'UTC') + make_interval(secs => %s))" % connection.quote(plus_seconds.to_f)
43
+ else
44
+ connection.quote(Time.now.utc + plus_seconds)
45
+ end
46
+ end
47
+
48
+ # For the few places where ActiveRecord wants a Time and not an SQL expression. The epoch
49
+ # comes back as a plain number, so there is no timezone for anybody to reinterpret
50
+ def self.now(connection)
51
+ return Time.now.utc unless postgres?(connection)
52
+ Time.at(connection.select_value("SELECT EXTRACT(EPOCH FROM clock_timestamp())").to_r).utc
53
+ end
54
+
55
+ def self.postgres?(connection)
56
+ /^postgres/i.match?(connection.adapter_name)
57
+ end
58
+
59
+ class Store < Struct.new(:key, :model, :verify_lock_held)
13
60
  def lookup
14
- model.where(idempotent_request_key: key).where("expire_at > ?", Time.now).first&.idempotent_response_payload
61
+ model.where(idempotent_request_key: key)
62
+ .where("expire_at > #{Idempo::ActiveRecordBackend.now_sql(model.connection)}")
63
+ .first&.idempotent_response_payload
15
64
  end
16
65
 
17
66
  def store(data:, ttl:)
18
67
  # MySQL does not support datetime with subsecont precision, so ceil() it is
19
- expire_at = Time.now.utc + ttl.ceil
68
+ expire_at = Idempo::ActiveRecordBackend.now(model.connection) + ttl.ceil
20
69
  model.transaction do
70
+ # The DELETE comes first on purpose. On SQLite it takes the database write lock
71
+ # at the very start of the transaction, so the read done by verify_lock_held
72
+ # below is not a read-then-write upgrade - those cannot be resolved by waiting
73
+ # and fail with SQLITE_BUSY_SNAPSHOT instead.
21
74
  model.where(idempotent_request_key: key).delete_all
75
+
76
+ # With a lease lock our lock can expire while the request is still running - if
77
+ # app.call takes longer than the lease, another request may already have taken
78
+ # over generating this response. We then have no guarantee that our response was
79
+ # generated exclusively, so the only sensible thing to do is to skip the write
80
+ # entirely - including the DELETE, which is why this rolls back.
81
+ raise ActiveRecord::Rollback if verify_lock_held && !verify_lock_held.call
82
+
22
83
  model.create(idempotent_request_key: key, idempotent_response_payload: data, expire_at: expire_at)
23
84
  end
24
85
  true
@@ -58,8 +119,84 @@ class Idempo::ActiveRecordBackend
58
119
  end
59
120
  end
60
121
 
61
- def initialize
122
+ # SQLite has no advisory locks - and barely any write concurrency - but it does give us
123
+ # atomic single-statement upserts, and that is all a lease lock needs. We insert a row
124
+ # carrying a random fencing token into `idempo_locks`; the unique index on the key is
125
+ # what makes the acquisition mutually exclusive. The row carries an expiry so a lock left
126
+ # behind by a crashed process can be taken over later, and the token ensures we only ever
127
+ # release - or write under - a lock we are still holding ourselves.
128
+ #
129
+ # This is the same scheme the RedisBackend uses (SET NX PX + token, delete by token),
130
+ # just expressed as one INSERT ... ON CONFLICT DO UPDATE ... WHERE, which the database
131
+ # applies atomically. Unlike an advisory lock this one is not bound to a connection,
132
+ # which also makes it usable through a transaction-pooling proxy such as PGBouncer.
133
+ #
134
+ # One instance serves every request of the process when passed via `lock:`, and requests
135
+ # with different keys run concurrently - so the tokens are kept per key.
136
+ class TokenLock
137
+ def initialize(ttl_seconds: LOCK_TTL_SECONDS)
138
+ @ttl_seconds = ttl_seconds
139
+ @tokens = {}
140
+ @tokens_mutex = Mutex.new
141
+ end
142
+
143
+ def acquire(connection, based_on_str)
144
+ token = SecureRandom.hex(16)
145
+ key = derive_lock_key(based_on_str)
146
+
147
+ # A single statement, so the database applies it atomically. It inserts when no lock
148
+ # row exists, overwrites the row when the previous lease has expired, and does
149
+ # nothing at all when the lock is currently held by somebody else.
150
+ sql = <<~SQL
151
+ INSERT INTO idempo_locks (idempotent_request_key, lock_token, expire_at)
152
+ VALUES (#{connection.quote(key)}, #{connection.quote(token)}, #{Idempo::ActiveRecordBackend.now_sql(connection, plus_seconds: @ttl_seconds)})
153
+ ON CONFLICT (idempotent_request_key) DO UPDATE SET
154
+ lock_token = excluded.lock_token,
155
+ expire_at = excluded.expire_at
156
+ WHERE idempo_locks.expire_at <= #{Idempo::ActiveRecordBackend.now_sql(connection)}
157
+ SQL
158
+ did_acquire = connection.exec_update(sql, "Idempo lock acquire") == 1
159
+ @tokens_mutex.synchronize { @tokens[key] = token } if did_acquire
160
+ did_acquire
161
+ end
162
+
163
+ def release(connection, based_on_str)
164
+ key = derive_lock_key(based_on_str)
165
+ token = @tokens_mutex.synchronize { @tokens.delete(key) } or return
166
+ connection.exec_delete(<<~SQL, "Idempo lock release")
167
+ DELETE FROM idempo_locks
168
+ WHERE idempotent_request_key = #{connection.quote(key)}
169
+ AND lock_token = #{connection.quote(token)}
170
+ SQL
171
+ end
172
+
173
+ # Tells whether the lease taken in `acquire` is still ours and has not expired yet
174
+ def held?(connection, based_on_str)
175
+ key = derive_lock_key(based_on_str)
176
+ token = @tokens_mutex.synchronize { @tokens[key] } or return false
177
+ row = connection.select_value(<<~SQL, "Idempo lock check")
178
+ SELECT 1 FROM idempo_locks
179
+ WHERE idempotent_request_key = #{connection.quote(key)}
180
+ AND lock_token = #{connection.quote(token)}
181
+ AND expire_at > #{Idempo::ActiveRecordBackend.now_sql(connection)}
182
+ SQL
183
+ !row.nil?
184
+ end
185
+
186
+ def derive_lock_key(from_str)
187
+ Digest::SHA1.base64digest(from_str)
188
+ end
189
+ end
190
+
191
+ # Pass `lock:` to override the lock implementation Idempo picks for your database - for
192
+ # example to use the TokenLock on PostgreSQL behind PGBouncer, where advisory locks are
193
+ # not usable. The object needs to respond to `acquire(connection, key)` and
194
+ # `release(connection, key)`, and may respond to `held?(connection, key)`. The same
195
+ # object serves every request, so any state it keeps must be safe to share between them.
196
+ def initialize(lock: nil)
62
197
  require "active_record"
198
+ require "securerandom"
199
+ @lock = lock
63
200
  @memory_lock = Idempo::MemoryLock.new
64
201
  end
65
202
 
@@ -70,6 +207,12 @@ class Idempo::ActiveRecordBackend
70
207
  end
71
208
  end
72
209
 
210
+ def lock_model
211
+ @lock_model_class ||= Class.new(ActiveRecord::Base) do
212
+ self.table_name = "idempo_locks"
213
+ end
214
+ end
215
+
73
216
  def with_idempotency_key(request_key)
74
217
  # We need to use an in-memory lock because database advisory locks are
75
218
  # reentrant. Both Postgres and MySQL allow multiple acquisitions of the
@@ -83,11 +226,14 @@ class Idempo::ActiveRecordBackend
83
226
  @memory_lock.with(request_key) do
84
227
  db_safe_key = Digest::SHA1.base64digest(request_key)
85
228
  ActiveRecord::Base.connection_pool.with_connection do |connection|
86
- database_lock = lock_implementation_for_connection(connection)
229
+ database_lock = @lock || lock_implementation_for_connection(connection)
87
230
  raise Idempo::ConcurrentRequest unless database_lock.acquire(connection, request_key)
88
231
 
89
232
  begin
90
- yield(Store.new(db_safe_key, model))
233
+ # Only lease-based locks can be lost while the request is running - an advisory
234
+ # lock is held by the connection for as long as we keep it
235
+ verify_lock_held = -> { database_lock.held?(connection, request_key) } if database_lock.respond_to?(:held?)
236
+ yield(Store.new(db_safe_key, model, verify_lock_held))
91
237
  ensure
92
238
  database_lock.release(connection, request_key)
93
239
  end
@@ -97,7 +243,12 @@ class Idempo::ActiveRecordBackend
97
243
 
98
244
  # Deletes expired cached Idempo responses from the database, in batches
99
245
  def prune!
100
- model.where("expire_at < ?", Time.now).in_batches.delete_all
246
+ model.where("expire_at < #{self.class.now_sql(model.connection)}").in_batches.delete_all
247
+ # Locks for keys which are never requested again would linger otherwise. Guarded
248
+ # because the table may not exist yet on an installation which predates it.
249
+ if lock_model.table_exists?
250
+ lock_model.where("expire_at < #{self.class.now_sql(lock_model.connection)}").in_batches.delete_all
251
+ end
101
252
  end
102
253
 
103
254
  private
@@ -107,8 +258,10 @@ class Idempo::ActiveRecordBackend
107
258
  MysqlLock.new
108
259
  elsif /^postgres/i.match?(connection.adapter_name)
109
260
  PostgresLock.new
261
+ elsif /^sqlite/i.match?(connection.adapter_name)
262
+ TokenLock.new
110
263
  else
111
- raise "Unsupported database driver #{model.connection.adapter_name.downcase} - we don't know whether it supports advisory locks"
264
+ raise "Unsupported database driver #{connection.adapter_name.downcase} - we don't know whether it supports advisory locks"
112
265
  end
113
266
  end
114
267
  end
@@ -14,8 +14,13 @@ class Idempo::MemoryLock
14
14
  @in_progress << request_key
15
15
  end
16
16
  end
17
- yield
18
- ensure
19
- @requests_in_flight_mutex.synchronize { @in_progress.delete(request_key) }
17
+
18
+ # Only the request which got the key in may remove it again - a rejected request
19
+ # must leave the key of the request still in flight alone
20
+ begin
21
+ yield
22
+ ensure
23
+ @requests_in_flight_mutex.synchronize { @in_progress.delete(request_key) }
24
+ end
20
25
  end
21
26
  end
@@ -1,10 +1,20 @@
1
- module Idempo::RequestFingerprint
1
+ # frozen_string_literal: true
2
+
3
+ class Idempo::RequestFingerprint
4
+ RAILS_SESSION_COOKIE_PATTERN = /\A_[a-z0-9_]+_session\z/i
5
+
6
+ # Maintains backward compatibility: Idempo::RequestFingerprint can be passed
7
+ # directly as the compute_fingerprint_via: value (the default) since it responds to .call.
2
8
  def self.call(idempotency_key, rack_request)
9
+ new.call(idempotency_key, rack_request)
10
+ end
11
+
12
+ def call(idempotency_key, rack_request)
3
13
  d = Digest::SHA256.new
4
14
  d << idempotency_key << "\n"
5
15
  d << rack_request.url << "\n"
6
16
  d << rack_request.request_method << "\n"
7
- d << rack_request.get_header("HTTP_AUTHORIZATION").to_s << "\n"
17
+ d << extract_user_identity(rack_request).to_s << "\n"
8
18
 
9
19
  # Under Rack 3.0 the rack.input may or may not be rewindable (this is done to support
10
20
  # streaming HTTP request bodies). If we know a request body is rewindable we can read it
@@ -18,7 +28,62 @@ module Idempo::RequestFingerprint
18
28
  Base64.strict_encode64(d.digest)
19
29
  end
20
30
 
21
- def self.read_and_rewind(source_io, to_destination_io)
31
+ # Extracts a value identifying the user from the request. This value gets
32
+ # included in the request fingerprint hash. Without user identity in the
33
+ # fingerprint, two different users sending the same idempotency key to the
34
+ # same endpoint would receive each other's cached responses — leaking
35
+ # sensitive data across user boundaries (similar to the Railway CDN caching
36
+ # incident of March 2026, where responses keyed only on method+URL were
37
+ # served to the wrong users).
38
+ #
39
+ # The default implementation tries two strategies, in order:
40
+ #
41
+ # 1. If an Authorization header is present (Bearer token, Basic auth, etc.),
42
+ # its full value is used. This is the common case for API applications.
43
+ # Different tokens produce different fingerprints, so requests from
44
+ # different users are naturally separated.
45
+ #
46
+ # 2. If no Authorization header is present, we look for a Rails-style session
47
+ # cookie (matching the pattern `_<appname>_session`). This covers the
48
+ # common case of Rails applications using cookie-based authentication,
49
+ # where the Authorization header is typically empty for all users. The
50
+ # encrypted session cookie value differs per user session, so it serves
51
+ # as a user identity signal. The cookie value is stable from the client's
52
+ # perspective across retries (the client resends the same cookie string
53
+ # until it receives a Set-Cookie with a new value), which is what matters
54
+ # for idempotency — the retry sends the same fingerprint as the original.
55
+ #
56
+ # If neither signal is available (no Authorization header and no Rails
57
+ # session cookie), the fingerprint will only contain the idempotency key,
58
+ # URL, method, and body. This is acceptable for unauthenticated endpoints
59
+ # but DANGEROUS for authenticated endpoints using other identity mechanisms
60
+ # (custom headers like X-API-Key, non-Rails session cookies, etc.).
61
+ #
62
+ # To handle those cases, subclass and override this method:
63
+ #
64
+ # class MyFingerprint < Idempo::RequestFingerprint
65
+ # private
66
+ # def extract_user_identity(rack_request)
67
+ # rack_request.get_header("HTTP_X_API_KEY")
68
+ # end
69
+ # end
70
+ #
71
+ # use Idempo, compute_fingerprint_via: MyFingerprint.new
72
+ #
73
+ def extract_user_identity(rack_request)
74
+ auth = rack_request.get_header("HTTP_AUTHORIZATION").to_s
75
+ return auth unless auth.empty?
76
+ extract_rails_session_cookie(rack_request)
77
+ end
78
+
79
+ def extract_rails_session_cookie(rack_request)
80
+ rack_request.cookies.each do |name, value|
81
+ return value if name.match?(RAILS_SESSION_COOKIE_PATTERN)
82
+ end
83
+ nil
84
+ end
85
+
86
+ def read_and_rewind(source_io, to_destination_io)
22
87
  while (chunk = source_io.read(1024 * 65))
23
88
  to_destination_io << chunk
24
89
  end
@@ -29,26 +29,26 @@ class Idempo::ResponseStore
29
29
  now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
30
30
  items_to_delete = remove_lower_than(@expiries, now, &:expire_at)
31
31
  items_to_delete.each do |expiry_handle|
32
- @values.delete(expiry_handle.key) if @values[expiry_handle.key] && @values[expiry_handle.key].expire_at < now
32
+ @values.delete(expiry_handle.key) if @values[expiry_handle.key] && @values[expiry_handle.key].expire_at <= now
33
33
  end
34
34
  end
35
35
 
36
36
  private
37
37
 
38
- def binary_insert(array, item, &property_getter)
38
+ def binary_insert(array, item)
39
39
  at_i = array.bsearch_index do |stored_item|
40
40
  yield(stored_item) <= yield(item)
41
41
  end
42
42
  at_i ? array.insert(at_i, item) : array.push(item)
43
43
  end
44
44
 
45
- def remove_lower_than(array, threshold_value, &property_getter)
45
+ def remove_lower_than(array, threshold_value)
46
46
  at_i = array.bsearch_index do |stored_item|
47
47
  yield(stored_item) <= threshold_value
48
48
  end
49
49
  if at_i
50
50
  array[at_i..array.length].tap do |_deleted_items|
51
- array.replace(array[0..at_i])
51
+ array.replace(array[0...at_i])
52
52
  end
53
53
  else
54
54
  []
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  class Idempo
4
- VERSION = "1.3.1"
4
+ VERSION = "1.5.1"
5
5
  end
metadata CHANGED
@@ -1,15 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: idempo
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.3.1
4
+ version: 1.5.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Julik Tarkhanov
8
8
  - Pablo Crivella
9
- autorequire:
10
9
  bindir: exe
11
10
  cert_chain: []
12
- date: 2025-08-07 00:00:00.000000000 Z
11
+ date: 2026-10-20 00:00:00.000000000 Z
13
12
  dependencies:
14
13
  - !ruby/object:Gem::Dependency
15
14
  name: msgpack
@@ -81,20 +80,6 @@ dependencies:
81
80
  - - "~>"
82
81
  - !ruby/object:Gem::Version
83
82
  version: '3.0'
84
- - !ruby/object:Gem::Dependency
85
- name: redis
86
- requirement: !ruby/object:Gem::Requirement
87
- requirements:
88
- - - "~>"
89
- - !ruby/object:Gem::Version
90
- version: '4'
91
- type: :development
92
- prerelease: false
93
- version_requirements: !ruby/object:Gem::Requirement
94
- requirements:
95
- - - "~>"
96
- - !ruby/object:Gem::Version
97
- version: '4'
98
83
  - !ruby/object:Gem::Dependency
99
84
  name: rack-test
100
85
  requirement: !ruby/object:Gem::Requirement
@@ -124,7 +109,7 @@ dependencies:
124
109
  - !ruby/object:Gem::Version
125
110
  version: '0'
126
111
  - !ruby/object:Gem::Dependency
127
- name: mysql2
112
+ name: sqlite3
128
113
  requirement: !ruby/object:Gem::Requirement
129
114
  requirements:
130
115
  - - ">="
@@ -138,7 +123,7 @@ dependencies:
138
123
  - !ruby/object:Gem::Version
139
124
  version: '0'
140
125
  - !ruby/object:Gem::Dependency
141
- name: pg
126
+ name: railties
142
127
  requirement: !ruby/object:Gem::Requirement
143
128
  requirements:
144
129
  - - ">="
@@ -155,30 +140,16 @@ dependencies:
155
140
  name: standard
156
141
  requirement: !ruby/object:Gem::Requirement
157
142
  requirements:
158
- - - ">="
159
- - !ruby/object:Gem::Version
160
- version: '0'
161
- type: :development
162
- prerelease: false
163
- version_requirements: !ruby/object:Gem::Requirement
164
- requirements:
165
- - - ">="
166
- - !ruby/object:Gem::Version
167
- version: '0'
168
- - !ruby/object:Gem::Dependency
169
- name: appraisal
170
- requirement: !ruby/object:Gem::Requirement
171
- requirements:
172
- - - ">="
143
+ - - '='
173
144
  - !ruby/object:Gem::Version
174
- version: '0'
145
+ version: 1.37.0
175
146
  type: :development
176
147
  prerelease: false
177
148
  version_requirements: !ruby/object:Gem::Requirement
178
149
  requirements:
179
- - - ">="
150
+ - - '='
180
151
  - !ruby/object:Gem::Version
181
- version: '0'
152
+ version: 1.37.0
182
153
  description: Provides idempotency keys for Rack applications.
183
154
  email:
184
155
  - me@julik.nl
@@ -189,10 +160,9 @@ extra_rdoc_files: []
189
160
  files:
190
161
  - ".github/workflows/ci.yml"
191
162
  - ".gitignore"
192
- - ".rubocop.yml"
193
163
  - ".standard.yml"
194
- - Appraisals
195
164
  - CHANGELOG.md
165
+ - CONTRIBUTING.md
196
166
  - Gemfile
197
167
  - LICENSE.txt
198
168
  - README.md
@@ -202,6 +172,8 @@ files:
202
172
  - examples/custom_locking.rb
203
173
  - examples/jwt_iss_fingerprint.rb
204
174
  - idempo.gemspec
175
+ - lib/generators/idempo/install/install_generator.rb
176
+ - lib/generators/idempo/install/templates/install_migration.rb.tt
205
177
  - lib/idempo.rb
206
178
  - lib/idempo/active_record_backend.rb
207
179
  - lib/idempo/concurrent_request_error_app.rb
@@ -220,7 +192,6 @@ metadata:
220
192
  homepage_uri: https://github.com/julik/idempo
221
193
  source_code_uri: https://github.com/julik/idempo
222
194
  changelog_uri: https://github.com/julik/idempo/blob/main/CHANGELOG.md
223
- post_install_message:
224
195
  rdoc_options: []
225
196
  require_paths:
226
197
  - lib
@@ -228,15 +199,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
228
199
  requirements:
229
200
  - - ">="
230
201
  - !ruby/object:Gem::Version
231
- version: 2.4.0
202
+ version: 2.7.0
232
203
  required_rubygems_version: !ruby/object:Gem::Requirement
233
204
  requirements:
234
205
  - - ">="
235
206
  - !ruby/object:Gem::Version
236
207
  version: '0'
237
208
  requirements: []
238
- rubygems_version: 3.1.6
239
- signing_key:
209
+ rubygems_version: 3.6.2
240
210
  specification_version: 4
241
211
  summary: Idempotency keys for all.
242
212
  test_files: []
data/.rubocop.yml DELETED
@@ -1,5 +0,0 @@
1
- inherit_gem:
2
- wetransfer_style: ruby/default.yml
3
-
4
- AllCops:
5
- TargetRubyVersion: 2.7
data/Appraisals DELETED
@@ -1,9 +0,0 @@
1
- appraise "rack-3" do
2
- gem "rack", ">= 3.0"
3
- gem "activerecord", "~> 7.0", "< 9.0"
4
- end
5
-
6
- appraise "rack-2" do
7
- gem "rack", ">= 2.0", "< 3.0"
8
- gem "activerecord", "~> 6", "< 7.0"
9
- end