idempo 1.0.0 → 1.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/ci.yml +37 -59
- data/.rubocop.yml +3 -0
- data/.standard.yml +1 -0
- data/CHANGELOG.md +12 -0
- data/README.md +9 -3
- data/Rakefile +1 -0
- data/examples/custom_locking.rb +22 -0
- data/examples/jwt_iss_fingerprint.rb +41 -0
- data/idempo.gemspec +14 -14
- data/lib/idempo/active_record_backend.rb +17 -12
- data/lib/idempo/concurrent_request_error_app.rb +2 -2
- data/lib/idempo/malformed_key_error_app.rb +2 -2
- data/lib/idempo/memory_backend.rb +6 -2
- data/lib/idempo/redis_backend.rb +8 -4
- data/lib/idempo/request_fingerprint.rb +3 -3
- data/lib/idempo/response_store.rb +2 -2
- data/lib/idempo/version.rb +1 -1
- data/lib/idempo.rb +27 -27
- metadata +7 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 9bfbbd03a37973e1309f1c9d3a0aa1032cbfdfaae73f78f275a7fd9800e5e9ed
|
4
|
+
data.tar.gz: 54e1bb92e23d7003e7d4205e0964e94ab7288d342f2032fd4f7f2a280a2341f1
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 9be3229d0844358a342b5fcc65a885e96f0ae7ae73ebcfe91b74046ed7ddc3e90d836d2fbe047f22434f9bd15beda36d5f0d0eb5bf4f8f597996fa7006a00101
|
7
|
+
data.tar.gz: a9933869e79cd56229907bafe38b8c135357eb1e3b9ff8954e51f74f4a171b33dee8e5322a9178782c95e4686d96104d9ff4fbc4d801b5ff12066591cd68cd7b
|
data/.github/workflows/ci.yml
CHANGED
@@ -7,22 +7,10 @@ on:
|
|
7
7
|
env:
|
8
8
|
BUNDLE_PATH: vendor/bundle
|
9
9
|
|
10
|
-
services:
|
11
|
-
mysql:
|
12
|
-
image: mysql:5.7
|
13
|
-
env:
|
14
|
-
MYSQL_ALLOW_EMPTY_PASSWORD: yes
|
15
|
-
ports:
|
16
|
-
- 3306
|
17
|
-
options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3
|
18
|
-
redis:
|
19
|
-
image: redis
|
20
|
-
options: --health-cmd "redis-cli ping" --health-interval 10s --health-timeout 5s --health-retries 5
|
21
|
-
|
22
10
|
jobs:
|
23
11
|
lint:
|
24
12
|
name: Code Style
|
25
|
-
runs-on: ubuntu-
|
13
|
+
runs-on: ubuntu-22.04
|
26
14
|
if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository
|
27
15
|
strategy:
|
28
16
|
matrix:
|
@@ -30,73 +18,63 @@ jobs:
|
|
30
18
|
- '2.7'
|
31
19
|
steps:
|
32
20
|
- name: Checkout
|
33
|
-
uses: actions/checkout@
|
21
|
+
uses: actions/checkout@v4
|
34
22
|
- name: Setup Ruby
|
35
23
|
uses: ruby/setup-ruby@v1
|
36
24
|
with:
|
37
25
|
ruby-version: ${{ matrix.ruby }}
|
38
|
-
|
39
|
-
uses: actions/cache@v2
|
40
|
-
with:
|
41
|
-
path: Gemfile.lock
|
42
|
-
key: ${{ runner.os }}-gemlock-${{ matrix.ruby }}-${{ hashFiles('Gemfile', 'idempo.gemspec') }}
|
43
|
-
restore-keys: |
|
44
|
-
${{ runner.os }}-gemlock-${{ matrix.ruby }}-
|
45
|
-
- name: Bundle Cache
|
46
|
-
id: cache-gems
|
47
|
-
uses: actions/cache@v2
|
48
|
-
with:
|
49
|
-
path: vendor/bundle
|
50
|
-
key: ${{ runner.os }}-gems-${{ matrix.ruby }}-${{ hashFiles('Gemfile', 'Gemfile.lock', 'idempo.gemspec') }}
|
51
|
-
restore-keys: |
|
52
|
-
${{ runner.os }}-gems-${{ matrix.ruby }}-
|
53
|
-
${{ runner.os }}-gems-
|
54
|
-
- name: Bundle Install
|
55
|
-
if: steps.cache-gems.outputs.cache-hit != 'true'
|
56
|
-
run: bundle install --jobs 4 --retry 3
|
26
|
+
bundler-cache: true
|
57
27
|
- name: Rubocop Cache
|
58
|
-
uses: actions/cache@
|
28
|
+
uses: actions/cache@v3
|
59
29
|
with:
|
60
30
|
path: ~/.cache/rubocop_cache
|
61
31
|
key: ${{ runner.os }}-rubocop-${{ hashFiles('.rubocop.yml') }}
|
62
32
|
restore-keys: |
|
63
33
|
${{ runner.os }}-rubocop-
|
64
|
-
- name:
|
65
|
-
run: bundle exec
|
34
|
+
- name: Standard (Lint)
|
35
|
+
run: bundle exec rake standard
|
66
36
|
test:
|
67
37
|
name: Specs
|
68
|
-
runs-on: ubuntu-
|
38
|
+
runs-on: ubuntu-22.04
|
69
39
|
if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository
|
70
40
|
strategy:
|
71
41
|
matrix:
|
72
42
|
ruby:
|
73
|
-
- '2.
|
74
|
-
- '3.
|
43
|
+
- '2.7'
|
44
|
+
- '3.2'
|
45
|
+
services:
|
46
|
+
mysql:
|
47
|
+
image: mysql:5.7
|
48
|
+
env:
|
49
|
+
MYSQL_ALLOW_EMPTY_PASSWORD: yes
|
50
|
+
ports:
|
51
|
+
- 3306:3306
|
52
|
+
options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3
|
53
|
+
postgres:
|
54
|
+
image: postgres
|
55
|
+
env:
|
56
|
+
POSTGRES_PASSWORD: postgres
|
57
|
+
options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
|
58
|
+
ports:
|
59
|
+
- 5432:5432
|
60
|
+
redis:
|
61
|
+
image: redis
|
62
|
+
options: --health-cmd "redis-cli ping" --health-interval 10s --health-timeout 5s --health-retries 5
|
63
|
+
ports:
|
64
|
+
- 6379:6379
|
75
65
|
steps:
|
76
66
|
- name: Checkout
|
77
|
-
uses: actions/checkout@
|
67
|
+
uses: actions/checkout@v4
|
78
68
|
- name: Setup Ruby
|
79
69
|
uses: ruby/setup-ruby@v1
|
80
70
|
with:
|
81
71
|
ruby-version: ${{ matrix.ruby }}
|
82
|
-
|
83
|
-
uses: actions/cache@v2
|
84
|
-
with:
|
85
|
-
path: Gemfile.lock
|
86
|
-
key: ${{ runner.os }}-gemlock-${{ matrix.ruby }}-${{ hashFiles('Gemfile', 'idempo.gemspec') }}
|
87
|
-
restore-keys: |
|
88
|
-
${{ runner.os }}-gemlock-${{ matrix.ruby }}-
|
89
|
-
- name: Bundle Cache
|
90
|
-
id: cache-gems
|
91
|
-
uses: actions/cache@v2
|
92
|
-
with:
|
93
|
-
path: vendor/bundle
|
94
|
-
key: ${{ runner.os }}-gems-${{ matrix.ruby }}-${{ hashFiles('Gemfile', 'Gemfile.lock', 'idempo.gemspec') }}
|
95
|
-
restore-keys: |
|
96
|
-
${{ runner.os }}-gems-${{ matrix.ruby }}-
|
97
|
-
${{ runner.os }}-gems-
|
98
|
-
- name: Bundle Install
|
99
|
-
if: steps.cache-gems.outputs.cache-hit != 'true'
|
100
|
-
run: bundle install --jobs 4 --retry 3
|
72
|
+
bundler-cache: true
|
101
73
|
- name: RSpec
|
102
74
|
run: bundle exec rspec
|
75
|
+
env:
|
76
|
+
MYSQL_HOST: 127.0.0.1
|
77
|
+
MYSQL_PORT: 3306
|
78
|
+
PGHOST: localhost
|
79
|
+
PGUSER: postgres
|
80
|
+
PGPASSWORD: postgres
|
data/.rubocop.yml
CHANGED
data/.standard.yml
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
ruby_version: 2.7
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,15 @@
|
|
1
|
+
## [1.1.0] - 2024-02-22
|
2
|
+
|
3
|
+
- Use modern ActiveRecord migration options for better Rails 7.x compatibility
|
4
|
+
- Ensure Github actions CI can run and uses Postgres appropriately
|
5
|
+
- Add examples for more sophisticated use cases
|
6
|
+
- Implement `#prune!` on storage backends
|
7
|
+
- Reformat all code using [standard](https://github.com/standardrb/standard) instead of wetransfer_style as it is both more relaxed and more modern
|
8
|
+
|
9
|
+
## [1.0.0] - 2023-10-27
|
10
|
+
|
11
|
+
- Release 1.0 as the API can be considered stable and the gem has been in production for years
|
12
|
+
|
1
13
|
## [0.2.0] - 2022-04-08
|
2
14
|
|
3
15
|
- Allow setting the global default TTL for the cached responses
|
data/README.md
CHANGED
@@ -42,7 +42,7 @@ If you run only one Puma on one server (so multiple threads but one process) the
|
|
42
42
|
* It uses a `Set` with a `Mutex` around it to store requests in progress
|
43
43
|
* It uses a sorted array for expiration and cached responses.
|
44
44
|
|
45
|
-
Needless to say, if your server terminates or restarts all the data
|
45
|
+
Needless to say, if your server terminates or restarts all the data disappears with it. This backend will also only work if you are running one Puma process (or other single-process server, and just one instance of it).
|
46
46
|
|
47
47
|
## Using your database for idempotency keys (via ActiveRecord)
|
48
48
|
|
@@ -71,9 +71,11 @@ config.middleware.insert Idempo, backend: Idempo::ActiveRecordBackend.new
|
|
71
71
|
In your regular tasks (cron or Rake) you will want to add a call to delete old Idempo responses (there is an index on `expire_at`):
|
72
72
|
|
73
73
|
```ruby
|
74
|
-
Idempo::ActiveRecordBackend.new.
|
74
|
+
Idempo::ActiveRecordBackend.new.prune!
|
75
75
|
```
|
76
76
|
|
77
|
+
If you need to use Idempo with PGBouncer you will need to write your own locking implementation based on fencing tokens or similar.
|
78
|
+
|
77
79
|
## Using Redis for idempotency keys
|
78
80
|
|
79
81
|
Redis is a near-perfect data store for idempotency keys, but it can have race conditions with locks if your application runs for too long or crashes very often. If you have Redis, initialize Idempo using the `RedisBackend`:
|
@@ -88,7 +90,7 @@ If you have a configured Redis connection pool (and you should) - pass it to the
|
|
88
90
|
config.middleware.insert Idempo, backend: Idempo::RedisBackend.new(config.redis_connection_pool)
|
89
91
|
```
|
90
92
|
|
91
|
-
All data stored in Redis will have TTLs and will expire automatically.
|
93
|
+
All data stored in Redis will have TTLs and will expire automatically. Redis scripts ensure that updates to the stored idempotent responses and locking happen atomically.
|
92
94
|
|
93
95
|
|
94
96
|
## Installation
|
@@ -107,6 +109,10 @@ Or install it yourself as:
|
|
107
109
|
|
108
110
|
$ gem install idempo
|
109
111
|
|
112
|
+
## More advanced use cases
|
113
|
+
|
114
|
+
Check out the files in the `examples/` directory to see a few customisations you can do.
|
115
|
+
|
110
116
|
## Development
|
111
117
|
|
112
118
|
After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
data/Rakefile
CHANGED
@@ -0,0 +1,22 @@
|
|
1
|
+
# Sometimes you might need a different locking strategy than advisory locks,
|
2
|
+
# but still use the database-backed storage for idempotent responses. This can arise
|
3
|
+
# if you are using pgbouncer for instance, where advisory locks are not available
|
4
|
+
# when using the "transaction mode". You can modify the backend to use a different
|
5
|
+
# locking mechanism, but keep the rest.
|
6
|
+
|
7
|
+
class ActiveRecordBackendWithDistributedLock < Idempo::ActiveRecordBackend
|
8
|
+
class LocksViaService
|
9
|
+
def acquire(_conn, based_on_str)
|
10
|
+
LockingService.acquire("idempo-lk-#{based_on_str}")
|
11
|
+
end
|
12
|
+
|
13
|
+
def release(_conn, based_on_str)
|
14
|
+
LockingService.release("idempo-lk-#{based_on_str}")
|
15
|
+
true
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
def lock_implementation_for_connection(_connection)
|
20
|
+
LocksViaService.new
|
21
|
+
end
|
22
|
+
end
|
@@ -0,0 +1,41 @@
|
|
1
|
+
# Sometimes authentication is done using a Bearer token with a signature, or using another token format
|
2
|
+
# which includes some form of expiration. This means that every time a request is made, the `Authorization`
|
3
|
+
# HTTP header may have a different value, and thus the request fingerprint could change every time,
|
4
|
+
# even though the idempotency key is the same.
|
5
|
+
# For this case, a custom fingerprinting function can be used. For example, if the bearer token is
|
6
|
+
# generated in JWT format by the client, it may include the `iss` (issuer) claim, identifying the
|
7
|
+
# specific device. This identifier can then be used instead of the entire Authorization header.
|
8
|
+
|
9
|
+
module FingerprinterWithIssuerClaim
|
10
|
+
def self.call(idempotency_key, rack_request)
|
11
|
+
d = Digest::SHA256.new
|
12
|
+
d << idempotency_key << "\n"
|
13
|
+
d << rack_request.url << "\n"
|
14
|
+
d << rack_request.request_method << "\n"
|
15
|
+
d << extract_jwt_iss_claim(rack_request) << "\n"
|
16
|
+
while (chunk = rack_request.env["rack.input"].read(1024 * 65))
|
17
|
+
d << chunk
|
18
|
+
end
|
19
|
+
Base64.strict_encode64(d.digest)
|
20
|
+
ensure
|
21
|
+
rack_request.env["rack.input"].rewind
|
22
|
+
end
|
23
|
+
|
24
|
+
def self.extract_jwt_iss_claim(rack_request)
|
25
|
+
header_value = rack_request.get_header("HTTP_AUTHORIZATION").to_s
|
26
|
+
return header_value unless header_value.start_with?("Bearer ")
|
27
|
+
|
28
|
+
jwt = header_value.delete_prefix("Bearer ")
|
29
|
+
# This is decoding without verification, but in this case it is reasonably safe
|
30
|
+
# as we are not actually authenticating the request - just using the `iss` claim.
|
31
|
+
# It can make the app slightly more sensitive to replay attacks but since the request
|
32
|
+
# is idempotent, an already executed (and authenticated) request that generated a
|
33
|
+
# cached response is reasonably safe to serve out.
|
34
|
+
unverified_claims, _unverified_header = JWT.decode(jwt, _key = nil, _verify = false)
|
35
|
+
unverified_claims.fetch("iss")
|
36
|
+
rescue
|
37
|
+
# If we fail to pick up the claim or anything else - assume the request is non-idempotent
|
38
|
+
# as treating it otherwise may create a replay attack
|
39
|
+
SecureRandom.bytes(32)
|
40
|
+
end
|
41
|
+
end
|
data/idempo.gemspec
CHANGED
@@ -3,21 +3,21 @@
|
|
3
3
|
require_relative "lib/idempo/version"
|
4
4
|
|
5
5
|
Gem::Specification.new do |spec|
|
6
|
-
spec.name
|
7
|
-
spec.version
|
8
|
-
spec.authors
|
9
|
-
spec.email
|
10
|
-
|
11
|
-
spec.summary
|
12
|
-
spec.description
|
13
|
-
spec.homepage
|
14
|
-
spec.license
|
6
|
+
spec.name = "idempo"
|
7
|
+
spec.version = Idempo::VERSION
|
8
|
+
spec.authors = ["Julik Tarkhanov", "Pablo Crivella"]
|
9
|
+
spec.email = ["me@julik.nl", "pablocrivella@gmail.com"]
|
10
|
+
|
11
|
+
spec.summary = "Idempotency keys for all."
|
12
|
+
spec.description = "Provides idempotency keys for Rack applications."
|
13
|
+
spec.homepage = "https://github.com/julik/idempo"
|
14
|
+
spec.license = "MIT"
|
15
15
|
spec.required_ruby_version = Gem::Requirement.new(">= 2.4.0")
|
16
16
|
|
17
17
|
# Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
|
18
18
|
# to allow pushing to a single host or delete this section to allow pushing to any host.
|
19
19
|
if spec.respond_to?(:metadata)
|
20
|
-
spec.metadata[
|
20
|
+
spec.metadata["allowed_push_host"] = "https://rubygems.org"
|
21
21
|
else
|
22
22
|
raise "RubyGems 2.0 or newer is required to protect against public gem pushes."
|
23
23
|
end
|
@@ -31,14 +31,14 @@ Gem::Specification.new do |spec|
|
|
31
31
|
spec.files = Dir.chdir(File.expand_path(__dir__)) do
|
32
32
|
`git ls-files -z`.split("\x0").reject { |f| f.match(%r{\A(?:test|spec|features)/}) }
|
33
33
|
end
|
34
|
-
spec.bindir
|
35
|
-
spec.executables
|
34
|
+
spec.bindir = "exe"
|
35
|
+
spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
|
36
36
|
spec.require_paths = ["lib"]
|
37
37
|
|
38
38
|
# Uncomment to register a new dependency of your gem
|
39
39
|
spec.add_dependency "rack"
|
40
40
|
spec.add_dependency "msgpack"
|
41
|
-
spec.add_dependency "measurometer",
|
41
|
+
spec.add_dependency "measurometer", "~> 1.3"
|
42
42
|
|
43
43
|
spec.add_development_dependency "rake", "~> 13.0"
|
44
44
|
spec.add_development_dependency "rspec", "~> 3.0"
|
@@ -47,7 +47,7 @@ Gem::Specification.new do |spec|
|
|
47
47
|
spec.add_development_dependency "activerecord"
|
48
48
|
spec.add_development_dependency "mysql2"
|
49
49
|
spec.add_development_dependency "pg"
|
50
|
-
spec.add_development_dependency "
|
50
|
+
spec.add_development_dependency "standard"
|
51
51
|
|
52
52
|
# For more information and examples about making a new gem, checkout our
|
53
53
|
# guide at: https://bundler.io/guides/creating_gem.html
|
@@ -1,17 +1,17 @@
|
|
1
1
|
# This backend currently only works with mysql2 since it uses advisory locks
|
2
2
|
class Idempo::ActiveRecordBackend
|
3
3
|
def self.create_table(via_migration)
|
4
|
-
via_migration.create_table
|
5
|
-
t.string :idempotent_request_key, index:
|
4
|
+
via_migration.create_table "idempo_responses", charset: "utf8mb4", collation: "utf8mb4_unicode_ci" do |t|
|
5
|
+
t.string :idempotent_request_key, index: {unique: true}, null: false
|
6
6
|
t.datetime :expire_at, index: true, null: false # Needs an index for cleanup
|
7
|
-
t.binary :idempotent_response_payload,
|
7
|
+
t.binary :idempotent_response_payload, limit: Idempo::SAVED_RESPONSE_BODY_SIZE_LIMIT
|
8
8
|
t.timestamps
|
9
9
|
end
|
10
10
|
end
|
11
11
|
|
12
12
|
class Store < Struct.new(:key, :model)
|
13
13
|
def lookup
|
14
|
-
model.where(idempotent_request_key: key).where(
|
14
|
+
model.where(idempotent_request_key: key).where("expire_at > ?", Time.now).first&.idempotent_response_payload
|
15
15
|
end
|
16
16
|
|
17
17
|
def store(data:, ttl:)
|
@@ -27,18 +27,18 @@ class Idempo::ActiveRecordBackend
|
|
27
27
|
|
28
28
|
class PostgresLock
|
29
29
|
def acquire(conn, based_on_str)
|
30
|
-
acquisition_result = conn.select_value(
|
31
|
-
[true,
|
30
|
+
acquisition_result = conn.select_value("SELECT pg_try_advisory_lock(%d)" % derive_lock_key(based_on_str))
|
31
|
+
[true, "t"].include?(acquisition_result)
|
32
32
|
end
|
33
33
|
|
34
34
|
def release(conn, based_on_str)
|
35
|
-
conn.select_value(
|
35
|
+
conn.select_value("SELECT pg_advisory_unlock(%d)" % derive_lock_key(based_on_str))
|
36
36
|
end
|
37
37
|
|
38
38
|
def derive_lock_key(from_str)
|
39
39
|
# The key must be a single bigint (signed long)
|
40
40
|
hash_bytes = Digest::SHA1.digest(from_str)
|
41
|
-
hash_bytes[0...8].
|
41
|
+
hash_bytes[0...8].unpack1("l_")
|
42
42
|
end
|
43
43
|
end
|
44
44
|
|
@@ -59,13 +59,13 @@ class Idempo::ActiveRecordBackend
|
|
59
59
|
end
|
60
60
|
|
61
61
|
def initialize
|
62
|
-
require
|
62
|
+
require "active_record"
|
63
63
|
end
|
64
64
|
|
65
65
|
# Allows the model to be defined lazily without having to require active_record when this module gets loaded
|
66
66
|
def model
|
67
67
|
@model_class ||= Class.new(ActiveRecord::Base) do
|
68
|
-
self.table_name =
|
68
|
+
self.table_name = "idempo_responses"
|
69
69
|
end
|
70
70
|
end
|
71
71
|
|
@@ -82,12 +82,17 @@ class Idempo::ActiveRecordBackend
|
|
82
82
|
end
|
83
83
|
end
|
84
84
|
|
85
|
+
# Deletes expired cached Idempo responses from the database, in batches
|
86
|
+
def prune!
|
87
|
+
model.where("expire_at < ?", Time.now).in_batches.delete_all
|
88
|
+
end
|
89
|
+
|
85
90
|
private
|
86
91
|
|
87
92
|
def lock_implementation_for_connection(connection)
|
88
|
-
if
|
93
|
+
if /^mysql2/i.match?(connection.adapter_name)
|
89
94
|
MysqlLock.new
|
90
|
-
elsif
|
95
|
+
elsif /^postgres/i.match?(connection.adapter_name)
|
91
96
|
PostgresLock.new
|
92
97
|
else
|
93
98
|
raise "Unsupported database driver #{model.connection.adapter_name.downcase} - we don't know whether it supports advisory locks"
|
@@ -1,4 +1,4 @@
|
|
1
|
-
require
|
1
|
+
require "json"
|
2
2
|
|
3
3
|
class Idempo::ConcurrentRequestErrorApp
|
4
4
|
RETRY_AFTER_SECONDS = 2.to_s
|
@@ -10,6 +10,6 @@ class Idempo::ConcurrentRequestErrorApp
|
|
10
10
|
message: "Another request with this idempotency key is still in progress, please try again later"
|
11
11
|
}
|
12
12
|
}
|
13
|
-
[429, {
|
13
|
+
[429, {"Retry-After" => RETRY_AFTER_SECONDS, "Content-Type" => "application/json"}, [JSON.pretty_generate(res)]]
|
14
14
|
end
|
15
15
|
end
|
@@ -1,4 +1,4 @@
|
|
1
|
-
require
|
1
|
+
require "json"
|
2
2
|
|
3
3
|
class Idempo::MalformedKeyErrorApp
|
4
4
|
def self.call(env)
|
@@ -8,6 +8,6 @@ class Idempo::MalformedKeyErrorApp
|
|
8
8
|
message: "The Idempotency-Key header provided was empty or malformed"
|
9
9
|
}
|
10
10
|
}
|
11
|
-
[400, {
|
11
|
+
[400, {"Content-Type" => "application/json"}, [JSON.pretty_generate(res)]]
|
12
12
|
end
|
13
13
|
end
|
@@ -1,7 +1,7 @@
|
|
1
1
|
class Idempo::MemoryBackend
|
2
2
|
def initialize
|
3
|
-
require
|
4
|
-
require_relative
|
3
|
+
require "set"
|
4
|
+
require_relative "response_store"
|
5
5
|
|
6
6
|
@requests_in_flight_mutex = Mutex.new
|
7
7
|
@in_progress = Set.new
|
@@ -42,4 +42,8 @@ class Idempo::MemoryBackend
|
|
42
42
|
@requests_in_flight_mutex.synchronize { @in_progress.delete(request_key) }
|
43
43
|
end
|
44
44
|
end
|
45
|
+
|
46
|
+
def prune!
|
47
|
+
@response_store.prune
|
48
|
+
end
|
45
49
|
end
|
data/lib/idempo/redis_backend.rb
CHANGED
@@ -54,7 +54,7 @@ class Idempo::RedisBackend
|
|
54
54
|
Idempo::RedisBackend.eval_or_evalsha(r, SET_WITH_TTL_IF_LOCK_STILL_HELD_SCRIPT, keys: keys, argv: argv)
|
55
55
|
end
|
56
56
|
|
57
|
-
Measurometer.increment_counter(
|
57
|
+
Measurometer.increment_counter("idempo.redis_lock_state_when_saving_response", 1, state: outcome_of_save)
|
58
58
|
end
|
59
59
|
end
|
60
60
|
|
@@ -65,8 +65,8 @@ class Idempo::RedisBackend
|
|
65
65
|
end
|
66
66
|
|
67
67
|
def initialize(redis_or_connection_pool = Redis.new)
|
68
|
-
require
|
69
|
-
require
|
68
|
+
require "redis"
|
69
|
+
require "securerandom"
|
70
70
|
@redis_pool = redis_or_connection_pool.respond_to?(:with) ? redis_or_connection_pool : NullPool.new(redis_or_connection_pool)
|
71
71
|
end
|
72
72
|
|
@@ -84,10 +84,14 @@ class Idempo::RedisBackend
|
|
84
84
|
outcome_of_del = @redis_pool.with do |r|
|
85
85
|
Idempo::RedisBackend.eval_or_evalsha(r, DELETE_BY_KEY_AND_VALUE_SCRIPT, keys: [lock_key], argv: [token])
|
86
86
|
end
|
87
|
-
Measurometer.increment_counter(
|
87
|
+
Measurometer.increment_counter("idempo.redis_lock_state_when_releasing_lock", 1, state: outcome_of_del)
|
88
88
|
end
|
89
89
|
end
|
90
90
|
|
91
|
+
def prune!
|
92
|
+
# Do nothing
|
93
|
+
end
|
94
|
+
|
91
95
|
def self.eval_or_evalsha(redis, script_code, keys:, argv:)
|
92
96
|
script_sha = Digest::SHA1.hexdigest(script_code)
|
93
97
|
redis.evalsha(script_sha, keys: keys, argv: argv)
|
@@ -4,12 +4,12 @@ module Idempo::RequestFingerprint
|
|
4
4
|
d << idempotency_key << "\n"
|
5
5
|
d << rack_request.url << "\n"
|
6
6
|
d << rack_request.request_method << "\n"
|
7
|
-
d << rack_request.get_header(
|
8
|
-
while chunk = rack_request.env[
|
7
|
+
d << rack_request.get_header("HTTP_AUTHORIZATION").to_s << "\n"
|
8
|
+
while (chunk = rack_request.env["rack.input"].read(1024 * 65))
|
9
9
|
d << chunk
|
10
10
|
end
|
11
11
|
Base64.strict_encode64(d.digest)
|
12
12
|
ensure
|
13
|
-
rack_request.env[
|
13
|
+
rack_request.env["rack.input"].rewind
|
14
14
|
end
|
15
15
|
end
|
@@ -25,8 +25,6 @@ class Idempo::ResponseStore
|
|
25
25
|
nil
|
26
26
|
end
|
27
27
|
|
28
|
-
private
|
29
|
-
|
30
28
|
def prune
|
31
29
|
now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
32
30
|
items_to_delete = remove_lower_than(@expiries, now, &:expire_at)
|
@@ -35,6 +33,8 @@ class Idempo::ResponseStore
|
|
35
33
|
end
|
36
34
|
end
|
37
35
|
|
36
|
+
private
|
37
|
+
|
38
38
|
def binary_insert(array, item, &property_getter)
|
39
39
|
at_i = array.bsearch_index do |stored_item|
|
40
40
|
yield(stored_item) <= yield(item)
|
data/lib/idempo/version.rb
CHANGED
data/lib/idempo.rb
CHANGED
@@ -1,19 +1,19 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
4
|
-
require
|
5
|
-
require
|
6
|
-
require
|
7
|
-
require
|
8
|
-
require
|
3
|
+
require "base64"
|
4
|
+
require "digest"
|
5
|
+
require "json"
|
6
|
+
require "measurometer"
|
7
|
+
require "msgpack"
|
8
|
+
require "zlib"
|
9
9
|
|
10
|
-
require_relative "idempo/version"
|
11
|
-
require_relative "idempo/request_fingerprint"
|
12
|
-
require_relative "idempo/memory_backend"
|
13
|
-
require_relative "idempo/redis_backend"
|
14
10
|
require_relative "idempo/active_record_backend"
|
15
|
-
require_relative "idempo/malformed_key_error_app"
|
16
11
|
require_relative "idempo/concurrent_request_error_app"
|
12
|
+
require_relative "idempo/malformed_key_error_app"
|
13
|
+
require_relative "idempo/memory_backend"
|
14
|
+
require_relative "idempo/redis_backend"
|
15
|
+
require_relative "idempo/request_fingerprint"
|
16
|
+
require_relative "idempo/version"
|
17
17
|
|
18
18
|
class Idempo
|
19
19
|
DEFAULT_TTL_SECONDS = 30
|
@@ -37,44 +37,44 @@ class Idempo
|
|
37
37
|
def call(env)
|
38
38
|
req = Rack::Request.new(env)
|
39
39
|
return @app.call(env) if request_verb_idempotent?(req)
|
40
|
-
return @app.call(env) unless idempotency_key_header = extract_idempotency_key_from(env)
|
40
|
+
return @app.call(env) unless (idempotency_key_header = extract_idempotency_key_from(env))
|
41
41
|
|
42
42
|
# The RFC requires that the Idempotency-Key header value is enclosed in quotes
|
43
43
|
idempotency_key_header_value = unquote(idempotency_key_header)
|
44
|
-
raise MalformedIdempotencyKey if idempotency_key_header_value ==
|
44
|
+
raise MalformedIdempotencyKey if idempotency_key_header_value == ""
|
45
45
|
|
46
46
|
request_key = @fingerprint_calculator.call(idempotency_key_header_value, req)
|
47
47
|
|
48
48
|
@backend.with_idempotency_key(request_key) do |store|
|
49
|
-
if stored_response = store.lookup
|
50
|
-
Measurometer.increment_counter(
|
49
|
+
if (stored_response = store.lookup)
|
50
|
+
Measurometer.increment_counter("idempo.responses_served_from", 1, from: "store")
|
51
51
|
return from_persisted_response(stored_response)
|
52
52
|
end
|
53
53
|
|
54
54
|
status, headers, body = @app.call(env)
|
55
55
|
|
56
|
-
expires_in_seconds = (headers.delete(
|
56
|
+
expires_in_seconds = (headers.delete("X-Idempo-Persist-For-Seconds") || @persist_for_seconds).to_i
|
57
57
|
if response_may_be_persisted?(status, headers, body)
|
58
58
|
# Body is replaced with a cached version since a Rack response body is not rewindable
|
59
59
|
marshaled_response, body = serialize_response(status, headers, body)
|
60
60
|
store.store(data: marshaled_response, ttl: expires_in_seconds)
|
61
61
|
end
|
62
62
|
|
63
|
-
Measurometer.increment_counter(
|
63
|
+
Measurometer.increment_counter("idempo.responses_served_from", 1, from: "freshly-generated")
|
64
64
|
[status, headers, body]
|
65
65
|
end
|
66
66
|
rescue MalformedIdempotencyKey
|
67
|
-
Measurometer.increment_counter(
|
67
|
+
Measurometer.increment_counter("idempo.responses_served_from", 1, from: "malformed-idempotency-key")
|
68
68
|
@malformed_key_error_app.call(env)
|
69
69
|
rescue ConcurrentRequest
|
70
|
-
Measurometer.increment_counter(
|
70
|
+
Measurometer.increment_counter("idempo.responses_served_from", 1, from: "conflict-concurrent-request")
|
71
71
|
@concurrent_request_error_app.call(env)
|
72
72
|
end
|
73
73
|
|
74
74
|
private
|
75
75
|
|
76
76
|
def from_persisted_response(marshaled_response)
|
77
|
-
if marshaled_response[-2
|
77
|
+
if marshaled_response[-2..] != ":1"
|
78
78
|
raise Error, "Unknown serialization of the marshaled response"
|
79
79
|
else
|
80
80
|
MessagePack.unpack(Zlib.inflate(marshaled_response[0..-3]))
|
@@ -84,18 +84,18 @@ class Idempo
|
|
84
84
|
def serialize_response(status, headers, rack_response_body)
|
85
85
|
# Buffer the Rack response body, we can only do that once (it is non-rewindable)
|
86
86
|
body_chunks = []
|
87
|
-
rack_response_body.each { |chunk|
|
87
|
+
rack_response_body.each { |chunk| body_chunks << chunk.dup }
|
88
88
|
rack_response_body.close if rack_response_body.respond_to?(:close)
|
89
89
|
|
90
90
|
# Only keep headers which are strings
|
91
91
|
stringified_headers = headers.each_with_object({}) do |(header, value), filtered|
|
92
|
-
filtered[header] = value if !header.start_with?(
|
92
|
+
filtered[header] = value if !header.start_with?("rack.") && value.is_a?(String)
|
93
93
|
end
|
94
94
|
|
95
95
|
message_packed_str = MessagePack.pack([status, stringified_headers, body_chunks])
|
96
96
|
deflated_message_packed_str = Zlib.deflate(message_packed_str) + ":1"
|
97
|
-
Measurometer.increment_counter(
|
98
|
-
Measurometer.add_distribution_value(
|
97
|
+
Measurometer.increment_counter("idempo.response_total_generated_bytes", deflated_message_packed_str.bytesize)
|
98
|
+
Measurometer.add_distribution_value("idempo.response_size_bytes", deflated_message_packed_str.bytesize)
|
99
99
|
|
100
100
|
# Add the version specifier at the end, because slicing a string in Ruby at the end
|
101
101
|
# (when we unserialize our response again) does a realloc, while slicing at the start
|
@@ -104,14 +104,14 @@ class Idempo
|
|
104
104
|
end
|
105
105
|
|
106
106
|
def response_may_be_persisted?(status, headers, body)
|
107
|
-
return false if headers.delete(
|
107
|
+
return false if headers.delete("X-Idempo-Policy") == "no-store"
|
108
108
|
return false unless status_may_be_persisted?(status)
|
109
109
|
return false unless body_size_within_limit?(headers, body)
|
110
110
|
true
|
111
111
|
end
|
112
112
|
|
113
113
|
def body_size_within_limit?(response_headers, body)
|
114
|
-
return response_headers[
|
114
|
+
return response_headers["Content-Length"].to_i <= SAVED_RESPONSE_BODY_SIZE_LIMIT if response_headers["Content-Length"]
|
115
115
|
|
116
116
|
return false unless body.is_a?(Array) # Arbitrary iterable of unknown size
|
117
117
|
|
@@ -132,7 +132,7 @@ class Idempo
|
|
132
132
|
end
|
133
133
|
|
134
134
|
def extract_idempotency_key_from(env)
|
135
|
-
env[
|
135
|
+
env["HTTP_IDEMPOTENCY_KEY"] || env["HTTP_X_IDEMPOTENCY_KEY"]
|
136
136
|
end
|
137
137
|
|
138
138
|
def request_verb_idempotent?(request)
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: idempo
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Julik Tarkhanov
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: exe
|
11
11
|
cert_chain: []
|
12
|
-
date:
|
12
|
+
date: 2024-02-22 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: rack
|
@@ -152,7 +152,7 @@ dependencies:
|
|
152
152
|
- !ruby/object:Gem::Version
|
153
153
|
version: '0'
|
154
154
|
- !ruby/object:Gem::Dependency
|
155
|
-
name:
|
155
|
+
name: standard
|
156
156
|
requirement: !ruby/object:Gem::Requirement
|
157
157
|
requirements:
|
158
158
|
- - ">="
|
@@ -176,6 +176,7 @@ files:
|
|
176
176
|
- ".github/workflows/ci.yml"
|
177
177
|
- ".gitignore"
|
178
178
|
- ".rubocop.yml"
|
179
|
+
- ".standard.yml"
|
179
180
|
- CHANGELOG.md
|
180
181
|
- Gemfile
|
181
182
|
- LICENSE.txt
|
@@ -183,6 +184,8 @@ files:
|
|
183
184
|
- Rakefile
|
184
185
|
- bin/console
|
185
186
|
- bin/setup
|
187
|
+
- examples/custom_locking.rb
|
188
|
+
- examples/jwt_iss_fingerprint.rb
|
186
189
|
- idempo.gemspec
|
187
190
|
- lib/idempo.rb
|
188
191
|
- lib/idempo/active_record_backend.rb
|
@@ -216,7 +219,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
216
219
|
- !ruby/object:Gem::Version
|
217
220
|
version: '0'
|
218
221
|
requirements: []
|
219
|
-
rubygems_version: 3.
|
222
|
+
rubygems_version: 3.1.6
|
220
223
|
signing_key:
|
221
224
|
specification_version: 4
|
222
225
|
summary: Idempotency keys for all.
|