idempo 0.1.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +14 -0
- data/.rubocop.yml +2 -0
- data/CHANGELOG.md +5 -0
- data/Gemfile +5 -0
- data/LICENSE.txt +21 -0
- data/README.md +50 -0
- data/Rakefile +8 -0
- data/bin/console +15 -0
- data/bin/setup +8 -0
- data/idempo.gemspec +53 -0
- data/lib/idempo/active_record_backend.rb +54 -0
- data/lib/idempo/memory_backend.rb +45 -0
- data/lib/idempo/redis_backend.rb +104 -0
- data/lib/idempo/response_store.rb +57 -0
- data/lib/idempo/version.rb +5 -0
- data/lib/idempo.rb +169 -0
- metadata +205 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: c746ccbd1875f6a87ff6eabf4aa2566180fae441fa70eada8e7e703612ecd003
|
4
|
+
data.tar.gz: fd2365ee176c8f2c5667f78721f7886a9511a9d45a8699ce3d5b7b83ecaff17a
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 1ede112b38e43dd06261438216a4afe4c5ef08348cb462a38ae2acf9a6b0f6cd530b74dbe469dd4889ef3364e1880803764bb80c65d94954715c19dd50eb67cd
|
7
|
+
data.tar.gz: 69c3d9946da4cc27dbf2b5eeace3d577d62de3c15ca0750b66cbfa0854af7ec991e0da6cfd6d3a1a365337f7f6a0f364da3d9793ab94d13fd7792b026c2b68b7
|
data/.gitignore
ADDED
data/.rubocop.yml
ADDED
data/CHANGELOG.md
ADDED
data/Gemfile
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
The MIT License (MIT)
|
2
|
+
|
3
|
+
Copyright (c) 2021 Julik Tarkhanov
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
7
|
+
in the Software without restriction, including without limitation the rights
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
10
|
+
furnished to do so, subject to the following conditions:
|
11
|
+
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
13
|
+
all copies or substantial portions of the Software.
|
14
|
+
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
|
+
THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,50 @@
|
|
1
|
+
# Idempo
|
2
|
+
|
3
|
+
A relatively straightforward idempotency keys gem. If your client sends the `Idempotency-Key` or `X-Idempotency-Key` header to your Rack
|
4
|
+
application, and the response can be cached, Idempo will provide both a concurrent request lock and a cache for idempotent responses. If
|
5
|
+
the idempotent response is already saved for this idempotency key and request fingerprint, the cached response is going to be served
|
6
|
+
instead of calling your application.
|
7
|
+
|
8
|
+
## Installation
|
9
|
+
|
10
|
+
Add this line to your application's Gemfile:
|
11
|
+
|
12
|
+
```ruby
|
13
|
+
gem 'idempo'
|
14
|
+
```
|
15
|
+
|
16
|
+
And then execute:
|
17
|
+
|
18
|
+
$ bundle install
|
19
|
+
|
20
|
+
Or install it yourself as:
|
21
|
+
|
22
|
+
$ gem install idempo
|
23
|
+
|
24
|
+
## Usage
|
25
|
+
|
26
|
+
Idempo supports a number of backends, we recommend using Redis if you have multiple application servers / dynos and MemoryBackend if you are only using one single Puma worker. To initialize with Redis as backend pass the `backend:` parameter when adding the middleware:
|
27
|
+
|
28
|
+
```ruby
|
29
|
+
use Idempo, backend: Idempo::RedisBackend.new(Rails.application.config.redis_connection_pool)
|
30
|
+
```
|
31
|
+
|
32
|
+
and to initialize with a memory store as backend:
|
33
|
+
|
34
|
+
```ruby
|
35
|
+
use Idempo, backend: Idempo::MemoryBackend.new)
|
36
|
+
```
|
37
|
+
|
38
|
+
## Development
|
39
|
+
|
40
|
+
After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
41
|
+
|
42
|
+
To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
|
43
|
+
|
44
|
+
## Contributing
|
45
|
+
|
46
|
+
Bug reports and pull requests are welcome on GitHub at https://github.com/julik/idempo.
|
47
|
+
|
48
|
+
## License
|
49
|
+
|
50
|
+
The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
|
data/Rakefile
ADDED
data/bin/console
ADDED
@@ -0,0 +1,15 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
# frozen_string_literal: true
|
3
|
+
|
4
|
+
require "bundler/setup"
|
5
|
+
require "idempo"
|
6
|
+
|
7
|
+
# You can add fixtures and/or initialization code here to make experimenting
|
8
|
+
# with your gem easier. You can also use a different console, if you like.
|
9
|
+
|
10
|
+
# (If you use this, don't forget to add pry to your Gemfile!)
|
11
|
+
# require "pry"
|
12
|
+
# Pry.start
|
13
|
+
|
14
|
+
require "irb"
|
15
|
+
IRB.start(__FILE__)
|
data/bin/setup
ADDED
data/idempo.gemspec
ADDED
@@ -0,0 +1,53 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require_relative "lib/idempo/version"
|
4
|
+
|
5
|
+
Gem::Specification.new do |spec|
|
6
|
+
spec.name = "idempo"
|
7
|
+
spec.version = Idempo::VERSION
|
8
|
+
spec.authors = ["Julik Tarkhanov", "Pablo Crivella"]
|
9
|
+
spec.email = ["me@julik.nl", "pablocrivella@gmail.com"]
|
10
|
+
|
11
|
+
spec.summary = "Idempotency keys for all."
|
12
|
+
spec.description = "Provides idempotency keys for Rack applications."
|
13
|
+
spec.homepage = "https://github.com/julik/idempo"
|
14
|
+
spec.license = "MIT"
|
15
|
+
spec.required_ruby_version = Gem::Requirement.new(">= 2.4.0")
|
16
|
+
|
17
|
+
# Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
|
18
|
+
# to allow pushing to a single host or delete this section to allow pushing to any host.
|
19
|
+
if spec.respond_to?(:metadata)
|
20
|
+
spec.metadata['allowed_push_host'] = "https://rubygems.org"
|
21
|
+
else
|
22
|
+
raise "RubyGems 2.0 or newer is required to protect against public gem pushes."
|
23
|
+
end
|
24
|
+
|
25
|
+
spec.metadata["homepage_uri"] = spec.homepage
|
26
|
+
spec.metadata["source_code_uri"] = spec.homepage
|
27
|
+
spec.metadata["changelog_uri"] = "https://github.com/julik/idempo/blob/main/CHANGELOG.md"
|
28
|
+
|
29
|
+
# Specify which files should be added to the gem when it is released.
|
30
|
+
# The `git ls-files -z` loads the files in the RubyGem that have been added into git.
|
31
|
+
spec.files = Dir.chdir(File.expand_path(__dir__)) do
|
32
|
+
`git ls-files -z`.split("\x0").reject { |f| f.match(%r{\A(?:test|spec|features)/}) }
|
33
|
+
end
|
34
|
+
spec.bindir = "exe"
|
35
|
+
spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
|
36
|
+
spec.require_paths = ["lib"]
|
37
|
+
|
38
|
+
# Uncomment to register a new dependency of your gem
|
39
|
+
spec.add_dependency "rack"
|
40
|
+
spec.add_dependency "msgpack"
|
41
|
+
spec.add_dependency "measurometer", '~> 1.3'
|
42
|
+
|
43
|
+
spec.add_development_dependency "rake", "~> 13.0"
|
44
|
+
spec.add_development_dependency "rspec", "~> 3.0"
|
45
|
+
spec.add_development_dependency "redis", "~> 4"
|
46
|
+
spec.add_development_dependency "rack-test"
|
47
|
+
spec.add_development_dependency "activerecord"
|
48
|
+
spec.add_development_dependency "mysql2"
|
49
|
+
spec.add_development_dependency "wetransfer_style"
|
50
|
+
|
51
|
+
# For more information and examples about making a new gem, checkout our
|
52
|
+
# guide at: https://bundler.io/guides/creating_gem.html
|
53
|
+
end
|
@@ -0,0 +1,54 @@
|
|
1
|
+
# This backend currently only works with mysql2 since it uses advisory locks
|
2
|
+
class Idempo::ActiveRecordBackend
|
3
|
+
def self.create_table(via_migration)
|
4
|
+
via_migration.create_table 'idempo_responses', charset: 'utf8mb4', collation: 'utf8mb4_unicode_ci' do |t|
|
5
|
+
t.string :idempotent_request_key, index: true, unique: true, null: false
|
6
|
+
t.datetime :expire_at, index: true, null: false
|
7
|
+
t.binary :idempotent_response_payload, size: :medium
|
8
|
+
t.timestamps
|
9
|
+
end
|
10
|
+
end
|
11
|
+
|
12
|
+
class Store < Struct.new(:key, :model)
|
13
|
+
def lookup
|
14
|
+
model.where(idempotent_request_key: key).where('expire_at > ?', Time.now).first&.idempotent_response_payload
|
15
|
+
end
|
16
|
+
|
17
|
+
def store(data:, ttl:)
|
18
|
+
# MySQL does not support datetime with subsecont precision, so ceil() it is
|
19
|
+
expire_at = Time.now.utc + ttl.ceil
|
20
|
+
model.transaction do
|
21
|
+
model.where(idempotent_request_key: key).delete_all
|
22
|
+
model.create(idempotent_request_key: key, idempotent_response_payload: data, expire_at: expire_at)
|
23
|
+
end
|
24
|
+
true
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
def initialize
|
29
|
+
require 'active_record'
|
30
|
+
end
|
31
|
+
|
32
|
+
# Allows the model to be defined lazily without having to require active_record when this module gets loaded
|
33
|
+
def model
|
34
|
+
@model_class ||= Class.new(ActiveRecord::Base) do
|
35
|
+
self.table_name = 'idempo_responses'
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
def with_idempotency_key(request_key)
|
40
|
+
db_safe_key = Base64.strict_encode64(request_key)
|
41
|
+
|
42
|
+
lock_name = "idempo_%s" % db_safe_key[0..48]
|
43
|
+
quoted_lock_name = model.connection.quote(lock_name) # Note there is a limit of 64 bytes on the lock name
|
44
|
+
did_acquire = model.connection.select_value("SELECT GET_LOCK(%s, %d)" % [quoted_lock_name, 0])
|
45
|
+
|
46
|
+
raise Idempo::ConcurrentRequest unless did_acquire == 1
|
47
|
+
|
48
|
+
begin
|
49
|
+
yield(Store.new(db_safe_key, model))
|
50
|
+
ensure
|
51
|
+
model.connection.select_value("SELECT RELEASE_LOCK(%s)" % quoted_lock_name)
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
@@ -0,0 +1,45 @@
|
|
1
|
+
class Idempo::MemoryBackend
|
2
|
+
def initialize
|
3
|
+
require 'set'
|
4
|
+
require_relative 'response_store'
|
5
|
+
|
6
|
+
@requests_in_flight_mutex = Mutex.new
|
7
|
+
@in_progress = Set.new
|
8
|
+
@store_mutex = Mutex.new
|
9
|
+
@response_store = Idempo::ResponseStore.new
|
10
|
+
end
|
11
|
+
|
12
|
+
class Store < Struct.new(:store_mutex, :response_store, :key, keyword_init: true)
|
13
|
+
def lookup
|
14
|
+
store_mutex.synchronize do
|
15
|
+
response_store.lookup(key)
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
def store(data:, ttl:)
|
20
|
+
store_mutex.synchronize do
|
21
|
+
response_store.save(key, data, ttl)
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
def with_idempotency_key(request_key)
|
27
|
+
did_insert = @requests_in_flight_mutex.synchronize do
|
28
|
+
if @in_progress.include?(request_key)
|
29
|
+
false
|
30
|
+
else
|
31
|
+
@in_progress << request_key
|
32
|
+
true
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
raise Idempo::ConcurrentRequest unless did_insert
|
37
|
+
|
38
|
+
store = Store.new(store_mutex: @store_mutex, response_store: @response_store, key: request_key)
|
39
|
+
begin
|
40
|
+
yield(store)
|
41
|
+
ensure
|
42
|
+
@requests_in_flight_mutex.synchronize { @in_progress.delete(request_key) }
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
@@ -0,0 +1,104 @@
|
|
1
|
+
class Idempo::RedisBackend
|
2
|
+
# The TTL value for the lock, which is used if the process
|
3
|
+
# holding the lock crashes or gets killed
|
4
|
+
LOCK_TTL_SECONDS = 5 * 60
|
5
|
+
|
6
|
+
# See https://redis.io/topics/distlock
|
7
|
+
DELETE_BY_KEY_AND_VALUE_SCRIPT = <<~EOL
|
8
|
+
redis.replicate_commands()
|
9
|
+
if redis.call("get",KEYS[1]) == ARGV[1] then
|
10
|
+
-- we are still holding the lock, release it
|
11
|
+
redis.call("del",KEYS[1])
|
12
|
+
return "ok"
|
13
|
+
else
|
14
|
+
-- someone else holds the lock or it has expired
|
15
|
+
return "lock_lost"
|
16
|
+
end
|
17
|
+
EOL
|
18
|
+
|
19
|
+
# See https://redis.io/topics/distlock as well as a rebuttal in
|
20
|
+
# https://martin.kleppmann.com/2016/02/08/how-to-do-distributed-locking.html
|
21
|
+
SET_WITH_TTL_IF_LOCK_STILL_HELD_SCRIPT = <<~EOL
|
22
|
+
redis.replicate_commands()
|
23
|
+
if redis.call("get", KEYS[1]) == ARGV[1] then
|
24
|
+
-- we are still holding the lock, we can go ahead and set it
|
25
|
+
redis.call("set", KEYS[2], ARGV[2], "px", ARGV[3])
|
26
|
+
return "ok"
|
27
|
+
else
|
28
|
+
return "lock_lost"
|
29
|
+
end
|
30
|
+
EOL
|
31
|
+
|
32
|
+
class Store < Struct.new(:redis_pool, :key, :lock_redis_key, :lock_token, keyword_init: true)
|
33
|
+
def lookup
|
34
|
+
response_redis_key = "idempo:response:#{key}"
|
35
|
+
redis_pool.with do |r|
|
36
|
+
bin_str = r.get(response_redis_key)
|
37
|
+
bin_str&.force_encoding(Encoding::BINARY)
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
def store(data:, ttl:)
|
42
|
+
response_redis_key = "idempo:response:#{key}"
|
43
|
+
ttl_millis = (ttl * 1000.0).round
|
44
|
+
|
45
|
+
# We save our payload using a script, and we will _only_ save it if our lock is still held.
|
46
|
+
# If our lock expires during the request - for example our app.call takes too long -
|
47
|
+
# we might have lost it, and another request has already saved a payload on our behalf. At this point
|
48
|
+
# we have no guarantee that our response was generated exclusively, or that the response that was generated
|
49
|
+
# by our "competitor" is equal to ours, or that a "competing" request is not holding our lock and executing the
|
50
|
+
# same workload as we just did. The only sensible thing to do when we encounter this is to actually _skip_ the write.
|
51
|
+
keys = [lock_redis_key, response_redis_key]
|
52
|
+
argv = [lock_token, data.force_encoding(Encoding::BINARY), ttl_millis]
|
53
|
+
outcome_of_save = redis_pool.with do |r|
|
54
|
+
Idempo::RedisBackend.eval_or_evalsha(r, SET_WITH_TTL_IF_LOCK_STILL_HELD_SCRIPT, keys: keys, argv: argv)
|
55
|
+
end
|
56
|
+
|
57
|
+
Measurometer.increment_counter('idempo.redis_lock_when_storing', 1, outcome: outcome_of_save)
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
class NullPool < Struct.new(:redis)
|
62
|
+
def with
|
63
|
+
yield redis
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
def initialize(redis_or_connection_pool)
|
68
|
+
require 'redis'
|
69
|
+
require 'securerandom'
|
70
|
+
@redis_pool = redis_or_connection_pool.respond_to?(:with) ? redis_or_connection_pool : NullPool.new(redis_or_connection_pool)
|
71
|
+
end
|
72
|
+
|
73
|
+
def with_idempotency_key(request_key)
|
74
|
+
lock_key = "idempo:lock:#{request_key}"
|
75
|
+
token = SecureRandom.bytes(32)
|
76
|
+
did_acquire = @redis_pool.with { |r| r.set(lock_key, token, nx: true, ex: LOCK_TTL_SECONDS) }
|
77
|
+
|
78
|
+
raise Idempo::ConcurrentRequest unless did_acquire
|
79
|
+
|
80
|
+
begin
|
81
|
+
store = Store.new(redis_pool: @redis_pool, lock_redis_key: lock_key, lock_token: token, key: request_key)
|
82
|
+
yield(store)
|
83
|
+
ensure
|
84
|
+
outcome_of_del = @redis_pool.with do |r|
|
85
|
+
Idempo::RedisBackend.eval_or_evalsha(r, DELETE_BY_KEY_AND_VALUE_SCRIPT, keys: [lock_key], argv: [token])
|
86
|
+
end
|
87
|
+
Measurometer.increment_counter('idempo_redis_release_lock', 1, outcome: outcome_of_del)
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
91
|
+
def self.eval_or_evalsha(redis, script_code, keys:, argv:)
|
92
|
+
script_sha = Digest::SHA1.hexdigest(script_code)
|
93
|
+
redis.evalsha(script_sha, keys: keys, argv: argv)
|
94
|
+
rescue Redis::CommandError => e
|
95
|
+
if e.message.include? "NOSCRIPT"
|
96
|
+
# The Redis server has never seen this script before. Needs to run only once in the entire lifetime
|
97
|
+
# of the Redis server, until the script changes - in which case it will be loaded under a different SHA
|
98
|
+
redis.script(:load, script_code)
|
99
|
+
retry
|
100
|
+
else
|
101
|
+
raise e
|
102
|
+
end
|
103
|
+
end
|
104
|
+
end
|
@@ -0,0 +1,57 @@
|
|
1
|
+
class Idempo::ResponseStore
|
2
|
+
ExpiryHandle = Struct.new(:key, :expire_at)
|
3
|
+
StoredResponse = Struct.new(:key, :expire_at, :payload)
|
4
|
+
|
5
|
+
def initialize
|
6
|
+
@values = {}
|
7
|
+
@expiries = []
|
8
|
+
end
|
9
|
+
|
10
|
+
def save(key, value, expire_in)
|
11
|
+
prune
|
12
|
+
exp = expire_in + Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
13
|
+
res = StoredResponse.new(key, exp, value)
|
14
|
+
expiry_handle = ExpiryHandle.new(key, exp)
|
15
|
+
binary_insert(@expiries, expiry_handle, &:expire_at)
|
16
|
+
@values[key] = res
|
17
|
+
end
|
18
|
+
|
19
|
+
def lookup(key)
|
20
|
+
prune
|
21
|
+
stored = @values[key]
|
22
|
+
return unless stored
|
23
|
+
return stored.payload if stored.expire_at > Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
24
|
+
@values.delete(key)
|
25
|
+
nil
|
26
|
+
end
|
27
|
+
|
28
|
+
private
|
29
|
+
|
30
|
+
def prune
|
31
|
+
now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
32
|
+
items_to_delete = remove_lower_than(@expiries, now, &:expire_at)
|
33
|
+
items_to_delete.each do |expiry_handle|
|
34
|
+
@values.delete(expiry_handle.key) if @values[expiry_handle.key] && @values[expiry_handle.key].expire_at < now
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
def binary_insert(array, item, &property_getter)
|
39
|
+
at_i = array.bsearch_index do |stored_item|
|
40
|
+
yield(stored_item) <= yield(item)
|
41
|
+
end
|
42
|
+
at_i ? array.insert(at_i, item) : array.push(item)
|
43
|
+
end
|
44
|
+
|
45
|
+
def remove_lower_than(array, threshold_value, &property_getter)
|
46
|
+
at_i = array.bsearch_index do |stored_item|
|
47
|
+
yield(stored_item) <= threshold_value
|
48
|
+
end
|
49
|
+
if at_i
|
50
|
+
array[at_i..array.length].tap do |_deleted_items|
|
51
|
+
array.replace(array[0..at_i])
|
52
|
+
end
|
53
|
+
else
|
54
|
+
[]
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
data/lib/idempo.rb
ADDED
@@ -0,0 +1,169 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'zlib'
|
4
|
+
require 'msgpack'
|
5
|
+
require 'base64'
|
6
|
+
require 'digest'
|
7
|
+
require 'json'
|
8
|
+
require 'measurometer'
|
9
|
+
|
10
|
+
require_relative "idempo/version"
|
11
|
+
require_relative "idempo/memory_backend"
|
12
|
+
require_relative "idempo/redis_backend"
|
13
|
+
require_relative "idempo/active_record_backend"
|
14
|
+
|
15
|
+
class Idempo
|
16
|
+
DEFAULT_TTL = 30
|
17
|
+
SAVED_RESPONSE_BODY_SIZE_LIMIT = 4 * 1024 * 1024
|
18
|
+
|
19
|
+
class Error < StandardError; end
|
20
|
+
|
21
|
+
class ConcurrentRequest < Error; end
|
22
|
+
|
23
|
+
class MalformedIdempotencyKey < Error; end
|
24
|
+
|
25
|
+
def initialize(app, backend: MemoryBackend.new)
|
26
|
+
@backend = backend
|
27
|
+
@app = app
|
28
|
+
end
|
29
|
+
|
30
|
+
def call(env)
|
31
|
+
req = Rack::Request.new(env)
|
32
|
+
return @app.call(env) if request_verb_idempotent?(req)
|
33
|
+
return @app.call(env) unless idempotency_key_header = extract_idempotency_key_from(env)
|
34
|
+
|
35
|
+
# The RFC requires that the Idempotency-Key header value is enclosed in quotes
|
36
|
+
idempotency_key_header = unquote(idempotency_key_header)
|
37
|
+
raise MalformedIdempotencyKey if idempotency_key_header == ''
|
38
|
+
|
39
|
+
fingerprint = compute_request_fingerprint(req)
|
40
|
+
request_key = "#{idempotency_key_header}_#{fingerprint}"
|
41
|
+
|
42
|
+
@backend.with_idempotency_key(request_key) do |store|
|
43
|
+
if stored_response = store.lookup
|
44
|
+
Measurometer.increment_counter('idempo.served', 1, via: 'cached')
|
45
|
+
return from_persisted_response(stored_response)
|
46
|
+
end
|
47
|
+
|
48
|
+
status, headers, body = @app.call(env)
|
49
|
+
|
50
|
+
if response_may_be_persisted?(status, headers, body)
|
51
|
+
expires_in_seconds = (headers.delete('X-Idempo-Persist-For-Seconds') || DEFAULT_TTL).to_i
|
52
|
+
# Body is replaced with a cached version since a Rack response body is not rewindable
|
53
|
+
marshaled_response, body = serialize_response(status, headers, body)
|
54
|
+
store.store(data: marshaled_response, ttl: expires_in_seconds)
|
55
|
+
end
|
56
|
+
|
57
|
+
Measurometer.increment_counter('idempo.served', 1, via: 'stored')
|
58
|
+
[status, headers, body]
|
59
|
+
end
|
60
|
+
rescue MalformedIdempotencyKey
|
61
|
+
res = {
|
62
|
+
ok: false,
|
63
|
+
error: {
|
64
|
+
message: "The Idempotency-Key header provided was empty"
|
65
|
+
}
|
66
|
+
}
|
67
|
+
[400, {'Content-Type' => 'application/json'}, [JSON.pretty_generate(res)]]
|
68
|
+
rescue ConcurrentRequest
|
69
|
+
Measurometer.increment_counter('idempo.served', 1, via: 'concurrent-request-error')
|
70
|
+
res = {
|
71
|
+
ok: false,
|
72
|
+
error: {
|
73
|
+
message: "Another request with this idempotency key is still in progress, please try again later"
|
74
|
+
}
|
75
|
+
}
|
76
|
+
[429, {'Retry-After' => '2', 'Content-Type' => 'application/json'}, [JSON.pretty_generate(res)]]
|
77
|
+
end
|
78
|
+
|
79
|
+
private
|
80
|
+
|
81
|
+
def from_persisted_response(marshaled_response)
|
82
|
+
if marshaled_response[-2..-1] != ':1'
|
83
|
+
raise Error, "Unknown serialization of the marshaled response"
|
84
|
+
else
|
85
|
+
MessagePack.unpack(Zlib.inflate(marshaled_response[0..-3]))
|
86
|
+
end
|
87
|
+
end
|
88
|
+
|
89
|
+
def serialize_response(status, headers, rack_response_body)
|
90
|
+
# Buffer the Rack response body, we can only do that once (it is non-rewindable)
|
91
|
+
body_chunks = []
|
92
|
+
rack_response_body.each { |chunk| body_chunks << chunk.dup }
|
93
|
+
rack_response_body.close if rack_response_body.respond_to?(:close)
|
94
|
+
|
95
|
+
# Only keep headers which are strings
|
96
|
+
stringified_headers = headers.each_with_object({}) do |(header, value), filtered|
|
97
|
+
filtered[header] = value if value.is_a?(String)
|
98
|
+
end
|
99
|
+
|
100
|
+
message_packed_str = MessagePack.pack([status, stringified_headers, body_chunks])
|
101
|
+
deflated_message_packed_str = Zlib.deflate(message_packed_str) + ":1"
|
102
|
+
Measurometer.increment_counter('idempo.response_total_generated_bytes', deflated_message_packed_str.bytesize)
|
103
|
+
Measurometer.add_distribution_value('idempo.response_size_bytes', deflated_message_packed_str.bytesize)
|
104
|
+
|
105
|
+
# Add the version specifier at the end, because slicing a string in Ruby at the end
|
106
|
+
# (when we unserialize our response again) does a realloc, while slicing at the start
|
107
|
+
# does not
|
108
|
+
[deflated_message_packed_str, body_chunks]
|
109
|
+
end
|
110
|
+
|
111
|
+
def response_may_be_persisted?(status, headers, body)
|
112
|
+
return false if headers.delete('X-Idempo-Policy') == 'no-store'
|
113
|
+
return false unless status_may_be_persisted?(status)
|
114
|
+
return false unless body_size_within_limit?(headers, body)
|
115
|
+
true
|
116
|
+
end
|
117
|
+
|
118
|
+
def body_size_within_limit?(response_headers, body)
|
119
|
+
return response_headers['Content-Length'].to_i <= SAVED_RESPONSE_BODY_SIZE_LIMIT if response_headers['Content-Length']
|
120
|
+
|
121
|
+
return false unless body.is_a?(Array) # Arbitrary iterable of unknown size
|
122
|
+
|
123
|
+
precomputed_body_size = body.inject(0) { |sum, chunk| sum + chunk.bytesize }
|
124
|
+
precomputed_body_size <= SAVED_RESPONSE_BODY_SIZE_LIMIT
|
125
|
+
end
|
126
|
+
|
127
|
+
def status_may_be_persisted?(status)
|
128
|
+
case status
|
129
|
+
when 200..400
|
130
|
+
true
|
131
|
+
when 429, 425
|
132
|
+
false
|
133
|
+
when 400..499
|
134
|
+
true
|
135
|
+
else
|
136
|
+
false
|
137
|
+
end
|
138
|
+
end
|
139
|
+
|
140
|
+
def compute_request_fingerprint(req)
|
141
|
+
d = Digest::SHA256.new
|
142
|
+
d << req.url << "\n"
|
143
|
+
d << req.request_method << "\n"
|
144
|
+
while chunk = req.env['rack.input'].read(1024 * 65)
|
145
|
+
d << chunk
|
146
|
+
end
|
147
|
+
Base64.strict_encode64(d.digest)
|
148
|
+
ensure
|
149
|
+
req.env['rack.input'].rewind
|
150
|
+
end
|
151
|
+
|
152
|
+
def extract_idempotency_key_from(env)
|
153
|
+
env['HTTP_IDEMPOTENCY_KEY'] || env['HTTP_X_IDEMPOTENCY_KEY']
|
154
|
+
end
|
155
|
+
|
156
|
+
def request_verb_idempotent?(request)
|
157
|
+
request.get? || request.head? || request.options?
|
158
|
+
end
|
159
|
+
|
160
|
+
def unquote(str)
|
161
|
+
# Do not use regular expressions so that we don't have to thing about a catastrophic lookahead
|
162
|
+
double_quote = '"'
|
163
|
+
if str.start_with?(double_quote) && str.end_with?(double_quote)
|
164
|
+
str[1..-2]
|
165
|
+
else
|
166
|
+
str
|
167
|
+
end
|
168
|
+
end
|
169
|
+
end
|
metadata
ADDED
@@ -0,0 +1,205 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: idempo
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Julik Tarkhanov
|
8
|
+
- Pablo Crivella
|
9
|
+
autorequire:
|
10
|
+
bindir: exe
|
11
|
+
cert_chain: []
|
12
|
+
date: 2021-10-19 00:00:00.000000000 Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: rack
|
16
|
+
requirement: !ruby/object:Gem::Requirement
|
17
|
+
requirements:
|
18
|
+
- - ">="
|
19
|
+
- !ruby/object:Gem::Version
|
20
|
+
version: '0'
|
21
|
+
type: :runtime
|
22
|
+
prerelease: false
|
23
|
+
version_requirements: !ruby/object:Gem::Requirement
|
24
|
+
requirements:
|
25
|
+
- - ">="
|
26
|
+
- !ruby/object:Gem::Version
|
27
|
+
version: '0'
|
28
|
+
- !ruby/object:Gem::Dependency
|
29
|
+
name: msgpack
|
30
|
+
requirement: !ruby/object:Gem::Requirement
|
31
|
+
requirements:
|
32
|
+
- - ">="
|
33
|
+
- !ruby/object:Gem::Version
|
34
|
+
version: '0'
|
35
|
+
type: :runtime
|
36
|
+
prerelease: false
|
37
|
+
version_requirements: !ruby/object:Gem::Requirement
|
38
|
+
requirements:
|
39
|
+
- - ">="
|
40
|
+
- !ruby/object:Gem::Version
|
41
|
+
version: '0'
|
42
|
+
- !ruby/object:Gem::Dependency
|
43
|
+
name: measurometer
|
44
|
+
requirement: !ruby/object:Gem::Requirement
|
45
|
+
requirements:
|
46
|
+
- - "~>"
|
47
|
+
- !ruby/object:Gem::Version
|
48
|
+
version: '1.3'
|
49
|
+
type: :runtime
|
50
|
+
prerelease: false
|
51
|
+
version_requirements: !ruby/object:Gem::Requirement
|
52
|
+
requirements:
|
53
|
+
- - "~>"
|
54
|
+
- !ruby/object:Gem::Version
|
55
|
+
version: '1.3'
|
56
|
+
- !ruby/object:Gem::Dependency
|
57
|
+
name: rake
|
58
|
+
requirement: !ruby/object:Gem::Requirement
|
59
|
+
requirements:
|
60
|
+
- - "~>"
|
61
|
+
- !ruby/object:Gem::Version
|
62
|
+
version: '13.0'
|
63
|
+
type: :development
|
64
|
+
prerelease: false
|
65
|
+
version_requirements: !ruby/object:Gem::Requirement
|
66
|
+
requirements:
|
67
|
+
- - "~>"
|
68
|
+
- !ruby/object:Gem::Version
|
69
|
+
version: '13.0'
|
70
|
+
- !ruby/object:Gem::Dependency
|
71
|
+
name: rspec
|
72
|
+
requirement: !ruby/object:Gem::Requirement
|
73
|
+
requirements:
|
74
|
+
- - "~>"
|
75
|
+
- !ruby/object:Gem::Version
|
76
|
+
version: '3.0'
|
77
|
+
type: :development
|
78
|
+
prerelease: false
|
79
|
+
version_requirements: !ruby/object:Gem::Requirement
|
80
|
+
requirements:
|
81
|
+
- - "~>"
|
82
|
+
- !ruby/object:Gem::Version
|
83
|
+
version: '3.0'
|
84
|
+
- !ruby/object:Gem::Dependency
|
85
|
+
name: redis
|
86
|
+
requirement: !ruby/object:Gem::Requirement
|
87
|
+
requirements:
|
88
|
+
- - "~>"
|
89
|
+
- !ruby/object:Gem::Version
|
90
|
+
version: '4'
|
91
|
+
type: :development
|
92
|
+
prerelease: false
|
93
|
+
version_requirements: !ruby/object:Gem::Requirement
|
94
|
+
requirements:
|
95
|
+
- - "~>"
|
96
|
+
- !ruby/object:Gem::Version
|
97
|
+
version: '4'
|
98
|
+
- !ruby/object:Gem::Dependency
|
99
|
+
name: rack-test
|
100
|
+
requirement: !ruby/object:Gem::Requirement
|
101
|
+
requirements:
|
102
|
+
- - ">="
|
103
|
+
- !ruby/object:Gem::Version
|
104
|
+
version: '0'
|
105
|
+
type: :development
|
106
|
+
prerelease: false
|
107
|
+
version_requirements: !ruby/object:Gem::Requirement
|
108
|
+
requirements:
|
109
|
+
- - ">="
|
110
|
+
- !ruby/object:Gem::Version
|
111
|
+
version: '0'
|
112
|
+
- !ruby/object:Gem::Dependency
|
113
|
+
name: activerecord
|
114
|
+
requirement: !ruby/object:Gem::Requirement
|
115
|
+
requirements:
|
116
|
+
- - ">="
|
117
|
+
- !ruby/object:Gem::Version
|
118
|
+
version: '0'
|
119
|
+
type: :development
|
120
|
+
prerelease: false
|
121
|
+
version_requirements: !ruby/object:Gem::Requirement
|
122
|
+
requirements:
|
123
|
+
- - ">="
|
124
|
+
- !ruby/object:Gem::Version
|
125
|
+
version: '0'
|
126
|
+
- !ruby/object:Gem::Dependency
|
127
|
+
name: mysql2
|
128
|
+
requirement: !ruby/object:Gem::Requirement
|
129
|
+
requirements:
|
130
|
+
- - ">="
|
131
|
+
- !ruby/object:Gem::Version
|
132
|
+
version: '0'
|
133
|
+
type: :development
|
134
|
+
prerelease: false
|
135
|
+
version_requirements: !ruby/object:Gem::Requirement
|
136
|
+
requirements:
|
137
|
+
- - ">="
|
138
|
+
- !ruby/object:Gem::Version
|
139
|
+
version: '0'
|
140
|
+
- !ruby/object:Gem::Dependency
|
141
|
+
name: wetransfer_style
|
142
|
+
requirement: !ruby/object:Gem::Requirement
|
143
|
+
requirements:
|
144
|
+
- - ">="
|
145
|
+
- !ruby/object:Gem::Version
|
146
|
+
version: '0'
|
147
|
+
type: :development
|
148
|
+
prerelease: false
|
149
|
+
version_requirements: !ruby/object:Gem::Requirement
|
150
|
+
requirements:
|
151
|
+
- - ">="
|
152
|
+
- !ruby/object:Gem::Version
|
153
|
+
version: '0'
|
154
|
+
description: Provides idempotency keys for Rack applications.
|
155
|
+
email:
|
156
|
+
- me@julik.nl
|
157
|
+
- pablocrivella@gmail.com
|
158
|
+
executables: []
|
159
|
+
extensions: []
|
160
|
+
extra_rdoc_files: []
|
161
|
+
files:
|
162
|
+
- ".gitignore"
|
163
|
+
- ".rubocop.yml"
|
164
|
+
- CHANGELOG.md
|
165
|
+
- Gemfile
|
166
|
+
- LICENSE.txt
|
167
|
+
- README.md
|
168
|
+
- Rakefile
|
169
|
+
- bin/console
|
170
|
+
- bin/setup
|
171
|
+
- idempo.gemspec
|
172
|
+
- lib/idempo.rb
|
173
|
+
- lib/idempo/active_record_backend.rb
|
174
|
+
- lib/idempo/memory_backend.rb
|
175
|
+
- lib/idempo/redis_backend.rb
|
176
|
+
- lib/idempo/response_store.rb
|
177
|
+
- lib/idempo/version.rb
|
178
|
+
homepage: https://github.com/julik/idempo
|
179
|
+
licenses:
|
180
|
+
- MIT
|
181
|
+
metadata:
|
182
|
+
allowed_push_host: https://rubygems.org
|
183
|
+
homepage_uri: https://github.com/julik/idempo
|
184
|
+
source_code_uri: https://github.com/julik/idempo
|
185
|
+
changelog_uri: https://github.com/julik/idempo/blob/main/CHANGELOG.md
|
186
|
+
post_install_message:
|
187
|
+
rdoc_options: []
|
188
|
+
require_paths:
|
189
|
+
- lib
|
190
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
191
|
+
requirements:
|
192
|
+
- - ">="
|
193
|
+
- !ruby/object:Gem::Version
|
194
|
+
version: 2.4.0
|
195
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
196
|
+
requirements:
|
197
|
+
- - ">="
|
198
|
+
- !ruby/object:Gem::Version
|
199
|
+
version: '0'
|
200
|
+
requirements: []
|
201
|
+
rubygems_version: 3.0.3
|
202
|
+
signing_key:
|
203
|
+
specification_version: 4
|
204
|
+
summary: Idempotency keys for all.
|
205
|
+
test_files: []
|