idempo 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +14 -0
- data/.rubocop.yml +2 -0
- data/CHANGELOG.md +5 -0
- data/Gemfile +5 -0
- data/LICENSE.txt +21 -0
- data/README.md +50 -0
- data/Rakefile +8 -0
- data/bin/console +15 -0
- data/bin/setup +8 -0
- data/idempo.gemspec +53 -0
- data/lib/idempo/active_record_backend.rb +54 -0
- data/lib/idempo/memory_backend.rb +45 -0
- data/lib/idempo/redis_backend.rb +104 -0
- data/lib/idempo/response_store.rb +57 -0
- data/lib/idempo/version.rb +5 -0
- data/lib/idempo.rb +169 -0
- metadata +205 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: c746ccbd1875f6a87ff6eabf4aa2566180fae441fa70eada8e7e703612ecd003
|
4
|
+
data.tar.gz: fd2365ee176c8f2c5667f78721f7886a9511a9d45a8699ce3d5b7b83ecaff17a
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 1ede112b38e43dd06261438216a4afe4c5ef08348cb462a38ae2acf9a6b0f6cd530b74dbe469dd4889ef3364e1880803764bb80c65d94954715c19dd50eb67cd
|
7
|
+
data.tar.gz: 69c3d9946da4cc27dbf2b5eeace3d577d62de3c15ca0750b66cbfa0854af7ec991e0da6cfd6d3a1a365337f7f6a0f364da3d9793ab94d13fd7792b026c2b68b7
|
data/.gitignore
ADDED
data/.rubocop.yml
ADDED
data/CHANGELOG.md
ADDED
data/Gemfile
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
The MIT License (MIT)
|
2
|
+
|
3
|
+
Copyright (c) 2021 Julik Tarkhanov
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
7
|
+
in the Software without restriction, including without limitation the rights
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
10
|
+
furnished to do so, subject to the following conditions:
|
11
|
+
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
13
|
+
all copies or substantial portions of the Software.
|
14
|
+
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
|
+
THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,50 @@
|
|
1
|
+
# Idempo
|
2
|
+
|
3
|
+
A relatively straightforward idempotency keys gem. If your client sends the `Idempotency-Key` or `X-Idempotency-Key` header to your Rack
|
4
|
+
application, and the response can be cached, Idempo will provide both a concurrent request lock and a cache for idempotent responses. If
|
5
|
+
the idempotent response is already saved for this idempotency key and request fingerprint, the cached response is going to be served
|
6
|
+
instead of calling your application.
|
7
|
+
|
8
|
+
## Installation
|
9
|
+
|
10
|
+
Add this line to your application's Gemfile:
|
11
|
+
|
12
|
+
```ruby
|
13
|
+
gem 'idempo'
|
14
|
+
```
|
15
|
+
|
16
|
+
And then execute:
|
17
|
+
|
18
|
+
$ bundle install
|
19
|
+
|
20
|
+
Or install it yourself as:
|
21
|
+
|
22
|
+
$ gem install idempo
|
23
|
+
|
24
|
+
## Usage
|
25
|
+
|
26
|
+
Idempo supports a number of backends, we recommend using Redis if you have multiple application servers / dynos and MemoryBackend if you are only using one single Puma worker. To initialize with Redis as backend pass the `backend:` parameter when adding the middleware:
|
27
|
+
|
28
|
+
```ruby
|
29
|
+
use Idempo, backend: Idempo::RedisBackend.new(Rails.application.config.redis_connection_pool)
|
30
|
+
```
|
31
|
+
|
32
|
+
and to initialize with a memory store as backend:
|
33
|
+
|
34
|
+
```ruby
|
35
|
+
use Idempo, backend: Idempo::MemoryBackend.new)
|
36
|
+
```
|
37
|
+
|
38
|
+
## Development
|
39
|
+
|
40
|
+
After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
41
|
+
|
42
|
+
To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
|
43
|
+
|
44
|
+
## Contributing
|
45
|
+
|
46
|
+
Bug reports and pull requests are welcome on GitHub at https://github.com/julik/idempo.
|
47
|
+
|
48
|
+
## License
|
49
|
+
|
50
|
+
The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
|
data/Rakefile
ADDED
data/bin/console
ADDED
@@ -0,0 +1,15 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
# frozen_string_literal: true
|
3
|
+
|
4
|
+
require "bundler/setup"
|
5
|
+
require "idempo"
|
6
|
+
|
7
|
+
# You can add fixtures and/or initialization code here to make experimenting
|
8
|
+
# with your gem easier. You can also use a different console, if you like.
|
9
|
+
|
10
|
+
# (If you use this, don't forget to add pry to your Gemfile!)
|
11
|
+
# require "pry"
|
12
|
+
# Pry.start
|
13
|
+
|
14
|
+
require "irb"
|
15
|
+
IRB.start(__FILE__)
|
data/bin/setup
ADDED
data/idempo.gemspec
ADDED
@@ -0,0 +1,53 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require_relative "lib/idempo/version"
|
4
|
+
|
5
|
+
Gem::Specification.new do |spec|
|
6
|
+
spec.name = "idempo"
|
7
|
+
spec.version = Idempo::VERSION
|
8
|
+
spec.authors = ["Julik Tarkhanov", "Pablo Crivella"]
|
9
|
+
spec.email = ["me@julik.nl", "pablocrivella@gmail.com"]
|
10
|
+
|
11
|
+
spec.summary = "Idempotency keys for all."
|
12
|
+
spec.description = "Provides idempotency keys for Rack applications."
|
13
|
+
spec.homepage = "https://github.com/julik/idempo"
|
14
|
+
spec.license = "MIT"
|
15
|
+
spec.required_ruby_version = Gem::Requirement.new(">= 2.4.0")
|
16
|
+
|
17
|
+
# Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
|
18
|
+
# to allow pushing to a single host or delete this section to allow pushing to any host.
|
19
|
+
if spec.respond_to?(:metadata)
|
20
|
+
spec.metadata['allowed_push_host'] = "https://rubygems.org"
|
21
|
+
else
|
22
|
+
raise "RubyGems 2.0 or newer is required to protect against public gem pushes."
|
23
|
+
end
|
24
|
+
|
25
|
+
spec.metadata["homepage_uri"] = spec.homepage
|
26
|
+
spec.metadata["source_code_uri"] = spec.homepage
|
27
|
+
spec.metadata["changelog_uri"] = "https://github.com/julik/idempo/blob/main/CHANGELOG.md"
|
28
|
+
|
29
|
+
# Specify which files should be added to the gem when it is released.
|
30
|
+
# The `git ls-files -z` loads the files in the RubyGem that have been added into git.
|
31
|
+
spec.files = Dir.chdir(File.expand_path(__dir__)) do
|
32
|
+
`git ls-files -z`.split("\x0").reject { |f| f.match(%r{\A(?:test|spec|features)/}) }
|
33
|
+
end
|
34
|
+
spec.bindir = "exe"
|
35
|
+
spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
|
36
|
+
spec.require_paths = ["lib"]
|
37
|
+
|
38
|
+
# Uncomment to register a new dependency of your gem
|
39
|
+
spec.add_dependency "rack"
|
40
|
+
spec.add_dependency "msgpack"
|
41
|
+
spec.add_dependency "measurometer", '~> 1.3'
|
42
|
+
|
43
|
+
spec.add_development_dependency "rake", "~> 13.0"
|
44
|
+
spec.add_development_dependency "rspec", "~> 3.0"
|
45
|
+
spec.add_development_dependency "redis", "~> 4"
|
46
|
+
spec.add_development_dependency "rack-test"
|
47
|
+
spec.add_development_dependency "activerecord"
|
48
|
+
spec.add_development_dependency "mysql2"
|
49
|
+
spec.add_development_dependency "wetransfer_style"
|
50
|
+
|
51
|
+
# For more information and examples about making a new gem, checkout our
|
52
|
+
# guide at: https://bundler.io/guides/creating_gem.html
|
53
|
+
end
|
@@ -0,0 +1,54 @@
|
|
1
|
+
# This backend currently only works with mysql2 since it uses advisory locks
|
2
|
+
class Idempo::ActiveRecordBackend
|
3
|
+
def self.create_table(via_migration)
|
4
|
+
via_migration.create_table 'idempo_responses', charset: 'utf8mb4', collation: 'utf8mb4_unicode_ci' do |t|
|
5
|
+
t.string :idempotent_request_key, index: true, unique: true, null: false
|
6
|
+
t.datetime :expire_at, index: true, null: false
|
7
|
+
t.binary :idempotent_response_payload, size: :medium
|
8
|
+
t.timestamps
|
9
|
+
end
|
10
|
+
end
|
11
|
+
|
12
|
+
class Store < Struct.new(:key, :model)
|
13
|
+
def lookup
|
14
|
+
model.where(idempotent_request_key: key).where('expire_at > ?', Time.now).first&.idempotent_response_payload
|
15
|
+
end
|
16
|
+
|
17
|
+
def store(data:, ttl:)
|
18
|
+
# MySQL does not support datetime with subsecont precision, so ceil() it is
|
19
|
+
expire_at = Time.now.utc + ttl.ceil
|
20
|
+
model.transaction do
|
21
|
+
model.where(idempotent_request_key: key).delete_all
|
22
|
+
model.create(idempotent_request_key: key, idempotent_response_payload: data, expire_at: expire_at)
|
23
|
+
end
|
24
|
+
true
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
def initialize
|
29
|
+
require 'active_record'
|
30
|
+
end
|
31
|
+
|
32
|
+
# Allows the model to be defined lazily without having to require active_record when this module gets loaded
|
33
|
+
def model
|
34
|
+
@model_class ||= Class.new(ActiveRecord::Base) do
|
35
|
+
self.table_name = 'idempo_responses'
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
def with_idempotency_key(request_key)
|
40
|
+
db_safe_key = Base64.strict_encode64(request_key)
|
41
|
+
|
42
|
+
lock_name = "idempo_%s" % db_safe_key[0..48]
|
43
|
+
quoted_lock_name = model.connection.quote(lock_name) # Note there is a limit of 64 bytes on the lock name
|
44
|
+
did_acquire = model.connection.select_value("SELECT GET_LOCK(%s, %d)" % [quoted_lock_name, 0])
|
45
|
+
|
46
|
+
raise Idempo::ConcurrentRequest unless did_acquire == 1
|
47
|
+
|
48
|
+
begin
|
49
|
+
yield(Store.new(db_safe_key, model))
|
50
|
+
ensure
|
51
|
+
model.connection.select_value("SELECT RELEASE_LOCK(%s)" % quoted_lock_name)
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
@@ -0,0 +1,45 @@
|
|
1
|
+
class Idempo::MemoryBackend
|
2
|
+
def initialize
|
3
|
+
require 'set'
|
4
|
+
require_relative 'response_store'
|
5
|
+
|
6
|
+
@requests_in_flight_mutex = Mutex.new
|
7
|
+
@in_progress = Set.new
|
8
|
+
@store_mutex = Mutex.new
|
9
|
+
@response_store = Idempo::ResponseStore.new
|
10
|
+
end
|
11
|
+
|
12
|
+
class Store < Struct.new(:store_mutex, :response_store, :key, keyword_init: true)
|
13
|
+
def lookup
|
14
|
+
store_mutex.synchronize do
|
15
|
+
response_store.lookup(key)
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
def store(data:, ttl:)
|
20
|
+
store_mutex.synchronize do
|
21
|
+
response_store.save(key, data, ttl)
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
def with_idempotency_key(request_key)
|
27
|
+
did_insert = @requests_in_flight_mutex.synchronize do
|
28
|
+
if @in_progress.include?(request_key)
|
29
|
+
false
|
30
|
+
else
|
31
|
+
@in_progress << request_key
|
32
|
+
true
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
raise Idempo::ConcurrentRequest unless did_insert
|
37
|
+
|
38
|
+
store = Store.new(store_mutex: @store_mutex, response_store: @response_store, key: request_key)
|
39
|
+
begin
|
40
|
+
yield(store)
|
41
|
+
ensure
|
42
|
+
@requests_in_flight_mutex.synchronize { @in_progress.delete(request_key) }
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
@@ -0,0 +1,104 @@
|
|
1
|
+
class Idempo::RedisBackend
|
2
|
+
# The TTL value for the lock, which is used if the process
|
3
|
+
# holding the lock crashes or gets killed
|
4
|
+
LOCK_TTL_SECONDS = 5 * 60
|
5
|
+
|
6
|
+
# See https://redis.io/topics/distlock
|
7
|
+
DELETE_BY_KEY_AND_VALUE_SCRIPT = <<~EOL
|
8
|
+
redis.replicate_commands()
|
9
|
+
if redis.call("get",KEYS[1]) == ARGV[1] then
|
10
|
+
-- we are still holding the lock, release it
|
11
|
+
redis.call("del",KEYS[1])
|
12
|
+
return "ok"
|
13
|
+
else
|
14
|
+
-- someone else holds the lock or it has expired
|
15
|
+
return "lock_lost"
|
16
|
+
end
|
17
|
+
EOL
|
18
|
+
|
19
|
+
# See https://redis.io/topics/distlock as well as a rebuttal in
|
20
|
+
# https://martin.kleppmann.com/2016/02/08/how-to-do-distributed-locking.html
|
21
|
+
SET_WITH_TTL_IF_LOCK_STILL_HELD_SCRIPT = <<~EOL
|
22
|
+
redis.replicate_commands()
|
23
|
+
if redis.call("get", KEYS[1]) == ARGV[1] then
|
24
|
+
-- we are still holding the lock, we can go ahead and set it
|
25
|
+
redis.call("set", KEYS[2], ARGV[2], "px", ARGV[3])
|
26
|
+
return "ok"
|
27
|
+
else
|
28
|
+
return "lock_lost"
|
29
|
+
end
|
30
|
+
EOL
|
31
|
+
|
32
|
+
class Store < Struct.new(:redis_pool, :key, :lock_redis_key, :lock_token, keyword_init: true)
|
33
|
+
def lookup
|
34
|
+
response_redis_key = "idempo:response:#{key}"
|
35
|
+
redis_pool.with do |r|
|
36
|
+
bin_str = r.get(response_redis_key)
|
37
|
+
bin_str&.force_encoding(Encoding::BINARY)
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
def store(data:, ttl:)
|
42
|
+
response_redis_key = "idempo:response:#{key}"
|
43
|
+
ttl_millis = (ttl * 1000.0).round
|
44
|
+
|
45
|
+
# We save our payload using a script, and we will _only_ save it if our lock is still held.
|
46
|
+
# If our lock expires during the request - for example our app.call takes too long -
|
47
|
+
# we might have lost it, and another request has already saved a payload on our behalf. At this point
|
48
|
+
# we have no guarantee that our response was generated exclusively, or that the response that was generated
|
49
|
+
# by our "competitor" is equal to ours, or that a "competing" request is not holding our lock and executing the
|
50
|
+
# same workload as we just did. The only sensible thing to do when we encounter this is to actually _skip_ the write.
|
51
|
+
keys = [lock_redis_key, response_redis_key]
|
52
|
+
argv = [lock_token, data.force_encoding(Encoding::BINARY), ttl_millis]
|
53
|
+
outcome_of_save = redis_pool.with do |r|
|
54
|
+
Idempo::RedisBackend.eval_or_evalsha(r, SET_WITH_TTL_IF_LOCK_STILL_HELD_SCRIPT, keys: keys, argv: argv)
|
55
|
+
end
|
56
|
+
|
57
|
+
Measurometer.increment_counter('idempo.redis_lock_when_storing', 1, outcome: outcome_of_save)
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
class NullPool < Struct.new(:redis)
|
62
|
+
def with
|
63
|
+
yield redis
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
def initialize(redis_or_connection_pool)
|
68
|
+
require 'redis'
|
69
|
+
require 'securerandom'
|
70
|
+
@redis_pool = redis_or_connection_pool.respond_to?(:with) ? redis_or_connection_pool : NullPool.new(redis_or_connection_pool)
|
71
|
+
end
|
72
|
+
|
73
|
+
def with_idempotency_key(request_key)
|
74
|
+
lock_key = "idempo:lock:#{request_key}"
|
75
|
+
token = SecureRandom.bytes(32)
|
76
|
+
did_acquire = @redis_pool.with { |r| r.set(lock_key, token, nx: true, ex: LOCK_TTL_SECONDS) }
|
77
|
+
|
78
|
+
raise Idempo::ConcurrentRequest unless did_acquire
|
79
|
+
|
80
|
+
begin
|
81
|
+
store = Store.new(redis_pool: @redis_pool, lock_redis_key: lock_key, lock_token: token, key: request_key)
|
82
|
+
yield(store)
|
83
|
+
ensure
|
84
|
+
outcome_of_del = @redis_pool.with do |r|
|
85
|
+
Idempo::RedisBackend.eval_or_evalsha(r, DELETE_BY_KEY_AND_VALUE_SCRIPT, keys: [lock_key], argv: [token])
|
86
|
+
end
|
87
|
+
Measurometer.increment_counter('idempo_redis_release_lock', 1, outcome: outcome_of_del)
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
91
|
+
def self.eval_or_evalsha(redis, script_code, keys:, argv:)
|
92
|
+
script_sha = Digest::SHA1.hexdigest(script_code)
|
93
|
+
redis.evalsha(script_sha, keys: keys, argv: argv)
|
94
|
+
rescue Redis::CommandError => e
|
95
|
+
if e.message.include? "NOSCRIPT"
|
96
|
+
# The Redis server has never seen this script before. Needs to run only once in the entire lifetime
|
97
|
+
# of the Redis server, until the script changes - in which case it will be loaded under a different SHA
|
98
|
+
redis.script(:load, script_code)
|
99
|
+
retry
|
100
|
+
else
|
101
|
+
raise e
|
102
|
+
end
|
103
|
+
end
|
104
|
+
end
|
@@ -0,0 +1,57 @@
|
|
1
|
+
class Idempo::ResponseStore
|
2
|
+
ExpiryHandle = Struct.new(:key, :expire_at)
|
3
|
+
StoredResponse = Struct.new(:key, :expire_at, :payload)
|
4
|
+
|
5
|
+
def initialize
|
6
|
+
@values = {}
|
7
|
+
@expiries = []
|
8
|
+
end
|
9
|
+
|
10
|
+
def save(key, value, expire_in)
|
11
|
+
prune
|
12
|
+
exp = expire_in + Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
13
|
+
res = StoredResponse.new(key, exp, value)
|
14
|
+
expiry_handle = ExpiryHandle.new(key, exp)
|
15
|
+
binary_insert(@expiries, expiry_handle, &:expire_at)
|
16
|
+
@values[key] = res
|
17
|
+
end
|
18
|
+
|
19
|
+
def lookup(key)
|
20
|
+
prune
|
21
|
+
stored = @values[key]
|
22
|
+
return unless stored
|
23
|
+
return stored.payload if stored.expire_at > Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
24
|
+
@values.delete(key)
|
25
|
+
nil
|
26
|
+
end
|
27
|
+
|
28
|
+
private
|
29
|
+
|
30
|
+
def prune
|
31
|
+
now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
32
|
+
items_to_delete = remove_lower_than(@expiries, now, &:expire_at)
|
33
|
+
items_to_delete.each do |expiry_handle|
|
34
|
+
@values.delete(expiry_handle.key) if @values[expiry_handle.key] && @values[expiry_handle.key].expire_at < now
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
def binary_insert(array, item, &property_getter)
|
39
|
+
at_i = array.bsearch_index do |stored_item|
|
40
|
+
yield(stored_item) <= yield(item)
|
41
|
+
end
|
42
|
+
at_i ? array.insert(at_i, item) : array.push(item)
|
43
|
+
end
|
44
|
+
|
45
|
+
def remove_lower_than(array, threshold_value, &property_getter)
|
46
|
+
at_i = array.bsearch_index do |stored_item|
|
47
|
+
yield(stored_item) <= threshold_value
|
48
|
+
end
|
49
|
+
if at_i
|
50
|
+
array[at_i..array.length].tap do |_deleted_items|
|
51
|
+
array.replace(array[0..at_i])
|
52
|
+
end
|
53
|
+
else
|
54
|
+
[]
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
data/lib/idempo.rb
ADDED
@@ -0,0 +1,169 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'zlib'
|
4
|
+
require 'msgpack'
|
5
|
+
require 'base64'
|
6
|
+
require 'digest'
|
7
|
+
require 'json'
|
8
|
+
require 'measurometer'
|
9
|
+
|
10
|
+
require_relative "idempo/version"
|
11
|
+
require_relative "idempo/memory_backend"
|
12
|
+
require_relative "idempo/redis_backend"
|
13
|
+
require_relative "idempo/active_record_backend"
|
14
|
+
|
15
|
+
class Idempo
|
16
|
+
DEFAULT_TTL = 30
|
17
|
+
SAVED_RESPONSE_BODY_SIZE_LIMIT = 4 * 1024 * 1024
|
18
|
+
|
19
|
+
class Error < StandardError; end
|
20
|
+
|
21
|
+
class ConcurrentRequest < Error; end
|
22
|
+
|
23
|
+
class MalformedIdempotencyKey < Error; end
|
24
|
+
|
25
|
+
def initialize(app, backend: MemoryBackend.new)
|
26
|
+
@backend = backend
|
27
|
+
@app = app
|
28
|
+
end
|
29
|
+
|
30
|
+
def call(env)
|
31
|
+
req = Rack::Request.new(env)
|
32
|
+
return @app.call(env) if request_verb_idempotent?(req)
|
33
|
+
return @app.call(env) unless idempotency_key_header = extract_idempotency_key_from(env)
|
34
|
+
|
35
|
+
# The RFC requires that the Idempotency-Key header value is enclosed in quotes
|
36
|
+
idempotency_key_header = unquote(idempotency_key_header)
|
37
|
+
raise MalformedIdempotencyKey if idempotency_key_header == ''
|
38
|
+
|
39
|
+
fingerprint = compute_request_fingerprint(req)
|
40
|
+
request_key = "#{idempotency_key_header}_#{fingerprint}"
|
41
|
+
|
42
|
+
@backend.with_idempotency_key(request_key) do |store|
|
43
|
+
if stored_response = store.lookup
|
44
|
+
Measurometer.increment_counter('idempo.served', 1, via: 'cached')
|
45
|
+
return from_persisted_response(stored_response)
|
46
|
+
end
|
47
|
+
|
48
|
+
status, headers, body = @app.call(env)
|
49
|
+
|
50
|
+
if response_may_be_persisted?(status, headers, body)
|
51
|
+
expires_in_seconds = (headers.delete('X-Idempo-Persist-For-Seconds') || DEFAULT_TTL).to_i
|
52
|
+
# Body is replaced with a cached version since a Rack response body is not rewindable
|
53
|
+
marshaled_response, body = serialize_response(status, headers, body)
|
54
|
+
store.store(data: marshaled_response, ttl: expires_in_seconds)
|
55
|
+
end
|
56
|
+
|
57
|
+
Measurometer.increment_counter('idempo.served', 1, via: 'stored')
|
58
|
+
[status, headers, body]
|
59
|
+
end
|
60
|
+
rescue MalformedIdempotencyKey
|
61
|
+
res = {
|
62
|
+
ok: false,
|
63
|
+
error: {
|
64
|
+
message: "The Idempotency-Key header provided was empty"
|
65
|
+
}
|
66
|
+
}
|
67
|
+
[400, {'Content-Type' => 'application/json'}, [JSON.pretty_generate(res)]]
|
68
|
+
rescue ConcurrentRequest
|
69
|
+
Measurometer.increment_counter('idempo.served', 1, via: 'concurrent-request-error')
|
70
|
+
res = {
|
71
|
+
ok: false,
|
72
|
+
error: {
|
73
|
+
message: "Another request with this idempotency key is still in progress, please try again later"
|
74
|
+
}
|
75
|
+
}
|
76
|
+
[429, {'Retry-After' => '2', 'Content-Type' => 'application/json'}, [JSON.pretty_generate(res)]]
|
77
|
+
end
|
78
|
+
|
79
|
+
private
|
80
|
+
|
81
|
+
def from_persisted_response(marshaled_response)
|
82
|
+
if marshaled_response[-2..-1] != ':1'
|
83
|
+
raise Error, "Unknown serialization of the marshaled response"
|
84
|
+
else
|
85
|
+
MessagePack.unpack(Zlib.inflate(marshaled_response[0..-3]))
|
86
|
+
end
|
87
|
+
end
|
88
|
+
|
89
|
+
def serialize_response(status, headers, rack_response_body)
|
90
|
+
# Buffer the Rack response body, we can only do that once (it is non-rewindable)
|
91
|
+
body_chunks = []
|
92
|
+
rack_response_body.each { |chunk| body_chunks << chunk.dup }
|
93
|
+
rack_response_body.close if rack_response_body.respond_to?(:close)
|
94
|
+
|
95
|
+
# Only keep headers which are strings
|
96
|
+
stringified_headers = headers.each_with_object({}) do |(header, value), filtered|
|
97
|
+
filtered[header] = value if value.is_a?(String)
|
98
|
+
end
|
99
|
+
|
100
|
+
message_packed_str = MessagePack.pack([status, stringified_headers, body_chunks])
|
101
|
+
deflated_message_packed_str = Zlib.deflate(message_packed_str) + ":1"
|
102
|
+
Measurometer.increment_counter('idempo.response_total_generated_bytes', deflated_message_packed_str.bytesize)
|
103
|
+
Measurometer.add_distribution_value('idempo.response_size_bytes', deflated_message_packed_str.bytesize)
|
104
|
+
|
105
|
+
# Add the version specifier at the end, because slicing a string in Ruby at the end
|
106
|
+
# (when we unserialize our response again) does a realloc, while slicing at the start
|
107
|
+
# does not
|
108
|
+
[deflated_message_packed_str, body_chunks]
|
109
|
+
end
|
110
|
+
|
111
|
+
def response_may_be_persisted?(status, headers, body)
|
112
|
+
return false if headers.delete('X-Idempo-Policy') == 'no-store'
|
113
|
+
return false unless status_may_be_persisted?(status)
|
114
|
+
return false unless body_size_within_limit?(headers, body)
|
115
|
+
true
|
116
|
+
end
|
117
|
+
|
118
|
+
def body_size_within_limit?(response_headers, body)
|
119
|
+
return response_headers['Content-Length'].to_i <= SAVED_RESPONSE_BODY_SIZE_LIMIT if response_headers['Content-Length']
|
120
|
+
|
121
|
+
return false unless body.is_a?(Array) # Arbitrary iterable of unknown size
|
122
|
+
|
123
|
+
precomputed_body_size = body.inject(0) { |sum, chunk| sum + chunk.bytesize }
|
124
|
+
precomputed_body_size <= SAVED_RESPONSE_BODY_SIZE_LIMIT
|
125
|
+
end
|
126
|
+
|
127
|
+
def status_may_be_persisted?(status)
|
128
|
+
case status
|
129
|
+
when 200..400
|
130
|
+
true
|
131
|
+
when 429, 425
|
132
|
+
false
|
133
|
+
when 400..499
|
134
|
+
true
|
135
|
+
else
|
136
|
+
false
|
137
|
+
end
|
138
|
+
end
|
139
|
+
|
140
|
+
def compute_request_fingerprint(req)
|
141
|
+
d = Digest::SHA256.new
|
142
|
+
d << req.url << "\n"
|
143
|
+
d << req.request_method << "\n"
|
144
|
+
while chunk = req.env['rack.input'].read(1024 * 65)
|
145
|
+
d << chunk
|
146
|
+
end
|
147
|
+
Base64.strict_encode64(d.digest)
|
148
|
+
ensure
|
149
|
+
req.env['rack.input'].rewind
|
150
|
+
end
|
151
|
+
|
152
|
+
def extract_idempotency_key_from(env)
|
153
|
+
env['HTTP_IDEMPOTENCY_KEY'] || env['HTTP_X_IDEMPOTENCY_KEY']
|
154
|
+
end
|
155
|
+
|
156
|
+
def request_verb_idempotent?(request)
|
157
|
+
request.get? || request.head? || request.options?
|
158
|
+
end
|
159
|
+
|
160
|
+
def unquote(str)
|
161
|
+
# Do not use regular expressions so that we don't have to thing about a catastrophic lookahead
|
162
|
+
double_quote = '"'
|
163
|
+
if str.start_with?(double_quote) && str.end_with?(double_quote)
|
164
|
+
str[1..-2]
|
165
|
+
else
|
166
|
+
str
|
167
|
+
end
|
168
|
+
end
|
169
|
+
end
|
metadata
ADDED
@@ -0,0 +1,205 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: idempo
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Julik Tarkhanov
|
8
|
+
- Pablo Crivella
|
9
|
+
autorequire:
|
10
|
+
bindir: exe
|
11
|
+
cert_chain: []
|
12
|
+
date: 2021-10-19 00:00:00.000000000 Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: rack
|
16
|
+
requirement: !ruby/object:Gem::Requirement
|
17
|
+
requirements:
|
18
|
+
- - ">="
|
19
|
+
- !ruby/object:Gem::Version
|
20
|
+
version: '0'
|
21
|
+
type: :runtime
|
22
|
+
prerelease: false
|
23
|
+
version_requirements: !ruby/object:Gem::Requirement
|
24
|
+
requirements:
|
25
|
+
- - ">="
|
26
|
+
- !ruby/object:Gem::Version
|
27
|
+
version: '0'
|
28
|
+
- !ruby/object:Gem::Dependency
|
29
|
+
name: msgpack
|
30
|
+
requirement: !ruby/object:Gem::Requirement
|
31
|
+
requirements:
|
32
|
+
- - ">="
|
33
|
+
- !ruby/object:Gem::Version
|
34
|
+
version: '0'
|
35
|
+
type: :runtime
|
36
|
+
prerelease: false
|
37
|
+
version_requirements: !ruby/object:Gem::Requirement
|
38
|
+
requirements:
|
39
|
+
- - ">="
|
40
|
+
- !ruby/object:Gem::Version
|
41
|
+
version: '0'
|
42
|
+
- !ruby/object:Gem::Dependency
|
43
|
+
name: measurometer
|
44
|
+
requirement: !ruby/object:Gem::Requirement
|
45
|
+
requirements:
|
46
|
+
- - "~>"
|
47
|
+
- !ruby/object:Gem::Version
|
48
|
+
version: '1.3'
|
49
|
+
type: :runtime
|
50
|
+
prerelease: false
|
51
|
+
version_requirements: !ruby/object:Gem::Requirement
|
52
|
+
requirements:
|
53
|
+
- - "~>"
|
54
|
+
- !ruby/object:Gem::Version
|
55
|
+
version: '1.3'
|
56
|
+
- !ruby/object:Gem::Dependency
|
57
|
+
name: rake
|
58
|
+
requirement: !ruby/object:Gem::Requirement
|
59
|
+
requirements:
|
60
|
+
- - "~>"
|
61
|
+
- !ruby/object:Gem::Version
|
62
|
+
version: '13.0'
|
63
|
+
type: :development
|
64
|
+
prerelease: false
|
65
|
+
version_requirements: !ruby/object:Gem::Requirement
|
66
|
+
requirements:
|
67
|
+
- - "~>"
|
68
|
+
- !ruby/object:Gem::Version
|
69
|
+
version: '13.0'
|
70
|
+
- !ruby/object:Gem::Dependency
|
71
|
+
name: rspec
|
72
|
+
requirement: !ruby/object:Gem::Requirement
|
73
|
+
requirements:
|
74
|
+
- - "~>"
|
75
|
+
- !ruby/object:Gem::Version
|
76
|
+
version: '3.0'
|
77
|
+
type: :development
|
78
|
+
prerelease: false
|
79
|
+
version_requirements: !ruby/object:Gem::Requirement
|
80
|
+
requirements:
|
81
|
+
- - "~>"
|
82
|
+
- !ruby/object:Gem::Version
|
83
|
+
version: '3.0'
|
84
|
+
- !ruby/object:Gem::Dependency
|
85
|
+
name: redis
|
86
|
+
requirement: !ruby/object:Gem::Requirement
|
87
|
+
requirements:
|
88
|
+
- - "~>"
|
89
|
+
- !ruby/object:Gem::Version
|
90
|
+
version: '4'
|
91
|
+
type: :development
|
92
|
+
prerelease: false
|
93
|
+
version_requirements: !ruby/object:Gem::Requirement
|
94
|
+
requirements:
|
95
|
+
- - "~>"
|
96
|
+
- !ruby/object:Gem::Version
|
97
|
+
version: '4'
|
98
|
+
- !ruby/object:Gem::Dependency
|
99
|
+
name: rack-test
|
100
|
+
requirement: !ruby/object:Gem::Requirement
|
101
|
+
requirements:
|
102
|
+
- - ">="
|
103
|
+
- !ruby/object:Gem::Version
|
104
|
+
version: '0'
|
105
|
+
type: :development
|
106
|
+
prerelease: false
|
107
|
+
version_requirements: !ruby/object:Gem::Requirement
|
108
|
+
requirements:
|
109
|
+
- - ">="
|
110
|
+
- !ruby/object:Gem::Version
|
111
|
+
version: '0'
|
112
|
+
- !ruby/object:Gem::Dependency
|
113
|
+
name: activerecord
|
114
|
+
requirement: !ruby/object:Gem::Requirement
|
115
|
+
requirements:
|
116
|
+
- - ">="
|
117
|
+
- !ruby/object:Gem::Version
|
118
|
+
version: '0'
|
119
|
+
type: :development
|
120
|
+
prerelease: false
|
121
|
+
version_requirements: !ruby/object:Gem::Requirement
|
122
|
+
requirements:
|
123
|
+
- - ">="
|
124
|
+
- !ruby/object:Gem::Version
|
125
|
+
version: '0'
|
126
|
+
- !ruby/object:Gem::Dependency
|
127
|
+
name: mysql2
|
128
|
+
requirement: !ruby/object:Gem::Requirement
|
129
|
+
requirements:
|
130
|
+
- - ">="
|
131
|
+
- !ruby/object:Gem::Version
|
132
|
+
version: '0'
|
133
|
+
type: :development
|
134
|
+
prerelease: false
|
135
|
+
version_requirements: !ruby/object:Gem::Requirement
|
136
|
+
requirements:
|
137
|
+
- - ">="
|
138
|
+
- !ruby/object:Gem::Version
|
139
|
+
version: '0'
|
140
|
+
- !ruby/object:Gem::Dependency
|
141
|
+
name: wetransfer_style
|
142
|
+
requirement: !ruby/object:Gem::Requirement
|
143
|
+
requirements:
|
144
|
+
- - ">="
|
145
|
+
- !ruby/object:Gem::Version
|
146
|
+
version: '0'
|
147
|
+
type: :development
|
148
|
+
prerelease: false
|
149
|
+
version_requirements: !ruby/object:Gem::Requirement
|
150
|
+
requirements:
|
151
|
+
- - ">="
|
152
|
+
- !ruby/object:Gem::Version
|
153
|
+
version: '0'
|
154
|
+
description: Provides idempotency keys for Rack applications.
|
155
|
+
email:
|
156
|
+
- me@julik.nl
|
157
|
+
- pablocrivella@gmail.com
|
158
|
+
executables: []
|
159
|
+
extensions: []
|
160
|
+
extra_rdoc_files: []
|
161
|
+
files:
|
162
|
+
- ".gitignore"
|
163
|
+
- ".rubocop.yml"
|
164
|
+
- CHANGELOG.md
|
165
|
+
- Gemfile
|
166
|
+
- LICENSE.txt
|
167
|
+
- README.md
|
168
|
+
- Rakefile
|
169
|
+
- bin/console
|
170
|
+
- bin/setup
|
171
|
+
- idempo.gemspec
|
172
|
+
- lib/idempo.rb
|
173
|
+
- lib/idempo/active_record_backend.rb
|
174
|
+
- lib/idempo/memory_backend.rb
|
175
|
+
- lib/idempo/redis_backend.rb
|
176
|
+
- lib/idempo/response_store.rb
|
177
|
+
- lib/idempo/version.rb
|
178
|
+
homepage: https://github.com/julik/idempo
|
179
|
+
licenses:
|
180
|
+
- MIT
|
181
|
+
metadata:
|
182
|
+
allowed_push_host: https://rubygems.org
|
183
|
+
homepage_uri: https://github.com/julik/idempo
|
184
|
+
source_code_uri: https://github.com/julik/idempo
|
185
|
+
changelog_uri: https://github.com/julik/idempo/blob/main/CHANGELOG.md
|
186
|
+
post_install_message:
|
187
|
+
rdoc_options: []
|
188
|
+
require_paths:
|
189
|
+
- lib
|
190
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
191
|
+
requirements:
|
192
|
+
- - ">="
|
193
|
+
- !ruby/object:Gem::Version
|
194
|
+
version: 2.4.0
|
195
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
196
|
+
requirements:
|
197
|
+
- - ">="
|
198
|
+
- !ruby/object:Gem::Version
|
199
|
+
version: '0'
|
200
|
+
requirements: []
|
201
|
+
rubygems_version: 3.0.3
|
202
|
+
signing_key:
|
203
|
+
specification_version: 4
|
204
|
+
summary: Idempotency keys for all.
|
205
|
+
test_files: []
|