idempo 0.1.1

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: c746ccbd1875f6a87ff6eabf4aa2566180fae441fa70eada8e7e703612ecd003
4
+ data.tar.gz: fd2365ee176c8f2c5667f78721f7886a9511a9d45a8699ce3d5b7b83ecaff17a
5
+ SHA512:
6
+ metadata.gz: 1ede112b38e43dd06261438216a4afe4c5ef08348cb462a38ae2acf9a6b0f6cd530b74dbe469dd4889ef3364e1880803764bb80c65d94954715c19dd50eb67cd
7
+ data.tar.gz: 69c3d9946da4cc27dbf2b5eeace3d577d62de3c15ca0750b66cbfa0854af7ec991e0da6cfd6d3a1a365337f7f6a0f364da3d9793ab94d13fd7792b026c2b68b7
data/.gitignore ADDED
@@ -0,0 +1,14 @@
1
+ /.bundle/
2
+ /.yardoc
3
+ /_yardoc/
4
+ /coverage/
5
+ /doc/
6
+ /pkg/
7
+ /spec/reports/
8
+ /tmp/
9
+
10
+ # rspec failure tracking
11
+ .rspec_status
12
+
13
+ # Gem lock not required for libraries
14
+ Gemfile.lock
data/.rubocop.yml ADDED
@@ -0,0 +1,2 @@
1
+ inherit_gem:
2
+ wetransfer_style: ruby/default.yml
data/CHANGELOG.md ADDED
@@ -0,0 +1,5 @@
1
+ ## [Unreleased]
2
+
3
+ ## [0.1.0] - 2021-10-14
4
+
5
+ - Initial release
data/Gemfile ADDED
@@ -0,0 +1,5 @@
1
+ # frozen_string_literal: true
2
+
3
+ source "https://rubygems.org"
4
+
5
+ gemspec
data/LICENSE.txt ADDED
@@ -0,0 +1,21 @@
1
+ The MIT License (MIT)
2
+
3
+ Copyright (c) 2021 Julik Tarkhanov
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in
13
+ all copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
+ THE SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,50 @@
1
+ # Idempo
2
+
3
+ A relatively straightforward idempotency keys gem. If your client sends the `Idempotency-Key` or `X-Idempotency-Key` header to your Rack
4
+ application, and the response can be cached, Idempo will provide both a concurrent request lock and a cache for idempotent responses. If
5
+ the idempotent response is already saved for this idempotency key and request fingerprint, the cached response is going to be served
6
+ instead of calling your application.
7
+
8
+ ## Installation
9
+
10
+ Add this line to your application's Gemfile:
11
+
12
+ ```ruby
13
+ gem 'idempo'
14
+ ```
15
+
16
+ And then execute:
17
+
18
+ $ bundle install
19
+
20
+ Or install it yourself as:
21
+
22
+ $ gem install idempo
23
+
24
+ ## Usage
25
+
26
+ Idempo supports a number of backends, we recommend using Redis if you have multiple application servers / dynos and MemoryBackend if you are only using one single Puma worker. To initialize with Redis as backend pass the `backend:` parameter when adding the middleware:
27
+
28
+ ```ruby
29
+ use Idempo, backend: Idempo::RedisBackend.new(Rails.application.config.redis_connection_pool)
30
+ ```
31
+
32
+ and to initialize with a memory store as backend:
33
+
34
+ ```ruby
35
+ use Idempo, backend: Idempo::MemoryBackend.new)
36
+ ```
37
+
38
+ ## Development
39
+
40
+ After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
41
+
42
+ To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
43
+
44
+ ## Contributing
45
+
46
+ Bug reports and pull requests are welcome on GitHub at https://github.com/julik/idempo.
47
+
48
+ ## License
49
+
50
+ The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
data/Rakefile ADDED
@@ -0,0 +1,8 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "bundler/gem_tasks"
4
+ require "rspec/core/rake_task"
5
+
6
+ RSpec::Core::RakeTask.new(:spec)
7
+
8
+ task default: :spec
data/bin/console ADDED
@@ -0,0 +1,15 @@
1
+ #!/usr/bin/env ruby
2
+ # frozen_string_literal: true
3
+
4
+ require "bundler/setup"
5
+ require "idempo"
6
+
7
+ # You can add fixtures and/or initialization code here to make experimenting
8
+ # with your gem easier. You can also use a different console, if you like.
9
+
10
+ # (If you use this, don't forget to add pry to your Gemfile!)
11
+ # require "pry"
12
+ # Pry.start
13
+
14
+ require "irb"
15
+ IRB.start(__FILE__)
data/bin/setup ADDED
@@ -0,0 +1,8 @@
1
+ #!/usr/bin/env bash
2
+ set -euo pipefail
3
+ IFS=$'\n\t'
4
+ set -vx
5
+
6
+ bundle install
7
+
8
+ # Do any other automated setup that you need to do here
data/idempo.gemspec ADDED
@@ -0,0 +1,53 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "lib/idempo/version"
4
+
5
+ Gem::Specification.new do |spec|
6
+ spec.name = "idempo"
7
+ spec.version = Idempo::VERSION
8
+ spec.authors = ["Julik Tarkhanov", "Pablo Crivella"]
9
+ spec.email = ["me@julik.nl", "pablocrivella@gmail.com"]
10
+
11
+ spec.summary = "Idempotency keys for all."
12
+ spec.description = "Provides idempotency keys for Rack applications."
13
+ spec.homepage = "https://github.com/julik/idempo"
14
+ spec.license = "MIT"
15
+ spec.required_ruby_version = Gem::Requirement.new(">= 2.4.0")
16
+
17
+ # Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
18
+ # to allow pushing to a single host or delete this section to allow pushing to any host.
19
+ if spec.respond_to?(:metadata)
20
+ spec.metadata['allowed_push_host'] = "https://rubygems.org"
21
+ else
22
+ raise "RubyGems 2.0 or newer is required to protect against public gem pushes."
23
+ end
24
+
25
+ spec.metadata["homepage_uri"] = spec.homepage
26
+ spec.metadata["source_code_uri"] = spec.homepage
27
+ spec.metadata["changelog_uri"] = "https://github.com/julik/idempo/blob/main/CHANGELOG.md"
28
+
29
+ # Specify which files should be added to the gem when it is released.
30
+ # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
31
+ spec.files = Dir.chdir(File.expand_path(__dir__)) do
32
+ `git ls-files -z`.split("\x0").reject { |f| f.match(%r{\A(?:test|spec|features)/}) }
33
+ end
34
+ spec.bindir = "exe"
35
+ spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
36
+ spec.require_paths = ["lib"]
37
+
38
+ # Uncomment to register a new dependency of your gem
39
+ spec.add_dependency "rack"
40
+ spec.add_dependency "msgpack"
41
+ spec.add_dependency "measurometer", '~> 1.3'
42
+
43
+ spec.add_development_dependency "rake", "~> 13.0"
44
+ spec.add_development_dependency "rspec", "~> 3.0"
45
+ spec.add_development_dependency "redis", "~> 4"
46
+ spec.add_development_dependency "rack-test"
47
+ spec.add_development_dependency "activerecord"
48
+ spec.add_development_dependency "mysql2"
49
+ spec.add_development_dependency "wetransfer_style"
50
+
51
+ # For more information and examples about making a new gem, checkout our
52
+ # guide at: https://bundler.io/guides/creating_gem.html
53
+ end
@@ -0,0 +1,54 @@
1
+ # This backend currently only works with mysql2 since it uses advisory locks
2
+ class Idempo::ActiveRecordBackend
3
+ def self.create_table(via_migration)
4
+ via_migration.create_table 'idempo_responses', charset: 'utf8mb4', collation: 'utf8mb4_unicode_ci' do |t|
5
+ t.string :idempotent_request_key, index: true, unique: true, null: false
6
+ t.datetime :expire_at, index: true, null: false
7
+ t.binary :idempotent_response_payload, size: :medium
8
+ t.timestamps
9
+ end
10
+ end
11
+
12
+ class Store < Struct.new(:key, :model)
13
+ def lookup
14
+ model.where(idempotent_request_key: key).where('expire_at > ?', Time.now).first&.idempotent_response_payload
15
+ end
16
+
17
+ def store(data:, ttl:)
18
+ # MySQL does not support datetime with subsecont precision, so ceil() it is
19
+ expire_at = Time.now.utc + ttl.ceil
20
+ model.transaction do
21
+ model.where(idempotent_request_key: key).delete_all
22
+ model.create(idempotent_request_key: key, idempotent_response_payload: data, expire_at: expire_at)
23
+ end
24
+ true
25
+ end
26
+ end
27
+
28
+ def initialize
29
+ require 'active_record'
30
+ end
31
+
32
+ # Allows the model to be defined lazily without having to require active_record when this module gets loaded
33
+ def model
34
+ @model_class ||= Class.new(ActiveRecord::Base) do
35
+ self.table_name = 'idempo_responses'
36
+ end
37
+ end
38
+
39
+ def with_idempotency_key(request_key)
40
+ db_safe_key = Base64.strict_encode64(request_key)
41
+
42
+ lock_name = "idempo_%s" % db_safe_key[0..48]
43
+ quoted_lock_name = model.connection.quote(lock_name) # Note there is a limit of 64 bytes on the lock name
44
+ did_acquire = model.connection.select_value("SELECT GET_LOCK(%s, %d)" % [quoted_lock_name, 0])
45
+
46
+ raise Idempo::ConcurrentRequest unless did_acquire == 1
47
+
48
+ begin
49
+ yield(Store.new(db_safe_key, model))
50
+ ensure
51
+ model.connection.select_value("SELECT RELEASE_LOCK(%s)" % quoted_lock_name)
52
+ end
53
+ end
54
+ end
@@ -0,0 +1,45 @@
1
+ class Idempo::MemoryBackend
2
+ def initialize
3
+ require 'set'
4
+ require_relative 'response_store'
5
+
6
+ @requests_in_flight_mutex = Mutex.new
7
+ @in_progress = Set.new
8
+ @store_mutex = Mutex.new
9
+ @response_store = Idempo::ResponseStore.new
10
+ end
11
+
12
+ class Store < Struct.new(:store_mutex, :response_store, :key, keyword_init: true)
13
+ def lookup
14
+ store_mutex.synchronize do
15
+ response_store.lookup(key)
16
+ end
17
+ end
18
+
19
+ def store(data:, ttl:)
20
+ store_mutex.synchronize do
21
+ response_store.save(key, data, ttl)
22
+ end
23
+ end
24
+ end
25
+
26
+ def with_idempotency_key(request_key)
27
+ did_insert = @requests_in_flight_mutex.synchronize do
28
+ if @in_progress.include?(request_key)
29
+ false
30
+ else
31
+ @in_progress << request_key
32
+ true
33
+ end
34
+ end
35
+
36
+ raise Idempo::ConcurrentRequest unless did_insert
37
+
38
+ store = Store.new(store_mutex: @store_mutex, response_store: @response_store, key: request_key)
39
+ begin
40
+ yield(store)
41
+ ensure
42
+ @requests_in_flight_mutex.synchronize { @in_progress.delete(request_key) }
43
+ end
44
+ end
45
+ end
@@ -0,0 +1,104 @@
1
+ class Idempo::RedisBackend
2
+ # The TTL value for the lock, which is used if the process
3
+ # holding the lock crashes or gets killed
4
+ LOCK_TTL_SECONDS = 5 * 60
5
+
6
+ # See https://redis.io/topics/distlock
7
+ DELETE_BY_KEY_AND_VALUE_SCRIPT = <<~EOL
8
+ redis.replicate_commands()
9
+ if redis.call("get",KEYS[1]) == ARGV[1] then
10
+ -- we are still holding the lock, release it
11
+ redis.call("del",KEYS[1])
12
+ return "ok"
13
+ else
14
+ -- someone else holds the lock or it has expired
15
+ return "lock_lost"
16
+ end
17
+ EOL
18
+
19
+ # See https://redis.io/topics/distlock as well as a rebuttal in
20
+ # https://martin.kleppmann.com/2016/02/08/how-to-do-distributed-locking.html
21
+ SET_WITH_TTL_IF_LOCK_STILL_HELD_SCRIPT = <<~EOL
22
+ redis.replicate_commands()
23
+ if redis.call("get", KEYS[1]) == ARGV[1] then
24
+ -- we are still holding the lock, we can go ahead and set it
25
+ redis.call("set", KEYS[2], ARGV[2], "px", ARGV[3])
26
+ return "ok"
27
+ else
28
+ return "lock_lost"
29
+ end
30
+ EOL
31
+
32
+ class Store < Struct.new(:redis_pool, :key, :lock_redis_key, :lock_token, keyword_init: true)
33
+ def lookup
34
+ response_redis_key = "idempo:response:#{key}"
35
+ redis_pool.with do |r|
36
+ bin_str = r.get(response_redis_key)
37
+ bin_str&.force_encoding(Encoding::BINARY)
38
+ end
39
+ end
40
+
41
+ def store(data:, ttl:)
42
+ response_redis_key = "idempo:response:#{key}"
43
+ ttl_millis = (ttl * 1000.0).round
44
+
45
+ # We save our payload using a script, and we will _only_ save it if our lock is still held.
46
+ # If our lock expires during the request - for example our app.call takes too long -
47
+ # we might have lost it, and another request has already saved a payload on our behalf. At this point
48
+ # we have no guarantee that our response was generated exclusively, or that the response that was generated
49
+ # by our "competitor" is equal to ours, or that a "competing" request is not holding our lock and executing the
50
+ # same workload as we just did. The only sensible thing to do when we encounter this is to actually _skip_ the write.
51
+ keys = [lock_redis_key, response_redis_key]
52
+ argv = [lock_token, data.force_encoding(Encoding::BINARY), ttl_millis]
53
+ outcome_of_save = redis_pool.with do |r|
54
+ Idempo::RedisBackend.eval_or_evalsha(r, SET_WITH_TTL_IF_LOCK_STILL_HELD_SCRIPT, keys: keys, argv: argv)
55
+ end
56
+
57
+ Measurometer.increment_counter('idempo.redis_lock_when_storing', 1, outcome: outcome_of_save)
58
+ end
59
+ end
60
+
61
+ class NullPool < Struct.new(:redis)
62
+ def with
63
+ yield redis
64
+ end
65
+ end
66
+
67
+ def initialize(redis_or_connection_pool)
68
+ require 'redis'
69
+ require 'securerandom'
70
+ @redis_pool = redis_or_connection_pool.respond_to?(:with) ? redis_or_connection_pool : NullPool.new(redis_or_connection_pool)
71
+ end
72
+
73
+ def with_idempotency_key(request_key)
74
+ lock_key = "idempo:lock:#{request_key}"
75
+ token = SecureRandom.bytes(32)
76
+ did_acquire = @redis_pool.with { |r| r.set(lock_key, token, nx: true, ex: LOCK_TTL_SECONDS) }
77
+
78
+ raise Idempo::ConcurrentRequest unless did_acquire
79
+
80
+ begin
81
+ store = Store.new(redis_pool: @redis_pool, lock_redis_key: lock_key, lock_token: token, key: request_key)
82
+ yield(store)
83
+ ensure
84
+ outcome_of_del = @redis_pool.with do |r|
85
+ Idempo::RedisBackend.eval_or_evalsha(r, DELETE_BY_KEY_AND_VALUE_SCRIPT, keys: [lock_key], argv: [token])
86
+ end
87
+ Measurometer.increment_counter('idempo_redis_release_lock', 1, outcome: outcome_of_del)
88
+ end
89
+ end
90
+
91
+ def self.eval_or_evalsha(redis, script_code, keys:, argv:)
92
+ script_sha = Digest::SHA1.hexdigest(script_code)
93
+ redis.evalsha(script_sha, keys: keys, argv: argv)
94
+ rescue Redis::CommandError => e
95
+ if e.message.include? "NOSCRIPT"
96
+ # The Redis server has never seen this script before. Needs to run only once in the entire lifetime
97
+ # of the Redis server, until the script changes - in which case it will be loaded under a different SHA
98
+ redis.script(:load, script_code)
99
+ retry
100
+ else
101
+ raise e
102
+ end
103
+ end
104
+ end
@@ -0,0 +1,57 @@
1
+ class Idempo::ResponseStore
2
+ ExpiryHandle = Struct.new(:key, :expire_at)
3
+ StoredResponse = Struct.new(:key, :expire_at, :payload)
4
+
5
+ def initialize
6
+ @values = {}
7
+ @expiries = []
8
+ end
9
+
10
+ def save(key, value, expire_in)
11
+ prune
12
+ exp = expire_in + Process.clock_gettime(Process::CLOCK_MONOTONIC)
13
+ res = StoredResponse.new(key, exp, value)
14
+ expiry_handle = ExpiryHandle.new(key, exp)
15
+ binary_insert(@expiries, expiry_handle, &:expire_at)
16
+ @values[key] = res
17
+ end
18
+
19
+ def lookup(key)
20
+ prune
21
+ stored = @values[key]
22
+ return unless stored
23
+ return stored.payload if stored.expire_at > Process.clock_gettime(Process::CLOCK_MONOTONIC)
24
+ @values.delete(key)
25
+ nil
26
+ end
27
+
28
+ private
29
+
30
+ def prune
31
+ now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
32
+ items_to_delete = remove_lower_than(@expiries, now, &:expire_at)
33
+ items_to_delete.each do |expiry_handle|
34
+ @values.delete(expiry_handle.key) if @values[expiry_handle.key] && @values[expiry_handle.key].expire_at < now
35
+ end
36
+ end
37
+
38
+ def binary_insert(array, item, &property_getter)
39
+ at_i = array.bsearch_index do |stored_item|
40
+ yield(stored_item) <= yield(item)
41
+ end
42
+ at_i ? array.insert(at_i, item) : array.push(item)
43
+ end
44
+
45
+ def remove_lower_than(array, threshold_value, &property_getter)
46
+ at_i = array.bsearch_index do |stored_item|
47
+ yield(stored_item) <= threshold_value
48
+ end
49
+ if at_i
50
+ array[at_i..array.length].tap do |_deleted_items|
51
+ array.replace(array[0..at_i])
52
+ end
53
+ else
54
+ []
55
+ end
56
+ end
57
+ end
@@ -0,0 +1,5 @@
1
+ # frozen_string_literal: true
2
+
3
+ class Idempo
4
+ VERSION = "0.1.1"
5
+ end
data/lib/idempo.rb ADDED
@@ -0,0 +1,169 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'zlib'
4
+ require 'msgpack'
5
+ require 'base64'
6
+ require 'digest'
7
+ require 'json'
8
+ require 'measurometer'
9
+
10
+ require_relative "idempo/version"
11
+ require_relative "idempo/memory_backend"
12
+ require_relative "idempo/redis_backend"
13
+ require_relative "idempo/active_record_backend"
14
+
15
+ class Idempo
16
+ DEFAULT_TTL = 30
17
+ SAVED_RESPONSE_BODY_SIZE_LIMIT = 4 * 1024 * 1024
18
+
19
+ class Error < StandardError; end
20
+
21
+ class ConcurrentRequest < Error; end
22
+
23
+ class MalformedIdempotencyKey < Error; end
24
+
25
+ def initialize(app, backend: MemoryBackend.new)
26
+ @backend = backend
27
+ @app = app
28
+ end
29
+
30
+ def call(env)
31
+ req = Rack::Request.new(env)
32
+ return @app.call(env) if request_verb_idempotent?(req)
33
+ return @app.call(env) unless idempotency_key_header = extract_idempotency_key_from(env)
34
+
35
+ # The RFC requires that the Idempotency-Key header value is enclosed in quotes
36
+ idempotency_key_header = unquote(idempotency_key_header)
37
+ raise MalformedIdempotencyKey if idempotency_key_header == ''
38
+
39
+ fingerprint = compute_request_fingerprint(req)
40
+ request_key = "#{idempotency_key_header}_#{fingerprint}"
41
+
42
+ @backend.with_idempotency_key(request_key) do |store|
43
+ if stored_response = store.lookup
44
+ Measurometer.increment_counter('idempo.served', 1, via: 'cached')
45
+ return from_persisted_response(stored_response)
46
+ end
47
+
48
+ status, headers, body = @app.call(env)
49
+
50
+ if response_may_be_persisted?(status, headers, body)
51
+ expires_in_seconds = (headers.delete('X-Idempo-Persist-For-Seconds') || DEFAULT_TTL).to_i
52
+ # Body is replaced with a cached version since a Rack response body is not rewindable
53
+ marshaled_response, body = serialize_response(status, headers, body)
54
+ store.store(data: marshaled_response, ttl: expires_in_seconds)
55
+ end
56
+
57
+ Measurometer.increment_counter('idempo.served', 1, via: 'stored')
58
+ [status, headers, body]
59
+ end
60
+ rescue MalformedIdempotencyKey
61
+ res = {
62
+ ok: false,
63
+ error: {
64
+ message: "The Idempotency-Key header provided was empty"
65
+ }
66
+ }
67
+ [400, {'Content-Type' => 'application/json'}, [JSON.pretty_generate(res)]]
68
+ rescue ConcurrentRequest
69
+ Measurometer.increment_counter('idempo.served', 1, via: 'concurrent-request-error')
70
+ res = {
71
+ ok: false,
72
+ error: {
73
+ message: "Another request with this idempotency key is still in progress, please try again later"
74
+ }
75
+ }
76
+ [429, {'Retry-After' => '2', 'Content-Type' => 'application/json'}, [JSON.pretty_generate(res)]]
77
+ end
78
+
79
+ private
80
+
81
+ def from_persisted_response(marshaled_response)
82
+ if marshaled_response[-2..-1] != ':1'
83
+ raise Error, "Unknown serialization of the marshaled response"
84
+ else
85
+ MessagePack.unpack(Zlib.inflate(marshaled_response[0..-3]))
86
+ end
87
+ end
88
+
89
+ def serialize_response(status, headers, rack_response_body)
90
+ # Buffer the Rack response body, we can only do that once (it is non-rewindable)
91
+ body_chunks = []
92
+ rack_response_body.each { |chunk| body_chunks << chunk.dup }
93
+ rack_response_body.close if rack_response_body.respond_to?(:close)
94
+
95
+ # Only keep headers which are strings
96
+ stringified_headers = headers.each_with_object({}) do |(header, value), filtered|
97
+ filtered[header] = value if value.is_a?(String)
98
+ end
99
+
100
+ message_packed_str = MessagePack.pack([status, stringified_headers, body_chunks])
101
+ deflated_message_packed_str = Zlib.deflate(message_packed_str) + ":1"
102
+ Measurometer.increment_counter('idempo.response_total_generated_bytes', deflated_message_packed_str.bytesize)
103
+ Measurometer.add_distribution_value('idempo.response_size_bytes', deflated_message_packed_str.bytesize)
104
+
105
+ # Add the version specifier at the end, because slicing a string in Ruby at the end
106
+ # (when we unserialize our response again) does a realloc, while slicing at the start
107
+ # does not
108
+ [deflated_message_packed_str, body_chunks]
109
+ end
110
+
111
+ def response_may_be_persisted?(status, headers, body)
112
+ return false if headers.delete('X-Idempo-Policy') == 'no-store'
113
+ return false unless status_may_be_persisted?(status)
114
+ return false unless body_size_within_limit?(headers, body)
115
+ true
116
+ end
117
+
118
+ def body_size_within_limit?(response_headers, body)
119
+ return response_headers['Content-Length'].to_i <= SAVED_RESPONSE_BODY_SIZE_LIMIT if response_headers['Content-Length']
120
+
121
+ return false unless body.is_a?(Array) # Arbitrary iterable of unknown size
122
+
123
+ precomputed_body_size = body.inject(0) { |sum, chunk| sum + chunk.bytesize }
124
+ precomputed_body_size <= SAVED_RESPONSE_BODY_SIZE_LIMIT
125
+ end
126
+
127
+ def status_may_be_persisted?(status)
128
+ case status
129
+ when 200..400
130
+ true
131
+ when 429, 425
132
+ false
133
+ when 400..499
134
+ true
135
+ else
136
+ false
137
+ end
138
+ end
139
+
140
+ def compute_request_fingerprint(req)
141
+ d = Digest::SHA256.new
142
+ d << req.url << "\n"
143
+ d << req.request_method << "\n"
144
+ while chunk = req.env['rack.input'].read(1024 * 65)
145
+ d << chunk
146
+ end
147
+ Base64.strict_encode64(d.digest)
148
+ ensure
149
+ req.env['rack.input'].rewind
150
+ end
151
+
152
+ def extract_idempotency_key_from(env)
153
+ env['HTTP_IDEMPOTENCY_KEY'] || env['HTTP_X_IDEMPOTENCY_KEY']
154
+ end
155
+
156
+ def request_verb_idempotent?(request)
157
+ request.get? || request.head? || request.options?
158
+ end
159
+
160
+ def unquote(str)
161
+ # Do not use regular expressions so that we don't have to thing about a catastrophic lookahead
162
+ double_quote = '"'
163
+ if str.start_with?(double_quote) && str.end_with?(double_quote)
164
+ str[1..-2]
165
+ else
166
+ str
167
+ end
168
+ end
169
+ end
metadata ADDED
@@ -0,0 +1,205 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: idempo
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.1
5
+ platform: ruby
6
+ authors:
7
+ - Julik Tarkhanov
8
+ - Pablo Crivella
9
+ autorequire:
10
+ bindir: exe
11
+ cert_chain: []
12
+ date: 2021-10-19 00:00:00.000000000 Z
13
+ dependencies:
14
+ - !ruby/object:Gem::Dependency
15
+ name: rack
16
+ requirement: !ruby/object:Gem::Requirement
17
+ requirements:
18
+ - - ">="
19
+ - !ruby/object:Gem::Version
20
+ version: '0'
21
+ type: :runtime
22
+ prerelease: false
23
+ version_requirements: !ruby/object:Gem::Requirement
24
+ requirements:
25
+ - - ">="
26
+ - !ruby/object:Gem::Version
27
+ version: '0'
28
+ - !ruby/object:Gem::Dependency
29
+ name: msgpack
30
+ requirement: !ruby/object:Gem::Requirement
31
+ requirements:
32
+ - - ">="
33
+ - !ruby/object:Gem::Version
34
+ version: '0'
35
+ type: :runtime
36
+ prerelease: false
37
+ version_requirements: !ruby/object:Gem::Requirement
38
+ requirements:
39
+ - - ">="
40
+ - !ruby/object:Gem::Version
41
+ version: '0'
42
+ - !ruby/object:Gem::Dependency
43
+ name: measurometer
44
+ requirement: !ruby/object:Gem::Requirement
45
+ requirements:
46
+ - - "~>"
47
+ - !ruby/object:Gem::Version
48
+ version: '1.3'
49
+ type: :runtime
50
+ prerelease: false
51
+ version_requirements: !ruby/object:Gem::Requirement
52
+ requirements:
53
+ - - "~>"
54
+ - !ruby/object:Gem::Version
55
+ version: '1.3'
56
+ - !ruby/object:Gem::Dependency
57
+ name: rake
58
+ requirement: !ruby/object:Gem::Requirement
59
+ requirements:
60
+ - - "~>"
61
+ - !ruby/object:Gem::Version
62
+ version: '13.0'
63
+ type: :development
64
+ prerelease: false
65
+ version_requirements: !ruby/object:Gem::Requirement
66
+ requirements:
67
+ - - "~>"
68
+ - !ruby/object:Gem::Version
69
+ version: '13.0'
70
+ - !ruby/object:Gem::Dependency
71
+ name: rspec
72
+ requirement: !ruby/object:Gem::Requirement
73
+ requirements:
74
+ - - "~>"
75
+ - !ruby/object:Gem::Version
76
+ version: '3.0'
77
+ type: :development
78
+ prerelease: false
79
+ version_requirements: !ruby/object:Gem::Requirement
80
+ requirements:
81
+ - - "~>"
82
+ - !ruby/object:Gem::Version
83
+ version: '3.0'
84
+ - !ruby/object:Gem::Dependency
85
+ name: redis
86
+ requirement: !ruby/object:Gem::Requirement
87
+ requirements:
88
+ - - "~>"
89
+ - !ruby/object:Gem::Version
90
+ version: '4'
91
+ type: :development
92
+ prerelease: false
93
+ version_requirements: !ruby/object:Gem::Requirement
94
+ requirements:
95
+ - - "~>"
96
+ - !ruby/object:Gem::Version
97
+ version: '4'
98
+ - !ruby/object:Gem::Dependency
99
+ name: rack-test
100
+ requirement: !ruby/object:Gem::Requirement
101
+ requirements:
102
+ - - ">="
103
+ - !ruby/object:Gem::Version
104
+ version: '0'
105
+ type: :development
106
+ prerelease: false
107
+ version_requirements: !ruby/object:Gem::Requirement
108
+ requirements:
109
+ - - ">="
110
+ - !ruby/object:Gem::Version
111
+ version: '0'
112
+ - !ruby/object:Gem::Dependency
113
+ name: activerecord
114
+ requirement: !ruby/object:Gem::Requirement
115
+ requirements:
116
+ - - ">="
117
+ - !ruby/object:Gem::Version
118
+ version: '0'
119
+ type: :development
120
+ prerelease: false
121
+ version_requirements: !ruby/object:Gem::Requirement
122
+ requirements:
123
+ - - ">="
124
+ - !ruby/object:Gem::Version
125
+ version: '0'
126
+ - !ruby/object:Gem::Dependency
127
+ name: mysql2
128
+ requirement: !ruby/object:Gem::Requirement
129
+ requirements:
130
+ - - ">="
131
+ - !ruby/object:Gem::Version
132
+ version: '0'
133
+ type: :development
134
+ prerelease: false
135
+ version_requirements: !ruby/object:Gem::Requirement
136
+ requirements:
137
+ - - ">="
138
+ - !ruby/object:Gem::Version
139
+ version: '0'
140
+ - !ruby/object:Gem::Dependency
141
+ name: wetransfer_style
142
+ requirement: !ruby/object:Gem::Requirement
143
+ requirements:
144
+ - - ">="
145
+ - !ruby/object:Gem::Version
146
+ version: '0'
147
+ type: :development
148
+ prerelease: false
149
+ version_requirements: !ruby/object:Gem::Requirement
150
+ requirements:
151
+ - - ">="
152
+ - !ruby/object:Gem::Version
153
+ version: '0'
154
+ description: Provides idempotency keys for Rack applications.
155
+ email:
156
+ - me@julik.nl
157
+ - pablocrivella@gmail.com
158
+ executables: []
159
+ extensions: []
160
+ extra_rdoc_files: []
161
+ files:
162
+ - ".gitignore"
163
+ - ".rubocop.yml"
164
+ - CHANGELOG.md
165
+ - Gemfile
166
+ - LICENSE.txt
167
+ - README.md
168
+ - Rakefile
169
+ - bin/console
170
+ - bin/setup
171
+ - idempo.gemspec
172
+ - lib/idempo.rb
173
+ - lib/idempo/active_record_backend.rb
174
+ - lib/idempo/memory_backend.rb
175
+ - lib/idempo/redis_backend.rb
176
+ - lib/idempo/response_store.rb
177
+ - lib/idempo/version.rb
178
+ homepage: https://github.com/julik/idempo
179
+ licenses:
180
+ - MIT
181
+ metadata:
182
+ allowed_push_host: https://rubygems.org
183
+ homepage_uri: https://github.com/julik/idempo
184
+ source_code_uri: https://github.com/julik/idempo
185
+ changelog_uri: https://github.com/julik/idempo/blob/main/CHANGELOG.md
186
+ post_install_message:
187
+ rdoc_options: []
188
+ require_paths:
189
+ - lib
190
+ required_ruby_version: !ruby/object:Gem::Requirement
191
+ requirements:
192
+ - - ">="
193
+ - !ruby/object:Gem::Version
194
+ version: 2.4.0
195
+ required_rubygems_version: !ruby/object:Gem::Requirement
196
+ requirements:
197
+ - - ">="
198
+ - !ruby/object:Gem::Version
199
+ version: '0'
200
+ requirements: []
201
+ rubygems_version: 3.0.3
202
+ signing_key:
203
+ specification_version: 4
204
+ summary: Idempotency keys for all.
205
+ test_files: []