idempo 0.1.3 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 057a7d3947de8a08ae62e9f7de2f2b05fae6607c1662aac78c7b12544050f41d
-  data.tar.gz: d0d8a84c2b208fd27b2301f975a3cf582d0c7b9e1b572d61574febf1f9f31cf4
+  metadata.gz: 96b24842e2d4d301bcf7a1470790a392fa1443a82ab2634e3aa81adbd173828c
+  data.tar.gz: 02243d85940b4e9f110fc6460eb925533d39ca62e331237b8e5ad4934f250703
 SHA512:
-  metadata.gz: bb98caf618c0c0c8987de2bd3e9e3a277c0d23d72d7b24ae90c22a678e44675bfb5a23468d23d0b73e9ccc7ec8e90704df553852b72a346b4f28e00d7a441fcc
-  data.tar.gz: 4c66050b6e219abd0922bd9a3e72740344c78c65bf0d35af08e2f423f4f2b835da377662b785802a37db4806f996e565027f1e3bdceff9d5344d59fa03344930
+  metadata.gz: b931753fa35dc613bf54416010c0c12a29855dedf1d9076958e7ef60e7594b1d90d53deae18dc5e5c9de6468916175ca49761413a54054c0c0523af552c29604
+  data.tar.gz: d32efc792665bd2695b39f3b98280196d9046e2e5b8affee072023f5e438477f064a1010927bf99fd755cf657d85796f19c28c51ba0086b10645ba95e5e2b66b

data/.github/workflows/ci.yml

@@ -0,0 +1,102 @@
+name: CI
+
+on:
+  - push
+  - pull_request
+
+env:
+  BUNDLE_PATH: vendor/bundle
+
+services:
+  mysql:
+    image: mysql:5.7
+    env:
+        MYSQL_ALLOW_EMPTY_PASSWORD: yes
+    ports:
+        - 3306
+    options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3
+  redis:
+    image: redis
+    options: --health-cmd "redis-cli ping" --health-interval 10s --health-timeout 5s --health-retries 5
+
+jobs:
+  lint:
+    name: Code Style
+    runs-on: ubuntu-18.04
+    if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository
+    strategy:
+      matrix:
+        ruby:
+          - '2.7'
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Setup Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+      - name: Gemfile Cache
+        uses: actions/cache@v2
+        with:
+          path: Gemfile.lock
+          key: ${{ runner.os }}-gemlock-${{ matrix.ruby }}-${{ hashFiles('Gemfile', 'idempo.gemspec') }}
+          restore-keys: |
+            ${{ runner.os }}-gemlock-${{ matrix.ruby }}-
+      - name: Bundle Cache
+        id: cache-gems
+        uses: actions/cache@v2
+        with:
+          path: vendor/bundle
+          key: ${{ runner.os }}-gems-${{ matrix.ruby }}-${{ hashFiles('Gemfile', 'Gemfile.lock', 'idempo.gemspec') }}
+          restore-keys: |
+            ${{ runner.os }}-gems-${{ matrix.ruby }}-
+            ${{ runner.os }}-gems-
+      - name: Bundle Install
+        if: steps.cache-gems.outputs.cache-hit != 'true'
+        run: bundle install --jobs 4 --retry 3
+      - name: Rubocop Cache
+        uses: actions/cache@v2
+        with:
+          path: ~/.cache/rubocop_cache
+          key: ${{ runner.os }}-rubocop-${{ hashFiles('.rubocop.yml') }}
+          restore-keys: |
+            ${{ runner.os }}-rubocop-
+      - name: Rubocop
+        run: bundle exec rubocop
+  test:
+    name: Specs
+    runs-on: ubuntu-18.04
+    if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository
+    strategy:
+      matrix:
+        ruby:
+          - '2.6'
+          - '3.0'
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Setup Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+      - name: Gemfile Cache
+        uses: actions/cache@v2
+        with:
+          path: Gemfile.lock
+          key: ${{ runner.os }}-gemlock-${{ matrix.ruby }}-${{ hashFiles('Gemfile', 'idempo.gemspec') }}
+          restore-keys: |
+            ${{ runner.os }}-gemlock-${{ matrix.ruby }}-
+      - name: Bundle Cache
+        id: cache-gems
+        uses: actions/cache@v2
+        with:
+          path: vendor/bundle
+          key: ${{ runner.os }}-gems-${{ matrix.ruby }}-${{ hashFiles('Gemfile', 'Gemfile.lock', 'idempo.gemspec') }}
+          restore-keys: |
+            ${{ runner.os }}-gems-${{ matrix.ruby }}-
+            ${{ runner.os }}-gems-
+      - name: Bundle Install
+        if: steps.cache-gems.outputs.cache-hit != 'true'
+        run: bundle install --jobs 4 --retry 3
+      - name: RSpec
+        run: bundle exec rspec

data/CHANGELOG.md

@@ -1,4 +1,8 @@
-## [Unreleased]
+## [0.2.0] - 2022-04-08
+
+- Allow setting the global default TTL for the cached responses
+- Allow customisation of the request key computation (so that the client can decide whether to include/exclude `Authorization` and the like)
+- Extract the error response generating apps into separate modules, to make them easier to override
 
 ## [0.1.0] - 2021-10-14
 

data/README.md

@@ -5,22 +5,6 @@ application, and the response can be cached, Idempo will provide both a concurre
 the idempotent response is already saved for this idempotency key and request fingerprint, the cached response is going to be served
 instead of calling your application.
 
-## Installation
-
-Add this line to your application's Gemfile:
-
-```ruby
-gem 'idempo'
-```
-
-And then execute:
-
-    $ bundle install
-
-Or install it yourself as:
-
-    $ gem install idempo
-
 ## Usage
 
 Idempo supports a number of backends, we recommend using Redis if you have multiple application servers / dynos and MemoryBackend if you are only using one single Puma worker. To initialize with Redis as backend pass the `backend:` parameter when adding the middleware:
@@ -32,7 +16,7 @@ use Idempo, backend: Idempo::RedisBackend.new(Rails.application.config.redis_con
 and to initialize with a memory store as backend:
 
 ```ruby
-use Idempo, backend: Idempo::MemoryBackend.new
+use Idempo
 ```
 
 In principle, the following requests qualify to be cached used the idempotency key:
@@ -42,12 +26,86 @@ In principle, the following requests qualify to be cached used the idempotency k
 
 The default time for storing the cache is 30 seconds from the moment the request has finished generating. The response is going to be buffered, then serialized using msgpack, then deflated. Idempo will not cache the response if its size cannot be known in advance, and if the size of the response body exceeds a reasonable size (4 MB is our limit for the time being) - this is to prevent your storage from filling up with very large responses.
 
-## Controlling the behavior of Idempo
+## Controlling the behavior of Idempo from your application
 
-You can control the behavior of Idempo using special headers:
+You can control the behavior of Idempo using special response headers:
 
 * Set `X-Idempo-Policy` to `no-store` to disable retention of the response even though it otherwise could be cached
-* Set `X-Idempo-Persist-For-Seconds` to decimal number of seconds to store your response fo 
+* Set `X-Idempo-Persist-For-Seconds` to a decimal number of seconds to store your response for. If your response contains time-sensitive data you might need to tweak the storage time.
+
+Idempo supports a number of data stores (here they are called "backends") - `MemoryBackend`, `ActiveRecordBackend`, `RedisBackend`.
+
+## Using memory for idempotency keys
+
+If you run only one Puma on one server (so multiple threads but one process) the `MemoryBackend` will work fine for you.
+
+* It uses a `Set` with a `Mutex` around it to store requests in progress
+* It uses a sorted array for expiration and cached responses.
+
+Needless to say, if your server terminates or restarts all the data goes dead with it. However 
+
+## Using your database for idempotency keys (via ActiveRecord)
+
+The relational database you already have is a perfectly fine place to store idempotency key locks and responses. A requirement for that is that your database supports some form of advisory locking - both PostgreSQL and MySQL do. First you will need to create a table for the records. The table is going to be called `idempo_responses`, and you need to add a migration in your Rails project for it:
+
+```bash
+$ rails g migration add_idempo_responses
+```
+
+and then add a migration like this:
+
+```ruby
+class AddIdempoResponses < ActiveRecord::Migration[7.0]
+  def change
+    Idempo::ActiveRecordBackend.create_table(self)
+  end
+end
+```
+
+Then configure Idempo to use the backend (in your `application.rb`):
+
+```ruby
+config.middleware.insert Idempo, backend: Idempo::ActiveRecordBackend.new
+```
+
+In your regular tasks (cron or Rake) you will want to add a call to delete old Idempo responses (there is an index on `expire_at`):
+
+```ruby
+Idempo::ActiveRecordBackend.new.model.where('expire_at_ < ?', Time.now).delete_all
+```
+
+## Using Redis for idempotency keys
+
+Redis is a near-perfect data store for idempotency keys, but it can have race conditions with locks if your application runs for too long or crashes very often. If you have Redis, initialize Idempo using the `RedisBackend`:
+
+```ruby
+use Idempo, backend: Idempo::RedisBackend.new
+```
+
+If you have a configured Redis connection pool (and you should) - pass it to the initializer:
+
+```ruby
+config.middleware.insert Idempo, backend: Idempo::RedisBackend.new(config.redis_connection_pool)
+```
+
+All data stored in Redis will have TTLs and will expire automatically.
+
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'idempo'
+```
+
+And then execute:
+
+    $ bundle install
+
+Or install it yourself as:
+
+    $ gem install idempo
 
 ## Development
 

data/idempo.gemspec

@@ -46,6 +46,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rack-test"
   spec.add_development_dependency "activerecord"
   spec.add_development_dependency "mysql2"
+  spec.add_development_dependency "pg"
   spec.add_development_dependency "wetransfer_style"
 
   # For more information and examples about making a new gem, checkout our

data/lib/idempo/active_record_backend.rb

@@ -3,7 +3,7 @@ class Idempo::ActiveRecordBackend
   def self.create_table(via_migration)
     via_migration.create_table 'idempo_responses', charset: 'utf8mb4', collation: 'utf8mb4_unicode_ci' do |t|
       t.string :idempotent_request_key, index: true, unique: true, null: false
-      t.datetime :expire_at, index: true, null: false
+      t.datetime :expire_at, index: true, null: false # Needs an index for cleanup
       t.binary :idempotent_response_payload, size: :medium
       t.timestamps
     end
@@ -25,6 +25,39 @@ class Idempo::ActiveRecordBackend
     end
   end
 
+  class PostgresLock
+    def acquire(conn, based_on_str)
+      acquisition_result = conn.select_value('SELECT pg_try_advisory_lock(%d)' % derive_lock_key(based_on_str))
+      [true, 't'].include?(acquisition_result)
+    end
+
+    def release(conn, based_on_str)
+      conn.select_value('SELECT pg_advisory_unlock(%d)' % derive_lock_key(based_on_str))
+    end
+
+    def derive_lock_key(from_str)
+      # The key must be a single bigint (signed long)
+      hash_bytes = Digest::SHA1.digest(from_str)
+      hash_bytes[0...8].unpack('l_').first
+    end
+  end
+
+  class MysqlLock
+    def acquire(connection, based_on_str)
+      did_acquire = connection.select_value("SELECT GET_LOCK(%s, %d)" % [connection.quote(derive_lock_name(based_on_str)), 0])
+      did_acquire == 1
+    end
+
+    def release(connection, based_on_str)
+      connection.select_value("SELECT RELEASE_LOCK(%s)" % connection.quote(derive_lock_name(based_on_str)))
+    end
+
+    def derive_lock_name(from_str)
+      db_safe_key = Base64.strict_encode64(from_str)
+      "idempo_%s" % db_safe_key[0...57] # Note there is a limit of 64 bytes on the lock name
+    end
+  end
+
   def initialize
     require 'active_record'
   end
@@ -37,18 +70,27 @@ class Idempo::ActiveRecordBackend
   end
 
   def with_idempotency_key(request_key)
-    db_safe_key = Base64.strict_encode64(request_key)
-
-    lock_name = "idempo_%s" % db_safe_key[0..48]
-    quoted_lock_name = model.connection.quote(lock_name) # Note there is a limit of 64 bytes on the lock name
-    did_acquire = model.connection.select_value("SELECT GET_LOCK(%s, %d)" % [quoted_lock_name, 0])
+    db_safe_key = Digest::SHA1.base64digest(request_key)
+    lock = lock_implementation_for_connection(model.connection)
 
-    raise Idempo::ConcurrentRequest unless did_acquire == 1
+    raise Idempo::ConcurrentRequest unless lock.acquire(model.connection, request_key)
 
     begin
       yield(Store.new(db_safe_key, model))
     ensure
-      model.connection.select_value("SELECT RELEASE_LOCK(%s)" % quoted_lock_name)
+      lock.release(model.connection, request_key)
+    end
+  end
+
+  private
+
+  def lock_implementation_for_connection(connection)
+    if connection.adapter_name =~ /^mysql2/i
+      MysqlLock.new
+    elsif connection.adapter_name =~ /^postgres/i
+      PostgresLock.new
+    else
+      raise "Unsupported database driver #{model.connection.adapter_name.downcase} - we don't know whether it supports advisory locks"
     end
   end
 end

data/lib/idempo/concurrent_request_error_app.rb

@@ -0,0 +1,15 @@
+require 'json'
+
+class Idempo::ConcurrentRequestErrorApp
+  RETRY_AFTER_SECONDS = 2.to_s
+
+  def self.call(env)
+    res = {
+      ok: false,
+      error: {
+        message: "Another request with this idempotency key is still in progress, please try again later"
+      }
+    }
+    [429, {'Retry-After' => RETRY_AFTER_SECONDS, 'Content-Type' => 'application/json'}, [JSON.pretty_generate(res)]]
+  end
+end

data/lib/idempo/malformed_key_error_app.rb

@@ -0,0 +1,13 @@
+require 'json'
+
+class Idempo::MalformedKeyErrorApp
+  def self.call(env)
+    res = {
+      ok: false,
+      error: {
+        message: "The Idempotency-Key header provided was empty or malformed"
+      }
+    }
+    [400, {'Content-Type' => 'application/json'}, [JSON.pretty_generate(res)]]
+  end
+end

data/lib/idempo/redis_backend.rb

@@ -64,7 +64,7 @@ class Idempo::RedisBackend
     end
   end
 
-  def initialize(redis_or_connection_pool)
+  def initialize(redis_or_connection_pool = Redis.new)
     require 'redis'
     require 'securerandom'
     @redis_pool = redis_or_connection_pool.respond_to?(:with) ? redis_or_connection_pool : NullPool.new(redis_or_connection_pool)

data/lib/idempo/request_fingerprint.rb

@@ -0,0 +1,15 @@
+module Idempo::RequestFingerprint
+  def self.call(idempotency_key, rack_request)
+    d = Digest::SHA256.new
+    d << idempotency_key << "\n"
+    d << rack_request.url << "\n"
+    d << rack_request.request_method << "\n"
+    d << rack_request.get_header('HTTP_AUTHORIZATION').to_s << "\n"
+    while chunk = rack_request.env['rack.input'].read(1024 * 65)
+      d << chunk
+    end
+    Base64.strict_encode64(d.digest)
+  ensure
+    rack_request.env['rack.input'].rewind
+  end
+end

data/lib/idempo/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 class Idempo
-  VERSION = "0.1.3"
+  VERSION = "0.2.0"
 end

data/lib/idempo.rb

@@ -8,12 +8,15 @@ require 'json'
 require 'measurometer'
 
 require_relative "idempo/version"
+require_relative "idempo/request_fingerprint"
 require_relative "idempo/memory_backend"
 require_relative "idempo/redis_backend"
 require_relative "idempo/active_record_backend"
+require_relative "idempo/malformed_key_error_app"
+require_relative "idempo/concurrent_request_error_app"
 
 class Idempo
-  DEFAULT_TTL = 30
+  DEFAULT_TTL_SECONDS = 30
   SAVED_RESPONSE_BODY_SIZE_LIMIT = 4 * 1024 * 1024
 
   class Error < StandardError; end
@@ -22,9 +25,13 @@ class Idempo
 
   class MalformedIdempotencyKey < Error; end
 
-  def initialize(app, backend: MemoryBackend.new)
+  def initialize(app, backend: MemoryBackend.new, malformed_key_error_app: MalformedKeyErrorApp, compute_fingerprint_via: RequestFingerprint, concurrent_request_error_app: ConcurrentRequestErrorApp, persist_for_seconds: DEFAULT_TTL_SECONDS)
     @backend = backend
     @app = app
+    @concurrent_request_error_app = concurrent_request_error_app
+    @malformed_key_error_app = malformed_key_error_app
+    @fingerprint_calculator = compute_fingerprint_via
+    @persist_for_seconds = persist_for_seconds.to_i
   end
 
   def call(env)
@@ -33,11 +40,10 @@ class Idempo
     return @app.call(env) unless idempotency_key_header = extract_idempotency_key_from(env)
 
     # The RFC requires that the Idempotency-Key header value is enclosed in quotes
-    idempotency_key_header = unquote(idempotency_key_header)
-    raise MalformedIdempotencyKey if idempotency_key_header == ''
+    idempotency_key_header_value = unquote(idempotency_key_header)
+    raise MalformedIdempotencyKey if idempotency_key_header_value == ''
 
-    fingerprint = compute_request_fingerprint(req)
-    request_key = "#{idempotency_key_header}_#{fingerprint}"
+    request_key = @fingerprint_calculator.call(idempotency_key_header_value, req)
 
     @backend.with_idempotency_key(request_key) do |store|
       if stored_response = store.lookup
@@ -47,8 +53,8 @@ class Idempo
 
       status, headers, body = @app.call(env)
 
+      expires_in_seconds = (headers.delete('X-Idempo-Persist-For-Seconds') || @persist_for_seconds).to_i
       if response_may_be_persisted?(status, headers, body)
-        expires_in_seconds = (headers.delete('X-Idempo-Persist-For-Seconds') || DEFAULT_TTL).to_i
         # Body is replaced with a cached version since a Rack response body is not rewindable
         marshaled_response, body = serialize_response(status, headers, body)
         store.store(data: marshaled_response, ttl: expires_in_seconds)
@@ -58,22 +64,11 @@ class Idempo
       [status, headers, body]
     end
   rescue MalformedIdempotencyKey
-    res = {
-      ok: false,
-      error: {
-        message: "The Idempotency-Key header provided was empty"
-      }
-    }
-    [400, {'Content-Type' => 'application/json'}, [JSON.pretty_generate(res)]]
+    Measurometer.increment_counter('idempo.responses_served_from', 1, from: 'malformed-idempotency-key')
+    @malformed_key_error_app.call(env)
   rescue ConcurrentRequest
     Measurometer.increment_counter('idempo.responses_served_from', 1, from: 'conflict-concurrent-request')
-    res = {
-      ok: false,
-      error: {
-        message: "Another request with this idempotency key is still in progress, please try again later"
-      }
-    }
-    [429, {'Retry-After' => '2', 'Content-Type' => 'application/json'}, [JSON.pretty_generate(res)]]
+    @concurrent_request_error_app.call(env)
   end
 
   private
@@ -94,7 +89,7 @@ class Idempo
 
     # Only keep headers which are strings
     stringified_headers = headers.each_with_object({}) do |(header, value), filtered|
-      filtered[header] = value if value.is_a?(String)
+      filtered[header] = value if !header.start_with?('rack.') && value.is_a?(String)
     end
 
     message_packed_str = MessagePack.pack([status, stringified_headers, body_chunks])
@@ -120,8 +115,7 @@ class Idempo
 
     return false unless body.is_a?(Array) # Arbitrary iterable of unknown size
 
-    precomputed_body_size = body.inject(0) { |sum, chunk| sum + chunk.bytesize }
-    precomputed_body_size <= SAVED_RESPONSE_BODY_SIZE_LIMIT
+    sum_of_string_bytesizes(body) <= SAVED_RESPONSE_BODY_SIZE_LIMIT
   end
 
   def status_may_be_persisted?(status)
@@ -137,19 +131,6 @@ class Idempo
     end
   end
 
-  def compute_request_fingerprint(req)
-    d = Digest::SHA256.new
-    d << req.url << "\n"
-    d << req.request_method << "\n"
-    d << req.get_header('HTTP_AUTHORIZATION').to_s << "\n"
-    while chunk = req.env['rack.input'].read(1024 * 65)
-      d << chunk
-    end
-    Base64.strict_encode64(d.digest)
-  ensure
-    req.env['rack.input'].rewind
-  end
-
   def extract_idempotency_key_from(env)
     env['HTTP_IDEMPOTENCY_KEY'] || env['HTTP_X_IDEMPOTENCY_KEY']
   end
@@ -158,8 +139,12 @@ class Idempo
     request.get? || request.head? || request.options?
   end
 
+  def sum_of_string_bytesizes(in_array)
+    in_array.inject(0) { |sum, chunk| sum + chunk.bytesize }
+  end
+
   def unquote(str)
-    # Do not use regular expressions so that we don't have to thing about a catastrophic lookahead
+    # Do not use regular expressions so that we don't have to think about a catastrophic lookahead
     double_quote = '"'
     if str.start_with?(double_quote) && str.end_with?(double_quote)
       str[1..-2]

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: idempo
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.2.0
 platform: ruby
 authors:
 - Julik Tarkhanov
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-10-20 00:00:00.000000000 Z
+date: 2022-04-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rack
@@ -137,6 +137,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: wetransfer_style
   requirement: !ruby/object:Gem::Requirement
@@ -159,6 +173,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/ci.yml"
 - ".gitignore"
 - ".rubocop.yml"
 - CHANGELOG.md
@@ -171,8 +186,11 @@ files:
 - idempo.gemspec
 - lib/idempo.rb
 - lib/idempo/active_record_backend.rb
+- lib/idempo/concurrent_request_error_app.rb
+- lib/idempo/malformed_key_error_app.rb
 - lib/idempo/memory_backend.rb
 - lib/idempo/redis_backend.rb
+- lib/idempo/request_fingerprint.rb
 - lib/idempo/response_store.rb
 - lib/idempo/version.rb
 homepage: https://github.com/julik/idempo