api_keys 0.5.1 → 0.7.0

data/lib/api_keys/helpers/token_session.rb

@@ -23,14 +23,36 @@ module ApiKeys
     # Deleting the payload from the session only consumes it in that session.
     # A copy of the session taken before retrieval (a copied CookieStore
     # cookie, or two requests racing with the same cookie) would still carry
-    # it. So each handoff also has a random ID, and the first retrieval
-    # records an atomic consumption marker for that ID in `Rails.cache`
-    # (`write(..., unless_exist: true)`). Any later retrieval of the same
-    # handoff finds the marker and gets nil. This guarantee needs a cache store
-    # that every application process shares and that implements
-    # `unless_exist:` atomically. With `ActiveSupport::Cache::NullStore` the
-    # marker cannot be recorded, so deleting the payload from the session is
-    # the only protection, as it was before 0.5.1.
+    # it. Two more layers make retrieval one-time across copies:
+    #
+    # 1. Durable (0.6.0): the first retrieval of a key's handoff claims the
+    #    key's `token_shown_at` column with one conditional UPDATE
+    #    (`... WHERE id = ? AND token_shown_at IS NULL`). The database makes
+    #    exactly one claim succeed, and a cache flush cannot undo it. Zero rows
+    #    changed (already shown, or the key is gone) returns nil. Installations
+    #    without the column skip this layer and behave as 0.5.1.
+    #
+    #    The claim is only durable once it commits, so it must run outside any
+    #    database transaction: the plaintext is returned before an enclosing
+    #    transaction could roll the claim back. A retrieval attempted inside an
+    #    open transaction on the ApiKey connection is therefore refused,
+    #    whatever the cache: it withholds the token with `:inside_transaction`,
+    #    records nothing, and leaves the handoff in the session, so the host
+    #    can retrieve it again outside the transaction. The check is Active
+    #    Record's transaction manager on that connection, so a transaction
+    #    opened with raw SQL (`BEGIN`) is not detected. The dashboard opens no
+    #    transaction itself; a host `around_action` that wraps its requests in
+    #    one sees the refusal.
+    # 2. Cache marker (0.5.1): each handoff has a random ID, and retrieval
+    #    records it in `Rails.cache` with `write(..., unless_exist: true)`;
+    #    a later retrieval of the same handoff finds the marker and gets nil.
+    #    With the column it is a second layer (it also covers handoffs that
+    #    name no key); without it, it is the only one, and it needs a cache
+    #    store every process shares, with an atomic `unless_exist:`, that
+    #    keeps the marker for the handoff lifetime. With
+    #    `ActiveSupport::Cache::NullStore` the marker cannot be recorded.
+    #
+    # Any failure in either layer withholds the token (fail closed).
     #
     # @example In your controller
     #   # After creating a key:
@@ -61,6 +83,48 @@ module ApiKeys
       HANDOFF_TTL = 10.minutes
       HANDOFF_ID_PATTERN = /\A[A-Za-z0-9_-]{22,64}\z/
       CONSUMPTION_CACHE_NAMESPACE = "api_keys:v1:token_handoff_consumed"
+
+      # The outcome of `retrieve_once_with_status`. `token` is set only when
+      # `status` is `:retrieved`. The other statuses are:
+      # - `:already_shown`: an authentic handoff for this key whose token an
+      #   earlier retrieval already released (a replayed copy of the session).
+      # - `:withheld`: the database or cache could not record the retrieval,
+      #   so the token is withheld rather than risk a second release.
+      # - `:inside_transaction`: retrieval ran inside an open Active Record
+      #   transaction on the ApiKey connection, where its one-time claim could
+      #   be rolled back after the token was shown. The token is withheld,
+      #   nothing is recorded, and the handoff stays in the session for a
+      #   retry outside the transaction.
+      # - `:missing`: no usable handoff for this key (none stored, expired,
+      #   tampered, or bound to another key).
+      #
+      # Inspection never prints the token.
+      Retrieval = Struct.new(:token, :status) do
+        def retrieved?
+          status == :retrieved
+        end
+
+        def already_shown?
+          status == :already_shown
+        end
+
+        # Struct's default inspection, string, pretty-print and JSON forms
+        # would all print the plaintext token; these say only whether there
+        # is one. Read it through #token.
+        def inspect
+          "#<#{self.class.name} status=#{status.inspect} token=#{token.nil? ? 'nil' : '[REDACTED]'}>"
+        end
+
+        alias_method :to_s, :inspect
+
+        def pretty_print(printer)
+          printer.text(inspect)
+        end
+
+        def as_json(*)
+          { "status" => status.to_s, "token" => (token.nil? ? nil : "[REDACTED]") }
+        end
+      end
       ENCRYPTION_CIPHER = "aes-256-gcm"
       ENCRYPTION_SALT = "api_keys/token_session/v2"
       ENCRYPTION_PURPOSE = "api_keys.token_session"
@@ -111,26 +175,46 @@ module ApiKeys
         # @param key [Symbol] Optional custom session key (default: :api_keys_new_token)
         # @return [String, nil] The token, or nil if not present
         def retrieve_once(session, key: DEFAULT_SESSION_KEY, api_key: nil, api_key_id: nil)
+          retrieve_once_with_status(session, key: key, api_key: api_key, api_key_id: api_key_id).token
+        end
+
+        # Same as `retrieve_once`, and also says why no token came back, so a
+        # page can tell a replayed handoff ("already shown") from a missing
+        # one. The token is consumed exactly as `retrieve_once` consumes it.
+        #
+        # @return [Retrieval]
+        def retrieve_once_with_status(session, key: DEFAULT_SESSION_KEY, api_key: nil, api_key_id: nil)
           payload = session.delete(key)
           expected_id = normalize_api_key_id(api_key_id || (api_key.id if api_key.respond_to?(:id)))
 
           # Plain string payloads from older versions remain readable only when
           # the caller does not request ID binding.
-          return payload if expected_id.nil? && valid_token_payload?(payload)
+          return Retrieval.new(payload, :retrieved).freeze if expected_id.nil? && valid_token_payload?(payload)
           decoded_payload = decode_payload(payload)
-          return nil unless decoded_payload
+          return Retrieval.new(nil, :missing).freeze unless decoded_payload
+
+          token = decoded_payload["token"] || decoded_payload[:token]
+          stored_id = decoded_payload["api_key_id"] || decoded_payload[:api_key_id]
+          bound_elsewhere = expected_id && stored_id.to_s != expected_id.to_s
 
           # Any authentic retrieval attempt uses the handoff up, matching the
           # session deletion above: a mismatched or replayed attempt never
           # leaves the handoff available to a later one.
-          return nil unless consume_handoff(decoded_payload)
+          status = consume_handoff(decoded_payload)
 
-          token = decoded_payload["token"] || decoded_payload[:token]
-          stored_id = decoded_payload["api_key_id"] || decoded_payload[:api_key_id]
-          return nil if expected_id && stored_id.to_s != expected_id.to_s
-          return nil unless valid_token_payload?(token)
+          # A handoff bound to another key says nothing about this one.
+          return Retrieval.new(nil, :missing).freeze if bound_elsewhere
 
-          token
+          if status == :inside_transaction
+            # Nothing was recorded, so the handoff goes back into the session
+            # and a retrieval outside the transaction can still release it.
+            session[key] = payload
+            return Retrieval.new(nil, :inside_transaction).freeze
+          end
+          return Retrieval.new(nil, status).freeze unless status == :consumed
+          return Retrieval.new(nil, :missing).freeze unless valid_token_payload?(token)
+
+          Retrieval.new(token, :retrieved).freeze
         end
 
         # Check if a token is available in the session without removing it.
@@ -196,23 +280,99 @@ module ApiKeys
           nil
         end
 
-        # Records the consumption marker. Answers true only to the first
-        # consumer of a handoff; every later one, and any cache failure, is
-        # treated as "no token" (fail closed).
+        # Uses the handoff up, durably first and then in the cache. Answers
+        # :consumed only to the first consumer of a handoff; a later one gets
+        # :already_shown, and any failure :withheld (fail closed).
         def consume_handoff(decoded_payload)
+          stored_id = decoded_payload["api_key_id"]
+          return refuse_inside_transaction(stored_id) if claim_could_roll_back?(stored_id)
+
+          durable = claim_durably(decoded_payload)
+          return durable unless durable.nil? || durable == :consumed
+
+          consume_cache_marker(decoded_payload)
+        rescue StandardError => error
+          log_warn "[ApiKeys Security] Token handoff could not be checked against the database (#{error.class}); withholding the token."
+          :withheld
+        end
+
+        # Whether this handoff has a durable claim that an open transaction on
+        # the api_keys connection could roll back after the token was shown.
+        # Raises on database errors (the caller withholds the token).
+        def claim_could_roll_back?(stored_id)
+          return false if stored_id.nil? || !defined?(::ApiKeys::ApiKey)
+          return false unless ::ApiKeys::ApiKey.column_names.include?(::ApiKeys::ApiKey::TOKEN_SHOWN_AT_COLUMN)
+
+          ::ApiKeys::ApiKey.connection_pool.with_connection(&:transaction_open?)
+        end
+
+        # Inside a transaction the claim is not durable until the host commits,
+        # and the token would be returned before that. No cache can stand in
+        # for it: a process-local store protects one process, and after a
+        # rollback another process would release the token again. So the
+        # retrieval is refused and records nothing (no claim, no marker). Only
+        # a read tells an already-used handoff apart, which is not put back.
+        def refuse_inside_transaction(stored_id)
+          return :already_shown if ::ApiKeys::ApiKey.token_handoff_claimable?(stored_id) == false
+
+          log_warn "[ApiKeys Security] Token handoff retrieval ran inside an open database transaction, where its " \
+                   "one-time claim could be rolled back; withholding the token and leaving the handoff in place. " \
+                   "Retrieve it outside the transaction."
+          :inside_transaction
+        end
+
+        # The durable layer: the key's `token_shown_at`, claimed by one
+        # conditional UPDATE. nil when there is no durable layer for this
+        # handoff (the column does not exist yet, or the handoff names no key).
+        def claim_durably(decoded_payload)
+          stored_id = decoded_payload["api_key_id"]
+          return nil if stored_id.nil? || !defined?(::ApiKeys::ApiKey)
+
+          case ::ApiKeys::ApiKey.claim_token_handoff!(stored_id)
+          when nil then nil
+          when true then :consumed
+          else :already_shown
+          end
+        rescue StandardError => error
+          log_warn "[ApiKeys Security] Token handoff could not be recorded in the database (#{error.class}); withholding the token."
+          :withheld
+        end
+
+        # The cache layer from 0.5.1: an atomic marker named by the handoff ID.
+        def consume_cache_marker(decoded_payload)
           cache = consumption_cache
-          return true unless cache
+          return :consumed unless cache
 
           marker = consumption_marker_key(decoded_payload)
-          return true unless marker
+          return :consumed unless marker
+          return :consumed if cache.write(marker, true, unless_exist: true, expires_in: HANDOFF_TTL)
 
-          cache.write(marker, true, unless_exist: true, expires_in: HANDOFF_TTL) ? true : false
+          # Refused: normally the marker is already there (a replay), but some
+          # stores also answer false on error, so only a marker that is
+          # really present counts as "already shown".
+          cache.exist?(marker) ? :already_shown : :withheld
         rescue StandardError => error
           log_warn "[ApiKeys Security] Token handoff consumption could not be recorded (#{error.class}); withholding the token."
-          false
+          :withheld
         end
 
+        # Read-only: whether either layer says the handoff was used up. A
+        # failed check reads as consumed (fail closed).
         def handoff_consumed?(decoded_payload)
+          durably_consumed?(decoded_payload) || cache_marker_present?(decoded_payload)
+        end
+
+        def durably_consumed?(decoded_payload)
+          stored_id = decoded_payload["api_key_id"]
+          return false if stored_id.nil? || !defined?(::ApiKeys::ApiKey)
+
+          ::ApiKeys::ApiKey.token_handoff_claimable?(stored_id) == false
+        rescue StandardError => error
+          log_warn "[ApiKeys Security] Token handoff consumption could not be checked in the database (#{error.class})."
+          true
+        end
+
+        def cache_marker_present?(decoded_payload)
           cache = consumption_cache
           return false unless cache
 

data/lib/api_keys/helpers/view_helpers.rb

@@ -19,6 +19,40 @@ module ApiKeys
     #   end
     #
     module ViewHelpers
+      # Plain-language reasons for every code ApiKeys::ApiKey::REFUSAL_CODES can
+      # stamp on a key. Written to finish the sentence "...refused, because".
+      REFUSAL_REASONS = {
+        "revoked_key" => "the key has been revoked",
+        "expired_key" => "the key has expired",
+        "unknown_key_type" => "the key's type is no longer configured",
+        "unknown_environment" => "the key's environment is no longer configured",
+        "environment_misconfigured" => "the key's environment could not be verified",
+        "environment_mismatch" => "the key was used in the wrong environment",
+        "origin_not_allowed" => "the request came from a web origin the key does not allow",
+        "ip_not_allowed" => "the request came from an IP address the key does not allow",
+        "restriction_misconfigured" => "the key's stored request restrictions could not be verified",
+        "missing_scope" => "the key lacks a permission the endpoint requires",
+        "untrusted_key" => "the person behind the key no longer stands behind it"
+      }.freeze
+      UNKNOWN_REFUSAL_REASON = "the request was refused for an unrecognized reason"
+
+      # Explains the most recently recorded refusal of a key: a request that
+      # presented the key's real token and was refused by a policy. Never echoes
+      # the stored code itself, so a tampered value cannot reach the page.
+      #
+      # @param api_key [ApiKeys::ApiKey] The API key
+      # @return [String, nil] A reason, or nil when no refusal has been recorded
+      #
+      # @example
+      #   "Last refused because #{api_key_refusal_reason(@key)}"
+      #   # => "Last refused because the request came from a web origin the key does not allow"
+      #
+      def api_key_refusal_reason(api_key)
+        return nil if api_key.last_refused_at.nil?
+
+        REFUSAL_REASONS.fetch(api_key.last_refusal_code.to_s, UNKNOWN_REFUSAL_REASON)
+      end
+
       # Returns the status of an API key as a symbol.
       #
       # @param api_key [ApiKeys::ApiKey] The API key

data/lib/api_keys/idempotency.rb

@@ -0,0 +1,155 @@
+# frozen_string_literal: true
+
+require "active_support/concern"
+require_relative "authentication"
+
+module ApiKeys
+  # Writes that run once per Idempotency-Key, for an API controller that
+  # authenticates with `authenticate_api_key!`
+  # (draft-ietf-httpapi-idempotency-key-header, ApiKeys::IdempotencyKey):
+  #
+  #   class Api::BaseController < ActionController::API
+  #     include ApiKeys::Controller
+  #     include ApiKeys::Idempotency
+  #
+  #     before_action :authenticate_api_key!
+  #     around_action :run_once_per_idempotency_key
+  #
+  #     private
+  #
+  #     # The JSON for a record: the first answer and every retry use it.
+  #     def resource_json(record, **options)
+  #       record.as_json(only: %i[id name])
+  #     end
+  #
+  #     # Whether this API key may read the record now. A retry answers only
+  #     # records it may read: a record moved to another owner is not found.
+  #     def idempotent_replay_allowed?(record)
+  #       record.owner == current_api_owner
+  #     end
+  #   end
+  #
+  #   class Api::ProjectsController < Api::BaseController
+  #     def create
+  #       render_resource current_api_owner.projects.create!(project_params), status: :created
+  #     end
+  #   end
+  #
+  # The around_action runs outside any transaction the app opens: the key's
+  # claim must commit on its own (ApiKeys::IdempotencyKey.hold refuses).
+  #
+  # Every request but GET and HEAD needs the header. A write answers with
+  # `render_resource`, so the key keeps the record it made or changed, and a
+  # retry with the same key and the same request answers that record as it
+  # is now, with `Idempotent-Replayed: true`. Refusals render through
+  # `render_unauthorized`, like the gem's other refusals:
+  #
+  #   400 idempotency_key_required   no header, or longer than 255 characters
+  #   409 idempotency_key_in_use     a request with this key still runs
+  #   422 idempotency_key_reused     the key was used for another request
+  module Idempotency
+    extend ActiveSupport::Concern
+    include ApiKeys::Authentication
+
+    REPLAYED_HEADER = "Idempotent-Replayed"
+    # The methods a host defines: the gem cannot know how to render a record,
+    # or who may read it.
+    HOST_METHODS = %i[resource_json idempotent_replay_allowed?].freeze
+
+    private
+
+    # The around_action. A write and its answer commit together; a refused
+    # write (any status but 2xx) or an error commits nothing and leaves the
+    # key free for the next attempt.
+    def run_once_per_idempotency_key
+      return yield if reading_request?
+      raise ApiKeys::Error, "run authenticate_api_key! before run_once_per_idempotency_key" unless current_api_key
+
+      require_idempotency_host_methods!
+
+      key = request.headers[ApiKeys::IdempotencyKey::HEADER].to_s
+      if key.blank? || key.length > ApiKeys::IdempotencyKey::MAX_LENGTH
+        return render_unauthorized(
+          error_code: :idempotency_key_required, status: :bad_request,
+          message: "Send an Idempotency-Key header with every write: a unique text of " \
+                   "1 to #{ApiKeys::IdempotencyKey::MAX_LENGTH} characters, the same when you retry."
+        )
+      end
+
+      ApiKeys::IdempotencyKey.hold(current_api_key, key) do |record|
+        next replay_idempotent_request(record) if record.answered?
+
+        record.update!(request_digest: idempotency_request_digest)
+        yield
+        response.successful? ? record.answer!(response.status, @api_keys_idempotent_resource) : raise(ActiveRecord::Rollback)
+      end
+    rescue ApiKeys::IdempotencyKey::Busy
+      render_unauthorized(error_code: :idempotency_key_in_use, status: :conflict,
+                          message: "A request with this Idempotency-Key is still running. Retry in a moment.")
+    end
+
+    # A write's answer: the record it made or changed, as `resource_json`
+    # renders it. The key keeps the record, so a retry answers the same way.
+    # Options go to `resource_json` for this answer only: a secret shown
+    # once is never in a retry's answer.
+    def render_resource(record, status: :ok, **options)
+      @api_keys_idempotent_resource = record
+      render json: resource_json(record, **options), status: status
+    end
+
+    # The JSON for a record. Define it in the host controller.
+    def resource_json(_record, **_options)
+      raise NotImplementedError,
+            "#{self.class.name} must define resource_json(record, **options) for ApiKeys::Idempotency"
+    end
+
+    # Whether current_api_key may read +record+ now. Define it in the host
+    # controller. A retry runs no action, so this is the only check before a
+    # record's current state goes back to the client.
+    def idempotent_replay_allowed?(_record)
+      raise NotImplementedError,
+            "#{self.class.name} must define idempotent_replay_allowed?(record) for ApiKeys::Idempotency: " \
+            "whether current_api_key may read the record that a retry answers with"
+    end
+
+    # Both host methods are checked on the first write, not on the first
+    # retry, so a missing one shows in development and tests.
+    def require_idempotency_host_methods!
+      missing = HOST_METHODS.select { |name| method(name).owner == ApiKeys::Idempotency }
+      return if missing.empty?
+
+      raise NotImplementedError, "#{self.class.name} must define #{missing.join(' and ')} for ApiKeys::Idempotency: " \
+                                 "resource_json(record, **options) renders a record, and " \
+                                 "idempotent_replay_allowed?(record) answers whether current_api_key may read it"
+    end
+
+    # The answer to a retry: the record as it is now, never a stored body,
+    # so it is current and holds no secret. A record that no longer exists,
+    # or that this API key may no longer read, is not found
+    # (ActiveRecord::RecordNotFound, which Rails answers 404).
+    def replay_idempotent_request(record)
+      if record.request_digest != idempotency_request_digest
+        return render_unauthorized(error_code: :idempotency_key_reused, status: 422,
+                                   message: "This Idempotency-Key was used for another request. Use a new key.")
+      end
+
+      resource = record.resource
+      unless resource && idempotent_replay_allowed?(resource)
+        raise ActiveRecord::RecordNotFound, "the record of this Idempotency-Key is not found for this API key"
+      end
+
+      response.headers[REPLAYED_HEADER] = "true"
+      render json: resource_json(resource), status: record.response_status
+    end
+
+    # The request a key stands for: the method, the path, the query, and
+    # the body. The query and the body count apart, so a name that moves
+    # from one to the other makes another request.
+    def idempotency_request_digest
+      @api_keys_idempotency_request_digest ||= ApiKeys::IdempotencyKey.digest_of(
+        request.request_method, request.path,
+        { "query" => request.query_parameters, "body" => request.request_parameters }
+      )
+    end
+  end
+end

data/lib/api_keys/jobs/update_stats_job.rb

@@ -9,6 +9,11 @@ module ApiKeys
   module Jobs
     # Background job to update API key usage statistics (last_used_at, requests_count).
     # Enqueued by the Authentication concern after a successful request.
+    #
+    # With `refusal_code:` it records refused-but-identified traffic instead
+    # (last_refused_at, last_refusal_code, refusals_count). The two halves never
+    # share a write: a refusal is not a use, so it can never move last_used_at
+    # or requests_count, whatever the code says.
     class UpdateStatsJob < ActiveJob::Base
       include ApiKeys::Logging # Include logging helpers
 
@@ -21,7 +26,9 @@ module ApiKeys
       #
       # @param api_key_id [Integer, String] The ID of the ApiKey to update.
       # @param timestamp [Time] The timestamp of the request (when it was authenticated).
-      def perform(api_key_id, timestamp)
+      # @param refusal_code [String, nil] Present when the request was refused
+      #   after the key was identified; one of ApiKeys::ApiKey::REFUSAL_CODES.
+      def perform(api_key_id, timestamp, refusal_code: nil)
         api_key = ApiKey.find_by(id: api_key_id)
 
         unless api_key
@@ -40,6 +47,13 @@ module ApiKeys
           return
         end
 
+        # Any refusal marker at all, even a blank or unrecognized one, takes the
+        # refusal branch: the usage branch runs only for a genuine success.
+        unless refusal_code.nil?
+          record_refusal(api_key, timestamp, refusal_code)
+          return
+        end
+
         log_debug "[ApiKeys::Jobs::UpdateStatsJob] Updating stats for ApiKey ID: #{api_key_id}"
 
         # Jobs may execute out of order. Update only when this event is newer so
@@ -68,6 +82,36 @@ module ApiKeys
         log_error "[ApiKeys::Jobs::UpdateStatsJob] Unexpected error processing ApiKey ID: #{api_key_id} (#{error.class})."
         # Consider re-raising or using a dead-letter queue strategy depending on job system
       end
+
+      private
+
+      # Stamps a refusal. An installation that has not run
+      # `rails generate api_keys:add_key_activity` yet skips it quietly, so a
+      # deploy that ships code before the migration never fails these jobs.
+      def record_refusal(api_key, timestamp, refusal_code)
+        unless ApiKey.refusal_stats_columns?
+          log_debug "[ApiKeys::Jobs::UpdateStatsJob] Refusal stats columns are missing; skipping. " \
+                    "Run: rails generate api_keys:add_key_activity && rails db:migrate"
+          return
+        end
+
+        code = refusal_code.to_s
+        unless ApiKey::REFUSAL_CODES.include?(code)
+          log_warn "[ApiKeys::Jobs::UpdateStatsJob] Ignored an unrecognized refusal code for ApiKey ID: #{api_key.id}."
+          return
+        end
+
+        # Same ordering guard as last_used_at: an older refusal processed late
+        # cannot overwrite a newer one, and the code always matches its timestamp.
+        ApiKey
+          .where(id: api_key.id)
+          .where("last_refused_at IS NULL OR last_refused_at < ?", timestamp)
+          .update_all(last_refused_at: timestamp, last_refusal_code: code)
+
+        # Counted under the same switch as requests_count, which is also the only
+        # mode in which no refusal is debounced away, so the count is exact.
+        ApiKey.increment_counter(:refusals_count, api_key.id) if ApiKeys.configuration.track_requests_count
+      end
     end
   end
 end