end_point_blank 0.6.0 → 0.12.0

data/lib/end_point_blank/token_unavailable_error.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+require_relative "target_url"
+
+module EndPointBlank
+  # Reopened with the same superclass in end_point_blank.rb; declared here too
+  # so this file can be required on its own.
+  class Error < StandardError; end
+
+  # Raised by {Authorization.header} when no access token can be obtained for
+  # an outbound call to a provider.
+  #
+  # There is deliberately no fallback. Before sc-1469 a failed mint silently
+  # produced "Basic base64(client_id:client_secret)" instead, which sent this
+  # service's own credential to whichever provider it was calling (and to
+  # that provider's intake). A client must never do that, so the call cannot
+  # be authorized and the caller has to decide what to do: retry, degrade, or
+  # fail its own request.
+  #
+  # `base_url` is the URL the token was wanted for with its userinfo, query
+  # and fragment removed ({TargetUrl.strip}), or nil when it could not be
+  # parsed. The raw value is never kept: the caller already has it, and a
+  # field on an exception reaches error reporting as surely as the message
+  # does.
+  #
+  # `failure` is the {AccessTokens::Failure} recorded for the mint, when one
+  # was recorded, so a handler can branch on `failure.outcome`
+  # (:credential_rejected, :request_rejected, :server_error,
+  # :transport_error) exactly as it would after calling
+  # {AccessTokens.last_failure} itself.
+  #
+  # A mint that raised is constructed with `unexpected: true` and reported as
+  # :transport_error, with the exception as `cause` (Ruby sets it when this
+  # is raised from the rescue). Its message is deliberately not copied into
+  # this one.
+  class TokenUnavailableError < Error
+    attr_reader :base_url, :failure
+
+    # @param base_url [String] the URL the token was wanted for
+    # @param failure [AccessTokens::Failure, nil] why the mint failed
+    # @param unexpected [Boolean] true when the mint raised rather than
+    #   reporting a failure
+    def initialize(base_url, failure = nil, unexpected: false)
+      @base_url = TargetUrl.strip(base_url)
+      @failure = failure
+      @unexpected = unexpected
+      super(build_message)
+    end
+
+    # @return [Boolean] true when the mint raised; `cause` holds what it raised
+    def unexpected?
+      @unexpected
+    end
+
+    # @return [Symbol, nil] the failure outcome, or nil when none was recorded
+    def outcome
+      failure&.outcome
+    end
+
+    # @return [Integer, nil] intake's HTTP status, or nil when it never answered
+    def status
+      failure&.status
+    end
+
+    private
+
+    UNPARSEABLE_URL = "the requested URL (not shown: it could not be parsed)"
+    private_constant :UNPARSEABLE_URL
+
+    # One fixed text per outcome, the same in every EndPointBlank SDK. Never
+    # intake's response body (Failure#reason) or an exception message: the
+    # message is what reaches logs and error reporting, and neither is ours
+    # to vouch for. Failure#reason stays available on #failure.
+    def reason # rubocop:disable Metrics/MethodLength
+      # Before the outcome, because a mint that raised is also :transport_error.
+      return "the token request failed unexpectedly" if unexpected?
+
+      http = status ? " (HTTP #{status})" : ""
+
+      case outcome
+      when :credential_rejected
+        "intake rejected this application's client credential#{http}; " \
+          "retrying cannot help -- re-issue the credential"
+      when :request_rejected
+        "intake refused the token request#{http}; check the URL and that a grant covers the target"
+      when :server_error
+        "intake failed to issue a token#{http}; this may be transient"
+      when :transport_error
+        "intake could not be reached (timeout, connection refused or retries exhausted); " \
+          "this may be transient"
+      else
+        "the token request failed for an unknown reason"
+      end
+    end
+
+    def build_message
+      "Could not mint an EndPointBlank access token for #{base_url || UNPARSEABLE_URL}: #{reason}. " \
+        "EndPointBlank never sends this service's client_id/client_secret to a provider, " \
+        "so there is no Basic-auth fallback and the call must not be made without a token."
+    end
+  end
+end

data/lib/end_point_blank/unauthorized_error.rb

@@ -1,4 +1,26 @@
+# frozen_string_literal: true
+
+require "json"
+
 module EndPointBlank
+  # Raised when a request fails authentication or authorization.
+  #
+  # This error is intentionally not logged by the middleware, as unauthorized
+  # access attempts are expected to occur.
+  #
+  # `status` carries the status intake answered the authenticate or authorize
+  # call with, so a handler can tell the two refusals apart: 401 means the
+  # credential was not accepted and the integrator should check it, 403 means
+  # the credential was fine but no grant covers this endpoint and the
+  # integrator should ask for one. Collapsing both to 401 sends them to debug
+  # the wrong thing. The other SDKs each carry the same value under their own
+  # idiomatic name -- `statusCode` in JS, `getStatusCode()` in Java,
+  # `status_code` in Python; this is Ruby's.
+  #
+  # It defaults to 401 so that `UnauthorizedError.new(message)` keeps working
+  # unchanged, and because 401 is the safe reading of a refusal that arrives
+  # with no status attached. The concerns never lean on that default: they pass
+  # intake's status, or 503 when intake did not answer at all.
   class UnauthorizedError < StandardError
     attr_reader :status
 
@@ -6,5 +28,63 @@ module EndPointBlank
       super(message)
       @status = status
     end
+
+    # Builds the error for a non-201 answer to intake's authenticate or
+    # authorize call. `action` is "Authentication" or "Authorization".
+    #
+    # One method rather than a copy per concern. `Rails::Authenticated` and
+    # `Rails::Authorized` were two transcriptions of one decision, and two
+    # copies is how one path acquires a fix the other does not. That is not
+    # hypothetical here: `authorize!` passed intake's status and read the body
+    # defensively, while `authenticate!` used `raise UnauthorizedError,
+    # "message"` -- the two-argument `raise Class, message` form, which
+    # structurally cannot pass a status however willing this class is to accept
+    # one -- and parsed the body before it had checked there was a body to
+    # parse. Both are fixed here, once.
+    #
+    # @param result [#status, #body, nil] intake's answer, or nil if it did not
+    #   answer at all.
+    # @param action [String] the word naming what was attempted.
+    # @return [UnauthorizedError]
+    def self.refusal_from(result, action)
+      if result.nil?
+        # Intake never answered at all, so nothing refused this caller and 401
+        # would blame a credential that was never judged. 503 says the true
+        # thing -- the check could not be made -- and is what the other four
+        # SDKs already send for this same case: JS and Java
+        # `response ? response.status : 503`, Python's `refusal_from`, and
+        # Elixir's literal `send_resp(503, ...)`. It is also the one case the
+        # "service unavailable" wording is actually true for.
+        #
+        # The wording deliberately has no "#{action} failed:" prefix, because
+        # that is what `authorize!` has always sent for this case and this
+        # method must not change what the working path produces for any input.
+        # The other SDKs prefix it; the status, which is what a handler
+        # branches on, agrees with them.
+        return new("#{action} service unavailable", 503)
+      end
+
+      # Intake's verdict verbatim. 401 tells an integrator to check the
+      # credential, 403 tells them to ask for a grant.
+      new("#{action} failed: #{reason_from(result)}", result.status)
+    end
+
+    # intake sends `{"error": "..."}`, but the SDK reaches it through a proxy
+    # and a WAF or load balancer can answer with an HTML page intake never
+    # generated. An unparseable body is therefore an expected case, not a
+    # swallowed failure: it falls back to the raw body so the operator still
+    # sees exactly what came back rather than an empty reason.
+    def self.reason_from(result)
+      parsed =
+        begin
+          JSON.parse(result.body)
+        rescue StandardError
+          nil
+        end
+
+      detail = parsed.is_a?(Hash) ? parsed["error"] : nil
+      detail || result.body
+    end
+    private_class_method :reason_from
   end
-end
+end

data/lib/end_point_blank/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module EndPointBlank
-  VERSION = "0.6.0"
+  VERSION = "0.12.0"
 end

data/lib/end_point_blank/writers/delayed_writer.rb

@@ -16,14 +16,46 @@ module EndPointBlank
     module DelayedWriter
       MAX_QUEUE_SIZE = 1000
       WARN_THROTTLE_SECONDS = 30
+      # Payloads per outbound request. One request per payload would multiply
+      # the host application's outbound traffic by its own request rate; the
+      # drain exists to amortise that.
+      BATCH_SIZE = 6
       # Fallback thread count used when Configuration#worker_count is unset,
       # preserving the previously-hardcoded pool size.
       DEFAULT_WORKER_COUNT = 2
+      # Applied after a worker iteration raises, doubling with each consecutive
+      # failure up to the cap. Surviving a defect must not mean retrying as fast
+      # as the CPU allows: a persistent failure should read as a slow, loud
+      # retry, not as a silent hot loop.
+      WORKER_BACKOFF_SECONDS = 0.1
+      MAX_WORKER_BACKOFF_SECONDS = 30
+
+      # sc-376: eager, so `queue`/`enqueue_mutex` are never nil for any
+      # worker thread. Every real includer (RequestWriter, ResponseWriter,
+      # ExceptionWriter, LogWriter) calls `super()` here before
+      # `start_threads`, so this always runs first, single-threaded, before
+      # any worker exists. An includer whose `initialize` skips `super`
+      # (some of this module's own specs, deliberately) never reaches this;
+      # `queue`/`enqueue_mutex` fall back to lazy `||=` for those.
+      def initialize
+        @queue = Queue.new
+        @enqueue_mutex = Mutex.new
+      end
 
       def direct_writer
         @direct_writer ||= DirectWriter.new(url)
       end
 
+      # `@queue ||= Queue.new` is a read, a nil check, then a write -- not
+      # atomic. Two threads racing this before either assigns could each
+      # allocate their own Queue, and the loser's is an orphan nothing will
+      # ever read from again. In production only this class's own worker
+      # threads could ever race here (every writer is a Singleton whose
+      # `initialize`, and so `super()` above, completes before any caller
+      # can reach `enqueue`), and `initialize` above now sets `@queue`
+      # before `start_threads` spawns a single one of them. The `||=` stays
+      # only as a fallback for an includer that skips this module's
+      # `initialize` entirely -- see the comment there.
       def queue
         @queue ||= Queue.new
       end
@@ -36,27 +68,7 @@ module EndPointBlank
         @threads = []
 
         worker_count.times do
-          @threads << Thread.new do
-            loop do
-              payload = queue.pop
-              payloads = [payload]
-              while (payload = pop_additional)
-                payloads << payload
-              end
-
-              payloads.compact!
-              while payloads.any?
-                list = payloads[0..5]
-                response = direct_writer.write(list)
-                if response.status < 299
-                  on_success(response) if respond_to?(:on_success)
-                elsif respond_to?(:on_failure)
-                  on_failure(response)
-                end
-                payloads -= list
-              end
-            end
-          end
+          @threads << Thread.new { run_worker }
         end
       end
 
@@ -78,8 +90,106 @@ module EndPointBlank
         EndPointBlank.logger.warn(message)
       end
 
+      # Logs a delivery or worker-loop error, through the same seam. Everything
+      # a worker recovers from goes through here: recovering quietly would trade
+      # one silent failure for another.
+      def log_error(message)
+        EndPointBlank.logger.error(message)
+      end
+
       private
 
+      # The body of a worker thread.
+      #
+      # A thread that dies here takes every future payload in the process with
+      # it - silently, until a restart - which is far worse than losing the
+      # batch in flight. So the loop catches every StandardError, says what it
+      # caught, and carries on.
+      #
+      # What it deliberately does not catch is anything outside StandardError:
+      # SystemExit, Interrupt, SignalException, NoMemoryError. Those mean the
+      # process itself is going down or is already broken, and a telemetry
+      # worker has no business arguing with that.
+      def run_worker
+        consecutive_failures = 0
+
+        loop do
+          drain_once
+          consecutive_failures = 0
+        rescue StandardError => e
+          consecutive_failures += 1
+          note_worker_error(e, consecutive_failures)
+          sleep(worker_backoff(consecutive_failures))
+        end
+      end
+
+      # Blocks for the next payload, then takes everything else already waiting
+      # so a burst leaves as a few batches rather than one request per payload.
+      #
+      # The batches are cut by position, and that is the point. This used to
+      # take a prefix and then remove it with `payloads -= list`, and Array#-
+      # removes every element *equal to* one in the batch rather than the ones
+      # actually sent: seven byte-identical payloads with a batch size of six
+      # meant six delivered and all seven gone. Nothing counted the loss and the
+      # queue drained normally, so it read as if nothing had happened.
+      #
+      # Equal payloads are ordinary, not exotic. Payloads are hashes, and two
+      # requests to the same endpoint from the same application in the same
+      # environment differ only in high-cardinality fields; `sent_at` at
+      # millisecond precision is not a reliable discriminator at ingest volumes.
+      # Slicing by index makes equality irrelevant rather than merely handled.
+      def drain_once
+        payloads = [queue.pop]
+        while (payload = pop_additional)
+          payloads << payload
+        end
+
+        payloads.compact!
+        payloads.each_slice(BATCH_SIZE) { |list| deliver_batch(list) }
+      end
+
+      def deliver_batch(list)
+        response = direct_writer.write(list)
+        return note_unanswered(list) if response.nil?
+
+        if response.status < 299
+          on_success(response) if respond_to?(:on_success)
+        elsif respond_to?(:on_failure)
+          on_failure(response)
+        end
+      end
+
+      # nil is what Commands::Http returns once its retries are exhausted. It is
+      # not a status - it is the absence of an answer - so it is reported rather
+      # than compared: on_failure hears about it with nil, meaning "nothing
+      # answered", and the batch it cost us is named in the log either way.
+      def note_unanswered(list)
+        log_error(
+          "[EndPointBlank] no response from #{url} after retries; " \
+          "#{list.size} payload(s) in that batch are lost"
+        )
+        on_failure(nil) if respond_to?(:on_failure)
+      end
+
+      def note_worker_error(error, consecutive_failures)
+        log_error(
+          "[EndPointBlank] send worker recovered from #{error.class}: #{error.message} " \
+          "(consecutive failure #{consecutive_failures}); the batch in flight is lost, " \
+          "retrying in #{worker_backoff(consecutive_failures).round(1)}s at #{Array(error.backtrace).first}"
+        )
+      end
+
+      def worker_backoff(consecutive_failures)
+        [
+          WORKER_BACKOFF_SECONDS * (2**(consecutive_failures - 1)),
+          MAX_WORKER_BACKOFF_SECONDS
+        ].min
+      end
+
+      # Same shape and same fallback rationale as `queue` above, and worse in
+      # kind if it ever raced: two threads synchronizing on *different* Mutex
+      # objects are not synchronized at all, so the critical section in
+      # `enqueue_one` would silently stop being one.
       def enqueue_mutex
         @enqueue_mutex ||= Mutex.new
       end

data/lib/end_point_blank/writers/direct_writer.rb

@@ -10,7 +10,7 @@ module EndPointBlank
       end
 
       def write(list)
-        auth = EndPointBlank::Authorization.header
+        auth = EndPointBlank::Authorization.intake_header
         EndPointBlank::Commands::Http.post(@url, auth, { payload: list })
       end
     end

data/lib/end_point_blank/writers/exception_writer.rb

@@ -12,6 +12,7 @@ module EndPointBlank
       attr_reader :url
 
       def initialize
+        super()
         @url = EndPointBlank::Configuration.instance.errors_url
         start_threads
       end
@@ -20,10 +21,18 @@ module EndPointBlank
         instance.write(exception)
       end
 
+      # Builds an error payload with or without an in-flight request. `env` is
+      # nil whenever an exception is reported from outside `EnvStore`'s scope
+      # (a background job, a worker, a boot-time failure) -- that is precisely
+      # the case this method exists to handle, not an edge case to special-case
+      # away. `Rack::Request.new(env)` was built unconditionally here even
+      # though nothing about an error report requires a request: version
+      # detection (Commands::VersionFinder) reads request params/path and is
+      # meaningless without one, so it is skipped rather than attempted and
+      # rescued. Its result was never even merged into the returned hash, so
+      # skipping it changes nothing for the in-request case either.
       def payload(exception)
         env = ::EndPointBlank::Rack::EnvStore.get
-        request = ::Rack::Request.new(env)
-        version = Commands::VersionFinder.new.find(request)
         {
           app_name: app_name,
           uuid: request_uuid(env),

data/lib/end_point_blank/writers/log_writer.rb

@@ -12,6 +12,7 @@ module EndPointBlank
       attr_reader :url
 
       def initialize
+        super()
         @url = EndPointBlank::Configuration.instance.logs_url
         start_threads
       end
@@ -61,7 +62,6 @@ module EndPointBlank
             stamped_http_method: rack_req.request_method
           )
         end
-        puts "Writing log: #{json}"
         enqueue(json)
       end
     end

data/lib/end_point_blank/writers/request_writer.rb

@@ -12,6 +12,7 @@ module EndPointBlank
       attr_reader :url
 
       def initialize
+        super()
         @url = EndPointBlank::Configuration.instance.requests_url
         start_threads
       end

data/lib/end_point_blank/writers/response_writer.rb

@@ -12,6 +12,7 @@ module EndPointBlank
       attr_reader :url
 
       def initialize
+        super()
         @url = EndPointBlank::Configuration.instance.responses_url
         start_threads
       end

data/lib/end_point_blank/writers/shared.rb

@@ -1,8 +1,17 @@
+require "securerandom"
+
 module EndPointBlank
   module Writers
     module Shared
       attr_accessor :url
 
+      # Where a minted request_uuid is cached on the Rack env, when the env
+      # carries no request id of its own. Distinct from
+      # "action_dispatch.request_id" so a later Rails middleware that sets
+      # the real key is never shadowed by ours -- we only ever read that key
+      # first and fall back to this one.
+      GENERATED_UUID_KEY = "end_point_blank.generated_request_uuid".freeze
+
       def configuration
         Configuration.instance
       end
@@ -24,11 +33,33 @@ module EndPointBlank
       end
 
       # Rails' ActionDispatch::Request#uuid simply reads this same Rack env
-      # key, so this is behavior-preserving when running under Rails, and
-      # gracefully returns nil when it isn't (plain Rack / Sinatra), instead
-      # of requiring actionpack's ActionDispatch::Request to be loaded.
+      # key, so reading it directly is behavior-preserving when running under
+      # Rails, without requiring actionpack's ActionDispatch::Request to be
+      # loaded.
+      #
+      # Outside Rails (plain Rack / Sinatra) nothing sets that key. Falling
+      # through to nil there used to be described as "graceful", but it is
+      # not: intake requires `uuid` on every error/log/request/response row
+      # and refuses the ones that lack it, so a nil uuid is a silently
+      # dropped row, not a tolerated one. Minting a fresh uuid instead means
+      # the row still lands -- correlating with nothing is strictly better
+      # than not existing.
+      #
+      # The mint is cached back onto `env`, not just returned. RequestWriter
+      # and ResponseWriter both call this with the *same* env object for one
+      # HTTP request -- EnvStore hands back whatever `set/1` was given at the
+      # top of the middleware, unchanged, for the life of the request. Without
+      # caching, each writer's call under plain Rack would mint its own fresh
+      # uuid, and the request row and the response row for a single
+      # interaction would carry two different, unrelated ids -- silently
+      # defeating the one thing these columns exist for. When `env` is nil
+      # (an exception reported with no request in flight at all) there is no
+      # shared object to cache onto and nothing to correlate across, so a
+      # standalone mint every time is correct.
       def request_uuid(env)
-        env && env["action_dispatch.request_id"]
+        return SecureRandom.uuid unless env
+
+        env["action_dispatch.request_id"] || (env[GENERATED_UUID_KEY] ||= SecureRandom.uuid)
       end
 
       def apply_masking(payload, record_type)