crumbtrail 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: fc7d1bce77e6988e2adaf35ddd5767cad8d7730981d023e198b0a6c1c99dae4c
+  data.tar.gz: 9f3f63783428f5d08c5b2cfb7c42455a965d98f2aa84fe301608fd76a3519e0a
+SHA512:
+  metadata.gz: 7484aa90ddacaeb4b52b455bcbe48ceb051705dc6aaf96b85d5414058809699dd6069963055df2f4526c4119ee39996d79587c1a19787c7c0106b7436cd0c236
+  data.tar.gz: 343f693391c31004a28af66519940a6c5fd9255ccff050cf621787cfb794d12838b20aa5398e0effe99d353fbc8db26aa2b9e3ebdd45597d215d4abfcfa840c8

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Crumbtrail
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,38 @@
+# Capture Rack and Rails requests
+
+Requires Ruby 3.2 or later and Rack 2 or 3. This package is source only until a RubyGems release is published. In a checkout, run `gem build crumbtrail.gemspec` and install the resulting gem. Applications can use a Bundler `path` dependency on this directory during local verification.
+
+```ruby
+require 'crumbtrail'
+sender = Crumbtrail::Sender.new(endpoint: ENV.fetch('CRUMBTRAIL_ENDPOINT'), key: ENV.fetch('CRUMBTRAIL_INGEST_KEY'), logger: Rails.logger)
+# config.ru, after the application's normal requires
+use Crumbtrail::Middleware, sink: sender, service: 'api', routes: ->(env) { env['PATH_INFO'].start_with?('/api/') && !env['PATH_INFO'].start_with?('/api/auth') }
+at_exit { sender.close(timeout: 5) }
+```
+
+For Rails, use the same keyword arguments with `config.middleware.use Crumbtrail::Middleware` in your application configuration. Initialize the sender inside each worker process after any application server fork.
+
+Both valid `x-crumbtrail-session-id` and `x-crumbtrail-request-id` headers are required. The browser SDK must register that session. No routes are captured unless the `routes` predicate returns true. Upgrade requests and responses that execute or arrange Rack hijacking bypass capture. A server advertising hijack capability still captures ordinary requests. Query strings, headers and raw paths are never captured. The optional `route` callback can return an application configured template such as `/api/orders/:id`. Its default is `/`. Do not return a path containing actual parameter values.
+
+JSON capture preserves bounded numeric operands, booleans, null, short enums and currency codes. Sensitive names and other strings are redacted, and a sensitive name redacts its whole subtree without walking it. A number is kept only when it stays inside the safe integer range, has at most six integer digits, and is not a valid card number. That cap is below the length of a phone number, a national identifier or an account number, so those are withheld even under an innocent field name. Strings are kept only as a short lowercase word, a three letter uppercase code, or at most six digits. The Ruby, Go and ASP.NET Core packages all run `test-fixtures/backend-body/cases.json`, so the three agree on every case in it. Bodies over 16 KiB are withheld with `truncated` state. Malformed, ambiguous, or structurally excessive JSON has `invalid` state. Non JSON or empty bodies have `missing` state. The request is observed as the application reads it. Unread bodies are missing. A nonempty request body is withheld as truncated unless it exactly matches a declared Content Length, or a full read or EOF proves completeness when no length was declared. Response chunks are forwarded unchanged and captured when Rack enumerates the body. Closing the body finalizes delivery once. Callable Rack streaming bodies receive the original stream unchanged and report missing response body evidence. Their request evidence and response status are still recorded. Enumerable response failures with partial bytes report truncated response evidence.
+
+The queue holds 64 batches and refuses new batches when full. A refused batch is a hole in the session, so the request records a `buffer_overflow` capture gap naming how many events were lost. Each request holds at most 200 database events plus request boundary events and reports an event limit gap.
+
+The optional `cert_store` accepts an OpenSSL certificate store for private trust roots and keeps certificate verification enabled. A sender uses HTTPS and does not follow redirects. It retries network errors, 429 and server errors up to four attempts. Any other status is permanent: repeating it cannot help, so the sender records a `delivery_failed` capture gap and writes one line to `logger`, or to `warn` when no logger was given. Without that line a revoked key looks exactly like a working SDK with an empty project.
+
+The cloud can also answer 202 with `{"capture":"shed"}` and a retry window, which means it accepted the request and discarded the evidence. The sender pauses delivery for the window, counts what it drops, and sends one capture gap naming the shed reason once the window passes.
+
+`close(timeout: 5)` waits at most five seconds for the queue to drain, then cancels the worker, and returns whether draining completed. A retry backoff cannot hold process exit open past the timeout the caller asked for. Abrupt process termination can still lose queued events.
+
+## Capture ActiveRecord commands
+
+Requires ActiveSupport and ActiveRecord from the application. Install once during initialization:
+
+```ruby
+require 'crumbtrail/active_record'
+Crumbtrail::ActiveRecord.install(engine: 'postgres')
+```
+
+Accepted engines are `postgres`, `mysql`, and `sqlite`. Installing twice with the same engine is a no op; installing a second, different engine raises `ArgumentError` rather than silently keeping the first. The adapter records operation, duration, statement sequence, reported row count, and exception class within captured requests, including queries during response enumeration. Statement order is the order the application issued statements in, so both the sequence and the timestamp are taken before the statement runs; `durationMs` recovers the completion time. Rails schema and query cache statements are ignored so they cannot spend the request's event budget. It deliberately withholds SQL, bindings and row values. It does not claim before/after snapshots, transaction correlation or database read evidence. `Crumbtrail::ActiveRecord.uninstall` removes the subscription.
+
+Run `pnpm test:ruby` from the repository root, or `bundle exec ruby -Ilib -Itest test/capture_test.rb` here, with Rack, ActiveRecord, SQLite3 and Minitest installed. Set `CRUMBTRAIL_CAPTURE_CONTRACT_OUTPUT=/tmp/ruby-capture.json` to export the real Rack test's batches for consumer contract checks.

data/lib/crumbtrail/active_record.rb

@@ -0,0 +1,88 @@
+require 'crumbtrail'
+require 'active_support/notifications'
+
+module Crumbtrail
+  module ActiveRecord
+    # Query text and bindings are intentionally withheld. This works across SQL dialects
+    # without treating a partial SQL parser as a privacy boundary.
+    ENGINES = %w[postgres mysql sqlite].freeze
+    MUTEX = Mutex.new
+    # Rails issues schema reflection and query cache hits inside application requests. They are
+    # not statements the application asked for, and they spend the per request event budget the
+    # application's own statements need.
+    IGNORED_NAMES = %w[SCHEMA CACHE].freeze
+    MAX_PENDING = 1024
+    OPERATIONS = %w[SELECT INSERT UPDATE DELETE].freeze
+    OPERATION = /\A(?:SELECT|INSERT|UPDATE|DELETE|BEGIN|COMMIT|ROLLBACK|SAVEPOINT|RELEASE)\b/i
+
+    # An evented subscriber, so the sequence and the timestamp are both taken when the statement
+    # is issued rather than when it completes. Stamping at completion sorts a slow query after
+    # faster ones issued after it, and leaves `seq` contradicting `t`.
+    class Subscriber
+      def initialize(engine)
+        @engine = engine
+      end
+
+      def start(_name, id, payload)
+        return if ignore?(payload)
+        context = Crumbtrail.current
+        return unless context
+        pending = (Thread.current[:crumbtrail_sql_pending] ||= {})
+        # A statement whose completion notification never arrives would otherwise retain its
+        # context for the life of the thread.
+        return if pending.size >= MAX_PENDING
+        pending[id] = [context, context.next_sequence, Crumbtrail.now, Process.clock_gettime(Process::CLOCK_MONOTONIC)]
+      rescue StandardError
+        nil
+      end
+
+      def finish(_name, id, payload)
+        state = Thread.current[:crumbtrail_sql_pending]&.delete(id)
+        return unless state
+        context, sequence, at, started = state
+        sql = payload[:sql].to_s
+        sql = '' if sql.bytesize > 32_768
+        operation = sql.lstrip[OPERATION]&.upcase || 'OTHER'
+        data = { engine: @engine, op: OPERATIONS.include?(operation) ? operation.downcase : 'other', table: nil,
+                 shape: '[statement omitted]',
+                 rowCount: payload[:row_count].is_a?(Integer) ? payload[:row_count] : nil,
+                 rowEvidence: 'not_captured',
+                 durationMs: (Process.clock_gettime(Process::CLOCK_MONOTONIC) - started) * 1000,
+                 cached: !!payload[:cached] }
+        error = payload[:exception_object]
+        data.merge!(errorName: error.class.name, code: nil, category: 'unknown') if error
+        context.database(error ? 'db.error' : 'db.statement', data, sequence: sequence, time: at)
+      rescue StandardError
+        nil
+      end
+
+      private
+
+      def ignore?(payload)
+        IGNORED_NAMES.include?(payload[:name].to_s) || !!payload[:cached]
+      end
+    end
+
+    def self.install(engine:)
+      raise ArgumentError, 'Unsupported database engine' unless ENGINES.include?(engine)
+      MUTEX.synchronize do
+        if @subscription
+          # A second install with a different engine used to be discarded in silence, and every
+          # event after it named the first engine.
+          raise ArgumentError, "Crumbtrail ActiveRecord capture is already installed for #{@engine}" unless @engine == engine
+          return @subscription
+        end
+        @engine = engine
+        @subscription = ActiveSupport::Notifications.subscribe('sql.active_record', Subscriber.new(engine))
+      end
+    end
+
+    def self.uninstall
+      MUTEX.synchronize do
+        ActiveSupport::Notifications.unsubscribe(@subscription) if @subscription
+        @subscription = nil
+        @engine = nil
+      end
+    end
+  end
+end

data/lib/crumbtrail/body.rb

@@ -0,0 +1,109 @@
+require 'json'
+
+module Crumbtrail
+  # Structured body policy. Ruby, Go and the ASP.NET Core package implement the same rules and
+  # are driven by the same corpus in `test-fixtures/backend-body/cases.json`, so a divergence
+  # between them fails a test instead of quietly producing three different bodies.
+  module Body
+    LIMIT = 16_384
+    POLICY = 'crumbtrail.backend-redaction.v1'
+    MAX_NESTING = 8
+    MAX_KEYS = 64
+    MAX_ITEMS = 40
+    # Well below a phone number (10), a national insurance number (9) and a card (13 to 19).
+    MAX_INTEGER_DIGITS = 6
+    SAFE_INTEGER = 9_007_199_254_740_991
+    DENIED = /password|passwd|passphrase|passcode|secret|token|auth|card|cvv|cvc|ssn|email|phone|address|iban|account|birth|credential|creds|cookie|session|privatekey|apikey|accesskey|securitycode|verificationcode|connection|routingnumber|taxid|nationalid|sortcode|name|postal|payload|beforejson|afterjson|mobile|contact|diagnosis|medical|patient|prescription|gender|ethnic|religion|salary|income|identifier|username|passport|insurance|beneficiary|guardian|occupation|citizen|latitude|longitude|coordinate|geolocation|province|country|street/i
+    # Short words that appear inside innocent identifiers ("capacity" contains "city"), so they
+    # are matched as whole words rather than as substrings.
+    DENIED_WORD = /\A(?:pwd|pin|pan|otp|pass|sid|dob|zip|jwt|mfa|csrf|xsrf|city|town|geo|cell|race|sex|age|location|lat|lng|lon|gps)s?[0-9]*\z/i
+    SAFE_STRING = /\A(?:[a-z][a-z_]{0,22}|[A-Z]{3}|[0-9]{1,#{MAX_INTEGER_DIGITS}})\z/
+    TOKEN_STRING = /(?:sk|pk|rk|ghp|gho|ghu|ghs|glpat|xox[baprs])[-_][a-zA-Z0-9_.=-]{12,}/i
+    FIELD_NAME = /\A[a-zA-Z_][a-zA-Z0-9_]*\z/
+    REDACTED = '[REDACTED]'.freeze
+
+    class Invalid < StandardError; end
+
+    def self.capture(bytes, truncated = false)
+      return [nil, 'truncated'] if truncated || bytes.bytesize > LIMIT
+      return [nil, 'missing'] if bytes.empty?
+      removed = [false]
+      value = JSON.parse(bytes, max_nesting: MAX_NESTING, allow_duplicate_key: false)
+      value = walk(value, '', removed)
+      encoded = JSON.generate(value)
+      return [nil, 'truncated'] if encoded.bytesize > LIMIT
+      [encoded, removed[0] ? 'redacted' : 'captured']
+    rescue JSON::ParserError, JSON::GeneratorError, JSON::NestingError, Invalid, EncodingError
+      [nil, 'invalid']
+    end
+
+    # A sensitive key is redacted whole. The subtree under it is never walked, so a malformed
+    # or oversized value inside a secret cannot turn the whole body into `invalid` and cannot
+    # be inspected on the way to being dropped.
+    def self.walk(value, key, removed)
+      unless sensitive?(key)
+        case value
+        when Hash
+          raise Invalid if value.size > MAX_KEYS
+          return value.to_h do |k, v|
+            raise Invalid unless k.bytesize <= 64 && k.match?(FIELD_NAME)
+            [k, walk(v, k, removed)]
+          end
+        when Array
+          raise Invalid if value.size > MAX_ITEMS
+          return value.map { |v| walk(v, key, removed) }
+        when Numeric
+          return value if number?(value)
+        when TrueClass, FalseClass, NilClass
+          return value
+        when String
+          return value if value.match?(SAFE_STRING) && !value.match?(TOKEN_STRING)
+        end
+      end
+      removed[0] = true
+      REDACTED
+    end
+
+    def self.sensitive?(key)
+      words = key.gsub(/([a-z0-9])([A-Z])/, '\1 \2').split(/[^a-zA-Z0-9]+/)
+      key.gsub(/[^a-zA-Z0-9]/, '').match?(DENIED) || words.any? { |word| word.match?(DENIED_WORD) }
+    end
+
+    # The integer digit cap already excludes every card length. Luhn stays because it also
+    # catches a card smuggled across a decimal point, where the integer part is short.
+    def self.number?(value)
+      return false unless value.finite?
+      return false if value.abs > SAFE_INTEGER
+      return false if value.to_i.abs.to_s.length > MAX_INTEGER_DIGITS
+      !luhn?(digits(value))
+    end
+
+    def self.digits(value)
+      value.abs.to_s.gsub(/[^0-9]/, '')
+    end
+
+    def self.luhn?(digits)
+      return false unless digits.length.between?(13, 19)
+      sum = 0
+      twice = false
+      digits.reverse.each_char do |char|
+        n = char.ord - 48
+        if twice
+          n *= 2
+          n -= 9 if n > 9
+        end
+        sum += n
+        twice = !twice
+      end
+      (sum % 10).zero?
+    end
+
+    def self.json?(type)
+      type.to_s.split(';', 2).first.to_s.strip.match?(%r{\Aapplication/(?:json|[^/;\s]+\+json)\z}i)
+    end
+
+    def self.metadata(field, state)
+      { policy: POLICY, fields: state == 'redacted' ? [{ path: field, reason: 'backend_structured_profile', action: 'redacted' }] : [] }
+    end
+  end
+end

data/lib/crumbtrail/event.rb

@@ -0,0 +1,23 @@
+module Crumbtrail
+  VERSION = '0.1.0'
+  SCHEMA_VERSION = 1
+  PLATFORM = 'ruby'
+  SDK = { name: 'crumbtrail-ruby', version: VERSION }.freeze
+
+  def self.now
+    (Time.now.to_f * 1000).to_i
+  end
+
+  # Wire envelope for every event. `schemaVersion`, `platform` and `sdk` are required of every
+  # SDK that is not built on crumbtrail-core: without them ingest defaults the event to
+  # `platform: "web"` and no reader can tell a Rails request apart from a browser one.
+  # `capabilities` and `target` are the remaining envelope fields. This SDK never populates
+  # them, and they are accepted here so the wire contract fixtures can be exercised through
+  # the real serializer rather than a copy of it.
+  def self.event(time, kind, data, platform: PLATFORM, sdk: SDK, capabilities: nil, target: nil)
+    event = { t: time, k: kind, d: data, schemaVersion: SCHEMA_VERSION, platform: platform, sdk: sdk }
+    event[:capabilities] = capabilities if capabilities && !capabilities.empty?
+    event[:target] = target if target && !target.empty?
+    event
+  end
+end

data/lib/crumbtrail/sender.rb

@@ -0,0 +1,191 @@
+require 'net/http'
+require 'uri'
+require 'json'
+require_relative 'event'
+
+module Crumbtrail
+  class Sender
+    # 429 and 5xx are the only answers worth repeating the identical batch into. A 404 names a
+    # session the cloud does not have, and every later batch for that id is refused the same
+    # way, so retrying it four times only delays the gap that says the evidence is gone.
+    RETRYABLE = [429].freeze
+    # Shed reasons authored by the capture edge. An unrecognised reason is recorded as a plain
+    # delivery failure rather than passed through as an invented classification.
+    SHED_REASONS = %w[kill_switch sessions_per_hour bytes_per_day rate_limited_ingest
+                      rate_limited_session_start trial_expired payment_failed upgrade_required].freeze
+    EVENTS_PATH = '/api/events'.freeze
+    ATTEMPTS = 4
+    MAX_SHED_SECONDS = 300
+    MAX_RESPONSE_BYTES = 4096
+
+    def initialize(endpoint:, key:, cert_store: nil, logger: nil)
+      @endpoint = URI(endpoint)
+      raise ArgumentError, 'Crumbtrail requires HTTPS without credentials, query or fragment' unless @endpoint.is_a?(URI::HTTPS) && @endpoint.host && !@endpoint.userinfo && !@endpoint.query && !@endpoint.fragment
+      raise ArgumentError, 'Crumbtrail ingest key is required' if key.to_s.strip.empty? || key.match?(/[[:cntrl:]]/)
+      @key = key
+      @cert_store = cert_store
+      @logger = logger
+      @queue = SizedQueue.new(64)
+      @monitor = Mutex.new
+      @wake = ConditionVariable.new
+      @stopping = false
+      @shed_until = nil
+      @shed = nil
+      @worker = Thread.new do
+        while (batch = @queue.pop)
+          begin
+            deliver(batch[0], batch[1], batch[2])
+          rescue Exception
+            # A worker killed by anything other than a StandardError would leave `close` joining
+            # a dead thread and reporting a clean drain over an empty queue.
+            nil
+          end
+        end
+      end
+      @worker.report_on_exception = false
+    end
+
+    def enqueue(batch)
+      session = (batch[:sessionId] || batch['sessionId']).to_s
+      events = batch[:events] || batch['events'] || []
+      @queue.push([session, JSON.generate(batch), events.size], true)
+      true
+    rescue ThreadError, ClosedQueueError, JSON::GeneratorError
+      false
+    end
+
+    # Closing the queue allows queued batches to drain within the caller's deadline. Past the
+    # deadline the worker is cancelled, so a retry backoff cannot hold process exit open past
+    # the timeout the caller asked for.
+    def close(timeout: 5)
+      @queue.close
+      return true if @worker.join(timeout)
+      cancel
+      @worker.join(1)
+      false
+    end
+
+    private
+
+    def cancel
+      @monitor.synchronize do
+        @stopping = true
+        @wake.broadcast
+      end
+    end
+
+    def stopping?
+      @monitor.synchronize { @stopping }
+    end
+
+    # Interruptible backoff. `sleep` here blocks process exit for the whole retry budget.
+    def pause(seconds)
+      @monitor.synchronize do
+        next if @stopping
+        @wake.wait(@monitor, seconds)
+      end
+    end
+
+    def deliver(session, payload, count, gap: false)
+      return if stopping?
+      if shedding?
+        @shed[:dropped] += count unless gap
+        return
+      end
+      flush_shed_gap unless gap
+      ATTEMPTS.times do |attempt|
+        return if stopping?
+        response = begin
+          post(payload)
+        rescue StandardError
+          nil # No response at all. A network failure is retried per the queue policy.
+        end
+        if response
+          code = response.code.to_i
+          if code == 202 && (directive = shed_directive(response))
+            begin_shed(session, directive, count) unless gap
+            return
+          end
+          return if code / 100 == 2
+          unless RETRYABLE.include?(code) || code >= 500
+            refuse(session, count, code) unless gap
+            return
+          end
+        end
+        break if attempt == ATTEMPTS - 1
+        pause(0.25 * (attempt + 1))
+      end
+      refuse(session, count, nil) unless gap
+    end
+
+    def post(payload)
+      uri = @endpoint.dup
+      uri.path = EVENTS_PATH
+      request = Net::HTTP::Post.new(uri)
+      request['Authorization'] = "Bearer #{@key}"
+      request['Content-Type'] = 'application/json'
+      request.body = payload
+      options = { use_ssl: true, open_timeout: 2, read_timeout: 3, write_timeout: 3 }
+      options[:cert_store] = @cert_store if @cert_store
+      Net::HTTP.start(uri.host, uri.port, **options) { |http| http.request(request) }
+    end
+
+    # A 202 passes any "is this a success" test while the cloud has already discarded the
+    # evidence. Reading the body is the only way to tell the two apart.
+    def shed_directive(response)
+      body = response.body.to_s
+      return nil if body.empty? || body.bytesize > MAX_RESPONSE_BYTES
+      parsed = JSON.parse(body)
+      return nil unless parsed.is_a?(Hash) && parsed['capture'] == 'shed'
+      seconds = parsed['retryAfterSeconds']
+      seconds = response['Retry-After'].to_i unless seconds.is_a?(Numeric)
+      { reason: SHED_REASONS.include?(parsed['reason']) ? parsed['reason'] : 'delivery_failed',
+        seconds: seconds.to_f.clamp(0, MAX_SHED_SECONDS) }
+    rescue JSON::ParserError, TypeError
+      nil
+    end
+
+    def begin_shed(session, directive, count)
+      @shed = { session: session, reason: directive[:reason], dropped: count }
+      @monitor.synchronize { @shed_until = Process.clock_gettime(Process::CLOCK_MONOTONIC) + directive[:seconds] }
+      log("Crumbtrail capture is shed at the endpoint (#{directive[:reason]}); pausing delivery for #{directive[:seconds].round}s")
+    end
+
+    def shedding?
+      deadline = @monitor.synchronize { @shed_until }
+      !deadline.nil? && Process.clock_gettime(Process::CLOCK_MONOTONIC) < deadline
+    end
+
+    # The gap for a shed window cannot be delivered during the window. It is held and sent on
+    # the first batch after Retry-After has passed.
+    def flush_shed_gap
+      return if @shed.nil? || shedding?
+      shed = @shed
+      @shed = nil
+      @monitor.synchronize { @shed_until = nil }
+      send_gap(shed[:session], shed[:reason], shed[:dropped], 'capture shed by the endpoint')
+    end
+
+    def refuse(session, count, code)
+      detail = code ? "HTTP #{code}" : 'retry budget exhausted'
+      log("Crumbtrail refused #{count} captured event(s): #{detail}")
+      send_gap(session, 'delivery_failed', count, detail)
+    end
+
+    def send_gap(session, reason, count, detail)
+      return if session.to_s.empty? || count.to_i <= 0
+      event = Crumbtrail.event(Crumbtrail.now, 'capture_gap',
+                               { kind: 'capture_gap', surface: 'backend_request', reason: reason,
+                                 sessionId: session, droppedEventCount: count, detail: detail })
+      deliver(session, JSON.generate(sessionId: session, events: [event]), 1, gap: true)
+    rescue StandardError
+      nil
+    end
+
+    def log(message)
+      @logger ? @logger.warn(message) : warn(message)
+    rescue StandardError
+      nil
+    end
+  end
+end

data/lib/crumbtrail.rb

@@ -0,0 +1,282 @@
+require_relative 'crumbtrail/event'
+require_relative 'crumbtrail/body'
+require_relative 'crumbtrail/sender'
+
+module Crumbtrail
+  EVENT_BUDGET = 200
+  BOUNDARY_KINDS = %w[backend.req.start backend.req.end backend.req.error].freeze
+
+  class Context
+    attr_reader :session_id, :request_id
+    def initialize(session_id, request_id, sink)
+      @session_id, @request_id, @sink = session_id, request_id, sink
+      @events = []
+      @dropped = 0
+      @sequence = 0
+    end
+    def add(kind, data, time = Crumbtrail.now)
+      if @events.size < EVENT_BUDGET || BOUNDARY_KINDS.include?(kind)
+        @events << Crumbtrail.event(time, kind, data.merge(requestId: @request_id, sessionId: @session_id))
+      else
+        @dropped += 1
+      end
+    end
+    # Statement order is the order the application issued statements in, so both the sequence
+    # and the timestamp are taken before the statement runs. Stamping either at completion
+    # sorts a slow query after faster ones issued after it, and makes `seq` contradict `t`.
+    def next_sequence
+      @sequence += 1
+    end
+    def database(kind, data, sequence: nil, time: nil)
+      time ||= Crumbtrail.now
+      add(kind, data.merge(seq: sequence || next_sequence, t: time), time)
+    end
+    def gap(reason, dropped, surface, detail = nil)
+      data = { kind: 'capture_gap', surface: surface, reason: reason, requestId: @request_id,
+               sessionId: @session_id, droppedEventCount: dropped }
+      data[:detail] = detail if detail
+      Crumbtrail.event(Crumbtrail.now, 'capture_gap', data)
+    end
+    def flush
+      @events << gap('scan_budget_exceeded', @dropped, 'backend_request') if @dropped > 0
+      @events.sort_by! { |event| [event[:t], event[:k] == 'backend.req.start' ? 0 : 1] }
+      refused = 0
+      @events.each_slice(20) do |events|
+        refused += events.size unless @sink.enqueue(sessionId: @session_id, events: events)
+      end
+      # A batch the sink refused is a hole in the session. Declaring it costs one event; leaving
+      # it implicit lets a burst drop `backend.req.end` and leaves a request that never
+      # terminated looking exactly like a request that never happened.
+      if refused > 0
+        @sink.enqueue(sessionId: @session_id, events: [gap('buffer_overflow', refused, 'queue', 'sink queue full')])
+      end
+    rescue StandardError
+      nil
+    ensure
+      @events.clear
+    end
+  end
+
+  def self.current
+    Thread.current[:crumbtrail_context]
+  end
+  def self.with_context(context)
+    previous = current
+    Thread.current[:crumbtrail_context] = context
+    yield
+  ensure
+    Thread.current[:crumbtrail_context] = previous
+  end
+
+  class Input
+    attr_reader :bytes, :truncated, :complete
+    def initialize(io)
+      @io, @bytes, @truncated, @complete = io, ''.b, false, false
+      # Rack does not require the input stream to be rewindable, and application code branches
+      # on `respond_to?(:rewind)`. Advertising a method the wrapped stream does not have turns
+      # a working request into a NoMethodError this middleware introduced. `close` and `size`
+      # are delegated for the same reason: the wrapper must not remove capability from the
+      # object it replaces.
+      define_singleton_method(:rewind) { rewind! } if io.respond_to?(:rewind)
+      define_singleton_method(:close) { @io.close } if io.respond_to?(:close)
+      define_singleton_method(:size) { @io.size } if io.respond_to?(:size)
+    end
+    def keep(data)
+      return data unless data
+      remaining = Body::LIMIT - @bytes.bytesize
+      @bytes << data.b.byteslice(0, remaining)
+      @truncated ||= data.bytesize > remaining
+      data
+    end
+    def read(*args)
+      data = @io.read(*args)
+      @complete = true if args.empty? || args[0].nil? || data.nil? || data.bytesize < args[0]
+      keep(data)
+    end
+    def gets(*args)
+      data = @io.gets(*args)
+      @complete = true if data.nil?
+      keep(data)
+    end
+    def each
+      return enum_for(:each) unless block_given?
+      @io.each { |data| yield keep(data) }
+      @complete = true
+    end
+    private
+    def rewind!
+      result = @io.rewind
+      @bytes.clear
+      @truncated = false
+      @complete = false
+      result
+    end
+  end
+
+  class ResponseBody
+    def initialize(body, context, finish)
+      @body, @context, @finish = body, context, finish
+      @bytes, @truncated, @finished, @complete = ''.b, false, false, false
+      # Rack::Files answers `to_path` so the server can hand the descriptor to sendfile. Hiding
+      # it costs every static response its fast path. The bytes never pass through Ruby on that
+      # path, so the response body is honestly reported as missing rather than invented.
+      define_singleton_method(:to_path) { @body.to_path } if body.respond_to?(:to_path)
+      define_singleton_method(:to_ary) { buffered } if body.respond_to?(:to_ary)
+    end
+    def each
+      return enum_for(:each) unless block_given?
+      Crumbtrail.with_context(@context) do
+        begin
+          @body.each do |chunk|
+            keep(chunk)
+            yield chunk
+          end
+          @complete = true
+        rescue Exception => error
+          finish(error)
+          raise
+        ensure
+          finish
+        end
+      end
+    end
+    def close
+      @body.close if @body.respond_to?(:close)
+    ensure
+      finish
+    end
+    private
+    def keep(chunk)
+      remaining = Body::LIMIT - @bytes.bytesize
+      @bytes << chunk.b.byteslice(0, remaining)
+      @truncated ||= chunk.bytesize > remaining
+    end
+    def buffered
+      chunks = Crumbtrail.with_context(@context) { @body.to_ary }
+      chunks.each { |chunk| keep(chunk) }
+      @complete = true
+      chunks
+    ensure
+      finish
+    end
+    def finish(error = nil)
+      @truncated = true if (error || !@complete) && !@bytes.empty?
+      return if @finished
+      @finished = true
+      @finish.call(@bytes, @truncated, error)
+    rescue StandardError
+      nil
+    end
+  end
+
+  class CallableResponseBody
+    def initialize(body, context, finish)
+      @body, @context, @finish, @finished = body, context, finish, false
+    end
+    def call(stream)
+      Crumbtrail.with_context(@context) do
+        begin
+          result = @body.call(stream)
+        rescue Exception => error
+          finish('', false, error)
+          raise
+        end
+        finish('', false)
+        result
+      end
+    end
+    def close
+      @body.close if @body.respond_to?(:close)
+    ensure
+      finish('', false)
+    end
+    private
+    def finish(bytes, truncated, error = nil)
+      return if @finished
+      @finished = true
+      @finish.call(bytes, truncated, error, false)
+    rescue StandardError
+      nil
+    end
+  end
+
+  class Middleware
+    ID = /\A[A-Za-z0-9][A-Za-z0-9._-]{0,127}\z/
+    def initialize(app, sink:, service:, routes: nil, route: nil)
+      @app, @sink, @service, @routes, @route = app, sink, service, routes, route
+    end
+    def call(env)
+      eligible = begin
+        @routes && @routes.call(env) && env['HTTP_UPGRADE'].to_s.empty? && ID.match?(env['HTTP_X_CRUMBTRAIL_SESSION_ID'].to_s) && ID.match?(env['HTTP_X_CRUMBTRAIL_REQUEST_ID'].to_s)
+      rescue StandardError
+        false
+      end
+      return @app.call(env) unless eligible
+      context = Context.new(env['HTTP_X_CRUMBTRAIL_SESSION_ID'], env['HTTP_X_CRUMBTRAIL_REQUEST_ID'], @sink)
+      original = env['rack.input']
+      original_hijack = env['rack.hijack']
+      hijacked = false
+      if original_hijack.respond_to?(:call)
+        env['rack.hijack'] = lambda do |*args, &block|
+          result = original_hijack.call(*args, &block)
+          hijacked = true
+          result
+        end
+      end
+      input = Input.new(original) if original && Body.json?(env['CONTENT_TYPE'])
+      env['rack.input'] = input if input
+      started, started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC), Crumbtrail.now
+      base = { method: env['REQUEST_METHOD'], url: '/', route: '/', pathname: '/', service: @service,
+               correlation: { status: 'linked', sessionIdSource: 'header', requestIdSource: 'header' } }
+      status, headers, body = nil
+      finish = lambda do |bytes, truncated, error = nil, observed = true|
+        template = begin
+          value = @route&.call(env)
+          value.is_a?(String) && value.bytesize <= 512 && value.match?(%r{\A/[a-zA-Z0-9_{}:.* /-]*\z}) ? value : '/'
+        rescue StandardError
+          '/'
+        end
+        base.merge!(url: template, route: template, pathname: template)
+        length = env['CONTENT_LENGTH'].to_s
+        complete = input && (length.match?(/\A[0-9]+\z/) ? input.bytes.bytesize == length.to_i : input.complete)
+        request_body, request_state = input && !input.bytes.empty? ? Body.capture(input.bytes, input.truncated || !complete) : [nil, 'missing']
+        body_allowed = env['REQUEST_METHOD'] != 'HEAD' && status != 204 && status != 304 && !(status && status < 200)
+        content_length = (headers && (headers['content-length'] || headers['Content-Length'])).to_s
+        if observed && body_allowed && content_length.match?(/\A[0-9]+\z/)
+          truncated ||= bytes.bytesize != content_length.to_i
+        end
+        response_body, response_state = observed && body_allowed && Body.json?(headers && (headers['content-type'] || headers['Content-Type'])) ? Body.capture(bytes, truncated) : [nil, 'missing']
+        context.add('backend.req.start', base.merge(body: request_body, requestBodyState: request_state, redaction: Body.metadata('body', request_state)), started_at)
+        context.add('backend.req.error', base.merge(error: { name: error.class.name })) if error
+        context.add('backend.req.end', base.merge(statusCode: status || 500, durationMs: (Process.clock_gettime(Process::CLOCK_MONOTONIC) - started) * 1000,
+          responseBody: response_body, responseBodyState: response_state, responseBodyTruncated: truncated, redaction: Body.metadata('responseBody', response_state)))
+        context.flush
+      ensure
+        env['rack.input'] = original
+        env['rack.hijack'] = original_hijack if original_hijack
+      end
+      begin
+        status, headers, body = Crumbtrail.with_context(context) { @app.call(env) }
+      rescue Exception => error
+        begin
+          if hijacked
+            env['rack.input'] = original
+            env['rack.hijack'] = original_hijack if original_hijack
+          else
+            finish.call('', false, error)
+          end
+        rescue StandardError
+          nil
+        end
+        raise
+      end
+      if hijacked || headers&.fetch('rack.hijack', nil)
+        env['rack.input'] = original
+        env['rack.hijack'] = original_hijack if original_hijack
+        return [status, headers, body]
+      end
+      wrapper = body.respond_to?(:each) ? ResponseBody : CallableResponseBody
+      [status, headers, wrapper.new(body, context, finish)]
+    end
+  end
+end

metadata

@@ -0,0 +1,69 @@
+--- !ruby/object:Gem::Specification
+name: crumbtrail
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Crumbtrail
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2026-09-05 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.21.2
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.21.2
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '3'
+description:
+email:
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE
+- README.md
+- lib/crumbtrail.rb
+- lib/crumbtrail/active_record.rb
+- lib/crumbtrail/body.rb
+- lib/crumbtrail/event.rb
+- lib/crumbtrail/sender.rb
+homepage: https://github.com/CrumbtrailDev/crumbtrail-cli
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '3.2'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.5.22
+signing_key:
+specification_version: 4
+summary: Correlated Rack request and ActiveRecord evidence for Crumbtrail
+test_files: []