devbench 0.7.0 → 0.8.1

data/lib/devbench/middleware.rb

@@ -3,10 +3,13 @@
 require_relative 'trace'
 require_relative 'current'
 require_relative 'reporter'
+require_relative 'request_signals'
 
 module Devbench
   # Rack middleware: accepts the correlation id and makes it available for the
-  # duration of the request, and reports exceptions that escape the app.
+  # duration of the request, reports exceptions that escape the app, and
+  # reports what a request does without raising (Capability 2b: a 5xx with
+  # no exception, a request over 3 s, a route's 429 spike; RequestSignals).
   #
   # Never rejects a request, never alters the response body, and never raises
   # anything of its own. An exception from the application is reported and
@@ -19,6 +22,10 @@ module Devbench
   class Middleware
     HANDLED_HEADER = 'x-adt-handled'
     EXPOSE_HEADER = 'Access-Control-Expose-Headers'
+    # Set while the outermost instance observes a request, so a second
+    # instance in the stack (an app that inserted it twice, an engine's own)
+    # does not count the request twice.
+    OBSERVING_KEY = :__adt_observing_request
 
     def initialize(app)
       @app = app
@@ -34,23 +41,31 @@ module Devbench
         nil
       end
 
+      observing = Thread.current[OBSERVING_KEY] != true
+      started = observing ? RequestSignals.clock.monotonic_ms : nil
+
       # Scoped even without a trace: identity set during this request must
       # end with it, and a thread's next request must not inherit it.
       status, headers, body = Current.with(trace) do
+        Thread.current[OBSERVING_KEY] = true if observing
         result = begin
           @app.call(env)
         rescue Exception => e # rubocop:disable Lint/RescueException
           # Reported inside the scope so the report carries this request's
           # trace and user. capture never raises; if it somehow did, the
           # application's exception would be lost, which is the one outcome
-          # this must not have.
+          # this must not have. A request that raised is not evaluated as a
+          # request signal: the exception is its report.
           report(e, env)
           raise e
+        ensure
+          Thread.current[OBSERVING_KEY] = nil if observing
         end
 
         # Rails' ShowExceptions turns an exception into a 500 page itself, so
         # nothing escapes to here. It leaves the exception in env.
         report(env['action_dispatch.exception'], env)
+        observe(env, result, started) if observing
 
         trace.nil? ? result : mark_handled(result)
       end
@@ -60,6 +75,47 @@ module Devbench
 
     private
 
+    # Classifies the finished request (failed / slow / nothing) and feeds the
+    # 429 spike detector. Inside the request's scope, so a signal carries its
+    # trace and user. Duration runs from entry to the application handing
+    # back its response; streaming a body after that is not counted.
+    def observe(env, result, started)
+      finished = RequestSignals.clock
+      duration_ms = (finished.monotonic_ms - started).floor
+      status, headers, = result
+      RequestSignals.observe(
+        status: status.is_a?(Integer) ? status : Integer(status.to_s, 10),
+        method: env['REQUEST_METHOD'],
+        route: symbol(env),
+        duration_ms: duration_ms,
+        streaming: streaming?(env, status.to_i, headers),
+        # Rails rendering a 500 itself is an exception: reported, or one the
+        # app chose to ignore (Devbench.ignore_exceptions), which must not
+        # come back as an http_500 either.
+        exception_reported: Current.exception_reported? || !env['action_dispatch.exception'].nil?,
+        finished_ms: finished.wall_ms
+      )
+    rescue StandardError, SystemStackError
+      # Never fail a response over instrumentation.
+      nil
+    end
+
+    # A 101 upgrade, a hijacked connection (a websocket takes the socket) or
+    # a text/event-stream response: its duration is a connection's lifetime,
+    # never a slow request.
+    def streaming?(env, status, headers)
+      return true if status == 101 || !env['rack.hijack_io'].nil?
+      return false unless headers.respond_to?(:each)
+
+      streaming = false
+      headers.each do |name, value|
+        key = name.to_s.downcase
+        streaming ||= key == 'rack.hijack'
+        streaming ||= key == 'content-type' && value.to_s.strip.downcase.start_with?('text/event-stream')
+      end
+      streaming
+    end
+
     def report(error, env)
       return if error.nil?
 

data/lib/devbench/rails_hooks.rb

@@ -3,7 +3,7 @@
 require_relative 'reporter'
 
 module Devbench
-  # The Rails-side exception hooks (docs/SERVER_SDK_SPEC.md, Capability 2).
+  # The Rails-side exception hooks (docs/server-sdk/unhandled-failures.md, Capability 2).
   #
   # Plain Ruby: nothing here names a Rails constant, so this file loads
   # without Rails. Devbench::Railtie installs the hooks against the real

data/lib/devbench/railtie.rb

@@ -3,6 +3,7 @@
 require_relative 'middleware'
 require_relative 'rails_hooks'
 require_relative 'sidekiq_hooks'
+require_relative 'active_job_hooks'
 require_relative 'view_helper'
 
 module Devbench
@@ -29,7 +30,7 @@ module Devbench
       Railtie.auto_insert(app.config.middleware)
     end
 
-    # Puts the request's trace on every Rails log line (SERVER_SDK_SPEC,
+    # Puts the request's trace on every Rails log line (docs/server-sdk/trace-propagation.md,
     # Capability 1, requirement 2), so the sidecar can hand triage the server
     # lines of one user action. Appended to whatever the app already tags
     # with — e.g. [:request_id] — never replacing it. Before the middleware
@@ -65,6 +66,11 @@ module Devbench
         if defined?(::ActiveSupport::Notifications)
           Devbench::RailsHooks.install_active_job(::ActiveSupport::Notifications)
         end
+        # The request's trace into ActiveJob jobs, on any adapter. on_load
+        # runs now if ActiveJob::Base is already loaded, else when it loads.
+        if defined?(::ActiveSupport) && ::ActiveSupport.respond_to?(:on_load)
+          ::ActiveSupport.on_load(:active_job) { Devbench::ActiveJobHooks.install(self) }
+        end
         # Native Sidekiq jobs never reach the ActiveJob hook. after_initialize
         # runs after every gem is required, so Gemfile order does not matter.
         Devbench::SidekiqHooks.install if defined?(::Sidekiq)

data/lib/devbench/reporter.rb

@@ -6,7 +6,7 @@ require_relative 'transport'
 
 module Devbench
   # Turns a failure into a report — the control message of
-  # docs/SERVER_SDK_SPEC.md, "Control socket — v1" — and hands it to the
+  # docs/server-sdk/control-socket.md, "Control socket — v1" — and hands it to the
   # active transport (Devbench.transport): direct to Dev Bench when a DSN is
   # set, else the local sidecar's socket.
   #
@@ -19,7 +19,7 @@ module Devbench
   module Reporter
     DEFAULT_SOCKET = '/tmp/adt-sidecar.sock'
 
-    # docs/SERVER_SDK_SPEC.md, "Control socket — v1".
+    # docs/server-sdk/control-socket.md, "Control socket — v1".
     MAX_MESSAGE = 2000
     MAX_HANDLED_MESSAGE = 500
 
@@ -111,7 +111,13 @@ module Devbench
       # would replace the application's own exception, so nothing is.
       def capture(error, context:, handled:, symbol: nil)
         return nil unless error.is_a?(Exception)
-        return nil if reported?(error) || ignored?(error)
+        return nil if ignored?(error)
+
+        # The request (or job) this happened in has reported an exception,
+        # whichever hook sent it first: a 5xx after it is not also a
+        # `failed` request (Capability 2b).
+        note_exception
+        return nil if reported?(error)
 
         # Marked before sending, so a second hook reached while this one is
         # still writing does not send it again.
@@ -142,15 +148,21 @@ module Devbench
         false
       end
 
-      # The `exception` report for an error, without sending it. Used by
-      # the self-test (Devbench.test!), which sends one synthetic report
-      # straight to ingest, built exactly as a real one would be.
+      # The `exception` report for an error, built exactly as a real one
+      # would be, without sending it. (The self-test no longer uses it: it
+      # POSTs /v1/check and sends no report.)
       def report_for(error, context:, handled:, symbol: nil)
         exception_payload(error, context, handled, symbol)
       end
 
       private
 
+      def note_exception
+        Current.record_exception if Current.scoped?
+      rescue StandardError, SystemStackError
+        nil
+      end
+
       def mark_reported(error)
         error.instance_variable_set(REPORTED_IVAR, true)
       rescue FrozenError
@@ -177,7 +189,7 @@ module Devbench
       # message is detail.
       def handled_payload(error, symbol, reason)
         {
-          v: 1,
+          v: PROTOCOL_VERSION,
           kind: 'handled_failure',
           symbol: symbol,
           error: safely { error.class.name },
@@ -190,7 +202,7 @@ module Devbench
 
       def exception_payload(error, context, handled, symbol)
         {
-          v: 1,
+          v: PROTOCOL_VERSION,
           kind: 'exception',
           context: CONTEXTS.include?(context) ? context : 'explicit',
           handled: handled ? true : false,

data/lib/devbench/request_signals.rb

@@ -0,0 +1,359 @@
+# frozen_string_literal: true
+
+require_relative 'fingerprint'
+
+module Devbench
+  # Request signals (docs/server-sdk/failed-and-slow-requests.md, "Capability
+  # 2b — Failed and slow requests"; DECISIONS #171): what a request does
+  # without raising.
+  #
+  #   * failed: the application answered 500–599 and no exception was
+  #     reported during the request;
+  #   * slow: it answered 100–499, not streaming, after more than 3,000 ms;
+  #   * rate_limit_spike: one route's 429s in one unix minute met the spike
+  #     rule (at least ten, and at least 10% of that route's requests).
+  #
+  # A port of the backend's internal/reqsignal (and the Go SDK's copy,
+  # sdk/server/go/reqsignal.go). A port can drift, and a drifted port reports
+  # a different set of problems by language, so it is held to the shared
+  # vectors (testdata/request_signal_vectors.json) every implementation must
+  # reproduce. Change nothing here without changing those in lockstep.
+  #
+  # Devbench::Middleware observes; this module classifies, builds and keeps
+  # the per-process spike detector. Nothing here raises into the
+  # application.
+  module RequestSignals
+    # The thresholds: constants, not configuration. Tenant-configurable
+    # thresholds are a later issue (DECISIONS #171), and every SDK must apply
+    # the same numbers or the same app reports differently by language.
+    SLOW_THRESHOLD_MS = 3000
+    SPIKE_WINDOW_SECONDS = 60
+    SPIKE_MIN_COUNT = 10
+    SPIKE_MIN_PERCENT = 10
+    MAX_ROUTES_PER_WINDOW = 512
+    MAX_ROUTE_CHARS = 255
+
+    # A request matched to no route pattern: one literal, never the raw path
+    # (a path carries ids and slugs).
+    UNKNOWN_ROUTE = '*'
+    # Any method outside the standard set, so a client sending invented
+    # methods cannot mint fingerprints.
+    OTHER_METHOD = 'OTHER'
+    METHODS = %w[GET HEAD POST PUT PATCH DELETE OPTIONS TRACE CONNECT].freeze
+
+    # The `signal` field of the `request` control message.
+    FAILED = 'failed'
+    SLOW = 'slow'
+    RATE_LIMIT_SPIKE = 'rate_limit_spike'
+
+    # Everything the direct transport needs to count a request signal: the
+    # count's kind, the fingerprint's type and message, the template text
+    # kept for evidence, and the raw exemplar (scrubbed at egress).
+    Built = Struct.new(:kind, :type, :message, :text, :exemplar)
+
+    # One (method, route) whose window met the spike rule. `window` is the
+    # window's start, unix seconds.
+    Spike = Struct.new(:method, :route, :window, :count, :total)
+
+    # The clock the middleware reads: monotonic milliseconds for a request's
+    # duration, wall milliseconds for the spike window. Replaceable only so
+    # tests can pin a duration to the millisecond.
+    module SystemClock
+      def self.monotonic_ms
+        Process.clock_gettime(Process::CLOCK_MONOTONIC, :float_millisecond)
+      end
+
+      def self.wall_ms
+        Process.clock_gettime(Process::CLOCK_REALTIME, :millisecond)
+      end
+    end
+
+    # The 429 spike rule for one process's requests. Thread-safe: every
+    # request's thread observes, the flush thread ticks.
+    #
+    # One window is open at a time. An observation in a later window closes
+    # the open one first; one in an earlier window (a clock step back) counts
+    # in the open window. Memory is bounded by MAX_ROUTES_PER_WINDOW, and
+    # nothing is kept past the open window. A forked child starts empty: the
+    # parent reports what it saw.
+    class SpikeDetector
+      Tally = Struct.new(:count, :total)
+
+      def initialize
+        @lock = Mutex.new
+        reset
+      end
+
+      # Records one finished request at t_ms (unix milliseconds) and returns
+      # the spikes of any window it closed.
+      def observe(t_ms, method, route, status)
+        w = RequestSignals.window_of(t_ms)
+        key = [RequestSignals.normalize_method(method), RequestSignals.normalize_route(route)].freeze
+
+        @lock.synchronize do
+          check_fork
+          closed = []
+          if @window.nil?
+            start(w)
+          elsif w > @window
+            closed = evaluate
+            start(w)
+          end
+
+          tally = @routes[key]
+          if tally.nil?
+            return closed if @routes.size >= MAX_ROUTES_PER_WINDOW
+
+            tally = @routes[key] = Tally.new(0, 0)
+          end
+          tally.total += 1
+          tally.count += 1 if status == 429
+          closed
+        end
+      end
+
+      # Closes the open window if now_ms is past it (at each flush) and
+      # returns its spikes.
+      def tick(now_ms)
+        @lock.synchronize do
+          check_fork
+          return [] if @window.nil? || RequestSignals.window_of(now_ms) <= @window
+
+          out = evaluate
+          reset
+          out
+        end
+      end
+
+      # Evaluates the open window as it stands (process exit, a new
+      # configuration) and closes it.
+      def finish
+        @lock.synchronize do
+          check_fork
+          return [] if @window.nil?
+
+          out = evaluate
+          reset
+          out
+        end
+      end
+
+      private
+
+      def reset
+        @pid = Process.pid
+        @window = nil
+        @routes = {}
+      end
+
+      def check_fork
+        reset unless @pid == Process.pid
+      end
+
+      def start(window)
+        @window = window
+        @routes = {}
+      end
+
+      # The open window's spikes, sorted by method then route (byte order,
+      # as Go's string comparison). Caller holds @lock.
+      def evaluate
+        @routes.filter_map do |(method, route), tally|
+          next unless RequestSignals.spike?(tally.count, tally.total)
+
+          Spike.new(method, route, @window * SPIKE_WINDOW_SECONDS, tally.count, tally.total)
+        end.sort_by { |s| [s.method.b, s.route.b] }
+      end
+    end
+
+    class << self
+      attr_writer :clock
+
+      def clock
+        @clock || SystemClock
+      end
+
+      # The process's spike detector, in both transports.
+      def detector
+        @detector ||= SpikeDetector.new
+      end
+
+      # The signal one finished request produces: FAILED, SLOW or nil. At
+      # most one per request. 429s are not classified here; they feed the
+      # spike detector.
+      def classify(status:, duration_ms:, exception_reported:, streaming:)
+        return nil if status < 100 || status > 599
+        # A 5xx after a reported exception: the exception is the report.
+        return (exception_reported ? nil : FAILED) if status >= 500
+        return nil if streaming || status == 101
+
+        duration_ms > SLOW_THRESHOLD_MS ? SLOW : nil
+      end
+
+      # Uppercased and trimmed; anything outside the standard set is OTHER.
+      def normalize_method(method)
+        m = Fingerprint.trim(utf8(method)).upcase
+        METHODS.include?(m) ? m : OTHER_METHOD
+      end
+
+      # Trimmed and bounded to MAX_ROUTE_CHARS code points; empty is
+      # UNKNOWN_ROUTE.
+      def normalize_route(route)
+        r = Fingerprint.trim(utf8(route))
+        return UNKNOWN_ROUTE if r.empty?
+
+        r.length > MAX_ROUTE_CHARS ? r[0, MAX_ROUTE_CHARS] : r
+      end
+
+      # Fingerprint inputs, template text and exemplar for a request signal,
+      # as the sidecar builds them from the `request` control message. Nil
+      # for a signal/status pair a sensor must refuse.
+      def build(signal:, method:, route:, status:, duration_ms: 0, count: 0, total: 0)
+        status = status.to_i
+        message = "#{normalize_method(method)} #{normalize_route(route)}"
+        case signal
+        when FAILED
+          return nil unless status.between?(500, 599)
+
+          type = "http_#{status}"
+          new_built('error', type, message, "#{type}: #{message} in #{non_negative(duration_ms)} ms")
+        when SLOW
+          return nil unless status.between?(100, 499)
+
+          type = 'slow_request'
+          new_built('degradation', type, message,
+                    "#{type}: #{message} in #{non_negative(duration_ms)} ms (status #{status})")
+        when RATE_LIMIT_SPIKE
+          type = 'http_429_spike'
+          new_built('error', type, message,
+                    "#{type}: #{message} #{non_negative(count)} of #{non_negative(total)} " \
+                    'requests answered 429 in one minute')
+        end
+      end
+
+      # The rule, in integer arithmetic: at least SPIKE_MIN_COUNT 429s, and
+      # at least SPIKE_MIN_PERCENT of the route's requests in the window.
+      def spike?(count, total)
+        count >= SPIKE_MIN_COUNT && count * 100 >= total * SPIKE_MIN_PERCENT
+      end
+
+      # Floors toward negative infinity (Ruby's integer division does), so a
+      # pre-1970 clock still maps to one window per minute.
+      def window_of(t_ms)
+        t_ms.to_i / (SPIKE_WINDOW_SECONDS * 1000)
+      end
+
+      # ---- Reporting (called by Devbench::Middleware and the flushes)
+
+      # Classifies a finished request, reports its signal, and feeds the spike
+      # detector. Called inside the request's scope, so a signal carries its
+      # trace and user. Never raises.
+      def observe(status:, method:, route:, duration_ms:, streaming:, exception_reported:, finished_ms:)
+        return nil unless status.is_a?(Integer)
+
+        method = normalize_method(method)
+        route = normalize_route(route)
+        signal = classify(status: status, duration_ms: duration_ms,
+                          exception_reported: exception_reported, streaming: streaming)
+        emit(request_payload(signal, method, route, status, duration_ms)) unless signal.nil?
+        return nil unless status.between?(100, 599)
+
+        install_exit_hook
+        report_spikes(detector.observe(finished_ms, method, route, status))
+        start_flusher if status == 429
+        nil
+      rescue StandardError, SystemStackError
+        nil
+      end
+
+      # Closes the spike window if the clock has left it (at a flush) and
+      # reports its spikes to `transport`.
+      def tick!(transport = nil)
+        report_spikes(detector.tick(clock.wall_ms), transport)
+      rescue StandardError, SystemStackError
+        nil
+      end
+
+      # Evaluates the open window as it stands and reports its spikes (process
+      # exit, a replaced configuration). Returns how many were reported.
+      def finish!(transport = nil)
+        report_spikes(detector.finish, transport)
+      rescue StandardError, SystemStackError
+        0
+      end
+
+      # The `request` control message for a spike: an aggregate of many
+      # requests, so no trace and no user.
+      def spike_payload(spike)
+        { v: PROTOCOL_VERSION, kind: 'request', signal: RATE_LIMIT_SPIKE, method: spike.method,
+          route: spike.route, status: 429, count: spike.count, total: spike.total }
+      end
+
+      private
+
+      def new_built(kind, type, message, exemplar)
+        Built.new(kind, type, message, "#{type} at #{message}", exemplar)
+      end
+
+      def non_negative(n)
+        n = n.to_i
+        n.negative? ? 0 : n
+      end
+
+      def utf8(text)
+        text.to_s.encode('UTF-8', invalid: :replace, undef: :replace).scrub
+      end
+
+      def request_payload(signal, method, route, status, duration_ms)
+        {
+          v: PROTOCOL_VERSION, kind: 'request', signal: signal, method: method, route: route,
+          status: status, duration_ms: non_negative(duration_ms),
+          trace: safely { Current.trace&.to_s }, user: safely { Current.user }
+        }.compact
+      end
+
+      def report_spikes(spikes, transport = nil)
+        spikes.each { |s| emit(spike_payload(s), transport) }
+        spikes.length
+      end
+
+      def emit(payload, transport = nil)
+        (transport || Devbench.transport).deliver(payload)
+      rescue StandardError, SystemStackError
+        nil
+      end
+
+      # Direct mode: a process whose only traffic is a burst of 429s has
+      # counted nothing, so its flush thread may not be running; without it
+      # the spike would wait for the next request or the process's exit.
+      def start_flusher
+        transport = Devbench.transport
+        transport.start if transport.respond_to?(:start)
+      rescue StandardError, SystemStackError
+        nil
+      end
+
+      # The open window is evaluated at process exit, in either transport:
+      # the spikes are delivered, then direct mode flushes them (bounded, as
+      # the transport's own exit flush is). Inherited by forked children,
+      # which report their own window.
+      def install_exit_hook
+        return if @exit_hook_installed
+
+        @exit_hook_installed = true
+        at_exit do
+          reported = finish!
+          Devbench.transport.flush!(timeout: DirectTransport::EXIT_TIMEOUT) if reported.positive?
+        rescue StandardError, SystemStackError
+          nil
+        end
+      end
+
+      def safely
+        yield
+      rescue StandardError, SystemStackError
+        nil
+      end
+    end
+  end
+end

data/lib/devbench/scrub.rb

@@ -25,6 +25,25 @@ module Devbench
     end
     private_class_method :go_re
 
+    # The card and phone rules (#186; internal/scrub/numbers.go, the
+    # `numbers` vectors): a candidate regex plus checks on the whole
+    # candidate, run in the gsub block. A rejected candidate is left as
+    # written and matching carries on after it; it is never retried shorter.
+    # No lookaround in the candidates (the Go SDK is RE2), so every SDK finds
+    # the same ones.
+    CARD = '<redacted:card>'
+    PHONE = '<redacted:phone>'
+    CARD_CANDIDATE = go_re('\b(?:\d[ \-]*?){13,19}\b')
+    # Group 1 is the character before the number (or the start of the text):
+    # a phone number is not glued to a letter, digit, `_` or `-`, which keeps
+    # a UUID's tail and `ord_123456789` out. Group 2 is the candidate.
+    PHONE_CANDIDATE = /(\A|[^A-Za-z0-9_\-])(\+?\d[\d\t\n\f\r ().\-]{7,}\d)/
+    ISO_DATE_PREFIX = /\A\d{4}-\d{2}-\d{2}(?:\z|\D)/
+    DECIMAL = /\A\d+\.\d+\z/
+    WORD_BYTE = /\A[A-Za-z0-9_]/
+    # E.164 caps a phone number at 15 digits; anything longer is an id.
+    PHONE_DIGITS = (7..15)
+
     # Ordered most-specific first, and the order is load-bearing: see the
     # comments on internal/scrub's rules (a credential containing an email,
     # a phone pattern eating an SSN).
@@ -39,11 +58,14 @@ module Devbench
       [go_re('\b(password|passwd|pwd|secret|api[_\-]?key|access[_\-]?token|refresh[_\-]?token|' \
              'client[_\-]?secret|private[_\-]?key|session[_\-]?id|csrf[_\-]?token)' \
              "(#{WS}*[:=]#{WS}*)(\"[^\"]*\"|'[^']*'|#{NOT_WS}+)", Regexp::IGNORECASE), '\1\2<redacted:secret>'],
-      [go_re('\b[^\t\n\f\r <>@]+@[^\t\n\f\r <>@]+\.[A-Za-z]{2,}\b'), '<redacted:email>'],
+      # The local part stops at `=` and `:` (#148): `customer=sam@x.com`
+      # keeps its key. Same class as the template's email rule.
+      [go_re('\b[^\t\n\f\r <>@=:]+@[^\t\n\f\r <>@]+\.[A-Za-z]{2,}\b'), '<redacted:email>'],
       [go_re('\b\d{3}-\d{2}-\d{4}\b'), '<redacted:ssn>'],
-      [go_re('\b(?:\d[ \-]*?){13,19}\b'), '<redacted:card>'],
+      # The card and phone rules (#186) judge each candidate in the block.
+      [CARD_CANDIDATE, ->(m) { card?(m[0]) ? CARD : m[0] }],
       [go_re('\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b'), '<redacted:ip>'],
-      [/\+?\d[\d\t\n\f\r ().\-]{7,}\d/, '<redacted:phone>']
+      [PHONE_CANDIDATE, ->(m) { phone?(m[2], m.string[m.end(0)].to_s) ? "#{m[1]}#{PHONE}" : m[0] }]
     ].freeze
 
     # internal/sidecar/trace.go's traceRe.
@@ -75,7 +97,33 @@ module Devbench
     class << self
       # internal/scrub.Text: every shape rule, in order.
       def text(line)
-        RULES.reduce(utf8(line)) { |out, (re, with)| out.gsub(re, with) }
+        RULES.reduce(utf8(line)) do |out, (re, with)|
+          with.is_a?(Proc) ? out.gsub(re) { with.call(Regexp.last_match) } : out.gsub(re, with)
+        end
+      end
+
+      # A card: its digits pass the Luhn check, and it is not a bare 13-digit
+      # run starting with 1 (a millisecond timestamp).
+      def card?(candidate)
+        digits = candidate.delete('^0-9')
+        return false if digits == candidate && timestamp?(digits)
+
+        luhn?(digits)
+      end
+
+      # A phone, judged with the character after it ('' at the end of the
+      # text; one character is all the check reads). Not one when a letter, digit
+      # or `_` follows; it starts with an ISO date; it is a decimal; it has
+      # fewer than 7 or more than 15 digits; or it is a bare 10- or 13-digit
+      # run starting with 1 (a timestamp).
+      def phone?(candidate, after)
+        return false if WORD_BYTE.match?(after)
+        return false if ISO_DATE_PREFIX.match?(candidate) || DECIMAL.match?(candidate)
+
+        digits = candidate.delete('^0-9')
+        return false unless PHONE_DIGITS.cover?(digits.length)
+
+        !(digits == candidate && timestamp?(digits))
       end
 
       # internal/scrub.Prose: runs of two or more capitalised words that are
@@ -153,6 +201,26 @@ module Devbench
 
       private
 
+      # A Unix timestamp in seconds or milliseconds (2001 to 2033).
+      def timestamp?(digits)
+        (digits.length == 10 || digits.length == 13) && digits.start_with?('1')
+      end
+
+      def luhn?(digits)
+        return false if digits.empty?
+
+        sum = 0
+        digits.reverse.each_char.with_index do |c, i|
+          d = c.ord - 48
+          if i.odd?
+            d *= 2
+            d -= 9 if d > 9
+          end
+          sum += d
+        end
+        (sum % 10).zero?
+      end
+
       # Base redaction (template + scrub), the prose heuristic unless the
       # shape is marked 'full', then field and term rules.
       def redact_body(body, rule, policy)