@provos/ironcurtain 0.10.0 → 0.12.0

package/dist/docker/mitm-proxy.js

@@ -19,10 +19,14 @@ import * as net from 'node:net';
 import { existsSync, unlinkSync } from 'node:fs';
 import forge from 'node-forge';
 import { randomSerialNumber } from './ca.js';
-import { isEndpointAllowed, shouldRewriteBody, } from './provider-config.js';
+import { isCapturableEndpoint, isEndpointAllowed, shouldRewriteBody, } from './provider-config.js';
 import { DENY_ALL_VALIDATOR } from './package-validator.js';
 import { validateDomain } from './proxy-tools.js';
+import { PassThrough } from 'node:stream';
+import { getTokenStreamBus } from './token-stream-bus.js';
+import { SseExtractorTransform } from './sse-extractor.js';
 import * as logger from '../logger.js';
+import { beginCaptureExchange, createResponseCaptureInlet } from './trajectory-tap.js';
 /** Connection reset errors are routine during proxy shutdown or client disconnect. */
 function isConnectionReset(err) {
     return err.code === 'ECONNRESET' || err.code === 'EPIPE';
@@ -30,6 +34,47 @@ function isConnectionReset(err) {
 const MAX_REWRITE_BODY_BYTES = 10 * 1024 * 1024; // 10 MB
 /** Max Content-Length for which we buffer solely for 401 retry (1 MB). */
 const MAX_RETRY_BODY_BYTES = 1 * 1024 * 1024;
+/**
+ * Max bytes buffered from a non-streaming JSON response body for token stream
+ * extraction. The response is still streamed to the client unchanged; only
+ * the extraction side stops capturing once the cap is hit. This protects the
+ * proxy from unbounded memory growth on very large completions.
+ */
+export const MAX_JSON_RESPONSE_CAPTURE_BYTES = 2 * 1024 * 1024; // 2 MB
+/**
+ * Create a size-bounded capture for a JSON response body.
+ *
+ * Up to `maxBytes` of data is accumulated into an internal chunk array.
+ * Once that threshold is crossed, any already-buffered chunks are released
+ * (so large responses do not pin memory) and further chunks are ignored.
+ * The caller is still responsible for piping the stream through to the
+ * client -- this helper only manages the extraction-side buffer.
+ */
+export function createBoundedJsonResponseCapture(maxBytes = MAX_JSON_RESPONSE_CAPTURE_BYTES) {
+    let chunks = [];
+    let capturedBytes = 0;
+    return {
+        onData(chunk) {
+            if (capturedBytes > maxBytes)
+                return;
+            capturedBytes += chunk.length;
+            if (capturedBytes > maxBytes) {
+                chunks = [];
+                return;
+            }
+            chunks.push(chunk);
+        },
+        onEnd(onComplete) {
+            if (capturedBytes > maxBytes) {
+                onComplete(null);
+                return;
+            }
+            const body = Buffer.concat(chunks);
+            chunks = [];
+            onComplete(body);
+        },
+    };
+}
 /**
  * Buffers the entire request body into a single Buffer.
  * Rejects if the body exceeds maxBytes.
@@ -52,17 +97,262 @@ function bufferRequestBody(req, maxBytes) {
         req.on('error', reject);
     });
 }
+/**
+ * Maps a target hostname to the SSE provider for token stream parsing.
+ * Only Anthropic and OpenAI hosts are recognized; everything else
+ * falls back to 'unknown'.
+ */
+export function resolveSseProvider(hostname) {
+    if (hostname === 'api.anthropic.com' || hostname === 'platform.claude.com') {
+        return 'anthropic';
+    }
+    if (hostname === 'api.openai.com') {
+        return 'openai';
+    }
+    return 'unknown';
+}
+/** Maximum characters for tool_result content before truncation. */
+const MAX_TOOL_RESULT_CONTENT_LEN = 500;
+/** Truncate tool result content to MAX_TOOL_RESULT_CONTENT_LEN, appending ellipsis if needed. */
+function truncateToolResult(text) {
+    return text.length > MAX_TOOL_RESULT_CONTENT_LEN ? text.slice(0, MAX_TOOL_RESULT_CONTENT_LEN) + '\u2026' : text;
+}
+/** Returns true for LLM messages endpoints that carry multi-turn request bodies. */
+function isLlmMessagesEndpoint(path) {
+    const p = path.split('?')[0];
+    return p === '/v1/messages' || p === '/v1/chat/completions';
+}
+/**
+ * Extract tool_result blocks from a parsed Anthropic/OpenAI request body
+ * and push them as TokenStreamEvents to the bus.
+ *
+ * Anthropic format: messages[].role === 'user', content[].type === 'tool_result'
+ * OpenAI format: messages[].role === 'tool', with tool_call_id
+ *
+ * Only extracts from the trailing user/tool messages to avoid re-emitting
+ * historical tool results (the full conversation history is sent with every request).
+ */
+function extractTrailingToolMessages(messages) {
+    // Walk backward from the end to find the contiguous block of
+    // user (with tool_result) and tool messages at the tail.
+    const trailing = [];
+    for (let i = messages.length - 1; i >= 0; i--) {
+        const msg = messages[i];
+        if (typeof msg !== 'object' || msg === null)
+            break;
+        const role = msg['role'];
+        if (role === 'user' || role === 'tool') {
+            trailing.push(msg);
+        }
+        else {
+            break;
+        }
+    }
+    trailing.reverse();
+    return trailing;
+}
+export function extractToolResults(parsed, sessionId) {
+    const messages = parsed['messages'];
+    if (!Array.isArray(messages))
+        return;
+    // Fetch the singleton bus lazily so `resetTokenStreamBus()` between tests
+    // is honored at call time (not import time).
+    const bus = getTokenStreamBus();
+    const now = Date.now();
+    // Only extract from the LAST user/tool messages to avoid re-emitting
+    // historical tool results from earlier turns (the full conversation
+    // history is sent with every API request).
+    const lastMessages = extractTrailingToolMessages(messages);
+    for (const msg of lastMessages) {
+        if (typeof msg !== 'object' || msg === null)
+            continue;
+        const record = msg;
+        // Anthropic format: role=user, content array with tool_result blocks
+        if (record['role'] === 'user') {
+            const content = record['content'];
+            if (!Array.isArray(content))
+                continue;
+            for (const block of content) {
+                if (typeof block !== 'object' || block === null)
+                    continue;
+                const b = block;
+                if (b['type'] !== 'tool_result')
+                    continue;
+                const toolUseId = typeof b['tool_use_id'] === 'string' ? b['tool_use_id'] : '';
+                const isError = b['is_error'] === true;
+                const rawContent = b['content'];
+                let text = '';
+                if (typeof rawContent === 'string') {
+                    text = rawContent;
+                }
+                else if (Array.isArray(rawContent)) {
+                    // Content can be an array of {type: 'text', text: '...'} blocks
+                    const parts = [];
+                    for (const part of rawContent) {
+                        if (typeof part === 'object' && part !== null && part['type'] === 'text') {
+                            const t = part['text'];
+                            if (typeof t === 'string')
+                                parts.push(t);
+                        }
+                    }
+                    text = parts.join('\n');
+                }
+                text = truncateToolResult(text);
+                bus.push(sessionId, {
+                    kind: 'tool_result',
+                    toolUseId,
+                    toolName: '',
+                    content: text,
+                    isError,
+                    timestamp: now,
+                });
+            }
+            continue;
+        }
+        // OpenAI format: role=tool, content is a string
+        if (record['role'] === 'tool') {
+            const toolCallId = typeof record['tool_call_id'] === 'string' ? record['tool_call_id'] : '';
+            let text = typeof record['content'] === 'string' ? record['content'] : '';
+            const isError = false; // OpenAI doesn't have an is_error field
+            text = truncateToolResult(text);
+            bus.push(sessionId, {
+                kind: 'tool_result',
+                toolUseId: toolCallId,
+                toolName: '',
+                content: text,
+                isError,
+                timestamp: now,
+            });
+        }
+    }
+}
+/**
+ * Extract token stream events from a non-streaming JSON response body.
+ * Used when the upstream returns application/json instead of text/event-stream.
+ *
+ * Emits message_start (model), text_delta (content), and message_end (usage).
+ */
+export function extractFromJsonResponse(body, sessionId) {
+    let parsed;
+    try {
+        parsed = JSON.parse(body.toString());
+    }
+    catch {
+        return; // Not valid JSON -- nothing to extract
+    }
+    // Fetch the singleton bus lazily so `resetTokenStreamBus()` between tests
+    // is honored at call time (not import time).
+    const bus = getTokenStreamBus();
+    const now = Date.now();
+    // Anthropic format: { model, content: [{type: 'text', text: '...'}], usage: {input_tokens, output_tokens}, stop_reason }
+    const model = typeof parsed['model'] === 'string' ? parsed['model'] : null;
+    if (model) {
+        bus.push(sessionId, { kind: 'message_start', model, timestamp: now });
+    }
+    // Extract text from content blocks (Anthropic) or choices (OpenAI)
+    const content = parsed['content'];
+    if (Array.isArray(content)) {
+        // Anthropic: content is an array of blocks
+        for (const block of content) {
+            if (typeof block === 'object' && block !== null) {
+                const b = block;
+                if (b['type'] === 'text' && typeof b['text'] === 'string') {
+                    bus.push(sessionId, { kind: 'text_delta', text: b['text'], timestamp: now });
+                }
+            }
+        }
+    }
+    const choices = parsed['choices'];
+    if (Array.isArray(choices)) {
+        // OpenAI: choices[].message.content
+        for (const choice of choices) {
+            if (typeof choice === 'object' && choice !== null) {
+                const c = choice;
+                const message = c['message'];
+                if (typeof message === 'object' && message !== null) {
+                    const m = message;
+                    if (typeof m['content'] === 'string') {
+                        bus.push(sessionId, { kind: 'text_delta', text: m['content'], timestamp: now });
+                    }
+                }
+            }
+        }
+    }
+    // Usage and stop reason
+    const usage = parsed['usage'];
+    const stopReason = typeof parsed['stop_reason'] === 'string' ? parsed['stop_reason'] : 'stop';
+    let inputTokens = 0;
+    let outputTokens = 0;
+    if (typeof usage === 'object' && usage !== null) {
+        const u = usage;
+        // Anthropic: input_tokens/output_tokens; OpenAI: prompt_tokens/completion_tokens
+        inputTokens =
+            typeof u['input_tokens'] === 'number'
+                ? u['input_tokens']
+                : typeof u['prompt_tokens'] === 'number'
+                    ? u['prompt_tokens']
+                    : 0;
+        outputTokens =
+            typeof u['output_tokens'] === 'number'
+                ? u['output_tokens']
+                : typeof u['completion_tokens'] === 'number'
+                    ? u['completion_tokens']
+                    : 0;
+    }
+    bus.push(sessionId, {
+        kind: 'message_end',
+        stopReason,
+        inputTokens,
+        outputTokens,
+        timestamp: now,
+    });
+}
 export function createMitmProxy(options) {
+    // Token stream extractor installation is gated on `tokenSessionId`.
+    // This is a mutable per-proxy value that callers flip around each
+    // active agent (see `setTokenSessionId` on the returned handle). When
+    // `undefined`, all extractor-install sites skip `bus.push()` entirely
+    // — no sentinel value is published. The module-scoped singleton bus is
+    // fetched lazily at each push site so `resetTokenStreamBus()` between
+    // tests is honored.
+    let tokenSessionId = options.initialTokenSessionId;
+    /**
+     * Capture-attribution session ID. Decoupled from `tokenSessionId` so
+     * a future extractor-related flip cannot reattribute in-flight
+     * captured exchanges. See §11 in the capture design.
+     */
+    let captureSessionId;
+    /** Persona stamped onto subsequent captured ExchangeRecords. */
+    let capturePersona;
+    const agentKind = options.agentKind;
+    const captureWriter = options.capture;
+    const captureRecordedAgentName = options.recordedAgentName;
+    const captureWorkflowRunId = options.workflowRunId;
+    const captureBundleId = options.bundleId;
     // Parse CA cert and key from PEM
     const caCert = forge.pki.certificateFromPem(options.ca.certPem);
     const caKey = forge.pki.privateKeyFromPem(options.ca.keyPem);
+    // Hostnames are case-insensitive (RFC 1035 §2.3.3, RFC 7230 §5.4). Normalize
+    // to lowercase at every storage and lookup site so the allowlist cannot be
+    // bypassed by varying the case (e.g. `CONNECT API.TEST.COM:443` while the
+    // provider is registered as `api.test.com`).
+    function normalizeHost(hostname) {
+        return hostname.toLowerCase();
+    }
     // Build host → provider lookup
     const providersByHost = new Map();
     for (const mapping of options.providers) {
-        providersByHost.set(mapping.config.host, mapping);
+        providersByHost.set(normalizeHost(mapping.config.host), mapping);
     }
     // Dynamically added passthrough hosts (no key swap, no endpoint filtering)
     const passthroughHosts = new Set();
+    // Escape hatch: when IRONCURTAIN_MITM_ALLOW_ALL_HOSTS is set to "1" or "true",
+    // any host that is NOT a configured provider or registry is treated as a
+    // passthrough tunnel. Known providers/registries still flow through the
+    // normal MITM path (TLS termination, credential swap, endpoint filtering) —
+    // this flag only widens the catch-all for unknown destinations. Defeats
+    // egress filtering for unknown hosts; only use in trusted environments.
+    const allowAllHosts = process.env.IRONCURTAIN_MITM_ALLOW_ALL_HOSTS === '1' || process.env.IRONCURTAIN_MITM_ALLOW_ALL_HOSTS === 'true';
     // Certificate cache: hostname → { ctx, expiresAt }
     const certCache = new Map();
     // Renew leaf certs 1 hour before they expire
@@ -104,14 +394,23 @@ export function createMitmProxy(options) {
     const registriesByHost = new Map();
     if (options.registries) {
         for (const reg of options.registries) {
-            registriesByHost.set(reg.host, reg);
+            registriesByHost.set(normalizeHost(reg.host), reg);
             if (reg.mirrorHosts) {
                 for (const mirror of reg.mirrorHosts) {
-                    registriesByHost.set(mirror, reg);
+                    registriesByHost.set(normalizeHost(mirror), reg);
                 }
             }
         }
     }
+    // True when a host is statically known to the proxy. Gates the wildcard
+    // escape hatch so known providers/registries always follow their dedicated
+    // MITM/registry path, never the passthrough tunnel. Accepts the hostname
+    // as given (case may vary on the wire); normalization happens here so all
+    // callers can trust the result regardless of input case.
+    function isKnownStaticHost(hostname) {
+        const h = normalizeHost(hostname);
+        return providersByHost.has(h) || registriesByHost.has(h);
+    }
     // AllowedVersionCache for tarball backstop
     const allowedVersionCache = new Map();
     const activeClientSockets = new Set();
@@ -130,6 +429,42 @@ export function createMitmProxy(options) {
             socket.end('HTTP/1.1 400 Bad Request\r\n\r\n');
         }
     });
+    // Fast-reject WebSocket upgrades on TLS-intercepted provider hosts.
+    //
+    // Codex CLI ≥ 0.142 talks to its inference endpoint over
+    // `wss://chatgpt.com/backend-api/codex/responses` (HTTP `GET` with
+    // `Upgrade: websocket`) and only falls back to the legacy `POST` + SSE
+    // path when the WS connection fails at the TRANSPORT level. Without this
+    // listener Node delivers the upgrade request to the `'request'` handler
+    // below, which forwards it via `https.request()`; the upstream's `101
+    // Switching Protocols` is then dropped on the floor (Node fires only
+    // `'close'` on a client request with no `'upgrade'` listener), and the
+    // agent's WS client sits on an open socket until its ~15s connect
+    // timeout, retrying ~7× (~112s) before falling back.
+    //
+    // We deliberately DESTROY the socket rather than replying with an HTTP
+    // 4xx/5xx: tungstenite surfaces a non-101 HTTP response as a fatal
+    // handshake error to the Codex turn (observed: `turn.failed` with the
+    // body text), whereas a connection-level failure (reset / EOF before
+    // handshake) is treated as a retryable transport error and triggers the
+    // POST/SSE fallback — the same path the timeout used to reach, just in
+    // milliseconds instead of ~15s per attempt.
+    //
+    // This is intentionally a fast-fail, NOT a bridge: bridging WS for
+    // provider hosts would let inference traffic bypass the SSE trajectory
+    // capture / token-stream extraction paths. A proper key-swapping WS
+    // bridge (with WS-frame reassembly for capture) is a follow-up; until
+    // then the agent's HTTP fallback is the supported transport.
+    innerServer.on('upgrade', (req, socket) => {
+        const meta = socketMetadata.get(req.socket);
+        const targetHost = meta?.host ?? 'unknown';
+        logger.info(`[mitm-proxy] REJECTED ${req.headers.upgrade ?? 'upgrade'} ${targetHost}${req.url ?? ''} ` +
+            '(WebSocket not bridged for MITM provider hosts; agent should fall back to HTTP)');
+        socket.on('error', () => {
+            /* swallow — socket is being torn down */
+        });
+        socket.destroy();
+    });
     // Lazy-loaded registry handler to avoid circular imports.
     // Cached after first import so subsequent calls are synchronous.
     let registryHandlerModule;
@@ -193,6 +528,45 @@ export function createMitmProxy(options) {
         }
         const provider = meta.provider;
         const { method, url: path, headers } = clientReq;
+        // Per-handler capture-attribution snapshot. Reading once at handler
+        // entry keeps every captured byte tied to the session active when
+        // the request arrived, even if `setCaptureSessionId(...)` flips
+        // mid-exchange. The snapshot is per-closure so HTTP keep-alive
+        // concurrent exchanges on the same TLS socket get independent
+        // attribution (§11 parallel-keep-alive correctness).
+        const sidAtCapture = captureSessionId;
+        let captureHandle;
+        // Capture-endpoint gate: only training-relevant completion endpoints are
+        // recorded. Host-shared housekeeping traffic (registry pagination,
+        // telemetry batches, settings/eval pings) is excluded so it never
+        // pollutes the corpus. Strictly subtractive — when capture is off or the
+        // endpoint isn't capturable, `captureHandle` stays undefined and
+        // `attachCaptureBranch` is a no-op; forwarding, key swap, and the
+        // 401-retry path are unaffected. The `captureWriter` short-circuit keeps
+        // a disabled proxy at zero per-request cost (no header clone, no glob
+        // match). See §3 integration point 4 of the capture design doc.
+        if (captureWriter && sidAtCapture && isCapturableEndpoint(provider.config, method, path)) {
+            // Snapshot the original (pre-key-swap) headers for the capture
+            // record. `modifiedHeaders` (below) gets the real key injected by
+            // `validateAndSwapApiKey` — we must capture from `headers`, not
+            // `modifiedHeaders`. The writer applies a second redaction pass
+            // defensively.
+            const captureRequestHeaders = { ...headers };
+            captureHandle = beginCaptureExchange({
+                writer: captureWriter,
+                sessionId: sidAtCapture,
+                persona: capturePersona,
+                workflowRunId: captureWorkflowRunId,
+                bundleId: captureBundleId,
+                recordedAgentName: captureRecordedAgentName,
+                host: targetHost,
+                path: String(path),
+                method: String(method),
+                requestHeaders: captureRequestHeaders,
+                requestContentEncoding: clientReq.headers['content-encoding']?.toLowerCase(),
+                requestStartedAt: Date.now(),
+            });
+        }
         // 1. Endpoint filtering
         if (!isEndpointAllowed(provider.config, method, path)) {
             logger.info(`[mitm-proxy] BLOCKED ${method} ${targetHost}${path}`);
@@ -219,11 +593,14 @@ export function createMitmProxy(options) {
         const needsRewrite = shouldRewriteBody(provider.config, method, path);
         // 401 retry only makes sense when we swapped our own managed credential
         const canRetryAuth = keyResult.swapped && !!provider.tokenManager;
-        // Only buffer for retry when the body is small enough (known Content-Length
-        // under 1MB). Large or chunked bodies stream through without retry support
-        // to avoid memory overhead and 413 rejections on large payloads.
+        // Only buffer for retry when the body is absent or small enough (known
+        // Content-Length under 1MB). Large or chunked bodies stream through without
+        // retry support to avoid memory overhead and 413 rejections on large
+        // payloads. Bodyless requests (notably Codex's model-list GET) still need
+        // the OAuth 401 refresh path.
         const contentLength = parseInt(clientReq.headers['content-length'] ?? '', 10);
-        const retryBufferOk = canRetryAuth && Number.isFinite(contentLength) && contentLength <= MAX_RETRY_BODY_BYTES;
+        const hasNoRequestBody = clientReq.headers['content-length'] === undefined && clientReq.headers['transfer-encoding'] === undefined;
+        const retryBufferOk = canRetryAuth && (hasNoRequestBody || (Number.isFinite(contentLength) && contentLength <= MAX_RETRY_BODY_BYTES));
         const needsBuffer = needsRewrite || retryBufferOk;
         /**
          * Sends the request upstream with optional body override and 401 retry.
@@ -294,7 +671,140 @@ export function createMitmProxy(options) {
                 clientRes.writeHead(upstreamRes.statusCode ?? 502, upstreamRes.headers);
                 clientRes.flushHeaders();
                 clientRes.socket?.setNoDelay(true);
-                upstreamRes.pipe(clientRes);
+                const contentType = upstreamRes.headers['content-type'] ?? '';
+                // Pin this response's events to the id active when it started,
+                // so an orchestrator mid-flight flip can't split one stream.
+                const sidAtAttach = tokenSessionId;
+                // Trajectory-capture response branch. The 401 refresh-retry
+                // path above already returned before reaching here, so the
+                // captureTap is bound only to the SUCCESSFUL upstream
+                // response — never the drained 401 (§5 "401 refresh-retry
+                // semantics").
+                //
+                // The capture branch is FANNED OUT from the upstream response
+                // (it does NOT sit in-series on the forwarding pipeline).
+                // Anthropic serves `/v1/messages` SSE with `content-encoding:
+                // gzip` by default; the forwarding path must continue to
+                // deliver the raw compressed bytes to the agent (its SDK
+                // decompresses transparently), while the capture path needs
+                // decompressed bytes so the reassembler can parse SSE events.
+                // See §3 and §6 invariant 6 of the design doc.
+                attachCaptureBranch(upstreamRes, captureHandle, sidAtCapture);
+                if (sidAtAttach && contentType.includes('text/event-stream')) {
+                    const tokenBus = getTokenStreamBus();
+                    const sseProvider = resolveSseProvider(targetHost);
+                    const extractor = new SseExtractorTransform(sseProvider, (event) => {
+                        tokenBus.push(sidAtAttach, event);
+                    });
+                    upstreamRes.pipe(extractor).pipe(clientRes);
+                }
+                else if (sidAtAttach && isLlmMessagesEndpoint(path) && contentType.includes('application/json')) {
+                    // Non-streaming JSON response: buffer for extraction while piping to client.
+                    // The passthrough stream always forwards the full response to the client.
+                    // Capture is bounded at MAX_JSON_RESPONSE_CAPTURE_BYTES: once exceeded,
+                    // we stop accumulating chunks (and skip extraction at end-of-stream),
+                    // but the passthrough listener stays attached so the stream completes
+                    // normally and the client still sees every byte.
+                    const passthrough = new PassThrough();
+                    const capture = createBoundedJsonResponseCapture();
+                    passthrough.on('data', (chunk) => capture.onData(chunk));
+                    passthrough.on('end', () => {
+                        capture.onEnd((body) => {
+                            if (body === null)
+                                return; // overflowed -- skip extraction
+                            try {
+                                extractFromJsonResponse(body, sidAtAttach);
+                            }
+                            catch {
+                                // Extraction errors must never affect the forwarding path
+                            }
+                        });
+                    });
+                    upstreamRes.pipe(passthrough).pipe(clientRes);
+                }
+                else {
+                    upstreamRes.pipe(clientRes);
+                }
+                /**
+                 * Attach the trajectory capture branch to `upstreamRes` via a
+                 * fan-out tee. Bytes the agent receives are unaffected. When
+                 * `content-encoding` is supported we decompress before feeding
+                 * the captureTap; unsupported encodings poison the session
+                 * with `unsupported-encoding`.
+                 */
+                function attachCaptureBranch(upstream, handle, sessionId) {
+                    if (!handle || !sessionId)
+                        return;
+                    const captureTap = handle.attachResponse({
+                        statusCode: upstream.statusCode ?? 502,
+                        headers: upstream.headers,
+                    });
+                    const inlet = createResponseCaptureInlet({
+                        captureTap,
+                        contentEncoding: upstream.headers['content-encoding'],
+                        captureHandle: handle,
+                        onPoison: (reason) => {
+                            try {
+                                captureWriter?.markSessionPoisoned(sessionId, reason);
+                            }
+                            catch {
+                                /* swallow — capture must never disturb forwarding */
+                            }
+                        },
+                    });
+                    // Tee upstream bytes into the capture inlet via data
+                    // listeners. This pattern coexists with the sibling
+                    // `upstreamRes.pipe(...)` for forwarding because Node
+                    // broadcasts `data` events to all listeners, and pipe is
+                    // implemented on top of the same event. Errors / aborts on
+                    // the upstream stream are propagated to the inlet so the
+                    // captureTap can settle cleanly (or surface a poison).
+                    upstream.on('data', (chunk) => {
+                        if (!inlet.writableEnded && !inlet.destroyed) {
+                            inlet.write(chunk);
+                        }
+                    });
+                    upstream.on('end', () => {
+                        if (!inlet.writableEnded && !inlet.destroyed) {
+                            inlet.end();
+                        }
+                    });
+                    upstream.on('error', () => {
+                        // On upstream 'error'/'aborted' a terminal event may already
+                        // be buffered in a decompressor inlet. Prefer a graceful
+                        // `inlet.end()` flush (NOT destroy) so zlib `_flush` emits the
+                        // buffered tail and the tap's `canFinalize()` can recover a
+                        // complete-but-aborted stream; `canFinalize()` is the primary
+                        // recovery mechanism (codex is uncompressed, so the inlet is
+                        // the captureTap itself and `end()` simply triggers finalize).
+                        // Genuine decompressor corruption surfaces as a zlib 'error'
+                        // on the inlet, which poisons `reassembly-failure` via
+                        // `createResponseCaptureInlet`'s own handler. Reserve
+                        // `inlet.destroy()` for the case where the inlet cannot be
+                        // ended (already errored).
+                        if (!inlet.writableEnded && !inlet.destroyed) {
+                            inlet.end();
+                        }
+                    });
+                    upstream.on('close', () => {
+                        // If `end` already fired, the inlet has been ended. If we
+                        // got here without `end` and without an `error`, this is a
+                        // graceful close-after-data (e.g. the client disconnected
+                        // after the completion). Call `inlet.end()` — NOT
+                        // `inlet.destroy()` — so a decompressor inlet runs zlib
+                        // `_flush`, emits its buffered tail + `end` to the
+                        // captureTap, and the reassembler sees its terminal event.
+                        // `inlet.destroy(err)` is reserved for genuine upstream
+                        // `error` (handled above), which would discard the
+                        // decompressor's pending tail. The tap's `canFinalize()`
+                        // lifecycle then writes a faithful record for a
+                        // complete-but-socket-aborted stream, or poisons
+                        // mid-stream-abort only if the terminal event never landed.
+                        if (!inlet.writableEnded && !inlet.destroyed) {
+                            inlet.end();
+                        }
+                    });
+                }
             });
             activeUpstreamRequests.add(upstreamReq);
             upstreamReq.on('close', () => activeUpstreamRequests.delete(upstreamReq));
@@ -324,7 +834,23 @@ export function createMitmProxy(options) {
                     log(`[mitm-proxy] client request error: ${err.message}`);
                     upstreamReq.destroy();
                 });
-                clientReq.pipe(upstreamReq);
+                // Trajectory-capture tee: when capture is active and we are NOT
+                // buffering for rewrite/retry, install a PassThrough between
+                // clientReq and upstreamReq so we can observe each request chunk
+                // on the way through. The tap's data listener never modifies
+                // bytes; the forwarding pipe stays line-rate. Adding a 'data'
+                // listener directly to clientReq would flip it to flowing mode
+                // and race the upstream pipe — the PassThrough avoids that.
+                if (captureHandle) {
+                    const reqTap = new PassThrough();
+                    const handle = captureHandle;
+                    reqTap.on('data', (chunk) => handle.pushRequestChunk(chunk));
+                    reqTap.on('end', () => handle.finishRequest());
+                    clientReq.pipe(reqTap).pipe(upstreamReq);
+                }
+                else {
+                    clientReq.pipe(upstreamReq);
+                }
             }
         }
         /**
@@ -370,17 +896,34 @@ export function createMitmProxy(options) {
                     }
                     return;
                 }
+                // Trajectory-capture: feed the buffered raw body into the
+                // capture record. This is the pre-rewrite, pre-key-swap body —
+                // exactly what the agent emitted. See §3 / §6 invariant #2.
+                captureHandle?.setRequestBody(rawBody);
                 let finalBody = rawBody;
+                // Extract tool_result events from the request body for token stream observation.
+                // Snapshot here (before rewrite) so a concurrent setTokenSessionId
+                // flip can't split this request's events across two ids.
+                const sidForToolResults = tokenSessionId;
+                if (sidForToolResults && isLlmMessagesEndpoint(path)) {
+                    try {
+                        const bodyParsed = JSON.parse(rawBody.toString());
+                        extractToolResults(bodyParsed, sidForToolResults);
+                    }
+                    catch {
+                        // Parse failure is fine -- the rewriter below will also attempt to parse
+                    }
+                }
                 if (needsRewrite) {
                     const rewriter = provider.config.requestRewriter;
                     const reqMethod = method;
                     const reqPath = path;
                     try {
                         const parsed = JSON.parse(rawBody.toString());
-                        const result = rewriter(parsed, { method: reqMethod, path: reqPath });
+                        const result = rewriter(parsed, { method: reqMethod, path: reqPath, agentKind });
                         if (result) {
                             finalBody = Buffer.from(JSON.stringify(result.modified));
-                            logger.info(`[mitm-proxy] POST ${targetHost}${path} - stripped server-side tools: ${result.stripped.join(', ')}`);
+                            logger.info(`[mitm-proxy] POST ${targetHost}${path} - stripped tools: ${result.stripped.join(', ')}`);
                         }
                     }
                     catch {
@@ -505,11 +1048,29 @@ export function createMitmProxy(options) {
         clientReq.pipe(upstreamReq);
     }
     // Outer server - UDS listener, handles CONNECT and plain HTTP proxy requests
-    const outerServer = http.createServer((req, res) => {
-        // Handle plain HTTP proxy requests for passthrough domains.
+    const outerServer = http.createServer();
+    // Connection-source guard (tcp-hostonly topology): destroy sockets from
+    // outside the allowed subnet before any HTTP parsing happens. Covers
+    // plain requests, CONNECT, and WebSocket upgrades alike since all ride
+    // on the same TCP connections.
+    if (options.allowRemoteAddress) {
+        const allowRemoteAddress = options.allowRemoteAddress;
+        outerServer.on('connection', (socket) => {
+            if (!allowRemoteAddress(socket.remoteAddress)) {
+                socket.destroy();
+            }
+        });
+    }
+    outerServer.on('request', (req, res) => {
+        // Handle plain HTTP proxy requests for passthrough and Debian registry domains.
         // HTTP proxy clients send absolute URLs: "GET http://host:port/path".
+        // Only Debian registries are included here because apt uses plain HTTP
+        // by default. npm/PyPI use HTTPS CONNECT and go through the registry-proxy
+        // validation path instead.
         const parsed = req.url ? tryParseProxyUrl(req.url) : null;
-        if (parsed && passthroughHosts.has(parsed.hostname)) {
+        const isDebianRegistry = parsed ? registriesByHost.get(parsed.hostname)?.type === 'debian' : false;
+        const isWildcardEligible = !!parsed && allowAllHosts && !isKnownStaticHost(parsed.hostname);
+        if (parsed && (passthroughHosts.has(parsed.hostname) || isDebianRegistry || isWildcardEligible)) {
             forwardPlainHttpPassthrough(req, res, parsed.hostname, parsed.port, parsed.path);
             return;
         }
@@ -526,7 +1087,12 @@ export function createMitmProxy(options) {
     outerServer.on('connect', (req, clientSocket, head) => {
         const url = req.url ?? '';
         const colonIndex = url.lastIndexOf(':');
-        const host = colonIndex > 0 ? url.substring(0, colonIndex) : url;
+        const rawHost = colonIndex > 0 ? url.substring(0, colonIndex) : url;
+        // Normalize so the allowlist is case-insensitive (RFC 1035 §2.3.3).
+        // Critical when the wildcard escape hatch is enabled — without this,
+        // `CONNECT API.TEST.COM:443` would miss the provider map and be treated
+        // as an unknown host, opening a raw tunnel that bypasses MITM.
+        const host = normalizeHost(rawHost);
         const port = colonIndex > 0 ? parseInt(url.substring(colonIndex + 1), 10) : 443;
         const connId = ++connectionId;
         // Handle client socket errors early to prevent uncaught 'error' events
@@ -541,14 +1107,17 @@ export function createMitmProxy(options) {
         // 1. Check allowlist (providers, registries, and dynamic passthrough)
         const provider = providersByHost.get(host);
         const registry = registriesByHost.get(host);
-        const isPassthrough = !provider && !registry && passthroughHosts.has(host);
+        const isDynamicPassthrough = passthroughHosts.has(host);
+        const isWildcardEligible = allowAllHosts && !isKnownStaticHost(host);
+        const isPassthrough = !provider && !registry && (isDynamicPassthrough || isWildcardEligible);
         if (!provider && !registry && !isPassthrough) {
             logger.info(`[mitm-proxy] #${connId} DENIED CONNECT ${host}:${port}`);
             clientSocket.write('HTTP/1.1 403 Forbidden\r\n\r\n');
             clientSocket.destroy();
             return;
         }
-        const connType = provider ? 'provider' : registry ? 'registry' : 'passthrough';
+        const passthroughKind = isDynamicPassthrough ? 'passthrough' : 'passthrough-wildcard';
+        const connType = provider ? 'provider' : registry ? 'registry' : passthroughKind;
         // 2. For passthrough domains, create a raw TCP tunnel (no MITM).
         //    The proxy just pipes bytes bidirectionally — this supports both
         //    plain HTTP and WebSocket connections through CONNECT.
@@ -644,7 +1213,8 @@ export function createMitmProxy(options) {
     // of routing through the normal 'request' handler.
     outerServer.on('upgrade', (req, clientSocket, head) => {
         const parsed = req.url ? tryParseProxyUrl(req.url) : null;
-        if (!parsed || !passthroughHosts.has(parsed.hostname)) {
+        const isWildcardEligible = !!parsed && allowAllHosts && !isKnownStaticHost(parsed.hostname);
+        if (!parsed || (!passthroughHosts.has(parsed.hostname) && !isWildcardEligible)) {
             clientSocket.end('HTTP/1.1 403 Forbidden\r\nConnection: close\r\nContent-Length: 0\r\n\r\n');
             return;
         }
@@ -824,17 +1394,18 @@ export function createMitmProxy(options) {
     const hostController = {
         addHost(domain) {
             validateDomain(domain);
-            if (providersByHost.has(domain))
+            const h = normalizeHost(domain);
+            if (providersByHost.has(h))
                 return false;
-            if (passthroughHosts.has(domain))
+            if (passthroughHosts.has(h))
                 return false;
-            passthroughHosts.add(domain);
+            passthroughHosts.add(h);
             // Pre-warm cert cache for the new domain
-            getOrCreateSecureContext(domain);
+            getOrCreateSecureContext(h);
             return true;
         },
         removeHost(domain) {
-            return passthroughHosts.delete(domain);
+            return passthroughHosts.delete(normalizeHost(domain));
         },
         listHosts() {
             return {
@@ -936,10 +1507,25 @@ export function createMitmProxy(options) {
     }
     return {
         hosts: hostController,
+        setTokenSessionId(id) {
+            tokenSessionId = id;
+        },
+        setCaptureSessionId(id) {
+            captureSessionId = id;
+        },
+        setCapturePersona(persona) {
+            capturePersona = persona;
+        },
         async start() {
+            // Emit the egress-filter downgrade notice once per actual listening
+            // proxy. Construction-time logging would fire even for proxies that
+            // are created and discarded (tests, aborted setup).
+            if (allowAllHosts) {
+                logger.warn('[mitm-proxy] IRONCURTAIN_MITM_ALLOW_ALL_HOSTS is set — unknown hosts will be tunneled (egress filtering DISABLED for unknown hosts)');
+            }
             // Pre-warm cert cache for all configured providers and registries
             for (const mapping of options.providers) {
-                getOrCreateSecureContext(mapping.config.host);
+                getOrCreateSecureContext(normalizeHost(mapping.config.host));
             }
             for (const host of registriesByHost.keys()) {
                 getOrCreateSecureContext(host);