@playcraft/cli 0.1.86 → 0.1.87

package/dist/workspace-runtime/codex/model-broker.js

@@ -1,29 +1,50 @@
-import { createHash, randomBytes, timingSafeEqual } from 'node:crypto';
+import { createHash, randomBytes, randomUUID, timingSafeEqual } from 'node:crypto';
 import { createServer } from 'node:http';
 import { Readable, Transform } from 'node:stream';
-import { pipeline } from 'node:stream/promises';
+import { finished, pipeline } from 'node:stream/promises';
 import { basename, dirname } from 'node:path';
 import { z } from 'zod';
 import { workspaceContextFromRoot } from '../workspaces/registry.js';
-import { PLATFORM_ROUTES, modelResponsesPath, platformEndpoint, workspaceExecutionTraceOriginSchema } from '@playcraft/common/cloud-workspace/runtime';
+import { PLATFORM_ROUTES, modelResponsesPath, platformEndpoint, workspaceExecutionTraceOriginSchema, } from '@playcraft/common/cloud-workspace/runtime';
 import { WorkspaceGenerationProvenanceBroker } from './generation-provenance-broker.js';
 import { WorkspaceRemixReferencesBroker } from './remix-references-broker.js';
 import { WorkspaceExecutionContextBroker } from './execution-context-broker.js';
 import { WorkspaceSkillDiscoveryBroker } from './skill-discovery-broker.js';
+import { ModelBrokerDiagnostics } from './model-broker-diagnostics.js';
+import { MODEL_REQUEST_BUDGET_MS, ModelBrokerRequestError, modelBrokerFailure, readModelRequestBody, } from './model-broker-request.js';
+import { WorkspaceModelCapture, captureSubmittedResponse, } from './model-capture.js';
 const credentialSchema = z.object({
-    token: z.string().min(1), expiresAt: z.string().datetime(),
-    configId: z.string().uuid(), model: z.string().min(1).max(128),
+    token: z.string().min(1),
+    expiresAt: z.string().datetime(),
+    configId: z.string().uuid(),
+    model: z.string().min(1).max(128),
     supportsReasoning: z.boolean().optional(),
     claims: z.object({
-        environment: z.string(), userId: z.string(), workspaceId: z.string(), runtimeId: z.string(),
-        generation: z.number().int(), runtimeEpoch: z.number().int(),
-        traceOrigin: z.custom(value => workspaceExecutionTraceOriginSchema.safeParse(value).success).optional(),
+        environment: z.string(),
+        userId: z.string(),
+        workspaceId: z.string(),
+        runtimeId: z.string(),
+        generation: z.number().int(),
+        runtimeEpoch: z.number().int(),
+        traceOrigin: z
+            .custom((value) => workspaceExecutionTraceOriginSchema.safeParse(value).success)
+            .optional(),
+        capturePolicy: z
+            .object({
+            enabled: z.literal(true),
+            policyVersion: z.literal('workspace-model-capture-v1'),
+            maxRequestBytes: z.number().int().positive(),
+            maxResponseBytes: z.number().int().positive(),
+            expiresAt: z.string().datetime(),
+        })
+            .optional(),
     }),
 });
 const MAX_MODEL_REQUEST_BYTES = 128 * 1024 * 1024;
 /** Keeps renewable execution credentials in control memory, outside Codex and project files. */
 export class WorkspaceModelBroker {
     state;
+    requestBudgetMs;
     server;
     baseUrl = '';
     entries = new Map();
@@ -35,12 +56,15 @@ export class WorkspaceModelBroker {
     references;
     executionContext;
     skills;
-    constructor(state) {
+    capture;
+    constructor(state, requestBudgetMs = MODEL_REQUEST_BUDGET_MS) {
         this.state = state;
+        this.requestBudgetMs = requestBudgetMs;
         this.generation = new WorkspaceGenerationProvenanceBroker(state);
         this.references = new WorkspaceRemixReferencesBroker(state);
         this.executionContext = new WorkspaceExecutionContextBroker(state);
         this.skills = new WorkspaceSkillDiscoveryBroker(state);
+        this.capture = new WorkspaceModelCapture(state);
     }
     async start() {
         if (this.server)
@@ -57,6 +81,7 @@ export class WorkspaceModelBroker {
         if (!address || typeof address === 'string')
             throw new Error('model_broker_unavailable');
         this.baseUrl = `http://127.0.0.1:${address.port}`;
+        await this.capture.start();
     }
     async threadConfig(cwd, modelConfigId, sessionId, reasoningEffort) {
         const { activate, ...settings } = await this.prepareThreadConfig(cwd, modelConfigId, sessionId, reasoningEffort);
@@ -69,24 +94,44 @@ export class WorkspaceModelBroker {
         const { workspaceId } = await workspaceContextFromRoot(this.state, cwd);
         if (modelConfigId !== undefined && !z.string().uuid().safeParse(modelConfigId).success)
             throw new Error('model_config_denied');
-        const routeKey = sessionId ? `${workspaceId}/${createHash('sha256').update(sessionId).digest('hex').slice(0, 32)}` : workspaceId;
+        const routeKey = sessionId
+            ? `${workspaceId}/${createHash('sha256').update(sessionId).digest('hex').slice(0, 32)}`
+            : workspaceId;
         let route = this.routes.get(routeKey);
-        const selected = modelConfigId ?? (route?.epoch === this.state.runtimeEpoch ? route.selected?.entry.modelConfigId : undefined);
+        const selected = modelConfigId ??
+            (route?.epoch === this.state.runtimeEpoch ? route.selected?.entry.modelConfigId : undefined);
         const cacheKey = `${workspaceId}:${selected ?? 'default'}:${reasoningEffort ?? 'default'}:${sessionId ?? ''}:${clientRequestId ?? ''}`;
-        let entry = this.entries.get(cacheKey) ?? [...this.entries.values()].find(item => item.workspaceId === workspaceId && item.epoch === this.state.runtimeEpoch && item.modelConfigId === selected
-            && item.sessionId === sessionId && item.clientRequestId === clientRequestId
-            && item.reasoningEffort === reasoningEffort);
+        let entry = this.entries.get(cacheKey) ??
+            [...this.entries.values()].find((item) => item.workspaceId === workspaceId &&
+                item.epoch === this.state.runtimeEpoch &&
+                item.modelConfigId === selected &&
+                item.sessionId === sessionId &&
+                item.clientRequestId === clientRequestId &&
+                item.reasoningEffort === reasoningEffort);
         if (!entry || entry.epoch !== this.state.runtimeEpoch) {
-            entry = { workspaceId, epoch: this.state.runtimeEpoch, cacheKey, modelConfigId: selected, reasoningEffort, sessionId, clientRequestId };
+            entry = {
+                workspaceId,
+                epoch: this.state.runtimeEpoch,
+                cacheKey,
+                modelConfigId: selected,
+                reasoningEffort,
+                sessionId,
+                clientRequestId,
+            };
             this.entries.set(cacheKey, entry);
         }
         // Resolve current model metadata and permission at every native turn boundary.
         const credential = await this.credential(entry, true);
         const boundaryId = sessionId && clientRequestId ? await this.generation.prepare() : undefined;
         if (!route || route.epoch !== this.state.runtimeEpoch) {
-            route = { workspaceId, epoch: this.state.runtimeEpoch, capability: randomBytes(32).toString('base64url'),
-                generationCapability: randomBytes(32).toString('base64url'), referencesCapability: randomBytes(32).toString('base64url'),
-                skillsCapability: randomBytes(32).toString('base64url') };
+            route = {
+                workspaceId,
+                epoch: this.state.runtimeEpoch,
+                capability: randomBytes(32).toString('base64url'),
+                generationCapability: randomBytes(32).toString('base64url'),
+                referencesCapability: randomBytes(32).toString('base64url'),
+                skillsCapability: randomBytes(32).toString('base64url'),
+            };
             this.routes.set(routeKey, route);
         }
         // Each session pins its prepared model; shared credential renewal cannot change an active turn.
@@ -95,50 +140,87 @@ export class WorkspaceModelBroker {
                 if (!this.valid(entry) || this.routes.get(routeKey) !== route)
                     throw new Error('model_access_denied');
                 route.lastAdmissionReason = undefined;
-                route.selected = { entry, model: credential.model,
-                    ...(sessionId && clientRequestId ? { feedback: { sessionId, clientRequestId, cwd, boundaryId, readOnly,
-                            ...(originCwd ? { originCwd } : {}) } } : {}) };
+                route.selected = {
+                    entry,
+                    model: credential.model,
+                    ...(sessionId && clientRequestId
+                        ? {
+                            feedback: {
+                                sessionId,
+                                clientRequestId,
+                                cwd,
+                                boundaryId,
+                                readOnly,
+                                ...(originCwd ? { originCwd } : {}),
+                            },
+                        }
+                        : {}),
+                };
             },
             model: credential.model,
             modelConfigId: credential.configId,
             modelProvider: 'playcraft_workspace',
             config: {
-                ...(sessionId ? { 'shell_environment_policy.set': {
-                        PLAYCRAFT_AGENT_GENERATION_ENDPOINT: `${this.baseUrl}/${routeKey}/generation-provenance`,
-                        PLAYCRAFT_AGENT_GENERATION_CAPABILITY: route.generationCapability,
-                        PLAYCRAFT_REMIX_REFERENCES_ENDPOINT: `${this.baseUrl}/${routeKey}/remix-references`,
-                        PLAYCRAFT_REMIX_REFERENCES_CAPABILITY: route.referencesCapability,
-                        PLAYCRAFT_SKILLS_ENDPOINT: `${this.baseUrl}/${routeKey}/skill-discovery`,
-                        PLAYCRAFT_SKILLS_CAPABILITY: route.skillsCapability,
-                    } } : {}),
+                ...(sessionId
+                    ? {
+                        'shell_environment_policy.set': {
+                            PLAYCRAFT_AGENT_GENERATION_ENDPOINT: `${this.baseUrl}/${routeKey}/generation-provenance`,
+                            PLAYCRAFT_AGENT_GENERATION_CAPABILITY: route.generationCapability,
+                            PLAYCRAFT_REMIX_REFERENCES_ENDPOINT: `${this.baseUrl}/${routeKey}/remix-references`,
+                            PLAYCRAFT_REMIX_REFERENCES_CAPABILITY: route.referencesCapability,
+                            PLAYCRAFT_SKILLS_ENDPOINT: `${this.baseUrl}/${routeKey}/skill-discovery`,
+                            PLAYCRAFT_SKILLS_CAPABILITY: route.skillsCapability,
+                        },
+                    }
+                    : {}),
                 'model_providers.playcraft_workspace': {
-                    name: 'PlayCraft Workspace', base_url: `${this.baseUrl}/${routeKey}`,
-                    wire_api: 'responses', requires_openai_auth: false,
+                    name: 'PlayCraft Workspace',
+                    base_url: `${this.baseUrl}/${routeKey}`,
+                    wire_api: 'responses',
+                    requires_openai_auth: false,
+                    request_max_retries: 0,
+                    stream_max_retries: 0,
+                    stream_idle_timeout_ms: MODEL_REQUEST_BUDGET_MS,
+                    supports_websockets: false,
                     http_headers: { Authorization: `Bearer ${route.capability}` },
                 },
+                'features.unbounded_connection_retries': false,
             },
         };
     }
     valid(entry) {
-        return Boolean(this.server && !this.state.journal.destroyed()
-            && this.state.runtimeEpoch === entry.epoch && this.state.journal.getWorkspace(entry.workspaceId)
-            && this.entries.get(entry.cacheKey) === entry);
+        return Boolean(this.server &&
+            !this.state.journal.destroyed() &&
+            this.state.runtimeEpoch === entry.epoch &&
+            this.state.journal.getWorkspace(entry.workspaceId) &&
+            this.entries.get(entry.cacheKey) === entry);
     }
     /** Called immediately before the native turn/start write, within this broker's process clock. */
     markCodexDispatched(cwd, sessionId, clientRequestId) {
-        const workspaceId = /^(basic|remix-)/.test(basename(cwd)) ? basename(dirname(cwd)) : basename(cwd);
+        const workspaceId = /^(basic|remix-)/.test(basename(cwd))
+            ? basename(dirname(cwd))
+            : basename(cwd);
         const route = this.routes.get(`${workspaceId}/${createHash('sha256').update(sessionId).digest('hex').slice(0, 32)}`);
         const entry = route?.selected?.entry;
-        if (!entry || !this.valid(entry) || entry.sessionId !== sessionId || entry.clientRequestId !== clientRequestId)
+        if (!entry ||
+            !this.valid(entry) ||
+            entry.sessionId !== sessionId ||
+            entry.clientRequestId !== clientRequestId)
             return;
         entry.codexDispatchedAt = new Date().toISOString();
         entry.codexDispatchMark = performance.now();
     }
-    async credential(entry, refresh = false) {
+    async credential(entry, refresh = false, signal) {
+        signal?.throwIfAborted();
         if (!this.valid(entry))
             throw new Error('model_access_denied');
-        if (!refresh && entry.credential && Date.parse(entry.credential.expiresAt) > Date.now() + 60_000)
+        if (!refresh &&
+            entry.credential &&
+            Date.parse(entry.credential.expiresAt) > Date.now() + 60_000)
             return entry.credential;
+        // A model request owns its renewal cancellation; it must not abort another caller's renewal.
+        if (signal)
+            return this.renew(entry, signal);
         let pending = this.pending.get(entry);
         if (!pending) {
             pending = this.renew(entry).finally(() => this.pending.delete(entry));
@@ -146,34 +228,76 @@ export class WorkspaceModelBroker {
         }
         return pending;
     }
-    async renew(entry) {
+    async renew(entry, signal) {
         const config = this.state.config;
         const endpoint = platformEndpoint(config.publicBaseUrl, PLATFORM_ROUTES.modelCredentials);
-        const response = await fetch(endpoint, {
-            method: 'POST', redirect: 'error', signal: AbortSignal.timeout(10_000),
-            headers: { authorization: `Bearer ${config.controlToken}`, 'content-type': 'application/json', 'playcraft-env': config.environment },
-            body: JSON.stringify({ runtimeId: config.runtimeId, generation: config.generation,
-                runtimeEpoch: entry.epoch, workspaceId: entry.workspaceId,
-                ...(entry.sessionId ? { sessionId: entry.sessionId } : {}),
-                ...(entry.clientRequestId ? { clientRequestId: entry.clientRequestId } : {}),
-                ...(entry.modelConfigId ? { modelConfigId: entry.modelConfigId } : {}),
-                ...(entry.reasoningEffort ? { reasoningEffort: entry.reasoningEffort } : {}) }),
-        });
-        if (!response.ok) {
-            const error = await response.json().catch(() => ({}));
-            throw new Error(['model_config_required', 'model_config_denied', 'reasoning_effort_denied', 'reasoning_effort_unsupported'].includes(error.reason ?? '') ? error.reason : 'model_credential_unavailable');
+        const timeout = new AbortController();
+        const timer = setTimeout(() => timeout.abort(new ModelBrokerRequestError(504, 'model_credential_timeout')), 10_000);
+        timer.unref();
+        try {
+            const response = await fetch(endpoint, {
+                method: 'POST',
+                redirect: 'error',
+                signal: signal ? AbortSignal.any([signal, timeout.signal]) : timeout.signal,
+                headers: {
+                    authorization: `Bearer ${config.controlToken}`,
+                    'content-type': 'application/json',
+                    'playcraft-env': config.environment,
+                    ...(this.capture.supported ? { 'x-playcraft-model-capture-protocol': '1' } : {}),
+                },
+                body: JSON.stringify({
+                    runtimeId: config.runtimeId,
+                    generation: config.generation,
+                    runtimeEpoch: entry.epoch,
+                    workspaceId: entry.workspaceId,
+                    ...(entry.sessionId ? { sessionId: entry.sessionId } : {}),
+                    ...(entry.clientRequestId ? { clientRequestId: entry.clientRequestId } : {}),
+                    ...(entry.modelConfigId ? { modelConfigId: entry.modelConfigId } : {}),
+                    ...(entry.reasoningEffort ? { reasoningEffort: entry.reasoningEffort } : {}),
+                }),
+            });
+            if (!response.ok) {
+                const error = (await response.json().catch(() => ({})));
+                signal?.throwIfAborted();
+                timeout.signal.throwIfAborted();
+                throw new ModelBrokerRequestError(response.status, [
+                    'model_config_required',
+                    'model_config_denied',
+                    'reasoning_effort_denied',
+                    'reasoning_effort_unsupported',
+                ].includes(error.reason ?? '')
+                    ? error.reason
+                    : 'model_credential_unavailable');
+            }
+            const parsed = credentialSchema.safeParse(await response.json());
+            if (!parsed.success)
+                throw new ModelBrokerRequestError(502, 'model_credential_invalid');
+            const credential = parsed.data;
+            const claims = credential.claims;
+            if (!this.valid(entry) ||
+                claims.workspaceId !== entry.workspaceId ||
+                claims.runtimeId !== config.runtimeId ||
+                claims.userId !== config.userId ||
+                claims.environment !== config.environment ||
+                claims.generation !== config.generation ||
+                claims.runtimeEpoch !== entry.epoch ||
+                (entry.modelConfigId !== undefined && credential.configId !== entry.modelConfigId) ||
+                Date.parse(credential.expiresAt) <= Date.now() + 1000)
+                throw new ModelBrokerRequestError(403, 'model_credential_scope_mismatch');
+            entry.modelConfigId = credential.configId;
+            entry.credential = credential;
+            return credential;
+        }
+        catch (error) {
+            signal?.throwIfAborted();
+            timeout.signal.throwIfAborted();
+            if (error instanceof ModelBrokerRequestError)
+                throw error;
+            throw new ModelBrokerRequestError(503, 'model_credential_unavailable', error);
+        }
+        finally {
+            clearTimeout(timer);
         }
-        const credential = credentialSchema.parse(await response.json());
-        const claims = credential.claims;
-        if (!this.valid(entry) || claims.workspaceId !== entry.workspaceId || claims.runtimeId !== config.runtimeId
-            || claims.userId !== config.userId || claims.environment !== config.environment
-            || claims.generation !== config.generation || claims.runtimeEpoch !== entry.epoch
-            || (entry.modelConfigId !== undefined && credential.configId !== entry.modelConfigId)
-            || Date.parse(credential.expiresAt) <= Date.now() + 1000)
-            throw new Error('model_credential_scope_mismatch');
-        entry.modelConfigId = credential.configId;
-        entry.credential = credential;
-        return credential;
     }
     async forward(request, response) {
         const receivedMark = performance.now();
@@ -182,7 +306,19 @@ export class WorkspaceModelBroker {
         const abort = new AbortController();
         this.requests.add(abort);
         let activeRoute;
-        const cancel = () => abort.abort();
+        let capturedRequest;
+        let capturedResponse;
+        let requestEnded = false;
+        let diagnostics;
+        let failure;
+        let deadlineTimer;
+        let credentialTimer;
+        const cancel = () => {
+            if (response.writableFinished)
+                return;
+            diagnostics?.abort('downstream_closed');
+            abort.abort();
+        };
         response.once('close', cancel);
         try {
             const match = /^\/([0-9a-f-]{36}(?:\/[0-9a-f]{32})?)\/(responses|generation-provenance|remix-references|execution-context|skill-discovery\/(?:match|selection|artifact|activate))$/.exec(request.url ?? '');
@@ -194,10 +330,21 @@ export class WorkspaceModelBroker {
             const contextRequest = match?.[2] === 'execution-context';
             const referencesRequest = match?.[2] === 'remix-references';
             const skillsRequest = match?.[2]?.startsWith('skill-discovery/');
-            const expected = Buffer.from(route ? `Bearer ${generationRequest || contextRequest ? route.generationCapability
-                : referencesRequest ? route.referencesCapability : skillsRequest ? route.skillsCapability : route.capability}` : '');
-            if (request.method !== 'POST' || !entry || !expected.length || presented.length !== expected.length
-                || !timingSafeEqual(presented, expected) || !this.valid(entry)) {
+            const expected = Buffer.from(route
+                ? `Bearer ${generationRequest || contextRequest
+                    ? route.generationCapability
+                    : referencesRequest
+                        ? route.referencesCapability
+                        : skillsRequest
+                            ? route.skillsCapability
+                            : route.capability}`
+                : '');
+            if (request.method !== 'POST' ||
+                !entry ||
+                !expected.length ||
+                presented.length !== expected.length ||
+                !timingSafeEqual(presented, expected) ||
+                !this.valid(entry)) {
                 response.writeHead(403);
                 response.end();
                 return;
@@ -239,13 +386,50 @@ export class WorkspaceModelBroker {
                 await this.skills.forward(request, response, selected.feedback, () => this.valid(entry) && route?.selected === selected, action);
                 return;
             }
+            const modelCallId = randomUUID();
+            diagnostics = new ModelBrokerDiagnostics({
+                environment: this.state.config.environment,
+                runtimeId: this.state.config.runtimeId,
+                generation: this.state.config.generation,
+                runtimeEpoch: entry.epoch,
+                workspaceId: entry.workspaceId,
+                sessionId: entry.sessionId,
+                clientRequestId: entry.clientRequestId,
+                configId: entry.modelConfigId,
+                modelCallId,
+                requestBudgetMs: this.requestBudgetMs,
+            });
+            const expire = () => {
+                if (abort.signal.aborted)
+                    return;
+                diagnostics?.abort('deadline_exceeded');
+                abort.abort(new ModelBrokerRequestError(504, 'model_request_deadline_exceeded'));
+            };
+            const checkDeadline = () => {
+                if (performance.now() - receivedMark >= this.requestBudgetMs)
+                    expire();
+                abort.signal.throwIfAborted();
+            };
+            deadlineTimer = setTimeout(expire, Math.max(0, this.requestBudgetMs - (performance.now() - receivedMark)));
+            deadlineTimer.unref();
             const rejectOversizedBody = () => {
-                response.writeHead(413, { 'content-type': 'application/json', 'cache-control': 'no-store' });
-                response.end(JSON.stringify({ error: { message: 'model_request_too_large',
-                        type: 'invalid_request_error', code: 'model_request_too_large', maxBytes: MAX_MODEL_REQUEST_BYTES } }));
+                response.writeHead(413, {
+                    'content-type': 'application/json',
+                    'cache-control': 'no-store',
+                });
+                response.end(JSON.stringify({
+                    error: {
+                        message: 'model_request_too_large',
+                        type: 'invalid_request_error',
+                        code: 'model_request_too_large',
+                        maxBytes: MAX_MODEL_REQUEST_BYTES,
+                    },
+                }));
             };
             const declaredLength = request.headers['content-length'];
-            if (declaredLength !== undefined && /^\d+$/.test(declaredLength) && Number(declaredLength) > MAX_MODEL_REQUEST_BYTES) {
+            if (declaredLength !== undefined &&
+                /^\d+$/.test(declaredLength) &&
+                Number(declaredLength) > MAX_MODEL_REQUEST_BYTES) {
                 rejectOversizedBody();
                 return;
             }
@@ -254,23 +438,52 @@ export class WorkspaceModelBroker {
             active.add(abort);
             this.sessionRequests.set(route, active);
             const credentialMark = performance.now();
-            const credential = await this.credential(entry);
+            const credential = await this.credential(entry, false, abort.signal);
+            checkDeadline();
             const credentialWaitMs = elapsed(credentialMark);
+            const candidateOrigin = credential.claims.traceOrigin;
+            const origin = candidateOrigin?.sessionId === entry.sessionId &&
+                candidateOrigin?.clientRequestId === entry.clientRequestId
+                ? candidateOrigin
+                : undefined;
+            diagnostics.bind({
+                configId: credential.configId,
+                runtimeOperationId: origin?.runtimeOperationId,
+            });
+            const captured = origin ? this.capture.begin(modelCallId, credential) : undefined;
+            capturedRequest = captured?.artifact('codex_request', request.headers['content-type'] ?? 'application/json');
+            capturedResponse = captured?.artifact('codex_response');
+            if (capturedResponse)
+                captureSubmittedResponse(response, capturedResponse);
+            diagnostics.enter('request_body');
             const requestReadMark = performance.now();
             const chunks = [];
             let size = 0;
-            for await (const chunk of request) {
+            await readModelRequestBody(request, abort.signal, (chunk) => {
+                diagnostics?.requestChunk(chunk.length);
+                capturedRequest?.append(chunk);
                 size += chunk.length;
                 if (size > MAX_MODEL_REQUEST_BYTES) {
-                    rejectOversizedBody();
-                    return;
+                    throw new ModelBrokerRequestError(413, 'model_request_too_large');
                 }
                 chunks.push(Buffer.from(chunk));
-            }
+            });
+            checkDeadline();
             const requestReadMs = elapsed(requestReadMark);
+            capturedRequest?.finish({ sourceEnd: 'eof', protocolTerminal: 'not_applicable' });
+            requestEnded = true;
+            diagnostics.enter('prepare');
             if (!this.valid(entry))
                 throw new Error('model_access_denied');
-            const body = JSON.parse(Buffer.concat(chunks).toString('utf8'));
+            let body;
+            try {
+                body = JSON.parse(Buffer.concat(chunks).toString('utf8'));
+            }
+            catch {
+                checkDeadline();
+                throw new ModelBrokerRequestError(400, 'model_request_invalid_json');
+            }
+            checkDeadline();
             if (!body || typeof body !== 'object' || Array.isArray(body)) {
                 response.writeHead(400);
                 response.end();
@@ -288,41 +501,82 @@ export class WorkspaceModelBroker {
                 const reasoning = body.reasoning;
                 const modelName = selected.model.split('/').pop() ?? selected.model;
                 const effort = entry.reasoningEffort === 'max' && /^o\d|gpt-[5-9]/i.test(modelName)
-                    ? 'xhigh' : entry.reasoningEffort;
+                    ? 'xhigh'
+                    : entry.reasoningEffort;
                 body.reasoning = {
-                    ...(reasoning && typeof reasoning === 'object' && !Array.isArray(reasoning) ? reasoning : {}),
+                    ...(reasoning && typeof reasoning === 'object' && !Array.isArray(reasoning)
+                        ? reasoning
+                        : {}),
                     effort,
                 };
             }
-            const expires = setTimeout(cancel, Math.max(0, Date.parse(credential.expiresAt) - Date.now()));
+            credentialTimer = setTimeout(() => {
+                if (abort.signal.aborted)
+                    return;
+                diagnostics?.abort('credential_expired');
+                abort.abort(new ModelBrokerRequestError(401, 'model_credential_expired'));
+            }, Math.max(0, Date.parse(credential.expiresAt) - Date.now()));
+            credentialTimer.unref();
             const dispatchedAt = new Date().toISOString();
             const dispatchedMark = performance.now();
             const beforeDispatchMs = elapsed(receivedMark);
-            const candidateOrigin = credential.claims.traceOrigin;
-            const origin = candidateOrigin?.sessionId === entry.sessionId && candidateOrigin?.clientRequestId === entry.clientRequestId
-                ? candidateOrigin : undefined;
-            const broker = { receivedAt, dispatchedAt,
-                credentialWaitMs, requestReadMs, beforeDispatchMs };
+            const broker = {
+                receivedAt,
+                dispatchedAt,
+                credentialWaitMs,
+                requestReadMs,
+                beforeDispatchMs,
+            };
             let attemptId;
             let outcome = 'error';
             try {
+                const requestBody = JSON.stringify(body);
+                checkDeadline();
+                diagnostics.enter('upstream_headers');
                 const upstream = await fetch(platformEndpoint(this.state.config.publicBaseUrl, modelResponsesPath(credential.configId)), {
-                    method: 'POST', redirect: 'error', signal: abort.signal,
-                    headers: { authorization: `Bearer ${credential.token}`, 'content-type': 'application/json',
-                        ...(origin ? { 'x-playcraft-model-journey': JSON.stringify({ version: 1,
-                                runtimeOperationId: origin.runtimeOperationId, sessionId: origin.sessionId, clientRequestId: origin.clientRequestId,
-                                ...(entry.codexDispatchedAt ? { codexDispatchedAt: entry.codexDispatchedAt } : {}),
-                                brokerReceivedAt: receivedAt, brokerDispatchedAt: dispatchedAt, credentialWaitMs, requestReadMs, beforeDispatchMs,
-                                ...(entry.codexDispatchMark === undefined ? {} : { codexDispatchToBrokerMs: Math.max(0, Math.round(receivedMark - entry.codexDispatchMark)) }),
-                            }) } : {}),
-                        ...(selected?.feedback ? {
-                            'x-playcraft-cloud-session-id': selected.feedback.sessionId,
-                            'x-playcraft-client-request-id': selected.feedback.clientRequestId,
-                        } : {}) },
-                    body: JSON.stringify(body),
+                    method: 'POST',
+                    redirect: 'error',
+                    signal: abort.signal,
+                    headers: {
+                        authorization: `Bearer ${credential.token}`,
+                        'content-type': 'application/json',
+                        'x-playcraft-model-call-id': modelCallId,
+                        'x-request-id': modelCallId,
+                        ...(origin
+                            ? {
+                                'x-playcraft-model-journey': JSON.stringify({
+                                    version: 1,
+                                    runtimeOperationId: origin.runtimeOperationId,
+                                    sessionId: origin.sessionId,
+                                    clientRequestId: origin.clientRequestId,
+                                    ...(entry.codexDispatchedAt
+                                        ? { codexDispatchedAt: entry.codexDispatchedAt }
+                                        : {}),
+                                    brokerReceivedAt: receivedAt,
+                                    brokerDispatchedAt: dispatchedAt,
+                                    credentialWaitMs,
+                                    requestReadMs,
+                                    beforeDispatchMs,
+                                    ...(entry.codexDispatchMark === undefined
+                                        ? {}
+                                        : {
+                                            codexDispatchToBrokerMs: Math.max(0, Math.round(receivedMark - entry.codexDispatchMark)),
+                                        }),
+                                }),
+                            }
+                            : {}),
+                        ...(selected?.feedback
+                            ? {
+                                'x-playcraft-cloud-session-id': selected.feedback.sessionId,
+                                'x-playcraft-client-request-id': selected.feedback.clientRequestId,
+                            }
+                            : {}),
+                    },
+                    body: requestBody,
                 });
                 broker.responseHeadersAt = new Date().toISOString();
                 broker.proxyResponseHeadersMs = elapsed(dispatchedMark);
+                diagnostics.headers(upstream.status, upstream.headers);
                 const reference = upstream.headers.get('x-model-attempt-id');
                 if (reference && z.string().uuid().safeParse(reference).success)
                     attemptId = reference;
@@ -334,24 +588,51 @@ export class WorkspaceModelBroker {
                     route.lastAdmissionReason = undefined;
                 else if (admissionReason)
                     route.lastAdmissionReason = admissionReason;
-                response.writeHead(upstream.status, {
-                    'content-type': upstream.headers.get('content-type') ?? 'application/json',
-                    'cache-control': 'no-store',
-                    ...(admissionReason ? { 'x-model-admission-reason': admissionReason } : {}),
-                    ...(retryAfter ? { 'retry-after': retryAfter } : {}),
-                });
-                if (upstream.body)
-                    await pipeline(Readable.fromWeb(upstream.body), new Transform({
+                const forwardHeaders = () => {
+                    if (response.headersSent)
+                        return;
+                    response.writeHead(upstream.status, {
+                        'content-type': upstream.headers.get('content-type') ?? 'application/json',
+                        'cache-control': 'no-store',
+                        ...(admissionReason ? { 'x-model-admission-reason': admissionReason } : {}),
+                        ...(retryAfter ? { 'retry-after': retryAfter } : {}),
+                    });
+                };
+                const requestDiagnostics = diagnostics;
+                if (upstream.body) {
+                    const observed = new Transform({
                         transform(chunk, _encoding, callback) {
+                            forwardHeaders();
+                            requestDiagnostics.responseChunk(chunk.length);
+                            capturedResponse?.upstreamObserved(chunk.length);
                             if (chunk.length && broker.firstBodyByteAt === undefined) {
                                 broker.firstBodyByteAt = new Date().toISOString();
                                 broker.proxyFirstBodyByteMs = elapsed(dispatchedMark);
                             }
                             callback(null, chunk);
                         },
-                    }), response);
-                else
+                    });
+                    // Keep an uncommitted HTTP response writable so a pre-body deadline can return 504.
+                    const drained = finished(response, { cleanup: true });
+                    const source = Readable.fromWeb(upstream.body);
+                    const transferred = pipeline(source, observed, {
+                        signal: abort.signal,
+                    });
+                    observed.once('end', forwardHeaders);
+                    observed.pipe(response);
+                    try {
+                        await Promise.all([transferred, drained]);
+                    }
+                    catch (error) {
+                        source.destroy();
+                        observed.destroy();
+                        throw error;
+                    }
+                }
+                else {
+                    forwardHeaders();
                     response.end();
+                }
                 outcome = 'completed';
             }
             catch (error) {
@@ -359,26 +640,63 @@ export class WorkspaceModelBroker {
                 throw error;
             }
             finally {
-                clearTimeout(expires);
+                clearTimeout(credentialTimer);
                 broker.completedAt = new Date().toISOString();
                 broker.proxyResponseDurationMs = elapsed(dispatchedMark);
                 if (origin && attemptId) {
                     // Telemetry is sent after transfer and cannot delay or change the Codex model response.
-                    void this.reportObservation(credential.token, { version: 1, attemptId,
-                        runtimeOperationId: origin.runtimeOperationId, sessionId: origin.sessionId,
-                        clientRequestId: origin.clientRequestId, broker, outcome });
+                    void this.reportObservation(credential.token, {
+                        version: 1,
+                        attemptId,
+                        runtimeOperationId: origin.runtimeOperationId,
+                        sessionId: origin.sessionId,
+                        clientRequestId: origin.clientRequestId,
+                        broker,
+                        outcome,
+                    });
                 }
             }
         }
-        catch {
-            if (!response.headersSent) {
-                response.writeHead(503, { 'content-type': 'application/json', 'cache-control': 'no-store' });
-                response.end(JSON.stringify({ error: { message: 'model_credential_unavailable', type: 'server_error' } }));
+        catch (error) {
+            if (deadlineTimer &&
+                !abort.signal.aborted &&
+                performance.now() - receivedMark >= this.requestBudgetMs) {
+                diagnostics?.abort('deadline_exceeded');
+                abort.abort(new ModelBrokerRequestError(504, 'model_request_deadline_exceeded'));
+            }
+            failure = error;
+            const rejected = modelBrokerFailure(error, abort.signal);
+            if (!response.headersSent && !response.destroyed) {
+                response.writeHead(rejected.status, {
+                    'content-type': 'application/json',
+                    'cache-control': 'no-store',
+                    ...(!request.complete ? { connection: 'close' } : {}),
+                });
+                response.end(JSON.stringify({
+                    error: {
+                        message: rejected.code,
+                        type: rejected.status === 499
+                            ? 'request_cancelled'
+                            : rejected.status < 500
+                                ? 'invalid_request_error'
+                                : 'server_error',
+                        code: rejected.code,
+                        ...(rejected.status === 413 ? { maxBytes: MAX_MODEL_REQUEST_BYTES } : {}),
+                    },
+                }));
             }
             else
                 response.destroy();
         }
         finally {
+            clearTimeout(deadlineTimer);
+            clearTimeout(credentialTimer);
+            diagnostics?.finish(response.statusCode, response.writableFinished, abort.signal.aborted, failure);
+            if (!requestEnded)
+                capturedRequest?.finish({
+                    sourceEnd: abort.signal.aborted ? 'aborted' : 'io_error',
+                    protocolTerminal: 'unknown',
+                });
             response.off('close', cancel);
             this.requests.delete(abort);
             if (activeRoute) {
@@ -392,12 +710,17 @@ export class WorkspaceModelBroker {
     async reportObservation(token, input) {
         try {
             const response = await fetch(platformEndpoint(this.state.config.publicBaseUrl, '/api/llm-proxy/runtime-observations'), {
-                method: 'POST', redirect: 'error', signal: AbortSignal.timeout(3_000),
-                headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' }, body: JSON.stringify(input),
+                method: 'POST',
+                redirect: 'error',
+                signal: AbortSignal.timeout(3_000),
+                headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' },
+                body: JSON.stringify(input),
             });
             await response.body?.cancel();
         }
-        catch { /* Missing observations remain unknown; never replay a model request for telemetry. */ }
+        catch {
+            /* Missing observations remain unknown; never replay a model request for telemetry. */
+        }
     }
     cancelSession(cwd, sessionId) {
         const routeKey = `${/^(basic|remix-)/.test(basename(cwd)) ? basename(dirname(cwd)) : basename(cwd)}/${createHash('sha256').update(sessionId).digest('hex').slice(0, 32)}`;
@@ -424,7 +747,8 @@ export class WorkspaceModelBroker {
             request.abort();
         if (server) {
             server.closeAllConnections();
-            await new Promise(resolve => server.close(() => resolve()));
+            await new Promise((resolve) => server.close(() => resolve()));
         }
+        await this.capture.close();
     }
 }