@playcraft/cli 0.1.85-beta → 0.1.87

package/dist/workspace-runtime/codex/model-broker.js

@@ -1,7 +1,7 @@
 import { createHash, randomBytes, randomUUID, timingSafeEqual } from 'node:crypto';
 import { createServer } from 'node:http';
 import { Readable, Transform } from 'node:stream';
-import { pipeline } from 'node:stream/promises';
+import { finished, pipeline } from 'node:stream/promises';
 import { basename, dirname } from 'node:path';
 import { z } from 'zod';
 import { workspaceContextFromRoot } from '../workspaces/registry.js';
@@ -10,6 +10,8 @@ import { WorkspaceGenerationProvenanceBroker } from './generation-provenance-bro
 import { WorkspaceRemixReferencesBroker } from './remix-references-broker.js';
 import { WorkspaceExecutionContextBroker } from './execution-context-broker.js';
 import { WorkspaceSkillDiscoveryBroker } from './skill-discovery-broker.js';
+import { ModelBrokerDiagnostics } from './model-broker-diagnostics.js';
+import { MODEL_REQUEST_BUDGET_MS, ModelBrokerRequestError, modelBrokerFailure, readModelRequestBody, } from './model-broker-request.js';
 import { WorkspaceModelCapture, captureSubmittedResponse, } from './model-capture.js';
 const credentialSchema = z.object({
     token: z.string().min(1),
@@ -42,6 +44,7 @@ const MAX_MODEL_REQUEST_BYTES = 128 * 1024 * 1024;
 /** Keeps renewable execution credentials in control memory, outside Codex and project files. */
 export class WorkspaceModelBroker {
     state;
+    requestBudgetMs;
     server;
     baseUrl = '';
     entries = new Map();
@@ -54,8 +57,9 @@ export class WorkspaceModelBroker {
     executionContext;
     skills;
     capture;
-    constructor(state) {
+    constructor(state, requestBudgetMs = MODEL_REQUEST_BUDGET_MS) {
         this.state = state;
+        this.requestBudgetMs = requestBudgetMs;
         this.generation = new WorkspaceGenerationProvenanceBroker(state);
         this.references = new WorkspaceRemixReferencesBroker(state);
         this.executionContext = new WorkspaceExecutionContextBroker(state);
@@ -174,8 +178,13 @@ export class WorkspaceModelBroker {
                     base_url: `${this.baseUrl}/${routeKey}`,
                     wire_api: 'responses',
                     requires_openai_auth: false,
+                    request_max_retries: 0,
+                    stream_max_retries: 0,
+                    stream_idle_timeout_ms: MODEL_REQUEST_BUDGET_MS,
+                    supports_websockets: false,
                     http_headers: { Authorization: `Bearer ${route.capability}` },
                 },
+                'features.unbounded_connection_retries': false,
             },
         };
     }
@@ -201,13 +210,17 @@ export class WorkspaceModelBroker {
         entry.codexDispatchedAt = new Date().toISOString();
         entry.codexDispatchMark = performance.now();
     }
-    async credential(entry, refresh = false) {
+    async credential(entry, refresh = false, signal) {
+        signal?.throwIfAborted();
         if (!this.valid(entry))
             throw new Error('model_access_denied');
         if (!refresh &&
             entry.credential &&
             Date.parse(entry.credential.expiresAt) > Date.now() + 60_000)
             return entry.credential;
+        // A model request owns its renewal cancellation; it must not abort another caller's renewal.
+        if (signal)
+            return this.renew(entry, signal);
         let pending = this.pending.get(entry);
         if (!pending) {
             pending = this.renew(entry).finally(() => this.pending.delete(entry));
@@ -215,56 +228,76 @@ export class WorkspaceModelBroker {
         }
         return pending;
     }
-    async renew(entry) {
+    async renew(entry, signal) {
         const config = this.state.config;
         const endpoint = platformEndpoint(config.publicBaseUrl, PLATFORM_ROUTES.modelCredentials);
-        const response = await fetch(endpoint, {
-            method: 'POST',
-            redirect: 'error',
-            signal: AbortSignal.timeout(10_000),
-            headers: {
-                authorization: `Bearer ${config.controlToken}`,
-                'content-type': 'application/json',
-                'playcraft-env': config.environment,
-                ...(this.capture.supported ? { 'x-playcraft-model-capture-protocol': '1' } : {}),
-            },
-            body: JSON.stringify({
-                runtimeId: config.runtimeId,
-                generation: config.generation,
-                runtimeEpoch: entry.epoch,
-                workspaceId: entry.workspaceId,
-                ...(entry.sessionId ? { sessionId: entry.sessionId } : {}),
-                ...(entry.clientRequestId ? { clientRequestId: entry.clientRequestId } : {}),
-                ...(entry.modelConfigId ? { modelConfigId: entry.modelConfigId } : {}),
-                ...(entry.reasoningEffort ? { reasoningEffort: entry.reasoningEffort } : {}),
-            }),
-        });
-        if (!response.ok) {
-            const error = (await response.json().catch(() => ({})));
-            throw new Error([
-                'model_config_required',
-                'model_config_denied',
-                'reasoning_effort_denied',
-                'reasoning_effort_unsupported',
-            ].includes(error.reason ?? '')
-                ? error.reason
-                : 'model_credential_unavailable');
+        const timeout = new AbortController();
+        const timer = setTimeout(() => timeout.abort(new ModelBrokerRequestError(504, 'model_credential_timeout')), 10_000);
+        timer.unref();
+        try {
+            const response = await fetch(endpoint, {
+                method: 'POST',
+                redirect: 'error',
+                signal: signal ? AbortSignal.any([signal, timeout.signal]) : timeout.signal,
+                headers: {
+                    authorization: `Bearer ${config.controlToken}`,
+                    'content-type': 'application/json',
+                    'playcraft-env': config.environment,
+                    ...(this.capture.supported ? { 'x-playcraft-model-capture-protocol': '1' } : {}),
+                },
+                body: JSON.stringify({
+                    runtimeId: config.runtimeId,
+                    generation: config.generation,
+                    runtimeEpoch: entry.epoch,
+                    workspaceId: entry.workspaceId,
+                    ...(entry.sessionId ? { sessionId: entry.sessionId } : {}),
+                    ...(entry.clientRequestId ? { clientRequestId: entry.clientRequestId } : {}),
+                    ...(entry.modelConfigId ? { modelConfigId: entry.modelConfigId } : {}),
+                    ...(entry.reasoningEffort ? { reasoningEffort: entry.reasoningEffort } : {}),
+                }),
+            });
+            if (!response.ok) {
+                const error = (await response.json().catch(() => ({})));
+                signal?.throwIfAborted();
+                timeout.signal.throwIfAborted();
+                throw new ModelBrokerRequestError(response.status, [
+                    'model_config_required',
+                    'model_config_denied',
+                    'reasoning_effort_denied',
+                    'reasoning_effort_unsupported',
+                ].includes(error.reason ?? '')
+                    ? error.reason
+                    : 'model_credential_unavailable');
+            }
+            const parsed = credentialSchema.safeParse(await response.json());
+            if (!parsed.success)
+                throw new ModelBrokerRequestError(502, 'model_credential_invalid');
+            const credential = parsed.data;
+            const claims = credential.claims;
+            if (!this.valid(entry) ||
+                claims.workspaceId !== entry.workspaceId ||
+                claims.runtimeId !== config.runtimeId ||
+                claims.userId !== config.userId ||
+                claims.environment !== config.environment ||
+                claims.generation !== config.generation ||
+                claims.runtimeEpoch !== entry.epoch ||
+                (entry.modelConfigId !== undefined && credential.configId !== entry.modelConfigId) ||
+                Date.parse(credential.expiresAt) <= Date.now() + 1000)
+                throw new ModelBrokerRequestError(403, 'model_credential_scope_mismatch');
+            entry.modelConfigId = credential.configId;
+            entry.credential = credential;
+            return credential;
+        }
+        catch (error) {
+            signal?.throwIfAborted();
+            timeout.signal.throwIfAborted();
+            if (error instanceof ModelBrokerRequestError)
+                throw error;
+            throw new ModelBrokerRequestError(503, 'model_credential_unavailable', error);
+        }
+        finally {
+            clearTimeout(timer);
         }
-        const credential = credentialSchema.parse(await response.json());
-        const claims = credential.claims;
-        if (!this.valid(entry) ||
-            claims.workspaceId !== entry.workspaceId ||
-            claims.runtimeId !== config.runtimeId ||
-            claims.userId !== config.userId ||
-            claims.environment !== config.environment ||
-            claims.generation !== config.generation ||
-            claims.runtimeEpoch !== entry.epoch ||
-            (entry.modelConfigId !== undefined && credential.configId !== entry.modelConfigId) ||
-            Date.parse(credential.expiresAt) <= Date.now() + 1000)
-            throw new Error('model_credential_scope_mismatch');
-        entry.modelConfigId = credential.configId;
-        entry.credential = credential;
-        return credential;
     }
     async forward(request, response) {
         const receivedMark = performance.now();
@@ -276,7 +309,16 @@ export class WorkspaceModelBroker {
         let capturedRequest;
         let capturedResponse;
         let requestEnded = false;
-        const cancel = () => abort.abort();
+        let diagnostics;
+        let failure;
+        let deadlineTimer;
+        let credentialTimer;
+        const cancel = () => {
+            if (response.writableFinished)
+                return;
+            diagnostics?.abort('downstream_closed');
+            abort.abort();
+        };
         response.once('close', cancel);
         try {
             const match = /^\/([0-9a-f-]{36}(?:\/[0-9a-f]{32})?)\/(responses|generation-provenance|remix-references|execution-context|skill-discovery\/(?:match|selection|artifact|activate))$/.exec(request.url ?? '');
@@ -344,6 +386,32 @@ export class WorkspaceModelBroker {
                 await this.skills.forward(request, response, selected.feedback, () => this.valid(entry) && route?.selected === selected, action);
                 return;
             }
+            const modelCallId = randomUUID();
+            diagnostics = new ModelBrokerDiagnostics({
+                environment: this.state.config.environment,
+                runtimeId: this.state.config.runtimeId,
+                generation: this.state.config.generation,
+                runtimeEpoch: entry.epoch,
+                workspaceId: entry.workspaceId,
+                sessionId: entry.sessionId,
+                clientRequestId: entry.clientRequestId,
+                configId: entry.modelConfigId,
+                modelCallId,
+                requestBudgetMs: this.requestBudgetMs,
+            });
+            const expire = () => {
+                if (abort.signal.aborted)
+                    return;
+                diagnostics?.abort('deadline_exceeded');
+                abort.abort(new ModelBrokerRequestError(504, 'model_request_deadline_exceeded'));
+            };
+            const checkDeadline = () => {
+                if (performance.now() - receivedMark >= this.requestBudgetMs)
+                    expire();
+                abort.signal.throwIfAborted();
+            };
+            deadlineTimer = setTimeout(expire, Math.max(0, this.requestBudgetMs - (performance.now() - receivedMark)));
+            deadlineTimer.unref();
             const rejectOversizedBody = () => {
                 response.writeHead(413, {
                     'content-type': 'application/json',
@@ -370,36 +438,52 @@ export class WorkspaceModelBroker {
             active.add(abort);
             this.sessionRequests.set(route, active);
             const credentialMark = performance.now();
-            const credential = await this.credential(entry);
+            const credential = await this.credential(entry, false, abort.signal);
+            checkDeadline();
             const credentialWaitMs = elapsed(credentialMark);
-            const modelCallId = randomUUID();
-            const candidateCaptureOrigin = credential.claims.traceOrigin;
-            const captured = candidateCaptureOrigin?.sessionId === entry.sessionId &&
-                candidateCaptureOrigin?.clientRequestId === entry.clientRequestId
-                ? this.capture.begin(modelCallId, credential)
+            const candidateOrigin = credential.claims.traceOrigin;
+            const origin = candidateOrigin?.sessionId === entry.sessionId &&
+                candidateOrigin?.clientRequestId === entry.clientRequestId
+                ? candidateOrigin
                 : undefined;
+            diagnostics.bind({
+                configId: credential.configId,
+                runtimeOperationId: origin?.runtimeOperationId,
+            });
+            const captured = origin ? this.capture.begin(modelCallId, credential) : undefined;
             capturedRequest = captured?.artifact('codex_request', request.headers['content-type'] ?? 'application/json');
             capturedResponse = captured?.artifact('codex_response');
             if (capturedResponse)
                 captureSubmittedResponse(response, capturedResponse);
+            diagnostics.enter('request_body');
             const requestReadMark = performance.now();
             const chunks = [];
             let size = 0;
-            for await (const chunk of request) {
+            await readModelRequestBody(request, abort.signal, (chunk) => {
+                diagnostics?.requestChunk(chunk.length);
                 capturedRequest?.append(chunk);
                 size += chunk.length;
                 if (size > MAX_MODEL_REQUEST_BYTES) {
-                    rejectOversizedBody();
-                    return;
+                    throw new ModelBrokerRequestError(413, 'model_request_too_large');
                 }
                 chunks.push(Buffer.from(chunk));
-            }
+            });
+            checkDeadline();
             const requestReadMs = elapsed(requestReadMark);
             capturedRequest?.finish({ sourceEnd: 'eof', protocolTerminal: 'not_applicable' });
             requestEnded = true;
+            diagnostics.enter('prepare');
             if (!this.valid(entry))
                 throw new Error('model_access_denied');
-            const body = JSON.parse(Buffer.concat(chunks).toString('utf8'));
+            let body;
+            try {
+                body = JSON.parse(Buffer.concat(chunks).toString('utf8'));
+            }
+            catch {
+                checkDeadline();
+                throw new ModelBrokerRequestError(400, 'model_request_invalid_json');
+            }
+            checkDeadline();
             if (!body || typeof body !== 'object' || Array.isArray(body)) {
                 response.writeHead(400);
                 response.end();
@@ -426,15 +510,16 @@ export class WorkspaceModelBroker {
                     effort,
                 };
             }
-            const expires = setTimeout(cancel, Math.max(0, Date.parse(credential.expiresAt) - Date.now()));
+            credentialTimer = setTimeout(() => {
+                if (abort.signal.aborted)
+                    return;
+                diagnostics?.abort('credential_expired');
+                abort.abort(new ModelBrokerRequestError(401, 'model_credential_expired'));
+            }, Math.max(0, Date.parse(credential.expiresAt) - Date.now()));
+            credentialTimer.unref();
             const dispatchedAt = new Date().toISOString();
             const dispatchedMark = performance.now();
             const beforeDispatchMs = elapsed(receivedMark);
-            const candidateOrigin = credential.claims.traceOrigin;
-            const origin = candidateOrigin?.sessionId === entry.sessionId &&
-                candidateOrigin?.clientRequestId === entry.clientRequestId
-                ? candidateOrigin
-                : undefined;
             const broker = {
                 receivedAt,
                 dispatchedAt,
@@ -445,6 +530,9 @@ export class WorkspaceModelBroker {
             let attemptId;
             let outcome = 'error';
             try {
+                const requestBody = JSON.stringify(body);
+                checkDeadline();
+                diagnostics.enter('upstream_headers');
                 const upstream = await fetch(platformEndpoint(this.state.config.publicBaseUrl, modelResponsesPath(credential.configId)), {
                     method: 'POST',
                     redirect: 'error',
@@ -453,6 +541,7 @@ export class WorkspaceModelBroker {
                         authorization: `Bearer ${credential.token}`,
                         'content-type': 'application/json',
                         'x-playcraft-model-call-id': modelCallId,
+                        'x-request-id': modelCallId,
                         ...(origin
                             ? {
                                 'x-playcraft-model-journey': JSON.stringify({
@@ -483,10 +572,11 @@ export class WorkspaceModelBroker {
                             }
                             : {}),
                     },
-                    body: JSON.stringify(body),
+                    body: requestBody,
                 });
                 broker.responseHeadersAt = new Date().toISOString();
                 broker.proxyResponseHeadersMs = elapsed(dispatchedMark);
+                diagnostics.headers(upstream.status, upstream.headers);
                 const reference = upstream.headers.get('x-model-attempt-id');
                 if (reference && z.string().uuid().safeParse(reference).success)
                     attemptId = reference;
@@ -498,15 +588,22 @@ export class WorkspaceModelBroker {
                     route.lastAdmissionReason = undefined;
                 else if (admissionReason)
                     route.lastAdmissionReason = admissionReason;
-                response.writeHead(upstream.status, {
-                    'content-type': upstream.headers.get('content-type') ?? 'application/json',
-                    'cache-control': 'no-store',
-                    ...(admissionReason ? { 'x-model-admission-reason': admissionReason } : {}),
-                    ...(retryAfter ? { 'retry-after': retryAfter } : {}),
-                });
-                if (upstream.body)
-                    await pipeline(Readable.fromWeb(upstream.body), new Transform({
+                const forwardHeaders = () => {
+                    if (response.headersSent)
+                        return;
+                    response.writeHead(upstream.status, {
+                        'content-type': upstream.headers.get('content-type') ?? 'application/json',
+                        'cache-control': 'no-store',
+                        ...(admissionReason ? { 'x-model-admission-reason': admissionReason } : {}),
+                        ...(retryAfter ? { 'retry-after': retryAfter } : {}),
+                    });
+                };
+                const requestDiagnostics = diagnostics;
+                if (upstream.body) {
+                    const observed = new Transform({
                         transform(chunk, _encoding, callback) {
+                            forwardHeaders();
+                            requestDiagnostics.responseChunk(chunk.length);
                             capturedResponse?.upstreamObserved(chunk.length);
                             if (chunk.length && broker.firstBodyByteAt === undefined) {
                                 broker.firstBodyByteAt = new Date().toISOString();
@@ -514,9 +611,28 @@ export class WorkspaceModelBroker {
                             }
                             callback(null, chunk);
                         },
-                    }), response);
-                else
+                    });
+                    // Keep an uncommitted HTTP response writable so a pre-body deadline can return 504.
+                    const drained = finished(response, { cleanup: true });
+                    const source = Readable.fromWeb(upstream.body);
+                    const transferred = pipeline(source, observed, {
+                        signal: abort.signal,
+                    });
+                    observed.once('end', forwardHeaders);
+                    observed.pipe(response);
+                    try {
+                        await Promise.all([transferred, drained]);
+                    }
+                    catch (error) {
+                        source.destroy();
+                        observed.destroy();
+                        throw error;
+                    }
+                }
+                else {
+                    forwardHeaders();
                     response.end();
+                }
                 outcome = 'completed';
             }
             catch (error) {
@@ -524,7 +640,7 @@ export class WorkspaceModelBroker {
                 throw error;
             }
             finally {
-                clearTimeout(expires);
+                clearTimeout(credentialTimer);
                 broker.completedAt = new Date().toISOString();
                 broker.proxyResponseDurationMs = elapsed(dispatchedMark);
                 if (origin && attemptId) {
@@ -541,20 +657,41 @@ export class WorkspaceModelBroker {
                 }
             }
         }
-        catch {
-            if (!response.headersSent) {
-                response.writeHead(503, {
+        catch (error) {
+            if (deadlineTimer &&
+                !abort.signal.aborted &&
+                performance.now() - receivedMark >= this.requestBudgetMs) {
+                diagnostics?.abort('deadline_exceeded');
+                abort.abort(new ModelBrokerRequestError(504, 'model_request_deadline_exceeded'));
+            }
+            failure = error;
+            const rejected = modelBrokerFailure(error, abort.signal);
+            if (!response.headersSent && !response.destroyed) {
+                response.writeHead(rejected.status, {
                     'content-type': 'application/json',
                     'cache-control': 'no-store',
+                    ...(!request.complete ? { connection: 'close' } : {}),
                 });
                 response.end(JSON.stringify({
-                    error: { message: 'model_credential_unavailable', type: 'server_error' },
+                    error: {
+                        message: rejected.code,
+                        type: rejected.status === 499
+                            ? 'request_cancelled'
+                            : rejected.status < 500
+                                ? 'invalid_request_error'
+                                : 'server_error',
+                        code: rejected.code,
+                        ...(rejected.status === 413 ? { maxBytes: MAX_MODEL_REQUEST_BYTES } : {}),
+                    },
                 }));
             }
             else
                 response.destroy();
         }
         finally {
+            clearTimeout(deadlineTimer);
+            clearTimeout(credentialTimer);
+            diagnostics?.finish(response.statusCode, response.writableFinished, abort.signal.aborted, failure);
             if (!requestEnded)
                 capturedRequest?.finish({
                     sourceEnd: abort.signal.aborted ? 'aborted' : 'io_error',

package/dist/workspace-runtime/codex/model-capture.js

@@ -237,6 +237,7 @@ export class CaptureCall {
     token;
     tokenExpires = 0;
     uploading = false;
+    pendingUpload = false;
     stopped = false;
     reservedBytes = 0;
     removing;
@@ -297,7 +298,7 @@ export class CaptureCall {
                 record.parts = partNo + 1;
                 await this.persist();
             }
-            catch (error) {
+            catch {
                 if (!written)
                     this.release(bytes.length);
                 this.failedStages.add(record.stage);
@@ -400,7 +401,11 @@ export class CaptureCall {
         });
         if (!response.ok) {
             await response.body?.cancel();
-            if ([401, 403, 404, 410].includes(response.status))
+            if ([401, 403, 404, 410].includes(response.status) ||
+                (response.status >= 400 &&
+                    response.status < 500 &&
+                    response.status !== 408 &&
+                    response.status !== 429))
                 await this.remove();
             throw new Error('capture_authority_unavailable');
         }
@@ -422,6 +427,12 @@ export class CaptureCall {
         });
         if (!response.ok) {
             const terminal = response.status === 410 ||
+                (response.status >= 400 &&
+                    response.status < 500 &&
+                    response.status !== 401 &&
+                    response.status !== 403 &&
+                    response.status !== 408 &&
+                    response.status !== 429) ||
                 ((response.status === 401 || response.status === 403) && this.tokenExpires > 0);
             if (response.status === 401 || response.status === 403) {
                 this.token = undefined;
@@ -445,9 +456,15 @@ export class CaptureCall {
         }
     }
     async upload() {
-        if (this.uploading || this.stopped)
+        if (this.stopped)
+            return;
+        if (this.uploading) {
+            this.pendingUpload = true;
             return;
+        }
         this.uploading = true;
+        this.pendingUpload = false;
+        let shouldRetry = false;
         try {
             await this.writes;
             if (this.stopped)
@@ -456,6 +473,7 @@ export class CaptureCall {
                 await this.remove();
                 return;
             }
+            let completedStages = 0;
             await uploadSlot(async () => {
                 if (!this.manifest.identity.captureCallRowId) {
                     const response = await this.send('/register', {
@@ -476,12 +494,18 @@ export class CaptureCall {
                 if (status.call?.callRowId !== this.manifest.identity.captureCallRowId ||
                     status.call.modelCallId !== this.manifest.identity.modelCallId ||
                     !Array.isArray(status.artifacts)) {
+                    await this.remove();
                     throw new Error('capture_status_identity_mismatch');
                 }
                 for (const record of this.manifest.artifacts) {
                     if (!record.end)
                         continue;
                     const remote = status.artifacts.find((item) => item.stage === record.stage);
+                    if (remote?.archiveState &&
+                        ['failed', 'expired', 'deleted'].includes(remote.archiveState)) {
+                        completedStages++;
+                        continue;
+                    }
                     const confirmed = new Map((remote?.confirmedParts ?? []).map((part) => [part.partNo, part]));
                     const hash = createHash('sha256');
                     for (let partNo = 0; partNo < record.parts; partNo++) {
@@ -490,8 +514,10 @@ export class CaptureCall {
                         const sha256 = createHash('sha256').update(bytes).digest('hex');
                         const part = confirmed.get(partNo);
                         if (part) {
-                            if (part.sha256 !== sha256 || part.rawBytes !== bytes.length)
+                            if (part.sha256 !== sha256 || part.rawBytes !== bytes.length) {
+                                await this.remove();
                                 throw new Error('capture_part_confirmation_mismatch');
+                            }
                             continue;
                         }
                         const response = await this.send(`/artifacts/${record.stage}/chunks/${partNo}`, {
@@ -508,7 +534,14 @@ export class CaptureCall {
                     }
                     const sha256 = hash.digest('hex');
                     if (remote?.archiveState === 'archived') {
-                        this.verifyArchive(record, sha256, remote);
+                        try {
+                            this.verifyArchive(record, sha256, remote);
+                        }
+                        catch (error) {
+                            await this.remove();
+                            throw error;
+                        }
+                        completedStages++;
                         continue;
                     }
                     const response = await this.send(`/artifacts/${record.stage}/finalize`, {
@@ -521,10 +554,20 @@ export class CaptureCall {
                             sha256,
                         }),
                     });
-                    this.verifyArchive(record, sha256, (await response.json()));
+                    const finalized = (await response.json());
+                    try {
+                        this.verifyArchive(record, sha256, finalized);
+                    }
+                    catch (error) {
+                        await this.remove();
+                        throw error;
+                    }
+                    completedStages++;
                 }
             });
-            if ((this.manifest.artifacts.length === 2 || this.artifacts.size === 0) &&
+            if (!this.pendingUpload &&
+                (this.manifest.artifacts.length === 2 || this.artifacts.size === 0) &&
+                completedStages === this.manifest.artifacts.length &&
                 this.manifest.artifacts.every((item) => item.end)) {
                 await this.remove();
                 return;
@@ -532,14 +575,20 @@ export class CaptureCall {
         }
         catch {
             /* Retry only archival transport, never model execution. */
+            shouldRetry = !this.stopped;
         }
         finally {
             this.uploading = false;
             if (!this.stopped) {
-                this.retry = setTimeout(() => {
+                if (this.pendingUpload && !shouldRetry) {
                     void this.upload();
-                }, 5_000);
-                this.retry.unref();
+                }
+                else if (shouldRetry) {
+                    this.retry = setTimeout(() => {
+                        void this.upload();
+                    }, 5_000);
+                    this.retry.unref();
+                }
             }
         }
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@playcraft/cli",
-  "version": "0.1.85-beta",
+  "version": "0.1.87",
   "private": false,
   "type": "module",
   "engines": {
@@ -58,8 +58,8 @@
     "@gltf-transform/core": "^4.3.0",
     "@gltf-transform/extensions": "^4.3.0",
     "@gltf-transform/functions": "^4.3.0",
-    "@playcraft/build": "0.1.85-beta",
-    "@playcraft/common": "0.1.85-beta",
+    "@playcraft/build": "0.1.87",
+    "@playcraft/common": "0.1.87",
     "better-sqlite3-pinned": "npm:better-sqlite3@13.0.3",
     "chokidar": "^4.0.3",
     "commander": "^13.1.0",