@nodetool-ai/runtime 0.8.1 → 0.8.3

package/dist/context.js

@@ -1,22 +1,14 @@
-/**
- * ProcessingContext – runtime context for node execution.
- *
- * Port of src/nodetool/workflows/processing_context.py.
- *
- * Provides:
- *   - Message queue for emitting ProcessingMessages.
- *   - Cache get/set interface.
- *   - Output normalization (sanitize memory URIs, etc.).
- *   - Asset handling with pluggable storage adapters.
- */
-import { buildAssetGenerationMetadata, isShortResourceId, packageAssetHttpPath, parsePackageAssetUri } from "@nodetool-ai/protocol";
+import { getRunTraceScope } from "./run-trace-context.js";
+import { withSpan } from "./tracing-helpers.js";
+import { traceStorageAdapter } from "./traced-storage.js";
+import { assetFileNameCandidates, buildAssetGenerationMetadata, isShortResourceId, packageAssetHttpPath, parsePackageAssetUri } from "@nodetool-ai/protocol";
 import { AgentMemory } from "./agent-memory.js";
 import { recordInvocationAsset, recordInvocationCost } from "./invocation-account.js";
 import { VariableChannel } from "./variable-channel.js";
 import { loadMediaRefBytes } from "./media-ref-bytes.js";
 import { objectTrackingRequestSchema, parseObjectTrackingResult } from "./providers/object-tracking.js";
 import { requestVideoToAudio, VIDEO_TO_AUDIO_TASK, VideoToAudioRequest } from "./providers/video-to-audio.js";
-import { encodeRawImageRef } from "./image-codec.js";
+import { encodeRawImageRef, validateImageEvidence } from "./image-codec.js";
 import { extForImageMime, sniffImageMime } from "./providers/image-mime.js";
 import { GenerationScopeError, isAuthoritativeProviderTerminalError, runWithGenerationReceipt } from "./generation-receipt.js";
 import { generationRegistry } from "./generation-registry.js";
@@ -28,11 +20,6 @@ import { getNodeBuiltinSync } from "@nodetool-ai/config";
 // lazily so this module loads in browser / Edge runtimes. `resolveWorkspacePath`
 // and the `randomUUID` fallback both degrade gracefully when these are
 // unavailable.
-/**
- * Secret values shorter than this are not recorded for masking: redacting a
- * two-character value would blank unrelated text wherever it happens to occur.
- */
-const MIN_MASKABLE_SECRET_LENGTH = 8;
 /**
  * ProcessingContext variable key carrying the provider/model the currently
  * running agent loop is itself talking to. Every loop that executes tools
@@ -481,6 +468,11 @@ export class ProcessingContext {
     _secretResolver = null;
     /** Every secret value the resolver handed out this run (see {@link ProcessingContext.getResolvedSecretValues}). */
     _resolvedSecrets = new Set();
+    /** Identity shared by all work beneath an app operation. */
+    appRunContext;
+    runTraceContext;
+    appRunCostAccount;
+    _appRunDocuments = new Map();
     /** Fetch function used by HTTP helpers. */
     _fetch;
     /** Optional temporary URL resolver for stored assets. */
@@ -526,9 +518,15 @@ export class ProcessingContext {
         this.assetOutputMode = opts.assetOutputMode ?? "native";
         this.persistOutputAssets = opts.persistOutputAssets ?? true;
         this._generationLifecycle = opts.generationLifecycle ?? null;
+        this.appRunContext = opts.appRunContext ? Object.freeze({ ...opts.appRunContext }) : null;
+        this.runTraceContext = opts.runTraceContext ?? getRunTraceScope() ?? null;
+        if (this.runTraceContext?.secretValues instanceof Set) {
+            this._resolvedSecrets = this.runTraceContext.secretValues;
+        }
+        this.appRunCostAccount = opts.appRunCostAccount ?? (this.appRunContext ? { llmCostUsd: 0, unpriced: false } : null);
         this.cache = opts.cache ?? new MemoryCache();
-        this.storage = opts.storage ?? null;
-        this.assetStorage = opts.assetStorage ?? null;
+        this.storage = traceStorageAdapter(opts.storage ?? null);
+        this.assetStorage = traceStorageAdapter(opts.assetStorage ?? null);
         this.workspaceStorage = opts.workspaceStorage ?? null;
         if (opts.onMessage) {
             this._messageListeners.add(opts.onMessage);
@@ -575,12 +573,12 @@ export class ProcessingContext {
      */
     copy(opts) {
         const next = new ProcessingContext({
-            jobId: this.jobId,
+            jobId: opts?.jobId ?? this.jobId,
             projectId: this.projectId,
-            workflowId: this.workflowId,
+            workflowId: opts?.workflowId !== undefined ? opts.workflowId : this.workflowId,
             threadId: this.threadId,
             userId: this.userId,
-            workspaceDir: this.workspaceDir,
+            workspace: opts?.workspace !== undefined ? opts.workspace : this.workspace,
             assetOutputMode: this.assetOutputMode,
             persistOutputAssets: this.persistOutputAssets,
             cache: this.cache,
@@ -596,7 +594,11 @@ export class ProcessingContext {
             tempUrlResolver: this._tempUrlResolver ?? undefined,
             modelInterfaces: this._modelInterfaces ?? undefined,
             retainMessageQueue: this._retainMessageQueue,
-            triggerEvent: this.triggerEvent
+            triggerEvent: this.triggerEvent,
+            generationLifecycle: opts?.generationLifecycle ?? this._generationLifecycle ?? undefined,
+            appRunContext: opts?.appRunContext ?? this.appRunContext ?? undefined,
+            runTraceContext: opts?.runTraceContext ?? this.runTraceContext ?? undefined,
+            appRunCostAccount: !opts?.appRunContext || opts.appRunContext.appRunId === this.appRunContext?.appRunId ? this.appRunCostAccount ?? undefined : undefined
         });
         if (opts?.inheritMessageListeners !== false) {
             for (const listener of this._messageListeners) {
@@ -606,6 +608,12 @@ export class ProcessingContext {
         if (opts?.shareMemory) {
             next.memory = this.memory;
         }
+        if (!opts?.runTraceContext) {
+            next._resolvedSecrets = this._resolvedSecrets;
+        }
+        if (next.appRunContext?.appRunId === this.appRunContext?.appRunId) {
+            next._appRunDocuments = this._appRunDocuments;
+        }
         next.signal = this.signal;
         next._providerResolver = this._providerResolver;
         next._modelInterfaces = this._modelInterfaces;
@@ -813,7 +821,7 @@ export class ProcessingContext {
         if (!this._secretResolver)
             return null;
         const value = await this._secretResolver(key, this.userId);
-        if (value != null && value.length >= MIN_MASKABLE_SECRET_LENGTH) {
+        if (value != null && value.length > 0) {
             this._resolvedSecrets.add(value);
         }
         return value ?? null;
@@ -1041,6 +1049,26 @@ export class ProcessingContext {
     getTotalCost() {
         return this._totalCost;
     }
+    /** Null means at least one LLM call could not be priced. */
+    getAppRunLlmCost() {
+        return this.appRunCostAccount && !this.appRunCostAccount.unpriced
+            ? this.appRunCostAccount.llmCostUsd
+            : null;
+    }
+    getAppRunDocuments() {
+        return [...this._appRunDocuments.values()].map((reference) => ({
+            ...reference
+        }));
+    }
+    recordAppRunDocument(kind, document) {
+        if (this.appRunContext && document) {
+            this._appRunDocuments.set(`${kind}:${document.id}`, {
+                kind,
+                id: document.id
+            });
+        }
+        return document;
+    }
     getOperationCosts() {
         return this._operationCosts;
     }
@@ -1199,7 +1227,8 @@ export class ProcessingContext {
     /** Create a persisted sketch (image document) owned by the current user. */
     async createImageDocument(args) {
         const fn = this.requireModelInterface("createImageDocument");
-        return fn({ userId: this.userId, ...args });
+        const document = await fn({ userId: this.userId, ...args });
+        return this.recordAppRunDocument("sketch", document);
     }
     /** Load a persisted timeline sequence owned by the current user. */
     async getTimelineSequence(id) {
@@ -1209,12 +1238,14 @@ export class ProcessingContext {
     /** Create a persisted timeline sequence owned by the current user. */
     async createTimelineSequence(sequence) {
         const fn = this.requireModelInterface("createTimelineSequence");
-        return fn({ userId: this.userId, sequence });
+        const document = await fn({ userId: this.userId, sequence });
+        return this.recordAppRunDocument("timeline", document);
     }
     /** Replace a persisted timeline sequence's document. */
     async updateTimelineSequence(id, sequence) {
         const fn = this.requireModelInterface("updateTimelineSequence");
-        return fn({ userId: this.userId, id, sequence });
+        const document = await fn({ userId: this.userId, id, sequence });
+        return this.recordAppRunDocument("timeline", document);
     }
     /** Load a persisted script owned by the current user. */
     async getScript(id) {
@@ -1224,12 +1255,14 @@ export class ProcessingContext {
     /** Create a persisted script owned by the current user. */
     async createScript(args) {
         const fn = this.requireModelInterface("createScript");
-        return fn({ userId: this.userId, ...args });
+        const document = await fn({ userId: this.userId, ...args });
+        return this.recordAppRunDocument("script", document);
     }
     /** Replace a persisted script's document (and optional timeline link). */
     async updateScript(id, args) {
         const fn = this.requireModelInterface("updateScript");
-        return fn({ userId: this.userId, id, ...args });
+        const document = await fn({ userId: this.userId, id, ...args });
+        return this.recordAppRunDocument("script", document);
     }
     /** Load a persisted storyboard owned by the current user. */
     async getStoryboard(id) {
@@ -1249,12 +1282,14 @@ export class ProcessingContext {
     /** Create a persisted storyboard owned by the current user. */
     async createStoryboard(args) {
         const fn = this.requireModelInterface("createStoryboard");
-        return fn({ userId: this.userId, ...args });
+        const document = await fn({ userId: this.userId, ...args });
+        return this.recordAppRunDocument("storyboard", document);
     }
     /** Replace a persisted storyboard's document (and optional timeline link). */
     async updateStoryboard(id, args) {
         const fn = this.requireModelInterface("updateStoryboard");
-        return fn({ userId: this.userId, id, ...args });
+        const document = await fn({ userId: this.userId, id, ...args });
+        return this.recordAppRunDocument("storyboard", document);
     }
     /** The current user's entity library, narrowed by the given filters. */
     async listEntities(args = {}) {
@@ -1289,7 +1324,7 @@ export class ProcessingContext {
      * `null` when that interface is not configured.
      */
     async getAssetInfo(assetId) {
-        const fn = this._modelInterfaces?.getAssetInfo;
+        const fn = this.modelInterfaces()?.getAssetInfo;
         if (!fn) {
             return null;
         }
@@ -1507,8 +1542,8 @@ export class ProcessingContext {
      * from the server's `/api/storage` route. Returns the bytes plus an `attempts`
      * trail for callers that build detailed errors; `null` bytes means unresolved.
      */
-    async resolveAssetBytes(assetId) {
-        const idCandidates = this.parseAssetIdCandidates(assetId);
+    async resolveAssetBytes(assetId, options) {
+        let idCandidates = this.parseAssetIdCandidates(assetId);
         const trimmed = assetId.trim();
         const attempts = [];
         const adapters = this.assetAdapters();
@@ -1595,6 +1630,33 @@ export class ProcessingContext {
                 }
             }
         }
+        // An asset row names the stored file exactly (`<owner>/<id>.<ext>`), so a
+        // reference that resolves to one is read by key and never by listing.
+        let assetRowFound = false;
+        if (this.hasModelInterface("getAssetInfo")) {
+            let asset = null;
+            const ownerPrefix = `${this.userId}/`;
+            const rowIdCandidates = new Set(idCandidates);
+            for (const candidate of idCandidates) {
+                if (candidate.startsWith(ownerPrefix)) {
+                    rowIdCandidates.add(candidate.slice(ownerPrefix.length));
+                }
+            }
+            for (const candidate of rowIdCandidates) {
+                asset = await this.getAssetInfo(candidate);
+                if (asset) {
+                    break;
+                }
+            }
+            if (asset) {
+                assetRowFound = true;
+                idCandidates = assetFileNameCandidates(asset.id, asset.content_type);
+            }
+            else if (options?.requireOwnedAsset) {
+                attempts.push(`asset unavailable or not owned: ${trimmed}`);
+                return { bytes: null, attempts };
+            }
+        }
         for (const adapter of adapters) {
             for (const candidate of idCandidates) {
                 for (const key of this.assetKeysFor(candidate)) {
@@ -1606,9 +1668,10 @@ export class ProcessingContext {
             }
             // Extension-tolerant fallback: locate the stored file whose name starts
             // with the id, since the URN extension may differ from the one on disk
-            // (e.g. jpeg vs jpg). Only reached when the exact-key lookups all miss.
+            // (e.g. jpeg vs jpg). Only reached when the exact-key lookups all miss
+            // and no asset row named the file.
             const bareId = idCandidates[idCandidates.length - 1];
-            if (bareId) {
+            if (bareId && !assetRowFound) {
                 for (const prefix of this.assetListingPrefixes(bareId)) {
                     try {
                         const listing = await adapter.list(prefix);
@@ -1876,8 +1939,9 @@ export class ProcessingContext {
         // Strict, anchored sanitizer for the URIs that may flow into the asset /
         // storage retrieval path. Accept asset references, the browser-facing
         // storage route, and URIs from configured storage adapters. External URLs
-        // and unrecognized schemes stay out of the storage adapter.
-        const RESOLVABLE_URI_RE = /^(?:asset:\/\/|\/?api\/storage\/|(?:memory|file|s3|supabase):\/\/)[A-Za-z0-9._~/%-]+$/;
+        // and unrecognized schemes stay out of the storage adapter. `:` is allowed
+        // for named owner ids such as `projects/personal:<user_id>/assets/…`.
+        const RESOLVABLE_URI_RE = /^(?:asset:\/\/|\/?api\/storage\/|(?:memory|file|s3|supabase):\/\/)[A-Za-z0-9._~/%:-]+$/;
         const isResolvableUri = (uri) => RESOLVABLE_URI_RE.test(uri);
         // An `asset://` reference that cannot be dereferenced must not reach the
         // provider: providers `fetch` a non-data URI, and the SSRF guard then
@@ -2003,12 +2067,82 @@ export class ProcessingContext {
         }
         return resolved;
     }
+    async resolveRequiredImageEvidence(messages, required = []) {
+        const evidence = new Map();
+        const resolve = async (ref) => {
+            const identity = ref.uri?.startsWith("data:")
+                ? "inline image"
+                : ref.uri ?? ref.asset_id ?? "inline image";
+            const cached = ref.uri ? evidence.get(ref.uri) : undefined;
+            if (cached) {
+                return cached;
+            }
+            try {
+                const bytes = await loadMediaRefBytes(ref, this);
+                if (!bytes?.length) {
+                    throw new Error("no image bytes");
+                }
+                const mimeType = sniffImageMime(bytes);
+                if (!mimeType ||
+                    !["image/png", "image/jpeg", "image/webp", "image/gif"].includes(mimeType)) {
+                    throw new Error("unsupported image format");
+                }
+                await validateImageEvidence(bytes);
+                const block = {
+                    type: "image_url",
+                    image: {
+                        uri: `data:${mimeType};base64,${Buffer.from(bytes).toString("base64")}`,
+                        mimeType
+                    }
+                };
+                if (ref.uri) {
+                    evidence.set(ref.uri, block);
+                }
+                return block;
+            }
+            catch (error) {
+                throw new Error(`Required image evidence unavailable: ${identity} (${error instanceof Error ? error.message : String(error)})`);
+            }
+        };
+        for (const uri of required) {
+            await resolve({ type: "image", uri });
+        }
+        const result = [];
+        for (const message of messages) {
+            if (!Array.isArray(message.content)) {
+                result.push(message);
+                continue;
+            }
+            const content = [];
+            let imageIndex = 0;
+            for (const part of message.content) {
+                if (part.type === "image_url") {
+                    imageIndex++;
+                    content.push(await resolve(part.image), {
+                        type: "text",
+                        text: `[required image ${part.image.uri && !part.image.uri.startsWith("data:")
+                            ? part.image.uri
+                            : `inline image ${imageIndex}`}]`
+                    });
+                }
+                else {
+                    content.push(part);
+                }
+            }
+            result.push({ ...message, content });
+        }
+        return result;
+    }
     async dispatchCapability(provider, req) {
         const params = req.params ?? {};
         switch (req.capability) {
             case "generate_message":
                 return provider.generateMessageTraced({
-                    messages: await this.resolveMessageMediaUris(params.messages ?? []),
+                    messages: params.strict_image_evidence === true
+                        ? await this.resolveRequiredImageEvidence(params.messages ?? [], Array.isArray(params.required_images)
+                            ? params.required_images.filter(isString)
+                            : [])
+                        : await this.resolveMessageMediaUris(params.messages ?? []),
                     model: req.model,
                     tools: params.tools,
                     toolChoice: params.tool_choice,
@@ -2163,6 +2297,22 @@ export class ProcessingContext {
                 return provider.vectorizeImage(params.image, {
                     model: { id: req.model, name: req.model, provider: req.provider }
                 });
+            case "estimate_depth":
+                return provider.estimateDepth(params.image, {
+                    model: { id: req.model, name: req.model, provider: req.provider },
+                    signal: isAbortSignal(params.signal) ? params.signal : this.signal
+                });
+            case "rerank":
+                return provider.rerank({
+                    model: req.model,
+                    query: isString(params.query) ? params.query : "",
+                    // Every entry stays, so result indices point into the caller's list.
+                    documents: Array.isArray(params.documents)
+                        ? params.documents.map((doc) => (isString(doc) ? doc : String(doc)))
+                        : [],
+                    topK: isNumber(params.top_k) ? params.top_k : undefined,
+                    signal: isAbortSignal(params.signal) ? params.signal : this.signal
+                });
             case "track_object": {
                 const request = objectTrackingRequestSchema.parse(params);
                 if (!provider.getCapabilities().includes("track_object")) {
@@ -2175,12 +2325,18 @@ export class ProcessingContext {
                     throw new Error("The tracking source video could not be loaded.");
                 }
                 signal.throwIfAborted();
-                const output = await provider.trackObject(video, { ...request, model: req.model, signal });
+                const output = await provider.trackObject(video, {
+                    ...request,
+                    model: req.model,
+                    signal
+                });
                 signal.throwIfAborted();
                 return parseObjectTrackingResult(output, request);
             }
             case "video_to_video": {
-                const signal = isAbortSignal(params.signal) ? params.signal : this.signal;
+                const signal = isAbortSignal(params.signal)
+                    ? params.signal
+                    : this.signal;
                 const referenceImages = coerceByteList(params.reference_images ?? params.referenceImages);
                 const referenceAssetIds = coerceStringList(params.reference_asset_ids ?? params.referenceAssetIds);
                 signal.throwIfAborted();
@@ -2378,137 +2534,174 @@ export class ProcessingContext {
      */
     async runGenerationWith(req, call, opts) {
         const id = req.id ?? randomUUID();
-        const startedAt = Date.now();
-        const origin = this.generationOrigin(req, req.origin);
-        const controller = new AbortController();
-        const signals = [controller.signal, this.signal];
-        if (req.signal)
-            signals.push(req.signal);
-        const signal = AbortSignal.any(signals);
-        const accept = opts?.onGenerationAccepted ??
-            this._generationLifecycle?.onGenerationAccepted;
-        const terminal = opts?.onGenerationTerminal ??
-            this._generationLifecycle?.onGenerationTerminal;
-        let durableAccepted = false;
-        let terminalAttempted = false;
-        try {
-            const accepted = await accept?.({ generationId: id, request: req });
-            if (accepted?.skipProvider) {
-                throw new GenerationAlreadyAcceptedError(accepted.existingGenerationId ?? id);
-            }
-            durableAccepted = accepted?.durable === true;
-            generationRegistry.register(id, {
-                userId: this.userId,
-                abort: () => controller.abort(new Error("Generation cancelled"))
-            });
-            this.emitPrediction("running", req, id, null, undefined, startedAt, {
-                origin
-            });
-            const provider = opts?.withoutProvider
-                ? null
-                : await this.getProvider(req.provider);
-            const { value: output, receipt: recorded, providerResult } = await runWithGenerationReceipt(() => call(provider, signal), {
-                providerRequestOptions: accepted ?? undefined,
-                onProviderRequestAccepted: opts?.onProviderRequestAccepted ??
-                    this._generationLifecycle?.onProviderRequestAccepted,
-                onProviderRequestBound: opts?.onProviderRequestBound ??
-                    this._generationLifecycle?.onProviderRequestBound
-            });
-            const encoded = encodedAudioOf(output);
-            const persistedAssets = req.persist
-                ? await this.persistGenerationBytes(id, req, encoded ? [encoded.data] : mediaBuffers(output), encoded?.mimeType)
-                : [];
-            const assets = persistedAssets.filter((asset) => asset !== null);
-            const late = opts?.receiptAfterPersist?.() ?? null;
-            const receipt = late || recorded ? { ...(recorded ?? {}), ...(late ?? {}) } : null;
-            const assetIds = assets
-                .map((asset) => asset.asset_id)
-                .filter((assetId) => typeof assetId === "string");
-            const durableAssetIds = persistedAssets.map((asset) => asset?.asset_id ?? null);
+        return withSpan("generation", { "generation.id": id, "generation.provider": req.provider, "generation.kind": req.capability }, async () => {
+            const startedAt = Date.now();
+            const origin = this.generationOrigin(req, req.origin);
+            const controller = new AbortController();
+            const signals = [controller.signal, this.signal];
+            if (req.signal)
+                signals.push(req.signal);
+            const signal = AbortSignal.any(signals);
+            const accept = opts?.onGenerationAccepted ??
+                this._generationLifecycle?.onGenerationAccepted;
+            const terminal = opts?.onGenerationTerminal ??
+                this._generationLifecycle?.onGenerationTerminal;
+            let durableAccepted = false;
+            let terminalAttempted = false;
             try {
-                terminalAttempted = true;
-                await terminal?.({
-                    generationId: id,
-                    request: req,
-                    status: "completed",
-                    output: providerResult ?? output,
-                    receipt,
-                    assetIds: durableAssetIds
+                const accepted = await accept?.({ generationId: id, request: req });
+                if (accepted?.skipProvider) {
+                    throw new GenerationAlreadyAcceptedError(accepted.existingGenerationId ?? id);
+                }
+                durableAccepted = accepted?.durable === true;
+                generationRegistry.register(id, {
+                    userId: this.userId,
+                    abort: () => controller.abort(new Error("Generation cancelled"))
                 });
-            }
-            catch (terminalError) {
-                this.emit({
-                    type: "log_update",
-                    node_id: req.nodeId ?? "",
-                    node_name: req.nodeId ?? "",
-                    workflow_id: req.workflowId ?? this.workflowId ?? null,
-                    severity: "warning",
-                    content: `Generation ${id} completed but durable finalization will be recovered: ${terminalError instanceof Error ? terminalError.message : String(terminalError)}`
+                this.emitPrediction("running", req, id, null, undefined, startedAt, {
+                    origin
                 });
-            }
-            this.emitPrediction("completed", req, id, generationResultData(req.capability, output), undefined, startedAt, { origin, asset_ids: assetIds, receipt });
-            generationRegistry.settle(id, {
-                status: "completed",
-                asset_ids: assetIds,
-                receipt
-            });
-            return {
-                id,
-                output,
-                assets,
-                receipt,
-                duration_ms: Date.now() - startedAt
-            };
-        }
-        catch (error) {
-            const cause = error instanceof GenerationScopeError ? error.cause : error;
-            const receipt = error instanceof GenerationScopeError ? error.receipt : null;
-            const providerResult = error instanceof GenerationScopeError
-                ? error.providerResult
-                : undefined;
-            const message = cause instanceof Error ? cause.message : String(cause);
-            if (cause instanceof GenerationAlreadyAcceptedError) {
-                throw cause;
-            }
-            const status = signal.aborted
-                ? durableAccepted
-                    ? "running"
-                    : "cancelled"
-                : durableAccepted && !isAuthoritativeProviderTerminalError(cause)
-                    ? "running"
-                    : "failed";
-            if (durableAccepted && !terminalAttempted) {
+                const provider = opts?.withoutProvider
+                    ? null
+                    : await this.getProvider(req.provider);
+                const { value: output, receipt: recorded, providerResult } = await runWithGenerationReceipt(() => call(provider, signal), {
+                    providerRequestOptions: accepted ?? undefined,
+                    onProviderRequestAccepted: opts?.onProviderRequestAccepted ??
+                        this._generationLifecycle?.onProviderRequestAccepted,
+                    onProviderRequestBound: opts?.onProviderRequestBound ??
+                        this._generationLifecycle?.onProviderRequestBound
+                });
+                const encoded = encodedAudioOf(output);
+                const persistedAssets = req.persist
+                    ? await this.persistGenerationBytes(id, req, encoded ? [encoded.data] : mediaBuffers(output), encoded?.mimeType)
+                    : [];
+                const assets = persistedAssets.filter((asset) => asset !== null);
+                let delivery;
+                if (opts?.finalizeOutput) {
+                    try {
+                        delivery = await opts.finalizeOutput(output, assets);
+                    }
+                    catch (error) {
+                        delivery = {
+                            status: "failed",
+                            error: error instanceof Error ? error.message : String(error)
+                        };
+                    }
+                }
+                const deliveryFields = {};
+                if (delivery) {
+                    deliveryFields.delivery = delivery;
+                }
+                const late = opts?.receiptAfterPersist?.() ?? null;
+                const receipt = late || recorded ? { ...(recorded ?? {}), ...(late ?? {}) } : null;
+                const assetIds = assets
+                    .map((asset) => asset.asset_id)
+                    .filter((assetId) => typeof assetId === "string");
+                const durableAssetIds = persistedAssets.map((asset) => asset?.asset_id ?? null);
                 try {
                     terminalAttempted = true;
                     await terminal?.({
                         generationId: id,
                         request: req,
-                        status: isAuthoritativeProviderTerminalError(cause)
-                            ? "failed"
-                            : "recovering",
-                        output: providerResult,
-                        error: message,
+                        status: "completed",
+                        output: providerResult ?? output,
                         receipt,
-                        assetIds: []
+                        assetIds: durableAssetIds,
+                        ...deliveryFields
                     });
                 }
-                catch {
-                    // The provider observation remains authoritative. The durable row
-                    // can be finalized by the recovery worker.
+                catch (terminalError) {
+                    this.emit({
+                        type: "log_update",
+                        node_id: req.nodeId ?? "",
+                        node_name: req.nodeId ?? "",
+                        workflow_id: req.workflowId ?? this.workflowId ?? null,
+                        severity: "warning",
+                        content: `Generation ${id} completed but durable finalization will be recovered: ${terminalError instanceof Error ? terminalError.message : String(terminalError)}`
+                    });
                 }
+                this.emitPrediction("completed", req, id, delivery ? { delivery } : generationResultData(req.capability, output), undefined, startedAt, { origin, asset_ids: assetIds, receipt });
+                generationRegistry.settle(id, {
+                    status: "completed",
+                    asset_ids: assetIds,
+                    receipt,
+                    ...deliveryFields
+                });
+                return {
+                    id,
+                    output,
+                    assets,
+                    receipt,
+                    duration_ms: Date.now() - startedAt,
+                    ...deliveryFields
+                };
             }
-            this.emitPrediction(status, req, id, null, message, startedAt, {
-                origin,
-                receipt
-            });
-            generationRegistry.settle(id, {
-                status,
-                error: message,
-                asset_ids: [],
-                receipt
-            });
-            throw cause;
-        }
+            catch (error) {
+                const cause = error instanceof GenerationScopeError ? error.cause : error;
+                const receipt = error instanceof GenerationScopeError ? error.receipt : null;
+                const providerResult = error instanceof GenerationScopeError
+                    ? error.providerResult
+                    : undefined;
+                const message = cause instanceof Error ? cause.message : String(cause);
+                if (cause instanceof GenerationAlreadyAcceptedError) {
+                    throw cause;
+                }
+                const status = signal.aborted
+                    ? durableAccepted
+                        ? "running"
+                        : "cancelled"
+                    : durableAccepted && !isAuthoritativeProviderTerminalError(cause)
+                        ? "running"
+                        : "failed";
+                if (durableAccepted && !terminalAttempted) {
+                    try {
+                        terminalAttempted = true;
+                        await terminal?.({
+                            generationId: id,
+                            request: req,
+                            status: isAuthoritativeProviderTerminalError(cause)
+                                ? "failed"
+                                : "recovering",
+                            output: providerResult,
+                            error: message,
+                            receipt,
+                            assetIds: []
+                        });
+                    }
+                    catch {
+                        // The provider observation remains authoritative. The durable row
+                        // can be finalized by the recovery worker.
+                    }
+                }
+                if (this.appRunContext && !durableAccepted && !terminalAttempted) {
+                    try {
+                        terminalAttempted = true;
+                        await terminal?.({
+                            generationId: id,
+                            request: req,
+                            status: signal.aborted ? "cancelled" : "failed",
+                            error: message,
+                            receipt,
+                            assetIds: []
+                        });
+                    }
+                    catch {
+                        // Preserve the provider failure; the live ledger can retry its write.
+                    }
+                }
+                this.emitPrediction(status, req, id, null, message, startedAt, {
+                    origin,
+                    receipt
+                });
+                generationRegistry.settle(id, {
+                    status,
+                    error: message,
+                    asset_ids: [],
+                    receipt
+                });
+                throw cause;
+            }
+        });
     }
     /**
      * Save a generation's media output as assets, stamping the generation id
@@ -2763,6 +2956,22 @@ export class ProcessingContext {
                     // retried by the recovery worker.
                 }
             }
+            if (this.appRunContext && !durableAccepted && !terminalAttempted) {
+                try {
+                    terminalAttempted = true;
+                    await terminal?.({
+                        generationId: id,
+                        request: req,
+                        status: signal.aborted ? "cancelled" : "failed",
+                        error: message,
+                        receipt,
+                        assetIds: []
+                    });
+                }
+                catch {
+                    // Preserve the provider failure; the live ledger can retry its write.
+                }
+            }
             this.emitPrediction(status, req, id, null, message, startedAt, {
                 origin,
                 receipt