@nodetool-ai/websocket 0.7.0-rc.23 → 0.8.2

package/dist/session/inference.js

@@ -0,0 +1,1143 @@
+import { randomUUID } from "node:crypto";
+import { createLogger } from "@nodetool-ai/config";
+import { getModelUnitPrice } from "@nodetool-ai/model-pricing";
+import { Asset, Prediction, Project, TimelineSequence, findFinishResourceIds, assertStoryboardClipGenerationAllowed } from "@nodetool-ai/models";
+import { extractPricingParams } from "@nodetool-ai/node-sdk/pricing-params";
+import { resolveNodetoolDelegate } from "@nodetool-ai/protocol";
+import { GenerationAlreadyAcceptedError } from "@nodetool-ai/runtime";
+import { calculateChatCost, detectImageMime, expandEntitiesForGeneration, generateStructured, isToolCall, IMAGE_MIME_TO_EXT, messageText } from "@nodetool-ai/runtime";
+import { linkGenerationAssets } from "@nodetool-ai/execution";
+import { admitSpend, releaseSpend, reserveSpend } from "../credit-gate.js";
+import { retrieveAssetBytes } from "../lib/asset-paths.js";
+import { trimVideoWindow } from "../lib/media.js";
+import { resolveImageSize } from "../lib/media-size.js";
+import { getAssetAdapter } from "../lib/storage.js";
+import { isFiniteNumber, isRecord, isString } from "../lib/wire-values.js";
+import { createGenerationRun, generateSpeechBytes, generateMusicBytes } from "./media-generation.js";
+const log = createLogger("nodetool.websocket.runner");
+/**
+ * Strip sentence punctuation after an `entity://` id. A loop, not `/\.+$/`:
+ * the regex backtracks quadratically on a long dot run followed by another
+ * character, and the prompt is user input.
+ */
+function trimTrailingDots(value) {
+    let end = value.length;
+    while (end > 0 && value[end - 1] === ".")
+        end -= 1;
+    return value.slice(0, end);
+}
+async function resolveVideoEditReferenceIds(userId, req) {
+    const ownedResolver = entityRefResolver(userId);
+    const assets = new Map();
+    const requestedEntities = new Set([
+        ...(req.entityIds ?? []),
+        ...Array.from(req.prompt.matchAll(/entity:\/\/([A-Za-z0-9._~-]+)/g), (match) => trimTrailingDots(match[1]))
+    ]);
+    const resolvedEntityIds = new Set();
+    const resolver = {
+        getAssetInfo: async (id) => {
+            let pending = assets.get(id);
+            if (!pending) {
+                pending = ownedResolver.getAssetInfo(id);
+                assets.set(id, pending);
+            }
+            const asset = await pending;
+            const marker = asset?.metadata?.nodetool_entity;
+            if (requestedEntities.has(id) && isRecord(marker) && isString(marker.name) && marker.name.trim()) {
+                resolvedEntityIds.add(id);
+            }
+            return asset;
+        }
+    };
+    const entityIds = [...new Set(req.entityIds ?? [])];
+    const entityReferenceAssetIds = [];
+    for (const id of entityIds) {
+        const asset = await resolver.getAssetInfo(id);
+        const marker = asset?.metadata?.nodetool_entity;
+        if (!isRecord(marker) || !isString(marker.name) || !marker.name.trim()) {
+            throw new Error(`Entity reference ${id} was not found`);
+        }
+        const referenceAssetId = isString(marker.reference_asset_id)
+            ? marker.reference_asset_id.trim()
+            : "";
+        entityReferenceAssetIds.push(referenceAssetId || id);
+    }
+    const expanded = await expandEntitiesForGeneration([req.prompt, ...entityIds.map((id) => `entity://${id}`)].join(" "), resolver);
+    const assetIds = [...new Set([
+            ...(req.referenceAssetIds ?? []),
+            ...(req.referenceImages ?? []).map((ref) => referenceAssetId(ref, "image")),
+            ...entityReferenceAssetIds,
+            ...expanded.referenceImages.map((ref) => ref.uri.slice("asset://".length, ref.uri.lastIndexOf(".")))
+        ])];
+    return { prompt: expanded.prompt, assetIds, entityIds: [...resolvedEntityIds] };
+}
+/**
+ * Chars per token for the up-front spend estimate. Deliberately low (real
+ * English averages nearer 4) so the estimate over-books rather than under.
+ */
+const ESTIMATE_CHARS_PER_TOKEN = 3;
+/** Output budget assumed when a request names no `max_tokens`. */
+const ESTIMATE_DEFAULT_OUTPUT_TOKENS = 4096;
+/**
+ * What one image block is worth to the estimate. A reference image costs on
+ * the order of a thousand tokens across the providers that take them; the
+ * estimate only has to over-book, and the real token cost is recorded when the
+ * call returns.
+ */
+const IMAGE_ESTIMATE_CHARS = 4000;
+/** Characters a message contributes to the estimate, text blocks included. */
+function messageChars(content) {
+    if (typeof content === "string") {
+        return content.length;
+    }
+    // A block's shape is only checked as `{ type: string }` + passthrough on the
+    // wire, so a `{ type: "text" }` with no `text` reaches here. Reading `.length`
+    // off it threw before the spend estimate was taken — a crash on the admission
+    // path, not a refusal.
+    return content.reduce((sum, block) => sum +
+        (block.type === "text" && typeof block.text === "string"
+            ? block.text.length
+            : IMAGE_ESTIMATE_CHARS), 0);
+}
+/**
+ * A conservative up-front price for one text generation, in USD: every
+ * character the messages carry counted as input tokens, plus the request's
+ * whole output budget as output tokens, at the delegate model's rate.
+ *
+ * It over-estimates on purpose. The figure is what gets *reserved* for the
+ * duration of the call, and the real cost replaces it afterwards — an
+ * over-booking that is released beats letting concurrent calls each admit
+ * against a balance none of them has spent yet.
+ */
+export function estimateDirectTextSpend(req) {
+    const delegate = req.provider === "nodetool" ? resolveNodetoolDelegate(req.model) : null;
+    const modelId = delegate?.model ?? req.model;
+    const providerId = delegate?.provider ?? req.provider;
+    const chars = req.messages.reduce((sum, m) => sum + messageChars(m.content), 0);
+    const inputTokens = Math.ceil(chars / ESTIMATE_CHARS_PER_TOKEN);
+    const outputTokens = req.maxTokens ?? ESTIMATE_DEFAULT_OUTPUT_TOKENS;
+    try {
+        return calculateChatCost(modelId, inputTokens, outputTokens, 0, providerId);
+    }
+    catch {
+        // An unpriced model estimates at zero: the gate still admits against the
+        // balance, and the real cost is recorded when the call returns.
+        return 0;
+    }
+}
+/**
+ * Entity-mention resolver over the Asset model, scoped to one user. Backs
+ * `expandEntitiesForGeneration` on every direct-generation surface.
+ */
+export function entityRefResolver(userId) {
+    return {
+        getAssetInfo: async (assetId) => {
+            const asset = await Asset.find(userId, assetId);
+            if (!asset)
+                return null;
+            return {
+                id: asset.id,
+                content_type: asset.content_type,
+                name: asset.name,
+                metadata: asset.metadata ?? null
+            };
+        }
+    };
+}
+/**
+ * Resolve entity-derived reference images to provider input bytes. A ref
+ * whose asset is gone (or reads back empty) contributes nothing — the same
+ * drop rule as an unresolvable mention.
+ */
+export async function resolveEntityReferenceImages(userId, refs) {
+    const out = [];
+    for (const ref of refs) {
+        const bare = ref.uri.slice("asset://".length);
+        const assetId = bare.slice(0, bare.lastIndexOf("."));
+        if (!assetId)
+            continue;
+        const asset = await Asset.find(userId, assetId);
+        if (!asset)
+            continue;
+        const bytes = await retrieveAssetBytes(getAssetAdapter(), userId, asset.id, asset.content_type);
+        if (!bytes || bytes.length === 0)
+            continue;
+        out.push(bytes);
+    }
+    return out;
+}
+/**
+ * Read one owned asset's bytes, with the descriptive errors the generation
+ * paths surface verbatim to callers.
+ */
+export async function retrieveSourceAssetBytes(userId, assetId) {
+    const asset = await Asset.find(userId, assetId);
+    if (!asset) {
+        throw new Error(`Source asset not found: ${assetId}`);
+    }
+    const bytes = await retrieveAssetBytes(getAssetAdapter(), userId, assetId, asset.content_type);
+    if (!bytes) {
+        throw new Error(`Source asset bytes not found: ${assetId}`);
+    }
+    return bytes;
+}
+async function resolveReferenceAssets(userId, refs, kind) {
+    const resolved = [];
+    for (const ref of refs) {
+        const assetId = referenceAssetId(ref, kind);
+        const asset = await Asset.find(userId, assetId);
+        if (!asset)
+            throw new Error(`Reference ${kind} asset ${assetId} was not found`);
+        if (!asset.content_type.startsWith(`${kind}/`)) {
+            throw new Error(`Reference ${kind} asset ${assetId} is not ${kind === "image" ? "an image" : "a video"}`);
+        }
+        const bytes = await retrieveAssetBytes(getAssetAdapter(), userId, asset.id, asset.content_type);
+        if (!bytes || bytes.length === 0) {
+            throw new Error(`Reference ${kind} asset ${assetId} is empty`);
+        }
+        resolved.push(bytes);
+    }
+    return resolved;
+}
+function referenceAssetId(ref, kind) {
+    if (!isRecord(ref) || (ref.type !== undefined && ref.type !== kind)) {
+        throw new Error(`Expected reference ${kind} objects`);
+    }
+    const assetId = isString(ref.asset_id) && ref.asset_id.length > 0
+        ? ref.asset_id
+        : isString(ref.uri) && ref.uri.startsWith("asset://")
+            ? ref.uri.slice("asset://".length).split(".")[0]
+            : "";
+    if (!assetId) {
+        throw new Error(`Reference ${kind} is missing an asset id`);
+    }
+    return assetId;
+}
+/**
+ * Refuse a bare prompt for a model the catalog lists as editing-only. Such a
+ * model needs an input image, and its provider rejects the request with an
+ * error that names neither the cause nor the fix (AtlasCloud's edit
+ * endpoints answer "index 0 out of range"). An unlisted model, or one with
+ * no declared tasks, is left to the provider.
+ */
+async function assertTextToImageModel(provider, modelId) {
+    if (typeof provider.getAvailableImageModels !== "function") {
+        return;
+    }
+    const model = (await provider.getAvailableImageModels()).find((candidate) => candidate.id === modelId);
+    const tasks = model?.supportedTasks;
+    if (model && tasks?.length && !tasks.includes("text_to_image")) {
+        throw new Error(`${model.name || modelId} only edits images and needs an input image. ` +
+            "Choose a model that creates images from text.");
+    }
+}
+/**
+ * One-shot model calls that skip the chat thread and the workflow runner:
+ * the streamed `inference` command, and the direct text / media / speech
+ * generation the sketch, timeline and Studio surfaces drive over RPC.
+ */
+export class DirectInferenceHandler {
+    session;
+    deps;
+    constructor(session, deps) {
+        this.session = session;
+        this.deps = deps;
+    }
+    async handleInference(data, requestSeq, signal) {
+        const providerId = isString(data.provider)
+            ? data.provider
+            : this.deps.defaults.provider;
+        const model = isString(data.model) ? data.model : this.deps.defaults.model;
+        const rawMessages = Array.isArray(data.messages) ? data.messages : [];
+        log.debug("Inference request", {
+            model,
+            provider: providerId,
+            messages: rawMessages.length
+        });
+        const messages = rawMessages.map((m) => {
+            const msg = m;
+            return {
+                role: (isString(msg.role)
+                    ? msg.role
+                    : "user"),
+                content: isString(msg.content)
+                    ? msg.content
+                    : Array.isArray(msg.content)
+                        ? msg.content
+                        : "",
+                toolCallId: isString(msg.toolCallId) ? msg.toolCallId : null,
+                toolCalls: Array.isArray(msg.toolCalls)
+                    ? msg.toolCalls
+                    : null,
+                threadId: null
+            };
+        });
+        if (!this.session.resolveProvider) {
+            await this.session.send({
+                type: "error",
+                message: "No provider resolver configured"
+            });
+            return;
+        }
+        const rawTools = Array.isArray(data.tools) ? data.tools : [];
+        const tools = rawTools
+            .map((t) => {
+            const tool = t;
+            return {
+                name: isString(tool.name) ? tool.name : "",
+                description: isString(tool.description)
+                    ? tool.description
+                    : undefined,
+                inputSchema: typeof tool.inputSchema === "object"
+                    ? tool.inputSchema
+                    : undefined
+            };
+        })
+            .filter((t) => t.name.length > 0);
+        const provider = await this.session.resolveProvider(providerId, this.session.requireUserId());
+        for await (const item of provider.generateMessagesTraced({
+            messages,
+            model,
+            tools: tools.length > 0 ? tools : undefined,
+            signal
+        })) {
+            if (requestSeq !== this.deps.currentRequestSeq())
+                break; // cancelled
+            if (isToolCall(item)) {
+                log.info("Tool call", { tool: item.name, args: item.args });
+                await this.session.send({
+                    type: "tool_call",
+                    id: item.id,
+                    name: item.name,
+                    args: item.args,
+                    seq: requestSeq
+                });
+            }
+            else if (item.type === "chunk") {
+                await this.session.send({ ...item, seq: requestSeq });
+            }
+        }
+        if (requestSeq === this.deps.currentRequestSeq()) {
+            log.debug("Inference complete");
+            await this.session.send({ type: "inference_done", seq: requestSeq });
+        }
+    }
+    /**
+     * Run a one-shot text generation and return the answer — no chat thread, no
+     * job row, no workflow. The text twin of `runDirectMediaGeneration`, and
+     * what a surface calls when it needs a model to write or decide one thing.
+     *
+     * With a schema the call is structured output: the model is forced through
+     * one tool whose input schema is that shape, and the parsed object comes
+     * back in `data`. `generateStructured` owns that mechanism, shared with the
+     * Director node and the agent nodes, so a schema answered here and a schema
+     * answered in a workflow are answered the same way.
+     */
+    async runDirectTextGeneration(req) {
+        if (!this.session.resolveProvider) {
+            throw new Error("No provider resolver configured");
+        }
+        if (!req.model) {
+            throw new Error("model is required");
+        }
+        if (req.messages.length === 0) {
+            throw new Error("prompt or messages is required");
+        }
+        const userId = this.session.requireUserId();
+        const provider = await this.session.resolveProvider(req.provider, userId);
+        if (req.provider !== "nodetool") {
+            // BYOK: the user's own keys, never metered.
+            return this.runDirectTextGenerationInner(req, provider);
+        }
+        // NodeTool's managed provider: admit against the balance (including
+        // in-flight reservations), hold the estimate for the duration of the call
+        // so concurrent requests admit against each other, and record the real
+        // token cost as a prediction row so the balance decrements.
+        const estimatedUsd = estimateDirectTextSpend(req);
+        const decision = await admitSpend(userId, estimatedUsd, [req.model]);
+        if (!decision.allowed) {
+            throw new Error(decision.reason);
+        }
+        const reservationKey = `text:${randomUUID()}`;
+        reserveSpend(userId, reservationKey, estimatedUsd);
+        try {
+            const result = await this.runDirectTextGenerationInner(req, provider);
+            // The estimate deliberately over-books, so the tracked token cost is
+            // the charge — never the reservation.
+            const cost = provider.getTotalCost();
+            if (cost > 0) {
+                try {
+                    await Prediction.create({
+                        user_id: userId,
+                        provider: "nodetool",
+                        model: req.model,
+                        node_type: req.schema ? "direct.structured" : "direct.text",
+                        cost,
+                        currency: "USD",
+                        billing_unit: "tokens",
+                        quantity: 1,
+                        workflow_id: null,
+                        node_id: "",
+                        status: "completed"
+                    });
+                }
+                catch (err) {
+                    this.session.logError("direct text cost persistence failed", err);
+                }
+            }
+            return result;
+        }
+        finally {
+            releaseSpend(userId, reservationKey);
+        }
+    }
+    /**
+     * The provider call itself. Runs under an abort signal registered on the
+     * connection, so a `stop` command or a dropped socket interrupts a model
+     * that is still generating instead of billing for an answer nobody reads.
+     */
+    async runDirectTextGenerationInner(req, provider) {
+        const messages = req.messages.map((m) => ({
+            role: (m.role === "system" || m.role === "assistant" || m.role === "tool"
+                ? m.role
+                : "user"),
+            content: m.content,
+            toolCallId: null,
+            toolCalls: null,
+            threadId: null
+        }));
+        const abort = new AbortController();
+        const deregister = this.deps.registerAbort(abort);
+        try {
+            if (req.schema) {
+                const data = await generateStructured(provider, {
+                    messages,
+                    model: req.model,
+                    maxTokens: req.maxTokens,
+                    toolName: req.schemaName,
+                    toolDescription: req.schemaDescription,
+                    schema: req.schema,
+                    signal: abort.signal
+                });
+                return { text: "", data };
+            }
+            const result = await provider.generateMessageTraced({
+                messages,
+                model: req.model,
+                maxTokens: req.maxTokens,
+                signal: abort.signal
+            });
+            return { text: messageText(result.content), data: null };
+        }
+        finally {
+            deregister();
+        }
+    }
+    /**
+     * Run a one-shot media-generation request (text-to-image, image-to-image,
+     * text-to-video, or text-to-audio) and return the produced asset ids.
+     * Mirrors the image / image_edit / video / audio branches of
+     * `handleMediaGenerationMessage` but skips the chat-thread machinery —
+     * the caller wants asset ids, not a streamed Message row.
+     *
+     * Used by the `generate_media` RPC for the sketch editor's direct-gen
+     * image layers and the timeline's direct-gen video / audio clips; the
+     * chat-path equivalents stay in `handleMediaGenerationMessage` for now.
+     */
+    async runDirectMediaGeneration(req) {
+        if (!this.session.resolveProvider) {
+            throw new Error("No provider resolver configured");
+        }
+        if (!req.model) {
+            throw new Error("model is required");
+        }
+        if (!req.prompt || !req.prompt.trim()) {
+            throw new Error("prompt is required");
+        }
+        if (req.mode !== "video_edit" && (req.referenceAssetIds?.length || req.entityIds?.length)) {
+            throw new Error("Entity and reference asset ids require video_edit mode");
+        }
+        if (req.capability === "reference_to_video") {
+            if (req.mode !== "video") {
+                throw new Error("reference_to_video requires video mode");
+            }
+            if (req.sourceAssetId) {
+                throw new Error("reference_to_video uses reference_images and reference_videos, not source_asset_id");
+            }
+            if (!req.referenceImages?.length && !req.referenceVideos?.length) {
+                throw new Error("reference_to_video requires at least one reference image or video");
+            }
+            if (req.useReferenceVideoAudio && !req.referenceVideos?.length) {
+                throw new Error("Reference video audio requires at least one reference video");
+            }
+        }
+        else if ((req.referenceImages?.length && req.mode !== "video_edit") ||
+            req.referenceVideos?.length ||
+            req.useReferenceVideoAudio !== undefined) {
+            throw new Error("Reference inputs require the reference_to_video capability");
+        }
+        const userId = this.session.requireUserId();
+        if (req.projectId && req.projectId !== "default") {
+            await Project.requireOwned(userId, req.projectId);
+        }
+        await assertTimelineGenerationAllowed(userId, req);
+        const videoEditReferenceIds = req.mode === "video_edit"
+            ? await resolveVideoEditReferenceIds(userId, req)
+            : undefined;
+        if (videoEditReferenceIds) {
+            await assertTimelineGenerationAllowed(userId, req, videoEditReferenceIds.assetIds);
+        }
+        const provider = await this.session.resolveProvider(req.provider, userId);
+        const videoEditReferences = videoEditReferenceIds ? {
+            ...videoEditReferenceIds,
+            images: await resolveReferenceAssets(userId, videoEditReferenceIds.assetIds.map(asset_id => ({ asset_id })), "image")
+        } : undefined;
+        if (req.mode === "video_edit" || req.mode === "video_extend") {
+            const extension = req.mode === "video_extend";
+            const task = extension ? "extend_video" : "video_to_video";
+            const context = req.sourceContext;
+            if (!req.sourceAssetId) {
+                throw new Error(`source_asset_id is required for ${req.mode}`);
+            }
+            if (extension &&
+                (!context ||
+                    (req.extensionMode !== "start" && req.extensionMode !== "end") ||
+                    !Number.isFinite(req.durationSeconds) ||
+                    (req.durationSeconds ?? 0) <= 0 ||
+                    (req.variations !== undefined && req.variations !== 1))) {
+                throw new Error("video_extend requires source_context, start/end extension_mode, a positive duration and one variation");
+            }
+            if (context && context.sourceAssetId !== req.sourceAssetId) {
+                throw new Error("source_context does not match source_asset_id");
+            }
+            if (context &&
+                (!Number.isFinite(context.sourceStartMs) ||
+                    !Number.isFinite(context.sourceEndMs) ||
+                    context.sourceStartMs < 0 ||
+                    context.sourceEndMs <= context.sourceStartMs ||
+                    !Number.isFinite(context.timelineDurationMs) ||
+                    context.timelineDurationMs <= 0 ||
+                    !Number.isFinite(context.speedMultiplier) ||
+                    context.speedMultiplier <= 0 ||
+                    (!extension && context.speedMultiplier !== 1) ||
+                    Math.abs(context.sourceEndMs -
+                        (context.sourceStartMs +
+                            context.timelineDurationMs * context.speedMultiplier)) > 0.5)) {
+                throw new Error("source_context contains an invalid constant-speed window");
+            }
+            if (typeof provider.getCapabilities === "function" &&
+                !provider.getCapabilities().includes(task)) {
+                throw new Error(`Provider ${req.provider} does not support the ${task} task`);
+            }
+            if (typeof provider.getAvailableVideoModels === "function") {
+                const model = (await provider.getAvailableVideoModels()).find((candidate) => candidate.id === req.model);
+                if (!model?.supportedTasks?.includes(task)) {
+                    throw new Error(`Model ${req.model} does not support the ${task} task`);
+                }
+                if (videoEditReferences?.assetIds.length &&
+                    !model.supportedTasks.includes("video_to_video_reference")) {
+                    throw new Error(`Model ${req.model} does not support video edit reference images`);
+                }
+                if (extension &&
+                    model.durations?.length &&
+                    !model.durations.includes(req.durationSeconds ?? 0)) {
+                    throw new Error("Choose a supported extension duration.");
+                }
+            }
+            else if (videoEditReferences?.assetIds.length) {
+                throw new Error(`Model ${req.model} has no video edit reference capabilities`);
+            }
+        }
+        if (req.provider !== "nodetool") {
+            // BYOK: the user's own keys, never metered.
+            try {
+                return await this.runDirectMediaGenerationInner(req, provider, undefined, videoEditReferences);
+            }
+            catch (error) {
+                if (error instanceof GenerationAlreadyAcceptedError) {
+                    return {
+                        asset_ids: [],
+                        existing_generation_id: error.generationId
+                    };
+                }
+                throw error;
+            }
+        }
+        // NodeTool's managed provider: admit against the balance (including
+        // in-flight reservations), reserve the unit-price estimate for the
+        // duration of the call, and record the spend as a prediction row so the
+        // balance actually decrements. Cost is the larger of the delegate's own
+        // tracked cost and the unit-price estimate — fal-style delegates bill
+        // per unit and track nothing themselves.
+        const variations = Math.max(1, Math.min(Number(req.variations ?? 1), 8));
+        const effectiveDurationSeconds = req.mode === "video_extend"
+            ? req.durationSeconds
+            : req.sourceContext
+                ? req.sourceContext.timelineDurationMs / 1000
+                : req.durationSeconds;
+        // What the request states about the job, in the vocabulary the catalogs
+        // bill in — a per-second video model prices the clip asked for, not one
+        // second of it.
+        const priceParams = extractPricingParams({
+            resolution: req.resolution,
+            duration_seconds: effectiveDurationSeconds,
+            width: req.width,
+            height: req.height
+        });
+        const unit = getModelUnitPrice({ id: req.model, provider: "nodetool" }, priceParams);
+        const unitPrice = unit && !unit.declined && isFiniteNumber(unit.unit_price)
+            ? unit.unit_price
+            : 0;
+        const estimatedUsd = unitPrice * variations;
+        const decision = await admitSpend(userId, estimatedUsd, [req.model]);
+        if (!decision.allowed) {
+            throw new Error(decision.reason);
+        }
+        const reservationKey = `media:${randomUUID()}`;
+        reserveSpend(userId, reservationKey, estimatedUsd);
+        try {
+            // The generation seam writes the row (docs/media-generation-tracking-
+            // design.md § 8, S5). The managed provider's charge is the larger of the
+            // delegate's own tracked cost and the unit-price estimate — fal-style
+            // delegates bill per unit and track nothing themselves — and reaches the
+            // row as the receipt, which wins over the catalog.
+            const generationIds = [];
+            let result;
+            try {
+                result = await this.runDirectMediaGenerationInner(req, provider, {
+                    generationIds,
+                    statedCost: (tracked) => {
+                        const cost = Math.max(tracked, estimatedUsd);
+                        return cost > 0
+                            ? {
+                                amount: cost,
+                                currency: "USD",
+                                billing_unit: unit?.billing_unit ?? null,
+                                quantity: variations,
+                                unit_price: cost / variations
+                            }
+                            : null;
+                    }
+                }, videoEditReferences);
+            }
+            catch (error) {
+                if (error instanceof GenerationAlreadyAcceptedError) {
+                    return {
+                        asset_ids: [],
+                        existing_generation_id: error.generationId
+                    };
+                }
+                throw error;
+            }
+            // The tracker writes best-effort and says so in its own log; the
+            // managed path is the one that decrements a balance, so a row that did
+            // not land is reported on the session as well.
+            for (const id of generationIds) {
+                try {
+                    const row = await Prediction.findForUser(userId, id);
+                    if (!row || row.status !== "completed") {
+                        throw new Error(`generation ${id} has no completed row`);
+                    }
+                }
+                catch (err) {
+                    this.session.logError("direct media cost persistence failed", err);
+                    break;
+                }
+            }
+            return result;
+        }
+        finally {
+            releaseSpend(userId, reservationKey);
+        }
+    }
+    async runDirectMediaGenerationInner(req, provider, metering, videoEditReferences) {
+        const userId = this.session.requireUserId();
+        const variations = Math.max(1, Math.min(Number(req.variations ?? 1), 8));
+        const { generate, seamAssetId, storeAsset } = createGenerationRun({
+            userId,
+            providerId: req.provider,
+            modelId: req.model,
+            provider,
+            origin: { surface: "rpc", request_id: req.requestId ?? null },
+            workflowId: null,
+            projectId: req.projectId ?? null,
+            assetNamePrefix: req.mode,
+            destination: req.destination,
+            // The row names the RPC mode the way it always did.
+            nodeType: () => `direct.${req.mode}`,
+            // The delegate's running total, read once the assets are stored, as the
+            // row always recorded it: a fal-style delegate tracks nothing and the
+            // estimate wins. A test double may be a bare object; only a real
+            // provider tracks.
+            receiptAfterPersist: () => {
+                if (!metering)
+                    return null;
+                const tracked = typeof provider.getTotalCost === "function"
+                    ? provider.getTotalCost()
+                    : 0;
+                const stated = metering.statedCost(tracked);
+                return stated ? { cost: stated } : null;
+            },
+            onGenerationId: (id) => metering?.generationIds.push(id)
+        });
+        // Entity mentions in the prompt (`entity://<id>`, written by @-mention
+        // pickers) expand against the library here: name inline, descriptor into
+        // a Consistency references block, reference image routed into the
+        // generation inputs below — the same rule node prompts get through
+        // mapPromptAssetsToInputs. A mention that resolves to no entity drops.
+        const { prompt, referenceImages } = videoEditReferences
+            ? { prompt: videoEditReferences.prompt, referenceImages: [] }
+            : await expandEntitiesForGeneration(req.prompt, entityRefResolver(userId));
+        const entityImageBytes = await resolveEntityReferenceImages(userId, referenceImages);
+        // Image modes are pixel-addressed on several providers (GPT Image's
+        // `size`): derive explicit dimensions from the resolution tier + aspect
+        // ratio when the caller sent none — the same numbers the generation nodes
+        // always computed. Caller-supplied pixels win.
+        const imageSize = req.mode === "image" || req.mode === "image_edit"
+            ? (resolveImageSize(req.resolution, req.aspectRatio) ?? undefined)
+            : undefined;
+        const width = req.width ?? imageSize?.width;
+        const height = req.height ?? imageSize?.height;
+        if (req.mode === "video") {
+            const videoModel = {
+                id: req.model,
+                name: req.model,
+                provider: req.provider
+            };
+            const videoParams = {
+                prompt,
+                aspect_ratio: req.aspectRatio ?? null,
+                resolution: req.resolution ?? null,
+                duration_seconds: req.durationSeconds ?? null
+            };
+            let generated;
+            if (req.capability === "reference_to_video") {
+                const referenceImages = await resolveReferenceAssets(userId, req.referenceImages ?? [], "image");
+                const referenceVideos = await resolveReferenceAssets(userId, req.referenceVideos ?? [], "video");
+                generated = await generate("reference_to_video", {
+                    ...videoParams,
+                    reference_images: referenceImages,
+                    reference_videos: referenceVideos,
+                    use_reference_video_audio: req.useReferenceVideoAudio
+                }, { mime: "video/mp4" }, (abort) => provider.referenceToVideo({ images: referenceImages, videos: referenceVideos }, {
+                    model: videoModel,
+                    prompt,
+                    durationSeconds: req.durationSeconds ?? null,
+                    aspectRatio: req.aspectRatio ?? null,
+                    resolution: req.resolution ?? null,
+                    useReferenceVideoAudio: req.useReferenceVideoAudio,
+                    signal: abort
+                }));
+            }
+            else if (req.sourceAssetId) {
+                // A source image turns the request into image-to-video: the image is
+                // the frame the animation starts from.
+                const sourceBytes = await retrieveSourceAssetBytes(userId, req.sourceAssetId);
+                const i2vParams = {
+                    model: videoModel,
+                    prompt,
+                    aspectRatio: req.aspectRatio ?? null,
+                    resolution: req.resolution ?? null,
+                    durationSeconds: req.durationSeconds ?? null
+                };
+                generated = await generate("image_to_video", { ...videoParams, images: [sourceBytes] }, { mime: "video/mp4" }, (abort) => provider.imageToVideo(sourceBytes, { ...i2vParams, signal: abort }));
+            }
+            else {
+                const params = {
+                    model: videoModel,
+                    prompt,
+                    durationSeconds: req.durationSeconds ?? null
+                };
+                generated = await generate("text_to_video", videoParams, { mime: "video/mp4" }, (abort) => provider.textToVideo({ ...params, signal: abort }));
+            }
+            const assetId = seamAssetId(generated) ??
+                (await storeAsset(generated.output, "video/mp4", "mp4"));
+            return { asset_ids: [assetId] };
+        }
+        if (req.mode === "video_edit" || req.mode === "video_extend") {
+            if (!req.sourceAssetId) {
+                throw new Error(`source_asset_id is required for ${req.mode}`);
+            }
+            const sourceBytes = await retrieveSourceAssetBytes(userId, req.sourceAssetId);
+            const editSource = req.sourceContext
+                ? await trimVideoWindow(sourceBytes, req.sourceContext.sourceStartMs, req.sourceContext.sourceEndMs)
+                : sourceBytes;
+            const videoModel = {
+                id: req.model,
+                name: req.model,
+                provider: req.provider
+            };
+            if (req.mode === "video_extend") {
+                if (!req.extensionMode || !req.durationSeconds) {
+                    throw new Error("Extension mode and duration are required.");
+                }
+                const params = {
+                    model: videoModel,
+                    prompt,
+                    mode: req.extensionMode,
+                    durationSeconds: req.durationSeconds
+                };
+                const generated = await generate("extend_video", {
+                    prompt,
+                    video: editSource,
+                    mode: params.mode,
+                    duration_seconds: params.durationSeconds,
+                    source_context: req.sourceContext
+                }, { mime: "video/mp4" }, () => provider.extendVideo(editSource, params));
+                const assetId = seamAssetId(generated) ??
+                    (await storeAsset(generated.output, "video/mp4", "mp4"));
+                return { asset_ids: [assetId] };
+            }
+            const editParams = {
+                prompt,
+                strength: req.strength ?? null,
+                duration_seconds: req.sourceContext
+                    ? req.sourceContext.timelineDurationMs / 1000
+                    : (req.durationSeconds ?? null),
+                resolution: req.resolution ?? null,
+                video: editSource
+            };
+            if (req.sourceContext) {
+                editParams.source_context = req.sourceContext;
+            }
+            const references = videoEditReferences?.assetIds.length
+                ? {
+                    referenceImages: videoEditReferences.images,
+                    referenceAssetIds: videoEditReferences.assetIds
+                }
+                : {};
+            if (videoEditReferences?.assetIds.length) {
+                editParams.reference_images = videoEditReferences.images;
+                editParams.reference_asset_ids = videoEditReferences.assetIds;
+            }
+            const referenceProvenance = {
+                referenceAssetIds: videoEditReferences?.assetIds ?? [],
+                entityIds: videoEditReferences?.entityIds ?? []
+            };
+            editParams.entity_ids = referenceProvenance.entityIds;
+            editParams.media_edit_references = referenceProvenance;
+            const generated = await generate("video_to_video", editParams, { mime: "video/mp4" }, (signal) => provider.videoToVideo(editSource, {
+                ...references,
+                signal,
+                model: videoModel,
+                prompt,
+                strength: req.strength ?? null,
+                durationSeconds: req.sourceContext
+                    ? req.sourceContext.timelineDurationMs / 1000
+                    : (req.durationSeconds ?? null),
+                resolution: req.resolution ?? null
+            }));
+            const assetId = seamAssetId(generated) ??
+                (await storeAsset(generated.output, "video/mp4", "mp4"));
+            const result = { asset_ids: [assetId] };
+            if (referenceProvenance.referenceAssetIds.length || referenceProvenance.entityIds.length) {
+                result.media_edit_references = referenceProvenance;
+            }
+            return result;
+        }
+        if (req.mode === "audio" || req.mode === "music") {
+            const supportedFormats = new Set([
+                "mp3",
+                "wav",
+                "flac",
+                "ogg",
+                "aac",
+                "pcm"
+            ]);
+            const requestedFormat = req.audioFormat && supportedFormats.has(req.audioFormat)
+                ? req.audioFormat
+                : null;
+            const audioGenerationId = randomUUID();
+            const generated = await generate(req.mode === "music" ? "text_to_music" : "text_to_speech", req.mode === "music"
+                ? { prompt, duration_seconds: req.durationSeconds }
+                : {
+                    text: prompt,
+                    voice: req.voice,
+                    speed: req.speed,
+                    language: req.language,
+                    instructions: req.instructions,
+                    audio_format: requestedFormat
+                }, null, async () => {
+                const speech = req.mode === "music"
+                    ? await generateMusicBytes(provider, {
+                        model: {
+                            id: req.model,
+                            name: req.model,
+                            provider: req.provider
+                        },
+                        prompt,
+                        durationSeconds: req.durationSeconds
+                    })
+                    : await generateSpeechBytes(provider, {
+                        text: prompt,
+                        model: req.model,
+                        voice: req.voice,
+                        speed: req.speed,
+                        language: req.language,
+                        instructions: req.instructions,
+                        audioFormat: requestedFormat
+                    });
+                if (!speech)
+                    throw new Error("Provider returned no audio data");
+                return storeAsset(speech.bytes, speech.mimeType, speech.ext);
+            }, audioGenerationId);
+            await linkGenerationAssets([audioGenerationId], [generated.output]);
+            return { asset_ids: [generated.output] };
+        }
+        const imageModel = {
+            id: req.model,
+            name: req.model,
+            provider: req.provider
+        };
+        let generated;
+        if (req.mode === "image" && entityImageBytes.length > 0) {
+            // A mentioned entity carries a reference image: the generation becomes
+            // an edit against those images, mirroring how node prompts with
+            // entity images route through ImageToImage. The provider throws when
+            // the chosen model cannot take input images.
+            const params = {
+                model: imageModel,
+                prompt,
+                targetWidth: width ?? null,
+                targetHeight: height ?? null,
+                aspectRatio: req.aspectRatio ?? null,
+                resolution: req.resolution ?? null,
+                strength: req.strength ?? null,
+                numInferenceSteps: req.numInferenceSteps ?? null
+            };
+            generated = await generate("image_to_image", {
+                prompt,
+                target_width: width ?? null,
+                target_height: height ?? null,
+                aspect_ratio: req.aspectRatio ?? null,
+                resolution: req.resolution ?? null,
+                strength: req.strength ?? null,
+                num_images: variations,
+                images: entityImageBytes
+            }, {}, (abort) => provider.imageToImages(entityImageBytes, { ...params, signal: abort }, variations));
+        }
+        else if (req.mode === "image") {
+            await assertTextToImageModel(provider, req.model);
+            const params = {
+                model: imageModel,
+                prompt,
+                width,
+                height,
+                aspectRatio: req.aspectRatio ?? null,
+                resolution: req.resolution ?? null,
+                seed: req.seed ?? null
+            };
+            generated = await generate("text_to_image", {
+                prompt,
+                width,
+                height,
+                aspect_ratio: req.aspectRatio ?? null,
+                resolution: req.resolution ?? null,
+                seed: req.seed ?? null,
+                num_images: variations
+            }, {}, (abort) => provider.textToImages({ ...params, signal: abort }, variations));
+        }
+        else if (req.mode === "inpaint") {
+            if (!req.sourceAssetId) {
+                throw new Error("source_asset_id is required for inpaint");
+            }
+            if (!req.maskAssetId) {
+                throw new Error("mask_asset_id is required for inpaint");
+            }
+            const adapter = getAssetAdapter();
+            const [sourceAsset, maskAsset] = await Promise.all([
+                Asset.find(userId, req.sourceAssetId),
+                Asset.find(userId, req.maskAssetId)
+            ]);
+            if (!sourceAsset)
+                throw new Error(`Source asset not found: ${req.sourceAssetId}`);
+            if (!maskAsset)
+                throw new Error(`Mask asset not found: ${req.maskAssetId}`);
+            const [sourceBytes, maskBytes] = await Promise.all([
+                retrieveAssetBytes(adapter, userId, req.sourceAssetId, sourceAsset.content_type),
+                retrieveAssetBytes(adapter, userId, req.maskAssetId, maskAsset.content_type)
+            ]);
+            if (!sourceBytes)
+                throw new Error(`Source asset bytes not found: ${req.sourceAssetId}`);
+            if (!maskBytes)
+                throw new Error(`Mask asset bytes not found: ${req.maskAssetId}`);
+            const params = {
+                model: imageModel,
+                prompt,
+                targetWidth: req.width ?? null,
+                targetHeight: req.height ?? null,
+                aspectRatio: req.aspectRatio ?? null,
+                resolution: req.resolution ?? null,
+                strength: req.strength ?? null,
+                numInferenceSteps: req.numInferenceSteps ?? null,
+                mask: maskBytes
+            };
+            generated = await generate("inpainting", {
+                prompt,
+                target_width: req.width ?? null,
+                target_height: req.height ?? null,
+                strength: req.strength ?? null,
+                num_images: variations,
+                images: [sourceBytes],
+                mask: maskBytes
+            }, {}, (abort) => provider.inpaintImages([sourceBytes], { ...params, signal: abort }, variations));
+        }
+        else {
+            if (!req.sourceAssetId) {
+                throw new Error("source_asset_id is required for image_edit");
+            }
+            const sourceAsset = await Asset.find(userId, req.sourceAssetId);
+            if (!sourceAsset) {
+                throw new Error(`Source asset not found: ${req.sourceAssetId}`);
+            }
+            const sourceBytes = await retrieveAssetBytes(getAssetAdapter(), userId, req.sourceAssetId, sourceAsset.content_type);
+            if (!sourceBytes) {
+                throw new Error(`Source asset bytes not found: ${req.sourceAssetId}`);
+            }
+            const params = {
+                model: imageModel,
+                prompt,
+                targetWidth: width ?? null,
+                targetHeight: height ?? null,
+                aspectRatio: req.aspectRatio ?? null,
+                resolution: req.resolution ?? null,
+                strength: req.strength ?? null,
+                numInferenceSteps: req.numInferenceSteps ?? null
+            };
+            generated = await generate("image_to_image", {
+                prompt,
+                target_width: width ?? null,
+                target_height: height ?? null,
+                aspect_ratio: req.aspectRatio ?? null,
+                resolution: req.resolution ?? null,
+                strength: req.strength ?? null,
+                num_inference_steps: req.numInferenceSteps ?? null,
+                num_images: variations,
+                images: [sourceBytes, ...entityImageBytes]
+            }, {}, (abort) => provider.imageToImages([sourceBytes, ...entityImageBytes], { ...params, signal: abort }, variations));
+        }
+        const assetIds = [];
+        for (const [index, bytes] of generated.output.entries()) {
+            const mimeType = detectImageMime(bytes);
+            assetIds.push(seamAssetId(generated, index) ??
+                (await storeAsset(bytes, mimeType, IMAGE_MIME_TO_EXT[mimeType] ?? "png")));
+        }
+        return { asset_ids: assetIds };
+    }
+    /**
+     * Transcribe a stored audio asset to word-level caption timing. Mirrors the
+     * provider path used by the ASR node but skips the workflow machinery — the
+     * caller (Studio transcript beats) wants `{ word, startMs, endMs }[]` back in
+     * one shot. Timestamps are returned in milliseconds relative to the start of
+     * the audio.
+     */
+    async runDirectTranscription(req) {
+        if (!this.session.resolveProvider) {
+            throw new Error("No provider resolver configured");
+        }
+        if (!req.model) {
+            throw new Error("model is required");
+        }
+        if (!req.assetId) {
+            throw new Error("asset_id is required");
+        }
+        const userId = this.session.requireUserId();
+        if (req.provider === "nodetool") {
+            const creditDecision = await admitSpend(userId, 0, [req.model]);
+            if (!creditDecision.allowed) {
+                throw new Error(creditDecision.reason);
+            }
+        }
+        const asset = await Asset.find(userId, req.assetId);
+        if (!asset) {
+            throw new Error(`Audio asset not found: ${req.assetId}`);
+        }
+        const bytes = await retrieveAssetBytes(getAssetAdapter(), userId, req.assetId, asset.content_type);
+        if (!bytes) {
+            throw new Error(`Audio asset bytes not found: ${req.assetId}`);
+        }
+        const provider = await this.session.resolveProvider(req.provider, userId);
+        const result = await provider.automaticSpeechRecognition({
+            audio: bytes,
+            model: req.model,
+            language: req.language,
+            word_timestamps: true
+        });
+        // Metered provider: record what the delegate tracked so the spend lands
+        // in the ledger the balance is computed from. Best-effort, never throws.
+        if (req.provider === "nodetool" && provider.getTotalCost() > 0) {
+            try {
+                await Prediction.create({
+                    user_id: userId,
+                    provider: "nodetool",
+                    model: req.model,
+                    node_type: "direct.transcription",
+                    cost: provider.getTotalCost(),
+                    currency: "USD",
+                    workflow_id: null,
+                    node_id: "",
+                    status: "completed"
+                });
+            }
+            catch (err) {
+                this.session.logError("direct transcription cost persistence failed", err);
+            }
+        }
+        const words = (result.chunks ?? [])
+            .map((chunk) => ({
+            word: chunk.text.trim(),
+            startMs: Math.round(chunk.timestamp[0] * 1000),
+            endMs: Math.round(chunk.timestamp[1] * 1000)
+        }))
+            .filter((w) => w.word.length > 0);
+        return { text: result.text, words };
+    }
+}
+/** Resolve protection from authorized persisted documents, never client-reported policy. */
+async function assertTimelineGenerationAllowed(userId, req, resolvedReferenceIds = []) {
+    const context = req.timelineContext ?? (req.sourceContext ? { sequenceId: req.sourceContext.sequenceId, sourceClipId: req.sourceContext.clipId } : undefined);
+    if (!context) {
+        return;
+    }
+    const sequenceIds = await findFinishResourceIds("timeline", context.sequenceId, userId, req.projectId ?? undefined);
+    if (!req.timelineContext && sequenceIds.length === 0) {
+        return;
+    }
+    if (sequenceIds.length !== 1) {
+        throw new Error("Timeline generation context was not found uniquely in the caller's project.");
+    }
+    const sequence = await TimelineSequence.findById(sequenceIds[0]);
+    if (!sequence || sequence.user_id !== userId || (req.projectId && req.projectId !== sequence.project_id)) {
+        throw new Error("Timeline generation context was not found in the caller's project.");
+    }
+    const clips = sequence.toDocument().clips;
+    const resolveClip = (id) => {
+        const matches = clips.filter(clip => clip.id === id || (/^[a-f0-9]{12}$/.test(id) && clip.id.startsWith(id)));
+        if (matches.length > 1) {
+            throw new Error("Timeline generation clip prefix is ambiguous.");
+        }
+        return matches[0];
+    };
+    const source = context.sourceClipId ? resolveClip(context.sourceClipId) : undefined;
+    if (context.sourceClipId && (!source || source.currentAssetId !== req.sourceAssetId)) {
+        throw new Error("Timeline generation source changed or was not found. Save and retry.");
+    }
+    const target = context.targetClipId ? resolveClip(context.targetClipId) : undefined;
+    const referencedAssets = new Set([
+        ...(req.referenceAssetIds ?? []),
+        ...resolvedReferenceIds,
+        ...(req.referenceImages ?? []).map(ref => referenceAssetId(ref, "image")),
+        ...(req.referenceVideos ?? []).map(ref => referenceAssetId(ref, "video"))
+    ]);
+    const referencedClips = clips.filter(clip => clip.currentAssetId && referencedAssets.has(clip.currentAssetId));
+    for (const clip of [source, target, ...referencedClips]) {
+        if (!clip?.storyboardBoardId) {
+            continue;
+        }
+        const capability = req.mode === "video" ? (req.capability ?? (req.sourceAssetId ? "image_to_video" : "text_to_video"))
+            : req.mode === "video_edit" || req.mode === "video_extend" ? "video_to_video"
+                : req.mode === "image_edit" || req.mode === "inpaint" ? "image_to_image" : "text_to_image";
+        await assertStoryboardClipGenerationAllowed(userId, sequence.project_id, clip, capability);
+    }
+}
+//# sourceMappingURL=inference.js.map