@nodetool-ai/runtime 0.7.0-rc.23 → 0.8.2

package/dist/prompt-asset-refs.js

@@ -0,0 +1,751 @@
+/**
+ * Inline asset references in prompt text.
+ *
+ * Prompt-style surfaces (the Prompt composer, chat) let users mention an asset
+ * with `@`, which encodes as an `asset://<id>.<ext>` token inside the plain
+ * prompt string. Text-generation tasks expand these tokens into multimodal
+ * image / audio message blocks before the provider call. This module is the
+ * shared, provider-agnostic parser so other task types (e.g. image-to-image)
+ * can pull the same referenced media out of a prompt and route it to a typed
+ * input instead of leaving a dangling `asset://` URI in the text.
+ *
+ * The low-level parsing is resolution-free — callers hand the verbatim
+ * `asset://` URI to {@link loadMediaRefBytes} / `resolveMessageMediaUris`,
+ * which dereference it to bytes / a data URI just before it is consumed. The
+ * higher-level {@link mapPromptAssetsToInputs} does resolve, so provider nodes
+ * (FAL / KIE / Replicate / image-to-image) can route a mentioned image / audio
+ * / video into a typed asset input with bytes already in hand.
+ *
+ * A mentioned plain-text document (`.md`, `.txt`, `.csv`, `.json`, …) has no
+ * typed input to route to — {@link inlineTextAssetRefs} substitutes its decoded
+ * contents into the prompt text in place of the token.
+ */
+import { encodeBase64, loadMediaRefBytes } from "./media-ref-bytes.js";
+import { isCallable, isObjectLike, isString } from "@nodetool-ai/protocol";
+/** Matches an inline `asset://<id>.<ext>` token (greedy over the URI charset). */
+const ASSET_URI_RE = /asset:\/\/[A-Za-z0-9._~\-/]+/g;
+const IMAGE_EXT_MIME = {
+    png: "image/png",
+    jpg: "image/jpeg",
+    jpeg: "image/jpeg",
+    gif: "image/gif",
+    webp: "image/webp",
+    bmp: "image/bmp",
+    svg: "image/svg+xml"
+};
+const AUDIO_EXT_MIME = {
+    mp3: "audio/mpeg",
+    mpeg: "audio/mpeg",
+    wav: "audio/wav",
+    ogg: "audio/ogg",
+    m4a: "audio/mp4",
+    aac: "audio/aac",
+    flac: "audio/flac",
+    opus: "audio/opus"
+};
+const VIDEO_EXT_MIME = {
+    mp4: "video/mp4",
+    webm: "video/webm",
+    mov: "video/quicktime",
+    mkv: "video/x-matroska",
+    avi: "video/x-msvideo"
+};
+/**
+ * Plain-text document extensions. Unlike media, a text mention has no typed
+ * input to route to — its decoded content is inlined into the prompt string in
+ * place of the token (see {@link inlineTextAssetRefs}).
+ */
+const TEXT_EXT_MIME = {
+    txt: "text/plain",
+    text: "text/plain",
+    md: "text/markdown",
+    markdown: "text/markdown",
+    csv: "text/csv",
+    tsv: "text/tab-separated-values",
+    json: "application/json",
+    jsonl: "application/json",
+    ndjson: "application/json",
+    xml: "application/xml",
+    yaml: "application/yaml",
+    yml: "application/yaml",
+    html: "text/html",
+    htm: "text/html",
+    log: "text/plain",
+    rst: "text/x-rst",
+    ini: "text/plain",
+    toml: "text/plain",
+    srt: "text/plain",
+    vtt: "text/vtt"
+};
+/** Lowercased extension of an `asset://<id>.<ext>` token (without the dot). */
+function tokenExt(token) {
+    const noScheme = token.slice("asset://".length);
+    const primary = noScheme.split(/[?#]/)[0];
+    return (primary.split(".").pop() ?? "").toLowerCase();
+}
+/**
+ * Classify an `asset://<id>.<ext>` token by its extension. Image, audio and
+ * video are provider-consumable media (reference-to-video / lipsync models take
+ * a video clip); everything else (text, unknown) returns null so the reference
+ * is left for the text-inlining pass or kept as literal text in the prompt.
+ */
+export function classifyAssetToken(token) {
+    if (!token.startsWith("asset://"))
+        return null;
+    const ext = tokenExt(token);
+    // Object.hasOwn, not `in`: `in` walks the prototype chain, so an extension of
+    // "__proto__"/"toString"/"constructor" would match an inherited key and
+    // return a non-string (object/function) mime, fabricating a bogus media ref.
+    if (Object.hasOwn(IMAGE_EXT_MIME, ext))
+        return { kind: "image", mime: IMAGE_EXT_MIME[ext] };
+    if (Object.hasOwn(AUDIO_EXT_MIME, ext))
+        return { kind: "audio", mime: AUDIO_EXT_MIME[ext] };
+    if (Object.hasOwn(VIDEO_EXT_MIME, ext))
+        return { kind: "video", mime: VIDEO_EXT_MIME[ext] };
+    return null;
+}
+/**
+ * Classify an `asset://<id>.<ext>` token as a plain-text document. Returns null
+ * for non-text tokens (media, folders, unknown) so they are left untouched by
+ * the text-inlining pass.
+ */
+export function classifyTextToken(token) {
+    if (!token.startsWith("asset://"))
+        return null;
+    const ext = tokenExt(token);
+    return Object.hasOwn(TEXT_EXT_MIME, ext)
+        ? { mime: TEXT_EXT_MIME[ext] }
+        : null;
+}
+/**
+ * Pick a media extension for a content type so a stored asset can be turned
+ * back into a classifiable `asset://<id>.<ext>` token. Returns null for content
+ * types that aren't a recognized image/audio/video kind (folders, documents).
+ */
+function extForContentType(contentType) {
+    const ct = (contentType ?? "").toLowerCase().split(";")[0].trim();
+    const slash = ct.indexOf("/");
+    if (slash <= 0)
+        return null;
+    const top = ct.slice(0, slash);
+    let sub = ct.slice(slash + 1).split("+")[0];
+    if (ct === "audio/mpeg" || ct === "audio/mp3")
+        sub = "mp3";
+    else if (ct === "audio/mp4")
+        sub = "m4a";
+    else if (ct === "video/quicktime")
+        sub = "mov";
+    if (top === "image" && Object.hasOwn(IMAGE_EXT_MIME, sub))
+        return sub;
+    if (top === "audio" && Object.hasOwn(AUDIO_EXT_MIME, sub))
+        return sub;
+    if (top === "video" && Object.hasOwn(VIDEO_EXT_MIME, sub))
+        return sub;
+    return null;
+}
+/**
+ * Reverse of the ext→mime tables: the extension whose mime matches. Tries the
+ * media tables first (via {@link extForContentType}, which also normalizes
+ * `+suffix` and audio/video aliases), then falls back to a text-document match.
+ * Returns null for an unrecognized mime.
+ */
+function extForMime(mime) {
+    const viaMedia = extForContentType(mime);
+    if (viaMedia)
+        return viaMedia;
+    const ct = (mime ?? "").toLowerCase().split(";")[0].trim();
+    for (const [ext, m] of Object.entries(TEXT_EXT_MIME)) {
+        if (m === ct)
+            return ext;
+    }
+    return null;
+}
+/** Fallback extension per ref `type` when the mime is missing/unrecognized. */
+const DEFAULT_EXT_BY_KIND = {
+    image: "png",
+    audio: "mp3",
+    video: "mp4",
+    document: "txt",
+    text: "txt"
+};
+/** Pick an extension for a ref from its mime, else its declared `type`. */
+function extForRef(ref) {
+    const mime = isString(ref.mimeType)
+        ? ref.mimeType
+        : isString(ref.content_type)
+            ? ref.content_type
+            : "";
+    const byMime = mime ? extForMime(mime) : null;
+    if (byMime)
+        return byMime;
+    const kind = isString(ref.type) ? ref.type.toLowerCase() : "";
+    return Object.hasOwn(DEFAULT_EXT_BY_KIND, kind)
+        ? DEFAULT_EXT_BY_KIND[kind]
+        : null;
+}
+/**
+ * Convert a media / document ref — the `{ type, uri, asset_id, mimeType }` shape
+ * image / audio / video / document outputs carry — into the inline
+ * `asset://<id>.<ext>` token the prompt-expansion machinery understands.
+ *
+ * This lets an asset wired into a Prompt node's `{{ variable }}` flow through
+ * the same path as an inline `@`-mention: once rendered, the token is picked up
+ * by {@link expandAssetReferences} (LLM message blocks), `mapPromptAssetsToInputs`
+ * (typed provider inputs), or {@link inlineTextAssetRefs} (text documents),
+ * instead of stringifying to `"[object Object]"`.
+ *
+ * Returns null for values that aren't asset refs (strings, numbers, plain
+ * objects with no `uri`/`asset_id`) so callers fall back to normal stringify.
+ * A ref pointing only at a non-asset uri (http / data / file) yields that uri
+ * verbatim — it won't route as media, but it still beats `"[object Object]"`.
+ */
+export function assetRefToPromptToken(value) {
+    if (!isObjectLike(value))
+        return null;
+    const ref = value;
+    const uri = isString(ref.uri) ? ref.uri.trim() : "";
+    const assetId = isString(ref.asset_id) ? ref.asset_id.trim() : "";
+    if (uri.startsWith("asset://")) {
+        // Drop any query/hash; keep the id (and any existing extension) intact.
+        const id = uri.slice("asset://".length).split(/[?#]/)[0];
+        if (!id)
+            return null;
+        if (id.includes("."))
+            return `asset://${id}`;
+        const ext = extForRef(ref);
+        return ext ? `asset://${id}.${ext}` : `asset://${id}`;
+    }
+    if (assetId) {
+        const ext = extForRef(ref);
+        return ext ? `asset://${assetId}.${ext}` : `asset://${assetId}`;
+    }
+    // Asset-less ref that still points somewhere resolvable.
+    if (uri)
+        return uri;
+    return null;
+}
+/**
+ * Trim trailing dots (sentence punctuation) off a token. A character loop, not
+ * `/\.+$/` — that regex backtracks polynomially on long dot runs, which CodeQL
+ * flags as a ReDoS risk on prompt-derived input.
+ */
+function trimTrailingDots(token) {
+    let end = token.length;
+    while (end > 0 && token[end - 1] === ".") {
+        end--;
+    }
+    return token.slice(0, end);
+}
+/**
+ * Collapse horizontal-whitespace gaps left behind by token substitution:
+ * runs of 2+ spaces/tabs become one space, horizontal whitespace before a
+ * newline is dropped, and the result is trimmed. A single pass instead of
+ * `/[ \t]{2,}/` + `/[ \t]+\n/` — those backtrack polynomially on long
+ * space/tab runs (a ReDoS risk on prompt-derived input).
+ */
+function collapseGapWhitespace(text) {
+    let out = "";
+    let run = ""; // pending run of spaces/tabs
+    for (const ch of text) {
+        if (ch === " " || ch === "\t") {
+            run += ch;
+            continue;
+        }
+        if (ch === "\n") {
+            // Drop horizontal whitespace before a newline.
+            run = "";
+            out += "\n";
+            continue;
+        }
+        if (run) {
+            out += run.length > 1 ? " " : run;
+            run = "";
+        }
+        out += ch;
+    }
+    if (run) {
+        out += run.length > 1 ? " " : run;
+    }
+    return out.trim();
+}
+/**
+ * Scan a prompt for `asset://` tokens in source order, yielding each token with
+ * its source offset. Trailing dots are treated as sentence punctuation (not
+ * part of the extension), so `asset://a.png.` yields `asset://a.png` and leaves
+ * the period in the surrounding text. Classification is left to the caller.
+ */
+function scanAssetTokens(prompt) {
+    const out = [];
+    ASSET_URI_RE.lastIndex = 0;
+    let match;
+    while ((match = ASSET_URI_RE.exec(prompt)) !== null) {
+        const token = trimTrailingDots(match[0]);
+        out.push({ token, index: match.index });
+    }
+    return out;
+}
+/**
+ * Find every classifiable media reference in a prompt, in source order.
+ *
+ * Unclassifiable tokens (no/unknown extension, e.g. `asset://doc.txt`) are
+ * skipped and remain literal text — text documents are handled separately by
+ * {@link findTextAssetRefs} / {@link inlineTextAssetRefs}.
+ */
+export function findAssetRefs(prompt) {
+    const refs = [];
+    for (const { token, index } of scanAssetTokens(prompt)) {
+        const classification = classifyAssetToken(token);
+        if (!classification)
+            continue;
+        refs.push({
+            uri: token,
+            kind: classification.kind,
+            mime: classification.mime,
+            index,
+            length: token.length
+        });
+    }
+    return refs;
+}
+/** Find every plain-text document reference in a prompt, in source order. */
+export function findTextAssetRefs(prompt) {
+    const refs = [];
+    for (const { token, index } of scanAssetTokens(prompt)) {
+        const classification = classifyTextToken(token);
+        if (!classification)
+            continue;
+        refs.push({ uri: token, mime: classification.mime, index, length: token.length });
+    }
+    return refs;
+}
+/**
+ * Replace each plain-text document mention in `prompt` with the asset's decoded
+ * UTF-8 contents, inline. A mention that can't be resolved to bytes is left as
+ * its literal token (never silently dropped). Media mentions are untouched.
+ */
+export async function inlineTextAssetRefs(prompt, context) {
+    if (!context || !prompt.includes("asset://"))
+        return prompt;
+    const refs = findTextAssetRefs(prompt);
+    if (refs.length === 0)
+        return prompt;
+    const decoder = new TextDecoder();
+    const contents = await Promise.all(refs.map(async (ref) => {
+        const bytes = await loadMediaRefBytes({ type: "text", uri: ref.uri }, context);
+        // A resolved-but-empty asset must inline as "" (not fall back to the
+        // literal token); only a failed resolution (null bytes) keeps the token.
+        return bytes ? decoder.decode(bytes) : null;
+    }));
+    let result = "";
+    let cursor = 0;
+    refs.forEach((ref, i) => {
+        if (ref.index < cursor)
+            return;
+        result += prompt.slice(cursor, ref.index);
+        // Inline the decoded text, or keep the literal token when resolution failed.
+        result += contents[i] ?? prompt.slice(ref.index, ref.index + ref.length);
+        cursor = ref.index + ref.length;
+    });
+    result += prompt.slice(cursor);
+    return result;
+}
+/** Convenience: only the image references in a prompt. */
+export function findImageAssetRefs(prompt) {
+    return findAssetRefs(prompt).filter((ref) => ref.kind === "image");
+}
+/**
+ * Split a prompt containing inline `asset://<id>.<ext>` references into a
+ * multimodal content array: text segments interleaved with image / audio
+ * blocks that carry the `asset://` URI verbatim. The blocks are NOT resolved
+ * here — {@link ProcessingContext.resolveMessageMediaUris} dereferences the
+ * URIs to data URIs just before the provider call. Tokens that aren't a
+ * supported chat media type (video, unknown) stay as literal text so the
+ * model still sees the mention.
+ */
+export function expandAssetReferences(prompt) {
+    const refs = findAssetRefs(prompt);
+    if (refs.length === 0) {
+        return [{ type: "text", text: prompt }];
+    }
+    const parts = [];
+    const pushText = (text) => {
+        if (text)
+            parts.push({ type: "text", text });
+    };
+    let cursor = 0;
+    for (const ref of refs) {
+        pushText(prompt.slice(cursor, ref.index));
+        if (ref.kind === "image") {
+            parts.push({
+                type: "image_url",
+                image: { uri: ref.uri, mimeType: ref.mime }
+            });
+            // Keep the literal token next to the block, the way the video branch
+            // below does. The block itself resolves to a data URI before the
+            // provider call, so without the token the model sees pixels it has no
+            // handle for and cannot name the image in a later `edit_image` call.
+            pushText(ref.uri);
+        }
+        else if (ref.kind === "audio") {
+            parts.push({
+                type: "audio",
+                audio: { uri: ref.uri, mimeType: ref.mime }
+            });
+            pushText(ref.uri);
+        }
+        else {
+            // Video (and any other non-chat media): keep the mention as literal text
+            // so the model still sees the reference rather than a mislabeled block.
+            pushText(ref.uri);
+        }
+        cursor = ref.index + ref.length;
+    }
+    pushText(prompt.slice(cursor));
+    return parts.length > 0 ? parts : [{ type: "text", text: prompt }];
+}
+/**
+ * Rewrite the given references inside the prompt, substituting each with the
+ * string `replace(ref)` returns (return `""` to drop it). Refs must come from
+ * {@link findAssetRefs} on the same string (they carry the source offsets).
+ * Horizontal whitespace left behind is collapsed so the result reads cleanly;
+ * newlines are preserved.
+ */
+function replaceAssetRefs(prompt, refs, replace) {
+    if (refs.length === 0)
+        return prompt;
+    let result = "";
+    let cursor = 0;
+    for (const ref of refs) {
+        if (ref.index < cursor)
+            continue;
+        result += prompt.slice(cursor, ref.index);
+        result += replace(ref);
+        cursor = ref.index + ref.length;
+    }
+    result += prompt.slice(cursor);
+    return collapseGapWhitespace(result);
+}
+/**
+ * Remove the given references from the prompt, collapsing the horizontal
+ * whitespace left behind so the remaining text reads as a clean instruction.
+ * Refs must come from {@link findAssetRefs} on the same string. Newlines are
+ * preserved.
+ */
+export function stripAssetRefs(prompt, refs) {
+    return replaceAssetRefs(prompt, refs, () => "");
+}
+/**
+ * Expand any folder mention in `text` into the `asset://<id>.<ext>` tokens of
+ * its members of an accepted kind, recursively. A folder mention is a bare
+ * `asset://<id>` (no extension) that {@link ProcessingContext.listFolderAssets}
+ * resolves to a folder; non-folder bare tokens and ordinary media tokens are
+ * left untouched. The expanded tokens then flow through the normal matcher, so
+ * a referenced folder fills as many of the node's media inputs as it can.
+ */
+async function expandFolderRefs(text, context, acceptedKinds) {
+    if (!context || !text.includes("asset://"))
+        return text;
+    // Bare `asset://<id>` tokens (no extension) are folder candidates.
+    const candidates = [];
+    ASSET_URI_RE.lastIndex = 0;
+    let match;
+    while ((match = ASSET_URI_RE.exec(text)) !== null) {
+        const token = match[0];
+        const id = token.slice("asset://".length);
+        if (id.includes("."))
+            continue; // has an extension → a media ref, not a folder
+        candidates.push({ index: match.index, length: token.length, id });
+    }
+    if (candidates.length === 0)
+        return text;
+    const replacements = new Map();
+    for (const candidate of candidates) {
+        const entries = await context.listFolderAssets(candidate.id);
+        if (!entries)
+            continue; // not a folder → leave the token alone
+        const uris = [];
+        for (const entry of entries) {
+            const ext = extForContentType(entry.content_type);
+            if (!ext)
+                continue;
+            const classification = classifyAssetToken(`asset://${entry.id}.${ext}`);
+            if (!classification || !acceptedKinds.has(classification.kind))
+                continue;
+            uris.push(`asset://${entry.id}.${ext}`);
+        }
+        replacements.set(candidate.index, uris.join(" "));
+    }
+    if (replacements.size === 0)
+        return text;
+    let result = "";
+    let cursor = 0;
+    for (const candidate of candidates) {
+        const replacement = replacements.get(candidate.index);
+        if (replacement === undefined)
+            continue;
+        result += text.slice(cursor, candidate.index);
+        result += replacement;
+        cursor = candidate.index + candidate.length;
+    }
+    result += text.slice(cursor);
+    return collapseGapWhitespace(result);
+}
+/** Matches an inline `entity://<id>` token (an entity mention from `@`). */
+const ENTITY_URI_RE = /entity:\/\/[A-Za-z0-9._~-]+/g;
+/** Read the entity marker off asset metadata, or null when absent/malformed. */
+function readEntityMarker(metadata) {
+    const raw = metadata?.["nodetool_entity"];
+    if (!isObjectLike(raw))
+        return null;
+    const obj = raw;
+    const name = isString(obj.name) ? obj.name.trim() : "";
+    if (!name)
+        return null;
+    const descriptor = isString(obj.descriptor) ? obj.descriptor.trim() : "";
+    const referenceAssetId = isString(obj.reference_asset_id) ? obj.reference_asset_id.trim() : "";
+    return { name, descriptor, referenceAssetId };
+}
+/**
+ * Expand `entity://<id>` mentions (entities are library assets tagged with a
+ * `nodetool_entity` marker; the `@`-mention pickers encode them as this token).
+ *
+ * Each token is replaced inline with the entity's **name** so the sentence
+ * still reads naturally, and every mentioned entity contributes once to a
+ * trailing `Consistency references:` block carrying its canonical descriptor —
+ * the same shape the Apply Entities node produces. With `includeImageRefs`,
+ * each entity's reference image is appended as an `asset://<id>.<ext>` token so
+ * the normal media-mention machinery routes it into a typed image input.
+ *
+ * Resolution happens here, at generation time, so later edits to an entity's
+ * descriptor or image propagate to every prompt that mentions it. A token that
+ * cannot be resolved (unknown id, no marker, no context) is dropped.
+ */
+export async function expandEntityRefs(text, context, includeImageRefs) {
+    if (!text.includes("entity://"))
+        return text;
+    const tokens = [];
+    ENTITY_URI_RE.lastIndex = 0;
+    let match;
+    while ((match = ENTITY_URI_RE.exec(text)) !== null) {
+        // Trailing dots are sentence punctuation, not part of the id.
+        const token = trimTrailingDots(match[0]);
+        tokens.push({
+            index: match.index,
+            length: token.length,
+            id: token.slice("entity://".length)
+        });
+    }
+    if (tokens.length === 0)
+        return text;
+    // Resolve each unique id once; a mention of the same entity twice reads as
+    // its name twice but contributes a single consistency line / image ref.
+    const resolved = new Map();
+    for (const token of tokens) {
+        if (resolved.has(token.id))
+            continue;
+        const info = isCallable(context?.getAssetInfo)
+            ? await context.getAssetInfo(token.id)
+            : null;
+        const marker = info ? readEntityMarker(info.metadata) : null;
+        if (!info || !marker) {
+            resolved.set(token.id, null);
+            continue;
+        }
+        const image = includeImageRefs && marker.referenceAssetId && marker.referenceAssetId !== info.id
+            ? await context?.getAssetInfo(marker.referenceAssetId)
+            : info;
+        const ext = image?.content_type.startsWith("image/")
+            ? extForContentType(image.content_type)
+            : null;
+        resolved.set(token.id, {
+            marker,
+            imageToken: image && ext ? `asset://${image.id}.${ext}` : null
+        });
+    }
+    let result = "";
+    let cursor = 0;
+    const consistencyLines = [];
+    const imageTokens = [];
+    const contributed = new Set();
+    for (const token of tokens) {
+        result += text.slice(cursor, token.index);
+        cursor = token.index + token.length;
+        const entity = resolved.get(token.id);
+        if (!entity)
+            continue; // unresolvable mention → dropped
+        result += entity.marker.name;
+        if (contributed.has(token.id))
+            continue;
+        contributed.add(token.id);
+        if (entity.marker.descriptor) {
+            consistencyLines.push(`- ${entity.marker.name}: ${entity.marker.descriptor}`);
+        }
+        if (includeImageRefs && entity.imageToken) {
+            imageTokens.push(entity.imageToken);
+        }
+    }
+    result += text.slice(cursor);
+    result = collapseGapWhitespace(result);
+    if (consistencyLines.length > 0) {
+        result += `\n\nConsistency references:\n${consistencyLines.join("\n")}`;
+    }
+    if (imageTokens.length > 0) {
+        result += `\n${imageTokens.join(" ")}`;
+    }
+    return result;
+}
+/**
+ * Expand the entity mentions in a generation prompt and separate what the
+ * provider call needs from it: the seasoned text (names inline + a
+ * Consistency references block) and the entities' reference images as typed
+ * refs — never left in the text as dangling tokens.
+ *
+ * This is the direct-generation counterpart of what node prompts get through
+ * {@link mapPromptAssetsToInputs}: same expansion rule, but returning the
+ * routed images instead of writing them onto node input fields. Image
+ * mentions that were already in the caller's text are left untouched — only
+ * tokens this expansion itself produced are extracted and stripped.
+ */
+export async function expandEntitiesForGeneration(prompt, resolver) {
+    const preExisting = new Set(findImageAssetRefs(prompt).map((r) => r.uri));
+    const expanded = await expandEntityRefs(prompt, resolver, true);
+    if (expanded === prompt) {
+        return { prompt, referenceImages: [] };
+    }
+    const referenceImages = findImageAssetRefs(expanded).filter((ref) => !preExisting.has(ref.uri));
+    return {
+        prompt: referenceImages.length > 0
+            ? stripAssetRefs(expanded, referenceImages).trimEnd()
+            : expanded,
+        referenceImages
+    };
+}
+/**
+ * Map media mentioned inline in a node's text inputs onto its typed media
+ * inputs. This is the generalized form of what text-generation tasks do when
+ * they expand `asset://` references into image / audio message blocks.
+ *
+ * A mention may be a single asset or a folder: folder mentions are expanded
+ * recursively to their member assets first (see {@link expandFolderRefs}), so a
+ * referenced folder fills as many inputs as it can.
+ *
+ * For every media kind the node actually accepts (derived from `assetFields`):
+ *   - mentions of that kind are pulled out of the text inputs, in order;
+ *   - each still-empty input of that kind is filled with the next mention,
+ *     resolved to bytes (`data`) so the provider's normal `data` path uploads
+ *     it — no provider needs to learn the `asset://` scheme;
+ *   - `list[...]` inputs absorb all remaining mentions of their kind;
+ *   - each placed mention is rewritten in the text to the input label it now
+ *     lives in (`image_url`, `reference_image_urls[0]`, …) so the model can
+ *     tell which slot holds which image; mentions that could not be placed
+ *     (an input was already wired, or capacity ran out) are dropped rather
+ *     than left as a dangling URI.
+ *
+ * Plain-text document mentions are inlined into the prompt as their decoded
+ * contents (see {@link inlineTextAssetRefs}). This applies even when the node
+ * has no media inputs at all (e.g. a text-to-image prompt referencing a brief).
+ *
+ * Inputs that already carry a source are left untouched. Returns a flat map of
+ * field name → replacement value (cleaned string for text fields, an
+ * {@link InjectedAssetRef} or array for asset fields); fields that did not
+ * change are absent, so callers do `overrides[name] ?? instance[name]`.
+ */
+export async function mapPromptAssetsToInputs(textFields, assetFields, context) {
+    const overrides = {};
+    const acceptedKinds = new Set(assetFields.map((f) => f.kind));
+    // Expand entity mentions first (they inject descriptor text and, when the
+    // node accepts images, an `asset://` reference-image token), then folder
+    // mentions (only meaningful when the node has media inputs to receive them),
+    // and work off the expanded text. The original value is kept to decide
+    // whether the field actually changed (a folder that expanded to nothing
+    // still rewrites text).
+    const expandedByField = new Map();
+    for (const tf of textFields) {
+        const withEntities = await expandEntityRefs(tf.value, context, acceptedKinds.has("image"));
+        expandedByField.set(tf.name, assetFields.length > 0
+            ? await expandFolderRefs(withEntities, context, acceptedKinds)
+            : withEntities);
+    }
+    // A node with no typed media inputs still inlines plain-text document
+    // mentions into its prompt — that's the only transformation that applies.
+    if (assetFields.length === 0) {
+        for (const tf of textFields) {
+            const inlined = await inlineTextAssetRefs(expandedByField.get(tf.name) ?? tf.value, context);
+            if (inlined !== tf.value)
+                overrides[tf.name] = inlined;
+        }
+        return overrides;
+    }
+    // Refs of accepted kinds, queued per kind in source order across all text
+    // fields, plus the per-field ref list used for relabeling.
+    const queues = {
+        image: [],
+        audio: [],
+        video: []
+    };
+    const refsByField = new Map();
+    for (const tf of textFields) {
+        const refs = findAssetRefs(expandedByField.get(tf.name) ?? tf.value).filter((r) => acceptedKinds.has(r.kind));
+        if (refs.length === 0)
+            continue;
+        refsByField.set(tf.name, refs);
+        for (const ref of refs)
+            queues[ref.kind].push(ref);
+    }
+    const resolveRef = async (ref) => {
+        const injected = {
+            type: ref.kind,
+            uri: ref.uri,
+            mimeType: ref.mime
+        };
+        const bytes = await loadMediaRefBytes({ type: ref.kind, uri: ref.uri }, context);
+        if (bytes && bytes.length > 0) {
+            injected.data = encodeBase64(bytes);
+        }
+        return injected;
+    };
+    // Fill empty inputs in declaration order from the matching kind's queue,
+    // recording which input label each placed mention was routed into.
+    const labelByRef = new Map();
+    const cursor = { image: 0, audio: 0, video: 0 };
+    for (const field of assetFields) {
+        if (field.hasSource)
+            continue;
+        const queue = queues[field.kind];
+        if (cursor[field.kind] >= queue.length)
+            continue;
+        const label = field.label ?? field.name;
+        if (field.list) {
+            const remaining = queue.slice(cursor[field.kind]);
+            cursor[field.kind] = queue.length;
+            const resolved = [];
+            for (let i = 0; i < remaining.length; i++) {
+                labelByRef.set(remaining[i], `${label}[${i}]`);
+                resolved.push(await resolveRef(remaining[i]));
+            }
+            if (resolved.length > 0)
+                overrides[field.name] = resolved;
+        }
+        else {
+            const ref = queue[cursor[field.kind]++];
+            labelByRef.set(ref, label);
+            overrides[field.name] = await resolveRef(ref);
+        }
+    }
+    // Rewrite each text input off its (folder-)expanded value: a placed media
+    // mention becomes a reference to the input label it now lives in; an unplaced
+    // one is dropped. Then inline any plain-text document mentions into the
+    // result — done after the media rewrite so the document's own whitespace
+    // isn't touched by the gap-collapsing in `replaceAssetRefs`. Compared against
+    // the original so a folder that expanded to nothing still rewrites the field.
+    for (const tf of textFields) {
+        const expanded = expandedByField.get(tf.name) ?? tf.value;
+        const refs = refsByField.get(tf.name) ?? [];
+        const relabeled = replaceAssetRefs(expanded, refs, (ref) => labelByRef.get(ref) ?? "");
+        const rewritten = await inlineTextAssetRefs(relabeled, context);
+        if (rewritten !== tf.value)
+            overrides[tf.name] = rewritten;
+    }
+    return overrides;
+}
+//# sourceMappingURL=prompt-asset-refs.js.map