@aexol/spectral 0.9.212 → 0.9.215

package/dist/sdk/ai/utils/image-source.d.ts

@@ -0,0 +1,121 @@
+/**
+ * Image source resolution for provider serialization.
+ *
+ * `ImageContent` can now carry either inline base64 (`data`) or a hosted
+ * `url`. Providers (OpenRouter, AWS Bedrock, OpenAI, ...) can only fetch
+ * images from PUBLIC urls — a `http://localhost:9002/...` MinIO url in dev is
+ * unreachable for them. This module resolves an `ImageContent` block into
+ * something a provider can actually consume:
+ *
+ * - `data` present            -> `data:<mimeType>;base64,<data>` (unchanged, legacy safe)
+ * - `url` public              -> passed through as-is (no base64 in the context!)
+ * - `url` relative            -> resolved against the backend base url first
+ *                                (`/generated-images/<id>` is what the backend returns)
+ * - `url` local/private       -> fetched and inlined **at send time only**
+ *
+ * The inlining is deliberately scoped to a single request: the fetched bytes
+ * are written into a throw-away copy of the message list, never back into the
+ * session, so base64 never accumulates in persisted history.
+ *
+ * @module @spectral/ai/utils/image-source
+ */
+import type { Context, ImageContent } from "../types.js";
+/** Don't inline absurd payloads — 25 MiB of base64 would blow the request. */
+export declare const DEFAULT_MAX_INLINE_BYTES: number;
+/** Timeout for fetching a local image that has to be inlined. */
+export declare const DEFAULT_INLINE_FETCH_TIMEOUT_MS = 15000;
+export interface ResolveImageOptions {
+    /** Abort the inline fetch (e.g. when the request is cancelled). */
+    signal?: AbortSignal;
+    timeoutMs?: number;
+    maxInlineBytes?: number;
+    /** Injectable for tests / alternative runtimes. */
+    fetchImpl?: typeof fetch;
+    /**
+     * Base url (usually the backend url) used to turn a relative image url
+     * (`/generated-images/<id>`) into an absolute one. May be a lazy getter so
+     * callers only pay the cost (e.g. reading config) when a relative url is
+     * actually present.
+     */
+    baseUrl?: string | (() => string | undefined | Promise<string | undefined>);
+    onWarn?: (message: string) => void;
+}
+export interface ResolvedImageSource {
+    /** Value to place in `image_url.url` (data URI or http(s) url). */
+    url: string;
+    /** True when the bytes were fetched and base64-inlined. */
+    inlined: boolean;
+    mimeType: string;
+    /**
+     * Raw base64 payload WITHOUT the `data:` prefix, matching the
+     * `ImageContent.data` contract. Only set when the bytes are known
+     * (inline `data` present, or fetched from a local url).
+     */
+    data?: string;
+}
+/**
+ * True when `rawUrl` has no scheme, i.e. it is relative and therefore
+ * unusable for a provider until it is joined with a base url.
+ */
+export declare function isRelativeImageUrl(rawUrl: string): boolean;
+/**
+ * Join a relative image url (`/generated-images/<id>`) with the backend base
+ * url. Absolute urls and missing base urls are returned untouched.
+ */
+export declare function resolveImageUrlAgainstBase(rawUrl: string, baseUrl?: string): string;
+/**
+ * True when a provider is unlikely to be able to fetch this URL itself and
+ * the image must therefore be inlined as base64 before sending.
+ *
+ * Covers: localhost, `*.local`, `*.localhost`, loopback IPs (v4 + v6),
+ * RFC1918 private ranges, link-local and CGNAT ranges, and non-http(s)
+ * schemes (except `data:`/`blob:` which need no fetch at all), plus
+ * container/metadata/reserved hosts and relative urls that still need to be
+ * resolved against a base url.
+ */
+export declare function isLocalOrPrivateImageUrl(rawUrl: string): boolean;
+export declare function hasInlineImageData(image: ImageContent): image is ImageContent & {
+    data: string;
+};
+/** Build a `data:` URI from base64 payload + mime type (idempotent). */
+export declare function dataUriForImage(data: string, mimeType: string): string;
+/**
+ * `ImageContent.data` is base64 WITHOUT a `data:` prefix, but legacy
+ * sessions may carry a full data URI. Strip it so consumers doing
+ * `Buffer.from(data, "base64")` get real bytes.
+ */
+export declare function base64FromImageData(data: string): string;
+/**
+ * Resolve an image block into a provider-sendable url.
+ *
+ * Returns `null` when the block carries neither data nor url (nothing sendable).
+ * Local/private urls are fetched and inlined; failures fall back to the raw
+ * url so the caller still emits *something* rather than dropping the image.
+ */
+export declare function resolveImageForProvider(image: ImageContent, options?: ResolveImageOptions): Promise<ResolvedImageSource | null>;
+/**
+ * Pre-pass used right before a request is built: any image block that only has
+ * a local/private `url` gets its bytes fetched and attached as `data` on a
+ * throw-away copy of the context.
+ *
+ * The returned context is a shallow clone (messages copied, unchanged messages
+ * shared by reference) and is discarded after the single request completes, so
+ * base64 never lands in session history. When there is nothing to inline the
+ * original context object is returned unchanged.
+ */
+export declare function resolveImagesForContext(context: Context, options?: ResolveImageOptions): Promise<Context>;
+/**
+ * Same as `resolveImagesForContext` but for the images API input list.
+ */
+export declare function resolveImagesForInput(input: Array<{
+    type: string;
+}>, options?: ResolveImageOptions): Promise<Array<{
+    type: string;
+}>>;
+/**
+ * Synchronous serializer used inside the (sync) param builders: after the
+ * `resolveImagesForContext` pre-pass there should be nothing local left, so
+ * this only has to choose between an existing `data` payload and a public url.
+ */
+export declare function imageSourceUrlForProvider(image: ImageContent, baseUrl?: string): string | undefined;
+//# sourceMappingURL=image-source.d.ts.map

package/dist/sdk/ai/utils/image-source.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"image-source.d.ts","sourceRoot":"","sources":["../../../../src/sdk/ai/utils/image-source.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,OAAO,KAAK,EAAE,OAAO,EAAE,YAAY,EAAW,MAAM,aAAa,CAAC;AAElE,8EAA8E;AAC9E,eAAO,MAAM,wBAAwB,QAAmB,CAAC;AAEzD,iEAAiE;AACjE,eAAO,MAAM,+BAA+B,QAAS,CAAC;AAEtD,MAAM,WAAW,mBAAmB;IACnC,mEAAmE;IACnE,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,mDAAmD;IACnD,SAAS,CAAC,EAAE,OAAO,KAAK,CAAC;IACzB;;;;;OAKG;IACH,OAAO,CAAC,EAAE,MAAM,GAAG,CAAC,MAAM,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC;IAC5E,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;CACnC;AAED,MAAM,WAAW,mBAAmB;IACnC,mEAAmE;IACnE,GAAG,EAAE,MAAM,CAAC;IACZ,2DAA2D;IAC3D,OAAO,EAAE,OAAO,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB;;;;OAIG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;CACd;AAuCD;;;GAGG;AACH,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAI1D;AAED;;;GAGG;AACH,wBAAgB,0BAA0B,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,CAUnF;AAmBD;;;;;;;;;GASG;AACH,wBAAgB,wBAAwB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CA4BhE;AAcD,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,YAAY,GAAG,KAAK,IAAI,YAAY,GAAG;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,CAEhG;AAED,wEAAwE;AACxE,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM,CAGtE;AAED;;;;GAIG;AACH,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAIxD;AAoBD;;;;;;GAMG;AACH,wBAAsB,uBAAuB,CAC5C,KAAK,EAAE,YAAY,EACnB,OAAO,GAAE,mBAAwB,GAC/B,OAAO,CAAC,mBAAmB,GAAG,IAAI,CAAC,CAgErC;AAcD;;;;;;;;;GASG;AACH,wBAAsB,uBAAuB,CAC5C,OAAO,EAAE,OAAO,EAChB,OAAO,GAAE,mBAAwB,GAC/B,OAAO,CAAC,OAAO,CAAC,CAoDlB;AAED;;GAEG;AACH,wBAAsB,qBAAqB,CAC1C,KAAK,EAAE,KAAK,CAAC;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC,EAC9B,OAAO,GAAE,mBAAwB,GAC/B,OAAO,CAAC,KAAK,CAAC;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC,CAAC,CAsBlC;AAED;;;;GAIG;AACH,wBAAgB,yBAAyB,CAAC,KAAK,EAAE,YAAY,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAUnG"}

package/dist/sdk/ai/utils/image-source.js

@@ -0,0 +1,410 @@
+/**
+ * Image source resolution for provider serialization.
+ *
+ * `ImageContent` can now carry either inline base64 (`data`) or a hosted
+ * `url`. Providers (OpenRouter, AWS Bedrock, OpenAI, ...) can only fetch
+ * images from PUBLIC urls — a `http://localhost:9002/...` MinIO url in dev is
+ * unreachable for them. This module resolves an `ImageContent` block into
+ * something a provider can actually consume:
+ *
+ * - `data` present            -> `data:<mimeType>;base64,<data>` (unchanged, legacy safe)
+ * - `url` public              -> passed through as-is (no base64 in the context!)
+ * - `url` relative            -> resolved against the backend base url first
+ *                                (`/generated-images/<id>` is what the backend returns)
+ * - `url` local/private       -> fetched and inlined **at send time only**
+ *
+ * The inlining is deliberately scoped to a single request: the fetched bytes
+ * are written into a throw-away copy of the message list, never back into the
+ * session, so base64 never accumulates in persisted history.
+ *
+ * @module @spectral/ai/utils/image-source
+ */
+/** Don't inline absurd payloads — 25 MiB of base64 would blow the request. */
+export const DEFAULT_MAX_INLINE_BYTES = 25 * 1024 * 1024;
+/** Timeout for fetching a local image that has to be inlined. */
+export const DEFAULT_INLINE_FETCH_TIMEOUT_MS = 15_000;
+function isDigitRun(value) {
+    return /^\d+$/.test(value);
+}
+function isPrivateIpv4(hostname) {
+    const parts = hostname.split(".");
+    if (parts.length !== 4 || !parts.every(isDigitRun))
+        return false;
+    const [a, b] = parts.map((p) => Number(p));
+    // 0.0.0.0/8, 10/8, 127/8 (loopback), 169.254/16 (link-local), 172.16/12, 192.168/16
+    if (a === 0 || a === 10 || a === 127)
+        return true;
+    if (a === 169 && b === 254)
+        return true;
+    if (a === 172 && b >= 16 && b <= 31)
+        return true;
+    if (a === 192 && b === 168)
+        return true;
+    // 100.64/10 CGNAT
+    if (a === 100 && b >= 64 && b <= 127)
+        return true;
+    return false;
+}
+/**
+ * Host suffixes that never resolve publicly: container/dev hosts
+ * (`host.docker.internal`), cloud metadata (`metadata.google.internal`),
+ * home networks (`*.home.arpa`) and reserved TLDs (`*.test`, `*.example`,
+ * `*.invalid`). A provider can never fetch these -> inline them.
+ */
+const LOCAL_HOST_SUFFIXES = [
+    "docker.internal",
+    ".internal",
+    ".localhost",
+    ".local",
+    ".home.arpa",
+    ".test",
+    ".example",
+    ".invalid",
+];
+const ABSOLUTE_URL_SCHEME = /^[a-z][a-z0-9+.-]*:/i;
+/**
+ * True when `rawUrl` has no scheme, i.e. it is relative and therefore
+ * unusable for a provider until it is joined with a base url.
+ */
+export function isRelativeImageUrl(rawUrl) {
+    const trimmed = typeof rawUrl === "string" ? rawUrl.trim() : "";
+    if (!trimmed)
+        return false;
+    return !ABSOLUTE_URL_SCHEME.test(trimmed);
+}
+/**
+ * Join a relative image url (`/generated-images/<id>`) with the backend base
+ * url. Absolute urls and missing base urls are returned untouched.
+ */
+export function resolveImageUrlAgainstBase(rawUrl, baseUrl) {
+    const trimmed = typeof rawUrl === "string" ? rawUrl.trim() : "";
+    if (!trimmed || !isRelativeImageUrl(trimmed))
+        return trimmed;
+    const base = typeof baseUrl === "string" ? baseUrl.trim() : "";
+    if (!base)
+        return trimmed;
+    try {
+        return new URL(trimmed, base).toString();
+    }
+    catch {
+        return trimmed;
+    }
+}
+/**
+ * Loopback / unique-local (fc00::/7) / link-local (fe80::/10) IPv6 are
+ * unreachable for a provider; every other literal (e.g. `2001:4860::8888`)
+ * is public and can be passed through.
+ */
+function isPrivateIpv6(hostname) {
+    if (hostname === "::" || hostname === "::1" || hostname === "0:0:0:0:0:0:0:1")
+        return true;
+    const mapped = hostname.match(/^::ffff:(\d+\.\d+\.\d+\.\d+)$/);
+    if (mapped)
+        return isPrivateIpv4(mapped[1]);
+    const head = hostname.split(":")[0] ?? "";
+    // fc00::/7 (fc00-fdff) unique local
+    if (/^f[cd][0-9a-f]{0,2}$/.test(head))
+        return true;
+    // fe80::/10 (fe80-febf) link local
+    if (/^fe[89ab][0-9a-f]?$/.test(head))
+        return true;
+    return false;
+}
+/**
+ * True when a provider is unlikely to be able to fetch this URL itself and
+ * the image must therefore be inlined as base64 before sending.
+ *
+ * Covers: localhost, `*.local`, `*.localhost`, loopback IPs (v4 + v6),
+ * RFC1918 private ranges, link-local and CGNAT ranges, and non-http(s)
+ * schemes (except `data:`/`blob:` which need no fetch at all), plus
+ * container/metadata/reserved hosts and relative urls that still need to be
+ * resolved against a base url.
+ */
+export function isLocalOrPrivateImageUrl(rawUrl) {
+    const trimmed = typeof rawUrl === "string" ? rawUrl.trim() : "";
+    if (!trimmed)
+        return false;
+    // Relative (`/generated-images/x`) or unparseable: the provider would only
+    // ever see a path fragment, so this needs resolution/inlining.
+    if (isRelativeImageUrl(trimmed))
+        return true;
+    let url;
+    try {
+        url = new URL(trimmed);
+    }
+    catch {
+        // Unparseable -> treat like a relative url: unusable for the provider.
+        return true;
+    }
+    const protocol = url.protocol.toLowerCase();
+    if (protocol === "data:" || protocol === "blob:")
+        return false;
+    if (protocol !== "http:" && protocol !== "https:")
+        return true;
+    const hostname = url.hostname.toLowerCase().replace(/^\[/, "").replace(/\]$/, "");
+    if (hostname === "localhost" || hostname === "localhost.localdomain")
+        return true;
+    if (LOCAL_HOST_SUFFIXES.some((suffix) => hostname.endsWith(suffix)))
+        return true;
+    // IPv6 literal: explicit ranges only — a public literal must NOT be
+    // inlined (it has no dot, so the single-label rule below would misfire).
+    if (hostname.includes(":"))
+        return isPrivateIpv6(hostname);
+    if (isPrivateIpv4(hostname))
+        return true;
+    // Single-label hostnames (docker service names, "minio", ...) are not
+    // publicly resolvable -> inline them.
+    if (!hostname.includes("."))
+        return true;
+    return false;
+}
+async function resolveBaseUrlOption(baseUrl) {
+    if (!baseUrl)
+        return undefined;
+    if (typeof baseUrl === "string")
+        return baseUrl;
+    try {
+        return await baseUrl();
+    }
+    catch {
+        return undefined;
+    }
+}
+export function hasInlineImageData(image) {
+    return typeof image.data === "string" && image.data.length > 0;
+}
+/** Build a `data:` URI from base64 payload + mime type (idempotent). */
+export function dataUriForImage(data, mimeType) {
+    if (data.startsWith("data:"))
+        return data;
+    return `data:${mimeType || "image/png"};base64,${data}`;
+}
+/**
+ * `ImageContent.data` is base64 WITHOUT a `data:` prefix, but legacy
+ * sessions may carry a full data URI. Strip it so consumers doing
+ * `Buffer.from(data, "base64")` get real bytes.
+ */
+export function base64FromImageData(data) {
+    if (!data.startsWith("data:"))
+        return data;
+    const comma = data.indexOf(",");
+    return comma === -1 ? data : data.slice(comma + 1);
+}
+function bytesToBase64(bytes) {
+    const maybeBuffer = globalThis
+        .Buffer;
+    if (maybeBuffer)
+        return maybeBuffer.from(bytes).toString("base64");
+    let binary = "";
+    const CHUNK = 0x8000;
+    for (let i = 0; i < bytes.length; i += CHUNK) {
+        binary += String.fromCharCode(...bytes.subarray(i, i + CHUNK));
+    }
+    return btoa(binary);
+}
+function normalizeMimeType(value, fallback) {
+    if (!value)
+        return fallback || "image/png";
+    const base = value.split(";")[0].trim().toLowerCase();
+    return base || fallback || "image/png";
+}
+/**
+ * Resolve an image block into a provider-sendable url.
+ *
+ * Returns `null` when the block carries neither data nor url (nothing sendable).
+ * Local/private urls are fetched and inlined; failures fall back to the raw
+ * url so the caller still emits *something* rather than dropping the image.
+ */
+export async function resolveImageForProvider(image, options = {}) {
+    if (hasInlineImageData(image)) {
+        // Keep the stored data URI intact (mime may differ from `mimeType`),
+        // but expose the raw base64 so copies stay contract-compliant.
+        const raw = base64FromImageData(image.data);
+        return {
+            url: image.data.startsWith("data:") ? image.data : dataUriForImage(raw, image.mimeType),
+            inlined: true,
+            mimeType: image.mimeType,
+            data: raw,
+        };
+    }
+    let url = typeof image.url === "string" ? image.url.trim() : "";
+    if (!url)
+        return null;
+    // Relative urls (`/generated-images/<id>`) must be joined with the backend
+    // base url before anything else — otherwise the provider gets a path.
+    if (isRelativeImageUrl(url)) {
+        url = resolveImageUrlAgainstBase(url, await resolveBaseUrlOption(options.baseUrl));
+    }
+    if (!isLocalOrPrivateImageUrl(url)) {
+        return { url, inlined: false, mimeType: image.mimeType };
+    }
+    const doFetch = options.fetchImpl ?? fetch;
+    const timeoutMs = options.timeoutMs ?? DEFAULT_INLINE_FETCH_TIMEOUT_MS;
+    const maxInlineBytes = options.maxInlineBytes ?? DEFAULT_MAX_INLINE_BYTES;
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), timeoutMs);
+    const onAbort = () => controller.abort();
+    options.signal?.addEventListener("abort", onAbort, { once: true });
+    try {
+        // An already-aborted signal never fires the listener above.
+        if (options.signal?.aborted)
+            throw new Error("aborted");
+        const response = await doFetch(url, { signal: controller.signal });
+        if (!response.ok)
+            throw new Error(`HTTP ${response.status}`);
+        // Reject oversized payloads from the declared length BEFORE reading
+        // the body — otherwise the whole payload lands in memory first.
+        const declaredLength = response.headers?.get?.("content-length");
+        if (declaredLength !== null && declaredLength !== undefined) {
+            const declared = Number(declaredLength);
+            if (Number.isFinite(declared) && declared > maxInlineBytes) {
+                throw new Error(`image too large to inline (${declared} bytes)`);
+            }
+        }
+        const buffer = await response.arrayBuffer();
+        if (buffer.byteLength > maxInlineBytes) {
+            throw new Error(`image too large to inline (${buffer.byteLength} bytes)`);
+        }
+        const mimeType = normalizeMimeType(response.headers?.get?.("content-type"), image.mimeType);
+        const data = bytesToBase64(new Uint8Array(buffer));
+        return {
+            url: dataUriForImage(data, mimeType),
+            inlined: true,
+            mimeType,
+            data,
+        };
+    }
+    catch (error) {
+        const reason = error instanceof Error ? error.message : String(error);
+        options.onWarn?.(`Could not inline local image ${url} (${reason}); sending url as-is`);
+        return { url, inlined: false, mimeType: image.mimeType };
+    }
+    finally {
+        clearTimeout(timer);
+        options.signal?.removeEventListener("abort", onAbort);
+    }
+}
+function inlineImageContentBlock(block, resolved) {
+    const image = block;
+    const patch = resolved.get(image);
+    if (patch === undefined)
+        return block;
+    // Copy-on-write: never mutate the block stored in the session.
+    const copy = { ...image, ...patch };
+    return copy;
+}
+/**
+ * Pre-pass used right before a request is built: any image block that only has
+ * a local/private `url` gets its bytes fetched and attached as `data` on a
+ * throw-away copy of the context.
+ *
+ * The returned context is a shallow clone (messages copied, unchanged messages
+ * shared by reference) and is discarded after the single request completes, so
+ * base64 never lands in session history. When there is nothing to inline the
+ * original context object is returned unchanged.
+ */
+export async function resolveImagesForContext(context, options = {}) {
+    const pending = [];
+    for (const message of context.messages) {
+        if (message.role === "user" && Array.isArray(message.content)) {
+            for (const block of message.content) {
+                if (block.type === "image" && !hasInlineImageData(block))
+                    pending.push(block);
+            }
+        }
+        else if (message.role === "toolResult") {
+            for (const block of message.content) {
+                if (block.type === "image" && !hasInlineImageData(block))
+                    pending.push(block);
+            }
+        }
+    }
+    if (pending.length === 0)
+        return context;
+    const resolved = new Map();
+    await Promise.all(pending.map(async (image) => {
+        const source = await resolveImageForProvider(image, options);
+        if (!source)
+            return;
+        const originalUrl = typeof image.url === "string" ? image.url.trim() : "";
+        const patch = {};
+        if (source.inlined) {
+            // `data` must be raw base64 (no `data:` prefix) — see
+            // `ImageContent.data` and `base64FromImageData`.
+            if (source.data)
+                patch.data = source.data;
+            if (source.mimeType && source.mimeType !== image.mimeType)
+                patch.mimeType = source.mimeType;
+        }
+        else if (source.url !== originalUrl) {
+            // e.g. a relative url resolved against the backend base url.
+            patch.url = source.url;
+        }
+        if (Object.keys(patch).length > 0)
+            resolved.set(image, patch);
+    }));
+    if (resolved.size === 0)
+        return context;
+    const messages = context.messages.map((message) => {
+        if (message.role === "user" && Array.isArray(message.content)) {
+            return {
+                ...message,
+                content: message.content.map((block) => inlineImageContentBlock(block, resolved)),
+            };
+        }
+        if (message.role === "toolResult") {
+            return {
+                ...message,
+                content: message.content.map((block) => inlineImageContentBlock(block, resolved)),
+            };
+        }
+        return message;
+    });
+    return { ...context, messages };
+}
+/**
+ * Same as `resolveImagesForContext` but for the images API input list.
+ */
+export async function resolveImagesForInput(input, options = {}) {
+    const images = input.filter((item) => item.type === "image");
+    if (images.length === 0)
+        return input;
+    let changed = false;
+    const resolvedInputs = await Promise.all(input.map(async (item) => {
+        if (item.type !== "image")
+            return item;
+        const image = item;
+        const source = await resolveImageForProvider(image, options);
+        if (!source)
+            return item;
+        if (source.inlined && source.data) {
+            changed = true;
+            return { ...image, data: source.data, mimeType: source.mimeType };
+        }
+        if (!source.inlined && source.url !== (typeof image.url === "string" ? image.url.trim() : "")) {
+            changed = true;
+            return { ...image, url: source.url };
+        }
+        return item;
+    }));
+    return changed ? resolvedInputs : input;
+}
+/**
+ * Synchronous serializer used inside the (sync) param builders: after the
+ * `resolveImagesForContext` pre-pass there should be nothing local left, so
+ * this only has to choose between an existing `data` payload and a public url.
+ */
+export function imageSourceUrlForProvider(image, baseUrl) {
+    if (hasInlineImageData(image))
+        return dataUriForImage(image.data, image.mimeType);
+    const url = resolveImageUrlAgainstBase(image.url ?? "", baseUrl);
+    if (!url)
+        return undefined;
+    // Still relative after resolution (`/generated-images/<id>` with no
+    // backend base url configured): a provider would only receive a path
+    // fragment and answer HTTP 400, so drop the block instead of sending
+    // something it can never fetch.
+    if (isRelativeImageUrl(url))
+        return undefined;
+    return url;
+}

package/dist/sdk/coding-agent/core/extensions/native-extensions.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"native-extensions.d.ts","sourceRoot":"","sources":["../../../../../src/sdk/coding-agent/core/extensions/native-extensions.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,MAAM,WAAW,uBAAuB;IACtC,qEAAqE;IACrE,EAAE,EAAE,MAAM,CAAC;IACX,4DAA4D;IAC5D,KAAK,EAAE,MAAM,CAAC;IACd,yCAAyC;IACzC,WAAW,EAAE,MAAM,CAAC;IACpB,mDAAmD;IACnD,QAAQ,EAAE,MAAM,GAAG,YAAY,GAAG,aAAa,CAAC;IAChD;;;OAGG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB,oEAAoE;IACpE,cAAc,EAAE,OAAO,CAAC;IACxB,mCAAmC;IACnC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;CACjB;AAED;;;;;;GAMG;AACH,eAAO,MAAM,iBAAiB,EAAE,uBAAuB,EAkJtD,CAAC;AAEF;;GAEG;AACH,wBAAgB,kBAAkB,CAAC,EAAE,EAAE,MAAM,GAAG,uBAAuB,GAAG,SAAS,CAElF;AAED;;;GAGG;AACH,wBAAgB,8BAA8B,CAAC,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,EAAE,UAAU,EAAE,GAAG,CAAC,MAAM,CAAC,GAAG,MAAM,EAAE,CAI1G;AAED;;GAEG;AACH,wBAAgB,yBAAyB,CAAC,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,EAAE,UAAU,EAAE,GAAG,CAAC,MAAM,CAAC,GAAG,KAAK,CAAC;IAClG,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,cAAc,EAAE,OAAO,CAAC;IACxB,OAAO,EAAE,OAAO,CAAC;IACjB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;CACjB,CAAC,CAUD"}
+{"version":3,"file":"native-extensions.d.ts","sourceRoot":"","sources":["../../../../../src/sdk/coding-agent/core/extensions/native-extensions.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,MAAM,WAAW,uBAAuB;IACtC,qEAAqE;IACrE,EAAE,EAAE,MAAM,CAAC;IACX,4DAA4D;IAC5D,KAAK,EAAE,MAAM,CAAC;IACd,yCAAyC;IACzC,WAAW,EAAE,MAAM,CAAC;IACpB,mDAAmD;IACnD,QAAQ,EAAE,MAAM,GAAG,YAAY,GAAG,aAAa,CAAC;IAChD;;;OAGG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB,oEAAoE;IACpE,cAAc,EAAE,OAAO,CAAC;IACxB,mCAAmC;IACnC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;CACjB;AAED;;;;;;GAMG;AACH,eAAO,MAAM,iBAAiB,EAAE,uBAAuB,EAiKtD,CAAC;AAEF;;GAEG;AACH,wBAAgB,kBAAkB,CAAC,EAAE,EAAE,MAAM,GAAG,uBAAuB,GAAG,SAAS,CAElF;AAED;;;GAGG;AACH,wBAAgB,8BAA8B,CAAC,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,EAAE,UAAU,EAAE,GAAG,CAAC,MAAM,CAAC,GAAG,MAAM,EAAE,CAI1G;AAED;;GAEG;AACH,wBAAgB,yBAAyB,CAAC,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,EAAE,UAAU,EAAE,GAAG,CAAC,MAAM,CAAC,GAAG,KAAK,CAAC;IAClG,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,cAAc,EAAE,OAAO,CAAC;IACxB,OAAO,EAAE,OAAO,CAAC;IACjB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;CACjB,CAAC,CAUD"}

package/dist/sdk/coding-agent/core/extensions/native-extensions.js

@@ -119,6 +119,20 @@ export const NATIVE_EXTENSIONS = [
         defaultEnabled: true,
         tags: ["screenshot", "desktop", "vision", "screen"],
     },
+    {
+        id: "desktop-control",
+        label: "Desktop Control",
+        description: "Computer use with a safety layer: moves the mouse, clicks, drags, scrolls, types text, presses hotkeys and " +
+            "focuses windows via native OS APIs (macOS: JXA/CGEvent, Windows: PowerShell/SendInput, Linux: xdotool/wtype). " +
+            "Every mutating action returns before/after screenshots for verification. Enabling the extension is the " +
+            "explicit consent, so actions run without a confirmation prompt; set SPECTRAL_DESKTOP_REQUIRE_CONFIRM=1 to " +
+            "opt back into a per-session prompt. Supports SPECTRAL_DESKTOP_DRY_RUN and SPECTRAL_DESKTOP_ALLOWLIST. " +
+            "Off by default because it injects real input.",
+        category: "automation",
+        entryPath: "src/extensions/desktop-control/index.ts",
+        defaultEnabled: false,
+        tags: ["desktop", "computer-use", "mouse", "keyboard", "automation", "dangerous"],
+    },
     {
         id: "code-order",
         label: "Code Order",

package/dist/sdk/coding-agent/core/extensions/types.d.ts

@@ -83,7 +83,16 @@ export interface ExtensionUIDialogOptions {
     signal?: AbortSignal;
     /** Timeout in milliseconds. Dialog auto-dismisses with live countdown display. */
     timeout?: number;
+    /**
+     * Optional side-channel reporting how the dialog ended, so callers can
+     * distinguish an explicit human decision (`allow`/`deny`) from an auto-deny
+     * (`timeout`, `no_tty`, `cancelled`) where no user answered. Modes that
+     * cannot report outcomes simply never call it.
+     */
+    onOutcome?: (outcome: ExtensionConfirmOutcome) => void;
 }
+/** How a confirm dialog ended (see ExtensionUIDialogOptions.onOutcome). */
+export type ExtensionConfirmOutcome = "allow" | "deny" | "timeout" | "no_tty" | "cancelled";
 /** Placement for extension widgets. */
 export type WidgetPlacement = "aboveEditor" | "belowEditor";
 /** Options for extension widgets. */