@nodetool-ai/runtime 0.7.0-rc.23 → 0.8.1

package/dist/providers/gemini-provider.js

@@ -1,41 +1,384 @@
 import { createLogger } from "@nodetool-ai/config";
 import { BaseProvider } from "./base-provider.js";
+import { geminiContextExceeded } from "./context-exceeded.js";
+import { sniffAudioMime } from "./audio-mime.js";
+import { sniffVideoMime } from "./video-mime.js";
+import { detectImageMime } from "./image-mime.js";
+import { safeFetch } from "./safe-url.js";
+import { ContentFilterRefusal, isContentFilterRefusal } from "./content-filter.js";
+import { isBoolean, isFiniteNumber, isNonEmptyString, isString } from "@nodetool-ai/protocol";
 const log = createLogger("nodetool.runtime.providers.gemini");
+import { WEB_SEARCH_TOOL_NAME } from "./types.js";
 const GEMINI_API_BASE = "https://generativelanguage.googleapis.com/v1beta";
+const GEMINI_UPLOAD_BASE = `${GEMINI_API_BASE.replace("/v1beta", "")}/upload/v1beta/files`;
+/**
+ * Largest payload sent as `inlineData`. Gemini caps a generateContent request
+ * at 20 MB total; anything above this goes through the Files API instead, which
+ * leaves room for the rest of the request.
+ */
+export const GEMINI_INLINE_VIDEO_MAX_BYTES = 19 * 1024 * 1024;
+/** Polls of `files/{name}` before an uploaded file is declared stuck. */
+const GEMINI_FILE_MAX_POLLS = 60;
+const GEMINI_FILE_POLL_INTERVAL_MS = 2_000;
+/** Veo accepts at most three `asset` reference images per request. */
+const VEO_MAX_REFERENCE_IMAGES = 3;
+/** Veo requires 8-second clips for reference images and first/last frames. */
+const VEO_CONSTRAINED_DURATION_SECONDS = 8;
+/** Veo appends a fixed 7-second continuation to the input video. */
+const VEO_EXTENSION_SECONDS = 7;
+/** Omni Flash accepts at most three reference clips. */
+const OMNI_MAX_REFERENCE_VIDEOS = 3;
+/**
+ * Video models and the tasks each accepts. Veo 3.1 Lite takes no reference
+ * images, last frame, or extension. Omni Flash runs through the Interactions
+ * API and edits existing video; its duration follows the prompt.
+ *
+ * Veo lists no `durations`: callers apply that list to extension length too,
+ * and Veo generates 4, 6, or 8 seconds but extends by 7 only.
+ */
+const GEMINI_VIDEO_MODELS = [
+    {
+        id: "veo-3.1-generate-preview",
+        name: "Veo 3.1 Preview",
+        provider: "gemini",
+        supportedTasks: [
+            "text_to_video",
+            "image_to_video",
+            "reference_to_video",
+            "extend_video",
+            "extend_video_end"
+        ],
+        resolutions: ["720p", "1080p", "4k"],
+        aspectRatios: ["16:9", "9:16"]
+    },
+    {
+        id: "veo-3.1-fast-generate-preview",
+        name: "Veo 3.1 Fast Preview",
+        provider: "gemini",
+        supportedTasks: [
+            "text_to_video",
+            "image_to_video",
+            "reference_to_video",
+            "extend_video",
+            "extend_video_end"
+        ],
+        resolutions: ["720p", "1080p", "4k"],
+        aspectRatios: ["16:9", "9:16"]
+    },
+    {
+        id: "veo-3.1-lite-generate-preview",
+        name: "Veo 3.1 Lite Preview",
+        provider: "gemini",
+        supportedTasks: ["text_to_video", "image_to_video"],
+        resolutions: ["720p", "1080p"],
+        aspectRatios: ["16:9", "9:16"]
+    },
+    {
+        id: "gemini-omni-1.1-flash",
+        name: "Gemini Omni Flash",
+        provider: "gemini",
+        supportedTasks: [
+            "text_to_video",
+            "image_to_video",
+            "reference_to_video",
+            "video_to_video"
+        ],
+        resolutions: ["360p", "720p", "1080p", "4k"],
+        aspectRatios: ["16:9", "9:16"]
+    }
+];
+function isOmniVideoModel(modelId) {
+    return modelId.startsWith("gemini-omni-");
+}
+function requireVideoTask(modelId, task) {
+    const model = GEMINI_VIDEO_MODELS.find((entry) => entry.id === modelId);
+    if (!model?.supportedTasks?.includes(task)) {
+        throw new Error(`Gemini model ${modelId} does not support ${task}`);
+    }
+}
+function combineSignals(signal, timeoutSeconds) {
+    const timeout = timeoutSeconds && timeoutSeconds > 0
+        ? AbortSignal.timeout(timeoutSeconds * 1000)
+        : undefined;
+    if (!signal)
+        return timeout;
+    if (!timeout)
+        return signal;
+    return AbortSignal.any([signal, timeout]);
+}
+/** The image object Veo reads for `image`, `lastFrame`, and references. */
+function veoImage(bytes) {
+    return {
+        bytesBase64Encoded: Buffer.from(bytes).toString("base64"),
+        mimeType: detectImageMime(bytes)
+    };
+}
+/** Drop `; charset=…`/`; codecs=…` parameters from a Content-Type header. */
+function stripMimeParams(value) {
+    const mime = value?.split(";")[0].trim();
+    return mime || undefined;
+}
+/**
+ * Normalize an audio mime to one Gemini's `inlineData` accepts. Gemini lists
+ * audio/wav, audio/mp3, audio/aiff, audio/aac, audio/ogg and audio/flac; the
+ * common `audio/mpeg` label is remapped to `audio/mp3`. Falls back to
+ * `audio/mp3` when the type is unknown.
+ */
+function geminiAudioMime(mime) {
+    if (!mime)
+        return "audio/mp3";
+    if (mime === "audio/mpeg" || mime === "audio/mpga")
+        return "audio/mp3";
+    return mime;
+}
+/**
+ * Gemini's native/custom tool combination contract is versioned by model
+ * family. Keep this check deliberately based on the model id supplied for the
+ * request: model discovery is optional and callers may use a newly released
+ * Gemini 3 id before it appears in the catalog.
+ */
+function isGemini3Model(model) {
+    const modelId = model.split("/").pop()?.toLowerCase() ?? model.toLowerCase();
+    return /^gemini-3(?:[.-]|$)/.test(modelId);
+}
 // Gemini's function-declaration schema is a strict subset of OpenAPI 3.0.
-// It rejects JSON-Schema-only fields like `additionalProperties`, `$schema`,
-// `$id`, `$ref`, `definitions`, `patternProperties`, etc. Any one of these
+// It rejects JSON-Schema-only fields like `const`, `additionalProperties`,
+// `$schema`, `$ref`, `definitions`, `patternProperties`, etc. Any one of these
 // anywhere in the tree causes a 400 that aborts the entire tool batch, so we
-// recursively strip them before sending.
+// recursively strip them before sending. Zod 4's `z.toJSONSchema` (draft
+// 2020-12) emits several of them — `const` for every `z.literal()`, `$ref` +
+// `$defs` for every reused schema — so tools defined in Zod hit this.
 const GEMINI_UNSUPPORTED_SCHEMA_KEYS = new Set([
     "additionalProperties",
     "$schema",
     "$id",
     "$ref",
+    "$defs",
+    "$comment",
     "definitions",
     "patternProperties",
+    "propertyNames",
     "unevaluatedProperties",
+    "unevaluatedItems",
     "dependentSchemas",
-    "dependentRequired"
+    "dependentRequired",
+    "exclusiveMinimum",
+    "exclusiveMaximum",
+    "const",
+    "allOf",
+    "oneOf",
+    "not",
+    "if",
+    "then",
+    "else",
+    "prefixItems",
+    "additionalItems",
+    "contains",
+    "minContains",
+    "maxContains",
+    "uniqueItems",
+    "multipleOf",
+    "examples",
+    "readOnly",
+    "writeOnly",
+    "deprecated",
+    "contentEncoding",
+    "contentMediaType"
 ]);
-function sanitizeGeminiSchema(value) {
-    if (Array.isArray(value)) {
-        return value.map(sanitizeGeminiSchema);
+/** Keywords whose value is data, not a schema — never recurse into them. */
+const GEMINI_DATA_KEYS = new Set([
+    "enum",
+    "const",
+    "default",
+    "example",
+    "examples"
+]);
+function isArraySchemaType(type) {
+    if (isString(type))
+        return type.toLowerCase() === "array";
+    if (Array.isArray(type)) {
+        return type.some((t) => isString(t) && t.toLowerCase() === "array");
+    }
+    return false;
+}
+function isPlainObject(value) {
+    return !!value && typeof value === "object" && !Array.isArray(value);
+}
+/** The JSON Schema type name for a primitive literal, if it has one. */
+function primitiveSchemaType(value) {
+    if (isString(value))
+        return "string";
+    if (isBoolean(value))
+        return "boolean";
+    if (isFiniteNumber(value)) {
+        return Number.isInteger(value) ? "integer" : "number";
     }
-    if (value && typeof value === "object") {
-        const out = {};
-        for (const [k, v] of Object.entries(value)) {
-            if (GEMINI_UNSUPPORTED_SCHEMA_KEYS.has(k))
+    return undefined;
+}
+function resolveJsonPointer(root, pointer) {
+    if (pointer === "#" || pointer === "")
+        return { found: true, value: root };
+    if (!pointer.startsWith("#/"))
+        return { found: false, value: undefined };
+    let cursor = root;
+    for (const rawSegment of pointer.slice(2).split("/")) {
+        const segment = decodeURIComponent(rawSegment)
+            .replace(/~1/g, "/")
+            .replace(/~0/g, "~");
+        if (Array.isArray(cursor)) {
+            const index = Number(segment);
+            if (!Number.isInteger(index) || index < 0 || index >= cursor.length) {
+                return { found: false, value: undefined };
+            }
+            cursor = cursor[index];
+            continue;
+        }
+        if (!isPlainObject(cursor) || !(segment in cursor)) {
+            return { found: false, value: undefined };
+        }
+        cursor = cursor[segment];
+    }
+    return { found: true, value: cursor };
+}
+function inlineGeminiRefs(node, root, seen) {
+    if (Array.isArray(node)) {
+        return node.map((item) => inlineGeminiRefs(item, root, seen));
+    }
+    // SAFETY: a schema node that is neither a list nor an object is a JSON
+    // scalar — this walker is only ever handed a decoded JSON Schema.
+    if (!isPlainObject(node))
+        return node;
+    if (isString(node.$ref)) {
+        const { $ref, ...rest } = node;
+        if (seen.has($ref))
+            return { type: "object", ...rest };
+        const { found, value } = resolveJsonPointer(root, $ref);
+        if (!found || !isPlainObject(value))
+            return { type: "object", ...rest };
+        // SAFETY: both inputs are objects, so both results are objects too.
+        const resolved = inlineGeminiRefs(value, root, new Set([...seen, $ref]));
+        // SAFETY: as above.
+        const overrides = inlineGeminiRefs(rest, root, seen);
+        return { ...resolved, ...overrides };
+    }
+    const out = {};
+    for (const [key, value] of Object.entries(node)) {
+        out[key] = GEMINI_DATA_KEYS.has(key)
+            ? // SAFETY: a data key's value is carried through verbatim; it is part of
+                // the same decoded JSON Schema.
+                value
+            : inlineGeminiRefs(value, root, seen);
+    }
+    return out;
+}
+/** Fold `allOf` members into the parent schema; parent keys win. */
+function mergeAllOf(out, members) {
+    for (const member of members) {
+        const sanitized = sanitizeSchemaNode(member);
+        if (!isPlainObject(sanitized))
+            continue;
+        for (const [key, value] of Object.entries(sanitized)) {
+            if (key === "properties" && isPlainObject(value)) {
+                // SAFETY: both sides are `properties` maps of the same schema.
+                out.properties = {
+                    ...value,
+                    ...(out.properties ?? {})
+                };
+                continue;
+            }
+            if (key === "required" && Array.isArray(value)) {
+                const existing = Array.isArray(out.required) ? out.required : [];
+                out.required = [...new Set([...existing, ...value])];
                 continue;
-            out[k] = sanitizeGeminiSchema(v);
+            }
+            if (out[key] === undefined)
+                out[key] = value;
+        }
+    }
+    return out;
+}
+function sanitizeSchemaNode(value) {
+    if (Array.isArray(value))
+        return value.map(sanitizeSchemaNode);
+    // SAFETY: as in `inlineGeminiRefs` — a non-list, non-object node is a scalar.
+    if (!isPlainObject(value))
+        return value;
+    const out = {};
+    for (const [key, nested] of Object.entries(value)) {
+        if (GEMINI_UNSUPPORTED_SCHEMA_KEYS.has(key))
+            continue;
+        if (key === "properties" && isPlainObject(nested)) {
+            const properties = {};
+            // Property *names* are data — a property called "const" must survive.
+            for (const [name, sub] of Object.entries(nested)) {
+                properties[name] = sanitizeSchemaNode(sub);
+            }
+            out.properties = properties;
+            continue;
+        }
+        if (key === "items") {
+            out.items = sanitizeSchemaNode(Array.isArray(nested) ? (nested[0] ?? { type: "string" }) : nested);
+            continue;
+        }
+        if (key === "anyOf" && Array.isArray(nested)) {
+            out.anyOf = nested.map(sanitizeSchemaNode);
+            continue;
         }
-        return out;
+        out[key] = GEMINI_DATA_KEYS.has(key)
+            ? // SAFETY: a data key's value is carried through verbatim; it is part of
+                // the same decoded JSON Schema.
+                nested
+            : sanitizeSchemaNode(nested);
+    }
+    // `oneOf` means the same thing to a model as `anyOf`, which Gemini accepts.
+    if (out.anyOf === undefined && Array.isArray(value.oneOf)) {
+        out.anyOf = value.oneOf.map(sanitizeSchemaNode);
+    }
+    if (Array.isArray(value.allOf))
+        mergeAllOf(out, value.allOf);
+    // `const` is what Zod emits for a literal. A single-value `enum` says the
+    // same thing in Gemini's dialect, but only for strings — its `enum` is a
+    // list of strings — so other literals keep the constraint in the description.
+    if (value.const !== undefined && out.enum === undefined) {
+        const literalType = primitiveSchemaType(value.const);
+        if (literalType === "string") {
+            // SAFETY: `primitiveSchemaType` proved the literal is a string.
+            out.enum = [value.const];
+            out.type ??= "string";
+        }
+        else if (literalType) {
+            out.type ??= literalType;
+            const hint = `Must be ${JSON.stringify(value.const)}.`;
+            out.description =
+                isNonEmptyString(out.description)
+                    ? `${out.description} ${hint}`
+                    : hint;
+        }
+    }
+    // Gemini's `type` is one string; JSON Schema allows a union. `["x","null"]`
+    // is Zod's optional/nullable shape and maps onto `nullable`.
+    if (Array.isArray(out.type)) {
+        const named = out.type.filter((t) => typeof t === "string" && t.toLowerCase() !== "null");
+        if (named.length < out.type.length)
+            out.nullable = true;
+        if (named.length > 0)
+            out.type = named[0];
+        else
+            delete out.type;
+    }
+    // Gemini rejects an array schema that omits `items` ("items: missing
+    // field"). JSON Schema allows it, so backfill a permissive default.
+    if (isArraySchemaType(out.type) && out.items === undefined) {
+        out.items = { type: "string" };
     }
-    return value;
+    return out;
+}
+function sanitizeGeminiSchema(value) {
+    return sanitizeSchemaNode(inlineGeminiRefs(value, value, new Set()));
 }
 function sanitizeToolName(name) {
     let sanitized = (name ?? "").trim();
-    sanitized = sanitized.replace(/[^a-zA-Z0-9_.:-]/g, "_");
+    sanitized = sanitized.replace(/[^a-zA-Z0-9_-]/g, "_");
     sanitized = sanitized.replace(/_+/g, "_");
     if (!sanitized)
         sanitized = "_tool";
@@ -47,12 +390,174 @@ function sanitizeToolName(name) {
         sanitized = "_tool";
     return sanitized;
 }
+function appendGeminiContent(contents, content) {
+    const previous = contents[contents.length - 1];
+    if (previous?.role === content.role) {
+        previous.parts.push(...content.parts);
+    }
+    else {
+        contents.push(content);
+    }
+}
+/**
+ * Sentinel Gemini accepts in place of a real thought signature. Gemini 3
+ * rejects a request whose history replays a function call with no signature —
+ * which is every turn recorded before signatures were persisted, and every
+ * call a caller injected by hand. The documented escape hatch is this literal:
+ * https://ai.google.dev/gemini-api/docs/generate-content/thought-signatures
+ * Reasoning continuity is lost for that call, so it is only ever a fallback
+ * for a signature we do not have.
+ */
+const GEMINI_UNSIGNED_CALL_SENTINEL = "skip_thought_signature_validator";
+/**
+ * Gemini validates that the *first* functionCall part of each model turn
+ * carries a signature. Stamp the sentinel on any that reaches us without one
+ * so an unsigned history fails soft (degraded reasoning) instead of 400.
+ */
+function signUnsignedFunctionCalls(contents) {
+    for (const content of contents) {
+        if (content.role !== "model")
+            continue;
+        const first = content.parts.find((p) => p.functionCall !== undefined);
+        if (first && !first.thoughtSignature) {
+            first.thoughtSignature = GEMINI_UNSIGNED_CALL_SENTINEL;
+        }
+    }
+}
+function geminiResponseError(data) {
+    if (data.error?.message)
+        return new Error(`Gemini API error: ${data.error.message}`);
+    if (data.promptFeedback?.blockReason) {
+        const detail = data.promptFeedback.blockReasonMessage
+            ? `: ${data.promptFeedback.blockReasonMessage}`
+            : "";
+        return new Error(`Gemini prompt blocked (${data.promptFeedback.blockReason})${detail}`);
+    }
+    return null;
+}
+function parseGeminiResponse(value) {
+    if (!isPlainObject(value)) {
+        throw new Error("Gemini returned an invalid response envelope");
+    }
+    const response = value;
+    if (response.candidates !== undefined &&
+        !Array.isArray(response.candidates)) {
+        throw new Error("Gemini returned invalid candidates");
+    }
+    for (const candidate of response.candidates ?? []) {
+        if (candidate.content?.parts !== undefined &&
+            !Array.isArray(candidate.content.parts)) {
+            throw new Error("Gemini returned invalid candidate parts");
+        }
+    }
+    return response;
+}
+/**
+ * Decode a `:generateContent` body: validate the envelope, raise the API's own
+ * error or block reason, and return the first candidate's parts.
+ *
+ * The live contract probe and the checked-in raw-response fixtures
+ * (`packages/runtime/tests/fixtures/provider-contract/`) run this same
+ * function, so a wire change surfaces here rather than only on a real call.
+ */
+export function decodeGeminiGenerateContent(value) {
+    const data = parseGeminiResponse(value);
+    const dataError = geminiResponseError(data);
+    if (dataError)
+        throw dataError;
+    const candidate = data.candidates?.[0];
+    if (!candidate?.content?.parts) {
+        throw new Error("Gemini returned no candidates");
+    }
+    return {
+        parts: candidate.content.parts,
+        finishReason: candidate.finishReason ?? null,
+        usage: data.usageMetadata
+    };
+}
+/**
+ * Decode one page of `GET /models`. Throws when the page carries no `models`
+ * array; {@link GeminiProvider.getAvailableLanguageModels} treats that the same
+ * way it treats an unreachable listing.
+ */
+export function decodeGeminiModelsPage(value) {
+    if (!isPlainObject(value)) {
+        throw new Error("Gemini model page is not an object");
+    }
+    const page = value;
+    if (!Array.isArray(page.models)) {
+        throw new Error("Gemini model page has no `models` array");
+    }
+    return page;
+}
+function normalizeEmbedding(values) {
+    const norm = Math.sqrt(values.reduce((sum, value) => sum + value * value, 0));
+    return norm > 0 ? values.map((value) => value / norm) : values;
+}
+function abortError(signal) {
+    return signal?.reason instanceof Error
+        ? signal.reason
+        : new DOMException("Aborted", "AbortError");
+}
+async function* decodeGeminiSse(body, signal) {
+    const reader = body.getReader();
+    const decoder = new TextDecoder();
+    let buffer = "";
+    try {
+        while (true) {
+            if (signal?.aborted)
+                throw abortError(signal);
+            const { done, value } = await reader.read();
+            buffer += decoder.decode(value, { stream: !done });
+            const events = buffer.split(/\r?\n\r?\n/);
+            buffer = events.pop() ?? "";
+            if (done && buffer.trim()) {
+                events.push(buffer);
+                buffer = "";
+            }
+            for (const eventText of events) {
+                const data = eventText
+                    .split(/\r?\n/)
+                    .filter((line) => line.startsWith("data:"))
+                    .map((line) => line.slice(5).trimStart())
+                    .join("\n")
+                    .trim();
+                if (!data || data === "[DONE]")
+                    continue;
+                let parsed;
+                try {
+                    parsed = JSON.parse(data);
+                }
+                catch (error) {
+                    throw new Error("Gemini returned malformed SSE JSON", {
+                        cause: error
+                    });
+                }
+                if (!isPlainObject(parsed)) {
+                    throw new Error("Gemini returned an invalid SSE event");
+                }
+                yield parseGeminiResponse(parsed);
+            }
+            if (done)
+                break;
+        }
+    }
+    finally {
+        // Stop the underlying connection whenever the consumer bails early (abort
+        // or `break`); releasing the lock alone leaves the HTTP body undrained.
+        await reader
+            .cancel(signal?.aborted ? signal.reason : undefined)
+            .catch(() => undefined);
+        reader.releaseLock();
+    }
+}
 export class GeminiProvider extends BaseProvider {
     static requiredSecrets() {
         return ["GEMINI_API_KEY"];
     }
     apiKey;
     _fetch;
+    _sleep;
     constructor(secrets, options = {}) {
         super("gemini");
         const apiKey = secrets.GEMINI_API_KEY;
@@ -61,6 +566,9 @@ export class GeminiProvider extends BaseProvider {
         }
         this.apiKey = apiKey;
         this._fetch = options.fetchFn ?? globalThis.fetch.bind(globalThis);
+        this._sleep =
+            options.sleepFn ??
+                ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));
     }
     getContainerEnv() {
         return { GEMINI_API_KEY: this.apiKey };
@@ -68,33 +576,55 @@ export class GeminiProvider extends BaseProvider {
     async hasToolSupport(_model) {
         return true;
     }
+    get supportsNativeWebSearch() {
+        return true;
+    }
+    /** Gemini reads a whole clip: see {@link videoContentToGeminiPart}. */
+    get supportsVideoInput() {
+        return true;
+    }
     // ---------------------------------------------------------------------------
     // Model listing
     // ---------------------------------------------------------------------------
     async getAvailableLanguageModels() {
-        const url = `${GEMINI_API_BASE}/models?key=${this.apiKey}`;
-        let response;
+        const items = [];
+        let pageToken;
         try {
-            response = await this._fetch(url);
+            do {
+                const query = new URLSearchParams({
+                    key: this.apiKey,
+                    pageSize: "1000"
+                });
+                if (pageToken)
+                    query.set("pageToken", pageToken);
+                const response = await this._fetch(`${GEMINI_API_BASE}/models?${query}`);
+                if (!response.ok)
+                    return [];
+                const payload = decodeGeminiModelsPage(await response.json());
+                items.push(...(payload.models ?? []));
+                pageToken = payload.nextPageToken;
+            } while (pageToken);
         }
         catch {
             return [];
         }
-        if (!response.ok)
-            return [];
-        const payload = (await response.json());
-        const items = payload.models ?? [];
+        const seen = new Set();
         return items
             .filter((m) => (m.supportedGenerationMethods ?? []).includes("generateContent"))
             .filter((m) => !!m.name)
+            .filter((m) => !/(embedding|aqa|imagen|veo|image|tts)/i.test(m.name ?? ""))
             .map((m) => {
             const id = m.name.split("/").pop();
+            if (seen.has(id))
+                return null;
+            seen.add(id);
             return {
                 id,
                 name: m.displayName ?? id,
                 provider: "gemini"
             };
-        });
+        })
+            .filter((model) => model !== null);
     }
     // ---------------------------------------------------------------------------
     // Message conversion helpers
@@ -107,26 +637,43 @@ export class GeminiProvider extends BaseProvider {
             const img = content.image;
             let base64Data;
             let mimeType = img.mimeType ?? "image/jpeg";
-            if (img.data) {
-                if (typeof img.data === "string") {
-                    base64Data = img.data;
+            const parseImageDataUri = (uri) => {
+                const idx = uri.indexOf(",");
+                if (idx < 0)
+                    throw new Error("Invalid image data URI");
+                const header = uri.slice(5, idx);
+                mimeType = header.split(";")[0] || mimeType;
+                return uri.slice(idx + 1);
+            };
+            if (isNonEmptyString(img.data) ||
+                (img.data instanceof Uint8Array && img.data.length > 0)) {
+                if (isString(img.data)) {
+                    // Inline data may itself be a data: URI — strip the prefix and take
+                    // the real mime type from it rather than shipping the header as
+                    // base64 payload.
+                    base64Data = img.data.startsWith("data:")
+                        ? parseImageDataUri(img.data)
+                        : img.data;
                 }
                 else {
                     base64Data = Buffer.from(img.data).toString("base64");
                 }
             }
             else if (img.uri) {
-                if (img.uri.startsWith("data:")) {
-                    const idx = img.uri.indexOf(",");
-                    const header = img.uri.slice(5, idx);
-                    mimeType = header.split(";")[0] || mimeType;
-                    base64Data = img.uri.slice(idx + 1);
+                // resolveUri turns asset file:// URIs (what the chat pipeline produces)
+                // into data: URIs; http(s) URIs pass through to safeFetch.
+                const resolvedUri = img.uri.startsWith("data:")
+                    ? img.uri
+                    : await this.resolveUri(img.uri);
+                if (resolvedUri.startsWith("data:")) {
+                    base64Data = parseImageDataUri(resolvedUri);
                 }
                 else {
-                    const resp = await this._fetch(img.uri);
+                    const resp = await safeFetch(resolvedUri, undefined, 5, this._fetch);
                     if (!resp.ok)
                         throw new Error(`Failed to fetch image: ${resp.status}`);
-                    mimeType = resp.headers.get("content-type") ?? mimeType;
+                    mimeType =
+                        stripMimeParams(resp.headers.get("content-type")) ?? mimeType;
                     base64Data = Buffer.from(await resp.arrayBuffer()).toString("base64");
                 }
             }
@@ -138,87 +685,271 @@ export class GeminiProvider extends BaseProvider {
         if (content.type === "audio") {
             const aud = content.audio;
             let base64Data;
-            let mimeType = aud.mimeType ?? "audio/mp3";
-            if (aud.data) {
-                if (typeof aud.data === "string") {
-                    base64Data = aud.data;
+            let mimeType = aud.mimeType;
+            const parseAudioDataUri = (uri) => {
+                const idx = uri.indexOf(",");
+                if (idx < 0)
+                    throw new Error("Invalid audio data URI");
+                const header = uri.slice(5, idx);
+                mimeType = mimeType ?? header.split(";")[0];
+                return uri.slice(idx + 1);
+            };
+            if (isNonEmptyString(aud.data) ||
+                (aud.data instanceof Uint8Array && aud.data.length > 0)) {
+                if (isString(aud.data)) {
+                    base64Data = aud.data.startsWith("data:")
+                        ? parseAudioDataUri(aud.data)
+                        : aud.data;
+                    mimeType =
+                        mimeType ?? sniffAudioMime(Buffer.from(base64Data, "base64"));
                 }
                 else {
-                    base64Data = Buffer.from(aud.data).toString("base64");
+                    const bytes = Buffer.from(aud.data);
+                    base64Data = bytes.toString("base64");
+                    mimeType = mimeType ?? sniffAudioMime(bytes);
                 }
             }
             else if (aud.uri) {
-                if (aud.uri.startsWith("data:")) {
-                    const idx = aud.uri.indexOf(",");
-                    const header = aud.uri.slice(5, idx);
-                    mimeType = header.split(";")[0] || mimeType;
-                    base64Data = aud.uri.slice(idx + 1);
+                const resolvedUri = aud.uri.startsWith("data:")
+                    ? aud.uri
+                    : await this.resolveUri(aud.uri);
+                if (resolvedUri.startsWith("data:")) {
+                    base64Data = parseAudioDataUri(resolvedUri);
                 }
                 else {
-                    const resp = await this._fetch(aud.uri);
+                    const resp = await safeFetch(resolvedUri, undefined, 5, this._fetch);
                     if (!resp.ok)
                         throw new Error(`Failed to fetch audio: ${resp.status}`);
-                    mimeType = resp.headers.get("content-type") ?? mimeType;
-                    base64Data = Buffer.from(await resp.arrayBuffer()).toString("base64");
+                    const bytes = Buffer.from(await resp.arrayBuffer());
+                    mimeType =
+                        stripMimeParams(resp.headers.get("content-type")) ??
+                            mimeType ??
+                            sniffAudioMime(bytes);
+                    base64Data = bytes.toString("base64");
                 }
             }
             else {
                 base64Data = "";
             }
-            return { inlineData: { mimeType, data: base64Data } };
+            return {
+                inlineData: { mimeType: geminiAudioMime(mimeType), data: base64Data }
+            };
+        }
+        if (content.type === "video") {
+            return await this.videoContentToGeminiPart(content.video);
         }
         return { text: "[unsupported content type]" };
     }
+    /**
+     * Source a video's bytes the way the image and audio branches do, then send
+     * them inline when small enough and through the Files API when not.
+     */
+    async videoContentToGeminiPart(video) {
+        let bytes;
+        let mimeType = video.mimeType;
+        const parseVideoDataUri = (uri) => {
+            const idx = uri.indexOf(",");
+            if (idx < 0)
+                throw new Error("Invalid video data URI");
+            const header = uri.slice(5, idx);
+            mimeType = mimeType ?? stripMimeParams(header.split(";base64")[0]);
+            return Buffer.from(uri.slice(idx + 1), "base64");
+        };
+        if (isNonEmptyString(video.data) ||
+            (video.data instanceof Uint8Array && video.data.length > 0)) {
+            if (isString(video.data)) {
+                bytes = video.data.startsWith("data:")
+                    ? parseVideoDataUri(video.data)
+                    : Buffer.from(video.data, "base64");
+            }
+            else {
+                bytes = Buffer.from(video.data);
+            }
+        }
+        else if (video.uri) {
+            const resolvedUri = video.uri.startsWith("data:")
+                ? video.uri
+                : await this.resolveUri(video.uri);
+            if (resolvedUri.startsWith("data:")) {
+                bytes = parseVideoDataUri(resolvedUri);
+            }
+            else {
+                const resp = await safeFetch(resolvedUri, undefined, 5, this._fetch);
+                if (!resp.ok)
+                    throw new Error(`Failed to fetch video: ${resp.status}`);
+                mimeType =
+                    stripMimeParams(resp.headers.get("content-type")) ?? mimeType;
+                bytes = Buffer.from(await resp.arrayBuffer());
+            }
+        }
+        else {
+            bytes = Buffer.alloc(0);
+        }
+        const resolvedMime = mimeType ?? sniffVideoMime(bytes);
+        if (bytes.length > GEMINI_INLINE_VIDEO_MAX_BYTES) {
+            const fileUri = await this.uploadFileToGemini(bytes, resolvedMime);
+            return { fileData: { mimeType: resolvedMime, fileUri } };
+        }
+        return {
+            inlineData: { mimeType: resolvedMime, data: bytes.toString("base64") }
+        };
+    }
+    /**
+     * Upload bytes with the Files API resumable protocol and wait until the file
+     * is ACTIVE — Gemini rejects a `fileData` part that references a file still
+     * being processed. Returns the file's uri.
+     */
+    async uploadFileToGemini(bytes, mimeType) {
+        const startResp = await this._fetch(`${GEMINI_UPLOAD_BASE}?key=${this.apiKey}`, {
+            method: "POST",
+            headers: {
+                "X-Goog-Upload-Protocol": "resumable",
+                "X-Goog-Upload-Command": "start",
+                "X-Goog-Upload-Header-Content-Length": String(bytes.length),
+                "X-Goog-Upload-Header-Content-Type": mimeType,
+                "Content-Type": "application/json"
+            },
+            body: JSON.stringify({
+                file: { display_name: `nodetool-upload.${mimeType.split("/")[1]}` }
+            })
+        });
+        if (!startResp.ok) {
+            const errText = await startResp.text();
+            throw new Error(`Gemini file upload failed to start ${startResp.status}: ${errText}`);
+        }
+        const uploadUrl = startResp.headers.get("x-goog-upload-url");
+        if (!uploadUrl) {
+            throw new Error("Gemini file upload returned no X-Goog-Upload-URL");
+        }
+        const uploadResp = await this._fetch(uploadUrl, {
+            method: "POST",
+            headers: {
+                "Content-Length": String(bytes.length),
+                "X-Goog-Upload-Offset": "0",
+                "X-Goog-Upload-Command": "upload, finalize"
+            },
+            body: new Blob([new Uint8Array(bytes)], {
+                type: mimeType
+            })
+        });
+        if (!uploadResp.ok) {
+            const errText = await uploadResp.text();
+            throw new Error(`Gemini file upload failed ${uploadResp.status}: ${errText}`);
+        }
+        const uploaded = (await uploadResp.json());
+        let file = uploaded.file;
+        if (!file?.name || !file.uri) {
+            throw new Error("Gemini file upload returned no file record");
+        }
+        const fileUri = file.uri;
+        const fileName = file.name;
+        for (let poll = 0; poll < GEMINI_FILE_MAX_POLLS; poll++) {
+            if (file?.state === "ACTIVE")
+                return fileUri;
+            if (file?.state === "FAILED") {
+                throw new Error(`Gemini file processing failed: ${file.error?.message ?? "unknown error"}`);
+            }
+            await this._sleep(GEMINI_FILE_POLL_INTERVAL_MS);
+            const pollResp = await this._fetch(`${GEMINI_API_BASE}/${fileName}`, {
+                headers: { "x-goog-api-key": this.apiKey }
+            });
+            if (!pollResp.ok) {
+                const errText = await pollResp.text();
+                throw new Error(`Gemini file poll failed ${pollResp.status}: ${errText}`);
+            }
+            file = (await pollResp.json());
+        }
+        if (file?.state === "ACTIVE")
+            return fileUri;
+        throw new Error(`Gemini file ${fileName} did not become ACTIVE in time`);
+    }
     /**
      * Convert our Message array into Gemini contents + optional system instruction.
      */
-    async convertMessages(messages) {
+    async convertMessages(messages, nameMap = new Map()) {
         let systemInstruction;
         const contents = [];
+        // Gemini correlates a tool result to its call by the function *name*, not
+        // by id (and our tool-call ids are synthesized — never valid Gemini
+        // function names). Map each tool-call id back to its function name so the
+        // `functionResponse.name` below matches the earlier `functionCall.name`.
+        const toolCallNames = new Map();
+        for (const m of messages) {
+            if (m.role === "assistant" && m.toolCalls) {
+                for (const tc of m.toolCalls) {
+                    if (tc.id)
+                        toolCallNames.set(tc.id, nameMap.get(tc.name) ?? tc.name);
+                }
+            }
+        }
         for (const msg of messages) {
             if (msg.role === "system") {
-                systemInstruction =
-                    typeof msg.content === "string"
-                        ? msg.content
-                        : (msg.content ?? [])
-                            .filter((c) => c.type === "text")
-                            .map((c) => c.text)
-                            .join(" ");
+                const instruction = isString(msg.content)
+                    ? msg.content
+                    : (msg.content ?? [])
+                        .filter((c) => c.type === "text")
+                        .map((c) => c.text)
+                        .join(" ");
+                systemInstruction = systemInstruction
+                    ? `${systemInstruction}\n${instruction}`
+                    : instruction;
                 continue;
             }
             if (msg.role === "tool") {
-                // Tool result → model role with functionResponse part
-                const responseText = typeof msg.content === "string"
+                // Tool result → user role with functionResponse part. The name must
+                // match the originating functionCall's name, resolved from the call id.
+                const responseText = isString(msg.content)
                     ? msg.content
                     : JSON.stringify(msg.content);
-                contents.push({
-                    role: "user",
-                    parts: [
-                        {
-                            functionResponse: {
-                                name: msg.toolCallId ?? "unknown",
-                                response: { result: responseText }
-                            }
-                        }
-                    ]
-                });
+                const functionName = (msg.toolCallId ? toolCallNames.get(msg.toolCallId) : undefined) ??
+                    msg.toolCallId ??
+                    "unknown";
+                const responsePart = {
+                    functionResponse: {
+                        name: functionName,
+                        id: msg.toolCallId ?? undefined,
+                        response: { result: responseText }
+                    }
+                };
+                // Merge parallel tool results into a single user turn so the request
+                // keeps alternating user/model roles.
+                const prev = contents[contents.length - 1];
+                if (prev &&
+                    prev.role === "user" &&
+                    prev.parts.length > 0 &&
+                    prev.parts.every((p) => p.functionResponse !== undefined)) {
+                    prev.parts.push(responsePart);
+                }
+                else {
+                    appendGeminiContent(contents, {
+                        role: "user",
+                        parts: [responsePart]
+                    });
+                }
                 continue;
             }
             if (msg.role === "assistant") {
                 // If we have raw Gemini parts (with thought content), replay them exactly
                 if (msg._rawGeminiParts && Array.isArray(msg._rawGeminiParts)) {
-                    contents.push({
+                    // Copy: the contents we build get merged and stamped below, and the
+                    // message's own parts must survive that untouched for the next turn.
+                    appendGeminiContent(contents, {
                         role: "model",
-                        parts: msg._rawGeminiParts
+                        parts: msg._rawGeminiParts.map((part) => ({
+                            ...part
+                        }))
                     });
                     continue;
                 }
                 const parts = [];
-                // Tool calls
                 if (msg.toolCalls && msg.toolCalls.length > 0) {
                     for (const tc of msg.toolCalls) {
                         const part = {
-                            functionCall: { name: tc.name, args: tc.args }
+                            functionCall: {
+                                id: tc.id,
+                                name: nameMap.get(tc.name) ?? tc.name,
+                                args: tc.args
+                            }
                         };
                         if (tc.thought_signature) {
                             part.thoughtSignature = tc.thought_signature;
@@ -226,8 +957,7 @@ export class GeminiProvider extends BaseProvider {
                         parts.push(part);
                     }
                 }
-                // Text / content
-                if (typeof msg.content === "string" && msg.content) {
+                if (isNonEmptyString(msg.content)) {
                     parts.push({ text: msg.content });
                 }
                 else if (Array.isArray(msg.content)) {
@@ -236,13 +966,12 @@ export class GeminiProvider extends BaseProvider {
                     }
                 }
                 if (parts.length > 0) {
-                    contents.push({ role: "model", parts });
+                    appendGeminiContent(contents, { role: "model", parts });
                 }
                 continue;
             }
-            // user
             const parts = [];
-            if (typeof msg.content === "string") {
+            if (isString(msg.content)) {
                 parts.push({ text: msg.content });
             }
             else if (Array.isArray(msg.content)) {
@@ -251,17 +980,26 @@ export class GeminiProvider extends BaseProvider {
                 }
             }
             if (parts.length > 0) {
-                contents.push({ role: "user", parts });
+                appendGeminiContent(contents, { role: "user", parts });
             }
         }
+        signUnsignedFunctionCalls(contents);
         return { contents, systemInstruction };
     }
     formatTools(tools) {
+        return this.formatToolsForModel(tools, "gemini-3");
+    }
+    formatToolsForModel(tools, model) {
         const nameMap = new Map();
         const reverseMap = new Map();
         const usedNames = new Set();
         const declarations = [];
         for (const tool of tools) {
+            if ((tool.name === WEB_SEARCH_TOOL_NAME &&
+                this.usesNativeWebSearch(model, tools)) ||
+                tool.type === "code_interpreter") {
+                continue;
+            }
             const original = tool.name;
             let unique = sanitizeToolName(original);
             let suffix = 2;
@@ -273,7 +1011,10 @@ export class GeminiProvider extends BaseProvider {
             usedNames.add(unique);
             nameMap.set(original, unique);
             reverseMap.set(unique, original);
-            const rawParameters = tool.inputSchema ?? { type: "object", properties: {} };
+            const rawParameters = tool.inputSchema ?? {
+                type: "object",
+                properties: {}
+            };
             declarations.push({
                 name: unique,
                 description: tool.description ?? "",
@@ -286,20 +1027,82 @@ export class GeminiProvider extends BaseProvider {
             reverseMap
         };
     }
+    usesNativeWebSearch(model, tools) {
+        const hasWebSearch = tools.some((tool) => tool.name === WEB_SEARCH_TOOL_NAME);
+        if (!hasWebSearch)
+            return false;
+        const hasCustomTools = this.hasCustomFunctionTools(tools);
+        return !hasCustomTools || isGemini3Model(model);
+    }
+    hasCustomFunctionTools(tools) {
+        return tools.some((tool) => tool.name !== WEB_SEARCH_TOOL_NAME &&
+            tool.type !== "code_interpreter");
+    }
+    validateToolCombination(model, tools) {
+        if (isGemini3Model(model))
+            return;
+        const hasCodeExecution = tools.some((tool) => tool.type === "code_interpreter");
+        if (hasCodeExecution && this.hasCustomFunctionTools(tools)) {
+            throw new Error(`Gemini code execution cannot be combined with other tools on ${model}; ` +
+                "only Gemini 3 models support this combination");
+        }
+    }
+    /**
+     * Fill in `tools` and `toolConfig` on a request body.
+     *
+     * Built-in tools (googleSearch, codeExecution) run server side. When they are
+     * combined with functionDeclarations, Gemini rejects the request unless
+     * `toolConfig.includeServerSideToolInvocations` is set — the built-in calls
+     * and their results have to be echoed back into the conversation for the
+     * function-calling loop to stay coherent.
+     */
+    applyTools(body, model, tools, geminiTools, nameMap, toolChoice) {
+        if (geminiTools.length > 0) {
+            body.tools = geminiTools;
+        }
+        let hasBuiltIn = false;
+        if (this.usesNativeWebSearch(model, tools)) {
+            body.tools = [...(body.tools ?? []), { googleSearch: {} }];
+            hasBuiltIn = true;
+        }
+        if (tools.some((tool) => tool.type === "code_interpreter")) {
+            body.tools = [...(body.tools ?? []), { codeExecution: {} }];
+            hasBuiltIn = true;
+        }
+        const toolConfig = {};
+        if (hasBuiltIn && geminiTools.length > 0) {
+            toolConfig.includeServerSideToolInvocations = true;
+        }
+        if (toolChoice &&
+            (toolChoice === "any" ? geminiTools.length > 0 : nameMap.has(toolChoice))) {
+            const selected = toolChoice === "any"
+                ? undefined
+                : (nameMap.get(toolChoice) ?? sanitizeToolName(toolChoice));
+            const functionCallingConfig = {
+                mode: "ANY"
+            };
+            if (selected) {
+                functionCallingConfig.allowedFunctionNames = [selected];
+            }
+            toolConfig.functionCallingConfig = functionCallingConfig;
+        }
+        if (Object.keys(toolConfig).length > 0) {
+            body.toolConfig = toolConfig;
+        }
+    }
     // ---------------------------------------------------------------------------
     // Non-streaming generation
     // ---------------------------------------------------------------------------
     async generateMessage(args) {
-        const { model, tools = [], maxTokens = 16384, temperature, topP } = args;
-        const { contents, systemInstruction } = await this.convertMessages(args.messages);
-        const { geminiTools, reverseMap } = this.formatTools(tools);
+        const { model, tools = [], maxTokens = 16384, temperature, topP, presencePenalty, frequencyPenalty } = args;
+        this.validateToolCombination(model, tools);
+        const { geminiTools, nameMap, reverseMap } = this.formatToolsForModel(tools, model);
+        const { contents, systemInstruction } = await this.convertMessages(args.messages, nameMap);
         const body = { contents };
         if (systemInstruction) {
             body.systemInstruction = { parts: [{ text: systemInstruction }] };
         }
-        if (geminiTools.length > 0) {
-            body.tools = geminiTools;
-        }
+        this.applyTools(body, model, tools, geminiTools, nameMap, args.toolChoice);
         const generationConfig = {
             maxOutputTokens: maxTokens
         };
@@ -307,13 +1110,19 @@ export class GeminiProvider extends BaseProvider {
             generationConfig.temperature = temperature;
         if (topP != null)
             generationConfig.topP = topP;
+        if (presencePenalty != null)
+            generationConfig.presencePenalty = presencePenalty;
+        if (frequencyPenalty != null)
+            generationConfig.frequencyPenalty = frequencyPenalty;
         body.generationConfig = generationConfig;
         log.debug("Gemini request", { model });
         const url = `${GEMINI_API_BASE}/models/${model}:generateContent?key=${this.apiKey}`;
+        this.recordRequestPayload(body);
         const response = await this._fetch(url, {
             method: "POST",
             headers: { "Content-Type": "application/json" },
-            body: JSON.stringify(body)
+            body: JSON.stringify(body),
+            signal: args.signal
         });
         if (!response.ok) {
             const text = await response.text();
@@ -323,30 +1132,33 @@ export class GeminiProvider extends BaseProvider {
             });
             throw new Error(`Gemini API error ${response.status}: ${text}`);
         }
-        const data = (await response.json());
-        if (data.error) {
-            throw new Error(`Gemini API error: ${data.error.message}`);
-        }
-        const candidate = data.candidates?.[0];
-        if (!candidate?.content?.parts) {
-            throw new Error("Gemini returned no candidates");
-        }
-        return this.extractMessage(candidate.content.parts, reverseMap);
+        const decoded = decodeGeminiGenerateContent(await response.json());
+        this.trackGeminiUsage(model, decoded.usage);
+        return this.extractMessage(decoded.parts, reverseMap);
+    }
+    /** Record token usage from a Gemini usageMetadata block (if present). */
+    trackGeminiUsage(model, usage) {
+        if (!usage)
+            return;
+        this.trackUsage(model, {
+            inputTokens: usage.promptTokenCount ?? 0,
+            outputTokens: (usage.candidatesTokenCount ?? 0) + (usage.thoughtsTokenCount ?? 0),
+            cachedTokens: usage.cachedContentTokenCount ?? 0
+        });
     }
     // ---------------------------------------------------------------------------
     // Streaming generation
     // ---------------------------------------------------------------------------
     async *generateMessages(args) {
-        const { model, tools = [], maxTokens = 16384, temperature, topP } = args;
-        const { contents, systemInstruction } = await this.convertMessages(args.messages);
-        const { geminiTools, reverseMap } = this.formatTools(tools);
+        const { model, tools = [], maxTokens = 16384, temperature, topP, presencePenalty, frequencyPenalty } = args;
+        this.validateToolCombination(model, tools);
+        const { geminiTools, nameMap, reverseMap } = this.formatToolsForModel(tools, model);
+        const { contents, systemInstruction } = await this.convertMessages(args.messages, nameMap);
         const body = { contents };
         if (systemInstruction) {
             body.systemInstruction = { parts: [{ text: systemInstruction }] };
         }
-        if (geminiTools.length > 0) {
-            body.tools = geminiTools;
-        }
+        this.applyTools(body, model, tools, geminiTools, nameMap, args.toolChoice);
         const generationConfig = {
             maxOutputTokens: maxTokens
         };
@@ -354,13 +1166,19 @@ export class GeminiProvider extends BaseProvider {
             generationConfig.temperature = temperature;
         if (topP != null)
             generationConfig.topP = topP;
+        if (presencePenalty != null)
+            generationConfig.presencePenalty = presencePenalty;
+        if (frequencyPenalty != null)
+            generationConfig.frequencyPenalty = frequencyPenalty;
         body.generationConfig = generationConfig;
         log.debug("Gemini request", { model });
         const url = `${GEMINI_API_BASE}/models/${model}:streamGenerateContent?alt=sse&key=${this.apiKey}`;
+        this.recordRequestPayload(body);
         const response = await this._fetch(url, {
             method: "POST",
             headers: { "Content-Type": "application/json" },
-            body: JSON.stringify(body)
+            body: JSON.stringify(body),
+            signal: args.signal
         });
         if (!response.ok) {
             const text = await response.text();
@@ -373,71 +1191,51 @@ export class GeminiProvider extends BaseProvider {
         if (!response.body) {
             throw new Error("Gemini streaming response has no body");
         }
-        const reader = response.body.getReader();
-        const decoder = new TextDecoder();
-        let buffer = "";
         // Accumulate all parts across SSE events for raw replay.
         // Gemini thinking models emit thought parts and function calls across
         // separate SSE events, but they must all be sent back together.
         const allParts = [];
         const pendingToolCalls = [];
-        try {
-            while (true) {
-                const { done, value } = await reader.read();
-                if (done)
-                    break;
-                buffer += decoder.decode(value, { stream: true });
-                const lines = buffer.split("\n");
-                buffer = lines.pop() ?? "";
-                for (const line of lines) {
-                    if (!line.startsWith("data: "))
-                        continue;
-                    const jsonStr = line.slice(6).trim();
-                    if (!jsonStr || jsonStr === "[DONE]")
-                        continue;
-                    let event;
-                    try {
-                        event = JSON.parse(jsonStr);
-                    }
-                    catch {
-                        continue;
-                    }
-                    const parts = event.candidates?.[0]?.content?.parts;
-                    if (!parts)
-                        continue;
-                    for (const part of parts) {
-                        allParts.push(part);
-                        if (part.text !== undefined && !part.thought) {
-                            const chunk = {
-                                type: "chunk",
-                                content: part.text,
-                                done: false
-                            };
-                            yield chunk;
-                        }
-                        else if (part.functionCall) {
-                            const originalName = reverseMap.get(part.functionCall.name) ??
-                                part.functionCall.name;
-                            const toolCall = {
-                                id: `call_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`,
-                                name: originalName,
-                                args: part.functionCall.args ?? {}
-                            };
-                            if (part.thoughtSignature) {
-                                toolCall.thought_signature = part.thoughtSignature;
-                            }
-                            pendingToolCalls.push(toolCall);
-                        }
-                        // Thought text parts are silently accumulated in allParts
+        // Gemini SSE reports CUMULATIVE usageMetadata; keep the last one seen and
+        // record it once after the stream (accumulating each event would over-count).
+        let lastUsage;
+        for await (const event of decodeGeminiSse(response.body, args.signal)) {
+            const eventError = geminiResponseError(event);
+            if (eventError)
+                throw eventError;
+            if (event.usageMetadata)
+                lastUsage = event.usageMetadata;
+            const parts = event.candidates?.[0]?.content?.parts;
+            if (!parts)
+                continue;
+            for (const part of parts) {
+                allParts.push(part);
+                if (part.text !== undefined && !part.thought) {
+                    const chunk = {
+                        type: "chunk",
+                        content: part.text,
+                        done: false
+                    };
+                    yield chunk;
+                }
+                else if (part.functionCall) {
+                    const originalName = reverseMap.get(part.functionCall.name) ?? part.functionCall.name;
+                    const toolCall = {
+                        id: part.functionCall.id ??
+                            `call_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`,
+                        name: originalName,
+                        args: part.functionCall.args ?? {}
+                    };
+                    if (part.thoughtSignature) {
+                        toolCall.thought_signature = part.thoughtSignature;
                     }
+                    pendingToolCalls.push(toolCall);
                 }
             }
         }
-        finally {
-            reader.releaseLock();
-        }
+        this.trackGeminiUsage(model, lastUsage);
         // Attach accumulated raw parts to tool calls for thought replay
-        const hasThoughts = allParts.some((p) => p.thought);
+        const hasThoughts = allParts.some((p) => p.thought || p.thoughtSignature);
         for (const tc of pendingToolCalls) {
             if (hasThoughts) {
                 tc._rawGeminiParts = allParts;
@@ -459,13 +1257,14 @@ export class GeminiProvider extends BaseProvider {
         const textParts = [];
         const toolCalls = [];
         for (const part of parts) {
-            if (part.text !== undefined) {
+            if (part.text !== undefined && !part.thought) {
                 textParts.push(part.text);
             }
             else if (part.functionCall) {
                 const originalName = reverseMap.get(part.functionCall.name) ?? part.functionCall.name;
                 const tc = {
-                    id: `call_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`,
+                    id: part.functionCall.id ??
+                        `call_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`,
                     name: originalName,
                     args: part.functionCall.args ?? {}
                 };
@@ -475,7 +1274,7 @@ export class GeminiProvider extends BaseProvider {
                 toolCalls.push(tc);
             }
         }
-        const hasThoughts = parts.some((p) => p.thought);
+        const hasThoughts = parts.some((p) => p.thought || p.thoughtSignature);
         const msg = {
             role: "assistant",
             content: textParts.join("") || null,
@@ -492,11 +1291,50 @@ export class GeminiProvider extends BaseProvider {
     async getAvailableImageModels() {
         return [
             {
-                id: "gemini-2.0-flash-preview-image-generation",
-                name: "Gemini 2.0 Flash Image Gen",
-                provider: "gemini"
+                id: "gemini-3.1-flash-image",
+                name: "Gemini 3.1 Flash Image",
+                provider: "gemini",
+                supportedTasks: ["text_to_image", "image_to_image"],
+                aspectRatios: [
+                    "1:1",
+                    "2:3",
+                    "3:2",
+                    "3:4",
+                    "4:3",
+                    "4:5",
+                    "5:4",
+                    "9:16",
+                    "16:9",
+                    "21:9"
+                ],
+                resolutions: ["1K", "2K", "4K"]
             },
-            { id: "imagen-3.0-generate-002", name: "Imagen 3.0", provider: "gemini" }
+            {
+                id: "gemini-3.1-flash-lite-image",
+                name: "Gemini 3.1 Flash-Lite Image",
+                provider: "gemini",
+                supportedTasks: ["text_to_image", "image_to_image"],
+                aspectRatios: ["1:1", "2:3", "3:2", "3:4", "4:3", "9:16", "16:9"]
+            },
+            {
+                id: "gemini-3-pro-image",
+                name: "Gemini 3 Pro Image",
+                provider: "gemini",
+                supportedTasks: ["text_to_image", "image_to_image"],
+                aspectRatios: [
+                    "1:1",
+                    "2:3",
+                    "3:2",
+                    "3:4",
+                    "4:3",
+                    "4:5",
+                    "5:4",
+                    "9:16",
+                    "16:9",
+                    "21:9"
+                ],
+                resolutions: ["1K", "2K", "4K"]
+            }
         ];
     }
     async getAvailableTTSModels() {
@@ -533,6 +1371,30 @@ export class GeminiProvider extends BaseProvider {
             "Sulafat"
         ];
         return [
+            {
+                id: "gemini-3.8-flash-tts",
+                name: "Gemini 3.8 Flash TTS",
+                provider: "gemini",
+                voices
+            },
+            {
+                id: "gemini-3.8-flash-lite-tts",
+                name: "Gemini 3.8 Flash-Lite TTS",
+                provider: "gemini",
+                voices
+            },
+            {
+                id: "gemini-3.1-flash-tts-preview",
+                name: "Gemini 3.1 Flash TTS Preview",
+                provider: "gemini",
+                voices
+            },
+            {
+                id: "gemini-2.5-flash-preview-tts",
+                name: "Gemini 2.5 Flash TTS",
+                provider: "gemini",
+                voices
+            },
             {
                 id: "gemini-2.5-pro-preview-tts",
                 name: "Gemini 2.5 Pro TTS",
@@ -543,27 +1405,51 @@ export class GeminiProvider extends BaseProvider {
     }
     async getAvailableASRModels() {
         return [
-            { id: "gemini-2.0-flash", name: "Gemini 2.0 Flash", provider: "gemini" },
-            { id: "gemini-2.5-flash", name: "Gemini 2.5 Flash", provider: "gemini" }
+            {
+                // The transcription-specific model. It answers `generateContent`, so
+                // it runs through the same path as the general models below.
+                id: "gemini-3.5-transcribe",
+                name: "Gemini 3.5 Transcribe",
+                provider: "gemini"
+            },
+            {
+                id: "gemini-3.5-flash",
+                name: "Gemini 3.5 Flash",
+                provider: "gemini"
+            },
+            {
+                id: "gemini-3.1-flash-lite",
+                name: "Gemini 3.1 Flash-Lite",
+                provider: "gemini"
+            }
         ];
     }
     async getAvailableVideoModels() {
+        return [...GEMINI_VIDEO_MODELS];
+    }
+    async getAvailableMusicModels() {
         return [
             {
-                id: "veo-3.1-generate-preview",
-                name: "Veo 3.1 Preview",
-                provider: "gemini"
+                id: "lyria-3.5",
+                name: "Lyria 3.5",
+                provider: "gemini",
+                supportedTasks: ["text_to_music"]
             },
-            { id: "veo-2.0-generate-001", name: "Veo 2.0", provider: "gemini" }
+            {
+                id: "lyria-3-clip-preview",
+                name: "Lyria 3 Clip Preview",
+                provider: "gemini",
+                supportedTasks: ["text_to_music"]
+            }
         ];
     }
     async getAvailableEmbeddingModels() {
         return [
             {
-                id: "text-embedding-004",
-                name: "Text Embedding 004",
+                id: "gemini-embedding-2",
+                name: "Gemini Embedding 2",
                 provider: "gemini",
-                dimensions: 768
+                dimensions: 3072
             },
             {
                 id: "gemini-embedding-001",
@@ -581,7 +1467,7 @@ export class GeminiProvider extends BaseProvider {
         if (!text || (Array.isArray(text) && text.length === 0)) {
             throw new Error("text must not be empty");
         }
-        const texts = typeof text === "string" ? [text] : text;
+        const texts = isString(text) ? [text] : text;
         // Gemini embedContent supports a single content; batch by calling per text
         const embeddings = [];
         for (const t of texts) {
@@ -605,7 +1491,9 @@ export class GeminiProvider extends BaseProvider {
             if (!data.embedding?.values) {
                 throw new Error("No embedding returned from Gemini API");
             }
-            embeddings.push(data.embedding.values);
+            embeddings.push(dimensions && dimensions < 3072
+                ? normalizeEmbedding(data.embedding.values)
+                : data.embedding.values);
         }
         return embeddings;
     }
@@ -619,11 +1507,20 @@ export class GeminiProvider extends BaseProvider {
         const modelId = params.model.id;
         if (modelId.startsWith("gemini-")) {
             // Use generateContent with IMAGE response modality
+            const imageConfig = {};
+            if (params.aspectRatio)
+                imageConfig.aspectRatio = params.aspectRatio;
+            if (params.resolution)
+                imageConfig.imageSize = params.resolution;
+            const generationConfig = {
+                responseModalities: ["IMAGE", "TEXT"]
+            };
+            if (Object.keys(imageConfig).length > 0) {
+                generationConfig.imageConfig = imageConfig;
+            }
             const body = {
                 contents: [{ role: "user", parts: [{ text: params.prompt }] }],
-                generationConfig: {
-                    responseModalities: ["IMAGE", "TEXT"]
-                }
+                generationConfig
             };
             const url = `${GEMINI_API_BASE}/models/${modelId}:generateContent?key=${this.apiKey}`;
             const response = await this._fetch(url, {
@@ -635,7 +1532,7 @@ export class GeminiProvider extends BaseProvider {
                 const errText = await response.text();
                 throw new Error(`Gemini text-to-image failed ${response.status}: ${errText}`);
             }
-            const data = (await response.json());
+            const data = parseGeminiResponse(await response.json());
             const parts = data.candidates?.[0]?.content?.parts;
             if (!parts)
                 throw new Error("No candidates in response");
@@ -646,33 +1543,13 @@ export class GeminiProvider extends BaseProvider {
             }
             throw new Error("No image data returned in response");
         }
-        // Imagen models use generateImages endpoint
-        const body = {
-            instances: [{ prompt: params.prompt }],
-            parameters: { sampleCount: 1 }
-        };
-        const url = `${GEMINI_API_BASE}/models/${modelId}:generateImages?key=${this.apiKey}`;
-        const response = await this._fetch(url, {
-            method: "POST",
-            headers: { "Content-Type": "application/json" },
-            body: JSON.stringify(body)
-        });
-        if (!response.ok) {
-            const errText = await response.text();
-            throw new Error(`Gemini image generation failed ${response.status}: ${errText}`);
-        }
-        const data = (await response.json());
-        // Try predictions format first (Vertex-style), then generatedImages
-        const b64 = data.predictions?.[0]?.bytesBase64Encoded ??
-            data.generatedImages?.[0]?.image?.imageBytes;
-        if (!b64)
-            throw new Error("No image data in response");
-        return Uint8Array.from(Buffer.from(b64, "base64"));
+        // Google shut down Imagen in the Gemini API; `predict` now answers 404.
+        throw new Error(`Model ${modelId} is not available. Use a gemini-*-image model for text-to-image.`);
     }
     // ---------------------------------------------------------------------------
     // Image-to-image
     // ---------------------------------------------------------------------------
-    async imageToImage(image, params) {
+    async imageToImage(images, params) {
         if (!params.prompt) {
             throw new Error("The input prompt cannot be empty.");
         }
@@ -680,20 +1557,36 @@ export class GeminiProvider extends BaseProvider {
         if (!modelId.startsWith("gemini-")) {
             throw new Error(`Model ${modelId} does not support image-to-image. Only gemini-* models supported.`);
         }
-        const imageBase64 = Buffer.from(image).toString("base64");
+        const imageParts = images
+            .filter((b) => b && b.length > 0)
+            .map((b) => ({
+            inlineData: {
+                mimeType: "image/png",
+                data: Buffer.from(b).toString("base64")
+            }
+        }));
+        if (imageParts.length === 0) {
+            throw new Error("At least one input image is required");
+        }
+        const imageConfig = {};
+        if (params.aspectRatio)
+            imageConfig.aspectRatio = params.aspectRatio;
+        if (params.resolution)
+            imageConfig.imageSize = params.resolution;
+        const generationConfig = {
+            responseModalities: ["IMAGE", "TEXT"]
+        };
+        if (Object.keys(imageConfig).length > 0) {
+            generationConfig.imageConfig = imageConfig;
+        }
         const body = {
             contents: [
                 {
                     role: "user",
-                    parts: [
-                        { text: params.prompt },
-                        { inlineData: { mimeType: "image/png", data: imageBase64 } }
-                    ]
+                    parts: [{ text: params.prompt }, ...imageParts]
                 }
             ],
-            generationConfig: {
-                responseModalities: ["IMAGE", "TEXT"]
-            }
+            generationConfig
         };
         const url = `${GEMINI_API_BASE}/models/${modelId}:generateContent?key=${this.apiKey}`;
         const response = await this._fetch(url, {
@@ -705,7 +1598,7 @@ export class GeminiProvider extends BaseProvider {
             const errText = await response.text();
             throw new Error(`Gemini image-to-image failed ${response.status}: ${errText}`);
         }
-        const data = (await response.json());
+        const data = parseGeminiResponse(await response.json());
         const parts = data.candidates?.[0]?.content?.parts;
         if (!parts)
             throw new Error("No candidates in response");
@@ -720,7 +1613,14 @@ export class GeminiProvider extends BaseProvider {
     // Text-to-speech
     // ---------------------------------------------------------------------------
     async *textToSpeech(args) {
-        const { text, model, voice = "Puck" } = args;
+        const { model, voice = "Puck" } = args;
+        const text = [
+            args.language ? `Speak in ${args.language}.` : undefined,
+            args.instructions,
+            args.text
+        ]
+            .filter((value) => value !== undefined)
+            .join(" ");
         const body = {
             contents: [{ role: "user", parts: [{ text }] }],
             generationConfig: {
@@ -742,7 +1642,7 @@ export class GeminiProvider extends BaseProvider {
             const errText = await response.text();
             throw new Error(`Gemini TTS failed ${response.status}: ${errText}`);
         }
-        const data = (await response.json());
+        const data = parseGeminiResponse(await response.json());
         const parts = data.candidates?.[0]?.content?.parts;
         if (!parts)
             throw new Error("No audio in response");
@@ -763,31 +1663,34 @@ export class GeminiProvider extends BaseProvider {
         if (!audio || audio.length === 0) {
             throw new Error("audio must not be empty");
         }
-        // Detect MIME type from audio header
-        let mimeType = "audio/wav";
+        if (audio.length > 20 * 1024 * 1024) {
+            throw new Error("Gemini inline audio is limited to 20 MB; upload the audio with the File API first");
+        }
+        // Detect MIME type from the audio header.
+        let mimeType = geminiAudioMime(sniffAudioMime(audio));
         if (audio[0] === 0x52 &&
             audio[1] === 0x49 &&
             audio[2] === 0x46 &&
             audio[3] === 0x46) {
-            mimeType = "audio/wav"; // RIFF
+            mimeType = "audio/wav";
         }
         else if (audio[0] === 0x49 && audio[1] === 0x44 && audio[2] === 0x33) {
-            mimeType = "audio/mp3"; // ID3
+            mimeType = "audio/mp3";
         }
         else if (audio[0] === 0xff && (audio[1] === 0xfb || audio[1] === 0xf3)) {
-            mimeType = "audio/mp3"; // MPEG sync
+            mimeType = "audio/mp3";
         }
         else if (audio[0] === 0x66 &&
             audio[1] === 0x4c &&
             audio[2] === 0x61 &&
             audio[3] === 0x43) {
-            mimeType = "audio/flac"; // fLaC
+            mimeType = "audio/flac";
         }
         else if (audio[0] === 0x4f &&
             audio[1] === 0x67 &&
             audio[2] === 0x67 &&
             audio[3] === 0x53) {
-            mimeType = "audio/ogg"; // OggS
+            mimeType = "audio/ogg";
         }
         let promptText = args.prompt ?? "Transcribe this audio to text.";
         if (language) {
@@ -818,7 +1721,7 @@ export class GeminiProvider extends BaseProvider {
             const errText = await response.text();
             throw new Error(`Gemini ASR failed ${response.status}: ${errText}`);
         }
-        const data = (await response.json());
+        const data = parseGeminiResponse(await response.json());
         const parts = data.candidates?.[0]?.content?.parts;
         if (!parts)
             return { text: "" };
@@ -828,142 +1731,423 @@ export class GeminiProvider extends BaseProvider {
             .join("");
         return { text };
     }
-    // ---------------------------------------------------------------------------
-    // Text-to-video (Veo models — async operation with polling)
-    // ---------------------------------------------------------------------------
-    async textToVideo(params) {
-        if (!params.prompt) {
-            throw new Error("The input prompt cannot be empty.");
-        }
-        const modelId = params.model.id;
-        if (!modelId.startsWith("veo-")) {
-            throw new Error(`Model ${modelId} is not a Veo model. Only Veo models support text-to-video.`);
-        }
-        const body = {
-            instances: [{ prompt: params.prompt }]
-        };
+    buildVideoParameters(params) {
         const parameters = {};
-        if (params.negativePrompt)
+        if (params.negativePrompt) {
             parameters.negativePrompt = params.negativePrompt;
-        if (params.aspectRatio)
+        }
+        if (params.aspectRatio) {
             parameters.aspectRatio = params.aspectRatio;
-        if (params.seed != null)
+        }
+        if (params.resolution) {
+            parameters.resolution = params.resolution;
+        }
+        if (params.durationSeconds != null) {
+            parameters.durationSeconds = params.durationSeconds;
+        }
+        if (params.seed != null) {
             parameters.seed = params.seed;
-        if (Object.keys(parameters).length > 0)
-            body.parameters = parameters;
-        // Initiate async generation
-        const url = `${GEMINI_API_BASE}/models/${modelId}:generateVideos?key=${this.apiKey}`;
-        const response = await this._fetch(url, {
-            method: "POST",
-            headers: { "Content-Type": "application/json" },
-            body: JSON.stringify(body)
-        });
-        if (!response.ok) {
-            const errText = await response.text();
-            throw new Error(`Gemini video generation failed ${response.status}: ${errText}`);
         }
-        const operation = (await response.json());
-        // Poll for completion
-        const maxWait = 600_000; // 10 minutes
+        return parameters;
+    }
+    getVideoUri(operation) {
+        return (operation.response?.generateVideoResponse?.generatedSamples?.[0]?.video
+            ?.uri ?? operation.response?.generatedVideos?.[0]?.video?.uri);
+    }
+    /**
+     * The refusal to raise when an operation came back with no video, or null
+     * when nothing says the filter is why. A filtered take leaves the operation
+     * successful and the samples empty, so without this the caller sees only
+     * "No video URI in response" — indistinguishable from a broken response, and
+     * fatal to a run that has already paid for its other shots.
+     */
+    videoContentFilterRefusal(operation, modelId) {
+        const rai = operation.response?.generateVideoResponse ?? operation.response;
+        const reasons = rai?.raiMediaFilteredReasons ?? [];
+        const filtered = rai?.raiMediaFilteredCount ?? 0;
+        if (filtered <= 0 && reasons.length === 0)
+            return null;
+        const detail = reasons.length
+            ? reasons.join("; ")
+            : "no reason given by the provider";
+        return new ContentFilterRefusal(`Veo filtered every generated video for this prompt: ${detail}`, { provider: "gemini", model: modelId, reasons });
+    }
+    async waitForVideoOperation(operation, timeoutSeconds, signal) {
+        const maxWait = timeoutSeconds && timeoutSeconds > 0 ? timeoutSeconds * 1000 : 600_000;
         const pollInterval = 10_000;
         let elapsed = 0;
         let current = operation;
         while (!current.done && elapsed < maxWait) {
-            await new Promise((r) => setTimeout(r, pollInterval));
+            await new Promise((resolve, reject) => {
+                const onAbort = () => {
+                    clearTimeout(timer);
+                    reject(abortError(signal));
+                };
+                const timer = setTimeout(() => {
+                    signal?.removeEventListener("abort", onAbort);
+                    resolve();
+                }, pollInterval);
+                signal?.addEventListener("abort", onAbort, { once: true });
+            });
             elapsed += pollInterval;
-            if (!current.name)
+            if (!current.name) {
                 throw new Error("No operation name for polling");
-            const pollUrl = `${GEMINI_API_BASE}/${current.name}?key=${this.apiKey}`;
-            const pollResp = await this._fetch(pollUrl);
+            }
+            const pollResp = await this._fetch(`${GEMINI_API_BASE}/${current.name}`, {
+                headers: { "x-goog-api-key": this.apiKey },
+                signal
+            });
             if (!pollResp.ok) {
                 const errText = await pollResp.text();
                 throw new Error(`Poll failed ${pollResp.status}: ${errText}`);
             }
             current = (await pollResp.json());
         }
-        if (!current.done)
+        if (!current.done) {
             throw new Error("Video generation timed out");
-        const videoUri = current.response?.generatedVideos?.[0]?.video?.uri;
-        if (!videoUri)
+        }
+        if (current.error?.message) {
+            // Vertex answers a filtered prompt through the operation's error slot
+            // ("videos were filtered out because they violated ... usage
+            // guidelines"), so the refusal has to be recognized here too.
+            if (isContentFilterRefusal(current.error.message)) {
+                throw new ContentFilterRefusal(current.error.message, {
+                    provider: "gemini"
+                });
+            }
+            throw new Error(`Gemini video generation failed: ${current.error.message}`);
+        }
+        return current;
+    }
+    async downloadGeminiMedia(uri, signal) {
+        const hostname = new URL(uri).hostname;
+        const headers = hostname === "generativelanguage.googleapis.com"
+            ? { "x-goog-api-key": this.apiKey }
+            : undefined;
+        const response = await safeFetch(uri, { headers, signal }, 5, this._fetch);
+        if (!response.ok) {
+            throw new Error(`Gemini media download failed: ${response.status}`);
+        }
+        return new Uint8Array(await response.arrayBuffer());
+    }
+    /**
+     * Submit a Veo `predictLongRunning` request, wait for the operation, and
+     * download the first generated video.
+     */
+    async runVeoOperation(modelId, body, label, options) {
+        const signal = combineSignals(options.signal, options.timeoutSeconds);
+        this.recordRequestPayload(body);
+        const response = await this._fetch(`${GEMINI_API_BASE}/models/${modelId}:predictLongRunning`, {
+            method: "POST",
+            headers: {
+                "Content-Type": "application/json",
+                "x-goog-api-key": this.apiKey
+            },
+            body: JSON.stringify(body),
+            signal
+        });
+        if (!response.ok) {
+            const errText = await response.text();
+            throw new Error(`Gemini ${label} failed ${response.status}: ${errText}`);
+        }
+        const operation = await this.waitForVideoOperation((await response.json()), options.timeoutSeconds, signal);
+        const videoUri = this.getVideoUri(operation);
+        if (!videoUri) {
+            const refusal = this.videoContentFilterRefusal(operation, modelId);
+            if (refusal)
+                throw refusal;
             throw new Error("No video URI in response");
-        // Download the video
-        const dlResp = await this._fetch(`${videoUri}&key=${this.apiKey}`);
-        if (!dlResp.ok)
-            throw new Error(`Video download failed: ${dlResp.status}`);
-        return new Uint8Array(await dlResp.arrayBuffer());
+        }
+        return this.downloadGeminiMedia(videoUri, signal);
+    }
+    /** Send one request to the Interactions API and return the finished interaction. */
+    async createInteraction(body, label, modelId, signal) {
+        this.recordRequestPayload(body);
+        const response = await this._fetch(`${GEMINI_API_BASE}/interactions`, {
+            method: "POST",
+            headers: {
+                "Content-Type": "application/json",
+                "x-goog-api-key": this.apiKey
+            },
+            body: JSON.stringify(body),
+            signal
+        });
+        if (!response.ok) {
+            const errText = await response.text();
+            if (isContentFilterRefusal(errText)) {
+                throw new ContentFilterRefusal(errText, {
+                    provider: "gemini",
+                    model: modelId
+                });
+            }
+            throw new Error(`Gemini ${label} failed ${response.status}: ${errText}`);
+        }
+        const interaction = (await response.json());
+        if (interaction.status && interaction.status !== "completed") {
+            throw new Error(`Gemini ${label} ended with status ${interaction.status}`);
+        }
+        return interaction;
+    }
+    /** The first `type` output of an interaction, inline or behind a file uri. */
+    async interactionMedia(interaction, type, signal) {
+        for (const step of interaction.steps ?? []) {
+            for (const content of step.content ?? []) {
+                if (content.type !== type)
+                    continue;
+                if (content.data) {
+                    return {
+                        data: new Uint8Array(Buffer.from(content.data, "base64")),
+                        mimeType: content.mime_type
+                    };
+                }
+                if (content.uri) {
+                    return {
+                        data: await this.downloadGeminiMedia(content.uri, signal),
+                        mimeType: content.mime_type
+                    };
+                }
+            }
+        }
+        throw new Error(`Gemini returned no ${type} output`);
+    }
+    /** An Interactions API video block: inline, or uploaded when too large. */
+    async omniVideoInput(video) {
+        const mimeType = sniffVideoMime(video);
+        if (video.length > GEMINI_INLINE_VIDEO_MAX_BYTES) {
+            const uri = await this.uploadFileToGemini(video, mimeType);
+            return { type: "video", uri, mime_type: mimeType };
+        }
+        return {
+            type: "video",
+            data: Buffer.from(video).toString("base64"),
+            mime_type: mimeType
+        };
+    }
+    /** Generate or edit a video with Gemini Omni through the Interactions API. */
+    async runOmniVideo(modelId, input, options) {
+        const responseFormat = { type: "video" };
+        if (options.aspectRatio)
+            responseFormat.aspect_ratio = options.aspectRatio;
+        if (options.resolution)
+            responseFormat.resolution = options.resolution;
+        const signal = combineSignals(options.signal, options.timeoutSeconds);
+        const interaction = await this.createInteraction({ model: modelId, input, response_format: responseFormat }, "video generation", modelId, signal);
+        return (await this.interactionMedia(interaction, "video", signal)).data;
+    }
+    // ---------------------------------------------------------------------------
+    // Text-to-video (Veo: long-running operation; Omni: Interactions API)
+    // ---------------------------------------------------------------------------
+    async textToVideo(params) {
+        if (!params.prompt) {
+            throw new Error("The input prompt cannot be empty.");
+        }
+        const modelId = params.model.id;
+        if (isOmniVideoModel(modelId)) {
+            return this.runOmniVideo(modelId, [{ type: "text", text: params.prompt }], params);
+        }
+        if (!modelId.startsWith("veo-")) {
+            throw new Error(`Model ${modelId} is not a Gemini video model. Use a Veo or Gemini Omni model for text-to-video.`);
+        }
+        const body = {
+            instances: [{ prompt: params.prompt }]
+        };
+        const parameters = this.buildVideoParameters(params);
+        if (Object.keys(parameters).length > 0) {
+            body.parameters = parameters;
+        }
+        return this.runVeoOperation(modelId, body, "video generation", params);
     }
     // ---------------------------------------------------------------------------
-    // Image-to-video (Veo models)
+    // Image-to-video
     // ---------------------------------------------------------------------------
     async imageToVideo(image, params) {
         if (!image || image.length === 0) {
             throw new Error("Input image cannot be empty.");
         }
         const modelId = params.model.id;
+        const prompt = params.prompt || "Animate this image";
+        if (isOmniVideoModel(modelId)) {
+            if (params.endImage?.length) {
+                throw new Error(`Gemini model ${modelId} does not accept a last frame`);
+            }
+            return this.runOmniVideo(modelId, [
+                {
+                    type: "image",
+                    data: Buffer.from(image).toString("base64"),
+                    mime_type: detectImageMime(image)
+                },
+                { type: "text", text: prompt }
+            ], params);
+        }
         if (!modelId.startsWith("veo-")) {
-            throw new Error(`Model ${modelId} is not a Veo model. Only Veo models support image-to-video.`);
+            throw new Error(`Model ${modelId} is not a Gemini video model. Use a Veo or Gemini Omni model for image-to-video.`);
+        }
+        const instance = {
+            prompt,
+            image: veoImage(image)
+        };
+        const parameters = this.buildVideoParameters(params);
+        if (params.endImage?.length) {
+            // Veo interpolates between the two frames only in an 8-second clip.
+            instance.lastFrame = veoImage(params.endImage);
+            parameters.durationSeconds ??= VEO_CONSTRAINED_DURATION_SECONDS;
+        }
+        const body = { instances: [instance] };
+        if (Object.keys(parameters).length > 0) {
+            body.parameters = parameters;
+        }
+        return this.runVeoOperation(modelId, body, "image-to-video", params);
+    }
+    // ---------------------------------------------------------------------------
+    // Reference-to-video
+    // ---------------------------------------------------------------------------
+    async referenceToVideo(inputs, params) {
+        if (!params.prompt) {
+            throw new Error("The input prompt cannot be empty.");
+        }
+        const modelId = params.model.id;
+        requireVideoTask(modelId, "reference_to_video");
+        const images = inputs.images.filter((bytes) => bytes.length > 0);
+        const videos = inputs.videos.filter((bytes) => bytes.length > 0);
+        if ((inputs.audios ?? []).some((bytes) => bytes.length > 0)) {
+            throw new Error(`Gemini model ${modelId} does not accept reference audio`);
+        }
+        if (images.length === 0 && videos.length === 0) {
+            throw new Error("At least one reference image or video is required");
+        }
+        if (isOmniVideoModel(modelId)) {
+            if (videos.length > OMNI_MAX_REFERENCE_VIDEOS) {
+                throw new Error(`Gemini model ${modelId} accepts at most ${OMNI_MAX_REFERENCE_VIDEOS} reference videos`);
+            }
+            const input = images.map((bytes) => ({
+                type: "image",
+                data: Buffer.from(bytes).toString("base64"),
+                mime_type: detectImageMime(bytes)
+            }));
+            for (const video of videos) {
+                input.push(await this.omniVideoInput(video));
+            }
+            input.push({ type: "text", text: params.prompt });
+            return this.runOmniVideo(modelId, input, params);
+        }
+        if (videos.length > 0) {
+            throw new Error(`Gemini model ${modelId} accepts reference images only`);
+        }
+        if (images.length > VEO_MAX_REFERENCE_IMAGES) {
+            throw new Error(`Gemini model ${modelId} accepts at most ${VEO_MAX_REFERENCE_IMAGES} reference images`);
         }
-        const imageBase64 = Buffer.from(image).toString("base64");
-        const prompt = params.prompt ?? "Animate this image";
+        const parameters = this.buildVideoParameters(params);
+        parameters.durationSeconds ??= VEO_CONSTRAINED_DURATION_SECONDS;
         const body = {
             instances: [
                 {
-                    prompt,
-                    image: { bytesBase64Encoded: imageBase64, mimeType: "image/png" }
+                    prompt: params.prompt,
+                    referenceImages: images.map((bytes) => ({
+                        image: veoImage(bytes),
+                        referenceType: "asset"
+                    }))
                 }
-            ]
+            ],
+            parameters
         };
-        const parameters = {};
-        if (params.negativePrompt)
-            parameters.negativePrompt = params.negativePrompt;
-        if (params.aspectRatio)
-            parameters.aspectRatio = params.aspectRatio;
-        if (params.seed != null)
-            parameters.seed = params.seed;
-        if (Object.keys(parameters).length > 0)
-            body.parameters = parameters;
-        const url = `${GEMINI_API_BASE}/models/${modelId}:generateVideos?key=${this.apiKey}`;
-        const response = await this._fetch(url, {
-            method: "POST",
-            headers: { "Content-Type": "application/json" },
-            body: JSON.stringify(body)
+        return this.runVeoOperation(modelId, body, "reference-to-video", params);
+    }
+    // ---------------------------------------------------------------------------
+    // Video extension (Veo) and editing (Omni)
+    // ---------------------------------------------------------------------------
+    async extendVideo(video, params) {
+        if (video.length === 0) {
+            throw new Error("The input video is empty.");
+        }
+        const modelId = params.model.id;
+        requireVideoTask(modelId, "extend_video");
+        if (params.mode !== "end") {
+            throw new Error(`Gemini model ${modelId} extends videos at the end only`);
+        }
+        if (params.durationSeconds !== VEO_EXTENSION_SECONDS) {
+            throw new Error(`Gemini model ${modelId} extends a video by exactly ${VEO_EXTENSION_SECONDS} seconds`);
+        }
+        const body = {
+            instances: [
+                {
+                    prompt: params.prompt,
+                    video: {
+                        inlineData: {
+                            mimeType: sniffVideoMime(video),
+                            data: Buffer.from(video).toString("base64")
+                        }
+                    }
+                }
+            ],
+            // Veo extends 720p input only and returns the input plus the new seconds.
+            parameters: { numberOfVideos: 1, resolution: "720p" }
+        };
+        return this.runVeoOperation(modelId, body, "video extension", {
+            signal: params.signal
         });
-        if (!response.ok) {
-            const errText = await response.text();
-            throw new Error(`Gemini image-to-video failed ${response.status}: ${errText}`);
+    }
+    async videoToVideo(video, params) {
+        if (video.length === 0) {
+            throw new Error("The input video is empty.");
         }
-        const operation = (await response.json());
-        // Poll for completion
-        const maxWait = 600_000;
-        const pollInterval = 10_000;
-        let elapsed = 0;
-        let current = operation;
-        while (!current.done && elapsed < maxWait) {
-            await new Promise((r) => setTimeout(r, pollInterval));
-            elapsed += pollInterval;
-            if (!current.name)
-                throw new Error("No operation name for polling");
-            const pollUrl = `${GEMINI_API_BASE}/${current.name}?key=${this.apiKey}`;
-            const pollResp = await this._fetch(pollUrl);
-            if (!pollResp.ok) {
-                const errText = await pollResp.text();
-                throw new Error(`Poll failed ${pollResp.status}: ${errText}`);
-            }
-            current = (await pollResp.json());
+        if (!params.prompt) {
+            throw new Error("The edit instruction cannot be empty.");
         }
-        if (!current.done)
-            throw new Error("Image-to-video generation timed out");
-        const videoUri = current.response?.generatedVideos?.[0]?.video?.uri;
-        if (!videoUri)
-            throw new Error("No video URI in response");
-        const dlResp = await this._fetch(`${videoUri}&key=${this.apiKey}`);
-        if (!dlResp.ok)
-            throw new Error(`Video download failed: ${dlResp.status}`);
-        return new Uint8Array(await dlResp.arrayBuffer());
+        const modelId = params.model.id;
+        requireVideoTask(modelId, "video_to_video");
+        const input = [await this.omniVideoInput(video)];
+        for (const bytes of params.referenceImages ?? []) {
+            if (bytes.length === 0)
+                continue;
+            input.push({
+                type: "image",
+                data: Buffer.from(bytes).toString("base64"),
+                mime_type: detectImageMime(bytes)
+            });
+        }
+        input.push({ type: "text", text: params.prompt });
+        return this.runOmniVideo(modelId, input, params);
+    }
+    // ---------------------------------------------------------------------------
+    // Text-to-music (Lyria, Interactions API)
+    // ---------------------------------------------------------------------------
+    async textToMusic(params) {
+        if (!params.prompt) {
+            throw new Error("The input prompt cannot be empty.");
+        }
+        const modelId = params.model.id;
+        if (!modelId.startsWith("lyria-")) {
+            throw new Error(`Model ${modelId} is not a Lyria model. Use a Lyria model for text-to-music.`);
+        }
+        // Lyria takes length and lyrics from the prompt; it has no fields for them.
+        const input = [
+            params.prompt,
+            params.durationSeconds
+                ? `Length: about ${Math.round(params.durationSeconds)} seconds.`
+                : undefined,
+            params.lyrics ? `Lyrics:\n${params.lyrics}` : undefined
+        ]
+            .filter((part) => part !== undefined)
+            .join("\n\n");
+        const body = { model: modelId, input };
+        // Lyria answers MP3 by default. Only Lyria 3.5 returns WAV, on request.
+        if (params.audioFormat === "wav" && modelId === "lyria-3.5") {
+            body.response_format = { type: "audio" };
+        }
+        const signal = combineSignals(undefined, params.timeoutSeconds);
+        const interaction = await this.createInteraction(body, "music generation", modelId, signal);
+        const audio = await this.interactionMedia(interaction, "audio", signal);
+        return {
+            data: audio.data,
+            mimeType: audio.mimeType ?? sniffAudioMime(audio.data)
+        };
     }
     // ---------------------------------------------------------------------------
     // Error detection
     // ---------------------------------------------------------------------------
+    isContextExceededError(error) {
+        return geminiContextExceeded(error) !== null;
+    }
     isContextLengthError(error) {
         const msg = String(error).toLowerCase();
         return (msg.includes("context length") ||