@spinabot/brigade 1.9.0 → 1.11.0

package/dist/agents/media-understanding/gemini-adapter.js

@@ -0,0 +1,343 @@
+/**
+ * Gemini (Google) media-understanding adapter — talks to the Generative
+ * Language REST API at `generativelanguage.googleapis.com/v1beta` directly.
+ *
+ * Two paths:
+ *   • VIDEO → the Files API. Inline base64 caps out well below real video
+ *     sizes, so video is UPLOADED first (resumable upload), POLLED until the
+ *     file state flips to ACTIVE (Gemini transcodes/indexes asynchronously),
+ *     then referenced from `generateContent` via a `file_data` part holding
+ *     the returned `fileUri`. This is the only way large video works.
+ *   • image / pdf → an inline `inline_data` part (base64) + the prompt, sent
+ *     straight to `generateContent`. These are small enough to inline.
+ *
+ * The Google key rides in the query string (`?key=…`), matching Brigade's
+ * existing validator (`providers/validate-key.ts`) and the Gemini convention
+ * — there is no auth header. Every call takes an injectable `fetchFn` so the
+ * adapter is exercised with zero real network.
+ */
+import { MediaUnderstandingProviderError, } from "./types.js";
+/** Canonical Gemini API base — keep on the trusted Google host. */
+export const DEFAULT_GEMINI_BASE_URL = "https://generativelanguage.googleapis.com/v1beta";
+/**
+ * Default models per kind. Current Gemini multimodal models that read video,
+ * images and PDFs. `gemini-2.5-flash` is the cheap/fast default; callers can
+ * override per call or via config.
+ */
+export const DEFAULT_GEMINI_MODELS = {
+    video: "gemini-2.5-pro",
+    image: "gemini-2.5-flash",
+    pdf: "gemini-2.5-flash",
+    audio: "gemini-2.5-flash",
+};
+const DEFAULT_PROMPTS = {
+    video: "Describe this video in detail: the scenes, actions, on-screen text, and any spoken words.",
+    image: "Describe this image in detail.",
+    pdf: "Read this document and summarize its contents, preserving key facts, figures, and structure.",
+    audio: "Transcribe this audio.",
+};
+/** Per-request timeouts. Video upload + generation can be slow; bound each. */
+const UPLOAD_TIMEOUT_MS = 120_000;
+const GENERATE_TIMEOUT_MS = 180_000;
+const POLL_TIMEOUT_MS = 15_000;
+/** Total wall-clock budget waiting for an uploaded file to become ACTIVE. */
+const ACTIVE_POLL_BUDGET_MS = 180_000;
+/** Delay between file-state polls. */
+const POLL_INTERVAL_MS = 2_000;
+/** Hard ceiling on `maxTokens`. */
+const MAX_OUTPUT_TOKENS_CEILING = 32_000;
+/**
+ * Build the optional `generationConfig` for a Gemini request. When no
+ * `maxTokens` is given we omit it entirely (let the model use its default);
+ * otherwise clamp it to a sane window.
+ */
+function generationConfig(maxTokens) {
+    if (typeof maxTokens !== "number" || !Number.isFinite(maxTokens))
+        return undefined;
+    const v = Math.max(256, Math.min(MAX_OUTPUT_TOKENS_CEILING, Math.floor(maxTokens)));
+    return { maxOutputTokens: v };
+}
+/**
+ * Strip a leading `models/` so callers can pass either `gemini-2.5-pro` or
+ * `models/gemini-2.5-pro`; we always rebuild the `models/<id>:method` path.
+ */
+function normalizeModelId(model) {
+    return model.replace(/^models\//, "").trim();
+}
+function resolveModel(kind, model) {
+    const trimmed = model?.trim();
+    if (trimmed)
+        return normalizeModelId(trimmed);
+    return DEFAULT_GEMINI_MODELS[kind] ?? DEFAULT_GEMINI_MODELS.image;
+}
+function resolvePrompt(kind, prompt) {
+    const trimmed = prompt?.trim();
+    return trimmed || DEFAULT_PROMPTS[kind];
+}
+/** Compose the caller signal with a per-request timeout. */
+function withTimeout(signal, ms) {
+    const timeoutSignal = AbortSignal.timeout(ms);
+    if (!signal)
+        return timeoutSignal;
+    return AbortSignal.any([signal, timeoutSignal]);
+}
+const defaultSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms).unref?.());
+/** Extract the concatenated text from a generateContent response payload. */
+function extractGeneratedText(payload) {
+    const candidates = payload
+        ?.candidates;
+    const parts = candidates?.[0]?.content?.parts ?? [];
+    return parts
+        .map((p) => (typeof p?.text === "string" ? p.text.trim() : ""))
+        .filter(Boolean)
+        .join("\n")
+        .trim();
+}
+async function readErrorMessage(res) {
+    try {
+        const body = (await res.json());
+        if (body?.error?.message)
+            return body.error.message;
+    }
+    catch {
+        /* fall through */
+    }
+    return `HTTP ${res.status}`;
+}
+/* ─────────────────────────── inline path (image / pdf) ─────────────────────────── */
+async function generateInline(params) {
+    const fetchFn = params.fetchFn ?? fetch;
+    const baseUrl = (params.baseUrl ?? DEFAULT_GEMINI_BASE_URL).replace(/\/+$/, "");
+    const model = resolveModel(params.kind, params.model);
+    const prompt = resolvePrompt(params.kind, params.prompt);
+    const url = `${baseUrl}/models/${model}:generateContent?key=${encodeURIComponent(params.apiKey)}`;
+    const genCfg = generationConfig(params.maxTokens);
+    const body = {
+        contents: [
+            {
+                role: "user",
+                parts: [
+                    { text: prompt },
+                    {
+                        inline_data: {
+                            mime_type: params.mimeType,
+                            data: params.bytes.toString("base64"),
+                        },
+                    },
+                ],
+            },
+        ],
+        ...(genCfg ? { generationConfig: genCfg } : {}),
+    };
+    let res;
+    try {
+        res = await fetchFn(url, {
+            method: "POST",
+            headers: { "content-type": "application/json" },
+            body: JSON.stringify(body),
+            signal: withTimeout(params.signal, GENERATE_TIMEOUT_MS),
+        });
+    }
+    catch (err) {
+        throw new MediaUnderstandingProviderError("google", `Gemini request failed: ${err instanceof Error ? err.message : String(err)}`);
+    }
+    if (!res.ok) {
+        throw new MediaUnderstandingProviderError("google", `Gemini error: ${await readErrorMessage(res)}`, res.status);
+    }
+    const payload = await res.json().catch(() => ({}));
+    const text = extractGeneratedText(payload);
+    if (!text) {
+        throw new MediaUnderstandingProviderError("google", "Gemini returned no text for the media.");
+    }
+    return { text, provider: "google", model };
+}
+/**
+ * Upload bytes via the Gemini Files API resumable protocol:
+ *   1. POST to the upload endpoint with `x-goog-upload-protocol: resumable`
+ *      + `x-goog-upload-command: start` and the file metadata → the response
+ *      carries an `x-goog-upload-url` to PUT the bytes to.
+ *   2. PUT the bytes to that URL with `x-goog-upload-command: upload, finalize`
+ *      → the response body is the `{ file: {...} }` resource.
+ */
+async function uploadFile(params) {
+    const fetchFn = params.fetchFn ?? fetch;
+    const baseUrl = (params.baseUrl ?? DEFAULT_GEMINI_BASE_URL).replace(/\/+$/, "");
+    // The upload endpoint shares the host but lives under `/upload/v1beta/files`.
+    // Derive it from the base (which ends in `/v1beta`) so a region/test override
+    // of the base carries through.
+    const uploadUrl = `${baseUrl.replace(/\/v1beta$/, "")}/upload/v1beta/files?key=${encodeURIComponent(params.apiKey)}`;
+    const numBytes = params.bytes.length;
+    // Step 1 — start a resumable upload session.
+    let startRes;
+    try {
+        startRes = await fetchFn(uploadUrl, {
+            method: "POST",
+            headers: {
+                "x-goog-upload-protocol": "resumable",
+                "x-goog-upload-command": "start",
+                "x-goog-upload-header-content-length": String(numBytes),
+                "x-goog-upload-header-content-type": params.mimeType,
+                "content-type": "application/json",
+            },
+            body: JSON.stringify({ file: { display_name: "brigade-media" } }),
+            signal: withTimeout(params.signal, UPLOAD_TIMEOUT_MS),
+        });
+    }
+    catch (err) {
+        throw new MediaUnderstandingProviderError("google", `Gemini file upload (start) failed: ${err instanceof Error ? err.message : String(err)}`);
+    }
+    if (!startRes.ok) {
+        throw new MediaUnderstandingProviderError("google", `Gemini file upload (start) error: ${await readErrorMessage(startRes)}`, startRes.status);
+    }
+    const sessionUrl = startRes.headers.get("x-goog-upload-url") ?? startRes.headers.get("X-Goog-Upload-URL");
+    if (!sessionUrl) {
+        throw new MediaUnderstandingProviderError("google", "Gemini file upload did not return an upload session URL.");
+    }
+    // Step 2 — upload + finalize the bytes.
+    let putRes;
+    try {
+        putRes = await fetchFn(sessionUrl, {
+            method: "POST",
+            headers: {
+                "content-length": String(numBytes),
+                "x-goog-upload-offset": "0",
+                "x-goog-upload-command": "upload, finalize",
+            },
+            body: new Uint8Array(params.bytes),
+            signal: withTimeout(params.signal, UPLOAD_TIMEOUT_MS),
+        });
+    }
+    catch (err) {
+        throw new MediaUnderstandingProviderError("google", `Gemini file upload (finalize) failed: ${err instanceof Error ? err.message : String(err)}`);
+    }
+    if (!putRes.ok) {
+        throw new MediaUnderstandingProviderError("google", `Gemini file upload (finalize) error: ${await readErrorMessage(putRes)}`, putRes.status);
+    }
+    const uploaded = (await putRes.json().catch(() => ({})));
+    const file = uploaded.file;
+    if (!file?.uri || !file?.name) {
+        throw new MediaUnderstandingProviderError("google", "Gemini file upload returned no file URI.");
+    }
+    return {
+        uri: file.uri,
+        name: file.name,
+        state: file.state ?? "PROCESSING",
+        mimeType: file.mimeType ?? params.mimeType,
+    };
+}
+/** GET the current state of an uploaded file (`files/<id>`). */
+async function getFileState(fileName, params) {
+    const fetchFn = params.fetchFn ?? fetch;
+    const baseUrl = (params.baseUrl ?? DEFAULT_GEMINI_BASE_URL).replace(/\/+$/, "");
+    // `fileName` is the resource name like `files/abc123`. Build `<base>/files/abc123`.
+    const rel = fileName.startsWith("files/") ? fileName : `files/${fileName}`;
+    const url = `${baseUrl}/${rel}?key=${encodeURIComponent(params.apiKey)}`;
+    let res;
+    try {
+        res = await fetchFn(url, {
+            method: "GET",
+            signal: withTimeout(params.signal, POLL_TIMEOUT_MS),
+        });
+    }
+    catch (err) {
+        throw new MediaUnderstandingProviderError("google", `Gemini file status poll failed: ${err instanceof Error ? err.message : String(err)}`);
+    }
+    if (!res.ok) {
+        throw new MediaUnderstandingProviderError("google", `Gemini file status error: ${await readErrorMessage(res)}`, res.status);
+    }
+    const body = (await res.json().catch(() => ({})));
+    return {
+        state: body.state ?? "PROCESSING",
+        ...(body.uri ? { uri: body.uri } : {}),
+        ...(body.mimeType ? { mimeType: body.mimeType } : {}),
+    };
+}
+/** Poll a freshly-uploaded file until it reaches ACTIVE (or fails / times out). */
+async function waitForActive(file, params) {
+    const sleep = params.sleepFn ?? defaultSleep;
+    if (file.state === "ACTIVE")
+        return file;
+    const deadline = Date.now() + ACTIVE_POLL_BUDGET_MS;
+    let current = file;
+    for (;;) {
+        if (current.state === "ACTIVE")
+            return current;
+        if (current.state === "FAILED") {
+            throw new MediaUnderstandingProviderError("google", "Gemini failed to process the uploaded video (file state FAILED).");
+        }
+        if (Date.now() >= deadline) {
+            throw new MediaUnderstandingProviderError("google", "Gemini did not finish processing the video within the time budget. Try a shorter clip.");
+        }
+        await sleep(POLL_INTERVAL_MS);
+        const next = await getFileState(current.name, params);
+        current = {
+            name: current.name,
+            uri: next.uri ?? current.uri,
+            mimeType: next.mimeType ?? current.mimeType,
+            state: next.state,
+        };
+    }
+}
+/** Reference an uploaded (ACTIVE) file from generateContent via a file_data part. */
+async function generateFromFile(file, params) {
+    const fetchFn = params.fetchFn ?? fetch;
+    const baseUrl = (params.baseUrl ?? DEFAULT_GEMINI_BASE_URL).replace(/\/+$/, "");
+    const model = resolveModel(params.kind, params.model);
+    const prompt = resolvePrompt(params.kind, params.prompt);
+    const url = `${baseUrl}/models/${model}:generateContent?key=${encodeURIComponent(params.apiKey)}`;
+    const genCfg = generationConfig(params.maxTokens);
+    const body = {
+        contents: [
+            {
+                role: "user",
+                parts: [
+                    { text: prompt },
+                    { file_data: { file_uri: file.uri, mime_type: file.mimeType } },
+                ],
+            },
+        ],
+        ...(genCfg ? { generationConfig: genCfg } : {}),
+    };
+    let res;
+    try {
+        res = await fetchFn(url, {
+            method: "POST",
+            headers: { "content-type": "application/json" },
+            body: JSON.stringify(body),
+            signal: withTimeout(params.signal, GENERATE_TIMEOUT_MS),
+        });
+    }
+    catch (err) {
+        throw new MediaUnderstandingProviderError("google", `Gemini generate (from file) failed: ${err instanceof Error ? err.message : String(err)}`);
+    }
+    if (!res.ok) {
+        throw new MediaUnderstandingProviderError("google", `Gemini generate error: ${await readErrorMessage(res)}`, res.status);
+    }
+    const payload = await res.json().catch(() => ({}));
+    const text = extractGeneratedText(payload);
+    if (!text) {
+        throw new MediaUnderstandingProviderError("google", "Gemini returned no text for the video.");
+    }
+    return { text, provider: "google", model };
+}
+/* ─────────────────────────── entry point ─────────────────────────── */
+/**
+ * Run a Gemini media-understanding request. Video goes through the Files API
+ * (upload → poll ACTIVE → generate); image/pdf go inline. Returns the model's
+ * textual answer.
+ */
+export async function runGemini(params) {
+    if (!params.apiKey) {
+        throw new MediaUnderstandingProviderError("google", "No Google/Gemini API key configured.");
+    }
+    if (params.kind === "video") {
+        const uploaded = await uploadFile(params);
+        const active = await waitForActive(uploaded, params);
+        return generateFromFile(active, params);
+    }
+    // image / pdf / audio → inline data.
+    return generateInline(params);
+}
+// Exported for tests.
+export { extractGeneratedText, normalizeModelId };
+//# sourceMappingURL=gemini-adapter.js.map

package/dist/agents/media-understanding/gemini-adapter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"gemini-adapter.js","sourceRoot":"","sources":["../../../src/agents/media-understanding/gemini-adapter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,EACN,+BAA+B,GAG/B,MAAM,YAAY,CAAC;AAEpB,mEAAmE;AACnE,MAAM,CAAC,MAAM,uBAAuB,GAAG,kDAAkD,CAAC;AAE1F;;;;GAIG;AACH,MAAM,CAAC,MAAM,qBAAqB,GAAuE;IACxG,KAAK,EAAE,gBAAgB;IACvB,KAAK,EAAE,kBAAkB;IACzB,GAAG,EAAE,kBAAkB;IACvB,KAAK,EAAE,kBAAkB;CACzB,CAAC;AAEF,MAAM,eAAe,GAA2C;IAC/D,KAAK,EAAE,2FAA2F;IAClG,KAAK,EAAE,gCAAgC;IACvC,GAAG,EAAE,8FAA8F;IACnG,KAAK,EAAE,wBAAwB;CAC/B,CAAC;AAEF,+EAA+E;AAC/E,MAAM,iBAAiB,GAAG,OAAO,CAAC;AAClC,MAAM,mBAAmB,GAAG,OAAO,CAAC;AACpC,MAAM,eAAe,GAAG,MAAM,CAAC;AAC/B,6EAA6E;AAC7E,MAAM,qBAAqB,GAAG,OAAO,CAAC;AACtC,sCAAsC;AACtC,MAAM,gBAAgB,GAAG,KAAK,CAAC;AAkB/B,mCAAmC;AACnC,MAAM,yBAAyB,GAAG,MAAM,CAAC;AAEzC;;;;GAIG;AACH,SAAS,gBAAgB,CAAC,SAAkB;IAC3C,IAAI,OAAO,SAAS,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC;QAAE,OAAO,SAAS,CAAC;IACnF,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,yBAAyB,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;IACpF,OAAO,EAAE,eAAe,EAAE,CAAC,EAAE,CAAC;AAC/B,CAAC;AAED;;;GAGG;AACH,SAAS,gBAAgB,CAAC,KAAa;IACtC,OAAO,KAAK,CAAC,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;AAC9C,CAAC;AAED,SAAS,YAAY,CAAC,IAA4B,EAAE,KAAc;IACjE,MAAM,OAAO,GAAG,KAAK,EAAE,IAAI,EAAE,CAAC;IAC9B,IAAI,OAAO;QAAE,OAAO,gBAAgB,CAAC,OAAO,CAAC,CAAC;IAC9C,OAAO,qBAAqB,CAAC,IAAI,CAAC,IAAI,qBAAqB,CAAC,KAAK,CAAC;AACnE,CAAC;AAED,SAAS,aAAa,CAAC,IAA4B,EAAE,MAAe;IACnE,MAAM,OAAO,GAAG,MAAM,EAAE,IAAI,EAAE,CAAC;IAC/B,OAAO,OAAO,IAAI,eAAe,CAAC,IAAI,CAAC,CAAC;AACzC,CAAC;AAED,4DAA4D;AAC5D,SAAS,WAAW,CAAC,MAA+B,EAAE,EAAU;IAC/D,MAAM,aAAa,GAAG,WAAW,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAC9C,IAAI,CAAC,MAAM;QAAE,OAAO,aAAa,CAAC;IAClC,OAAO,WAAW,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC,CAAC;AACjD,CAAC;AAED,MAAM,YAAY,GAAG,CAAC,EAAU,EAAiB,EAAE,CAClD,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;AAE7D,6EAA6E;AAC7E,SAAS,oBAAoB,CAAC,OAAgB;IAC7C,MAAM,UAAU,GAAI,OAAsF;QACzG,EAAE,UAAU,CAAC;IACd,MAAM,KAAK,GAAG,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,IAAI,EAAE,CAAC;IACpD,OAAO,KAAK;SACV,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,EAAE,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;SAC9D,MAAM,CAAC,OAAO,CAAC;SACf,IAAI,CAAC,IAAI,CAAC;SACV,IAAI,EAAE,CAAC;AACV,CAAC;AAED,KAAK,UAAU,gBAAgB,CAAC,GAAa;IAC5C,IAAI,CAAC;QACJ,MAAM,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAAqC,CAAC;QACpE,IAAI,IAAI,EAAE,KAAK,EAAE,OAAO;YAAE,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC;IACrD,CAAC;IAAC,MAAM,CAAC;QACR,kBAAkB;IACnB,CAAC;IACD,OAAO,QAAQ,GAAG,CAAC,MAAM,EAAE,CAAC;AAC7B,CAAC;AAED,uFAAuF;AAEvF,KAAK,UAAU,cAAc,CAAC,MAA2B;IACxD,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,KAAK,CAAC;IACxC,MAAM,OAAO,GAAG,CAAC,MAAM,CAAC,OAAO,IAAI,uBAAuB,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IAChF,MAAM,KAAK,GAAG,YAAY,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;IACtD,MAAM,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;IACzD,MAAM,GAAG,GAAG,GAAG,OAAO,WAAW,KAAK,wBAAwB,kBAAkB,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;IAClG,MAAM,MAAM,GAAG,gBAAgB,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IAClD,MAAM,IAAI,GAAG;QACZ,QAAQ,EAAE;YACT;gBACC,IAAI,EAAE,MAAM;gBACZ,KAAK,EAAE;oBACN,EAAE,IAAI,EAAE,MAAM,EAAE;oBAChB;wBACC,WAAW,EAAE;4BACZ,SAAS,EAAE,MAAM,CAAC,QAAQ;4BAC1B,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC;yBACrC;qBACD;iBACD;aACD;SACD;QACD,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC/C,CAAC;IACF,IAAI,GAAa,CAAC;IAClB,IAAI,CAAC;QACJ,GAAG,GAAG,MAAM,OAAO,CAAC,GAAG,EAAE;YACxB,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE;YAC/C,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;YAC1B,MAAM,EAAE,WAAW,CAAC,MAAM,CAAC,MAAM,EAAE,mBAAmB,CAAC;SACvD,CAAC,CAAC;IACJ,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACd,MAAM,IAAI,+BAA+B,CACxC,QAAQ,EACR,0BAA0B,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAC5E,CAAC;IACH,CAAC;IACD,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;QACb,MAAM,IAAI,+BAA+B,CAAC,QAAQ,EAAE,iBAAiB,MAAM,gBAAgB,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IACjH,CAAC;IACD,MAAM,OAAO,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACnD,MAAM,IAAI,GAAG,oBAAoB,CAAC,OAAO,CAAC,CAAC;IAC3C,IAAI,CAAC,IAAI,EAAE,CAAC;QACX,MAAM,IAAI,+BAA+B,CAAC,QAAQ,EAAE,wCAAwC,CAAC,CAAC;IAC/F,CAAC;IACD,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC;AAC5C,CAAC;AAWD;;;;;;;GAOG;AACH,KAAK,UAAU,UAAU,CAAC,MAA2B;IACpD,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,KAAK,CAAC;IACxC,MAAM,OAAO,GAAG,CAAC,MAAM,CAAC,OAAO,IAAI,uBAAuB,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IAChF,8EAA8E;IAC9E,8EAA8E;IAC9E,+BAA+B;IAC/B,MAAM,SAAS,GAAG,GAAG,OAAO,CAAC,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC,4BAA4B,kBAAkB,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;IACrH,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC;IAErC,6CAA6C;IAC7C,IAAI,QAAkB,CAAC;IACvB,IAAI,CAAC;QACJ,QAAQ,GAAG,MAAM,OAAO,CAAC,SAAS,EAAE;YACnC,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACR,wBAAwB,EAAE,WAAW;gBACrC,uBAAuB,EAAE,OAAO;gBAChC,qCAAqC,EAAE,MAAM,CAAC,QAAQ,CAAC;gBACvD,mCAAmC,EAAE,MAAM,CAAC,QAAQ;gBACpD,cAAc,EAAE,kBAAkB;aAClC;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,EAAE,YAAY,EAAE,eAAe,EAAE,EAAE,CAAC;YACjE,MAAM,EAAE,WAAW,CAAC,MAAM,CAAC,MAAM,EAAE,iBAAiB,CAAC;SACrD,CAAC,CAAC;IACJ,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACd,MAAM,IAAI,+BAA+B,CACxC,QAAQ,EACR,sCAAsC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CACxF,CAAC;IACH,CAAC;IACD,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QAClB,MAAM,IAAI,+BAA+B,CACxC,QAAQ,EACR,qCAAqC,MAAM,gBAAgB,CAAC,QAAQ,CAAC,EAAE,EACvE,QAAQ,CAAC,MAAM,CACf,CAAC;IACH,CAAC;IACD,MAAM,UAAU,GACf,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,IAAI,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;IACxF,IAAI,CAAC,UAAU,EAAE,CAAC;QACjB,MAAM,IAAI,+BAA+B,CACxC,QAAQ,EACR,0DAA0D,CAC1D,CAAC;IACH,CAAC;IAED,wCAAwC;IACxC,IAAI,MAAgB,CAAC;IACrB,IAAI,CAAC;QACJ,MAAM,GAAG,MAAM,OAAO,CAAC,UAAU,EAAE;YAClC,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACR,gBAAgB,EAAE,MAAM,CAAC,QAAQ,CAAC;gBAClC,sBAAsB,EAAE,GAAG;gBAC3B,uBAAuB,EAAE,kBAAkB;aAC3C;YACD,IAAI,EAAE,IAAI,UAAU,CAAC,MAAM,CAAC,KAAK,CAAC;YAClC,MAAM,EAAE,WAAW,CAAC,MAAM,CAAC,MAAM,EAAE,iBAAiB,CAAC;SACrD,CAAC,CAAC;IACJ,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACd,MAAM,IAAI,+BAA+B,CACxC,QAAQ,EACR,yCAAyC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAC3F,CAAC;IACH,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC;QAChB,MAAM,IAAI,+BAA+B,CACxC,QAAQ,EACR,wCAAwC,MAAM,gBAAgB,CAAC,MAAM,CAAC,EAAE,EACxE,MAAM,CAAC,MAAM,CACb,CAAC;IACH,CAAC;IACD,MAAM,QAAQ,GAAG,CAAC,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAEtD,CAAC;IACF,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC;IAC3B,IAAI,CAAC,IAAI,EAAE,GAAG,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC;QAC/B,MAAM,IAAI,+BAA+B,CACxC,QAAQ,EACR,0CAA0C,CAC1C,CAAC;IACH,CAAC;IACD,OAAO;QACN,GAAG,EAAE,IAAI,CAAC,GAAG;QACb,IAAI,EAAE,IAAI,CAAC,IAAI;QACf,KAAK,EAAE,IAAI,CAAC,KAAK,IAAI,YAAY;QACjC,QAAQ,EAAE,IAAI,CAAC,QAAQ,IAAI,MAAM,CAAC,QAAQ;KAC1C,CAAC;AACH,CAAC;AAED,gEAAgE;AAChE,KAAK,UAAU,YAAY,CAC1B,QAAgB,EAChB,MAA2B;IAE3B,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,KAAK,CAAC;IACxC,MAAM,OAAO,GAAG,CAAC,MAAM,CAAC,OAAO,IAAI,uBAAuB,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IAChF,oFAAoF;IACpF,MAAM,GAAG,GAAG,QAAQ,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,QAAQ,EAAE,CAAC;IAC3E,MAAM,GAAG,GAAG,GAAG,OAAO,IAAI,GAAG,QAAQ,kBAAkB,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;IACzE,IAAI,GAAa,CAAC;IAClB,IAAI,CAAC;QACJ,GAAG,GAAG,MAAM,OAAO,CAAC,GAAG,EAAE;YACxB,MAAM,EAAE,KAAK;YACb,MAAM,EAAE,WAAW,CAAC,MAAM,CAAC,MAAM,EAAE,eAAe,CAAC;SACnD,CAAC,CAAC;IACJ,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACd,MAAM,IAAI,+BAA+B,CACxC,QAAQ,EACR,mCAAmC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CACrF,CAAC;IACH,CAAC;IACD,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;QACb,MAAM,IAAI,+BAA+B,CACxC,QAAQ,EACR,6BAA6B,MAAM,gBAAgB,CAAC,GAAG,CAAC,EAAE,EAC1D,GAAG,CAAC,MAAM,CACV,CAAC;IACH,CAAC;IACD,MAAM,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAI/C,CAAC;IACF,OAAO;QACN,KAAK,EAAE,IAAI,CAAC,KAAK,IAAI,YAAY;QACjC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACtC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACrD,CAAC;AACH,CAAC;AAED,mFAAmF;AACnF,KAAK,UAAU,aAAa,CAC3B,IAAkB,EAClB,MAA2B;IAE3B,MAAM,KAAK,GAAG,MAAM,CAAC,OAAO,IAAI,YAAY,CAAC;IAC7C,IAAI,IAAI,CAAC,KAAK,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC;IACzC,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,qBAAqB,CAAC;IACpD,IAAI,OAAO,GAAG,IAAI,CAAC;IACnB,SAAS,CAAC;QACT,IAAI,OAAO,CAAC,KAAK,KAAK,QAAQ;YAAE,OAAO,OAAO,CAAC;QAC/C,IAAI,OAAO,CAAC,KAAK,KAAK,QAAQ,EAAE,CAAC;YAChC,MAAM,IAAI,+BAA+B,CACxC,QAAQ,EACR,kEAAkE,CAClE,CAAC;QACH,CAAC;QACD,IAAI,IAAI,CAAC,GAAG,EAAE,IAAI,QAAQ,EAAE,CAAC;YAC5B,MAAM,IAAI,+BAA+B,CACxC,QAAQ,EACR,wFAAwF,CACxF,CAAC;QACH,CAAC;QACD,MAAM,KAAK,CAAC,gBAAgB,CAAC,CAAC;QAC9B,MAAM,IAAI,GAAG,MAAM,YAAY,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACtD,OAAO,GAAG;YACT,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,GAAG,EAAE,IAAI,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG;YAC5B,QAAQ,EAAE,IAAI,CAAC,QAAQ,IAAI,OAAO,CAAC,QAAQ;YAC3C,KAAK,EAAE,IAAI,CAAC,KAAK;SACjB,CAAC;IACH,CAAC;AACF,CAAC;AAED,qFAAqF;AACrF,KAAK,UAAU,gBAAgB,CAC9B,IAAkB,EAClB,MAA2B;IAE3B,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,KAAK,CAAC;IACxC,MAAM,OAAO,GAAG,CAAC,MAAM,CAAC,OAAO,IAAI,uBAAuB,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IAChF,MAAM,KAAK,GAAG,YAAY,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;IACtD,MAAM,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;IACzD,MAAM,GAAG,GAAG,GAAG,OAAO,WAAW,KAAK,wBAAwB,kBAAkB,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;IAClG,MAAM,MAAM,GAAG,gBAAgB,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IAClD,MAAM,IAAI,GAAG;QACZ,QAAQ,EAAE;YACT;gBACC,IAAI,EAAE,MAAM;gBACZ,KAAK,EAAE;oBACN,EAAE,IAAI,EAAE,MAAM,EAAE;oBAChB,EAAE,SAAS,EAAE,EAAE,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,SAAS,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE;iBAC/D;aACD;SACD;QACD,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC/C,CAAC;IACF,IAAI,GAAa,CAAC;IAClB,IAAI,CAAC;QACJ,GAAG,GAAG,MAAM,OAAO,CAAC,GAAG,EAAE;YACxB,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE;YAC/C,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;YAC1B,MAAM,EAAE,WAAW,CAAC,MAAM,CAAC,MAAM,EAAE,mBAAmB,CAAC;SACvD,CAAC,CAAC;IACJ,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACd,MAAM,IAAI,+BAA+B,CACxC,QAAQ,EACR,uCAAuC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CACzF,CAAC;IACH,CAAC;IACD,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;QACb,MAAM,IAAI,+BAA+B,CACxC,QAAQ,EACR,0BAA0B,MAAM,gBAAgB,CAAC,GAAG,CAAC,EAAE,EACvD,GAAG,CAAC,MAAM,CACV,CAAC;IACH,CAAC;IACD,MAAM,OAAO,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACnD,MAAM,IAAI,GAAG,oBAAoB,CAAC,OAAO,CAAC,CAAC;IAC3C,IAAI,CAAC,IAAI,EAAE,CAAC;QACX,MAAM,IAAI,+BAA+B,CAAC,QAAQ,EAAE,wCAAwC,CAAC,CAAC;IAC/F,CAAC;IACD,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC;AAC5C,CAAC;AAED,yEAAyE;AAEzE;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,MAA2B;IAC1D,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QACpB,MAAM,IAAI,+BAA+B,CAAC,QAAQ,EAAE,sCAAsC,CAAC,CAAC;IAC7F,CAAC;IACD,IAAI,MAAM,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;QAC7B,MAAM,QAAQ,GAAG,MAAM,UAAU,CAAC,MAAM,CAAC,CAAC;QAC1C,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QACrD,OAAO,gBAAgB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACzC,CAAC;IACD,qCAAqC;IACrC,OAAO,cAAc,CAAC,MAAM,CAAC,CAAC;AAC/B,CAAC;AAED,sBAAsB;AACtB,OAAO,EAAE,oBAAoB,EAAE,gBAAgB,EAAE,CAAC"}

package/dist/agents/media-understanding/index.d.ts

@@ -0,0 +1,58 @@
+/**
+ * Media-understanding subsystem — single entry point + provider selection.
+ *
+ * `runMediaUnderstanding({ kind, bytes, mimeType, prompt, provider?, model?,
+ * cfg, fetchImpl? })` resolves a capable provider that has a configured key,
+ * calls its REST API directly with the media + prompt, and returns the model's
+ * TEXT answer. This is what lets `analyze_media` understand VIDEO (Gemini Files
+ * API) and native/scanned PDFs (Anthropic document blocks / Gemini) even though
+ * Pi's tool-result channel carries only text + image.
+ *
+ * Selection (`resolveMediaUnderstandingProvider`):
+ *   • video → Gemini (the only adapter with video; via the Files API).
+ *   • pdf   → prefer Anthropic (native + OCR for scanned), else Gemini.
+ *   • image → Anthropic, then Gemini (bespoke REST), then the Pi path (`pi`) —
+ *     a one-shot `completeSimple` against ANY keyed provider with an
+ *     image-capable model (OpenAI / Groq / Mistral / OpenRouter / xAI / Ollama
+ *     / …), so image understanding is no longer limited to google + anthropic.
+ *   • audio → Gemini ONLY. Pi's content model is text + image (`Model.input` is
+ *     `("text"|"image")[]`) — NO provider Pi can drive accepts an audio block,
+ *     so the Pi path is deliberately NOT in the audio chain: routing a voice
+ *     note through it would stuff audio bytes into an IMAGE block and the
+ *     provider would reject it (HTTP 400). Audio understanding is Gemini's real
+ *     capability (inline audio); with no Google key the caller gets a clean
+ *     "needs a Google/Gemini key" message instead of a 400.
+ * A `cfg.preferredProvider[kind]` override wins when that provider has a key.
+ * When NO provider with a key can handle the kind, a clear
+ * `MediaUnderstandingUnavailableError` is thrown.
+ */
+import { type MediaUnderstandingConfig, type MediaUnderstandingKind, type MediaUnderstandingProviderId, type RunMediaUnderstandingRequest, type RunMediaUnderstandingResult } from "./types.js";
+export { MediaUnderstandingProviderError, MediaUnderstandingUnavailableError, type MediaUnderstandingConfig, type MediaUnderstandingKind, type MediaUnderstandingModel, type MediaUnderstandingProviderId, type PiCompleteFn, type PiCompleteRequest, type RunMediaUnderstandingRequest, type RunMediaUnderstandingResult, } from "./types.js";
+export { DEFAULT_GEMINI_MODELS, DEFAULT_GEMINI_BASE_URL } from "./gemini-adapter.js";
+export { DEFAULT_ANTHROPIC_MODEL, DEFAULT_ANTHROPIC_BASE_URL } from "./anthropic-adapter.js";
+export { resolvePiModel, runPi, modelAcceptsImage, defaultPiComplete } from "./pi-adapter.js";
+/**
+ * Pick a provider that (a) can handle `kind` and (b) has a resolved key.
+ * Honors `cfg.preferredProvider[kind]` first when that provider is both
+ * capable and keyed. For IMAGE / AUDIO, when neither google nor anthropic is
+ * keyed but the Pi path can resolve a capable model for SOME keyed provider,
+ * returns the virtual `"pi"` provider so every configured provider works.
+ * Returns `undefined` when nothing qualifies.
+ */
+export declare function resolveMediaUnderstandingProvider(kind: MediaUnderstandingKind, cfg: MediaUnderstandingConfig): MediaUnderstandingProviderId | undefined;
+/**
+ * Run a media-understanding request against a capable, keyed provider and
+ * return the textual answer. Throws `MediaUnderstandingUnavailableError` when
+ * no provider can serve the kind, or `MediaUnderstandingProviderError` when the
+ * chosen provider's API call fails.
+ */
+export declare function runMediaUnderstanding(req: RunMediaUnderstandingRequest): Promise<RunMediaUnderstandingResult>;
+/**
+ * A provider error is RETRYABLE (worth a retry / a fallover) when it is a rate
+ * limit (429) or a server/transport error (5xx, or a transport throw with no
+ * status). A 4xx other than 429 (bad request, auth) is NOT retryable — retrying
+ * or falling over wouldn't help and would waste calls. `Unavailable` errors are
+ * retryable in the fallover sense (the NEXT provider might serve the kind).
+ */
+export declare function isRetryableError(err: unknown): boolean;
+//# sourceMappingURL=index.d.ts.map

package/dist/agents/media-understanding/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/agents/media-understanding/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAKH,OAAO,EAGN,KAAK,wBAAwB,EAC7B,KAAK,sBAAsB,EAC3B,KAAK,4BAA4B,EACjC,KAAK,4BAA4B,EACjC,KAAK,2BAA2B,EAChC,MAAM,YAAY,CAAC;AAEpB,OAAO,EACN,+BAA+B,EAC/B,kCAAkC,EAClC,KAAK,wBAAwB,EAC7B,KAAK,sBAAsB,EAC3B,KAAK,uBAAuB,EAC5B,KAAK,4BAA4B,EACjC,KAAK,YAAY,EACjB,KAAK,iBAAiB,EACtB,KAAK,4BAA4B,EACjC,KAAK,2BAA2B,GAChC,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,qBAAqB,EAAE,uBAAuB,EAAE,MAAM,qBAAqB,CAAC;AACrF,OAAO,EAAE,uBAAuB,EAAE,0BAA0B,EAAE,MAAM,wBAAwB,CAAC;AAC7F,OAAO,EAAE,cAAc,EAAE,KAAK,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AA2E9F;;;;;;;GAOG;AACH,wBAAgB,iCAAiC,CAChD,IAAI,EAAE,sBAAsB,EAC5B,GAAG,EAAE,wBAAwB,GAC3B,4BAA4B,GAAG,SAAS,CAS1C;AAED;;;;;GAKG;AACH,wBAAsB,qBAAqB,CAC1C,GAAG,EAAE,4BAA4B,GAC/B,OAAO,CAAC,2BAA2B,CAAC,CAuDtC;AAgGD;;;;;;GAMG;AACH,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,OAAO,GAAG,OAAO,CAStD"}