@opencode/ai 2.0.14 → 2.0.16

package/dist/protocols/google-images.js

@@ -1,12 +1,18 @@
-import { Effect, Encoding, Schema } from "effect";
-import { Headers, HttpClientRequest } from "effect/unstable/http";
-import { GeneratedImage, ImageModel, ImageResponse, } from "../image.js";
-import { Auth } from "../route/auth.js";
-import { AIError, Usage, mergeHttpOptions, mergeJsonRecords } from "../schema/index.js";
+import { Effect, Schema } from "effect";
+import { ImageModel, ImageResponse } from "../image.js";
+import { MediaProtocol } from "../route/media-protocol.js";
+import { MediaRoute } from "../route/media.js";
+import { ProviderID, mergeJsonRecords } from "../schema/index.js";
 import { ProviderShared } from "./shared.js";
-import { ImageInputs } from "./utils/image-input.js";
+import { GeminiGenerateContent } from "./utils/gemini-generate-content.js";
+import { MediaInput } from "./utils/media-input.js";
 const ADAPTER = "google-images";
+const NAME = "Google Images";
+const PROVIDER = ProviderID.make("google");
 export const DEFAULT_BASE_URL = "https://generativelanguage.googleapis.com/v1beta";
+// ---------------------------------------------------------------------------
+// 2. Response schema
+// ---------------------------------------------------------------------------
 const GoogleUsage = Schema.StructWithRest(Schema.Struct({
     cachedContentTokenCount: Schema.optional(Schema.Number),
     thoughtsTokenCount: Schema.optional(Schema.Number),
@@ -41,141 +47,160 @@ const GoogleImageResponse = Schema.Struct({
     responseId: Schema.optional(Schema.String),
     promptFeedback: Schema.optional(Schema.Unknown),
 });
-const nativeOptions = (options) => {
-    const { aspectRatio, imageSize, seed, thinkingLevel, includeThoughts, ...native } = options ?? {};
-    const image = {
-        aspectRatio,
-        imageSize,
-    };
-    const thinkingConfig = {
-        thinkingLevel,
-        includeThoughts,
-    };
+// ---------------------------------------------------------------------------
+// 5. Request body construction
+// ---------------------------------------------------------------------------
+const generationConfig = (request) => {
+    const { imageSize, thinkingLevel, includeThoughts, ...native } = request.providerOptions ?? {};
+    const imageConfig = { aspectRatio: request.aspectRatio, imageSize };
+    const thinkingConfig = { thinkingLevel, includeThoughts };
     return (mergeJsonRecords({
         responseModalities: ["IMAGE"],
-        imageConfig: Object.values(image).some((value) => value !== undefined) ? image : undefined,
-        seed,
+        imageConfig: Object.values(imageConfig).some((value) => value !== undefined) ? imageConfig : undefined,
+        seed: request.seed,
         thinkingConfig: Object.values(thinkingConfig).some((value) => value !== undefined) ? thinkingConfig : undefined,
     }, native) ?? { responseModalities: ["IMAGE"] });
 };
-const applyQuery = (url, query) => {
-    if (!query)
-        return url;
-    const next = new URL(url);
-    Object.entries(query).forEach(([key, value]) => next.searchParams.set(key, value));
-    return next.toString();
-};
-export const model = (input) => {
-    const route = {
-        id: ADAPTER,
-        generate: Effect.fn("GoogleImages.generate")(function* (request, execute) {
-            const imageParts = yield* Effect.forEach(request.images ?? [], googleImagePart);
-            const http = mergeHttpOptions(request.model.http, request.http);
-            const requestBody = mergeJsonRecords({
-                contents: [{ role: "user", parts: [{ text: request.prompt }, ...imageParts] }],
-                generationConfig: nativeOptions(request.options),
-            }, http?.body);
-            const text = ProviderShared.encodeJson(requestBody);
-            const url = applyQuery(`${(input.baseURL ?? DEFAULT_BASE_URL).replace(/\/$/, "")}/models/${request.model.id}:generateContent`, http?.query);
-            const headers = yield* Auth.toEffect(input.auth)({
-                request,
-                method: "POST",
-                url,
-                body: text,
-                headers: Headers.fromInput({ ...input.headers, ...http?.headers }),
-            });
-            const response = yield* execute(HttpClientRequest.post(url).pipe(HttpClientRequest.setHeaders(headers), HttpClientRequest.bodyText(text, "application/json")));
-            const output = yield* ProviderShared.imageResponse(ADAPTER, "Google Images", response);
-            const decoded = yield* Schema.decodeUnknownEffect(Schema.fromJsonString(GoogleImageResponse))(output.body).pipe(Effect.mapError((cause) => output.invalid("Google Images returned an invalid response", cause)));
-            const candidates = decoded.candidates ?? [];
-            const candidateMetadata = candidates.map((candidate, candidateIndex) => ({
-                index: candidate.index ?? candidateIndex,
-                finishReason: candidate.finishReason,
-                finishMessage: candidate.finishMessage,
-                safetyRatings: candidate.safetyRatings,
-                citationMetadata: candidate.citationMetadata,
-                groundingMetadata: candidate.groundingMetadata,
-                parts: (candidate.content?.parts ?? []).map((part) => part.inlineData === undefined
-                    ? {
-                        type: "text",
-                        text: part.text,
-                        thought: part.thought,
-                        thoughtSignature: part.thoughtSignature,
-                    }
-                    : {
-                        type: "inlineData",
-                        mediaType: part.inlineData.mimeType,
-                        thought: part.thought,
-                        thoughtSignature: part.thoughtSignature,
-                    }),
-            }));
-            const encoded = candidates.flatMap((candidate, candidateIndex) => (candidate.content?.parts ?? []).flatMap((part, partIndex) => part.inlineData === undefined || part.thought === true
-                ? []
-                : [{ candidate, candidateIndex, partIndex, inlineData: part.inlineData }]));
-            const images = yield* Effect.forEach(encoded, (item) => Effect.fromResult(Encoding.decodeBase64(item.inlineData.data)).pipe(Effect.mapError((cause) => output.invalid(`Google Images candidate ${item.candidateIndex} part ${item.partIndex} contains invalid base64 data`, cause)), Effect.map((data) => new GeneratedImage({
-                mediaType: item.inlineData.mimeType,
-                data,
-                providerMetadata: {
-                    google: {
-                        candidateIndex: item.candidate.index ?? item.candidateIndex,
-                        partIndex: item.partIndex,
-                        finishReason: item.candidate.finishReason,
-                        safetyRatings: item.candidate.safetyRatings,
-                        citationMetadata: item.candidate.citationMetadata,
-                        groundingMetadata: item.candidate.groundingMetadata,
-                        thoughtSignature: item.candidate.content?.parts[item.partIndex]?.thoughtSignature,
-                    },
+const fromRequest = Effect.fn("GoogleImages.fromRequest")(function* (request) {
+    if (request.n !== undefined && request.n > 1)
+        return yield* ProviderShared.unsupportedOperation({
+            operation: "image.n",
+            provider: PROVIDER,
+            route: ADAPTER,
+            message: `${NAME} generates one image per request; call it once per image instead of n=${request.n}`,
+        });
+    const parts = yield* Effect.forEach(request.images ?? [], (image) => GeminiGenerateContent.mediaPart(NAME, image));
+    return MediaProtocol.json(mergeJsonRecords({
+        contents: [{ role: "user", parts: [{ text: request.prompt }, ...parts] }],
+        generationConfig: generationConfig(request),
+    }, request.http?.body) ?? {});
+});
+// ---------------------------------------------------------------------------
+// 6. Response decoding
+// ---------------------------------------------------------------------------
+const decodeResponse = Effect.fn("GoogleImages.decodeResponse")(function* (response) {
+    const output = yield* MediaProtocol.decodeJson(ADAPTER, NAME, GoogleImageResponse)(response);
+    const decoded = output.value;
+    const candidates = decoded.candidates ?? [];
+    const candidateMetadata = candidates.map((candidate, candidateIndex) => ({
+        index: candidate.index ?? candidateIndex,
+        finishReason: candidate.finishReason,
+        finishMessage: candidate.finishMessage,
+        safetyRatings: candidate.safetyRatings,
+        citationMetadata: candidate.citationMetadata,
+        groundingMetadata: candidate.groundingMetadata,
+        parts: (candidate.content?.parts ?? []).map((part) => part.inlineData === undefined
+            ? { type: "text", text: part.text, thought: part.thought, thoughtSignature: part.thoughtSignature }
+            : {
+                type: "inlineData",
+                mediaType: part.inlineData.mimeType,
+                thought: part.thought,
+                thoughtSignature: part.thoughtSignature,
+            }),
+    }));
+    // Thought parts are drafts; only non-thought inline data is a final image.
+    const encoded = candidates.flatMap((candidate, candidateIndex) => (candidate.content?.parts ?? []).flatMap((part, partIndex) => part.inlineData === undefined || part.thought === true
+        ? []
+        : [
+            {
+                candidate,
+                candidateIndex,
+                partIndex,
+                inlineData: part.inlineData,
+                thoughtSignature: part.thoughtSignature,
+            },
+        ]));
+    const images = yield* Effect.forEach(encoded, (item) => MediaInput.decodedAsset(output.invalid, `${NAME} candidate ${item.candidateIndex} part ${item.partIndex}`, item.inlineData.data, item.inlineData.mimeType, {
+        providerMetadata: {
+            google: {
+                candidateIndex: item.candidate.index ?? item.candidateIndex,
+                partIndex: item.partIndex,
+                finishReason: item.candidate.finishReason,
+                safetyRatings: item.candidate.safetyRatings,
+                citationMetadata: item.candidate.citationMetadata,
+                groundingMetadata: item.candidate.groundingMetadata,
+                thoughtSignature: item.thoughtSignature,
+            },
+        },
+    }));
+    if (images.length === 0) {
+        const finishReasons = candidates.flatMap((candidate) => candidate.finishReason === undefined ? [] : [candidate.finishReason]);
+        return yield* output.invalid(`${NAME} returned no final images${finishReasons.length === 0 ? "" : ` (finish reasons: ${finishReasons.join(", ")})`}; inspect body for prompt feedback and candidate details`);
+    }
+    // Candidates that stopped for a safety or policy reason are partial results, not a silent drop.
+    const notices = [
+        ...(decoded.promptFeedback === undefined
+            ? []
+            : [
+                {
+                    type: "filtered",
+                    message: `${NAME} reported prompt feedback`,
+                    providerMetadata: { google: { promptFeedback: decoded.promptFeedback } },
                 },
-            }))));
-            if (images.length === 0) {
-                const finishReasons = candidates.flatMap((candidate) => candidate.finishReason === undefined ? [] : [candidate.finishReason]);
-                return yield* output.invalid(`Google Images returned no final images${finishReasons.length === 0 ? "" : ` (finish reasons: ${finishReasons.join(", ")})`}; inspect body for prompt feedback and candidate details`);
-            }
-            const usage = decoded.usageMetadata;
-            const outputTokens = usage?.candidatesTokenCount === undefined
-                ? undefined
-                : usage.candidatesTokenCount + (usage.thoughtsTokenCount ?? 0);
-            return new ImageResponse({
-                images,
-                usage: usage === undefined
-                    ? undefined
-                    : new Usage({
-                        inputTokens: usage.promptTokenCount,
-                        outputTokens,
-                        nonCachedInputTokens: ProviderShared.subtractTokens(usage.promptTokenCount, usage.cachedContentTokenCount),
-                        cacheReadInputTokens: usage.cachedContentTokenCount,
-                        reasoningTokens: usage.thoughtsTokenCount,
-                        totalTokens: ProviderShared.totalTokens(usage.promptTokenCount, outputTokens, usage.totalTokenCount),
-                        providerMetadata: { google: usage },
-                    }),
-                providerMetadata: {
-                    google: {
-                        modelVersion: decoded.modelVersion,
-                        responseId: decoded.responseId,
-                        promptFeedback: decoded.promptFeedback,
-                        candidates: candidateMetadata,
+            ]),
+        ...candidates.flatMap((candidate, index) => candidate.finishReason === undefined || candidate.finishReason === "STOP"
+            ? []
+            : [
+                {
+                    type: "filtered",
+                    message: `${NAME} candidate ${candidate.index ?? index} finished with ${candidate.finishReason}${candidate.finishMessage === undefined ? "" : `: ${candidate.finishMessage}`}`,
+                    providerMetadata: {
+                        google: {
+                            candidateIndex: candidate.index ?? index,
+                            finishReason: candidate.finishReason,
+                            finishMessage: candidate.finishMessage,
+                            safetyRatings: candidate.safetyRatings,
+                        },
                     },
                 },
-            });
-        }),
-    };
-    return ImageModel.make({ id: input.id, provider: "google", route, http: input.http });
-};
-const googleImagePart = (image) => {
-    if (image.type === "bytes")
-        return Effect.succeed({ inlineData: { mimeType: image.mediaType, data: Encoding.encodeBase64(image.data) } });
-    if (image.type === "file-uri")
-        return Effect.succeed({ fileData: { mimeType: image.mediaType, fileUri: image.uri } });
-    if (image.type === "url")
-        return ImageInputs.decodeDataUrl(image.url).pipe(Effect.flatMap((decoded) => {
-            if (decoded === undefined)
-                return Effect.fail(ImageInputs.invalid("Google generateContent does not fetch public image URLs; use bytes, a data URL, or a Gemini file URI"));
-            return Effect.succeed({
-                inlineData: { mimeType: decoded.mediaType, data: Encoding.encodeBase64(decoded.data) },
-            });
-        }));
-    return Effect.fail(ImageInputs.invalid("Google generateContent requires Gemini file URIs rather than provider file IDs"));
-};
+            ]),
+    ];
+    const usage = decoded.usageMetadata;
+    const outputTokens = usage?.candidatesTokenCount === undefined ? undefined : usage.candidatesTokenCount + (usage.thoughtsTokenCount ?? 0);
+    return new ImageResponse({
+        images,
+        notices: notices.length === 0 ? undefined : notices,
+        usage: usage === undefined
+            ? undefined
+            : {
+                type: "tokens",
+                input: usage.promptTokenCount,
+                output: outputTokens,
+                total: ProviderShared.totalTokens(usage.promptTokenCount, outputTokens, usage.totalTokenCount),
+                details: {
+                    reasoningTokens: usage.thoughtsTokenCount,
+                    cacheReadInputTokens: usage.cachedContentTokenCount,
+                    google: usage,
+                },
+            },
+        providerMetadata: {
+            google: {
+                modelVersion: decoded.modelVersion,
+                responseId: decoded.responseId,
+                promptFeedback: decoded.promptFeedback,
+                candidates: candidateMetadata,
+            },
+        },
+    });
+});
+// ---------------------------------------------------------------------------
+// 7. Protocol and route
+// ---------------------------------------------------------------------------
+export const protocol = MediaProtocol.inline({
+    id: ADAPTER,
+    name: NAME,
+    unsupported: ["mask", "size", "format"],
+    body: { from: fromRequest },
+    response: { decode: decodeResponse },
+});
+export const model = (input) => ImageModel.fromRoute({
+    id: ADAPTER,
+    provider: PROVIDER,
+    protocol,
+    baseURL: DEFAULT_BASE_URL,
+    path: ({ request }) => `/models/${request.model.id}:generateContent`,
+}, input);
 export const GoogleImages = {
+    protocol,
     model,
 };

package/dist/protocols/google-speech.d.ts

@@ -0,0 +1,130 @@
+import { MediaProtocol } from "../route/media-protocol.js";
+import { MediaRoute } from "../route/media.js";
+import { SpeechModel, type SpeechRequestFor } from "../speech.js";
+import { GeminiGenerateContent } from "./utils/gemini-generate-content.js";
+import { SpeechStream } from "./utils/speech-stream.js";
+export declare const DEFAULT_BASE_URL = "https://generativelanguage.googleapis.com/v1beta";
+/** Style is directed in the text itself, and `speechConfig.multiSpeakerVoiceConfig` excludes `voice`. */
+export type GoogleSpeechOptions = {
+    readonly temperature?: number;
+    readonly seed?: number;
+    readonly speechConfig?: {
+        readonly multiSpeakerVoiceConfig?: {
+            readonly speakerVoiceConfigs: ReadonlyArray<{
+                readonly speaker: string;
+                readonly voiceConfig: {
+                    readonly prebuiltVoiceConfig: {
+                        readonly voiceName: string;
+                    };
+                };
+            }>;
+        };
+    };
+} & Record<string, unknown>;
+export type Request = SpeechRequestFor<GoogleSpeechOptions>;
+interface State extends SpeechStream.Audio, GeminiGenerateContent.Metadata {
+    readonly mimeType?: string;
+}
+export declare const protocol: MediaProtocol.Streamed<Request, {
+    readonly type: "audio-delta";
+    readonly chunk: Uint8Array<ArrayBufferLike>;
+} | {
+    readonly type: "timestamps";
+    readonly items: readonly {
+        readonly text: string;
+        readonly startSeconds: number;
+        readonly endSeconds: number;
+    }[];
+} | {
+    readonly type: "finish";
+    readonly audio: import("../media.js").Asset;
+    readonly providerMetadata?: {
+        readonly [x: string]: {
+            readonly [x: string]: unknown;
+        };
+    } | undefined;
+    readonly usage?: {
+        readonly type: "tokens";
+        readonly input?: number | undefined;
+        readonly output?: number | undefined;
+        readonly total?: number | undefined;
+        readonly details?: {
+            readonly [x: string]: unknown;
+        } | undefined;
+    } | {
+        readonly type: "seconds";
+        readonly seconds: number;
+    } | {
+        readonly type: "characters";
+        readonly characters: number;
+    } | {
+        readonly type: "credits";
+        readonly credits: number;
+    } | {
+        readonly type: "compute";
+        readonly seconds: number;
+    } | undefined;
+    readonly notices?: readonly {
+        readonly type: "other" | "moderated" | "filtered";
+        readonly message: string;
+        readonly providerMetadata?: {
+            readonly [x: string]: {
+                readonly [x: string]: unknown;
+            };
+        } | undefined;
+    }[] | undefined;
+}, string, State>;
+export declare const model: (input: MediaRoute.ModelInput) => SpeechModel<GoogleSpeechOptions>;
+export declare const GoogleSpeech: {
+    readonly protocol: MediaProtocol.Streamed<Request, {
+        readonly type: "audio-delta";
+        readonly chunk: Uint8Array<ArrayBufferLike>;
+    } | {
+        readonly type: "timestamps";
+        readonly items: readonly {
+            readonly text: string;
+            readonly startSeconds: number;
+            readonly endSeconds: number;
+        }[];
+    } | {
+        readonly type: "finish";
+        readonly audio: import("../media.js").Asset;
+        readonly providerMetadata?: {
+            readonly [x: string]: {
+                readonly [x: string]: unknown;
+            };
+        } | undefined;
+        readonly usage?: {
+            readonly type: "tokens";
+            readonly input?: number | undefined;
+            readonly output?: number | undefined;
+            readonly total?: number | undefined;
+            readonly details?: {
+                readonly [x: string]: unknown;
+            } | undefined;
+        } | {
+            readonly type: "seconds";
+            readonly seconds: number;
+        } | {
+            readonly type: "characters";
+            readonly characters: number;
+        } | {
+            readonly type: "credits";
+            readonly credits: number;
+        } | {
+            readonly type: "compute";
+            readonly seconds: number;
+        } | undefined;
+        readonly notices?: readonly {
+            readonly type: "other" | "moderated" | "filtered";
+            readonly message: string;
+            readonly providerMetadata?: {
+                readonly [x: string]: {
+                    readonly [x: string]: unknown;
+                };
+            } | undefined;
+        }[] | undefined;
+    }, string, State>;
+    readonly model: (input: MediaRoute.ModelInput) => SpeechModel<GoogleSpeechOptions>;
+};
+export {};

package/dist/protocols/google-speech.js

@@ -0,0 +1,84 @@
+import { Effect, Schema } from "effect";
+import { MediaProtocol } from "../route/media-protocol.js";
+import { MediaRoute } from "../route/media.js";
+import { ProviderID, mergeJsonRecords } from "../schema/index.js";
+import { SpeechModel } from "../speech.js";
+import { GeminiGenerateContent } from "./utils/gemini-generate-content.js";
+import { SpeechStream } from "./utils/speech-stream.js";
+const ADAPTER = "google-speech";
+const NAME = "Google Speech";
+const PROVIDER = ProviderID.make("google");
+export const DEFAULT_BASE_URL = "https://generativelanguage.googleapis.com/v1beta";
+const DEFAULT_SAMPLE_RATE = 24000;
+// ---------------------------------------------------------------------------
+// 3. Streaming event schema
+// ---------------------------------------------------------------------------
+const GenerateContentChunk = GeminiGenerateContent.chunk(Schema.Struct({
+    text: Schema.optional(Schema.String),
+    inlineData: Schema.optional(Schema.Struct({ mimeType: Schema.String, data: Schema.Uint8ArrayFromBase64 })),
+}));
+const decodeChunk = MediaProtocol.decodeFrame(ADAPTER, NAME, GenerateContentChunk);
+// ---------------------------------------------------------------------------
+// 5. Request body construction
+// ---------------------------------------------------------------------------
+const fromRequest = Effect.fn("GoogleSpeech.fromRequest")(function* (request) {
+    if (request.format !== undefined && request.format !== "pcm")
+        return yield* SpeechStream.unsupportedFormat(PROVIDER, ADAPTER, `${NAME} only returns raw PCM; request format "pcm" or omit it, then wrap the samples yourself`);
+    const voiceName = SpeechStream.voiceID(request.voice);
+    return MediaProtocol.json(mergeJsonRecords({
+        contents: [{ role: "user", parts: [{ text: request.text }] }],
+        generationConfig: mergeJsonRecords({
+            responseModalities: ["AUDIO"],
+            speechConfig: {
+                voiceConfig: voiceName === undefined ? undefined : { prebuiltVoiceConfig: { voiceName } },
+                languageCode: request.language,
+            },
+        }, request.providerOptions),
+    }, request.http?.body) ?? {});
+});
+// ---------------------------------------------------------------------------
+// 6. Stream parsing
+// ---------------------------------------------------------------------------
+const step = Effect.fn("GoogleSpeech.step")(function* (state, frame) {
+    const chunk = yield* decodeChunk(frame);
+    const blocked = GeminiGenerateContent.blocked(NAME, chunk, frame);
+    if (blocked !== undefined)
+        return yield* blocked;
+    const audio = (chunk.candidates?.[0]?.content?.parts ?? []).flatMap((part) => part.inlineData === undefined ? [] : [part.inlineData]);
+    const next = { ...GeminiGenerateContent.track(state, chunk), mimeType: state.mimeType ?? audio[0]?.mimeType };
+    return [next, audio.flatMap((part) => SpeechStream.delta(next, part.data)[1])];
+});
+const finish = (state) => {
+    const sampleRate = SpeechStream.sampleRate(state.mimeType) ?? DEFAULT_SAMPLE_RATE;
+    return SpeechStream.finish(ADAPTER, state, {
+        ...SpeechStream.pcm("pcm_s16le", sampleRate, state.mimeType ?? `audio/L16;codec=pcm;rate=${sampleRate}`),
+        usage: GeminiGenerateContent.usage(state.usage),
+        providerMetadata: GeminiGenerateContent.providerMetadata(state),
+        detail: state.finishReason === undefined ? undefined : `finish reason: ${state.finishReason}`,
+    });
+};
+// ---------------------------------------------------------------------------
+// 7. Protocol and route
+// ---------------------------------------------------------------------------
+export const protocol = MediaProtocol.stream({
+    id: ADAPTER,
+    name: NAME,
+    unsupported: ["instructions", "speed", "timestamps"],
+    body: { from: fromRequest },
+    frames: (bytes, context) => GeminiGenerateContent.frames(bytes, context.request.mode),
+    initial: () => ({ chunks: [] }),
+    step,
+    finish,
+});
+export const model = (input) => SpeechModel.fromRoute({
+    id: ADAPTER,
+    provider: PROVIDER,
+    protocol,
+    baseURL: DEFAULT_BASE_URL,
+    // Only `gemini-3.1-flash-tts-preview` and later stream; earlier TTS models reject `streamGenerateContent`.
+    path: ({ request }) => GeminiGenerateContent.path(request.model.id, request.mode),
+}, input);
+export const GoogleSpeech = {
+    protocol,
+    model,
+};