@effect-uai/google 0.3.0 → 0.5.0

package/src/GeminiSynthesizer.ts

@@ -0,0 +1,214 @@
+import { Context, Effect, Layer, Match, Redacted, Schema, Stream } from "effect"
+import { HttpClient, HttpClientRequest } from "effect/unstable/http"
+import * as AiError from "@effect-uai/core/AiError"
+import type { AudioBlob, AudioChunk, AudioFormat } from "@effect-uai/core/Audio"
+import {
+  type CommonStreamSynthesizeRequest,
+  type CommonSynthesizeRequest,
+  SpeechSynthesizer,
+  type SpeechSynthesizerService,
+} from "@effect-uai/core/SpeechSynthesizer"
+import {
+  decodeBase64Audio,
+  httpStatusError,
+  transportFailure,
+  wrapPcmAsWav,
+} from "./geminiSpeechCodec.js"
+import type { GeminiTtsModel, GeminiVoiceName } from "./models.js"
+
+// ---------------------------------------------------------------------------
+// Public types
+// ---------------------------------------------------------------------------
+
+export type GeminiSynthesizeRequest = Omit<CommonSynthesizeRequest, "model" | "voiceId"> & {
+  readonly model: GeminiTtsModel
+  readonly voiceId: GeminiVoiceName
+}
+
+export type GeminiSynthesizerService = {
+  readonly synthesize: (r: GeminiSynthesizeRequest) => Effect.Effect<AudioBlob, AiError.AiError>
+  readonly streamSynthesis: (
+    r: GeminiSynthesizeRequest,
+  ) => Stream.Stream<AudioChunk, AiError.AiError>
+  readonly streamSynthesisFrom: SpeechSynthesizerService["streamSynthesisFrom"]
+}
+
+export class GeminiSynthesizer extends Context.Service<
+  GeminiSynthesizer,
+  GeminiSynthesizerService
+>()("@betalyra/effect-uai/providers/google/GeminiSynthesizer") {}
+
+export type Config = { readonly apiKey: Redacted.Redacted; readonly baseUrl?: string }
+
+// ---------------------------------------------------------------------------
+// Codec — Gemini TTS always emits 24 kHz mono signed-16 PCM. Caller picks
+// raw bytes or WAV-wrapped.
+// ---------------------------------------------------------------------------
+
+const PCM_RATE = 24000
+const PCM_CHANNELS = 1
+
+export const realizeOutput = Match.type<AudioFormat["container"]>().pipe(
+  Match.when("raw", () =>
+    Effect.succeed([
+      { container: "raw", encoding: "pcm_s16le", sampleRate: PCM_RATE, channels: PCM_CHANNELS },
+      (b: Uint8Array) => b,
+    ] as const),
+  ),
+  Match.when("wav", () =>
+    Effect.succeed([
+      { container: "wav", encoding: "pcm_s16le", sampleRate: PCM_RATE, channels: PCM_CHANNELS },
+      (b: Uint8Array) => wrapPcmAsWav(b, PCM_RATE, PCM_CHANNELS),
+    ] as const),
+  ),
+  Match.whenOr("mp3", "opus", "aac", "flac", "ogg", "webm", (c) =>
+    Effect.fail(
+      new AiError.Unsupported({
+        provider: "gemini",
+        capability: "outputFormat",
+        reason: `Gemini TTS emits raw 16-bit PCM at 24 kHz. Request "raw" or "wav"; ${c} is unavailable.`,
+      }),
+    ),
+  ),
+  Match.exhaustive,
+)
+
+// ---------------------------------------------------------------------------
+// Wire
+// ---------------------------------------------------------------------------
+
+const Wire = Schema.Struct({
+  candidates: Schema.optional(
+    Schema.Array(
+      Schema.Struct({
+        content: Schema.optional(
+          Schema.Struct({
+            parts: Schema.optional(
+              Schema.Array(
+                Schema.Struct({
+                  inlineData: Schema.optional(
+                    Schema.Struct({
+                      mimeType: Schema.optional(Schema.String),
+                      data: Schema.String,
+                    }),
+                  ),
+                }),
+              ),
+            ),
+          }),
+        ),
+      }),
+    ),
+  ),
+})
+const decodeWire = Schema.decodeUnknownEffect(Wire)
+
+const findInline = (wire: typeof Wire.Type) =>
+  (wire.candidates ?? [])
+    .flatMap((c) => c.content?.parts ?? [])
+    .find((p) => p.inlineData !== undefined)?.inlineData
+
+const ttsBody = (r: GeminiSynthesizeRequest) => ({
+  contents: [{ parts: [{ text: r.text }] }],
+  generationConfig: {
+    responseModalities: ["AUDIO"],
+    speechConfig: { voiceConfig: { prebuiltVoiceConfig: { voiceName: r.voiceId } } },
+  },
+})
+
+// ---------------------------------------------------------------------------
+// Service implementation
+// ---------------------------------------------------------------------------
+
+const baseUrl = (cfg: Config) => cfg.baseUrl ?? "https://generativelanguage.googleapis.com/v1beta"
+
+const synthesizeImpl = (cfg: Config) => (request: GeminiSynthesizeRequest) =>
+  Effect.gen(function* () {
+    const client = yield* HttpClient.HttpClient
+    const [format, wrap] = yield* realizeOutput(request.outputFormat?.container ?? "raw")
+    const httpRequest = HttpClientRequest.post(
+      `${baseUrl(cfg)}/models/${request.model}:generateContent`,
+    ).pipe(
+      HttpClientRequest.setHeader("x-goog-api-key", Redacted.value(cfg.apiKey)),
+      HttpClientRequest.bodyJsonUnsafe(ttsBody(request)),
+    )
+    const response = yield* client.execute(httpRequest).pipe(Effect.mapError(transportFailure))
+    if (response.status >= 400) {
+      const text = yield* response.text.pipe(Effect.orElseSucceed(() => ""))
+      return yield* httpStatusError(response.status, text)
+    }
+    const json = yield* response.json.pipe(Effect.mapError(transportFailure))
+    const inline = yield* decodeWire(json).pipe(
+      Effect.map(findInline),
+      Effect.mapError((cause) => new AiError.GenerationFailed({ provider: "gemini", raw: cause })),
+      Effect.flatMap((found) =>
+        found === undefined
+          ? Effect.fail(
+              new AiError.GenerationFailed({
+                provider: "gemini",
+                raw: {
+                  message: "Gemini TTS response had no inlineData audio part.",
+                  hint: "Likely a prompt-safety rejection or the model didn't honour responseModalities.",
+                  response: json,
+                },
+              }),
+            )
+          : Effect.succeed(found),
+      ),
+    )
+    const pcm = yield* decodeBase64Audio(inline.data)
+    return { format, bytes: wrap(pcm) } satisfies AudioBlob
+  })
+
+const unsupportedStreamFrom: SpeechSynthesizerService["streamSynthesisFrom"] = () =>
+  Stream.fail(
+    new AiError.Unsupported({
+      provider: "gemini",
+      capability: "streamSynthesisFrom",
+      reason:
+        "Gemini TTS via generateContent is sync-only. Use Cloud Text-to-Speech (Chirp 3 HD) for incremental text-in.",
+    }),
+  )
+
+// ---------------------------------------------------------------------------
+// Constructors
+// ---------------------------------------------------------------------------
+
+export const make = (cfg: Config) =>
+  Effect.map(
+    HttpClient.HttpClient.asEffect(),
+    (client): GeminiSynthesizerService => ({
+      synthesize: (r) =>
+        synthesizeImpl(cfg)(r).pipe(Effect.provideService(HttpClient.HttpClient, client)),
+      streamSynthesis: (r) =>
+        Stream.fromEffect(synthesizeImpl(cfg)(r)).pipe(
+          Stream.map((b): AudioChunk => ({ bytes: b.bytes })),
+          Stream.provideService(HttpClient.HttpClient, client),
+        ),
+      streamSynthesisFrom: unsupportedStreamFrom,
+    }),
+  )
+
+/**
+ * Layer registers both `GeminiSynthesizer` and the generic
+ * `SpeechSynthesizer`. Does NOT register `TtsIncrementalText` — Gemini
+ * TTS is sync-only at the wire level, so `streamSynthesisFrom` against
+ * this Layer alone is a compile-time error.
+ */
+export const layer = (cfg: Config) =>
+  Layer.merge(
+    Layer.effect(GeminiSynthesizer, make(cfg)),
+    Layer.effect(
+      SpeechSynthesizer,
+      Effect.map(
+        make(cfg),
+        (s): SpeechSynthesizerService => ({
+          synthesize: (req: CommonSynthesizeRequest) =>
+            s.synthesize(req as GeminiSynthesizeRequest),
+          streamSynthesis: (req: CommonSynthesizeRequest) =>
+            s.streamSynthesis(req as GeminiSynthesizeRequest),
+          streamSynthesisFrom: s.streamSynthesisFrom,
+        }),
+      ),
+    ),
+  )

package/src/GeminiTranscriber.test.ts

@@ -0,0 +1,136 @@
+import { Effect, Layer, Redacted, Stream } from "effect"
+import { FetchHttpClient } from "effect/unstable/http"
+import { describe, expect, expectTypeOf, it } from "vitest"
+import type * as AiError from "@effect-uai/core/AiError"
+import type { AudioSource } from "@effect-uai/core/Audio"
+import type { TranscriptResult } from "@effect-uai/core/Transcript"
+import * as Transcriber from "@effect-uai/core/Transcriber"
+import * as GeminiTranscriber from "./GeminiTranscriber.js"
+
+const cfg: GeminiTranscriber.Config = { apiKey: Redacted.make("test-key") }
+const live = Layer.provide(GeminiTranscriber.layer(cfg), FetchHttpClient.layer)
+
+const dummyAudio: AudioSource = {
+  _tag: "bytes",
+  bytes: new Uint8Array([0, 1, 2, 3]),
+  mimeType: "audio/wav",
+}
+
+describe("GeminiTranscriber.buildPrompt", () => {
+  it("uses the language-agnostic prompt when no language is set", () => {
+    expect(
+      GeminiTranscriber.buildPrompt({ audio: dummyAudio, model: "gemini-2.5-flash" }),
+    ).toContain("Transcribe the audio verbatim. Return only the transcript text")
+  })
+
+  it("includes the language when set", () => {
+    expect(
+      GeminiTranscriber.buildPrompt({
+        audio: dummyAudio,
+        model: "gemini-2.5-flash",
+        language: "pt-BR",
+      }),
+    ).toContain("in pt-BR")
+  })
+
+  it("appends a free-text prompt as context", () => {
+    const out = GeminiTranscriber.buildPrompt({
+      audio: dummyAudio,
+      model: "gemini-2.5-flash",
+      prompt: "Engineering meeting; expect technical jargon.",
+    })
+    expect(out).toContain("Context: Engineering meeting")
+  })
+
+  it("appends a terms list as biasing vocabulary", () => {
+    const out = GeminiTranscriber.buildPrompt({
+      audio: dummyAudio,
+      model: "gemini-2.5-flash",
+      prompt: { terms: ["Effect", "Lyria", "MusicGenerator"] },
+    })
+    expect(out).toContain("Effect, Lyria, MusicGenerator")
+  })
+
+  it("omits the biasing block when the terms list is empty", () => {
+    const out = GeminiTranscriber.buildPrompt({
+      audio: dummyAudio,
+      model: "gemini-2.5-flash",
+      prompt: { terms: [] },
+    })
+    expect(out).not.toContain("terms / names")
+  })
+})
+
+describe("GeminiTranscriber capability guards (runtime)", () => {
+  it("fails Unsupported for wordTimestamps", async () => {
+    const program = GeminiTranscriber.GeminiTranscriber.use((s) =>
+      s.transcribe({
+        audio: dummyAudio,
+        model: "gemini-2.5-flash",
+        wordTimestamps: true,
+      }),
+    )
+    const exit = await Effect.runPromiseExit(program.pipe(Effect.provide(live)))
+    expect(exit._tag).toBe("Failure")
+    if (exit._tag === "Failure") {
+      expect(JSON.stringify(exit.cause)).toContain("Unsupported")
+      expect(JSON.stringify(exit.cause)).toContain("wordTimestamps")
+    }
+  })
+
+  it("fails Unsupported for diarization", async () => {
+    const program = GeminiTranscriber.GeminiTranscriber.use((s) =>
+      s.transcribe({
+        audio: dummyAudio,
+        model: "gemini-2.5-flash",
+        diarization: true,
+      }),
+    )
+    const exit = await Effect.runPromiseExit(program.pipe(Effect.provide(live)))
+    expect(exit._tag).toBe("Failure")
+    if (exit._tag === "Failure") {
+      expect(JSON.stringify(exit.cause)).toContain("Unsupported")
+      expect(JSON.stringify(exit.cause)).toContain("diarization")
+    }
+  })
+
+  it("streamTranscriptionFrom returns an Unsupported stream", async () => {
+    const program = GeminiTranscriber.GeminiTranscriber.use((s) =>
+      Stream.runDrain(
+        s.streamTranscriptionFrom(Stream.fromIterable([new Uint8Array([0])]), {
+          model: "gemini-2.5-flash",
+          inputFormat: { container: "raw", encoding: "pcm_s16le", sampleRate: 16000 },
+        }),
+      ),
+    )
+    const exit = await Effect.runPromiseExit(program.pipe(Effect.provide(live)))
+    expect(exit._tag).toBe("Failure")
+    if (exit._tag === "Failure") {
+      expect(JSON.stringify(exit.cause)).toContain("Unsupported")
+      expect(JSON.stringify(exit.cause)).toContain("streamTranscriptionFrom")
+    }
+  })
+})
+
+describe("GeminiTranscriber Layer (compile-time)", () => {
+  it("leaves `SttStreaming` unsatisfied when using `streamTranscriptionFrom` against this Layer", () => {
+    const audio: Stream.Stream<Uint8Array> = Stream.fromIterable([new Uint8Array([0])])
+    const events = audio.pipe(
+      Transcriber.streamTranscriptionFrom({
+        model: "gemini-2.5-flash",
+        inputFormat: { container: "raw", encoding: "pcm_s16le", sampleRate: 16000 },
+      }),
+    )
+    const provided = Stream.runDrain(events).pipe(Effect.provide(live))
+    expectTypeOf(provided).toEqualTypeOf<
+      Effect.Effect<void, AiError.AiError, Transcriber.SttStreaming>
+    >()
+  })
+
+  it("sync `transcribe` requires no marker", () => {
+    const t = Transcriber.transcribe({ audio: dummyAudio, model: "gemini-2.5-flash" }).pipe(
+      Effect.provide(live),
+    )
+    expectTypeOf(t).toEqualTypeOf<Effect.Effect<TranscriptResult, AiError.AiError, never>>()
+  })
+})

package/src/GeminiTranscriber.ts

@@ -0,0 +1,204 @@
+import { Context, Effect, Layer, Match, Redacted, Schema, Stream } from "effect"
+import { HttpClient, HttpClientRequest } from "effect/unstable/http"
+import * as AiError from "@effect-uai/core/AiError"
+import type { TranscriptResult } from "@effect-uai/core/Transcript"
+import {
+  type CommonTranscribeRequest,
+  Transcriber,
+  type TranscriberService,
+} from "@effect-uai/core/Transcriber"
+import { audioSourceToInlineData, httpStatusError, transportFailure } from "./geminiSpeechCodec.js"
+import type { GeminiSttModel } from "./models.js"
+
+// ---------------------------------------------------------------------------
+// Public types
+// ---------------------------------------------------------------------------
+
+/**
+ * Gemini transcription rides on `:generateContent` — the model receives
+ * audio plus a textual "transcribe verbatim" prompt and returns plain
+ * text. There is no native structured timestamp / diarization channel,
+ * so `wordTimestamps` / `diarization` short-circuit with `Unsupported`.
+ */
+export type GeminiTranscribeRequest = Omit<CommonTranscribeRequest, "model"> & {
+  readonly model: GeminiSttModel
+}
+
+export type GeminiTranscriberService = {
+  readonly transcribe: (
+    r: GeminiTranscribeRequest,
+  ) => Effect.Effect<TranscriptResult, AiError.AiError>
+  readonly streamTranscriptionFrom: TranscriberService["streamTranscriptionFrom"]
+}
+
+export class GeminiTranscriber extends Context.Service<
+  GeminiTranscriber,
+  GeminiTranscriberService
+>()("@betalyra/effect-uai/providers/google/GeminiTranscriber") {}
+
+export type Config = { readonly apiKey: Redacted.Redacted; readonly baseUrl?: string }
+
+// ---------------------------------------------------------------------------
+// Prompt builder + capability guards
+// ---------------------------------------------------------------------------
+
+const unsupported = (capability: string, reason: string) =>
+  new AiError.Unsupported({ provider: "gemini", capability, reason })
+
+const ensureSupported = Match.type<GeminiTranscribeRequest>().pipe(
+  Match.when({ wordTimestamps: true }, () =>
+    Effect.fail(
+      unsupported(
+        "wordTimestamps",
+        "Gemini's prompt-driven transcription returns plain text — no per-word timing. Use Cloud Speech-to-Text (Chirp 2) for word timestamps.",
+      ),
+    ),
+  ),
+  Match.when({ diarization: true }, () =>
+    Effect.fail(
+      unsupported(
+        "diarization",
+        "Gemini's prompt-driven transcription does not surface speaker IDs. Use Cloud Speech-to-Text (Chirp 2) for diarization.",
+      ),
+    ),
+  ),
+  Match.orElse(() => Effect.void),
+)
+
+const biasingHint = (r: GeminiTranscribeRequest) => {
+  if (r.prompt === undefined) return ""
+  if (typeof r.prompt === "string") return `\nContext: ${r.prompt}`
+  return r.prompt.terms.length === 0
+    ? ""
+    : `\nThe following terms / names appear in the audio: ${r.prompt.terms.join(", ")}.`
+}
+
+export const buildPrompt = (r: GeminiTranscribeRequest) =>
+  (r.language !== undefined
+    ? `Transcribe the audio verbatim in ${r.language}. Return only the transcript text, no preamble.`
+    : "Transcribe the audio verbatim. Return only the transcript text, no preamble.") +
+  biasingHint(r)
+
+// ---------------------------------------------------------------------------
+// Wire
+// ---------------------------------------------------------------------------
+
+const Wire = Schema.Struct({
+  candidates: Schema.optional(
+    Schema.Array(
+      Schema.Struct({
+        content: Schema.optional(
+          Schema.Struct({
+            parts: Schema.optional(
+              Schema.Array(Schema.Struct({ text: Schema.optional(Schema.String) })),
+            ),
+          }),
+        ),
+      }),
+    ),
+  ),
+})
+const decodeWire = Schema.decodeUnknownEffect(Wire)
+
+const collectText = (wire: typeof Wire.Type) =>
+  (wire.candidates ?? [])
+    .flatMap((c) => c.content?.parts ?? [])
+    .flatMap((p) => (p.text !== undefined ? [p.text] : []))
+    .join("\n")
+    .trim()
+
+// ---------------------------------------------------------------------------
+// Service implementation
+// ---------------------------------------------------------------------------
+
+const baseUrl = (cfg: Config) => cfg.baseUrl ?? "https://generativelanguage.googleapis.com/v1beta"
+
+const transcribeImpl = (cfg: Config) => (request: GeminiTranscribeRequest) =>
+  Effect.gen(function* () {
+    yield* ensureSupported(request)
+    const client = yield* HttpClient.HttpClient
+    const inline = yield* audioSourceToInlineData(request.audio)
+    const httpRequest = HttpClientRequest.post(
+      `${baseUrl(cfg)}/models/${request.model}:generateContent`,
+    ).pipe(
+      HttpClientRequest.setHeader("x-goog-api-key", Redacted.value(cfg.apiKey)),
+      HttpClientRequest.bodyJsonUnsafe({
+        contents: [{ parts: [{ text: buildPrompt(request) }, { inlineData: inline }] }],
+      }),
+    )
+    const response = yield* client.execute(httpRequest).pipe(Effect.mapError(transportFailure))
+    if (response.status >= 400) {
+      const text = yield* response.text.pipe(Effect.orElseSucceed(() => ""))
+      return yield* httpStatusError(response.status, text)
+    }
+    const json = yield* response.json.pipe(Effect.mapError(transportFailure))
+    const text = yield* decodeWire(json).pipe(
+      Effect.map(collectText),
+      Effect.mapError((cause) => new AiError.GenerationFailed({ provider: "gemini", raw: cause })),
+      Effect.flatMap((t) =>
+        t.length === 0
+          ? Effect.fail(
+              new AiError.GenerationFailed({
+                provider: "gemini",
+                raw: {
+                  message: "Gemini transcription response had no text part.",
+                  hint: "Likely a prompt-safety rejection or the model returned only metadata.",
+                  response: json,
+                },
+              }),
+            )
+          : Effect.succeed(t),
+      ),
+    )
+    return {
+      text,
+      ...(request.language !== undefined && { languageCode: request.language }),
+      raw: json,
+    } satisfies TranscriptResult
+  })
+
+const unsupportedStreamFrom: TranscriberService["streamTranscriptionFrom"] = () =>
+  Stream.fail(
+    new AiError.Unsupported({
+      provider: "gemini",
+      capability: "streamTranscriptionFrom",
+      reason:
+        "Gemini transcription via generateContent is sync-only. Use Cloud Speech-to-Text (StreamingRecognize) for live transcription.",
+    }),
+  )
+
+// ---------------------------------------------------------------------------
+// Constructors
+// ---------------------------------------------------------------------------
+
+export const make = (cfg: Config) =>
+  Effect.map(
+    HttpClient.HttpClient.asEffect(),
+    (client): GeminiTranscriberService => ({
+      transcribe: (r) =>
+        transcribeImpl(cfg)(r).pipe(Effect.provideService(HttpClient.HttpClient, client)),
+      streamTranscriptionFrom: unsupportedStreamFrom,
+    }),
+  )
+
+/**
+ * Layer registers both `GeminiTranscriber` and the generic
+ * `Transcriber`. Does NOT register `SttStreaming` — Gemini's transcription
+ * is sync-only, so `streamTranscriptionFrom` against this Layer alone
+ * is a compile-time error.
+ */
+export const layer = (cfg: Config) =>
+  Layer.merge(
+    Layer.effect(GeminiTranscriber, make(cfg)),
+    Layer.effect(
+      Transcriber,
+      Effect.map(
+        make(cfg),
+        (s): TranscriberService => ({
+          transcribe: (req: CommonTranscribeRequest) =>
+            s.transcribe(req as GeminiTranscribeRequest),
+          streamTranscriptionFrom: s.streamTranscriptionFrom,
+        }),
+      ),
+    ),
+  )