@effect-uai/google 0.2.0 → 0.4.0

package/src/GeminiTranscriber.test.ts

@@ -0,0 +1,136 @@
+import { Effect, Layer, Redacted, Stream } from "effect"
+import { FetchHttpClient } from "effect/unstable/http"
+import { describe, expect, expectTypeOf, it } from "vitest"
+import type * as AiError from "@effect-uai/core/AiError"
+import type { AudioSource } from "@effect-uai/core/Audio"
+import type { TranscriptResult } from "@effect-uai/core/Transcript"
+import * as Transcriber from "@effect-uai/core/Transcriber"
+import * as GeminiTranscriber from "./GeminiTranscriber.js"
+
+const cfg: GeminiTranscriber.Config = { apiKey: Redacted.make("test-key") }
+const live = Layer.provide(GeminiTranscriber.layer(cfg), FetchHttpClient.layer)
+
+const dummyAudio: AudioSource = {
+  _tag: "bytes",
+  bytes: new Uint8Array([0, 1, 2, 3]),
+  mimeType: "audio/wav",
+}
+
+describe("GeminiTranscriber.buildPrompt", () => {
+  it("uses the language-agnostic prompt when no language is set", () => {
+    expect(
+      GeminiTranscriber.buildPrompt({ audio: dummyAudio, model: "gemini-2.5-flash" }),
+    ).toContain("Transcribe the audio verbatim. Return only the transcript text")
+  })
+
+  it("includes the language when set", () => {
+    expect(
+      GeminiTranscriber.buildPrompt({
+        audio: dummyAudio,
+        model: "gemini-2.5-flash",
+        language: "pt-BR",
+      }),
+    ).toContain("in pt-BR")
+  })
+
+  it("appends a free-text prompt as context", () => {
+    const out = GeminiTranscriber.buildPrompt({
+      audio: dummyAudio,
+      model: "gemini-2.5-flash",
+      prompt: "Engineering meeting; expect technical jargon.",
+    })
+    expect(out).toContain("Context: Engineering meeting")
+  })
+
+  it("appends a terms list as biasing vocabulary", () => {
+    const out = GeminiTranscriber.buildPrompt({
+      audio: dummyAudio,
+      model: "gemini-2.5-flash",
+      prompt: { terms: ["Effect", "Lyria", "MusicGenerator"] },
+    })
+    expect(out).toContain("Effect, Lyria, MusicGenerator")
+  })
+
+  it("omits the biasing block when the terms list is empty", () => {
+    const out = GeminiTranscriber.buildPrompt({
+      audio: dummyAudio,
+      model: "gemini-2.5-flash",
+      prompt: { terms: [] },
+    })
+    expect(out).not.toContain("terms / names")
+  })
+})
+
+describe("GeminiTranscriber capability guards (runtime)", () => {
+  it("fails Unsupported for wordTimestamps", async () => {
+    const program = GeminiTranscriber.GeminiTranscriber.use((s) =>
+      s.transcribe({
+        audio: dummyAudio,
+        model: "gemini-2.5-flash",
+        wordTimestamps: true,
+      }),
+    )
+    const exit = await Effect.runPromiseExit(program.pipe(Effect.provide(live)))
+    expect(exit._tag).toBe("Failure")
+    if (exit._tag === "Failure") {
+      expect(JSON.stringify(exit.cause)).toContain("Unsupported")
+      expect(JSON.stringify(exit.cause)).toContain("wordTimestamps")
+    }
+  })
+
+  it("fails Unsupported for diarization", async () => {
+    const program = GeminiTranscriber.GeminiTranscriber.use((s) =>
+      s.transcribe({
+        audio: dummyAudio,
+        model: "gemini-2.5-flash",
+        diarization: true,
+      }),
+    )
+    const exit = await Effect.runPromiseExit(program.pipe(Effect.provide(live)))
+    expect(exit._tag).toBe("Failure")
+    if (exit._tag === "Failure") {
+      expect(JSON.stringify(exit.cause)).toContain("Unsupported")
+      expect(JSON.stringify(exit.cause)).toContain("diarization")
+    }
+  })
+
+  it("streamTranscriptionFrom returns an Unsupported stream", async () => {
+    const program = GeminiTranscriber.GeminiTranscriber.use((s) =>
+      Stream.runDrain(
+        s.streamTranscriptionFrom(Stream.fromIterable([new Uint8Array([0])]), {
+          model: "gemini-2.5-flash",
+          inputFormat: { container: "raw", encoding: "pcm_s16le", sampleRate: 16000 },
+        }),
+      ),
+    )
+    const exit = await Effect.runPromiseExit(program.pipe(Effect.provide(live)))
+    expect(exit._tag).toBe("Failure")
+    if (exit._tag === "Failure") {
+      expect(JSON.stringify(exit.cause)).toContain("Unsupported")
+      expect(JSON.stringify(exit.cause)).toContain("streamTranscriptionFrom")
+    }
+  })
+})
+
+describe("GeminiTranscriber Layer (compile-time)", () => {
+  it("leaves `SttStreaming` unsatisfied when using `streamTranscriptionFrom` against this Layer", () => {
+    const audio: Stream.Stream<Uint8Array> = Stream.fromIterable([new Uint8Array([0])])
+    const events = audio.pipe(
+      Transcriber.streamTranscriptionFrom({
+        model: "gemini-2.5-flash",
+        inputFormat: { container: "raw", encoding: "pcm_s16le", sampleRate: 16000 },
+      }),
+    )
+    const provided = Stream.runDrain(events).pipe(Effect.provide(live))
+    expectTypeOf(provided).toEqualTypeOf<
+      Effect.Effect<void, AiError.AiError, Transcriber.SttStreaming>
+    >()
+  })
+
+  it("sync `transcribe` requires no marker", () => {
+    const t = Transcriber.transcribe({ audio: dummyAudio, model: "gemini-2.5-flash" }).pipe(
+      Effect.provide(live),
+    )
+    expectTypeOf(t).toEqualTypeOf<Effect.Effect<TranscriptResult, AiError.AiError, never>>()
+  })
+})

package/src/GeminiTranscriber.ts

@@ -0,0 +1,204 @@
+import { Context, Effect, Layer, Match, Redacted, Schema, Stream } from "effect"
+import { HttpClient, HttpClientRequest } from "effect/unstable/http"
+import * as AiError from "@effect-uai/core/AiError"
+import type { TranscriptResult } from "@effect-uai/core/Transcript"
+import {
+  type CommonTranscribeRequest,
+  Transcriber,
+  type TranscriberService,
+} from "@effect-uai/core/Transcriber"
+import { audioSourceToInlineData, httpStatusError, transportFailure } from "./geminiSpeechCodec.js"
+import type { GeminiSttModel } from "./models.js"
+
+// ---------------------------------------------------------------------------
+// Public types
+// ---------------------------------------------------------------------------
+
+/**
+ * Gemini transcription rides on `:generateContent` — the model receives
+ * audio plus a textual "transcribe verbatim" prompt and returns plain
+ * text. There is no native structured timestamp / diarization channel,
+ * so `wordTimestamps` / `diarization` short-circuit with `Unsupported`.
+ */
+export type GeminiTranscribeRequest = Omit<CommonTranscribeRequest, "model"> & {
+  readonly model: GeminiSttModel
+}
+
+export type GeminiTranscriberService = {
+  readonly transcribe: (
+    r: GeminiTranscribeRequest,
+  ) => Effect.Effect<TranscriptResult, AiError.AiError>
+  readonly streamTranscriptionFrom: TranscriberService["streamTranscriptionFrom"]
+}
+
+export class GeminiTranscriber extends Context.Service<
+  GeminiTranscriber,
+  GeminiTranscriberService
+>()("@betalyra/effect-uai/providers/google/GeminiTranscriber") {}
+
+export type Config = { readonly apiKey: Redacted.Redacted; readonly baseUrl?: string }
+
+// ---------------------------------------------------------------------------
+// Prompt builder + capability guards
+// ---------------------------------------------------------------------------
+
+const unsupported = (capability: string, reason: string) =>
+  new AiError.Unsupported({ provider: "gemini", capability, reason })
+
+const ensureSupported = Match.type<GeminiTranscribeRequest>().pipe(
+  Match.when({ wordTimestamps: true }, () =>
+    Effect.fail(
+      unsupported(
+        "wordTimestamps",
+        "Gemini's prompt-driven transcription returns plain text — no per-word timing. Use Cloud Speech-to-Text (Chirp 2) for word timestamps.",
+      ),
+    ),
+  ),
+  Match.when({ diarization: true }, () =>
+    Effect.fail(
+      unsupported(
+        "diarization",
+        "Gemini's prompt-driven transcription does not surface speaker IDs. Use Cloud Speech-to-Text (Chirp 2) for diarization.",
+      ),
+    ),
+  ),
+  Match.orElse(() => Effect.void),
+)
+
+const biasingHint = (r: GeminiTranscribeRequest) => {
+  if (r.prompt === undefined) return ""
+  if (typeof r.prompt === "string") return `\nContext: ${r.prompt}`
+  return r.prompt.terms.length === 0
+    ? ""
+    : `\nThe following terms / names appear in the audio: ${r.prompt.terms.join(", ")}.`
+}
+
+export const buildPrompt = (r: GeminiTranscribeRequest) =>
+  (r.language !== undefined
+    ? `Transcribe the audio verbatim in ${r.language}. Return only the transcript text, no preamble.`
+    : "Transcribe the audio verbatim. Return only the transcript text, no preamble.") +
+  biasingHint(r)
+
+// ---------------------------------------------------------------------------
+// Wire
+// ---------------------------------------------------------------------------
+
+const Wire = Schema.Struct({
+  candidates: Schema.optional(
+    Schema.Array(
+      Schema.Struct({
+        content: Schema.optional(
+          Schema.Struct({
+            parts: Schema.optional(
+              Schema.Array(Schema.Struct({ text: Schema.optional(Schema.String) })),
+            ),
+          }),
+        ),
+      }),
+    ),
+  ),
+})
+const decodeWire = Schema.decodeUnknownEffect(Wire)
+
+const collectText = (wire: typeof Wire.Type) =>
+  (wire.candidates ?? [])
+    .flatMap((c) => c.content?.parts ?? [])
+    .flatMap((p) => (p.text !== undefined ? [p.text] : []))
+    .join("\n")
+    .trim()
+
+// ---------------------------------------------------------------------------
+// Service implementation
+// ---------------------------------------------------------------------------
+
+const baseUrl = (cfg: Config) => cfg.baseUrl ?? "https://generativelanguage.googleapis.com/v1beta"
+
+const transcribeImpl = (cfg: Config) => (request: GeminiTranscribeRequest) =>
+  Effect.gen(function* () {
+    yield* ensureSupported(request)
+    const client = yield* HttpClient.HttpClient
+    const inline = yield* audioSourceToInlineData(request.audio)
+    const httpRequest = HttpClientRequest.post(
+      `${baseUrl(cfg)}/models/${request.model}:generateContent`,
+    ).pipe(
+      HttpClientRequest.setHeader("x-goog-api-key", Redacted.value(cfg.apiKey)),
+      HttpClientRequest.bodyJsonUnsafe({
+        contents: [{ parts: [{ text: buildPrompt(request) }, { inlineData: inline }] }],
+      }),
+    )
+    const response = yield* client.execute(httpRequest).pipe(Effect.mapError(transportFailure))
+    if (response.status >= 400) {
+      const text = yield* response.text.pipe(Effect.orElseSucceed(() => ""))
+      return yield* Effect.fail(httpStatusError(response.status, text))
+    }
+    const json = yield* response.json.pipe(Effect.mapError(transportFailure))
+    const text = yield* decodeWire(json).pipe(
+      Effect.map(collectText),
+      Effect.mapError((cause) => new AiError.GenerationFailed({ provider: "gemini", raw: cause })),
+      Effect.flatMap((t) =>
+        t.length === 0
+          ? Effect.fail(
+              new AiError.GenerationFailed({
+                provider: "gemini",
+                raw: {
+                  message: "Gemini transcription response had no text part.",
+                  hint: "Likely a prompt-safety rejection or the model returned only metadata.",
+                  response: json,
+                },
+              }),
+            )
+          : Effect.succeed(t),
+      ),
+    )
+    return {
+      text,
+      ...(request.language !== undefined && { languageCode: request.language }),
+      raw: json,
+    } satisfies TranscriptResult
+  })
+
+const unsupportedStreamFrom: TranscriberService["streamTranscriptionFrom"] = () =>
+  Stream.fail(
+    new AiError.Unsupported({
+      provider: "gemini",
+      capability: "streamTranscriptionFrom",
+      reason:
+        "Gemini transcription via generateContent is sync-only. Use Cloud Speech-to-Text (StreamingRecognize) for live transcription.",
+    }),
+  )
+
+// ---------------------------------------------------------------------------
+// Constructors
+// ---------------------------------------------------------------------------
+
+export const make = (cfg: Config) =>
+  Effect.map(
+    HttpClient.HttpClient.asEffect(),
+    (client): GeminiTranscriberService => ({
+      transcribe: (r) =>
+        transcribeImpl(cfg)(r).pipe(Effect.provideService(HttpClient.HttpClient, client)),
+      streamTranscriptionFrom: unsupportedStreamFrom,
+    }),
+  )
+
+/**
+ * Layer registers both `GeminiTranscriber` and the generic
+ * `Transcriber`. Does NOT register `SttStreaming` — Gemini's transcription
+ * is sync-only, so `streamTranscriptionFrom` against this Layer alone
+ * is a compile-time error.
+ */
+export const layer = (cfg: Config) =>
+  Layer.merge(
+    Layer.effect(GeminiTranscriber, make(cfg)),
+    Layer.effect(
+      Transcriber,
+      Effect.map(
+        make(cfg),
+        (s): TranscriberService => ({
+          transcribe: (req: CommonTranscribeRequest) =>
+            s.transcribe(req as GeminiTranscribeRequest),
+          streamTranscriptionFrom: s.streamTranscriptionFrom,
+        }),
+      ),
+    ),
+  )

package/src/LyriaGenerator.test.ts

@@ -0,0 +1,162 @@
+import { Effect, Layer, Redacted, Stream } from "effect"
+import { FetchHttpClient } from "effect/unstable/http"
+import { describe, expect, expectTypeOf, it } from "vitest"
+import type * as AiError from "@effect-uai/core/AiError"
+import * as MusicGenerator from "@effect-uai/core/MusicGenerator"
+import { promptsInput } from "@effect-uai/core/Music"
+import * as LyriaGenerator from "./LyriaGenerator.js"
+
+const cfg: LyriaGenerator.Config = { apiKey: Redacted.make("test-key") }
+// FetchHttpClient is required for `make`, but these tests only exercise the
+// compile-time gating and runtime Unsupported branches — no real HTTP is made.
+const live = Layer.provide(LyriaGenerator.layer(cfg), FetchHttpClient.layer)
+
+describe("buildPrompt", () => {
+  it("returns a plain string when prompts is a string", () => {
+    const out = LyriaGenerator.buildPrompt({
+      model: "lyria-3-clip-preview",
+      prompts: "upbeat indie pop",
+    })
+    expect(out).toBe("upbeat indie pop")
+  })
+
+  it("joins weighted prompts with weight annotations", () => {
+    const out = LyriaGenerator.buildPrompt({
+      model: "lyria-3-clip-preview",
+      prompts: [
+        { text: "minimal techno", weight: 1.0 },
+        { text: "1980s synthwave", weight: 0.3 },
+      ],
+    })
+    expect(out).toContain("minimal techno")
+    expect(out).toContain("1980s synthwave (weight 0.3)")
+  })
+
+  it("omits the weight annotation for weight === 1", () => {
+    const out = LyriaGenerator.buildPrompt({
+      model: "lyria-3-clip-preview",
+      prompts: [{ text: "ambient", weight: 1 }],
+    })
+    expect(out).toBe("ambient")
+  })
+
+  it("splices in bpm, scale, duration, and instrumental hints", () => {
+    const out = LyriaGenerator.buildPrompt({
+      model: "lyria-3-clip-preview",
+      prompts: "house",
+      bpm: 124,
+      scale: "C_MAJOR",
+      durationSeconds: 30,
+      instrumental: true,
+    })
+    expect(out).toContain("Instrumental only")
+    expect(out).toContain("BPM: 124")
+    expect(out).toContain("Key/scale: C_MAJOR")
+    expect(out).toContain("Target duration: 30s")
+  })
+
+  it("appends a lyrics block when lyrics is set and instrumental is not true", () => {
+    const out = LyriaGenerator.buildPrompt({
+      model: "lyria-3-clip-preview",
+      prompts: "indie rock",
+      lyrics: "[Verse]\nhello world",
+    })
+    expect(out).toContain("Lyrics:")
+    expect(out).toContain("[Verse]")
+  })
+
+  it("drops the lyrics block when instrumental is true", () => {
+    const out = LyriaGenerator.buildPrompt({
+      model: "lyria-3-clip-preview",
+      prompts: "indie rock",
+      lyrics: "[Verse]\nhello world",
+      instrumental: true,
+    })
+    expect(out).not.toContain("Lyrics:")
+  })
+})
+
+describe("containerToMimeType", () => {
+  it("maps mp3 and wav", async () => {
+    expect(await Effect.runPromise(LyriaGenerator.containerToMimeType("mp3"))).toBe("audio/mp3")
+    expect(await Effect.runPromise(LyriaGenerator.containerToMimeType("wav"))).toBe("audio/wav")
+  })
+
+  it("fails Unsupported for ogg / opus / flac / aac / webm / raw", async () => {
+    const exit = await Effect.runPromiseExit(LyriaGenerator.containerToMimeType("ogg"))
+    expect(exit._tag).toBe("Failure")
+    if (exit._tag === "Failure") {
+      expect(JSON.stringify(exit.cause)).toContain("Unsupported")
+      expect(JSON.stringify(exit.cause)).toContain("outputFormat")
+    }
+  })
+})
+
+describe("realizedFormat", () => {
+  it("reports 44.1 kHz stereo for both mp3 and wav", () => {
+    expect(LyriaGenerator.realizedFormat("audio/mp3")).toEqual({
+      container: "mp3",
+      encoding: "mp3",
+      sampleRate: 44100,
+      channels: 2,
+    })
+    expect(LyriaGenerator.realizedFormat("audio/wav")).toEqual({
+      container: "wav",
+      encoding: "pcm_s16le",
+      sampleRate: 44100,
+      channels: 2,
+    })
+  })
+})
+
+describe("LyriaGenerator capability guards (runtime)", () => {
+  it("streamGenerationFrom returns an Unsupported stream", async () => {
+    const program = LyriaGenerator.LyriaGenerator.use((s) =>
+      Stream.runDrain(
+        s.streamGenerationFrom(Stream.fromIterable([promptsInput([{ text: "x" }])]), {
+          model: "lyria-3-clip-preview",
+          prompts: "",
+        }),
+      ),
+    )
+    const exit = await Effect.runPromiseExit(program.pipe(Effect.provide(live)))
+    expect(exit._tag).toBe("Failure")
+    if (exit._tag === "Failure") {
+      expect(JSON.stringify(exit.cause)).toContain("Unsupported")
+      expect(JSON.stringify(exit.cause)).toContain("streamGenerationFrom")
+    }
+  })
+})
+
+describe("LyriaGenerator Layer (compile-time)", () => {
+  it("leaves `MusicInteractiveSession` unsatisfied when using `streamGenerationFrom` against this Layer", () => {
+    const inputs = Stream.fromIterable([promptsInput([{ text: "techno" }])])
+    const audio = inputs.pipe(
+      MusicGenerator.streamGenerationFrom({
+        model: "lyria-3-clip-preview",
+        prompts: "",
+      }),
+    )
+    const provided = Stream.runDrain(audio).pipe(Effect.provide(live))
+    expectTypeOf(provided).toEqualTypeOf<
+      Effect.Effect<void, AiError.AiError, MusicGenerator.MusicInteractiveSession>
+    >()
+  })
+
+  it("sync `generate` and chunked `streamGeneration` require no marker", () => {
+    const gen = MusicGenerator.generate({
+      model: "lyria-3-clip-preview",
+      prompts: "ambient",
+    }).pipe(Effect.provide(live))
+    expectTypeOf(gen).toEqualTypeOf<
+      Effect.Effect<import("@effect-uai/core/Music").MusicResult, AiError.AiError, never>
+    >()
+    const _stream = Stream.runCollect(
+      MusicGenerator.streamGeneration({
+        model: "lyria-3-clip-preview",
+        prompts: "ambient",
+      }),
+    ).pipe(Effect.provide(live))
+    expect(_stream).toBeDefined()
+  })
+})