@kuralle-syrinx/gemini 4.1.0 → 4.3.0

package/package.json

@@ -1,18 +1,36 @@
 {
   "name": "@kuralle-syrinx/gemini",
-  "version": "4.1.0",
+  "version": "4.3.0",
   "private": false,
-  "type": "module",
+  "description": "Google Gemini TTS adapter for Syrinx",
+  "keywords": [
+    "voice",
+    "voice-agent",
+    "speech",
+    "syrinx",
+    "gemini",
+    "text-to-speech"
+  ],
   "license": "MIT",
+  "homepage": "https://github.com/kuralle/syrinx#readme",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/kuralle/syrinx.git",
+    "directory": "packages/gemini"
+  },
+  "bugs": {
+    "url": "https://github.com/kuralle/syrinx/issues"
+  },
+  "type": "module",
   "main": "./src/index.ts",
   "types": "./src/index.ts",
   "dependencies": {
     "@google/genai": "^1.0.0",
-    "@kuralle-syrinx/core": "4.1.0"
+    "@kuralle-syrinx/core": "4.3.0"
   },
   "devDependencies": {
     "typescript": "^5.7.0",
-    "vitest": "^2.1.0"
+    "vitest": "^3.2.6"
   },
   "scripts": {
     "typecheck": "tsc --noEmit",

package/src/index.ts

@@ -48,7 +48,7 @@ export type GeminiTTSModel =
 
 const DEFAULT_MODEL: GeminiTTSModel = "gemini-3.1-flash-tts-preview";
 const DEFAULT_VOICE: GeminiVoice = "Kore";
-const SAMPLE_RATE = 24000;
+const DEFAULT_SAMPLE_RATE = 24000;
 
 // =============================================================================
 // Plugin
@@ -60,15 +60,24 @@ export class GeminiTTSPlugin implements VoicePlugin {
   private model: string = DEFAULT_MODEL;
   private voiceName: string = DEFAULT_VOICE;
   private instruction = "";
+  private languageCode: string | undefined;
   private timeoutMs = 45_000;
+  private sampleRate = DEFAULT_SAMPLE_RATE;
+  private generationConfig: Record<string, unknown> | undefined;
   // One controller per in-flight turn. A single shared field let a second turn's
   // synthesize() overwrite the first's controller, so a barge-in on turn N aborted
   // only turn N+1 and turn N's stale audio could still stream after the interrupt.
   private readonly abortControllers = new Map<string, AbortController>();
   private textByContextId = new Map<string, string>();
+  /** Synthesized character counts per contextId — billed on successful tts.end. */
+  private charactersByContextId = new Map<string, number>();
   private retryConfig: RetryConfig = readRetryConfig({});
   private disposers: Array<() => void> = [];
-  private readonly audioFormat: AudioFormat = { encoding: "pcm_s16le", sampleRateHz: SAMPLE_RATE, channels: 1 };
+  private audioFormat: AudioFormat = {
+    encoding: "pcm_s16le",
+    sampleRateHz: DEFAULT_SAMPLE_RATE,
+    channels: 1,
+  };
 
   async initialize(bus: PipelineBus, config: PluginConfig): Promise<void> {
     this.bus = bus;
@@ -76,8 +85,13 @@ export class GeminiTTSPlugin implements VoicePlugin {
     this.model = optionalStringConfig(config, "model") ?? DEFAULT_MODEL;
     this.voiceName = optionalStringConfig(config, "voice_name") ?? DEFAULT_VOICE;
     this.instruction = optionalStringConfig(config, "instruction") ?? "";
+    this.languageCode =
+      optionalStringConfig(config, "language_code") ?? optionalStringConfig(config, "language");
     this.timeoutMs = readPositiveInteger(config["timeout_ms"], 45_000);
+    this.sampleRate = readPositiveInteger(config["sample_rate"], DEFAULT_SAMPLE_RATE);
+    this.generationConfig = readPlainObject(config["generation_config"]);
     this.retryConfig = readRetryConfig(config);
+    this.audioFormat = { encoding: "pcm_s16le", sampleRateHz: this.sampleRate, channels: 1 };
     assertAudioFormat(this.audioFormat);
 
     // Accumulate streaming text deltas; Gemini TTS returns chunked audio for
@@ -95,15 +109,18 @@ export class GeminiTTSPlugin implements VoicePlugin {
         this.textByContextId.delete(donePkt.contextId);
         const text = donePkt.text || buffered;
         if (!text.trim()) {
+          this.charactersByContextId.delete(donePkt.contextId);
           this.emitEnd(donePkt.contextId);
           return;
         }
+        this.charactersByContextId.set(donePkt.contextId, text.length);
         await this.synthesize(text, donePkt.contextId);
       }),
 
       // Listen for TTS interrupts — abort only the interrupted turn's synthesis.
       bus.on("interrupt.tts", (pkt) => {
         const ctxId = (pkt as { contextId: string }).contextId;
+        this.charactersByContextId.delete(ctxId);
         const controller = this.abortControllers.get(ctxId);
         if (controller) {
           controller.abort();
@@ -134,23 +151,25 @@ export class GeminiTTSPlugin implements VoicePlugin {
     for (let attempt = 1; attempt <= this.retryConfig.maxAttempts; attempt += 1) {
       try {
         const audioChunks = await this.synthesizeOnce(text, contextId, signal);
-        if (!signal.aborted && audioChunks > 0) {
-          this.bus?.push(Route.Main, {
-            kind: "tts.end",
-            contextId,
-            timestampMs: Date.now(),
-          });
+        if (signal.aborted) {
+          this.charactersByContextId.delete(contextId);
+          return;
         }
-        if (!signal.aborted && audioChunks === 0) {
-          throw new Error("Gemini TTS returned no audio chunks");
+        if (audioChunks > 0) {
+          this.emitEnd(contextId);
+          return;
         }
-        return;
+        throw new Error("Gemini TTS returned no audio chunks");
       } catch (err) {
-        if (signal.aborted) return;
+        if (signal.aborted) {
+          this.charactersByContextId.delete(contextId);
+          return;
+        }
 
         const category = categorizeTtsError(err);
         const recoverable = isRecoverable(category);
         if (!recoverable || attempt >= this.retryConfig.maxAttempts) {
+          this.charactersByContextId.delete(contextId);
           this.bus?.push(Route.Critical, {
             kind: "tts.error",
             contextId,
@@ -176,6 +195,19 @@ export class GeminiTTSPlugin implements VoicePlugin {
   }
 
   private emitEnd(contextId: string): void {
+    const characters = this.charactersByContextId.get(contextId) ?? 0;
+    this.charactersByContextId.delete(contextId);
+    if (characters > 0) {
+      this.bus?.push(Route.Background, {
+        kind: "usage.recorded",
+        contextId,
+        timestampMs: Date.now(),
+        stage: "tts",
+        provider: "gemini",
+        model: this.model,
+        characters,
+      });
+    }
     this.bus?.push(Route.Main, {
       kind: "tts.end",
       contextId,
@@ -189,19 +221,23 @@ export class GeminiTTSPlugin implements VoicePlugin {
 
     const client = new GoogleGenAI({ apiKey: this.apiKey });
 
+    const speechConfig: Record<string, unknown> = {
+      voiceConfig: {
+        prebuiltVoiceConfig: {
+          voiceName: this.voiceName,
+        },
+      },
+    };
+    if (this.languageCode) speechConfig["languageCode"] = this.languageCode;
+
     const response = await withTimeout(
       client.models.generateContent({
         model: this.model,
         contents: [{ parts: [{ text: this.instruction ? `${this.instruction}: ${text}` : text }] }],
         config: {
           responseModalities: ["AUDIO"],
-          speechConfig: {
-            voiceConfig: {
-              prebuiltVoiceConfig: {
-                voiceName: this.voiceName,
-              },
-            },
-          },
+          speechConfig,
+          ...this.generationConfig,
           abortSignal: signal,
         },
       }),
@@ -226,7 +262,7 @@ export class GeminiTTSPlugin implements VoicePlugin {
           contextId,
           timestampMs: Date.now(),
           audio: audioUint8,
-          sampleRateHz: SAMPLE_RATE,
+          sampleRateHz: this.sampleRate,
         });
         audioChunks++;
       }
@@ -239,12 +275,14 @@ export class GeminiTTSPlugin implements VoicePlugin {
   flush(): void {
     for (const controller of this.abortControllers.values()) controller.abort();
     this.abortControllers.clear();
+    this.charactersByContextId.clear();
   }
 
   async close(): Promise<void> {
     this.flush();
     for (const dispose of this.disposers.splice(0)) dispose();
     this.textByContextId.clear();
+    this.charactersByContextId.clear();
     this.bus = null;
   }
 }
@@ -285,3 +323,10 @@ function readPositiveInteger(value: unknown, fallback: number): number {
   const integer = Math.floor(value);
   return integer > 0 ? integer : fallback;
 }
+
+function readPlainObject(value: unknown): Record<string, unknown> | undefined {
+  if (value !== null && typeof value === "object" && !Array.isArray(value)) {
+    return value as Record<string, unknown>;
+  }
+  return undefined;
+}

package/src/index.test.ts

@@ -1,115 +0,0 @@
-// SPDX-License-Identifier: MIT
-
-import { afterEach, describe, expect, it, vi } from "vitest";
-import {
-  PipelineBusImpl,
-  Route,
-  type TextToSpeechAudioPacket,
-  type TtsErrorPacket,
-} from "@kuralle-syrinx/core";
-import { GeminiTTSPlugin } from "./index.js";
-
-const generateContent = vi.fn();
-
-vi.mock("@google/genai", () => ({
-  GoogleGenAI: vi.fn().mockImplementation(() => ({
-    models: { generateContent },
-  })),
-}));
-
-afterEach(() => {
-  generateContent.mockReset();
-});
-
-describe("GeminiTTSPlugin", () => {
-  it("emits tts.error for odd-length PCM16 without throwing into the bus pump", async () => {
-    generateContent.mockResolvedValue({
-      candidates: [{
-        content: {
-          parts: [{
-            inlineData: {
-              mimeType: "audio/pcm",
-              data: Buffer.from(new Uint8Array([1, 2, 3])).toString("base64"),
-            },
-          }],
-        },
-      }],
-    });
-
-    const bus = new PipelineBusImpl();
-    const started = bus.start();
-    const plugin = new GeminiTTSPlugin();
-    const audio: TextToSpeechAudioPacket[] = [];
-    const errors: TtsErrorPacket[] = [];
-    bus.on("tts.audio", (pkt) => {
-      audio.push(pkt as TextToSpeechAudioPacket);
-    });
-    bus.on("tts.error", (pkt) => {
-      errors.push(pkt as TtsErrorPacket);
-    });
-
-    await plugin.initialize(bus, { api_key: "test-gemini-key" });
-    bus.push(Route.Main, {
-      kind: "tts.done",
-      contextId: "turn-odd-pcm",
-      timestampMs: Date.now(),
-      text: "Hello.",
-    });
-    await new Promise((resolve) => setTimeout(resolve, 50));
-
-    expect(audio).toEqual([]);
-    expect(errors).toEqual([
-      expect.objectContaining({
-        kind: "tts.error",
-        contextId: "turn-odd-pcm",
-        component: "tts",
-        cause: expect.objectContaining({
-          message: expect.stringMatching(/even number of bytes/i),
-        }),
-      }),
-    ]);
-
-    await plugin.close();
-    bus.stop();
-    await started;
-  });
-
-  it("sends the raw text to Gemini by default (no hardcoded persona lead-in)", async () => {
-    generateContent.mockResolvedValue({
-      candidates: [{ content: { parts: [{ inlineData: { mimeType: "audio/pcm", data: Buffer.from(new Uint8Array([1, 2, 3, 4])).toString("base64") } }] } }],
-    });
-    const bus = new PipelineBusImpl();
-    const started = bus.start();
-    const plugin = new GeminiTTSPlugin();
-    await plugin.initialize(bus, { api_key: "test-gemini-key" });
-    bus.push(Route.Main, { kind: "tts.done", contextId: "t1", timestampMs: Date.now(), text: "Add Biology 101." });
-    await new Promise((resolve) => setTimeout(resolve, 50));
-
-    expect(generateContent).toHaveBeenCalledTimes(1);
-    const arg = generateContent.mock.calls[0]![0] as { contents: Array<{ parts: Array<{ text: string }> }> };
-    expect(arg.contents[0]!.parts[0]!.text).toBe("Add Biology 101.");
-
-    await plugin.close();
-    bus.stop();
-    await started;
-  });
-
-  it("prepends a configured instruction lead-in when provided", async () => {
-    generateContent.mockResolvedValue({
-      candidates: [{ content: { parts: [{ inlineData: { mimeType: "audio/pcm", data: Buffer.from(new Uint8Array([1, 2, 3, 4])).toString("base64") } }] } }],
-    });
-    const bus = new PipelineBusImpl();
-    const started = bus.start();
-    const plugin = new GeminiTTSPlugin();
-    await plugin.initialize(bus, { api_key: "test-gemini-key", instruction: "Read aloud warmly" });
-    bus.push(Route.Main, { kind: "tts.done", contextId: "t2", timestampMs: Date.now(), text: "Hi." });
-    await new Promise((resolve) => setTimeout(resolve, 50));
-
-    const arg = generateContent.mock.calls[0]![0] as { contents: Array<{ parts: Array<{ text: string }> }> };
-    expect(arg.contents[0]!.parts[0]!.text).toBe("Read aloud warmly: Hi.");
-
-    await plugin.close();
-    bus.stop();
-    await started;
-  });
-});

package/tsconfig.json

@@ -1,21 +0,0 @@
-{
-  "compilerOptions": {
-    "target": "ES2022",
-    "module": "NodeNext",
-    "moduleResolution": "NodeNext",
-    "lib": ["ES2022"],
-    "strict": true,
-    "noUncheckedIndexedAccess": true,
-    "noImplicitReturns": true,
-    "declaration": true,
-    "declarationMap": true,
-    "sourceMap": true,
-    "outDir": "./dist",
-    "rootDir": "./src",
-    "esModuleInterop": true,
-    "skipLibCheck": true,
-    "forceConsistentCasingInFileNames": true
-  },
-  "include": ["src/**/*.ts"],
-  "exclude": ["node_modules", "dist"]
-}