@kuralle-syrinx/grok 2.1.0

package/src/stt.test.ts

@@ -0,0 +1,244 @@
+// SPDX-License-Identifier: MIT
+
+import { afterEach, describe, expect, it } from "vitest";
+import { WebSocketServer, type WebSocket } from "ws";
+import {
+  PipelineBusImpl,
+  Route,
+  type SttErrorPacket,
+  type SttInterimPacket,
+  type SttResultPacket,
+} from "@kuralle-syrinx/core";
+
+import { GrokSTTPlugin } from "./stt.js";
+
+let servers: WebSocketServer[] = [];
+
+afterEach(async () => {
+  await Promise.all(
+    servers.splice(0).map(
+      (server) =>
+        new Promise<void>((resolve) => {
+          for (const client of server.clients) client.terminate();
+          server.close(() => resolve());
+        }),
+    ),
+  );
+});
+
+async function createLocalServer(onConnection: (socket: WebSocket) => void): Promise<string> {
+  const server = await new Promise<WebSocketServer>((resolve) => {
+    let nextServer: WebSocketServer;
+    nextServer = new WebSocketServer({ port: 0 }, () => {
+      resolve(nextServer);
+    });
+  });
+  servers.push(server);
+  server.on("connection", onConnection);
+  const address = server.address();
+  if (!address || typeof address === "string") throw new Error("Expected TCP server address");
+  return `ws://127.0.0.1:${address.port}/stt`;
+}
+
+async function waitFor<T>(items: T[], count = 1): Promise<void> {
+  for (let i = 0; i < 50; i += 1) {
+    if (items.length >= count) return;
+    await new Promise((resolve) => setTimeout(resolve, 10));
+  }
+}
+
+describe("GrokSTTPlugin", () => {
+  it("waits for transcript.created before sending binary audio", async () => {
+    const binaryFrames: Uint8Array[] = [];
+    const controlMessages: string[] = [];
+    const endpointUrl = await createLocalServer((socket) => {
+      socket.on("message", (data, isBinary) => {
+        if (isBinary) {
+          binaryFrames.push(new Uint8Array(data as Buffer));
+          return;
+        }
+        controlMessages.push(data.toString());
+      });
+      socket.send(JSON.stringify({ type: "transcript.created" }));
+    });
+
+    const bus = new PipelineBusImpl();
+    const started = bus.start();
+    const plugin = new GrokSTTPlugin();
+
+    await plugin.initialize(bus, {
+      api_key: "test",
+      endpoint_url: endpointUrl,
+      sample_rate: 16000,
+    });
+    bus.push(Route.Main, {
+      kind: "stt.audio",
+      contextId: "turn-1",
+      timestampMs: Date.now(),
+      audio: new Uint8Array(640),
+    });
+    await waitFor(binaryFrames);
+
+    expect(binaryFrames).toEqual([new Uint8Array(640)]);
+    expect(controlMessages).toEqual([]);
+
+    await plugin.close();
+    await waitFor(controlMessages);
+    expect(controlMessages).toContain(JSON.stringify({ type: "audio.done" }));
+    bus.stop();
+    await started;
+  });
+
+  it("maps transcript.partial to stt.interim and stt.result", async () => {
+    const endpointUrl = await createLocalServer((socket) => {
+      socket.send(JSON.stringify({ type: "transcript.created" }));
+      socket.on("message", (data, isBinary) => {
+        if (!isBinary) return;
+        socket.send(JSON.stringify({
+          type: "transcript.partial",
+          text: "hello",
+          is_final: false,
+        }));
+        socket.send(JSON.stringify({
+          type: "transcript.partial",
+          text: "hello world",
+          is_final: true,
+          speech_final: false,
+          words: [{ word: "hello" }, { word: "world" }],
+          start: 0,
+          duration: 1.2,
+          end_of_turn_confidence: 0.91,
+        }));
+        socket.send(JSON.stringify({
+          type: "transcript.partial",
+          text: "hello world",
+          is_final: true,
+          speech_final: true,
+          end_of_turn_confidence: 0.95,
+        }));
+      });
+    });
+
+    const bus = new PipelineBusImpl();
+    const started = bus.start();
+    const plugin = new GrokSTTPlugin();
+    const interims: Array<{ text: string }> = [];
+    const finals: SttResultPacket[] = [];
+    const turnCompletes: Array<{ kind: string }> = [];
+    bus.on("stt.interim", (pkt) => {
+      interims.push({ text: (pkt as SttInterimPacket).text });
+    });
+    bus.on("stt.result", (pkt) => {
+      finals.push(pkt as SttResultPacket);
+    });
+    bus.on("eos.turn_complete", (pkt) => {
+      turnCompletes.push(pkt as { kind: string });
+    });
+
+    await plugin.initialize(bus, {
+      api_key: "test",
+      endpoint_url: endpointUrl,
+      sample_rate: 16000,
+    });
+    bus.push(Route.Main, {
+      kind: "user.audio_received",
+      contextId: "turn-2",
+      timestampMs: Date.now(),
+      audio: new Uint8Array(320),
+    });
+    await waitFor(finals, 2);
+    await waitFor(turnCompletes);
+
+    expect(interims).toEqual([{ text: "hello" }]);
+    expect(finals[0]).toEqual(
+      expect.objectContaining({
+        kind: "stt.result",
+        contextId: "turn-2",
+        text: "hello world",
+        confidence: 0.91,
+        provider: expect.objectContaining({
+          name: "grok",
+          speechFinal: false,
+          words: [{ word: "hello" }, { word: "world" }],
+          start: 0,
+          duration: 1.2,
+        }),
+      }),
+    );
+    expect(finals[1]).toEqual(
+      expect.objectContaining({
+        text: "hello world",
+        confidence: 0.95,
+        provider: expect.objectContaining({ speechFinal: true }),
+      }),
+    );
+    expect(turnCompletes).toHaveLength(1);
+
+    await plugin.close();
+    bus.stop();
+    await started;
+  });
+
+  it("sends audio.done on stt.finalize", async () => {
+    const controlMessages: string[] = [];
+    const endpointUrl = await createLocalServer((socket) => {
+      socket.send(JSON.stringify({ type: "transcript.created" }));
+      socket.on("message", (data, isBinary) => {
+        if (!isBinary) controlMessages.push(data.toString());
+      });
+    });
+
+    const bus = new PipelineBusImpl();
+    const started = bus.start();
+    const plugin = new GrokSTTPlugin();
+
+    await plugin.initialize(bus, {
+      api_key: "test",
+      endpoint_url: endpointUrl,
+      sample_rate: 16000,
+    });
+    bus.push(Route.Main, {
+      kind: "stt.finalize",
+      contextId: "turn-3",
+      timestampMs: Date.now(),
+    });
+    await waitFor(controlMessages);
+
+    expect(JSON.parse(controlMessages[0]!)).toEqual({ type: "audio.done" });
+
+    await plugin.close();
+    bus.stop();
+    await started;
+  });
+
+  it("maps provider error frames to stt.error", async () => {
+    const endpointUrl = await createLocalServer((socket) => {
+      socket.send(JSON.stringify({ type: "error", message: "bad audio format" }));
+    });
+    const bus = new PipelineBusImpl();
+    const started = bus.start();
+    const plugin = new GrokSTTPlugin();
+    const errors: SttErrorPacket[] = [];
+    bus.on("stt.error", (pkt) => {
+      errors.push(pkt as SttErrorPacket);
+    });
+
+    await plugin.initialize(bus, {
+      api_key: "test",
+      endpoint_url: endpointUrl,
+      sample_rate: 16000,
+    });
+    await waitFor(errors);
+
+    expect(errors[0]).toEqual(
+      expect.objectContaining({
+        kind: "stt.error",
+        cause: expect.objectContaining({ message: "bad audio format" }),
+      }),
+    );
+
+    await plugin.close();
+    bus.stop();
+    await started;
+  });
+});

package/src/stt.ts

@@ -0,0 +1,264 @@
+// SPDX-License-Identifier: MIT
+
+import type { PipelineBus } from "@kuralle-syrinx/core";
+import {
+  Route,
+  type AudioFormat,
+  type PluginConfig,
+  type SttErrorPacket,
+  type VoicePlugin,
+  assertAudioFormat,
+  assertAudioPayload,
+  categorizeSttError,
+  isRecoverable,
+  optionalStringConfig,
+  readProviderRetryConfig,
+  requireStringConfig,
+} from "@kuralle-syrinx/core";
+import { WebSocketConnection, type SocketFactory } from "@kuralle-syrinx/ws";
+
+const AUDIO_DONE = JSON.stringify({ type: "audio.done" });
+
+export class GrokSTTPlugin implements VoicePlugin {
+  readonly endpointingCapability = {
+    owner: "provider_stt" as const,
+    disableConfig: {
+      emit_eos_on_final: false,
+      finalize_on_speech_final: false,
+    },
+  };
+
+  constructor(private readonly socketFactory?: SocketFactory) {}
+
+  private bus: PipelineBus | null = null;
+  private conn: WebSocketConnection | null = null;
+  private apiKey = "";
+  private sampleRate = 16000;
+  private language = "en";
+  private endpointUrl = "wss://api.x.ai/v1/stt";
+  private encoding = "pcm";
+  private interimResults = true;
+  private endpointing = 10;
+  private smartTurn: number | undefined;
+  private smartTurnTimeoutMs: number | undefined;
+  private diarize = false;
+  private keyterm: string | undefined;
+  private emitEosOnFinal = true;
+  private audioFormat: AudioFormat = { encoding: "pcm_s16le", sampleRateHz: 16000, channels: 1 };
+  private currentContextId = "";
+  private transcriptReady = false;
+  private disposers: Array<() => void> = [];
+
+  async initialize(bus: PipelineBus, config: PluginConfig): Promise<void> {
+    this.bus = bus;
+    this.apiKey = requireStringConfig(config, "api_key");
+    this.sampleRate = (config["sample_rate"] as number) ?? 16000;
+    this.language = optionalStringConfig(config, "language") ?? "en";
+    this.endpointUrl = optionalStringConfig(config, "endpoint_url") ?? this.endpointUrl;
+    this.encoding = optionalStringConfig(config, "encoding") ?? this.encoding;
+    this.interimResults = (config["interim_results"] as boolean) ?? true;
+    this.endpointing = (config["endpointing"] as number) ?? 10;
+    this.smartTurn = typeof config["smart_turn"] === "number" ? config["smart_turn"] : undefined;
+    this.smartTurnTimeoutMs =
+      typeof config["smart_turn_timeout"] === "number" ? config["smart_turn_timeout"] : undefined;
+    this.diarize = (config["diarize"] as boolean) ?? false;
+    this.keyterm = optionalStringConfig(config, "keyterm");
+    this.emitEosOnFinal = (config["emit_eos_on_final"] as boolean) ?? true;
+    this.audioFormat = { encoding: "pcm_s16le", sampleRateHz: this.sampleRate, channels: 1 };
+    assertAudioFormat(this.audioFormat);
+
+    this.conn = new WebSocketConnection({
+      url: () => {
+        const params = new URLSearchParams({
+          sample_rate: String(this.sampleRate),
+          encoding: this.encoding,
+          interim_results: String(this.interimResults),
+          language: this.language,
+          endpointing: String(this.endpointing),
+        });
+        if (this.smartTurn !== undefined) params.set("smart_turn", String(this.smartTurn));
+        if (this.smartTurnTimeoutMs !== undefined) {
+          params.set("smart_turn_timeout", String(this.smartTurnTimeoutMs));
+        }
+        if (this.diarize) params.set("diarize", "true");
+        if (this.keyterm) params.set("keyterm", this.keyterm);
+        const separator = this.endpointUrl.includes("?") ? "&" : "?";
+        return `${this.endpointUrl}${separator}${params.toString()}`;
+      },
+      headers: { Authorization: `Bearer ${this.apiKey}` },
+      socketFactory: this.socketFactory ?? (await defaultSocketFactory()),
+      retry: readProviderRetryConfig(config),
+      replayBufferSize: (config["replay_buffer_size"] as number) ?? 64,
+      onMessage: (data) => {
+        if (typeof data === "string") this.handleProviderMessage(data);
+      },
+      onConnectionLost: (err) => {
+        this.transcriptReady = false;
+        this.emitError(this.currentContextId, err);
+      },
+    });
+    await this.conn.connect();
+
+    this.disposers.push(
+      bus.on("stt.audio", (pkt: unknown) => {
+        void this.handleAudioPacket(pkt as { audio: Uint8Array; contextId?: string });
+      }),
+      bus.on("user.audio_received", (pkt: unknown) => {
+        void this.handleAudioPacket(pkt as { audio: Uint8Array; contextId?: string });
+      }),
+      bus.on("turn.change", (pkt: unknown) => {
+        this.currentContextId = (pkt as { contextId: string }).contextId;
+      }),
+      bus.on("interrupt.stt", () => {
+        this.currentContextId = "";
+      }),
+      bus.on("stt.finalize", () => {
+        void this.sendAudioDone();
+      }),
+    );
+  }
+
+  private async handleAudioPacket(pkt: { audio: Uint8Array; contextId?: string }): Promise<void> {
+    if (pkt.contextId) this.currentContextId = pkt.contextId;
+    await this.sendAudio(pkt.audio, this.currentContextId);
+  }
+
+  async sendAudio(audio: Uint8Array, contextId = this.currentContextId): Promise<boolean> {
+    if (audio.byteLength === 0) return true;
+    try {
+      assertAudioPayload(this.audioFormat, audio);
+      if (!this.conn) throw new Error("Grok STT is not connected");
+      await this.conn.ensureReady();
+      if (!this.transcriptReady) return false;
+      this.conn.send(audio);
+      return true;
+    } catch (err) {
+      this.emitError(contextId, err instanceof Error ? err : new Error(String(err)));
+      return false;
+    }
+  }
+
+  private async sendAudioDone(): Promise<void> {
+    try {
+      await this.conn?.ensureReady();
+      if (this.conn?.isReady) this.conn.send(AUDIO_DONE);
+    } catch (err) {
+      this.emitError(this.currentContextId, err instanceof Error ? err : new Error(String(err)));
+    }
+  }
+
+  private handleProviderMessage(data: string): void {
+    let msg: Record<string, unknown>;
+    try {
+      msg = JSON.parse(data) as Record<string, unknown>;
+    } catch (err) {
+      this.emitError(
+        this.currentContextId,
+        new Error(`Grok STT provider sent malformed JSON: ${err instanceof Error ? err.message : String(err)}`),
+      );
+      return;
+    }
+
+    const type = typeof msg["type"] === "string" ? msg["type"] : "";
+    switch (type) {
+      case "transcript.created":
+        this.transcriptReady = true;
+        return;
+      case "transcript.partial":
+        this.handleTranscriptPartial(msg);
+        return;
+      case "transcript.done":
+        return;
+      case "error":
+        this.emitError(
+          this.currentContextId,
+          new Error(typeof msg["message"] === "string" ? msg["message"] : "Grok STT provider error"),
+        );
+        return;
+      default:
+        return;
+    }
+  }
+
+  private handleTranscriptPartial(msg: Record<string, unknown>): void {
+    const text = typeof msg["text"] === "string" ? msg["text"].trim() : "";
+    if (!text) return;
+
+    const isFinal = msg["is_final"] === true;
+    const speechFinal = msg["speech_final"] === true;
+    const confidence =
+      typeof msg["end_of_turn_confidence"] === "number" ? msg["end_of_turn_confidence"] : 0;
+    const provider: Record<string, unknown> = {
+      name: "grok",
+      model: "stt",
+      region: "global",
+      speechFinal,
+      words: msg["words"],
+      start: msg["start"],
+      duration: msg["duration"],
+    };
+
+    if (isFinal) {
+      this.bus?.push(Route.Main, {
+        kind: "stt.result",
+        contextId: this.currentContextId,
+        timestampMs: Date.now(),
+        text,
+        confidence,
+        language: this.language,
+        provider,
+      });
+      if (this.emitEosOnFinal && speechFinal) {
+        this.bus?.push(Route.Main, {
+          kind: "eos.turn_complete",
+          contextId: this.currentContextId,
+          timestampMs: Date.now(),
+          text,
+          transcripts: [],
+        });
+      }
+      return;
+    }
+
+    this.bus?.push(Route.Main, {
+      kind: "stt.interim",
+      contextId: this.currentContextId,
+      timestampMs: Date.now(),
+      text,
+    });
+  }
+
+  private emitError(contextId: string, err: Error): void {
+    const category = categorizeSttError(err);
+    const packet: SttErrorPacket = {
+      kind: "stt.error",
+      contextId,
+      timestampMs: Date.now(),
+      component: "stt",
+      category,
+      cause: err,
+      isRecoverable: isRecoverable(category),
+    };
+    this.bus?.push(Route.Critical, packet);
+  }
+
+  async close(): Promise<void> {
+    for (const dispose of this.disposers.splice(0)) dispose();
+    if (this.conn?.isReady) {
+      try {
+        this.conn.send(AUDIO_DONE);
+      } catch {
+        // best effort
+      }
+    }
+    await this.conn?.close();
+    this.conn = null;
+    this.bus = null;
+    this.transcriptReady = false;
+  }
+}
+
+async function defaultSocketFactory(): Promise<SocketFactory> {
+  const mod = await import("@kuralle-syrinx/ws/node");
+  return mod.createNodeWsSocket;
+}

package/src/tts.test.ts

@@ -0,0 +1,160 @@
+// SPDX-License-Identifier: MIT
+
+import { afterEach, describe, expect, it } from "vitest";
+import { WebSocketServer, type WebSocket } from "ws";
+import {
+  PipelineBusImpl,
+  Route,
+  type TextToSpeechAudioPacket,
+  type TextToSpeechEndPacket,
+} from "@kuralle-syrinx/core";
+
+import { bytesToBase64 } from "@kuralle-syrinx/realtime";
+import { GrokTTSPlugin } from "./tts.js";
+
+let servers: WebSocketServer[] = [];
+
+afterEach(async () => {
+  await Promise.all(
+    servers.splice(0).map(
+      (server) =>
+        new Promise<void>((resolve) => {
+          for (const client of server.clients) client.terminate();
+          server.close(() => resolve());
+        }),
+    ),
+  );
+});
+
+async function createLocalServer(
+  onConnection: (socket: WebSocket, requestUrl: string, authHeader: string) => void,
+): Promise<string> {
+  const server = await new Promise<WebSocketServer>((resolve) => {
+    let next: WebSocketServer;
+    next = new WebSocketServer({ port: 0 }, () => resolve(next));
+  });
+  servers.push(server);
+  server.on("connection", (socket, request) => {
+    const header = request.headers["authorization"];
+    onConnection(
+      socket,
+      request.url ?? "",
+      Array.isArray(header) ? header[0] ?? "" : header ?? "",
+    );
+  });
+  const address = server.address();
+  if (!address || typeof address === "string") throw new Error("Expected TCP server address");
+  return `ws://127.0.0.1:${address.port}/v1/tts`;
+}
+
+async function waitForCondition(predicate: () => boolean, timeoutMs = 1000): Promise<void> {
+  const startedAt = Date.now();
+  while (Date.now() - startedAt < timeoutMs) {
+    if (predicate()) return;
+    await new Promise((resolve) => setTimeout(resolve, 10));
+  }
+  throw new Error("Timed out waiting for Grok TTS test condition");
+}
+
+describe("GrokTTSPlugin", () => {
+  it("streams text.delta and maps audio.delta to tts.audio", async () => {
+    const received: Array<Record<string, unknown>> = [];
+    const endpointUrl = await createLocalServer((socket, requestUrl, authHeader) => {
+      expect(requestUrl).toContain("voice=eve");
+      expect(requestUrl).toContain("codec=pcm");
+      expect(requestUrl).toContain("sample_rate=16000");
+      expect(authHeader).toBe("Bearer test-xai-key");
+      socket.on("message", (data) => {
+        const msg = JSON.parse(data.toString()) as Record<string, unknown>;
+        received.push(msg);
+        if (msg["type"] === "text.delta") {
+          socket.send(JSON.stringify({
+            type: "audio.delta",
+            delta: bytesToBase64(new Uint8Array([1, 2, 3, 4])),
+          }));
+        }
+        if (msg["type"] === "text.done") {
+          socket.send(JSON.stringify({ type: "audio.done" }));
+        }
+      });
+    });
+
+    const bus = new PipelineBusImpl();
+    const started = bus.start();
+    const plugin = new GrokTTSPlugin();
+    const audio: TextToSpeechAudioPacket[] = [];
+    const ends: TextToSpeechEndPacket[] = [];
+    bus.on("tts.audio", (pkt) => {
+      audio.push(pkt as TextToSpeechAudioPacket);
+    });
+    bus.on("tts.end", (pkt) => {
+      ends.push(pkt as TextToSpeechEndPacket);
+    });
+
+    await plugin.initialize(bus, {
+      api_key: "test-xai-key",
+      endpoint_url: endpointUrl,
+      voice_id: "eve",
+      sample_rate: 16000,
+    });
+    bus.push(Route.Main, {
+      kind: "tts.text",
+      contextId: "turn-1",
+      timestampMs: Date.now(),
+      text: "Hello there.",
+    });
+    bus.push(Route.Main, { kind: "tts.done", contextId: "turn-1", timestampMs: Date.now() });
+    await waitForCondition(() => ends.length >= 1);
+
+    expect(received).toEqual([
+      expect.objectContaining({ type: "text.delta", delta: "Hello there." }),
+      expect.objectContaining({ type: "text.done" }),
+    ]);
+    expect(audio).toEqual([
+      expect.objectContaining({
+        contextId: "turn-1",
+        sampleRateHz: 16000,
+        audio: new Uint8Array([1, 2, 3, 4]),
+        provider: expect.objectContaining({ name: "grok", model: "eve" }),
+      }),
+    ]);
+    expect(ends).toEqual([expect.objectContaining({ contextId: "turn-1" })]);
+
+    await plugin.close();
+    bus.stop();
+    await started;
+  });
+
+  it("sends text.clear on interrupt", async () => {
+    const received: Array<Record<string, unknown>> = [];
+    const endpointUrl = await createLocalServer((socket) => {
+      socket.on("message", (data) => {
+        received.push(JSON.parse(data.toString()) as Record<string, unknown>);
+      });
+    });
+
+    const bus = new PipelineBusImpl();
+    const started = bus.start();
+    const plugin = new GrokTTSPlugin();
+
+    await plugin.initialize(bus, { api_key: "test-xai-key", endpoint_url: endpointUrl });
+    bus.push(Route.Main, {
+      kind: "tts.text",
+      contextId: "turn-x",
+      timestampMs: Date.now(),
+      text: "Interrupt me.",
+    });
+    await waitForCondition(() => received.some((m) => m["type"] === "text.delta"));
+    bus.push(Route.Critical, { kind: "interrupt.tts", contextId: "turn-x", timestampMs: Date.now() });
+    await waitForCondition(() => received.some((m) => m["type"] === "text.clear"));
+
+    expect(received).toEqual([
+      expect.objectContaining({ type: "text.delta", delta: "Interrupt me." }),
+      expect.objectContaining({ type: "text.clear" }),
+    ]);
+
+    await plugin.close();
+    bus.stop();
+    await started;
+  });
+});