@kuralle-syrinx/grok 4.2.0 → 4.4.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@kuralle-syrinx/grok",
-  "version": "4.2.0",
+  "version": "4.4.0",
   "private": false,
   "description": "xAI Grok adapters for Syrinx — STT, TTS, and realtime",
   "keywords": [
@@ -32,16 +32,17 @@
     "./realtime": "./src/from-grok-realtime.ts"
   },
   "dependencies": {
-    "@kuralle-syrinx/core": "4.2.0",
-    "@kuralle-syrinx/realtime": "4.2.0",
-    "@kuralle-syrinx/tts-core": "4.2.0",
-    "@kuralle-syrinx/ws": "4.2.0"
+    "@kuralle-syrinx/realtime": "4.4.0",
+    "@kuralle-syrinx/core": "4.4.0",
+    "@kuralle-syrinx/tts-core": "4.4.0",
+    "@kuralle-syrinx/stt-core": "4.4.0",
+    "@kuralle-syrinx/ws": "4.4.0"
   },
   "devDependencies": {
     "@types/ws": "^8.5.0",
     "typescript": "^5.7.0",
-    "vitest": "^2.1.0",
-    "ws": "^8.18.0"
+    "vitest": "^3.2.6",
+    "ws": "^8.21.0"
   },
   "scripts": {
     "typecheck": "tsc --noEmit",

package/src/from-grok-realtime.ts

@@ -25,6 +25,19 @@ export interface GrokRealtimeOptions {
   readonly instructions?: string;
   readonly inputRateHz?: number;
   readonly outputRateHz?: number;
+  /** Input audio format. Defaults to `{ type: "audio/pcm", rate: inputRateHz }`. */
+  readonly inputAudioFormat?: Record<string, unknown>;
+  /** Output audio format. Defaults to `{ type: "audio/pcm", rate: outputRateHz }`. */
+  readonly outputAudioFormat?: Record<string, unknown>;
+  readonly temperature?: number;
+  readonly modalities?: readonly string[];
+  readonly inputTranscription?: Record<string, unknown> | boolean;
+  readonly toolChoice?: string | Record<string, unknown>;
+  /**
+   * Merged last into `session.update.session` for provider-specific fields the
+   * adapter does not enumerate. Overrides same-key defaults when both set.
+   */
+  readonly sessionConfig?: Record<string, unknown>;
 }
 
 export function fromGrokRealtime(opts: GrokRealtimeOptions): RealtimeAdapter {
@@ -52,6 +65,14 @@ export function fromGrokRealtime(opts: GrokRealtimeOptions): RealtimeAdapter {
       const turnDetection =
         "turnDetection" in opts ? opts.turnDetection : { type: "server_vad" };
 
+      const inputAudio: Record<string, unknown> = {
+        format: opts.inputAudioFormat ?? { type: "audio/pcm", rate: inputRateHz },
+      };
+      if (opts.inputTranscription !== undefined) {
+        inputAudio["transcription"] =
+          opts.inputTranscription === true ? {} : opts.inputTranscription;
+      }
+
       const session: Record<string, unknown> = {
         voice,
         turn_detection: turnDetection,
@@ -62,14 +83,29 @@ export function fromGrokRealtime(opts: GrokRealtimeOptions): RealtimeAdapter {
           parameters: t.parameters,
         })),
         audio: {
-          input: { format: { type: "audio/pcm", rate: inputRateHz } },
-          output: { format: { type: "audio/pcm", rate: outputRateHz }, voice },
+          input: inputAudio,
+          output: {
+            format: opts.outputAudioFormat ?? { type: "audio/pcm", rate: outputRateHz },
+            voice,
+          },
         },
       };
 
       if (opts.instructions !== undefined) {
         session["instructions"] = opts.instructions;
       }
+      if (opts.temperature !== undefined) {
+        session["temperature"] = opts.temperature;
+      }
+      if (opts.modalities !== undefined) {
+        session["output_modalities"] = opts.modalities;
+      }
+      if (opts.toolChoice !== undefined) {
+        session["tool_choice"] = opts.toolChoice;
+      }
+      if (opts.sessionConfig) {
+        Object.assign(session, opts.sessionConfig);
+      }
 
       return session;
     },

package/src/stt.ts

@@ -1,264 +1,209 @@
 // SPDX-License-Identifier: MIT
+//
+// Grok (xAI) STT Plugin. The streaming lifecycle lives in @kuralle-syrinx/stt-core. This
+// file is the Grok wire protocol: connect URL + query knobs, audio.done finalize, and
+// decode of transcript.created / transcript.partial (is_final / duration / speech_final).
 
-import type { PipelineBus } from "@kuralle-syrinx/core";
 import {
-  Route,
-  type AudioFormat,
-  type PluginConfig,
-  type SttErrorPacket,
-  type VoicePlugin,
   assertAudioFormat,
-  assertAudioPayload,
-  categorizeSttError,
-  isRecoverable,
   optionalStringConfig,
   readProviderRetryConfig,
   requireStringConfig,
+  type AudioFormat,
+  type PipelineBus,
+  type PluginConfig,
+  type VoicePlugin,
 } from "@kuralle-syrinx/core";
-import { WebSocketConnection, type SocketFactory } from "@kuralle-syrinx/ws";
+import {
+  defaultNodeSocketFactory,
+  startStreamingSttSession,
+  type SttEvent,
+  type SttWireProtocol,
+  type StreamingSttSession,
+} from "@kuralle-syrinx/stt-core";
+import type { SocketData, SocketFactory } from "@kuralle-syrinx/ws";
 
 const AUDIO_DONE = JSON.stringify({ type: "audio.done" });
 
-export class GrokSTTPlugin implements VoicePlugin {
-  readonly endpointingCapability = {
-    owner: "provider_stt" as const,
-    disableConfig: {
-      emit_eos_on_final: false,
-      finalize_on_speech_final: false,
-    },
-  };
-
-  constructor(private readonly socketFactory?: SocketFactory) {}
-
-  private bus: PipelineBus | null = null;
-  private conn: WebSocketConnection | null = null;
-  private apiKey = "";
-  private sampleRate = 16000;
-  private language = "en";
-  private endpointUrl = "wss://api.x.ai/v1/stt";
-  private encoding = "pcm";
-  private interimResults = true;
-  private endpointing = 10;
-  private smartTurn: number | undefined;
-  private smartTurnTimeoutMs: number | undefined;
-  private diarize = false;
-  private keyterm: string | undefined;
-  private emitEosOnFinal = true;
-  private audioFormat: AudioFormat = { encoding: "pcm_s16le", sampleRateHz: 16000, channels: 1 };
-  private currentContextId = "";
-  private transcriptReady = false;
-  private disposers: Array<() => void> = [];
-
-  async initialize(bus: PipelineBus, config: PluginConfig): Promise<void> {
-    this.bus = bus;
-    this.apiKey = requireStringConfig(config, "api_key");
-    this.sampleRate = (config["sample_rate"] as number) ?? 16000;
-    this.language = optionalStringConfig(config, "language") ?? "en";
-    this.endpointUrl = optionalStringConfig(config, "endpoint_url") ?? this.endpointUrl;
-    this.encoding = optionalStringConfig(config, "encoding") ?? this.encoding;
-    this.interimResults = (config["interim_results"] as boolean) ?? true;
-    this.endpointing = (config["endpointing"] as number) ?? 10;
-    this.smartTurn = typeof config["smart_turn"] === "number" ? config["smart_turn"] : undefined;
-    this.smartTurnTimeoutMs =
-      typeof config["smart_turn_timeout"] === "number" ? config["smart_turn_timeout"] : undefined;
-    this.diarize = (config["diarize"] as boolean) ?? false;
-    this.keyterm = optionalStringConfig(config, "keyterm");
-    this.emitEosOnFinal = (config["emit_eos_on_final"] as boolean) ?? true;
-    this.audioFormat = { encoding: "pcm_s16le", sampleRateHz: this.sampleRate, channels: 1 };
-    assertAudioFormat(this.audioFormat);
+class GrokSttWireProtocol implements SttWireProtocol {
+  private ready = false;
 
-    this.conn = new WebSocketConnection({
-      url: () => {
-        const params = new URLSearchParams({
-          sample_rate: String(this.sampleRate),
-          encoding: this.encoding,
-          interim_results: String(this.interimResults),
-          language: this.language,
-          endpointing: String(this.endpointing),
-        });
-        if (this.smartTurn !== undefined) params.set("smart_turn", String(this.smartTurn));
-        if (this.smartTurnTimeoutMs !== undefined) {
-          params.set("smart_turn_timeout", String(this.smartTurnTimeoutMs));
-        }
-        if (this.diarize) params.set("diarize", "true");
-        if (this.keyterm) params.set("keyterm", this.keyterm);
-        const separator = this.endpointUrl.includes("?") ? "&" : "?";
-        return `${this.endpointUrl}${separator}${params.toString()}`;
-      },
-      headers: { Authorization: `Bearer ${this.apiKey}` },
-      socketFactory: this.socketFactory ?? (await defaultSocketFactory()),
-      retry: readProviderRetryConfig(config),
-      replayBufferSize: (config["replay_buffer_size"] as number) ?? 64,
-      onMessage: (data) => {
-        if (typeof data === "string") this.handleProviderMessage(data);
-      },
-      onConnectionLost: (err) => {
-        this.transcriptReady = false;
-        this.emitError(this.currentContextId, err);
-      },
-    });
-    await this.conn.connect();
-
-    this.disposers.push(
-      bus.on("stt.audio", (pkt: unknown) => {
-        void this.handleAudioPacket(pkt as { audio: Uint8Array; contextId?: string });
-      }),
-      bus.on("user.audio_received", (pkt: unknown) => {
-        void this.handleAudioPacket(pkt as { audio: Uint8Array; contextId?: string });
-      }),
-      bus.on("turn.change", (pkt: unknown) => {
-        this.currentContextId = (pkt as { contextId: string }).contextId;
-      }),
-      bus.on("interrupt.stt", () => {
-        this.currentContextId = "";
-      }),
-      bus.on("stt.finalize", () => {
-        void this.sendAudioDone();
-      }),
-    );
+  isReady(): boolean {
+    return this.ready;
   }
 
-  private async handleAudioPacket(pkt: { audio: Uint8Array; contextId?: string }): Promise<void> {
-    if (pkt.contextId) this.currentContextId = pkt.contextId;
-    await this.sendAudio(pkt.audio, this.currentContextId);
+  onConnectionLost(): void {
+    this.ready = false;
   }
 
-  async sendAudio(audio: Uint8Array, contextId = this.currentContextId): Promise<boolean> {
-    if (audio.byteLength === 0) return true;
-    try {
-      assertAudioPayload(this.audioFormat, audio);
-      if (!this.conn) throw new Error("Grok STT is not connected");
-      await this.conn.ensureReady();
-      if (!this.transcriptReady) return false;
-      this.conn.send(audio);
-      return true;
-    } catch (err) {
-      this.emitError(contextId, err instanceof Error ? err : new Error(String(err)));
-      return false;
-    }
+  encodeFinalize(): SocketData[] {
+    return [AUDIO_DONE];
   }
 
-  private async sendAudioDone(): Promise<void> {
-    try {
-      await this.conn?.ensureReady();
-      if (this.conn?.isReady) this.conn.send(AUDIO_DONE);
-    } catch (err) {
-      this.emitError(this.currentContextId, err instanceof Error ? err : new Error(String(err)));
-    }
+  encodeClose(): SocketData[] {
+    return [AUDIO_DONE];
   }
 
-  private handleProviderMessage(data: string): void {
+  decode(data: SocketData, _isBinary: boolean): readonly SttEvent[] {
+    if (typeof data !== "string") return [];
     let msg: Record<string, unknown>;
     try {
       msg = JSON.parse(data) as Record<string, unknown>;
     } catch (err) {
-      this.emitError(
-        this.currentContextId,
-        new Error(`Grok STT provider sent malformed JSON: ${err instanceof Error ? err.message : String(err)}`),
-      );
-      return;
+      return [
+        {
+          type: "error",
+          error: new Error(
+            `Grok STT provider sent malformed JSON: ${err instanceof Error ? err.message : String(err)}`,
+          ),
+        },
+      ];
     }
 
     const type = typeof msg["type"] === "string" ? msg["type"] : "";
     switch (type) {
       case "transcript.created":
-        this.transcriptReady = true;
-        return;
+        this.ready = true;
+        return [];
       case "transcript.partial":
-        this.handleTranscriptPartial(msg);
-        return;
+        return this.decodePartial(msg);
       case "transcript.done":
-        return;
+        return [];
       case "error":
-        this.emitError(
-          this.currentContextId,
-          new Error(typeof msg["message"] === "string" ? msg["message"] : "Grok STT provider error"),
-        );
-        return;
+        return [
+          {
+            type: "error",
+            error: new Error(
+              typeof msg["message"] === "string" ? msg["message"] : "Grok STT provider error",
+            ),
+          },
+        ];
       default:
-        return;
+        return [];
     }
   }
 
-  private handleTranscriptPartial(msg: Record<string, unknown>): void {
+  private decodePartial(msg: Record<string, unknown>): readonly SttEvent[] {
     const text = typeof msg["text"] === "string" ? msg["text"].trim() : "";
-    if (!text) return;
+    if (!text) return [];
 
     const isFinal = msg["is_final"] === true;
     const speechFinal = msg["speech_final"] === true;
     const confidence =
       typeof msg["end_of_turn_confidence"] === "number" ? msg["end_of_turn_confidence"] : 0;
-    const provider: Record<string, unknown> = {
-      name: "grok",
-      model: "stt",
-      region: "global",
-      speechFinal,
-      words: msg["words"],
-      start: msg["start"],
-      duration: msg["duration"],
-    };
+    // contextId is stamped by the session from the audio/turn context.
+    const contextId = "";
 
-    if (isFinal) {
-      this.bus?.push(Route.Main, {
-        kind: "stt.result",
-        contextId: this.currentContextId,
-        timestampMs: Date.now(),
+    if (!isFinal) {
+      return [{ type: "interim", contextId, text }];
+    }
+
+    return [
+      {
+        type: "final",
+        contextId,
         text,
         confidence,
-        language: this.language,
-        provider,
-      });
-      if (this.emitEosOnFinal && speechFinal) {
-        this.bus?.push(Route.Main, {
-          kind: "eos.turn_complete",
-          contextId: this.currentContextId,
-          timestampMs: Date.now(),
-          text,
-          transcripts: [],
-        });
-      }
-      return;
-    }
+        speechFinal,
+        audioSeconds: typeof msg["duration"] === "number" ? msg["duration"] : undefined,
+        provider: {
+          speechFinal,
+          words: msg["words"],
+          start: msg["start"],
+          duration: msg["duration"],
+        },
+      },
+    ];
+  }
+}
+
+export class GrokSTTPlugin implements VoicePlugin {
+  readonly endpointingCapability = {
+    owner: "provider_stt" as const,
+    disableConfig: {
+      emit_eos_on_final: false,
+      finalize_on_speech_final: false,
+    },
+  };
+
+  constructor(private readonly socketFactory?: SocketFactory) {}
 
-    this.bus?.push(Route.Main, {
-      kind: "stt.interim",
-      contextId: this.currentContextId,
-      timestampMs: Date.now(),
-      text,
+  private session: StreamingSttSession | null = null;
+
+  async initialize(bus: PipelineBus, config: PluginConfig): Promise<void> {
+    const apiKey = requireStringConfig(config, "api_key");
+    const sampleRate = (config["sample_rate"] as number) ?? 16000;
+    const language = optionalStringConfig(config, "language") ?? "en";
+    const endpointUrl = optionalStringConfig(config, "endpoint_url") ?? "wss://api.x.ai/v1/stt";
+    const encoding = optionalStringConfig(config, "encoding") ?? "pcm";
+    const interimResults = (config["interim_results"] as boolean) ?? true;
+    const endpointing = (config["endpointing"] as number) ?? 10;
+    const smartTurn = typeof config["smart_turn"] === "number" ? config["smart_turn"] : undefined;
+    const smartTurnTimeoutMs =
+      typeof config["smart_turn_timeout"] === "number" ? config["smart_turn_timeout"] : undefined;
+    const diarize = (config["diarize"] as boolean) ?? false;
+    const keyterm = optionalStringConfig(config, "keyterm");
+    const emitEosOnFinal = (config["emit_eos_on_final"] as boolean) ?? true;
+    const queryParams = readPlainObject(config["query_params"]);
+    const audioFormat: AudioFormat = { encoding: "pcm_s16le", sampleRateHz: sampleRate, channels: 1 };
+    assertAudioFormat(audioFormat);
+
+    this.session = await startStreamingSttSession(bus, {
+      protocol: new GrokSttWireProtocol(),
+      provider: { name: "grok", model: "stt", region: "global" },
+      format: audioFormat,
+      language,
+      emitEosOnFinal,
+      url: () => {
+        const params = new URLSearchParams({
+          sample_rate: String(sampleRate),
+          encoding,
+          interim_results: String(interimResults),
+          language,
+          endpointing: String(endpointing),
+        });
+        if (smartTurn !== undefined) params.set("smart_turn", String(smartTurn));
+        if (smartTurnTimeoutMs !== undefined) {
+          params.set("smart_turn_timeout", String(smartTurnTimeoutMs));
+        }
+        if (diarize) params.set("diarize", "true");
+        if (keyterm) params.set("keyterm", keyterm);
+        applyQueryParams(params, queryParams);
+        const separator = endpointUrl.includes("?") ? "&" : "?";
+        return `${endpointUrl}${separator}${params.toString()}`;
+      },
+      headers: { Authorization: `Bearer ${apiKey}` },
+      socketFactory: this.socketFactory ?? (await defaultNodeSocketFactory()),
+      retry: readProviderRetryConfig(config),
+      replayBufferSize: (config["replay_buffer_size"] as number) ?? 64,
+      metricPrefix: "stt.grok",
     });
   }
 
-  private emitError(contextId: string, err: Error): void {
-    const category = categorizeSttError(err);
-    const packet: SttErrorPacket = {
-      kind: "stt.error",
-      contextId,
-      timestampMs: Date.now(),
-      component: "stt",
-      category,
-      cause: err,
-      isRecoverable: isRecoverable(category),
-    };
-    this.bus?.push(Route.Critical, packet);
+  async close(): Promise<void> {
+    await this.session?.dispose();
+    this.session = null;
   }
+}
 
-  async close(): Promise<void> {
-    for (const dispose of this.disposers.splice(0)) dispose();
-    if (this.conn?.isReady) {
-      try {
-        this.conn.send(AUDIO_DONE);
-      } catch {
-        // best effort
-      }
-    }
-    await this.conn?.close();
-    this.conn = null;
-    this.bus = null;
-    this.transcriptReady = false;
+function readPlainObject(value: unknown): Record<string, unknown> | undefined {
+  if (value !== null && typeof value === "object" && !Array.isArray(value)) {
+    return value as Record<string, unknown>;
   }
+  return undefined;
 }
 
-async function defaultSocketFactory(): Promise<SocketFactory> {
-  const mod = await import("@kuralle-syrinx/ws/node");
-  return mod.createNodeWsSocket;
+function applyQueryParams(params: URLSearchParams, extra: Record<string, unknown> | undefined): void {
+  if (!extra) return;
+  for (const [key, value] of Object.entries(extra)) {
+    if (value === undefined || value === null) continue;
+    if (Array.isArray(value)) {
+      for (const item of value) {
+        if (item === undefined || item === null) continue;
+        params.append(key, String(item));
+      }
+      continue;
+    }
+    params.set(key, String(value));
+  }
 }

package/src/tts.ts

@@ -7,6 +7,7 @@
 // Grok relies on the engine's streaming carry.
 
 import {
+  Route,
   assertAudioFormat,
   optionalStringConfig,
   readProviderRetryConfig,
@@ -35,13 +36,17 @@ class GrokWireProtocol implements WireProtocol {
   // to the current one. (Audio for an interrupted context is dropped by the engine's
   // cancelled-key tracking, which subsumes Grok's old `clearedPending` race guard.)
   private current: AttributionKey | null = null;
+  private readonly charactersByKey = new Map<AttributionKey, number>();
+
+  constructor(private readonly modelId: string) {}
 
   attributionFor(contextId: string): { key: AttributionKey; contextId: string } {
     this.current = attributionKey(contextId);
     return { key: this.current, contextId };
   }
 
-  encodeText(_key: AttributionKey, text: string): SocketData[] {
+  encodeText(key: AttributionKey, text: string): SocketData[] {
+    this.charactersByKey.set(key, (this.charactersByKey.get(key) ?? 0) + text.length);
     return [JSON.stringify({ type: "text.delta", delta: text })];
   }
 
@@ -49,7 +54,8 @@ class GrokWireProtocol implements WireProtocol {
     return [JSON.stringify({ type: "text.done" })];
   }
 
-  encodeCancel(): SocketData[] {
+  encodeCancel(key: AttributionKey): SocketData[] {
+    this.charactersByKey.delete(key);
     return [JSON.stringify({ type: "text.clear" })];
   }
 
@@ -78,14 +84,38 @@ class GrokWireProtocol implements WireProtocol {
         }
       }
       case "audio.done": {
+        const characters = key ? (this.charactersByKey.get(key) ?? 0) : 0;
+        if (key) this.charactersByKey.delete(key);
         this.current = null;
-        return key ? [{ type: "context_end", key }] : [];
+        if (!key) return [];
+        const events: WireEvent[] = [];
+        // Sideband before context_end so the engine still has the key→context mapping.
+        if (characters > 0) {
+          const modelId = this.modelId;
+          events.push({
+            type: "sideband",
+            key,
+            route: Route.Background,
+            build: (ctxId, timestampMs) => ({
+              kind: "usage.recorded",
+              contextId: ctxId,
+              timestampMs,
+              stage: "tts" as const,
+              provider: "grok",
+              model: modelId,
+              characters,
+            }),
+          });
+        }
+        events.push({ type: "context_end", key });
+        return events;
       }
       case "audio.clear":
         // Provider acknowledged a text.clear. Audio for the interrupted context is already
         // dropped via the engine's cancelled tracking; nothing more to do.
         return [];
       case "error":
+        if (key) this.charactersByKey.delete(key);
         return key ? [{ type: "error", key, error: grokProviderError(msg) }] : [];
       default:
         return [];
@@ -104,11 +134,13 @@ export class GrokTTSPlugin implements VoicePlugin {
     const language = optionalStringConfig(config, "language") ?? "en";
     const endpointUrl = optionalStringConfig(config, "endpoint_url") ?? "wss://api.x.ai/v1/tts";
     const sampleRate = readPositiveInteger(config["sample_rate"], 16000);
+    const codec = optionalStringConfig(config, "codec") ?? "pcm";
+    const bitRate = readPositiveIntegerOrUndefined(config["bit_rate"]);
     const audioFormat: AudioFormat = { encoding: "pcm_s16le", sampleRateHz: sampleRate, channels: 1 };
     assertAudioFormat(audioFormat);
 
     this.session = await startStreamingTtsSession(bus, {
-      protocol: new GrokWireProtocol(),
+      protocol: new GrokWireProtocol(voiceId),
       provider: { name: "grok", model: voiceId, region: "global" },
       format: audioFormat,
       sampleRateHz: sampleRate,
@@ -116,9 +148,10 @@ export class GrokTTSPlugin implements VoicePlugin {
         const params = new URLSearchParams({
           language,
           voice: voiceId,
-          codec: "pcm",
+          codec,
           sample_rate: String(sampleRate),
         });
+        if (bitRate !== undefined) params.set("bit_rate", String(bitRate));
         const separator = endpointUrl.includes("?") ? "&" : "?";
         return `${endpointUrl}${separator}${params.toString()}`;
       },
@@ -152,6 +185,12 @@ function readPositiveInteger(value: unknown, fallback: number): number {
   return integer > 0 ? integer : fallback;
 }
 
+function readPositiveIntegerOrUndefined(value: unknown): number | undefined {
+  if (typeof value !== "number" || !Number.isFinite(value)) return undefined;
+  const integer = Math.floor(value);
+  return integer > 0 ? integer : undefined;
+}
+
 function readNonNegativeInteger(value: unknown, fallback: number): number {
   if (typeof value !== "number" || !Number.isFinite(value)) return fallback;
   const integer = Math.floor(value);