@tanstack/ai-grok 0.8.4 → 0.8.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/esm/adapters/image.js +26 -14
- package/dist/esm/adapters/image.js.map +1 -1
- package/dist/esm/adapters/text.js +2 -2
- package/dist/esm/adapters/text.js.map +1 -1
- package/dist/esm/adapters/transcription.js +7 -3
- package/dist/esm/adapters/transcription.js.map +1 -1
- package/dist/esm/adapters/tts.js +4 -1
- package/dist/esm/adapters/tts.js.map +1 -1
- package/dist/esm/realtime/adapter.js +19 -12
- package/dist/esm/realtime/adapter.js.map +1 -1
- package/dist/esm/utils/audio.js +1 -0
- package/dist/esm/utils/audio.js.map +1 -1
- package/package.json +5 -5
- package/src/adapters/image.ts +25 -16
- package/src/adapters/text.ts +2 -2
- package/src/adapters/transcription.ts +3 -2
- package/src/realtime/adapter.ts +25 -14
- package/src/utils/audio.ts +1 -0
|
@@ -5,10 +5,11 @@ import { generateId } from "@tanstack/ai-utils";
|
|
|
5
5
|
import { withGrokDefaults, getGrokApiKeyFromEnv } from "../utils/client.js";
|
|
6
6
|
import { validatePrompt, validateImageSize, validateNumberOfImages } from "../image/image-provider-options.js";
|
|
7
7
|
class GrokImageAdapter extends BaseImageAdapter {
|
|
8
|
+
kind = "image";
|
|
9
|
+
name = "grok";
|
|
10
|
+
client;
|
|
8
11
|
constructor(config, model) {
|
|
9
12
|
super(model, {});
|
|
10
|
-
this.kind = "image";
|
|
11
|
-
this.name = "grok";
|
|
12
13
|
this.client = new OpenAI(withGrokDefaults(config));
|
|
13
14
|
}
|
|
14
15
|
async generateImages(options) {
|
|
@@ -16,11 +17,13 @@ class GrokImageAdapter extends BaseImageAdapter {
|
|
|
16
17
|
validatePrompt({ prompt });
|
|
17
18
|
validateImageSize(model, size);
|
|
18
19
|
validateNumberOfImages(model, numberOfImages);
|
|
20
|
+
const resolvedSize = size;
|
|
19
21
|
const request = {
|
|
20
22
|
model,
|
|
21
23
|
prompt,
|
|
22
24
|
n: numberOfImages ?? 1,
|
|
23
|
-
size,
|
|
25
|
+
...resolvedSize !== void 0 && { size: resolvedSize },
|
|
26
|
+
stream: false,
|
|
24
27
|
...modelOptions
|
|
25
28
|
};
|
|
26
29
|
try {
|
|
@@ -28,18 +31,25 @@ class GrokImageAdapter extends BaseImageAdapter {
|
|
|
28
31
|
`activity=image provider=${this.name} model=${model} n=${request.n ?? 1} size=${request.size ?? "default"}`,
|
|
29
32
|
{ provider: this.name, model }
|
|
30
33
|
);
|
|
31
|
-
const response = await this.client.images.generate(
|
|
32
|
-
...request,
|
|
33
|
-
stream: false
|
|
34
|
-
});
|
|
34
|
+
const response = await this.client.images.generate(request);
|
|
35
35
|
const images = (response.data ?? []).flatMap(
|
|
36
36
|
(item) => {
|
|
37
37
|
const revisedPrompt = item.revised_prompt;
|
|
38
38
|
if (item.b64_json) {
|
|
39
|
-
return [
|
|
39
|
+
return [
|
|
40
|
+
{
|
|
41
|
+
b64Json: item.b64_json,
|
|
42
|
+
...revisedPrompt !== void 0 && { revisedPrompt }
|
|
43
|
+
}
|
|
44
|
+
];
|
|
40
45
|
}
|
|
41
46
|
if (item.url) {
|
|
42
|
-
return [
|
|
47
|
+
return [
|
|
48
|
+
{
|
|
49
|
+
url: item.url,
|
|
50
|
+
...revisedPrompt !== void 0 && { revisedPrompt }
|
|
51
|
+
}
|
|
52
|
+
];
|
|
43
53
|
}
|
|
44
54
|
return [];
|
|
45
55
|
}
|
|
@@ -48,11 +58,13 @@ class GrokImageAdapter extends BaseImageAdapter {
|
|
|
48
58
|
id: generateId(this.name),
|
|
49
59
|
model,
|
|
50
60
|
images,
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
61
|
+
...response.usage && {
|
|
62
|
+
usage: {
|
|
63
|
+
inputTokens: response.usage.input_tokens,
|
|
64
|
+
outputTokens: response.usage.output_tokens,
|
|
65
|
+
totalTokens: response.usage.total_tokens
|
|
66
|
+
}
|
|
67
|
+
}
|
|
56
68
|
};
|
|
57
69
|
} catch (error) {
|
|
58
70
|
options.logger.errors(`${this.name}.generateImages fatal`, {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"image.js","sources":["../../../src/adapters/image.ts"],"sourcesContent":["import OpenAI from 'openai'\nimport { BaseImageAdapter } from '@tanstack/ai/adapters'\nimport { toRunErrorPayload } from '@tanstack/ai/adapter-internals'\nimport { generateId } from '@tanstack/ai-utils'\nimport { getGrokApiKeyFromEnv, withGrokDefaults } from '../utils/client'\nimport {\n validateImageSize,\n validateNumberOfImages,\n validatePrompt,\n} from '../image/image-provider-options'\nimport type {\n GeneratedImage,\n ImageGenerationOptions,\n ImageGenerationResult,\n} from '@tanstack/ai'\nimport type OpenAI_SDK from 'openai'\nimport type { GrokImageModel } from '../model-meta'\nimport type {\n GrokImageModelProviderOptionsByName,\n GrokImageModelSizeByName,\n GrokImageProviderOptions,\n} from '../image/image-provider-options'\nimport type { GrokClientConfig } from '../utils'\n\n/**\n * Configuration for Grok image adapter\n */\nexport interface GrokImageConfig extends GrokClientConfig {}\n\n/**\n * Grok Image Generation Adapter\n *\n * Tree-shakeable adapter for Grok image generation functionality.\n * Supports grok-2-image-1212 model.\n *\n * Features:\n * - Model-specific type-safe provider options\n * - Size validation per model\n * - Number of images validation\n */\nexport class GrokImageAdapter<\n TModel extends GrokImageModel,\n> extends BaseImageAdapter<\n TModel,\n GrokImageProviderOptions,\n GrokImageModelProviderOptionsByName,\n GrokImageModelSizeByName\n> {\n readonly kind = 'image' as const\n readonly name = 'grok' as const\n\n protected client: OpenAI\n\n constructor(config: GrokImageConfig, model: TModel) {\n super(model, {})\n this.client = new OpenAI(withGrokDefaults(config))\n }\n\n async generateImages(\n options: ImageGenerationOptions<GrokImageProviderOptions>,\n ): Promise<ImageGenerationResult> {\n const { model, prompt, numberOfImages, size, modelOptions } = options\n\n validatePrompt({ prompt, model })\n validateImageSize(model, size)\n validateNumberOfImages(model, numberOfImages)\n\n const request: OpenAI_SDK.Images.
|
|
1
|
+
{"version":3,"file":"image.js","sources":["../../../src/adapters/image.ts"],"sourcesContent":["import OpenAI from 'openai'\nimport { BaseImageAdapter } from '@tanstack/ai/adapters'\nimport { toRunErrorPayload } from '@tanstack/ai/adapter-internals'\nimport { generateId } from '@tanstack/ai-utils'\nimport { getGrokApiKeyFromEnv, withGrokDefaults } from '../utils/client'\nimport {\n validateImageSize,\n validateNumberOfImages,\n validatePrompt,\n} from '../image/image-provider-options'\nimport type {\n GeneratedImage,\n ImageGenerationOptions,\n ImageGenerationResult,\n} from '@tanstack/ai'\nimport type OpenAI_SDK from 'openai'\nimport type { GrokImageModel } from '../model-meta'\nimport type {\n GrokImageModelProviderOptionsByName,\n GrokImageModelSizeByName,\n GrokImageProviderOptions,\n} from '../image/image-provider-options'\nimport type { GrokClientConfig } from '../utils'\n\n/**\n * Configuration for Grok image adapter\n */\nexport interface GrokImageConfig extends GrokClientConfig {}\n\n/**\n * Grok Image Generation Adapter\n *\n * Tree-shakeable adapter for Grok image generation functionality.\n * Supports grok-2-image-1212 model.\n *\n * Features:\n * - Model-specific type-safe provider options\n * - Size validation per model\n * - Number of images validation\n */\nexport class GrokImageAdapter<\n TModel extends GrokImageModel,\n> extends BaseImageAdapter<\n TModel,\n GrokImageProviderOptions,\n GrokImageModelProviderOptionsByName,\n GrokImageModelSizeByName\n> {\n override readonly kind = 'image' as const\n readonly name = 'grok' as const\n\n protected client: OpenAI\n\n constructor(config: GrokImageConfig, model: TModel) {\n super(model, {})\n this.client = new OpenAI(withGrokDefaults(config))\n }\n\n async generateImages(\n options: ImageGenerationOptions<GrokImageProviderOptions>,\n ): Promise<ImageGenerationResult> {\n const { model, prompt, numberOfImages, size, modelOptions } = options\n\n validatePrompt({ prompt, model })\n validateImageSize(model, size)\n validateNumberOfImages(model, numberOfImages)\n\n const resolvedSize = size as OpenAI_SDK.Images.ImageGenerateParams['size']\n const request: OpenAI_SDK.Images.ImageGenerateParamsNonStreaming = {\n model,\n prompt,\n n: numberOfImages ?? 1,\n ...(resolvedSize !== undefined && { size: resolvedSize }),\n stream: false,\n ...modelOptions,\n }\n\n try {\n options.logger.request(\n `activity=image provider=${this.name} model=${model} n=${request.n ?? 1} size=${request.size ?? 'default'}`,\n { provider: this.name, model },\n )\n const response = await this.client.images.generate(request)\n\n const images: Array<GeneratedImage> = (response.data ?? []).flatMap(\n (item): Array<GeneratedImage> => {\n const revisedPrompt = item.revised_prompt\n if (item.b64_json) {\n return [\n {\n b64Json: item.b64_json,\n ...(revisedPrompt !== undefined && { revisedPrompt }),\n },\n ]\n }\n if (item.url) {\n return [\n {\n url: item.url,\n ...(revisedPrompt !== undefined && { revisedPrompt }),\n },\n ]\n }\n return []\n },\n )\n\n return {\n id: generateId(this.name),\n model,\n images,\n ...(response.usage && {\n usage: {\n inputTokens: response.usage.input_tokens,\n outputTokens: response.usage.output_tokens,\n totalTokens: response.usage.total_tokens,\n },\n }),\n }\n } catch (error: unknown) {\n options.logger.errors(`${this.name}.generateImages fatal`, {\n error: toRunErrorPayload(error, `${this.name}.generateImages failed`),\n source: `${this.name}.generateImages`,\n })\n throw error\n }\n }\n}\n\n/**\n * Creates a Grok image adapter with explicit API key.\n * Type resolution happens here at the call site.\n *\n * @param model - The model name (e.g., 'grok-2-image-1212')\n * @param apiKey - Your xAI API key\n * @param config - Optional additional configuration\n * @returns Configured Grok image adapter instance with resolved types\n *\n * @example\n * ```typescript\n * const adapter = createGrokImage('grok-2-image-1212', \"xai-...\");\n *\n * const result = await generateImage({\n * adapter,\n * prompt: 'A cute baby sea otter'\n * });\n * ```\n */\nexport function createGrokImage<TModel extends GrokImageModel>(\n model: TModel,\n apiKey: string,\n config?: Omit<GrokImageConfig, 'apiKey'>,\n): GrokImageAdapter<TModel> {\n return new GrokImageAdapter({ apiKey, ...config }, model)\n}\n\n/**\n * Creates a Grok image adapter with automatic API key detection from environment variables.\n * Type resolution happens here at the call site.\n *\n * Looks for `XAI_API_KEY` in:\n * - `process.env` (Node.js)\n * - `window.env` (Browser with injected env)\n *\n * @param model - The model name (e.g., 'grok-2-image-1212')\n * @param config - Optional configuration (excluding apiKey which is auto-detected)\n * @returns Configured Grok image adapter instance with resolved types\n * @throws Error if XAI_API_KEY is not found in environment\n *\n * @example\n * ```typescript\n * // Automatically uses XAI_API_KEY from environment\n * const adapter = grokImage('grok-2-image-1212');\n *\n * const result = await generateImage({\n * adapter,\n * prompt: 'A beautiful sunset over mountains'\n * });\n * ```\n */\nexport function grokImage<TModel extends GrokImageModel>(\n model: TModel,\n config?: Omit<GrokImageConfig, 'apiKey'>,\n): GrokImageAdapter<TModel> {\n const apiKey = getGrokApiKeyFromEnv()\n return createGrokImage(model, apiKey, config)\n}\n"],"names":[],"mappings":";;;;;;AAwCO,MAAM,yBAEH,iBAKR;AAAA,EACkB,OAAO;AAAA,EAChB,OAAO;AAAA,EAEN;AAAA,EAEV,YAAY,QAAyB,OAAe;AAClD,UAAM,OAAO,EAAE;AACf,SAAK,SAAS,IAAI,OAAO,iBAAiB,MAAM,CAAC;AAAA,EACnD;AAAA,EAEA,MAAM,eACJ,SACgC;AAChC,UAAM,EAAE,OAAO,QAAQ,gBAAgB,MAAM,iBAAiB;AAE9D,mBAAe,EAAE,OAAc,CAAC;AAChC,sBAAkB,OAAO,IAAI;AAC7B,2BAAuB,OAAO,cAAc;AAE5C,UAAM,eAAe;AACrB,UAAM,UAA6D;AAAA,MACjE;AAAA,MACA;AAAA,MACA,GAAG,kBAAkB;AAAA,MACrB,GAAI,iBAAiB,UAAa,EAAE,MAAM,aAAA;AAAA,MAC1C,QAAQ;AAAA,MACR,GAAG;AAAA,IAAA;AAGL,QAAI;AACF,cAAQ,OAAO;AAAA,QACb,2BAA2B,KAAK,IAAI,UAAU,KAAK,MAAM,QAAQ,KAAK,CAAC,SAAS,QAAQ,QAAQ,SAAS;AAAA,QACzG,EAAE,UAAU,KAAK,MAAM,MAAA;AAAA,MAAM;AAE/B,YAAM,WAAW,MAAM,KAAK,OAAO,OAAO,SAAS,OAAO;AAE1D,YAAM,UAAiC,SAAS,QAAQ,CAAA,GAAI;AAAA,QAC1D,CAAC,SAAgC;AAC/B,gBAAM,gBAAgB,KAAK;AAC3B,cAAI,KAAK,UAAU;AACjB,mBAAO;AAAA,cACL;AAAA,gBACE,SAAS,KAAK;AAAA,gBACd,GAAI,kBAAkB,UAAa,EAAE,cAAA;AAAA,cAAc;AAAA,YACrD;AAAA,UAEJ;AACA,cAAI,KAAK,KAAK;AACZ,mBAAO;AAAA,cACL;AAAA,gBACE,KAAK,KAAK;AAAA,gBACV,GAAI,kBAAkB,UAAa,EAAE,cAAA;AAAA,cAAc;AAAA,YACrD;AAAA,UAEJ;AACA,iBAAO,CAAA;AAAA,QACT;AAAA,MAAA;AAGF,aAAO;AAAA,QACL,IAAI,WAAW,KAAK,IAAI;AAAA,QACxB;AAAA,QACA;AAAA,QACA,GAAI,SAAS,SAAS;AAAA,UACpB,OAAO;AAAA,YACL,aAAa,SAAS,MAAM;AAAA,YAC5B,cAAc,SAAS,MAAM;AAAA,YAC7B,aAAa,SAAS,MAAM;AAAA,UAAA;AAAA,QAC9B;AAAA,MACF;AAAA,IAEJ,SAAS,OAAgB;AACvB,cAAQ,OAAO,OAAO,GAAG,KAAK,IAAI,yBAAyB;AAAA,QACzD,OAAO,kBAAkB,OAAO,GAAG,KAAK,IAAI,wBAAwB;AAAA,QACpE,QAAQ,GAAG,KAAK,IAAI;AAAA,MAAA,CACrB;AACD,YAAM;AAAA,IACR;AAAA,EACF;AACF;AAqBO,SAAS,gBACd,OACA,QACA,QAC0B;AAC1B,SAAO,IAAI,iBAAiB,EAAE,QAAQ,GAAG,OAAA,GAAU,KAAK;AAC1D;AA0BO,SAAS,UACd,OACA,QAC0B;AAC1B,QAAM,SAAS,qBAAA;AACf,SAAO,gBAAgB,OAAO,QAAQ,MAAM;AAC9C;"}
|
|
@@ -2,10 +2,10 @@ import OpenAI from "openai";
|
|
|
2
2
|
import { OpenAIBaseChatCompletionsTextAdapter } from "@tanstack/openai-base";
|
|
3
3
|
import { withGrokDefaults, getGrokApiKeyFromEnv } from "../utils/client.js";
|
|
4
4
|
class GrokTextAdapter extends OpenAIBaseChatCompletionsTextAdapter {
|
|
5
|
+
kind = "text";
|
|
6
|
+
name = "grok";
|
|
5
7
|
constructor(config, model) {
|
|
6
8
|
super(model, "grok", new OpenAI(withGrokDefaults(config)));
|
|
7
|
-
this.kind = "text";
|
|
8
|
-
this.name = "grok";
|
|
9
9
|
}
|
|
10
10
|
/**
|
|
11
11
|
* Surfaces xAI reasoning deltas on Grok reasoning models. The DeepSeek-style
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"text.js","sources":["../../../src/adapters/text.ts"],"sourcesContent":["import OpenAI from 'openai'\nimport { OpenAIBaseChatCompletionsTextAdapter } from '@tanstack/openai-base'\nimport { getGrokApiKeyFromEnv, withGrokDefaults } from '../utils/client'\nimport type {\n GROK_CHAT_MODELS,\n GrokChatModelToolCapabilitiesByName,\n ResolveInputModalities,\n ResolveProviderOptions,\n} from '../model-meta'\nimport type { Modality } from '@tanstack/ai'\nimport type { GrokMessageMetadataByModality } from '../message-types'\nimport type { GrokClientConfig } from '../utils'\n\n/**\n * Resolve tool capabilities for a specific Grok model.\n */\ntype ResolveToolCapabilities<TModel extends string> =\n TModel extends keyof GrokChatModelToolCapabilitiesByName\n ? NonNullable<GrokChatModelToolCapabilitiesByName[TModel]>\n : readonly []\n\n/**\n * Configuration for Grok text adapter\n */\nexport interface GrokTextConfig extends GrokClientConfig {}\n\n/**\n * Alias for TextProviderOptions for external use\n */\nexport type { ExternalTextProviderOptions as GrokTextProviderOptions } from '../text/text-provider-options'\n\n/**\n * Grok Text (Chat) Adapter\n *\n * Tree-shakeable adapter for Grok chat/text completion functionality.\n * Uses OpenAI-compatible Chat Completions API (not Responses API).\n *\n * Delegates implementation to {@link OpenAIBaseChatCompletionsTextAdapter}\n * from `@tanstack/openai-base` and threads Grok-specific tool-capability\n * typing through the 5th generic of the base class.\n */\nexport class GrokTextAdapter<\n TModel extends (typeof GROK_CHAT_MODELS)[number],\n TProviderOptions extends Record<string, any> = ResolveProviderOptions<TModel>,\n TInputModalities extends ReadonlyArray<Modality> =\n ResolveInputModalities<TModel>,\n TToolCapabilities extends ReadonlyArray<string> =\n ResolveToolCapabilities<TModel>,\n> extends OpenAIBaseChatCompletionsTextAdapter<\n TModel,\n TProviderOptions,\n TInputModalities,\n GrokMessageMetadataByModality,\n TToolCapabilities\n> {\n readonly kind = 'text' as const\n readonly name = 'grok' as const\n\n constructor(config: GrokTextConfig, model: TModel) {\n super(model, 'grok', new OpenAI(withGrokDefaults(config)))\n }\n\n /**\n * Surfaces xAI reasoning deltas on Grok reasoning models. The DeepSeek-style\n * convention puts the chain-of-thought on `delta.reasoning_content`; some\n * Grok variants also populate `delta.reasoning`. Reading both keeps\n * reasoning flowing through the base's REASONING_* lifecycle for both\n * `chatStream` and `structuredOutputStream`.\n */\n protected override extractReasoning(\n chunk: OpenAI.Chat.Completions.ChatCompletionChunk,\n ): { text: string } | undefined {\n const delta = chunk.choices[0]?.delta as\n | { reasoning?: unknown; reasoning_content?: unknown }\n | undefined\n const raw = delta?.reasoning_content ?? delta?.reasoning\n if (typeof raw === 'string' && raw.length > 0) {\n return { text: raw }\n }\n return undefined\n }\n}\n\n/**\n * Creates a Grok text adapter with explicit API key.\n * Type resolution happens here at the call site.\n *\n * @param model - The model name (e.g., 'grok-3', 'grok-4')\n * @param apiKey - Your xAI API key\n * @param config - Optional additional configuration\n * @returns Configured Grok text adapter instance with resolved types\n *\n * @example\n * ```typescript\n * const adapter = createGrokText('grok-3', \"xai-...\");\n * // adapter has type-safe providerOptions for grok-3\n * ```\n */\nexport function createGrokText<\n TModel extends (typeof GROK_CHAT_MODELS)[number],\n>(\n model: TModel,\n apiKey: string,\n config?: Omit<GrokTextConfig, 'apiKey'>,\n): GrokTextAdapter<TModel> {\n return new GrokTextAdapter({ apiKey, ...config }, model)\n}\n\n/**\n * Creates a Grok text adapter with automatic API key detection from environment variables.\n * Type resolution happens here at the call site.\n *\n * Looks for `XAI_API_KEY` in:\n * - `process.env` (Node.js)\n * - `window.env` (Browser with injected env)\n *\n * @param model - The model name (e.g., 'grok-3', 'grok-4')\n * @param config - Optional configuration (excluding apiKey which is auto-detected)\n * @returns Configured Grok text adapter instance with resolved types\n * @throws Error if XAI_API_KEY is not found in environment\n *\n * @example\n * ```typescript\n * // Automatically uses XAI_API_KEY from environment\n * const adapter = grokText('grok-3');\n *\n * const stream = chat({\n * adapter,\n * messages: [{ role: \"user\", content: \"Hello!\" }]\n * });\n * ```\n */\nexport function grokText<TModel extends (typeof GROK_CHAT_MODELS)[number]>(\n model: TModel,\n config?: Omit<GrokTextConfig, 'apiKey'>,\n): GrokTextAdapter<TModel> {\n const apiKey = getGrokApiKeyFromEnv()\n return createGrokText(model, apiKey, config)\n}\n"],"names":[],"mappings":";;;AAyCO,MAAM,wBAOH,qCAMR;AAAA,
|
|
1
|
+
{"version":3,"file":"text.js","sources":["../../../src/adapters/text.ts"],"sourcesContent":["import OpenAI from 'openai'\nimport { OpenAIBaseChatCompletionsTextAdapter } from '@tanstack/openai-base'\nimport { getGrokApiKeyFromEnv, withGrokDefaults } from '../utils/client'\nimport type {\n GROK_CHAT_MODELS,\n GrokChatModelToolCapabilitiesByName,\n ResolveInputModalities,\n ResolveProviderOptions,\n} from '../model-meta'\nimport type { Modality } from '@tanstack/ai'\nimport type { GrokMessageMetadataByModality } from '../message-types'\nimport type { GrokClientConfig } from '../utils'\n\n/**\n * Resolve tool capabilities for a specific Grok model.\n */\ntype ResolveToolCapabilities<TModel extends string> =\n TModel extends keyof GrokChatModelToolCapabilitiesByName\n ? NonNullable<GrokChatModelToolCapabilitiesByName[TModel]>\n : readonly []\n\n/**\n * Configuration for Grok text adapter\n */\nexport interface GrokTextConfig extends GrokClientConfig {}\n\n/**\n * Alias for TextProviderOptions for external use\n */\nexport type { ExternalTextProviderOptions as GrokTextProviderOptions } from '../text/text-provider-options'\n\n/**\n * Grok Text (Chat) Adapter\n *\n * Tree-shakeable adapter for Grok chat/text completion functionality.\n * Uses OpenAI-compatible Chat Completions API (not Responses API).\n *\n * Delegates implementation to {@link OpenAIBaseChatCompletionsTextAdapter}\n * from `@tanstack/openai-base` and threads Grok-specific tool-capability\n * typing through the 5th generic of the base class.\n */\nexport class GrokTextAdapter<\n TModel extends (typeof GROK_CHAT_MODELS)[number],\n TProviderOptions extends Record<string, any> = ResolveProviderOptions<TModel>,\n TInputModalities extends ReadonlyArray<Modality> =\n ResolveInputModalities<TModel>,\n TToolCapabilities extends ReadonlyArray<string> =\n ResolveToolCapabilities<TModel>,\n> extends OpenAIBaseChatCompletionsTextAdapter<\n TModel,\n TProviderOptions,\n TInputModalities,\n GrokMessageMetadataByModality,\n TToolCapabilities\n> {\n override readonly kind = 'text' as const\n override readonly name = 'grok' as const\n\n constructor(config: GrokTextConfig, model: TModel) {\n super(model, 'grok', new OpenAI(withGrokDefaults(config)))\n }\n\n /**\n * Surfaces xAI reasoning deltas on Grok reasoning models. The DeepSeek-style\n * convention puts the chain-of-thought on `delta.reasoning_content`; some\n * Grok variants also populate `delta.reasoning`. Reading both keeps\n * reasoning flowing through the base's REASONING_* lifecycle for both\n * `chatStream` and `structuredOutputStream`.\n */\n protected override extractReasoning(\n chunk: OpenAI.Chat.Completions.ChatCompletionChunk,\n ): { text: string } | undefined {\n const delta = chunk.choices[0]?.delta as\n | { reasoning?: unknown; reasoning_content?: unknown }\n | undefined\n const raw = delta?.reasoning_content ?? delta?.reasoning\n if (typeof raw === 'string' && raw.length > 0) {\n return { text: raw }\n }\n return undefined\n }\n}\n\n/**\n * Creates a Grok text adapter with explicit API key.\n * Type resolution happens here at the call site.\n *\n * @param model - The model name (e.g., 'grok-3', 'grok-4')\n * @param apiKey - Your xAI API key\n * @param config - Optional additional configuration\n * @returns Configured Grok text adapter instance with resolved types\n *\n * @example\n * ```typescript\n * const adapter = createGrokText('grok-3', \"xai-...\");\n * // adapter has type-safe providerOptions for grok-3\n * ```\n */\nexport function createGrokText<\n TModel extends (typeof GROK_CHAT_MODELS)[number],\n>(\n model: TModel,\n apiKey: string,\n config?: Omit<GrokTextConfig, 'apiKey'>,\n): GrokTextAdapter<TModel> {\n return new GrokTextAdapter({ apiKey, ...config }, model)\n}\n\n/**\n * Creates a Grok text adapter with automatic API key detection from environment variables.\n * Type resolution happens here at the call site.\n *\n * Looks for `XAI_API_KEY` in:\n * - `process.env` (Node.js)\n * - `window.env` (Browser with injected env)\n *\n * @param model - The model name (e.g., 'grok-3', 'grok-4')\n * @param config - Optional configuration (excluding apiKey which is auto-detected)\n * @returns Configured Grok text adapter instance with resolved types\n * @throws Error if XAI_API_KEY is not found in environment\n *\n * @example\n * ```typescript\n * // Automatically uses XAI_API_KEY from environment\n * const adapter = grokText('grok-3');\n *\n * const stream = chat({\n * adapter,\n * messages: [{ role: \"user\", content: \"Hello!\" }]\n * });\n * ```\n */\nexport function grokText<TModel extends (typeof GROK_CHAT_MODELS)[number]>(\n model: TModel,\n config?: Omit<GrokTextConfig, 'apiKey'>,\n): GrokTextAdapter<TModel> {\n const apiKey = getGrokApiKeyFromEnv()\n return createGrokText(model, apiKey, config)\n}\n"],"names":[],"mappings":";;;AAyCO,MAAM,wBAOH,qCAMR;AAAA,EACkB,OAAO;AAAA,EACP,OAAO;AAAA,EAEzB,YAAY,QAAwB,OAAe;AACjD,UAAM,OAAO,QAAQ,IAAI,OAAO,iBAAiB,MAAM,CAAC,CAAC;AAAA,EAC3D;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASmB,iBACjB,OAC8B;AAC9B,UAAM,QAAQ,MAAM,QAAQ,CAAC,GAAG;AAGhC,UAAM,MAAM,OAAO,qBAAqB,OAAO;AAC/C,QAAI,OAAO,QAAQ,YAAY,IAAI,SAAS,GAAG;AAC7C,aAAO,EAAE,MAAM,IAAA;AAAA,IACjB;AACA,WAAO;AAAA,EACT;AACF;AAiBO,SAAS,eAGd,OACA,QACA,QACyB;AACzB,SAAO,IAAI,gBAAgB,EAAE,QAAQ,GAAG,OAAA,GAAU,KAAK;AACzD;AA0BO,SAAS,SACd,OACA,QACyB;AACzB,QAAM,SAAS,qBAAA;AACf,SAAO,eAAe,OAAO,QAAQ,MAAM;AAC7C;"}
|
|
@@ -5,9 +5,12 @@ import "@tanstack/openai-base";
|
|
|
5
5
|
import { toAudioFile } from "../utils/audio.js";
|
|
6
6
|
const DEFAULT_GROK_BASE_URL = "https://api.x.ai/v1";
|
|
7
7
|
class GrokTranscriptionAdapter extends BaseTranscriptionAdapter {
|
|
8
|
+
name = "grok";
|
|
9
|
+
apiKey;
|
|
10
|
+
baseURL;
|
|
11
|
+
defaultHeaders;
|
|
8
12
|
constructor(config, model) {
|
|
9
13
|
super(model, config);
|
|
10
|
-
this.name = "grok";
|
|
11
14
|
this.apiKey = config.apiKey;
|
|
12
15
|
this.baseURL = (config.baseURL ?? DEFAULT_GROK_BASE_URL).replace(/\/+$/, "");
|
|
13
16
|
this.defaultHeaders = config.defaultHeaders ?? {};
|
|
@@ -50,13 +53,14 @@ class GrokTranscriptionAdapter extends BaseTranscriptionAdapter {
|
|
|
50
53
|
return tw;
|
|
51
54
|
}
|
|
52
55
|
);
|
|
56
|
+
const resolvedLanguage = data.language ?? language;
|
|
53
57
|
return {
|
|
54
58
|
id: generateId(this.name),
|
|
55
59
|
model,
|
|
56
60
|
text: data.text,
|
|
57
|
-
language:
|
|
61
|
+
...resolvedLanguage !== void 0 && { language: resolvedLanguage },
|
|
58
62
|
duration: data.duration,
|
|
59
|
-
words
|
|
63
|
+
...words !== void 0 && { words }
|
|
60
64
|
};
|
|
61
65
|
} catch (error) {
|
|
62
66
|
logger.errors("grok.transcribe fatal", {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"transcription.js","sources":["../../../src/adapters/transcription.ts"],"sourcesContent":["import { BaseTranscriptionAdapter } from '@tanstack/ai/adapters'\nimport { generateId, getGrokApiKeyFromEnv, toAudioFile } from '../utils'\nimport type {\n TranscriptionOptions,\n TranscriptionResult,\n TranscriptionWord,\n} from '@tanstack/ai'\nimport type { GrokTranscriptionModel } from '../model-meta'\nimport type { GrokTranscriptionProviderOptions } from '../audio/transcription-provider-options'\n\n/**\n * Grok-specific extension of `TranscriptionWord` that surfaces the extra\n * fields xAI returns when diarization / confidence are enabled. The base\n * cross-provider `TranscriptionWord` contract doesn't include these, so\n * callers who know they're using Grok can narrow with:\n *\n * ```ts\n * const words = result.words as Array<GrokTranscriptionWord> | undefined\n * ```\n */\nexport interface GrokTranscriptionWord extends TranscriptionWord {\n /** Model confidence for the word, when xAI returns one. */\n confidence?: number\n /** Speaker index, populated when `modelOptions.diarize === true`. */\n speaker?: number\n}\n\nconst DEFAULT_GROK_BASE_URL = 'https://api.x.ai/v1'\n\n/**\n * Configuration for the Grok transcription adapter.\n *\n * Uses direct `fetch` rather than the OpenAI SDK because xAI's `/v1/stt`\n * endpoint is not OpenAI-compatible.\n */\nexport interface GrokTranscriptionConfig {\n apiKey: string\n baseURL?: string\n /** Additional headers to merge into every request (e.g., test IDs). */\n defaultHeaders?: Record<string, string>\n}\n\n/**\n * xAI STT response shape from `POST /v1/stt`.\n * Grok returns word-level timestamps only; no segment array.\n */\ninterface GrokSTTWord {\n text: string\n start: number\n end: number\n confidence?: number\n speaker?: number\n}\n\ninterface GrokSTTResponse {\n text: string\n language?: string\n duration?: number\n words?: Array<GrokSTTWord>\n channels?: Array<unknown>\n}\n\n/**\n * Grok Speech-to-Text Adapter.\n *\n * Talks to `POST {baseURL}/stt` per\n * https://docs.x.ai/developers/rest-api-reference/inference/voice\n */\nexport class GrokTranscriptionAdapter<\n TModel extends GrokTranscriptionModel,\n> extends BaseTranscriptionAdapter<TModel, GrokTranscriptionProviderOptions> {\n readonly name = 'grok' as const\n\n private readonly apiKey: string\n private readonly baseURL: string\n private readonly defaultHeaders: Record<string, string>\n\n constructor(config: GrokTranscriptionConfig, model: TModel) {\n super(model, config)\n this.apiKey = config.apiKey\n this.baseURL = (config.baseURL ?? DEFAULT_GROK_BASE_URL).replace(/\\/+$/, '')\n this.defaultHeaders = config.defaultHeaders ?? {}\n }\n\n async transcribe(\n options: TranscriptionOptions<GrokTranscriptionProviderOptions>,\n ): Promise<TranscriptionResult> {\n const { logger } = options\n const { model, audio, language, modelOptions } = options\n\n logger.request(\n `activity=generateTranscription provider=grok model=${model}`,\n { provider: 'grok', model },\n )\n\n const file = toAudioFile(audio, modelOptions?.audio_format)\n const form = buildTranscriptionFormData({ file, language, modelOptions })\n\n try {\n const response = await fetch(`${this.baseURL}/stt`, {\n method: 'POST',\n headers: {\n // `defaultHeaders` first so Authorization always wins.\n ...this.defaultHeaders,\n Authorization: `Bearer ${this.apiKey}`,\n },\n body: form,\n })\n\n if (!response.ok) {\n const errorText = await response.text()\n throw new Error(\n `Grok transcription request failed: ${response.status} ${errorText}`,\n )\n }\n\n const data = (await response.json()) as GrokSTTResponse\n\n const words: Array<TranscriptionWord> | undefined = data.words?.map(\n (w) => {\n // Construct a GrokTranscriptionWord so that `confidence` and\n // `speaker` (when xAI returns them under `diarize` / confidence\n // mode) are preserved on the result. The returned array is typed\n // as `Array<TranscriptionWord>` per the cross-provider contract;\n // callers who want the extras narrow via `as Array<GrokTranscriptionWord>`.\n const tw: GrokTranscriptionWord = {\n word: w.text,\n start: w.start,\n end: w.end,\n }\n if (w.confidence !== undefined) tw.confidence = w.confidence\n if (w.speaker !== undefined) tw.speaker = w.speaker\n return tw\n },\n )\n\n return {\n id: generateId(this.name),\n model,\n text: data.text,\n language: data.language ?? language,\n duration: data.duration,\n words,\n }\n } catch (error) {\n logger.errors('grok.transcribe fatal', {\n error,\n source: 'grok.transcribe',\n })\n throw error\n }\n }\n}\n\n/**\n * Build the multipart/form-data body for `POST /v1/stt`, coercing SDK-level\n * model options into xAI's wire format (booleans as `'true'`/`'false'`\n * strings, numeric fields stringified, etc.).\n *\n * Wire-field mapping:\n * - `modelOptions.inverse_text_normalization` → `format` (xAI's chosen\n * wire-field name for the ITN boolean; the SDK surfaces it under the\n * clearer `inverse_text_normalization` key).\n * - `modelOptions.audio_format`, `sample_rate`, `multichannel`, `channels`,\n * `diarize` map to same-named form fields.\n */\nexport function buildTranscriptionFormData(options: {\n file: File\n language: string | undefined\n modelOptions: GrokTranscriptionProviderOptions | undefined\n}): FormData {\n const { file, language, modelOptions } = options\n const form = new FormData()\n form.set('file', file)\n if (language) form.set('language', language)\n if (modelOptions?.audio_format !== undefined) {\n form.set('audio_format', modelOptions.audio_format)\n }\n if (modelOptions?.sample_rate !== undefined) {\n form.set('sample_rate', String(modelOptions.sample_rate))\n }\n if (modelOptions?.inverse_text_normalization !== undefined) {\n form.set(\n 'format',\n modelOptions.inverse_text_normalization ? 'true' : 'false',\n )\n }\n if (modelOptions?.multichannel !== undefined) {\n form.set('multichannel', modelOptions.multichannel ? 'true' : 'false')\n }\n if (modelOptions?.channels !== undefined) {\n form.set('channels', String(modelOptions.channels))\n }\n if (modelOptions?.diarize !== undefined) {\n form.set('diarize', modelOptions.diarize ? 'true' : 'false')\n }\n return form\n}\n\n/**\n * Creates a Grok transcription adapter with an explicit API key.\n *\n * @example\n * ```typescript\n * const adapter = createGrokTranscription('grok-stt', 'xai-...')\n * const result = await generateTranscription({\n * adapter,\n * audio: audioFile,\n * language: 'en',\n * })\n * ```\n */\nexport function createGrokTranscription<TModel extends GrokTranscriptionModel>(\n model: TModel,\n apiKey: string,\n config?: Omit<GrokTranscriptionConfig, 'apiKey'>,\n): GrokTranscriptionAdapter<TModel> {\n return new GrokTranscriptionAdapter({ apiKey, ...config }, model)\n}\n\n/**\n * Creates a Grok transcription adapter, reading the API key from\n * `XAI_API_KEY` in the environment.\n *\n * @throws Error if `XAI_API_KEY` is not set.\n */\nexport function grokTranscription<TModel extends GrokTranscriptionModel>(\n model: TModel,\n config?: Omit<GrokTranscriptionConfig, 'apiKey'>,\n): GrokTranscriptionAdapter<TModel> {\n const apiKey = getGrokApiKeyFromEnv()\n return createGrokTranscription(model, apiKey, config)\n}\n"],"names":[],"mappings":";;;;;AA2BA,MAAM,wBAAwB;AAyCvB,MAAM,iCAEH,yBAAmE;AAAA,EAO3E,YAAY,QAAiC,OAAe;AAC1D,UAAM,OAAO,MAAM;AAPrB,SAAS,OAAO;AAQd,SAAK,SAAS,OAAO;AACrB,SAAK,WAAW,OAAO,WAAW,uBAAuB,QAAQ,QAAQ,EAAE;AAC3E,SAAK,iBAAiB,OAAO,kBAAkB,CAAA;AAAA,EACjD;AAAA,EAEA,MAAM,WACJ,SAC8B;AAC9B,UAAM,EAAE,WAAW;AACnB,UAAM,EAAE,OAAO,OAAO,UAAU,iBAAiB;AAEjD,WAAO;AAAA,MACL,sDAAsD,KAAK;AAAA,MAC3D,EAAE,UAAU,QAAQ,MAAA;AAAA,IAAM;AAG5B,UAAM,OAAO,YAAY,OAAO,cAAc,YAAY;AAC1D,UAAM,OAAO,2BAA2B,EAAE,MAAM,UAAU,cAAc;AAExE,QAAI;AACF,YAAM,WAAW,MAAM,MAAM,GAAG,KAAK,OAAO,QAAQ;AAAA,QAClD,QAAQ;AAAA,QACR,SAAS;AAAA;AAAA,UAEP,GAAG,KAAK;AAAA,UACR,eAAe,UAAU,KAAK,MAAM;AAAA,QAAA;AAAA,QAEtC,MAAM;AAAA,MAAA,CACP;AAED,UAAI,CAAC,SAAS,IAAI;AAChB,cAAM,YAAY,MAAM,SAAS,KAAA;AACjC,cAAM,IAAI;AAAA,UACR,sCAAsC,SAAS,MAAM,IAAI,SAAS;AAAA,QAAA;AAAA,MAEtE;AAEA,YAAM,OAAQ,MAAM,SAAS,KAAA;AAE7B,YAAM,QAA8C,KAAK,OAAO;AAAA,QAC9D,CAAC,MAAM;AAML,gBAAM,KAA4B;AAAA,YAChC,MAAM,EAAE;AAAA,YACR,OAAO,EAAE;AAAA,YACT,KAAK,EAAE;AAAA,UAAA;AAET,cAAI,EAAE,eAAe,OAAW,IAAG,aAAa,EAAE;AAClD,cAAI,EAAE,YAAY,OAAW,IAAG,UAAU,EAAE;AAC5C,iBAAO;AAAA,QACT;AAAA,MAAA;AAGF,aAAO;AAAA,QACL,IAAI,WAAW,KAAK,IAAI;AAAA,QACxB;AAAA,QACA,MAAM,KAAK;AAAA,QACX,UAAU,KAAK,YAAY;AAAA,QAC3B,UAAU,KAAK;AAAA,QACf;AAAA,MAAA;AAAA,IAEJ,SAAS,OAAO;AACd,aAAO,OAAO,yBAAyB;AAAA,QACrC;AAAA,QACA,QAAQ;AAAA,MAAA,CACT;AACD,YAAM;AAAA,IACR;AAAA,EACF;AACF;AAcO,SAAS,2BAA2B,SAI9B;AACX,QAAM,EAAE,MAAM,UAAU,aAAA,IAAiB;AACzC,QAAM,OAAO,IAAI,SAAA;AACjB,OAAK,IAAI,QAAQ,IAAI;AACrB,MAAI,SAAU,MAAK,IAAI,YAAY,QAAQ;AAC3C,MAAI,cAAc,iBAAiB,QAAW;AAC5C,SAAK,IAAI,gBAAgB,aAAa,YAAY;AAAA,EACpD;AACA,MAAI,cAAc,gBAAgB,QAAW;AAC3C,SAAK,IAAI,eAAe,OAAO,aAAa,WAAW,CAAC;AAAA,EAC1D;AACA,MAAI,cAAc,+BAA+B,QAAW;AAC1D,SAAK;AAAA,MACH;AAAA,MACA,aAAa,6BAA6B,SAAS;AAAA,IAAA;AAAA,EAEvD;AACA,MAAI,cAAc,iBAAiB,QAAW;AAC5C,SAAK,IAAI,gBAAgB,aAAa,eAAe,SAAS,OAAO;AAAA,EACvE;AACA,MAAI,cAAc,aAAa,QAAW;AACxC,SAAK,IAAI,YAAY,OAAO,aAAa,QAAQ,CAAC;AAAA,EACpD;AACA,MAAI,cAAc,YAAY,QAAW;AACvC,SAAK,IAAI,WAAW,aAAa,UAAU,SAAS,OAAO;AAAA,EAC7D;AACA,SAAO;AACT;AAeO,SAAS,wBACd,OACA,QACA,QACkC;AAClC,SAAO,IAAI,yBAAyB,EAAE,QAAQ,GAAG,OAAA,GAAU,KAAK;AAClE;AAQO,SAAS,kBACd,OACA,QACkC;AAClC,QAAM,SAAS,qBAAA;AACf,SAAO,wBAAwB,OAAO,QAAQ,MAAM;AACtD;"}
|
|
1
|
+
{"version":3,"file":"transcription.js","sources":["../../../src/adapters/transcription.ts"],"sourcesContent":["import { BaseTranscriptionAdapter } from '@tanstack/ai/adapters'\nimport { generateId, getGrokApiKeyFromEnv, toAudioFile } from '../utils'\nimport type {\n TranscriptionOptions,\n TranscriptionResult,\n TranscriptionWord,\n} from '@tanstack/ai'\nimport type { GrokTranscriptionModel } from '../model-meta'\nimport type { GrokTranscriptionProviderOptions } from '../audio/transcription-provider-options'\n\n/**\n * Grok-specific extension of `TranscriptionWord` that surfaces the extra\n * fields xAI returns when diarization / confidence are enabled. The base\n * cross-provider `TranscriptionWord` contract doesn't include these, so\n * callers who know they're using Grok can narrow with:\n *\n * ```ts\n * const words = result.words as Array<GrokTranscriptionWord> | undefined\n * ```\n */\nexport interface GrokTranscriptionWord extends TranscriptionWord {\n /** Model confidence for the word, when xAI returns one. */\n confidence?: number\n /** Speaker index, populated when `modelOptions.diarize === true`. */\n speaker?: number\n}\n\nconst DEFAULT_GROK_BASE_URL = 'https://api.x.ai/v1'\n\n/**\n * Configuration for the Grok transcription adapter.\n *\n * Uses direct `fetch` rather than the OpenAI SDK because xAI's `/v1/stt`\n * endpoint is not OpenAI-compatible.\n */\nexport interface GrokTranscriptionConfig {\n apiKey: string\n baseURL?: string\n /** Additional headers to merge into every request (e.g., test IDs). */\n defaultHeaders?: Record<string, string>\n}\n\n/**\n * xAI STT response shape from `POST /v1/stt`.\n * Grok returns word-level timestamps only; no segment array.\n */\ninterface GrokSTTWord {\n text: string\n start: number\n end: number\n confidence?: number\n speaker?: number\n}\n\ninterface GrokSTTResponse {\n text: string\n language?: string\n duration?: number\n words?: Array<GrokSTTWord>\n channels?: Array<unknown>\n}\n\n/**\n * Grok Speech-to-Text Adapter.\n *\n * Talks to `POST {baseURL}/stt` per\n * https://docs.x.ai/developers/rest-api-reference/inference/voice\n */\nexport class GrokTranscriptionAdapter<\n TModel extends GrokTranscriptionModel,\n> extends BaseTranscriptionAdapter<TModel, GrokTranscriptionProviderOptions> {\n readonly name = 'grok' as const\n\n private readonly apiKey: string\n private readonly baseURL: string\n private readonly defaultHeaders: Record<string, string>\n\n constructor(config: GrokTranscriptionConfig, model: TModel) {\n super(model, config)\n this.apiKey = config.apiKey\n this.baseURL = (config.baseURL ?? DEFAULT_GROK_BASE_URL).replace(/\\/+$/, '')\n this.defaultHeaders = config.defaultHeaders ?? {}\n }\n\n async transcribe(\n options: TranscriptionOptions<GrokTranscriptionProviderOptions>,\n ): Promise<TranscriptionResult> {\n const { logger } = options\n const { model, audio, language, modelOptions } = options\n\n logger.request(\n `activity=generateTranscription provider=grok model=${model}`,\n { provider: 'grok', model },\n )\n\n const file = toAudioFile(audio, modelOptions?.audio_format)\n const form = buildTranscriptionFormData({ file, language, modelOptions })\n\n try {\n const response = await fetch(`${this.baseURL}/stt`, {\n method: 'POST',\n headers: {\n // `defaultHeaders` first so Authorization always wins.\n ...this.defaultHeaders,\n Authorization: `Bearer ${this.apiKey}`,\n },\n body: form,\n })\n\n if (!response.ok) {\n const errorText = await response.text()\n throw new Error(\n `Grok transcription request failed: ${response.status} ${errorText}`,\n )\n }\n\n const data = (await response.json()) as GrokSTTResponse\n\n const words: Array<TranscriptionWord> | undefined = data.words?.map(\n (w) => {\n // Construct a GrokTranscriptionWord so that `confidence` and\n // `speaker` (when xAI returns them under `diarize` / confidence\n // mode) are preserved on the result. The returned array is typed\n // as `Array<TranscriptionWord>` per the cross-provider contract;\n // callers who want the extras narrow via `as Array<GrokTranscriptionWord>`.\n const tw: GrokTranscriptionWord = {\n word: w.text,\n start: w.start,\n end: w.end,\n }\n if (w.confidence !== undefined) tw.confidence = w.confidence\n if (w.speaker !== undefined) tw.speaker = w.speaker\n return tw\n },\n )\n\n const resolvedLanguage = data.language ?? language\n return {\n id: generateId(this.name),\n model,\n text: data.text,\n ...(resolvedLanguage !== undefined && { language: resolvedLanguage }),\n duration: data.duration,\n ...(words !== undefined && { words }),\n }\n } catch (error) {\n logger.errors('grok.transcribe fatal', {\n error,\n source: 'grok.transcribe',\n })\n throw error\n }\n }\n}\n\n/**\n * Build the multipart/form-data body for `POST /v1/stt`, coercing SDK-level\n * model options into xAI's wire format (booleans as `'true'`/`'false'`\n * strings, numeric fields stringified, etc.).\n *\n * Wire-field mapping:\n * - `modelOptions.inverse_text_normalization` → `format` (xAI's chosen\n * wire-field name for the ITN boolean; the SDK surfaces it under the\n * clearer `inverse_text_normalization` key).\n * - `modelOptions.audio_format`, `sample_rate`, `multichannel`, `channels`,\n * `diarize` map to same-named form fields.\n */\nexport function buildTranscriptionFormData(options: {\n file: File\n language: string | undefined\n modelOptions: GrokTranscriptionProviderOptions | undefined\n}): FormData {\n const { file, language, modelOptions } = options\n const form = new FormData()\n form.set('file', file)\n if (language) form.set('language', language)\n if (modelOptions?.audio_format !== undefined) {\n form.set('audio_format', modelOptions.audio_format)\n }\n if (modelOptions?.sample_rate !== undefined) {\n form.set('sample_rate', String(modelOptions.sample_rate))\n }\n if (modelOptions?.inverse_text_normalization !== undefined) {\n form.set(\n 'format',\n modelOptions.inverse_text_normalization ? 'true' : 'false',\n )\n }\n if (modelOptions?.multichannel !== undefined) {\n form.set('multichannel', modelOptions.multichannel ? 'true' : 'false')\n }\n if (modelOptions?.channels !== undefined) {\n form.set('channels', String(modelOptions.channels))\n }\n if (modelOptions?.diarize !== undefined) {\n form.set('diarize', modelOptions.diarize ? 'true' : 'false')\n }\n return form\n}\n\n/**\n * Creates a Grok transcription adapter with an explicit API key.\n *\n * @example\n * ```typescript\n * const adapter = createGrokTranscription('grok-stt', 'xai-...')\n * const result = await generateTranscription({\n * adapter,\n * audio: audioFile,\n * language: 'en',\n * })\n * ```\n */\nexport function createGrokTranscription<TModel extends GrokTranscriptionModel>(\n model: TModel,\n apiKey: string,\n config?: Omit<GrokTranscriptionConfig, 'apiKey'>,\n): GrokTranscriptionAdapter<TModel> {\n return new GrokTranscriptionAdapter({ apiKey, ...config }, model)\n}\n\n/**\n * Creates a Grok transcription adapter, reading the API key from\n * `XAI_API_KEY` in the environment.\n *\n * @throws Error if `XAI_API_KEY` is not set.\n */\nexport function grokTranscription<TModel extends GrokTranscriptionModel>(\n model: TModel,\n config?: Omit<GrokTranscriptionConfig, 'apiKey'>,\n): GrokTranscriptionAdapter<TModel> {\n const apiKey = getGrokApiKeyFromEnv()\n return createGrokTranscription(model, apiKey, config)\n}\n"],"names":[],"mappings":";;;;;AA2BA,MAAM,wBAAwB;AAyCvB,MAAM,iCAEH,yBAAmE;AAAA,EAClE,OAAO;AAAA,EAEC;AAAA,EACA;AAAA,EACA;AAAA,EAEjB,YAAY,QAAiC,OAAe;AAC1D,UAAM,OAAO,MAAM;AACnB,SAAK,SAAS,OAAO;AACrB,SAAK,WAAW,OAAO,WAAW,uBAAuB,QAAQ,QAAQ,EAAE;AAC3E,SAAK,iBAAiB,OAAO,kBAAkB,CAAA;AAAA,EACjD;AAAA,EAEA,MAAM,WACJ,SAC8B;AAC9B,UAAM,EAAE,WAAW;AACnB,UAAM,EAAE,OAAO,OAAO,UAAU,iBAAiB;AAEjD,WAAO;AAAA,MACL,sDAAsD,KAAK;AAAA,MAC3D,EAAE,UAAU,QAAQ,MAAA;AAAA,IAAM;AAG5B,UAAM,OAAO,YAAY,OAAO,cAAc,YAAY;AAC1D,UAAM,OAAO,2BAA2B,EAAE,MAAM,UAAU,cAAc;AAExE,QAAI;AACF,YAAM,WAAW,MAAM,MAAM,GAAG,KAAK,OAAO,QAAQ;AAAA,QAClD,QAAQ;AAAA,QACR,SAAS;AAAA;AAAA,UAEP,GAAG,KAAK;AAAA,UACR,eAAe,UAAU,KAAK,MAAM;AAAA,QAAA;AAAA,QAEtC,MAAM;AAAA,MAAA,CACP;AAED,UAAI,CAAC,SAAS,IAAI;AAChB,cAAM,YAAY,MAAM,SAAS,KAAA;AACjC,cAAM,IAAI;AAAA,UACR,sCAAsC,SAAS,MAAM,IAAI,SAAS;AAAA,QAAA;AAAA,MAEtE;AAEA,YAAM,OAAQ,MAAM,SAAS,KAAA;AAE7B,YAAM,QAA8C,KAAK,OAAO;AAAA,QAC9D,CAAC,MAAM;AAML,gBAAM,KAA4B;AAAA,YAChC,MAAM,EAAE;AAAA,YACR,OAAO,EAAE;AAAA,YACT,KAAK,EAAE;AAAA,UAAA;AAET,cAAI,EAAE,eAAe,OAAW,IAAG,aAAa,EAAE;AAClD,cAAI,EAAE,YAAY,OAAW,IAAG,UAAU,EAAE;AAC5C,iBAAO;AAAA,QACT;AAAA,MAAA;AAGF,YAAM,mBAAmB,KAAK,YAAY;AAC1C,aAAO;AAAA,QACL,IAAI,WAAW,KAAK,IAAI;AAAA,QACxB;AAAA,QACA,MAAM,KAAK;AAAA,QACX,GAAI,qBAAqB,UAAa,EAAE,UAAU,iBAAA;AAAA,QAClD,UAAU,KAAK;AAAA,QACf,GAAI,UAAU,UAAa,EAAE,MAAA;AAAA,MAAM;AAAA,IAEvC,SAAS,OAAO;AACd,aAAO,OAAO,yBAAyB;AAAA,QACrC;AAAA,QACA,QAAQ;AAAA,MAAA,CACT;AACD,YAAM;AAAA,IACR;AAAA,EACF;AACF;AAcO,SAAS,2BAA2B,SAI9B;AACX,QAAM,EAAE,MAAM,UAAU,aAAA,IAAiB;AACzC,QAAM,OAAO,IAAI,SAAA;AACjB,OAAK,IAAI,QAAQ,IAAI;AACrB,MAAI,SAAU,MAAK,IAAI,YAAY,QAAQ;AAC3C,MAAI,cAAc,iBAAiB,QAAW;AAC5C,SAAK,IAAI,gBAAgB,aAAa,YAAY;AAAA,EACpD;AACA,MAAI,cAAc,gBAAgB,QAAW;AAC3C,SAAK,IAAI,eAAe,OAAO,aAAa,WAAW,CAAC;AAAA,EAC1D;AACA,MAAI,cAAc,+BAA+B,QAAW;AAC1D,SAAK;AAAA,MACH;AAAA,MACA,aAAa,6BAA6B,SAAS;AAAA,IAAA;AAAA,EAEvD;AACA,MAAI,cAAc,iBAAiB,QAAW;AAC5C,SAAK,IAAI,gBAAgB,aAAa,eAAe,SAAS,OAAO;AAAA,EACvE;AACA,MAAI,cAAc,aAAa,QAAW;AACxC,SAAK,IAAI,YAAY,OAAO,aAAa,QAAQ,CAAC;AAAA,EACpD;AACA,MAAI,cAAc,YAAY,QAAW;AACvC,SAAK,IAAI,WAAW,aAAa,UAAU,SAAS,OAAO;AAAA,EAC7D;AACA,SAAO;AACT;AAeO,SAAS,wBACd,OACA,QACA,QACkC;AAClC,SAAO,IAAI,yBAAyB,EAAE,QAAQ,GAAG,OAAA,GAAU,KAAK;AAClE;AAQO,SAAS,kBACd,OACA,QACkC;AAClC,QAAM,SAAS,qBAAA;AACf,SAAO,wBAAwB,OAAO,QAAQ,MAAM;AACtD;"}
|
package/dist/esm/adapters/tts.js
CHANGED
|
@@ -5,9 +5,12 @@ import "@tanstack/openai-base";
|
|
|
5
5
|
import { arrayBufferToBase64 } from "../utils/audio.js";
|
|
6
6
|
const DEFAULT_GROK_BASE_URL = "https://api.x.ai/v1";
|
|
7
7
|
class GrokSpeechAdapter extends BaseTTSAdapter {
|
|
8
|
+
name = "grok";
|
|
9
|
+
apiKey;
|
|
10
|
+
baseURL;
|
|
11
|
+
defaultHeaders;
|
|
8
12
|
constructor(config, model) {
|
|
9
13
|
super(model, config);
|
|
10
|
-
this.name = "grok";
|
|
11
14
|
this.apiKey = config.apiKey;
|
|
12
15
|
this.baseURL = (config.baseURL ?? DEFAULT_GROK_BASE_URL).replace(/\/+$/, "");
|
|
13
16
|
this.defaultHeaders = config.defaultHeaders ?? {};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tts.js","sources":["../../../src/adapters/tts.ts"],"sourcesContent":["import { BaseTTSAdapter } from '@tanstack/ai/adapters'\nimport { arrayBufferToBase64, generateId, getGrokApiKeyFromEnv } from '../utils'\nimport type { TTSOptions, TTSResult } from '@tanstack/ai'\nimport type { GrokTTSModel } from '../model-meta'\nimport type {\n GrokTTSCodec,\n GrokTTSProviderOptions,\n GrokTTSVoice,\n} from '../audio/tts-provider-options'\n\nconst DEFAULT_GROK_BASE_URL = 'https://api.x.ai/v1'\n\n/**\n * Configuration for the Grok TTS adapter.\n *\n * Unlike chat/image/summarize adapters, TTS does not use the OpenAI SDK\n * because xAI's `/v1/tts` endpoint is not OpenAI-compatible. This config\n * is a minimal subset suitable for direct `fetch` calls.\n */\nexport interface GrokSpeechConfig {\n apiKey: string\n baseURL?: string\n /** Additional headers to merge into every request (e.g., test IDs). */\n defaultHeaders?: Record<string, string>\n}\n\n/**\n * Grok Text-to-Speech Adapter.\n *\n * Talks to `POST {baseURL}/tts` per\n * https://docs.x.ai/developers/model-capabilities/audio/text-to-speech\n */\nexport class GrokSpeechAdapter<\n TModel extends GrokTTSModel,\n> extends BaseTTSAdapter<TModel, GrokTTSProviderOptions> {\n readonly name = 'grok' as const\n\n private readonly apiKey: string\n private readonly baseURL: string\n private readonly defaultHeaders: Record<string, string>\n\n constructor(config: GrokSpeechConfig, model: TModel) {\n super(model, config)\n this.apiKey = config.apiKey\n this.baseURL = (config.baseURL ?? DEFAULT_GROK_BASE_URL).replace(/\\/+$/, '')\n this.defaultHeaders = config.defaultHeaders ?? {}\n }\n\n async generateSpeech(\n options: TTSOptions<GrokTTSProviderOptions>,\n ): Promise<TTSResult> {\n const { logger } = options\n const { model, text, voice, format, modelOptions } = options\n\n logger.request(`activity=generateSpeech provider=grok model=${model}`, {\n provider: 'grok',\n model,\n })\n\n const { body, codec, sampleRateForContentType } = buildTTSRequestBody({\n text,\n voice,\n format,\n modelOptions,\n })\n\n try {\n const response = await fetch(`${this.baseURL}/tts`, {\n method: 'POST',\n headers: {\n // `defaultHeaders` first so the adapter's Authorization / Content-Type\n // always win — otherwise a caller-supplied `Authorization` header\n // could silently clobber the bearer token.\n ...this.defaultHeaders,\n Authorization: `Bearer ${this.apiKey}`,\n 'Content-Type': 'application/json',\n },\n body: JSON.stringify(body),\n })\n\n if (!response.ok) {\n const errorText = await response.text()\n throw new Error(\n `Grok TTS request failed: ${response.status} ${errorText}`,\n )\n }\n\n const arrayBuffer = await response.arrayBuffer()\n const audio = arrayBufferToBase64(arrayBuffer)\n\n return {\n id: generateId(this.name),\n model,\n audio,\n format: codec,\n contentType: getContentType(codec, sampleRateForContentType),\n }\n } catch (error) {\n logger.errors('grok.generateSpeech fatal', {\n error,\n source: 'grok.generateSpeech',\n })\n throw error\n }\n }\n}\n\n/**\n * Build the JSON body for `POST /v1/tts`, resolving codec / sample-rate / voice\n * defaults in one place.\n *\n * Returns the request `body`, the resolved `codec`, and the `sampleRateForContentType`\n * used by the caller to label the response via `getContentType`.\n */\nexport function buildTTSRequestBody(options: {\n text: string\n voice: string | undefined\n format: TTSOptions['format'] | undefined\n modelOptions: GrokTTSProviderOptions | undefined\n}): {\n body: Record<string, unknown>\n codec: GrokTTSCodec\n sampleRateForContentType: number\n} {\n const { text, voice, format, modelOptions } = options\n\n const codec = pickCodec(modelOptions?.codec, format)\n\n // Only forward `sample_rate` when either:\n // - the caller explicitly set `modelOptions.sample_rate`, or\n // - the codec's Content-Type carries the rate (pcm → audio/L16;rate=…).\n // For mp3/wav/opus/aac/flac we leave sample_rate unset so xAI's server\n // default applies.\n const callerSampleRate = modelOptions?.sample_rate\n // Default sample rate documented in GrokTTSProviderOptions is 24000 Hz —\n // used only when we MUST attach a rate to the contentType (pcm) and the\n // caller didn't pick one.\n const pcmDefault = 24000\n const needsRateInContentType = codec === 'pcm'\n\n const outputFormat: Record<string, unknown> = { codec }\n if (callerSampleRate !== undefined) {\n outputFormat.sample_rate = callerSampleRate\n } else if (needsRateInContentType) {\n outputFormat.sample_rate = pcmDefault\n }\n if (codec === 'mp3' && modelOptions?.bit_rate !== undefined) {\n outputFormat.bit_rate = modelOptions.bit_rate\n }\n\n // pcm embeds the rate in `audio/L16;rate=…`; mulaw/alaw embed it in\n // `audio/PCMU;rate=…` / `audio/PCMA;rate=…` when non-default. mp3/wav\n // don't carry a rate parameter so the value is unused for those.\n const sampleRateForContentType = callerSampleRate ?? pcmDefault\n\n const body: Record<string, unknown> = {\n text,\n voice_id: (voice as GrokTTSVoice | undefined) ?? 'eve',\n language: modelOptions?.language ?? 'en',\n output_format: outputFormat,\n }\n if (modelOptions?.optimize_streaming_latency !== undefined) {\n body.optimize_streaming_latency = modelOptions.optimize_streaming_latency\n }\n if (modelOptions?.text_normalization !== undefined) {\n body.text_normalization = modelOptions.text_normalization\n }\n\n return { body, codec, sampleRateForContentType }\n}\n\n/**\n * Maps the cross-provider `TTSOptions.format` onto Grok's supported codecs.\n * `opus`, `aac`, and `flac` are not supported by xAI TTS (which only exposes\n * mp3/wav/pcm/mulaw/alaw) — we fall back to mp3. An explicit\n * `modelOptions.codec` always wins.\n */\nfunction pickCodec(\n codecOverride: GrokTTSCodec | undefined,\n format: TTSOptions['format'] | undefined,\n): GrokTTSCodec {\n if (codecOverride) return codecOverride\n if (!format) return 'mp3'\n switch (format) {\n case 'mp3':\n case 'wav':\n case 'pcm':\n return format\n case 'flac':\n case 'opus':\n case 'aac':\n return 'mp3'\n default:\n return 'mp3'\n }\n}\n\nexport function getContentType(\n codec: GrokTTSCodec,\n sampleRate: number,\n): string {\n switch (codec) {\n case 'mp3':\n return 'audio/mpeg'\n case 'wav':\n return 'audio/wav'\n case 'pcm':\n // `audio/L16` requires a `rate` parameter per RFC 3551/3555.\n return `audio/L16;rate=${sampleRate}`\n case 'mulaw':\n // `audio/basic` is 8 kHz mono by RFC 2046 registration. For non-8kHz\n // streams xAI still produces mulaw-encoded bytes at the requested\n // rate, but the registered MIME can't carry that rate — so we use\n // the non-standard but commonly-supported `audio/PCMU;rate=…` (RFC 3551\n // RTP payload name) whenever the caller asked for a rate other than\n // 8000, and keep `audio/basic` for the standard 8kHz case.\n return sampleRate === 8000\n ? 'audio/basic'\n : `audio/PCMU;rate=${sampleRate}`\n case 'alaw':\n return sampleRate === 8000\n ? 'audio/x-alaw-basic'\n : `audio/PCMA;rate=${sampleRate}`\n }\n}\n\n/**\n * Creates a Grok speech (TTS) adapter with an explicit API key.\n *\n * @example\n * ```typescript\n * const adapter = createGrokSpeech('grok-tts', 'xai-...')\n * const result = await generateSpeech({\n * adapter,\n * text: 'Hello from Grok',\n * voice: 'eve',\n * })\n * ```\n */\nexport function createGrokSpeech<TModel extends GrokTTSModel>(\n model: TModel,\n apiKey: string,\n config?: Omit<GrokSpeechConfig, 'apiKey'>,\n): GrokSpeechAdapter<TModel> {\n return new GrokSpeechAdapter({ apiKey, ...config }, model)\n}\n\n/**\n * Creates a Grok speech (TTS) adapter, reading the API key from\n * `XAI_API_KEY` in the environment.\n *\n * @throws Error if `XAI_API_KEY` is not set.\n */\nexport function grokSpeech<TModel extends GrokTTSModel>(\n model: TModel,\n config?: Omit<GrokSpeechConfig, 'apiKey'>,\n): GrokSpeechAdapter<TModel> {\n const apiKey = getGrokApiKeyFromEnv()\n return createGrokSpeech(model, apiKey, config)\n}\n"],"names":[],"mappings":";;;;;AAUA,MAAM,wBAAwB;AAsBvB,MAAM,0BAEH,eAA+C;AAAA,EAOvD,YAAY,QAA0B,OAAe;AACnD,UAAM,OAAO,MAAM;AAPrB,SAAS,OAAO;AAQd,SAAK,SAAS,OAAO;AACrB,SAAK,WAAW,OAAO,WAAW,uBAAuB,QAAQ,QAAQ,EAAE;AAC3E,SAAK,iBAAiB,OAAO,kBAAkB,CAAA;AAAA,EACjD;AAAA,EAEA,MAAM,eACJ,SACoB;AACpB,UAAM,EAAE,WAAW;AACnB,UAAM,EAAE,OAAO,MAAM,OAAO,QAAQ,iBAAiB;AAErD,WAAO,QAAQ,+CAA+C,KAAK,IAAI;AAAA,MACrE,UAAU;AAAA,MACV;AAAA,IAAA,CACD;AAED,UAAM,EAAE,MAAM,OAAO,yBAAA,IAA6B,oBAAoB;AAAA,MACpE;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IAAA,CACD;AAED,QAAI;AACF,YAAM,WAAW,MAAM,MAAM,GAAG,KAAK,OAAO,QAAQ;AAAA,QAClD,QAAQ;AAAA,QACR,SAAS;AAAA;AAAA;AAAA;AAAA,UAIP,GAAG,KAAK;AAAA,UACR,eAAe,UAAU,KAAK,MAAM;AAAA,UACpC,gBAAgB;AAAA,QAAA;AAAA,QAElB,MAAM,KAAK,UAAU,IAAI;AAAA,MAAA,CAC1B;AAED,UAAI,CAAC,SAAS,IAAI;AAChB,cAAM,YAAY,MAAM,SAAS,KAAA;AACjC,cAAM,IAAI;AAAA,UACR,4BAA4B,SAAS,MAAM,IAAI,SAAS;AAAA,QAAA;AAAA,MAE5D;AAEA,YAAM,cAAc,MAAM,SAAS,YAAA;AACnC,YAAM,QAAQ,oBAAoB,WAAW;AAE7C,aAAO;AAAA,QACL,IAAI,WAAW,KAAK,IAAI;AAAA,QACxB;AAAA,QACA;AAAA,QACA,QAAQ;AAAA,QACR,aAAa,eAAe,OAAO,wBAAwB;AAAA,MAAA;AAAA,IAE/D,SAAS,OAAO;AACd,aAAO,OAAO,6BAA6B;AAAA,QACzC;AAAA,QACA,QAAQ;AAAA,MAAA,CACT;AACD,YAAM;AAAA,IACR;AAAA,EACF;AACF;AASO,SAAS,oBAAoB,SASlC;AACA,QAAM,EAAE,MAAM,OAAO,QAAQ,iBAAiB;AAE9C,QAAM,QAAQ,UAAU,cAAc,OAAO,MAAM;AAOnD,QAAM,mBAAmB,cAAc;AAIvC,QAAM,aAAa;AACnB,QAAM,yBAAyB,UAAU;AAEzC,QAAM,eAAwC,EAAE,MAAA;AAChD,MAAI,qBAAqB,QAAW;AAClC,iBAAa,cAAc;AAAA,EAC7B,WAAW,wBAAwB;AACjC,iBAAa,cAAc;AAAA,EAC7B;AACA,MAAI,UAAU,SAAS,cAAc,aAAa,QAAW;AAC3D,iBAAa,WAAW,aAAa;AAAA,EACvC;AAKA,QAAM,2BAA2B,oBAAoB;AAErD,QAAM,OAAgC;AAAA,IACpC;AAAA,IACA,UAAW,SAAsC;AAAA,IACjD,UAAU,cAAc,YAAY;AAAA,IACpC,eAAe;AAAA,EAAA;AAEjB,MAAI,cAAc,+BAA+B,QAAW;AAC1D,SAAK,6BAA6B,aAAa;AAAA,EACjD;AACA,MAAI,cAAc,uBAAuB,QAAW;AAClD,SAAK,qBAAqB,aAAa;AAAA,EACzC;AAEA,SAAO,EAAE,MAAM,OAAO,yBAAA;AACxB;AAQA,SAAS,UACP,eACA,QACc;AACd,MAAI,cAAe,QAAO;AAC1B,MAAI,CAAC,OAAQ,QAAO;AACpB,UAAQ,QAAA;AAAA,IACN,KAAK;AAAA,IACL,KAAK;AAAA,IACL,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AAAA,IACL,KAAK;AAAA,IACL,KAAK;AACH,aAAO;AAAA,IACT;AACE,aAAO;AAAA,EAAA;AAEb;AAEO,SAAS,eACd,OACA,YACQ;AACR,UAAQ,OAAA;AAAA,IACN,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AAEH,aAAO,kBAAkB,UAAU;AAAA,IACrC,KAAK;AAOH,aAAO,eAAe,MAClB,gBACA,mBAAmB,UAAU;AAAA,IACnC,KAAK;AACH,aAAO,eAAe,MAClB,uBACA,mBAAmB,UAAU;AAAA,EAAA;AAEvC;AAeO,SAAS,iBACd,OACA,QACA,QAC2B;AAC3B,SAAO,IAAI,kBAAkB,EAAE,QAAQ,GAAG,OAAA,GAAU,KAAK;AAC3D;AAQO,SAAS,WACd,OACA,QAC2B;AAC3B,QAAM,SAAS,qBAAA;AACf,SAAO,iBAAiB,OAAO,QAAQ,MAAM;AAC/C;"}
|
|
1
|
+
{"version":3,"file":"tts.js","sources":["../../../src/adapters/tts.ts"],"sourcesContent":["import { BaseTTSAdapter } from '@tanstack/ai/adapters'\nimport { arrayBufferToBase64, generateId, getGrokApiKeyFromEnv } from '../utils'\nimport type { TTSOptions, TTSResult } from '@tanstack/ai'\nimport type { GrokTTSModel } from '../model-meta'\nimport type {\n GrokTTSCodec,\n GrokTTSProviderOptions,\n GrokTTSVoice,\n} from '../audio/tts-provider-options'\n\nconst DEFAULT_GROK_BASE_URL = 'https://api.x.ai/v1'\n\n/**\n * Configuration for the Grok TTS adapter.\n *\n * Unlike chat/image/summarize adapters, TTS does not use the OpenAI SDK\n * because xAI's `/v1/tts` endpoint is not OpenAI-compatible. This config\n * is a minimal subset suitable for direct `fetch` calls.\n */\nexport interface GrokSpeechConfig {\n apiKey: string\n baseURL?: string\n /** Additional headers to merge into every request (e.g., test IDs). */\n defaultHeaders?: Record<string, string>\n}\n\n/**\n * Grok Text-to-Speech Adapter.\n *\n * Talks to `POST {baseURL}/tts` per\n * https://docs.x.ai/developers/model-capabilities/audio/text-to-speech\n */\nexport class GrokSpeechAdapter<\n TModel extends GrokTTSModel,\n> extends BaseTTSAdapter<TModel, GrokTTSProviderOptions> {\n readonly name = 'grok' as const\n\n private readonly apiKey: string\n private readonly baseURL: string\n private readonly defaultHeaders: Record<string, string>\n\n constructor(config: GrokSpeechConfig, model: TModel) {\n super(model, config)\n this.apiKey = config.apiKey\n this.baseURL = (config.baseURL ?? DEFAULT_GROK_BASE_URL).replace(/\\/+$/, '')\n this.defaultHeaders = config.defaultHeaders ?? {}\n }\n\n async generateSpeech(\n options: TTSOptions<GrokTTSProviderOptions>,\n ): Promise<TTSResult> {\n const { logger } = options\n const { model, text, voice, format, modelOptions } = options\n\n logger.request(`activity=generateSpeech provider=grok model=${model}`, {\n provider: 'grok',\n model,\n })\n\n const { body, codec, sampleRateForContentType } = buildTTSRequestBody({\n text,\n voice,\n format,\n modelOptions,\n })\n\n try {\n const response = await fetch(`${this.baseURL}/tts`, {\n method: 'POST',\n headers: {\n // `defaultHeaders` first so the adapter's Authorization / Content-Type\n // always win — otherwise a caller-supplied `Authorization` header\n // could silently clobber the bearer token.\n ...this.defaultHeaders,\n Authorization: `Bearer ${this.apiKey}`,\n 'Content-Type': 'application/json',\n },\n body: JSON.stringify(body),\n })\n\n if (!response.ok) {\n const errorText = await response.text()\n throw new Error(\n `Grok TTS request failed: ${response.status} ${errorText}`,\n )\n }\n\n const arrayBuffer = await response.arrayBuffer()\n const audio = arrayBufferToBase64(arrayBuffer)\n\n return {\n id: generateId(this.name),\n model,\n audio,\n format: codec,\n contentType: getContentType(codec, sampleRateForContentType),\n }\n } catch (error) {\n logger.errors('grok.generateSpeech fatal', {\n error,\n source: 'grok.generateSpeech',\n })\n throw error\n }\n }\n}\n\n/**\n * Build the JSON body for `POST /v1/tts`, resolving codec / sample-rate / voice\n * defaults in one place.\n *\n * Returns the request `body`, the resolved `codec`, and the `sampleRateForContentType`\n * used by the caller to label the response via `getContentType`.\n */\nexport function buildTTSRequestBody(options: {\n text: string\n voice: string | undefined\n format: TTSOptions['format'] | undefined\n modelOptions: GrokTTSProviderOptions | undefined\n}): {\n body: Record<string, unknown>\n codec: GrokTTSCodec\n sampleRateForContentType: number\n} {\n const { text, voice, format, modelOptions } = options\n\n const codec = pickCodec(modelOptions?.codec, format)\n\n // Only forward `sample_rate` when either:\n // - the caller explicitly set `modelOptions.sample_rate`, or\n // - the codec's Content-Type carries the rate (pcm → audio/L16;rate=…).\n // For mp3/wav/opus/aac/flac we leave sample_rate unset so xAI's server\n // default applies.\n const callerSampleRate = modelOptions?.sample_rate\n // Default sample rate documented in GrokTTSProviderOptions is 24000 Hz —\n // used only when we MUST attach a rate to the contentType (pcm) and the\n // caller didn't pick one.\n const pcmDefault = 24000\n const needsRateInContentType = codec === 'pcm'\n\n const outputFormat: Record<string, unknown> = { codec }\n if (callerSampleRate !== undefined) {\n outputFormat.sample_rate = callerSampleRate\n } else if (needsRateInContentType) {\n outputFormat.sample_rate = pcmDefault\n }\n if (codec === 'mp3' && modelOptions?.bit_rate !== undefined) {\n outputFormat.bit_rate = modelOptions.bit_rate\n }\n\n // pcm embeds the rate in `audio/L16;rate=…`; mulaw/alaw embed it in\n // `audio/PCMU;rate=…` / `audio/PCMA;rate=…` when non-default. mp3/wav\n // don't carry a rate parameter so the value is unused for those.\n const sampleRateForContentType = callerSampleRate ?? pcmDefault\n\n const body: Record<string, unknown> = {\n text,\n voice_id: (voice as GrokTTSVoice | undefined) ?? 'eve',\n language: modelOptions?.language ?? 'en',\n output_format: outputFormat,\n }\n if (modelOptions?.optimize_streaming_latency !== undefined) {\n body.optimize_streaming_latency = modelOptions.optimize_streaming_latency\n }\n if (modelOptions?.text_normalization !== undefined) {\n body.text_normalization = modelOptions.text_normalization\n }\n\n return { body, codec, sampleRateForContentType }\n}\n\n/**\n * Maps the cross-provider `TTSOptions.format` onto Grok's supported codecs.\n * `opus`, `aac`, and `flac` are not supported by xAI TTS (which only exposes\n * mp3/wav/pcm/mulaw/alaw) — we fall back to mp3. An explicit\n * `modelOptions.codec` always wins.\n */\nfunction pickCodec(\n codecOverride: GrokTTSCodec | undefined,\n format: TTSOptions['format'] | undefined,\n): GrokTTSCodec {\n if (codecOverride) return codecOverride\n if (!format) return 'mp3'\n switch (format) {\n case 'mp3':\n case 'wav':\n case 'pcm':\n return format\n case 'flac':\n case 'opus':\n case 'aac':\n return 'mp3'\n default:\n return 'mp3'\n }\n}\n\nexport function getContentType(\n codec: GrokTTSCodec,\n sampleRate: number,\n): string {\n switch (codec) {\n case 'mp3':\n return 'audio/mpeg'\n case 'wav':\n return 'audio/wav'\n case 'pcm':\n // `audio/L16` requires a `rate` parameter per RFC 3551/3555.\n return `audio/L16;rate=${sampleRate}`\n case 'mulaw':\n // `audio/basic` is 8 kHz mono by RFC 2046 registration. For non-8kHz\n // streams xAI still produces mulaw-encoded bytes at the requested\n // rate, but the registered MIME can't carry that rate — so we use\n // the non-standard but commonly-supported `audio/PCMU;rate=…` (RFC 3551\n // RTP payload name) whenever the caller asked for a rate other than\n // 8000, and keep `audio/basic` for the standard 8kHz case.\n return sampleRate === 8000\n ? 'audio/basic'\n : `audio/PCMU;rate=${sampleRate}`\n case 'alaw':\n return sampleRate === 8000\n ? 'audio/x-alaw-basic'\n : `audio/PCMA;rate=${sampleRate}`\n }\n}\n\n/**\n * Creates a Grok speech (TTS) adapter with an explicit API key.\n *\n * @example\n * ```typescript\n * const adapter = createGrokSpeech('grok-tts', 'xai-...')\n * const result = await generateSpeech({\n * adapter,\n * text: 'Hello from Grok',\n * voice: 'eve',\n * })\n * ```\n */\nexport function createGrokSpeech<TModel extends GrokTTSModel>(\n model: TModel,\n apiKey: string,\n config?: Omit<GrokSpeechConfig, 'apiKey'>,\n): GrokSpeechAdapter<TModel> {\n return new GrokSpeechAdapter({ apiKey, ...config }, model)\n}\n\n/**\n * Creates a Grok speech (TTS) adapter, reading the API key from\n * `XAI_API_KEY` in the environment.\n *\n * @throws Error if `XAI_API_KEY` is not set.\n */\nexport function grokSpeech<TModel extends GrokTTSModel>(\n model: TModel,\n config?: Omit<GrokSpeechConfig, 'apiKey'>,\n): GrokSpeechAdapter<TModel> {\n const apiKey = getGrokApiKeyFromEnv()\n return createGrokSpeech(model, apiKey, config)\n}\n"],"names":[],"mappings":";;;;;AAUA,MAAM,wBAAwB;AAsBvB,MAAM,0BAEH,eAA+C;AAAA,EAC9C,OAAO;AAAA,EAEC;AAAA,EACA;AAAA,EACA;AAAA,EAEjB,YAAY,QAA0B,OAAe;AACnD,UAAM,OAAO,MAAM;AACnB,SAAK,SAAS,OAAO;AACrB,SAAK,WAAW,OAAO,WAAW,uBAAuB,QAAQ,QAAQ,EAAE;AAC3E,SAAK,iBAAiB,OAAO,kBAAkB,CAAA;AAAA,EACjD;AAAA,EAEA,MAAM,eACJ,SACoB;AACpB,UAAM,EAAE,WAAW;AACnB,UAAM,EAAE,OAAO,MAAM,OAAO,QAAQ,iBAAiB;AAErD,WAAO,QAAQ,+CAA+C,KAAK,IAAI;AAAA,MACrE,UAAU;AAAA,MACV;AAAA,IAAA,CACD;AAED,UAAM,EAAE,MAAM,OAAO,yBAAA,IAA6B,oBAAoB;AAAA,MACpE;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IAAA,CACD;AAED,QAAI;AACF,YAAM,WAAW,MAAM,MAAM,GAAG,KAAK,OAAO,QAAQ;AAAA,QAClD,QAAQ;AAAA,QACR,SAAS;AAAA;AAAA;AAAA;AAAA,UAIP,GAAG,KAAK;AAAA,UACR,eAAe,UAAU,KAAK,MAAM;AAAA,UACpC,gBAAgB;AAAA,QAAA;AAAA,QAElB,MAAM,KAAK,UAAU,IAAI;AAAA,MAAA,CAC1B;AAED,UAAI,CAAC,SAAS,IAAI;AAChB,cAAM,YAAY,MAAM,SAAS,KAAA;AACjC,cAAM,IAAI;AAAA,UACR,4BAA4B,SAAS,MAAM,IAAI,SAAS;AAAA,QAAA;AAAA,MAE5D;AAEA,YAAM,cAAc,MAAM,SAAS,YAAA;AACnC,YAAM,QAAQ,oBAAoB,WAAW;AAE7C,aAAO;AAAA,QACL,IAAI,WAAW,KAAK,IAAI;AAAA,QACxB;AAAA,QACA;AAAA,QACA,QAAQ;AAAA,QACR,aAAa,eAAe,OAAO,wBAAwB;AAAA,MAAA;AAAA,IAE/D,SAAS,OAAO;AACd,aAAO,OAAO,6BAA6B;AAAA,QACzC;AAAA,QACA,QAAQ;AAAA,MAAA,CACT;AACD,YAAM;AAAA,IACR;AAAA,EACF;AACF;AASO,SAAS,oBAAoB,SASlC;AACA,QAAM,EAAE,MAAM,OAAO,QAAQ,iBAAiB;AAE9C,QAAM,QAAQ,UAAU,cAAc,OAAO,MAAM;AAOnD,QAAM,mBAAmB,cAAc;AAIvC,QAAM,aAAa;AACnB,QAAM,yBAAyB,UAAU;AAEzC,QAAM,eAAwC,EAAE,MAAA;AAChD,MAAI,qBAAqB,QAAW;AAClC,iBAAa,cAAc;AAAA,EAC7B,WAAW,wBAAwB;AACjC,iBAAa,cAAc;AAAA,EAC7B;AACA,MAAI,UAAU,SAAS,cAAc,aAAa,QAAW;AAC3D,iBAAa,WAAW,aAAa;AAAA,EACvC;AAKA,QAAM,2BAA2B,oBAAoB;AAErD,QAAM,OAAgC;AAAA,IACpC;AAAA,IACA,UAAW,SAAsC;AAAA,IACjD,UAAU,cAAc,YAAY;AAAA,IACpC,eAAe;AAAA,EAAA;AAEjB,MAAI,cAAc,+BAA+B,QAAW;AAC1D,SAAK,6BAA6B,aAAa;AAAA,EACjD;AACA,MAAI,cAAc,uBAAuB,QAAW;AAClD,SAAK,qBAAqB,aAAa;AAAA,EACzC;AAEA,SAAO,EAAE,MAAM,OAAO,yBAAA;AACxB;AAQA,SAAS,UACP,eACA,QACc;AACd,MAAI,cAAe,QAAO;AAC1B,MAAI,CAAC,OAAQ,QAAO;AACpB,UAAQ,QAAA;AAAA,IACN,KAAK;AAAA,IACL,KAAK;AAAA,IACL,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AAAA,IACL,KAAK;AAAA,IACL,KAAK;AACH,aAAO;AAAA,IACT;AACE,aAAO;AAAA,EAAA;AAEb;AAEO,SAAS,eACd,OACA,YACQ;AACR,UAAQ,OAAA;AAAA,IACN,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AAEH,aAAO,kBAAkB,UAAU;AAAA,IACrC,KAAK;AAOH,aAAO,eAAe,MAClB,gBACA,mBAAmB,UAAU;AAAA,IACnC,KAAK;AACH,aAAO,eAAe,MAClB,uBACA,mBAAmB,UAAU;AAAA,EAAA;AAEvC;AAeO,SAAS,iBACd,OACA,QACA,QAC2B;AAC3B,SAAO,IAAI,kBAAkB,EAAE,QAAQ,GAAG,OAAA,GAAU,KAAK;AAC3D;AAQO,SAAS,WACd,OACA,QAC2B;AAC3B,QAAM,SAAS,qBAAA;AACf,SAAO,iBAAiB,OAAO,QAAQ,MAAM;AAC/C;"}
|
|
@@ -49,7 +49,8 @@ async function createWebRTCConnection(token, logger) {
|
|
|
49
49
|
let outputSource = null;
|
|
50
50
|
let localStream = null;
|
|
51
51
|
let audioElement = null;
|
|
52
|
-
|
|
52
|
+
const channel = pc.createDataChannel("oai-events");
|
|
53
|
+
let dataChannel = channel;
|
|
53
54
|
let currentMode = "idle";
|
|
54
55
|
let currentMessageId = null;
|
|
55
56
|
let isTornDown = false;
|
|
@@ -66,7 +67,6 @@ async function createWebRTCConnection(token, logger) {
|
|
|
66
67
|
}
|
|
67
68
|
}
|
|
68
69
|
}
|
|
69
|
-
dataChannel = pc.createDataChannel("oai-events");
|
|
70
70
|
let dataChannelOpened = false;
|
|
71
71
|
let rejectDataChannelReady = null;
|
|
72
72
|
let dataChannelReadyTimeout = null;
|
|
@@ -88,7 +88,7 @@ async function createWebRTCConnection(token, logger) {
|
|
|
88
88
|
);
|
|
89
89
|
}
|
|
90
90
|
}, 15e3);
|
|
91
|
-
|
|
91
|
+
channel.onopen = () => {
|
|
92
92
|
dataChannelOpened = true;
|
|
93
93
|
if (dataChannelReadyTimeout !== null) {
|
|
94
94
|
clearTimeout(dataChannelReadyTimeout);
|
|
@@ -100,7 +100,7 @@ async function createWebRTCConnection(token, logger) {
|
|
|
100
100
|
resolve();
|
|
101
101
|
};
|
|
102
102
|
});
|
|
103
|
-
|
|
103
|
+
channel.onmessage = (event) => {
|
|
104
104
|
try {
|
|
105
105
|
const message = JSON.parse(event.data);
|
|
106
106
|
const messageRecord = message !== null && typeof message === "object" ? message : {};
|
|
@@ -119,7 +119,7 @@ async function createWebRTCConnection(token, logger) {
|
|
|
119
119
|
});
|
|
120
120
|
}
|
|
121
121
|
};
|
|
122
|
-
|
|
122
|
+
channel.onerror = (error) => {
|
|
123
123
|
if (isTornDown) return;
|
|
124
124
|
logger.errors("grok.realtime fatal", {
|
|
125
125
|
error,
|
|
@@ -134,7 +134,7 @@ async function createWebRTCConnection(token, logger) {
|
|
|
134
134
|
}
|
|
135
135
|
emit("error", { error: dcErr });
|
|
136
136
|
};
|
|
137
|
-
|
|
137
|
+
channel.onclose = () => {
|
|
138
138
|
if (isTornDown) return;
|
|
139
139
|
if (!dataChannelOpened) {
|
|
140
140
|
rejectDataChannelReady?.(new Error("Data channel closed before opening"));
|
|
@@ -461,7 +461,9 @@ async function createWebRTCConnection(token, logger) {
|
|
|
461
461
|
currentMode = "listening";
|
|
462
462
|
emit("mode_change", { mode: "listening" });
|
|
463
463
|
}
|
|
464
|
-
emit("interrupted", {
|
|
464
|
+
emit("interrupted", {
|
|
465
|
+
...currentMessageId !== null && { messageId: currentMessageId }
|
|
466
|
+
});
|
|
465
467
|
break;
|
|
466
468
|
case "error": {
|
|
467
469
|
const errorObj = readObject(event, "error") ?? {};
|
|
@@ -480,6 +482,7 @@ async function createWebRTCConnection(token, logger) {
|
|
|
480
482
|
emit("error", { error: err });
|
|
481
483
|
break;
|
|
482
484
|
}
|
|
485
|
+
case void 0:
|
|
483
486
|
default:
|
|
484
487
|
logger.provider("grok.realtime unhandled server event", {
|
|
485
488
|
type: event.type
|
|
@@ -592,7 +595,7 @@ async function createWebRTCConnection(token, logger) {
|
|
|
592
595
|
`provider=grok direction=out type=${readString(event, "type") ?? "<unknown>"}`,
|
|
593
596
|
{ frame: event }
|
|
594
597
|
);
|
|
595
|
-
|
|
598
|
+
channel.send(JSON.stringify(event));
|
|
596
599
|
}
|
|
597
600
|
pendingEvents.length = 0;
|
|
598
601
|
} catch (error) {
|
|
@@ -733,13 +736,17 @@ async function createWebRTCConnection(token, logger) {
|
|
|
733
736
|
sendEvent({ type: "response.cancel" });
|
|
734
737
|
currentMode = "listening";
|
|
735
738
|
emit("mode_change", { mode: "listening" });
|
|
736
|
-
emit("interrupted", {
|
|
739
|
+
emit("interrupted", {
|
|
740
|
+
...currentMessageId !== null && { messageId: currentMessageId }
|
|
741
|
+
});
|
|
737
742
|
},
|
|
738
743
|
on(event, handler) {
|
|
739
|
-
|
|
740
|
-
|
|
744
|
+
let handlers = eventHandlers.get(event);
|
|
745
|
+
if (!handlers) {
|
|
746
|
+
handlers = /* @__PURE__ */ new Set();
|
|
747
|
+
eventHandlers.set(event, handlers);
|
|
741
748
|
}
|
|
742
|
-
|
|
749
|
+
handlers.add(handler);
|
|
743
750
|
return () => {
|
|
744
751
|
eventHandlers.get(event)?.delete(handler);
|
|
745
752
|
};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"adapter.js","sources":["../../../src/realtime/adapter.ts"],"sourcesContent":["import { resolveDebugOption } from '@tanstack/ai/adapter-internals'\nimport type {\n AnyClientTool,\n AudioVisualization,\n RealtimeEvent,\n RealtimeEventHandler,\n RealtimeMessage,\n RealtimeMode,\n RealtimeSessionConfig,\n RealtimeStatus,\n RealtimeToken,\n} from '@tanstack/ai'\nimport type { InternalLogger } from '@tanstack/ai/adapter-internals'\nimport type { RealtimeAdapter, RealtimeConnection } from './realtime-contract'\nimport type { GrokRealtimeOptions } from './types'\n\nconst GROK_REALTIME_URL = 'https://api.x.ai/v1/realtime'\n\n/**\n * Runtime-checked field readers for untyped server events. Replace the\n * drive-by `event.X as string` / `event.X as Record<string, unknown>` casts\n * with readers that return `undefined` when the shape doesn't match, so a\n * malformed frame can't throw a TypeError inside `handleServerEvent`.\n */\nfunction readString(\n obj: Record<string, unknown>,\n key: string,\n): string | undefined {\n const value = obj[key]\n return typeof value === 'string' ? value : undefined\n}\n\nfunction readObject(\n obj: Record<string, unknown>,\n key: string,\n): Record<string, unknown> | undefined {\n const value = obj[key]\n return value && typeof value === 'object' && !Array.isArray(value)\n ? (value as Record<string, unknown>)\n : undefined\n}\n\nfunction readObjectArray(\n obj: Record<string, unknown>,\n key: string,\n): Array<Record<string, unknown>> | undefined {\n const value = obj[key]\n if (!Array.isArray(value)) return undefined\n return value.filter(\n (item): item is Record<string, unknown> =>\n item !== null && typeof item === 'object' && !Array.isArray(item),\n )\n}\n\ntype RealtimeServerError = Error & {\n code?: string\n type?: string\n param?: string\n}\n\n/**\n * Creates a Grok realtime adapter for client-side use.\n *\n * Uses WebRTC for browser connections (default). Mirrors the OpenAI realtime\n * adapter because xAI's Voice Agent API is OpenAI-realtime-compatible — the\n * only differences are the endpoint URL and default model.\n *\n * @example\n * ```typescript\n * import { RealtimeClient } from '@tanstack/ai-client'\n * import { grokRealtime } from '@tanstack/ai-grok'\n *\n * const client = new RealtimeClient({\n * getToken: () => fetch('/api/realtime-token').then(r => r.json()),\n * adapter: grokRealtime(),\n * })\n * ```\n */\nexport function grokRealtime(\n options: GrokRealtimeOptions = {},\n): RealtimeAdapter {\n const connectionMode = options.connectionMode ?? 'webrtc'\n const logger = resolveDebugOption(options.debug)\n\n return {\n provider: 'grok',\n\n async connect(\n token: RealtimeToken,\n _clientTools?: ReadonlyArray<AnyClientTool>,\n ): Promise<RealtimeConnection> {\n const model = token.config.model ?? 'grok-voice-fast-1.0'\n logger.request(`activity=realtime provider=grok model=${model}`, {\n provider: 'grok',\n model,\n })\n\n if (connectionMode === 'webrtc') {\n return createWebRTCConnection(token, logger)\n }\n const error = new Error('WebSocket connection mode not yet implemented')\n logger.errors('grok.realtime fatal', {\n error,\n source: 'grok.realtime',\n })\n throw error\n },\n }\n}\n\n/**\n * Creates a WebRTC connection to xAI's realtime API.\n */\nasync function createWebRTCConnection(\n token: RealtimeToken,\n logger: InternalLogger,\n): Promise<RealtimeConnection> {\n const model = token.config.model ?? 'grok-voice-fast-1.0'\n const eventHandlers = new Map<RealtimeEvent, Set<RealtimeEventHandler<any>>>()\n\n const pc = new RTCPeerConnection()\n\n let audioContext: AudioContext | null = null\n let inputAnalyser: AnalyserNode | null = null\n let outputAnalyser: AnalyserNode | null = null\n let inputSource: MediaStreamAudioSourceNode | null = null\n let outputSource: MediaStreamAudioSourceNode | null = null\n let localStream: MediaStream | null = null\n\n let audioElement: HTMLAudioElement | null = null\n\n let dataChannel: RTCDataChannel | null = null\n\n let currentMode: RealtimeMode = 'idle'\n let currentMessageId: string | null = null\n\n // Flipped by `teardownConnection`. Guards `sendEvent` so post-disconnect\n // calls (e.g. a React `useEffect` cleanup flushing queued events) are\n // logged and skipped instead of silently piling up in `pendingEvents`.\n let isTornDown = false\n\n // Outbound events queued while the data channel isn't yet open. Declared\n // here (rather than next to `sendEvent`) so `teardownConnection` — which\n // lives higher up and can run from the SDP-path catch before `sendEvent`\n // is defined — can drain it without hitting the TDZ.\n const pendingEvents: Array<Record<string, unknown>> = []\n\n // Tracks whether we've sent the first session.update. On the first update\n // we attach a default input_audio_transcription so the server will emit\n // user transcripts unless the caller opts out via\n // `providerOptions.inputAudioTranscription = null | false`.\n let hasSentInitialSessionUpdate = false\n\n // Size hints for the fallback buffers returned when an analyser isn't yet\n // populated. We return a *fresh* `Uint8Array` on each call so a caller\n // that draws into it (e.g. a canvas visualiser zeroing the buffer) can't\n // mutate a shared module-level instance for every other consumer.\n const FALLBACK_FREQUENCY_BIN_COUNT = 1024\n const FALLBACK_TIME_DOMAIN_SIZE = 2048\n const FALLBACK_TIME_DOMAIN_FILL = 128\n\n function emit<TEvent extends RealtimeEvent>(\n event: TEvent,\n payload: Parameters<RealtimeEventHandler<TEvent>>[0],\n ) {\n const handlers = eventHandlers.get(event)\n if (handlers) {\n for (const handler of handlers) {\n handler(payload)\n }\n }\n }\n\n dataChannel = pc.createDataChannel('oai-events')\n\n let dataChannelOpened = false\n let rejectDataChannelReady: ((reason: unknown) => void) | null = null\n let dataChannelReadyTimeout: ReturnType<typeof setTimeout> | null = null\n\n const dataChannelReady = new Promise<void>((resolve, reject) => {\n rejectDataChannelReady = (reason) => {\n if (dataChannelReadyTimeout !== null) {\n clearTimeout(dataChannelReadyTimeout)\n dataChannelReadyTimeout = null\n }\n // One-shot: null out so later state transitions don't reject twice.\n rejectDataChannelReady = null\n reject(reason)\n }\n\n dataChannelReadyTimeout = setTimeout(() => {\n if (!dataChannelOpened) {\n rejectDataChannelReady?.(\n new Error(\n 'Data channel did not open within 15000ms — aborting connection',\n ),\n )\n }\n }, 15000)\n\n dataChannel!.onopen = () => {\n dataChannelOpened = true\n if (dataChannelReadyTimeout !== null) {\n clearTimeout(dataChannelReadyTimeout)\n dataChannelReadyTimeout = null\n }\n // Once resolved, rejecting is a no-op — null out so teardown paths\n // don't attempt a redundant reject on an already-settled promise.\n rejectDataChannelReady = null\n flushPendingEvents()\n emit('status_change', { status: 'connected' as RealtimeStatus })\n resolve()\n }\n })\n\n dataChannel.onmessage = (event) => {\n try {\n const message = JSON.parse(event.data)\n const messageRecord: Record<string, unknown> =\n message !== null && typeof message === 'object' ? message : {}\n logger.provider(\n `provider=grok direction=in type=${readString(messageRecord, 'type') ?? '<unknown>'}`,\n { frame: messageRecord },\n )\n handleServerEvent(messageRecord)\n } catch (parseErr) {\n logger.errors('grok.realtime fatal', {\n error: parseErr,\n source: 'grok.realtime',\n })\n emit('error', {\n error:\n parseErr instanceof Error ? parseErr : new Error(String(parseErr)),\n })\n }\n }\n\n dataChannel.onerror = (error) => {\n // Closing the peer connection cascades into `onerror`/`onclose` on the\n // data channel. Once teardown has started, re-surfacing those as\n // `emit('error')` is noise that confuses consumers (they just called\n // `disconnect()` — they don't want an error toast for it).\n if (isTornDown) return\n logger.errors('grok.realtime fatal', {\n error,\n source: 'grok.realtime',\n })\n // RTCErrorEvent exposes a typed `.error`; fall back to the event type\n // name, then to a string representation, so the emitted error message\n // doesn't end up as \"[object Event]\".\n // `onerror` always fires with an Event (often an RTCErrorEvent), so we\n // can read it via the untyped helpers without first proving object-ness.\n const errorRecord = error as unknown as Record<string, unknown>\n const rtcError = readObject(errorRecord, 'error')\n const msg =\n (rtcError && readString(rtcError, 'message')) ?? (error.type || 'unknown')\n const dcErr = new Error(`Data channel error: ${msg}`)\n if (!dataChannelOpened) {\n rejectDataChannelReady?.(dcErr)\n }\n emit('error', { error: dcErr })\n }\n\n dataChannel.onclose = () => {\n // Same rationale as `onerror` above: `pc.close()` during teardown\n // cascades to the data channel's `onclose`. If we've already started\n // teardown, there's nothing to do here.\n if (isTornDown) return\n if (!dataChannelOpened) {\n rejectDataChannelReady?.(new Error('Data channel closed before opening'))\n }\n }\n\n pc.ontrack = (event) => {\n if (event.track.kind === 'audio' && event.streams[0]) {\n setupOutputAudioAnalysis(event.streams[0])\n }\n }\n\n // `status_change` has a single source of truth: `onconnectionstatechange`\n // (the higher-level aggregate state). `oniceconnectionstatechange` is\n // responsible only for rejecting `dataChannelReady` on ICE failures so we\n // surface them without waiting for the 15s timeout.\n pc.onconnectionstatechange = () => {\n const state = pc.connectionState\n logger.provider(`provider=grok pc.connectionState=${state}`, {\n state,\n })\n if (state === 'failed' || state === 'disconnected' || state === 'closed') {\n // Suppress the `status_change` emission when teardown is in progress:\n // the user-facing `disconnect()` already emits `status_change: 'idle'`\n // and then calls `teardownConnection()` → `pc.close()`, which fires\n // `onconnectionstatechange` with state === 'closed'. Without this\n // guard listeners would see two `idle` events per disconnect.\n if (!isTornDown) {\n emit('status_change', {\n status:\n state === 'failed'\n ? ('error' as RealtimeStatus)\n : ('idle' as RealtimeStatus),\n })\n }\n if (!dataChannelOpened) {\n // Reject on any terminal-ish pre-open state so callers don't hang\n // for the full 15s timeout. The reject is one-shot — subsequent\n // state changes become no-ops via the null-out in\n // `rejectDataChannelReady`.\n const message =\n state === 'failed'\n ? `PeerConnection failed before data channel opened`\n : `PeerConnection entered state '${state}' before data channel opened`\n rejectDataChannelReady?.(new Error(message))\n }\n // Auto-teardown on `failed`: without this the mic track, pc, and\n // AudioContext stay allocated after a fatal connection failure, so the\n // browser's mic indicator stays on and the user sees a broken\n // \"connected mic\" state. `closed` already means pc was torn down\n // (usually by teardownConnection itself) so nothing extra to do.\n // `disconnected` is transient per the WebRTC spec and may recover, so\n // we leave resources in place. `teardownConnection` is idempotent so\n // a subsequent consumer `disconnect()` remains safe.\n if (state === 'failed' && !isTornDown) {\n void teardownConnection()\n }\n }\n }\n\n pc.oniceconnectionstatechange = () => {\n const state = pc.iceConnectionState\n logger.provider(`provider=grok pc.iceConnectionState=${state}`, {\n state,\n })\n if (\n !dataChannelOpened &&\n (state === 'failed' || state === 'closed' || state === 'disconnected')\n ) {\n const message =\n state === 'failed'\n ? `ICE connection failed before data channel opened`\n : `ICE connection entered state '${state}' before data channel opened`\n rejectDataChannelReady?.(new Error(message))\n }\n }\n\n /**\n * Tear down every resource we may have allocated so the mic/pc/audio\n * nodes/audio element don't leak on a failed connect. Safe to call from\n * any point after `new RTCPeerConnection()`; each branch null-guards and\n * swallows errors because cascading closes (e.g. `pc.close()` closing the\n * data channel implicitly) are expected.\n *\n * Shared between the SDP-path catch, the post-SDP catch, and (implicitly\n * via idempotency) the `disconnect()` entry point.\n */\n async function teardownConnection() {\n // Flip the teardown flag BEFORE any awaits so handlers that fire during\n // `await audioContext.close()` (or any other async step below) can guard\n // on it — otherwise a late `pc.onconnectionstatechange` or `pc.ontrack`\n // can allocate new resources or re-emit `status_change: idle` after the\n // user-facing `disconnect()` already emitted one.\n isTornDown = true\n\n // Drop any queued events the caller sent before the data channel opened\n // up front. Without this they'd accumulate across reconnect attempts\n // (each connect allocates a fresh closure, but a caller holding the old\n // `connection` reference could otherwise keep appending forever). Done\n // at the top — before the awaits below — so `sendEvent` calls racing\n // with teardown don't push into a list we're about to drain.\n pendingEvents.length = 0\n\n // Clear the data-channel-open timeout / reject the readiness promise\n // if it's still pending. `rejectDataChannelReady` is one-shot and nulls\n // itself on first call, so calling it from `disconnect()` after a\n // successful open is a no-op.\n rejectDataChannelReady?.(\n new Error('Connection torn down before data channel opened'),\n )\n\n if (localStream) {\n for (const track of localStream.getTracks()) {\n track.stop()\n }\n localStream = null\n }\n\n // Output audio (populated by `pc.ontrack` → setupOutputAudioAnalysis,\n // which may have fired during SDP negotiation before we threw).\n if (audioElement) {\n try {\n audioElement.pause()\n } catch {\n // ignore — element may already be unloaded\n }\n audioElement.srcObject = null\n audioElement = null\n }\n if (outputSource) {\n try {\n outputSource.disconnect()\n } catch {\n // ignore\n }\n outputSource = null\n }\n if (outputAnalyser) {\n try {\n outputAnalyser.disconnect()\n } catch {\n // ignore\n }\n outputAnalyser = null\n }\n\n // Input audio (populated by setupInputAudioAnalysis after SDP).\n if (inputSource) {\n try {\n inputSource.disconnect()\n } catch {\n // ignore\n }\n inputSource = null\n }\n if (inputAnalyser) {\n try {\n inputAnalyser.disconnect()\n } catch {\n // ignore\n }\n inputAnalyser = null\n }\n\n if (dataChannel) {\n try {\n dataChannel.close()\n } catch {\n // ignore — channel may already be closed by pc.close()\n }\n dataChannel = null\n }\n\n try {\n pc.close()\n } catch {\n // ignore — pc may already be closed\n }\n\n if (audioContext) {\n try {\n await audioContext.close()\n } catch {\n // ignore — context may already be closed\n }\n audioContext = null\n }\n }\n\n // xAI requires an audio track in the SDP offer, same as OpenAI realtime.\n //\n // This try/catch also covers `getUserMedia` failure (e.g. the user denies\n // microphone permission). `pc` + `dataChannel` are already allocated above\n // and the 15s `dataChannelReady` timeout is already armed, so we MUST\n // teardown on failure here — otherwise they leak until the tab closes.\n // `teardownConnection` is idempotent and null-safe (runs fine even if the\n // mic was never acquired).\n try {\n try {\n localStream = await navigator.mediaDevices.getUserMedia({\n audio: {\n echoCancellation: true,\n noiseSuppression: true,\n sampleRate: 24000,\n },\n })\n } catch (error) {\n logger.errors('grok.realtime fatal', {\n error,\n source: 'grok.realtime.getUserMedia',\n })\n // Re-throw with the descriptive message callers rely on. Teardown runs\n // in the outer catch below.\n throw new Error(\n `Microphone access required for realtime voice: ${error instanceof Error ? error.message : error}`,\n )\n }\n\n for (const track of localStream.getAudioTracks()) {\n pc.addTrack(track, localStream)\n }\n\n const offer = await pc.createOffer()\n await pc.setLocalDescription(offer)\n\n const sdpResponse = await fetch(`${GROK_REALTIME_URL}?model=${model}`, {\n method: 'POST',\n headers: {\n Authorization: `Bearer ${token.token}`,\n 'Content-Type': 'application/sdp',\n },\n body: offer.sdp,\n })\n\n if (!sdpResponse.ok) {\n const errorText = await sdpResponse.text()\n const error = new Error(\n `Failed to establish WebRTC connection: ${sdpResponse.status} - ${errorText}`,\n )\n logger.errors('grok.realtime fatal', {\n error,\n source: 'grok.realtime.sdp',\n status: sdpResponse.status,\n })\n throw error\n }\n\n const answerSdp = await sdpResponse.text()\n await pc.setRemoteDescription({ type: 'answer', sdp: answerSdp })\n } catch (err) {\n await teardownConnection()\n throw err\n }\n\n // Second cleanup scope: after SDP succeeds we still have to set up input\n // audio analysis and wait for the data channel to open. Both can fail\n // (AudioContext allocation, 15s timeout, ICE failure, pc.close from the\n // other end, etc.) and those failures must NOT leave the mic/pc/audio\n // nodes running.\n try {\n setupInputAudioAnalysis(localStream)\n await dataChannelReady\n } catch (err) {\n await teardownConnection()\n throw err\n }\n\n function handleServerEvent(event: Record<string, unknown>) {\n const type = readString(event, 'type')\n\n switch (type) {\n case 'session.created':\n case 'session.updated':\n break\n\n case 'input_audio_buffer.speech_started':\n currentMode = 'listening'\n emit('mode_change', { mode: 'listening' })\n break\n\n case 'input_audio_buffer.speech_stopped':\n currentMode = 'thinking'\n emit('mode_change', { mode: 'thinking' })\n break\n\n case 'input_audio_buffer.committed':\n break\n\n case 'conversation.item.input_audio_transcription.completed': {\n const transcript = readString(event, 'transcript')\n if (transcript === undefined) break\n emit('transcript', { role: 'user', transcript, isFinal: true })\n break\n }\n\n case 'response.created':\n // Reset message id so a tool-only response (which never emits\n // response.output_item.added for a message) can't reuse the previous\n // turn's id when `response.done` later inspects this flag.\n currentMessageId = null\n currentMode = 'thinking'\n emit('mode_change', { mode: 'thinking' })\n break\n\n case 'response.output_item.added': {\n const item = readObject(event, 'item')\n if (item && readString(item, 'type') === 'message') {\n const id = readString(item, 'id')\n if (id !== undefined) currentMessageId = id\n }\n break\n }\n\n // xAI realtime per docs uses `response.output_audio_transcript.*`;\n // accept the legacy OpenAI-realtime `response.audio_transcript.*` as\n // an alias so this adapter stays compatible across protocol versions.\n case 'response.output_audio_transcript.delta':\n case 'response.audio_transcript.delta': {\n const delta = readString(event, 'delta')\n if (delta === undefined) break\n emit('transcript', {\n role: 'assistant',\n transcript: delta,\n isFinal: false,\n })\n break\n }\n\n case 'response.output_audio_transcript.done':\n case 'response.audio_transcript.done': {\n const transcript = readString(event, 'transcript')\n if (transcript === undefined) break\n emit('transcript', { role: 'assistant', transcript, isFinal: true })\n break\n }\n\n // xAI realtime per docs uses `response.text.*`; accept the legacy\n // OpenAI-realtime `response.output_text.*` as an alias.\n case 'response.text.delta':\n case 'response.output_text.delta': {\n const delta = readString(event, 'delta')\n if (delta === undefined) break\n emit('transcript', {\n role: 'assistant',\n transcript: delta,\n isFinal: false,\n })\n break\n }\n\n case 'response.text.done':\n case 'response.output_text.done': {\n const text = readString(event, 'text')\n if (text === undefined) break\n emit('transcript', {\n role: 'assistant',\n transcript: text,\n isFinal: true,\n })\n break\n }\n\n // xAI realtime per docs uses `response.output_audio.*`; accept the\n // legacy OpenAI-realtime `response.audio.*` as an alias.\n case 'response.output_audio.delta':\n case 'response.audio.delta':\n if (currentMode !== 'speaking') {\n currentMode = 'speaking'\n emit('mode_change', { mode: 'speaking' })\n }\n break\n\n case 'response.output_audio.done':\n case 'response.audio.done':\n break\n\n case 'response.function_call_arguments.done': {\n // Only `call_id` is valid for `sendToolResult` correlation. Falling\n // back to `item_id` would produce a tool-call id the server doesn't\n // recognise when the result is posted back, silently dropping the\n // tool execution. If `call_id` is missing we surface an error event\n // so the UI can react instead of pretending the tool call succeeded.\n const callId = readString(event, 'call_id')\n const name = readString(event, 'name') ?? ''\n const args = readString(event, 'arguments') ?? ''\n if (!callId) {\n logger.errors(\n 'grok.realtime tool_call missing call_id — dropping tool_call',\n {\n source: 'grok.realtime',\n event_type: 'response.function_call_arguments.done',\n item_id: event.item_id,\n },\n )\n emit('error', {\n error: new Error(\n 'Realtime tool call missing call_id; tool will not execute',\n ),\n })\n break\n }\n try {\n const input = JSON.parse(args)\n emit('tool_call', { toolCallId: callId, toolName: name, input })\n } catch {\n emit('tool_call', { toolCallId: callId, toolName: name, input: args })\n }\n break\n }\n\n case 'response.done': {\n const response = readObject(event, 'response') ?? {}\n const output = readObjectArray(response, 'output')\n\n // Only transition back to `listening` if the user hasn't already\n // stopped capture — otherwise we'd override their explicit `idle`\n // state and re-arm the mic visualisation.\n if (currentMode !== 'idle') {\n currentMode = 'listening'\n emit('mode_change', { mode: 'listening' })\n }\n\n if (currentMessageId) {\n const message: RealtimeMessage = {\n id: currentMessageId,\n role: 'assistant',\n timestamp: Date.now(),\n parts: [],\n }\n\n for (const item of output ?? []) {\n if (readString(item, 'type') !== 'message') continue\n const content = readObjectArray(item, 'content')\n if (!content) continue\n for (const part of content) {\n const partType = readString(part, 'type')\n if (partType === 'audio') {\n const transcript = readString(part, 'transcript')\n if (transcript) {\n message.parts.push({ type: 'audio', transcript })\n }\n } else if (partType === 'text') {\n const content = readString(part, 'text')\n if (content) {\n message.parts.push({ type: 'text', content })\n }\n }\n }\n }\n\n emit('message_complete', { message })\n currentMessageId = null\n }\n break\n }\n\n case 'conversation.item.truncated':\n // Assistant playback was interrupted — flip mode back to `listening`\n // unless the user already called `stopAudioCapture()` (idle). Without\n // this the visualisation would stay stuck on `speaking` even though\n // no audio is playing.\n if (currentMode !== 'idle') {\n currentMode = 'listening'\n emit('mode_change', { mode: 'listening' })\n }\n emit('interrupted', { messageId: currentMessageId ?? undefined })\n break\n\n case 'error': {\n // The realtime server's `error` envelope isn't guaranteed to carry\n // an `error` object at all (network-layer corruption, protocol\n // drift, etc.). Validate shape before dereferencing so a malformed\n // payload can't throw a TypeError inside this handler and stop the\n // switch from running for the rest of the session.\n const errorObj = readObject(event, 'error') ?? {}\n const message =\n readString(errorObj, 'message') ?? 'Unknown realtime server error'\n const err: RealtimeServerError = new Error(message)\n // Preserve `code` / `type` / `param` on the Error as extra props so\n // consumers can branch on them without re-parsing the raw event.\n const code = readString(errorObj, 'code')\n if (code !== undefined) err.code = code\n const errType = readString(errorObj, 'type')\n if (errType !== undefined) err.type = errType\n const param = readString(errorObj, 'param')\n if (param !== undefined) err.param = param\n logger.errors('grok.realtime server error', {\n ...errorObj,\n source: 'grok.realtime server',\n })\n emit('error', { error: err })\n break\n }\n\n default:\n // The xAI realtime protocol is a moving target; log unhandled event\n // types at provider level so they're visible during debugging without\n // emitting a user-visible error.\n logger.provider('grok.realtime unhandled server event', {\n type: event.type,\n })\n break\n }\n }\n\n function setupOutputAudioAnalysis(stream: MediaStream) {\n // Bail out if teardown has already started. `pc.ontrack` can fire\n // asynchronously after `teardownConnection()` has flipped `isTornDown`\n // (e.g. a remote track arriving mid-close); without this guard we'd\n // allocate a fresh AudioContext / audio element that nothing would ever\n // clean up.\n if (isTornDown) return\n\n // Tear down any prior output audio before allocating new resources.\n // `pc.ontrack` can fire multiple times over the lifetime of a session\n // (e.g. after renegotiation), and without this we'd leak audio elements\n // and analyser nodes.\n if (audioElement) {\n try {\n audioElement.pause()\n } catch {\n // ignore — element may already be unloaded\n }\n audioElement.srcObject = null\n audioElement = null\n }\n if (outputSource) {\n try {\n outputSource.disconnect()\n } catch {\n // ignore — may already be disconnected\n }\n outputSource = null\n }\n if (outputAnalyser) {\n try {\n outputAnalyser.disconnect()\n } catch {\n // ignore\n }\n outputAnalyser = null\n }\n\n audioElement = new Audio()\n audioElement.srcObject = stream\n audioElement.autoplay = true\n audioElement.play().catch((e) => {\n // Autoplay is commonly blocked until the user interacts with the page\n // (browser gesture requirement). Surfacing this as a fatal `error`\n // event makes the UI render a red/error state even though the\n // connection is healthy — the page just needs a click. Log at a\n // dedicated source tag so it's debuggable, but don't emit `error`.\n logger.errors('grok.realtime audio autoplay blocked', {\n error: e,\n source: 'grok.realtime.audio_permission_required',\n })\n })\n\n if (!audioContext) {\n audioContext = new AudioContext()\n }\n\n if (audioContext.state === 'suspended') {\n audioContext.resume().catch((err) => {\n // Same rationale as the autoplay catch: `resume()` failure usually\n // means the user hasn't interacted yet. Logging only — no error\n // emit — so the UI doesn't go into a fatal state for a recoverable\n // condition.\n logger.errors('grok.realtime audioContext.resume failed', {\n error: err,\n source: 'grok.realtime',\n })\n })\n }\n\n outputAnalyser = audioContext.createAnalyser()\n outputAnalyser.fftSize = 2048\n outputAnalyser.smoothingTimeConstant = 0.3\n\n outputSource = audioContext.createMediaStreamSource(stream)\n outputSource.connect(outputAnalyser)\n }\n\n function setupInputAudioAnalysis(stream: MediaStream) {\n // Defensive symmetry with `setupOutputAudioAnalysis`. Today this is\n // only called inline after SDP negotiation, but keeping the guard\n // means any future caller path (e.g. renegotiation) won't leak a fresh\n // AudioContext after teardown.\n if (isTornDown) return\n\n if (!audioContext) {\n audioContext = new AudioContext()\n }\n\n if (audioContext.state === 'suspended') {\n audioContext.resume().catch((err) => {\n // Same rationale as in setupOutputAudioAnalysis: a suspended\n // AudioContext usually resumes after a user gesture. Log only —\n // surfacing this as a fatal error makes the UI look broken for a\n // recoverable condition.\n logger.errors('grok.realtime audioContext.resume failed', {\n error: err,\n source: 'grok.realtime',\n })\n })\n }\n\n inputAnalyser = audioContext.createAnalyser()\n inputAnalyser.fftSize = 2048\n inputAnalyser.smoothingTimeConstant = 0.3\n\n inputSource = audioContext.createMediaStreamSource(stream)\n inputSource.connect(inputAnalyser)\n }\n\n function sendEvent(event: Record<string, unknown>) {\n if (isTornDown) {\n // The caller is holding onto a `connection` object after `disconnect()`\n // (or a failed connect). Silently queueing would leak memory and the\n // events would never flush. Log + drop so the misuse is visible in\n // debug mode without escalating to a throw — throwing from a React\n // useEffect cleanup path can break teardown ordering in the UI.\n logger.errors('grok.realtime sendEvent after disconnect', {\n eventType: readString(event, 'type') ?? '<unknown>',\n source: 'grok.realtime',\n })\n return\n }\n if (dataChannel?.readyState === 'open') {\n logger.provider(\n `provider=grok direction=out type=${readString(event, 'type') ?? '<unknown>'}`,\n { frame: event },\n )\n // Mirror the try/catch in `flushPendingEvents` — `dataChannel.send`\n // can synchronously throw if the channel flipped to `closing` between\n // our readyState check and this call, or if `JSON.stringify` chokes\n // on a caller-supplied payload. Log + emit error instead of letting\n // the exception propagate up through public `sendText` / `sendImage`\n // / `updateSession` call sites.\n try {\n dataChannel.send(JSON.stringify(event))\n } catch (error) {\n logger.errors('grok.realtime sendEvent failed', {\n error,\n eventType: readString(event, 'type') ?? '<unknown>',\n source: 'grok.realtime',\n })\n emit('error', {\n error: error instanceof Error ? error : new Error(String(error)),\n })\n }\n } else {\n pendingEvents.push(event)\n }\n }\n\n function flushPendingEvents() {\n try {\n for (const event of pendingEvents) {\n logger.provider(\n `provider=grok direction=out type=${readString(event, 'type') ?? '<unknown>'}`,\n { frame: event },\n )\n dataChannel!.send(JSON.stringify(event))\n }\n pendingEvents.length = 0\n } catch (error) {\n // A send failure here (e.g. dataChannel went from 'open' back to\n // 'closing' mid-flush, or JSON.stringify on a caller-provided event\n // threw) would otherwise be silently swallowed. By the time we're\n // called, `onopen` has already resolved `dataChannelReady`, so the\n // consumer-facing signal is `emit('error')` — try rejectDataChannelReady\n // as a defensive belt-and-braces in case this ever runs pre-resolve.\n logger.errors('grok.realtime flushPendingEvents failed', {\n error,\n source: 'grok.realtime',\n })\n const err = error instanceof Error ? error : new Error(String(error))\n rejectDataChannelReady?.(err)\n emit('error', { error: err })\n }\n }\n\n const connection: RealtimeConnection = {\n async disconnect() {\n // Reuse the same teardown path as the failed-connect branches so\n // every cleanup site stays in sync (input analyser, output analyser,\n // output source, audio element, etc.).\n await teardownConnection()\n emit('status_change', { status: 'idle' as RealtimeStatus })\n },\n\n async startAudioCapture() {\n if (localStream) {\n for (const track of localStream.getAudioTracks()) {\n track.enabled = true\n }\n }\n currentMode = 'listening'\n emit('mode_change', { mode: 'listening' })\n },\n\n stopAudioCapture() {\n if (localStream) {\n for (const track of localStream.getAudioTracks()) {\n track.enabled = false\n }\n }\n currentMode = 'idle'\n emit('mode_change', { mode: 'idle' })\n },\n\n sendText(text: string) {\n sendEvent({\n type: 'conversation.item.create',\n item: {\n type: 'message',\n role: 'user',\n content: [{ type: 'input_text', text }],\n },\n })\n sendEvent({ type: 'response.create' })\n },\n\n sendImage(imageData: string, mimeType: string) {\n // Accept:\n // - http(s):// URLs → forward as-is\n // - data: URIs (e.g. from FileReader.readAsDataURL) → forward as-is\n // so we don't double-wrap into `data:image/png;base64,data:image/png;base64,…`\n // - bare base64 → wrap in `data:${mimeType};base64,…`\n const isAlreadyUrlOrDataUri =\n imageData.startsWith('http://') ||\n imageData.startsWith('https://') ||\n imageData.startsWith('data:')\n const imageContent = {\n type: 'input_image',\n // The OpenAI-realtime content part (which this adapter mirrors) nests\n // the URL under an `image_url: { url: ... }` object, not a bare\n // string.\n image_url: {\n url: isAlreadyUrlOrDataUri\n ? imageData\n : `data:${mimeType};base64,${imageData}`,\n },\n }\n\n sendEvent({\n type: 'conversation.item.create',\n item: {\n type: 'message',\n role: 'user',\n content: [imageContent],\n },\n })\n sendEvent({ type: 'response.create' })\n },\n\n sendToolResult(callId: string, result: string) {\n sendEvent({\n type: 'conversation.item.create',\n item: {\n type: 'function_call_output',\n call_id: callId,\n output: result,\n },\n })\n sendEvent({ type: 'response.create' })\n },\n\n updateSession(config: Partial<RealtimeSessionConfig>) {\n const sessionUpdate: Record<string, unknown> = {}\n\n if (config.instructions) {\n sessionUpdate.instructions = config.instructions\n }\n\n if (config.voice) {\n sessionUpdate.voice = config.voice\n }\n\n if (config.vadMode) {\n if (config.vadMode === 'semantic') {\n sessionUpdate.turn_detection = {\n type: 'semantic_vad',\n eagerness: config.semanticEagerness ?? 'medium',\n }\n } else if (config.vadMode === 'server') {\n sessionUpdate.turn_detection = {\n type: 'server_vad',\n threshold: config.vadConfig?.threshold ?? 0.5,\n prefix_padding_ms: config.vadConfig?.prefixPaddingMs ?? 300,\n silence_duration_ms: config.vadConfig?.silenceDurationMs ?? 500,\n }\n } else {\n sessionUpdate.turn_detection = null\n }\n }\n\n if (config.tools !== undefined) {\n sessionUpdate.tools = config.tools.map((t) => ({\n type: 'function',\n name: t.name,\n description: t.description,\n parameters: t.inputSchema ?? { type: 'object', properties: {} },\n }))\n sessionUpdate.tool_choice = 'auto'\n }\n\n if (config.outputModalities) {\n sessionUpdate.modalities = config.outputModalities\n }\n\n if (config.temperature !== undefined) {\n sessionUpdate.temperature = config.temperature\n }\n\n if (config.maxOutputTokens !== undefined) {\n sessionUpdate.max_response_output_tokens = config.maxOutputTokens\n }\n\n // Let callers forward an explicit `input_audio_transcription` value\n // through `providerOptions` — including `null` / `false` to disable\n // the feature. Only apply our `grok-stt` default on the first\n // session.update and only if the caller hasn't set it themselves.\n const providerOptions: Record<string, unknown> =\n config.providerOptions ?? {}\n const callerTranscription =\n 'inputAudioTranscription' in providerOptions\n ? providerOptions.inputAudioTranscription\n : 'input_audio_transcription' in providerOptions\n ? providerOptions.input_audio_transcription\n : undefined\n if (callerTranscription !== undefined) {\n sessionUpdate.input_audio_transcription =\n callerTranscription === false ? null : callerTranscription\n } else if (!hasSentInitialSessionUpdate) {\n sessionUpdate.input_audio_transcription = { model: 'grok-stt' }\n }\n\n if (Object.keys(sessionUpdate).length > 0) {\n sendEvent({\n type: 'session.update',\n session: sessionUpdate,\n })\n hasSentInitialSessionUpdate = true\n }\n },\n\n interrupt() {\n sendEvent({ type: 'response.cancel' })\n currentMode = 'listening'\n emit('mode_change', { mode: 'listening' })\n emit('interrupted', { messageId: currentMessageId ?? undefined })\n },\n\n on<TEvent extends RealtimeEvent>(\n event: TEvent,\n handler: RealtimeEventHandler<TEvent>,\n ): () => void {\n if (!eventHandlers.has(event)) {\n eventHandlers.set(event, new Set())\n }\n eventHandlers.get(event)!.add(handler)\n\n return () => {\n eventHandlers.get(event)?.delete(handler)\n }\n },\n\n getAudioVisualization(): AudioVisualization {\n function calculateLevel(analyser: AnalyserNode): number {\n const data = new Uint8Array(analyser.fftSize)\n analyser.getByteTimeDomainData(data)\n\n let maxDeviation = 0\n for (const sample of data) {\n const deviation = Math.abs(sample - 128)\n if (deviation > maxDeviation) {\n maxDeviation = deviation\n }\n }\n\n const normalized = maxDeviation / 128\n return Math.min(1, normalized * 1.5)\n }\n\n return {\n get inputLevel() {\n if (!inputAnalyser) return 0\n return calculateLevel(inputAnalyser)\n },\n\n get outputLevel() {\n if (!outputAnalyser) return 0\n return calculateLevel(outputAnalyser)\n },\n\n getInputFrequencyData() {\n if (!inputAnalyser)\n return new Uint8Array(FALLBACK_FREQUENCY_BIN_COUNT)\n const data = new Uint8Array(inputAnalyser.frequencyBinCount)\n inputAnalyser.getByteFrequencyData(data)\n return data\n },\n\n getOutputFrequencyData() {\n if (!outputAnalyser)\n return new Uint8Array(FALLBACK_FREQUENCY_BIN_COUNT)\n const data = new Uint8Array(outputAnalyser.frequencyBinCount)\n outputAnalyser.getByteFrequencyData(data)\n return data\n },\n\n getInputTimeDomainData() {\n if (!inputAnalyser)\n return new Uint8Array(FALLBACK_TIME_DOMAIN_SIZE).fill(\n FALLBACK_TIME_DOMAIN_FILL,\n )\n const data = new Uint8Array(inputAnalyser.fftSize)\n inputAnalyser.getByteTimeDomainData(data)\n return data\n },\n\n getOutputTimeDomainData() {\n if (!outputAnalyser)\n return new Uint8Array(FALLBACK_TIME_DOMAIN_SIZE).fill(\n FALLBACK_TIME_DOMAIN_FILL,\n )\n const data = new Uint8Array(outputAnalyser.fftSize)\n outputAnalyser.getByteTimeDomainData(data)\n return data\n },\n\n get inputSampleRate() {\n return 24000\n },\n\n get outputSampleRate() {\n return 24000\n },\n }\n },\n }\n\n // `dataChannelReady` was already awaited inside the post-SDP try/catch\n // above so we can short-circuit on failures with full teardown.\n return connection\n}\n"],"names":["content"],"mappings":";AAgBA,MAAM,oBAAoB;AAQ1B,SAAS,WACP,KACA,KACoB;AACpB,QAAM,QAAQ,IAAI,GAAG;AACrB,SAAO,OAAO,UAAU,WAAW,QAAQ;AAC7C;AAEA,SAAS,WACP,KACA,KACqC;AACrC,QAAM,QAAQ,IAAI,GAAG;AACrB,SAAO,SAAS,OAAO,UAAU,YAAY,CAAC,MAAM,QAAQ,KAAK,IAC5D,QACD;AACN;AAEA,SAAS,gBACP,KACA,KAC4C;AAC5C,QAAM,QAAQ,IAAI,GAAG;AACrB,MAAI,CAAC,MAAM,QAAQ,KAAK,EAAG,QAAO;AAClC,SAAO,MAAM;AAAA,IACX,CAAC,SACC,SAAS,QAAQ,OAAO,SAAS,YAAY,CAAC,MAAM,QAAQ,IAAI;AAAA,EAAA;AAEtE;AA0BO,SAAS,aACd,UAA+B,IACd;AACjB,QAAM,iBAAiB,QAAQ,kBAAkB;AACjD,QAAM,SAAS,mBAAmB,QAAQ,KAAK;AAE/C,SAAO;AAAA,IACL,UAAU;AAAA,IAEV,MAAM,QACJ,OACA,cAC6B;AAC7B,YAAM,QAAQ,MAAM,OAAO,SAAS;AACpC,aAAO,QAAQ,yCAAyC,KAAK,IAAI;AAAA,QAC/D,UAAU;AAAA,QACV;AAAA,MAAA,CACD;AAED,UAAI,mBAAmB,UAAU;AAC/B,eAAO,uBAAuB,OAAO,MAAM;AAAA,MAC7C;AACA,YAAM,QAAQ,IAAI,MAAM,+CAA+C;AACvE,aAAO,OAAO,uBAAuB;AAAA,QACnC;AAAA,QACA,QAAQ;AAAA,MAAA,CACT;AACD,YAAM;AAAA,IACR;AAAA,EAAA;AAEJ;AAKA,eAAe,uBACb,OACA,QAC6B;AAC7B,QAAM,QAAQ,MAAM,OAAO,SAAS;AACpC,QAAM,oCAAoB,IAAA;AAE1B,QAAM,KAAK,IAAI,kBAAA;AAEf,MAAI,eAAoC;AACxC,MAAI,gBAAqC;AACzC,MAAI,iBAAsC;AAC1C,MAAI,cAAiD;AACrD,MAAI,eAAkD;AACtD,MAAI,cAAkC;AAEtC,MAAI,eAAwC;AAE5C,MAAI,cAAqC;AAEzC,MAAI,cAA4B;AAChC,MAAI,mBAAkC;AAKtC,MAAI,aAAa;AAMjB,QAAM,gBAAgD,CAAA;AAMtD,MAAI,8BAA8B;AAMlC,QAAM,+BAA+B;AACrC,QAAM,4BAA4B;AAClC,QAAM,4BAA4B;AAElC,WAAS,KACP,OACA,SACA;AACA,UAAM,WAAW,cAAc,IAAI,KAAK;AACxC,QAAI,UAAU;AACZ,iBAAW,WAAW,UAAU;AAC9B,gBAAQ,OAAO;AAAA,MACjB;AAAA,IACF;AAAA,EACF;AAEA,gBAAc,GAAG,kBAAkB,YAAY;AAE/C,MAAI,oBAAoB;AACxB,MAAI,yBAA6D;AACjE,MAAI,0BAAgE;AAEpE,QAAM,mBAAmB,IAAI,QAAc,CAAC,SAAS,WAAW;AAC9D,6BAAyB,CAAC,WAAW;AACnC,UAAI,4BAA4B,MAAM;AACpC,qBAAa,uBAAuB;AACpC,kCAA0B;AAAA,MAC5B;AAEA,+BAAyB;AACzB,aAAO,MAAM;AAAA,IACf;AAEA,8BAA0B,WAAW,MAAM;AACzC,UAAI,CAAC,mBAAmB;AACtB;AAAA,UACE,IAAI;AAAA,YACF;AAAA,UAAA;AAAA,QACF;AAAA,MAEJ;AAAA,IACF,GAAG,IAAK;AAER,gBAAa,SAAS,MAAM;AAC1B,0BAAoB;AACpB,UAAI,4BAA4B,MAAM;AACpC,qBAAa,uBAAuB;AACpC,kCAA0B;AAAA,MAC5B;AAGA,+BAAyB;AACzB,yBAAA;AACA,WAAK,iBAAiB,EAAE,QAAQ,YAAA,CAA+B;AAC/D,cAAA;AAAA,IACF;AAAA,EACF,CAAC;AAED,cAAY,YAAY,CAAC,UAAU;AACjC,QAAI;AACF,YAAM,UAAU,KAAK,MAAM,MAAM,IAAI;AACrC,YAAM,gBACJ,YAAY,QAAQ,OAAO,YAAY,WAAW,UAAU,CAAA;AAC9D,aAAO;AAAA,QACL,mCAAmC,WAAW,eAAe,MAAM,KAAK,WAAW;AAAA,QACnF,EAAE,OAAO,cAAA;AAAA,MAAc;AAEzB,wBAAkB,aAAa;AAAA,IACjC,SAAS,UAAU;AACjB,aAAO,OAAO,uBAAuB;AAAA,QACnC,OAAO;AAAA,QACP,QAAQ;AAAA,MAAA,CACT;AACD,WAAK,SAAS;AAAA,QACZ,OACE,oBAAoB,QAAQ,WAAW,IAAI,MAAM,OAAO,QAAQ,CAAC;AAAA,MAAA,CACpE;AAAA,IACH;AAAA,EACF;AAEA,cAAY,UAAU,CAAC,UAAU;AAK/B,QAAI,WAAY;AAChB,WAAO,OAAO,uBAAuB;AAAA,MACnC;AAAA,MACA,QAAQ;AAAA,IAAA,CACT;AAMD,UAAM,cAAc;AACpB,UAAM,WAAW,WAAW,aAAa,OAAO;AAChD,UAAM,OACH,YAAY,WAAW,UAAU,SAAS,OAAO,MAAM,QAAQ;AAClE,UAAM,QAAQ,IAAI,MAAM,uBAAuB,GAAG,EAAE;AACpD,QAAI,CAAC,mBAAmB;AACtB,+BAAyB,KAAK;AAAA,IAChC;AACA,SAAK,SAAS,EAAE,OAAO,MAAA,CAAO;AAAA,EAChC;AAEA,cAAY,UAAU,MAAM;AAI1B,QAAI,WAAY;AAChB,QAAI,CAAC,mBAAmB;AACtB,+BAAyB,IAAI,MAAM,oCAAoC,CAAC;AAAA,IAC1E;AAAA,EACF;AAEA,KAAG,UAAU,CAAC,UAAU;AACtB,QAAI,MAAM,MAAM,SAAS,WAAW,MAAM,QAAQ,CAAC,GAAG;AACpD,+BAAyB,MAAM,QAAQ,CAAC,CAAC;AAAA,IAC3C;AAAA,EACF;AAMA,KAAG,0BAA0B,MAAM;AACjC,UAAM,QAAQ,GAAG;AACjB,WAAO,SAAS,oCAAoC,KAAK,IAAI;AAAA,MAC3D;AAAA,IAAA,CACD;AACD,QAAI,UAAU,YAAY,UAAU,kBAAkB,UAAU,UAAU;AAMxE,UAAI,CAAC,YAAY;AACf,aAAK,iBAAiB;AAAA,UACpB,QACE,UAAU,WACL,UACA;AAAA,QAAA,CACR;AAAA,MACH;AACA,UAAI,CAAC,mBAAmB;AAKtB,cAAM,UACJ,UAAU,WACN,qDACA,iCAAiC,KAAK;AAC5C,iCAAyB,IAAI,MAAM,OAAO,CAAC;AAAA,MAC7C;AASA,UAAI,UAAU,YAAY,CAAC,YAAY;AACrC,aAAK,mBAAA;AAAA,MACP;AAAA,IACF;AAAA,EACF;AAEA,KAAG,6BAA6B,MAAM;AACpC,UAAM,QAAQ,GAAG;AACjB,WAAO,SAAS,uCAAuC,KAAK,IAAI;AAAA,MAC9D;AAAA,IAAA,CACD;AACD,QACE,CAAC,sBACA,UAAU,YAAY,UAAU,YAAY,UAAU,iBACvD;AACA,YAAM,UACJ,UAAU,WACN,qDACA,iCAAiC,KAAK;AAC5C,+BAAyB,IAAI,MAAM,OAAO,CAAC;AAAA,IAC7C;AAAA,EACF;AAYA,iBAAe,qBAAqB;AAMlC,iBAAa;AAQb,kBAAc,SAAS;AAMvB;AAAA,MACE,IAAI,MAAM,iDAAiD;AAAA,IAAA;AAG7D,QAAI,aAAa;AACf,iBAAW,SAAS,YAAY,aAAa;AAC3C,cAAM,KAAA;AAAA,MACR;AACA,oBAAc;AAAA,IAChB;AAIA,QAAI,cAAc;AAChB,UAAI;AACF,qBAAa,MAAA;AAAA,MACf,QAAQ;AAAA,MAER;AACA,mBAAa,YAAY;AACzB,qBAAe;AAAA,IACjB;AACA,QAAI,cAAc;AAChB,UAAI;AACF,qBAAa,WAAA;AAAA,MACf,QAAQ;AAAA,MAER;AACA,qBAAe;AAAA,IACjB;AACA,QAAI,gBAAgB;AAClB,UAAI;AACF,uBAAe,WAAA;AAAA,MACjB,QAAQ;AAAA,MAER;AACA,uBAAiB;AAAA,IACnB;AAGA,QAAI,aAAa;AACf,UAAI;AACF,oBAAY,WAAA;AAAA,MACd,QAAQ;AAAA,MAER;AACA,oBAAc;AAAA,IAChB;AACA,QAAI,eAAe;AACjB,UAAI;AACF,sBAAc,WAAA;AAAA,MAChB,QAAQ;AAAA,MAER;AACA,sBAAgB;AAAA,IAClB;AAEA,QAAI,aAAa;AACf,UAAI;AACF,oBAAY,MAAA;AAAA,MACd,QAAQ;AAAA,MAER;AACA,oBAAc;AAAA,IAChB;AAEA,QAAI;AACF,SAAG,MAAA;AAAA,IACL,QAAQ;AAAA,IAER;AAEA,QAAI,cAAc;AAChB,UAAI;AACF,cAAM,aAAa,MAAA;AAAA,MACrB,QAAQ;AAAA,MAER;AACA,qBAAe;AAAA,IACjB;AAAA,EACF;AAUA,MAAI;AACF,QAAI;AACF,oBAAc,MAAM,UAAU,aAAa,aAAa;AAAA,QACtD,OAAO;AAAA,UACL,kBAAkB;AAAA,UAClB,kBAAkB;AAAA,UAClB,YAAY;AAAA,QAAA;AAAA,MACd,CACD;AAAA,IACH,SAAS,OAAO;AACd,aAAO,OAAO,uBAAuB;AAAA,QACnC;AAAA,QACA,QAAQ;AAAA,MAAA,CACT;AAGD,YAAM,IAAI;AAAA,QACR,kDAAkD,iBAAiB,QAAQ,MAAM,UAAU,KAAK;AAAA,MAAA;AAAA,IAEpG;AAEA,eAAW,SAAS,YAAY,kBAAkB;AAChD,SAAG,SAAS,OAAO,WAAW;AAAA,IAChC;AAEA,UAAM,QAAQ,MAAM,GAAG,YAAA;AACvB,UAAM,GAAG,oBAAoB,KAAK;AAElC,UAAM,cAAc,MAAM,MAAM,GAAG,iBAAiB,UAAU,KAAK,IAAI;AAAA,MACrE,QAAQ;AAAA,MACR,SAAS;AAAA,QACP,eAAe,UAAU,MAAM,KAAK;AAAA,QACpC,gBAAgB;AAAA,MAAA;AAAA,MAElB,MAAM,MAAM;AAAA,IAAA,CACb;AAED,QAAI,CAAC,YAAY,IAAI;AACnB,YAAM,YAAY,MAAM,YAAY,KAAA;AACpC,YAAM,QAAQ,IAAI;AAAA,QAChB,0CAA0C,YAAY,MAAM,MAAM,SAAS;AAAA,MAAA;AAE7E,aAAO,OAAO,uBAAuB;AAAA,QACnC;AAAA,QACA,QAAQ;AAAA,QACR,QAAQ,YAAY;AAAA,MAAA,CACrB;AACD,YAAM;AAAA,IACR;AAEA,UAAM,YAAY,MAAM,YAAY,KAAA;AACpC,UAAM,GAAG,qBAAqB,EAAE,MAAM,UAAU,KAAK,WAAW;AAAA,EAClE,SAAS,KAAK;AACZ,UAAM,mBAAA;AACN,UAAM;AAAA,EACR;AAOA,MAAI;AACF,4BAAwB,WAAW;AACnC,UAAM;AAAA,EACR,SAAS,KAAK;AACZ,UAAM,mBAAA;AACN,UAAM;AAAA,EACR;AAEA,WAAS,kBAAkB,OAAgC;AACzD,UAAM,OAAO,WAAW,OAAO,MAAM;AAErC,YAAQ,MAAA;AAAA,MACN,KAAK;AAAA,MACL,KAAK;AACH;AAAA,MAEF,KAAK;AACH,sBAAc;AACd,aAAK,eAAe,EAAE,MAAM,YAAA,CAAa;AACzC;AAAA,MAEF,KAAK;AACH,sBAAc;AACd,aAAK,eAAe,EAAE,MAAM,WAAA,CAAY;AACxC;AAAA,MAEF,KAAK;AACH;AAAA,MAEF,KAAK,yDAAyD;AAC5D,cAAM,aAAa,WAAW,OAAO,YAAY;AACjD,YAAI,eAAe,OAAW;AAC9B,aAAK,cAAc,EAAE,MAAM,QAAQ,YAAY,SAAS,MAAM;AAC9D;AAAA,MACF;AAAA,MAEA,KAAK;AAIH,2BAAmB;AACnB,sBAAc;AACd,aAAK,eAAe,EAAE,MAAM,WAAA,CAAY;AACxC;AAAA,MAEF,KAAK,8BAA8B;AACjC,cAAM,OAAO,WAAW,OAAO,MAAM;AACrC,YAAI,QAAQ,WAAW,MAAM,MAAM,MAAM,WAAW;AAClD,gBAAM,KAAK,WAAW,MAAM,IAAI;AAChC,cAAI,OAAO,OAAW,oBAAmB;AAAA,QAC3C;AACA;AAAA,MACF;AAAA;AAAA;AAAA;AAAA,MAKA,KAAK;AAAA,MACL,KAAK,mCAAmC;AACtC,cAAM,QAAQ,WAAW,OAAO,OAAO;AACvC,YAAI,UAAU,OAAW;AACzB,aAAK,cAAc;AAAA,UACjB,MAAM;AAAA,UACN,YAAY;AAAA,UACZ,SAAS;AAAA,QAAA,CACV;AACD;AAAA,MACF;AAAA,MAEA,KAAK;AAAA,MACL,KAAK,kCAAkC;AACrC,cAAM,aAAa,WAAW,OAAO,YAAY;AACjD,YAAI,eAAe,OAAW;AAC9B,aAAK,cAAc,EAAE,MAAM,aAAa,YAAY,SAAS,MAAM;AACnE;AAAA,MACF;AAAA;AAAA;AAAA,MAIA,KAAK;AAAA,MACL,KAAK,8BAA8B;AACjC,cAAM,QAAQ,WAAW,OAAO,OAAO;AACvC,YAAI,UAAU,OAAW;AACzB,aAAK,cAAc;AAAA,UACjB,MAAM;AAAA,UACN,YAAY;AAAA,UACZ,SAAS;AAAA,QAAA,CACV;AACD;AAAA,MACF;AAAA,MAEA,KAAK;AAAA,MACL,KAAK,6BAA6B;AAChC,cAAM,OAAO,WAAW,OAAO,MAAM;AACrC,YAAI,SAAS,OAAW;AACxB,aAAK,cAAc;AAAA,UACjB,MAAM;AAAA,UACN,YAAY;AAAA,UACZ,SAAS;AAAA,QAAA,CACV;AACD;AAAA,MACF;AAAA;AAAA;AAAA,MAIA,KAAK;AAAA,MACL,KAAK;AACH,YAAI,gBAAgB,YAAY;AAC9B,wBAAc;AACd,eAAK,eAAe,EAAE,MAAM,WAAA,CAAY;AAAA,QAC1C;AACA;AAAA,MAEF,KAAK;AAAA,MACL,KAAK;AACH;AAAA,MAEF,KAAK,yCAAyC;AAM5C,cAAM,SAAS,WAAW,OAAO,SAAS;AAC1C,cAAM,OAAO,WAAW,OAAO,MAAM,KAAK;AAC1C,cAAM,OAAO,WAAW,OAAO,WAAW,KAAK;AAC/C,YAAI,CAAC,QAAQ;AACX,iBAAO;AAAA,YACL;AAAA,YACA;AAAA,cACE,QAAQ;AAAA,cACR,YAAY;AAAA,cACZ,SAAS,MAAM;AAAA,YAAA;AAAA,UACjB;AAEF,eAAK,SAAS;AAAA,YACZ,OAAO,IAAI;AAAA,cACT;AAAA,YAAA;AAAA,UACF,CACD;AACD;AAAA,QACF;AACA,YAAI;AACF,gBAAM,QAAQ,KAAK,MAAM,IAAI;AAC7B,eAAK,aAAa,EAAE,YAAY,QAAQ,UAAU,MAAM,OAAO;AAAA,QACjE,QAAQ;AACN,eAAK,aAAa,EAAE,YAAY,QAAQ,UAAU,MAAM,OAAO,MAAM;AAAA,QACvE;AACA;AAAA,MACF;AAAA,MAEA,KAAK,iBAAiB;AACpB,cAAM,WAAW,WAAW,OAAO,UAAU,KAAK,CAAA;AAClD,cAAM,SAAS,gBAAgB,UAAU,QAAQ;AAKjD,YAAI,gBAAgB,QAAQ;AAC1B,wBAAc;AACd,eAAK,eAAe,EAAE,MAAM,YAAA,CAAa;AAAA,QAC3C;AAEA,YAAI,kBAAkB;AACpB,gBAAM,UAA2B;AAAA,YAC/B,IAAI;AAAA,YACJ,MAAM;AAAA,YACN,WAAW,KAAK,IAAA;AAAA,YAChB,OAAO,CAAA;AAAA,UAAC;AAGV,qBAAW,QAAQ,UAAU,IAAI;AAC/B,gBAAI,WAAW,MAAM,MAAM,MAAM,UAAW;AAC5C,kBAAM,UAAU,gBAAgB,MAAM,SAAS;AAC/C,gBAAI,CAAC,QAAS;AACd,uBAAW,QAAQ,SAAS;AAC1B,oBAAM,WAAW,WAAW,MAAM,MAAM;AACxC,kBAAI,aAAa,SAAS;AACxB,sBAAM,aAAa,WAAW,MAAM,YAAY;AAChD,oBAAI,YAAY;AACd,0BAAQ,MAAM,KAAK,EAAE,MAAM,SAAS,YAAY;AAAA,gBAClD;AAAA,cACF,WAAW,aAAa,QAAQ;AAC9B,sBAAMA,WAAU,WAAW,MAAM,MAAM;AACvC,oBAAIA,UAAS;AACX,0BAAQ,MAAM,KAAK,EAAE,MAAM,QAAQ,SAAAA,UAAS;AAAA,gBAC9C;AAAA,cACF;AAAA,YACF;AAAA,UACF;AAEA,eAAK,oBAAoB,EAAE,SAAS;AACpC,6BAAmB;AAAA,QACrB;AACA;AAAA,MACF;AAAA,MAEA,KAAK;AAKH,YAAI,gBAAgB,QAAQ;AAC1B,wBAAc;AACd,eAAK,eAAe,EAAE,MAAM,YAAA,CAAa;AAAA,QAC3C;AACA,aAAK,eAAe,EAAE,WAAW,oBAAoB,QAAW;AAChE;AAAA,MAEF,KAAK,SAAS;AAMZ,cAAM,WAAW,WAAW,OAAO,OAAO,KAAK,CAAA;AAC/C,cAAM,UACJ,WAAW,UAAU,SAAS,KAAK;AACrC,cAAM,MAA2B,IAAI,MAAM,OAAO;AAGlD,cAAM,OAAO,WAAW,UAAU,MAAM;AACxC,YAAI,SAAS,OAAW,KAAI,OAAO;AACnC,cAAM,UAAU,WAAW,UAAU,MAAM;AAC3C,YAAI,YAAY,OAAW,KAAI,OAAO;AACtC,cAAM,QAAQ,WAAW,UAAU,OAAO;AAC1C,YAAI,UAAU,OAAW,KAAI,QAAQ;AACrC,eAAO,OAAO,8BAA8B;AAAA,UAC1C,GAAG;AAAA,UACH,QAAQ;AAAA,QAAA,CACT;AACD,aAAK,SAAS,EAAE,OAAO,IAAA,CAAK;AAC5B;AAAA,MACF;AAAA,MAEA;AAIE,eAAO,SAAS,wCAAwC;AAAA,UACtD,MAAM,MAAM;AAAA,QAAA,CACb;AACD;AAAA,IAAA;AAAA,EAEN;AAEA,WAAS,yBAAyB,QAAqB;AAMrD,QAAI,WAAY;AAMhB,QAAI,cAAc;AAChB,UAAI;AACF,qBAAa,MAAA;AAAA,MACf,QAAQ;AAAA,MAER;AACA,mBAAa,YAAY;AACzB,qBAAe;AAAA,IACjB;AACA,QAAI,cAAc;AAChB,UAAI;AACF,qBAAa,WAAA;AAAA,MACf,QAAQ;AAAA,MAER;AACA,qBAAe;AAAA,IACjB;AACA,QAAI,gBAAgB;AAClB,UAAI;AACF,uBAAe,WAAA;AAAA,MACjB,QAAQ;AAAA,MAER;AACA,uBAAiB;AAAA,IACnB;AAEA,mBAAe,IAAI,MAAA;AACnB,iBAAa,YAAY;AACzB,iBAAa,WAAW;AACxB,iBAAa,KAAA,EAAO,MAAM,CAAC,MAAM;AAM/B,aAAO,OAAO,wCAAwC;AAAA,QACpD,OAAO;AAAA,QACP,QAAQ;AAAA,MAAA,CACT;AAAA,IACH,CAAC;AAED,QAAI,CAAC,cAAc;AACjB,qBAAe,IAAI,aAAA;AAAA,IACrB;AAEA,QAAI,aAAa,UAAU,aAAa;AACtC,mBAAa,OAAA,EAAS,MAAM,CAAC,QAAQ;AAKnC,eAAO,OAAO,4CAA4C;AAAA,UACxD,OAAO;AAAA,UACP,QAAQ;AAAA,QAAA,CACT;AAAA,MACH,CAAC;AAAA,IACH;AAEA,qBAAiB,aAAa,eAAA;AAC9B,mBAAe,UAAU;AACzB,mBAAe,wBAAwB;AAEvC,mBAAe,aAAa,wBAAwB,MAAM;AAC1D,iBAAa,QAAQ,cAAc;AAAA,EACrC;AAEA,WAAS,wBAAwB,QAAqB;AAKpD,QAAI,WAAY;AAEhB,QAAI,CAAC,cAAc;AACjB,qBAAe,IAAI,aAAA;AAAA,IACrB;AAEA,QAAI,aAAa,UAAU,aAAa;AACtC,mBAAa,OAAA,EAAS,MAAM,CAAC,QAAQ;AAKnC,eAAO,OAAO,4CAA4C;AAAA,UACxD,OAAO;AAAA,UACP,QAAQ;AAAA,QAAA,CACT;AAAA,MACH,CAAC;AAAA,IACH;AAEA,oBAAgB,aAAa,eAAA;AAC7B,kBAAc,UAAU;AACxB,kBAAc,wBAAwB;AAEtC,kBAAc,aAAa,wBAAwB,MAAM;AACzD,gBAAY,QAAQ,aAAa;AAAA,EACnC;AAEA,WAAS,UAAU,OAAgC;AACjD,QAAI,YAAY;AAMd,aAAO,OAAO,4CAA4C;AAAA,QACxD,WAAW,WAAW,OAAO,MAAM,KAAK;AAAA,QACxC,QAAQ;AAAA,MAAA,CACT;AACD;AAAA,IACF;AACA,QAAI,aAAa,eAAe,QAAQ;AACtC,aAAO;AAAA,QACL,oCAAoC,WAAW,OAAO,MAAM,KAAK,WAAW;AAAA,QAC5E,EAAE,OAAO,MAAA;AAAA,MAAM;AAQjB,UAAI;AACF,oBAAY,KAAK,KAAK,UAAU,KAAK,CAAC;AAAA,MACxC,SAAS,OAAO;AACd,eAAO,OAAO,kCAAkC;AAAA,UAC9C;AAAA,UACA,WAAW,WAAW,OAAO,MAAM,KAAK;AAAA,UACxC,QAAQ;AAAA,QAAA,CACT;AACD,aAAK,SAAS;AAAA,UACZ,OAAO,iBAAiB,QAAQ,QAAQ,IAAI,MAAM,OAAO,KAAK,CAAC;AAAA,QAAA,CAChE;AAAA,MACH;AAAA,IACF,OAAO;AACL,oBAAc,KAAK,KAAK;AAAA,IAC1B;AAAA,EACF;AAEA,WAAS,qBAAqB;AAC5B,QAAI;AACF,iBAAW,SAAS,eAAe;AACjC,eAAO;AAAA,UACL,oCAAoC,WAAW,OAAO,MAAM,KAAK,WAAW;AAAA,UAC5E,EAAE,OAAO,MAAA;AAAA,QAAM;AAEjB,oBAAa,KAAK,KAAK,UAAU,KAAK,CAAC;AAAA,MACzC;AACA,oBAAc,SAAS;AAAA,IACzB,SAAS,OAAO;AAOd,aAAO,OAAO,2CAA2C;AAAA,QACvD;AAAA,QACA,QAAQ;AAAA,MAAA,CACT;AACD,YAAM,MAAM,iBAAiB,QAAQ,QAAQ,IAAI,MAAM,OAAO,KAAK,CAAC;AACpE,+BAAyB,GAAG;AAC5B,WAAK,SAAS,EAAE,OAAO,IAAA,CAAK;AAAA,IAC9B;AAAA,EACF;AAEA,QAAM,aAAiC;AAAA,IACrC,MAAM,aAAa;AAIjB,YAAM,mBAAA;AACN,WAAK,iBAAiB,EAAE,QAAQ,OAAA,CAA0B;AAAA,IAC5D;AAAA,IAEA,MAAM,oBAAoB;AACxB,UAAI,aAAa;AACf,mBAAW,SAAS,YAAY,kBAAkB;AAChD,gBAAM,UAAU;AAAA,QAClB;AAAA,MACF;AACA,oBAAc;AACd,WAAK,eAAe,EAAE,MAAM,YAAA,CAAa;AAAA,IAC3C;AAAA,IAEA,mBAAmB;AACjB,UAAI,aAAa;AACf,mBAAW,SAAS,YAAY,kBAAkB;AAChD,gBAAM,UAAU;AAAA,QAClB;AAAA,MACF;AACA,oBAAc;AACd,WAAK,eAAe,EAAE,MAAM,OAAA,CAAQ;AAAA,IACtC;AAAA,IAEA,SAAS,MAAc;AACrB,gBAAU;AAAA,QACR,MAAM;AAAA,QACN,MAAM;AAAA,UACJ,MAAM;AAAA,UACN,MAAM;AAAA,UACN,SAAS,CAAC,EAAE,MAAM,cAAc,MAAM;AAAA,QAAA;AAAA,MACxC,CACD;AACD,gBAAU,EAAE,MAAM,mBAAmB;AAAA,IACvC;AAAA,IAEA,UAAU,WAAmB,UAAkB;AAM7C,YAAM,wBACJ,UAAU,WAAW,SAAS,KAC9B,UAAU,WAAW,UAAU,KAC/B,UAAU,WAAW,OAAO;AAC9B,YAAM,eAAe;AAAA,QACnB,MAAM;AAAA;AAAA;AAAA;AAAA,QAIN,WAAW;AAAA,UACT,KAAK,wBACD,YACA,QAAQ,QAAQ,WAAW,SAAS;AAAA,QAAA;AAAA,MAC1C;AAGF,gBAAU;AAAA,QACR,MAAM;AAAA,QACN,MAAM;AAAA,UACJ,MAAM;AAAA,UACN,MAAM;AAAA,UACN,SAAS,CAAC,YAAY;AAAA,QAAA;AAAA,MACxB,CACD;AACD,gBAAU,EAAE,MAAM,mBAAmB;AAAA,IACvC;AAAA,IAEA,eAAe,QAAgB,QAAgB;AAC7C,gBAAU;AAAA,QACR,MAAM;AAAA,QACN,MAAM;AAAA,UACJ,MAAM;AAAA,UACN,SAAS;AAAA,UACT,QAAQ;AAAA,QAAA;AAAA,MACV,CACD;AACD,gBAAU,EAAE,MAAM,mBAAmB;AAAA,IACvC;AAAA,IAEA,cAAc,QAAwC;AACpD,YAAM,gBAAyC,CAAA;AAE/C,UAAI,OAAO,cAAc;AACvB,sBAAc,eAAe,OAAO;AAAA,MACtC;AAEA,UAAI,OAAO,OAAO;AAChB,sBAAc,QAAQ,OAAO;AAAA,MAC/B;AAEA,UAAI,OAAO,SAAS;AAClB,YAAI,OAAO,YAAY,YAAY;AACjC,wBAAc,iBAAiB;AAAA,YAC7B,MAAM;AAAA,YACN,WAAW,OAAO,qBAAqB;AAAA,UAAA;AAAA,QAE3C,WAAW,OAAO,YAAY,UAAU;AACtC,wBAAc,iBAAiB;AAAA,YAC7B,MAAM;AAAA,YACN,WAAW,OAAO,WAAW,aAAa;AAAA,YAC1C,mBAAmB,OAAO,WAAW,mBAAmB;AAAA,YACxD,qBAAqB,OAAO,WAAW,qBAAqB;AAAA,UAAA;AAAA,QAEhE,OAAO;AACL,wBAAc,iBAAiB;AAAA,QACjC;AAAA,MACF;AAEA,UAAI,OAAO,UAAU,QAAW;AAC9B,sBAAc,QAAQ,OAAO,MAAM,IAAI,CAAC,OAAO;AAAA,UAC7C,MAAM;AAAA,UACN,MAAM,EAAE;AAAA,UACR,aAAa,EAAE;AAAA,UACf,YAAY,EAAE,eAAe,EAAE,MAAM,UAAU,YAAY,CAAA,EAAC;AAAA,QAAE,EAC9D;AACF,sBAAc,cAAc;AAAA,MAC9B;AAEA,UAAI,OAAO,kBAAkB;AAC3B,sBAAc,aAAa,OAAO;AAAA,MACpC;AAEA,UAAI,OAAO,gBAAgB,QAAW;AACpC,sBAAc,cAAc,OAAO;AAAA,MACrC;AAEA,UAAI,OAAO,oBAAoB,QAAW;AACxC,sBAAc,6BAA6B,OAAO;AAAA,MACpD;AAMA,YAAM,kBACJ,OAAO,mBAAmB,CAAA;AAC5B,YAAM,sBACJ,6BAA6B,kBACzB,gBAAgB,0BAChB,+BAA+B,kBAC7B,gBAAgB,4BAChB;AACR,UAAI,wBAAwB,QAAW;AACrC,sBAAc,4BACZ,wBAAwB,QAAQ,OAAO;AAAA,MAC3C,WAAW,CAAC,6BAA6B;AACvC,sBAAc,4BAA4B,EAAE,OAAO,WAAA;AAAA,MACrD;AAEA,UAAI,OAAO,KAAK,aAAa,EAAE,SAAS,GAAG;AACzC,kBAAU;AAAA,UACR,MAAM;AAAA,UACN,SAAS;AAAA,QAAA,CACV;AACD,sCAA8B;AAAA,MAChC;AAAA,IACF;AAAA,IAEA,YAAY;AACV,gBAAU,EAAE,MAAM,mBAAmB;AACrC,oBAAc;AACd,WAAK,eAAe,EAAE,MAAM,YAAA,CAAa;AACzC,WAAK,eAAe,EAAE,WAAW,oBAAoB,QAAW;AAAA,IAClE;AAAA,IAEA,GACE,OACA,SACY;AACZ,UAAI,CAAC,cAAc,IAAI,KAAK,GAAG;AAC7B,sBAAc,IAAI,OAAO,oBAAI,IAAA,CAAK;AAAA,MACpC;AACA,oBAAc,IAAI,KAAK,EAAG,IAAI,OAAO;AAErC,aAAO,MAAM;AACX,sBAAc,IAAI,KAAK,GAAG,OAAO,OAAO;AAAA,MAC1C;AAAA,IACF;AAAA,IAEA,wBAA4C;AAC1C,eAAS,eAAe,UAAgC;AACtD,cAAM,OAAO,IAAI,WAAW,SAAS,OAAO;AAC5C,iBAAS,sBAAsB,IAAI;AAEnC,YAAI,eAAe;AACnB,mBAAW,UAAU,MAAM;AACzB,gBAAM,YAAY,KAAK,IAAI,SAAS,GAAG;AACvC,cAAI,YAAY,cAAc;AAC5B,2BAAe;AAAA,UACjB;AAAA,QACF;AAEA,cAAM,aAAa,eAAe;AAClC,eAAO,KAAK,IAAI,GAAG,aAAa,GAAG;AAAA,MACrC;AAEA,aAAO;AAAA,QACL,IAAI,aAAa;AACf,cAAI,CAAC,cAAe,QAAO;AAC3B,iBAAO,eAAe,aAAa;AAAA,QACrC;AAAA,QAEA,IAAI,cAAc;AAChB,cAAI,CAAC,eAAgB,QAAO;AAC5B,iBAAO,eAAe,cAAc;AAAA,QACtC;AAAA,QAEA,wBAAwB;AACtB,cAAI,CAAC;AACH,mBAAO,IAAI,WAAW,4BAA4B;AACpD,gBAAM,OAAO,IAAI,WAAW,cAAc,iBAAiB;AAC3D,wBAAc,qBAAqB,IAAI;AACvC,iBAAO;AAAA,QACT;AAAA,QAEA,yBAAyB;AACvB,cAAI,CAAC;AACH,mBAAO,IAAI,WAAW,4BAA4B;AACpD,gBAAM,OAAO,IAAI,WAAW,eAAe,iBAAiB;AAC5D,yBAAe,qBAAqB,IAAI;AACxC,iBAAO;AAAA,QACT;AAAA,QAEA,yBAAyB;AACvB,cAAI,CAAC;AACH,mBAAO,IAAI,WAAW,yBAAyB,EAAE;AAAA,cAC/C;AAAA,YAAA;AAEJ,gBAAM,OAAO,IAAI,WAAW,cAAc,OAAO;AACjD,wBAAc,sBAAsB,IAAI;AACxC,iBAAO;AAAA,QACT;AAAA,QAEA,0BAA0B;AACxB,cAAI,CAAC;AACH,mBAAO,IAAI,WAAW,yBAAyB,EAAE;AAAA,cAC/C;AAAA,YAAA;AAEJ,gBAAM,OAAO,IAAI,WAAW,eAAe,OAAO;AAClD,yBAAe,sBAAsB,IAAI;AACzC,iBAAO;AAAA,QACT;AAAA,QAEA,IAAI,kBAAkB;AACpB,iBAAO;AAAA,QACT;AAAA,QAEA,IAAI,mBAAmB;AACrB,iBAAO;AAAA,QACT;AAAA,MAAA;AAAA,IAEJ;AAAA,EAAA;AAKF,SAAO;AACT;"}
|
|
1
|
+
{"version":3,"file":"adapter.js","sources":["../../../src/realtime/adapter.ts"],"sourcesContent":["import { resolveDebugOption } from '@tanstack/ai/adapter-internals'\nimport type {\n AnyClientTool,\n AudioVisualization,\n RealtimeEvent,\n RealtimeEventHandler,\n RealtimeMessage,\n RealtimeMode,\n RealtimeSessionConfig,\n RealtimeStatus,\n RealtimeToken,\n} from '@tanstack/ai'\nimport type { InternalLogger } from '@tanstack/ai/adapter-internals'\nimport type { RealtimeAdapter, RealtimeConnection } from './realtime-contract'\nimport type { GrokRealtimeOptions } from './types'\n\nconst GROK_REALTIME_URL = 'https://api.x.ai/v1/realtime'\n\n/**\n * Runtime-checked field readers for untyped server events. Replace the\n * drive-by `event.X as string` / `event.X as Record<string, unknown>` casts\n * with readers that return `undefined` when the shape doesn't match, so a\n * malformed frame can't throw a TypeError inside `handleServerEvent`.\n */\nfunction readString(\n obj: Record<string, unknown>,\n key: string,\n): string | undefined {\n const value = obj[key]\n return typeof value === 'string' ? value : undefined\n}\n\nfunction readObject(\n obj: Record<string, unknown>,\n key: string,\n): Record<string, unknown> | undefined {\n const value = obj[key]\n return value && typeof value === 'object' && !Array.isArray(value)\n ? (value as Record<string, unknown>)\n : undefined\n}\n\nfunction readObjectArray(\n obj: Record<string, unknown>,\n key: string,\n): Array<Record<string, unknown>> | undefined {\n const value = obj[key]\n if (!Array.isArray(value)) return undefined\n return value.filter(\n (item): item is Record<string, unknown> =>\n item !== null && typeof item === 'object' && !Array.isArray(item),\n )\n}\n\ntype RealtimeServerError = Error & {\n code?: string\n type?: string\n param?: string\n}\n\n/**\n * Creates a Grok realtime adapter for client-side use.\n *\n * Uses WebRTC for browser connections (default). Mirrors the OpenAI realtime\n * adapter because xAI's Voice Agent API is OpenAI-realtime-compatible — the\n * only differences are the endpoint URL and default model.\n *\n * @example\n * ```typescript\n * import { RealtimeClient } from '@tanstack/ai-client'\n * import { grokRealtime } from '@tanstack/ai-grok'\n *\n * const client = new RealtimeClient({\n * getToken: () => fetch('/api/realtime-token').then(r => r.json()),\n * adapter: grokRealtime(),\n * })\n * ```\n */\nexport function grokRealtime(\n options: GrokRealtimeOptions = {},\n): RealtimeAdapter {\n const connectionMode = options.connectionMode ?? 'webrtc'\n const logger = resolveDebugOption(options.debug)\n\n return {\n provider: 'grok',\n\n async connect(\n token: RealtimeToken,\n _clientTools?: ReadonlyArray<AnyClientTool>,\n ): Promise<RealtimeConnection> {\n const model = token.config.model ?? 'grok-voice-fast-1.0'\n logger.request(`activity=realtime provider=grok model=${model}`, {\n provider: 'grok',\n model,\n })\n\n if (connectionMode === 'webrtc') {\n return createWebRTCConnection(token, logger)\n }\n const error = new Error('WebSocket connection mode not yet implemented')\n logger.errors('grok.realtime fatal', {\n error,\n source: 'grok.realtime',\n })\n throw error\n },\n }\n}\n\n/**\n * Creates a WebRTC connection to xAI's realtime API.\n */\nasync function createWebRTCConnection(\n token: RealtimeToken,\n logger: InternalLogger,\n): Promise<RealtimeConnection> {\n const model = token.config.model ?? 'grok-voice-fast-1.0'\n const eventHandlers = new Map<RealtimeEvent, Set<RealtimeEventHandler<any>>>()\n\n const pc = new RTCPeerConnection()\n\n let audioContext: AudioContext | null = null\n let inputAnalyser: AnalyserNode | null = null\n let outputAnalyser: AnalyserNode | null = null\n let inputSource: MediaStreamAudioSourceNode | null = null\n let outputSource: MediaStreamAudioSourceNode | null = null\n let localStream: MediaStream | null = null\n\n let audioElement: HTMLAudioElement | null = null\n\n // Captured into a const so closures see a non-nullable reference (teardown\n // re-points the outer `dataChannel` to null, but in-flight closures still\n // need to close their own channel).\n const channel = pc.createDataChannel('oai-events')\n let dataChannel: RTCDataChannel | null = channel\n\n let currentMode: RealtimeMode = 'idle'\n let currentMessageId: string | null = null\n\n // Flipped by `teardownConnection`. Guards `sendEvent` so post-disconnect\n // calls (e.g. a React `useEffect` cleanup flushing queued events) are\n // logged and skipped instead of silently piling up in `pendingEvents`.\n let isTornDown = false\n\n // Outbound events queued while the data channel isn't yet open. Declared\n // here (rather than next to `sendEvent`) so `teardownConnection` — which\n // lives higher up and can run from the SDP-path catch before `sendEvent`\n // is defined — can drain it without hitting the TDZ.\n const pendingEvents: Array<Record<string, unknown>> = []\n\n // Tracks whether we've sent the first session.update. On the first update\n // we attach a default input_audio_transcription so the server will emit\n // user transcripts unless the caller opts out via\n // `providerOptions.inputAudioTranscription = null | false`.\n let hasSentInitialSessionUpdate = false\n\n // Size hints for the fallback buffers returned when an analyser isn't yet\n // populated. We return a *fresh* `Uint8Array` on each call so a caller\n // that draws into it (e.g. a canvas visualiser zeroing the buffer) can't\n // mutate a shared module-level instance for every other consumer.\n const FALLBACK_FREQUENCY_BIN_COUNT = 1024\n const FALLBACK_TIME_DOMAIN_SIZE = 2048\n const FALLBACK_TIME_DOMAIN_FILL = 128\n\n function emit<TEvent extends RealtimeEvent>(\n event: TEvent,\n payload: Parameters<RealtimeEventHandler<TEvent>>[0],\n ) {\n const handlers = eventHandlers.get(event)\n if (handlers) {\n for (const handler of handlers) {\n handler(payload)\n }\n }\n }\n\n let dataChannelOpened = false\n let rejectDataChannelReady: ((reason: unknown) => void) | null = null\n let dataChannelReadyTimeout: ReturnType<typeof setTimeout> | null = null\n\n const dataChannelReady = new Promise<void>((resolve, reject) => {\n rejectDataChannelReady = (reason) => {\n if (dataChannelReadyTimeout !== null) {\n clearTimeout(dataChannelReadyTimeout)\n dataChannelReadyTimeout = null\n }\n // One-shot: null out so later state transitions don't reject twice.\n rejectDataChannelReady = null\n reject(reason)\n }\n\n dataChannelReadyTimeout = setTimeout(() => {\n if (!dataChannelOpened) {\n rejectDataChannelReady?.(\n new Error(\n 'Data channel did not open within 15000ms — aborting connection',\n ),\n )\n }\n }, 15000)\n\n channel.onopen = () => {\n dataChannelOpened = true\n if (dataChannelReadyTimeout !== null) {\n clearTimeout(dataChannelReadyTimeout)\n dataChannelReadyTimeout = null\n }\n // Once resolved, rejecting is a no-op — null out so teardown paths\n // don't attempt a redundant reject on an already-settled promise.\n rejectDataChannelReady = null\n flushPendingEvents()\n emit('status_change', { status: 'connected' as RealtimeStatus })\n resolve()\n }\n })\n\n channel.onmessage = (event) => {\n try {\n const message = JSON.parse(event.data)\n const messageRecord: Record<string, unknown> =\n message !== null && typeof message === 'object' ? message : {}\n logger.provider(\n `provider=grok direction=in type=${readString(messageRecord, 'type') ?? '<unknown>'}`,\n { frame: messageRecord },\n )\n handleServerEvent(messageRecord)\n } catch (parseErr) {\n logger.errors('grok.realtime fatal', {\n error: parseErr,\n source: 'grok.realtime',\n })\n emit('error', {\n error:\n parseErr instanceof Error ? parseErr : new Error(String(parseErr)),\n })\n }\n }\n\n channel.onerror = (error) => {\n // Closing the peer connection cascades into `onerror`/`onclose` on the\n // data channel. Once teardown has started, re-surfacing those as\n // `emit('error')` is noise that confuses consumers (they just called\n // `disconnect()` — they don't want an error toast for it).\n if (isTornDown) return\n logger.errors('grok.realtime fatal', {\n error,\n source: 'grok.realtime',\n })\n // RTCErrorEvent exposes a typed `.error`; fall back to the event type\n // name, then to a string representation, so the emitted error message\n // doesn't end up as \"[object Event]\".\n // `onerror` always fires with an Event (often an RTCErrorEvent), so we\n // can read it via the untyped helpers without first proving object-ness.\n // eslint-disable-next-line no-restricted-syntax -- RTCErrorEvent is a typed DOM class that does not structurally overlap Record<string, unknown>; we duck-type it via readObject/readString\n const errorRecord = error as unknown as Record<string, unknown>\n const rtcError = readObject(errorRecord, 'error')\n const msg =\n (rtcError && readString(rtcError, 'message')) ?? (error.type || 'unknown')\n const dcErr = new Error(`Data channel error: ${msg}`)\n if (!dataChannelOpened) {\n rejectDataChannelReady?.(dcErr)\n }\n emit('error', { error: dcErr })\n }\n\n channel.onclose = () => {\n // Same rationale as `onerror` above: `pc.close()` during teardown\n // cascades to the data channel's `onclose`. If we've already started\n // teardown, there's nothing to do here.\n if (isTornDown) return\n if (!dataChannelOpened) {\n rejectDataChannelReady?.(new Error('Data channel closed before opening'))\n }\n }\n\n pc.ontrack = (event) => {\n if (event.track.kind === 'audio' && event.streams[0]) {\n setupOutputAudioAnalysis(event.streams[0])\n }\n }\n\n // `status_change` has a single source of truth: `onconnectionstatechange`\n // (the higher-level aggregate state). `oniceconnectionstatechange` is\n // responsible only for rejecting `dataChannelReady` on ICE failures so we\n // surface them without waiting for the 15s timeout.\n pc.onconnectionstatechange = () => {\n const state = pc.connectionState\n logger.provider(`provider=grok pc.connectionState=${state}`, {\n state,\n })\n if (state === 'failed' || state === 'disconnected' || state === 'closed') {\n // Suppress the `status_change` emission when teardown is in progress:\n // the user-facing `disconnect()` already emits `status_change: 'idle'`\n // and then calls `teardownConnection()` → `pc.close()`, which fires\n // `onconnectionstatechange` with state === 'closed'. Without this\n // guard listeners would see two `idle` events per disconnect.\n if (!isTornDown) {\n emit('status_change', {\n status:\n state === 'failed'\n ? ('error' as RealtimeStatus)\n : ('idle' as RealtimeStatus),\n })\n }\n if (!dataChannelOpened) {\n // Reject on any terminal-ish pre-open state so callers don't hang\n // for the full 15s timeout. The reject is one-shot — subsequent\n // state changes become no-ops via the null-out in\n // `rejectDataChannelReady`.\n const message =\n state === 'failed'\n ? `PeerConnection failed before data channel opened`\n : `PeerConnection entered state '${state}' before data channel opened`\n rejectDataChannelReady?.(new Error(message))\n }\n // Auto-teardown on `failed`: without this the mic track, pc, and\n // AudioContext stay allocated after a fatal connection failure, so the\n // browser's mic indicator stays on and the user sees a broken\n // \"connected mic\" state. `closed` already means pc was torn down\n // (usually by teardownConnection itself) so nothing extra to do.\n // `disconnected` is transient per the WebRTC spec and may recover, so\n // we leave resources in place. `teardownConnection` is idempotent so\n // a subsequent consumer `disconnect()` remains safe.\n if (state === 'failed' && !isTornDown) {\n void teardownConnection()\n }\n }\n }\n\n pc.oniceconnectionstatechange = () => {\n const state = pc.iceConnectionState\n logger.provider(`provider=grok pc.iceConnectionState=${state}`, {\n state,\n })\n if (\n !dataChannelOpened &&\n (state === 'failed' || state === 'closed' || state === 'disconnected')\n ) {\n const message =\n state === 'failed'\n ? `ICE connection failed before data channel opened`\n : `ICE connection entered state '${state}' before data channel opened`\n rejectDataChannelReady?.(new Error(message))\n }\n }\n\n /**\n * Tear down every resource we may have allocated so the mic/pc/audio\n * nodes/audio element don't leak on a failed connect. Safe to call from\n * any point after `new RTCPeerConnection()`; each branch null-guards and\n * swallows errors because cascading closes (e.g. `pc.close()` closing the\n * data channel implicitly) are expected.\n *\n * Shared between the SDP-path catch, the post-SDP catch, and (implicitly\n * via idempotency) the `disconnect()` entry point.\n */\n async function teardownConnection() {\n // Flip the teardown flag BEFORE any awaits so handlers that fire during\n // `await audioContext.close()` (or any other async step below) can guard\n // on it — otherwise a late `pc.onconnectionstatechange` or `pc.ontrack`\n // can allocate new resources or re-emit `status_change: idle` after the\n // user-facing `disconnect()` already emitted one.\n isTornDown = true\n\n // Drop any queued events the caller sent before the data channel opened\n // up front. Without this they'd accumulate across reconnect attempts\n // (each connect allocates a fresh closure, but a caller holding the old\n // `connection` reference could otherwise keep appending forever). Done\n // at the top — before the awaits below — so `sendEvent` calls racing\n // with teardown don't push into a list we're about to drain.\n pendingEvents.length = 0\n\n // Clear the data-channel-open timeout / reject the readiness promise\n // if it's still pending. `rejectDataChannelReady` is one-shot and nulls\n // itself on first call, so calling it from `disconnect()` after a\n // successful open is a no-op.\n rejectDataChannelReady?.(\n new Error('Connection torn down before data channel opened'),\n )\n\n if (localStream) {\n for (const track of localStream.getTracks()) {\n track.stop()\n }\n localStream = null\n }\n\n // Output audio (populated by `pc.ontrack` → setupOutputAudioAnalysis,\n // which may have fired during SDP negotiation before we threw).\n if (audioElement) {\n try {\n audioElement.pause()\n } catch {\n // ignore — element may already be unloaded\n }\n audioElement.srcObject = null\n audioElement = null\n }\n if (outputSource) {\n try {\n outputSource.disconnect()\n } catch {\n // ignore\n }\n outputSource = null\n }\n if (outputAnalyser) {\n try {\n outputAnalyser.disconnect()\n } catch {\n // ignore\n }\n outputAnalyser = null\n }\n\n // Input audio (populated by setupInputAudioAnalysis after SDP).\n if (inputSource) {\n try {\n inputSource.disconnect()\n } catch {\n // ignore\n }\n inputSource = null\n }\n if (inputAnalyser) {\n try {\n inputAnalyser.disconnect()\n } catch {\n // ignore\n }\n inputAnalyser = null\n }\n\n if (dataChannel) {\n try {\n dataChannel.close()\n } catch {\n // ignore — channel may already be closed by pc.close()\n }\n dataChannel = null\n }\n\n try {\n pc.close()\n } catch {\n // ignore — pc may already be closed\n }\n\n if (audioContext) {\n try {\n await audioContext.close()\n } catch {\n // ignore — context may already be closed\n }\n audioContext = null\n }\n }\n\n // xAI requires an audio track in the SDP offer, same as OpenAI realtime.\n //\n // This try/catch also covers `getUserMedia` failure (e.g. the user denies\n // microphone permission). `pc` + `dataChannel` are already allocated above\n // and the 15s `dataChannelReady` timeout is already armed, so we MUST\n // teardown on failure here — otherwise they leak until the tab closes.\n // `teardownConnection` is idempotent and null-safe (runs fine even if the\n // mic was never acquired).\n try {\n try {\n localStream = await navigator.mediaDevices.getUserMedia({\n audio: {\n echoCancellation: true,\n noiseSuppression: true,\n sampleRate: 24000,\n },\n })\n } catch (error) {\n logger.errors('grok.realtime fatal', {\n error,\n source: 'grok.realtime.getUserMedia',\n })\n // Re-throw with the descriptive message callers rely on. Teardown runs\n // in the outer catch below.\n throw new Error(\n `Microphone access required for realtime voice: ${error instanceof Error ? error.message : error}`,\n )\n }\n\n for (const track of localStream.getAudioTracks()) {\n pc.addTrack(track, localStream)\n }\n\n const offer = await pc.createOffer()\n await pc.setLocalDescription(offer)\n\n const sdpResponse = await fetch(`${GROK_REALTIME_URL}?model=${model}`, {\n method: 'POST',\n headers: {\n Authorization: `Bearer ${token.token}`,\n 'Content-Type': 'application/sdp',\n },\n body: offer.sdp,\n })\n\n if (!sdpResponse.ok) {\n const errorText = await sdpResponse.text()\n const error = new Error(\n `Failed to establish WebRTC connection: ${sdpResponse.status} - ${errorText}`,\n )\n logger.errors('grok.realtime fatal', {\n error,\n source: 'grok.realtime.sdp',\n status: sdpResponse.status,\n })\n throw error\n }\n\n const answerSdp = await sdpResponse.text()\n await pc.setRemoteDescription({ type: 'answer', sdp: answerSdp })\n } catch (err) {\n await teardownConnection()\n throw err\n }\n\n // Second cleanup scope: after SDP succeeds we still have to set up input\n // audio analysis and wait for the data channel to open. Both can fail\n // (AudioContext allocation, 15s timeout, ICE failure, pc.close from the\n // other end, etc.) and those failures must NOT leave the mic/pc/audio\n // nodes running.\n try {\n setupInputAudioAnalysis(localStream)\n await dataChannelReady\n } catch (err) {\n await teardownConnection()\n throw err\n }\n\n function handleServerEvent(event: Record<string, unknown>) {\n const type = readString(event, 'type')\n\n switch (type) {\n case 'session.created':\n case 'session.updated':\n break\n\n case 'input_audio_buffer.speech_started':\n currentMode = 'listening'\n emit('mode_change', { mode: 'listening' })\n break\n\n case 'input_audio_buffer.speech_stopped':\n currentMode = 'thinking'\n emit('mode_change', { mode: 'thinking' })\n break\n\n case 'input_audio_buffer.committed':\n break\n\n case 'conversation.item.input_audio_transcription.completed': {\n const transcript = readString(event, 'transcript')\n if (transcript === undefined) break\n emit('transcript', { role: 'user', transcript, isFinal: true })\n break\n }\n\n case 'response.created':\n // Reset message id so a tool-only response (which never emits\n // response.output_item.added for a message) can't reuse the previous\n // turn's id when `response.done` later inspects this flag.\n currentMessageId = null\n currentMode = 'thinking'\n emit('mode_change', { mode: 'thinking' })\n break\n\n case 'response.output_item.added': {\n const item = readObject(event, 'item')\n if (item && readString(item, 'type') === 'message') {\n const id = readString(item, 'id')\n if (id !== undefined) currentMessageId = id\n }\n break\n }\n\n // xAI realtime per docs uses `response.output_audio_transcript.*`;\n // accept the legacy OpenAI-realtime `response.audio_transcript.*` as\n // an alias so this adapter stays compatible across protocol versions.\n case 'response.output_audio_transcript.delta':\n case 'response.audio_transcript.delta': {\n const delta = readString(event, 'delta')\n if (delta === undefined) break\n emit('transcript', {\n role: 'assistant',\n transcript: delta,\n isFinal: false,\n })\n break\n }\n\n case 'response.output_audio_transcript.done':\n case 'response.audio_transcript.done': {\n const transcript = readString(event, 'transcript')\n if (transcript === undefined) break\n emit('transcript', { role: 'assistant', transcript, isFinal: true })\n break\n }\n\n // xAI realtime per docs uses `response.text.*`; accept the legacy\n // OpenAI-realtime `response.output_text.*` as an alias.\n case 'response.text.delta':\n case 'response.output_text.delta': {\n const delta = readString(event, 'delta')\n if (delta === undefined) break\n emit('transcript', {\n role: 'assistant',\n transcript: delta,\n isFinal: false,\n })\n break\n }\n\n case 'response.text.done':\n case 'response.output_text.done': {\n const text = readString(event, 'text')\n if (text === undefined) break\n emit('transcript', {\n role: 'assistant',\n transcript: text,\n isFinal: true,\n })\n break\n }\n\n // xAI realtime per docs uses `response.output_audio.*`; accept the\n // legacy OpenAI-realtime `response.audio.*` as an alias.\n case 'response.output_audio.delta':\n case 'response.audio.delta':\n if (currentMode !== 'speaking') {\n currentMode = 'speaking'\n emit('mode_change', { mode: 'speaking' })\n }\n break\n\n case 'response.output_audio.done':\n case 'response.audio.done':\n break\n\n case 'response.function_call_arguments.done': {\n // Only `call_id` is valid for `sendToolResult` correlation. Falling\n // back to `item_id` would produce a tool-call id the server doesn't\n // recognise when the result is posted back, silently dropping the\n // tool execution. If `call_id` is missing we surface an error event\n // so the UI can react instead of pretending the tool call succeeded.\n const callId = readString(event, 'call_id')\n const name = readString(event, 'name') ?? ''\n const args = readString(event, 'arguments') ?? ''\n if (!callId) {\n logger.errors(\n 'grok.realtime tool_call missing call_id — dropping tool_call',\n {\n source: 'grok.realtime',\n event_type: 'response.function_call_arguments.done',\n item_id: event.item_id,\n },\n )\n emit('error', {\n error: new Error(\n 'Realtime tool call missing call_id; tool will not execute',\n ),\n })\n break\n }\n try {\n const input = JSON.parse(args)\n emit('tool_call', { toolCallId: callId, toolName: name, input })\n } catch {\n emit('tool_call', { toolCallId: callId, toolName: name, input: args })\n }\n break\n }\n\n case 'response.done': {\n const response = readObject(event, 'response') ?? {}\n const output = readObjectArray(response, 'output')\n\n // Only transition back to `listening` if the user hasn't already\n // stopped capture — otherwise we'd override their explicit `idle`\n // state and re-arm the mic visualisation.\n if (currentMode !== 'idle') {\n currentMode = 'listening'\n emit('mode_change', { mode: 'listening' })\n }\n\n if (currentMessageId) {\n const message: RealtimeMessage = {\n id: currentMessageId,\n role: 'assistant',\n timestamp: Date.now(),\n parts: [],\n }\n\n for (const item of output ?? []) {\n if (readString(item, 'type') !== 'message') continue\n const content = readObjectArray(item, 'content')\n if (!content) continue\n for (const part of content) {\n const partType = readString(part, 'type')\n if (partType === 'audio') {\n const transcript = readString(part, 'transcript')\n if (transcript) {\n message.parts.push({ type: 'audio', transcript })\n }\n } else if (partType === 'text') {\n const content = readString(part, 'text')\n if (content) {\n message.parts.push({ type: 'text', content })\n }\n }\n }\n }\n\n emit('message_complete', { message })\n currentMessageId = null\n }\n break\n }\n\n case 'conversation.item.truncated':\n // Assistant playback was interrupted — flip mode back to `listening`\n // unless the user already called `stopAudioCapture()` (idle). Without\n // this the visualisation would stay stuck on `speaking` even though\n // no audio is playing.\n if (currentMode !== 'idle') {\n currentMode = 'listening'\n emit('mode_change', { mode: 'listening' })\n }\n emit('interrupted', {\n ...(currentMessageId !== null && { messageId: currentMessageId }),\n })\n break\n\n case 'error': {\n // The realtime server's `error` envelope isn't guaranteed to carry\n // an `error` object at all (network-layer corruption, protocol\n // drift, etc.). Validate shape before dereferencing so a malformed\n // payload can't throw a TypeError inside this handler and stop the\n // switch from running for the rest of the session.\n const errorObj = readObject(event, 'error') ?? {}\n const message =\n readString(errorObj, 'message') ?? 'Unknown realtime server error'\n const err: RealtimeServerError = new Error(message)\n // Preserve `code` / `type` / `param` on the Error as extra props so\n // consumers can branch on them without re-parsing the raw event.\n const code = readString(errorObj, 'code')\n if (code !== undefined) err.code = code\n const errType = readString(errorObj, 'type')\n if (errType !== undefined) err.type = errType\n const param = readString(errorObj, 'param')\n if (param !== undefined) err.param = param\n logger.errors('grok.realtime server error', {\n ...errorObj,\n source: 'grok.realtime server',\n })\n emit('error', { error: err })\n break\n }\n\n case undefined:\n default:\n // The xAI realtime protocol is a moving target; log unhandled event\n // types at provider level so they're visible during debugging without\n // emitting a user-visible error. `undefined` shares the bucket because\n // a malformed event without a `type` field is just as unhandleable.\n logger.provider('grok.realtime unhandled server event', {\n type: event.type,\n })\n break\n }\n }\n\n function setupOutputAudioAnalysis(stream: MediaStream) {\n // Bail out if teardown has already started. `pc.ontrack` can fire\n // asynchronously after `teardownConnection()` has flipped `isTornDown`\n // (e.g. a remote track arriving mid-close); without this guard we'd\n // allocate a fresh AudioContext / audio element that nothing would ever\n // clean up.\n if (isTornDown) return\n\n // Tear down any prior output audio before allocating new resources.\n // `pc.ontrack` can fire multiple times over the lifetime of a session\n // (e.g. after renegotiation), and without this we'd leak audio elements\n // and analyser nodes.\n if (audioElement) {\n try {\n audioElement.pause()\n } catch {\n // ignore — element may already be unloaded\n }\n audioElement.srcObject = null\n audioElement = null\n }\n if (outputSource) {\n try {\n outputSource.disconnect()\n } catch {\n // ignore — may already be disconnected\n }\n outputSource = null\n }\n if (outputAnalyser) {\n try {\n outputAnalyser.disconnect()\n } catch {\n // ignore\n }\n outputAnalyser = null\n }\n\n audioElement = new Audio()\n audioElement.srcObject = stream\n audioElement.autoplay = true\n audioElement.play().catch((e) => {\n // Autoplay is commonly blocked until the user interacts with the page\n // (browser gesture requirement). Surfacing this as a fatal `error`\n // event makes the UI render a red/error state even though the\n // connection is healthy — the page just needs a click. Log at a\n // dedicated source tag so it's debuggable, but don't emit `error`.\n logger.errors('grok.realtime audio autoplay blocked', {\n error: e,\n source: 'grok.realtime.audio_permission_required',\n })\n })\n\n if (!audioContext) {\n audioContext = new AudioContext()\n }\n\n if (audioContext.state === 'suspended') {\n audioContext.resume().catch((err) => {\n // Same rationale as the autoplay catch: `resume()` failure usually\n // means the user hasn't interacted yet. Logging only — no error\n // emit — so the UI doesn't go into a fatal state for a recoverable\n // condition.\n logger.errors('grok.realtime audioContext.resume failed', {\n error: err,\n source: 'grok.realtime',\n })\n })\n }\n\n outputAnalyser = audioContext.createAnalyser()\n outputAnalyser.fftSize = 2048\n outputAnalyser.smoothingTimeConstant = 0.3\n\n outputSource = audioContext.createMediaStreamSource(stream)\n outputSource.connect(outputAnalyser)\n }\n\n function setupInputAudioAnalysis(stream: MediaStream) {\n // Defensive symmetry with `setupOutputAudioAnalysis`. Today this is\n // only called inline after SDP negotiation, but keeping the guard\n // means any future caller path (e.g. renegotiation) won't leak a fresh\n // AudioContext after teardown.\n if (isTornDown) return\n\n if (!audioContext) {\n audioContext = new AudioContext()\n }\n\n if (audioContext.state === 'suspended') {\n audioContext.resume().catch((err) => {\n // Same rationale as in setupOutputAudioAnalysis: a suspended\n // AudioContext usually resumes after a user gesture. Log only —\n // surfacing this as a fatal error makes the UI look broken for a\n // recoverable condition.\n logger.errors('grok.realtime audioContext.resume failed', {\n error: err,\n source: 'grok.realtime',\n })\n })\n }\n\n inputAnalyser = audioContext.createAnalyser()\n inputAnalyser.fftSize = 2048\n inputAnalyser.smoothingTimeConstant = 0.3\n\n inputSource = audioContext.createMediaStreamSource(stream)\n inputSource.connect(inputAnalyser)\n }\n\n function sendEvent(event: Record<string, unknown>) {\n if (isTornDown) {\n // The caller is holding onto a `connection` object after `disconnect()`\n // (or a failed connect). Silently queueing would leak memory and the\n // events would never flush. Log + drop so the misuse is visible in\n // debug mode without escalating to a throw — throwing from a React\n // useEffect cleanup path can break teardown ordering in the UI.\n logger.errors('grok.realtime sendEvent after disconnect', {\n eventType: readString(event, 'type') ?? '<unknown>',\n source: 'grok.realtime',\n })\n return\n }\n if (dataChannel?.readyState === 'open') {\n logger.provider(\n `provider=grok direction=out type=${readString(event, 'type') ?? '<unknown>'}`,\n { frame: event },\n )\n // Mirror the try/catch in `flushPendingEvents` — `dataChannel.send`\n // can synchronously throw if the channel flipped to `closing` between\n // our readyState check and this call, or if `JSON.stringify` chokes\n // on a caller-supplied payload. Log + emit error instead of letting\n // the exception propagate up through public `sendText` / `sendImage`\n // / `updateSession` call sites.\n try {\n dataChannel.send(JSON.stringify(event))\n } catch (error) {\n logger.errors('grok.realtime sendEvent failed', {\n error,\n eventType: readString(event, 'type') ?? '<unknown>',\n source: 'grok.realtime',\n })\n emit('error', {\n error: error instanceof Error ? error : new Error(String(error)),\n })\n }\n } else {\n pendingEvents.push(event)\n }\n }\n\n function flushPendingEvents() {\n try {\n for (const event of pendingEvents) {\n logger.provider(\n `provider=grok direction=out type=${readString(event, 'type') ?? '<unknown>'}`,\n { frame: event },\n )\n channel.send(JSON.stringify(event))\n }\n pendingEvents.length = 0\n } catch (error) {\n // A send failure here (e.g. dataChannel went from 'open' back to\n // 'closing' mid-flush, or JSON.stringify on a caller-provided event\n // threw) would otherwise be silently swallowed. By the time we're\n // called, `onopen` has already resolved `dataChannelReady`, so the\n // consumer-facing signal is `emit('error')` — try rejectDataChannelReady\n // as a defensive belt-and-braces in case this ever runs pre-resolve.\n logger.errors('grok.realtime flushPendingEvents failed', {\n error,\n source: 'grok.realtime',\n })\n const err = error instanceof Error ? error : new Error(String(error))\n rejectDataChannelReady?.(err)\n emit('error', { error: err })\n }\n }\n\n const connection: RealtimeConnection = {\n async disconnect() {\n // Reuse the same teardown path as the failed-connect branches so\n // every cleanup site stays in sync (input analyser, output analyser,\n // output source, audio element, etc.).\n await teardownConnection()\n emit('status_change', { status: 'idle' as RealtimeStatus })\n },\n\n async startAudioCapture() {\n if (localStream) {\n for (const track of localStream.getAudioTracks()) {\n track.enabled = true\n }\n }\n currentMode = 'listening'\n emit('mode_change', { mode: 'listening' })\n },\n\n stopAudioCapture() {\n if (localStream) {\n for (const track of localStream.getAudioTracks()) {\n track.enabled = false\n }\n }\n currentMode = 'idle'\n emit('mode_change', { mode: 'idle' })\n },\n\n sendText(text: string) {\n sendEvent({\n type: 'conversation.item.create',\n item: {\n type: 'message',\n role: 'user',\n content: [{ type: 'input_text', text }],\n },\n })\n sendEvent({ type: 'response.create' })\n },\n\n sendImage(imageData: string, mimeType: string) {\n // Accept:\n // - http(s):// URLs → forward as-is\n // - data: URIs (e.g. from FileReader.readAsDataURL) → forward as-is\n // so we don't double-wrap into `data:image/png;base64,data:image/png;base64,…`\n // - bare base64 → wrap in `data:${mimeType};base64,…`\n const isAlreadyUrlOrDataUri =\n imageData.startsWith('http://') ||\n imageData.startsWith('https://') ||\n imageData.startsWith('data:')\n const imageContent = {\n type: 'input_image',\n // The OpenAI-realtime content part (which this adapter mirrors) nests\n // the URL under an `image_url: { url: ... }` object, not a bare\n // string.\n image_url: {\n url: isAlreadyUrlOrDataUri\n ? imageData\n : `data:${mimeType};base64,${imageData}`,\n },\n }\n\n sendEvent({\n type: 'conversation.item.create',\n item: {\n type: 'message',\n role: 'user',\n content: [imageContent],\n },\n })\n sendEvent({ type: 'response.create' })\n },\n\n sendToolResult(callId: string, result: string) {\n sendEvent({\n type: 'conversation.item.create',\n item: {\n type: 'function_call_output',\n call_id: callId,\n output: result,\n },\n })\n sendEvent({ type: 'response.create' })\n },\n\n updateSession(config: Partial<RealtimeSessionConfig>) {\n const sessionUpdate: Record<string, unknown> = {}\n\n if (config.instructions) {\n sessionUpdate.instructions = config.instructions\n }\n\n if (config.voice) {\n sessionUpdate.voice = config.voice\n }\n\n if (config.vadMode) {\n if (config.vadMode === 'semantic') {\n sessionUpdate.turn_detection = {\n type: 'semantic_vad',\n eagerness: config.semanticEagerness ?? 'medium',\n }\n } else if (config.vadMode === 'server') {\n sessionUpdate.turn_detection = {\n type: 'server_vad',\n threshold: config.vadConfig?.threshold ?? 0.5,\n prefix_padding_ms: config.vadConfig?.prefixPaddingMs ?? 300,\n silence_duration_ms: config.vadConfig?.silenceDurationMs ?? 500,\n }\n } else {\n sessionUpdate.turn_detection = null\n }\n }\n\n if (config.tools !== undefined) {\n sessionUpdate.tools = config.tools.map((t) => ({\n type: 'function',\n name: t.name,\n description: t.description,\n parameters: t.inputSchema ?? { type: 'object', properties: {} },\n }))\n sessionUpdate.tool_choice = 'auto'\n }\n\n if (config.outputModalities) {\n sessionUpdate.modalities = config.outputModalities\n }\n\n if (config.temperature !== undefined) {\n sessionUpdate.temperature = config.temperature\n }\n\n if (config.maxOutputTokens !== undefined) {\n sessionUpdate.max_response_output_tokens = config.maxOutputTokens\n }\n\n // Let callers forward an explicit `input_audio_transcription` value\n // through `providerOptions` — including `null` / `false` to disable\n // the feature. Only apply our `grok-stt` default on the first\n // session.update and only if the caller hasn't set it themselves.\n const providerOptions: Record<string, unknown> =\n config.providerOptions ?? {}\n const callerTranscription =\n 'inputAudioTranscription' in providerOptions\n ? providerOptions.inputAudioTranscription\n : 'input_audio_transcription' in providerOptions\n ? providerOptions.input_audio_transcription\n : undefined\n if (callerTranscription !== undefined) {\n sessionUpdate.input_audio_transcription =\n callerTranscription === false ? null : callerTranscription\n } else if (!hasSentInitialSessionUpdate) {\n sessionUpdate.input_audio_transcription = { model: 'grok-stt' }\n }\n\n if (Object.keys(sessionUpdate).length > 0) {\n sendEvent({\n type: 'session.update',\n session: sessionUpdate,\n })\n hasSentInitialSessionUpdate = true\n }\n },\n\n interrupt() {\n sendEvent({ type: 'response.cancel' })\n currentMode = 'listening'\n emit('mode_change', { mode: 'listening' })\n emit('interrupted', {\n ...(currentMessageId !== null && { messageId: currentMessageId }),\n })\n },\n\n on<TEvent extends RealtimeEvent>(\n event: TEvent,\n handler: RealtimeEventHandler<TEvent>,\n ): () => void {\n let handlers = eventHandlers.get(event)\n if (!handlers) {\n handlers = new Set()\n eventHandlers.set(event, handlers)\n }\n handlers.add(handler)\n\n return () => {\n eventHandlers.get(event)?.delete(handler)\n }\n },\n\n getAudioVisualization(): AudioVisualization {\n function calculateLevel(analyser: AnalyserNode): number {\n const data = new Uint8Array(analyser.fftSize)\n analyser.getByteTimeDomainData(data)\n\n let maxDeviation = 0\n for (const sample of data) {\n const deviation = Math.abs(sample - 128)\n if (deviation > maxDeviation) {\n maxDeviation = deviation\n }\n }\n\n const normalized = maxDeviation / 128\n return Math.min(1, normalized * 1.5)\n }\n\n return {\n get inputLevel() {\n if (!inputAnalyser) return 0\n return calculateLevel(inputAnalyser)\n },\n\n get outputLevel() {\n if (!outputAnalyser) return 0\n return calculateLevel(outputAnalyser)\n },\n\n getInputFrequencyData() {\n if (!inputAnalyser)\n return new Uint8Array(FALLBACK_FREQUENCY_BIN_COUNT)\n const data = new Uint8Array(inputAnalyser.frequencyBinCount)\n inputAnalyser.getByteFrequencyData(data)\n return data\n },\n\n getOutputFrequencyData() {\n if (!outputAnalyser)\n return new Uint8Array(FALLBACK_FREQUENCY_BIN_COUNT)\n const data = new Uint8Array(outputAnalyser.frequencyBinCount)\n outputAnalyser.getByteFrequencyData(data)\n return data\n },\n\n getInputTimeDomainData() {\n if (!inputAnalyser)\n return new Uint8Array(FALLBACK_TIME_DOMAIN_SIZE).fill(\n FALLBACK_TIME_DOMAIN_FILL,\n )\n const data = new Uint8Array(inputAnalyser.fftSize)\n inputAnalyser.getByteTimeDomainData(data)\n return data\n },\n\n getOutputTimeDomainData() {\n if (!outputAnalyser)\n return new Uint8Array(FALLBACK_TIME_DOMAIN_SIZE).fill(\n FALLBACK_TIME_DOMAIN_FILL,\n )\n const data = new Uint8Array(outputAnalyser.fftSize)\n outputAnalyser.getByteTimeDomainData(data)\n return data\n },\n\n get inputSampleRate() {\n return 24000\n },\n\n get outputSampleRate() {\n return 24000\n },\n }\n },\n }\n\n // `dataChannelReady` was already awaited inside the post-SDP try/catch\n // above so we can short-circuit on failures with full teardown.\n return connection\n}\n"],"names":["content"],"mappings":";AAgBA,MAAM,oBAAoB;AAQ1B,SAAS,WACP,KACA,KACoB;AACpB,QAAM,QAAQ,IAAI,GAAG;AACrB,SAAO,OAAO,UAAU,WAAW,QAAQ;AAC7C;AAEA,SAAS,WACP,KACA,KACqC;AACrC,QAAM,QAAQ,IAAI,GAAG;AACrB,SAAO,SAAS,OAAO,UAAU,YAAY,CAAC,MAAM,QAAQ,KAAK,IAC5D,QACD;AACN;AAEA,SAAS,gBACP,KACA,KAC4C;AAC5C,QAAM,QAAQ,IAAI,GAAG;AACrB,MAAI,CAAC,MAAM,QAAQ,KAAK,EAAG,QAAO;AAClC,SAAO,MAAM;AAAA,IACX,CAAC,SACC,SAAS,QAAQ,OAAO,SAAS,YAAY,CAAC,MAAM,QAAQ,IAAI;AAAA,EAAA;AAEtE;AA0BO,SAAS,aACd,UAA+B,IACd;AACjB,QAAM,iBAAiB,QAAQ,kBAAkB;AACjD,QAAM,SAAS,mBAAmB,QAAQ,KAAK;AAE/C,SAAO;AAAA,IACL,UAAU;AAAA,IAEV,MAAM,QACJ,OACA,cAC6B;AAC7B,YAAM,QAAQ,MAAM,OAAO,SAAS;AACpC,aAAO,QAAQ,yCAAyC,KAAK,IAAI;AAAA,QAC/D,UAAU;AAAA,QACV;AAAA,MAAA,CACD;AAED,UAAI,mBAAmB,UAAU;AAC/B,eAAO,uBAAuB,OAAO,MAAM;AAAA,MAC7C;AACA,YAAM,QAAQ,IAAI,MAAM,+CAA+C;AACvE,aAAO,OAAO,uBAAuB;AAAA,QACnC;AAAA,QACA,QAAQ;AAAA,MAAA,CACT;AACD,YAAM;AAAA,IACR;AAAA,EAAA;AAEJ;AAKA,eAAe,uBACb,OACA,QAC6B;AAC7B,QAAM,QAAQ,MAAM,OAAO,SAAS;AACpC,QAAM,oCAAoB,IAAA;AAE1B,QAAM,KAAK,IAAI,kBAAA;AAEf,MAAI,eAAoC;AACxC,MAAI,gBAAqC;AACzC,MAAI,iBAAsC;AAC1C,MAAI,cAAiD;AACrD,MAAI,eAAkD;AACtD,MAAI,cAAkC;AAEtC,MAAI,eAAwC;AAK5C,QAAM,UAAU,GAAG,kBAAkB,YAAY;AACjD,MAAI,cAAqC;AAEzC,MAAI,cAA4B;AAChC,MAAI,mBAAkC;AAKtC,MAAI,aAAa;AAMjB,QAAM,gBAAgD,CAAA;AAMtD,MAAI,8BAA8B;AAMlC,QAAM,+BAA+B;AACrC,QAAM,4BAA4B;AAClC,QAAM,4BAA4B;AAElC,WAAS,KACP,OACA,SACA;AACA,UAAM,WAAW,cAAc,IAAI,KAAK;AACxC,QAAI,UAAU;AACZ,iBAAW,WAAW,UAAU;AAC9B,gBAAQ,OAAO;AAAA,MACjB;AAAA,IACF;AAAA,EACF;AAEA,MAAI,oBAAoB;AACxB,MAAI,yBAA6D;AACjE,MAAI,0BAAgE;AAEpE,QAAM,mBAAmB,IAAI,QAAc,CAAC,SAAS,WAAW;AAC9D,6BAAyB,CAAC,WAAW;AACnC,UAAI,4BAA4B,MAAM;AACpC,qBAAa,uBAAuB;AACpC,kCAA0B;AAAA,MAC5B;AAEA,+BAAyB;AACzB,aAAO,MAAM;AAAA,IACf;AAEA,8BAA0B,WAAW,MAAM;AACzC,UAAI,CAAC,mBAAmB;AACtB;AAAA,UACE,IAAI;AAAA,YACF;AAAA,UAAA;AAAA,QACF;AAAA,MAEJ;AAAA,IACF,GAAG,IAAK;AAER,YAAQ,SAAS,MAAM;AACrB,0BAAoB;AACpB,UAAI,4BAA4B,MAAM;AACpC,qBAAa,uBAAuB;AACpC,kCAA0B;AAAA,MAC5B;AAGA,+BAAyB;AACzB,yBAAA;AACA,WAAK,iBAAiB,EAAE,QAAQ,YAAA,CAA+B;AAC/D,cAAA;AAAA,IACF;AAAA,EACF,CAAC;AAED,UAAQ,YAAY,CAAC,UAAU;AAC7B,QAAI;AACF,YAAM,UAAU,KAAK,MAAM,MAAM,IAAI;AACrC,YAAM,gBACJ,YAAY,QAAQ,OAAO,YAAY,WAAW,UAAU,CAAA;AAC9D,aAAO;AAAA,QACL,mCAAmC,WAAW,eAAe,MAAM,KAAK,WAAW;AAAA,QACnF,EAAE,OAAO,cAAA;AAAA,MAAc;AAEzB,wBAAkB,aAAa;AAAA,IACjC,SAAS,UAAU;AACjB,aAAO,OAAO,uBAAuB;AAAA,QACnC,OAAO;AAAA,QACP,QAAQ;AAAA,MAAA,CACT;AACD,WAAK,SAAS;AAAA,QACZ,OACE,oBAAoB,QAAQ,WAAW,IAAI,MAAM,OAAO,QAAQ,CAAC;AAAA,MAAA,CACpE;AAAA,IACH;AAAA,EACF;AAEA,UAAQ,UAAU,CAAC,UAAU;AAK3B,QAAI,WAAY;AAChB,WAAO,OAAO,uBAAuB;AAAA,MACnC;AAAA,MACA,QAAQ;AAAA,IAAA,CACT;AAOD,UAAM,cAAc;AACpB,UAAM,WAAW,WAAW,aAAa,OAAO;AAChD,UAAM,OACH,YAAY,WAAW,UAAU,SAAS,OAAO,MAAM,QAAQ;AAClE,UAAM,QAAQ,IAAI,MAAM,uBAAuB,GAAG,EAAE;AACpD,QAAI,CAAC,mBAAmB;AACtB,+BAAyB,KAAK;AAAA,IAChC;AACA,SAAK,SAAS,EAAE,OAAO,MAAA,CAAO;AAAA,EAChC;AAEA,UAAQ,UAAU,MAAM;AAItB,QAAI,WAAY;AAChB,QAAI,CAAC,mBAAmB;AACtB,+BAAyB,IAAI,MAAM,oCAAoC,CAAC;AAAA,IAC1E;AAAA,EACF;AAEA,KAAG,UAAU,CAAC,UAAU;AACtB,QAAI,MAAM,MAAM,SAAS,WAAW,MAAM,QAAQ,CAAC,GAAG;AACpD,+BAAyB,MAAM,QAAQ,CAAC,CAAC;AAAA,IAC3C;AAAA,EACF;AAMA,KAAG,0BAA0B,MAAM;AACjC,UAAM,QAAQ,GAAG;AACjB,WAAO,SAAS,oCAAoC,KAAK,IAAI;AAAA,MAC3D;AAAA,IAAA,CACD;AACD,QAAI,UAAU,YAAY,UAAU,kBAAkB,UAAU,UAAU;AAMxE,UAAI,CAAC,YAAY;AACf,aAAK,iBAAiB;AAAA,UACpB,QACE,UAAU,WACL,UACA;AAAA,QAAA,CACR;AAAA,MACH;AACA,UAAI,CAAC,mBAAmB;AAKtB,cAAM,UACJ,UAAU,WACN,qDACA,iCAAiC,KAAK;AAC5C,iCAAyB,IAAI,MAAM,OAAO,CAAC;AAAA,MAC7C;AASA,UAAI,UAAU,YAAY,CAAC,YAAY;AACrC,aAAK,mBAAA;AAAA,MACP;AAAA,IACF;AAAA,EACF;AAEA,KAAG,6BAA6B,MAAM;AACpC,UAAM,QAAQ,GAAG;AACjB,WAAO,SAAS,uCAAuC,KAAK,IAAI;AAAA,MAC9D;AAAA,IAAA,CACD;AACD,QACE,CAAC,sBACA,UAAU,YAAY,UAAU,YAAY,UAAU,iBACvD;AACA,YAAM,UACJ,UAAU,WACN,qDACA,iCAAiC,KAAK;AAC5C,+BAAyB,IAAI,MAAM,OAAO,CAAC;AAAA,IAC7C;AAAA,EACF;AAYA,iBAAe,qBAAqB;AAMlC,iBAAa;AAQb,kBAAc,SAAS;AAMvB;AAAA,MACE,IAAI,MAAM,iDAAiD;AAAA,IAAA;AAG7D,QAAI,aAAa;AACf,iBAAW,SAAS,YAAY,aAAa;AAC3C,cAAM,KAAA;AAAA,MACR;AACA,oBAAc;AAAA,IAChB;AAIA,QAAI,cAAc;AAChB,UAAI;AACF,qBAAa,MAAA;AAAA,MACf,QAAQ;AAAA,MAER;AACA,mBAAa,YAAY;AACzB,qBAAe;AAAA,IACjB;AACA,QAAI,cAAc;AAChB,UAAI;AACF,qBAAa,WAAA;AAAA,MACf,QAAQ;AAAA,MAER;AACA,qBAAe;AAAA,IACjB;AACA,QAAI,gBAAgB;AAClB,UAAI;AACF,uBAAe,WAAA;AAAA,MACjB,QAAQ;AAAA,MAER;AACA,uBAAiB;AAAA,IACnB;AAGA,QAAI,aAAa;AACf,UAAI;AACF,oBAAY,WAAA;AAAA,MACd,QAAQ;AAAA,MAER;AACA,oBAAc;AAAA,IAChB;AACA,QAAI,eAAe;AACjB,UAAI;AACF,sBAAc,WAAA;AAAA,MAChB,QAAQ;AAAA,MAER;AACA,sBAAgB;AAAA,IAClB;AAEA,QAAI,aAAa;AACf,UAAI;AACF,oBAAY,MAAA;AAAA,MACd,QAAQ;AAAA,MAER;AACA,oBAAc;AAAA,IAChB;AAEA,QAAI;AACF,SAAG,MAAA;AAAA,IACL,QAAQ;AAAA,IAER;AAEA,QAAI,cAAc;AAChB,UAAI;AACF,cAAM,aAAa,MAAA;AAAA,MACrB,QAAQ;AAAA,MAER;AACA,qBAAe;AAAA,IACjB;AAAA,EACF;AAUA,MAAI;AACF,QAAI;AACF,oBAAc,MAAM,UAAU,aAAa,aAAa;AAAA,QACtD,OAAO;AAAA,UACL,kBAAkB;AAAA,UAClB,kBAAkB;AAAA,UAClB,YAAY;AAAA,QAAA;AAAA,MACd,CACD;AAAA,IACH,SAAS,OAAO;AACd,aAAO,OAAO,uBAAuB;AAAA,QACnC;AAAA,QACA,QAAQ;AAAA,MAAA,CACT;AAGD,YAAM,IAAI;AAAA,QACR,kDAAkD,iBAAiB,QAAQ,MAAM,UAAU,KAAK;AAAA,MAAA;AAAA,IAEpG;AAEA,eAAW,SAAS,YAAY,kBAAkB;AAChD,SAAG,SAAS,OAAO,WAAW;AAAA,IAChC;AAEA,UAAM,QAAQ,MAAM,GAAG,YAAA;AACvB,UAAM,GAAG,oBAAoB,KAAK;AAElC,UAAM,cAAc,MAAM,MAAM,GAAG,iBAAiB,UAAU,KAAK,IAAI;AAAA,MACrE,QAAQ;AAAA,MACR,SAAS;AAAA,QACP,eAAe,UAAU,MAAM,KAAK;AAAA,QACpC,gBAAgB;AAAA,MAAA;AAAA,MAElB,MAAM,MAAM;AAAA,IAAA,CACb;AAED,QAAI,CAAC,YAAY,IAAI;AACnB,YAAM,YAAY,MAAM,YAAY,KAAA;AACpC,YAAM,QAAQ,IAAI;AAAA,QAChB,0CAA0C,YAAY,MAAM,MAAM,SAAS;AAAA,MAAA;AAE7E,aAAO,OAAO,uBAAuB;AAAA,QACnC;AAAA,QACA,QAAQ;AAAA,QACR,QAAQ,YAAY;AAAA,MAAA,CACrB;AACD,YAAM;AAAA,IACR;AAEA,UAAM,YAAY,MAAM,YAAY,KAAA;AACpC,UAAM,GAAG,qBAAqB,EAAE,MAAM,UAAU,KAAK,WAAW;AAAA,EAClE,SAAS,KAAK;AACZ,UAAM,mBAAA;AACN,UAAM;AAAA,EACR;AAOA,MAAI;AACF,4BAAwB,WAAW;AACnC,UAAM;AAAA,EACR,SAAS,KAAK;AACZ,UAAM,mBAAA;AACN,UAAM;AAAA,EACR;AAEA,WAAS,kBAAkB,OAAgC;AACzD,UAAM,OAAO,WAAW,OAAO,MAAM;AAErC,YAAQ,MAAA;AAAA,MACN,KAAK;AAAA,MACL,KAAK;AACH;AAAA,MAEF,KAAK;AACH,sBAAc;AACd,aAAK,eAAe,EAAE,MAAM,YAAA,CAAa;AACzC;AAAA,MAEF,KAAK;AACH,sBAAc;AACd,aAAK,eAAe,EAAE,MAAM,WAAA,CAAY;AACxC;AAAA,MAEF,KAAK;AACH;AAAA,MAEF,KAAK,yDAAyD;AAC5D,cAAM,aAAa,WAAW,OAAO,YAAY;AACjD,YAAI,eAAe,OAAW;AAC9B,aAAK,cAAc,EAAE,MAAM,QAAQ,YAAY,SAAS,MAAM;AAC9D;AAAA,MACF;AAAA,MAEA,KAAK;AAIH,2BAAmB;AACnB,sBAAc;AACd,aAAK,eAAe,EAAE,MAAM,WAAA,CAAY;AACxC;AAAA,MAEF,KAAK,8BAA8B;AACjC,cAAM,OAAO,WAAW,OAAO,MAAM;AACrC,YAAI,QAAQ,WAAW,MAAM,MAAM,MAAM,WAAW;AAClD,gBAAM,KAAK,WAAW,MAAM,IAAI;AAChC,cAAI,OAAO,OAAW,oBAAmB;AAAA,QAC3C;AACA;AAAA,MACF;AAAA;AAAA;AAAA;AAAA,MAKA,KAAK;AAAA,MACL,KAAK,mCAAmC;AACtC,cAAM,QAAQ,WAAW,OAAO,OAAO;AACvC,YAAI,UAAU,OAAW;AACzB,aAAK,cAAc;AAAA,UACjB,MAAM;AAAA,UACN,YAAY;AAAA,UACZ,SAAS;AAAA,QAAA,CACV;AACD;AAAA,MACF;AAAA,MAEA,KAAK;AAAA,MACL,KAAK,kCAAkC;AACrC,cAAM,aAAa,WAAW,OAAO,YAAY;AACjD,YAAI,eAAe,OAAW;AAC9B,aAAK,cAAc,EAAE,MAAM,aAAa,YAAY,SAAS,MAAM;AACnE;AAAA,MACF;AAAA;AAAA;AAAA,MAIA,KAAK;AAAA,MACL,KAAK,8BAA8B;AACjC,cAAM,QAAQ,WAAW,OAAO,OAAO;AACvC,YAAI,UAAU,OAAW;AACzB,aAAK,cAAc;AAAA,UACjB,MAAM;AAAA,UACN,YAAY;AAAA,UACZ,SAAS;AAAA,QAAA,CACV;AACD;AAAA,MACF;AAAA,MAEA,KAAK;AAAA,MACL,KAAK,6BAA6B;AAChC,cAAM,OAAO,WAAW,OAAO,MAAM;AACrC,YAAI,SAAS,OAAW;AACxB,aAAK,cAAc;AAAA,UACjB,MAAM;AAAA,UACN,YAAY;AAAA,UACZ,SAAS;AAAA,QAAA,CACV;AACD;AAAA,MACF;AAAA;AAAA;AAAA,MAIA,KAAK;AAAA,MACL,KAAK;AACH,YAAI,gBAAgB,YAAY;AAC9B,wBAAc;AACd,eAAK,eAAe,EAAE,MAAM,WAAA,CAAY;AAAA,QAC1C;AACA;AAAA,MAEF,KAAK;AAAA,MACL,KAAK;AACH;AAAA,MAEF,KAAK,yCAAyC;AAM5C,cAAM,SAAS,WAAW,OAAO,SAAS;AAC1C,cAAM,OAAO,WAAW,OAAO,MAAM,KAAK;AAC1C,cAAM,OAAO,WAAW,OAAO,WAAW,KAAK;AAC/C,YAAI,CAAC,QAAQ;AACX,iBAAO;AAAA,YACL;AAAA,YACA;AAAA,cACE,QAAQ;AAAA,cACR,YAAY;AAAA,cACZ,SAAS,MAAM;AAAA,YAAA;AAAA,UACjB;AAEF,eAAK,SAAS;AAAA,YACZ,OAAO,IAAI;AAAA,cACT;AAAA,YAAA;AAAA,UACF,CACD;AACD;AAAA,QACF;AACA,YAAI;AACF,gBAAM,QAAQ,KAAK,MAAM,IAAI;AAC7B,eAAK,aAAa,EAAE,YAAY,QAAQ,UAAU,MAAM,OAAO;AAAA,QACjE,QAAQ;AACN,eAAK,aAAa,EAAE,YAAY,QAAQ,UAAU,MAAM,OAAO,MAAM;AAAA,QACvE;AACA;AAAA,MACF;AAAA,MAEA,KAAK,iBAAiB;AACpB,cAAM,WAAW,WAAW,OAAO,UAAU,KAAK,CAAA;AAClD,cAAM,SAAS,gBAAgB,UAAU,QAAQ;AAKjD,YAAI,gBAAgB,QAAQ;AAC1B,wBAAc;AACd,eAAK,eAAe,EAAE,MAAM,YAAA,CAAa;AAAA,QAC3C;AAEA,YAAI,kBAAkB;AACpB,gBAAM,UAA2B;AAAA,YAC/B,IAAI;AAAA,YACJ,MAAM;AAAA,YACN,WAAW,KAAK,IAAA;AAAA,YAChB,OAAO,CAAA;AAAA,UAAC;AAGV,qBAAW,QAAQ,UAAU,IAAI;AAC/B,gBAAI,WAAW,MAAM,MAAM,MAAM,UAAW;AAC5C,kBAAM,UAAU,gBAAgB,MAAM,SAAS;AAC/C,gBAAI,CAAC,QAAS;AACd,uBAAW,QAAQ,SAAS;AAC1B,oBAAM,WAAW,WAAW,MAAM,MAAM;AACxC,kBAAI,aAAa,SAAS;AACxB,sBAAM,aAAa,WAAW,MAAM,YAAY;AAChD,oBAAI,YAAY;AACd,0BAAQ,MAAM,KAAK,EAAE,MAAM,SAAS,YAAY;AAAA,gBAClD;AAAA,cACF,WAAW,aAAa,QAAQ;AAC9B,sBAAMA,WAAU,WAAW,MAAM,MAAM;AACvC,oBAAIA,UAAS;AACX,0BAAQ,MAAM,KAAK,EAAE,MAAM,QAAQ,SAAAA,UAAS;AAAA,gBAC9C;AAAA,cACF;AAAA,YACF;AAAA,UACF;AAEA,eAAK,oBAAoB,EAAE,SAAS;AACpC,6BAAmB;AAAA,QACrB;AACA;AAAA,MACF;AAAA,MAEA,KAAK;AAKH,YAAI,gBAAgB,QAAQ;AAC1B,wBAAc;AACd,eAAK,eAAe,EAAE,MAAM,YAAA,CAAa;AAAA,QAC3C;AACA,aAAK,eAAe;AAAA,UAClB,GAAI,qBAAqB,QAAQ,EAAE,WAAW,iBAAA;AAAA,QAAiB,CAChE;AACD;AAAA,MAEF,KAAK,SAAS;AAMZ,cAAM,WAAW,WAAW,OAAO,OAAO,KAAK,CAAA;AAC/C,cAAM,UACJ,WAAW,UAAU,SAAS,KAAK;AACrC,cAAM,MAA2B,IAAI,MAAM,OAAO;AAGlD,cAAM,OAAO,WAAW,UAAU,MAAM;AACxC,YAAI,SAAS,OAAW,KAAI,OAAO;AACnC,cAAM,UAAU,WAAW,UAAU,MAAM;AAC3C,YAAI,YAAY,OAAW,KAAI,OAAO;AACtC,cAAM,QAAQ,WAAW,UAAU,OAAO;AAC1C,YAAI,UAAU,OAAW,KAAI,QAAQ;AACrC,eAAO,OAAO,8BAA8B;AAAA,UAC1C,GAAG;AAAA,UACH,QAAQ;AAAA,QAAA,CACT;AACD,aAAK,SAAS,EAAE,OAAO,IAAA,CAAK;AAC5B;AAAA,MACF;AAAA,MAEA,KAAK;AAAA,MACL;AAKE,eAAO,SAAS,wCAAwC;AAAA,UACtD,MAAM,MAAM;AAAA,QAAA,CACb;AACD;AAAA,IAAA;AAAA,EAEN;AAEA,WAAS,yBAAyB,QAAqB;AAMrD,QAAI,WAAY;AAMhB,QAAI,cAAc;AAChB,UAAI;AACF,qBAAa,MAAA;AAAA,MACf,QAAQ;AAAA,MAER;AACA,mBAAa,YAAY;AACzB,qBAAe;AAAA,IACjB;AACA,QAAI,cAAc;AAChB,UAAI;AACF,qBAAa,WAAA;AAAA,MACf,QAAQ;AAAA,MAER;AACA,qBAAe;AAAA,IACjB;AACA,QAAI,gBAAgB;AAClB,UAAI;AACF,uBAAe,WAAA;AAAA,MACjB,QAAQ;AAAA,MAER;AACA,uBAAiB;AAAA,IACnB;AAEA,mBAAe,IAAI,MAAA;AACnB,iBAAa,YAAY;AACzB,iBAAa,WAAW;AACxB,iBAAa,KAAA,EAAO,MAAM,CAAC,MAAM;AAM/B,aAAO,OAAO,wCAAwC;AAAA,QACpD,OAAO;AAAA,QACP,QAAQ;AAAA,MAAA,CACT;AAAA,IACH,CAAC;AAED,QAAI,CAAC,cAAc;AACjB,qBAAe,IAAI,aAAA;AAAA,IACrB;AAEA,QAAI,aAAa,UAAU,aAAa;AACtC,mBAAa,OAAA,EAAS,MAAM,CAAC,QAAQ;AAKnC,eAAO,OAAO,4CAA4C;AAAA,UACxD,OAAO;AAAA,UACP,QAAQ;AAAA,QAAA,CACT;AAAA,MACH,CAAC;AAAA,IACH;AAEA,qBAAiB,aAAa,eAAA;AAC9B,mBAAe,UAAU;AACzB,mBAAe,wBAAwB;AAEvC,mBAAe,aAAa,wBAAwB,MAAM;AAC1D,iBAAa,QAAQ,cAAc;AAAA,EACrC;AAEA,WAAS,wBAAwB,QAAqB;AAKpD,QAAI,WAAY;AAEhB,QAAI,CAAC,cAAc;AACjB,qBAAe,IAAI,aAAA;AAAA,IACrB;AAEA,QAAI,aAAa,UAAU,aAAa;AACtC,mBAAa,OAAA,EAAS,MAAM,CAAC,QAAQ;AAKnC,eAAO,OAAO,4CAA4C;AAAA,UACxD,OAAO;AAAA,UACP,QAAQ;AAAA,QAAA,CACT;AAAA,MACH,CAAC;AAAA,IACH;AAEA,oBAAgB,aAAa,eAAA;AAC7B,kBAAc,UAAU;AACxB,kBAAc,wBAAwB;AAEtC,kBAAc,aAAa,wBAAwB,MAAM;AACzD,gBAAY,QAAQ,aAAa;AAAA,EACnC;AAEA,WAAS,UAAU,OAAgC;AACjD,QAAI,YAAY;AAMd,aAAO,OAAO,4CAA4C;AAAA,QACxD,WAAW,WAAW,OAAO,MAAM,KAAK;AAAA,QACxC,QAAQ;AAAA,MAAA,CACT;AACD;AAAA,IACF;AACA,QAAI,aAAa,eAAe,QAAQ;AACtC,aAAO;AAAA,QACL,oCAAoC,WAAW,OAAO,MAAM,KAAK,WAAW;AAAA,QAC5E,EAAE,OAAO,MAAA;AAAA,MAAM;AAQjB,UAAI;AACF,oBAAY,KAAK,KAAK,UAAU,KAAK,CAAC;AAAA,MACxC,SAAS,OAAO;AACd,eAAO,OAAO,kCAAkC;AAAA,UAC9C;AAAA,UACA,WAAW,WAAW,OAAO,MAAM,KAAK;AAAA,UACxC,QAAQ;AAAA,QAAA,CACT;AACD,aAAK,SAAS;AAAA,UACZ,OAAO,iBAAiB,QAAQ,QAAQ,IAAI,MAAM,OAAO,KAAK,CAAC;AAAA,QAAA,CAChE;AAAA,MACH;AAAA,IACF,OAAO;AACL,oBAAc,KAAK,KAAK;AAAA,IAC1B;AAAA,EACF;AAEA,WAAS,qBAAqB;AAC5B,QAAI;AACF,iBAAW,SAAS,eAAe;AACjC,eAAO;AAAA,UACL,oCAAoC,WAAW,OAAO,MAAM,KAAK,WAAW;AAAA,UAC5E,EAAE,OAAO,MAAA;AAAA,QAAM;AAEjB,gBAAQ,KAAK,KAAK,UAAU,KAAK,CAAC;AAAA,MACpC;AACA,oBAAc,SAAS;AAAA,IACzB,SAAS,OAAO;AAOd,aAAO,OAAO,2CAA2C;AAAA,QACvD;AAAA,QACA,QAAQ;AAAA,MAAA,CACT;AACD,YAAM,MAAM,iBAAiB,QAAQ,QAAQ,IAAI,MAAM,OAAO,KAAK,CAAC;AACpE,+BAAyB,GAAG;AAC5B,WAAK,SAAS,EAAE,OAAO,IAAA,CAAK;AAAA,IAC9B;AAAA,EACF;AAEA,QAAM,aAAiC;AAAA,IACrC,MAAM,aAAa;AAIjB,YAAM,mBAAA;AACN,WAAK,iBAAiB,EAAE,QAAQ,OAAA,CAA0B;AAAA,IAC5D;AAAA,IAEA,MAAM,oBAAoB;AACxB,UAAI,aAAa;AACf,mBAAW,SAAS,YAAY,kBAAkB;AAChD,gBAAM,UAAU;AAAA,QAClB;AAAA,MACF;AACA,oBAAc;AACd,WAAK,eAAe,EAAE,MAAM,YAAA,CAAa;AAAA,IAC3C;AAAA,IAEA,mBAAmB;AACjB,UAAI,aAAa;AACf,mBAAW,SAAS,YAAY,kBAAkB;AAChD,gBAAM,UAAU;AAAA,QAClB;AAAA,MACF;AACA,oBAAc;AACd,WAAK,eAAe,EAAE,MAAM,OAAA,CAAQ;AAAA,IACtC;AAAA,IAEA,SAAS,MAAc;AACrB,gBAAU;AAAA,QACR,MAAM;AAAA,QACN,MAAM;AAAA,UACJ,MAAM;AAAA,UACN,MAAM;AAAA,UACN,SAAS,CAAC,EAAE,MAAM,cAAc,MAAM;AAAA,QAAA;AAAA,MACxC,CACD;AACD,gBAAU,EAAE,MAAM,mBAAmB;AAAA,IACvC;AAAA,IAEA,UAAU,WAAmB,UAAkB;AAM7C,YAAM,wBACJ,UAAU,WAAW,SAAS,KAC9B,UAAU,WAAW,UAAU,KAC/B,UAAU,WAAW,OAAO;AAC9B,YAAM,eAAe;AAAA,QACnB,MAAM;AAAA;AAAA;AAAA;AAAA,QAIN,WAAW;AAAA,UACT,KAAK,wBACD,YACA,QAAQ,QAAQ,WAAW,SAAS;AAAA,QAAA;AAAA,MAC1C;AAGF,gBAAU;AAAA,QACR,MAAM;AAAA,QACN,MAAM;AAAA,UACJ,MAAM;AAAA,UACN,MAAM;AAAA,UACN,SAAS,CAAC,YAAY;AAAA,QAAA;AAAA,MACxB,CACD;AACD,gBAAU,EAAE,MAAM,mBAAmB;AAAA,IACvC;AAAA,IAEA,eAAe,QAAgB,QAAgB;AAC7C,gBAAU;AAAA,QACR,MAAM;AAAA,QACN,MAAM;AAAA,UACJ,MAAM;AAAA,UACN,SAAS;AAAA,UACT,QAAQ;AAAA,QAAA;AAAA,MACV,CACD;AACD,gBAAU,EAAE,MAAM,mBAAmB;AAAA,IACvC;AAAA,IAEA,cAAc,QAAwC;AACpD,YAAM,gBAAyC,CAAA;AAE/C,UAAI,OAAO,cAAc;AACvB,sBAAc,eAAe,OAAO;AAAA,MACtC;AAEA,UAAI,OAAO,OAAO;AAChB,sBAAc,QAAQ,OAAO;AAAA,MAC/B;AAEA,UAAI,OAAO,SAAS;AAClB,YAAI,OAAO,YAAY,YAAY;AACjC,wBAAc,iBAAiB;AAAA,YAC7B,MAAM;AAAA,YACN,WAAW,OAAO,qBAAqB;AAAA,UAAA;AAAA,QAE3C,WAAW,OAAO,YAAY,UAAU;AACtC,wBAAc,iBAAiB;AAAA,YAC7B,MAAM;AAAA,YACN,WAAW,OAAO,WAAW,aAAa;AAAA,YAC1C,mBAAmB,OAAO,WAAW,mBAAmB;AAAA,YACxD,qBAAqB,OAAO,WAAW,qBAAqB;AAAA,UAAA;AAAA,QAEhE,OAAO;AACL,wBAAc,iBAAiB;AAAA,QACjC;AAAA,MACF;AAEA,UAAI,OAAO,UAAU,QAAW;AAC9B,sBAAc,QAAQ,OAAO,MAAM,IAAI,CAAC,OAAO;AAAA,UAC7C,MAAM;AAAA,UACN,MAAM,EAAE;AAAA,UACR,aAAa,EAAE;AAAA,UACf,YAAY,EAAE,eAAe,EAAE,MAAM,UAAU,YAAY,CAAA,EAAC;AAAA,QAAE,EAC9D;AACF,sBAAc,cAAc;AAAA,MAC9B;AAEA,UAAI,OAAO,kBAAkB;AAC3B,sBAAc,aAAa,OAAO;AAAA,MACpC;AAEA,UAAI,OAAO,gBAAgB,QAAW;AACpC,sBAAc,cAAc,OAAO;AAAA,MACrC;AAEA,UAAI,OAAO,oBAAoB,QAAW;AACxC,sBAAc,6BAA6B,OAAO;AAAA,MACpD;AAMA,YAAM,kBACJ,OAAO,mBAAmB,CAAA;AAC5B,YAAM,sBACJ,6BAA6B,kBACzB,gBAAgB,0BAChB,+BAA+B,kBAC7B,gBAAgB,4BAChB;AACR,UAAI,wBAAwB,QAAW;AACrC,sBAAc,4BACZ,wBAAwB,QAAQ,OAAO;AAAA,MAC3C,WAAW,CAAC,6BAA6B;AACvC,sBAAc,4BAA4B,EAAE,OAAO,WAAA;AAAA,MACrD;AAEA,UAAI,OAAO,KAAK,aAAa,EAAE,SAAS,GAAG;AACzC,kBAAU;AAAA,UACR,MAAM;AAAA,UACN,SAAS;AAAA,QAAA,CACV;AACD,sCAA8B;AAAA,MAChC;AAAA,IACF;AAAA,IAEA,YAAY;AACV,gBAAU,EAAE,MAAM,mBAAmB;AACrC,oBAAc;AACd,WAAK,eAAe,EAAE,MAAM,YAAA,CAAa;AACzC,WAAK,eAAe;AAAA,QAClB,GAAI,qBAAqB,QAAQ,EAAE,WAAW,iBAAA;AAAA,MAAiB,CAChE;AAAA,IACH;AAAA,IAEA,GACE,OACA,SACY;AACZ,UAAI,WAAW,cAAc,IAAI,KAAK;AACtC,UAAI,CAAC,UAAU;AACb,uCAAe,IAAA;AACf,sBAAc,IAAI,OAAO,QAAQ;AAAA,MACnC;AACA,eAAS,IAAI,OAAO;AAEpB,aAAO,MAAM;AACX,sBAAc,IAAI,KAAK,GAAG,OAAO,OAAO;AAAA,MAC1C;AAAA,IACF;AAAA,IAEA,wBAA4C;AAC1C,eAAS,eAAe,UAAgC;AACtD,cAAM,OAAO,IAAI,WAAW,SAAS,OAAO;AAC5C,iBAAS,sBAAsB,IAAI;AAEnC,YAAI,eAAe;AACnB,mBAAW,UAAU,MAAM;AACzB,gBAAM,YAAY,KAAK,IAAI,SAAS,GAAG;AACvC,cAAI,YAAY,cAAc;AAC5B,2BAAe;AAAA,UACjB;AAAA,QACF;AAEA,cAAM,aAAa,eAAe;AAClC,eAAO,KAAK,IAAI,GAAG,aAAa,GAAG;AAAA,MACrC;AAEA,aAAO;AAAA,QACL,IAAI,aAAa;AACf,cAAI,CAAC,cAAe,QAAO;AAC3B,iBAAO,eAAe,aAAa;AAAA,QACrC;AAAA,QAEA,IAAI,cAAc;AAChB,cAAI,CAAC,eAAgB,QAAO;AAC5B,iBAAO,eAAe,cAAc;AAAA,QACtC;AAAA,QAEA,wBAAwB;AACtB,cAAI,CAAC;AACH,mBAAO,IAAI,WAAW,4BAA4B;AACpD,gBAAM,OAAO,IAAI,WAAW,cAAc,iBAAiB;AAC3D,wBAAc,qBAAqB,IAAI;AACvC,iBAAO;AAAA,QACT;AAAA,QAEA,yBAAyB;AACvB,cAAI,CAAC;AACH,mBAAO,IAAI,WAAW,4BAA4B;AACpD,gBAAM,OAAO,IAAI,WAAW,eAAe,iBAAiB;AAC5D,yBAAe,qBAAqB,IAAI;AACxC,iBAAO;AAAA,QACT;AAAA,QAEA,yBAAyB;AACvB,cAAI,CAAC;AACH,mBAAO,IAAI,WAAW,yBAAyB,EAAE;AAAA,cAC/C;AAAA,YAAA;AAEJ,gBAAM,OAAO,IAAI,WAAW,cAAc,OAAO;AACjD,wBAAc,sBAAsB,IAAI;AACxC,iBAAO;AAAA,QACT;AAAA,QAEA,0BAA0B;AACxB,cAAI,CAAC;AACH,mBAAO,IAAI,WAAW,yBAAyB,EAAE;AAAA,cAC/C;AAAA,YAAA;AAEJ,gBAAM,OAAO,IAAI,WAAW,eAAe,OAAO;AAClD,yBAAe,sBAAsB,IAAI;AACzC,iBAAO;AAAA,QACT;AAAA,QAEA,IAAI,kBAAkB;AACpB,iBAAO;AAAA,QACT;AAAA,QAEA,IAAI,mBAAmB;AACrB,iBAAO;AAAA,QACT;AAAA,MAAA;AAAA,IAEJ;AAAA,EAAA;AAKF,SAAO;AACT;"}
|
package/dist/esm/utils/audio.js
CHANGED
|
@@ -144,6 +144,7 @@ function arrayBufferToBase64(buffer) {
|
|
|
144
144
|
const end = Math.min(i + chunkSize, bytes.length);
|
|
145
145
|
binary += String.fromCharCode.apply(
|
|
146
146
|
null,
|
|
147
|
+
// eslint-disable-next-line no-restricted-syntax -- TS lib types String.fromCharCode.apply as Array<number> but runtime accepts any ArrayLike
|
|
147
148
|
bytes.subarray(i, end)
|
|
148
149
|
);
|
|
149
150
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"audio.js","sources":["../../../src/utils/audio.ts"],"sourcesContent":["/**\n * Coerce the various audio input shapes accepted by `TranscriptionOptions.audio`\n * into a `File` suitable for `multipart/form-data` uploads.\n *\n * For base64 string inputs we require an explicit MIME type — either via a\n * `data:<mime>;base64,<payload>` URI prefix, or via the caller-provided\n * `audioFormat` parameter. Bare base64 without either is rejected, because\n * silently defaulting to `audio/mpeg` misreports non-mp3 audio to the server.\n *\n * The same rule applies to raw `ArrayBuffer` inputs: the caller must supply\n * an `audioFormat` so we know what MIME type and extension to use.\n */\nexport function toAudioFile(\n audio: string | File | Blob | ArrayBuffer,\n audioFormat?: string,\n): File {\n if (typeof File !== 'undefined' && audio instanceof File) {\n // Prefer the caller-supplied `audioFormat` over a potentially empty or\n // incorrect `File.type` — callers pass `audioFormat` precisely because\n // they have more context than the browser does about the payload. If\n // neither is set, fall through to the Blob-style error path below.\n if (audioFormat) {\n const mimeType = toMimeType(audioFormat)\n return new File([audio], `audio.${extensionFor(mimeType)}`, {\n type: mimeType,\n })\n }\n if (audio.type) {\n return audio\n }\n throw new Error(\n 'toAudioFile cannot infer type for File input with empty .type — pass an explicit audioFormat (e.g. \"mp3\", \"wav\", \"audio/mpeg\")',\n )\n }\n\n if (typeof Blob !== 'undefined' && audio instanceof Blob) {\n // Mirror the ArrayBuffer / bare-base64 paths: prefer the explicit\n // audioFormat argument over the Blob's (often empty) .type. We refuse to\n // fall back to `application/octet-stream` because that mislabels audio\n // for the server.\n const mimeType = audioFormat\n ? toMimeType(audioFormat)\n : audio.type || undefined\n if (!mimeType) {\n throw new Error(\n 'toAudioFile cannot infer type for Blob input with empty .type — pass an explicit audioFormat (e.g. \"mp3\", \"wav\", \"audio/mpeg\")',\n )\n }\n return new File([audio], `audio.${extensionFor(mimeType)}`, {\n type: mimeType,\n })\n }\n\n if (audio instanceof ArrayBuffer) {\n if (!audioFormat) {\n throw new Error(\n 'toAudioFile cannot infer type for ArrayBuffer input — pass an explicit audioFormat (e.g. \"mp3\", \"wav\", \"audio/mpeg\")',\n )\n }\n const mimeType = toMimeType(audioFormat)\n return new File([audio], `audio.${extensionFor(mimeType)}`, {\n type: mimeType,\n })\n }\n\n if (typeof audio === 'string') {\n if (audio.startsWith('data:')) {\n const [header, base64Data] = audio.split(',')\n // Fail loudly on malformed data: URIs instead of silently defaulting\n // to `audio/mpeg` — the file's contract is that we never mislabel\n // audio for the server.\n const headerMatch = header?.match(/data:([^;]+)/)\n const uriMimeType = headerMatch?.[1]\n if (!uriMimeType) {\n throw new Error(\n 'Malformed data: URI in toAudioFile: cannot parse MIME type — expected data:<mime>[;charset=…][;base64],<payload>',\n )\n }\n if (base64Data === undefined || base64Data.trim() === '') {\n throw new Error(\n 'Malformed data: URI in toAudioFile: missing base64 payload after comma',\n )\n }\n // Caller-supplied `audioFormat` wins over the URI-embedded MIME: the\n // caller has more context (the URI MIME may be wrong, or a generic\n // `application/octet-stream`).\n const mimeType = audioFormat ? toMimeType(audioFormat) : uriMimeType\n const buffer = base64ToArrayBuffer(base64Data)\n return new File([buffer], `audio.${extensionFor(mimeType)}`, {\n type: mimeType,\n })\n }\n\n if (!audioFormat) {\n throw new Error(\n 'toAudioFile requires a data: URI (e.g. data:audio/wav;base64,...) or an explicit audioFormat argument — bare base64 strings have no MIME type to infer',\n )\n }\n\n const buffer = base64ToArrayBuffer(audio)\n const mimeType = toMimeType(audioFormat)\n return new File([buffer], `audio.${extensionFor(mimeType)}`, {\n type: mimeType,\n })\n }\n\n throw new Error('Invalid audio input type')\n}\n\nfunction toMimeType(audioFormat: string): string {\n // Accept either \"audio/…\" strings or bare extensions like \"mp3\".\n if (audioFormat.includes('/')) return audioFormat\n const ext = audioFormat.toLowerCase()\n switch (ext) {\n case 'mp3':\n return 'audio/mpeg'\n case 'wav':\n return 'audio/wav'\n case 'ogg':\n return 'audio/ogg'\n case 'opus':\n return 'audio/opus'\n case 'flac':\n return 'audio/flac'\n case 'aac':\n return 'audio/aac'\n case 'mp4':\n return 'audio/mp4'\n case 'm4a':\n return 'audio/mp4'\n case 'webm':\n return 'audio/webm'\n case 'pcm':\n return 'audio/L16'\n case 'mulaw':\n return 'audio/basic'\n case 'alaw':\n return 'audio/x-alaw-basic'\n default:\n return `audio/${ext}`\n }\n}\n\nfunction extensionFor(mimeType: string): string {\n switch (mimeType) {\n case 'audio/mpeg':\n return 'mp3'\n case 'audio/wav':\n case 'audio/x-wav':\n return 'wav'\n case 'audio/ogg':\n return 'ogg'\n case 'audio/opus':\n return 'opus'\n case 'audio/flac':\n return 'flac'\n case 'audio/aac':\n return 'aac'\n case 'audio/mp4':\n return 'm4a'\n case 'audio/webm':\n return 'webm'\n case 'audio/L16':\n return 'pcm'\n case 'audio/basic':\n return 'mulaw'\n case 'audio/x-alaw-basic':\n return 'alaw'\n default: {\n const slash = mimeType.indexOf('/')\n if (slash === -1) return 'bin'\n return mimeType.slice(slash + 1) || 'bin'\n }\n }\n}\n\n/**\n * Cross-runtime ArrayBuffer → base64 conversion.\n *\n * Uses Node's `Buffer` when available (fastest path on server) and falls\n * back to `btoa` + chunked `String.fromCharCode` everywhere else (browser,\n * Cloudflare Workers, Bun, Deno). Chunking is required because a very\n * large audio buffer spread into `String.fromCharCode(...bytes)` in one\n * call can hit `Maximum call stack size exceeded`.\n */\nexport function arrayBufferToBase64(buffer: ArrayBuffer): string {\n if (typeof Buffer !== 'undefined') {\n return Buffer.from(buffer).toString('base64')\n }\n const bytes = new Uint8Array(buffer)\n const chunkSize = 0x8000\n let binary = ''\n for (let i = 0; i < bytes.length; i += chunkSize) {\n const end = Math.min(i + chunkSize, bytes.length)\n binary += String.fromCharCode.apply(\n null,\n bytes.subarray(i, end) as unknown as Array<number>,\n )\n }\n return btoa(binary)\n}\n\nfunction base64ToArrayBuffer(base64: string): ArrayBuffer {\n let binary: string\n try {\n binary = atob(base64)\n } catch (err) {\n const msg = err instanceof Error ? err.message : String(err)\n throw new Error(`Invalid base64 input to toAudioFile: ${msg}`)\n }\n const buffer = new ArrayBuffer(binary.length)\n const bytes = new Uint8Array(buffer)\n for (let i = 0; i < binary.length; i++) {\n bytes[i] = binary.charCodeAt(i)\n }\n return buffer\n}\n"],"names":["mimeType","buffer"],"mappings":"AAYO,SAAS,YACd,OACA,aACM;AACN,MAAI,OAAO,SAAS,eAAe,iBAAiB,MAAM;AAKxD,QAAI,aAAa;AACf,YAAM,WAAW,WAAW,WAAW;AACvC,aAAO,IAAI,KAAK,CAAC,KAAK,GAAG,SAAS,aAAa,QAAQ,CAAC,IAAI;AAAA,QAC1D,MAAM;AAAA,MAAA,CACP;AAAA,IACH;AACA,QAAI,MAAM,MAAM;AACd,aAAO;AAAA,IACT;AACA,UAAM,IAAI;AAAA,MACR;AAAA,IAAA;AAAA,EAEJ;AAEA,MAAI,OAAO,SAAS,eAAe,iBAAiB,MAAM;AAKxD,UAAM,WAAW,cACb,WAAW,WAAW,IACtB,MAAM,QAAQ;AAClB,QAAI,CAAC,UAAU;AACb,YAAM,IAAI;AAAA,QACR;AAAA,MAAA;AAAA,IAEJ;AACA,WAAO,IAAI,KAAK,CAAC,KAAK,GAAG,SAAS,aAAa,QAAQ,CAAC,IAAI;AAAA,MAC1D,MAAM;AAAA,IAAA,CACP;AAAA,EACH;AAEA,MAAI,iBAAiB,aAAa;AAChC,QAAI,CAAC,aAAa;AAChB,YAAM,IAAI;AAAA,QACR;AAAA,MAAA;AAAA,IAEJ;AACA,UAAM,WAAW,WAAW,WAAW;AACvC,WAAO,IAAI,KAAK,CAAC,KAAK,GAAG,SAAS,aAAa,QAAQ,CAAC,IAAI;AAAA,MAC1D,MAAM;AAAA,IAAA,CACP;AAAA,EACH;AAEA,MAAI,OAAO,UAAU,UAAU;AAC7B,QAAI,MAAM,WAAW,OAAO,GAAG;AAC7B,YAAM,CAAC,QAAQ,UAAU,IAAI,MAAM,MAAM,GAAG;AAI5C,YAAM,cAAc,QAAQ,MAAM,cAAc;AAChD,YAAM,cAAc,cAAc,CAAC;AACnC,UAAI,CAAC,aAAa;AAChB,cAAM,IAAI;AAAA,UACR;AAAA,QAAA;AAAA,MAEJ;AACA,UAAI,eAAe,UAAa,WAAW,KAAA,MAAW,IAAI;AACxD,cAAM,IAAI;AAAA,UACR;AAAA,QAAA;AAAA,MAEJ;AAIA,YAAMA,YAAW,cAAc,WAAW,WAAW,IAAI;AACzD,YAAMC,UAAS,oBAAoB,UAAU;AAC7C,aAAO,IAAI,KAAK,CAACA,OAAM,GAAG,SAAS,aAAaD,SAAQ,CAAC,IAAI;AAAA,QAC3D,MAAMA;AAAAA,MAAA,CACP;AAAA,IACH;AAEA,QAAI,CAAC,aAAa;AAChB,YAAM,IAAI;AAAA,QACR;AAAA,MAAA;AAAA,IAEJ;AAEA,UAAM,SAAS,oBAAoB,KAAK;AACxC,UAAM,WAAW,WAAW,WAAW;AACvC,WAAO,IAAI,KAAK,CAAC,MAAM,GAAG,SAAS,aAAa,QAAQ,CAAC,IAAI;AAAA,MAC3D,MAAM;AAAA,IAAA,CACP;AAAA,EACH;AAEA,QAAM,IAAI,MAAM,0BAA0B;AAC5C;AAEA,SAAS,WAAW,aAA6B;AAE/C,MAAI,YAAY,SAAS,GAAG,EAAG,QAAO;AACtC,QAAM,MAAM,YAAY,YAAA;AACxB,UAAQ,KAAA;AAAA,IACN,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO;AAAA,IACT;AACE,aAAO,SAAS,GAAG;AAAA,EAAA;AAEzB;AAEA,SAAS,aAAa,UAA0B;AAC9C,UAAQ,UAAA;AAAA,IACN,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AAAA,IACL,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO;AAAA,IACT,SAAS;AACP,YAAM,QAAQ,SAAS,QAAQ,GAAG;AAClC,UAAI,UAAU,GAAI,QAAO;AACzB,aAAO,SAAS,MAAM,QAAQ,CAAC,KAAK;AAAA,IACtC;AAAA,EAAA;AAEJ;AAWO,SAAS,oBAAoB,QAA6B;AAC/D,MAAI,OAAO,WAAW,aAAa;AACjC,WAAO,OAAO,KAAK,MAAM,EAAE,SAAS,QAAQ;AAAA,EAC9C;AACA,QAAM,QAAQ,IAAI,WAAW,MAAM;AACnC,QAAM,YAAY;AAClB,MAAI,SAAS;AACb,WAAS,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK,WAAW;AAChD,UAAM,MAAM,KAAK,IAAI,IAAI,WAAW,MAAM,MAAM;AAChD,cAAU,OAAO,aAAa;AAAA,MAC5B;AAAA,MACA,MAAM,SAAS,GAAG,GAAG;AAAA,IAAA;AAAA,EAEzB;AACA,SAAO,KAAK,MAAM;AACpB;AAEA,SAAS,oBAAoB,QAA6B;AACxD,MAAI;AACJ,MAAI;AACF,aAAS,KAAK,MAAM;AAAA,EACtB,SAAS,KAAK;AACZ,UAAM,MAAM,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAC3D,UAAM,IAAI,MAAM,wCAAwC,GAAG,EAAE;AAAA,EAC/D;AACA,QAAM,SAAS,IAAI,YAAY,OAAO,MAAM;AAC5C,QAAM,QAAQ,IAAI,WAAW,MAAM;AACnC,WAAS,IAAI,GAAG,IAAI,OAAO,QAAQ,KAAK;AACtC,UAAM,CAAC,IAAI,OAAO,WAAW,CAAC;AAAA,EAChC;AACA,SAAO;AACT;"}
|
|
1
|
+
{"version":3,"file":"audio.js","sources":["../../../src/utils/audio.ts"],"sourcesContent":["/**\n * Coerce the various audio input shapes accepted by `TranscriptionOptions.audio`\n * into a `File` suitable for `multipart/form-data` uploads.\n *\n * For base64 string inputs we require an explicit MIME type — either via a\n * `data:<mime>;base64,<payload>` URI prefix, or via the caller-provided\n * `audioFormat` parameter. Bare base64 without either is rejected, because\n * silently defaulting to `audio/mpeg` misreports non-mp3 audio to the server.\n *\n * The same rule applies to raw `ArrayBuffer` inputs: the caller must supply\n * an `audioFormat` so we know what MIME type and extension to use.\n */\nexport function toAudioFile(\n audio: string | File | Blob | ArrayBuffer,\n audioFormat?: string,\n): File {\n if (typeof File !== 'undefined' && audio instanceof File) {\n // Prefer the caller-supplied `audioFormat` over a potentially empty or\n // incorrect `File.type` — callers pass `audioFormat` precisely because\n // they have more context than the browser does about the payload. If\n // neither is set, fall through to the Blob-style error path below.\n if (audioFormat) {\n const mimeType = toMimeType(audioFormat)\n return new File([audio], `audio.${extensionFor(mimeType)}`, {\n type: mimeType,\n })\n }\n if (audio.type) {\n return audio\n }\n throw new Error(\n 'toAudioFile cannot infer type for File input with empty .type — pass an explicit audioFormat (e.g. \"mp3\", \"wav\", \"audio/mpeg\")',\n )\n }\n\n if (typeof Blob !== 'undefined' && audio instanceof Blob) {\n // Mirror the ArrayBuffer / bare-base64 paths: prefer the explicit\n // audioFormat argument over the Blob's (often empty) .type. We refuse to\n // fall back to `application/octet-stream` because that mislabels audio\n // for the server.\n const mimeType = audioFormat\n ? toMimeType(audioFormat)\n : audio.type || undefined\n if (!mimeType) {\n throw new Error(\n 'toAudioFile cannot infer type for Blob input with empty .type — pass an explicit audioFormat (e.g. \"mp3\", \"wav\", \"audio/mpeg\")',\n )\n }\n return new File([audio], `audio.${extensionFor(mimeType)}`, {\n type: mimeType,\n })\n }\n\n if (audio instanceof ArrayBuffer) {\n if (!audioFormat) {\n throw new Error(\n 'toAudioFile cannot infer type for ArrayBuffer input — pass an explicit audioFormat (e.g. \"mp3\", \"wav\", \"audio/mpeg\")',\n )\n }\n const mimeType = toMimeType(audioFormat)\n return new File([audio], `audio.${extensionFor(mimeType)}`, {\n type: mimeType,\n })\n }\n\n if (typeof audio === 'string') {\n if (audio.startsWith('data:')) {\n const [header, base64Data] = audio.split(',')\n // Fail loudly on malformed data: URIs instead of silently defaulting\n // to `audio/mpeg` — the file's contract is that we never mislabel\n // audio for the server.\n const headerMatch = header?.match(/data:([^;]+)/)\n const uriMimeType = headerMatch?.[1]\n if (!uriMimeType) {\n throw new Error(\n 'Malformed data: URI in toAudioFile: cannot parse MIME type — expected data:<mime>[;charset=…][;base64],<payload>',\n )\n }\n if (base64Data === undefined || base64Data.trim() === '') {\n throw new Error(\n 'Malformed data: URI in toAudioFile: missing base64 payload after comma',\n )\n }\n // Caller-supplied `audioFormat` wins over the URI-embedded MIME: the\n // caller has more context (the URI MIME may be wrong, or a generic\n // `application/octet-stream`).\n const mimeType = audioFormat ? toMimeType(audioFormat) : uriMimeType\n const buffer = base64ToArrayBuffer(base64Data)\n return new File([buffer], `audio.${extensionFor(mimeType)}`, {\n type: mimeType,\n })\n }\n\n if (!audioFormat) {\n throw new Error(\n 'toAudioFile requires a data: URI (e.g. data:audio/wav;base64,...) or an explicit audioFormat argument — bare base64 strings have no MIME type to infer',\n )\n }\n\n const buffer = base64ToArrayBuffer(audio)\n const mimeType = toMimeType(audioFormat)\n return new File([buffer], `audio.${extensionFor(mimeType)}`, {\n type: mimeType,\n })\n }\n\n throw new Error('Invalid audio input type')\n}\n\nfunction toMimeType(audioFormat: string): string {\n // Accept either \"audio/…\" strings or bare extensions like \"mp3\".\n if (audioFormat.includes('/')) return audioFormat\n const ext = audioFormat.toLowerCase()\n switch (ext) {\n case 'mp3':\n return 'audio/mpeg'\n case 'wav':\n return 'audio/wav'\n case 'ogg':\n return 'audio/ogg'\n case 'opus':\n return 'audio/opus'\n case 'flac':\n return 'audio/flac'\n case 'aac':\n return 'audio/aac'\n case 'mp4':\n return 'audio/mp4'\n case 'm4a':\n return 'audio/mp4'\n case 'webm':\n return 'audio/webm'\n case 'pcm':\n return 'audio/L16'\n case 'mulaw':\n return 'audio/basic'\n case 'alaw':\n return 'audio/x-alaw-basic'\n default:\n return `audio/${ext}`\n }\n}\n\nfunction extensionFor(mimeType: string): string {\n switch (mimeType) {\n case 'audio/mpeg':\n return 'mp3'\n case 'audio/wav':\n case 'audio/x-wav':\n return 'wav'\n case 'audio/ogg':\n return 'ogg'\n case 'audio/opus':\n return 'opus'\n case 'audio/flac':\n return 'flac'\n case 'audio/aac':\n return 'aac'\n case 'audio/mp4':\n return 'm4a'\n case 'audio/webm':\n return 'webm'\n case 'audio/L16':\n return 'pcm'\n case 'audio/basic':\n return 'mulaw'\n case 'audio/x-alaw-basic':\n return 'alaw'\n default: {\n const slash = mimeType.indexOf('/')\n if (slash === -1) return 'bin'\n return mimeType.slice(slash + 1) || 'bin'\n }\n }\n}\n\n/**\n * Cross-runtime ArrayBuffer → base64 conversion.\n *\n * Uses Node's `Buffer` when available (fastest path on server) and falls\n * back to `btoa` + chunked `String.fromCharCode` everywhere else (browser,\n * Cloudflare Workers, Bun, Deno). Chunking is required because a very\n * large audio buffer spread into `String.fromCharCode(...bytes)` in one\n * call can hit `Maximum call stack size exceeded`.\n */\nexport function arrayBufferToBase64(buffer: ArrayBuffer): string {\n if (typeof Buffer !== 'undefined') {\n return Buffer.from(buffer).toString('base64')\n }\n const bytes = new Uint8Array(buffer)\n const chunkSize = 0x8000\n let binary = ''\n for (let i = 0; i < bytes.length; i += chunkSize) {\n const end = Math.min(i + chunkSize, bytes.length)\n binary += String.fromCharCode.apply(\n null,\n // eslint-disable-next-line no-restricted-syntax -- TS lib types String.fromCharCode.apply as Array<number> but runtime accepts any ArrayLike\n bytes.subarray(i, end) as unknown as Array<number>,\n )\n }\n return btoa(binary)\n}\n\nfunction base64ToArrayBuffer(base64: string): ArrayBuffer {\n let binary: string\n try {\n binary = atob(base64)\n } catch (err) {\n const msg = err instanceof Error ? err.message : String(err)\n throw new Error(`Invalid base64 input to toAudioFile: ${msg}`)\n }\n const buffer = new ArrayBuffer(binary.length)\n const bytes = new Uint8Array(buffer)\n for (let i = 0; i < binary.length; i++) {\n bytes[i] = binary.charCodeAt(i)\n }\n return buffer\n}\n"],"names":["mimeType","buffer"],"mappings":"AAYO,SAAS,YACd,OACA,aACM;AACN,MAAI,OAAO,SAAS,eAAe,iBAAiB,MAAM;AAKxD,QAAI,aAAa;AACf,YAAM,WAAW,WAAW,WAAW;AACvC,aAAO,IAAI,KAAK,CAAC,KAAK,GAAG,SAAS,aAAa,QAAQ,CAAC,IAAI;AAAA,QAC1D,MAAM;AAAA,MAAA,CACP;AAAA,IACH;AACA,QAAI,MAAM,MAAM;AACd,aAAO;AAAA,IACT;AACA,UAAM,IAAI;AAAA,MACR;AAAA,IAAA;AAAA,EAEJ;AAEA,MAAI,OAAO,SAAS,eAAe,iBAAiB,MAAM;AAKxD,UAAM,WAAW,cACb,WAAW,WAAW,IACtB,MAAM,QAAQ;AAClB,QAAI,CAAC,UAAU;AACb,YAAM,IAAI;AAAA,QACR;AAAA,MAAA;AAAA,IAEJ;AACA,WAAO,IAAI,KAAK,CAAC,KAAK,GAAG,SAAS,aAAa,QAAQ,CAAC,IAAI;AAAA,MAC1D,MAAM;AAAA,IAAA,CACP;AAAA,EACH;AAEA,MAAI,iBAAiB,aAAa;AAChC,QAAI,CAAC,aAAa;AAChB,YAAM,IAAI;AAAA,QACR;AAAA,MAAA;AAAA,IAEJ;AACA,UAAM,WAAW,WAAW,WAAW;AACvC,WAAO,IAAI,KAAK,CAAC,KAAK,GAAG,SAAS,aAAa,QAAQ,CAAC,IAAI;AAAA,MAC1D,MAAM;AAAA,IAAA,CACP;AAAA,EACH;AAEA,MAAI,OAAO,UAAU,UAAU;AAC7B,QAAI,MAAM,WAAW,OAAO,GAAG;AAC7B,YAAM,CAAC,QAAQ,UAAU,IAAI,MAAM,MAAM,GAAG;AAI5C,YAAM,cAAc,QAAQ,MAAM,cAAc;AAChD,YAAM,cAAc,cAAc,CAAC;AACnC,UAAI,CAAC,aAAa;AAChB,cAAM,IAAI;AAAA,UACR;AAAA,QAAA;AAAA,MAEJ;AACA,UAAI,eAAe,UAAa,WAAW,KAAA,MAAW,IAAI;AACxD,cAAM,IAAI;AAAA,UACR;AAAA,QAAA;AAAA,MAEJ;AAIA,YAAMA,YAAW,cAAc,WAAW,WAAW,IAAI;AACzD,YAAMC,UAAS,oBAAoB,UAAU;AAC7C,aAAO,IAAI,KAAK,CAACA,OAAM,GAAG,SAAS,aAAaD,SAAQ,CAAC,IAAI;AAAA,QAC3D,MAAMA;AAAAA,MAAA,CACP;AAAA,IACH;AAEA,QAAI,CAAC,aAAa;AAChB,YAAM,IAAI;AAAA,QACR;AAAA,MAAA;AAAA,IAEJ;AAEA,UAAM,SAAS,oBAAoB,KAAK;AACxC,UAAM,WAAW,WAAW,WAAW;AACvC,WAAO,IAAI,KAAK,CAAC,MAAM,GAAG,SAAS,aAAa,QAAQ,CAAC,IAAI;AAAA,MAC3D,MAAM;AAAA,IAAA,CACP;AAAA,EACH;AAEA,QAAM,IAAI,MAAM,0BAA0B;AAC5C;AAEA,SAAS,WAAW,aAA6B;AAE/C,MAAI,YAAY,SAAS,GAAG,EAAG,QAAO;AACtC,QAAM,MAAM,YAAY,YAAA;AACxB,UAAQ,KAAA;AAAA,IACN,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO;AAAA,IACT;AACE,aAAO,SAAS,GAAG;AAAA,EAAA;AAEzB;AAEA,SAAS,aAAa,UAA0B;AAC9C,UAAQ,UAAA;AAAA,IACN,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AAAA,IACL,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO;AAAA,IACT,SAAS;AACP,YAAM,QAAQ,SAAS,QAAQ,GAAG;AAClC,UAAI,UAAU,GAAI,QAAO;AACzB,aAAO,SAAS,MAAM,QAAQ,CAAC,KAAK;AAAA,IACtC;AAAA,EAAA;AAEJ;AAWO,SAAS,oBAAoB,QAA6B;AAC/D,MAAI,OAAO,WAAW,aAAa;AACjC,WAAO,OAAO,KAAK,MAAM,EAAE,SAAS,QAAQ;AAAA,EAC9C;AACA,QAAM,QAAQ,IAAI,WAAW,MAAM;AACnC,QAAM,YAAY;AAClB,MAAI,SAAS;AACb,WAAS,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK,WAAW;AAChD,UAAM,MAAM,KAAK,IAAI,IAAI,WAAW,MAAM,MAAM;AAChD,cAAU,OAAO,aAAa;AAAA,MAC5B;AAAA;AAAA,MAEA,MAAM,SAAS,GAAG,GAAG;AAAA,IAAA;AAAA,EAEzB;AACA,SAAO,KAAK,MAAM;AACpB;AAEA,SAAS,oBAAoB,QAA6B;AACxD,MAAI;AACJ,MAAI;AACF,aAAS,KAAK,MAAM;AAAA,EACtB,SAAS,KAAK;AACZ,UAAM,MAAM,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAC3D,UAAM,IAAI,MAAM,wCAAwC,GAAG,EAAE;AAAA,EAC/D;AACA,QAAM,SAAS,IAAI,YAAY,OAAO,MAAM;AAC5C,QAAM,QAAQ,IAAI,WAAW,MAAM;AACnC,WAAS,IAAI,GAAG,IAAI,OAAO,QAAQ,KAAK;AACtC,UAAM,CAAC,IAAI,OAAO,WAAW,CAAC;AAAA,EAChC;AACA,SAAO;AACT;"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@tanstack/ai-grok",
|
|
3
|
-
"version": "0.8.
|
|
3
|
+
"version": "0.8.5",
|
|
4
4
|
"description": "Grok (xAI) adapter for TanStack AI",
|
|
5
5
|
"author": "",
|
|
6
6
|
"license": "MIT",
|
|
@@ -36,17 +36,17 @@
|
|
|
36
36
|
"dependencies": {
|
|
37
37
|
"openai": "^6.9.1",
|
|
38
38
|
"@tanstack/ai-utils": "0.2.0",
|
|
39
|
-
"@tanstack/openai-base": "0.3.
|
|
39
|
+
"@tanstack/openai-base": "0.3.4"
|
|
40
40
|
},
|
|
41
41
|
"devDependencies": {
|
|
42
42
|
"@vitest/coverage-v8": "4.0.14",
|
|
43
43
|
"vite": "^7.3.3",
|
|
44
|
-
"@tanstack/ai": "0.20.
|
|
45
|
-
"@tanstack/ai-client": "0.11.
|
|
44
|
+
"@tanstack/ai": "0.20.1",
|
|
45
|
+
"@tanstack/ai-client": "0.11.3"
|
|
46
46
|
},
|
|
47
47
|
"peerDependencies": {
|
|
48
48
|
"zod": "^4.0.0",
|
|
49
|
-
"@tanstack/ai": "^0.20.
|
|
49
|
+
"@tanstack/ai": "^0.20.1"
|
|
50
50
|
},
|
|
51
51
|
"scripts": {
|
|
52
52
|
"build": "vite build",
|
package/src/adapters/image.ts
CHANGED
|
@@ -46,7 +46,7 @@ export class GrokImageAdapter<
|
|
|
46
46
|
GrokImageModelProviderOptionsByName,
|
|
47
47
|
GrokImageModelSizeByName
|
|
48
48
|
> {
|
|
49
|
-
readonly kind = 'image' as const
|
|
49
|
+
override readonly kind = 'image' as const
|
|
50
50
|
readonly name = 'grok' as const
|
|
51
51
|
|
|
52
52
|
protected client: OpenAI
|
|
@@ -65,11 +65,13 @@ export class GrokImageAdapter<
|
|
|
65
65
|
validateImageSize(model, size)
|
|
66
66
|
validateNumberOfImages(model, numberOfImages)
|
|
67
67
|
|
|
68
|
-
const
|
|
68
|
+
const resolvedSize = size as OpenAI_SDK.Images.ImageGenerateParams['size']
|
|
69
|
+
const request: OpenAI_SDK.Images.ImageGenerateParamsNonStreaming = {
|
|
69
70
|
model,
|
|
70
71
|
prompt,
|
|
71
72
|
n: numberOfImages ?? 1,
|
|
72
|
-
size:
|
|
73
|
+
...(resolvedSize !== undefined && { size: resolvedSize }),
|
|
74
|
+
stream: false,
|
|
73
75
|
...modelOptions,
|
|
74
76
|
}
|
|
75
77
|
|
|
@@ -78,19 +80,26 @@ export class GrokImageAdapter<
|
|
|
78
80
|
`activity=image provider=${this.name} model=${model} n=${request.n ?? 1} size=${request.size ?? 'default'}`,
|
|
79
81
|
{ provider: this.name, model },
|
|
80
82
|
)
|
|
81
|
-
const response = await this.client.images.generate(
|
|
82
|
-
...request,
|
|
83
|
-
stream: false,
|
|
84
|
-
})
|
|
83
|
+
const response = await this.client.images.generate(request)
|
|
85
84
|
|
|
86
85
|
const images: Array<GeneratedImage> = (response.data ?? []).flatMap(
|
|
87
86
|
(item): Array<GeneratedImage> => {
|
|
88
87
|
const revisedPrompt = item.revised_prompt
|
|
89
88
|
if (item.b64_json) {
|
|
90
|
-
return [
|
|
89
|
+
return [
|
|
90
|
+
{
|
|
91
|
+
b64Json: item.b64_json,
|
|
92
|
+
...(revisedPrompt !== undefined && { revisedPrompt }),
|
|
93
|
+
},
|
|
94
|
+
]
|
|
91
95
|
}
|
|
92
96
|
if (item.url) {
|
|
93
|
-
return [
|
|
97
|
+
return [
|
|
98
|
+
{
|
|
99
|
+
url: item.url,
|
|
100
|
+
...(revisedPrompt !== undefined && { revisedPrompt }),
|
|
101
|
+
},
|
|
102
|
+
]
|
|
94
103
|
}
|
|
95
104
|
return []
|
|
96
105
|
},
|
|
@@ -100,13 +109,13 @@ export class GrokImageAdapter<
|
|
|
100
109
|
id: generateId(this.name),
|
|
101
110
|
model,
|
|
102
111
|
images,
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
112
|
+
...(response.usage && {
|
|
113
|
+
usage: {
|
|
114
|
+
inputTokens: response.usage.input_tokens,
|
|
115
|
+
outputTokens: response.usage.output_tokens,
|
|
116
|
+
totalTokens: response.usage.total_tokens,
|
|
117
|
+
},
|
|
118
|
+
}),
|
|
110
119
|
}
|
|
111
120
|
} catch (error: unknown) {
|
|
112
121
|
options.logger.errors(`${this.name}.generateImages fatal`, {
|
package/src/adapters/text.ts
CHANGED
|
@@ -53,8 +53,8 @@ export class GrokTextAdapter<
|
|
|
53
53
|
GrokMessageMetadataByModality,
|
|
54
54
|
TToolCapabilities
|
|
55
55
|
> {
|
|
56
|
-
readonly kind = 'text' as const
|
|
57
|
-
readonly name = 'grok' as const
|
|
56
|
+
override readonly kind = 'text' as const
|
|
57
|
+
override readonly name = 'grok' as const
|
|
58
58
|
|
|
59
59
|
constructor(config: GrokTextConfig, model: TModel) {
|
|
60
60
|
super(model, 'grok', new OpenAI(withGrokDefaults(config)))
|
|
@@ -134,13 +134,14 @@ export class GrokTranscriptionAdapter<
|
|
|
134
134
|
},
|
|
135
135
|
)
|
|
136
136
|
|
|
137
|
+
const resolvedLanguage = data.language ?? language
|
|
137
138
|
return {
|
|
138
139
|
id: generateId(this.name),
|
|
139
140
|
model,
|
|
140
141
|
text: data.text,
|
|
141
|
-
language:
|
|
142
|
+
...(resolvedLanguage !== undefined && { language: resolvedLanguage }),
|
|
142
143
|
duration: data.duration,
|
|
143
|
-
words,
|
|
144
|
+
...(words !== undefined && { words }),
|
|
144
145
|
}
|
|
145
146
|
} catch (error) {
|
|
146
147
|
logger.errors('grok.transcribe fatal', {
|
package/src/realtime/adapter.ts
CHANGED
|
@@ -129,7 +129,11 @@ async function createWebRTCConnection(
|
|
|
129
129
|
|
|
130
130
|
let audioElement: HTMLAudioElement | null = null
|
|
131
131
|
|
|
132
|
-
|
|
132
|
+
// Captured into a const so closures see a non-nullable reference (teardown
|
|
133
|
+
// re-points the outer `dataChannel` to null, but in-flight closures still
|
|
134
|
+
// need to close their own channel).
|
|
135
|
+
const channel = pc.createDataChannel('oai-events')
|
|
136
|
+
let dataChannel: RTCDataChannel | null = channel
|
|
133
137
|
|
|
134
138
|
let currentMode: RealtimeMode = 'idle'
|
|
135
139
|
let currentMessageId: string | null = null
|
|
@@ -171,8 +175,6 @@ async function createWebRTCConnection(
|
|
|
171
175
|
}
|
|
172
176
|
}
|
|
173
177
|
|
|
174
|
-
dataChannel = pc.createDataChannel('oai-events')
|
|
175
|
-
|
|
176
178
|
let dataChannelOpened = false
|
|
177
179
|
let rejectDataChannelReady: ((reason: unknown) => void) | null = null
|
|
178
180
|
let dataChannelReadyTimeout: ReturnType<typeof setTimeout> | null = null
|
|
@@ -198,7 +200,7 @@ async function createWebRTCConnection(
|
|
|
198
200
|
}
|
|
199
201
|
}, 15000)
|
|
200
202
|
|
|
201
|
-
|
|
203
|
+
channel.onopen = () => {
|
|
202
204
|
dataChannelOpened = true
|
|
203
205
|
if (dataChannelReadyTimeout !== null) {
|
|
204
206
|
clearTimeout(dataChannelReadyTimeout)
|
|
@@ -213,7 +215,7 @@ async function createWebRTCConnection(
|
|
|
213
215
|
}
|
|
214
216
|
})
|
|
215
217
|
|
|
216
|
-
|
|
218
|
+
channel.onmessage = (event) => {
|
|
217
219
|
try {
|
|
218
220
|
const message = JSON.parse(event.data)
|
|
219
221
|
const messageRecord: Record<string, unknown> =
|
|
@@ -235,7 +237,7 @@ async function createWebRTCConnection(
|
|
|
235
237
|
}
|
|
236
238
|
}
|
|
237
239
|
|
|
238
|
-
|
|
240
|
+
channel.onerror = (error) => {
|
|
239
241
|
// Closing the peer connection cascades into `onerror`/`onclose` on the
|
|
240
242
|
// data channel. Once teardown has started, re-surfacing those as
|
|
241
243
|
// `emit('error')` is noise that confuses consumers (they just called
|
|
@@ -250,6 +252,7 @@ async function createWebRTCConnection(
|
|
|
250
252
|
// doesn't end up as "[object Event]".
|
|
251
253
|
// `onerror` always fires with an Event (often an RTCErrorEvent), so we
|
|
252
254
|
// can read it via the untyped helpers without first proving object-ness.
|
|
255
|
+
// eslint-disable-next-line no-restricted-syntax -- RTCErrorEvent is a typed DOM class that does not structurally overlap Record<string, unknown>; we duck-type it via readObject/readString
|
|
253
256
|
const errorRecord = error as unknown as Record<string, unknown>
|
|
254
257
|
const rtcError = readObject(errorRecord, 'error')
|
|
255
258
|
const msg =
|
|
@@ -261,7 +264,7 @@ async function createWebRTCConnection(
|
|
|
261
264
|
emit('error', { error: dcErr })
|
|
262
265
|
}
|
|
263
266
|
|
|
264
|
-
|
|
267
|
+
channel.onclose = () => {
|
|
265
268
|
// Same rationale as `onerror` above: `pc.close()` during teardown
|
|
266
269
|
// cascades to the data channel's `onclose`. If we've already started
|
|
267
270
|
// teardown, there's nothing to do here.
|
|
@@ -730,7 +733,9 @@ async function createWebRTCConnection(
|
|
|
730
733
|
currentMode = 'listening'
|
|
731
734
|
emit('mode_change', { mode: 'listening' })
|
|
732
735
|
}
|
|
733
|
-
emit('interrupted', {
|
|
736
|
+
emit('interrupted', {
|
|
737
|
+
...(currentMessageId !== null && { messageId: currentMessageId }),
|
|
738
|
+
})
|
|
734
739
|
break
|
|
735
740
|
|
|
736
741
|
case 'error': {
|
|
@@ -759,10 +764,12 @@ async function createWebRTCConnection(
|
|
|
759
764
|
break
|
|
760
765
|
}
|
|
761
766
|
|
|
767
|
+
case undefined:
|
|
762
768
|
default:
|
|
763
769
|
// The xAI realtime protocol is a moving target; log unhandled event
|
|
764
770
|
// types at provider level so they're visible during debugging without
|
|
765
|
-
// emitting a user-visible error.
|
|
771
|
+
// emitting a user-visible error. `undefined` shares the bucket because
|
|
772
|
+
// a malformed event without a `type` field is just as unhandleable.
|
|
766
773
|
logger.provider('grok.realtime unhandled server event', {
|
|
767
774
|
type: event.type,
|
|
768
775
|
})
|
|
@@ -928,7 +935,7 @@ async function createWebRTCConnection(
|
|
|
928
935
|
`provider=grok direction=out type=${readString(event, 'type') ?? '<unknown>'}`,
|
|
929
936
|
{ frame: event },
|
|
930
937
|
)
|
|
931
|
-
|
|
938
|
+
channel.send(JSON.stringify(event))
|
|
932
939
|
}
|
|
933
940
|
pendingEvents.length = 0
|
|
934
941
|
} catch (error) {
|
|
@@ -1117,17 +1124,21 @@ async function createWebRTCConnection(
|
|
|
1117
1124
|
sendEvent({ type: 'response.cancel' })
|
|
1118
1125
|
currentMode = 'listening'
|
|
1119
1126
|
emit('mode_change', { mode: 'listening' })
|
|
1120
|
-
emit('interrupted', {
|
|
1127
|
+
emit('interrupted', {
|
|
1128
|
+
...(currentMessageId !== null && { messageId: currentMessageId }),
|
|
1129
|
+
})
|
|
1121
1130
|
},
|
|
1122
1131
|
|
|
1123
1132
|
on<TEvent extends RealtimeEvent>(
|
|
1124
1133
|
event: TEvent,
|
|
1125
1134
|
handler: RealtimeEventHandler<TEvent>,
|
|
1126
1135
|
): () => void {
|
|
1127
|
-
|
|
1128
|
-
|
|
1136
|
+
let handlers = eventHandlers.get(event)
|
|
1137
|
+
if (!handlers) {
|
|
1138
|
+
handlers = new Set()
|
|
1139
|
+
eventHandlers.set(event, handlers)
|
|
1129
1140
|
}
|
|
1130
|
-
|
|
1141
|
+
handlers.add(handler)
|
|
1131
1142
|
|
|
1132
1143
|
return () => {
|
|
1133
1144
|
eventHandlers.get(event)?.delete(handler)
|
package/src/utils/audio.ts
CHANGED
|
@@ -194,6 +194,7 @@ export function arrayBufferToBase64(buffer: ArrayBuffer): string {
|
|
|
194
194
|
const end = Math.min(i + chunkSize, bytes.length)
|
|
195
195
|
binary += String.fromCharCode.apply(
|
|
196
196
|
null,
|
|
197
|
+
// eslint-disable-next-line no-restricted-syntax -- TS lib types String.fromCharCode.apply as Array<number> but runtime accepts any ArrayLike
|
|
197
198
|
bytes.subarray(i, end) as unknown as Array<number>,
|
|
198
199
|
)
|
|
199
200
|
}
|