@happyvertical/speech 0.96.1 → 0.98.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENT.md +42 -3
- package/README.md +335 -3
- package/dist/__tests__/fake-websocket.d.ts +38 -0
- package/dist/__tests__/fake-websocket.d.ts.map +1 -0
- package/dist/adapters/local/abort.d.ts +6 -0
- package/dist/adapters/local/abort.d.ts.map +1 -0
- package/dist/adapters/local/audio.d.ts +45 -0
- package/dist/adapters/local/audio.d.ts.map +1 -0
- package/dist/adapters/local/env.d.ts +15 -0
- package/dist/adapters/local/env.d.ts.map +1 -0
- package/dist/adapters/local/runtime.d.ts +58 -0
- package/dist/adapters/local/runtime.d.ts.map +1 -0
- package/dist/adapters/local/transcriber.d.ts +40 -0
- package/dist/adapters/local/transcriber.d.ts.map +1 -0
- package/dist/adapters/local/transformers-shim.d.ts +8 -0
- package/dist/adapters/local/transformers-shim.d.ts.map +1 -0
- package/dist/adapters/local/types.d.ts +92 -0
- package/dist/adapters/local/types.d.ts.map +1 -0
- package/dist/adapters/local/worker.d.ts +118 -0
- package/dist/adapters/local/worker.d.ts.map +1 -0
- package/dist/adapters/openai-compatible-transcriber.d.ts +23 -0
- package/dist/adapters/openai-compatible-transcriber.d.ts.map +1 -0
- package/dist/adapters/openai-compatible.d.ts +2 -1
- package/dist/adapters/openai-compatible.d.ts.map +1 -1
- package/dist/adapters/openai-realtime.d.ts +48 -0
- package/dist/adapters/openai-realtime.d.ts.map +1 -0
- package/dist/adapters/studio-server.d.ts.map +1 -1
- package/dist/adapters/voxtral-realtime.d.ts +44 -0
- package/dist/adapters/voxtral-realtime.d.ts.map +1 -0
- package/dist/chunks/registry-DfID41Ew.js +343 -0
- package/dist/chunks/registry-DfID41Ew.js.map +1 -0
- package/dist/index.d.ts +12 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2148 -85
- package/dist/index.js.map +1 -1
- package/dist/local.d.ts +21 -0
- package/dist/local.d.ts.map +1 -0
- package/dist/local.js +963 -0
- package/dist/local.js.map +1 -0
- package/dist/shared/audio.d.ts +47 -0
- package/dist/shared/audio.d.ts.map +1 -0
- package/dist/shared/env.d.ts +75 -0
- package/dist/shared/env.d.ts.map +1 -0
- package/dist/shared/errors.d.ts +3 -0
- package/dist/shared/errors.d.ts.map +1 -1
- package/dist/shared/factory.d.ts +1 -4
- package/dist/shared/factory.d.ts.map +1 -1
- package/dist/shared/http.d.ts +29 -3
- package/dist/shared/http.d.ts.map +1 -1
- package/dist/shared/pcm.d.ts +21 -0
- package/dist/shared/pcm.d.ts.map +1 -0
- package/dist/shared/realtime-auth.d.ts +34 -0
- package/dist/shared/realtime-auth.d.ts.map +1 -0
- package/dist/shared/realtime-session.d.ts +240 -0
- package/dist/shared/realtime-session.d.ts.map +1 -0
- package/dist/shared/registry.d.ts +9 -0
- package/dist/shared/registry.d.ts.map +1 -0
- package/dist/shared/retry.d.ts +31 -0
- package/dist/shared/retry.d.ts.map +1 -0
- package/dist/shared/streaming-client-secret.d.ts +90 -0
- package/dist/shared/streaming-client-secret.d.ts.map +1 -0
- package/dist/shared/streaming-factory.d.ts +65 -0
- package/dist/shared/streaming-factory.d.ts.map +1 -0
- package/dist/shared/streaming-types.d.ts +303 -0
- package/dist/shared/streaming-types.d.ts.map +1 -0
- package/dist/shared/turn-limit.d.ts +84 -0
- package/dist/shared/turn-limit.d.ts.map +1 -0
- package/dist/shared/types.d.ts +84 -6
- package/dist/shared/types.d.ts.map +1 -1
- package/dist/shared/usage.d.ts +35 -0
- package/dist/shared/usage.d.ts.map +1 -0
- package/dist/shared/websocket.d.ts +56 -0
- package/dist/shared/websocket.d.ts.map +1 -0
- package/metadata.json +7 -1
- package/package.json +19 -3
package/dist/index.js
CHANGED
|
@@ -1,45 +1,137 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
var SpeechConfigurationError = class extends SpeechError {
|
|
13
|
-
constructor(message, adapter) {
|
|
14
|
-
super(message, "SPEECH_CONFIGURATION_ERROR", adapter);
|
|
15
|
-
this.name = "SpeechConfigurationError";
|
|
16
|
-
}
|
|
17
|
-
};
|
|
18
|
-
var InvalidSpeechAdapterError = class extends SpeechError {
|
|
19
|
-
constructor(type, kind) {
|
|
20
|
-
super(`Invalid ${kind} speech adapter type: ${type}`, "INVALID_ADAPTER");
|
|
21
|
-
this.name = "InvalidSpeechAdapterError";
|
|
22
|
-
}
|
|
1
|
+
import { C as SpeechProviderError, S as SpeechError, _ as mimeTypeToAudioExtension, a as TRANSCRIBER_ENV_KEYS, b as InvalidSpeechAdapterError, c as hasTranscriberEnv, d as parseOptionalInteger, f as readEnv, g as DEFAULT_MAX_AUDIO_BYTES, h as reportSpeechUsage, i as registeredOptionalTranscribers, l as mergeHeaderInits, m as audioSecondsFromProviderUsage, n as optionalTranscriberEntry, o as defaultEnv, p as resolveTranscriberConfig, s as hasAnyEnv, t as getOptionalTranscriberFactory, u as parseHeadersEnv, v as normalizeAudioInput, x as SpeechConfigurationError, y as normalizeMimeType } from "./chunks/registry-DfID41Ew.js";
|
|
2
|
+
//#region src/shared/retry.ts
|
|
3
|
+
/**
|
|
4
|
+
* Retry with exponential backoff for speech provider calls. Retries only
|
|
5
|
+
* rate-limit (429) and server (5xx) responses, honours `Retry-After`, and
|
|
6
|
+
* stops immediately when the caller aborts.
|
|
7
|
+
*/
|
|
8
|
+
var DEFAULT_SPEECH_RETRY = {
|
|
9
|
+
maxRetries: 2,
|
|
10
|
+
initialDelayMs: 500,
|
|
11
|
+
maxDelayMs: 3e4
|
|
23
12
|
};
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
13
|
+
function resolveRetryOptions(retry, defaults = DEFAULT_SPEECH_RETRY) {
|
|
14
|
+
if (retry === false) return false;
|
|
15
|
+
if (retry === void 0) return defaults;
|
|
16
|
+
const base = defaults || DEFAULT_SPEECH_RETRY;
|
|
17
|
+
return {
|
|
18
|
+
maxRetries: Math.floor(finiteRetryValue("maxRetries", retry.maxRetries, base.maxRetries)),
|
|
19
|
+
initialDelayMs: finiteRetryValue("initialDelayMs", retry.initialDelayMs, base.initialDelayMs),
|
|
20
|
+
maxDelayMs: finiteRetryValue("maxDelayMs", retry.maxDelayMs, base.maxDelayMs)
|
|
21
|
+
};
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* Rejects non-finite values (`NaN`, `Infinity`): they would make the retry
|
|
25
|
+
* budget or delay cap unbounded. Negative values clamp to `0`.
|
|
26
|
+
*/
|
|
27
|
+
function finiteRetryValue(name, value, fallback) {
|
|
28
|
+
if (value === void 0) return Math.max(0, fallback);
|
|
29
|
+
if (typeof value !== "number" || !Number.isFinite(value)) throw new SpeechConfigurationError(`Speech retry option ${name} must be a finite number`);
|
|
30
|
+
return Math.max(0, value);
|
|
31
|
+
}
|
|
32
|
+
function isRetryableStatus(status) {
|
|
33
|
+
return status === 429 || status !== void 0 && status >= 500;
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* Parses a `Retry-After` header (delta-seconds or HTTP-date) into
|
|
37
|
+
* milliseconds. Returns `undefined` when absent or unparseable.
|
|
38
|
+
*/
|
|
39
|
+
function parseRetryAfter(value, now = Date.now()) {
|
|
40
|
+
if (!value?.trim()) return;
|
|
41
|
+
const trimmed = value.trim();
|
|
42
|
+
if (/^\d+(\.\d+)?$/.test(trimmed)) return Math.round(Number(trimmed) * 1e3);
|
|
43
|
+
const date = Date.parse(trimmed);
|
|
44
|
+
return Number.isNaN(date) ? void 0 : Math.max(0, date - now);
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* Runs `attempt` until it succeeds, throws a non-retryable error, or the
|
|
48
|
+
* retry budget is exhausted. Retryable errors are {@link SpeechProviderError}
|
|
49
|
+
* instances whose `status` is 429 or 5xx.
|
|
50
|
+
*/
|
|
51
|
+
async function withSpeechRetry(attempt, retry, signal) {
|
|
52
|
+
const maxRetries = retry ? retry.maxRetries : 0;
|
|
53
|
+
for (let attemptNumber = 0;; attemptNumber += 1) try {
|
|
54
|
+
return await attempt(attemptNumber);
|
|
55
|
+
} catch (error) {
|
|
56
|
+
if (!retry || attemptNumber >= maxRetries || signal?.aborted || !(error instanceof SpeechProviderError) || !isRetryableStatus(error.status)) throw error;
|
|
57
|
+
const backoff = Math.min(retry.maxDelayMs, retry.initialDelayMs * 2 ** attemptNumber);
|
|
58
|
+
const retryAfterMs = error.retryAfterMs;
|
|
59
|
+
if (retryAfterMs !== void 0 && retryAfterMs > retry.maxDelayMs) throw error;
|
|
60
|
+
await sleep(Math.max(backoff, retryAfterMs ?? 0), signal);
|
|
33
61
|
}
|
|
34
|
-
}
|
|
62
|
+
}
|
|
63
|
+
function sleep(ms, signal) {
|
|
64
|
+
return new Promise((resolve, reject) => {
|
|
65
|
+
if (signal?.aborted) {
|
|
66
|
+
reject(signal.reason);
|
|
67
|
+
return;
|
|
68
|
+
}
|
|
69
|
+
const onAbort = () => {
|
|
70
|
+
clearTimeout(timer);
|
|
71
|
+
reject(signal?.reason);
|
|
72
|
+
};
|
|
73
|
+
const timer = setTimeout(() => {
|
|
74
|
+
signal?.removeEventListener("abort", onAbort);
|
|
75
|
+
resolve();
|
|
76
|
+
}, ms);
|
|
77
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
78
|
+
});
|
|
79
|
+
}
|
|
35
80
|
//#endregion
|
|
36
81
|
//#region src/shared/http.ts
|
|
82
|
+
/**
|
|
83
|
+
* Parses a base URL, dropping any fragment and keeping any query string.
|
|
84
|
+
* Throws {@link SpeechConfigurationError} for a blank or unparseable value.
|
|
85
|
+
* The value is not echoed in the error because it may carry credentials.
|
|
86
|
+
*/
|
|
87
|
+
function parseBaseUrl(baseUrl) {
|
|
88
|
+
const trimmed = baseUrl.trim();
|
|
89
|
+
if (!trimmed) throw new SpeechConfigurationError("Speech adapter baseUrl is required");
|
|
90
|
+
let url;
|
|
91
|
+
try {
|
|
92
|
+
url = new URL(trimmed);
|
|
93
|
+
} catch {
|
|
94
|
+
throw new SpeechConfigurationError("Speech adapter baseUrl is not a valid absolute URL");
|
|
95
|
+
}
|
|
96
|
+
url.hash = "";
|
|
97
|
+
return url;
|
|
98
|
+
}
|
|
99
|
+
/**
|
|
100
|
+
* Returns `baseUrl` with a trailing `/` on its path so relative paths resolve
|
|
101
|
+
* beneath it. The query string is preserved and any fragment is dropped.
|
|
102
|
+
*/
|
|
37
103
|
function normalizeBaseUrl(baseUrl) {
|
|
38
|
-
|
|
39
|
-
|
|
104
|
+
const url = parseBaseUrl(baseUrl);
|
|
105
|
+
if (!url.pathname.endsWith("/")) url.pathname = `${url.pathname}/`;
|
|
106
|
+
return url.toString();
|
|
40
107
|
}
|
|
108
|
+
/**
|
|
109
|
+
* Resolves `path` against `baseUrl`. A relative path without its own query
|
|
110
|
+
* inherits the base URL's query (for example a gateway `api-version`); an
|
|
111
|
+
* absolute URL is returned unchanged.
|
|
112
|
+
*/
|
|
41
113
|
function resolveSpeechUrl(baseUrl, path) {
|
|
42
|
-
|
|
114
|
+
const base = new URL(normalizeBaseUrl(baseUrl));
|
|
115
|
+
const isAbsolute = /^[a-z][a-z\d+.-]*:/i.test(path);
|
|
116
|
+
const resolved = new URL(isAbsolute ? path : path.replace(/^\//, ""), base);
|
|
117
|
+
if (!isAbsolute && !resolved.search && base.search) resolved.search = base.search;
|
|
118
|
+
return resolved.toString();
|
|
119
|
+
}
|
|
120
|
+
/**
|
|
121
|
+
* Resolves an OpenAI-compatible endpoint from a base URL. Accepts a server
|
|
122
|
+
* root (`https://api.openai.com` → `/v1/<resource>`), an API root ending in a
|
|
123
|
+
* version segment (`http://gateway/stt/v1` → `/<resource>`), or the full
|
|
124
|
+
* endpoint URL (used as-is). Only the path is rewritten: the query string is
|
|
125
|
+
* preserved and any fragment is dropped.
|
|
126
|
+
*/
|
|
127
|
+
function resolveOpenAICompatibleUrl(baseUrl, resource) {
|
|
128
|
+
const trimmedResource = resource.replace(/^\/+|\/+$/g, "");
|
|
129
|
+
const url = parseBaseUrl(baseUrl);
|
|
130
|
+
const pathname = url.pathname.replace(/\/+$/, "");
|
|
131
|
+
if (pathname.endsWith(`/${trimmedResource}`)) url.pathname = pathname;
|
|
132
|
+
else if (/\/v\d+$/.test(pathname)) url.pathname = `${pathname}/${trimmedResource}`;
|
|
133
|
+
else url.pathname = `${pathname}/v1/${trimmedResource}`;
|
|
134
|
+
return url.toString();
|
|
43
135
|
}
|
|
44
136
|
function resolveFetch(fetchOverride) {
|
|
45
137
|
if (fetchOverride) return fetchOverride;
|
|
@@ -68,11 +160,21 @@ var HttpSpeechAdapter = class {
|
|
|
68
160
|
this.headers = options.headers;
|
|
69
161
|
this.timeoutMs = options.timeoutMs;
|
|
70
162
|
}
|
|
71
|
-
|
|
163
|
+
/**
|
|
164
|
+
* POSTs to `path` (relative to `baseUrl`, or an absolute URL). The optional
|
|
165
|
+
* `retry` policy retries 429/5xx responses; `timeoutMs` applies per attempt.
|
|
166
|
+
* The request body must be re-sendable (string, Blob, FormData) when
|
|
167
|
+
* retries are enabled.
|
|
168
|
+
*/
|
|
169
|
+
async post(adapterName, path, init, readResponse, retry = false) {
|
|
170
|
+
return withSpeechRetry(() => this.postOnce(adapterName, path, init, readResponse), retry, init.signal ?? void 0);
|
|
171
|
+
}
|
|
172
|
+
async postOnce(adapterName, path, init, readResponse) {
|
|
72
173
|
const timeoutController = this.timeoutMs && this.timeoutMs > 0 ? new AbortController() : void 0;
|
|
73
174
|
const timeout = timeoutController && this.timeoutMs ? setTimeout(() => timeoutController.abort(), this.timeoutMs) : void 0;
|
|
74
175
|
const requestSignal = composeAbortSignals(init.signal, timeoutController?.signal);
|
|
75
176
|
try {
|
|
177
|
+
requestSignal.signal?.throwIfAborted();
|
|
76
178
|
const response = await this.fetchImpl(resolveSpeechUrl(this.baseUrl, path), {
|
|
77
179
|
...init,
|
|
78
180
|
method: "POST",
|
|
@@ -82,7 +184,7 @@ var HttpSpeechAdapter = class {
|
|
|
82
184
|
}, init.headers),
|
|
83
185
|
signal: requestSignal.signal
|
|
84
186
|
});
|
|
85
|
-
await assertOk(response, adapterName);
|
|
187
|
+
await assertOk(response, adapterName, this.apiKey);
|
|
86
188
|
return await readResponse(response);
|
|
87
189
|
} finally {
|
|
88
190
|
if (timeout) clearTimeout(timeout);
|
|
@@ -122,26 +224,19 @@ function composeAbortSignals(...signals) {
|
|
|
122
224
|
cleanup
|
|
123
225
|
};
|
|
124
226
|
}
|
|
125
|
-
async function assertOk(response, adapterName) {
|
|
227
|
+
async function assertOk(response, adapterName, apiKey) {
|
|
126
228
|
if (response.ok) return;
|
|
127
229
|
const responseBody = await response.text().catch(() => void 0);
|
|
128
230
|
throw new SpeechProviderError(adapterName, `${adapterName} speech request failed with HTTP ${response.status}`, {
|
|
129
231
|
status: response.status,
|
|
130
|
-
responseBody
|
|
232
|
+
responseBody: redactSecret(responseBody, apiKey),
|
|
233
|
+
retryAfterMs: parseRetryAfter(response.headers.get("retry-after"))
|
|
131
234
|
});
|
|
132
235
|
}
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
const contentType = audio.contentType ?? (data.type || "application/octet-stream");
|
|
138
|
-
const blob = data.type === contentType ? data : new Blob([data], { type: contentType });
|
|
139
|
-
form.append(fieldName, blob, filename);
|
|
140
|
-
return;
|
|
141
|
-
}
|
|
142
|
-
const contentType = audio.contentType ?? "application/octet-stream";
|
|
143
|
-
const blobPart = data instanceof Uint8Array ? arrayBufferFromBytes(data) : data;
|
|
144
|
-
form.append(fieldName, new Blob([blobPart], { type: contentType }), filename);
|
|
236
|
+
/** Removes an API key from provider-supplied text before it is surfaced. */
|
|
237
|
+
function redactSecret(text, secret) {
|
|
238
|
+
if (!text || !secret) return text;
|
|
239
|
+
return text.split(secret).join("[REDACTED]");
|
|
145
240
|
}
|
|
146
241
|
function appendOptionalFormValue(form, name, value) {
|
|
147
242
|
if (value === void 0 || value === null) return;
|
|
@@ -280,15 +375,1556 @@ function parseHeaderNumber(response, headerName) {
|
|
|
280
375
|
return Number.isFinite(parsed) ? parsed : void 0;
|
|
281
376
|
}
|
|
282
377
|
//#endregion
|
|
378
|
+
//#region src/shared/websocket.ts
|
|
379
|
+
/**
|
|
380
|
+
* Minimal WebSocket surface used by streaming adapters. Browser, Node 22+
|
|
381
|
+
* (`globalThis.WebSocket`), Deno, Bun, and the `ws` package all satisfy it.
|
|
382
|
+
*/
|
|
383
|
+
/** True in browsers and web workers, where API keys must never be used. */
|
|
384
|
+
function isBrowserRuntime() {
|
|
385
|
+
const scope = globalThis;
|
|
386
|
+
if (scope.process?.versions?.node) return false;
|
|
387
|
+
return typeof scope.document !== "undefined" || typeof scope.WorkerGlobalScope !== "undefined";
|
|
388
|
+
}
|
|
389
|
+
/**
|
|
390
|
+
* Resolves a socket factory from explicit options, falling back to
|
|
391
|
+
* `globalThis.WebSocket`. Headers are only passed when present, so standard
|
|
392
|
+
* constructors that cannot send them are never handed an options object.
|
|
393
|
+
*/
|
|
394
|
+
function resolveWebSocketFactory(adapter, options) {
|
|
395
|
+
if (options.createWebSocket) return options.createWebSocket;
|
|
396
|
+
const Ctor = options.WebSocket ?? globalThis.WebSocket;
|
|
397
|
+
if (!Ctor) throw new SpeechConfigurationError("No WebSocket implementation available; pass `WebSocket` or `createWebSocket`", adapter);
|
|
398
|
+
return (url, init) => Object.keys(init.headers).length > 0 ? new Ctor(url, {
|
|
399
|
+
protocols: init.protocols,
|
|
400
|
+
headers: init.headers
|
|
401
|
+
}) : new Ctor(url, init.protocols);
|
|
402
|
+
}
|
|
403
|
+
/** Converts `http(s)://` to `ws(s)://`; leaves `ws(s)://` unchanged. */
|
|
404
|
+
function toWebSocketUrl(url, adapter) {
|
|
405
|
+
const parsed = new URL(url);
|
|
406
|
+
if (parsed.protocol === "http:") parsed.protocol = "ws:";
|
|
407
|
+
else if (parsed.protocol === "https:") parsed.protocol = "wss:";
|
|
408
|
+
else if (parsed.protocol !== "ws:" && parsed.protocol !== "wss:") throw new SpeechConfigurationError(`Unsupported streaming URL protocol: ${parsed.protocol}`, adapter);
|
|
409
|
+
return parsed.toString();
|
|
410
|
+
}
|
|
411
|
+
/** Converts a message payload to text; returns `undefined` for unsupported data. */
|
|
412
|
+
function messageDataToText(data) {
|
|
413
|
+
if (typeof data === "string") return data;
|
|
414
|
+
if (data instanceof ArrayBuffer) return new TextDecoder().decode(data);
|
|
415
|
+
if (ArrayBuffer.isView(data)) return new TextDecoder().decode(data);
|
|
416
|
+
}
|
|
417
|
+
//#endregion
|
|
418
|
+
//#region src/shared/realtime-auth.ts
|
|
419
|
+
/**
|
|
420
|
+
* Credential handling shared by realtime (WebSocket) adapters.
|
|
421
|
+
*
|
|
422
|
+
* Node callers authenticate with a long-lived `apiKey` in the
|
|
423
|
+
* `Authorization` header. Browsers cannot set WebSocket headers and must never
|
|
424
|
+
* hold a long-lived key, so they pass a short-lived `clientSecret`, which is
|
|
425
|
+
* sent as a WebSocket subprotocol.
|
|
426
|
+
*/
|
|
427
|
+
/** Subprotocol prefix that carries a client secret (OpenAI's convention). */
|
|
428
|
+
var CLIENT_SECRET_SUBPROTOCOL_PREFIX = "openai-insecure-api-key.";
|
|
429
|
+
/**
|
|
430
|
+
* Long-lived OpenAI-style keys (`sk-…`, `sk-proj-…`, `sk-svcacct-…`). A
|
|
431
|
+
* browser `clientSecret` that looks like one is refused: browsers accept only
|
|
432
|
+
* short-lived tokens such as OpenAI's `ek_…` client secrets.
|
|
433
|
+
*/
|
|
434
|
+
var LONG_LIVED_KEY = /^sk-/;
|
|
435
|
+
/** Throws when an adapter is given a long-lived `apiKey` in a browser. */
|
|
436
|
+
function assertNoBrowserApiKey(adapter, apiKey) {
|
|
437
|
+
if (apiKey && isBrowserRuntime()) throw new SpeechConfigurationError(`${adapter} refuses apiKey in a browser; mint a short-lived clientSecret on your server instead`, adapter);
|
|
438
|
+
}
|
|
439
|
+
/** Resolves a static or per-session client secret and validates its shape. */
|
|
440
|
+
async function resolveClientSecret(adapter, secret) {
|
|
441
|
+
const trimmed = (typeof secret === "function" ? await secret() : secret)?.trim();
|
|
442
|
+
if (trimmed === void 0) return;
|
|
443
|
+
if (!/^[\x21-\x7e]+$/.test(trimmed)) throw new SpeechConfigurationError("clientSecret must be a non-empty token without spaces", adapter);
|
|
444
|
+
if (isBrowserRuntime() && LONG_LIVED_KEY.test(trimmed)) throw new SpeechConfigurationError("clientSecret looks like a long-lived API key; browsers accept only short-lived tokens minted by your server (see createStreamingClientSecret)", adapter);
|
|
445
|
+
return trimmed;
|
|
446
|
+
}
|
|
447
|
+
/**
|
|
448
|
+
* Builds the per-session `connect` callback: refuses handshake headers in a
|
|
449
|
+
* browser, resolves the client secret, and sets `Authorization` from
|
|
450
|
+
* `apiKey` unless a header already supplies it.
|
|
451
|
+
*/
|
|
452
|
+
function realtimeConnector(options) {
|
|
453
|
+
if (isBrowserRuntime() && Object.keys(options.headers).length > 0) throw new SpeechConfigurationError("Browsers cannot send WebSocket headers; remove `headers` or connect from a server", options.adapter);
|
|
454
|
+
return async () => {
|
|
455
|
+
const secret = await resolveClientSecret(options.adapter, options.clientSecret);
|
|
456
|
+
const headers = { ...options.headers };
|
|
457
|
+
const secrets = headerSecrets(headers);
|
|
458
|
+
if (secret) secrets.push(secret);
|
|
459
|
+
if (options.apiKey) {
|
|
460
|
+
secrets.push(options.apiKey);
|
|
461
|
+
if (!hasHeader(headers, "authorization")) headers.authorization = `Bearer ${options.apiKey}`;
|
|
462
|
+
}
|
|
463
|
+
return {
|
|
464
|
+
url: options.url,
|
|
465
|
+
protocols: options.protocols(secret),
|
|
466
|
+
headers,
|
|
467
|
+
secrets
|
|
468
|
+
};
|
|
469
|
+
};
|
|
470
|
+
}
|
|
471
|
+
/** Values of other headers shorter than this are not treated as credentials. */
|
|
472
|
+
var MIN_HEADER_SECRET_LENGTH = 8;
|
|
473
|
+
/**
|
|
474
|
+
* Header names that carry credentials (`authorization`, `x-api-key`,
|
|
475
|
+
* `x-bf-vk`, `*-token`, `*-secret`, …). Their values are redacted at any
|
|
476
|
+
* length.
|
|
477
|
+
*/
|
|
478
|
+
var CREDENTIAL_HEADER = /auth|key|token|secret|password|credential|-vk$/i;
|
|
479
|
+
/**
|
|
480
|
+
* Values of caller-supplied headers to redact from provider error text.
|
|
481
|
+
* Credential-named headers are redacted whatever their length; for
|
|
482
|
+
* `Authorization`, the credential after the scheme is included too. Other
|
|
483
|
+
* headers may also carry tenant secrets, so their values are redacted when
|
|
484
|
+
* they are at least 8 characters long, which keeps short values such as
|
|
485
|
+
* `1` from mangling messages.
|
|
486
|
+
*/
|
|
487
|
+
function headerSecrets(headers) {
|
|
488
|
+
const secrets = [];
|
|
489
|
+
for (const [key, value] of Object.entries(mergeHeaderRecords(headers))) {
|
|
490
|
+
const credential = CREDENTIAL_HEADER.test(key);
|
|
491
|
+
const candidates = [value];
|
|
492
|
+
if (key.toLowerCase() === "authorization") candidates.push(value.replace(/^\S+\s+/, ""));
|
|
493
|
+
for (const candidate of candidates) {
|
|
494
|
+
const trimmed = candidate.trim();
|
|
495
|
+
const long = trimmed.length >= MIN_HEADER_SECRET_LENGTH;
|
|
496
|
+
if (trimmed && (credential || long) && !secrets.includes(trimmed)) secrets.push(trimmed);
|
|
497
|
+
}
|
|
498
|
+
}
|
|
499
|
+
return secrets.sort((left, right) => right.length - left.length);
|
|
500
|
+
}
|
|
501
|
+
function mergeHeaderRecords(...sets) {
|
|
502
|
+
const merged = {};
|
|
503
|
+
for (const set of sets) if (set) new Headers(set).forEach((value, key) => {
|
|
504
|
+
merged[key] = value;
|
|
505
|
+
});
|
|
506
|
+
return merged;
|
|
507
|
+
}
|
|
508
|
+
function hasHeader(headers, name) {
|
|
509
|
+
return Object.keys(headers).some((key) => key.toLowerCase() === name);
|
|
510
|
+
}
|
|
511
|
+
//#endregion
|
|
512
|
+
//#region src/shared/pcm.ts
|
|
513
|
+
/**
|
|
514
|
+
* Raw-audio helpers for streaming adapters: byte views, base64 encoding,
|
|
515
|
+
* duration from byte counts, and WAV/`audio/L16` unwrapping for the
|
|
516
|
+
* record-then-send wrapper. Nothing here decodes compressed audio or resamples.
|
|
517
|
+
*/
|
|
518
|
+
var ENCODING_BYTES_PER_SAMPLE = {
|
|
519
|
+
pcm16: 2,
|
|
520
|
+
g711_ulaw: 1,
|
|
521
|
+
g711_alaw: 1
|
|
522
|
+
};
|
|
523
|
+
/** Returns a byte view over a chunk without copying. */
|
|
524
|
+
function chunkToBytes(chunk, adapter) {
|
|
525
|
+
if (chunk instanceof Uint8Array) return chunk;
|
|
526
|
+
if (chunk instanceof ArrayBuffer) return new Uint8Array(chunk);
|
|
527
|
+
if (ArrayBuffer.isView(chunk)) return new Uint8Array(chunk.buffer, chunk.byteOffset, chunk.byteLength);
|
|
528
|
+
throw new SpeechConfigurationError("Streaming audio chunks must be Uint8Array, ArrayBuffer, or an ArrayBuffer view", adapter);
|
|
529
|
+
}
|
|
530
|
+
function encodeBase64(bytes) {
|
|
531
|
+
if (typeof Buffer !== "undefined") return Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength).toString("base64");
|
|
532
|
+
let binary = "";
|
|
533
|
+
const step = 32768;
|
|
534
|
+
for (let index = 0; index < bytes.length; index += step) binary += String.fromCharCode(...bytes.subarray(index, index + step));
|
|
535
|
+
return btoa(binary);
|
|
536
|
+
}
|
|
537
|
+
/** Bytes per second of audio in `format`. */
|
|
538
|
+
function bytesPerSecond(format) {
|
|
539
|
+
return format.sampleRate * format.channels * ENCODING_BYTES_PER_SAMPLE[format.encoding];
|
|
540
|
+
}
|
|
541
|
+
/** Audio duration in seconds for `bytes` of `format`, rounded to milliseconds. */
|
|
542
|
+
function audioSecondsForBytes(bytes, format) {
|
|
543
|
+
return Math.round(bytes / bytesPerSecond(format) * 1e3) / 1e3;
|
|
544
|
+
}
|
|
545
|
+
var WAV_TYPES = /* @__PURE__ */ new Set([
|
|
546
|
+
"audio/wav",
|
|
547
|
+
"audio/wave",
|
|
548
|
+
"audio/x-wav",
|
|
549
|
+
"audio/vnd.wave"
|
|
550
|
+
]);
|
|
551
|
+
var RAW_PCM_TYPES = /* @__PURE__ */ new Set([
|
|
552
|
+
"audio/pcm",
|
|
553
|
+
"audio/l16",
|
|
554
|
+
"audio/raw",
|
|
555
|
+
"audio/x-raw"
|
|
556
|
+
]);
|
|
557
|
+
/** Untyped bytes: the streaming transcriber's default format applies. */
|
|
558
|
+
var UNTYPED = "application/octet-stream";
|
|
559
|
+
var MULAW_TYPES = /* @__PURE__ */ new Set([
|
|
560
|
+
"audio/pcmu",
|
|
561
|
+
"audio/basic",
|
|
562
|
+
"audio/x-mulaw"
|
|
563
|
+
]);
|
|
564
|
+
var ALAW_TYPES = /* @__PURE__ */ new Set(["audio/pcma", "audio/x-alaw"]);
|
|
565
|
+
/**
|
|
566
|
+
* Interprets a finished recording as raw audio for a streaming session.
|
|
567
|
+
* Accepts PCM/G.711 WAV files, `audio/pcm`/`audio/L16` (with optional `rate`
|
|
568
|
+
* and `channels` parameters), `audio/pcmu`/`audio/pcma`, and untyped bytes.
|
|
569
|
+
* Compressed containers are rejected with a `SpeechConfigurationError`.
|
|
570
|
+
*/
|
|
571
|
+
function unwrapRawAudio(bytes, mimeType, adapter) {
|
|
572
|
+
const essence = normalizeMimeType(mimeType) ?? "application/octet-stream";
|
|
573
|
+
const params = parseMimeParameters(mimeType);
|
|
574
|
+
const rate = parsePositiveInt(params.rate);
|
|
575
|
+
const channels = parsePositiveInt(params.channels);
|
|
576
|
+
if (WAV_TYPES.has(essence) || isRiffWave(bytes)) return parseWav(bytes, adapter);
|
|
577
|
+
if (essence === UNTYPED) return {
|
|
578
|
+
bytes,
|
|
579
|
+
format: compact({
|
|
580
|
+
sampleRate: rate,
|
|
581
|
+
channels
|
|
582
|
+
})
|
|
583
|
+
};
|
|
584
|
+
if (RAW_PCM_TYPES.has(essence)) {
|
|
585
|
+
if (essence === "audio/l16" && bytes.byteLength % 2 !== 0) throw new SpeechConfigurationError(`audio/L16 payload has an odd byte length (${bytes.byteLength}); 16-bit samples need an even number of bytes`, adapter);
|
|
586
|
+
return {
|
|
587
|
+
bytes: essence === "audio/l16" ? swapBytePairs(bytes) : bytes,
|
|
588
|
+
format: compact({
|
|
589
|
+
encoding: "pcm16",
|
|
590
|
+
sampleRate: rate,
|
|
591
|
+
channels
|
|
592
|
+
})
|
|
593
|
+
};
|
|
594
|
+
}
|
|
595
|
+
if (MULAW_TYPES.has(essence)) return {
|
|
596
|
+
bytes,
|
|
597
|
+
format: compact({
|
|
598
|
+
encoding: "g711_ulaw",
|
|
599
|
+
sampleRate: rate,
|
|
600
|
+
channels
|
|
601
|
+
})
|
|
602
|
+
};
|
|
603
|
+
if (ALAW_TYPES.has(essence)) return {
|
|
604
|
+
bytes,
|
|
605
|
+
format: compact({
|
|
606
|
+
encoding: "g711_alaw",
|
|
607
|
+
sampleRate: rate,
|
|
608
|
+
channels
|
|
609
|
+
})
|
|
610
|
+
};
|
|
611
|
+
throw new SpeechConfigurationError(`${adapter} streams raw audio and cannot decode ${essence}; send PCM WAV or raw PCM, or use an HTTP transcriber for compressed recordings`, adapter);
|
|
612
|
+
}
|
|
613
|
+
/** Returns a copy with each 16-bit sample's bytes swapped. */
|
|
614
|
+
function swapBytePairs(bytes) {
|
|
615
|
+
const swapped = new Uint8Array(bytes.byteLength);
|
|
616
|
+
for (let index = 0; index + 1 < bytes.byteLength; index += 2) {
|
|
617
|
+
swapped[index] = bytes[index + 1] ?? 0;
|
|
618
|
+
swapped[index + 1] = bytes[index] ?? 0;
|
|
619
|
+
}
|
|
620
|
+
return swapped;
|
|
621
|
+
}
|
|
622
|
+
function isRiffWave(bytes) {
|
|
623
|
+
return bytes.byteLength >= 12 && ascii(bytes, 0, 4) === "RIFF" && ascii(bytes, 8, 4) === "WAVE";
|
|
624
|
+
}
|
|
625
|
+
function parseWav(bytes, adapter) {
|
|
626
|
+
if (!isRiffWave(bytes)) throw new SpeechConfigurationError("Invalid WAV header", adapter);
|
|
627
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
628
|
+
let offset = 12;
|
|
629
|
+
let format;
|
|
630
|
+
while (offset + 8 <= bytes.byteLength) {
|
|
631
|
+
const id = ascii(bytes, offset, 4);
|
|
632
|
+
const size = view.getUint32(offset + 4, true);
|
|
633
|
+
const body = offset + 8;
|
|
634
|
+
if (id === "fmt " && body + 16 <= bytes.byteLength) {
|
|
635
|
+
const audioFormat = view.getUint16(body, true);
|
|
636
|
+
const channels = view.getUint16(body + 2, true);
|
|
637
|
+
const sampleRate = view.getUint32(body + 4, true);
|
|
638
|
+
format = {
|
|
639
|
+
encoding: wavEncoding(audioFormat, view.getUint16(body + 14, true), adapter),
|
|
640
|
+
sampleRate,
|
|
641
|
+
channels
|
|
642
|
+
};
|
|
643
|
+
} else if (id === "data") {
|
|
644
|
+
if (!format) throw new SpeechConfigurationError("WAV data chunk precedes its fmt chunk", adapter);
|
|
645
|
+
const end = Math.min(bytes.byteLength, body + size);
|
|
646
|
+
return {
|
|
647
|
+
bytes: bytes.subarray(body, end),
|
|
648
|
+
format
|
|
649
|
+
};
|
|
650
|
+
}
|
|
651
|
+
offset = body + size + size % 2;
|
|
652
|
+
}
|
|
653
|
+
throw new SpeechConfigurationError("WAV file has no data chunk", adapter);
|
|
654
|
+
}
|
|
655
|
+
function wavEncoding(audioFormat, bitsPerSample, adapter) {
|
|
656
|
+
if ((audioFormat === 1 || audioFormat === 65534) && bitsPerSample === 16) return "pcm16";
|
|
657
|
+
if (audioFormat === 7 && bitsPerSample === 8) return "g711_ulaw";
|
|
658
|
+
if (audioFormat === 6 && bitsPerSample === 8) return "g711_alaw";
|
|
659
|
+
throw new SpeechConfigurationError(`Unsupported WAV encoding (format ${audioFormat}, ${bitsPerSample}-bit); expected 16-bit PCM or 8-bit G.711`, adapter);
|
|
660
|
+
}
|
|
661
|
+
function ascii(bytes, offset, length) {
|
|
662
|
+
return String.fromCharCode(...bytes.subarray(offset, offset + length));
|
|
663
|
+
}
|
|
664
|
+
function parseMimeParameters(mimeType) {
|
|
665
|
+
const params = {};
|
|
666
|
+
for (const part of mimeType.split(";").slice(1)) {
|
|
667
|
+
const [key, value] = part.split("=");
|
|
668
|
+
if (key && value) params[key.trim().toLowerCase()] = value.trim();
|
|
669
|
+
}
|
|
670
|
+
return params;
|
|
671
|
+
}
|
|
672
|
+
function parsePositiveInt(value) {
|
|
673
|
+
if (!value) return;
|
|
674
|
+
const parsed = Number.parseInt(value, 10);
|
|
675
|
+
return Number.isFinite(parsed) && parsed > 0 ? parsed : void 0;
|
|
676
|
+
}
|
|
677
|
+
function compact(format) {
|
|
678
|
+
return Object.fromEntries(Object.entries(format).filter(([, value]) => value !== void 0));
|
|
679
|
+
}
|
|
680
|
+
//#endregion
|
|
681
|
+
//#region src/shared/turn-limit.ts
|
|
682
|
+
/**
|
|
683
|
+
* Per-turn audio limits for realtime sessions.
|
|
684
|
+
*
|
|
685
|
+
* Some providers cap the audio of one turn rather than of a session: vLLM's
|
|
686
|
+
* realtime endpoint keeps a turn's audio and generated tokens in one model
|
|
687
|
+
* context (`max_model_len`) and clears it after each final commit. A
|
|
688
|
+
* {@link TurnSplitter} measures the audio of the current turn from byte counts
|
|
689
|
+
* and the negotiated format, and picks where to roll over to a new turn: at
|
|
690
|
+
* the first quiet stretch inside a window before the limit, or at the limit
|
|
691
|
+
* itself when no quiet stretch turns up. The silence check is a plain RMS
|
|
692
|
+
* energy test, not a voice-activity model.
|
|
693
|
+
*/
|
|
694
|
+
/** Default per-turn audio cap for providers that need one (`voxtral-realtime`). */
|
|
695
|
+
var DEFAULT_MAX_TURN_SECONDS = 270;
|
|
696
|
+
/** How long before the cap the rollover starts looking for silence. */
|
|
697
|
+
var DEFAULT_ROLLOVER_WINDOW_SECONDS = 15;
|
|
698
|
+
/** RMS level (0-1 of full scale, about -40 dBFS) below which audio is quiet. */
|
|
699
|
+
var DEFAULT_ROLLOVER_SILENCE_THRESHOLD = .01;
|
|
700
|
+
/** Quiet audio needed before the rollover commits. */
|
|
701
|
+
var DEFAULT_ROLLOVER_MIN_SILENCE_MS = 300;
|
|
702
|
+
/** Energy is measured over frames of this length. */
|
|
703
|
+
var ANALYSIS_FRAME_MS = 20;
|
|
704
|
+
var BYTES_PER_SAMPLE = {
|
|
705
|
+
pcm16: 2,
|
|
706
|
+
g711_ulaw: 1,
|
|
707
|
+
g711_alaw: 1
|
|
708
|
+
};
|
|
709
|
+
/**
|
|
710
|
+
* Resolves the turn limit from settings, highest priority first (session
|
|
711
|
+
* options, then adapter options, which already carry environment values),
|
|
712
|
+
* falling back to `defaultMaxTurnSeconds`. Returns `undefined` when no limit
|
|
713
|
+
* applies or `maxTurnSeconds` is `Infinity`.
|
|
714
|
+
*/
|
|
715
|
+
function resolveTurnLimit(adapter, defaultMaxTurnSeconds, ...settings) {
|
|
716
|
+
const maxTurnSeconds = settings.find((entry) => entry?.maxTurnSeconds !== void 0)?.maxTurnSeconds ?? defaultMaxTurnSeconds;
|
|
717
|
+
if (maxTurnSeconds === void 0) return;
|
|
718
|
+
if (typeof maxTurnSeconds !== "number" || Number.isNaN(maxTurnSeconds) || maxTurnSeconds <= 0) throw new SpeechConfigurationError(`maxTurnSeconds must be a positive number of seconds or Infinity (got ${String(maxTurnSeconds)})`, adapter);
|
|
719
|
+
if (maxTurnSeconds === Number.POSITIVE_INFINITY) return;
|
|
720
|
+
const rollover = settings.find((entry) => entry?.rollover !== void 0)?.rollover;
|
|
721
|
+
const tuning = (key) => settings.map((entry) => entry?.rollover).find((value) => typeof value === "object" && value !== null && value[key] !== void 0)?.[key];
|
|
722
|
+
const limit = {
|
|
723
|
+
maxTurnSeconds,
|
|
724
|
+
rollover: rollover !== false,
|
|
725
|
+
windowSeconds: tuning("windowSeconds") ?? 15,
|
|
726
|
+
silenceThreshold: tuning("silenceThreshold") ?? .01,
|
|
727
|
+
minSilenceMs: tuning("minSilenceMs") ?? 300
|
|
728
|
+
};
|
|
729
|
+
assertNonNegative(adapter, "rollover.windowSeconds", limit.windowSeconds);
|
|
730
|
+
assertNonNegative(adapter, "rollover.minSilenceMs", limit.minSilenceMs);
|
|
731
|
+
assertNonNegative(adapter, "rollover.silenceThreshold", limit.silenceThreshold);
|
|
732
|
+
if (limit.silenceThreshold > 1) throw new SpeechConfigurationError(`rollover.silenceThreshold is an RMS level from 0 to 1 (got ${limit.silenceThreshold})`, adapter);
|
|
733
|
+
return limit;
|
|
734
|
+
}
|
|
735
|
+
function assertNonNegative(adapter, name, value) {
|
|
736
|
+
if (typeof value !== "number" || !Number.isFinite(value) || value < 0) throw new SpeechConfigurationError(`${name} must be a non-negative finite number (got ${String(value)})`, adapter);
|
|
737
|
+
}
|
|
738
|
+
/** Bytes of one sample frame (one sample per channel). */
|
|
739
|
+
function blockAlign(format) {
|
|
740
|
+
return BYTES_PER_SAMPLE[format.encoding] * format.channels;
|
|
741
|
+
}
|
|
742
|
+
/** Whole sample frames in `seconds` of audio, as bytes (rounded down). */
|
|
743
|
+
function bytesForSeconds(seconds, format) {
|
|
744
|
+
const align = blockAlign(format);
|
|
745
|
+
return Math.floor(seconds * bytesPerSecond(format) / align + 1e-9) * align;
|
|
746
|
+
}
|
|
747
|
+
/** Digital silence in `format`: zero samples, encoded. */
|
|
748
|
+
function silence(bytes, format) {
|
|
749
|
+
const fill = format.encoding === "g711_ulaw" ? 255 : format.encoding === "g711_alaw" ? 213 : 0;
|
|
750
|
+
return new Uint8Array(bytes).fill(fill);
|
|
751
|
+
}
|
|
752
|
+
/** Bytes of audio one turn may hold: whole sample frames, at least one. */
|
|
753
|
+
function turnCapBytes(limit, format) {
|
|
754
|
+
return Math.max(blockAlign(format), bytesForSeconds(limit.maxTurnSeconds, format));
|
|
755
|
+
}
|
|
756
|
+
/** The public view of a resolved limit for a session format. */
|
|
757
|
+
function describeTurnLimit(limit, format) {
|
|
758
|
+
return limit && {
|
|
759
|
+
maxTurnSeconds: limit.maxTurnSeconds,
|
|
760
|
+
rollover: limit.rollover,
|
|
761
|
+
maxTurnBytes: turnCapBytes(limit, format)
|
|
762
|
+
};
|
|
763
|
+
}
|
|
764
|
+
/**
|
|
765
|
+
* Tracks the audio of the current turn and finds rollover points.
|
|
766
|
+
*
|
|
767
|
+
* Feed every chunk in write order to {@link split} (rollover on) or
|
|
768
|
+
* {@link add} (rollover off), and call {@link reset} whenever the turn ends
|
|
769
|
+
* for another reason (a manual commit).
|
|
770
|
+
*/
|
|
771
|
+
var TurnSplitter = class {
|
|
772
|
+
/** Bytes of the current turn so far. */
|
|
773
|
+
turnBytes = 0;
|
|
774
|
+
capBytes;
|
|
775
|
+
windowStartBytes;
|
|
776
|
+
frameBytes;
|
|
777
|
+
minSilenceBytes;
|
|
778
|
+
threshold;
|
|
779
|
+
encoding;
|
|
780
|
+
sampleBytes;
|
|
781
|
+
frameFill = 0;
|
|
782
|
+
frameSumSquares = 0;
|
|
783
|
+
frameSamples = 0;
|
|
784
|
+
/** Trailing bytes of a sample split across chunks. */
|
|
785
|
+
carry = [];
|
|
786
|
+
/** Consecutive quiet bytes ending at the last complete frame. */
|
|
787
|
+
quietRun = 0;
|
|
788
|
+
constructor(limit, format) {
|
|
789
|
+
const align = blockAlign(format);
|
|
790
|
+
this.capBytes = turnCapBytes(limit, format);
|
|
791
|
+
this.windowStartBytes = Math.max(0, this.capBytes - bytesForSeconds(limit.windowSeconds, format));
|
|
792
|
+
this.frameBytes = Math.max(align, bytesForSeconds(ANALYSIS_FRAME_MS / 1e3, format));
|
|
793
|
+
this.minSilenceBytes = Math.max(this.frameBytes, bytesForSeconds(limit.minSilenceMs / 1e3, format));
|
|
794
|
+
this.threshold = limit.silenceThreshold;
|
|
795
|
+
this.encoding = format.encoding;
|
|
796
|
+
this.sampleBytes = BYTES_PER_SAMPLE[format.encoding];
|
|
797
|
+
}
|
|
798
|
+
/** Ends the current turn: the next byte starts a new one. */
|
|
799
|
+
reset() {
|
|
800
|
+
this.turnBytes = 0;
|
|
801
|
+
this.frameFill = 0;
|
|
802
|
+
this.frameSumSquares = 0;
|
|
803
|
+
this.frameSamples = 0;
|
|
804
|
+
this.carry = [];
|
|
805
|
+
this.quietRun = 0;
|
|
806
|
+
}
|
|
807
|
+
/** Whether `bytes` more would push the current turn past the cap. */
|
|
808
|
+
wouldOverflow(bytes) {
|
|
809
|
+
return this.turnBytes + bytes > this.capBytes;
|
|
810
|
+
}
|
|
811
|
+
/** Counts `bytes` toward the current turn without analysing them. */
|
|
812
|
+
add(bytes) {
|
|
813
|
+
this.turnBytes += bytes;
|
|
814
|
+
}
|
|
815
|
+
/**
|
|
816
|
+
* Feeds one chunk and returns the offsets inside it where the turn should
|
|
817
|
+
* end (each in `1..chunk.byteLength`, ascending). The splitter resets at
|
|
818
|
+
* each offset, so bytes after it count toward the next turn.
|
|
819
|
+
*/
|
|
820
|
+
split(chunk) {
|
|
821
|
+
const cuts = [];
|
|
822
|
+
let offset = 0;
|
|
823
|
+
while (offset < chunk.byteLength) {
|
|
824
|
+
if (this.turnBytes < this.windowStartBytes) {
|
|
825
|
+
const skip = Math.min(chunk.byteLength - offset, this.windowStartBytes - this.turnBytes);
|
|
826
|
+
offset += skip;
|
|
827
|
+
this.turnBytes += skip;
|
|
828
|
+
if (this.turnBytes >= this.capBytes) {
|
|
829
|
+
cuts.push(offset);
|
|
830
|
+
this.reset();
|
|
831
|
+
}
|
|
832
|
+
continue;
|
|
833
|
+
}
|
|
834
|
+
const take = Math.min(chunk.byteLength - offset, this.frameBytes - this.frameFill, this.capBytes - this.turnBytes);
|
|
835
|
+
this.measure(chunk, offset, take);
|
|
836
|
+
offset += take;
|
|
837
|
+
this.turnBytes += take;
|
|
838
|
+
this.frameFill += take;
|
|
839
|
+
let cut = false;
|
|
840
|
+
if (this.frameFill >= this.frameBytes) {
|
|
841
|
+
const quiet = this.frameIsQuiet();
|
|
842
|
+
this.quietRun = quiet ? this.quietRun + this.frameFill : 0;
|
|
843
|
+
this.frameFill = 0;
|
|
844
|
+
this.frameSumSquares = 0;
|
|
845
|
+
this.frameSamples = 0;
|
|
846
|
+
cut = this.quietRun >= this.minSilenceBytes;
|
|
847
|
+
}
|
|
848
|
+
if (cut || this.turnBytes >= this.capBytes) {
|
|
849
|
+
cuts.push(offset);
|
|
850
|
+
this.reset();
|
|
851
|
+
}
|
|
852
|
+
}
|
|
853
|
+
return cuts;
|
|
854
|
+
}
|
|
855
|
+
frameIsQuiet() {
|
|
856
|
+
if (this.frameSamples === 0) return false;
|
|
857
|
+
return Math.sqrt(this.frameSumSquares / this.frameSamples) / 32768 < this.threshold;
|
|
858
|
+
}
|
|
859
|
+
measure(chunk, start, length) {
|
|
860
|
+
const end = start + length;
|
|
861
|
+
let index = start;
|
|
862
|
+
while (this.carry.length > 0 && index < end) {
|
|
863
|
+
this.carry.push(chunk[index] ?? 0);
|
|
864
|
+
index += 1;
|
|
865
|
+
if (this.carry.length === this.sampleBytes) {
|
|
866
|
+
this.addSample(this.carry);
|
|
867
|
+
this.carry = [];
|
|
868
|
+
}
|
|
869
|
+
}
|
|
870
|
+
for (; index + this.sampleBytes <= end; index += this.sampleBytes) this.addSampleAt(chunk, index);
|
|
871
|
+
for (; index < end; index += 1) this.carry.push(chunk[index] ?? 0);
|
|
872
|
+
}
|
|
873
|
+
addSample(bytes) {
|
|
874
|
+
this.addLinear(this.encoding === "pcm16" ? pcm16Sample(bytes[0] ?? 0, bytes[1] ?? 0) : this.decodeG711(bytes[0] ?? 0));
|
|
875
|
+
}
|
|
876
|
+
addSampleAt(chunk, index) {
|
|
877
|
+
this.addLinear(this.encoding === "pcm16" ? pcm16Sample(chunk[index] ?? 0, chunk[index + 1] ?? 0) : this.decodeG711(chunk[index] ?? 0));
|
|
878
|
+
}
|
|
879
|
+
addLinear(sample) {
|
|
880
|
+
this.frameSumSquares += sample * sample;
|
|
881
|
+
this.frameSamples += 1;
|
|
882
|
+
}
|
|
883
|
+
decodeG711(byte) {
|
|
884
|
+
return this.encoding === "g711_ulaw" ? decodeMuLaw(byte) : decodeALaw(byte);
|
|
885
|
+
}
|
|
886
|
+
};
|
|
887
|
+
/** Signed 16-bit little-endian sample. */
|
|
888
|
+
function pcm16Sample(low, high) {
|
|
889
|
+
const value = low | high << 8;
|
|
890
|
+
return value >= 32768 ? value - 65536 : value;
|
|
891
|
+
}
|
|
892
|
+
/** G.711 mu-law byte to a 16-bit linear sample (ITU-T G.711). */
|
|
893
|
+
function decodeMuLaw(byte) {
|
|
894
|
+
const value = ~byte & 255;
|
|
895
|
+
const exponent = value >> 4 & 7;
|
|
896
|
+
const magnitude = (((value & 15) << 3) + 132 << exponent) - 132;
|
|
897
|
+
return value & 128 ? -magnitude : magnitude;
|
|
898
|
+
}
|
|
899
|
+
/** G.711 A-law byte to a 16-bit linear sample (ITU-T G.711). */
|
|
900
|
+
function decodeALaw(byte) {
|
|
901
|
+
const value = byte ^ 85;
|
|
902
|
+
const exponent = value >> 4 & 7;
|
|
903
|
+
const mantissa = value & 15;
|
|
904
|
+
const magnitude = exponent === 0 ? (mantissa << 4) + 8 : (mantissa << 4) + 264 << exponent - 1;
|
|
905
|
+
return value & 128 ? magnitude : -magnitude;
|
|
906
|
+
}
|
|
907
|
+
var DRAIN_POLL_MS = 10;
|
|
908
|
+
var noop = () => void 0;
|
|
909
|
+
var RealtimeTranscriptionSession = class {
|
|
910
|
+
state = "connecting";
|
|
911
|
+
sessionId;
|
|
912
|
+
ready;
|
|
913
|
+
init;
|
|
914
|
+
provider;
|
|
915
|
+
resolveReady = noop;
|
|
916
|
+
rejectReady = noop;
|
|
917
|
+
socket;
|
|
918
|
+
opened = false;
|
|
919
|
+
secrets = [];
|
|
920
|
+
queue = [];
|
|
921
|
+
queued = 0;
|
|
922
|
+
pumping = false;
|
|
923
|
+
totalBytes = 0;
|
|
924
|
+
uncommittedBytes = 0;
|
|
925
|
+
outstandingCommits = 0;
|
|
926
|
+
/** Per outstanding commit, in order: whether it carried audio. */
|
|
927
|
+
commitHadAudio = [];
|
|
928
|
+
turnOpen = false;
|
|
929
|
+
/** `Date.now()` of the last server message; bounds inactivity waits. */
|
|
930
|
+
lastServerMessageAt = 0;
|
|
931
|
+
itemOrder = [];
|
|
932
|
+
pendingItems = /* @__PURE__ */ new Set();
|
|
933
|
+
vadStoppedItems = /* @__PURE__ */ new Set();
|
|
934
|
+
partials = /* @__PURE__ */ new Map();
|
|
935
|
+
finals = [];
|
|
936
|
+
listeners = /* @__PURE__ */ new Map();
|
|
937
|
+
error;
|
|
938
|
+
endPromise;
|
|
939
|
+
endWaiter;
|
|
940
|
+
connectTimer;
|
|
941
|
+
endTimer;
|
|
942
|
+
closeEmitted = false;
|
|
943
|
+
/** Measures the current turn in write order when a turn limit applies. */
|
|
944
|
+
splitter;
|
|
945
|
+
onAbort = () => {
|
|
946
|
+
this.fail(toError(this.init.signal?.reason, this.provider));
|
|
947
|
+
};
|
|
948
|
+
constructor(init) {
|
|
949
|
+
this.init = init;
|
|
950
|
+
this.provider = init.protocol.provider;
|
|
951
|
+
assertTurnLimitFitsProtocol(init.protocol, init.turnLimit);
|
|
952
|
+
if (init.turnLimit) this.splitter = new TurnSplitter(init.turnLimit, init.config.format);
|
|
953
|
+
this.ready = new Promise((resolve, reject) => {
|
|
954
|
+
this.resolveReady = resolve;
|
|
955
|
+
this.rejectReady = reject;
|
|
956
|
+
});
|
|
957
|
+
this.ready.catch(noop);
|
|
958
|
+
if (init.signal?.aborted) {
|
|
959
|
+
this.fail(toError(init.signal.reason, this.provider));
|
|
960
|
+
return;
|
|
961
|
+
}
|
|
962
|
+
init.signal?.addEventListener("abort", this.onAbort, { once: true });
|
|
963
|
+
this.connect();
|
|
964
|
+
}
|
|
965
|
+
get queuedBytes() {
|
|
966
|
+
return this.queued;
|
|
967
|
+
}
|
|
968
|
+
write(chunk) {
|
|
969
|
+
const promise = this.enqueueAudio(chunk);
|
|
970
|
+
promise.catch(noop);
|
|
971
|
+
return promise;
|
|
972
|
+
}
|
|
973
|
+
commit() {
|
|
974
|
+
if (this.isTerminal() || this.endPromise) return;
|
|
975
|
+
this.queue.push({
|
|
976
|
+
kind: "commit",
|
|
977
|
+
final: false
|
|
978
|
+
});
|
|
979
|
+
this.splitter?.reset();
|
|
980
|
+
this.pump();
|
|
981
|
+
}
|
|
982
|
+
end() {
|
|
983
|
+
if (this.endPromise) return this.endPromise;
|
|
984
|
+
if (this.error) {
|
|
985
|
+
this.endPromise = Promise.reject(this.error);
|
|
986
|
+
return this.endPromise;
|
|
987
|
+
}
|
|
988
|
+
this.state = "ending";
|
|
989
|
+
this.queue.push({
|
|
990
|
+
kind: "commit",
|
|
991
|
+
final: true
|
|
992
|
+
});
|
|
993
|
+
const settled = new Promise((resolve, reject) => {
|
|
994
|
+
this.endWaiter = {
|
|
995
|
+
resolve,
|
|
996
|
+
reject
|
|
997
|
+
};
|
|
998
|
+
});
|
|
999
|
+
this.armEndTimer();
|
|
1000
|
+
this.endPromise = settled.then(() => this.finish());
|
|
1001
|
+
this.pump();
|
|
1002
|
+
this.checkEnded();
|
|
1003
|
+
return this.endPromise;
|
|
1004
|
+
}
|
|
1005
|
+
/**
|
|
1006
|
+
* (Re)starts the end timeout. It measures provider inactivity: every server
|
|
1007
|
+
* message restarts it, so a long recording that is still being transcribed
|
|
1008
|
+
* is not cut off, while a silent provider still fails after `timeoutMs`.
|
|
1009
|
+
*/
|
|
1010
|
+
armEndTimer() {
|
|
1011
|
+
clearTimeout(this.endTimer);
|
|
1012
|
+
this.endTimer = setTimeout(() => {
|
|
1013
|
+
this.fail(new SpeechProviderError(this.provider, `${this.provider} timed out after ${this.init.timeoutMs} ms waiting for the final transcript`));
|
|
1014
|
+
}, this.init.timeoutMs);
|
|
1015
|
+
}
|
|
1016
|
+
abort(reason) {
|
|
1017
|
+
this.fail(toError(reason, this.provider));
|
|
1018
|
+
}
|
|
1019
|
+
on(event, listener) {
|
|
1020
|
+
let set = this.listeners.get(event);
|
|
1021
|
+
if (!set) {
|
|
1022
|
+
set = /* @__PURE__ */ new Set();
|
|
1023
|
+
this.listeners.set(event, set);
|
|
1024
|
+
}
|
|
1025
|
+
set.add(listener);
|
|
1026
|
+
return () => this.off(event, listener);
|
|
1027
|
+
}
|
|
1028
|
+
off(event, listener) {
|
|
1029
|
+
this.listeners.get(event)?.delete(listener);
|
|
1030
|
+
}
|
|
1031
|
+
enqueueAudio(chunk) {
|
|
1032
|
+
if (this.error) return Promise.reject(this.error);
|
|
1033
|
+
if (this.endPromise || this.isTerminal()) return Promise.reject(new SpeechError("Cannot write to a streaming session after end()", "SPEECH_SESSION_ENDED", this.provider));
|
|
1034
|
+
let bytes;
|
|
1035
|
+
try {
|
|
1036
|
+
bytes = chunkToBytes(chunk, this.provider);
|
|
1037
|
+
} catch (error) {
|
|
1038
|
+
return Promise.reject(error);
|
|
1039
|
+
}
|
|
1040
|
+
if (bytes.byteLength === 0) return Promise.resolve();
|
|
1041
|
+
if (this.queued + bytes.byteLength > this.init.maxBufferedBytes) return Promise.reject(new SpeechError(`Streaming write would queue more than maxBufferedBytes (${this.init.maxBufferedBytes}); await write() before sending more audio`, "SPEECH_BACKPRESSURE", this.provider));
|
|
1042
|
+
const splitter = this.splitter;
|
|
1043
|
+
let cuts = [];
|
|
1044
|
+
if (splitter && !this.init.turnLimit?.rollover) {
|
|
1045
|
+
if (splitter.wouldOverflow(bytes.byteLength)) return Promise.reject(new SpeechError(`Streaming write would push the current turn past maxTurnSeconds (${this.init.turnLimit?.maxTurnSeconds} s); call commit() before writing more audio`, "SPEECH_TURN_TOO_LONG", this.provider));
|
|
1046
|
+
splitter.add(bytes.byteLength);
|
|
1047
|
+
} else if (splitter) cuts = splitter.split(bytes);
|
|
1048
|
+
return new Promise((resolve, reject) => {
|
|
1049
|
+
const pushPart = (start, end) => {
|
|
1050
|
+
const part = bytes.subarray(start, end);
|
|
1051
|
+
this.queue.push({
|
|
1052
|
+
kind: "audio",
|
|
1053
|
+
base64: encodeBase64(part),
|
|
1054
|
+
bytes: part.byteLength,
|
|
1055
|
+
resolve: end === bytes.byteLength ? resolve : noop,
|
|
1056
|
+
reject
|
|
1057
|
+
});
|
|
1058
|
+
this.queued += part.byteLength;
|
|
1059
|
+
};
|
|
1060
|
+
let start = 0;
|
|
1061
|
+
for (const cut of cuts) {
|
|
1062
|
+
pushPart(start, cut);
|
|
1063
|
+
this.queue.push({
|
|
1064
|
+
kind: "commit",
|
|
1065
|
+
final: false
|
|
1066
|
+
});
|
|
1067
|
+
start = cut;
|
|
1068
|
+
}
|
|
1069
|
+
if (start < bytes.byteLength) pushPart(start, bytes.byteLength);
|
|
1070
|
+
this.pump();
|
|
1071
|
+
});
|
|
1072
|
+
}
|
|
1073
|
+
async connect() {
|
|
1074
|
+
this.connectTimer = setTimeout(() => {
|
|
1075
|
+
this.fail(new SpeechProviderError(this.provider, `${this.provider} connection timed out after ${this.init.connectTimeoutMs} ms`));
|
|
1076
|
+
}, this.init.connectTimeoutMs);
|
|
1077
|
+
try {
|
|
1078
|
+
const connection = await this.init.connect();
|
|
1079
|
+
if (this.isTerminal()) return;
|
|
1080
|
+
this.secrets = connection.secrets;
|
|
1081
|
+
const socket = this.init.createWebSocket(connection.url, {
|
|
1082
|
+
protocols: connection.protocols,
|
|
1083
|
+
headers: connection.headers
|
|
1084
|
+
});
|
|
1085
|
+
this.socket = socket;
|
|
1086
|
+
try {
|
|
1087
|
+
socket.binaryType = "arraybuffer";
|
|
1088
|
+
} catch {}
|
|
1089
|
+
socket.addEventListener("open", () => this.onOpen());
|
|
1090
|
+
socket.addEventListener("message", (event) => this.onMessage(event.data));
|
|
1091
|
+
socket.addEventListener("error", () => {
|
|
1092
|
+
this.fail(new SpeechProviderError(this.provider, `${this.provider} WebSocket error`));
|
|
1093
|
+
});
|
|
1094
|
+
socket.addEventListener("close", (event) => this.onClose(event));
|
|
1095
|
+
} catch (error) {
|
|
1096
|
+
this.fail(toError(error, this.provider));
|
|
1097
|
+
}
|
|
1098
|
+
}
|
|
1099
|
+
onOpen() {
|
|
1100
|
+
clearTimeout(this.connectTimer);
|
|
1101
|
+
if (this.isTerminal()) {
|
|
1102
|
+
this.closeSocket();
|
|
1103
|
+
return;
|
|
1104
|
+
}
|
|
1105
|
+
this.opened = true;
|
|
1106
|
+
for (const message of this.init.protocol.sessionMessages(this.init.config)) if (!this.send(message)) return;
|
|
1107
|
+
if (this.state === "connecting") this.state = "open";
|
|
1108
|
+
this.resolveReady();
|
|
1109
|
+
this.emit("open", { sessionId: this.sessionId });
|
|
1110
|
+
this.pump();
|
|
1111
|
+
}
|
|
1112
|
+
onMessage(data) {
|
|
1113
|
+
if (this.isTerminal()) return;
|
|
1114
|
+
this.lastServerMessageAt = Date.now();
|
|
1115
|
+
if (this.endWaiter) this.armEndTimer();
|
|
1116
|
+
const text = messageDataToText(data);
|
|
1117
|
+
if (text === void 0) return;
|
|
1118
|
+
let parsed;
|
|
1119
|
+
try {
|
|
1120
|
+
parsed = JSON.parse(text);
|
|
1121
|
+
} catch {
|
|
1122
|
+
return;
|
|
1123
|
+
}
|
|
1124
|
+
let events;
|
|
1125
|
+
try {
|
|
1126
|
+
events = this.init.protocol.parse(parsed);
|
|
1127
|
+
} catch (error) {
|
|
1128
|
+
this.fail(toError(error, this.provider));
|
|
1129
|
+
return;
|
|
1130
|
+
}
|
|
1131
|
+
for (const event of events) {
|
|
1132
|
+
if (this.isTerminal()) return;
|
|
1133
|
+
this.handle(event);
|
|
1134
|
+
}
|
|
1135
|
+
this.checkEnded();
|
|
1136
|
+
}
|
|
1137
|
+
handle(event) {
|
|
1138
|
+
switch (event.type) {
|
|
1139
|
+
case "session":
|
|
1140
|
+
this.sessionId = event.sessionId ?? this.sessionId;
|
|
1141
|
+
return;
|
|
1142
|
+
case "committed":
|
|
1143
|
+
if (event.itemId && !this.pendingItems.has(event.itemId)) {
|
|
1144
|
+
this.pendingItems.add(event.itemId);
|
|
1145
|
+
this.itemOrder.push(event.itemId);
|
|
1146
|
+
}
|
|
1147
|
+
if (this.init.protocol.commitAck === "committed" && !(event.itemId && this.vadStoppedItems.has(event.itemId))) this.ackCommit();
|
|
1148
|
+
return;
|
|
1149
|
+
case "commit_empty":
|
|
1150
|
+
this.ackCommit();
|
|
1151
|
+
return;
|
|
1152
|
+
case "partial": {
|
|
1153
|
+
const key = event.itemId ?? "";
|
|
1154
|
+
const text = (this.partials.get(key) ?? "") + event.delta;
|
|
1155
|
+
this.partials.set(key, text);
|
|
1156
|
+
this.emit("partial", {
|
|
1157
|
+
delta: event.delta,
|
|
1158
|
+
text,
|
|
1159
|
+
itemId: event.itemId
|
|
1160
|
+
});
|
|
1161
|
+
return;
|
|
1162
|
+
}
|
|
1163
|
+
case "final": {
|
|
1164
|
+
let serverEnded = false;
|
|
1165
|
+
if (this.init.protocol.commitAck === "final") {
|
|
1166
|
+
if (this.outstandingCommits === 0) serverEnded = true;
|
|
1167
|
+
else if (this.ackCommit() && event.audioConsumed === false) {
|
|
1168
|
+
this.fail(new SpeechProviderError(this.provider, `${this.provider} finished a turn without transcribing its audio (the server's turn state is out of sync, e.g. after a commit crossed a server-ended turn); start a new session`));
|
|
1169
|
+
return;
|
|
1170
|
+
}
|
|
1171
|
+
}
|
|
1172
|
+
if (event.itemId) this.pendingItems.delete(event.itemId);
|
|
1173
|
+
this.partials.delete(event.itemId ?? "");
|
|
1174
|
+
this.finals.push({
|
|
1175
|
+
itemId: event.itemId,
|
|
1176
|
+
text: event.text,
|
|
1177
|
+
providerUsage: event.providerUsage,
|
|
1178
|
+
raw: event.raw,
|
|
1179
|
+
arrival: this.finals.length
|
|
1180
|
+
});
|
|
1181
|
+
this.emit("final", {
|
|
1182
|
+
text: event.text,
|
|
1183
|
+
itemId: event.itemId,
|
|
1184
|
+
providerUsage: event.providerUsage,
|
|
1185
|
+
raw: event.raw
|
|
1186
|
+
});
|
|
1187
|
+
if (serverEnded) this.fail(new SpeechProviderError(this.provider, `${this.provider} ended a turn before it was committed (for example at a server generation limit); audio sent after that point was not transcribed. Call commit() more often and start a new session`));
|
|
1188
|
+
return;
|
|
1189
|
+
}
|
|
1190
|
+
case "speech_started":
|
|
1191
|
+
this.emit(event.type, {
|
|
1192
|
+
itemId: event.itemId,
|
|
1193
|
+
audioMs: event.audioMs
|
|
1194
|
+
});
|
|
1195
|
+
return;
|
|
1196
|
+
case "speech_stopped":
|
|
1197
|
+
if (event.itemId) this.vadStoppedItems.add(event.itemId);
|
|
1198
|
+
this.emit(event.type, {
|
|
1199
|
+
itemId: event.itemId,
|
|
1200
|
+
audioMs: event.audioMs
|
|
1201
|
+
});
|
|
1202
|
+
return;
|
|
1203
|
+
case "error":
|
|
1204
|
+
this.fail(new SpeechProviderError(this.provider, this.redact(`${this.provider} error${event.code ? ` (${event.code})` : ""}: ${event.message}`) ?? `${this.provider} error`, { responseBody: event.raw === void 0 ? void 0 : this.redact(JSON.stringify(event.raw)) }));
|
|
1205
|
+
return;
|
|
1206
|
+
}
|
|
1207
|
+
}
|
|
1208
|
+
/** Acknowledges the oldest outstanding commit; returns whether it carried audio. */
|
|
1209
|
+
ackCommit() {
|
|
1210
|
+
this.outstandingCommits = Math.max(0, this.outstandingCommits - 1);
|
|
1211
|
+
return this.commitHadAudio.shift() ?? false;
|
|
1212
|
+
}
|
|
1213
|
+
async pump() {
|
|
1214
|
+
if (this.pumping || !this.opened) return;
|
|
1215
|
+
this.pumping = true;
|
|
1216
|
+
try {
|
|
1217
|
+
while (this.queue.length > 0 && !this.isTerminal()) {
|
|
1218
|
+
await this.waitForDrain();
|
|
1219
|
+
if (!await this.openTurnIfNeeded()) break;
|
|
1220
|
+
const item = this.queue.shift();
|
|
1221
|
+
if (!item || this.isTerminal()) {
|
|
1222
|
+
if (item?.kind === "audio") item.reject(this.error ?? /* @__PURE__ */ new Error("Session closed"));
|
|
1223
|
+
break;
|
|
1224
|
+
}
|
|
1225
|
+
if (item.kind === "commit") {
|
|
1226
|
+
this.sendCommit(item.final);
|
|
1227
|
+
continue;
|
|
1228
|
+
}
|
|
1229
|
+
this.queued -= item.bytes;
|
|
1230
|
+
if (!this.send(this.init.protocol.appendMessage(item.base64))) {
|
|
1231
|
+
item.reject(this.error ?? /* @__PURE__ */ new Error("Session closed"));
|
|
1232
|
+
break;
|
|
1233
|
+
}
|
|
1234
|
+
this.totalBytes += item.bytes;
|
|
1235
|
+
this.uncommittedBytes += item.bytes;
|
|
1236
|
+
await this.waitForDrain();
|
|
1237
|
+
if (this.error) item.reject(this.error);
|
|
1238
|
+
else item.resolve();
|
|
1239
|
+
}
|
|
1240
|
+
} finally {
|
|
1241
|
+
this.pumping = false;
|
|
1242
|
+
}
|
|
1243
|
+
this.checkEnded();
|
|
1244
|
+
}
|
|
1245
|
+
sendCommit(final) {
|
|
1246
|
+
if (!(this.uncommittedBytes > 0 || final && this.init.protocol.endCommit === "always") || !this.padShortTurn()) return;
|
|
1247
|
+
if (this.send(this.init.protocol.commitMessage({ final }))) {
|
|
1248
|
+
this.commitHadAudio.push(this.uncommittedBytes > 0);
|
|
1249
|
+
this.uncommittedBytes = 0;
|
|
1250
|
+
this.outstandingCommits += 1;
|
|
1251
|
+
this.turnOpen = false;
|
|
1252
|
+
}
|
|
1253
|
+
}
|
|
1254
|
+
/**
|
|
1255
|
+
* In a session with a turn limit, tops up a non-empty turn shorter than the
|
|
1256
|
+
* protocol's `minCommitSeconds` with silence (not counted as streamed
|
|
1257
|
+
* audio). Rollover can leave such a tail, which the provider would
|
|
1258
|
+
* otherwise reject as empty and drop. Returns false when the send failed.
|
|
1259
|
+
*/
|
|
1260
|
+
padShortTurn() {
|
|
1261
|
+
const minSeconds = this.init.protocol.minCommitSeconds;
|
|
1262
|
+
if (!this.init.turnLimit || !minSeconds || this.uncommittedBytes === 0) return true;
|
|
1263
|
+
const format = this.init.config.format;
|
|
1264
|
+
const align = blockAlign(format);
|
|
1265
|
+
const missing = Math.ceil(minSeconds * bytesPerSecond(format) / align - 1e-9) * align - this.uncommittedBytes;
|
|
1266
|
+
if (missing <= 0) return true;
|
|
1267
|
+
if (!this.send(this.init.protocol.appendMessage(encodeBase64(silence(missing, format))))) return false;
|
|
1268
|
+
this.uncommittedBytes += missing;
|
|
1269
|
+
return true;
|
|
1270
|
+
}
|
|
1271
|
+
/**
|
|
1272
|
+
* For protocols with an explicit turn start: before the next audio, waits
|
|
1273
|
+
* for every outstanding turn to be acknowledged, then opens a new turn. Like
|
|
1274
|
+
* the end timeout, the wait is bounded by provider inactivity: it fails only
|
|
1275
|
+
* after `timeoutMs` without any server message. Returns false when the
|
|
1276
|
+
* session has failed.
|
|
1277
|
+
*/
|
|
1278
|
+
async openTurnIfNeeded() {
|
|
1279
|
+
const start = this.init.protocol.turnStartMessage;
|
|
1280
|
+
if (!start || this.turnOpen || this.queue[0]?.kind !== "audio") return true;
|
|
1281
|
+
const waitStartedAt = Date.now();
|
|
1282
|
+
while (!this.isTerminal() && this.outstandingCommits > 0) {
|
|
1283
|
+
const lastActivity = Math.max(waitStartedAt, this.lastServerMessageAt);
|
|
1284
|
+
if (Date.now() - lastActivity >= this.init.timeoutMs) {
|
|
1285
|
+
this.fail(new SpeechProviderError(this.provider, `${this.provider} timed out after ${this.init.timeoutMs} ms without a server message while waiting for the previous turn to finish`));
|
|
1286
|
+
break;
|
|
1287
|
+
}
|
|
1288
|
+
await new Promise((resolve) => setTimeout(resolve, DRAIN_POLL_MS));
|
|
1289
|
+
}
|
|
1290
|
+
if (this.isTerminal() || !this.send(start.call(this.init.protocol))) return false;
|
|
1291
|
+
this.turnOpen = true;
|
|
1292
|
+
return true;
|
|
1293
|
+
}
|
|
1294
|
+
/**
|
|
1295
|
+
* Waits until the socket's send buffer is at or below `highWaterMark`. A
|
|
1296
|
+
* socket that stays congested for `timeoutMs` fails the session, so a
|
|
1297
|
+
* stalled connection can never leave `write()` pending forever.
|
|
1298
|
+
*/
|
|
1299
|
+
async waitForDrain() {
|
|
1300
|
+
const deadline = Date.now() + this.init.timeoutMs;
|
|
1301
|
+
while (!this.isTerminal() && this.socket && this.socket.bufferedAmount > this.init.highWaterMark) {
|
|
1302
|
+
if (Date.now() >= deadline) {
|
|
1303
|
+
this.fail(new SpeechProviderError(this.provider, `${this.provider} socket did not drain below highWaterMark within ${this.init.timeoutMs} ms`));
|
|
1304
|
+
return;
|
|
1305
|
+
}
|
|
1306
|
+
await new Promise((resolve) => setTimeout(resolve, DRAIN_POLL_MS));
|
|
1307
|
+
}
|
|
1308
|
+
}
|
|
1309
|
+
send(message) {
|
|
1310
|
+
try {
|
|
1311
|
+
this.socket?.send(JSON.stringify(message));
|
|
1312
|
+
return true;
|
|
1313
|
+
} catch (error) {
|
|
1314
|
+
this.fail(toError(error, this.provider));
|
|
1315
|
+
return false;
|
|
1316
|
+
}
|
|
1317
|
+
}
|
|
1318
|
+
checkEnded() {
|
|
1319
|
+
if (!this.endWaiter || this.isTerminal() || !this.opened || this.pumping || this.queue.length > 0 || this.outstandingCommits > 0 || this.pendingItems.size > 0) return;
|
|
1320
|
+
clearTimeout(this.endTimer);
|
|
1321
|
+
const waiter = this.endWaiter;
|
|
1322
|
+
this.endWaiter = void 0;
|
|
1323
|
+
waiter.resolve();
|
|
1324
|
+
}
|
|
1325
|
+
async finish() {
|
|
1326
|
+
this.state = "closed";
|
|
1327
|
+
this.cleanup();
|
|
1328
|
+
this.closeSocket();
|
|
1329
|
+
const ordered = [...this.finals].sort((left, right) => this.rank(left) - this.rank(right));
|
|
1330
|
+
const segments = ordered.map((final) => final.text.trim()).filter(Boolean).map((text) => ({ text }));
|
|
1331
|
+
const providerUsage = sumUsage(ordered.map((final) => final.providerUsage));
|
|
1332
|
+
const durationSeconds = audioSecondsForBytes(this.totalBytes, this.init.config.format);
|
|
1333
|
+
const usage = {
|
|
1334
|
+
operation: "transcription",
|
|
1335
|
+
provider: this.provider,
|
|
1336
|
+
model: this.init.config.model,
|
|
1337
|
+
audioSeconds: audioSecondsFromProviderUsage(providerUsage) ?? durationSeconds,
|
|
1338
|
+
bytes: this.totalBytes,
|
|
1339
|
+
providerUsage
|
|
1340
|
+
};
|
|
1341
|
+
const result = {
|
|
1342
|
+
text: segments.map((segment) => segment.text).join(" "),
|
|
1343
|
+
language: this.init.config.language,
|
|
1344
|
+
durationSeconds,
|
|
1345
|
+
segments,
|
|
1346
|
+
provider: this.provider,
|
|
1347
|
+
model: this.init.config.model,
|
|
1348
|
+
raw: {
|
|
1349
|
+
sessionId: this.sessionId,
|
|
1350
|
+
finals: ordered.map((final) => final.raw)
|
|
1351
|
+
},
|
|
1352
|
+
usage
|
|
1353
|
+
};
|
|
1354
|
+
await reportSpeechUsage(usage, ...this.init.onUsage);
|
|
1355
|
+
return result;
|
|
1356
|
+
}
|
|
1357
|
+
rank(final) {
|
|
1358
|
+
const index = final.itemId ? this.itemOrder.indexOf(final.itemId) : -1;
|
|
1359
|
+
return index >= 0 ? index : this.itemOrder.length + final.arrival;
|
|
1360
|
+
}
|
|
1361
|
+
onClose(event) {
|
|
1362
|
+
clearTimeout(this.connectTimer);
|
|
1363
|
+
if (!this.isTerminal()) {
|
|
1364
|
+
const code = event.code ?? 1005;
|
|
1365
|
+
this.fail(new SpeechProviderError(this.provider, `${this.provider} socket closed unexpectedly (code ${code}${event.reason ? `: ${this.redact(event.reason)}` : ""})`));
|
|
1366
|
+
}
|
|
1367
|
+
if (!this.closeEmitted) {
|
|
1368
|
+
this.closeEmitted = true;
|
|
1369
|
+
this.emit("close", {
|
|
1370
|
+
code: event.code ?? 1005,
|
|
1371
|
+
reason: this.redact(event.reason ?? "") ?? ""
|
|
1372
|
+
});
|
|
1373
|
+
}
|
|
1374
|
+
}
|
|
1375
|
+
fail(error) {
|
|
1376
|
+
if (this.isTerminal()) return;
|
|
1377
|
+
this.error = error;
|
|
1378
|
+
this.state = "failed";
|
|
1379
|
+
this.cleanup();
|
|
1380
|
+
const queue = this.queue;
|
|
1381
|
+
this.queue = [];
|
|
1382
|
+
this.queued = 0;
|
|
1383
|
+
for (const item of queue) if (item.kind === "audio") item.reject(error);
|
|
1384
|
+
this.rejectReady(error);
|
|
1385
|
+
const waiter = this.endWaiter;
|
|
1386
|
+
this.endWaiter = void 0;
|
|
1387
|
+
waiter?.reject(error);
|
|
1388
|
+
this.emit("error", error);
|
|
1389
|
+
this.closeSocket();
|
|
1390
|
+
}
|
|
1391
|
+
cleanup() {
|
|
1392
|
+
clearTimeout(this.connectTimer);
|
|
1393
|
+
clearTimeout(this.endTimer);
|
|
1394
|
+
this.init.signal?.removeEventListener("abort", this.onAbort);
|
|
1395
|
+
}
|
|
1396
|
+
closeSocket() {
|
|
1397
|
+
try {
|
|
1398
|
+
this.socket?.close(1e3);
|
|
1399
|
+
} catch {}
|
|
1400
|
+
}
|
|
1401
|
+
emit(event, payload) {
|
|
1402
|
+
for (const listener of [...this.listeners.get(event) ?? []]) try {
|
|
1403
|
+
listener(payload);
|
|
1404
|
+
} catch (error) {
|
|
1405
|
+
if (event !== "error" && event !== "close") this.fail(toError(error, this.provider));
|
|
1406
|
+
}
|
|
1407
|
+
}
|
|
1408
|
+
isTerminal() {
|
|
1409
|
+
return this.state === "closed" || this.state === "failed";
|
|
1410
|
+
}
|
|
1411
|
+
redact(text) {
|
|
1412
|
+
let result = text;
|
|
1413
|
+
for (const secret of this.secrets) result = redactSecret(result, secret);
|
|
1414
|
+
return result;
|
|
1415
|
+
}
|
|
1416
|
+
};
|
|
1417
|
+
/**
|
|
1418
|
+
* Throws when a turn cap is below the provider's minimum commit length:
|
|
1419
|
+
* padding a short turn up to that minimum would then exceed the cap.
|
|
1420
|
+
*/
|
|
1421
|
+
function assertTurnLimitFitsProtocol(protocol, limit) {
|
|
1422
|
+
const minimum = protocol.minCommitSeconds;
|
|
1423
|
+
if (limit && minimum && limit.maxTurnSeconds < minimum) throw new SpeechConfigurationError(`maxTurnSeconds (${limit.maxTurnSeconds}) is below ${protocol.provider}'s minimum commit of ${minimum} s`, protocol.provider);
|
|
1424
|
+
}
|
|
1425
|
+
function toError(value, provider) {
|
|
1426
|
+
if (value instanceof Error) return value;
|
|
1427
|
+
return new SpeechError(value === void 0 ? "Streaming session aborted" : String(value), "SPEECH_ABORTED", provider);
|
|
1428
|
+
}
|
|
1429
|
+
/**
|
|
1430
|
+
* Sums numeric fields of per-turn provider usage blocks (recursively), keeping
|
|
1431
|
+
* the first value of non-numeric fields such as `type`.
|
|
1432
|
+
*/
|
|
1433
|
+
function sumUsage(blocks) {
|
|
1434
|
+
const present = blocks.filter((block) => Boolean(block) && typeof block === "object");
|
|
1435
|
+
if (present.length === 0) return;
|
|
1436
|
+
const total = {};
|
|
1437
|
+
for (const block of present) for (const [key, value] of Object.entries(block)) {
|
|
1438
|
+
const current = total[key];
|
|
1439
|
+
if (typeof value === "number") total[key] = (typeof current === "number" ? current : 0) + value;
|
|
1440
|
+
else if (value && typeof value === "object" && !Array.isArray(value)) total[key] = sumUsage([current, value]);
|
|
1441
|
+
else if (current === void 0) total[key] = value;
|
|
1442
|
+
}
|
|
1443
|
+
return total;
|
|
1444
|
+
}
|
|
1445
|
+
//#endregion
|
|
1446
|
+
//#region src/adapters/openai-realtime.ts
|
|
1447
|
+
/**
|
|
1448
|
+
* OpenAI Realtime transcription adapter (`openai-realtime`).
|
|
1449
|
+
*
|
|
1450
|
+
* Targets the GA Realtime API transcription session (no `OpenAI-Beta`
|
|
1451
|
+
* header): connect to `wss://api.openai.com/v1/realtime?intent=transcription`,
|
|
1452
|
+
* send `session.update` with `session.type: "transcription"`, stream
|
|
1453
|
+
* `input_audio_buffer.append`, and read
|
|
1454
|
+
* `conversation.item.input_audio_transcription.delta` / `.completed`.
|
|
1455
|
+
* Reference: https://developers.openai.com/api/docs/guides/realtime-transcription
|
|
1456
|
+
* and https://developers.openai.com/api/reference/resources/realtime/client-events
|
|
1457
|
+
*/
|
|
1458
|
+
var ADAPTER$1 = "openai-realtime";
|
|
1459
|
+
var OPENAI_REALTIME_DEFAULT_URL = "wss://api.openai.com/v1/realtime";
|
|
1460
|
+
var OPENAI_REALTIME_DEFAULT_MODEL = "gpt-4o-transcribe";
|
|
1461
|
+
/** GA realtime input formats: 24 kHz mono PCM16, or 8 kHz mono G.711. */
|
|
1462
|
+
var WIRE_FORMATS = {
|
|
1463
|
+
pcm16: {
|
|
1464
|
+
type: "audio/pcm",
|
|
1465
|
+
sampleRate: 24e3
|
|
1466
|
+
},
|
|
1467
|
+
g711_ulaw: {
|
|
1468
|
+
type: "audio/pcmu",
|
|
1469
|
+
sampleRate: 8e3
|
|
1470
|
+
},
|
|
1471
|
+
g711_alaw: {
|
|
1472
|
+
type: "audio/pcma",
|
|
1473
|
+
sampleRate: 8e3
|
|
1474
|
+
}
|
|
1475
|
+
};
|
|
1476
|
+
/** The OpenAI Realtime transcription wire protocol. */
|
|
1477
|
+
var openAIRealtimeProtocol = (extras = {}) => ({
|
|
1478
|
+
provider: ADAPTER$1,
|
|
1479
|
+
commitAck: "committed",
|
|
1480
|
+
endCommit: "if-audio",
|
|
1481
|
+
minCommitSeconds: .1,
|
|
1482
|
+
sessionMessages(config) {
|
|
1483
|
+
return [{
|
|
1484
|
+
type: "session.update",
|
|
1485
|
+
session: openAIRealtimeTranscriptionSession(config, extras)
|
|
1486
|
+
}];
|
|
1487
|
+
},
|
|
1488
|
+
appendMessage(base64Audio) {
|
|
1489
|
+
return {
|
|
1490
|
+
type: "input_audio_buffer.append",
|
|
1491
|
+
audio: base64Audio
|
|
1492
|
+
};
|
|
1493
|
+
},
|
|
1494
|
+
commitMessage() {
|
|
1495
|
+
return { type: "input_audio_buffer.commit" };
|
|
1496
|
+
},
|
|
1497
|
+
parse: parseOpenAIRealtimeEvent
|
|
1498
|
+
});
|
|
1499
|
+
/**
|
|
1500
|
+
* The GA transcription session object (`session.type: "transcription"`),
|
|
1501
|
+
* shared by `session.update` and the client-secret mint request.
|
|
1502
|
+
*/
|
|
1503
|
+
function openAIRealtimeTranscriptionSession(config, extras = {}) {
|
|
1504
|
+
const input = {
|
|
1505
|
+
format: config.format.encoding === "pcm16" ? {
|
|
1506
|
+
type: "audio/pcm",
|
|
1507
|
+
rate: config.format.sampleRate
|
|
1508
|
+
} : { type: WIRE_FORMATS[config.format.encoding].type },
|
|
1509
|
+
transcription: {
|
|
1510
|
+
...compactJson(extras.transcriptionOptions ?? {}),
|
|
1511
|
+
...compactJson({
|
|
1512
|
+
model: config.model,
|
|
1513
|
+
language: config.language,
|
|
1514
|
+
prompt: config.prompt
|
|
1515
|
+
})
|
|
1516
|
+
},
|
|
1517
|
+
turn_detection: turnDetectionToWire(config.turnDetection)
|
|
1518
|
+
};
|
|
1519
|
+
if (extras.noiseReduction) input.noise_reduction = { type: extras.noiseReduction };
|
|
1520
|
+
return compactJson({
|
|
1521
|
+
type: "transcription",
|
|
1522
|
+
audio: { input },
|
|
1523
|
+
include: extras.include
|
|
1524
|
+
});
|
|
1525
|
+
}
|
|
1526
|
+
/** Maps one OpenAI Realtime server event to normalised session events. */
|
|
1527
|
+
function parseOpenAIRealtimeEvent(message) {
|
|
1528
|
+
if (!message || typeof message !== "object") return [];
|
|
1529
|
+
const event = message;
|
|
1530
|
+
const itemId = stringField(event, "item_id");
|
|
1531
|
+
switch (event.type) {
|
|
1532
|
+
case "session.created":
|
|
1533
|
+
case "session.updated":
|
|
1534
|
+
case "transcription_session.created":
|
|
1535
|
+
case "transcription_session.updated": {
|
|
1536
|
+
const session = event.session;
|
|
1537
|
+
return [{
|
|
1538
|
+
type: "session",
|
|
1539
|
+
sessionId: session && stringField(session, "id")
|
|
1540
|
+
}];
|
|
1541
|
+
}
|
|
1542
|
+
case "input_audio_buffer.committed": return [{
|
|
1543
|
+
type: "committed",
|
|
1544
|
+
itemId
|
|
1545
|
+
}];
|
|
1546
|
+
case "input_audio_buffer.speech_started": return [{
|
|
1547
|
+
type: "speech_started",
|
|
1548
|
+
itemId,
|
|
1549
|
+
audioMs: numberField(event, "audio_start_ms")
|
|
1550
|
+
}];
|
|
1551
|
+
case "input_audio_buffer.speech_stopped": return [{
|
|
1552
|
+
type: "speech_stopped",
|
|
1553
|
+
itemId,
|
|
1554
|
+
audioMs: numberField(event, "audio_end_ms")
|
|
1555
|
+
}];
|
|
1556
|
+
case "conversation.item.input_audio_transcription.delta": {
|
|
1557
|
+
const delta = stringField(event, "delta");
|
|
1558
|
+
return delta ? [{
|
|
1559
|
+
type: "partial",
|
|
1560
|
+
itemId,
|
|
1561
|
+
delta
|
|
1562
|
+
}] : [];
|
|
1563
|
+
}
|
|
1564
|
+
case "conversation.item.input_audio_transcription.completed": {
|
|
1565
|
+
const usage = event.usage;
|
|
1566
|
+
return [{
|
|
1567
|
+
type: "final",
|
|
1568
|
+
itemId,
|
|
1569
|
+
text: stringField(event, "transcript") ?? "",
|
|
1570
|
+
providerUsage: usage && typeof usage === "object" && !Array.isArray(usage) ? usage : void 0,
|
|
1571
|
+
raw: event
|
|
1572
|
+
}];
|
|
1573
|
+
}
|
|
1574
|
+
case "conversation.item.input_audio_transcription.failed": {
|
|
1575
|
+
const error = event.error ?? {};
|
|
1576
|
+
return [{
|
|
1577
|
+
type: "error",
|
|
1578
|
+
message: stringField(error, "message") ?? "transcription failed",
|
|
1579
|
+
code: stringField(error, "code"),
|
|
1580
|
+
raw: event
|
|
1581
|
+
}];
|
|
1582
|
+
}
|
|
1583
|
+
case "error": {
|
|
1584
|
+
const error = event.error ?? {};
|
|
1585
|
+
const code = stringField(error, "code");
|
|
1586
|
+
if (code === "input_audio_buffer_commit_empty") return [{ type: "commit_empty" }];
|
|
1587
|
+
return [{
|
|
1588
|
+
type: "error",
|
|
1589
|
+
message: stringField(error, "message") ?? "unknown error",
|
|
1590
|
+
code,
|
|
1591
|
+
raw: event
|
|
1592
|
+
}];
|
|
1593
|
+
}
|
|
1594
|
+
default: return [];
|
|
1595
|
+
}
|
|
1596
|
+
}
|
|
1597
|
+
/**
|
|
1598
|
+
* Realtime transcriber for OpenAI and OpenAI-Realtime-compatible servers.
|
|
1599
|
+
*
|
|
1600
|
+
* Auth: `apiKey` is sent as `Authorization: Bearer` (Node only). In browsers,
|
|
1601
|
+
* pass `clientSecret` (an ephemeral client secret minted by your server); it
|
|
1602
|
+
* is sent as the `openai-insecure-api-key.<secret>` subprotocol. Supplying
|
|
1603
|
+
* `apiKey` or `headers` in a browser throws.
|
|
1604
|
+
*/
|
|
1605
|
+
var OpenAIRealtimeTranscriber = class {
|
|
1606
|
+
type = ADAPTER$1;
|
|
1607
|
+
audioFormat;
|
|
1608
|
+
options;
|
|
1609
|
+
url;
|
|
1610
|
+
createWebSocket;
|
|
1611
|
+
protocol;
|
|
1612
|
+
constructor(options = {}) {
|
|
1613
|
+
assertNoBrowserApiKey(ADAPTER$1, options.apiKey);
|
|
1614
|
+
this.options = options;
|
|
1615
|
+
this.url = resolveRealtimeUrl(options.baseUrl ?? "wss://api.openai.com/v1/realtime");
|
|
1616
|
+
this.createWebSocket = resolveWebSocketFactory(ADAPTER$1, options);
|
|
1617
|
+
this.audioFormat = resolveOpenAIRealtimeFormat(options.format);
|
|
1618
|
+
this.protocol = openAIRealtimeProtocol(options);
|
|
1619
|
+
resolveTurnLimit(ADAPTER$1, void 0, options);
|
|
1620
|
+
}
|
|
1621
|
+
/**
|
|
1622
|
+
* A turn limit applies only to `manual` sessions with `maxTurnSeconds` set;
|
|
1623
|
+
* under server or semantic VAD the provider ends turns itself.
|
|
1624
|
+
*/
|
|
1625
|
+
turnLimit(options = {}) {
|
|
1626
|
+
return describeTurnLimit(this.resolveTurnLimit(options), this.sessionFormat(options));
|
|
1627
|
+
}
|
|
1628
|
+
sessionFormat(options) {
|
|
1629
|
+
return resolveOpenAIRealtimeFormat(options.format?.encoding && options.format.encoding !== this.audioFormat.encoding ? options.format : {
|
|
1630
|
+
...this.audioFormat,
|
|
1631
|
+
...options.format
|
|
1632
|
+
});
|
|
1633
|
+
}
|
|
1634
|
+
resolveTurnLimit(options) {
|
|
1635
|
+
const limit = resolveTurnLimit(ADAPTER$1, this.protocol.maxTurnSeconds, options, this.options);
|
|
1636
|
+
if (this.turnDetection(options).type !== "manual") return;
|
|
1637
|
+
assertTurnLimitFitsProtocol(this.protocol, limit);
|
|
1638
|
+
return limit;
|
|
1639
|
+
}
|
|
1640
|
+
turnDetection(options) {
|
|
1641
|
+
return options.turnDetection ?? this.options.turnDetection ?? { type: "server_vad" };
|
|
1642
|
+
}
|
|
1643
|
+
start(options = {}) {
|
|
1644
|
+
const format = this.sessionFormat(options);
|
|
1645
|
+
const connect = realtimeConnector({
|
|
1646
|
+
adapter: ADAPTER$1,
|
|
1647
|
+
url: this.url,
|
|
1648
|
+
apiKey: this.options.apiKey,
|
|
1649
|
+
headers: mergeHeaderRecords(this.options.headers, options.headers),
|
|
1650
|
+
clientSecret: options.clientSecret ?? this.options.clientSecret,
|
|
1651
|
+
protocols: (secret) => secret ? ["realtime", `${CLIENT_SECRET_SUBPROTOCOL_PREFIX}${secret}`] : ["realtime"]
|
|
1652
|
+
});
|
|
1653
|
+
return new RealtimeTranscriptionSession({
|
|
1654
|
+
protocol: this.protocol,
|
|
1655
|
+
config: {
|
|
1656
|
+
model: options.model?.trim() || this.options.model?.trim() || "gpt-4o-transcribe",
|
|
1657
|
+
format,
|
|
1658
|
+
turnDetection: this.turnDetection(options),
|
|
1659
|
+
language: options.language ?? this.options.language,
|
|
1660
|
+
prompt: options.prompt ?? this.options.prompt
|
|
1661
|
+
},
|
|
1662
|
+
connect,
|
|
1663
|
+
createWebSocket: this.createWebSocket,
|
|
1664
|
+
timeoutMs: this.options.timeoutMs ?? 3e4,
|
|
1665
|
+
connectTimeoutMs: this.options.connectTimeoutMs ?? 1e4,
|
|
1666
|
+
highWaterMark: this.options.highWaterMark ?? 1048576,
|
|
1667
|
+
maxBufferedBytes: this.options.maxBufferedBytes ?? 16777216,
|
|
1668
|
+
signal: options.signal,
|
|
1669
|
+
onUsage: [this.options.onUsage, options.onUsage],
|
|
1670
|
+
turnLimit: this.resolveTurnLimit(options)
|
|
1671
|
+
});
|
|
1672
|
+
}
|
|
1673
|
+
};
|
|
1674
|
+
/**
|
|
1675
|
+
* Resolves a realtime WebSocket URL: `http(s)` becomes `ws(s)`, a server root
|
|
1676
|
+
* gets `/v1/realtime`, a versioned API root gets `/realtime`, and
|
|
1677
|
+
* `intent=transcription` is added unless an `intent` is already present. No
|
|
1678
|
+
* `model` query parameter is added: the model travels in `session.update`.
|
|
1679
|
+
*/
|
|
1680
|
+
function resolveRealtimeUrl(baseUrl) {
|
|
1681
|
+
const url = new URL(toWebSocketUrl(resolveOpenAICompatibleUrl(baseUrl, "realtime"), ADAPTER$1));
|
|
1682
|
+
if (!url.searchParams.has("intent")) url.searchParams.set("intent", "transcription");
|
|
1683
|
+
return url.toString();
|
|
1684
|
+
}
|
|
1685
|
+
function resolveOpenAIRealtimeFormat(format) {
|
|
1686
|
+
const encoding = format?.encoding ?? "pcm16";
|
|
1687
|
+
const wire = WIRE_FORMATS[encoding];
|
|
1688
|
+
if (!wire) throw new SpeechConfigurationError(`openai-realtime does not support encoding ${String(encoding)}`, ADAPTER$1);
|
|
1689
|
+
const resolved = {
|
|
1690
|
+
encoding,
|
|
1691
|
+
sampleRate: format?.sampleRate ?? wire.sampleRate,
|
|
1692
|
+
channels: format?.channels ?? 1
|
|
1693
|
+
};
|
|
1694
|
+
if (resolved.sampleRate !== wire.sampleRate || resolved.channels !== 1) throw new SpeechConfigurationError(`openai-realtime requires ${encoding} at ${wire.sampleRate} Hz mono (got ${resolved.sampleRate} Hz, ${resolved.channels} channel(s)); resample before streaming`, ADAPTER$1);
|
|
1695
|
+
return resolved;
|
|
1696
|
+
}
|
|
1697
|
+
function turnDetectionToWire(turnDetection) {
|
|
1698
|
+
switch (turnDetection.type) {
|
|
1699
|
+
case "manual": return null;
|
|
1700
|
+
case "server_vad": return compactJson({
|
|
1701
|
+
type: "server_vad",
|
|
1702
|
+
threshold: turnDetection.threshold,
|
|
1703
|
+
prefix_padding_ms: turnDetection.prefixPaddingMs,
|
|
1704
|
+
silence_duration_ms: turnDetection.silenceDurationMs
|
|
1705
|
+
});
|
|
1706
|
+
case "semantic_vad": return compactJson({
|
|
1707
|
+
type: "semantic_vad",
|
|
1708
|
+
eagerness: turnDetection.eagerness
|
|
1709
|
+
});
|
|
1710
|
+
default: throw new SpeechConfigurationError(`Unknown turn detection type: ${String(turnDetection.type)}`, ADAPTER$1);
|
|
1711
|
+
}
|
|
1712
|
+
}
|
|
1713
|
+
function stringField(record, key) {
|
|
1714
|
+
const value = record[key];
|
|
1715
|
+
return typeof value === "string" ? value : void 0;
|
|
1716
|
+
}
|
|
1717
|
+
function numberField(record, key) {
|
|
1718
|
+
const value = record[key];
|
|
1719
|
+
return typeof value === "number" ? value : void 0;
|
|
1720
|
+
}
|
|
1721
|
+
//#endregion
|
|
1722
|
+
//#region src/adapters/voxtral-realtime.ts
|
|
1723
|
+
/**
|
|
1724
|
+
* vLLM realtime transcription adapter (`voxtral-realtime`), for Mistral's
|
|
1725
|
+
* Voxtral Realtime models (e.g. `mistralai/Voxtral-Mini-4B-Realtime-2602`)
|
|
1726
|
+
* served by vLLM at `/v1/realtime`.
|
|
1727
|
+
*
|
|
1728
|
+
* vLLM's protocol borrows OpenAI event names but is not OpenAI-compatible.
|
|
1729
|
+
* Verified against vLLM's `entrypoints/speech_to_text/realtime` source and a
|
|
1730
|
+
* live server:
|
|
1731
|
+
*
|
|
1732
|
+
* 1. The server sends `session.created` (`{ id, created }`) on connect.
|
|
1733
|
+
* 2. The client sends `session.update` with a top-level `model` (no nested
|
|
1734
|
+
* `session` object, no acknowledgement; an unknown model yields an
|
|
1735
|
+
* `error` with code `model_not_found`).
|
|
1736
|
+
* 3. A non-final `input_audio_buffer.commit` starts generation for a turn.
|
|
1737
|
+
* 4. `input_audio_buffer.append` carries base64 PCM16 at 16 kHz mono. Audio
|
|
1738
|
+
* appended before the start commit is kept.
|
|
1739
|
+
* 5. The server streams `transcription.delta` (`{ delta }`, often empty) while
|
|
1740
|
+
* audio arrives, at roughly real-time speed.
|
|
1741
|
+
* 6. `input_audio_buffer.commit` with `final: true` ends the turn; the server
|
|
1742
|
+
* replies `transcription.done` (`{ text, usage }`, token usage only) and
|
|
1743
|
+
* clears its buffer. A further start commit opens the next turn on the same
|
|
1744
|
+
* socket.
|
|
1745
|
+
* 7. Errors arrive as `{ type: 'error', error: string, code }` and leave the
|
|
1746
|
+
* socket open; this adapter treats every error as fatal.
|
|
1747
|
+
* 8. A turn whose audio and tokens exceed `max_model_len` (~12.5 tokens per
|
|
1748
|
+
* audio second plus a 39-token prompt) ends in `error` `processing_error`
|
|
1749
|
+
* and can take the engine down. The limit is per turn: after a final
|
|
1750
|
+
* commit vLLM clears the turn's context. The session therefore rolls over
|
|
1751
|
+
* to a new turn before `maxTurnSeconds` (default 270) of audio.
|
|
1752
|
+
* 9. A stray final commit leaves a stale end marker: the next turn ends at once
|
|
1753
|
+
* with `prompt_tokens: 1` (any audio costs 39) and its audio is dropped.
|
|
1754
|
+
* The session fails closed on that, and on any `done` for a turn it did
|
|
1755
|
+
* not commit, rather than losing audio silently.
|
|
1756
|
+
*/
|
|
1757
|
+
var ADAPTER = "voxtral-realtime";
|
|
1758
|
+
/** vLLM's default served model name for Voxtral Mini Realtime. */
|
|
1759
|
+
var VOXTRAL_REALTIME_DEFAULT_MODEL = "mistralai/Voxtral-Mini-4B-Realtime-2602";
|
|
1760
|
+
/** The only input format vLLM's realtime endpoint accepts. */
|
|
1761
|
+
var VOXTRAL_REALTIME_AUDIO_FORMAT = Object.freeze({
|
|
1762
|
+
encoding: "pcm16",
|
|
1763
|
+
sampleRate: 16e3,
|
|
1764
|
+
channels: 1
|
|
1765
|
+
});
|
|
1766
|
+
/** The vLLM realtime wire protocol. */
|
|
1767
|
+
var voxtralRealtimeProtocol = () => ({
|
|
1768
|
+
provider: ADAPTER,
|
|
1769
|
+
commitAck: "final",
|
|
1770
|
+
endCommit: "if-audio",
|
|
1771
|
+
maxTurnSeconds: 270,
|
|
1772
|
+
sessionMessages(config) {
|
|
1773
|
+
return [{
|
|
1774
|
+
type: "session.update",
|
|
1775
|
+
model: config.model
|
|
1776
|
+
}];
|
|
1777
|
+
},
|
|
1778
|
+
turnStartMessage() {
|
|
1779
|
+
return { type: "input_audio_buffer.commit" };
|
|
1780
|
+
},
|
|
1781
|
+
appendMessage(base64Audio) {
|
|
1782
|
+
return {
|
|
1783
|
+
type: "input_audio_buffer.append",
|
|
1784
|
+
audio: base64Audio
|
|
1785
|
+
};
|
|
1786
|
+
},
|
|
1787
|
+
commitMessage() {
|
|
1788
|
+
return {
|
|
1789
|
+
type: "input_audio_buffer.commit",
|
|
1790
|
+
final: true
|
|
1791
|
+
};
|
|
1792
|
+
},
|
|
1793
|
+
parse: parseVoxtralRealtimeEvent
|
|
1794
|
+
});
|
|
1795
|
+
/** Maps one vLLM realtime server event to normalised session events. */
|
|
1796
|
+
function parseVoxtralRealtimeEvent(message) {
|
|
1797
|
+
if (!message || typeof message !== "object") return [];
|
|
1798
|
+
const event = message;
|
|
1799
|
+
switch (event.type) {
|
|
1800
|
+
case "session.created": return [{
|
|
1801
|
+
type: "session",
|
|
1802
|
+
sessionId: typeof event.id === "string" ? event.id : void 0
|
|
1803
|
+
}];
|
|
1804
|
+
case "transcription.delta": {
|
|
1805
|
+
const delta = typeof event.delta === "string" ? event.delta : "";
|
|
1806
|
+
return delta ? [{
|
|
1807
|
+
type: "partial",
|
|
1808
|
+
delta
|
|
1809
|
+
}] : [];
|
|
1810
|
+
}
|
|
1811
|
+
case "transcription.done": {
|
|
1812
|
+
const usage = event.usage && typeof event.usage === "object" && !Array.isArray(event.usage) ? event.usage : void 0;
|
|
1813
|
+
const promptTokens = usage?.prompt_tokens;
|
|
1814
|
+
return [{
|
|
1815
|
+
type: "final",
|
|
1816
|
+
text: typeof event.text === "string" ? event.text.trim() : "",
|
|
1817
|
+
providerUsage: usage,
|
|
1818
|
+
raw: event,
|
|
1819
|
+
audioConsumed: typeof promptTokens === "number" ? promptTokens > 1 : void 0
|
|
1820
|
+
}];
|
|
1821
|
+
}
|
|
1822
|
+
case "error": return [{
|
|
1823
|
+
type: "error",
|
|
1824
|
+
message: typeof event.error === "string" ? event.error : "unknown error",
|
|
1825
|
+
code: typeof event.code === "string" ? event.code : void 0,
|
|
1826
|
+
raw: event
|
|
1827
|
+
}];
|
|
1828
|
+
default: return [];
|
|
1829
|
+
}
|
|
1830
|
+
}
|
|
1831
|
+
/**
|
|
1832
|
+
* Realtime transcriber for Voxtral Realtime served by vLLM.
|
|
1833
|
+
*
|
|
1834
|
+
* - Audio: PCM16, 16 kHz, mono only; write whole samples (even byte counts).
|
|
1835
|
+
* - Turns: manual only. vLLM has no VAD; it streams partials continuously and
|
|
1836
|
+
* produces one final per `commit()` / `end()`, plus one per automatic
|
|
1837
|
+
* rollover when a turn reaches `maxTurnSeconds` (default 270).
|
|
1838
|
+
* - `language` and `prompt` are not sent: vLLM's realtime session accepts
|
|
1839
|
+
* only `model`.
|
|
1840
|
+
* - Auth: `apiKey` → `Authorization: Bearer` (Node only). vLLM has no
|
|
1841
|
+
* ephemeral tokens, so browsers must connect through the consumer's own
|
|
1842
|
+
* proxy or token gateway, passing that gateway's short-lived `clientSecret`
|
|
1843
|
+
* (sent as the `openai-insecure-api-key.<secret>` subprotocol next to
|
|
1844
|
+
* `realtime`; the gateway must echo `realtime`).
|
|
1845
|
+
*/
|
|
1846
|
+
var VoxtralRealtimeTranscriber = class {
|
|
1847
|
+
type = ADAPTER;
|
|
1848
|
+
audioFormat = { ...VOXTRAL_REALTIME_AUDIO_FORMAT };
|
|
1849
|
+
options;
|
|
1850
|
+
url;
|
|
1851
|
+
createWebSocket;
|
|
1852
|
+
protocol = voxtralRealtimeProtocol();
|
|
1853
|
+
constructor(options = {}) {
|
|
1854
|
+
assertNoBrowserApiKey(ADAPTER, options.apiKey);
|
|
1855
|
+
if (!options.baseUrl?.trim()) throw new SpeechConfigurationError("voxtral-realtime requires baseUrl (the vLLM server, e.g. http://localhost:8000 or its /v1 root)", ADAPTER);
|
|
1856
|
+
this.options = options;
|
|
1857
|
+
this.url = resolveVoxtralRealtimeUrl(options.baseUrl);
|
|
1858
|
+
this.createWebSocket = resolveWebSocketFactory(ADAPTER, options);
|
|
1859
|
+
assertFormat(options.format);
|
|
1860
|
+
assertTurnDetection(options.turnDetection);
|
|
1861
|
+
this.resolveTurnLimit();
|
|
1862
|
+
}
|
|
1863
|
+
turnLimit(options = {}) {
|
|
1864
|
+
assertFormat(options.format);
|
|
1865
|
+
return describeTurnLimit(this.resolveTurnLimit(options), VOXTRAL_REALTIME_AUDIO_FORMAT);
|
|
1866
|
+
}
|
|
1867
|
+
resolveTurnLimit(options = {}) {
|
|
1868
|
+
return resolveTurnLimit(ADAPTER, this.protocol.maxTurnSeconds, options, this.options);
|
|
1869
|
+
}
|
|
1870
|
+
start(options = {}) {
|
|
1871
|
+
assertFormat(options.format);
|
|
1872
|
+
assertTurnDetection(options.turnDetection);
|
|
1873
|
+
const turnLimit = this.resolveTurnLimit(options);
|
|
1874
|
+
const connect = realtimeConnector({
|
|
1875
|
+
adapter: ADAPTER,
|
|
1876
|
+
url: this.url,
|
|
1877
|
+
apiKey: this.options.apiKey,
|
|
1878
|
+
headers: mergeHeaderRecords(this.options.headers, options.headers),
|
|
1879
|
+
clientSecret: options.clientSecret ?? this.options.clientSecret,
|
|
1880
|
+
protocols: (secret) => secret ? ["realtime", `${CLIENT_SECRET_SUBPROTOCOL_PREFIX}${secret}`] : []
|
|
1881
|
+
});
|
|
1882
|
+
return new RealtimeTranscriptionSession({
|
|
1883
|
+
protocol: this.protocol,
|
|
1884
|
+
config: {
|
|
1885
|
+
model: options.model?.trim() || this.options.model?.trim() || "mistralai/Voxtral-Mini-4B-Realtime-2602",
|
|
1886
|
+
format: { ...VOXTRAL_REALTIME_AUDIO_FORMAT },
|
|
1887
|
+
turnDetection: { type: "manual" },
|
|
1888
|
+
language: options.language ?? this.options.language
|
|
1889
|
+
},
|
|
1890
|
+
connect,
|
|
1891
|
+
createWebSocket: this.createWebSocket,
|
|
1892
|
+
timeoutMs: this.options.timeoutMs ?? 3e4,
|
|
1893
|
+
connectTimeoutMs: this.options.connectTimeoutMs ?? 1e4,
|
|
1894
|
+
highWaterMark: this.options.highWaterMark ?? 1048576,
|
|
1895
|
+
maxBufferedBytes: this.options.maxBufferedBytes ?? 16777216,
|
|
1896
|
+
signal: options.signal,
|
|
1897
|
+
onUsage: [this.options.onUsage, options.onUsage],
|
|
1898
|
+
turnLimit
|
|
1899
|
+
});
|
|
1900
|
+
}
|
|
1901
|
+
};
|
|
1902
|
+
/**
|
|
1903
|
+
* Resolves vLLM's realtime WebSocket URL: `http(s)` becomes `ws(s)`, a server
|
|
1904
|
+
* root gets `/v1/realtime`, a versioned API root (e.g. a gateway's
|
|
1905
|
+
* `/stt/v1`) gets `/realtime`, and a URL ending in `/realtime` is kept.
|
|
1906
|
+
*/
|
|
1907
|
+
function resolveVoxtralRealtimeUrl(baseUrl) {
|
|
1908
|
+
return toWebSocketUrl(resolveOpenAICompatibleUrl(baseUrl, "realtime"), ADAPTER);
|
|
1909
|
+
}
|
|
1910
|
+
function assertFormat(format) {
|
|
1911
|
+
const expected = VOXTRAL_REALTIME_AUDIO_FORMAT;
|
|
1912
|
+
if (format && (format.encoding !== void 0 && format.encoding !== expected.encoding || format.sampleRate !== void 0 && format.sampleRate !== expected.sampleRate || format.channels !== void 0 && format.channels !== expected.channels)) throw new SpeechConfigurationError(`voxtral-realtime requires pcm16 at 16000 Hz mono (got ${format.encoding ?? "pcm16"} at ${format.sampleRate ?? 16e3} Hz, ${format.channels ?? 1} channel(s)); resample before streaming`, ADAPTER);
|
|
1913
|
+
}
|
|
1914
|
+
function assertTurnDetection(turnDetection) {
|
|
1915
|
+
if (turnDetection && turnDetection.type !== "manual") throw new SpeechConfigurationError(`voxtral-realtime has no server turn detection (got ${turnDetection.type}); use manual turns with commit()/end()`, ADAPTER);
|
|
1916
|
+
}
|
|
1917
|
+
//#endregion
|
|
283
1918
|
//#region src/adapters/openai-compatible.ts
|
|
284
1919
|
var OpenAICompatibleSpeechSynthesizer = class extends HttpSpeechAdapter {
|
|
285
1920
|
type = "openai-compatible";
|
|
1921
|
+
/** Explicit override, resolved relative to `baseUrl` (or absolute). */
|
|
286
1922
|
speechPath;
|
|
287
1923
|
defaultModel;
|
|
288
1924
|
defaultVoice;
|
|
289
1925
|
constructor(options) {
|
|
290
1926
|
super(options);
|
|
291
|
-
this.speechPath = options.speechPath
|
|
1927
|
+
this.speechPath = options.speechPath;
|
|
292
1928
|
this.defaultModel = options.defaultModel ?? "tts-1";
|
|
293
1929
|
this.defaultVoice = options.defaultVoice ?? "alloy";
|
|
294
1930
|
}
|
|
@@ -301,7 +1937,8 @@ var OpenAICompatibleSpeechSynthesizer = class extends HttpSpeechAdapter {
|
|
|
301
1937
|
response_format: outputFormat,
|
|
302
1938
|
speed: request.speed
|
|
303
1939
|
});
|
|
304
|
-
const
|
|
1940
|
+
const endpoint = this.speechPath ?? resolveOpenAICompatibleUrl(this.baseUrl, "audio/speech");
|
|
1941
|
+
const speech = await this.post(this.type, endpoint, {
|
|
305
1942
|
headers: { "content-type": "application/json" },
|
|
306
1943
|
body: JSON.stringify(payload),
|
|
307
1944
|
signal: request.signal
|
|
@@ -314,6 +1951,118 @@ var OpenAICompatibleSpeechSynthesizer = class extends HttpSpeechAdapter {
|
|
|
314
1951
|
}
|
|
315
1952
|
};
|
|
316
1953
|
//#endregion
|
|
1954
|
+
//#region src/adapters/openai-compatible-transcriber.ts
|
|
1955
|
+
var DEFAULT_MODEL = "whisper-1";
|
|
1956
|
+
/**
|
|
1957
|
+
* Models that only accept `json`/`text` and reject `verbose_json` and
|
|
1958
|
+
* `timestamp_granularities[]` (e.g. `gpt-4o-transcribe`,
|
|
1959
|
+
* `gpt-4o-mini-transcribe`, optionally gateway-prefixed like
|
|
1960
|
+
* `openai/gpt-4o-transcribe`).
|
|
1961
|
+
*/
|
|
1962
|
+
var JSON_ONLY_MODEL = /(^|\/)gpt-4o[\w.-]*-transcribe/i;
|
|
1963
|
+
/** Provider 400/422 bodies that indicate an unsupported format option. */
|
|
1964
|
+
var UNSUPPORTED_FORMAT_ERROR = /verbose_json|response_format|timestamp_granularit/i;
|
|
1965
|
+
function isJsonOnlyTranscriptionModel(model) {
|
|
1966
|
+
return JSON_ONLY_MODEL.test(model);
|
|
1967
|
+
}
|
|
1968
|
+
/**
|
|
1969
|
+
* OpenAI-compatible speech-to-text adapter (`POST <base>/audio/transcriptions`).
|
|
1970
|
+
* Works with OpenAI, Groq, Fireworks, LiteLLM/Bifrost gateways, vLLM,
|
|
1971
|
+
* speaches/faster-whisper-server, and whisper.cpp server.
|
|
1972
|
+
*
|
|
1973
|
+
* Server-side only: the adapter holds the API key.
|
|
1974
|
+
*/
|
|
1975
|
+
var OpenAICompatibleTranscriber = class extends HttpSpeechAdapter {
|
|
1976
|
+
type = "openai-compatible";
|
|
1977
|
+
endpoint;
|
|
1978
|
+
model;
|
|
1979
|
+
responseFormat;
|
|
1980
|
+
maxBytes;
|
|
1981
|
+
retry;
|
|
1982
|
+
onUsage;
|
|
1983
|
+
constructor(options) {
|
|
1984
|
+
super(options);
|
|
1985
|
+
this.endpoint = resolveOpenAICompatibleUrl(options.baseUrl, "audio/transcriptions");
|
|
1986
|
+
this.model = options.model?.trim() || DEFAULT_MODEL;
|
|
1987
|
+
this.responseFormat = options.responseFormat;
|
|
1988
|
+
this.maxBytes = options.maxBytes ?? 26214400;
|
|
1989
|
+
this.retry = resolveRetryOptions(options.retry);
|
|
1990
|
+
this.onUsage = options.onUsage;
|
|
1991
|
+
}
|
|
1992
|
+
async transcribe(request) {
|
|
1993
|
+
const audio = await normalizeAudioInput(request.audio, {
|
|
1994
|
+
mimeType: request.mimeType,
|
|
1995
|
+
maxBytes: request.maxBytes ?? this.maxBytes,
|
|
1996
|
+
adapter: this.type,
|
|
1997
|
+
signal: request.signal
|
|
1998
|
+
});
|
|
1999
|
+
const wire = this.resolveWireOptions(request);
|
|
2000
|
+
const send = (options) => {
|
|
2001
|
+
const form = new FormData();
|
|
2002
|
+
form.append("file", audio.blob, audio.filename);
|
|
2003
|
+
form.set("model", options.model);
|
|
2004
|
+
appendOptionalFormValue(form, "language", request.language);
|
|
2005
|
+
appendOptionalFormValue(form, "prompt", request.prompt);
|
|
2006
|
+
appendOptionalFormValue(form, "temperature", request.temperature);
|
|
2007
|
+
form.set("response_format", options.responseFormat);
|
|
2008
|
+
for (const granularity of options.timestampGranularities ?? []) form.append("timestamp_granularities[]", granularity);
|
|
2009
|
+
return this.post(this.type, this.endpoint, {
|
|
2010
|
+
body: form,
|
|
2011
|
+
headers: request.headers,
|
|
2012
|
+
signal: request.signal
|
|
2013
|
+
}, (response) => readTranscriptResponse(response, this.type), this.retry);
|
|
2014
|
+
};
|
|
2015
|
+
let transcript;
|
|
2016
|
+
try {
|
|
2017
|
+
transcript = await send(wire);
|
|
2018
|
+
} catch (error) {
|
|
2019
|
+
if (!isUnsupportedFormatError(error, wire)) throw error;
|
|
2020
|
+
transcript = await send({
|
|
2021
|
+
model: wire.model,
|
|
2022
|
+
responseFormat: "json"
|
|
2023
|
+
});
|
|
2024
|
+
}
|
|
2025
|
+
const providerUsage = extractProviderUsage(transcript.raw);
|
|
2026
|
+
const usage = {
|
|
2027
|
+
operation: "transcription",
|
|
2028
|
+
provider: this.type,
|
|
2029
|
+
model: transcript.model ?? wire.model,
|
|
2030
|
+
audioSeconds: transcript.durationSeconds ?? audioSecondsFromProviderUsage(providerUsage) ?? audio.durationSeconds,
|
|
2031
|
+
bytes: audio.bytes,
|
|
2032
|
+
providerUsage
|
|
2033
|
+
};
|
|
2034
|
+
const result = {
|
|
2035
|
+
...transcript,
|
|
2036
|
+
text: transcript.raw === void 0 ? transcript.text.trim() : transcript.text,
|
|
2037
|
+
provider: this.type,
|
|
2038
|
+
model: transcript.model ?? wire.model,
|
|
2039
|
+
usage
|
|
2040
|
+
};
|
|
2041
|
+
await reportSpeechUsage(usage, this.onUsage, request.onUsage);
|
|
2042
|
+
return result;
|
|
2043
|
+
}
|
|
2044
|
+
resolveWireOptions(request) {
|
|
2045
|
+
const model = request.model?.trim() || this.model;
|
|
2046
|
+
const jsonOnly = isJsonOnlyTranscriptionModel(model);
|
|
2047
|
+
let responseFormat = request.responseFormat ?? this.responseFormat ?? (jsonOnly ? "json" : "verbose_json");
|
|
2048
|
+
if (jsonOnly && responseFormat === "verbose_json") responseFormat = "json";
|
|
2049
|
+
const timestampGranularities = responseFormat === "verbose_json" && request.timestampGranularities?.length ? [...new Set(request.timestampGranularities)] : void 0;
|
|
2050
|
+
return {
|
|
2051
|
+
model,
|
|
2052
|
+
responseFormat,
|
|
2053
|
+
timestampGranularities
|
|
2054
|
+
};
|
|
2055
|
+
}
|
|
2056
|
+
};
|
|
2057
|
+
function isUnsupportedFormatError(error, wire) {
|
|
2058
|
+
return error instanceof SpeechProviderError && (error.status === 400 || error.status === 422) && (wire.responseFormat === "verbose_json" || Boolean(wire.timestampGranularities?.length)) && UNSUPPORTED_FORMAT_ERROR.test(error.responseBody ?? "");
|
|
2059
|
+
}
|
|
2060
|
+
function extractProviderUsage(raw) {
|
|
2061
|
+
if (!raw || typeof raw !== "object") return;
|
|
2062
|
+
const usage = raw.usage;
|
|
2063
|
+
return usage && typeof usage === "object" && !Array.isArray(usage) ? usage : void 0;
|
|
2064
|
+
}
|
|
2065
|
+
//#endregion
|
|
317
2066
|
//#region src/adapters/qwen3.ts
|
|
318
2067
|
var Qwen3SpeechSynthesizer = class extends HttpSpeechAdapter {
|
|
319
2068
|
type = "qwen3-tts";
|
|
@@ -349,11 +2098,19 @@ var StudioServerTranscriber = class extends HttpSpeechAdapter {
|
|
|
349
2098
|
this.transcribePath = options.transcribePath ?? "/v1/transcribe";
|
|
350
2099
|
}
|
|
351
2100
|
async transcribe(request) {
|
|
2101
|
+
const audio = await normalizeAudioInput(request.audio, {
|
|
2102
|
+
mimeType: request.mimeType,
|
|
2103
|
+
maxBytes: request.maxBytes,
|
|
2104
|
+
adapter: this.type,
|
|
2105
|
+
signal: request.signal,
|
|
2106
|
+
deriveExtension: false
|
|
2107
|
+
});
|
|
352
2108
|
const form = new FormData();
|
|
353
|
-
|
|
2109
|
+
form.append("audio", audio.blob, audio.filename);
|
|
354
2110
|
appendOptionalFormValue(form, "language", request.language);
|
|
355
2111
|
return this.post(this.type, this.transcribePath, {
|
|
356
2112
|
body: form,
|
|
2113
|
+
headers: request.headers,
|
|
357
2114
|
signal: request.signal
|
|
358
2115
|
}, (response) => readTranscriptResponse(response, this.type));
|
|
359
2116
|
}
|
|
@@ -376,6 +2133,143 @@ var StudioServerSpeechSynthesizer = class extends HttpSpeechAdapter {
|
|
|
376
2133
|
}
|
|
377
2134
|
};
|
|
378
2135
|
//#endregion
|
|
2136
|
+
//#region src/shared/streaming-factory.ts
|
|
2137
|
+
/**
|
|
2138
|
+
* Streaming transcriber factory, environment configuration, and the
|
|
2139
|
+
* record-then-send `Transcriber` wrapper.
|
|
2140
|
+
*/
|
|
2141
|
+
/** Streaming adapters `getStreamingTranscriber()` can build. */
|
|
2142
|
+
var STREAMING_TRANSCRIBER_TYPES = ["openai-realtime", "voxtral-realtime"];
|
|
2143
|
+
/** Environment variable names read for each streaming setting, in priority order. */
|
|
2144
|
+
var STREAMING_TRANSCRIBER_ENV_KEYS = {
|
|
2145
|
+
type: ["HAVE_SPEECH_STREAMING_TYPE"],
|
|
2146
|
+
baseUrl: ["HAVE_SPEECH_STREAMING_BASE_URL"],
|
|
2147
|
+
model: ["HAVE_SPEECH_STREAMING_MODEL"],
|
|
2148
|
+
apiKey: ["HAVE_SPEECH_STREAMING_API_KEY"],
|
|
2149
|
+
language: ["HAVE_SPEECH_STREAMING_LANGUAGE"],
|
|
2150
|
+
turnDetection: ["HAVE_SPEECH_STREAMING_TURN_DETECTION"],
|
|
2151
|
+
timeoutMs: ["HAVE_SPEECH_STREAMING_TIMEOUT", "HAVE_SPEECH_STREAMING_TIMEOUT_MS"],
|
|
2152
|
+
connectTimeoutMs: ["HAVE_SPEECH_STREAMING_CONNECT_TIMEOUT_MS"],
|
|
2153
|
+
headers: ["HAVE_SPEECH_STREAMING_HEADERS"],
|
|
2154
|
+
maxTurnSeconds: ["HAVE_SPEECH_STREAMING_MAX_TURN_SECONDS"]
|
|
2155
|
+
};
|
|
2156
|
+
function isStreamingTranscriberType(type) {
|
|
2157
|
+
return type !== void 0 && STREAMING_TRANSCRIBER_TYPES.includes(type);
|
|
2158
|
+
}
|
|
2159
|
+
/**
|
|
2160
|
+
* Creates a realtime streaming transcriber. Explicit options win over
|
|
2161
|
+
* `HAVE_SPEECH_STREAMING_*` environment variables. The type defaults to
|
|
2162
|
+
* `openai-realtime`.
|
|
2163
|
+
*/
|
|
2164
|
+
function getStreamingTranscriber(options = {}, context = {}) {
|
|
2165
|
+
const env = context.env ?? defaultEnv();
|
|
2166
|
+
const keys = STREAMING_TRANSCRIBER_ENV_KEYS;
|
|
2167
|
+
const type = options.type ?? readEnv(env, ...keys.type) ?? "openai-realtime";
|
|
2168
|
+
if (!isStreamingTranscriberType(type)) throw new InvalidSpeechAdapterError(type, "streaming STT");
|
|
2169
|
+
const headers = mergeHeaderInits(parseHeadersEnv(env, keys.headers), context.headers, options.headers);
|
|
2170
|
+
const resolved = {
|
|
2171
|
+
...options,
|
|
2172
|
+
type,
|
|
2173
|
+
baseUrl: (options.baseUrl ?? readEnv(env, ...keys.baseUrl))?.trim(),
|
|
2174
|
+
model: options.model ?? readEnv(env, ...keys.model),
|
|
2175
|
+
apiKey: options.apiKey ?? readEnv(env, ...keys.apiKey),
|
|
2176
|
+
language: options.language ?? readEnv(env, ...keys.language),
|
|
2177
|
+
turnDetection: options.turnDetection ?? parseTurnDetection(readEnv(env, ...keys.turnDetection)),
|
|
2178
|
+
timeoutMs: options.timeoutMs ?? parseOptionalInteger(readEnv(env, ...keys.timeoutMs)),
|
|
2179
|
+
connectTimeoutMs: options.connectTimeoutMs ?? parseOptionalInteger(readEnv(env, ...keys.connectTimeoutMs)),
|
|
2180
|
+
maxTurnSeconds: options.maxTurnSeconds ?? parseMaxTurnSeconds(readEnv(env, ...keys.maxTurnSeconds)),
|
|
2181
|
+
headers
|
|
2182
|
+
};
|
|
2183
|
+
switch (type) {
|
|
2184
|
+
case "openai-realtime": return new OpenAIRealtimeTranscriber({
|
|
2185
|
+
...resolved,
|
|
2186
|
+
type
|
|
2187
|
+
});
|
|
2188
|
+
case "voxtral-realtime": return new VoxtralRealtimeTranscriber({
|
|
2189
|
+
...resolved,
|
|
2190
|
+
type
|
|
2191
|
+
});
|
|
2192
|
+
default: throw new InvalidSpeechAdapterError(type, "streaming STT");
|
|
2193
|
+
}
|
|
2194
|
+
}
|
|
2195
|
+
function parseTurnDetection(value) {
|
|
2196
|
+
if (!value) return;
|
|
2197
|
+
const normalized = value.toLowerCase();
|
|
2198
|
+
if (normalized === "server_vad" || normalized === "semantic_vad") return { type: normalized };
|
|
2199
|
+
if (normalized === "manual" || normalized === "none") return { type: "manual" };
|
|
2200
|
+
throw new SpeechConfigurationError("HAVE_SPEECH_STREAMING_TURN_DETECTION must be server_vad, semantic_vad, or manual");
|
|
2201
|
+
}
|
|
2202
|
+
/**
|
|
2203
|
+
* Parses `HAVE_SPEECH_STREAMING_MAX_TURN_SECONDS`: a positive number of
|
|
2204
|
+
* seconds, or `Infinity` for no limit.
|
|
2205
|
+
*/
|
|
2206
|
+
function parseMaxTurnSeconds(value) {
|
|
2207
|
+
if (!value) return;
|
|
2208
|
+
const parsed = Number(value);
|
|
2209
|
+
if (Number.isNaN(parsed) || parsed <= 0) throw new SpeechConfigurationError("HAVE_SPEECH_STREAMING_MAX_TURN_SECONDS must be a positive number of seconds or Infinity");
|
|
2210
|
+
return parsed;
|
|
2211
|
+
}
|
|
2212
|
+
/**
|
|
2213
|
+
* Adapts a streaming transcriber to the request/response `Transcriber`
|
|
2214
|
+
* interface for record-then-send callers. Each `transcribe()` opens one
|
|
2215
|
+
* session, streams the recording, and returns `session.end()`.
|
|
2216
|
+
*
|
|
2217
|
+
* Input must be raw audio: a PCM16/G.711 WAV file, `audio/pcm` or
|
|
2218
|
+
* `audio/L16` (optionally with `rate`/`channels` parameters), `audio/pcmu`,
|
|
2219
|
+
* `audio/pcma`, or untyped bytes in the adapter's default format. WAV headers
|
|
2220
|
+
* and MIME parameters set the session format; compressed recordings are
|
|
2221
|
+
* rejected, so use an HTTP transcriber for WebM/MP4/Ogg.
|
|
2222
|
+
*
|
|
2223
|
+
* A clip longer than the adapter's per-turn limit (`maxTurnSeconds`) is
|
|
2224
|
+
* never sent as one turn: with rollover (the default) the session splits it
|
|
2225
|
+
* into several turns at quiet boundaries, and with `rollover: false` the clip
|
|
2226
|
+
* is rejected with `SPEECH_TURN_TOO_LONG` before connecting.
|
|
2227
|
+
*/
|
|
2228
|
+
function wrapStreamingTranscriber(streaming, options = {}) {
|
|
2229
|
+
const chunkBytes = options.chunkBytes ?? 64 * 1024;
|
|
2230
|
+
if (!Number.isInteger(chunkBytes) || chunkBytes <= 0) throw new SpeechConfigurationError("chunkBytes must be a positive integer", streaming.type);
|
|
2231
|
+
return {
|
|
2232
|
+
type: streaming.type,
|
|
2233
|
+
async transcribe(request) {
|
|
2234
|
+
const audio = await normalizeAudioInput(request.audio, {
|
|
2235
|
+
mimeType: request.mimeType,
|
|
2236
|
+
maxBytes: request.maxBytes ?? options.maxBytes ?? 26214400,
|
|
2237
|
+
adapter: streaming.type,
|
|
2238
|
+
signal: request.signal,
|
|
2239
|
+
deriveExtension: false
|
|
2240
|
+
});
|
|
2241
|
+
const raw = unwrapRawAudio(new Uint8Array(await audio.blob.arrayBuffer()), audio.mimeType, streaming.type);
|
|
2242
|
+
const declared = request.audio && typeof request.audio === "object" && "data" in request.audio ? request.audio : void 0;
|
|
2243
|
+
const format = {
|
|
2244
|
+
...raw.format,
|
|
2245
|
+
sampleRate: raw.format.sampleRate ?? declared?.sampleRate,
|
|
2246
|
+
channels: raw.format.channels ?? declared?.channels
|
|
2247
|
+
};
|
|
2248
|
+
const sessionOptions = {
|
|
2249
|
+
format: Object.fromEntries(Object.entries(format).filter(([, value]) => value !== void 0)),
|
|
2250
|
+
language: request.language,
|
|
2251
|
+
model: request.model,
|
|
2252
|
+
prompt: request.prompt,
|
|
2253
|
+
headers: request.headers,
|
|
2254
|
+
signal: request.signal,
|
|
2255
|
+
onUsage: request.onUsage,
|
|
2256
|
+
metadata: request.metadata,
|
|
2257
|
+
turnDetection: options.turnDetection ?? { type: "manual" }
|
|
2258
|
+
};
|
|
2259
|
+
const limit = streaming.turnLimit?.(sessionOptions);
|
|
2260
|
+
if (limit && !limit.rollover && raw.bytes.byteLength > limit.maxTurnBytes) throw new SpeechError(`The recording is longer than one turn may be (maxTurnSeconds ${limit.maxTurnSeconds} s) and rollover is disabled; enable rollover or split the recording`, "SPEECH_TURN_TOO_LONG", streaming.type);
|
|
2261
|
+
const session = streaming.start(sessionOptions);
|
|
2262
|
+
try {
|
|
2263
|
+
for (let offset = 0; offset < raw.bytes.byteLength; offset += chunkBytes) await session.write(raw.bytes.subarray(offset, offset + chunkBytes));
|
|
2264
|
+
} catch (error) {
|
|
2265
|
+
session.abort(error);
|
|
2266
|
+
throw error;
|
|
2267
|
+
}
|
|
2268
|
+
return session.end();
|
|
2269
|
+
}
|
|
2270
|
+
};
|
|
2271
|
+
}
|
|
2272
|
+
//#endregion
|
|
379
2273
|
//#region src/shared/factory.ts
|
|
380
2274
|
/**
|
|
381
2275
|
* Creates a speech service from explicit options and/or environment variables.
|
|
@@ -401,9 +2295,14 @@ async function getSpeech(options = {}, context = {}) {
|
|
|
401
2295
|
};
|
|
402
2296
|
}
|
|
403
2297
|
async function getTranscriber(options = {}, context = {}) {
|
|
2298
|
+
const streamingType = options.type ?? readEnv(context.env ?? defaultEnv(), ...TRANSCRIBER_ENV_KEYS.type);
|
|
2299
|
+
if (isStreamingTranscriberType(streamingType)) return getWrappedStreamingTranscriber(streamingType, options, context);
|
|
2300
|
+
const optional = await getRegisteredTranscriber(options, context);
|
|
2301
|
+
if (optional) return optional;
|
|
404
2302
|
const resolved = normalizeTranscriberOptions(options, context);
|
|
405
2303
|
switch (resolved.type) {
|
|
406
2304
|
case "studio-server": return new StudioServerTranscriber(resolved);
|
|
2305
|
+
case "openai-compatible": return new OpenAICompatibleTranscriber(resolved);
|
|
407
2306
|
default: throw new InvalidSpeechAdapterError(resolved.type ?? "unknown", "STT");
|
|
408
2307
|
}
|
|
409
2308
|
}
|
|
@@ -418,7 +2317,13 @@ async function getSpeechSynthesizer(options, context = {}) {
|
|
|
418
2317
|
}
|
|
419
2318
|
function getAvailableSpeechAdapters() {
|
|
420
2319
|
return {
|
|
421
|
-
transcribers: [
|
|
2320
|
+
transcribers: [
|
|
2321
|
+
"studio-server",
|
|
2322
|
+
"openai-compatible",
|
|
2323
|
+
...registeredOptionalTranscribers(),
|
|
2324
|
+
...STREAMING_TRANSCRIBER_TYPES
|
|
2325
|
+
],
|
|
2326
|
+
streamingTranscribers: [...STREAMING_TRANSCRIBER_TYPES],
|
|
422
2327
|
synthesizers: [
|
|
423
2328
|
"studio-server",
|
|
424
2329
|
"qwen3-tts",
|
|
@@ -426,6 +2331,43 @@ function getAvailableSpeechAdapters() {
|
|
|
426
2331
|
]
|
|
427
2332
|
};
|
|
428
2333
|
}
|
|
2334
|
+
/**
|
|
2335
|
+
* Builds a streaming adapter from `getTranscriber()` options and wraps it as
|
|
2336
|
+
* a record-then-send `Transcriber`. Settings come from explicit options, then
|
|
2337
|
+
* `options.streaming`, then `HAVE_SPEECH_STREAMING_*`.
|
|
2338
|
+
*/
|
|
2339
|
+
function getWrappedStreamingTranscriber(type, options, context) {
|
|
2340
|
+
return wrapStreamingTranscriber(getStreamingTranscriber({
|
|
2341
|
+
...options.streaming,
|
|
2342
|
+
...compactJson({
|
|
2343
|
+
type,
|
|
2344
|
+
baseUrl: options.baseUrl,
|
|
2345
|
+
apiKey: options.apiKey,
|
|
2346
|
+
model: options.model,
|
|
2347
|
+
headers: options.headers,
|
|
2348
|
+
timeoutMs: options.timeoutMs,
|
|
2349
|
+
onUsage: options.onUsage
|
|
2350
|
+
})
|
|
2351
|
+
}, {
|
|
2352
|
+
env: context.env,
|
|
2353
|
+
headers: context.headers
|
|
2354
|
+
}), { maxBytes: options.maxBytes });
|
|
2355
|
+
}
|
|
2356
|
+
/**
|
|
2357
|
+
* Builds opt-in adapters (e.g. `local`) registered by their subpath entry.
|
|
2358
|
+
* Returns `undefined` for built-in types.
|
|
2359
|
+
*/
|
|
2360
|
+
async function getRegisteredTranscriber(options, context) {
|
|
2361
|
+
const type = resolveTranscriberConfig(options, context).type;
|
|
2362
|
+
const entry = type ? optionalTranscriberEntry(type) : void 0;
|
|
2363
|
+
if (!type || !entry) return;
|
|
2364
|
+
const factory = getOptionalTranscriberFactory(type);
|
|
2365
|
+
if (!factory) throw new SpeechConfigurationError(`The '${type}' transcriber is opt-in: import '${entry}' before calling getTranscriber()`, type);
|
|
2366
|
+
return factory({
|
|
2367
|
+
...options,
|
|
2368
|
+
type
|
|
2369
|
+
}, context);
|
|
2370
|
+
}
|
|
429
2371
|
async function getOptionalTranscriber(options, context) {
|
|
430
2372
|
if (!options && !hasTranscriberEnv(context.env ?? defaultEnv())) return;
|
|
431
2373
|
return getTranscriber(options, context);
|
|
@@ -435,20 +2377,30 @@ async function getOptionalSpeechSynthesizer(options, context) {
|
|
|
435
2377
|
return getSpeechSynthesizer(options, context);
|
|
436
2378
|
}
|
|
437
2379
|
function normalizeTranscriberOptions(options = {}, context) {
|
|
438
|
-
const
|
|
439
|
-
const type =
|
|
440
|
-
|
|
441
|
-
if (
|
|
442
|
-
|
|
2380
|
+
const resolved = resolveTranscriberConfig(options, context);
|
|
2381
|
+
const type = resolved.type ?? "studio-server";
|
|
2382
|
+
if (type !== "studio-server" && type !== "openai-compatible") throw new InvalidSpeechAdapterError(type, "STT");
|
|
2383
|
+
if (!resolved.baseUrl) throw new SpeechConfigurationError("STT baseUrl is required", type);
|
|
2384
|
+
const shared = {
|
|
2385
|
+
baseUrl: resolved.baseUrl,
|
|
2386
|
+
fetch: resolved.fetch,
|
|
2387
|
+
headers: resolved.headers,
|
|
2388
|
+
apiKey: resolved.apiKey,
|
|
2389
|
+
timeoutMs: resolved.timeoutMs
|
|
2390
|
+
};
|
|
2391
|
+
if (type === "openai-compatible") return {
|
|
2392
|
+
...shared,
|
|
2393
|
+
type,
|
|
2394
|
+
model: resolved.model,
|
|
2395
|
+
responseFormat: options.responseFormat,
|
|
2396
|
+
maxBytes: resolved.maxBytes,
|
|
2397
|
+
retry: options.retry,
|
|
2398
|
+
onUsage: options.onUsage
|
|
2399
|
+
};
|
|
443
2400
|
return {
|
|
444
|
-
...
|
|
2401
|
+
...shared,
|
|
445
2402
|
type,
|
|
446
|
-
|
|
447
|
-
fetch: options.fetch ?? context.fetch,
|
|
448
|
-
headers: options.headers ?? context.headers,
|
|
449
|
-
apiKey: options.apiKey ?? readEnv(env, "HAVE_SPEECH_STT_API_KEY", "STT_API_KEY", "SPEECH_API_KEY"),
|
|
450
|
-
timeoutMs: options.timeoutMs ?? parseOptionalInteger(readEnv(env, "HAVE_SPEECH_STT_TIMEOUT_MS", "STT_TIMEOUT_MS")),
|
|
451
|
-
transcribePath: options.transcribePath ?? readEnv(env, "HAVE_SPEECH_STT_PATH", "STT_PATH")
|
|
2403
|
+
transcribePath: options.transcribePath ?? resolved.path
|
|
452
2404
|
};
|
|
453
2405
|
}
|
|
454
2406
|
function normalizeSpeechSynthesizerOptions(options, context) {
|
|
@@ -488,30 +2440,141 @@ function normalizeSpeechSynthesizerOptions(options, context) {
|
|
|
488
2440
|
};
|
|
489
2441
|
throw new InvalidSpeechAdapterError(type, "TTS");
|
|
490
2442
|
}
|
|
491
|
-
function hasTranscriberEnv(env) {
|
|
492
|
-
return hasAnyEnv(env, "HAVE_SPEECH_STT_TYPE", "HAVE_SPEECH_STT_ADAPTER", "HAVE_SPEECH_STT_BASE_URL", "STT_ADAPTER", "STT_BASE_URL");
|
|
493
|
-
}
|
|
494
2443
|
function hasSynthesizerEnv(env) {
|
|
495
2444
|
return hasAnyEnv(env, "HAVE_SPEECH_TTS_TYPE", "HAVE_SPEECH_TTS_ADAPTER", "HAVE_SPEECH_TTS_BASE_URL", "TTS_ADAPTER", "TTS_BASE_URL");
|
|
496
2445
|
}
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
2446
|
+
//#endregion
|
|
2447
|
+
//#region src/shared/streaming-client-secret.ts
|
|
2448
|
+
/**
|
|
2449
|
+
* Server-side minting of short-lived browser credentials for realtime
|
|
2450
|
+
* transcription.
|
|
2451
|
+
*
|
|
2452
|
+
* A browser must never hold the long-lived provider key. Your server calls
|
|
2453
|
+
* {@link createStreamingClientSecret} once per tenant session (after its own
|
|
2454
|
+
* auth check), records the returned tenant/session attribution, and hands the
|
|
2455
|
+
* browser only `value` and `expiresAt`. The browser passes `value` as the
|
|
2456
|
+
* streaming adapter's `clientSecret`.
|
|
2457
|
+
*
|
|
2458
|
+
* - `openai-realtime`: `POST /v1/realtime/client_secrets` with
|
|
2459
|
+
* `expires_after: { anchor: 'created_at', seconds }` and a
|
|
2460
|
+
* `session.type: "transcription"` config, returning an `ek_…` secret
|
|
2461
|
+
* (https://developers.openai.com/api/reference/resources/realtime/subresources/client_secrets/methods/create).
|
|
2462
|
+
* - `voxtral-realtime`: vLLM has no ephemeral-token mechanism, so this throws
|
|
2463
|
+
* `SpeechConfigurationError`. Browser Voxtral needs your own proxy or token
|
|
2464
|
+
* gateway in front of vLLM.
|
|
2465
|
+
*/
|
|
2466
|
+
/** Default client-secret lifetime: long enough to connect, short if leaked. */
|
|
2467
|
+
var DEFAULT_CLIENT_SECRET_TTL_SECONDS = 60;
|
|
2468
|
+
/** OpenAI's accepted `expires_after.seconds` range. */
|
|
2469
|
+
var CLIENT_SECRET_TTL_RANGE = {
|
|
2470
|
+
min: 10,
|
|
2471
|
+
max: 7200
|
|
2472
|
+
};
|
|
2473
|
+
var OPENAI_CLIENT_SECRET_ENDPOINT = "realtime/client_secrets";
|
|
2474
|
+
/**
|
|
2475
|
+
* Mints a short-lived, session-scoped client secret for browser streaming.
|
|
2476
|
+
* Server-side only: throws in a browser runtime, and for adapters without an
|
|
2477
|
+
* ephemeral-token mechanism (`voxtral-realtime`).
|
|
2478
|
+
*/
|
|
2479
|
+
async function createStreamingClientSecret(options = {}, context = {}) {
|
|
2480
|
+
const env = context.env ?? defaultEnv();
|
|
2481
|
+
const keys = STREAMING_TRANSCRIBER_ENV_KEYS;
|
|
2482
|
+
const type = options.type ?? readEnv(env, ...keys.type) ?? "openai-realtime";
|
|
2483
|
+
if (!isStreamingTranscriberType(type)) throw new InvalidSpeechAdapterError(type, "streaming STT");
|
|
2484
|
+
if (isBrowserRuntime()) throw new SpeechConfigurationError("createStreamingClientSecret runs on your server: it needs the long-lived API key, which must never reach a browser", type);
|
|
2485
|
+
if (type === "voxtral-realtime") throw new SpeechConfigurationError("voxtral-realtime cannot mint browser credentials: vLLM has no ephemeral-token mechanism. Put your own proxy or token gateway in front of vLLM, mint its short-lived tokens yourself, and never send the vLLM API key to a browser", type);
|
|
2486
|
+
const ttlSeconds = options.ttlSeconds ?? 60;
|
|
2487
|
+
if (!Number.isInteger(ttlSeconds) || ttlSeconds < CLIENT_SECRET_TTL_RANGE.min || ttlSeconds > CLIENT_SECRET_TTL_RANGE.max) throw new SpeechConfigurationError(`ttlSeconds must be an integer from ${CLIENT_SECRET_TTL_RANGE.min} to ${CLIENT_SECRET_TTL_RANGE.max}`, type);
|
|
2488
|
+
const apiKey = options.apiKey ?? readEnv(env, ...keys.apiKey);
|
|
2489
|
+
if (!apiKey) throw new SpeechConfigurationError("createStreamingClientSecret requires apiKey (or HAVE_SPEECH_STREAMING_API_KEY)", type);
|
|
2490
|
+
const model = options.model?.trim() || readEnv(env, ...keys.model)?.trim() || "gpt-4o-transcribe";
|
|
2491
|
+
const session = openAIRealtimeTranscriptionSession({
|
|
2492
|
+
model,
|
|
2493
|
+
format: resolveOpenAIRealtimeFormat(options.format),
|
|
2494
|
+
turnDetection: options.turnDetection ?? parseTurnDetection(readEnv(env, ...keys.turnDetection)) ?? { type: "server_vad" },
|
|
2495
|
+
language: options.language ?? readEnv(env, ...keys.language),
|
|
2496
|
+
prompt: options.prompt
|
|
2497
|
+
}, options);
|
|
2498
|
+
const endpoint = resolveClientSecretUrl(options.baseUrl?.trim() || readEnv(env, ...keys.baseUrl)?.trim() || "wss://api.openai.com/v1/realtime");
|
|
2499
|
+
const headers = mergeHeaderInits(parseHeadersEnv(env, keys.headers), options.headers);
|
|
2500
|
+
const body = await new ClientSecretClient({
|
|
2501
|
+
baseUrl: endpoint,
|
|
2502
|
+
apiKey,
|
|
2503
|
+
fetch: options.fetch,
|
|
2504
|
+
headers,
|
|
2505
|
+
timeoutMs: options.timeoutMs ?? parseOptionalInteger(readEnv(env, ...keys.timeoutMs)) ?? 1e4
|
|
2506
|
+
}).mint(endpoint, {
|
|
2507
|
+
expires_after: {
|
|
2508
|
+
anchor: "created_at",
|
|
2509
|
+
seconds: ttlSeconds
|
|
2510
|
+
},
|
|
2511
|
+
session
|
|
2512
|
+
}, options.signal).catch((error) => {
|
|
2513
|
+
throw redactProviderError(error, headerSecrets(headers));
|
|
2514
|
+
});
|
|
2515
|
+
const value = body.value;
|
|
2516
|
+
if (typeof value !== "string" || !value) throw new SpeechProviderError(type, "openai-realtime client secret response did not include a value");
|
|
2517
|
+
const expiresAtSeconds = body.expires_at;
|
|
2518
|
+
return {
|
|
2519
|
+
type,
|
|
2520
|
+
value,
|
|
2521
|
+
expiresAt: typeof expiresAtSeconds === "number" && Number.isFinite(expiresAtSeconds) ? expiresAtSeconds * 1e3 : Date.now() + ttlSeconds * 1e3,
|
|
2522
|
+
ttlSeconds,
|
|
2523
|
+
model,
|
|
2524
|
+
tenantId: options.tenantId,
|
|
2525
|
+
sessionId: options.sessionId,
|
|
2526
|
+
metadata: options.metadata,
|
|
2527
|
+
session: body.session && typeof body.session === "object" ? body.session : void 0
|
|
2528
|
+
};
|
|
505
2529
|
}
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
2530
|
+
/**
|
|
2531
|
+
* Resolves the client-secret endpoint from any accepted realtime base:
|
|
2532
|
+
* `ws(s)` becomes `http(s)`, a trailing `/realtime` and query are dropped,
|
|
2533
|
+
* then `/v1/realtime/client_secrets` (or `/realtime/client_secrets` under a
|
|
2534
|
+
* versioned root) is appended.
|
|
2535
|
+
*/
|
|
2536
|
+
function resolveClientSecretUrl(baseUrl) {
|
|
2537
|
+
const url = new URL(baseUrl);
|
|
2538
|
+
if (url.protocol === "ws:") url.protocol = "http:";
|
|
2539
|
+
else if (url.protocol === "wss:") url.protocol = "https:";
|
|
2540
|
+
url.search = "";
|
|
2541
|
+
url.hash = "";
|
|
2542
|
+
url.pathname = url.pathname.replace(/\/+$/, "").replace(/\/realtime$/, "");
|
|
2543
|
+
return resolveOpenAICompatibleUrl(url.toString(), OPENAI_CLIENT_SECRET_ENDPOINT);
|
|
510
2544
|
}
|
|
511
|
-
|
|
512
|
-
|
|
2545
|
+
/**
|
|
2546
|
+
* The HTTP layer redacts the API key from provider error bodies; this also
|
|
2547
|
+
* removes gateway credentials sent in `headers` (e.g. `x-bf-vk`).
|
|
2548
|
+
*/
|
|
2549
|
+
function redactProviderError(error, secrets) {
|
|
2550
|
+
if (!(error instanceof SpeechProviderError) || secrets.length === 0) return error;
|
|
2551
|
+
let responseBody = error.responseBody;
|
|
2552
|
+
let message = error.message;
|
|
2553
|
+
for (const secret of secrets) {
|
|
2554
|
+
responseBody = redactSecret(responseBody, secret);
|
|
2555
|
+
message = redactSecret(message, secret) ?? message;
|
|
2556
|
+
}
|
|
2557
|
+
if (responseBody === error.responseBody && message === error.message) return error;
|
|
2558
|
+
return new SpeechProviderError(error.adapter ?? "openai-realtime", message, {
|
|
2559
|
+
status: error.status,
|
|
2560
|
+
responseBody,
|
|
2561
|
+
retryAfterMs: error.retryAfterMs
|
|
2562
|
+
});
|
|
513
2563
|
}
|
|
2564
|
+
var ClientSecretClient = class extends HttpSpeechAdapter {
|
|
2565
|
+
mint(endpoint, payload, signal) {
|
|
2566
|
+
return this.post("openai-realtime", endpoint, {
|
|
2567
|
+
body: JSON.stringify(payload),
|
|
2568
|
+
headers: { "content-type": "application/json" },
|
|
2569
|
+
signal
|
|
2570
|
+
}, async (response) => {
|
|
2571
|
+
const json = await response.json().catch(() => void 0);
|
|
2572
|
+
if (!json || typeof json !== "object" || Array.isArray(json)) throw new SpeechProviderError("openai-realtime", "openai-realtime client secret response was not a JSON object");
|
|
2573
|
+
return json;
|
|
2574
|
+
});
|
|
2575
|
+
}
|
|
2576
|
+
};
|
|
514
2577
|
//#endregion
|
|
515
|
-
export { InvalidSpeechAdapterError, SpeechConfigurationError, SpeechError, SpeechProviderError, getAvailableSpeechAdapters, getSpeech, getSpeechSynthesizer, getTranscriber };
|
|
2578
|
+
export { CLIENT_SECRET_TTL_RANGE, DEFAULT_CLIENT_SECRET_TTL_SECONDS, DEFAULT_MAX_AUDIO_BYTES, DEFAULT_MAX_TURN_SECONDS, DEFAULT_ROLLOVER_MIN_SILENCE_MS, DEFAULT_ROLLOVER_SILENCE_THRESHOLD, DEFAULT_ROLLOVER_WINDOW_SECONDS, InvalidSpeechAdapterError, OPENAI_REALTIME_DEFAULT_MODEL, OPENAI_REALTIME_DEFAULT_URL, RealtimeTranscriptionSession, STREAMING_TRANSCRIBER_ENV_KEYS, STREAMING_TRANSCRIBER_TYPES, SpeechConfigurationError, SpeechError, SpeechProviderError, TRANSCRIBER_ENV_KEYS, VOXTRAL_REALTIME_AUDIO_FORMAT, VOXTRAL_REALTIME_DEFAULT_MODEL, audioSecondsForBytes, bytesPerSecond, createStreamingClientSecret, getAvailableSpeechAdapters, getSpeech, getSpeechSynthesizer, getStreamingTranscriber, getTranscriber, isBrowserRuntime, isStreamingTranscriberType, mimeTypeToAudioExtension, openAIRealtimeProtocol, openAIRealtimeTranscriptionSession, parseMaxTurnSeconds, parseOpenAIRealtimeEvent, parseVoxtralRealtimeEvent, resolveClientSecretUrl, resolveOpenAIRealtimeFormat, resolveRealtimeUrl, resolveVoxtralRealtimeUrl, unwrapRawAudio, voxtralRealtimeProtocol, wrapStreamingTranscriber };
|
|
516
2579
|
|
|
517
2580
|
//# sourceMappingURL=index.js.map
|