@providerkit/core 0.27.2 → 0.28.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -0
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -1
- package/dist/index.js.map +1 -1
- package/dist/presets.d.ts +50 -0
- package/dist/presets.d.ts.map +1 -1
- package/dist/presets.js +82 -0
- package/dist/presets.js.map +1 -1
- package/dist/providers/gemini-speech.d.ts +25 -0
- package/dist/providers/gemini-speech.d.ts.map +1 -0
- package/dist/providers/gemini-speech.js +172 -0
- package/dist/providers/gemini-speech.js.map +1 -0
- package/dist/providers/openai-speech.d.ts +23 -0
- package/dist/providers/openai-speech.d.ts.map +1 -0
- package/dist/providers/openai-speech.js +86 -0
- package/dist/providers/openai-speech.js.map +1 -0
- package/dist/providers/speech-factory.d.ts +35 -0
- package/dist/providers/speech-factory.d.ts.map +1 -0
- package/dist/providers/speech-factory.js +47 -0
- package/dist/providers/speech-factory.js.map +1 -0
- package/dist/speech-core.d.ts +184 -0
- package/dist/speech-core.d.ts.map +1 -0
- package/dist/speech-core.js +349 -0
- package/dist/speech-core.js.map +1 -0
- package/dist/speech.d.ts +6 -0
- package/dist/speech.d.ts.map +1 -0
- package/dist/speech.js +12 -0
- package/dist/speech.js.map +1 -0
- package/dist/transport.d.ts +42 -0
- package/dist/transport.d.ts.map +1 -1
- package/dist/transport.js +154 -11
- package/dist/transport.js.map +1 -1
- package/package.json +6 -1
- package/src/index.ts +14 -1
- package/src/presets.ts +113 -0
- package/src/providers/gemini-speech.ts +232 -0
- package/src/providers/openai-speech.ts +125 -0
- package/src/providers/speech-factory.ts +85 -0
- package/src/speech-core.ts +507 -0
- package/src/speech.ts +24 -0
- package/src/transport.ts +185 -11
|
@@ -0,0 +1,507 @@
|
|
|
1
|
+
// Speech: text in, one audio clip out. A second kind of client next to `Provider`,
|
|
2
|
+
// reached through `@providerkit/core/speech` so a chat-only app never loads it.
|
|
3
|
+
//
|
|
4
|
+
// This file holds what both wires share: the types, the error, and the byte-level
|
|
5
|
+
// checks. Two rules run through all of it.
|
|
6
|
+
//
|
|
7
|
+
// 1. A paid synthesis is never replayed. A speech reply arrives whole, so there is
|
|
8
|
+
// no "nothing was emitted yet" the way a chat stream has: a timeout or a dropped
|
|
9
|
+
// socket may land after the vendor started billing. One request per call.
|
|
10
|
+
// 2. Audio is named by its own bytes, not by what was asked for. Many servers ignore
|
|
11
|
+
// the format they were sent, and "WebM with an MP4 name" is not MP4.
|
|
12
|
+
import { ProviderError, streamError, type ErrorKind } from "./errors.ts";
|
|
13
|
+
import { withDeadline } from "./transport.ts";
|
|
14
|
+
import type { TokenUsage } from "./types.ts";
|
|
15
|
+
|
|
16
|
+
export const SPEECH_FORMATS = ["wav", "pcm", "mp3", "opus", "aac", "flac"] as const;
|
|
17
|
+
export type SpeechFormat = (typeof SPEECH_FORMATS)[number];
|
|
18
|
+
|
|
19
|
+
/** Which wire speaks. A third shape is one new file and one preset row. */
|
|
20
|
+
export type SpeechShape = "openai" | "gemini";
|
|
21
|
+
|
|
22
|
+
export interface SpeechRequest {
|
|
23
|
+
/** What to say. Trimmed first. Empty is refused before any request. */
|
|
24
|
+
text: string;
|
|
25
|
+
/** The voice as the endpoint spells it ("Kore", "alloy"). Absent: the provider's default. */
|
|
26
|
+
voice?: string;
|
|
27
|
+
/** Absent: the provider's default format, else "wav". */
|
|
28
|
+
format?: SpeechFormat;
|
|
29
|
+
/** The caller's Stop. Never retried and never turned into a timeout. */
|
|
30
|
+
signal?: AbortSignal;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/** Token counts the provider stated. `totalTokens` can exceed input plus output:
|
|
34
|
+
* the gap is what the provider counts and does not break out. */
|
|
35
|
+
export interface SpeechUsage extends TokenUsage {
|
|
36
|
+
totalTokens?: number;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export interface SpeechResult {
|
|
40
|
+
bytes: Uint8Array;
|
|
41
|
+
/** Read from the bytes, so it can differ from the format asked for. */
|
|
42
|
+
format: SpeechFormat;
|
|
43
|
+
/** The media type of `format`. */
|
|
44
|
+
mimeType: string;
|
|
45
|
+
/** Hz. Only when the bytes (a WAV header) or the request state it. */
|
|
46
|
+
sampleRate?: number;
|
|
47
|
+
/** Only when the bytes or the provider state it. */
|
|
48
|
+
channels?: number;
|
|
49
|
+
/** Exact, from a WAV header that checks out. Absent for every other format:
|
|
50
|
+
* measuring those takes a decoder, and a guess here would be wrong in the ledger. */
|
|
51
|
+
durationMs?: number;
|
|
52
|
+
/** Characters sent after trimming, for endpoints billed per character. */
|
|
53
|
+
characters: number;
|
|
54
|
+
/** Only when the provider reported it. Never estimated. */
|
|
55
|
+
usage?: SpeechUsage;
|
|
56
|
+
/** The model and voice that were asked for. */
|
|
57
|
+
model: string;
|
|
58
|
+
voice: string;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
export interface SpeechProvider {
|
|
62
|
+
readonly id: string;
|
|
63
|
+
readonly model: string;
|
|
64
|
+
/**
|
|
65
|
+
* Exactly one HTTP request. No retry, no fallback, no second key. Throws a
|
|
66
|
+
* `ProviderError`; a reply that arrived but holds no usable audio throws a
|
|
67
|
+
* `SpeechError`, which keeps the usage the provider billed for it.
|
|
68
|
+
*/
|
|
69
|
+
synthesize(request: SpeechRequest): Promise<SpeechResult>;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
export interface SpeechLimits {
|
|
73
|
+
/** Longest `text`, in UTF-16 units (JavaScript's `.length`). Default 4,096.
|
|
74
|
+
* Over it is refused before any request. Never silently cut. */
|
|
75
|
+
maxInputChars?: number;
|
|
76
|
+
/** Largest audio accepted, in bytes. Default 32 MiB. */
|
|
77
|
+
maxAudioBytes?: number;
|
|
78
|
+
/** One deadline for the whole call, headers and body. Default 120,000. */
|
|
79
|
+
timeoutMs?: number;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
export const DEFAULT_SPEECH_LIMITS = {
|
|
83
|
+
maxInputChars: 4_096,
|
|
84
|
+
maxAudioBytes: 32 * 1024 * 1024,
|
|
85
|
+
timeoutMs: 120_000,
|
|
86
|
+
} as const satisfies Required<SpeechLimits>;
|
|
87
|
+
|
|
88
|
+
type ProviderErrorOptions = NonNullable<ConstructorParameters<typeof ProviderError>[3]>;
|
|
89
|
+
|
|
90
|
+
/**
|
|
91
|
+
* A reply that arrived but holds no usable audio, or a failure the provider
|
|
92
|
+
* reported inside a 200. Carries the usage the provider stated for it, because it
|
|
93
|
+
* billed that: a caller that drops this loses the only record of the charge.
|
|
94
|
+
* Plain `ProviderError`s carry no usage.
|
|
95
|
+
*/
|
|
96
|
+
export class SpeechError extends ProviderError {
|
|
97
|
+
readonly usage?: SpeechUsage;
|
|
98
|
+
|
|
99
|
+
constructor(
|
|
100
|
+
provider: string,
|
|
101
|
+
kind: ErrorKind,
|
|
102
|
+
message: string,
|
|
103
|
+
opts: ProviderErrorOptions & { usage?: SpeechUsage } = {},
|
|
104
|
+
) {
|
|
105
|
+
const { usage, ...rest } = opts;
|
|
106
|
+
super(provider, kind, message, rest);
|
|
107
|
+
this.name = "SpeechError";
|
|
108
|
+
this.usage = usage;
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/** @internal Positive integers or a RangeError, at build time, never mid-call. */
|
|
113
|
+
export function resolveLimits(config: SpeechLimits): Required<SpeechLimits> {
|
|
114
|
+
const limits = {
|
|
115
|
+
maxInputChars: config.maxInputChars ?? DEFAULT_SPEECH_LIMITS.maxInputChars,
|
|
116
|
+
maxAudioBytes: config.maxAudioBytes ?? DEFAULT_SPEECH_LIMITS.maxAudioBytes,
|
|
117
|
+
timeoutMs: config.timeoutMs ?? DEFAULT_SPEECH_LIMITS.timeoutMs,
|
|
118
|
+
};
|
|
119
|
+
for (const [name, value] of Object.entries(limits)) {
|
|
120
|
+
if (!Number.isSafeInteger(value) || value < 1) {
|
|
121
|
+
throw new RangeError(`[providerkit] ${name} must be a positive integer`);
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
return limits;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
const refuse = (provider: string, code: string, message: string, kind: ErrorKind = "invalid") =>
|
|
128
|
+
new ProviderError(provider, kind, `${provider}: ${message}`, { code, shouldRetry: false });
|
|
129
|
+
|
|
130
|
+
/** Space to tilde. Anything else in a header value is refused or altered by a runtime. */
|
|
131
|
+
const HEADER_SAFE = /^[\x20-\x7e]*$/;
|
|
132
|
+
|
|
133
|
+
/**
|
|
134
|
+
* @internal Trim, bound and fill in defaults. Throws BEFORE any request, and never
|
|
135
|
+
* puts the text or the key in the message: they are the caller's, and may be private.
|
|
136
|
+
*/
|
|
137
|
+
export function prepareSpeech(
|
|
138
|
+
provider: string,
|
|
139
|
+
request: SpeechRequest,
|
|
140
|
+
defaults: {
|
|
141
|
+
/** Checked, never echoed. A key with a line break, a non-Latin-1 character or a zero-width
|
|
142
|
+
* one pasted in makes the runtime reject the header and quote the value in its error. */
|
|
143
|
+
apiKey: string;
|
|
144
|
+
voice?: string;
|
|
145
|
+
format?: SpeechFormat;
|
|
146
|
+
maxInputChars: number;
|
|
147
|
+
/** What this wire can make. */
|
|
148
|
+
formats: readonly SpeechFormat[];
|
|
149
|
+
},
|
|
150
|
+
): { text: string; voice: string; format: SpeechFormat } {
|
|
151
|
+
if (!HEADER_SAFE.test(defaults.apiKey)) {
|
|
152
|
+
throw refuse(
|
|
153
|
+
provider,
|
|
154
|
+
"key_invalid",
|
|
155
|
+
"the API key holds a character that cannot be sent in a header (a line break or a character outside plain ASCII). Paste the key again",
|
|
156
|
+
"auth",
|
|
157
|
+
);
|
|
158
|
+
}
|
|
159
|
+
const text = typeof request.text === "string" ? request.text.trim() : "";
|
|
160
|
+
if (!text) throw refuse(provider, "text_empty", "there is no text to speak");
|
|
161
|
+
if (text.length > defaults.maxInputChars) {
|
|
162
|
+
throw refuse(
|
|
163
|
+
provider,
|
|
164
|
+
"text_too_long",
|
|
165
|
+
`the text is ${text.length} characters and the limit is ${defaults.maxInputChars}`,
|
|
166
|
+
);
|
|
167
|
+
}
|
|
168
|
+
const voice = (request.voice ?? defaults.voice)?.trim();
|
|
169
|
+
if (!voice) throw refuse(provider, "voice_required", "this provider needs a voice name");
|
|
170
|
+
const format = request.format ?? defaults.format ?? "wav";
|
|
171
|
+
if (!defaults.formats.includes(format)) {
|
|
172
|
+
throw refuse(
|
|
173
|
+
provider,
|
|
174
|
+
"format_unsupported",
|
|
175
|
+
`this provider cannot make ${String(format)}. It makes ${defaults.formats.join(", ")}`,
|
|
176
|
+
);
|
|
177
|
+
}
|
|
178
|
+
return { text, voice, format };
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
const REPLAY_UNSAFE: ReadonlySet<ErrorKind> = new Set<ErrorKind>([
|
|
182
|
+
"network",
|
|
183
|
+
"timeout",
|
|
184
|
+
"overload",
|
|
185
|
+
"unknown",
|
|
186
|
+
]);
|
|
187
|
+
|
|
188
|
+
function neverRetry(err: ProviderError): ProviderError {
|
|
189
|
+
const opts = {
|
|
190
|
+
status: err.status,
|
|
191
|
+
code: err.code,
|
|
192
|
+
retryAfterMs: err.retryAfterMs,
|
|
193
|
+
resetAtMs: err.resetAtMs,
|
|
194
|
+
window: err.window,
|
|
195
|
+
shouldRetry: false,
|
|
196
|
+
body: err.body,
|
|
197
|
+
cause: err.cause,
|
|
198
|
+
};
|
|
199
|
+
return err instanceof SpeechError
|
|
200
|
+
? new SpeechError(err.provider, err.kind, err.message, { ...opts, usage: err.usage })
|
|
201
|
+
: new ProviderError(err.provider, err.kind, err.message, opts);
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
/**
|
|
205
|
+
* @internal The "could the vendor already have billed this?" rule, applied in one
|
|
206
|
+
* place so a new failure path cannot forget it.
|
|
207
|
+
*
|
|
208
|
+
* - The caller's Stop comes back untouched.
|
|
209
|
+
* - A timeout, a dead socket, a 5xx or anything unclassified may have landed after
|
|
210
|
+
* the vendor started work, so it is marked `shouldRetry: false`, and a vendor's
|
|
211
|
+
* `x-should-retry: true` does not undo that: the header says the vendor thinks a
|
|
212
|
+
* resend is fine, not that the first call was not billed.
|
|
213
|
+
* - Every other kind keeps what it came with. A 429 means the vendor refused before
|
|
214
|
+
* doing any work, so sending it again is safe, and `Retry-After` is honoured by
|
|
215
|
+
* the caller's own retry. `x-should-retry` is believed there, in either direction.
|
|
216
|
+
*/
|
|
217
|
+
export function noReplay(provider: string, error: unknown, signal?: AbortSignal): unknown {
|
|
218
|
+
if (signal?.aborted) return error;
|
|
219
|
+
const err = ProviderError.from(provider, error);
|
|
220
|
+
return REPLAY_UNSAFE.has(err.kind) && err.shouldRetry !== false ? neverRetry(err) : err;
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
/**
|
|
224
|
+
* @internal The one place a call runs: under one deadline, once, and with the
|
|
225
|
+
* no-replay rule applied to whatever it throws. Both wires go through here.
|
|
226
|
+
*/
|
|
227
|
+
export async function synthesizeOnce<T>(
|
|
228
|
+
provider: string,
|
|
229
|
+
timeoutMs: number,
|
|
230
|
+
request: SpeechRequest,
|
|
231
|
+
call: (signal: AbortSignal) => Promise<T>,
|
|
232
|
+
): Promise<T> {
|
|
233
|
+
try {
|
|
234
|
+
return await withDeadline(provider, timeoutMs, request.signal, call);
|
|
235
|
+
} catch (error) {
|
|
236
|
+
throw noReplay(provider, error, request.signal);
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
// ── failures ──────────────────────────────────────────────────────────────
|
|
241
|
+
|
|
242
|
+
/** @internal A reply that arrived but cannot be used. Never retried: it was billed. */
|
|
243
|
+
export function unusable(
|
|
244
|
+
provider: string,
|
|
245
|
+
kind: ErrorKind,
|
|
246
|
+
code: string,
|
|
247
|
+
message: string,
|
|
248
|
+
usage?: SpeechUsage,
|
|
249
|
+
): SpeechError {
|
|
250
|
+
return new SpeechError(provider, kind, `${provider}: ${message}`, {
|
|
251
|
+
code,
|
|
252
|
+
shouldRetry: false,
|
|
253
|
+
...(usage ? { usage } : {}),
|
|
254
|
+
});
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
/**
|
|
258
|
+
* @internal A failure the provider reported inside a 200 body (a gateway that
|
|
259
|
+
* answers 200 with a JSON error, an Interactions reply that says it failed). It is
|
|
260
|
+
* classified like any other failure, with the same words-over-status rules, and
|
|
261
|
+
* then the provider's words are dropped: they can echo the text that was sent.
|
|
262
|
+
*/
|
|
263
|
+
export function inBandFailure(
|
|
264
|
+
provider: string,
|
|
265
|
+
envelope: unknown,
|
|
266
|
+
usage?: SpeechUsage,
|
|
267
|
+
): SpeechError {
|
|
268
|
+
const found = streamError(provider, envelope);
|
|
269
|
+
return new SpeechError(provider, found.kind, `${provider}: the provider reported ${found.kind}`, {
|
|
270
|
+
status: found.status,
|
|
271
|
+
retryAfterMs: found.retryAfterMs,
|
|
272
|
+
resetAtMs: found.resetAtMs,
|
|
273
|
+
window: found.window,
|
|
274
|
+
...(usage ? { usage } : {}),
|
|
275
|
+
});
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
/** @internal */
|
|
279
|
+
export function asRecord(value: unknown): Record<string, unknown> | undefined {
|
|
280
|
+
return typeof value === "object" && value !== null && !Array.isArray(value)
|
|
281
|
+
? (value as Record<string, unknown>)
|
|
282
|
+
: undefined;
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
// ── bytes ─────────────────────────────────────────────────────────────────
|
|
286
|
+
|
|
287
|
+
/**
|
|
288
|
+
* @internal Strict base64 to bytes: the length is a multiple of 4, the alphabet is
|
|
289
|
+
* the standard one, padding sits only at the end. `atob` alone is too forgiving:
|
|
290
|
+
* it drops whitespace and accepts missing padding. `undefined` on anything else.
|
|
291
|
+
*/
|
|
292
|
+
export function decodeBase64(data: string): Uint8Array | undefined {
|
|
293
|
+
if (data.length === 0 || data.length % 4 !== 0 || !/^[A-Za-z0-9+/]+={0,2}$/.test(data)) {
|
|
294
|
+
return undefined;
|
|
295
|
+
}
|
|
296
|
+
let binary: string;
|
|
297
|
+
try {
|
|
298
|
+
binary = atob(data);
|
|
299
|
+
} catch {
|
|
300
|
+
return undefined;
|
|
301
|
+
}
|
|
302
|
+
const bytes = new Uint8Array(binary.length);
|
|
303
|
+
for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i);
|
|
304
|
+
return bytes;
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
function ascii(bytes: Uint8Array, at: number, length: number): string {
|
|
308
|
+
if (at < 0 || at + length > bytes.byteLength) return "";
|
|
309
|
+
let out = "";
|
|
310
|
+
for (let i = 0; i < length; i++) out += String.fromCharCode(bytes[at + i]!);
|
|
311
|
+
return out;
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
/** The first three bytes of an MPEG audio frame: Layer III is mp3, and a zero layer
|
|
315
|
+
* field behind twelve sync bits is AAC in ADTS. Layer I and II are neither. */
|
|
316
|
+
function mpegAudio(
|
|
317
|
+
b0: number | undefined,
|
|
318
|
+
b1: number | undefined,
|
|
319
|
+
b2: number | undefined,
|
|
320
|
+
): "mp3" | "aac" | undefined {
|
|
321
|
+
if (b0 !== 0xff || b1 === undefined || b2 === undefined || (b1 & 0xe0) !== 0xe0) {
|
|
322
|
+
return undefined;
|
|
323
|
+
}
|
|
324
|
+
const layer = (b1 >> 1) & 3;
|
|
325
|
+
if (layer === 0) return (b1 & 0xf0) === 0xf0 && ((b2 >> 2) & 15) < 13 ? "aac" : undefined;
|
|
326
|
+
const reservedVersion = ((b1 >> 3) & 3) === 1;
|
|
327
|
+
const reservedBitrate = b2 >> 4 === 15;
|
|
328
|
+
const reservedRate = ((b2 >> 2) & 3) === 3;
|
|
329
|
+
return layer === 1 && !reservedVersion && !reservedBitrate && !reservedRate ? "mp3" : undefined;
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
/** A format with a signature in its first bytes. Raw PCM has none. */
|
|
333
|
+
type ContainerFormat = Exclude<SpeechFormat, "pcm">;
|
|
334
|
+
|
|
335
|
+
/**
|
|
336
|
+
* @internal Which container the first bytes say this is, or `undefined`. Raw PCM
|
|
337
|
+
* has no signature, so it is never returned here. ADTS AAC and mp3 share a sync
|
|
338
|
+
* pattern and are told apart by their layer field.
|
|
339
|
+
*/
|
|
340
|
+
export function sniffFormat(bytes: Uint8Array): ContainerFormat | undefined {
|
|
341
|
+
if (ascii(bytes, 0, 4) === "RIFF" && ascii(bytes, 8, 4) === "WAVE") return "wav";
|
|
342
|
+
if (ascii(bytes, 0, 4) === "fLaC") return "flac";
|
|
343
|
+
if (ascii(bytes, 0, 4) === "OggS") {
|
|
344
|
+
// The first packet starts after the page's 27-byte header and its segment table.
|
|
345
|
+
return ascii(bytes, 27 + (bytes[26] ?? 0), 8) === "OpusHead" ? "opus" : undefined;
|
|
346
|
+
}
|
|
347
|
+
let at = 0;
|
|
348
|
+
if (ascii(bytes, 0, 3) === "ID3" && bytes.byteLength >= 10) {
|
|
349
|
+
// An ID3v2 tag in front of the audio: a 10-byte header whose size is four
|
|
350
|
+
// 7-bit bytes, plus a 10-byte footer when flag bit 4 says there is one.
|
|
351
|
+
const size =
|
|
352
|
+
((bytes[6]! & 0x7f) << 21) |
|
|
353
|
+
((bytes[7]! & 0x7f) << 14) |
|
|
354
|
+
((bytes[8]! & 0x7f) << 7) |
|
|
355
|
+
(bytes[9]! & 0x7f);
|
|
356
|
+
at = 10 + size + (bytes[5]! & 0x10 ? 10 : 0);
|
|
357
|
+
}
|
|
358
|
+
return mpegAudio(bytes[at], bytes[at + 1], bytes[at + 2]);
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
export interface WavInfo {
|
|
362
|
+
sampleRate: number;
|
|
363
|
+
channels: number;
|
|
364
|
+
bitsPerSample: number;
|
|
365
|
+
/** Samples per channel. */
|
|
366
|
+
frames: number;
|
|
367
|
+
durationMs: number;
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
/**
|
|
371
|
+
* Read a WAV file's own numbers, strictly. PCM only (format tag 1). The RIFF size
|
|
372
|
+
* is exact, every chunk sits inside the file, there is one `fmt ` before one
|
|
373
|
+
* non-empty `data` that holds whole frames, and nothing trails. Extra chunks such
|
|
374
|
+
* as `LIST` are skipped, and odd-sized chunks carry their pad byte. `undefined`
|
|
375
|
+
* on anything else. It reports what the bytes say; whether 24 kHz mono is
|
|
376
|
+
* acceptable is the caller's policy.
|
|
377
|
+
*/
|
|
378
|
+
export function readWav(bytes: Uint8Array): WavInfo | undefined {
|
|
379
|
+
if (ascii(bytes, 0, 4) !== "RIFF" || ascii(bytes, 8, 4) !== "WAVE") return undefined;
|
|
380
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
381
|
+
if (view.getUint32(4, true) !== bytes.byteLength - 8) return undefined;
|
|
382
|
+
let fmt: (Omit<WavInfo, "frames" | "durationMs"> & { blockAlign: number }) | undefined;
|
|
383
|
+
let dataBytes: number | undefined;
|
|
384
|
+
let offset = 12;
|
|
385
|
+
let chunks = 0;
|
|
386
|
+
while (offset < bytes.byteLength) {
|
|
387
|
+
if (++chunks > 1024 || bytes.byteLength - offset < 8) return undefined;
|
|
388
|
+
const name = ascii(bytes, offset, 4);
|
|
389
|
+
const size = view.getUint32(offset + 4, true);
|
|
390
|
+
const start = offset + 8;
|
|
391
|
+
const paddedEnd = start + size + (size % 2);
|
|
392
|
+
if (start + size > bytes.byteLength || paddedEnd > bytes.byteLength) return undefined;
|
|
393
|
+
if (name === "fmt ") {
|
|
394
|
+
if (fmt || (size !== 16 && size !== 18)) return undefined;
|
|
395
|
+
const channels = view.getUint16(start + 2, true);
|
|
396
|
+
const sampleRate = view.getUint32(start + 4, true);
|
|
397
|
+
const blockAlign = view.getUint16(start + 12, true);
|
|
398
|
+
const bitsPerSample = view.getUint16(start + 14, true);
|
|
399
|
+
if (
|
|
400
|
+
view.getUint16(start, true) !== 1 ||
|
|
401
|
+
channels < 1 ||
|
|
402
|
+
sampleRate < 1 ||
|
|
403
|
+
bitsPerSample === 0 ||
|
|
404
|
+
bitsPerSample % 8 !== 0 ||
|
|
405
|
+
blockAlign !== (channels * bitsPerSample) / 8 ||
|
|
406
|
+
view.getUint32(start + 8, true) !== sampleRate * blockAlign ||
|
|
407
|
+
(size === 18 && view.getUint16(start + 16, true) !== 0)
|
|
408
|
+
) {
|
|
409
|
+
return undefined;
|
|
410
|
+
}
|
|
411
|
+
fmt = { sampleRate, channels, bitsPerSample, blockAlign };
|
|
412
|
+
} else if (name === "data") {
|
|
413
|
+
if (!fmt || dataBytes !== undefined || size === 0 || size % fmt.blockAlign !== 0) {
|
|
414
|
+
return undefined;
|
|
415
|
+
}
|
|
416
|
+
dataBytes = size;
|
|
417
|
+
} else if (name === "RIFF" || name === "RIFX" || name === "RF64") {
|
|
418
|
+
return undefined;
|
|
419
|
+
}
|
|
420
|
+
offset = paddedEnd;
|
|
421
|
+
}
|
|
422
|
+
if (!fmt || dataBytes === undefined) return undefined;
|
|
423
|
+
const frames = dataBytes / fmt.blockAlign;
|
|
424
|
+
const { blockAlign: _blockAlign, ...info } = fmt;
|
|
425
|
+
return { ...info, frames, durationMs: (frames / fmt.sampleRate) * 1000 };
|
|
426
|
+
}
|
|
427
|
+
|
|
428
|
+
const MIME: Record<ContainerFormat, string> = {
|
|
429
|
+
wav: "audio/wav",
|
|
430
|
+
mp3: "audio/mpeg",
|
|
431
|
+
opus: "audio/ogg; codecs=opus",
|
|
432
|
+
aac: "audio/aac",
|
|
433
|
+
flac: "audio/flac",
|
|
434
|
+
};
|
|
435
|
+
|
|
436
|
+
/** @internal Everything a result needs besides the audio itself. */
|
|
437
|
+
export interface AudioExtras {
|
|
438
|
+
model: string;
|
|
439
|
+
voice: string;
|
|
440
|
+
characters: number;
|
|
441
|
+
usage?: SpeechUsage;
|
|
442
|
+
/** What the request or the provider stated, for raw PCM only. */
|
|
443
|
+
sampleRate?: number;
|
|
444
|
+
channels?: number;
|
|
445
|
+
}
|
|
446
|
+
|
|
447
|
+
/**
|
|
448
|
+
* @internal Bytes in, `SpeechResult` out. An empty body is `empty_audio`. A
|
|
449
|
+
* recognised container is accepted whatever was asked for, and `format` says what
|
|
450
|
+
* it really is. Raw PCM is the one case with no signature to read, so it is
|
|
451
|
+
* accepted only when PCM was asked for and the length is whole samples. Anything
|
|
452
|
+
* else is `bad_audio`.
|
|
453
|
+
*
|
|
454
|
+
* A WAV is labelled by its signature. Its numbers (rate, channels, duration) are
|
|
455
|
+
* filled in only when the header checks out, so a server whose header lies still
|
|
456
|
+
* gets its audio through, without a duration anyone could take as measured.
|
|
457
|
+
*/
|
|
458
|
+
export function describeAudio(
|
|
459
|
+
provider: string,
|
|
460
|
+
bytes: Uint8Array,
|
|
461
|
+
wanted: SpeechFormat,
|
|
462
|
+
extras: AudioExtras,
|
|
463
|
+
): SpeechResult {
|
|
464
|
+
if (bytes.byteLength === 0) {
|
|
465
|
+
throw unusable(provider, "overload", "empty_audio", "the reply holds no audio", extras.usage);
|
|
466
|
+
}
|
|
467
|
+
const base = {
|
|
468
|
+
bytes,
|
|
469
|
+
characters: extras.characters,
|
|
470
|
+
model: extras.model,
|
|
471
|
+
voice: extras.voice,
|
|
472
|
+
...(extras.usage ? { usage: extras.usage } : {}),
|
|
473
|
+
};
|
|
474
|
+
const sniffed = sniffFormat(bytes);
|
|
475
|
+
if (sniffed === "wav") {
|
|
476
|
+
const wav = readWav(bytes);
|
|
477
|
+
return {
|
|
478
|
+
...base,
|
|
479
|
+
format: "wav",
|
|
480
|
+
mimeType: MIME.wav,
|
|
481
|
+
...(wav
|
|
482
|
+
? { sampleRate: wav.sampleRate, channels: wav.channels, durationMs: wav.durationMs }
|
|
483
|
+
: {}),
|
|
484
|
+
};
|
|
485
|
+
}
|
|
486
|
+
if (sniffed) return { ...base, format: sniffed, mimeType: MIME[sniffed] };
|
|
487
|
+
if (wanted !== "pcm" || bytes.byteLength % 2 !== 0) {
|
|
488
|
+
throw unusable(
|
|
489
|
+
provider,
|
|
490
|
+
"unknown",
|
|
491
|
+
"bad_audio",
|
|
492
|
+
"the reply is not audio this package recognises",
|
|
493
|
+
extras.usage,
|
|
494
|
+
);
|
|
495
|
+
}
|
|
496
|
+
const { sampleRate, channels } = extras;
|
|
497
|
+
return {
|
|
498
|
+
...base,
|
|
499
|
+
format: "pcm",
|
|
500
|
+
mimeType: sampleRate ? `audio/L16;rate=${sampleRate}` : "audio/L16",
|
|
501
|
+
...(sampleRate ? { sampleRate } : {}),
|
|
502
|
+
...(channels ? { channels } : {}),
|
|
503
|
+
...(sampleRate && channels
|
|
504
|
+
? { durationMs: (bytes.byteLength / 2 / channels / sampleRate) * 1000 }
|
|
505
|
+
: {}),
|
|
506
|
+
};
|
|
507
|
+
}
|
package/src/speech.ts
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
// `@providerkit/core/speech`: text to speech, a second kind of client next to the chat
|
|
2
|
+
// `Provider`. It is its own entry point, and `index.ts` does not export it, so an app
|
|
3
|
+
// that only chats loads none of the speech code. Same reason `decisions.ts` stands apart.
|
|
4
|
+
//
|
|
5
|
+
// Named, not `export *`: speech-core.ts also holds helpers the two wires share, and a
|
|
6
|
+
// helper exported here is one this package can never change without a major version.
|
|
7
|
+
export {
|
|
8
|
+
DEFAULT_SPEECH_LIMITS,
|
|
9
|
+
readWav,
|
|
10
|
+
SPEECH_FORMATS,
|
|
11
|
+
SpeechError,
|
|
12
|
+
type SpeechFormat,
|
|
13
|
+
type SpeechLimits,
|
|
14
|
+
type SpeechProvider,
|
|
15
|
+
type SpeechRequest,
|
|
16
|
+
type SpeechResult,
|
|
17
|
+
type SpeechShape,
|
|
18
|
+
type SpeechUsage,
|
|
19
|
+
type WavInfo,
|
|
20
|
+
} from "./speech-core.ts";
|
|
21
|
+
export * from "./providers/openai-speech.ts";
|
|
22
|
+
export * from "./providers/gemini-speech.ts";
|
|
23
|
+
export * from "./providers/speech-factory.ts";
|
|
24
|
+
export { SPEECH_PRESET_IDS, type PresetSpeech } from "./presets.ts";
|