@sixb/azure-ai-foundry 0.2.4 → 0.2.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/speech.ts ADDED
@@ -0,0 +1,390 @@
1
+ import {
2
+ defineModelRateCard,
3
+ type JsonObject,
4
+ type ModelRateCard,
5
+ type ModelUsage,
6
+ rateModelCall,
7
+ type SpeechCostEstimator,
8
+ type SpeechModel,
9
+ type SpeechModelDefinition,
10
+ type SpeechModelRequest,
11
+ SpeechModelResponseError,
12
+ type SpeechModelResponseMetadata,
13
+ type SpeechModelResult,
14
+ UnsupportedModelFeatureError,
15
+ } from "@sixb/core/models"
16
+ import {
17
+ type AzureAIFoundryDeployment,
18
+ deploymentMetadata,
19
+ type FoundryDeployments,
20
+ type ResolvedDeployment,
21
+ } from "./discovery"
22
+ import { assertDeploymentKind } from "./kinds"
23
+ import type { AzureAIFoundryModelMetadata } from "./provider"
24
+ import { azurePublisher } from "./publisher"
25
+ import { abortable, type FoundryTransport, requestId } from "./transport"
26
+ import { counter, object, PREFIX, positiveInteger } from "./util"
27
+
28
+ export interface AzureAIFoundrySpeechOptions {
29
+ /** The voice used when a request names none. Default: `alloy`. */
30
+ readonly voice?: string
31
+ /** Deadline for one generation, response included. Default: 120,000 ms. */
32
+ readonly timeoutMs?: number
33
+ readonly rateCard?: ModelRateCard
34
+ readonly costEstimator?: SpeechCostEstimator
35
+ }
36
+
37
+ export interface AzureAIFoundrySpeechModel extends SpeechModel {
38
+ readonly metadata: AzureAIFoundryModelMetadata
39
+ resolve(options?: { readonly offline?: boolean }): Promise<AzureAIFoundrySpeechModel>
40
+ }
41
+
42
+ /** The `response_format` for each media type a request may ask for; PCM is not a file. */
43
+ const FORMATS: Readonly<Record<string, string>> = {
44
+ "audio/mpeg": "mp3",
45
+ "audio/ogg": "opus",
46
+ "audio/aac": "aac",
47
+ "audio/flac": "flac",
48
+ "audio/wav": "wav",
49
+ }
50
+ const OUTPUT_MEDIA_TYPES = Object.freeze(Object.keys(FORMATS))
51
+
52
+ /** Azure rejects longer input. */
53
+ const MAX_INPUT_CHARACTERS = 4096
54
+
55
+ /** 4,096 characters at speed 0.25 make about 20 minutes, 60 MiB of WAV. */
56
+ const MAX_AUDIO_BYTES = 128 * 1024 * 1024
57
+
58
+ /** One event carries a chunk of base64 audio; a longer one is not Azure's. */
59
+ const MAX_EVENT_BYTES = 16 * 1024 * 1024
60
+
61
+ /** AbortSignal.timeout's largest delay. */
62
+ const MAX_TIMEOUT_MS = 2 ** 31 - 1
63
+
64
+ /**
65
+ * Azure Retail Prices (verified 2026-11-13), by deployment SKU, USD per million text input and
66
+ * audio output tokens: meters gpt-4o-mini-tts-{txt-inp,aud-out}-{glbl,dz,rgnl} and, for the
67
+ * 2025-12-15 version, "gpt4o mn tts {txt in,aud out} {gl,dz} 1215".
68
+ */
69
+ const PUBLISHED: Readonly<Record<string, Readonly<Record<string, ModelRateCard>>>> = {
70
+ "2025-03-20": {
71
+ GlobalStandard: { currency: "USD", input: "0.6", output: "12" },
72
+ DataZoneStandard: { currency: "USD", input: "0.66", output: "13.2" },
73
+ Standard: { currency: "USD", input: "0.66", output: "13.2" },
74
+ },
75
+ "2025-12-15": {
76
+ GlobalStandard: { currency: "USD", input: "0.6", output: "12" },
77
+ DataZoneStandard: { currency: "USD", input: "0.66", output: "13.2" },
78
+ },
79
+ }
80
+
81
+ interface SpeechSettings {
82
+ readonly voice: string
83
+ readonly timeoutMs: number
84
+ readonly rateCard?: ModelRateCard
85
+ readonly costEstimator?: SpeechCostEstimator
86
+ }
87
+
88
+ interface SpeechResolution extends ResolvedDeployment {
89
+ readonly costEstimator?: SpeechCostEstimator
90
+ readonly rateCard?: ModelRateCard
91
+ }
92
+
93
+ export function createFoundrySpeech(
94
+ providerId: string,
95
+ modelId: string,
96
+ options: AzureAIFoundrySpeechOptions,
97
+ transport: FoundryTransport,
98
+ deployments: FoundryDeployments
99
+ ): AzureAIFoundrySpeechModel {
100
+ if (!modelId.trim() || modelId !== modelId.trim())
101
+ throw new TypeError(`${PREFIX} deploymentName must be nonempty without surrounding whitespace.`)
102
+ positiveInteger(options.timeoutMs, "timeoutMs")
103
+ if (options.timeoutMs !== undefined && options.timeoutMs > MAX_TIMEOUT_MS)
104
+ throw new TypeError(`${PREFIX} timeoutMs must be at most ${MAX_TIMEOUT_MS}.`)
105
+ if (options.voice !== undefined && (typeof options.voice !== "string" || !options.voice.trim()))
106
+ throw new TypeError(`${PREFIX} voice must be a nonblank string.`)
107
+ if (options.rateCard && options.costEstimator)
108
+ throw new TypeError(`${PREFIX} Configure either rateCard or costEstimator, not both.`)
109
+ // Validate and snapshot now: later edits to the caller's object change nothing.
110
+ const settings: SpeechSettings = Object.freeze({
111
+ voice: options.voice ?? "alloy",
112
+ timeoutMs: options.timeoutMs ?? 120_000,
113
+ ...(options.rateCard ? { rateCard: defineModelRateCard(options.rateCard) } : {}),
114
+ ...(options.costEstimator ? { costEstimator: options.costEstimator } : {}),
115
+ })
116
+ return speechBinding(providerId, modelId, settings, transport, deployments)
117
+ }
118
+
119
+ /** Discovery pins the deployed model for pricing; inference streams the audio as events. */
120
+ function speechBinding(
121
+ providerId: string,
122
+ modelId: string,
123
+ settings: SpeechSettings,
124
+ transport: FoundryTransport,
125
+ deployments: FoundryDeployments,
126
+ resolution?: SpeechResolution
127
+ ): AzureAIFoundrySpeechModel {
128
+ const deployment = resolution?.deployment
129
+ const definition: SpeechModelDefinition = Object.freeze({
130
+ kind: "speech",
131
+ providerId,
132
+ modelId,
133
+ ...(deployment ? { publisher: azurePublisher(deployment.modelPublisher) } : {}),
134
+ via: "Azure AI Foundry",
135
+ maxInputCharacters: MAX_INPUT_CHARACTERS,
136
+ outputMediaTypes: OUTPUT_MEDIA_TYPES,
137
+ instructions: true,
138
+ })
139
+
140
+ let pending: Promise<AzureAIFoundrySpeechModel> | undefined
141
+ const binding: AzureAIFoundrySpeechModel = Object.freeze({
142
+ providerId,
143
+ modelId,
144
+ definition,
145
+ defaultVoice: settings.voice,
146
+ metadata: deploymentMetadata(resolution),
147
+ ...(resolution?.costEstimator ? { costEstimator: resolution.costEstimator } : {}),
148
+ ...(resolution?.rateCard ? { rateCard: resolution.rateCard } : {}),
149
+
150
+ async resolve(input?: { readonly offline?: boolean }) {
151
+ if (resolution) return binding
152
+ const resolved = await deployments.resolve(modelId, input?.offline === true)
153
+ assertDeploymentKind(resolved.deployment, modelId, "speech")
154
+ // tts and tts-hd do not stream events, so they report no usage to price.
155
+ if (!/^gpt-4o-mini-tts$/i.test(resolved.deployment.modelName))
156
+ throw new UnsupportedModelFeatureError(
157
+ `${PREFIX} ${resolved.deployment.modelName} is not supported: deploy gpt-4o-mini-tts, which reports its usage.`
158
+ )
159
+ const rateCard = settings.costEstimator
160
+ ? undefined
161
+ : (settings.rateCard ?? publishedRateCard(resolved.deployment))
162
+ return speechBinding(providerId, modelId, settings, transport, deployments, {
163
+ ...resolved,
164
+ ...(rateCard ? { rateCard } : {}),
165
+ ...(settings.costEstimator || rateCard
166
+ ? { costEstimator: settings.costEstimator ?? speechEstimator(rateCard!) }
167
+ : {}),
168
+ })
169
+ },
170
+
171
+ async speak(request: SpeechModelRequest): Promise<SpeechModelResult> {
172
+ const signal = AbortSignal.any([
173
+ ...(request.signal ? [request.signal] : []),
174
+ AbortSignal.timeout(settings.timeoutMs),
175
+ ])
176
+ signal.throwIfAborted()
177
+ const mediaType = request.mediaType ?? "audio/mpeg"
178
+ const format = FORMATS[mediaType]
179
+ if (!format)
180
+ throw new TypeError(`${PREFIX} Speech media type '${mediaType}' cannot be requested.`)
181
+ if (
182
+ typeof request.text !== "string" ||
183
+ !request.text.trim() ||
184
+ request.text.length > MAX_INPUT_CHARACTERS
185
+ )
186
+ throw new TypeError(
187
+ `${PREFIX} Speech text must be 1 to ${MAX_INPUT_CHARACTERS} characters.`
188
+ )
189
+ if (!resolution) {
190
+ // A direct call pins the deployment once, as the runtime does before admission.
191
+ pending ??= binding.resolve().catch((error) => {
192
+ pending = undefined
193
+ throw error
194
+ })
195
+ const resolved = await abortable(() => pending!, signal)
196
+ return resolved.speak({ ...request, signal })
197
+ }
198
+
199
+ // Only the event stream reports usage; a plain audio reply carries no token counts.
200
+ const response = await transport.post(
201
+ JSON.stringify({
202
+ model: modelId,
203
+ input: request.text,
204
+ voice: request.voice ?? settings.voice,
205
+ response_format: format,
206
+ stream_format: "sse",
207
+ ...(request.instructions === undefined ? {} : { instructions: request.instructions }),
208
+ ...(request.speed === undefined ? {} : { speed: request.speed }),
209
+ }),
210
+ signal,
211
+ providerId,
212
+ modelId,
213
+ "speech"
214
+ )
215
+ const id = transport.redactText(response, requestId(response))
216
+ const stream = await readSpeechStream(response, signal)
217
+ const metadata: SpeechModelResponseMetadata = {
218
+ usage: speechUsage(stream.usage),
219
+ ...(id ? { providerIds: { requestId: id } } : {}),
220
+ }
221
+ // Azure billed the call whatever the stream holds: every failure keeps its metadata.
222
+ const failure = (reason: string) =>
223
+ new SpeechModelResponseError(`${PREFIX} ${reason}`, providerId, modelId, metadata)
224
+ if (stream.error !== undefined)
225
+ throw failure(
226
+ `Speech generation failed: ${transport.redactText(response, stream.error) ?? stream.error}`
227
+ )
228
+ if (stream.problem) throw failure(`Speech generation ${stream.problem}.`)
229
+ const audio = wavLengths(stream.audio, format)
230
+ if (!hasSignature(audio, format))
231
+ throw failure(`Speech generation returned no ${format} audio.`)
232
+ // The API has no language setting: the model reads the text as written.
233
+ const warnings =
234
+ request.language === undefined
235
+ ? []
236
+ : ["language: not supported; the model reads the text in its own language"]
237
+ return {
238
+ ...metadata,
239
+ output: { audio: { data: audio, mediaType } },
240
+ ...(warnings.length ? { warnings } : {}),
241
+ }
242
+ },
243
+ })
244
+ return binding
245
+ }
246
+
247
+ interface SpeechStream {
248
+ readonly audio: Uint8Array
249
+ readonly usage?: JsonObject
250
+ /** Azure's message from an `error` event. */
251
+ readonly error?: string
252
+ /** Why the stream cannot be trusted: cut off, oversized or malformed. */
253
+ readonly problem?: string
254
+ }
255
+
256
+ /**
257
+ * Read `speech.audio.delta` events as they arrive, decoding their base64 audio, until
258
+ * `speech.audio.done` brings the usage. Only a stream that reached it is complete.
259
+ */
260
+ async function readSpeechStream(response: Response, signal: AbortSignal): Promise<SpeechStream> {
261
+ const parts: Buffer[] = []
262
+ let bytes = 0
263
+ let usage: JsonObject | undefined
264
+ let done = false
265
+ let error: string | undefined
266
+ let problem: string | undefined
267
+ let data: string[] = []
268
+ let pending = ""
269
+
270
+ // An event is its `data:` lines up to a blank line, joined by newlines.
271
+ const dispatch = () => {
272
+ const text = data.join("\n")
273
+ data = []
274
+ if (!text || text === "[DONE]") return
275
+ let event: JsonObject | undefined
276
+ try {
277
+ event = object(JSON.parse(text))
278
+ } catch {
279
+ problem = "returned a malformed event"
280
+ return
281
+ }
282
+ if (event?.type === "speech.audio.delta" && typeof event.audio === "string" && !done) {
283
+ const chunk = Buffer.from(event.audio, "base64")
284
+ bytes += chunk.length
285
+ if (bytes > MAX_AUDIO_BYTES) problem = `exceeded ${MAX_AUDIO_BYTES} bytes of audio`
286
+ else parts.push(chunk)
287
+ } else if (event?.type === "speech.audio.done") {
288
+ done = true
289
+ usage = object(event.usage)
290
+ } else if (event?.type === "error") {
291
+ const detail = object(event.error)
292
+ error = typeof detail?.message === "string" ? detail.message : "error event"
293
+ }
294
+ }
295
+
296
+ const reader = response.body?.getReader()
297
+ const decoder = new TextDecoder()
298
+ try {
299
+ while (reader && !problem && error === undefined) {
300
+ const chunk = await abortable(() => reader.read(), signal).catch(() => undefined)
301
+ if (!chunk) {
302
+ if (signal.aborted) throw signal.reason
303
+ problem = "stream failed before it finished"
304
+ break
305
+ }
306
+ pending += decoder.decode(chunk.value, { stream: !chunk.done })
307
+ const lines = pending.split(/\r?\n/)
308
+ pending = chunk.done ? "" : lines.pop()!
309
+ if (pending.length > MAX_EVENT_BYTES) problem = "returned an oversized event"
310
+ for (const line of lines) {
311
+ if (line === "") dispatch()
312
+ else if (line.startsWith("data:")) data.push(line.slice(line.startsWith("data: ") ? 6 : 5))
313
+ if (problem || error !== undefined) break
314
+ }
315
+ if (chunk.done) {
316
+ dispatch()
317
+ break
318
+ }
319
+ }
320
+ } finally {
321
+ void reader?.cancel().catch(() => {})
322
+ reader?.releaseLock()
323
+ }
324
+ return {
325
+ audio: new Uint8Array(Buffer.concat(parts)),
326
+ ...(usage ? { usage } : {}),
327
+ ...(error === undefined ? {} : { error }),
328
+ ...(problem
329
+ ? { problem }
330
+ : error === undefined && !done
331
+ ? { problem: "stream ended before speech.audio.done" }
332
+ : {}),
333
+ }
334
+ }
335
+
336
+ /** Input tokens are text and instructions; every output token is audio. */
337
+ function speechUsage(raw: JsonObject | undefined): ModelUsage {
338
+ const inputTokens = counter(raw?.input_tokens)
339
+ const outputTokens = counter(raw?.output_tokens)
340
+ return {
341
+ ...(inputTokens === undefined ? {} : { inputTokens }),
342
+ ...(outputTokens === undefined ? {} : { outputTokens }),
343
+ ...(raw ? { raw } : {}),
344
+ }
345
+ }
346
+
347
+ /**
348
+ * A streamed WAV header leaves its RIFF and data sizes at 0xFFFFFFFF, unknown when it was sent.
349
+ * The whole file is here, so write the real sizes: some players refuse the placeholder.
350
+ */
351
+ function wavLengths(audio: Uint8Array, format: string): Uint8Array {
352
+ if (format !== "wav" || audio.length < 44 || ascii(audio, 36) !== "data") return audio
353
+ const view = new DataView(audio.buffer, audio.byteOffset, audio.byteLength)
354
+ if (view.getUint32(4, true) === 0xffffffff) view.setUint32(4, audio.length - 8, true)
355
+ if (view.getUint32(40, true) === 0xffffffff) view.setUint32(40, audio.length - 44, true)
356
+ return audio
357
+ }
358
+
359
+ /** The reply must be the format asked for: Sixb never transcodes. */
360
+ function hasSignature(audio: Uint8Array, format: string): boolean {
361
+ if (format === "wav") return ascii(audio, 0) === "RIFF" && ascii(audio, 8) === "WAVE"
362
+ if (format === "opus") return ascii(audio, 0) === "OggS"
363
+ if (format === "flac") return ascii(audio, 0) === "fLaC"
364
+ // MP3 starts with an ID3 tag or an MPEG frame sync; ADTS AAC is a sync with layer bits zero.
365
+ const sync = audio.length >= 2 && audio[0] === 0xff && (audio[1]! & 0xe0) === 0xe0
366
+ const layer = audio.length >= 2 ? audio[1]! & 0x06 : -1
367
+ if (format === "aac") return sync && layer === 0
368
+ return ascii(audio, 0).startsWith("ID3") || (sync && layer !== 0)
369
+ }
370
+
371
+ function ascii(bytes: Uint8Array, offset: number): string {
372
+ return String.fromCharCode(...bytes.subarray(offset, offset + 4))
373
+ }
374
+
375
+ /** The deployment's retail price, when Azure publishes one for its version and SKU. */
376
+ function publishedRateCard(deployment: AzureAIFoundryDeployment): ModelRateCard | undefined {
377
+ const skus =
378
+ /^gpt-4o-mini-tts$/i.test(deployment.modelName) &&
379
+ Object.hasOwn(PUBLISHED, deployment.modelVersion)
380
+ ? PUBLISHED[deployment.modelVersion]
381
+ : undefined
382
+ const card =
383
+ skus && Object.hasOwn(skus, deployment.sku.name) ? skus[deployment.sku.name] : undefined
384
+ return card ? defineModelRateCard(card) : undefined
385
+ }
386
+
387
+ function speechEstimator(rateCard: ModelRateCard): SpeechCostEstimator {
388
+ // The audio's length, so its output tokens, is unknown before the call: no reservation.
389
+ return { estimate: ({ usage }) => rateModelCall({ usage, rateCard }) }
390
+ }