@effect-uai/elevenlabs 0.6.0 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. package/dist/ElevenLabsMusicGenerator.d.mts +75 -0
  2. package/dist/ElevenLabsMusicGenerator.d.mts.map +1 -0
  3. package/dist/ElevenLabsMusicGenerator.mjs +191 -0
  4. package/dist/ElevenLabsMusicGenerator.mjs.map +1 -0
  5. package/dist/ElevenLabsMusicGenerator.test.mjs +98 -0
  6. package/dist/ElevenLabsMusicGenerator.test.mjs.map +1 -0
  7. package/dist/ElevenLabsSynthesizer.d.mts +10 -4
  8. package/dist/ElevenLabsSynthesizer.d.mts.map +1 -1
  9. package/dist/ElevenLabsSynthesizer.mjs +9 -33
  10. package/dist/ElevenLabsSynthesizer.mjs.map +1 -1
  11. package/dist/ElevenLabsTranscriber.d.mts +4 -4
  12. package/dist/ElevenLabsTranscriber.d.mts.map +1 -1
  13. package/dist/ElevenLabsTranscriber.mjs +9 -1
  14. package/dist/ElevenLabsTranscriber.mjs.map +1 -1
  15. package/dist/codec.d.mts +25 -2
  16. package/dist/codec.d.mts.map +1 -1
  17. package/dist/codec.mjs +18 -1
  18. package/dist/codec.mjs.map +1 -1
  19. package/dist/index.d.mts +4 -3
  20. package/dist/index.mjs +2 -1
  21. package/dist/models.d.mts +13 -1
  22. package/dist/models.d.mts.map +1 -1
  23. package/dist/musicCodec-CZni1gAQ.d.mts +42 -0
  24. package/dist/musicCodec-CZni1gAQ.d.mts.map +1 -0
  25. package/dist/musicCodec.d.mts +2 -0
  26. package/dist/musicCodec.mjs +38 -0
  27. package/dist/musicCodec.mjs.map +1 -0
  28. package/dist/realtimeStt.d.mts +2 -1
  29. package/dist/realtimeStt.d.mts.map +1 -1
  30. package/dist/realtimeStt.mjs +14 -3
  31. package/dist/realtimeStt.mjs.map +1 -1
  32. package/dist/realtimeTts.d.mts +8 -3
  33. package/dist/realtimeTts.d.mts.map +1 -1
  34. package/dist/realtimeTts.mjs +5 -3
  35. package/dist/realtimeTts.mjs.map +1 -1
  36. package/package.json +9 -3
  37. package/src/ElevenLabsMusicGenerator.test.ts +116 -0
  38. package/src/ElevenLabsMusicGenerator.ts +353 -0
  39. package/src/ElevenLabsSynthesizer.ts +17 -51
  40. package/src/ElevenLabsTranscriber.ts +11 -0
  41. package/src/codec.ts +47 -0
  42. package/src/index.ts +1 -0
  43. package/src/models.ts +17 -0
  44. package/src/musicCodec.ts +86 -0
  45. package/src/realtimeStt.ts +11 -0
  46. package/src/realtimeTts.ts +11 -0
  47. package/dist/ElevenLabsSynthesizer.test.mjs +0 -32
  48. package/dist/ElevenLabsSynthesizer.test.mjs.map +0 -1
  49. package/src/ElevenLabsSynthesizer.test.ts +0 -49
  50. /package/dist/{ElevenLabsSynthesizer.test.d.mts → ElevenLabsMusicGenerator.test.d.mts} +0 -0
@@ -0,0 +1,353 @@
1
+ import { Context, Duration, Effect, Layer, Redacted, Stream } from "effect"
2
+ import { HttpClient, HttpClientRequest } from "effect/unstable/http"
3
+ import * as AiError from "@effect-uai/core/AiError"
4
+ import type { AudioChunk, AudioFormat } from "@effect-uai/core/Audio"
5
+ import type {
6
+ CommonGenerateMusicRequest,
7
+ CommonStreamGenerateMusicRequest,
8
+ GenerateResult,
9
+ MusicResult,
10
+ MusicSessionInput,
11
+ MusicStreamEvent,
12
+ } from "@effect-uai/core/Music"
13
+ import { singleVariant } from "@effect-uai/core/Music"
14
+ import { MusicGenerator, type MusicGeneratorService } from "@effect-uai/core/MusicGenerator"
15
+ import { defaultFormat, formatToOutputSlug } from "./codec.js"
16
+ import type { ElevenLabsMusicModel } from "./models.js"
17
+ import {
18
+ decodeCompositionPlan,
19
+ type ElevenLabsCompositionPlan,
20
+ warnDroppedPromptModeHints,
21
+ wireCompositionPlan,
22
+ } from "./musicCodec.js"
23
+ import { type ElevenLabsRegion, resolveHost } from "./region.js"
24
+
25
+ // ---------------------------------------------------------------------------
26
+ // Public types
27
+ // ---------------------------------------------------------------------------
28
+
29
+ /**
30
+ * ElevenLabs-typed music request. Narrows `model` to
31
+ * `ElevenLabsMusicModel` and adds provider-specific extras:
32
+ *
33
+ * - `compositionPlan` — switch from prompt mode to plan mode. Mutually
34
+ * exclusive with `prompt` on the wire.
35
+ * - `forceInstrumental` — skip vocals. ElevenLabs has no `instrumental`
36
+ * on the cross-provider Common request (the field was removed in
37
+ * v0.7 because it has wildly different semantics per provider); the
38
+ * typed surface restores it.
39
+ * - `signWithC2pa` — opt-in C2PA Content Credentials, MP3 output only.
40
+ * - `respectSectionsDurations` — composition-plan mode only. False
41
+ * trades section-duration accuracy for quality + latency.
42
+ *
43
+ * Setting `compositionPlan` with a non-empty `prompt`, or with
44
+ * `duration`, fails `InvalidRequest` at the codec layer.
45
+ */
46
+ export type ElevenLabsMusicGenerateRequest = Omit<CommonGenerateMusicRequest, "model"> & {
47
+ readonly model?: ElevenLabsMusicModel
48
+ readonly compositionPlan?: ElevenLabsCompositionPlan
49
+ readonly forceInstrumental?: boolean
50
+ readonly signWithC2pa?: boolean
51
+ readonly respectSectionsDurations?: boolean
52
+ }
53
+
54
+ export type CreateCompositionPlanRequest = {
55
+ readonly prompt: string
56
+ readonly duration?: Duration.Duration
57
+ readonly sourceCompositionPlan?: ElevenLabsCompositionPlan
58
+ readonly model?: ElevenLabsMusicModel
59
+ }
60
+
61
+ export type ElevenLabsMusicGeneratorService = {
62
+ readonly generate: (
63
+ request: ElevenLabsMusicGenerateRequest,
64
+ ) => Effect.Effect<GenerateResult, AiError.AiError>
65
+ readonly streamGeneration: (
66
+ request: ElevenLabsMusicGenerateRequest,
67
+ ) => Stream.Stream<AudioChunk, AiError.AiError>
68
+ readonly streamGenerationFrom: <E, R>(
69
+ input: Stream.Stream<MusicSessionInput, E, R>,
70
+ request: CommonStreamGenerateMusicRequest,
71
+ ) => Stream.Stream<MusicStreamEvent, AiError.AiError | E, R>
72
+ /**
73
+ * Free endpoint (rate-limited, no credit cost): turn a prompt into a
74
+ * structured `MusicPrompt` plan. Feed the result back into
75
+ * `generate` / `streamGeneration` via `compositionPlan`.
76
+ */
77
+ readonly createCompositionPlan: (
78
+ request: CreateCompositionPlanRequest,
79
+ ) => Effect.Effect<ElevenLabsCompositionPlan, AiError.AiError>
80
+ }
81
+
82
+ export class ElevenLabsMusicGenerator extends Context.Service<
83
+ ElevenLabsMusicGenerator,
84
+ ElevenLabsMusicGeneratorService
85
+ >()("@betalyra/effect-uai/providers/elevenlabs/ElevenLabsMusicGenerator") {}
86
+
87
+ export type Config = {
88
+ readonly apiKey: Redacted.Redacted
89
+ readonly baseUrl?: string
90
+ readonly region?: ElevenLabsRegion
91
+ }
92
+
93
+ const DEFAULT_MODEL: ElevenLabsMusicModel = "music_v1"
94
+
95
+ // ---------------------------------------------------------------------------
96
+ // HTTP errors (music namespace; reuses the same status taxonomy)
97
+ // ---------------------------------------------------------------------------
98
+
99
+ const musicTransportFailure = (cause: unknown): AiError.AiError =>
100
+ new AiError.Unavailable({ provider: "elevenlabs-music", raw: cause })
101
+
102
+ const musicHttpStatusError = (status: number, body: string): AiError.AiError => {
103
+ if (status === 429) return new AiError.RateLimited({ provider: "elevenlabs-music", raw: body })
104
+ if (status === 408 || status === 504)
105
+ return new AiError.Timeout({ provider: "elevenlabs-music", raw: body })
106
+ if (status === 401)
107
+ return new AiError.AuthFailed({ provider: "elevenlabs-music", subtype: "auth", raw: body })
108
+ if (status === 403)
109
+ return new AiError.AuthFailed({
110
+ provider: "elevenlabs-music",
111
+ subtype: "permission",
112
+ raw: body,
113
+ })
114
+ if (status === 402)
115
+ return new AiError.AuthFailed({
116
+ provider: "elevenlabs-music",
117
+ subtype: "billing",
118
+ raw: body,
119
+ })
120
+ if (status >= 500)
121
+ return new AiError.Unavailable({ provider: "elevenlabs-music", status, raw: body })
122
+ return new AiError.InvalidRequest({ provider: "elevenlabs-music", raw: body })
123
+ }
124
+
125
+ // ---------------------------------------------------------------------------
126
+ // Wire body
127
+ // ---------------------------------------------------------------------------
128
+
129
+ /**
130
+ * Build the JSON body for `/v1/music` (sync + stream share the body).
131
+ *
132
+ * Mutual exclusion: `compositionPlan` and prompt-mode fields cannot
133
+ * coexist. When `compositionPlan` is set, `prompt` must be empty and
134
+ * `duration` must be unset (section durations live on the plan).
135
+ */
136
+ const buildBody = (request: ElevenLabsMusicGenerateRequest) =>
137
+ Effect.gen(function* () {
138
+ const model = request.model ?? DEFAULT_MODEL
139
+ if (request.compositionPlan !== undefined) {
140
+ if (request.prompt.length > 0) {
141
+ return yield* Effect.fail(
142
+ new AiError.InvalidRequest({
143
+ provider: "elevenlabs-music",
144
+ param: "prompt",
145
+ raw: "`prompt` and `compositionPlan` are mutually exclusive. Pass one or the other (compositionPlan carries its own per-section lyrics and styles).",
146
+ }),
147
+ )
148
+ }
149
+ if (request.duration !== undefined) {
150
+ return yield* Effect.fail(
151
+ new AiError.InvalidRequest({
152
+ provider: "elevenlabs-music",
153
+ param: "duration",
154
+ raw: "`duration` is ignored when `compositionPlan` is set; section durations live on the plan. Remove `duration`.",
155
+ }),
156
+ )
157
+ }
158
+ return {
159
+ composition_plan: wireCompositionPlan(request.compositionPlan),
160
+ model_id: model,
161
+ ...(request.seed !== undefined && { seed: request.seed }),
162
+ ...(request.forceInstrumental !== undefined && {
163
+ force_instrumental: request.forceInstrumental,
164
+ }),
165
+ ...(request.respectSectionsDurations !== undefined && {
166
+ respect_sections_durations: request.respectSectionsDurations,
167
+ }),
168
+ ...(request.signWithC2pa !== undefined && { sign_with_c2pa: request.signWithC2pa }),
169
+ }
170
+ }
171
+ yield* warnDroppedPromptModeHints(request as CommonGenerateMusicRequest)
172
+ return {
173
+ prompt: request.prompt,
174
+ model_id: model,
175
+ ...(request.duration !== undefined && {
176
+ music_length_ms: Duration.toMillis(request.duration),
177
+ }),
178
+ ...(request.seed !== undefined && { seed: request.seed }),
179
+ ...(request.forceInstrumental !== undefined && {
180
+ force_instrumental: request.forceInstrumental,
181
+ }),
182
+ ...(request.signWithC2pa !== undefined && { sign_with_c2pa: request.signWithC2pa }),
183
+ }
184
+ })
185
+
186
+ const buildMusicHttpRequest = (
187
+ cfg: Config,
188
+ request: ElevenLabsMusicGenerateRequest,
189
+ path: "" | "/stream",
190
+ ) =>
191
+ Effect.gen(function* () {
192
+ const format: AudioFormat = request.outputFormat ?? defaultFormat
193
+ const slug = yield* formatToOutputSlug(format)
194
+ const body = yield* buildBody(request)
195
+ const url = `${resolveHost(cfg)}/music${path}?output_format=${slug}`
196
+ const httpRequest = HttpClientRequest.post(url).pipe(
197
+ HttpClientRequest.setHeader("xi-api-key", Redacted.value(cfg.apiKey)),
198
+ HttpClientRequest.bodyJsonUnsafe(body),
199
+ )
200
+ return { httpRequest, format }
201
+ })
202
+
203
+ // ---------------------------------------------------------------------------
204
+ // Service implementation
205
+ // ---------------------------------------------------------------------------
206
+
207
+ const generateImpl =
208
+ (cfg: Config) =>
209
+ (
210
+ request: ElevenLabsMusicGenerateRequest,
211
+ ): Effect.Effect<GenerateResult, AiError.AiError, HttpClient.HttpClient> =>
212
+ Effect.gen(function* () {
213
+ const client = yield* HttpClient.HttpClient
214
+ const { httpRequest, format } = yield* buildMusicHttpRequest(cfg, request, "")
215
+ const response = yield* client
216
+ .execute(httpRequest)
217
+ .pipe(Effect.mapError(musicTransportFailure))
218
+ if (response.status >= 400) {
219
+ const text = yield* response.text.pipe(Effect.orElseSucceed(() => ""))
220
+ return yield* Effect.fail(musicHttpStatusError(response.status, text))
221
+ }
222
+ const buffer = yield* response.arrayBuffer.pipe(Effect.mapError(musicTransportFailure))
223
+ const bytes = new Uint8Array(buffer)
224
+ const songId = response.headers["song_id"] ?? response.headers["x-song-id"]
225
+ const result: MusicResult = {
226
+ audio: { format, bytes },
227
+ provider: "elevenlabs-music",
228
+ ...(typeof songId === "string" && { songId }),
229
+ ...(request.signWithC2pa === true && { watermark: "c2pa" as const }),
230
+ }
231
+ return singleVariant(result)
232
+ })
233
+
234
+ const streamGenerationImpl =
235
+ (cfg: Config) =>
236
+ (
237
+ request: ElevenLabsMusicGenerateRequest,
238
+ ): Stream.Stream<AudioChunk, AiError.AiError, HttpClient.HttpClient> =>
239
+ Stream.unwrap(
240
+ Effect.gen(function* () {
241
+ const client = yield* HttpClient.HttpClient
242
+ const { httpRequest } = yield* buildMusicHttpRequest(cfg, request, "/stream")
243
+ const response = yield* client
244
+ .execute(httpRequest)
245
+ .pipe(Effect.mapError(musicTransportFailure))
246
+ if (response.status >= 400) {
247
+ const text = yield* response.text.pipe(Effect.orElseSucceed(() => ""))
248
+ return Stream.fail(musicHttpStatusError(response.status, text))
249
+ }
250
+ return response.stream.pipe(
251
+ Stream.mapError(musicTransportFailure),
252
+ Stream.map((bytes): AudioChunk => ({ bytes })),
253
+ )
254
+ }),
255
+ )
256
+
257
+ /**
258
+ * ElevenLabs music has no bidirectional session at the wire level.
259
+ * Layer does not register `MusicInteractiveSession`, so callers using
260
+ * `MusicGenerator.streamGenerationFrom` get a compile-time error
261
+ * before this runtime fallback fires.
262
+ */
263
+ const streamGenerationFromUnsupported = <E, R>(
264
+ _input: Stream.Stream<MusicSessionInput, E, R>,
265
+ _request: CommonStreamGenerateMusicRequest,
266
+ ): Stream.Stream<MusicStreamEvent, AiError.AiError | E, R> =>
267
+ Stream.fail(
268
+ new AiError.Unsupported({
269
+ provider: "elevenlabs-music",
270
+ capability: "streamGenerationFrom",
271
+ reason:
272
+ "ElevenLabs music has no bidirectional session. Use `streamGeneration` for chunked output, or `createCompositionPlan` + `generate` for structured plans.",
273
+ }),
274
+ )
275
+
276
+ const createCompositionPlanImpl =
277
+ (cfg: Config) =>
278
+ (
279
+ request: CreateCompositionPlanRequest,
280
+ ): Effect.Effect<ElevenLabsCompositionPlan, AiError.AiError, HttpClient.HttpClient> =>
281
+ Effect.gen(function* () {
282
+ const client = yield* HttpClient.HttpClient
283
+ const body = {
284
+ prompt: request.prompt,
285
+ model_id: request.model ?? DEFAULT_MODEL,
286
+ ...(request.duration !== undefined && {
287
+ music_length_ms: Duration.toMillis(request.duration),
288
+ }),
289
+ ...(request.sourceCompositionPlan !== undefined && {
290
+ source_composition_plan: wireCompositionPlan(request.sourceCompositionPlan),
291
+ }),
292
+ }
293
+ const httpRequest = HttpClientRequest.post(`${resolveHost(cfg)}/music/plan`).pipe(
294
+ HttpClientRequest.setHeader("xi-api-key", Redacted.value(cfg.apiKey)),
295
+ HttpClientRequest.bodyJsonUnsafe(body),
296
+ )
297
+ const response = yield* client
298
+ .execute(httpRequest)
299
+ .pipe(Effect.mapError(musicTransportFailure))
300
+ if (response.status >= 400) {
301
+ const text = yield* response.text.pipe(Effect.orElseSucceed(() => ""))
302
+ return yield* Effect.fail(musicHttpStatusError(response.status, text))
303
+ }
304
+ const json = yield* response.json.pipe(Effect.mapError(musicTransportFailure))
305
+ return decodeCompositionPlan(json)
306
+ })
307
+
308
+ // ---------------------------------------------------------------------------
309
+ // Constructors
310
+ // ---------------------------------------------------------------------------
311
+
312
+ export const make = (
313
+ cfg: Config,
314
+ ): Effect.Effect<ElevenLabsMusicGeneratorService, never, HttpClient.HttpClient> =>
315
+ Effect.map(HttpClient.HttpClient.asEffect(), (client) => ({
316
+ generate: (request) =>
317
+ generateImpl(cfg)(request).pipe(Effect.provideService(HttpClient.HttpClient, client)),
318
+ streamGeneration: (request) =>
319
+ streamGenerationImpl(cfg)(request).pipe(Stream.provideService(HttpClient.HttpClient, client)),
320
+ streamGenerationFrom: streamGenerationFromUnsupported,
321
+ createCompositionPlan: (request) =>
322
+ createCompositionPlanImpl(cfg)(request).pipe(
323
+ Effect.provideService(HttpClient.HttpClient, client),
324
+ ),
325
+ }))
326
+
327
+ /**
328
+ * Layer that registers the provider-specific `ElevenLabsMusicGenerator`
329
+ * tag and the generic `MusicGenerator` tag.
330
+ *
331
+ * Does NOT register `MusicInteractiveSession`. ElevenLabs music has no
332
+ * bidirectional wire endpoint; `streamGenerationFrom` against this
333
+ * Layer alone fails to typecheck.
334
+ */
335
+ export const layer = (
336
+ cfg: Config,
337
+ ): Layer.Layer<ElevenLabsMusicGenerator | MusicGenerator, never, HttpClient.HttpClient> =>
338
+ Layer.merge(
339
+ Layer.effect(ElevenLabsMusicGenerator, make(cfg)),
340
+ Layer.effect(
341
+ MusicGenerator,
342
+ Effect.map(
343
+ make(cfg),
344
+ (s): MusicGeneratorService => ({
345
+ generate: (req: CommonGenerateMusicRequest) =>
346
+ s.generate({ ...req, model: req.model as ElevenLabsMusicModel }),
347
+ streamGeneration: (req: CommonStreamGenerateMusicRequest) =>
348
+ s.streamGeneration({ ...req, model: req.model as ElevenLabsMusicModel }),
349
+ streamGenerationFrom: s.streamGenerationFrom,
350
+ }),
351
+ ),
352
+ ),
353
+ )
@@ -1,4 +1,4 @@
1
- import { Array as Arr, Context, Effect, Layer, Redacted, Stream } from "effect"
1
+ import { Context, Effect, Layer, Redacted, Stream } from "effect"
2
2
  import { HttpClient, HttpClientRequest } from "effect/unstable/http"
3
3
  import * as Socket from "effect/unstable/socket/Socket"
4
4
  import type * as AiError from "@effect-uai/core/AiError"
@@ -7,7 +7,6 @@ import {
7
7
  type CommonStreamSynthesizeRequest,
8
8
  type CommonSynthesizeDialogueRequest,
9
9
  type CommonSynthesizeRequest,
10
- type CustomPronunciation,
11
10
  MultiSpeakerTts,
12
11
  SpeechSynthesizer,
13
12
  type SpeechSynthesizerService,
@@ -17,8 +16,11 @@ import {
17
16
  defaultFormat,
18
17
  formatToOutputSlug,
19
18
  httpStatusError,
19
+ type PronunciationDictionaryLocator,
20
+ rejectInlinePronunciations,
20
21
  transportFailure,
21
22
  type VoiceSettings,
23
+ wirePronunciationLocators,
22
24
  wireVoiceSettings,
23
25
  } from "./codec.js"
24
26
  import type { ElevenLabsTtsModel, ElevenLabsVoiceId } from "./models.js"
@@ -34,6 +36,11 @@ import { type ElevenLabsRegion, resolveHost } from "./region.js"
34
36
  * provider's prosody/timbre controls; `seed` makes generation
35
37
  * deterministic; `previousText` / `nextText` thread context across
36
38
  * sequential calls for natural prosody.
39
+ *
40
+ * `pronunciationDictionaryLocators` references pre-provisioned
41
+ * pronunciation dictionaries by ID. Inline `pronunciations` (IPA) on
42
+ * the Common request are not supported by ElevenLabs and fail
43
+ * `Unsupported`; use a dictionary for phonetic control.
37
44
  */
38
45
  export type ElevenLabsSynthesizeRequest = Omit<CommonSynthesizeRequest, "model" | "voiceId"> & {
39
46
  readonly model: ElevenLabsTtsModel
@@ -42,6 +49,7 @@ export type ElevenLabsSynthesizeRequest = Omit<CommonSynthesizeRequest, "model"
42
49
  readonly seed?: number
43
50
  readonly previousText?: string
44
51
  readonly nextText?: string
52
+ readonly pronunciationDictionaryLocators?: ReadonlyArray<PronunciationDictionaryLocator>
45
53
  }
46
54
 
47
55
  export type ElevenLabsSynthesizerService = {
@@ -65,64 +73,18 @@ export type Config = {
65
73
  readonly region?: ElevenLabsRegion
66
74
  }
67
75
 
68
- // ---------------------------------------------------------------------------
69
- // Pronunciations — SSML <phoneme> rewrite (gated to legacy models)
70
- // ---------------------------------------------------------------------------
71
-
72
- /**
73
- * Models that honor inline `<phoneme>` SSML tags. Other models silently
74
- * drop the tags server-side, so we don't bother rewriting the text for
75
- * them — the pronunciation overrides are silently ignored.
76
- *
77
- * Reference: https://elevenlabs.io/docs/best-practices/prompting/controls
78
- */
79
- const PHONEME_SUPPORTED_MODELS: ReadonlySet<string> = new Set([
80
- "eleven_flash_v2",
81
- "eleven_english_v1",
82
- "eleven_monolingual_v1",
83
- ])
84
-
85
- const wireAlphabet = (encoding: CustomPronunciation["encoding"]): string | undefined =>
86
- encoding === "ipa" ? "ipa" : encoding === "cmu-arpabet" ? "cmu-arpabet" : undefined
87
-
88
- const phonemeTag = (phrase: string, pronunciation: string, alphabet: string): string =>
89
- `<phoneme alphabet="${alphabet}" ph="${pronunciation}">${phrase}</phoneme>`
90
-
91
- const escapeForRegex = (s: string): string => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")
92
-
93
- /**
94
- * Apply pronunciation overrides to a text string. Only applies for
95
- * phoneme-supported models; only IPA and CMU Arpabet are honored
96
- * (x-sampa is silently dropped). Whole-word, case-insensitive
97
- * replacement.
98
- */
99
- const applyPronunciations = (
100
- text: string,
101
- model: string,
102
- pronunciations: ReadonlyArray<CustomPronunciation> | undefined,
103
- ): string => {
104
- if (pronunciations === undefined || !PHONEME_SUPPORTED_MODELS.has(model)) return text
105
- return Arr.reduce(pronunciations, text, (acc, p) => {
106
- const alphabet = wireAlphabet(p.encoding)
107
- if (alphabet === undefined) return acc
108
- return acc.replace(
109
- new RegExp(`\\b${escapeForRegex(p.phrase)}\\b`, "i"),
110
- phonemeTag(p.phrase, p.pronunciation, alphabet),
111
- )
112
- })
113
- }
114
-
115
76
  // ---------------------------------------------------------------------------
116
77
  // Codec — request → JSON body
117
78
  // ---------------------------------------------------------------------------
118
79
 
119
80
  const buildBody = (r: ElevenLabsSynthesizeRequest) => ({
120
- text: applyPronunciations(r.text, r.model, r.pronunciations),
81
+ text: r.text,
121
82
  model_id: r.model,
122
83
  ...(r.languageCode !== undefined && { language_code: r.languageCode }),
123
84
  ...(r.seed !== undefined && { seed: r.seed }),
124
85
  ...(r.previousText !== undefined && { previous_text: r.previousText }),
125
86
  ...(r.nextText !== undefined && { next_text: r.nextText }),
87
+ ...wirePronunciationLocators(r.pronunciationDictionaryLocators),
126
88
  ...(r.voiceSettings !== undefined && { voice_settings: wireVoiceSettings(r.voiceSettings) }),
127
89
  })
128
90
 
@@ -135,7 +97,7 @@ const DEFAULT_DIALOGUE_MODEL = "eleven_v3"
135
97
  const buildDialogueBody = (r: CommonSynthesizeDialogueRequest) => ({
136
98
  inputs: r.turns.map((t) => ({
137
99
  voice_id: t.voiceId,
138
- text: applyPronunciations(t.text, r.model, r.pronunciations),
100
+ text: t.text,
139
101
  })),
140
102
  model_id: r.model ?? DEFAULT_DIALOGUE_MODEL,
141
103
  ...(r.languageCode !== undefined && { language_code: r.languageCode }),
@@ -159,6 +121,7 @@ const buildDialogueHttpRequest = (
159
121
 
160
122
  const synthesizeDialogueImpl = (cfg: Config) => (request: CommonSynthesizeDialogueRequest) =>
161
123
  Effect.gen(function* () {
124
+ yield* rejectInlinePronunciations(request.pronunciations)
162
125
  const client = yield* HttpClient.HttpClient
163
126
  const { httpRequest, format } = yield* buildDialogueHttpRequest(cfg, request, "")
164
127
  const response = yield* client.execute(httpRequest).pipe(Effect.mapError(transportFailure))
@@ -173,6 +136,7 @@ const synthesizeDialogueImpl = (cfg: Config) => (request: CommonSynthesizeDialog
173
136
  const streamSynthesizeDialogueImpl = (cfg: Config) => (request: CommonSynthesizeDialogueRequest) =>
174
137
  Stream.unwrap(
175
138
  Effect.gen(function* () {
139
+ yield* rejectInlinePronunciations(request.pronunciations)
176
140
  const client = yield* HttpClient.HttpClient
177
141
  const { httpRequest } = yield* buildDialogueHttpRequest(cfg, request, "/stream")
178
142
  const response = yield* client.execute(httpRequest).pipe(Effect.mapError(transportFailure))
@@ -205,6 +169,7 @@ const buildHttpRequest = (cfg: Config, r: ElevenLabsSynthesizeRequest, path: ""
205
169
 
206
170
  const synthesizeImpl = (cfg: Config) => (request: ElevenLabsSynthesizeRequest) =>
207
171
  Effect.gen(function* () {
172
+ yield* rejectInlinePronunciations(request.pronunciations)
208
173
  const client = yield* HttpClient.HttpClient
209
174
  const { httpRequest, format } = yield* buildHttpRequest(cfg, request, "")
210
175
  const response = yield* client.execute(httpRequest).pipe(Effect.mapError(transportFailure))
@@ -219,6 +184,7 @@ const synthesizeImpl = (cfg: Config) => (request: ElevenLabsSynthesizeRequest) =
219
184
  const streamSynthesisImpl = (cfg: Config) => (request: ElevenLabsSynthesizeRequest) =>
220
185
  Stream.unwrap(
221
186
  Effect.gen(function* () {
187
+ yield* rejectInlinePronunciations(request.pronunciations)
222
188
  const client = yield* HttpClient.HttpClient
223
189
  const { httpRequest } = yield* buildHttpRequest(cfg, request, "/stream")
224
190
  const response = yield* client.execute(httpRequest).pipe(Effect.mapError(transportFailure))
@@ -2,6 +2,7 @@ import { Context, Effect, Layer, Redacted, Schema, Stream } from "effect"
2
2
  import { HttpClient, HttpClientRequest } from "effect/unstable/http"
3
3
  import * as Socket from "effect/unstable/socket/Socket"
4
4
  import * as AiError from "@effect-uai/core/AiError"
5
+ import * as Capabilities from "@effect-uai/core/Capabilities"
5
6
  import type { TranscriptResult, WordTimestamp } from "@effect-uai/core/Transcript"
6
7
  import {
7
8
  type CommonTranscribeRequest,
@@ -101,6 +102,16 @@ const buildForm = (request: ElevenLabsTranscribeRequest) =>
101
102
  if (request.numSpeakers !== undefined) form.set("num_speakers", String(request.numSpeakers))
102
103
  if (request.audioEvents !== undefined)
103
104
  form.set("tag_audio_events", request.audioEvents ? "true" : "false")
105
+ // Vocab biasing → Scribe `keyterms` (≤1000 terms, ≤50 chars). Each
106
+ // term is a repeated form field (`keyterms=a&keyterms=b`), matching
107
+ // the ElevenLabs SDK. Scribe has no free-form prompt field.
108
+ for (const term of request.biasingTerms ?? []) form.append("keyterms", term)
109
+ yield* Capabilities.warnDroppedWhen(request.prompt, {
110
+ provider: "elevenlabs",
111
+ capability: "prompt",
112
+ field: "prompt",
113
+ reason: "ElevenLabs Scribe has no free-form prompt field; bias via `biasingTerms`.",
114
+ })
104
115
  return form
105
116
  })
106
117
 
package/src/codec.ts CHANGED
@@ -1,6 +1,7 @@
1
1
  import { Array as Arr, Effect, Encoding, Match, Option, Result } from "effect"
2
2
  import * as AiError from "@effect-uai/core/AiError"
3
3
  import type { AudioFormat, AudioSource } from "@effect-uai/core/Audio"
4
+ import type { CustomPronunciation } from "@effect-uai/core/SpeechSynthesizer"
4
5
 
5
6
  // ---------------------------------------------------------------------------
6
7
  // Shared TTS voice-settings shape (sync + streaming use the same fields)
@@ -25,6 +26,52 @@ export const wireVoiceSettings = (v: VoiceSettings | undefined) =>
25
26
  ...(v.speed !== undefined && { speed: v.speed }),
26
27
  }
27
28
 
29
+ // ---------------------------------------------------------------------------
30
+ // Pronunciation: dictionary locators (supported) vs inline IPA (rejected)
31
+ // ---------------------------------------------------------------------------
32
+
33
+ /**
34
+ * Reference to a pre-provisioned ElevenLabs pronunciation dictionary.
35
+ * The caller provisions and versions the dictionary out-of-band
36
+ * (dashboard or the pronunciation-dictionary API); we only reference it.
37
+ */
38
+ export type PronunciationDictionaryLocator = {
39
+ readonly dictionaryId: string
40
+ readonly versionId: string
41
+ }
42
+
43
+ export const wirePronunciationLocators = (
44
+ locators: ReadonlyArray<PronunciationDictionaryLocator> | undefined,
45
+ ) =>
46
+ locators === undefined || locators.length === 0
47
+ ? undefined
48
+ : {
49
+ pronunciation_dictionary_locators: locators.map((l) => ({
50
+ pronunciation_dictionary_id: l.dictionaryId,
51
+ version_id: l.versionId,
52
+ })),
53
+ }
54
+
55
+ /**
56
+ * ElevenLabs has no stateless inline IPA path (legacy `<phoneme>` SSML
57
+ * is dropped; modern phonetics live in pre-uploaded dictionaries).
58
+ * Inline `pronunciations` are load-bearing (bucket 1), so reject rather
59
+ * than silently mispronounce. Use `pronunciationDictionaryLocators`.
60
+ */
61
+ export const rejectInlinePronunciations = (
62
+ pronunciations: ReadonlyArray<CustomPronunciation> | undefined,
63
+ ): Effect.Effect<void, AiError.AiError> =>
64
+ pronunciations !== undefined && pronunciations.length > 0
65
+ ? Effect.fail(
66
+ new AiError.Unsupported({
67
+ provider: "elevenlabs",
68
+ capability: "pronunciations",
69
+ reason:
70
+ "ElevenLabs has no stateless inline IPA path. Provision a pronunciation dictionary and pass `pronunciationDictionaryLocators`, or use a provider with inline phonemes (Inworld, Google).",
71
+ }),
72
+ )
73
+ : Effect.void
74
+
28
75
  // ---------------------------------------------------------------------------
29
76
  // AudioFormat → ElevenLabs `output_format` query value
30
77
  // ---------------------------------------------------------------------------
package/src/index.ts CHANGED
@@ -1,3 +1,4 @@
1
+ export * as ElevenLabsMusicGenerator from "./ElevenLabsMusicGenerator.js"
1
2
  export * as ElevenLabsSynthesizer from "./ElevenLabsSynthesizer.js"
2
3
  export * as ElevenLabsTranscriber from "./ElevenLabsTranscriber.js"
3
4
  export * as codec from "./codec.js"
package/src/models.ts CHANGED
@@ -44,3 +44,20 @@ export type ElevenLabsSttModel =
44
44
  * full catalog is fetched dynamically via `GET /v1/voices`.
45
45
  */
46
46
  export type ElevenLabsVoiceId = string
47
+
48
+ /**
49
+ * Known ElevenLabs music model identifiers.
50
+ *
51
+ * - `music_v1` — current API default (as of May 2026). General-purpose
52
+ * music composition.
53
+ * - `music_v2` — UI default; API plumbing in progress per the
54
+ * May 26 2026 v2 launch blog. Include in the literal union so
55
+ * callers can opt in when it ships.
56
+ *
57
+ * Reference: https://elevenlabs.io/docs/api-reference/music/compose
58
+ */
59
+ export type ElevenLabsMusicModel =
60
+ | "music_v1"
61
+ | "music_v2"
62
+ // eslint-disable-next-line @typescript-eslint/ban-types
63
+ | (string & {})