@effect-uai/elevenlabs 0.6.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/ElevenLabsMusicGenerator.d.mts +75 -0
- package/dist/ElevenLabsMusicGenerator.d.mts.map +1 -0
- package/dist/ElevenLabsMusicGenerator.mjs +191 -0
- package/dist/ElevenLabsMusicGenerator.mjs.map +1 -0
- package/dist/ElevenLabsMusicGenerator.test.mjs +98 -0
- package/dist/ElevenLabsMusicGenerator.test.mjs.map +1 -0
- package/dist/ElevenLabsSynthesizer.d.mts +10 -4
- package/dist/ElevenLabsSynthesizer.d.mts.map +1 -1
- package/dist/ElevenLabsSynthesizer.mjs +9 -33
- package/dist/ElevenLabsSynthesizer.mjs.map +1 -1
- package/dist/ElevenLabsTranscriber.d.mts +4 -4
- package/dist/ElevenLabsTranscriber.d.mts.map +1 -1
- package/dist/ElevenLabsTranscriber.mjs +9 -1
- package/dist/ElevenLabsTranscriber.mjs.map +1 -1
- package/dist/codec.d.mts +25 -2
- package/dist/codec.d.mts.map +1 -1
- package/dist/codec.mjs +18 -1
- package/dist/codec.mjs.map +1 -1
- package/dist/index.d.mts +4 -3
- package/dist/index.mjs +2 -1
- package/dist/models.d.mts +13 -1
- package/dist/models.d.mts.map +1 -1
- package/dist/musicCodec-CZni1gAQ.d.mts +42 -0
- package/dist/musicCodec-CZni1gAQ.d.mts.map +1 -0
- package/dist/musicCodec.d.mts +2 -0
- package/dist/musicCodec.mjs +38 -0
- package/dist/musicCodec.mjs.map +1 -0
- package/dist/realtimeStt.d.mts +2 -1
- package/dist/realtimeStt.d.mts.map +1 -1
- package/dist/realtimeStt.mjs +14 -3
- package/dist/realtimeStt.mjs.map +1 -1
- package/dist/realtimeTts.d.mts +8 -3
- package/dist/realtimeTts.d.mts.map +1 -1
- package/dist/realtimeTts.mjs +5 -3
- package/dist/realtimeTts.mjs.map +1 -1
- package/package.json +9 -3
- package/src/ElevenLabsMusicGenerator.test.ts +116 -0
- package/src/ElevenLabsMusicGenerator.ts +353 -0
- package/src/ElevenLabsSynthesizer.ts +17 -51
- package/src/ElevenLabsTranscriber.ts +11 -0
- package/src/codec.ts +47 -0
- package/src/index.ts +1 -0
- package/src/models.ts +17 -0
- package/src/musicCodec.ts +86 -0
- package/src/realtimeStt.ts +11 -0
- package/src/realtimeTts.ts +11 -0
- package/dist/ElevenLabsSynthesizer.test.mjs +0 -32
- package/dist/ElevenLabsSynthesizer.test.mjs.map +0 -1
- package/src/ElevenLabsSynthesizer.test.ts +0 -49
- /package/dist/{ElevenLabsSynthesizer.test.d.mts → ElevenLabsMusicGenerator.test.d.mts} +0 -0
|
@@ -0,0 +1,353 @@
|
|
|
1
|
+
import { Context, Duration, Effect, Layer, Redacted, Stream } from "effect"
|
|
2
|
+
import { HttpClient, HttpClientRequest } from "effect/unstable/http"
|
|
3
|
+
import * as AiError from "@effect-uai/core/AiError"
|
|
4
|
+
import type { AudioChunk, AudioFormat } from "@effect-uai/core/Audio"
|
|
5
|
+
import type {
|
|
6
|
+
CommonGenerateMusicRequest,
|
|
7
|
+
CommonStreamGenerateMusicRequest,
|
|
8
|
+
GenerateResult,
|
|
9
|
+
MusicResult,
|
|
10
|
+
MusicSessionInput,
|
|
11
|
+
MusicStreamEvent,
|
|
12
|
+
} from "@effect-uai/core/Music"
|
|
13
|
+
import { singleVariant } from "@effect-uai/core/Music"
|
|
14
|
+
import { MusicGenerator, type MusicGeneratorService } from "@effect-uai/core/MusicGenerator"
|
|
15
|
+
import { defaultFormat, formatToOutputSlug } from "./codec.js"
|
|
16
|
+
import type { ElevenLabsMusicModel } from "./models.js"
|
|
17
|
+
import {
|
|
18
|
+
decodeCompositionPlan,
|
|
19
|
+
type ElevenLabsCompositionPlan,
|
|
20
|
+
warnDroppedPromptModeHints,
|
|
21
|
+
wireCompositionPlan,
|
|
22
|
+
} from "./musicCodec.js"
|
|
23
|
+
import { type ElevenLabsRegion, resolveHost } from "./region.js"
|
|
24
|
+
|
|
25
|
+
// ---------------------------------------------------------------------------
|
|
26
|
+
// Public types
|
|
27
|
+
// ---------------------------------------------------------------------------
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* ElevenLabs-typed music request. Narrows `model` to
|
|
31
|
+
* `ElevenLabsMusicModel` and adds provider-specific extras:
|
|
32
|
+
*
|
|
33
|
+
* - `compositionPlan` — switch from prompt mode to plan mode. Mutually
|
|
34
|
+
* exclusive with `prompt` on the wire.
|
|
35
|
+
* - `forceInstrumental` — skip vocals. ElevenLabs has no `instrumental`
|
|
36
|
+
* on the cross-provider Common request (the field was removed in
|
|
37
|
+
* v0.7 because it has wildly different semantics per provider); the
|
|
38
|
+
* typed surface restores it.
|
|
39
|
+
* - `signWithC2pa` — opt-in C2PA Content Credentials, MP3 output only.
|
|
40
|
+
* - `respectSectionsDurations` — composition-plan mode only. False
|
|
41
|
+
* trades section-duration accuracy for quality + latency.
|
|
42
|
+
*
|
|
43
|
+
* Setting `compositionPlan` with a non-empty `prompt`, or with
|
|
44
|
+
* `duration`, fails `InvalidRequest` at the codec layer.
|
|
45
|
+
*/
|
|
46
|
+
export type ElevenLabsMusicGenerateRequest = Omit<CommonGenerateMusicRequest, "model"> & {
|
|
47
|
+
readonly model?: ElevenLabsMusicModel
|
|
48
|
+
readonly compositionPlan?: ElevenLabsCompositionPlan
|
|
49
|
+
readonly forceInstrumental?: boolean
|
|
50
|
+
readonly signWithC2pa?: boolean
|
|
51
|
+
readonly respectSectionsDurations?: boolean
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
export type CreateCompositionPlanRequest = {
|
|
55
|
+
readonly prompt: string
|
|
56
|
+
readonly duration?: Duration.Duration
|
|
57
|
+
readonly sourceCompositionPlan?: ElevenLabsCompositionPlan
|
|
58
|
+
readonly model?: ElevenLabsMusicModel
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
export type ElevenLabsMusicGeneratorService = {
|
|
62
|
+
readonly generate: (
|
|
63
|
+
request: ElevenLabsMusicGenerateRequest,
|
|
64
|
+
) => Effect.Effect<GenerateResult, AiError.AiError>
|
|
65
|
+
readonly streamGeneration: (
|
|
66
|
+
request: ElevenLabsMusicGenerateRequest,
|
|
67
|
+
) => Stream.Stream<AudioChunk, AiError.AiError>
|
|
68
|
+
readonly streamGenerationFrom: <E, R>(
|
|
69
|
+
input: Stream.Stream<MusicSessionInput, E, R>,
|
|
70
|
+
request: CommonStreamGenerateMusicRequest,
|
|
71
|
+
) => Stream.Stream<MusicStreamEvent, AiError.AiError | E, R>
|
|
72
|
+
/**
|
|
73
|
+
* Free endpoint (rate-limited, no credit cost): turn a prompt into a
|
|
74
|
+
* structured `MusicPrompt` plan. Feed the result back into
|
|
75
|
+
* `generate` / `streamGeneration` via `compositionPlan`.
|
|
76
|
+
*/
|
|
77
|
+
readonly createCompositionPlan: (
|
|
78
|
+
request: CreateCompositionPlanRequest,
|
|
79
|
+
) => Effect.Effect<ElevenLabsCompositionPlan, AiError.AiError>
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
export class ElevenLabsMusicGenerator extends Context.Service<
|
|
83
|
+
ElevenLabsMusicGenerator,
|
|
84
|
+
ElevenLabsMusicGeneratorService
|
|
85
|
+
>()("@betalyra/effect-uai/providers/elevenlabs/ElevenLabsMusicGenerator") {}
|
|
86
|
+
|
|
87
|
+
export type Config = {
|
|
88
|
+
readonly apiKey: Redacted.Redacted
|
|
89
|
+
readonly baseUrl?: string
|
|
90
|
+
readonly region?: ElevenLabsRegion
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
const DEFAULT_MODEL: ElevenLabsMusicModel = "music_v1"
|
|
94
|
+
|
|
95
|
+
// ---------------------------------------------------------------------------
|
|
96
|
+
// HTTP errors (music namespace; reuses the same status taxonomy)
|
|
97
|
+
// ---------------------------------------------------------------------------
|
|
98
|
+
|
|
99
|
+
const musicTransportFailure = (cause: unknown): AiError.AiError =>
|
|
100
|
+
new AiError.Unavailable({ provider: "elevenlabs-music", raw: cause })
|
|
101
|
+
|
|
102
|
+
const musicHttpStatusError = (status: number, body: string): AiError.AiError => {
|
|
103
|
+
if (status === 429) return new AiError.RateLimited({ provider: "elevenlabs-music", raw: body })
|
|
104
|
+
if (status === 408 || status === 504)
|
|
105
|
+
return new AiError.Timeout({ provider: "elevenlabs-music", raw: body })
|
|
106
|
+
if (status === 401)
|
|
107
|
+
return new AiError.AuthFailed({ provider: "elevenlabs-music", subtype: "auth", raw: body })
|
|
108
|
+
if (status === 403)
|
|
109
|
+
return new AiError.AuthFailed({
|
|
110
|
+
provider: "elevenlabs-music",
|
|
111
|
+
subtype: "permission",
|
|
112
|
+
raw: body,
|
|
113
|
+
})
|
|
114
|
+
if (status === 402)
|
|
115
|
+
return new AiError.AuthFailed({
|
|
116
|
+
provider: "elevenlabs-music",
|
|
117
|
+
subtype: "billing",
|
|
118
|
+
raw: body,
|
|
119
|
+
})
|
|
120
|
+
if (status >= 500)
|
|
121
|
+
return new AiError.Unavailable({ provider: "elevenlabs-music", status, raw: body })
|
|
122
|
+
return new AiError.InvalidRequest({ provider: "elevenlabs-music", raw: body })
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
// ---------------------------------------------------------------------------
|
|
126
|
+
// Wire body
|
|
127
|
+
// ---------------------------------------------------------------------------
|
|
128
|
+
|
|
129
|
+
/**
|
|
130
|
+
* Build the JSON body for `/v1/music` (sync + stream share the body).
|
|
131
|
+
*
|
|
132
|
+
* Mutual exclusion: `compositionPlan` and prompt-mode fields cannot
|
|
133
|
+
* coexist. When `compositionPlan` is set, `prompt` must be empty and
|
|
134
|
+
* `duration` must be unset (section durations live on the plan).
|
|
135
|
+
*/
|
|
136
|
+
const buildBody = (request: ElevenLabsMusicGenerateRequest) =>
|
|
137
|
+
Effect.gen(function* () {
|
|
138
|
+
const model = request.model ?? DEFAULT_MODEL
|
|
139
|
+
if (request.compositionPlan !== undefined) {
|
|
140
|
+
if (request.prompt.length > 0) {
|
|
141
|
+
return yield* Effect.fail(
|
|
142
|
+
new AiError.InvalidRequest({
|
|
143
|
+
provider: "elevenlabs-music",
|
|
144
|
+
param: "prompt",
|
|
145
|
+
raw: "`prompt` and `compositionPlan` are mutually exclusive. Pass one or the other (compositionPlan carries its own per-section lyrics and styles).",
|
|
146
|
+
}),
|
|
147
|
+
)
|
|
148
|
+
}
|
|
149
|
+
if (request.duration !== undefined) {
|
|
150
|
+
return yield* Effect.fail(
|
|
151
|
+
new AiError.InvalidRequest({
|
|
152
|
+
provider: "elevenlabs-music",
|
|
153
|
+
param: "duration",
|
|
154
|
+
raw: "`duration` is ignored when `compositionPlan` is set; section durations live on the plan. Remove `duration`.",
|
|
155
|
+
}),
|
|
156
|
+
)
|
|
157
|
+
}
|
|
158
|
+
return {
|
|
159
|
+
composition_plan: wireCompositionPlan(request.compositionPlan),
|
|
160
|
+
model_id: model,
|
|
161
|
+
...(request.seed !== undefined && { seed: request.seed }),
|
|
162
|
+
...(request.forceInstrumental !== undefined && {
|
|
163
|
+
force_instrumental: request.forceInstrumental,
|
|
164
|
+
}),
|
|
165
|
+
...(request.respectSectionsDurations !== undefined && {
|
|
166
|
+
respect_sections_durations: request.respectSectionsDurations,
|
|
167
|
+
}),
|
|
168
|
+
...(request.signWithC2pa !== undefined && { sign_with_c2pa: request.signWithC2pa }),
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
yield* warnDroppedPromptModeHints(request as CommonGenerateMusicRequest)
|
|
172
|
+
return {
|
|
173
|
+
prompt: request.prompt,
|
|
174
|
+
model_id: model,
|
|
175
|
+
...(request.duration !== undefined && {
|
|
176
|
+
music_length_ms: Duration.toMillis(request.duration),
|
|
177
|
+
}),
|
|
178
|
+
...(request.seed !== undefined && { seed: request.seed }),
|
|
179
|
+
...(request.forceInstrumental !== undefined && {
|
|
180
|
+
force_instrumental: request.forceInstrumental,
|
|
181
|
+
}),
|
|
182
|
+
...(request.signWithC2pa !== undefined && { sign_with_c2pa: request.signWithC2pa }),
|
|
183
|
+
}
|
|
184
|
+
})
|
|
185
|
+
|
|
186
|
+
const buildMusicHttpRequest = (
|
|
187
|
+
cfg: Config,
|
|
188
|
+
request: ElevenLabsMusicGenerateRequest,
|
|
189
|
+
path: "" | "/stream",
|
|
190
|
+
) =>
|
|
191
|
+
Effect.gen(function* () {
|
|
192
|
+
const format: AudioFormat = request.outputFormat ?? defaultFormat
|
|
193
|
+
const slug = yield* formatToOutputSlug(format)
|
|
194
|
+
const body = yield* buildBody(request)
|
|
195
|
+
const url = `${resolveHost(cfg)}/music${path}?output_format=${slug}`
|
|
196
|
+
const httpRequest = HttpClientRequest.post(url).pipe(
|
|
197
|
+
HttpClientRequest.setHeader("xi-api-key", Redacted.value(cfg.apiKey)),
|
|
198
|
+
HttpClientRequest.bodyJsonUnsafe(body),
|
|
199
|
+
)
|
|
200
|
+
return { httpRequest, format }
|
|
201
|
+
})
|
|
202
|
+
|
|
203
|
+
// ---------------------------------------------------------------------------
|
|
204
|
+
// Service implementation
|
|
205
|
+
// ---------------------------------------------------------------------------
|
|
206
|
+
|
|
207
|
+
const generateImpl =
|
|
208
|
+
(cfg: Config) =>
|
|
209
|
+
(
|
|
210
|
+
request: ElevenLabsMusicGenerateRequest,
|
|
211
|
+
): Effect.Effect<GenerateResult, AiError.AiError, HttpClient.HttpClient> =>
|
|
212
|
+
Effect.gen(function* () {
|
|
213
|
+
const client = yield* HttpClient.HttpClient
|
|
214
|
+
const { httpRequest, format } = yield* buildMusicHttpRequest(cfg, request, "")
|
|
215
|
+
const response = yield* client
|
|
216
|
+
.execute(httpRequest)
|
|
217
|
+
.pipe(Effect.mapError(musicTransportFailure))
|
|
218
|
+
if (response.status >= 400) {
|
|
219
|
+
const text = yield* response.text.pipe(Effect.orElseSucceed(() => ""))
|
|
220
|
+
return yield* Effect.fail(musicHttpStatusError(response.status, text))
|
|
221
|
+
}
|
|
222
|
+
const buffer = yield* response.arrayBuffer.pipe(Effect.mapError(musicTransportFailure))
|
|
223
|
+
const bytes = new Uint8Array(buffer)
|
|
224
|
+
const songId = response.headers["song_id"] ?? response.headers["x-song-id"]
|
|
225
|
+
const result: MusicResult = {
|
|
226
|
+
audio: { format, bytes },
|
|
227
|
+
provider: "elevenlabs-music",
|
|
228
|
+
...(typeof songId === "string" && { songId }),
|
|
229
|
+
...(request.signWithC2pa === true && { watermark: "c2pa" as const }),
|
|
230
|
+
}
|
|
231
|
+
return singleVariant(result)
|
|
232
|
+
})
|
|
233
|
+
|
|
234
|
+
const streamGenerationImpl =
|
|
235
|
+
(cfg: Config) =>
|
|
236
|
+
(
|
|
237
|
+
request: ElevenLabsMusicGenerateRequest,
|
|
238
|
+
): Stream.Stream<AudioChunk, AiError.AiError, HttpClient.HttpClient> =>
|
|
239
|
+
Stream.unwrap(
|
|
240
|
+
Effect.gen(function* () {
|
|
241
|
+
const client = yield* HttpClient.HttpClient
|
|
242
|
+
const { httpRequest } = yield* buildMusicHttpRequest(cfg, request, "/stream")
|
|
243
|
+
const response = yield* client
|
|
244
|
+
.execute(httpRequest)
|
|
245
|
+
.pipe(Effect.mapError(musicTransportFailure))
|
|
246
|
+
if (response.status >= 400) {
|
|
247
|
+
const text = yield* response.text.pipe(Effect.orElseSucceed(() => ""))
|
|
248
|
+
return Stream.fail(musicHttpStatusError(response.status, text))
|
|
249
|
+
}
|
|
250
|
+
return response.stream.pipe(
|
|
251
|
+
Stream.mapError(musicTransportFailure),
|
|
252
|
+
Stream.map((bytes): AudioChunk => ({ bytes })),
|
|
253
|
+
)
|
|
254
|
+
}),
|
|
255
|
+
)
|
|
256
|
+
|
|
257
|
+
/**
|
|
258
|
+
* ElevenLabs music has no bidirectional session at the wire level.
|
|
259
|
+
* Layer does not register `MusicInteractiveSession`, so callers using
|
|
260
|
+
* `MusicGenerator.streamGenerationFrom` get a compile-time error
|
|
261
|
+
* before this runtime fallback fires.
|
|
262
|
+
*/
|
|
263
|
+
const streamGenerationFromUnsupported = <E, R>(
|
|
264
|
+
_input: Stream.Stream<MusicSessionInput, E, R>,
|
|
265
|
+
_request: CommonStreamGenerateMusicRequest,
|
|
266
|
+
): Stream.Stream<MusicStreamEvent, AiError.AiError | E, R> =>
|
|
267
|
+
Stream.fail(
|
|
268
|
+
new AiError.Unsupported({
|
|
269
|
+
provider: "elevenlabs-music",
|
|
270
|
+
capability: "streamGenerationFrom",
|
|
271
|
+
reason:
|
|
272
|
+
"ElevenLabs music has no bidirectional session. Use `streamGeneration` for chunked output, or `createCompositionPlan` + `generate` for structured plans.",
|
|
273
|
+
}),
|
|
274
|
+
)
|
|
275
|
+
|
|
276
|
+
const createCompositionPlanImpl =
|
|
277
|
+
(cfg: Config) =>
|
|
278
|
+
(
|
|
279
|
+
request: CreateCompositionPlanRequest,
|
|
280
|
+
): Effect.Effect<ElevenLabsCompositionPlan, AiError.AiError, HttpClient.HttpClient> =>
|
|
281
|
+
Effect.gen(function* () {
|
|
282
|
+
const client = yield* HttpClient.HttpClient
|
|
283
|
+
const body = {
|
|
284
|
+
prompt: request.prompt,
|
|
285
|
+
model_id: request.model ?? DEFAULT_MODEL,
|
|
286
|
+
...(request.duration !== undefined && {
|
|
287
|
+
music_length_ms: Duration.toMillis(request.duration),
|
|
288
|
+
}),
|
|
289
|
+
...(request.sourceCompositionPlan !== undefined && {
|
|
290
|
+
source_composition_plan: wireCompositionPlan(request.sourceCompositionPlan),
|
|
291
|
+
}),
|
|
292
|
+
}
|
|
293
|
+
const httpRequest = HttpClientRequest.post(`${resolveHost(cfg)}/music/plan`).pipe(
|
|
294
|
+
HttpClientRequest.setHeader("xi-api-key", Redacted.value(cfg.apiKey)),
|
|
295
|
+
HttpClientRequest.bodyJsonUnsafe(body),
|
|
296
|
+
)
|
|
297
|
+
const response = yield* client
|
|
298
|
+
.execute(httpRequest)
|
|
299
|
+
.pipe(Effect.mapError(musicTransportFailure))
|
|
300
|
+
if (response.status >= 400) {
|
|
301
|
+
const text = yield* response.text.pipe(Effect.orElseSucceed(() => ""))
|
|
302
|
+
return yield* Effect.fail(musicHttpStatusError(response.status, text))
|
|
303
|
+
}
|
|
304
|
+
const json = yield* response.json.pipe(Effect.mapError(musicTransportFailure))
|
|
305
|
+
return decodeCompositionPlan(json)
|
|
306
|
+
})
|
|
307
|
+
|
|
308
|
+
// ---------------------------------------------------------------------------
|
|
309
|
+
// Constructors
|
|
310
|
+
// ---------------------------------------------------------------------------
|
|
311
|
+
|
|
312
|
+
export const make = (
|
|
313
|
+
cfg: Config,
|
|
314
|
+
): Effect.Effect<ElevenLabsMusicGeneratorService, never, HttpClient.HttpClient> =>
|
|
315
|
+
Effect.map(HttpClient.HttpClient.asEffect(), (client) => ({
|
|
316
|
+
generate: (request) =>
|
|
317
|
+
generateImpl(cfg)(request).pipe(Effect.provideService(HttpClient.HttpClient, client)),
|
|
318
|
+
streamGeneration: (request) =>
|
|
319
|
+
streamGenerationImpl(cfg)(request).pipe(Stream.provideService(HttpClient.HttpClient, client)),
|
|
320
|
+
streamGenerationFrom: streamGenerationFromUnsupported,
|
|
321
|
+
createCompositionPlan: (request) =>
|
|
322
|
+
createCompositionPlanImpl(cfg)(request).pipe(
|
|
323
|
+
Effect.provideService(HttpClient.HttpClient, client),
|
|
324
|
+
),
|
|
325
|
+
}))
|
|
326
|
+
|
|
327
|
+
/**
|
|
328
|
+
* Layer that registers the provider-specific `ElevenLabsMusicGenerator`
|
|
329
|
+
* tag and the generic `MusicGenerator` tag.
|
|
330
|
+
*
|
|
331
|
+
* Does NOT register `MusicInteractiveSession`. ElevenLabs music has no
|
|
332
|
+
* bidirectional wire endpoint; `streamGenerationFrom` against this
|
|
333
|
+
* Layer alone fails to typecheck.
|
|
334
|
+
*/
|
|
335
|
+
export const layer = (
|
|
336
|
+
cfg: Config,
|
|
337
|
+
): Layer.Layer<ElevenLabsMusicGenerator | MusicGenerator, never, HttpClient.HttpClient> =>
|
|
338
|
+
Layer.merge(
|
|
339
|
+
Layer.effect(ElevenLabsMusicGenerator, make(cfg)),
|
|
340
|
+
Layer.effect(
|
|
341
|
+
MusicGenerator,
|
|
342
|
+
Effect.map(
|
|
343
|
+
make(cfg),
|
|
344
|
+
(s): MusicGeneratorService => ({
|
|
345
|
+
generate: (req: CommonGenerateMusicRequest) =>
|
|
346
|
+
s.generate({ ...req, model: req.model as ElevenLabsMusicModel }),
|
|
347
|
+
streamGeneration: (req: CommonStreamGenerateMusicRequest) =>
|
|
348
|
+
s.streamGeneration({ ...req, model: req.model as ElevenLabsMusicModel }),
|
|
349
|
+
streamGenerationFrom: s.streamGenerationFrom,
|
|
350
|
+
}),
|
|
351
|
+
),
|
|
352
|
+
),
|
|
353
|
+
)
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { Context, Effect, Layer, Redacted, Stream } from "effect"
|
|
2
2
|
import { HttpClient, HttpClientRequest } from "effect/unstable/http"
|
|
3
3
|
import * as Socket from "effect/unstable/socket/Socket"
|
|
4
4
|
import type * as AiError from "@effect-uai/core/AiError"
|
|
@@ -7,7 +7,6 @@ import {
|
|
|
7
7
|
type CommonStreamSynthesizeRequest,
|
|
8
8
|
type CommonSynthesizeDialogueRequest,
|
|
9
9
|
type CommonSynthesizeRequest,
|
|
10
|
-
type CustomPronunciation,
|
|
11
10
|
MultiSpeakerTts,
|
|
12
11
|
SpeechSynthesizer,
|
|
13
12
|
type SpeechSynthesizerService,
|
|
@@ -17,8 +16,11 @@ import {
|
|
|
17
16
|
defaultFormat,
|
|
18
17
|
formatToOutputSlug,
|
|
19
18
|
httpStatusError,
|
|
19
|
+
type PronunciationDictionaryLocator,
|
|
20
|
+
rejectInlinePronunciations,
|
|
20
21
|
transportFailure,
|
|
21
22
|
type VoiceSettings,
|
|
23
|
+
wirePronunciationLocators,
|
|
22
24
|
wireVoiceSettings,
|
|
23
25
|
} from "./codec.js"
|
|
24
26
|
import type { ElevenLabsTtsModel, ElevenLabsVoiceId } from "./models.js"
|
|
@@ -34,6 +36,11 @@ import { type ElevenLabsRegion, resolveHost } from "./region.js"
|
|
|
34
36
|
* provider's prosody/timbre controls; `seed` makes generation
|
|
35
37
|
* deterministic; `previousText` / `nextText` thread context across
|
|
36
38
|
* sequential calls for natural prosody.
|
|
39
|
+
*
|
|
40
|
+
* `pronunciationDictionaryLocators` references pre-provisioned
|
|
41
|
+
* pronunciation dictionaries by ID. Inline `pronunciations` (IPA) on
|
|
42
|
+
* the Common request are not supported by ElevenLabs and fail
|
|
43
|
+
* `Unsupported`; use a dictionary for phonetic control.
|
|
37
44
|
*/
|
|
38
45
|
export type ElevenLabsSynthesizeRequest = Omit<CommonSynthesizeRequest, "model" | "voiceId"> & {
|
|
39
46
|
readonly model: ElevenLabsTtsModel
|
|
@@ -42,6 +49,7 @@ export type ElevenLabsSynthesizeRequest = Omit<CommonSynthesizeRequest, "model"
|
|
|
42
49
|
readonly seed?: number
|
|
43
50
|
readonly previousText?: string
|
|
44
51
|
readonly nextText?: string
|
|
52
|
+
readonly pronunciationDictionaryLocators?: ReadonlyArray<PronunciationDictionaryLocator>
|
|
45
53
|
}
|
|
46
54
|
|
|
47
55
|
export type ElevenLabsSynthesizerService = {
|
|
@@ -65,64 +73,18 @@ export type Config = {
|
|
|
65
73
|
readonly region?: ElevenLabsRegion
|
|
66
74
|
}
|
|
67
75
|
|
|
68
|
-
// ---------------------------------------------------------------------------
|
|
69
|
-
// Pronunciations — SSML <phoneme> rewrite (gated to legacy models)
|
|
70
|
-
// ---------------------------------------------------------------------------
|
|
71
|
-
|
|
72
|
-
/**
|
|
73
|
-
* Models that honor inline `<phoneme>` SSML tags. Other models silently
|
|
74
|
-
* drop the tags server-side, so we don't bother rewriting the text for
|
|
75
|
-
* them — the pronunciation overrides are silently ignored.
|
|
76
|
-
*
|
|
77
|
-
* Reference: https://elevenlabs.io/docs/best-practices/prompting/controls
|
|
78
|
-
*/
|
|
79
|
-
const PHONEME_SUPPORTED_MODELS: ReadonlySet<string> = new Set([
|
|
80
|
-
"eleven_flash_v2",
|
|
81
|
-
"eleven_english_v1",
|
|
82
|
-
"eleven_monolingual_v1",
|
|
83
|
-
])
|
|
84
|
-
|
|
85
|
-
const wireAlphabet = (encoding: CustomPronunciation["encoding"]): string | undefined =>
|
|
86
|
-
encoding === "ipa" ? "ipa" : encoding === "cmu-arpabet" ? "cmu-arpabet" : undefined
|
|
87
|
-
|
|
88
|
-
const phonemeTag = (phrase: string, pronunciation: string, alphabet: string): string =>
|
|
89
|
-
`<phoneme alphabet="${alphabet}" ph="${pronunciation}">${phrase}</phoneme>`
|
|
90
|
-
|
|
91
|
-
const escapeForRegex = (s: string): string => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")
|
|
92
|
-
|
|
93
|
-
/**
|
|
94
|
-
* Apply pronunciation overrides to a text string. Only applies for
|
|
95
|
-
* phoneme-supported models; only IPA and CMU Arpabet are honored
|
|
96
|
-
* (x-sampa is silently dropped). Whole-word, case-insensitive
|
|
97
|
-
* replacement.
|
|
98
|
-
*/
|
|
99
|
-
const applyPronunciations = (
|
|
100
|
-
text: string,
|
|
101
|
-
model: string,
|
|
102
|
-
pronunciations: ReadonlyArray<CustomPronunciation> | undefined,
|
|
103
|
-
): string => {
|
|
104
|
-
if (pronunciations === undefined || !PHONEME_SUPPORTED_MODELS.has(model)) return text
|
|
105
|
-
return Arr.reduce(pronunciations, text, (acc, p) => {
|
|
106
|
-
const alphabet = wireAlphabet(p.encoding)
|
|
107
|
-
if (alphabet === undefined) return acc
|
|
108
|
-
return acc.replace(
|
|
109
|
-
new RegExp(`\\b${escapeForRegex(p.phrase)}\\b`, "i"),
|
|
110
|
-
phonemeTag(p.phrase, p.pronunciation, alphabet),
|
|
111
|
-
)
|
|
112
|
-
})
|
|
113
|
-
}
|
|
114
|
-
|
|
115
76
|
// ---------------------------------------------------------------------------
|
|
116
77
|
// Codec — request → JSON body
|
|
117
78
|
// ---------------------------------------------------------------------------
|
|
118
79
|
|
|
119
80
|
const buildBody = (r: ElevenLabsSynthesizeRequest) => ({
|
|
120
|
-
text:
|
|
81
|
+
text: r.text,
|
|
121
82
|
model_id: r.model,
|
|
122
83
|
...(r.languageCode !== undefined && { language_code: r.languageCode }),
|
|
123
84
|
...(r.seed !== undefined && { seed: r.seed }),
|
|
124
85
|
...(r.previousText !== undefined && { previous_text: r.previousText }),
|
|
125
86
|
...(r.nextText !== undefined && { next_text: r.nextText }),
|
|
87
|
+
...wirePronunciationLocators(r.pronunciationDictionaryLocators),
|
|
126
88
|
...(r.voiceSettings !== undefined && { voice_settings: wireVoiceSettings(r.voiceSettings) }),
|
|
127
89
|
})
|
|
128
90
|
|
|
@@ -135,7 +97,7 @@ const DEFAULT_DIALOGUE_MODEL = "eleven_v3"
|
|
|
135
97
|
const buildDialogueBody = (r: CommonSynthesizeDialogueRequest) => ({
|
|
136
98
|
inputs: r.turns.map((t) => ({
|
|
137
99
|
voice_id: t.voiceId,
|
|
138
|
-
text:
|
|
100
|
+
text: t.text,
|
|
139
101
|
})),
|
|
140
102
|
model_id: r.model ?? DEFAULT_DIALOGUE_MODEL,
|
|
141
103
|
...(r.languageCode !== undefined && { language_code: r.languageCode }),
|
|
@@ -159,6 +121,7 @@ const buildDialogueHttpRequest = (
|
|
|
159
121
|
|
|
160
122
|
const synthesizeDialogueImpl = (cfg: Config) => (request: CommonSynthesizeDialogueRequest) =>
|
|
161
123
|
Effect.gen(function* () {
|
|
124
|
+
yield* rejectInlinePronunciations(request.pronunciations)
|
|
162
125
|
const client = yield* HttpClient.HttpClient
|
|
163
126
|
const { httpRequest, format } = yield* buildDialogueHttpRequest(cfg, request, "")
|
|
164
127
|
const response = yield* client.execute(httpRequest).pipe(Effect.mapError(transportFailure))
|
|
@@ -173,6 +136,7 @@ const synthesizeDialogueImpl = (cfg: Config) => (request: CommonSynthesizeDialog
|
|
|
173
136
|
const streamSynthesizeDialogueImpl = (cfg: Config) => (request: CommonSynthesizeDialogueRequest) =>
|
|
174
137
|
Stream.unwrap(
|
|
175
138
|
Effect.gen(function* () {
|
|
139
|
+
yield* rejectInlinePronunciations(request.pronunciations)
|
|
176
140
|
const client = yield* HttpClient.HttpClient
|
|
177
141
|
const { httpRequest } = yield* buildDialogueHttpRequest(cfg, request, "/stream")
|
|
178
142
|
const response = yield* client.execute(httpRequest).pipe(Effect.mapError(transportFailure))
|
|
@@ -205,6 +169,7 @@ const buildHttpRequest = (cfg: Config, r: ElevenLabsSynthesizeRequest, path: ""
|
|
|
205
169
|
|
|
206
170
|
const synthesizeImpl = (cfg: Config) => (request: ElevenLabsSynthesizeRequest) =>
|
|
207
171
|
Effect.gen(function* () {
|
|
172
|
+
yield* rejectInlinePronunciations(request.pronunciations)
|
|
208
173
|
const client = yield* HttpClient.HttpClient
|
|
209
174
|
const { httpRequest, format } = yield* buildHttpRequest(cfg, request, "")
|
|
210
175
|
const response = yield* client.execute(httpRequest).pipe(Effect.mapError(transportFailure))
|
|
@@ -219,6 +184,7 @@ const synthesizeImpl = (cfg: Config) => (request: ElevenLabsSynthesizeRequest) =
|
|
|
219
184
|
const streamSynthesisImpl = (cfg: Config) => (request: ElevenLabsSynthesizeRequest) =>
|
|
220
185
|
Stream.unwrap(
|
|
221
186
|
Effect.gen(function* () {
|
|
187
|
+
yield* rejectInlinePronunciations(request.pronunciations)
|
|
222
188
|
const client = yield* HttpClient.HttpClient
|
|
223
189
|
const { httpRequest } = yield* buildHttpRequest(cfg, request, "/stream")
|
|
224
190
|
const response = yield* client.execute(httpRequest).pipe(Effect.mapError(transportFailure))
|
|
@@ -2,6 +2,7 @@ import { Context, Effect, Layer, Redacted, Schema, Stream } from "effect"
|
|
|
2
2
|
import { HttpClient, HttpClientRequest } from "effect/unstable/http"
|
|
3
3
|
import * as Socket from "effect/unstable/socket/Socket"
|
|
4
4
|
import * as AiError from "@effect-uai/core/AiError"
|
|
5
|
+
import * as Capabilities from "@effect-uai/core/Capabilities"
|
|
5
6
|
import type { TranscriptResult, WordTimestamp } from "@effect-uai/core/Transcript"
|
|
6
7
|
import {
|
|
7
8
|
type CommonTranscribeRequest,
|
|
@@ -101,6 +102,16 @@ const buildForm = (request: ElevenLabsTranscribeRequest) =>
|
|
|
101
102
|
if (request.numSpeakers !== undefined) form.set("num_speakers", String(request.numSpeakers))
|
|
102
103
|
if (request.audioEvents !== undefined)
|
|
103
104
|
form.set("tag_audio_events", request.audioEvents ? "true" : "false")
|
|
105
|
+
// Vocab biasing → Scribe `keyterms` (≤1000 terms, ≤50 chars). Each
|
|
106
|
+
// term is a repeated form field (`keyterms=a&keyterms=b`), matching
|
|
107
|
+
// the ElevenLabs SDK. Scribe has no free-form prompt field.
|
|
108
|
+
for (const term of request.biasingTerms ?? []) form.append("keyterms", term)
|
|
109
|
+
yield* Capabilities.warnDroppedWhen(request.prompt, {
|
|
110
|
+
provider: "elevenlabs",
|
|
111
|
+
capability: "prompt",
|
|
112
|
+
field: "prompt",
|
|
113
|
+
reason: "ElevenLabs Scribe has no free-form prompt field; bias via `biasingTerms`.",
|
|
114
|
+
})
|
|
104
115
|
return form
|
|
105
116
|
})
|
|
106
117
|
|
package/src/codec.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { Array as Arr, Effect, Encoding, Match, Option, Result } from "effect"
|
|
2
2
|
import * as AiError from "@effect-uai/core/AiError"
|
|
3
3
|
import type { AudioFormat, AudioSource } from "@effect-uai/core/Audio"
|
|
4
|
+
import type { CustomPronunciation } from "@effect-uai/core/SpeechSynthesizer"
|
|
4
5
|
|
|
5
6
|
// ---------------------------------------------------------------------------
|
|
6
7
|
// Shared TTS voice-settings shape (sync + streaming use the same fields)
|
|
@@ -25,6 +26,52 @@ export const wireVoiceSettings = (v: VoiceSettings | undefined) =>
|
|
|
25
26
|
...(v.speed !== undefined && { speed: v.speed }),
|
|
26
27
|
}
|
|
27
28
|
|
|
29
|
+
// ---------------------------------------------------------------------------
|
|
30
|
+
// Pronunciation: dictionary locators (supported) vs inline IPA (rejected)
|
|
31
|
+
// ---------------------------------------------------------------------------
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* Reference to a pre-provisioned ElevenLabs pronunciation dictionary.
|
|
35
|
+
* The caller provisions and versions the dictionary out-of-band
|
|
36
|
+
* (dashboard or the pronunciation-dictionary API); we only reference it.
|
|
37
|
+
*/
|
|
38
|
+
export type PronunciationDictionaryLocator = {
|
|
39
|
+
readonly dictionaryId: string
|
|
40
|
+
readonly versionId: string
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
export const wirePronunciationLocators = (
|
|
44
|
+
locators: ReadonlyArray<PronunciationDictionaryLocator> | undefined,
|
|
45
|
+
) =>
|
|
46
|
+
locators === undefined || locators.length === 0
|
|
47
|
+
? undefined
|
|
48
|
+
: {
|
|
49
|
+
pronunciation_dictionary_locators: locators.map((l) => ({
|
|
50
|
+
pronunciation_dictionary_id: l.dictionaryId,
|
|
51
|
+
version_id: l.versionId,
|
|
52
|
+
})),
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* ElevenLabs has no stateless inline IPA path (legacy `<phoneme>` SSML
|
|
57
|
+
* is dropped; modern phonetics live in pre-uploaded dictionaries).
|
|
58
|
+
* Inline `pronunciations` are load-bearing (bucket 1), so reject rather
|
|
59
|
+
* than silently mispronounce. Use `pronunciationDictionaryLocators`.
|
|
60
|
+
*/
|
|
61
|
+
export const rejectInlinePronunciations = (
|
|
62
|
+
pronunciations: ReadonlyArray<CustomPronunciation> | undefined,
|
|
63
|
+
): Effect.Effect<void, AiError.AiError> =>
|
|
64
|
+
pronunciations !== undefined && pronunciations.length > 0
|
|
65
|
+
? Effect.fail(
|
|
66
|
+
new AiError.Unsupported({
|
|
67
|
+
provider: "elevenlabs",
|
|
68
|
+
capability: "pronunciations",
|
|
69
|
+
reason:
|
|
70
|
+
"ElevenLabs has no stateless inline IPA path. Provision a pronunciation dictionary and pass `pronunciationDictionaryLocators`, or use a provider with inline phonemes (Inworld, Google).",
|
|
71
|
+
}),
|
|
72
|
+
)
|
|
73
|
+
: Effect.void
|
|
74
|
+
|
|
28
75
|
// ---------------------------------------------------------------------------
|
|
29
76
|
// AudioFormat → ElevenLabs `output_format` query value
|
|
30
77
|
// ---------------------------------------------------------------------------
|
package/src/index.ts
CHANGED
package/src/models.ts
CHANGED
|
@@ -44,3 +44,20 @@ export type ElevenLabsSttModel =
|
|
|
44
44
|
* full catalog is fetched dynamically via `GET /v1/voices`.
|
|
45
45
|
*/
|
|
46
46
|
export type ElevenLabsVoiceId = string
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* Known ElevenLabs music model identifiers.
|
|
50
|
+
*
|
|
51
|
+
* - `music_v1` — current API default (as of May 2026). General-purpose
|
|
52
|
+
* music composition.
|
|
53
|
+
* - `music_v2` — UI default; API plumbing in progress per the
|
|
54
|
+
* May 26 2026 v2 launch blog. Include in the literal union so
|
|
55
|
+
* callers can opt in when it ships.
|
|
56
|
+
*
|
|
57
|
+
* Reference: https://elevenlabs.io/docs/api-reference/music/compose
|
|
58
|
+
*/
|
|
59
|
+
export type ElevenLabsMusicModel =
|
|
60
|
+
| "music_v1"
|
|
61
|
+
| "music_v2"
|
|
62
|
+
// eslint-disable-next-line @typescript-eslint/ban-types
|
|
63
|
+
| (string & {})
|