@ai-sdk/elevenlabs 3.0.52 → 3.0.53
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +9 -0
- package/dist/index.d.ts +139 -132
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +685 -863
- package/dist/index.js.map +1 -1
- package/package.json +8 -8
package/dist/index.js
CHANGED
|
@@ -1,881 +1,703 @@
|
|
|
1
|
-
|
|
2
|
-
import {
|
|
3
|
-
NoSuchModelError
|
|
4
|
-
} from "@ai-sdk/provider";
|
|
5
|
-
import {
|
|
6
|
-
loadApiKey,
|
|
7
|
-
withUserAgentSuffix
|
|
8
|
-
} from "@ai-sdk/provider-utils";
|
|
9
|
-
|
|
10
|
-
// src/elevenlabs-transcription-model.ts
|
|
11
|
-
import {
|
|
12
|
-
InvalidArgumentError,
|
|
13
|
-
UnsupportedFunctionalityError
|
|
14
|
-
} from "@ai-sdk/provider";
|
|
15
|
-
import {
|
|
16
|
-
combineHeaders,
|
|
17
|
-
convertBase64ToUint8Array,
|
|
18
|
-
convertToBase64,
|
|
19
|
-
connectToWebSocket,
|
|
20
|
-
createJsonResponseHandler,
|
|
21
|
-
mediaTypeToExtension,
|
|
22
|
-
parseProviderOptions,
|
|
23
|
-
postFormDataToApi,
|
|
24
|
-
safeParseJSON,
|
|
25
|
-
serializeModelOptions,
|
|
26
|
-
toWebSocketUrl,
|
|
27
|
-
waitForWebSocketBufferDrain,
|
|
28
|
-
WORKFLOW_SERIALIZE,
|
|
29
|
-
WORKFLOW_DESERIALIZE
|
|
30
|
-
} from "@ai-sdk/provider-utils";
|
|
31
|
-
import { z as z3 } from "zod/v4";
|
|
32
|
-
|
|
33
|
-
// src/elevenlabs-error.ts
|
|
1
|
+
import { InvalidArgumentError, NoSuchModelError, UnsupportedFunctionalityError } from "@ai-sdk/provider";
|
|
2
|
+
import { WORKFLOW_DESERIALIZE, WORKFLOW_SERIALIZE, combineHeaders, connectToWebSocket, convertBase64ToUint8Array, convertToBase64, createBinaryResponseHandler, createJsonErrorResponseHandler, createJsonResponseHandler, loadApiKey, mediaTypeToExtension, parseProviderOptions, postFormDataToApi, postJsonToApi, safeParseJSON, serializeModelOptions, toWebSocketUrl, waitForWebSocketBufferDrain, withUserAgentSuffix } from "@ai-sdk/provider-utils";
|
|
34
3
|
import { z } from "zod/v4";
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
errorSchema: elevenlabsErrorDataSchema,
|
|
44
|
-
errorToMessage: (data) => data.error.message
|
|
4
|
+
//#region src/elevenlabs-error.ts
|
|
5
|
+
const elevenlabsErrorDataSchema = z.object({ error: z.object({
|
|
6
|
+
message: z.string(),
|
|
7
|
+
code: z.number()
|
|
8
|
+
}) });
|
|
9
|
+
const elevenlabsFailedResponseHandler = createJsonErrorResponseHandler({
|
|
10
|
+
errorSchema: elevenlabsErrorDataSchema,
|
|
11
|
+
errorToMessage: (data) => data.error.message
|
|
45
12
|
});
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
13
|
+
//#endregion
|
|
14
|
+
//#region src/elevenlabs-transcription-model-options.ts
|
|
15
|
+
const elevenLabsTranscriptionModelOptionsSchema = z.object({
|
|
16
|
+
languageCode: z.string().nullish(),
|
|
17
|
+
tagAudioEvents: z.boolean().nullish().default(true),
|
|
18
|
+
numSpeakers: z.number().int().min(1).max(32).nullish(),
|
|
19
|
+
timestampsGranularity: z.enum([
|
|
20
|
+
"none",
|
|
21
|
+
"word",
|
|
22
|
+
"character"
|
|
23
|
+
]).nullish().default("word"),
|
|
24
|
+
diarize: z.boolean().nullish().default(false),
|
|
25
|
+
fileFormat: z.enum(["pcm_s16le_16", "other"]).nullish().default("other"),
|
|
26
|
+
streaming: z.object({
|
|
27
|
+
commitStrategy: z.enum(["manual", "vad"]).nullish(),
|
|
28
|
+
enableLogging: z.boolean().nullish(),
|
|
29
|
+
filterBackgroundAudio: z.boolean().nullish(),
|
|
30
|
+
includeLanguageDetection: z.boolean().nullish(),
|
|
31
|
+
includeTimestamps: z.boolean().nullish(),
|
|
32
|
+
keyterms: z.array(z.string().max(20)).max(50).nullish(),
|
|
33
|
+
minSilenceDurationMs: z.number().int().min(50).max(2e3).nullish(),
|
|
34
|
+
minSpeechDurationMs: z.number().int().min(50).max(2e3).nullish(),
|
|
35
|
+
noVerbatim: z.boolean().nullish(),
|
|
36
|
+
previousText: z.string().nullish(),
|
|
37
|
+
secondaryLanguages: z.array(z.string()).nullish(),
|
|
38
|
+
vadSilenceThresholdSecs: z.number().min(.3).max(3).nullish(),
|
|
39
|
+
vadThreshold: z.number().min(.1).max(.9).nullish()
|
|
40
|
+
}).nullish()
|
|
71
41
|
});
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
42
|
+
//#endregion
|
|
43
|
+
//#region src/elevenlabs-transcription-model.ts
|
|
44
|
+
const elevenLabsRealtimeErrorTypes = /* @__PURE__ */ new Set([
|
|
45
|
+
"auth_error",
|
|
46
|
+
"chunk_size_exceeded",
|
|
47
|
+
"commit_throttled",
|
|
48
|
+
"error",
|
|
49
|
+
"input_error",
|
|
50
|
+
"insufficient_audio_activity",
|
|
51
|
+
"queue_overflow",
|
|
52
|
+
"quota_exceeded",
|
|
53
|
+
"rate_limited",
|
|
54
|
+
"resource_exhausted",
|
|
55
|
+
"session_time_limit_exceeded",
|
|
56
|
+
"transcriber_error",
|
|
57
|
+
"unaccepted_terms"
|
|
88
58
|
]);
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
59
|
+
const elevenLabsLateFinalizationErrorTypes = /* @__PURE__ */ new Set([
|
|
60
|
+
"commit_throttled",
|
|
61
|
+
"input_error",
|
|
62
|
+
"insufficient_audio_activity"
|
|
93
63
|
]);
|
|
94
|
-
|
|
64
|
+
const finalCommitGracePeriodMs = 250;
|
|
95
65
|
function isRealtimeTranscriptionModelId(modelId) {
|
|
96
|
-
|
|
66
|
+
return modelId === "scribe_v2_realtime";
|
|
97
67
|
}
|
|
98
|
-
var ElevenLabsTranscriptionModel = class
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
}
|
|
247
|
-
const inputFormat = getElevenLabsRealtimeAudioFormat(
|
|
248
|
-
options.inputAudioFormat
|
|
249
|
-
);
|
|
250
|
-
const url = buildElevenLabsRealtimeTranscriptionUrl({
|
|
251
|
-
baseUrl: toWebSocketUrl(
|
|
252
|
-
this.config.url({
|
|
253
|
-
path: "/v1/speech-to-text/realtime",
|
|
254
|
-
modelId: this.modelId
|
|
255
|
-
})
|
|
256
|
-
),
|
|
257
|
-
inputFormat: inputFormat.audioFormat,
|
|
258
|
-
languageCode: elevenLabsOptions?.languageCode ?? void 0,
|
|
259
|
-
modelId: this.modelId,
|
|
260
|
-
streamingOptions
|
|
261
|
-
});
|
|
262
|
-
return {
|
|
263
|
-
request: { body: url.toString() },
|
|
264
|
-
response: {
|
|
265
|
-
timestamp: currentDate,
|
|
266
|
-
modelId: this.modelId
|
|
267
|
-
},
|
|
268
|
-
stream: createElevenLabsRealtimeTranscriptionStream({
|
|
269
|
-
abortSignal: options.abortSignal,
|
|
270
|
-
audio: options.audio,
|
|
271
|
-
headers: combineHeaders(this.config.headers?.(), options.headers),
|
|
272
|
-
includeLanguageDetection: streamingOptions?.includeLanguageDetection === true,
|
|
273
|
-
includeRawChunks: options.includeRawChunks,
|
|
274
|
-
includeTimestamps: streamingOptions?.includeTimestamps === true,
|
|
275
|
-
language: elevenLabsOptions?.languageCode ?? void 0,
|
|
276
|
-
previousText: streamingOptions?.previousText ?? void 0,
|
|
277
|
-
sampleRate: inputFormat.sampleRate,
|
|
278
|
-
url,
|
|
279
|
-
warnings,
|
|
280
|
-
webSocket: this.config.webSocket
|
|
281
|
-
})
|
|
282
|
-
};
|
|
283
|
-
}
|
|
68
|
+
var ElevenLabsTranscriptionModel = class ElevenLabsTranscriptionModel {
|
|
69
|
+
get provider() {
|
|
70
|
+
return this.config.provider;
|
|
71
|
+
}
|
|
72
|
+
static [WORKFLOW_SERIALIZE](model) {
|
|
73
|
+
return serializeModelOptions({
|
|
74
|
+
modelId: model.modelId,
|
|
75
|
+
config: model.config
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
static [WORKFLOW_DESERIALIZE](options) {
|
|
79
|
+
return new ElevenLabsTranscriptionModel(options.modelId, options.config);
|
|
80
|
+
}
|
|
81
|
+
constructor(modelId, config) {
|
|
82
|
+
this.modelId = modelId;
|
|
83
|
+
this.config = config;
|
|
84
|
+
this.specificationVersion = "v4";
|
|
85
|
+
}
|
|
86
|
+
async getArgs({ audio, mediaType, providerOptions }) {
|
|
87
|
+
const warnings = [];
|
|
88
|
+
const elevenlabsOptions = await parseProviderOptions({
|
|
89
|
+
provider: "elevenlabs",
|
|
90
|
+
providerOptions,
|
|
91
|
+
schema: elevenLabsTranscriptionModelOptionsSchema
|
|
92
|
+
});
|
|
93
|
+
if (elevenlabsOptions?.streaming != null) warnings.push({
|
|
94
|
+
type: "unsupported",
|
|
95
|
+
feature: "providerOptions.elevenlabs.streaming",
|
|
96
|
+
details: "ElevenLabs batch transcription does not support streaming options."
|
|
97
|
+
});
|
|
98
|
+
const formData = new FormData();
|
|
99
|
+
const blob = audio instanceof Uint8Array ? new Blob([audio]) : new Blob([convertBase64ToUint8Array(audio)]);
|
|
100
|
+
formData.append("model_id", this.modelId);
|
|
101
|
+
const fileExtension = mediaTypeToExtension(mediaType);
|
|
102
|
+
formData.append("file", new File([blob], "audio", { type: mediaType }), `audio.${fileExtension}`);
|
|
103
|
+
formData.append("diarize", "true");
|
|
104
|
+
if (elevenlabsOptions) {
|
|
105
|
+
const transcriptionModelOptions = {
|
|
106
|
+
language_code: elevenlabsOptions.languageCode ?? void 0,
|
|
107
|
+
tag_audio_events: elevenlabsOptions.tagAudioEvents ?? void 0,
|
|
108
|
+
num_speakers: elevenlabsOptions.numSpeakers ?? void 0,
|
|
109
|
+
timestamps_granularity: elevenlabsOptions.timestampsGranularity ?? void 0,
|
|
110
|
+
file_format: elevenlabsOptions.fileFormat ?? void 0
|
|
111
|
+
};
|
|
112
|
+
if (typeof elevenlabsOptions.diarize === "boolean") formData.append("diarize", String(elevenlabsOptions.diarize));
|
|
113
|
+
for (const key in transcriptionModelOptions) {
|
|
114
|
+
const value = transcriptionModelOptions[key];
|
|
115
|
+
if (value !== void 0) formData.append(key, String(value));
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
return {
|
|
119
|
+
formData,
|
|
120
|
+
warnings
|
|
121
|
+
};
|
|
122
|
+
}
|
|
123
|
+
async doGenerate(options) {
|
|
124
|
+
if (isRealtimeTranscriptionModelId(this.modelId)) throw new UnsupportedFunctionalityError({ functionality: `non-streaming transcription with ${this.modelId}` });
|
|
125
|
+
const currentDate = this.config._internal?.currentDate?.() ?? /* @__PURE__ */ new Date();
|
|
126
|
+
const { formData, warnings } = await this.getArgs(options);
|
|
127
|
+
const { value: response, responseHeaders, rawValue: rawResponse } = await postFormDataToApi({
|
|
128
|
+
url: this.config.url({
|
|
129
|
+
path: "/v1/speech-to-text",
|
|
130
|
+
modelId: this.modelId
|
|
131
|
+
}),
|
|
132
|
+
headers: combineHeaders(this.config.headers?.(), options.headers),
|
|
133
|
+
formData,
|
|
134
|
+
failedResponseHandler: elevenlabsFailedResponseHandler,
|
|
135
|
+
successfulResponseHandler: createJsonResponseHandler(elevenlabsTranscriptionResponseSchema),
|
|
136
|
+
abortSignal: options.abortSignal,
|
|
137
|
+
fetch: this.config.fetch
|
|
138
|
+
});
|
|
139
|
+
return {
|
|
140
|
+
text: response.text,
|
|
141
|
+
segments: response.words?.map((word) => ({
|
|
142
|
+
text: word.text,
|
|
143
|
+
startSecond: word.start ?? 0,
|
|
144
|
+
endSecond: word.end ?? 0
|
|
145
|
+
})) ?? [],
|
|
146
|
+
language: response.language_code,
|
|
147
|
+
durationInSeconds: response.words?.at(-1)?.end ?? void 0,
|
|
148
|
+
warnings,
|
|
149
|
+
response: {
|
|
150
|
+
timestamp: currentDate,
|
|
151
|
+
modelId: this.modelId,
|
|
152
|
+
headers: responseHeaders,
|
|
153
|
+
body: rawResponse
|
|
154
|
+
}
|
|
155
|
+
};
|
|
156
|
+
}
|
|
157
|
+
async doStream(options) {
|
|
158
|
+
if (!isRealtimeTranscriptionModelId(this.modelId)) throw new UnsupportedFunctionalityError({ functionality: `streaming transcription with ${this.modelId}` });
|
|
159
|
+
const currentDate = this.config._internal?.currentDate?.() ?? /* @__PURE__ */ new Date();
|
|
160
|
+
const elevenLabsOptions = await parseProviderOptions({
|
|
161
|
+
provider: "elevenlabs",
|
|
162
|
+
providerOptions: options.providerOptions,
|
|
163
|
+
schema: elevenLabsTranscriptionModelOptionsSchema
|
|
164
|
+
});
|
|
165
|
+
const streamingOptions = elevenLabsOptions?.streaming ?? void 0;
|
|
166
|
+
const warnings = [];
|
|
167
|
+
const rawElevenLabsOptions = options.providerOptions?.elevenlabs ?? {};
|
|
168
|
+
for (const option of [
|
|
169
|
+
"diarize",
|
|
170
|
+
"fileFormat",
|
|
171
|
+
"numSpeakers",
|
|
172
|
+
"tagAudioEvents",
|
|
173
|
+
"timestampsGranularity"
|
|
174
|
+
]) if (rawElevenLabsOptions[option] != null) warnings.push({
|
|
175
|
+
type: "unsupported",
|
|
176
|
+
feature: `providerOptions.elevenlabs.${option}`,
|
|
177
|
+
details: `ElevenLabs realtime transcription does not support ${option}.`
|
|
178
|
+
});
|
|
179
|
+
if (streamingOptions?.filterBackgroundAudio === true && (streamingOptions.includeTimestamps === true || streamingOptions.includeLanguageDetection === true)) throw new InvalidArgumentError({
|
|
180
|
+
argument: "providerOptions",
|
|
181
|
+
message: "providerOptions.elevenlabs.streaming.filterBackgroundAudio cannot be combined with includeTimestamps or includeLanguageDetection"
|
|
182
|
+
});
|
|
183
|
+
const inputFormat = getElevenLabsRealtimeAudioFormat(options.inputAudioFormat);
|
|
184
|
+
const url = buildElevenLabsRealtimeTranscriptionUrl({
|
|
185
|
+
baseUrl: toWebSocketUrl(this.config.url({
|
|
186
|
+
path: "/v1/speech-to-text/realtime",
|
|
187
|
+
modelId: this.modelId
|
|
188
|
+
})),
|
|
189
|
+
inputFormat: inputFormat.audioFormat,
|
|
190
|
+
languageCode: elevenLabsOptions?.languageCode ?? void 0,
|
|
191
|
+
modelId: this.modelId,
|
|
192
|
+
streamingOptions
|
|
193
|
+
});
|
|
194
|
+
return {
|
|
195
|
+
request: { body: url.toString() },
|
|
196
|
+
response: {
|
|
197
|
+
timestamp: currentDate,
|
|
198
|
+
modelId: this.modelId
|
|
199
|
+
},
|
|
200
|
+
stream: createElevenLabsRealtimeTranscriptionStream({
|
|
201
|
+
abortSignal: options.abortSignal,
|
|
202
|
+
audio: options.audio,
|
|
203
|
+
headers: combineHeaders(this.config.headers?.(), options.headers),
|
|
204
|
+
includeLanguageDetection: streamingOptions?.includeLanguageDetection === true,
|
|
205
|
+
includeRawChunks: options.includeRawChunks,
|
|
206
|
+
includeTimestamps: streamingOptions?.includeTimestamps === true,
|
|
207
|
+
language: elevenLabsOptions?.languageCode ?? void 0,
|
|
208
|
+
previousText: streamingOptions?.previousText ?? void 0,
|
|
209
|
+
sampleRate: inputFormat.sampleRate,
|
|
210
|
+
url,
|
|
211
|
+
warnings,
|
|
212
|
+
webSocket: this.config.webSocket
|
|
213
|
+
})
|
|
214
|
+
};
|
|
215
|
+
}
|
|
284
216
|
};
|
|
285
217
|
function getElevenLabsRealtimeAudioFormat(inputAudioFormat) {
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
218
|
+
const type = inputAudioFormat.type.toLowerCase();
|
|
219
|
+
const rate = inputAudioFormat.rate;
|
|
220
|
+
if (type === "audio/pcmu") {
|
|
221
|
+
if (rate != null && rate !== 8e3) throw new InvalidArgumentError({
|
|
222
|
+
argument: "inputAudioFormat",
|
|
223
|
+
message: "ElevenLabs only supports audio/pcmu at 8000 Hz"
|
|
224
|
+
});
|
|
225
|
+
return {
|
|
226
|
+
audioFormat: "ulaw_8000",
|
|
227
|
+
sampleRate: 8e3
|
|
228
|
+
};
|
|
229
|
+
}
|
|
230
|
+
const supportedPcmRates = [
|
|
231
|
+
8e3,
|
|
232
|
+
16e3,
|
|
233
|
+
22050,
|
|
234
|
+
24e3,
|
|
235
|
+
44100,
|
|
236
|
+
48e3
|
|
237
|
+
];
|
|
238
|
+
const pcmRate = rate ?? 16e3;
|
|
239
|
+
if (type !== "audio/pcm" || !supportedPcmRates.includes(pcmRate)) throw new InvalidArgumentError({
|
|
240
|
+
argument: "inputAudioFormat",
|
|
241
|
+
message: "ElevenLabs realtime transcription supports audio/pcm at 8000, 16000, 22050, 24000, 44100, or 48000 Hz, and audio/pcmu at 8000 Hz"
|
|
242
|
+
});
|
|
243
|
+
return {
|
|
244
|
+
audioFormat: `pcm_${pcmRate}`,
|
|
245
|
+
sampleRate: pcmRate
|
|
246
|
+
};
|
|
306
247
|
}
|
|
307
|
-
function buildElevenLabsRealtimeTranscriptionUrl({
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
vad_threshold: streamingOptions?.vadThreshold
|
|
330
|
-
};
|
|
331
|
-
for (const [key, value] of Object.entries(parameters)) {
|
|
332
|
-
if (value != null) {
|
|
333
|
-
url.searchParams.set(key, String(value));
|
|
334
|
-
}
|
|
335
|
-
}
|
|
336
|
-
for (const keyterm of streamingOptions?.keyterms ?? []) {
|
|
337
|
-
url.searchParams.append("keyterms", keyterm);
|
|
338
|
-
}
|
|
339
|
-
for (const secondaryLanguage of streamingOptions?.secondaryLanguages ?? []) {
|
|
340
|
-
url.searchParams.append("secondary_languages", secondaryLanguage);
|
|
341
|
-
}
|
|
342
|
-
return url;
|
|
248
|
+
function buildElevenLabsRealtimeTranscriptionUrl({ baseUrl, inputFormat, languageCode, modelId, streamingOptions }) {
|
|
249
|
+
const url = new URL(baseUrl);
|
|
250
|
+
url.searchParams.set("model_id", modelId);
|
|
251
|
+
url.searchParams.set("audio_format", inputFormat);
|
|
252
|
+
const includeDetailedCommit = streamingOptions?.includeTimestamps === true || streamingOptions?.includeLanguageDetection === true;
|
|
253
|
+
const parameters = {
|
|
254
|
+
commit_strategy: streamingOptions?.commitStrategy,
|
|
255
|
+
enable_logging: streamingOptions?.enableLogging,
|
|
256
|
+
filter_background_audio: streamingOptions?.filterBackgroundAudio,
|
|
257
|
+
include_language_detection: streamingOptions?.includeLanguageDetection,
|
|
258
|
+
include_timestamps: includeDetailedCommit ? true : streamingOptions?.includeTimestamps,
|
|
259
|
+
language_code: languageCode,
|
|
260
|
+
min_silence_duration_ms: streamingOptions?.minSilenceDurationMs,
|
|
261
|
+
min_speech_duration_ms: streamingOptions?.minSpeechDurationMs,
|
|
262
|
+
no_verbatim: streamingOptions?.noVerbatim,
|
|
263
|
+
vad_silence_threshold_secs: streamingOptions?.vadSilenceThresholdSecs,
|
|
264
|
+
vad_threshold: streamingOptions?.vadThreshold
|
|
265
|
+
};
|
|
266
|
+
for (const [key, value] of Object.entries(parameters)) if (value != null) url.searchParams.set(key, String(value));
|
|
267
|
+
for (const keyterm of streamingOptions?.keyterms ?? []) url.searchParams.append("keyterms", keyterm);
|
|
268
|
+
for (const secondaryLanguage of streamingOptions?.secondaryLanguages ?? []) url.searchParams.append("secondary_languages", secondaryLanguage);
|
|
269
|
+
return url;
|
|
343
270
|
}
|
|
344
|
-
function createElevenLabsRealtimeTranscriptionStream({
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
} : {}
|
|
544
|
-
});
|
|
545
|
-
}
|
|
546
|
-
timestampedCommitCount++;
|
|
547
|
-
if (includeTimestamps) {
|
|
548
|
-
finalSegments.push(
|
|
549
|
-
...timestampedWords.map((word) => ({
|
|
550
|
-
text: word.text ?? "",
|
|
551
|
-
startSecond: word.start,
|
|
552
|
-
endSecond: word.end
|
|
553
|
-
}))
|
|
554
|
-
);
|
|
555
|
-
}
|
|
556
|
-
if (endOfInput && timestampedCommitCount > committedEventsAtEndOfInput && (finalCommitEventCount == null || timestampedCommitCount >= finalCommitEventCount)) {
|
|
557
|
-
receivedPostInputCommit = true;
|
|
558
|
-
scheduleFinish();
|
|
559
|
-
}
|
|
560
|
-
break;
|
|
561
|
-
}
|
|
562
|
-
}
|
|
563
|
-
},
|
|
564
|
-
onProcessingError: finishWithError,
|
|
565
|
-
onSocketError: () => {
|
|
566
|
-
finishWithError(
|
|
567
|
-
new Error(
|
|
568
|
-
"ElevenLabs realtime transcription error." + (webSocket == null ? " Note: the native WebSocket implementation in browsers, Node.js, Deno, and Bun cannot send the xi-api-key header required by ElevenLabs. Pass a header-capable WebSocket implementation (e.g. the 'ws' package) via createElevenLabs({ webSocket })." : "")
|
|
569
|
-
)
|
|
570
|
-
);
|
|
571
|
-
},
|
|
572
|
-
url,
|
|
573
|
-
webSocket
|
|
574
|
-
});
|
|
575
|
-
},
|
|
576
|
-
cancel: () => {
|
|
577
|
-
if (finished) return;
|
|
578
|
-
finished = true;
|
|
579
|
-
cleanup();
|
|
580
|
-
}
|
|
581
|
-
});
|
|
271
|
+
function createElevenLabsRealtimeTranscriptionStream({ abortSignal, audio, headers, includeLanguageDetection, includeRawChunks, includeTimestamps, language, previousText, sampleRate, url, warnings, webSocket }) {
|
|
272
|
+
let finished = false;
|
|
273
|
+
let cleanup = () => {};
|
|
274
|
+
return new ReadableStream({
|
|
275
|
+
start: (controller) => {
|
|
276
|
+
let audioReader;
|
|
277
|
+
let connection;
|
|
278
|
+
let detectedLanguage = language;
|
|
279
|
+
let endOfInput = false;
|
|
280
|
+
let finalCommitGracePeriod;
|
|
281
|
+
let receivedPostInputCommit = false;
|
|
282
|
+
let segmentIndex = 0;
|
|
283
|
+
let sessionId;
|
|
284
|
+
let committedEventCount = 0;
|
|
285
|
+
let committedEventsAtEndOfInput = 0;
|
|
286
|
+
let finalCommitEventCount;
|
|
287
|
+
let timestampedCommitCount = 0;
|
|
288
|
+
const expectDetailedCommit = includeTimestamps || includeLanguageDetection;
|
|
289
|
+
const finalSegments = [];
|
|
290
|
+
const finalTexts = [];
|
|
291
|
+
cleanup = (closeCode) => {
|
|
292
|
+
clearTimeout(finalCommitGracePeriod);
|
|
293
|
+
if (audioReader != null) audioReader.cancel().catch(() => {});
|
|
294
|
+
else audio.cancel().catch(() => {});
|
|
295
|
+
connection?.close(closeCode);
|
|
296
|
+
};
|
|
297
|
+
const finishWithError = (error) => {
|
|
298
|
+
if (finished) return;
|
|
299
|
+
finished = true;
|
|
300
|
+
cleanup();
|
|
301
|
+
controller.error(error);
|
|
302
|
+
};
|
|
303
|
+
const finish = () => {
|
|
304
|
+
if (finished) return;
|
|
305
|
+
finished = true;
|
|
306
|
+
controller.enqueue({
|
|
307
|
+
type: "finish",
|
|
308
|
+
text: finalTexts.join(" ").trim(),
|
|
309
|
+
segments: finalSegments,
|
|
310
|
+
language: detectedLanguage,
|
|
311
|
+
durationInSeconds: finalSegments.at(-1)?.endSecond
|
|
312
|
+
});
|
|
313
|
+
controller.close();
|
|
314
|
+
cleanup(1e3);
|
|
315
|
+
};
|
|
316
|
+
const scheduleFinish = () => {
|
|
317
|
+
clearTimeout(finalCommitGracePeriod);
|
|
318
|
+
finalCommitGracePeriod = setTimeout(finish, finalCommitGracePeriodMs);
|
|
319
|
+
};
|
|
320
|
+
const sendAudio = async (socket) => {
|
|
321
|
+
audioReader = audio.getReader();
|
|
322
|
+
let firstChunk = true;
|
|
323
|
+
try {
|
|
324
|
+
while (true) {
|
|
325
|
+
const { done, value } = await audioReader.read();
|
|
326
|
+
if (done || finished) break;
|
|
327
|
+
socket.send(JSON.stringify({
|
|
328
|
+
message_type: "input_audio_chunk",
|
|
329
|
+
audio_base_64: convertToBase64(value),
|
|
330
|
+
commit: false,
|
|
331
|
+
sample_rate: sampleRate,
|
|
332
|
+
...firstChunk && previousText != null ? { previous_text: previousText } : {}
|
|
333
|
+
}));
|
|
334
|
+
firstChunk = false;
|
|
335
|
+
await waitForWebSocketBufferDrain(socket);
|
|
336
|
+
}
|
|
337
|
+
} finally {
|
|
338
|
+
audioReader.releaseLock();
|
|
339
|
+
audioReader = void 0;
|
|
340
|
+
}
|
|
341
|
+
if (!finished) {
|
|
342
|
+
committedEventsAtEndOfInput = committedEventCount;
|
|
343
|
+
endOfInput = true;
|
|
344
|
+
socket.send(JSON.stringify({
|
|
345
|
+
message_type: "input_audio_chunk",
|
|
346
|
+
audio_base_64: "",
|
|
347
|
+
commit: true,
|
|
348
|
+
sample_rate: sampleRate
|
|
349
|
+
}));
|
|
350
|
+
}
|
|
351
|
+
};
|
|
352
|
+
connection = connectToWebSocket({
|
|
353
|
+
abortSignal,
|
|
354
|
+
headers,
|
|
355
|
+
onAbort: finishWithError,
|
|
356
|
+
onClose: () => {
|
|
357
|
+
if (finished) return;
|
|
358
|
+
if (endOfInput && receivedPostInputCommit) {
|
|
359
|
+
finish();
|
|
360
|
+
return;
|
|
361
|
+
}
|
|
362
|
+
finishWithError(/* @__PURE__ */ new Error("ElevenLabs realtime transcription stream closed before completion."));
|
|
363
|
+
},
|
|
364
|
+
onMessageText: async (text) => {
|
|
365
|
+
const parsed = await safeParseJSON({ text });
|
|
366
|
+
if (!parsed.success) return;
|
|
367
|
+
const raw = parsed.value;
|
|
368
|
+
if (includeRawChunks) controller.enqueue({
|
|
369
|
+
type: "raw",
|
|
370
|
+
rawValue: raw
|
|
371
|
+
});
|
|
372
|
+
if (raw.message_type != null && elevenLabsRealtimeErrorTypes.has(raw.message_type)) {
|
|
373
|
+
if (endOfInput && (committedEventCount > 0 || timestampedCommitCount > 0) && elevenLabsLateFinalizationErrorTypes.has(raw.message_type)) {
|
|
374
|
+
receivedPostInputCommit = true;
|
|
375
|
+
finish();
|
|
376
|
+
return;
|
|
377
|
+
}
|
|
378
|
+
finishWithError(new Error(raw.error ?? "ElevenLabs realtime transcription error"));
|
|
379
|
+
return;
|
|
380
|
+
}
|
|
381
|
+
switch (raw.message_type) {
|
|
382
|
+
case "session_started": {
|
|
383
|
+
sessionId = raw.session_id;
|
|
384
|
+
controller.enqueue({
|
|
385
|
+
type: "stream-start",
|
|
386
|
+
warnings
|
|
387
|
+
});
|
|
388
|
+
const socket = connection?.socket;
|
|
389
|
+
if (socket == null) {
|
|
390
|
+
finishWithError(/* @__PURE__ */ new Error("WebSocket is not connected."));
|
|
391
|
+
break;
|
|
392
|
+
}
|
|
393
|
+
sendAudio(socket).catch(finishWithError);
|
|
394
|
+
break;
|
|
395
|
+
}
|
|
396
|
+
case "partial_transcript":
|
|
397
|
+
controller.enqueue({
|
|
398
|
+
type: "transcript-partial",
|
|
399
|
+
id: `${sessionId ?? "session"}:${segmentIndex}`,
|
|
400
|
+
text: raw.text ?? ""
|
|
401
|
+
});
|
|
402
|
+
break;
|
|
403
|
+
case "final_transcript":
|
|
404
|
+
case "committed_transcript": {
|
|
405
|
+
committedEventCount++;
|
|
406
|
+
const text = (raw.text ?? "").trim();
|
|
407
|
+
const id = `${sessionId ?? "session"}:${segmentIndex}`;
|
|
408
|
+
if (text.length > 0) {
|
|
409
|
+
finalTexts.push(text);
|
|
410
|
+
segmentIndex++;
|
|
411
|
+
controller.enqueue({
|
|
412
|
+
type: "transcript-final",
|
|
413
|
+
id,
|
|
414
|
+
text
|
|
415
|
+
});
|
|
416
|
+
}
|
|
417
|
+
if (endOfInput && committedEventCount > committedEventsAtEndOfInput) {
|
|
418
|
+
receivedPostInputCommit = true;
|
|
419
|
+
finalCommitEventCount = committedEventCount;
|
|
420
|
+
if (!expectDetailedCommit || raw.message_type === "final_transcript") scheduleFinish();
|
|
421
|
+
}
|
|
422
|
+
break;
|
|
423
|
+
}
|
|
424
|
+
case "final_transcript_with_timestamps":
|
|
425
|
+
case "committed_transcript_with_timestamps": {
|
|
426
|
+
const text = (raw.text ?? "").trim();
|
|
427
|
+
const timestampedWords = (raw.words ?? []).filter((word) => typeof word.start === "number" && typeof word.end === "number");
|
|
428
|
+
detectedLanguage = raw.language_code ?? detectedLanguage;
|
|
429
|
+
if (finalTexts[timestampedCommitCount] == null && text.length > 0) {
|
|
430
|
+
const id = `${sessionId ?? "session"}:${segmentIndex++}`;
|
|
431
|
+
finalTexts.push(text);
|
|
432
|
+
controller.enqueue({
|
|
433
|
+
type: "transcript-final",
|
|
434
|
+
id,
|
|
435
|
+
text,
|
|
436
|
+
...includeTimestamps ? {
|
|
437
|
+
startSecond: timestampedWords[0]?.start,
|
|
438
|
+
endSecond: timestampedWords.at(-1)?.end
|
|
439
|
+
} : {}
|
|
440
|
+
});
|
|
441
|
+
}
|
|
442
|
+
timestampedCommitCount++;
|
|
443
|
+
if (includeTimestamps) finalSegments.push(...timestampedWords.map((word) => ({
|
|
444
|
+
text: word.text ?? "",
|
|
445
|
+
startSecond: word.start,
|
|
446
|
+
endSecond: word.end
|
|
447
|
+
})));
|
|
448
|
+
if (endOfInput && timestampedCommitCount > committedEventsAtEndOfInput && (finalCommitEventCount == null || timestampedCommitCount >= finalCommitEventCount)) {
|
|
449
|
+
receivedPostInputCommit = true;
|
|
450
|
+
scheduleFinish();
|
|
451
|
+
}
|
|
452
|
+
break;
|
|
453
|
+
}
|
|
454
|
+
}
|
|
455
|
+
},
|
|
456
|
+
onProcessingError: finishWithError,
|
|
457
|
+
onSocketError: () => {
|
|
458
|
+
finishWithError(/* @__PURE__ */ new Error("ElevenLabs realtime transcription error." + (webSocket == null ? " Note: the native WebSocket implementation in browsers, Node.js, Deno, and Bun cannot send the xi-api-key header required by ElevenLabs. Pass a header-capable WebSocket implementation (e.g. the 'ws' package) via createElevenLabs({ webSocket })." : "")));
|
|
459
|
+
},
|
|
460
|
+
url,
|
|
461
|
+
webSocket
|
|
462
|
+
});
|
|
463
|
+
},
|
|
464
|
+
cancel: () => {
|
|
465
|
+
if (finished) return;
|
|
466
|
+
finished = true;
|
|
467
|
+
cleanup();
|
|
468
|
+
}
|
|
469
|
+
});
|
|
582
470
|
}
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
471
|
+
const elevenlabsTranscriptionResponseSchema = z.object({
|
|
472
|
+
language_code: z.string(),
|
|
473
|
+
language_probability: z.number(),
|
|
474
|
+
text: z.string(),
|
|
475
|
+
words: z.array(z.object({
|
|
476
|
+
text: z.string(),
|
|
477
|
+
type: z.enum([
|
|
478
|
+
"word",
|
|
479
|
+
"spacing",
|
|
480
|
+
"audio_event"
|
|
481
|
+
]),
|
|
482
|
+
start: z.number().nullish(),
|
|
483
|
+
end: z.number().nullish(),
|
|
484
|
+
speaker_id: z.string().nullish(),
|
|
485
|
+
characters: z.array(z.object({
|
|
486
|
+
text: z.string(),
|
|
487
|
+
start: z.number().nullish(),
|
|
488
|
+
end: z.number().nullish()
|
|
489
|
+
})).nullish()
|
|
490
|
+
})).nullish()
|
|
603
491
|
});
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
})
|
|
631
|
-
).max(3).optional(),
|
|
632
|
-
seed: z4.number().min(0).max(4294967295).optional(),
|
|
633
|
-
previousText: z4.string().optional(),
|
|
634
|
-
nextText: z4.string().optional(),
|
|
635
|
-
previousRequestIds: z4.array(z4.string()).max(3).optional(),
|
|
636
|
-
nextRequestIds: z4.array(z4.string()).max(3).optional(),
|
|
637
|
-
applyTextNormalization: z4.enum(["auto", "on", "off"]).optional(),
|
|
638
|
-
applyLanguageTextNormalization: z4.boolean().optional(),
|
|
639
|
-
enableLogging: z4.boolean().optional()
|
|
492
|
+
//#endregion
|
|
493
|
+
//#region src/elevenlabs-speech-model-options.ts
|
|
494
|
+
const elevenLabsSpeechModelOptionsSchema = z.object({
|
|
495
|
+
languageCode: z.string().optional(),
|
|
496
|
+
voiceSettings: z.object({
|
|
497
|
+
stability: z.number().min(0).max(1).optional(),
|
|
498
|
+
similarityBoost: z.number().min(0).max(1).optional(),
|
|
499
|
+
style: z.number().min(0).max(1).optional(),
|
|
500
|
+
useSpeakerBoost: z.boolean().optional()
|
|
501
|
+
}).optional(),
|
|
502
|
+
pronunciationDictionaryLocators: z.array(z.object({
|
|
503
|
+
pronunciationDictionaryId: z.string(),
|
|
504
|
+
versionId: z.string().optional()
|
|
505
|
+
})).max(3).optional(),
|
|
506
|
+
seed: z.number().min(0).max(4294967295).optional(),
|
|
507
|
+
previousText: z.string().optional(),
|
|
508
|
+
nextText: z.string().optional(),
|
|
509
|
+
previousRequestIds: z.array(z.string()).max(3).optional(),
|
|
510
|
+
nextRequestIds: z.array(z.string()).max(3).optional(),
|
|
511
|
+
applyTextNormalization: z.enum([
|
|
512
|
+
"auto",
|
|
513
|
+
"on",
|
|
514
|
+
"off"
|
|
515
|
+
]).optional(),
|
|
516
|
+
applyLanguageTextNormalization: z.boolean().optional(),
|
|
517
|
+
enableLogging: z.boolean().optional()
|
|
640
518
|
});
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
var ElevenLabsSpeechModel = class
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
if (Object.keys(voiceSettings).length > 0) {
|
|
756
|
-
requestBody.voice_settings = voiceSettings;
|
|
757
|
-
}
|
|
758
|
-
if (instructions) {
|
|
759
|
-
warnings.push({
|
|
760
|
-
type: "unsupported",
|
|
761
|
-
feature: "instructions",
|
|
762
|
-
details: `ElevenLabs speech models do not support instructions. Instructions parameter was ignored.`
|
|
763
|
-
});
|
|
764
|
-
}
|
|
765
|
-
return {
|
|
766
|
-
requestBody,
|
|
767
|
-
queryParams,
|
|
768
|
-
warnings,
|
|
769
|
-
voiceId: voice
|
|
770
|
-
};
|
|
771
|
-
}
|
|
772
|
-
async doGenerate(options) {
|
|
773
|
-
const currentDate = this.config._internal?.currentDate?.() ?? /* @__PURE__ */ new Date();
|
|
774
|
-
const { requestBody, queryParams, warnings, voiceId } = await this.getArgs(options);
|
|
775
|
-
const {
|
|
776
|
-
value: audio,
|
|
777
|
-
responseHeaders,
|
|
778
|
-
rawValue: rawResponse
|
|
779
|
-
} = await postJsonToApi({
|
|
780
|
-
url: (() => {
|
|
781
|
-
const baseUrl = this.config.url({
|
|
782
|
-
path: `/v1/text-to-speech/${voiceId}`,
|
|
783
|
-
modelId: this.modelId
|
|
784
|
-
});
|
|
785
|
-
const queryString = new URLSearchParams(queryParams).toString();
|
|
786
|
-
return queryString ? `${baseUrl}?${queryString}` : baseUrl;
|
|
787
|
-
})(),
|
|
788
|
-
headers: combineHeaders2(this.config.headers?.(), options.headers),
|
|
789
|
-
body: requestBody,
|
|
790
|
-
failedResponseHandler: elevenlabsFailedResponseHandler,
|
|
791
|
-
successfulResponseHandler: createBinaryResponseHandler(),
|
|
792
|
-
abortSignal: options.abortSignal,
|
|
793
|
-
fetch: this.config.fetch
|
|
794
|
-
});
|
|
795
|
-
return {
|
|
796
|
-
audio,
|
|
797
|
-
warnings,
|
|
798
|
-
request: {
|
|
799
|
-
body: JSON.stringify(requestBody)
|
|
800
|
-
},
|
|
801
|
-
response: {
|
|
802
|
-
timestamp: currentDate,
|
|
803
|
-
modelId: this.modelId,
|
|
804
|
-
headers: responseHeaders,
|
|
805
|
-
body: rawResponse
|
|
806
|
-
}
|
|
807
|
-
};
|
|
808
|
-
}
|
|
519
|
+
//#endregion
|
|
520
|
+
//#region src/elevenlabs-speech-model.ts
|
|
521
|
+
var ElevenLabsSpeechModel = class ElevenLabsSpeechModel {
|
|
522
|
+
get provider() {
|
|
523
|
+
return this.config.provider;
|
|
524
|
+
}
|
|
525
|
+
static [WORKFLOW_SERIALIZE](model) {
|
|
526
|
+
return serializeModelOptions({
|
|
527
|
+
modelId: model.modelId,
|
|
528
|
+
config: model.config
|
|
529
|
+
});
|
|
530
|
+
}
|
|
531
|
+
static [WORKFLOW_DESERIALIZE](options) {
|
|
532
|
+
return new ElevenLabsSpeechModel(options.modelId, options.config);
|
|
533
|
+
}
|
|
534
|
+
constructor(modelId, config) {
|
|
535
|
+
this.modelId = modelId;
|
|
536
|
+
this.config = config;
|
|
537
|
+
this.specificationVersion = "v4";
|
|
538
|
+
}
|
|
539
|
+
async getArgs({ text, voice = "21m00Tcm4TlvDq8ikWAM", outputFormat = "mp3_44100_128", instructions, language, speed, providerOptions }) {
|
|
540
|
+
const warnings = [];
|
|
541
|
+
const elevenLabsOptions = await parseProviderOptions({
|
|
542
|
+
provider: "elevenlabs",
|
|
543
|
+
providerOptions,
|
|
544
|
+
schema: elevenLabsSpeechModelOptionsSchema
|
|
545
|
+
});
|
|
546
|
+
const requestBody = {
|
|
547
|
+
text,
|
|
548
|
+
model_id: this.modelId
|
|
549
|
+
};
|
|
550
|
+
const queryParams = {};
|
|
551
|
+
if (outputFormat) queryParams.output_format = {
|
|
552
|
+
mp3: "mp3_44100_128",
|
|
553
|
+
mp3_32: "mp3_44100_32",
|
|
554
|
+
mp3_64: "mp3_44100_64",
|
|
555
|
+
mp3_96: "mp3_44100_96",
|
|
556
|
+
mp3_128: "mp3_44100_128",
|
|
557
|
+
mp3_192: "mp3_44100_192",
|
|
558
|
+
pcm: "pcm_44100",
|
|
559
|
+
pcm_16000: "pcm_16000",
|
|
560
|
+
pcm_22050: "pcm_22050",
|
|
561
|
+
pcm_24000: "pcm_24000",
|
|
562
|
+
pcm_44100: "pcm_44100",
|
|
563
|
+
ulaw: "ulaw_8000"
|
|
564
|
+
}[outputFormat] || outputFormat;
|
|
565
|
+
if (language) requestBody.language_code = language;
|
|
566
|
+
const voiceSettings = {};
|
|
567
|
+
if (speed != null) voiceSettings.speed = speed;
|
|
568
|
+
if (elevenLabsOptions) {
|
|
569
|
+
if (elevenLabsOptions.voiceSettings) {
|
|
570
|
+
if (elevenLabsOptions.voiceSettings.stability != null) voiceSettings.stability = elevenLabsOptions.voiceSettings.stability;
|
|
571
|
+
if (elevenLabsOptions.voiceSettings.similarityBoost != null) voiceSettings.similarity_boost = elevenLabsOptions.voiceSettings.similarityBoost;
|
|
572
|
+
if (elevenLabsOptions.voiceSettings.style != null) voiceSettings.style = elevenLabsOptions.voiceSettings.style;
|
|
573
|
+
if (elevenLabsOptions.voiceSettings.useSpeakerBoost != null) voiceSettings.use_speaker_boost = elevenLabsOptions.voiceSettings.useSpeakerBoost;
|
|
574
|
+
}
|
|
575
|
+
if (elevenLabsOptions.languageCode && !requestBody.language_code) requestBody.language_code = elevenLabsOptions.languageCode;
|
|
576
|
+
if (elevenLabsOptions.pronunciationDictionaryLocators) requestBody.pronunciation_dictionary_locators = elevenLabsOptions.pronunciationDictionaryLocators.map((locator) => ({
|
|
577
|
+
pronunciation_dictionary_id: locator.pronunciationDictionaryId,
|
|
578
|
+
...locator.versionId && { version_id: locator.versionId }
|
|
579
|
+
}));
|
|
580
|
+
if (elevenLabsOptions.seed != null) requestBody.seed = elevenLabsOptions.seed;
|
|
581
|
+
if (elevenLabsOptions.previousText) requestBody.previous_text = elevenLabsOptions.previousText;
|
|
582
|
+
if (elevenLabsOptions.nextText) requestBody.next_text = elevenLabsOptions.nextText;
|
|
583
|
+
if (elevenLabsOptions.previousRequestIds) requestBody.previous_request_ids = elevenLabsOptions.previousRequestIds;
|
|
584
|
+
if (elevenLabsOptions.nextRequestIds) requestBody.next_request_ids = elevenLabsOptions.nextRequestIds;
|
|
585
|
+
if (elevenLabsOptions.applyTextNormalization) requestBody.apply_text_normalization = elevenLabsOptions.applyTextNormalization;
|
|
586
|
+
if (elevenLabsOptions.applyLanguageTextNormalization != null) requestBody.apply_language_text_normalization = elevenLabsOptions.applyLanguageTextNormalization;
|
|
587
|
+
if (elevenLabsOptions.enableLogging != null) queryParams.enable_logging = String(elevenLabsOptions.enableLogging);
|
|
588
|
+
}
|
|
589
|
+
if (Object.keys(voiceSettings).length > 0) requestBody.voice_settings = voiceSettings;
|
|
590
|
+
if (instructions) warnings.push({
|
|
591
|
+
type: "unsupported",
|
|
592
|
+
feature: "instructions",
|
|
593
|
+
details: `ElevenLabs speech models do not support instructions. Instructions parameter was ignored.`
|
|
594
|
+
});
|
|
595
|
+
return {
|
|
596
|
+
requestBody,
|
|
597
|
+
queryParams,
|
|
598
|
+
warnings,
|
|
599
|
+
voiceId: voice
|
|
600
|
+
};
|
|
601
|
+
}
|
|
602
|
+
async doGenerate(options) {
|
|
603
|
+
const currentDate = this.config._internal?.currentDate?.() ?? /* @__PURE__ */ new Date();
|
|
604
|
+
const { requestBody, queryParams, warnings, voiceId } = await this.getArgs(options);
|
|
605
|
+
const { value: audio, responseHeaders, rawValue: rawResponse } = await postJsonToApi({
|
|
606
|
+
url: (() => {
|
|
607
|
+
const baseUrl = this.config.url({
|
|
608
|
+
path: `/v1/text-to-speech/${voiceId}`,
|
|
609
|
+
modelId: this.modelId
|
|
610
|
+
});
|
|
611
|
+
const queryString = new URLSearchParams(queryParams).toString();
|
|
612
|
+
return queryString ? `${baseUrl}?${queryString}` : baseUrl;
|
|
613
|
+
})(),
|
|
614
|
+
headers: combineHeaders(this.config.headers?.(), options.headers),
|
|
615
|
+
body: requestBody,
|
|
616
|
+
failedResponseHandler: elevenlabsFailedResponseHandler,
|
|
617
|
+
successfulResponseHandler: createBinaryResponseHandler(),
|
|
618
|
+
abortSignal: options.abortSignal,
|
|
619
|
+
fetch: this.config.fetch
|
|
620
|
+
});
|
|
621
|
+
return {
|
|
622
|
+
audio,
|
|
623
|
+
warnings,
|
|
624
|
+
request: { body: JSON.stringify(requestBody) },
|
|
625
|
+
response: {
|
|
626
|
+
timestamp: currentDate,
|
|
627
|
+
modelId: this.modelId,
|
|
628
|
+
headers: responseHeaders,
|
|
629
|
+
body: rawResponse
|
|
630
|
+
}
|
|
631
|
+
};
|
|
632
|
+
}
|
|
809
633
|
};
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
634
|
+
//#endregion
|
|
635
|
+
//#region src/version.ts
|
|
636
|
+
const VERSION = "3.0.53";
|
|
637
|
+
//#endregion
|
|
638
|
+
//#region src/elevenlabs-provider.ts
|
|
639
|
+
/**
|
|
640
|
+
* Create an ElevenLabs provider instance.
|
|
641
|
+
*/
|
|
815
642
|
function createElevenLabs(options = {}) {
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
modelType: "imageModel",
|
|
869
|
-
message: "ElevenLabs does not provide image models"
|
|
870
|
-
});
|
|
871
|
-
};
|
|
872
|
-
return provider;
|
|
643
|
+
const getHeaders = () => withUserAgentSuffix({
|
|
644
|
+
"xi-api-key": loadApiKey({
|
|
645
|
+
apiKey: options.apiKey,
|
|
646
|
+
environmentVariableName: "ELEVENLABS_API_KEY",
|
|
647
|
+
description: "ElevenLabs"
|
|
648
|
+
}),
|
|
649
|
+
...options.headers
|
|
650
|
+
}, `ai-sdk-elevenlabs/${VERSION}`);
|
|
651
|
+
const createTranscriptionModel = (modelId) => new ElevenLabsTranscriptionModel(modelId, {
|
|
652
|
+
provider: `elevenlabs.transcription`,
|
|
653
|
+
url: ({ path }) => `https://api.elevenlabs.io${path}`,
|
|
654
|
+
headers: getHeaders,
|
|
655
|
+
fetch: options.fetch,
|
|
656
|
+
webSocket: options.webSocket
|
|
657
|
+
});
|
|
658
|
+
const createSpeechModel = (modelId) => new ElevenLabsSpeechModel(modelId, {
|
|
659
|
+
provider: `elevenlabs.speech`,
|
|
660
|
+
url: ({ path }) => `https://api.elevenlabs.io${path}`,
|
|
661
|
+
headers: getHeaders,
|
|
662
|
+
fetch: options.fetch
|
|
663
|
+
});
|
|
664
|
+
const provider = function(modelId) {
|
|
665
|
+
return { transcription: createTranscriptionModel(modelId) };
|
|
666
|
+
};
|
|
667
|
+
provider.specificationVersion = "v4";
|
|
668
|
+
provider.transcription = createTranscriptionModel;
|
|
669
|
+
provider.transcriptionModel = createTranscriptionModel;
|
|
670
|
+
provider.speech = createSpeechModel;
|
|
671
|
+
provider.speechModel = createSpeechModel;
|
|
672
|
+
provider.languageModel = (modelId) => {
|
|
673
|
+
throw new NoSuchModelError({
|
|
674
|
+
modelId,
|
|
675
|
+
modelType: "languageModel",
|
|
676
|
+
message: "ElevenLabs does not provide language models"
|
|
677
|
+
});
|
|
678
|
+
};
|
|
679
|
+
provider.embeddingModel = (modelId) => {
|
|
680
|
+
throw new NoSuchModelError({
|
|
681
|
+
modelId,
|
|
682
|
+
modelType: "embeddingModel",
|
|
683
|
+
message: "ElevenLabs does not provide embedding models"
|
|
684
|
+
});
|
|
685
|
+
};
|
|
686
|
+
provider.textEmbeddingModel = provider.embeddingModel;
|
|
687
|
+
provider.imageModel = (modelId) => {
|
|
688
|
+
throw new NoSuchModelError({
|
|
689
|
+
modelId,
|
|
690
|
+
modelType: "imageModel",
|
|
691
|
+
message: "ElevenLabs does not provide image models"
|
|
692
|
+
});
|
|
693
|
+
};
|
|
694
|
+
return provider;
|
|
873
695
|
}
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
696
|
+
/**
|
|
697
|
+
* Default ElevenLabs provider instance.
|
|
698
|
+
*/
|
|
699
|
+
const elevenLabs = createElevenLabs();
|
|
700
|
+
//#endregion
|
|
701
|
+
export { VERSION, createElevenLabs, elevenLabs, elevenLabs as elevenlabs };
|
|
702
|
+
|
|
881
703
|
//# sourceMappingURL=index.js.map
|