@transcribe-api/sdk 0.1.2 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +389 -54
- package/index.js +1185 -1377
- package/package.json +15 -15
- package/worker.js +567 -301
package/worker.js
CHANGED
|
@@ -3,33 +3,44 @@ const MAX_BATCH_FILES = 10000;
|
|
|
3
3
|
const MAX_BATCH_TOTAL_SIZE_BYTES = 10 * 1024 * 1024 * 1024;
|
|
4
4
|
const MAX_SYNC_AUDIO_BYTES = 30 * 1024 * 1024;
|
|
5
5
|
const MAX_SYNC_AUDIO_SECONDS = 10 * 60;
|
|
6
|
-
const MULTIPART_UPLOAD_THRESHOLD_BYTES =
|
|
7
|
-
const
|
|
8
|
-
const
|
|
6
|
+
const MULTIPART_UPLOAD_THRESHOLD_BYTES = 128 * 1024 * 1024;
|
|
7
|
+
const DEFAULT_UPLOAD_CONCURRENCY = 1;
|
|
8
|
+
const MAX_UPLOAD_CONCURRENCY = 32;
|
|
9
9
|
const MAX_MULTIPART_ADAPTIVE_ATTEMPTS = 12;
|
|
10
10
|
const MULTIPART_IDLE_WAIT_MS = 50;
|
|
11
11
|
const MIN_POLLING_INTERVAL_SECONDS = 10;
|
|
12
|
+
const DEFAULT_POLLING_SPINNER_INTERVAL_MS = 150;
|
|
12
13
|
const TERMINAL_JOB_STATUSES = new Set(["completed", "failed", "insufficient_funds"]);
|
|
13
14
|
const BATCH_MP4_UNSUPPORTED_MESSAGE = "Batch uploads do not support .mp4 for MVP. Supported batch audio formats: mp3, mpeg, mpga, m4a, wav, webm.";
|
|
14
15
|
const BATCH_UNSUPPORTED_MESSAGE = "Unsupported batch audio format. Supported batch audio formats: mp3, mpeg, mpga, m4a, wav, webm.";
|
|
15
16
|
|
|
16
17
|
export class TranscribeAPIError extends Error {
|
|
17
|
-
constructor(
|
|
18
|
-
super(
|
|
18
|
+
constructor(message, { status = null, code = null, extra = null, response = null } = {}) {
|
|
19
|
+
super(message);
|
|
19
20
|
this.name = "TranscribeAPIError";
|
|
20
21
|
this.status = status;
|
|
21
22
|
this.code = code;
|
|
22
|
-
this.
|
|
23
|
-
this.response = response;
|
|
23
|
+
this.response = response || { message, ...(code ? { code } : {}) };
|
|
24
24
|
if (extra && typeof extra === "object") {
|
|
25
25
|
for (const [key, value] of Object.entries(extra)) {
|
|
26
|
-
if (key === "status" || key === "stack" || key === "name") {
|
|
26
|
+
if (key === "status" || key === "stack" || key === "name" || key === "message" || key === "code") {
|
|
27
27
|
continue;
|
|
28
28
|
}
|
|
29
29
|
this[key] = value;
|
|
30
30
|
}
|
|
31
31
|
}
|
|
32
32
|
}
|
|
33
|
+
|
|
34
|
+
toJSON() {
|
|
35
|
+
return {
|
|
36
|
+
message: this.message,
|
|
37
|
+
...(this.code ? { code: this.code } : {}),
|
|
38
|
+
...Object.fromEntries(
|
|
39
|
+
Object.entries(this)
|
|
40
|
+
.filter(([key]) => !["name", "status", "code", "response"].includes(key)),
|
|
41
|
+
),
|
|
42
|
+
};
|
|
43
|
+
}
|
|
33
44
|
}
|
|
34
45
|
|
|
35
46
|
function sleep(ms) {
|
|
@@ -72,7 +83,8 @@ async function retry(operation, { attempts = 3, baseDelayMs = 250 } = {}) {
|
|
|
72
83
|
return await operation(index);
|
|
73
84
|
} catch (error) {
|
|
74
85
|
lastError = error;
|
|
75
|
-
|
|
86
|
+
const retryable = isRetryableError(error);
|
|
87
|
+
if (!retryable || index === attempts - 1) {
|
|
76
88
|
throw error;
|
|
77
89
|
}
|
|
78
90
|
await sleep(baseDelayMs * (2 ** index));
|
|
@@ -81,17 +93,17 @@ async function retry(operation, { attempts = 3, baseDelayMs = 250 } = {}) {
|
|
|
81
93
|
throw lastError;
|
|
82
94
|
}
|
|
83
95
|
|
|
84
|
-
function
|
|
96
|
+
function normalizeUploadConcurrency(value) {
|
|
85
97
|
if (value === undefined || value === null || value === "") {
|
|
86
|
-
return
|
|
98
|
+
return DEFAULT_UPLOAD_CONCURRENCY;
|
|
87
99
|
}
|
|
88
100
|
const parsed = Number.parseInt(String(value), 10);
|
|
89
101
|
if (!Number.isInteger(parsed) || parsed < 1) {
|
|
90
|
-
throw new TranscribeAPIError("`
|
|
91
|
-
code: "
|
|
102
|
+
throw new TranscribeAPIError("`uploadConcurrency` must be an integer >= 1.", {
|
|
103
|
+
code: "invalid_upload_concurrency",
|
|
92
104
|
});
|
|
93
105
|
}
|
|
94
|
-
return Math.min(parsed,
|
|
106
|
+
return Math.min(parsed, MAX_UPLOAD_CONCURRENCY);
|
|
95
107
|
}
|
|
96
108
|
|
|
97
109
|
function normalizePollingConfig(polling) {
|
|
@@ -142,31 +154,39 @@ function contentTypeFromName(name = "") {
|
|
|
142
154
|
return "application/octet-stream";
|
|
143
155
|
}
|
|
144
156
|
|
|
145
|
-
function
|
|
157
|
+
function isRemoteBatchItem(input) {
|
|
146
158
|
return Boolean(input && typeof input === "object" && typeof input.url === "string");
|
|
147
159
|
}
|
|
148
160
|
|
|
149
|
-
function
|
|
150
|
-
return
|
|
161
|
+
function isFilesInput(input) {
|
|
162
|
+
return Array.isArray(input);
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
function normalizeLanguageCode(language, fieldName = "`language`") {
|
|
166
|
+
const value = String(language || "").trim().toLowerCase();
|
|
167
|
+
if (!value || value === "auto") {
|
|
168
|
+
return null;
|
|
169
|
+
}
|
|
170
|
+
if (!/^[a-z]{2}$/i.test(value)) {
|
|
171
|
+
throw new TranscribeAPIError(`${fieldName} must be a two-letter language code such as \`en\` or \`fr\`.`, {
|
|
172
|
+
code: "invalid_language",
|
|
173
|
+
});
|
|
174
|
+
}
|
|
175
|
+
return value;
|
|
151
176
|
}
|
|
152
177
|
|
|
153
178
|
function defaultReferenceId(index) {
|
|
154
|
-
return
|
|
179
|
+
return String(index + 1).padStart(5, "0");
|
|
155
180
|
}
|
|
156
181
|
|
|
157
182
|
function normalizeBatchInputItem(item, index) {
|
|
158
183
|
if (!item || typeof item !== "object" || Array.isArray(item)) {
|
|
159
|
-
throw new TranscribeAPIError("Each batch item must be an object with
|
|
184
|
+
throw new TranscribeAPIError("Each batch item must be an object with either `file` or `url`.", {
|
|
160
185
|
code: "invalid_batch_item",
|
|
161
186
|
});
|
|
162
187
|
}
|
|
163
188
|
|
|
164
|
-
const referenceId = String(item.reference_id || "").trim();
|
|
165
|
-
if (!referenceId) {
|
|
166
|
-
throw new TranscribeAPIError(`files[${index}].reference_id is required.`, {
|
|
167
|
-
code: "missing_reference_id",
|
|
168
|
-
});
|
|
169
|
-
}
|
|
189
|
+
const referenceId = String(item.reference_id || "").trim() || null;
|
|
170
190
|
|
|
171
191
|
const hasFile = Object.prototype.hasOwnProperty.call(item, "file");
|
|
172
192
|
const hasUrl = typeof item.url === "string" && item.url.trim();
|
|
@@ -186,11 +206,14 @@ function normalizeBatchInputItem(item, index) {
|
|
|
186
206
|
file: hasFile ? item.file : null,
|
|
187
207
|
url: hasUrl ? item.url.trim() : null,
|
|
188
208
|
durationEstimateSec: item.durationEstimateSec || item.duration_estimate_sec || null,
|
|
209
|
+
hasLanguage: Object.prototype.hasOwnProperty.call(item, "language"),
|
|
210
|
+
language: normalizeLanguageCode(item.language, `files[${index}].language`),
|
|
189
211
|
};
|
|
190
212
|
}
|
|
191
213
|
|
|
192
214
|
function uploadDescriptorForFile(referenceId, file) {
|
|
193
|
-
const descriptor = {
|
|
215
|
+
const descriptor = {};
|
|
216
|
+
if (referenceId) descriptor.reference_id = referenceId;
|
|
194
217
|
if (Number(file?.size || 0) >= MULTIPART_UPLOAD_THRESHOLD_BYTES) {
|
|
195
218
|
descriptor.size_bytes = Number(file.size || 0);
|
|
196
219
|
}
|
|
@@ -220,9 +243,10 @@ function uploadFromResponse(response, referenceId) {
|
|
|
220
243
|
}
|
|
221
244
|
|
|
222
245
|
function emitProgress(handler, event) {
|
|
223
|
-
if (typeof handler
|
|
224
|
-
|
|
246
|
+
if (typeof handler !== "function") {
|
|
247
|
+
return;
|
|
225
248
|
}
|
|
249
|
+
handler(event);
|
|
226
250
|
}
|
|
227
251
|
|
|
228
252
|
function renderProgressBar(loaded, total, width = 30) {
|
|
@@ -243,60 +267,151 @@ function formatBytes(bytes) {
|
|
|
243
267
|
}
|
|
244
268
|
|
|
245
269
|
function createSdkLoggerProgressHandler(logger = console) {
|
|
270
|
+
let activeProgressLine = false;
|
|
271
|
+
let lastRenderedLength = 0;
|
|
272
|
+
|
|
273
|
+
const writeLine = (line) => {
|
|
274
|
+
if (activeProgressLine && typeof process !== "undefined" && process?.stdout?.write) {
|
|
275
|
+
process.stdout.write("\r".padEnd(lastRenderedLength + 1, " "));
|
|
276
|
+
process.stdout.write("\n");
|
|
277
|
+
activeProgressLine = false;
|
|
278
|
+
lastRenderedLength = 0;
|
|
279
|
+
}
|
|
280
|
+
if (typeof logger?.log === "function") {
|
|
281
|
+
logger.log(line);
|
|
282
|
+
}
|
|
283
|
+
};
|
|
284
|
+
|
|
285
|
+
const writeProgress = (line) => {
|
|
286
|
+
if (typeof process !== "undefined" && process?.stdout?.write) {
|
|
287
|
+
const paddedLine = line.padEnd(lastRenderedLength, " ");
|
|
288
|
+
lastRenderedLength = paddedLine.length;
|
|
289
|
+
process.stdout.write(`\r${paddedLine}`);
|
|
290
|
+
activeProgressLine = true;
|
|
291
|
+
return;
|
|
292
|
+
}
|
|
293
|
+
if (typeof logger?.log === "function") {
|
|
294
|
+
logger.log(line);
|
|
295
|
+
}
|
|
296
|
+
};
|
|
297
|
+
|
|
246
298
|
return (event) => {
|
|
247
|
-
if (!event || typeof event !== "object"
|
|
299
|
+
if (!event || typeof event !== "object") {
|
|
248
300
|
return;
|
|
249
301
|
}
|
|
250
302
|
if (event.event === "upload_started") {
|
|
251
|
-
|
|
303
|
+
writeLine(`Uploading ${event.uploadFiles} file(s) for ${event.jobId}`);
|
|
252
304
|
return;
|
|
253
305
|
}
|
|
254
306
|
if (event.event === "upload_progress") {
|
|
255
307
|
const loaded = Number(event.batchLoaded ?? event.loaded ?? 0);
|
|
256
308
|
const total = Number(event.batchTotal ?? event.total ?? 0);
|
|
257
309
|
const percent = total ? ((loaded / total) * 100).toFixed(1) : "0.0";
|
|
258
|
-
|
|
310
|
+
writeProgress(`Uploading ${renderProgressBar(loaded, total)} ${percent}% (${formatBytes(loaded)} / ${formatBytes(total)})`);
|
|
311
|
+
if (total && loaded >= total && activeProgressLine && typeof process !== "undefined" && process?.stdout?.write) {
|
|
312
|
+
process.stdout.write("\n");
|
|
313
|
+
activeProgressLine = false;
|
|
314
|
+
lastRenderedLength = 0;
|
|
315
|
+
}
|
|
259
316
|
return;
|
|
260
317
|
}
|
|
261
|
-
if (event.event === "upload_completed"
|
|
262
|
-
|
|
318
|
+
if (event.event === "upload_completed") {
|
|
319
|
+
if (event.suppressLog) {
|
|
320
|
+
if (activeProgressLine && typeof process !== "undefined" && process?.stdout?.write) {
|
|
321
|
+
process.stdout.write("\n");
|
|
322
|
+
activeProgressLine = false;
|
|
323
|
+
lastRenderedLength = 0;
|
|
324
|
+
}
|
|
325
|
+
return;
|
|
326
|
+
}
|
|
327
|
+
writeLine(`Uploaded completed: ${JSON.stringify(event.response, null, 2)}`);
|
|
263
328
|
}
|
|
264
329
|
};
|
|
265
330
|
}
|
|
266
331
|
|
|
267
332
|
function createSdkPollingLogger(logger = console) {
|
|
268
|
-
let
|
|
333
|
+
let spinnerTimer = null;
|
|
334
|
+
let activeLine = false;
|
|
335
|
+
let currentFrameIndex = 0;
|
|
336
|
+
let currentStatus = "waiting";
|
|
337
|
+
let currentJobId = "unknown_job";
|
|
338
|
+
let lastRenderedLength = 0;
|
|
339
|
+
const frames = ["|", "/", "-", "\\"];
|
|
340
|
+
|
|
341
|
+
const stopSpinnerLine = () => {
|
|
342
|
+
if (spinnerTimer) {
|
|
343
|
+
clearInterval(spinnerTimer);
|
|
344
|
+
spinnerTimer = null;
|
|
345
|
+
}
|
|
346
|
+
if (activeLine && typeof process !== "undefined" && process?.stdout?.write) {
|
|
347
|
+
process.stdout.write("\n");
|
|
348
|
+
activeLine = false;
|
|
349
|
+
}
|
|
350
|
+
};
|
|
351
|
+
|
|
352
|
+
const renderLine = () => {
|
|
353
|
+
const frame = frames[currentFrameIndex % frames.length];
|
|
354
|
+
currentFrameIndex += 1;
|
|
355
|
+
const rawLine = `${frame} Polling ${currentJobId} (${currentStatus})`;
|
|
356
|
+
const paddedLine = rawLine.padEnd(lastRenderedLength, " ");
|
|
357
|
+
lastRenderedLength = paddedLine.length;
|
|
358
|
+
if (typeof process !== "undefined" && process?.stdout?.write) {
|
|
359
|
+
process.stdout.write(`\r${paddedLine}`);
|
|
360
|
+
activeLine = true;
|
|
361
|
+
return;
|
|
362
|
+
}
|
|
363
|
+
if (typeof logger?.log === "function") {
|
|
364
|
+
logger.log(rawLine);
|
|
365
|
+
}
|
|
366
|
+
};
|
|
367
|
+
|
|
368
|
+
const writeLine = (line) => {
|
|
369
|
+
stopSpinnerLine();
|
|
370
|
+
if (typeof logger?.log === "function") {
|
|
371
|
+
logger.log(line);
|
|
372
|
+
}
|
|
373
|
+
};
|
|
374
|
+
|
|
269
375
|
return {
|
|
270
376
|
start({ jobId, jobStatus }) {
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
if (jobStatus === lastStatus) {
|
|
377
|
+
currentJobId = jobId || currentJobId;
|
|
378
|
+
currentStatus = jobStatus || currentStatus;
|
|
379
|
+
currentFrameIndex = 0;
|
|
380
|
+
if (typeof process !== "undefined" && process?.stdout?.write) {
|
|
381
|
+
renderLine();
|
|
382
|
+
spinnerTimer = setInterval(renderLine, DEFAULT_POLLING_SPINNER_INTERVAL_MS);
|
|
278
383
|
return;
|
|
279
384
|
}
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
385
|
+
writeLine(`Polling ${currentJobId} (${currentStatus})`);
|
|
386
|
+
},
|
|
387
|
+
update({ jobStatus }) {
|
|
388
|
+
currentStatus = jobStatus || currentStatus;
|
|
389
|
+
if (!spinnerTimer && !(typeof process !== "undefined" && process?.stdout?.write)) {
|
|
390
|
+
writeLine(`Polling ${currentJobId} (${currentStatus})`);
|
|
283
391
|
}
|
|
284
392
|
},
|
|
285
393
|
finish({ jobStatus, resultUrl }) {
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
? `Polling complete: ${jobStatus} - ${resultUrl}`
|
|
289
|
-
: `Polling complete: ${jobStatus}`);
|
|
290
|
-
}
|
|
394
|
+
currentStatus = jobStatus || currentStatus;
|
|
395
|
+
writeLine(resultUrl || `Polling complete: ${currentStatus}`);
|
|
291
396
|
},
|
|
292
397
|
timeout({ timeoutSeconds, jobStatus }) {
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
}
|
|
398
|
+
currentStatus = jobStatus || currentStatus;
|
|
399
|
+
writeLine(`Polling stopped: timeout after ${timeoutSeconds}s (${currentStatus})`);
|
|
296
400
|
},
|
|
297
401
|
};
|
|
298
402
|
}
|
|
299
403
|
|
|
404
|
+
function logTerminalAsyncResult(result, logger = console) {
|
|
405
|
+
if (typeof logger?.log !== "function" || !result || typeof result !== "object") {
|
|
406
|
+
return;
|
|
407
|
+
}
|
|
408
|
+
if (result.result_url) {
|
|
409
|
+
logger.log(String(result.result_url));
|
|
410
|
+
return;
|
|
411
|
+
}
|
|
412
|
+
logger.log(JSON.stringify(result, null, 2));
|
|
413
|
+
}
|
|
414
|
+
|
|
300
415
|
function composeProgressHandler({ onProgress, showLogs = false, logger = console } = {}) {
|
|
301
416
|
const logHandler = showLogs ? createSdkLoggerProgressHandler(logger) : null;
|
|
302
417
|
if (!logHandler) {
|
|
@@ -511,7 +626,14 @@ async function estimateDurationFromFile(file) {
|
|
|
511
626
|
}
|
|
512
627
|
|
|
513
628
|
try {
|
|
514
|
-
|
|
629
|
+
let bytes;
|
|
630
|
+
if (typeof file?.readSlice === "function") {
|
|
631
|
+
bytes = new Uint8Array(await file.readSlice(0, headerLength));
|
|
632
|
+
} else if (typeof file?.slice === "function") {
|
|
633
|
+
bytes = new Uint8Array(await file.slice(0, headerLength).arrayBuffer());
|
|
634
|
+
} else {
|
|
635
|
+
return estimateDurationFromSize(file?.size);
|
|
636
|
+
}
|
|
515
637
|
const type = typeFromNameOrContentType(file?.name, file?.type || contentTypeFromName(file?.name));
|
|
516
638
|
const duration = type === "wav"
|
|
517
639
|
? parseWavDuration(bytes)
|
|
@@ -528,6 +650,30 @@ async function estimateDurationFromFile(file) {
|
|
|
528
650
|
return estimateDurationFromSize(file?.size);
|
|
529
651
|
}
|
|
530
652
|
|
|
653
|
+
async function fileFromPath(pathValue) {
|
|
654
|
+
const fs = await import("node:fs/promises");
|
|
655
|
+
const path = await import("node:path");
|
|
656
|
+
const stats = await fs.stat(pathValue);
|
|
657
|
+
const name = path.basename(pathValue);
|
|
658
|
+
return {
|
|
659
|
+
name,
|
|
660
|
+
size: stats.size,
|
|
661
|
+
type: contentTypeFromName(name),
|
|
662
|
+
path: pathValue,
|
|
663
|
+
async readSlice(start, end) {
|
|
664
|
+
const handle = await fs.open(pathValue, "r");
|
|
665
|
+
try {
|
|
666
|
+
const length = Math.max(0, end - start);
|
|
667
|
+
const buffer = Buffer.alloc(length);
|
|
668
|
+
const { bytesRead } = await handle.read(buffer, 0, length, start);
|
|
669
|
+
return buffer.subarray(0, bytesRead);
|
|
670
|
+
} finally {
|
|
671
|
+
await handle.close();
|
|
672
|
+
}
|
|
673
|
+
},
|
|
674
|
+
};
|
|
675
|
+
}
|
|
676
|
+
|
|
531
677
|
function makeFile(parts, name, type) {
|
|
532
678
|
if (typeof File !== "undefined") {
|
|
533
679
|
return new File(parts, name, { type });
|
|
@@ -544,14 +690,12 @@ async function normalizeFile(input, fallbackName = "audio.mp3") {
|
|
|
544
690
|
&& typeof input.name === "string"
|
|
545
691
|
&& Number.isFinite(Number(input.size))
|
|
546
692
|
&& typeof input.type === "string"
|
|
547
|
-
&& typeof input.slice === "function"
|
|
693
|
+
&& (typeof input.readSlice === "function" || typeof input.slice === "function")
|
|
548
694
|
) {
|
|
549
695
|
return input;
|
|
550
696
|
}
|
|
551
697
|
if (typeof input === "string") {
|
|
552
|
-
|
|
553
|
-
code: "unsupported_file_path",
|
|
554
|
-
});
|
|
698
|
+
return fileFromPath(input);
|
|
555
699
|
}
|
|
556
700
|
if (typeof File !== "undefined" && input instanceof File) {
|
|
557
701
|
return input;
|
|
@@ -562,7 +706,7 @@ async function normalizeFile(input, fallbackName = "audio.mp3") {
|
|
|
562
706
|
if (input?.data instanceof Uint8Array || input?.data instanceof ArrayBuffer) {
|
|
563
707
|
return makeFile([input.data], input.name || fallbackName, input.type || contentTypeFromName(input.name || fallbackName));
|
|
564
708
|
}
|
|
565
|
-
throw new TranscribeAPIError("Invalid file. Provide a File, Blob, or { data, name, type }.");
|
|
709
|
+
throw new TranscribeAPIError("Invalid file. Provide a path, File, Blob, or { data, name, type }.");
|
|
566
710
|
}
|
|
567
711
|
|
|
568
712
|
async function parseApiResponse(response) {
|
|
@@ -571,14 +715,24 @@ async function parseApiResponse(response) {
|
|
|
571
715
|
try {
|
|
572
716
|
data = text ? JSON.parse(text) : {};
|
|
573
717
|
} catch {
|
|
574
|
-
data =
|
|
718
|
+
data = text;
|
|
575
719
|
}
|
|
576
720
|
if (!response.ok) {
|
|
577
|
-
|
|
721
|
+
const body = data && typeof data === "object" ? data : {};
|
|
722
|
+
const message = body.message || body.error || text || `HTTP ${response.status}`;
|
|
723
|
+
const extra = { ...body };
|
|
724
|
+
delete extra.message;
|
|
725
|
+
delete extra.error;
|
|
726
|
+
delete extra.code;
|
|
727
|
+
throw new TranscribeAPIError(message, {
|
|
578
728
|
status: response.status,
|
|
579
|
-
code:
|
|
580
|
-
extra
|
|
581
|
-
response:
|
|
729
|
+
code: body.code || null,
|
|
730
|
+
extra,
|
|
731
|
+
response: {
|
|
732
|
+
message,
|
|
733
|
+
...(body.code ? { code: body.code } : {}),
|
|
734
|
+
...extra,
|
|
735
|
+
},
|
|
582
736
|
});
|
|
583
737
|
}
|
|
584
738
|
return data;
|
|
@@ -592,6 +746,68 @@ function normalizeHeaders(headers = {}) {
|
|
|
592
746
|
);
|
|
593
747
|
}
|
|
594
748
|
|
|
749
|
+
function isNodeStreamBody(body) {
|
|
750
|
+
return Boolean(body && typeof body === "object" && typeof body.pipe === "function");
|
|
751
|
+
}
|
|
752
|
+
|
|
753
|
+
async function putNodeStream(upload, streamBody, headers) {
|
|
754
|
+
const http = await import("node:http");
|
|
755
|
+
const https = await import("node:https");
|
|
756
|
+
const { pipeline } = await import("node:stream/promises");
|
|
757
|
+
const target = new URL(upload.url || upload);
|
|
758
|
+
const transport = target.protocol === "https:" ? https : http;
|
|
759
|
+
|
|
760
|
+
return new Promise((resolve, reject) => {
|
|
761
|
+
let settled = false;
|
|
762
|
+
const finishResolve = (value) => {
|
|
763
|
+
if (settled) {
|
|
764
|
+
return;
|
|
765
|
+
}
|
|
766
|
+
settled = true;
|
|
767
|
+
resolve(value);
|
|
768
|
+
};
|
|
769
|
+
const finishReject = (error) => {
|
|
770
|
+
if (settled) {
|
|
771
|
+
return;
|
|
772
|
+
}
|
|
773
|
+
settled = true;
|
|
774
|
+
reject(error);
|
|
775
|
+
};
|
|
776
|
+
const request = transport.request(target, {
|
|
777
|
+
method: "PUT",
|
|
778
|
+
headers,
|
|
779
|
+
agent: false,
|
|
780
|
+
}, (response) => {
|
|
781
|
+
response.resume();
|
|
782
|
+
response.on("error", finishReject);
|
|
783
|
+
response.on("end", () => {
|
|
784
|
+
finishResolve({
|
|
785
|
+
ok: response.statusCode >= 200 && response.statusCode < 300,
|
|
786
|
+
status: response.statusCode || 0,
|
|
787
|
+
headers: {
|
|
788
|
+
get(name) {
|
|
789
|
+
const value = response.headers[String(name || "").toLowerCase()];
|
|
790
|
+
return Array.isArray(value) ? value[0] : value || null;
|
|
791
|
+
},
|
|
792
|
+
},
|
|
793
|
+
});
|
|
794
|
+
});
|
|
795
|
+
});
|
|
796
|
+
|
|
797
|
+
request.on("error", finishReject);
|
|
798
|
+
request.on("socket", (socket) => {
|
|
799
|
+
socket.on("error", finishReject);
|
|
800
|
+
});
|
|
801
|
+
request.setTimeout(120000, () => {
|
|
802
|
+
request.destroy(new Error("stream_upload_timeout"));
|
|
803
|
+
});
|
|
804
|
+
if (typeof streamBody.on === "function") {
|
|
805
|
+
streamBody.on("error", finishReject);
|
|
806
|
+
}
|
|
807
|
+
pipeline(streamBody, request).catch(finishReject);
|
|
808
|
+
});
|
|
809
|
+
}
|
|
810
|
+
|
|
595
811
|
async function putObjectWithRetry(upload, body, {
|
|
596
812
|
onProgress,
|
|
597
813
|
loadedOffset = 0,
|
|
@@ -599,17 +815,39 @@ async function putObjectWithRetry(upload, body, {
|
|
|
599
815
|
contentLength = null,
|
|
600
816
|
progressMeta = null,
|
|
601
817
|
debugContext = null,
|
|
818
|
+
uploadLimiter = null,
|
|
602
819
|
} = {}) {
|
|
603
|
-
const response = await retry(async () => {
|
|
604
|
-
const
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
headers: normalizeHeaders({
|
|
820
|
+
const response = await retry(async (attemptIndex) => {
|
|
821
|
+
const executePut = async () => {
|
|
822
|
+
const resolvedBody = typeof body === "function" ? await body() : body;
|
|
823
|
+
const headers = normalizeHeaders({
|
|
608
824
|
...(upload.headers || {}),
|
|
609
825
|
...(contentLength !== null ? { "Content-Length": contentLength } : {}),
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
826
|
+
...(typeof body === "function" ? { Connection: "close" } : {}),
|
|
827
|
+
});
|
|
828
|
+
const putResponse = isNodeStreamBody(resolvedBody)
|
|
829
|
+
? await putNodeStream(upload, resolvedBody, headers)
|
|
830
|
+
: await fetch(upload.url || upload, {
|
|
831
|
+
method: "PUT",
|
|
832
|
+
headers,
|
|
833
|
+
body: resolvedBody,
|
|
834
|
+
});
|
|
835
|
+
if (!putResponse.ok) {
|
|
836
|
+
throw new TranscribeAPIError(`R2 upload failed with HTTP ${putResponse.status}.`, {
|
|
837
|
+
status: putResponse.status,
|
|
838
|
+
code: "upload_failed",
|
|
839
|
+
extra: {
|
|
840
|
+
...(progressMeta || {}),
|
|
841
|
+
...(debugContext || {}),
|
|
842
|
+
content_length: contentLength,
|
|
843
|
+
},
|
|
844
|
+
});
|
|
845
|
+
}
|
|
846
|
+
return putResponse;
|
|
847
|
+
};
|
|
848
|
+
const putResponse = uploadLimiter?.withToken
|
|
849
|
+
? await uploadLimiter.withToken(executePut)
|
|
850
|
+
: await executePut();
|
|
613
851
|
if (!putResponse.ok) {
|
|
614
852
|
throw new TranscribeAPIError(`R2 upload failed with HTTP ${putResponse.status}.`, {
|
|
615
853
|
status: putResponse.status,
|
|
@@ -626,7 +864,6 @@ async function putObjectWithRetry(upload, body, {
|
|
|
626
864
|
attempts: 5,
|
|
627
865
|
baseDelayMs: 1000,
|
|
628
866
|
});
|
|
629
|
-
|
|
630
867
|
const transferredBytes = Number(
|
|
631
868
|
contentLength
|
|
632
869
|
?? body?.size
|
|
@@ -644,6 +881,14 @@ async function putObjectWithRetry(upload, body, {
|
|
|
644
881
|
return response;
|
|
645
882
|
}
|
|
646
883
|
|
|
884
|
+
async function openFileBody(file, start = 0, end = null) {
|
|
885
|
+
if (!file?.path) {
|
|
886
|
+
return file;
|
|
887
|
+
}
|
|
888
|
+
const fs = await import("node:fs");
|
|
889
|
+
return fs.createReadStream(file.path, end === null ? { start } : { start, end: end - 1 });
|
|
890
|
+
}
|
|
891
|
+
|
|
647
892
|
async function runWithConcurrency(items, concurrency, worker) {
|
|
648
893
|
let index = 0;
|
|
649
894
|
const workerCount = Math.max(1, Math.min(items.length || 1, concurrency));
|
|
@@ -660,6 +905,41 @@ async function runWithConcurrency(items, concurrency, worker) {
|
|
|
660
905
|
await Promise.all(workers);
|
|
661
906
|
}
|
|
662
907
|
|
|
908
|
+
function createUploadConcurrencyLimiter(concurrency) {
|
|
909
|
+
const maxConcurrency = Math.max(1, normalizeUploadConcurrency(concurrency));
|
|
910
|
+
let active = 0;
|
|
911
|
+
const waiters = [];
|
|
912
|
+
|
|
913
|
+
const acquire = async () => {
|
|
914
|
+
if (active < maxConcurrency) {
|
|
915
|
+
active += 1;
|
|
916
|
+
return;
|
|
917
|
+
}
|
|
918
|
+
await new Promise((resolve) => waiters.push(resolve));
|
|
919
|
+
active += 1;
|
|
920
|
+
};
|
|
921
|
+
|
|
922
|
+
const release = () => {
|
|
923
|
+
active = Math.max(0, active - 1);
|
|
924
|
+
const next = waiters.shift();
|
|
925
|
+
if (next) {
|
|
926
|
+
next();
|
|
927
|
+
}
|
|
928
|
+
};
|
|
929
|
+
|
|
930
|
+
return {
|
|
931
|
+
concurrency: maxConcurrency,
|
|
932
|
+
async withToken(operation) {
|
|
933
|
+
await acquire();
|
|
934
|
+
try {
|
|
935
|
+
return await operation();
|
|
936
|
+
} finally {
|
|
937
|
+
release();
|
|
938
|
+
}
|
|
939
|
+
},
|
|
940
|
+
};
|
|
941
|
+
}
|
|
942
|
+
|
|
663
943
|
function multipartCompleteXml(parts) {
|
|
664
944
|
const rows = parts
|
|
665
945
|
.sort((a, b) => a.partNumber - b.partNumber)
|
|
@@ -670,18 +950,19 @@ function multipartCompleteXml(parts) {
|
|
|
670
950
|
|
|
671
951
|
async function uploadMultipart(upload, file, {
|
|
672
952
|
onProgress,
|
|
673
|
-
|
|
953
|
+
uploadConcurrency = DEFAULT_UPLOAD_CONCURRENCY,
|
|
674
954
|
onConcurrencyChange = null,
|
|
955
|
+
uploadLimiter = null,
|
|
675
956
|
} = {}) {
|
|
676
957
|
const completed = [];
|
|
677
|
-
const completedPartNumbers = new Set();
|
|
678
|
-
let completedBytes = 0;
|
|
679
958
|
const totalParts = upload.parts.length;
|
|
680
|
-
const
|
|
959
|
+
const completedPartNumbers = new Set(completed.map((part) => part.partNumber));
|
|
960
|
+
let completedBytes = 0;
|
|
961
|
+
const concurrency = normalizeUploadConcurrency(uploadConcurrency);
|
|
681
962
|
let targetConcurrency = concurrency;
|
|
682
963
|
let activeWorkers = 0;
|
|
683
964
|
let fatalError = null;
|
|
684
|
-
const pendingParts =
|
|
965
|
+
const pendingParts = upload.parts.filter((part) => !completedPartNumbers.has(part.part_number));
|
|
685
966
|
const partAttempts = new Map();
|
|
686
967
|
|
|
687
968
|
const workers = Array.from({ length: concurrency }, async (_, workerIndex) => {
|
|
@@ -696,7 +977,6 @@ async function uploadMultipart(upload, file, {
|
|
|
696
977
|
await sleep(MULTIPART_IDLE_WAIT_MS);
|
|
697
978
|
continue;
|
|
698
979
|
}
|
|
699
|
-
|
|
700
980
|
const part = pendingParts.shift();
|
|
701
981
|
if (!part) {
|
|
702
982
|
if (!activeWorkers) {
|
|
@@ -713,7 +993,7 @@ async function uploadMultipart(upload, file, {
|
|
|
713
993
|
try {
|
|
714
994
|
const response = await putObjectWithRetry(
|
|
715
995
|
{ url: part.url },
|
|
716
|
-
file.slice(start, end),
|
|
996
|
+
file.path ? (() => openFileBody(file, start, end)) : file.slice(start, end),
|
|
717
997
|
{
|
|
718
998
|
onProgress: null,
|
|
719
999
|
loadedOffset: 0,
|
|
@@ -731,6 +1011,7 @@ async function uploadMultipart(upload, file, {
|
|
|
731
1011
|
range_start: start,
|
|
732
1012
|
range_end_exclusive: end,
|
|
733
1013
|
},
|
|
1014
|
+
uploadLimiter,
|
|
734
1015
|
},
|
|
735
1016
|
);
|
|
736
1017
|
const etag = response.headers.get("ETag") || response.headers.get("etag");
|
|
@@ -754,7 +1035,8 @@ async function uploadMultipart(upload, file, {
|
|
|
754
1035
|
} catch (error) {
|
|
755
1036
|
const attempts = (partAttempts.get(part.part_number) || 0) + 1;
|
|
756
1037
|
partAttempts.set(part.part_number, attempts);
|
|
757
|
-
|
|
1038
|
+
const retryable = isRetryableError(error);
|
|
1039
|
+
if (retryable && attempts < MAX_MULTIPART_ADAPTIVE_ATTEMPTS) {
|
|
758
1040
|
const previousConcurrency = targetConcurrency;
|
|
759
1041
|
targetConcurrency = Math.max(1, Math.floor(targetConcurrency / 2));
|
|
760
1042
|
pendingParts.push(part);
|
|
@@ -775,19 +1057,32 @@ async function uploadMultipart(upload, file, {
|
|
|
775
1057
|
await sleep(Math.min(10000, 1000 * (2 ** Math.min(attempts - 1, 3))));
|
|
776
1058
|
continue;
|
|
777
1059
|
}
|
|
778
|
-
fatalError = new TranscribeAPIError(
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
1060
|
+
fatalError = new TranscribeAPIError(
|
|
1061
|
+
`Multipart upload failed for part ${part.part_number}/${totalParts}.`,
|
|
1062
|
+
{
|
|
1063
|
+
status: error?.status || null,
|
|
1064
|
+
code: error?.code || "multipart_part_upload_failed",
|
|
1065
|
+
extra: {
|
|
1066
|
+
cause_error: error?.error || error?.message || String(error),
|
|
1067
|
+
cause_code: error?.code || error?.cause?.code || null,
|
|
1068
|
+
cause_status: error?.status || null,
|
|
1069
|
+
upload_type: "multipart",
|
|
1070
|
+
part_number: part.part_number,
|
|
1071
|
+
total_parts: totalParts,
|
|
1072
|
+
chunk_bytes: chunkSize,
|
|
1073
|
+
completed_bytes_before_failure: completedBytes,
|
|
1074
|
+
multipart_concurrency: targetConcurrency,
|
|
1075
|
+
file_name: file?.name || null,
|
|
1076
|
+
file_size: file?.size || null,
|
|
1077
|
+
range_start: start,
|
|
1078
|
+
range_end_exclusive: end,
|
|
1079
|
+
attempts,
|
|
1080
|
+
},
|
|
785
1081
|
},
|
|
786
|
-
|
|
1082
|
+
);
|
|
787
1083
|
activeWorkers -= 1;
|
|
788
1084
|
return;
|
|
789
1085
|
}
|
|
790
|
-
|
|
791
1086
|
activeWorkers -= 1;
|
|
792
1087
|
}
|
|
793
1088
|
});
|
|
@@ -818,7 +1113,8 @@ async function uploadUsingInstructions(upload, file, options = {}) {
|
|
|
818
1113
|
if (upload.type === "multipart") {
|
|
819
1114
|
return uploadMultipart(upload, file, options);
|
|
820
1115
|
}
|
|
821
|
-
|
|
1116
|
+
const body = file.path ? (() => openFileBody(file)) : file;
|
|
1117
|
+
return putObjectWithRetry(upload, body, {
|
|
822
1118
|
onProgress: options.onProgress,
|
|
823
1119
|
loadedOffset: 0,
|
|
824
1120
|
totalBytes: file.size,
|
|
@@ -831,6 +1127,7 @@ async function uploadUsingInstructions(upload, file, options = {}) {
|
|
|
831
1127
|
file_name: file?.name || null,
|
|
832
1128
|
file_size: file?.size || null,
|
|
833
1129
|
},
|
|
1130
|
+
uploadLimiter: options.uploadLimiter || null,
|
|
834
1131
|
});
|
|
835
1132
|
}
|
|
836
1133
|
|
|
@@ -847,66 +1144,6 @@ function assertBatchLimits(totalFiles, totalSizeBytes) {
|
|
|
847
1144
|
}
|
|
848
1145
|
}
|
|
849
1146
|
|
|
850
|
-
class BigFileJob {
|
|
851
|
-
constructor(client, file, createResponse, options) {
|
|
852
|
-
this.client = client;
|
|
853
|
-
this.file = file;
|
|
854
|
-
this.createResponse = createResponse;
|
|
855
|
-
this.referenceId = options.referenceId || defaultReferenceId(0);
|
|
856
|
-
this.jobId = createResponse.job_id;
|
|
857
|
-
this.jobStatus = createResponse.job_status;
|
|
858
|
-
this.model = createResponse.model || options.model || null;
|
|
859
|
-
this.uploadInfo = uploadFromResponse(createResponse, this.referenceId);
|
|
860
|
-
this.options = options;
|
|
861
|
-
}
|
|
862
|
-
|
|
863
|
-
async upload({ onProgress } = {}) {
|
|
864
|
-
const progress = onProgress || this.options.onProgress;
|
|
865
|
-
if (!this.uploadInfo) {
|
|
866
|
-
return this.createResponse;
|
|
867
|
-
}
|
|
868
|
-
await uploadUsingInstructions(this.uploadInfo, this.file, {
|
|
869
|
-
onProgress: progress,
|
|
870
|
-
multipartConcurrency: this.options.multipartConcurrency,
|
|
871
|
-
onConcurrencyChange: (event) => {
|
|
872
|
-
if (typeof onProgress === "function" || typeof this.options.onProgress === "function") {
|
|
873
|
-
(onProgress || this.options.onProgress)({
|
|
874
|
-
uploadType: "multipart",
|
|
875
|
-
event: "adaptive_concurrency",
|
|
876
|
-
previousConcurrency: event.previousConcurrency,
|
|
877
|
-
multipartConcurrency: event.nextConcurrency,
|
|
878
|
-
partNumber: event.partNumber,
|
|
879
|
-
attempts: event.attempts,
|
|
880
|
-
error: event.error,
|
|
881
|
-
});
|
|
882
|
-
}
|
|
883
|
-
},
|
|
884
|
-
});
|
|
885
|
-
const completion = await this.client.jobs.complete(this.jobId);
|
|
886
|
-
emitProgress(progress, {
|
|
887
|
-
event: "upload_completed",
|
|
888
|
-
jobId: this.jobId,
|
|
889
|
-
response: completion,
|
|
890
|
-
suppressLog: Boolean(this.client.polling),
|
|
891
|
-
});
|
|
892
|
-
return completion;
|
|
893
|
-
}
|
|
894
|
-
}
|
|
895
|
-
|
|
896
|
-
class RemoteBigFileJob {
|
|
897
|
-
constructor(client, createResponse, options) {
|
|
898
|
-
this.client = client;
|
|
899
|
-
this.createResponse = createResponse;
|
|
900
|
-
this.jobId = createResponse.job_id;
|
|
901
|
-
this.jobStatus = createResponse.job_status;
|
|
902
|
-
this.model = createResponse.model || options.model || null;
|
|
903
|
-
}
|
|
904
|
-
|
|
905
|
-
async upload() {
|
|
906
|
-
return this.createResponse;
|
|
907
|
-
}
|
|
908
|
-
}
|
|
909
|
-
|
|
910
1147
|
class BatchJob {
|
|
911
1148
|
constructor(client, files, createResponse, options) {
|
|
912
1149
|
this.client = client;
|
|
@@ -930,6 +1167,7 @@ class BatchJob {
|
|
|
930
1167
|
const totalBytes = this.files.reduce((sum, item) => sum + Number(item?.file?.size || 0), 0);
|
|
931
1168
|
const loadedByReferenceId = new Map();
|
|
932
1169
|
const uploadableFiles = this.files.filter((item) => item.file);
|
|
1170
|
+
const uploadLimiter = this.options.uploadLimiter || createUploadConcurrencyLimiter(this.options.uploadConcurrency);
|
|
933
1171
|
|
|
934
1172
|
emitProgress(progress, {
|
|
935
1173
|
event: "upload_started",
|
|
@@ -939,10 +1177,7 @@ class BatchJob {
|
|
|
939
1177
|
totalBytes,
|
|
940
1178
|
});
|
|
941
1179
|
|
|
942
|
-
|
|
943
|
-
if (!item.file) {
|
|
944
|
-
continue;
|
|
945
|
-
}
|
|
1180
|
+
await runWithConcurrency(uploadableFiles, this.options.uploadConcurrency, async (item) => {
|
|
946
1181
|
const upload = this.uploadsByReferenceId.get(item.referenceId);
|
|
947
1182
|
if (!upload) {
|
|
948
1183
|
throw new TranscribeAPIError(`Missing upload instructions for \`${item.referenceId}\`.`, {
|
|
@@ -950,6 +1185,8 @@ class BatchJob {
|
|
|
950
1185
|
});
|
|
951
1186
|
}
|
|
952
1187
|
await uploadUsingInstructions(upload, item.file, {
|
|
1188
|
+
uploadConcurrency: this.options.uploadConcurrency,
|
|
1189
|
+
uploadLimiter,
|
|
953
1190
|
onProgress: (event) => {
|
|
954
1191
|
const fileLoaded = Math.max(0, Math.min(Number(event?.loaded || 0), Number(item.file.size || 0)));
|
|
955
1192
|
loadedByReferenceId.set(item.referenceId, fileLoaded);
|
|
@@ -968,8 +1205,7 @@ class BatchJob {
|
|
|
968
1205
|
});
|
|
969
1206
|
},
|
|
970
1207
|
});
|
|
971
|
-
}
|
|
972
|
-
|
|
1208
|
+
});
|
|
973
1209
|
const completion = await this.client.jobs.complete(this.jobId);
|
|
974
1210
|
emitProgress(progress, {
|
|
975
1211
|
event: "upload_completed",
|
|
@@ -979,7 +1215,7 @@ class BatchJob {
|
|
|
979
1215
|
batchTotal: totalBytes,
|
|
980
1216
|
totalFiles: this.files.length,
|
|
981
1217
|
uploadFiles: uploadableFiles.length,
|
|
982
|
-
suppressLog: Boolean(this.client.polling),
|
|
1218
|
+
suppressLog: Boolean(this.client.polling) && !TERMINAL_JOB_STATUSES.has(String(completion?.job_status || "")),
|
|
983
1219
|
});
|
|
984
1220
|
return completion;
|
|
985
1221
|
}
|
|
@@ -989,7 +1225,7 @@ export class TranscribeAPI {
|
|
|
989
1225
|
constructor({
|
|
990
1226
|
apiKey,
|
|
991
1227
|
baseUrl = DEFAULT_BASE_URL,
|
|
992
|
-
|
|
1228
|
+
uploadConcurrency = DEFAULT_UPLOAD_CONCURRENCY,
|
|
993
1229
|
showLogs = false,
|
|
994
1230
|
logger = console,
|
|
995
1231
|
polling = null,
|
|
@@ -1000,13 +1236,10 @@ export class TranscribeAPI {
|
|
|
1000
1236
|
|
|
1001
1237
|
this.apiKey = apiKey;
|
|
1002
1238
|
this.baseUrl = baseUrl.replace(/\/+$/, "");
|
|
1003
|
-
this.
|
|
1239
|
+
this.uploadConcurrency = normalizeUploadConcurrency(uploadConcurrency);
|
|
1004
1240
|
this.showLogs = Boolean(showLogs);
|
|
1005
1241
|
this.logger = logger || console;
|
|
1006
1242
|
this.polling = normalizePollingConfig(polling);
|
|
1007
|
-
this.batch = {
|
|
1008
|
-
transcribe: (options) => this.transcribeMany(options),
|
|
1009
|
-
};
|
|
1010
1243
|
this.jobs = {
|
|
1011
1244
|
createBigFile: (options) => this.createBigFileJob(options),
|
|
1012
1245
|
createBatch: (options) => this.createBatchJob(options),
|
|
@@ -1044,16 +1277,16 @@ export class TranscribeAPI {
|
|
|
1044
1277
|
}
|
|
1045
1278
|
|
|
1046
1279
|
const startedAt = Date.now();
|
|
1047
|
-
const
|
|
1280
|
+
const spinner = showLogs ? createSdkPollingLogger(logger) : null;
|
|
1048
1281
|
let lastJob = initialJob || null;
|
|
1049
|
-
|
|
1282
|
+
spinner?.start({
|
|
1050
1283
|
jobId,
|
|
1051
1284
|
jobStatus: lastJob?.job_status || "waiting",
|
|
1052
1285
|
});
|
|
1053
1286
|
|
|
1054
1287
|
while (!TERMINAL_JOB_STATUSES.has(String(lastJob?.job_status || ""))) {
|
|
1055
1288
|
if (effectivePolling.timeout !== null && (Date.now() - startedAt) >= effectivePolling.timeout * 1000) {
|
|
1056
|
-
|
|
1289
|
+
spinner?.timeout({
|
|
1057
1290
|
timeoutSeconds: effectivePolling.timeout,
|
|
1058
1291
|
jobStatus: lastJob?.job_status || "unknown",
|
|
1059
1292
|
});
|
|
@@ -1070,13 +1303,12 @@ export class TranscribeAPI {
|
|
|
1070
1303
|
|
|
1071
1304
|
await sleep(effectivePolling.interval * 1000);
|
|
1072
1305
|
lastJob = await this.jobs.get(jobId);
|
|
1073
|
-
|
|
1074
|
-
jobId,
|
|
1306
|
+
spinner?.update({
|
|
1075
1307
|
jobStatus: lastJob?.job_status || "unknown",
|
|
1076
1308
|
});
|
|
1077
1309
|
}
|
|
1078
1310
|
|
|
1079
|
-
|
|
1311
|
+
spinner?.finish({
|
|
1080
1312
|
jobStatus: lastJob?.job_status || "unknown",
|
|
1081
1313
|
resultUrl: lastJob?.result_url || null,
|
|
1082
1314
|
});
|
|
@@ -1084,96 +1316,152 @@ export class TranscribeAPI {
|
|
|
1084
1316
|
}
|
|
1085
1317
|
|
|
1086
1318
|
async transcribe({
|
|
1087
|
-
|
|
1319
|
+
files,
|
|
1088
1320
|
webhookUrl,
|
|
1089
1321
|
onProgress,
|
|
1090
1322
|
showLogs,
|
|
1091
1323
|
logger,
|
|
1092
1324
|
language,
|
|
1093
|
-
task,
|
|
1094
|
-
vadFilter,
|
|
1095
|
-
initialPrompt,
|
|
1096
|
-
vttGranularity,
|
|
1097
1325
|
exclude,
|
|
1098
|
-
|
|
1326
|
+
uploadConcurrency,
|
|
1099
1327
|
} = {}) {
|
|
1100
1328
|
const progress = composeProgressHandler({
|
|
1101
1329
|
onProgress,
|
|
1102
1330
|
showLogs: showLogs ?? this.showLogs,
|
|
1103
1331
|
logger: logger ?? this.logger,
|
|
1104
1332
|
});
|
|
1105
|
-
if (
|
|
1106
|
-
|
|
1107
|
-
file,
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
|
|
1333
|
+
if (isFilesInput(files)) {
|
|
1334
|
+
if (files.length === 0) {
|
|
1335
|
+
throw new TranscribeAPIError("Transcribe requires at least one file.", {
|
|
1336
|
+
code: "invalid_files",
|
|
1337
|
+
});
|
|
1338
|
+
}
|
|
1339
|
+
if (files.length > 1) {
|
|
1340
|
+
for (let index = 0; index < files.length; index += 1) {
|
|
1341
|
+
if (!String(files[index]?.reference_id || "").trim()) {
|
|
1342
|
+
throw new TranscribeAPIError(`files[${index}].reference_id is required when sending multiple files.`, {
|
|
1343
|
+
code: "missing_reference_id",
|
|
1344
|
+
});
|
|
1345
|
+
}
|
|
1346
|
+
}
|
|
1347
|
+
return this._transcribeAsync({
|
|
1348
|
+
files,
|
|
1349
|
+
webhookUrl,
|
|
1350
|
+
durationEstimateSec: undefined,
|
|
1351
|
+
onProgress,
|
|
1352
|
+
showLogs,
|
|
1118
1353
|
logger: logger ?? this.logger,
|
|
1119
|
-
|
|
1354
|
+
uploadConcurrency,
|
|
1355
|
+
language,
|
|
1356
|
+
exclude,
|
|
1120
1357
|
});
|
|
1121
1358
|
}
|
|
1122
|
-
return result;
|
|
1123
|
-
}
|
|
1124
1359
|
|
|
1125
|
-
|
|
1126
|
-
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
|
|
1139
|
-
|
|
1140
|
-
|
|
1360
|
+
const normalizedItem = normalizeBatchInputItem(files[0], 0);
|
|
1361
|
+
const normalizedSingleFile = normalizedItem.url ? null : await normalizeFile(normalizedItem.file);
|
|
1362
|
+
const estimatedDurationSec = normalizedItem.url
|
|
1363
|
+
? normalizedItem.durationEstimateSec
|
|
1364
|
+
: (normalizedItem.durationEstimateSec || await estimateDurationFromFile(normalizedSingleFile));
|
|
1365
|
+
const effectiveLanguage = normalizedItem.hasLanguage ? normalizedItem.language : normalizeLanguageCode(language);
|
|
1366
|
+
const isAsync = files.length > 1
|
|
1367
|
+
|| Boolean(normalizedItem.url)
|
|
1368
|
+
|| Boolean(webhookUrl)
|
|
1369
|
+
|| (
|
|
1370
|
+
normalizedSingleFile
|
|
1371
|
+
&& (
|
|
1372
|
+
normalizedSingleFile.size > MAX_SYNC_AUDIO_BYTES
|
|
1373
|
+
|| estimatedDurationSec > MAX_SYNC_AUDIO_SECONDS
|
|
1374
|
+
)
|
|
1375
|
+
);
|
|
1376
|
+
if (isAsync) {
|
|
1377
|
+
return this._transcribeAsync({
|
|
1378
|
+
files,
|
|
1379
|
+
webhookUrl,
|
|
1380
|
+
durationEstimateSec: estimatedDurationSec,
|
|
1381
|
+
onProgress,
|
|
1382
|
+
showLogs,
|
|
1141
1383
|
logger: logger ?? this.logger,
|
|
1142
|
-
|
|
1384
|
+
uploadConcurrency,
|
|
1385
|
+
language,
|
|
1386
|
+
exclude,
|
|
1143
1387
|
});
|
|
1144
1388
|
}
|
|
1145
|
-
|
|
1389
|
+
|
|
1390
|
+
return this.transcribeDirect({
|
|
1391
|
+
file: normalizedSingleFile,
|
|
1392
|
+
referenceId: normalizedItem.referenceId,
|
|
1393
|
+
language: effectiveLanguage,
|
|
1394
|
+
exclude,
|
|
1395
|
+
webhookUrl,
|
|
1396
|
+
showLogs: showLogs ?? this.showLogs,
|
|
1397
|
+
logger: logger ?? this.logger,
|
|
1398
|
+
});
|
|
1146
1399
|
}
|
|
1400
|
+
throw new TranscribeAPIError("`transcribe` requires a `files` array.", {
|
|
1401
|
+
code: "invalid_files",
|
|
1402
|
+
});
|
|
1403
|
+
}
|
|
1147
1404
|
|
|
1148
|
-
|
|
1149
|
-
|
|
1405
|
+
async _transcribeAsync({
|
|
1406
|
+
files,
|
|
1407
|
+
webhookUrl,
|
|
1408
|
+
durationEstimateSec,
|
|
1409
|
+
onProgress,
|
|
1410
|
+
showLogs,
|
|
1411
|
+
logger,
|
|
1412
|
+
uploadConcurrency,
|
|
1413
|
+
language,
|
|
1414
|
+
exclude,
|
|
1415
|
+
} = {}) {
|
|
1416
|
+
const progress = composeProgressHandler({
|
|
1417
|
+
onProgress,
|
|
1418
|
+
showLogs: showLogs ?? this.showLogs,
|
|
1419
|
+
logger: logger ?? this.logger,
|
|
1420
|
+
});
|
|
1421
|
+
const job = await this.createBatchJob({
|
|
1422
|
+
files,
|
|
1423
|
+
webhookUrl,
|
|
1424
|
+
durationEstimateSec,
|
|
1425
|
+
onProgress: progress,
|
|
1426
|
+
showLogs: false,
|
|
1427
|
+
uploadConcurrency,
|
|
1150
1428
|
language,
|
|
1151
|
-
task,
|
|
1152
|
-
vadFilter,
|
|
1153
|
-
initialPrompt,
|
|
1154
|
-
vttGranularity,
|
|
1155
1429
|
exclude,
|
|
1156
1430
|
});
|
|
1431
|
+
const result = await job.upload({ onProgress: progress });
|
|
1432
|
+
if (this.polling && !TERMINAL_JOB_STATUSES.has(String(result?.job_status || ""))) {
|
|
1433
|
+
return this.waitForJobCompletion(result.job_id, {
|
|
1434
|
+
polling: this.polling,
|
|
1435
|
+
showLogs: showLogs ?? this.showLogs,
|
|
1436
|
+
logger: logger ?? this.logger,
|
|
1437
|
+
initialJob: result,
|
|
1438
|
+
});
|
|
1439
|
+
}
|
|
1440
|
+
if ((showLogs ?? this.showLogs) && TERMINAL_JOB_STATUSES.has(String(result?.job_status || ""))) {
|
|
1441
|
+
logTerminalAsyncResult(result, logger ?? this.logger);
|
|
1442
|
+
}
|
|
1443
|
+
return result;
|
|
1157
1444
|
}
|
|
1158
1445
|
|
|
1159
1446
|
async transcribeDirect({
|
|
1160
1447
|
file,
|
|
1448
|
+
referenceId,
|
|
1161
1449
|
language,
|
|
1162
|
-
task,
|
|
1163
|
-
vadFilter,
|
|
1164
|
-
initialPrompt,
|
|
1165
|
-
vttGranularity,
|
|
1166
1450
|
exclude,
|
|
1451
|
+
webhookUrl,
|
|
1452
|
+
showLogs = false,
|
|
1453
|
+
logger = console,
|
|
1167
1454
|
} = {}) {
|
|
1168
1455
|
const normalizedFile = await normalizeFile(file);
|
|
1456
|
+
const directFile = normalizedFile.path
|
|
1457
|
+
? makeFile([await normalizedFile.readSlice(0, normalizedFile.size)], normalizedFile.name, normalizedFile.type)
|
|
1458
|
+
: normalizedFile;
|
|
1169
1459
|
const form = new FormData();
|
|
1170
|
-
form.set("
|
|
1460
|
+
if (String(referenceId || "").trim()) form.set("reference_id", String(referenceId).trim());
|
|
1461
|
+
form.set("file", directFile, directFile.name);
|
|
1171
1462
|
if (language) form.set("language", language);
|
|
1172
|
-
if (task) form.set("task", task);
|
|
1173
|
-
if (vadFilter !== undefined) form.set("vad_filter", String(Boolean(vadFilter)));
|
|
1174
|
-
if (initialPrompt) form.set("initial_prompt", initialPrompt);
|
|
1175
|
-
if (vttGranularity) form.set("vtt_granularity", vttGranularity);
|
|
1176
1463
|
if (exclude) form.set("exclude", Array.isArray(exclude) ? exclude.join(",") : exclude);
|
|
1464
|
+
if (webhookUrl) form.set("webhook_url", webhookUrl);
|
|
1177
1465
|
|
|
1178
1466
|
const response = await fetch(`${this.baseUrl}/transcribe`, {
|
|
1179
1467
|
method: "POST",
|
|
@@ -1182,7 +1470,19 @@ export class TranscribeAPI {
|
|
|
1182
1470
|
},
|
|
1183
1471
|
body: form,
|
|
1184
1472
|
});
|
|
1185
|
-
|
|
1473
|
+
const result = await parseApiResponse(response);
|
|
1474
|
+
if (this.polling && result && typeof result === "object" && result.job_status && !TERMINAL_JOB_STATUSES.has(String(result.job_status))) {
|
|
1475
|
+
return this.waitForJobCompletion(result.job_id, {
|
|
1476
|
+
polling: this.polling,
|
|
1477
|
+
showLogs,
|
|
1478
|
+
logger,
|
|
1479
|
+
initialJob: result,
|
|
1480
|
+
});
|
|
1481
|
+
}
|
|
1482
|
+
if (showLogs && typeof logger?.log === "function") {
|
|
1483
|
+
logger.log(typeof result === "string" ? result : JSON.stringify(result, null, 2));
|
|
1484
|
+
}
|
|
1485
|
+
return result;
|
|
1186
1486
|
}
|
|
1187
1487
|
|
|
1188
1488
|
async transcribeMany({
|
|
@@ -1192,31 +1492,21 @@ export class TranscribeAPI {
|
|
|
1192
1492
|
onProgress,
|
|
1193
1493
|
showLogs,
|
|
1194
1494
|
logger,
|
|
1195
|
-
|
|
1495
|
+
uploadConcurrency,
|
|
1496
|
+
language,
|
|
1497
|
+
exclude,
|
|
1196
1498
|
} = {}) {
|
|
1197
|
-
|
|
1198
|
-
onProgress,
|
|
1199
|
-
showLogs: showLogs ?? this.showLogs,
|
|
1200
|
-
logger: logger ?? this.logger,
|
|
1201
|
-
});
|
|
1202
|
-
const job = await this.createBatchJob({
|
|
1499
|
+
return this._transcribeAsync({
|
|
1203
1500
|
files,
|
|
1204
1501
|
webhookUrl,
|
|
1205
1502
|
durationEstimateSec,
|
|
1206
|
-
onProgress
|
|
1207
|
-
showLogs
|
|
1208
|
-
|
|
1503
|
+
onProgress,
|
|
1504
|
+
showLogs,
|
|
1505
|
+
logger: logger ?? this.logger,
|
|
1506
|
+
uploadConcurrency,
|
|
1507
|
+
language,
|
|
1508
|
+
exclude,
|
|
1209
1509
|
});
|
|
1210
|
-
const result = await job.upload({ onProgress: progress });
|
|
1211
|
-
if (this.polling && !TERMINAL_JOB_STATUSES.has(String(result?.job_status || ""))) {
|
|
1212
|
-
return this.waitForJobCompletion(result.job_id, {
|
|
1213
|
-
polling: this.polling,
|
|
1214
|
-
showLogs: showLogs ?? this.showLogs,
|
|
1215
|
-
logger: logger ?? this.logger,
|
|
1216
|
-
initialJob: result,
|
|
1217
|
-
});
|
|
1218
|
-
}
|
|
1219
|
-
return result;
|
|
1220
1510
|
}
|
|
1221
1511
|
|
|
1222
1512
|
async createBigFileJob({
|
|
@@ -1226,57 +1516,20 @@ export class TranscribeAPI {
|
|
|
1226
1516
|
onProgress,
|
|
1227
1517
|
showLogs,
|
|
1228
1518
|
logger,
|
|
1229
|
-
|
|
1519
|
+
uploadConcurrency,
|
|
1520
|
+
language,
|
|
1521
|
+
exclude,
|
|
1230
1522
|
} = {}) {
|
|
1231
|
-
|
|
1523
|
+
return this.createBatchJob({
|
|
1524
|
+
files: [file],
|
|
1525
|
+
webhookUrl,
|
|
1526
|
+
durationEstimateSec,
|
|
1232
1527
|
onProgress,
|
|
1233
|
-
showLogs
|
|
1528
|
+
showLogs,
|
|
1234
1529
|
logger: logger ?? this.logger,
|
|
1235
|
-
|
|
1236
|
-
|
|
1237
|
-
|
|
1238
|
-
const response = await this.requestJson("/transcribe", {
|
|
1239
|
-
method: "POST",
|
|
1240
|
-
body: {
|
|
1241
|
-
files: [{
|
|
1242
|
-
reference_id: defaultReferenceId(0),
|
|
1243
|
-
url: file.url,
|
|
1244
|
-
}],
|
|
1245
|
-
...(webhookUrl ? { webhook_url: webhookUrl } : {}),
|
|
1246
|
-
},
|
|
1247
|
-
retryable: true,
|
|
1248
|
-
});
|
|
1249
|
-
return new RemoteBigFileJob(this, response, {
|
|
1250
|
-
onProgress: progress,
|
|
1251
|
-
multipartConcurrency: normalizedConcurrency,
|
|
1252
|
-
});
|
|
1253
|
-
}
|
|
1254
|
-
|
|
1255
|
-
const normalizedFile = await normalizeFile(file);
|
|
1256
|
-
const estimate = durationEstimateSec || await estimateDurationFromFile(normalizedFile);
|
|
1257
|
-
const response = await this.requestJson("/transcribe", {
|
|
1258
|
-
method: "POST",
|
|
1259
|
-
body: {
|
|
1260
|
-
files: [
|
|
1261
|
-
estimate > MAX_SYNC_AUDIO_SECONDS || normalizedFile.size > MAX_SYNC_AUDIO_BYTES
|
|
1262
|
-
? uploadDescriptorForFile(defaultReferenceId(0), normalizedFile)
|
|
1263
|
-
: { reference_id: defaultReferenceId(0) },
|
|
1264
|
-
],
|
|
1265
|
-
...(webhookUrl ? { webhook_url: webhookUrl } : {}),
|
|
1266
|
-
},
|
|
1267
|
-
retryable: true,
|
|
1268
|
-
});
|
|
1269
|
-
emitProgress(progress, {
|
|
1270
|
-
event: "upload_urls_received",
|
|
1271
|
-
jobId: response.job_id,
|
|
1272
|
-
jobStatus: response.job_status,
|
|
1273
|
-
uploadCount: normalizeResponseUploads(response).length,
|
|
1274
|
-
totalFiles: 1,
|
|
1275
|
-
});
|
|
1276
|
-
return new BigFileJob(this, normalizedFile, response, {
|
|
1277
|
-
onProgress: progress,
|
|
1278
|
-
multipartConcurrency: normalizedConcurrency,
|
|
1279
|
-
referenceId: defaultReferenceId(0),
|
|
1530
|
+
uploadConcurrency,
|
|
1531
|
+
language,
|
|
1532
|
+
exclude,
|
|
1280
1533
|
});
|
|
1281
1534
|
}
|
|
1282
1535
|
|
|
@@ -1287,13 +1540,16 @@ export class TranscribeAPI {
|
|
|
1287
1540
|
onProgress,
|
|
1288
1541
|
showLogs,
|
|
1289
1542
|
logger,
|
|
1290
|
-
|
|
1543
|
+
uploadConcurrency,
|
|
1544
|
+
language,
|
|
1545
|
+
exclude,
|
|
1291
1546
|
} = {}) {
|
|
1292
1547
|
const progress = composeProgressHandler({
|
|
1293
1548
|
onProgress,
|
|
1294
1549
|
showLogs: showLogs ?? this.showLogs,
|
|
1295
1550
|
logger: logger ?? this.logger,
|
|
1296
1551
|
});
|
|
1552
|
+
const normalizedLanguage = normalizeLanguageCode(language);
|
|
1297
1553
|
if (!Array.isArray(files) || files.length === 0) {
|
|
1298
1554
|
throw new TranscribeAPIError("Batch upload requires at least one file.", { code: "invalid_files" });
|
|
1299
1555
|
}
|
|
@@ -1309,17 +1565,21 @@ export class TranscribeAPI {
|
|
|
1309
1565
|
referenceId: item.referenceId,
|
|
1310
1566
|
url: item.url,
|
|
1311
1567
|
durationEstimateSec: item.durationEstimateSec || durationEstimateSec || null,
|
|
1568
|
+
hasLanguage: item.hasLanguage,
|
|
1569
|
+
language: item.language,
|
|
1312
1570
|
});
|
|
1313
1571
|
continue;
|
|
1314
1572
|
}
|
|
1315
1573
|
|
|
1316
|
-
const
|
|
1317
|
-
assertSupportedBatchFormat(
|
|
1574
|
+
const file = await normalizeFile(item.file, `file_${String(index + 1).padStart(6, "0")}.mp3`);
|
|
1575
|
+
assertSupportedBatchFormat(file);
|
|
1318
1576
|
normalizedBatchItems.push({
|
|
1319
|
-
file
|
|
1320
|
-
referenceId: item.referenceId,
|
|
1577
|
+
file,
|
|
1578
|
+
referenceId: item.referenceId || defaultReferenceId(index),
|
|
1321
1579
|
url: null,
|
|
1322
|
-
durationEstimateSec: item.durationEstimateSec || await estimateDurationFromFile(
|
|
1580
|
+
durationEstimateSec: item.durationEstimateSec || await estimateDurationFromFile(file),
|
|
1581
|
+
hasLanguage: item.hasLanguage,
|
|
1582
|
+
language: item.language,
|
|
1323
1583
|
});
|
|
1324
1584
|
}
|
|
1325
1585
|
|
|
@@ -1331,12 +1591,18 @@ export class TranscribeAPI {
|
|
|
1331
1591
|
files: normalizedBatchItems.map((item) => (
|
|
1332
1592
|
item.url
|
|
1333
1593
|
? {
|
|
1334
|
-
reference_id: item.referenceId,
|
|
1594
|
+
...(item.referenceId ? { reference_id: item.referenceId } : {}),
|
|
1335
1595
|
url: item.url,
|
|
1596
|
+
...(item.hasLanguage ? { language: item.language } : {}),
|
|
1597
|
+
}
|
|
1598
|
+
: {
|
|
1599
|
+
...uploadDescriptorForFile(item.referenceId, item.file),
|
|
1600
|
+
...(item.hasLanguage ? { language: item.language } : {}),
|
|
1336
1601
|
}
|
|
1337
|
-
: uploadDescriptorForFile(item.referenceId, item.file)
|
|
1338
1602
|
)),
|
|
1603
|
+
...(normalizedLanguage ? { language: normalizedLanguage } : {}),
|
|
1339
1604
|
...(webhookUrl ? { webhook_url: webhookUrl } : {}),
|
|
1605
|
+
...(exclude ? { exclude: Array.isArray(exclude) ? exclude.join(",") : exclude } : {}),
|
|
1340
1606
|
},
|
|
1341
1607
|
retryable: true,
|
|
1342
1608
|
});
|
|
@@ -1350,7 +1616,7 @@ export class TranscribeAPI {
|
|
|
1350
1616
|
return new BatchJob(this, normalizedBatchItems, response, {
|
|
1351
1617
|
onProgress: progress,
|
|
1352
1618
|
webhookUrl,
|
|
1353
|
-
|
|
1619
|
+
uploadConcurrency: normalizeUploadConcurrency(uploadConcurrency ?? this.uploadConcurrency),
|
|
1354
1620
|
});
|
|
1355
1621
|
}
|
|
1356
1622
|
}
|