@vocalstack/mcp 1.2.0 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -18
- package/dist/handlers.js +422 -159
- package/dist/tools.json +216 -341
- package/package.json +1 -1
package/dist/handlers.js
CHANGED
|
@@ -2,12 +2,14 @@ import { createHash } from "node:crypto";
|
|
|
2
2
|
import { ToolError, bad } from "./errors.js";
|
|
3
3
|
export const MAX_FILE_BYTES = 10 * 1024 * 1024;
|
|
4
4
|
export const MAX_FILE_BASE64_CHARS = Math.floor(MAX_FILE_BYTES / 3) * 4 + 8 + 1024;
|
|
5
|
-
const WAIT_MAX_S = 60;
|
|
6
|
-
const
|
|
5
|
+
export const WAIT_MAX_S = 60;
|
|
6
|
+
export const WAITS_PER_KEY = 4;
|
|
7
7
|
const WAIT_POLL_MS = 1000;
|
|
8
8
|
const UPLOAD_TIMEOUT_MS = 15 * 60_000;
|
|
9
|
+
const LANGUAGES_TTL_MS = 60 * 60_000;
|
|
9
10
|
const NO_TRANSCRIPTION = "No transcription with that id for this account.";
|
|
10
11
|
const NO_BROADCAST = "No broadcast with that id for this account.";
|
|
12
|
+
const NO_SPEAKER = "No saved speaker with that id for this account; list_speakers shows them.";
|
|
11
13
|
const OPEN = new Set(["pending", "live", "paused", "processing"]);
|
|
12
14
|
const TERMINAL = new Set(["done", "failed", "stopped"]);
|
|
13
15
|
function put(o, k, v) {
|
|
@@ -17,6 +19,9 @@ function put(o, k, v) {
|
|
|
17
19
|
function path(...parts) {
|
|
18
20
|
return parts.map((p, i) => (i % 2 ? encodeURIComponent(p) : p)).join("");
|
|
19
21
|
}
|
|
22
|
+
function present(a, k) {
|
|
23
|
+
return a.raw[k] !== undefined && a.raw[k] !== null;
|
|
24
|
+
}
|
|
20
25
|
function notFound(e, message) {
|
|
21
26
|
if (e instanceof ToolError && (e.code === "not_found" || e.code === "forbidden")) {
|
|
22
27
|
throw new ToolError("not_found", message);
|
|
@@ -39,6 +44,28 @@ async function broadcast(ctx, id) {
|
|
|
39
44
|
notFound(e, NO_BROADCAST);
|
|
40
45
|
}
|
|
41
46
|
}
|
|
47
|
+
const languages = new Map();
|
|
48
|
+
async function targetLang(a, ctx, k) {
|
|
49
|
+
const l = a.str(k)?.toLowerCase();
|
|
50
|
+
if (l === undefined)
|
|
51
|
+
return undefined;
|
|
52
|
+
let known = languages.get(ctx.api.origin);
|
|
53
|
+
if (!known || Date.now() - known.at > LANGUAGES_TTL_MS) {
|
|
54
|
+
try {
|
|
55
|
+
const r = await ctx.api.get("/v1/languages", undefined, ctx.signal);
|
|
56
|
+
if (Array.isArray(r?.translation) && r.translation.length) {
|
|
57
|
+
known = { at: Date.now(), codes: new Set(r.translation.map((c) => String(c).toLowerCase())) };
|
|
58
|
+
languages.set(ctx.api.origin, known);
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
catch {
|
|
62
|
+
known = undefined;
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
if (known && !known.codes.has(l))
|
|
66
|
+
throw bad(`\`${k}\` ${JSON.stringify(l)} is not a language this API translates into`);
|
|
67
|
+
return l;
|
|
68
|
+
}
|
|
42
69
|
function jobStatus(job, t) {
|
|
43
70
|
const pr = job.progress ?? {};
|
|
44
71
|
const progress = { pct: pr.pct ?? 0, processedS: pr.processedS ?? 0, segmentsDone: pr.segmentsDone ?? 0 };
|
|
@@ -104,17 +131,9 @@ function firstOrder(lines) {
|
|
|
104
131
|
const order = [...first.entries()].sort((a, b) => a[1] - b[1] || a[0] - b[0]);
|
|
105
132
|
return new Map(order.map(([spk], i) => [spk, i + 1]));
|
|
106
133
|
}
|
|
107
|
-
function
|
|
108
|
-
const mode = snapshot.speakerLabels ?? t.settings?.speakerLabels ?? "auto";
|
|
109
|
-
const off = mode === "off";
|
|
110
|
-
const lines = raw.map((l) => ({ ...l, speaker: off ? -1 : l.speaker }));
|
|
111
|
-
const heard = new Set(raw.map((l) => l.speaker).filter((s) => s >= 0)).size;
|
|
112
|
-
const shown = mode === "on" || (mode === "auto" && heard > 1);
|
|
113
|
-
const names = new Map();
|
|
134
|
+
function numbering(lines, snapshot) {
|
|
114
135
|
const server = new Map();
|
|
115
136
|
for (const a of snapshot.speakers ?? []) {
|
|
116
|
-
if (typeof a.name === "string")
|
|
117
|
-
names.set(a.speaker, a.name);
|
|
118
137
|
if (typeof a.number === "number")
|
|
119
138
|
server.set(a.speaker, a.number);
|
|
120
139
|
}
|
|
@@ -129,6 +148,20 @@ function labelled(raw, snapshot, t) {
|
|
|
129
148
|
for (const [s] of rest)
|
|
130
149
|
numbers.set(s, ++top);
|
|
131
150
|
}
|
|
151
|
+
return numbers;
|
|
152
|
+
}
|
|
153
|
+
function labelled(raw, snapshot, t) {
|
|
154
|
+
const mode = snapshot.speakerLabels ?? t.settings?.speakerLabels ?? "auto";
|
|
155
|
+
const off = mode === "off";
|
|
156
|
+
const lines = raw.map((l) => ({ ...l, speaker: off ? -1 : l.speaker }));
|
|
157
|
+
const heard = new Set(raw.map((l) => l.speaker).filter((s) => s >= 0)).size;
|
|
158
|
+
const shown = mode === "on" || (mode === "auto" && heard > 1);
|
|
159
|
+
const names = new Map();
|
|
160
|
+
for (const a of snapshot.speakers ?? []) {
|
|
161
|
+
if (typeof a.name === "string")
|
|
162
|
+
names.set(a.speaker, a.name);
|
|
163
|
+
}
|
|
164
|
+
const numbers = numbering(lines, snapshot);
|
|
132
165
|
return lines.map((l) => {
|
|
133
166
|
let speakerName = "";
|
|
134
167
|
if (shown) {
|
|
@@ -188,11 +221,13 @@ async function snapshot(ctx, id) {
|
|
|
188
221
|
notFound(e, NO_TRANSCRIPTION);
|
|
189
222
|
}
|
|
190
223
|
}
|
|
191
|
-
function
|
|
192
|
-
|
|
224
|
+
function spokenSegments(snap) {
|
|
225
|
+
return (snap.segments ?? [])
|
|
193
226
|
.filter((s) => typeof s.text === "string" && s.text.trim())
|
|
194
227
|
.map((s) => ({ start: s.start, end: s.end, speaker: s.speaker ?? -1, text: s.text, language: s.language, words: s.words ?? [] }));
|
|
195
|
-
|
|
228
|
+
}
|
|
229
|
+
function segmentLines(snap, t) {
|
|
230
|
+
return labelled(spokenSegments(snap), snap, t);
|
|
196
231
|
}
|
|
197
232
|
function validKey(k) {
|
|
198
233
|
return k.length >= 1 && k.length <= 255 && /^[\x21-\x7e]+$/.test(k);
|
|
@@ -207,6 +242,20 @@ function stepKey(key, step) {
|
|
|
207
242
|
const k = `${key}:${step}`;
|
|
208
243
|
return k.length <= 255 ? k : `${createHash("sha256").update(key).digest("hex")}:${step}`;
|
|
209
244
|
}
|
|
245
|
+
function sourceOf(a) {
|
|
246
|
+
const url = present(a, "url");
|
|
247
|
+
const file = present(a, "fileBase64");
|
|
248
|
+
if (url && file)
|
|
249
|
+
throw bad("give either `url` or `fileBase64`, not both");
|
|
250
|
+
if (!url && !file)
|
|
251
|
+
throw bad(`give \`url\` (a public link) or \`fileBase64\` with \`filename\` (a file up to ${MAX_FILE_BYTES >> 20} MiB)`);
|
|
252
|
+
if (url) {
|
|
253
|
+
const stray = ["filename", "mimeType"].find((k) => present(a, k));
|
|
254
|
+
if (stray)
|
|
255
|
+
throw bad(`\`${stray}\` goes with \`fileBase64\`, not with \`url\``);
|
|
256
|
+
}
|
|
257
|
+
return file;
|
|
258
|
+
}
|
|
210
259
|
function transcribeOptions(a) {
|
|
211
260
|
const settings = {};
|
|
212
261
|
const spoken = a.str("spokenLanguage");
|
|
@@ -224,7 +273,7 @@ function transcribeOptions(a) {
|
|
|
224
273
|
return options;
|
|
225
274
|
}
|
|
226
275
|
function decodeFile(a) {
|
|
227
|
-
const tooBig = () => bad(`fileBase64 is over ${MAX_FILE_BYTES >> 20} MiB decoded; give
|
|
276
|
+
const tooBig = () => bad(`fileBase64 is over ${MAX_FILE_BYTES >> 20} MiB decoded; give \`url\` a link to the file instead`);
|
|
228
277
|
const raw = a.rawStr("fileBase64");
|
|
229
278
|
if (raw === undefined)
|
|
230
279
|
throw bad("`fileBase64` must be a string");
|
|
@@ -247,13 +296,14 @@ function decodeFile(a) {
|
|
|
247
296
|
throw bad("fileBase64 is not valid base64: Invalid input length.");
|
|
248
297
|
const bytes = Buffer.from(body, "base64");
|
|
249
298
|
if (bytes.length > MAX_FILE_BYTES)
|
|
250
|
-
throw bad(`the file is over ${MAX_FILE_BYTES >> 20} MiB; give
|
|
299
|
+
throw bad(`the file is over ${MAX_FILE_BYTES >> 20} MiB; give \`url\` a link to it instead`);
|
|
251
300
|
if (bytes.length === 0)
|
|
252
301
|
throw new ToolError("invalid_audio", "the file is empty");
|
|
253
302
|
return bytes;
|
|
254
303
|
}
|
|
255
|
-
async function
|
|
304
|
+
async function createTranscription(a, ctx) {
|
|
256
305
|
const key = idempotencyKey(a);
|
|
306
|
+
const file = sourceOf(a);
|
|
257
307
|
const options = transcribeOptions(a);
|
|
258
308
|
let res;
|
|
259
309
|
if (!file) {
|
|
@@ -282,27 +332,26 @@ async function transcribe(a, ctx, file) {
|
|
|
282
332
|
if (first?.url)
|
|
283
333
|
v.shareUrl = first.url;
|
|
284
334
|
}
|
|
285
|
-
v.next =
|
|
335
|
+
v.next = `Call get_transcription with this transcriptionId and waitS (up to ${WAIT_MAX_S}) to wait for the text.`;
|
|
286
336
|
if (res.headers.get("idempotent-replayed") === "true")
|
|
287
337
|
v.replayed = true;
|
|
288
338
|
return v;
|
|
289
339
|
}
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
return out;
|
|
340
|
+
let waitsOpen = 0;
|
|
341
|
+
function takeWait() {
|
|
342
|
+
if (waitsOpen >= WAITS_PER_KEY) {
|
|
343
|
+
throw new ToolError("rate_limited", `this key already has ${WAITS_PER_KEY} waits open; let one finish first`, {
|
|
344
|
+
retryAfterS: 5,
|
|
345
|
+
limit: WAITS_PER_KEY,
|
|
346
|
+
});
|
|
347
|
+
}
|
|
348
|
+
waitsOpen++;
|
|
349
|
+
let held = true;
|
|
350
|
+
return () => {
|
|
351
|
+
if (held)
|
|
352
|
+
waitsOpen--;
|
|
353
|
+
held = false;
|
|
354
|
+
};
|
|
306
355
|
}
|
|
307
356
|
function phaseText(job) {
|
|
308
357
|
const pr = job.progress ?? {};
|
|
@@ -336,45 +385,146 @@ function sleep(ms, signal) {
|
|
|
336
385
|
signal.addEventListener("abort", done, { once: true });
|
|
337
386
|
});
|
|
338
387
|
}
|
|
339
|
-
async function
|
|
340
|
-
const
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
388
|
+
async function waitFor(ctx, id, first, timeoutMs) {
|
|
389
|
+
const release = takeWait();
|
|
390
|
+
try {
|
|
391
|
+
const deadline = Date.now() + timeoutMs;
|
|
392
|
+
let job = first;
|
|
393
|
+
let lastSent;
|
|
394
|
+
let lastAt = 0;
|
|
395
|
+
for (;;) {
|
|
396
|
+
if (ctx.progress) {
|
|
397
|
+
const msg = phaseText(job);
|
|
398
|
+
const now = `${job.progress?.pct ?? 0}|${msg}`;
|
|
399
|
+
if (lastSent !== now && Date.now() - lastAt >= 1000) {
|
|
400
|
+
await ctx.progress(job.progress?.pct ?? 0, 100, msg).catch(() => { });
|
|
401
|
+
lastSent = now;
|
|
402
|
+
lastAt = Date.now();
|
|
403
|
+
}
|
|
404
|
+
}
|
|
405
|
+
const left = deadline - Date.now();
|
|
406
|
+
if (left <= 0 || ctx.signal.aborted)
|
|
407
|
+
return true;
|
|
408
|
+
await sleep(Math.min(WAIT_POLL_MS, left), ctx.signal);
|
|
409
|
+
if (ctx.signal.aborted)
|
|
410
|
+
return true;
|
|
411
|
+
try {
|
|
412
|
+
job = await ctx.api.get(path("/v1/transcriptions/", id, "/job"), undefined, ctx.signal);
|
|
413
|
+
}
|
|
414
|
+
catch (e) {
|
|
415
|
+
if (ctx.signal.aborted)
|
|
416
|
+
return true;
|
|
417
|
+
notFound(e, NO_TRANSCRIPTION);
|
|
358
418
|
}
|
|
419
|
+
if (TERMINAL.has(job.state))
|
|
420
|
+
return false;
|
|
359
421
|
}
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
422
|
+
}
|
|
423
|
+
finally {
|
|
424
|
+
release();
|
|
425
|
+
}
|
|
426
|
+
}
|
|
427
|
+
function family(l) {
|
|
428
|
+
return l.split(/[_-]/)[0].toLowerCase();
|
|
429
|
+
}
|
|
430
|
+
async function translatedLines(ctx, id, snap, t, lang, offset, limit) {
|
|
431
|
+
const spoken = t.spokenLanguage ?? undefined;
|
|
432
|
+
const sentences = (snap.sentences ?? [])
|
|
433
|
+
.filter((s) => s.status !== "dropped" && typeof s.text === "string" && s.text.trim())
|
|
434
|
+
.sort((x, y) => x.start - y.start || x.sentenceId - y.sentenceId);
|
|
435
|
+
const lines = labelled(sentences.map((s) => ({ start: s.start, end: s.end, speaker: s.speaker ?? -1, text: s.text, language: s.language })), snap, t);
|
|
436
|
+
const shown = sentences.slice(Math.min(offset, sentences.length), Math.min(offset + limit, sentences.length));
|
|
437
|
+
const have = new Map();
|
|
438
|
+
const missing = [];
|
|
439
|
+
for (const s of shown) {
|
|
440
|
+
const done = s.translations?.[lang];
|
|
441
|
+
if (typeof done === "string")
|
|
442
|
+
have.set(s.sentenceId, done);
|
|
443
|
+
else if (!(s.language && family(s.language) === family(lang)))
|
|
444
|
+
missing.push(s.sentenceId);
|
|
445
|
+
}
|
|
446
|
+
for (let i = 0; i < missing.length; i += 40) {
|
|
447
|
+
let r;
|
|
366
448
|
try {
|
|
367
|
-
|
|
449
|
+
r = await ctx.api.get(path("/v1/transcriptions/", id, "/translation"), { lang, sentences: missing.slice(i, i + 40).join(",") }, ctx.signal);
|
|
368
450
|
}
|
|
369
451
|
catch (e) {
|
|
370
|
-
if (
|
|
371
|
-
|
|
452
|
+
if (e instanceof ToolError && e.code === "invalid_options")
|
|
453
|
+
throw bad(`\`lang\` ${JSON.stringify(lang)} is not a language this API translates into`);
|
|
372
454
|
notFound(e, NO_TRANSCRIPTION);
|
|
373
455
|
}
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
456
|
+
for (const [k, v] of Object.entries(r.translations ?? {}))
|
|
457
|
+
if (typeof v === "string")
|
|
458
|
+
have.set(Number(k), v);
|
|
377
459
|
}
|
|
460
|
+
sentences.forEach((s, i) => {
|
|
461
|
+
const done = have.get(s.sentenceId);
|
|
462
|
+
if (done !== undefined) {
|
|
463
|
+
lines[i].text = done;
|
|
464
|
+
lines[i].language = lang;
|
|
465
|
+
}
|
|
466
|
+
else if (!(s.language && family(s.language) === family(lang))) {
|
|
467
|
+
lines[i].language = s.language ?? spoken ?? null;
|
|
468
|
+
}
|
|
469
|
+
});
|
|
470
|
+
return lines;
|
|
471
|
+
}
|
|
472
|
+
async function getTranscription(a, ctx) {
|
|
473
|
+
const id = a.id("transcriptionId");
|
|
474
|
+
const waitS = a.int("waitS", 0, 0, WAIT_MAX_S);
|
|
475
|
+
const format = a.oneOf("format", ["text", "segments", "none"], "text");
|
|
476
|
+
const lang = await targetLang(a, ctx, "lang");
|
|
477
|
+
const words = a.bool("includeWords");
|
|
478
|
+
if (words && lang !== undefined)
|
|
479
|
+
throw bad("`includeWords` is for the original language; leave it out with `lang`");
|
|
480
|
+
const offset = a.int("offset", 0, 0, 4294967295);
|
|
481
|
+
const limit = a.int("limit", 500, 1, 2000);
|
|
482
|
+
let t = await transcription(ctx, id);
|
|
483
|
+
let timedOut;
|
|
484
|
+
if (waitS > 0) {
|
|
485
|
+
timedOut = false;
|
|
486
|
+
if (t.job && !TERMINAL.has(t.job.state)) {
|
|
487
|
+
timedOut = await waitFor(ctx, id, t.job, waitS * 1000);
|
|
488
|
+
t = await transcription(ctx, id);
|
|
489
|
+
}
|
|
490
|
+
}
|
|
491
|
+
const status = statusOf(t);
|
|
492
|
+
try {
|
|
493
|
+
const links = await ctx.api.get(path("/v1/transcriptions/", id, "/share-links"), undefined, ctx.signal);
|
|
494
|
+
const first = links?.links?.[0];
|
|
495
|
+
if (first?.url)
|
|
496
|
+
status.shareUrl = first.url;
|
|
497
|
+
}
|
|
498
|
+
catch { }
|
|
499
|
+
const out = { status };
|
|
500
|
+
if (timedOut !== undefined)
|
|
501
|
+
out.timedOut = timedOut;
|
|
502
|
+
if (format === "none")
|
|
503
|
+
return out;
|
|
504
|
+
if (lang !== undefined) {
|
|
505
|
+
let notYet;
|
|
506
|
+
if (t.job && t.job.state !== "done")
|
|
507
|
+
notYet = `the transcription is ${t.job.state}; wait until it is done (get_transcription with waitS)`;
|
|
508
|
+
else if (!t.job && OPEN.has(t.state)) {
|
|
509
|
+
notYet = "the live transcription is still running; read the text so far without lang, or stop it (stop_broadcast) and translate it then";
|
|
510
|
+
}
|
|
511
|
+
if (notYet !== undefined) {
|
|
512
|
+
if (timedOut === true)
|
|
513
|
+
return out;
|
|
514
|
+
throw new ToolError("invalid_state", notYet);
|
|
515
|
+
}
|
|
516
|
+
const snap = await snapshot(ctx, id);
|
|
517
|
+
const spoken = t.spokenLanguage ?? undefined;
|
|
518
|
+
const same = spoken !== undefined && family(spoken) === family(lang);
|
|
519
|
+
const lines = same ? segmentLines(snap, t) : await translatedLines(ctx, id, snap, t, lang, offset, limit);
|
|
520
|
+
return { ...out, ...page(lines, format, false, offset, limit), lang, translated: !same };
|
|
521
|
+
}
|
|
522
|
+
const snap = await snapshot(ctx, id);
|
|
523
|
+
Object.assign(out, page(segmentLines(snap, t), format, words, offset, limit));
|
|
524
|
+
const partial = t.job ? !TERMINAL.has(t.job.state) : t.state !== "finalized";
|
|
525
|
+
if (partial)
|
|
526
|
+
out.partial = true;
|
|
527
|
+
return out;
|
|
378
528
|
}
|
|
379
529
|
async function listTranscriptions(a, ctx) {
|
|
380
530
|
const lifecycle = a.oneOf("lifecycle", ["active", "archived", "all"], "active");
|
|
@@ -429,7 +579,7 @@ async function only(a, ctx) {
|
|
|
429
579
|
await transcription(ctx, id);
|
|
430
580
|
return id;
|
|
431
581
|
}
|
|
432
|
-
async function
|
|
582
|
+
async function searchWords(a, ctx) {
|
|
433
583
|
const q = query(a);
|
|
434
584
|
const transcriptionId = await only(a, ctx);
|
|
435
585
|
const lang = a.str("lang");
|
|
@@ -451,7 +601,10 @@ async function search(a, ctx) {
|
|
|
451
601
|
});
|
|
452
602
|
return { query: r.query, queryEn: r.queryEn, hits };
|
|
453
603
|
}
|
|
454
|
-
async function
|
|
604
|
+
async function searchMeaning(a, ctx) {
|
|
605
|
+
const stray = ["lang", "limit"].find((k) => present(a, k));
|
|
606
|
+
if (stray)
|
|
607
|
+
throw bad(`\`${stray}\` applies to mode words only; leave it out with mode meaning`);
|
|
455
608
|
const q = query(a);
|
|
456
609
|
const transcriptionId = await only(a, ctx);
|
|
457
610
|
const r = await ctx.api.get("/v1/search/vector", { q, transcription: transcriptionId }, ctx.signal);
|
|
@@ -468,12 +621,12 @@ async function searchByMeaning(a, ctx) {
|
|
|
468
621
|
put(out, "searchesLeftThisHour", r.allowance?.remaining);
|
|
469
622
|
return out;
|
|
470
623
|
}
|
|
471
|
-
function
|
|
472
|
-
return
|
|
624
|
+
async function search(a, ctx) {
|
|
625
|
+
return a.oneOf("mode", ["words", "meaning"], "words") === "meaning" ? searchMeaning(a, ctx) : searchWords(a, ctx);
|
|
473
626
|
}
|
|
474
627
|
async function summary(a, ctx) {
|
|
475
628
|
const id = a.id("transcriptionId");
|
|
476
|
-
const lang = a
|
|
629
|
+
const lang = await targetLang(a, ctx, "lang");
|
|
477
630
|
const t = await transcription(ctx, id);
|
|
478
631
|
const pending = (message) => ({ state: "pending", message });
|
|
479
632
|
const s = await ctx.api.get(path("/v1/transcriptions/", id, "/summary"), { lang }, ctx.signal);
|
|
@@ -486,7 +639,7 @@ async function summary(a, ctx) {
|
|
|
486
639
|
return { state: "unavailable", message: "The transcription failed, so there is nothing to summarise." };
|
|
487
640
|
}
|
|
488
641
|
if ((t.job && !TERMINAL.has(t.job.state)) || (!t.job && OPEN.has(t.state))) {
|
|
489
|
-
return pending("The transcription is still running; wait for it (
|
|
642
|
+
return pending("The transcription is still running; wait for it (get_transcription with waitS), then ask again.");
|
|
490
643
|
}
|
|
491
644
|
if (s.state === "pending")
|
|
492
645
|
return pending("The summary is being written; ask again in a few seconds.");
|
|
@@ -511,78 +664,6 @@ async function summary(a, ctx) {
|
|
|
511
664
|
return pending("The summary is being written; ask again in a few seconds.");
|
|
512
665
|
}
|
|
513
666
|
}
|
|
514
|
-
async function translate(a, ctx) {
|
|
515
|
-
const id = a.id("transcriptionId");
|
|
516
|
-
const lang = a.reqStr("lang").toLowerCase();
|
|
517
|
-
const format = a.oneOf("format", ["text", "segments"], "text");
|
|
518
|
-
const offset = a.int("offset", 0, 0, 4294967295);
|
|
519
|
-
const limit = a.int("limit", 500, 1, 2000);
|
|
520
|
-
const t = await transcription(ctx, id);
|
|
521
|
-
if (t.job && t.job.state !== "done") {
|
|
522
|
-
throw new ToolError("invalid_state", `the transcription is ${t.job.state}; wait until it is done (wait_for_transcription)`);
|
|
523
|
-
}
|
|
524
|
-
if (!t.job && OPEN.has(t.state)) {
|
|
525
|
-
throw new ToolError("invalid_state", "the live transcription is still running; read the text so far with get_transcription, or stop it (stop_broadcast) and translate it then");
|
|
526
|
-
}
|
|
527
|
-
const snap = await snapshot(ctx, id);
|
|
528
|
-
const spoken = t.spokenLanguage ?? undefined;
|
|
529
|
-
const same = spoken !== undefined && family(spoken) === family(lang);
|
|
530
|
-
if (same) {
|
|
531
|
-
return { ...page(segmentLines(snap, t), format, false, offset, limit), lang, translated: false };
|
|
532
|
-
}
|
|
533
|
-
const sentences = (snap.sentences ?? [])
|
|
534
|
-
.filter((s) => s.status !== "dropped" && typeof s.text === "string" && s.text.trim())
|
|
535
|
-
.sort((x, y) => x.start - y.start || x.sentenceId - y.sentenceId);
|
|
536
|
-
const lines = labelled(sentences.map((s) => ({ start: s.start, end: s.end, speaker: s.speaker ?? -1, text: s.text, language: s.language })), snap, t);
|
|
537
|
-
const shown = sentences.slice(Math.min(offset, sentences.length), Math.min(offset + limit, sentences.length));
|
|
538
|
-
const have = new Map();
|
|
539
|
-
const missing = [];
|
|
540
|
-
for (const s of shown) {
|
|
541
|
-
const done = s.translations?.[lang];
|
|
542
|
-
if (typeof done === "string")
|
|
543
|
-
have.set(s.sentenceId, done);
|
|
544
|
-
else if (!(s.language && family(s.language) === family(lang)))
|
|
545
|
-
missing.push(s.sentenceId);
|
|
546
|
-
}
|
|
547
|
-
for (let i = 0; i < missing.length; i += 40) {
|
|
548
|
-
let r;
|
|
549
|
-
try {
|
|
550
|
-
r = await ctx.api.get(path("/v1/transcriptions/", id, "/translation"), { lang, sentences: missing.slice(i, i + 40).join(",") }, ctx.signal);
|
|
551
|
-
}
|
|
552
|
-
catch (e) {
|
|
553
|
-
if (e instanceof ToolError && e.code === "invalid_options")
|
|
554
|
-
throw bad(`\`lang\` ${JSON.stringify(lang)} is not a language this API translates into`);
|
|
555
|
-
notFound(e, NO_TRANSCRIPTION);
|
|
556
|
-
}
|
|
557
|
-
for (const [k, v] of Object.entries(r.translations ?? {}))
|
|
558
|
-
if (typeof v === "string")
|
|
559
|
-
have.set(Number(k), v);
|
|
560
|
-
}
|
|
561
|
-
sentences.forEach((s, i) => {
|
|
562
|
-
const done = have.get(s.sentenceId);
|
|
563
|
-
if (done !== undefined) {
|
|
564
|
-
lines[i].text = done;
|
|
565
|
-
lines[i].language = lang;
|
|
566
|
-
}
|
|
567
|
-
else if (!(s.language && family(s.language) === family(lang))) {
|
|
568
|
-
lines[i].language = s.language ?? spoken ?? null;
|
|
569
|
-
}
|
|
570
|
-
});
|
|
571
|
-
return { ...page(lines, format, false, offset, limit), lang, translated: true };
|
|
572
|
-
}
|
|
573
|
-
async function voices(a, ctx) {
|
|
574
|
-
const lang = a.reqStr("lang");
|
|
575
|
-
const cat = await ctx.api.get("/v1/voices/available", { lang }, ctx.signal);
|
|
576
|
-
const list = (cat.voices ?? []).map((v) => {
|
|
577
|
-
const o = { id: v.id, name: v.name, default: v.default, cloned: v.cloned, quality: v.quality };
|
|
578
|
-
put(o, "gender", v.gender);
|
|
579
|
-
put(o, "accent", v.accent);
|
|
580
|
-
if (typeof v.previewUrl === "string")
|
|
581
|
-
o.previewUrl = v.previewUrl.startsWith("/") ? ctx.api.origin + v.previewUrl : v.previewUrl;
|
|
582
|
-
return o;
|
|
583
|
-
});
|
|
584
|
-
return { lang: cat.lang, voices: list };
|
|
585
|
-
}
|
|
586
667
|
async function all(ctx, route, field, query = {}) {
|
|
587
668
|
const out = [];
|
|
588
669
|
let cursor;
|
|
@@ -607,6 +688,57 @@ async function speakers(_a, ctx) {
|
|
|
607
688
|
}),
|
|
608
689
|
};
|
|
609
690
|
}
|
|
691
|
+
function noSpeaker(number) {
|
|
692
|
+
return new ToolError("not_found", `This transcription has no Speaker ${number} who has spoken; get_transcription shows the speaker numbers.`);
|
|
693
|
+
}
|
|
694
|
+
async function saveSpeaker(a, ctx) {
|
|
695
|
+
const id = a.id("transcriptionId");
|
|
696
|
+
const number = a.int("speaker", 1, 1, 1000);
|
|
697
|
+
const name = a.reqStr("name");
|
|
698
|
+
if ([...name].length > 60)
|
|
699
|
+
throw bad("`name` must be 1 to 60 characters");
|
|
700
|
+
const consent = a.bool("consent");
|
|
701
|
+
await transcription(ctx, id);
|
|
702
|
+
const snap = await snapshot(ctx, id);
|
|
703
|
+
const spoken = spokenSegments(snap).filter((l) => l.speaker >= 0);
|
|
704
|
+
const numbers = numbering(spoken, snap);
|
|
705
|
+
const heard = new Set(spoken.map((l) => l.speaker));
|
|
706
|
+
const appearance = [...numbers.entries()]
|
|
707
|
+
.filter(([idx, n]) => n === number && heard.has(idx))
|
|
708
|
+
.map(([idx]) => idx)
|
|
709
|
+
.sort((x, y) => x - y)[0];
|
|
710
|
+
if (appearance === undefined)
|
|
711
|
+
throw noSpeaker(number);
|
|
712
|
+
let s;
|
|
713
|
+
try {
|
|
714
|
+
s = (await ctx.api.request("POST", "/v1/speakers/from-appearance", {
|
|
715
|
+
json: { transcriptionId: id, appearance, name, consent },
|
|
716
|
+
signal: ctx.signal,
|
|
717
|
+
})).data;
|
|
718
|
+
}
|
|
719
|
+
catch (e) {
|
|
720
|
+
if (e instanceof ToolError && e.code === "not_found" && e.message.includes("has not said anything"))
|
|
721
|
+
throw noSpeaker(number);
|
|
722
|
+
throw e;
|
|
723
|
+
}
|
|
724
|
+
const out = { speakerId: s.id, name: s.name, consent: s.consent, cloned: s.voice != null };
|
|
725
|
+
if (s.voice)
|
|
726
|
+
put(out, "voiceState", s.voice.state);
|
|
727
|
+
return out;
|
|
728
|
+
}
|
|
729
|
+
async function deleteSpeaker(a, ctx) {
|
|
730
|
+
const id = a.id("speakerId");
|
|
731
|
+
try {
|
|
732
|
+
await ctx.api.request("DELETE", path("/v1/speakers/", id), { signal: ctx.signal });
|
|
733
|
+
}
|
|
734
|
+
catch (e) {
|
|
735
|
+
if (e instanceof ToolError && (e.code === "not_found" || (e.code === "forbidden" && !e.details.requiredScope))) {
|
|
736
|
+
throw new ToolError("not_found", NO_SPEAKER);
|
|
737
|
+
}
|
|
738
|
+
throw e;
|
|
739
|
+
}
|
|
740
|
+
return { speakerId: id, deleted: true };
|
|
741
|
+
}
|
|
610
742
|
function onAir(b) {
|
|
611
743
|
const c = b.currentTranscription;
|
|
612
744
|
return c && ["live", "paused", "pending"].includes(c.state) ? c : undefined;
|
|
@@ -631,12 +763,7 @@ function brief(b) {
|
|
|
631
763
|
put(v, "lastRecordedAt", b.lastRecordedAt);
|
|
632
764
|
return v;
|
|
633
765
|
}
|
|
634
|
-
async function
|
|
635
|
-
const lifecycle = a.oneOf("lifecycle", ["active", "archived", "all"], "active");
|
|
636
|
-
const list = await all(ctx, "/v1/broadcasts", "broadcasts", { lifecycle });
|
|
637
|
-
return { broadcasts: list.map(brief) };
|
|
638
|
-
}
|
|
639
|
-
async function getBroadcast(a, ctx) {
|
|
766
|
+
async function oneBroadcast(a, ctx) {
|
|
640
767
|
const b = await broadcast(ctx, a.id("broadcastId"));
|
|
641
768
|
const v = brief(b);
|
|
642
769
|
v.transcriptions = (b.transcriptions ?? []).slice(0, 20).map((t) => {
|
|
@@ -645,7 +772,14 @@ async function getBroadcast(a, ctx) {
|
|
|
645
772
|
return o;
|
|
646
773
|
});
|
|
647
774
|
v.streams = (b.audioSources ?? []).map((s) => ({ name: s.name, url: s.url }));
|
|
648
|
-
return v;
|
|
775
|
+
return { broadcasts: [v] };
|
|
776
|
+
}
|
|
777
|
+
async function listBroadcasts(a, ctx) {
|
|
778
|
+
if (a.str("broadcastId") !== undefined)
|
|
779
|
+
return oneBroadcast(a, ctx);
|
|
780
|
+
const lifecycle = a.oneOf("lifecycle", ["active", "archived", "all"], "active");
|
|
781
|
+
const list = await all(ctx, "/v1/broadcasts", "broadcasts", { lifecycle });
|
|
782
|
+
return { broadcasts: list.map(brief) };
|
|
649
783
|
}
|
|
650
784
|
function streamUrl(a) {
|
|
651
785
|
const u = a.reqStr("streamUrl");
|
|
@@ -678,7 +812,49 @@ async function deleteTranscription(a, ctx) {
|
|
|
678
812
|
put(out, "restorableUntil", t.purgeAfter);
|
|
679
813
|
return out;
|
|
680
814
|
}
|
|
815
|
+
function renameTarget(a) {
|
|
816
|
+
const sid = present(a, "speakerId");
|
|
817
|
+
const tid = present(a, "transcriptionId");
|
|
818
|
+
const spk = present(a, "speaker");
|
|
819
|
+
if (sid && !tid && !spk)
|
|
820
|
+
return true;
|
|
821
|
+
if (!sid && tid && spk)
|
|
822
|
+
return false;
|
|
823
|
+
if (sid)
|
|
824
|
+
throw bad("give either `speakerId` (a saved speaker) or `transcriptionId` with `speaker`, not both");
|
|
825
|
+
if (tid)
|
|
826
|
+
throw bad("`speaker` is required with `transcriptionId`");
|
|
827
|
+
if (spk)
|
|
828
|
+
throw bad("`transcriptionId` is required with `speaker`");
|
|
829
|
+
throw bad("give `transcriptionId` with `speaker` (one transcription) or `speakerId` (a saved speaker)");
|
|
830
|
+
}
|
|
831
|
+
function noSavedSpeaker(e) {
|
|
832
|
+
if (e instanceof ToolError && (e.code === "not_found" || (e.code === "forbidden" && !e.details.requiredScope))) {
|
|
833
|
+
throw new ToolError("not_found", NO_SPEAKER);
|
|
834
|
+
}
|
|
835
|
+
throw e;
|
|
836
|
+
}
|
|
837
|
+
async function renameSavedSpeaker(a, ctx) {
|
|
838
|
+
const id = a.id("speakerId");
|
|
839
|
+
const given = a.raw.name;
|
|
840
|
+
if (typeof given !== "string")
|
|
841
|
+
throw bad("`name` must be a string");
|
|
842
|
+
const name = given.trim();
|
|
843
|
+
const n = [...name].length;
|
|
844
|
+
if (n === 0 || n > 60)
|
|
845
|
+
throw bad("a saved speaker's `name` must be 1 to 60 characters");
|
|
846
|
+
let s;
|
|
847
|
+
try {
|
|
848
|
+
s = (await ctx.api.request("PATCH", path("/v1/speakers/", id), { json: { name }, signal: ctx.signal })).data;
|
|
849
|
+
}
|
|
850
|
+
catch (e) {
|
|
851
|
+
noSavedSpeaker(e);
|
|
852
|
+
}
|
|
853
|
+
return { speakerId: s.id, name: s.name };
|
|
854
|
+
}
|
|
681
855
|
async function renameSpeaker(a, ctx) {
|
|
856
|
+
if (renameTarget(a))
|
|
857
|
+
return renameSavedSpeaker(a, ctx);
|
|
682
858
|
const id = a.id("transcriptionId");
|
|
683
859
|
const number = a.int("speaker", 1, 1, 1000);
|
|
684
860
|
const given = a.raw.name;
|
|
@@ -701,11 +877,83 @@ async function renameSpeaker(a, ctx) {
|
|
|
701
877
|
return { transcriptionId: id, speaker: number, name: name || null };
|
|
702
878
|
}
|
|
703
879
|
const VOICE_CLONING_NEEDS_CONSENT = "turning voiceCloning on needs voiceCloningConsent: true — confirm you have permission from everyone speaking";
|
|
704
|
-
|
|
705
|
-
const
|
|
880
|
+
function clonedVoicesTarget(a) {
|
|
881
|
+
const sid = present(a, "speakerId");
|
|
882
|
+
const tid = present(a, "transcriptionId");
|
|
883
|
+
if (sid && !tid)
|
|
884
|
+
return true;
|
|
885
|
+
if (tid && !sid)
|
|
886
|
+
return false;
|
|
887
|
+
if (sid)
|
|
888
|
+
throw bad("give either `transcriptionId` (one transcription) or `speakerId` (a saved speaker), not both");
|
|
889
|
+
throw bad("give `transcriptionId` (one transcription) or `speakerId` (a saved speaker)");
|
|
890
|
+
}
|
|
891
|
+
function onArg(a) {
|
|
706
892
|
const on = a.raw.on;
|
|
707
893
|
if (typeof on !== "boolean")
|
|
708
894
|
throw bad("`on` must be true or false");
|
|
895
|
+
return on;
|
|
896
|
+
}
|
|
897
|
+
async function savedSpeaker(ctx, id) {
|
|
898
|
+
try {
|
|
899
|
+
return await ctx.api.get(path("/v1/speakers/", id), undefined, ctx.signal);
|
|
900
|
+
}
|
|
901
|
+
catch (e) {
|
|
902
|
+
noSavedSpeaker(e);
|
|
903
|
+
}
|
|
904
|
+
}
|
|
905
|
+
async function patchSpeaker(ctx, id, body) {
|
|
906
|
+
try {
|
|
907
|
+
await ctx.api.request("PATCH", path("/v1/speakers/", id), { json: body, signal: ctx.signal });
|
|
908
|
+
}
|
|
909
|
+
catch (e) {
|
|
910
|
+
noSavedSpeaker(e);
|
|
911
|
+
}
|
|
912
|
+
}
|
|
913
|
+
async function setSpeakerClonedVoice(a, ctx) {
|
|
914
|
+
const id = a.id("speakerId");
|
|
915
|
+
const on = onArg(a);
|
|
916
|
+
const consent = a.bool("consent");
|
|
917
|
+
if (on && !consent) {
|
|
918
|
+
throw bad("Turning a saved speaker's cloned voice on needs consent: true, confirming that person agreed to have their voice cloned.");
|
|
919
|
+
}
|
|
920
|
+
const cur = await savedSpeaker(ctx, id);
|
|
921
|
+
if (on) {
|
|
922
|
+
if (cur.consent === "refused") {
|
|
923
|
+
throw new ToolError("invalid_state", "This speaker refused to be cloned and their audio was deleted; save them again from a transcription (save_speaker with consent) to clone them.");
|
|
924
|
+
}
|
|
925
|
+
let s;
|
|
926
|
+
try {
|
|
927
|
+
s = (await ctx.api.request("POST", path("/v1/speakers/", id, "/consent"), { json: { consent: "granted" }, signal: ctx.signal })).data;
|
|
928
|
+
}
|
|
929
|
+
catch (e) {
|
|
930
|
+
if (e instanceof ToolError && e.code === "invalid_state" && e.message.includes("no sample stored")) {
|
|
931
|
+
throw new ToolError("invalid_state", "This saved speaker has no stored sample to clone their voice from; save them again from a transcription (save_speaker with consent).");
|
|
932
|
+
}
|
|
933
|
+
noSavedSpeaker(e);
|
|
934
|
+
}
|
|
935
|
+
if (s?.voice && s.voice.enabled === false)
|
|
936
|
+
await patchSpeaker(ctx, id, { enabled: true });
|
|
937
|
+
}
|
|
938
|
+
else if (cur.voice?.enabled === true) {
|
|
939
|
+
await patchSpeaker(ctx, id, { enabled: false });
|
|
940
|
+
}
|
|
941
|
+
const s = await savedSpeaker(ctx, id);
|
|
942
|
+
const out = {
|
|
943
|
+
speakerId: s.id,
|
|
944
|
+
name: s.name,
|
|
945
|
+
consent: s.consent,
|
|
946
|
+
clonedVoice: s.voice?.enabled === true && s.consent === "granted",
|
|
947
|
+
};
|
|
948
|
+
if (s.voice)
|
|
949
|
+
put(out, "voiceState", s.voice.state);
|
|
950
|
+
return out;
|
|
951
|
+
}
|
|
952
|
+
async function setClonedVoices(a, ctx) {
|
|
953
|
+
if (clonedVoicesTarget(a))
|
|
954
|
+
return setSpeakerClonedVoice(a, ctx);
|
|
955
|
+
const id = a.id("transcriptionId");
|
|
956
|
+
const on = onArg(a);
|
|
709
957
|
const consent = a.bool("consent");
|
|
710
958
|
await transcription(ctx, id);
|
|
711
959
|
let t;
|
|
@@ -813,23 +1061,38 @@ async function stopBroadcast(a, ctx) {
|
|
|
813
1061
|
const after = await transcription(ctx, tid);
|
|
814
1062
|
return { ...liveStatus(after), broadcastId: b.id, link: b.shareUrl };
|
|
815
1063
|
}
|
|
1064
|
+
async function deleteBroadcast(a, ctx) {
|
|
1065
|
+
const id = a.id("broadcastId");
|
|
1066
|
+
await broadcast(ctx, id);
|
|
1067
|
+
let b;
|
|
1068
|
+
try {
|
|
1069
|
+
b = (await ctx.api.request("POST", path("/v1/broadcasts/", id, "/archive"), { signal: ctx.signal })).data;
|
|
1070
|
+
}
|
|
1071
|
+
catch (e) {
|
|
1072
|
+
if (e instanceof ToolError && e.code === "invalid_state" && e.message.includes("stop the recording first")) {
|
|
1073
|
+
throw new ToolError("invalid_state", "This broadcast is on air; stop it first (stop_broadcast), then delete it.", { broadcastId: id });
|
|
1074
|
+
}
|
|
1075
|
+
throw e;
|
|
1076
|
+
}
|
|
1077
|
+
const out = { broadcastId: b.id, name: b.name, deleted: true };
|
|
1078
|
+
put(out, "deletedAt", b.archivedAt);
|
|
1079
|
+
put(out, "restorableUntil", b.purgeAfter);
|
|
1080
|
+
return out;
|
|
1081
|
+
}
|
|
816
1082
|
export const handlers = {
|
|
817
|
-
|
|
818
|
-
transcribe_file: (a, ctx) => transcribe(a, ctx, true),
|
|
1083
|
+
create_transcription: createTranscription,
|
|
819
1084
|
get_transcription: getTranscription,
|
|
820
|
-
wait_for_transcription: wait,
|
|
821
1085
|
list_transcriptions: listTranscriptions,
|
|
822
1086
|
search_transcriptions: search,
|
|
823
|
-
search_by_meaning: searchByMeaning,
|
|
824
1087
|
get_summary: summary,
|
|
825
|
-
translate_transcription: translate,
|
|
826
1088
|
rename_speaker: renameSpeaker,
|
|
827
1089
|
delete_transcription: deleteTranscription,
|
|
828
|
-
list_voices: voices,
|
|
829
1090
|
set_cloned_voices: setClonedVoices,
|
|
830
1091
|
list_speakers: speakers,
|
|
1092
|
+
save_speaker: saveSpeaker,
|
|
1093
|
+
delete_speaker: deleteSpeaker,
|
|
831
1094
|
list_broadcasts: listBroadcasts,
|
|
832
|
-
get_broadcast: getBroadcast,
|
|
833
1095
|
start_broadcast: startBroadcast,
|
|
834
1096
|
stop_broadcast: stopBroadcast,
|
|
1097
|
+
delete_broadcast: deleteBroadcast,
|
|
835
1098
|
};
|