nixamp 0.26.6 → 0.27.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +91 -7
- package/dist/live-interpreter.d.ts +70 -0
- package/dist/live-interpreter.js +243 -0
- package/dist/live-voice.d.ts +5 -2
- package/dist/live-voice.js +36 -7
- package/dist/pcm-wav.d.ts +2 -0
- package/dist/pcm-wav.js +27 -0
- package/dist/server.d.ts +6 -0
- package/dist/server.js +136 -7
- package/dist/shared-translation.d.ts +55 -0
- package/dist/shared-translation.js +335 -0
- package/dist/translation-passes.d.ts +97 -0
- package/dist/translation-passes.js +227 -0
- package/dist/web-sites.d.ts +6 -0
- package/dist/web-sites.js +26 -0
- package/package.json +1 -1
- package/src/live-interpreter.ts +194 -0
- package/src/live-voice.ts +34 -7
- package/src/pcm-wav.ts +26 -0
- package/src/server.ts +96 -6
- package/src/shared-translation.ts +218 -0
- package/src/translation-passes.ts +232 -0
- package/src/web-sites.ts +29 -0
- package/web/dist/assets/{hls-3VKVEQE3-e7zQjVfV.js → hls-3VKVEQE3-P3sj2d3a.js} +1 -1
- package/web/dist/assets/{index-biLbPIu_.css → index-BmR7T0zX.css} +1 -1
- package/web/dist/assets/index-Do9VYiU8.js +3 -0
- package/web/dist/assets/{mpegts-Ci1G5S8S.js → mpegts-CZ1YoF1U.js} +1 -1
- package/web/dist/assets/{mpegts-LO6RVLD6-cj6SzTIe.js → mpegts-LO6RVLD6-DuNM0Zxm.js} +1 -1
- package/web/dist/index.html +20 -2
- package/web/dist/sw.js +6 -6
- package/web/dist/assets/index-Ckxp8OU8.js +0 -1
package/dist/server.d.ts
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
import { type IncomingMessage, type Server, type ServerResponse } from "node:http";
|
|
2
|
+
import { SharedTranslations } from "./shared-translation.ts";
|
|
3
|
+
import { TranslationPasses } from "./translation-passes.ts";
|
|
2
4
|
import { LiveVoice } from "./live-voice.ts";
|
|
3
5
|
import { Connections } from "./connections.ts";
|
|
4
6
|
import { Broadcaster, type Destination, type EncoderSettings } from "./broadcast.ts";
|
|
@@ -525,6 +527,8 @@ export declare function joinSubject(url: URL, options: Pick<HandlerOptions, "dir
|
|
|
525
527
|
export declare function joinDocument(shell: string, subject: JoinSubject, site: string): string;
|
|
526
528
|
export interface HandlerOptions {
|
|
527
529
|
web: string | null;
|
|
530
|
+
/** Branded clients on this same backend, keyed by an explicitly configured host. */
|
|
531
|
+
webSites?: ReadonlyMap<string, import("./web-sites.ts").WebSite>;
|
|
528
532
|
media: boolean;
|
|
529
533
|
version: string;
|
|
530
534
|
/** The key from the share link, or null to serve to anyone who can connect. */
|
|
@@ -705,6 +709,8 @@ export interface HandlerOptions {
|
|
|
705
709
|
/** Speech to text: a line said out loud, heard here. Needs the optional model. */
|
|
706
710
|
speech?: Speech;
|
|
707
711
|
liveVoice?: LiveVoice;
|
|
712
|
+
translationPasses?: TranslationPasses;
|
|
713
|
+
sharedTranslations?: SharedTranslations;
|
|
708
714
|
/** Translation: texts in another language, by a model here. The same optional library. */
|
|
709
715
|
translator?: Translator;
|
|
710
716
|
/** The transcript store: what was heard, kept under the media's identity. Where the accounts are. */
|
package/dist/server.js
CHANGED
|
@@ -18,6 +18,8 @@ import { spawn, spawnSync } from "node:child_process";
|
|
|
18
18
|
import { readFileSync } from "node:fs";
|
|
19
19
|
import { Readable } from "node:stream";
|
|
20
20
|
import { pipeline as pipeStream } from "node:stream/promises";
|
|
21
|
+
import { SharedTranslations } from "./shared-translation.js";
|
|
22
|
+
import { TranslationPasses } from "./translation-passes.js";
|
|
21
23
|
import { LiveVoice, LIVE_VOICE_LANGUAGES, LIVE_VOICE_MODEL } from "./live-voice.js";
|
|
22
24
|
import { Connections } from "./connections.js";
|
|
23
25
|
import { Broadcaster, DEFAULT_ENCODER, PRESETS, redact, } from "./broadcast.js";
|
|
@@ -1355,6 +1357,9 @@ export function createHandler(engine, options) {
|
|
|
1355
1357
|
return async function handle(request, response) {
|
|
1356
1358
|
const url = new URL(request.url ?? "/", "http://localhost");
|
|
1357
1359
|
const path = url.pathname;
|
|
1360
|
+
const webSite = options.webSites?.get((request.headers.host ?? "").toLowerCase());
|
|
1361
|
+
const web = webSite?.web ?? options.web;
|
|
1362
|
+
const eventSite = webSite?.site ?? options.invites?.site;
|
|
1358
1363
|
const key = options.key ?? null;
|
|
1359
1364
|
const behindProxy = options.behindProxy ?? false;
|
|
1360
1365
|
const listenKey = options.listenKey ?? null;
|
|
@@ -1444,7 +1449,7 @@ export function createHandler(engine, options) {
|
|
|
1444
1449
|
...(options.layouts ? { layouts: options.layouts } : {}),
|
|
1445
1450
|
...(options.rooms ? { rooms: options.rooms } : {}),
|
|
1446
1451
|
...(options.tickets ? { tickets: options.tickets } : {}),
|
|
1447
|
-
...(
|
|
1452
|
+
...(eventSite ? { site: eventSite } : {}),
|
|
1448
1453
|
...(options.invites?.email ? { email: options.invites.email } : {}),
|
|
1449
1454
|
}))
|
|
1450
1455
|
return;
|
|
@@ -2325,6 +2330,117 @@ export function createHandler(engine, options) {
|
|
|
2325
2330
|
}
|
|
2326
2331
|
return;
|
|
2327
2332
|
}
|
|
2333
|
+
if (path === "/api/v1/speech/shared" && request.method === "POST") {
|
|
2334
|
+
try {
|
|
2335
|
+
const service = options.sharedTranslations;
|
|
2336
|
+
const who = await options.accounts?.whoIs(tokenFrom(request.headers));
|
|
2337
|
+
if (!who)
|
|
2338
|
+
throw new SpeechError("Sign in to translate live audio.", 401);
|
|
2339
|
+
if (!service)
|
|
2340
|
+
throw new SpeechError("Shared translation is unavailable.", 503);
|
|
2341
|
+
const caller = callerOf(request.headers, request.socket.remoteAddress, options.behindProxy ?? false);
|
|
2342
|
+
if (!guard.check(`shared-join:${who.id}`, { allowed: 10, windowMs: 60_000 }).ok || !guard.check(`shared-ip:${caller}`, { allowed: 30, windowMs: 60_000 }).ok)
|
|
2343
|
+
throw new SpeechError("Too many live translation joins.", 429);
|
|
2344
|
+
if (!request.headers["content-type"]?.startsWith("application/json"))
|
|
2345
|
+
throw new SpeechError("Send a live source as JSON.", 415);
|
|
2346
|
+
let body;
|
|
2347
|
+
try {
|
|
2348
|
+
body = JSON.parse(await readBody(request, 4096));
|
|
2349
|
+
}
|
|
2350
|
+
catch {
|
|
2351
|
+
throw new SpeechError("Choose a live source and language.", 400);
|
|
2352
|
+
}
|
|
2353
|
+
if (typeof body?.source !== "string" || typeof body.language !== "string")
|
|
2354
|
+
throw new SpeechError("Choose a live source and language.", 400);
|
|
2355
|
+
const queued = [];
|
|
2356
|
+
let finished = false;
|
|
2357
|
+
const send = (event) => {
|
|
2358
|
+
if (!response.headersSent) {
|
|
2359
|
+
queued.push(event);
|
|
2360
|
+
return;
|
|
2361
|
+
}
|
|
2362
|
+
if (response.destroyed || response.writableLength > 512_000)
|
|
2363
|
+
throw new Error("slow listener");
|
|
2364
|
+
response.write(`data: ${JSON.stringify(event)}\n\n`);
|
|
2365
|
+
};
|
|
2366
|
+
const leave = await service.join(body.source, body.language, who.id, send, () => { finished = true; if (response.headersSent)
|
|
2367
|
+
response.end(); });
|
|
2368
|
+
if (response.destroyed) {
|
|
2369
|
+
leave();
|
|
2370
|
+
return;
|
|
2371
|
+
}
|
|
2372
|
+
response.writeHead(200, { ...CORS, "content-type": "text/event-stream", "cache-control": "no-store", "x-accel-buffering": "no" });
|
|
2373
|
+
for (const event of queued)
|
|
2374
|
+
send(event);
|
|
2375
|
+
if (finished) {
|
|
2376
|
+
leave();
|
|
2377
|
+
response.end();
|
|
2378
|
+
return;
|
|
2379
|
+
}
|
|
2380
|
+
const heartbeat = setInterval(() => { if (!response.destroyed)
|
|
2381
|
+
response.write(": listening\n\n"); }, 10_000);
|
|
2382
|
+
response.once("close", () => { clearInterval(heartbeat); leave(); });
|
|
2383
|
+
}
|
|
2384
|
+
catch (error) {
|
|
2385
|
+
if (response.headersSent)
|
|
2386
|
+
response.end();
|
|
2387
|
+
else
|
|
2388
|
+
json(response, error instanceof SpeechError ? error.status : 502, { error: error instanceof SpeechError ? error.message : "Shared translation is unavailable." });
|
|
2389
|
+
}
|
|
2390
|
+
return;
|
|
2391
|
+
}
|
|
2392
|
+
if (path === "/api/v1/translation-passes" || path.startsWith("/api/v1/translation-passes/")) {
|
|
2393
|
+
response.setHeader("cache-control", "no-store");
|
|
2394
|
+
try {
|
|
2395
|
+
const caller = callerOf(request.headers, request.socket.remoteAddress, options.behindProxy ?? false);
|
|
2396
|
+
if (!guard.check(`translation-pass-ip:${caller}`, { allowed: 120, windowMs: 60_000 }).ok)
|
|
2397
|
+
throw new SpeechError("Too many purchase requests.", 429);
|
|
2398
|
+
const service = options.translationPasses;
|
|
2399
|
+
if (!service || !options.accounts) {
|
|
2400
|
+
if (path === "/api/v1/translation-passes" && request.method === "GET")
|
|
2401
|
+
json(response, 200, { required: false, available: false, balanceMicros: 0, expires: null, plans: [], coins: [], orders: [] });
|
|
2402
|
+
else
|
|
2403
|
+
json(response, 503, { error: "Translation checkout is unavailable." });
|
|
2404
|
+
return;
|
|
2405
|
+
}
|
|
2406
|
+
const who = await options.accounts.whoIs(tokenFrom(request.headers));
|
|
2407
|
+
if (path === "/api/v1/translation-passes" && request.method === "GET") {
|
|
2408
|
+
json(response, 200, await service.access(who?.id));
|
|
2409
|
+
return;
|
|
2410
|
+
}
|
|
2411
|
+
if (!who)
|
|
2412
|
+
throw new SpeechError("Sign in to buy translated audio.", 401);
|
|
2413
|
+
if (!guard.check(`translation-pass-account:${who.id}`, { allowed: 30, windowMs: 60_000 }).ok)
|
|
2414
|
+
throw new SpeechError("Too many purchase requests.", 429);
|
|
2415
|
+
if (path === "/api/v1/translation-passes/checkout" && request.method === "POST") {
|
|
2416
|
+
if (!request.headers["content-type"]?.startsWith("application/json"))
|
|
2417
|
+
throw new SpeechError("Send a purchase as JSON.", 415);
|
|
2418
|
+
if (!guard.check(`translation-checkout-ip:${caller}`, { allowed: 10, windowMs: 3600_000 }).ok)
|
|
2419
|
+
throw new SpeechError("Too many new checkouts. Resume your pending purchase.", 429);
|
|
2420
|
+
let body;
|
|
2421
|
+
try {
|
|
2422
|
+
body = JSON.parse(await readBody(request, 1024));
|
|
2423
|
+
}
|
|
2424
|
+
catch {
|
|
2425
|
+
throw new SpeechError("Choose a translation pass.", 400);
|
|
2426
|
+
}
|
|
2427
|
+
if (!body || typeof body !== "object" || typeof body["plan"] !== "string" || typeof body["coin"] !== "string" || typeof body["requestKey"] !== "string")
|
|
2428
|
+
throw new SpeechError("Choose a translation pass and payment currency.", 400);
|
|
2429
|
+
json(response, 200, await service.checkout(who.id, body["plan"], body["coin"], body["requestKey"]));
|
|
2430
|
+
return;
|
|
2431
|
+
}
|
|
2432
|
+
const order = path.match(/^\/api\/v1\/translation-passes\/orders\/([^/]+)$/);
|
|
2433
|
+
if (order && request.method === "GET") {
|
|
2434
|
+
json(response, 200, await service.check(who.id, order[1]));
|
|
2435
|
+
return;
|
|
2436
|
+
}
|
|
2437
|
+
json(response, 405, { error: "GET your balance or purchase status; POST a checkout." });
|
|
2438
|
+
}
|
|
2439
|
+
catch (error) {
|
|
2440
|
+
json(response, error instanceof SpeechError ? error.status : 502, { error: error instanceof SpeechError ? error.message : "Translation purchases are temporarily unavailable." });
|
|
2441
|
+
}
|
|
2442
|
+
return;
|
|
2443
|
+
}
|
|
2328
2444
|
/*
|
|
2329
2445
|
* Speech to text: a WAV in, the words out. Signed in only -- the ear
|
|
2330
2446
|
* costs CPU and a trollbox line needs a name anyway -- and, given a
|
|
@@ -4809,8 +4925,8 @@ export function createHandler(engine, options) {
|
|
|
4809
4925
|
json(response, 404, { error: "no such endpoint" });
|
|
4810
4926
|
return;
|
|
4811
4927
|
}
|
|
4812
|
-
if (
|
|
4813
|
-
const direct = safeJoin(
|
|
4928
|
+
if (web !== null) {
|
|
4929
|
+
const direct = safeJoin(web, path);
|
|
4814
4930
|
if (direct === null) {
|
|
4815
4931
|
json(response, 400, { error: "bad path" });
|
|
4816
4932
|
return;
|
|
@@ -4821,8 +4937,8 @@ export function createHandler(engine, options) {
|
|
|
4821
4937
|
else if (isFile(join(direct, "index.html")))
|
|
4822
4938
|
file = join(direct, "index.html");
|
|
4823
4939
|
// A single-page app: any unknown path is the shell, and the client routes.
|
|
4824
|
-
else if (isFile(join(
|
|
4825
|
-
file = join(
|
|
4940
|
+
else if (isFile(join(web, "index.html")))
|
|
4941
|
+
file = join(web, "index.html");
|
|
4826
4942
|
if (file !== null) {
|
|
4827
4943
|
if (file.endsWith("index.html") && path.startsWith("/live/") && options.events) {
|
|
4828
4944
|
try {
|
|
@@ -4835,7 +4951,7 @@ export function createHandler(engine, options) {
|
|
|
4835
4951
|
if (event && (await options.events.canAccess(event, account?.id) || invited)) {
|
|
4836
4952
|
const shell = readIfPossible(file);
|
|
4837
4953
|
if (shell !== null) {
|
|
4838
|
-
html(response, 200, eventDocument(shell, event,
|
|
4954
|
+
html(response, 200, eventDocument(shell, event, eventSite ?? "https://backtoschool.help"));
|
|
4839
4955
|
return;
|
|
4840
4956
|
}
|
|
4841
4957
|
}
|
|
@@ -4846,7 +4962,7 @@ export function createHandler(engine, options) {
|
|
|
4846
4962
|
const subject = joinSubject(url, options);
|
|
4847
4963
|
const shell = subject ? readIfPossible(file) : null;
|
|
4848
4964
|
if (subject && shell !== null) {
|
|
4849
|
-
html(response, 200, joinDocument(shell, subject,
|
|
4965
|
+
html(response, 200, joinDocument(shell, subject, eventSite ?? "https://nixamp.com"));
|
|
4850
4966
|
return;
|
|
4851
4967
|
}
|
|
4852
4968
|
}
|
|
@@ -5257,6 +5373,8 @@ export async function serve(argv, version = "0.1.0") {
|
|
|
5257
5373
|
if (!options.noJingle)
|
|
5258
5374
|
playJingle(tools);
|
|
5259
5375
|
const web = options.web !== null ? resolve(options.web) : defaultWebDir();
|
|
5376
|
+
const { webSitesFromEnv } = await import("./web-sites.js");
|
|
5377
|
+
const webSites = webSitesFromEnv();
|
|
5260
5378
|
// The same keys this port used last time, so a link somebody was given
|
|
5261
5379
|
// still works after a restart -- and a server is restarted to pick up a new
|
|
5262
5380
|
// version, which is to say often. `--new-key` mints a fresh pair and forgets
|
|
@@ -5418,8 +5536,15 @@ export async function serve(argv, version = "0.1.0") {
|
|
|
5418
5536
|
// the first person to speak does not wait for the model to arrive; a box
|
|
5419
5537
|
// without the optional model never says it is ready, and answers 503.
|
|
5420
5538
|
const speech = accounts && process.env["NIXAMP_STT"] !== "off" ? new Speech() : undefined;
|
|
5539
|
+
// Account servers meter paid providers by default, including when checkout
|
|
5540
|
+
// is temporarily unconfigured. Self-hosters can explicitly sponsor usage.
|
|
5541
|
+
const translationPasses = accounts && pool && process.env["NIXAMP_TRANSLATION_BILLING"] !== "off"
|
|
5542
|
+
? new TranslationPasses({ db: pool, key: process.env["COINPAY_X402_KEY"] ?? "", site: nixampSite }) : undefined;
|
|
5543
|
+
if (translationPasses)
|
|
5544
|
+
void translationPasses.access().catch(() => console.error("nixamp: translation checkout could not warm; it will retry on demand"));
|
|
5421
5545
|
const liveVoice = accounts && process.env["NIXAMP_DUBBING"] !== "off" ? new LiveVoice({
|
|
5422
5546
|
...(pool ? { db: pool } : {}),
|
|
5547
|
+
...(translationPasses ? { billing: translationPasses } : {}),
|
|
5423
5548
|
dailyChars: Number(process.env["NIXAMP_DUB_DAILY_CHARS"] ?? 200_000),
|
|
5424
5549
|
dailyAudioSeconds: Number(process.env["NIXAMP_DUB_DAILY_AUDIO_SECONDS"] ?? 86_400),
|
|
5425
5550
|
userDailyChars: Number(process.env["NIXAMP_DUB_USER_DAILY_CHARS"] ?? 120_000),
|
|
@@ -5435,6 +5560,7 @@ export async function serve(argv, version = "0.1.0") {
|
|
|
5435
5560
|
// names pairs to load at boot (en-de,en-sv) so the first line does not
|
|
5436
5561
|
// wait; NIXAMP_MT=off turns it off.
|
|
5437
5562
|
const translator = accounts && process.env["NIXAMP_MT"] !== "off" ? new Translator() : undefined;
|
|
5563
|
+
const sharedTranslations = liveVoice && translator ? new SharedTranslations({ voice: liveVoice, translator, ...(translationPasses ? { billing: translationPasses } : {}) }) : undefined;
|
|
5438
5564
|
if (translator) {
|
|
5439
5565
|
const warm = (process.env["NIXAMP_MT_WARM"] ?? "").split(",").map((one) => one.trim()).filter(Boolean);
|
|
5440
5566
|
if (warm.length > 0) {
|
|
@@ -5820,6 +5946,7 @@ export async function serve(argv, version = "0.1.0") {
|
|
|
5820
5946
|
let publishable_;
|
|
5821
5947
|
const server = createServer(engine, {
|
|
5822
5948
|
web,
|
|
5949
|
+
webSites,
|
|
5823
5950
|
media: options.media,
|
|
5824
5951
|
owner,
|
|
5825
5952
|
channels,
|
|
@@ -5909,6 +6036,8 @@ export async function serve(argv, version = "0.1.0") {
|
|
|
5909
6036
|
...(trollbox ? { trollbox } : {}),
|
|
5910
6037
|
...(speech ? { speech } : {}),
|
|
5911
6038
|
...(liveVoice ? { liveVoice } : {}),
|
|
6039
|
+
...(translationPasses ? { translationPasses } : {}),
|
|
6040
|
+
...(sharedTranslations ? { sharedTranslations } : {}),
|
|
5912
6041
|
...(translator ? { translator } : {}),
|
|
5913
6042
|
...(transcripts ? { transcripts } : {}),
|
|
5914
6043
|
...(translations ? { translations } : {}),
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
import { type Caption } from "./live-interpreter.ts";
|
|
2
|
+
import { LiveVoice } from "./live-voice.ts";
|
|
3
|
+
import type { Translator } from "./translate.ts";
|
|
4
|
+
import type { TranslationMeter } from "./translation-passes.ts";
|
|
5
|
+
export declare function publicStreamAddress(address: string): boolean;
|
|
6
|
+
export declare function sharedSource(raw: string): URL;
|
|
7
|
+
export interface LiveInput {
|
|
8
|
+
stop(): void;
|
|
9
|
+
}
|
|
10
|
+
export type OpenLiveInput = (source: URL, pcm: (bytes: Buffer) => void, ended: () => void, signal: AbortSignal) => Promise<LiveInput>;
|
|
11
|
+
/** Resolve once and pin that public address; no redirects or ffmpeg URL fetches. */
|
|
12
|
+
export declare function liveInput(ffmpeg?: string[]): OpenLiveInput;
|
|
13
|
+
export type SharedEvent = {
|
|
14
|
+
type: "status";
|
|
15
|
+
listeners: number;
|
|
16
|
+
} | {
|
|
17
|
+
type: "line";
|
|
18
|
+
id: string;
|
|
19
|
+
line: Caption;
|
|
20
|
+
} | {
|
|
21
|
+
type: "audio";
|
|
22
|
+
id: string;
|
|
23
|
+
data: string;
|
|
24
|
+
} | {
|
|
25
|
+
type: "end";
|
|
26
|
+
id: string;
|
|
27
|
+
} | {
|
|
28
|
+
type: "error";
|
|
29
|
+
error: string;
|
|
30
|
+
};
|
|
31
|
+
type Member = {
|
|
32
|
+
by: string;
|
|
33
|
+
send: (event: SharedEvent) => void;
|
|
34
|
+
close: () => void;
|
|
35
|
+
};
|
|
36
|
+
export declare class SharedTranslations {
|
|
37
|
+
private readonly options;
|
|
38
|
+
private readonly feeds;
|
|
39
|
+
constructor(options: {
|
|
40
|
+
voice: LiveVoice;
|
|
41
|
+
translator: Pick<Translator, "translate">;
|
|
42
|
+
billing?: TranslationMeter;
|
|
43
|
+
open?: OpenLiveInput;
|
|
44
|
+
});
|
|
45
|
+
get size(): number;
|
|
46
|
+
join(raw: string, language: string, by: string, send: Member["send"], close: () => void): Promise<() => void>;
|
|
47
|
+
private leave;
|
|
48
|
+
private checkMembers;
|
|
49
|
+
private broadcast;
|
|
50
|
+
private stop;
|
|
51
|
+
private make;
|
|
52
|
+
private start;
|
|
53
|
+
private speak;
|
|
54
|
+
}
|
|
55
|
+
export {};
|
|
@@ -0,0 +1,335 @@
|
|
|
1
|
+
/** One live source/language pipeline, paid access for every listening account.
|
|
2
|
+
* Browsers receive the same PCM, never upload a replacement for a public feed. */
|
|
3
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
4
|
+
import { lookup } from "node:dns/promises";
|
|
5
|
+
import { BlockList, isIP } from "node:net";
|
|
6
|
+
import http from "node:http";
|
|
7
|
+
import https from "node:https";
|
|
8
|
+
import { spawn } from "node:child_process";
|
|
9
|
+
import { Interpreter } from "./live-interpreter.js";
|
|
10
|
+
import { LIVE_VOICE_LANGUAGES } from "./live-voice.js";
|
|
11
|
+
import { SpeechError } from "./speech.js";
|
|
12
|
+
const blocked = new BlockList();
|
|
13
|
+
for (const [ip, bits] of [["0.0.0.0", 8], ["10.0.0.0", 8], ["100.64.0.0", 10], ["127.0.0.0", 8], ["169.254.0.0", 16], ["172.16.0.0", 12], ["192.0.0.0", 24], ["192.0.2.0", 24], ["192.168.0.0", 16], ["198.18.0.0", 15], ["198.51.100.0", 24], ["203.0.113.0", 24], ["224.0.0.0", 4], ["240.0.0.0", 4]])
|
|
14
|
+
blocked.addSubnet(ip, bits, "ipv4");
|
|
15
|
+
const v6global = new BlockList();
|
|
16
|
+
v6global.addSubnet("2000::", 3, "ipv6");
|
|
17
|
+
blocked.addSubnet("2001:db8::", 32, "ipv6");
|
|
18
|
+
blocked.addSubnet("2002::", 16, "ipv6");
|
|
19
|
+
blocked.addSubnet("2001::", 32, "ipv6");
|
|
20
|
+
export function publicStreamAddress(address) {
|
|
21
|
+
const family = isIP(address);
|
|
22
|
+
return family === 4 ? !blocked.check(address, "ipv4") : family === 6 && v6global.check(address, "ipv6") && !blocked.check(address, "ipv6");
|
|
23
|
+
}
|
|
24
|
+
export function sharedSource(raw) {
|
|
25
|
+
let url;
|
|
26
|
+
try {
|
|
27
|
+
url = new URL(raw);
|
|
28
|
+
}
|
|
29
|
+
catch {
|
|
30
|
+
throw new SpeechError("Choose a live Nixamp channel.", 400);
|
|
31
|
+
}
|
|
32
|
+
if (!["http:", "https:"].includes(url.protocol) || url.username || url.password || url.hash || !/^\/api\/channels\/[\w-]{1,80}$/.test(url.pathname) || [...url.searchParams.keys()].some(key => key !== "k"))
|
|
33
|
+
throw new SpeechError("Choose a live Nixamp channel.", 400);
|
|
34
|
+
if (url.toString().length > 2048)
|
|
35
|
+
throw new SpeechError("Invalid live source.", 400);
|
|
36
|
+
url.searchParams.sort();
|
|
37
|
+
return url;
|
|
38
|
+
}
|
|
39
|
+
/** Resolve once and pin that public address; no redirects or ffmpeg URL fetches. */
|
|
40
|
+
export function liveInput(ffmpeg = ["ffmpeg"]) {
|
|
41
|
+
return async (source, pcm, ended, signal) => {
|
|
42
|
+
const hostname = source.hostname.replace(/^\[|\]$/g, "");
|
|
43
|
+
const addresses = await lookup(hostname, { all: true });
|
|
44
|
+
if (!addresses.length || addresses.some(one => !publicStreamAddress(one.address)))
|
|
45
|
+
throw new SpeechError("This live source is not publicly reachable.", 400);
|
|
46
|
+
signal.throwIfAborted();
|
|
47
|
+
const selected = addresses[0];
|
|
48
|
+
const child = spawn(ffmpeg[0], [...ffmpeg.slice(1), "-v", "error", "-nostats", "-protocol_whitelist", "pipe", "-readrate", "1", "-analyzeduration", "500000", "-probesize", "262144", "-i", "pipe:0", "-vn", "-ac", "1", "-ar", "16000", "-f", "s16le", "pipe:1"], { stdio: ["pipe", "pipe", "ignore"] });
|
|
49
|
+
let stopped = false;
|
|
50
|
+
const request = (source.protocol === "https:" ? https : http).get(source, {
|
|
51
|
+
signal, headers: { "user-agent": "Nixamp shared translated audio", accept: "audio/*,video/*,application/octet-stream" },
|
|
52
|
+
lookup: (_name, options, callback) => { if (typeof options === "object" && options.all)
|
|
53
|
+
callback(null, [selected]);
|
|
54
|
+
else
|
|
55
|
+
callback(null, selected.address, selected.family); },
|
|
56
|
+
});
|
|
57
|
+
const stop = () => { if (stopped)
|
|
58
|
+
return; stopped = true; request.destroy(); child.stdin.destroy(); child.kill("SIGKILL"); signal.removeEventListener("abort", stop); };
|
|
59
|
+
const end = () => { if (!stopped) {
|
|
60
|
+
stop();
|
|
61
|
+
ended();
|
|
62
|
+
} };
|
|
63
|
+
signal.addEventListener("abort", stop, { once: true });
|
|
64
|
+
child.stdout.on("data", pcm);
|
|
65
|
+
child.on("error", end);
|
|
66
|
+
child.on("close", end);
|
|
67
|
+
child.stdin.on("error", end);
|
|
68
|
+
request.on("response", response => {
|
|
69
|
+
if (response.statusCode !== 200) {
|
|
70
|
+
response.destroy();
|
|
71
|
+
end();
|
|
72
|
+
return;
|
|
73
|
+
}
|
|
74
|
+
response.on("error", end);
|
|
75
|
+
response.on("end", end);
|
|
76
|
+
response.pipe(child.stdin);
|
|
77
|
+
});
|
|
78
|
+
request.on("error", end);
|
|
79
|
+
request.setTimeout(15_000, end);
|
|
80
|
+
return { stop };
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
export class SharedTranslations {
|
|
84
|
+
options;
|
|
85
|
+
feeds = new Map();
|
|
86
|
+
constructor(options) {
|
|
87
|
+
this.options = options;
|
|
88
|
+
}
|
|
89
|
+
get size() { return this.feeds.size; }
|
|
90
|
+
async join(raw, language, by, send, close) {
|
|
91
|
+
if (!this.options.voice.available())
|
|
92
|
+
throw new SpeechError("Translated audio is unavailable.", 503);
|
|
93
|
+
const source = sharedSource(raw);
|
|
94
|
+
if (!LIVE_VOICE_LANGUAGES.has(language))
|
|
95
|
+
throw new SpeechError("Choose a supported audio language.", 400);
|
|
96
|
+
await this.options.billing?.require(by);
|
|
97
|
+
const connections = [...this.feeds.values()].reduce((total, feed) => total + [...feed.members].filter(member => member.by === by).length, 0);
|
|
98
|
+
if (connections >= 2)
|
|
99
|
+
throw new SpeechError("Translated audio is already open in two players for this account.", 429);
|
|
100
|
+
const id = `dub-${createHash("sha256").update(`${source}|${language}`).digest("hex").slice(0, 40)}`;
|
|
101
|
+
let feed = this.feeds.get(id);
|
|
102
|
+
if (!feed) {
|
|
103
|
+
if (this.feeds.size >= 4)
|
|
104
|
+
throw new SpeechError("Live translation is busy. Try again shortly.", 429);
|
|
105
|
+
feed = this.make(id, source, language);
|
|
106
|
+
this.feeds.set(id, feed);
|
|
107
|
+
const current = feed;
|
|
108
|
+
current.heartbeat = setInterval(() => void this.checkMembers(current), 10_000);
|
|
109
|
+
current.heartbeat.unref?.();
|
|
110
|
+
}
|
|
111
|
+
if (feed.members.size >= 1000)
|
|
112
|
+
throw new SpeechError("This translated stream is full. Try again shortly.", 429);
|
|
113
|
+
const member = { by, send, close };
|
|
114
|
+
feed.members.add(member);
|
|
115
|
+
this.broadcast(feed, { type: "status", listeners: feed.members.size });
|
|
116
|
+
if (feed.members.size === 1 && !feed.input)
|
|
117
|
+
void this.start(feed);
|
|
118
|
+
const current = feed;
|
|
119
|
+
return () => this.leave(current, member);
|
|
120
|
+
}
|
|
121
|
+
leave(feed, member) {
|
|
122
|
+
if (!feed.members.delete(member))
|
|
123
|
+
return;
|
|
124
|
+
if (!feed.members.size) {
|
|
125
|
+
if (feed.heartbeat)
|
|
126
|
+
clearInterval(feed.heartbeat);
|
|
127
|
+
feed.controller.abort();
|
|
128
|
+
feed.input?.stop();
|
|
129
|
+
feed.interpreter.reset();
|
|
130
|
+
feed.queue = [];
|
|
131
|
+
this.feeds.delete(feed.id);
|
|
132
|
+
}
|
|
133
|
+
else
|
|
134
|
+
this.broadcast(feed, { type: "status", listeners: feed.members.size });
|
|
135
|
+
}
|
|
136
|
+
async checkMembers(feed) {
|
|
137
|
+
const accounts = [...new Set([...feed.members].map(member => member.by))];
|
|
138
|
+
let funded = new Set(accounts), message = "Your translation balance is used up or expired. Buy another pass to continue.";
|
|
139
|
+
try {
|
|
140
|
+
if (this.options.billing?.eligible)
|
|
141
|
+
funded = new Set(await this.options.billing.eligible(accounts));
|
|
142
|
+
else if (this.options.billing) {
|
|
143
|
+
const results = await Promise.allSettled(accounts.map(by => this.options.billing.require(by)));
|
|
144
|
+
funded = new Set(accounts.filter((_by, index) => results[index]?.status === "fulfilled"));
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
catch {
|
|
148
|
+
funded.clear();
|
|
149
|
+
message = "Your translation balance could not be checked. Try again shortly.";
|
|
150
|
+
}
|
|
151
|
+
for (const member of [...feed.members])
|
|
152
|
+
if (accounts.includes(member.by) && !funded.has(member.by)) {
|
|
153
|
+
try {
|
|
154
|
+
member.send({ type: "error", error: message });
|
|
155
|
+
}
|
|
156
|
+
catch { /* disconnected */ }
|
|
157
|
+
finally {
|
|
158
|
+
this.leave(feed, member);
|
|
159
|
+
member.close();
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
broadcast(feed, event) {
|
|
164
|
+
for (const member of [...feed.members]) {
|
|
165
|
+
try {
|
|
166
|
+
member.send(event);
|
|
167
|
+
}
|
|
168
|
+
catch {
|
|
169
|
+
this.leave(feed, member);
|
|
170
|
+
member.close();
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
stop(feed, error) {
|
|
175
|
+
this.broadcast(feed, { type: "error", error });
|
|
176
|
+
for (const member of [...feed.members]) {
|
|
177
|
+
this.leave(feed, member);
|
|
178
|
+
member.close();
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
make(id, source, language) {
|
|
182
|
+
const feed = { id, source, language, members: new Set(), controller: new AbortController(), reservations: new Map(), queue: [], speaking: false, previous: Buffer.alloc(0), pending: Buffer.alloc(0), until: Date.now() };
|
|
183
|
+
// Every active account pays the same published access rate. API work is
|
|
184
|
+
// done once; additional listeners do not trigger recognition or synthesis.
|
|
185
|
+
feed.meter = {
|
|
186
|
+
require: async () => { if (!feed.members.size)
|
|
187
|
+
throw new SpeechError("No listeners.", 410); },
|
|
188
|
+
reserve: async (_by, kind, units) => {
|
|
189
|
+
const ids = [];
|
|
190
|
+
const accounts = [...new Set([...feed.members].map(member => member.by))];
|
|
191
|
+
const accepted = new Set();
|
|
192
|
+
const billing = this.options.billing;
|
|
193
|
+
if (billing?.reserveMany) {
|
|
194
|
+
for (const charged of await billing.reserveMany(accounts, kind, units)) {
|
|
195
|
+
ids.push(charged.id);
|
|
196
|
+
accepted.add(charged.by);
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
else {
|
|
200
|
+
const results = await Promise.allSettled(accounts.map(async (account) => ({ by: account, id: await billing?.reserve(account, kind, units) })));
|
|
201
|
+
let failure;
|
|
202
|
+
for (const result of results) {
|
|
203
|
+
if (result.status === "fulfilled") {
|
|
204
|
+
accepted.add(result.value.by);
|
|
205
|
+
if (result.value.id)
|
|
206
|
+
ids.push(result.value.id);
|
|
207
|
+
}
|
|
208
|
+
else if (!(result.reason instanceof SpeechError) || result.reason.status !== 402)
|
|
209
|
+
failure = result.reason;
|
|
210
|
+
}
|
|
211
|
+
if (failure) {
|
|
212
|
+
await Promise.all(ids.map(id => billing.refund(id)));
|
|
213
|
+
throw failure;
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
for (const member of [...feed.members])
|
|
217
|
+
if (accounts.includes(member.by) && !accepted.has(member.by)) {
|
|
218
|
+
try {
|
|
219
|
+
member.send({ type: "error", error: "Your translation balance is used up or expired. Buy another pass to continue." });
|
|
220
|
+
}
|
|
221
|
+
catch { /* disconnected */ }
|
|
222
|
+
finally {
|
|
223
|
+
this.leave(feed, member);
|
|
224
|
+
member.close();
|
|
225
|
+
}
|
|
226
|
+
}
|
|
227
|
+
if (!feed.members.size) {
|
|
228
|
+
await Promise.all(ids.map(id => this.options.billing.refund(id)));
|
|
229
|
+
throw new SpeechError("No funded listeners.", 402);
|
|
230
|
+
}
|
|
231
|
+
const reservation = randomUUID();
|
|
232
|
+
feed.reservations.set(reservation, ids);
|
|
233
|
+
return reservation;
|
|
234
|
+
},
|
|
235
|
+
commit: async (id) => { const ids = feed.reservations.get(id) ?? []; if (this.options.billing?.commitMany)
|
|
236
|
+
await this.options.billing.commitMany(ids);
|
|
237
|
+
else
|
|
238
|
+
await Promise.all(ids.map(id => this.options.billing.commit(id))); feed.reservations.delete(id); },
|
|
239
|
+
refund: async (id) => { const ids = feed.reservations.get(id) ?? []; if (this.options.billing?.refundMany)
|
|
240
|
+
await this.options.billing.refundMany(ids);
|
|
241
|
+
else
|
|
242
|
+
await Promise.all(ids.map(id => this.options.billing.refund(id))); feed.reservations.delete(id); },
|
|
243
|
+
};
|
|
244
|
+
let voices = [];
|
|
245
|
+
feed.interpreter = new Interpreter({
|
|
246
|
+
language: () => language, speakers: () => true, voices: () => voices, channel: () => id,
|
|
247
|
+
status: text => { if (!feed.controller.signal.aborted && !feed.members.size)
|
|
248
|
+
this.stop(feed, text); },
|
|
249
|
+
failed: () => this.stop(feed, "Live translation stopped. Enable it again to retry."),
|
|
250
|
+
lines: lines => {
|
|
251
|
+
feed.queue.push(...lines);
|
|
252
|
+
if (feed.queue.length > 24) {
|
|
253
|
+
this.stop(feed, "Live translation fell behind. Enable it again to retry.");
|
|
254
|
+
return;
|
|
255
|
+
}
|
|
256
|
+
if (!feed.speaking)
|
|
257
|
+
void this.speak(feed);
|
|
258
|
+
},
|
|
259
|
+
fetcher: (async (url, init) => {
|
|
260
|
+
feed.controller.signal.throwIfAborted();
|
|
261
|
+
const signal = AbortSignal.any([feed.controller.signal, ...(init?.signal ? [init.signal] : [])]);
|
|
262
|
+
if (String(url).includes("/speakers")) {
|
|
263
|
+
voices = await this.options.voice.voices();
|
|
264
|
+
return Response.json(await this.options.voice.hear(new Uint8Array(init?.body), id, signal, feed.meter));
|
|
265
|
+
}
|
|
266
|
+
const body = JSON.parse(String(init?.body));
|
|
267
|
+
return Response.json(await this.options.translator.translate(body.texts, body.from, body.to, { by: id, deadline: Date.now() + 10_000 }));
|
|
268
|
+
}),
|
|
269
|
+
});
|
|
270
|
+
return feed;
|
|
271
|
+
}
|
|
272
|
+
async start(feed) {
|
|
273
|
+
try {
|
|
274
|
+
const input = await (this.options.open ?? liveInput())(feed.source, bytes => {
|
|
275
|
+
if (feed.controller.signal.aborted)
|
|
276
|
+
return;
|
|
277
|
+
feed.pending = Buffer.concat([feed.pending, bytes]);
|
|
278
|
+
while (feed.pending.length >= 64_000) {
|
|
279
|
+
const fresh = feed.pending.subarray(0, 64_000);
|
|
280
|
+
feed.pending = feed.pending.subarray(64_000);
|
|
281
|
+
const pcm = Buffer.concat([feed.previous, fresh]);
|
|
282
|
+
feed.previous = pcm.subarray(-128_000);
|
|
283
|
+
feed.until += 2000;
|
|
284
|
+
if (Math.abs(Date.now() - feed.until) > 2000)
|
|
285
|
+
feed.until = Date.now();
|
|
286
|
+
const samples = new Float32Array(pcm.length / 2);
|
|
287
|
+
for (let index = 0; index < samples.length; index++)
|
|
288
|
+
samples[index] = pcm.readInt16LE(index * 2) / 32768;
|
|
289
|
+
feed.interpreter.push({ samples, at: feed.until - samples.length / 16, until: feed.until, freshAt: feed.until - 2000 });
|
|
290
|
+
}
|
|
291
|
+
}, () => this.stop(feed, "The live source stopped. Enable translated audio again when playback resumes."), feed.controller.signal);
|
|
292
|
+
if (feed.controller.signal.aborted)
|
|
293
|
+
input.stop();
|
|
294
|
+
else
|
|
295
|
+
feed.input = input;
|
|
296
|
+
}
|
|
297
|
+
catch (error) {
|
|
298
|
+
this.stop(feed, error instanceof SpeechError ? error.message : "The live source could not be opened.");
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
async speak(feed) {
|
|
302
|
+
feed.speaking = true;
|
|
303
|
+
try {
|
|
304
|
+
while (feed.queue.length && !feed.controller.signal.aborted) {
|
|
305
|
+
const line = feed.queue.shift();
|
|
306
|
+
if (Date.now() - line.until > 12_000)
|
|
307
|
+
throw new SpeechError("Live translation fell behind. Enable it again to retry.", 503);
|
|
308
|
+
const response = await this.options.voice.stream({ text: line.text, language: feed.language, channel: feed.id, speaker: line.speaker, voice: feed.interpreter.tracker.speakers.get(line.speaker ?? "")?.voice }, feed.id, feed.controller.signal, feed.meter);
|
|
309
|
+
const id = randomUUID();
|
|
310
|
+
this.broadcast(feed, { type: "line", id, line });
|
|
311
|
+
const reader = response.body.getReader();
|
|
312
|
+
try {
|
|
313
|
+
while (true) {
|
|
314
|
+
const { done, value } = await reader.read();
|
|
315
|
+
if (done)
|
|
316
|
+
break;
|
|
317
|
+
for (let at = 0; at < value.length; at += 16_384)
|
|
318
|
+
this.broadcast(feed, { type: "audio", id, data: Buffer.from(value.subarray(at, at + 16_384)).toString("base64") });
|
|
319
|
+
}
|
|
320
|
+
}
|
|
321
|
+
finally {
|
|
322
|
+
reader.releaseLock();
|
|
323
|
+
}
|
|
324
|
+
this.broadcast(feed, { type: "end", id });
|
|
325
|
+
}
|
|
326
|
+
}
|
|
327
|
+
catch (error) {
|
|
328
|
+
if (!feed.controller.signal.aborted)
|
|
329
|
+
this.stop(feed, error instanceof SpeechError ? error.message : "Translated audio was interrupted.");
|
|
330
|
+
}
|
|
331
|
+
finally {
|
|
332
|
+
feed.speaking = false;
|
|
333
|
+
}
|
|
334
|
+
}
|
|
335
|
+
}
|