nixamp 0.26.6 → 0.27.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +91 -7
- package/dist/live-interpreter.d.ts +70 -0
- package/dist/live-interpreter.js +243 -0
- package/dist/live-voice.d.ts +5 -2
- package/dist/live-voice.js +36 -7
- package/dist/pcm-wav.d.ts +2 -0
- package/dist/pcm-wav.js +27 -0
- package/dist/server.d.ts +6 -0
- package/dist/server.js +136 -7
- package/dist/shared-translation.d.ts +55 -0
- package/dist/shared-translation.js +335 -0
- package/dist/translation-passes.d.ts +97 -0
- package/dist/translation-passes.js +227 -0
- package/dist/web-sites.d.ts +6 -0
- package/dist/web-sites.js +26 -0
- package/package.json +1 -1
- package/src/live-interpreter.ts +194 -0
- package/src/live-voice.ts +34 -7
- package/src/pcm-wav.ts +26 -0
- package/src/server.ts +96 -6
- package/src/shared-translation.ts +218 -0
- package/src/translation-passes.ts +232 -0
- package/src/web-sites.ts +29 -0
- package/web/dist/assets/{hls-3VKVEQE3-e7zQjVfV.js → hls-3VKVEQE3-P3sj2d3a.js} +1 -1
- package/web/dist/assets/{index-biLbPIu_.css → index-BmR7T0zX.css} +1 -1
- package/web/dist/assets/index-Do9VYiU8.js +3 -0
- package/web/dist/assets/{mpegts-Ci1G5S8S.js → mpegts-CZ1YoF1U.js} +1 -1
- package/web/dist/assets/{mpegts-LO6RVLD6-cj6SzTIe.js → mpegts-LO6RVLD6-DuNM0Zxm.js} +1 -1
- package/web/dist/index.html +20 -2
- package/web/dist/sw.js +6 -6
- package/web/dist/assets/index-Ckxp8OU8.js +0 -1
package/src/live-voice.ts
CHANGED
|
@@ -4,6 +4,7 @@ import { SpeechError, decodeWav, encodeWav, quietSamples } from "./speech.ts";
|
|
|
4
4
|
import { speakerTurns, type ScribeResult, type SpeakerTranscript } from "./speaker-turns.ts";
|
|
5
5
|
import type { VoiceProfile } from "./voice-profile.ts";
|
|
6
6
|
import { Guard } from "./guard.ts";
|
|
7
|
+
import type { TranslationMeter } from "./translation-passes.ts";
|
|
7
8
|
import type { Queryable } from "./follows.ts";
|
|
8
9
|
|
|
9
10
|
export const LIVE_VOICE_MODEL = "eleven_flash_v2_5";
|
|
@@ -28,6 +29,7 @@ export class LiveVoice {
|
|
|
28
29
|
private readonly grants = new Map<string, { by: string; channel: string; expires: number; remaining: number }>();
|
|
29
30
|
private readonly activeBy = new Map<string, number>();
|
|
30
31
|
private readonly db?: Queryable;
|
|
32
|
+
private readonly billing?: TranslationMeter;
|
|
31
33
|
private schema: Promise<unknown> | null = null;
|
|
32
34
|
private readonly dailyChars: number;
|
|
33
35
|
private cleanupAt = 0;
|
|
@@ -36,7 +38,7 @@ export class LiveVoice {
|
|
|
36
38
|
private readonly userDailyChars: number;
|
|
37
39
|
private readonly userDailyAudioSeconds: number;
|
|
38
40
|
|
|
39
|
-
constructor(options: { apiKey?: string; fetcher?: typeof fetch; now?: () => number; charsPerMinute?: number; dailyChars?: number; dailyAudioSeconds?: number; userDailyChars?: number; userDailyAudioSeconds?: number; db?: Queryable } = {}) {
|
|
41
|
+
constructor(options: { apiKey?: string; fetcher?: typeof fetch; now?: () => number; charsPerMinute?: number; dailyChars?: number; dailyAudioSeconds?: number; userDailyChars?: number; userDailyAudioSeconds?: number; db?: Queryable; billing?: TranslationMeter } = {}) {
|
|
40
42
|
this.key = options.apiKey ?? process.env["ELEVENLABS_API_KEY"] ?? "";
|
|
41
43
|
this.fetcher = options.fetcher ?? fetch;
|
|
42
44
|
this.now = options.now ?? Date.now;
|
|
@@ -47,6 +49,7 @@ export class LiveVoice {
|
|
|
47
49
|
this.userDailyAudioSeconds = budget(options.userDailyAudioSeconds, 43_200);
|
|
48
50
|
this.requests = new Guard(this.now);
|
|
49
51
|
this.db = options.db;
|
|
52
|
+
this.billing = options.billing;
|
|
50
53
|
}
|
|
51
54
|
|
|
52
55
|
available(): boolean { return this.key !== ""; }
|
|
@@ -54,16 +57,19 @@ export class LiveVoice {
|
|
|
54
57
|
/** Optional diarization, billed only while a signed-in listener requests it.
|
|
55
58
|
* Rolling audio is bounded to 15 seconds, including overlap. Every second
|
|
56
59
|
* submitted (also repeated context) consumes the persistent provider budget. */
|
|
57
|
-
async hear(bytes: Uint8Array, by: string, signal?: AbortSignal): Promise<SpeakerTranscript> {
|
|
60
|
+
async hear(bytes: Uint8Array, by: string, signal?: AbortSignal, meter = this.billing): Promise<SpeakerTranscript> {
|
|
58
61
|
if (!this.available()) throw new SpeechError("speaker voices are unavailable", 503);
|
|
62
|
+
await meter?.require(by);
|
|
59
63
|
const wav = decodeWav(bytes);
|
|
60
64
|
const seconds = wav.samples.length / wav.rate;
|
|
61
65
|
if (wav.rate !== 16000 || wav.channels !== 1 || seconds < 0.2 || seconds > 15.1) throw new SpeechError("send up to 15 seconds of mono 16 kHz WAV", 400);
|
|
62
66
|
if (wav.samples.some(sample => !Number.isFinite(sample))) throw new SpeechError("invalid audio samples", 400);
|
|
63
|
-
if (!this.requests.check(`hear:${by}`, { allowed:
|
|
67
|
+
if (!this.requests.check(`hear:${by}`, { allowed: 36, windowMs: 60_000 }).ok) throw new SpeechError("too many speaker transcription requests", 429);
|
|
64
68
|
if (quietSamples(wav.samples)) return { language: "", seconds, turns: [] };
|
|
65
69
|
if (this.hearing.has(by) || this.hearing.size >= 4) throw new SpeechError("speaker transcription is busy", 429);
|
|
66
70
|
this.hearing.add(by);
|
|
71
|
+
let reservation: string | undefined;
|
|
72
|
+
let accepted = false;
|
|
67
73
|
try {
|
|
68
74
|
const billed = Math.ceil(seconds);
|
|
69
75
|
await this.reserve(`scribe:user:${by}`, billed, 300, 60_000);
|
|
@@ -80,12 +86,18 @@ export class LiveVoice {
|
|
|
80
86
|
form.set("tag_audio_events", "false");
|
|
81
87
|
form.set("timestamps_granularity", "word");
|
|
82
88
|
// No language_code: preserve the source language, including Spanish.
|
|
89
|
+
reservation = await meter?.reserve(by, "transcription", wav.samples.length);
|
|
83
90
|
const answer = await this.fetcher("https://api.elevenlabs.io/v1/speech-to-text", {
|
|
84
91
|
method: "POST", headers: { "xi-api-key": this.key }, body: form,
|
|
85
92
|
signal: signal ? AbortSignal.any([signal, AbortSignal.timeout(10_000)]) : AbortSignal.timeout(10_000),
|
|
86
93
|
});
|
|
87
94
|
if (!answer.ok) throw new SpeechError("speaker transcription could not run; check provider quota and permissions", answer.status === 429 ? 429 : 502);
|
|
95
|
+
accepted = true;
|
|
96
|
+
if (reservation) await meter!.commit(reservation);
|
|
88
97
|
return speakerTurns(await answer.json() as ScribeResult, wav);
|
|
98
|
+
} catch (error) {
|
|
99
|
+
if (reservation && !accepted) await meter!.refund(reservation);
|
|
100
|
+
throw error;
|
|
89
101
|
} finally { this.hearing.delete(by); }
|
|
90
102
|
}
|
|
91
103
|
|
|
@@ -110,6 +122,7 @@ export class LiveVoice {
|
|
|
110
122
|
async grant(by: string, channel: string): Promise<{ token: string; expires: number }> {
|
|
111
123
|
if (!/^[\w-]{1,80}$/.test(channel)) throw new SpeechError("choose a playback session or live channel", 400);
|
|
112
124
|
if (!this.requests.check(`grant:${by}`, { allowed: 10, windowMs: 60_000 }).ok) throw new SpeechError("too many audio authorization requests", 429);
|
|
125
|
+
await this.billing?.require(by);
|
|
113
126
|
for (const [token, grant] of this.grants) if (grant.expires <= this.now()) this.grants.delete(token);
|
|
114
127
|
if (this.grants.size >= 5000) throw new SpeechError("audio authorization is busy", 503);
|
|
115
128
|
const token = `nxd_${randomBytes(32).toString("base64url")}`;
|
|
@@ -185,8 +198,9 @@ export class LiveVoice {
|
|
|
185
198
|
}
|
|
186
199
|
|
|
187
200
|
/** Stream the first request immediately; concurrent listeners share its cached result. */
|
|
188
|
-
async stream(ask: VoiceRequest, by: string, signal?: AbortSignal): Promise<Response> {
|
|
201
|
+
async stream(ask: VoiceRequest, by: string, signal?: AbortSignal, meter = this.billing): Promise<Response> {
|
|
189
202
|
this.checkRequest(by);
|
|
203
|
+
await meter?.require(by);
|
|
190
204
|
const text = typeof ask.text === "string" ? ask.text.trim() : "";
|
|
191
205
|
if (!text || text.length > 600) throw new SpeechError("translated audio needs a caption of 1–600 characters", 400);
|
|
192
206
|
if (!LIVE_VOICE_LANGUAGES.has(ask.language)) throw new SpeechError("this language is not supported by Flash voices", 400);
|
|
@@ -202,9 +216,14 @@ export class LiveVoice {
|
|
|
202
216
|
const id = createHash("sha256").update(JSON.stringify([LIVE_VOICE_MODEL, voice.id, ask.language, text])).digest("hex");
|
|
203
217
|
for (const [key, item] of this.cache) if (item.until < this.now()) this.cache.delete(key);
|
|
204
218
|
const cached = this.cache.get(id);
|
|
205
|
-
if (cached) return new Response(new Uint8Array(cached.bytes), { headers: HEADERS });
|
|
206
219
|
const pending = this.pending.get(id);
|
|
207
|
-
if (
|
|
220
|
+
if (cached || pending) {
|
|
221
|
+
const bytes = cached?.bytes ?? await pending!;
|
|
222
|
+
signal?.throwIfAborted();
|
|
223
|
+
const paid = await meter?.reserve(by, "voice", text.length);
|
|
224
|
+
if (paid) await meter!.commit(paid);
|
|
225
|
+
return new Response(new Uint8Array(bytes), { headers: HEADERS });
|
|
226
|
+
}
|
|
208
227
|
if (this.pending.size >= 4) throw new SpeechError("translated audio is busy; waiting for the next caption", 429);
|
|
209
228
|
if ((this.activeBy.get(by) ?? 0) >= 2) throw new SpeechError("two voice requests are already active for this account", 429);
|
|
210
229
|
// Register before the first provider await, so identical requests cannot both bill.
|
|
@@ -215,9 +234,12 @@ export class LiveVoice {
|
|
|
215
234
|
this.activeBy.set(by, (this.activeBy.get(by) ?? 0) + 1);
|
|
216
235
|
const release = (): void => { this.pending.delete(id); this.activeBy.set(by, Math.max(0, (this.activeBy.get(by) ?? 1) - 1)); if (!this.activeBy.get(by)) this.activeBy.delete(by); };
|
|
217
236
|
void finished.catch(() => undefined);
|
|
237
|
+
let reservation: string | undefined;
|
|
238
|
+
let accepted = false;
|
|
218
239
|
try {
|
|
219
240
|
await this.charge(by, ask.channel ?? "direct", text.length);
|
|
220
241
|
signal?.throwIfAborted();
|
|
242
|
+
reservation = await meter?.reserve(by, "voice", text.length);
|
|
221
243
|
const response = await this.fetcher(`https://api.elevenlabs.io/v1/text-to-speech/${voice.id}/stream?output_format=pcm_16000`, {
|
|
222
244
|
method: "POST",
|
|
223
245
|
headers: { "xi-api-key": this.key, "content-type": "application/json" },
|
|
@@ -225,6 +247,9 @@ export class LiveVoice {
|
|
|
225
247
|
signal: signal ? AbortSignal.any([signal, AbortSignal.timeout(10_000)]) : AbortSignal.timeout(10_000),
|
|
226
248
|
});
|
|
227
249
|
if (!response.ok || !response.body) throw new SpeechError(response.status === 429 ? "ElevenLabs audio quota is temporarily exhausted" : "ElevenLabs could not generate audio; check the server key and quota", response.status === 429 ? 429 : 502);
|
|
250
|
+
// Once the provider accepts, aborting playback cannot refund heard audio.
|
|
251
|
+
accepted = true;
|
|
252
|
+
if (reservation) await meter!.commit(reservation);
|
|
228
253
|
const [play, keep] = response.body.tee();
|
|
229
254
|
void (async () => {
|
|
230
255
|
const reader = keep.getReader();
|
|
@@ -251,7 +276,9 @@ export class LiveVoice {
|
|
|
251
276
|
})();
|
|
252
277
|
return new Response(play, { headers: HEADERS });
|
|
253
278
|
} catch (error) {
|
|
254
|
-
fail(error); release();
|
|
279
|
+
fail(error); release();
|
|
280
|
+
if (reservation && !accepted) await meter!.refund(reservation);
|
|
281
|
+
throw error;
|
|
255
282
|
}
|
|
256
283
|
}
|
|
257
284
|
}
|
package/src/pcm-wav.ts
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
/** 16-bit mono PCM WAV bytes from samples in [-1, 1]. */
|
|
2
|
+
export function encodeWav(samples: Float32Array, rate = 16000): Uint8Array {
|
|
3
|
+
const bytes = new Uint8Array(44 + samples.length * 2);
|
|
4
|
+
const view = new DataView(bytes.buffer);
|
|
5
|
+
const ascii = (at: number, text: string): void => {
|
|
6
|
+
for (let i = 0; i < text.length; i++) bytes[at + i] = text.charCodeAt(i);
|
|
7
|
+
};
|
|
8
|
+
ascii(0, "RIFF");
|
|
9
|
+
view.setUint32(4, 36 + samples.length * 2, true);
|
|
10
|
+
ascii(8, "WAVE");
|
|
11
|
+
ascii(12, "fmt ");
|
|
12
|
+
view.setUint32(16, 16, true);
|
|
13
|
+
view.setUint16(20, 1, true);
|
|
14
|
+
view.setUint16(22, 1, true);
|
|
15
|
+
view.setUint32(24, rate, true);
|
|
16
|
+
view.setUint32(28, rate * 2, true);
|
|
17
|
+
view.setUint16(32, 2, true);
|
|
18
|
+
view.setUint16(34, 16, true);
|
|
19
|
+
ascii(36, "data");
|
|
20
|
+
view.setUint32(40, samples.length * 2, true);
|
|
21
|
+
for (let i = 0; i < samples.length; i++) {
|
|
22
|
+
const clipped = Math.max(-1, Math.min(1, samples[i] as number));
|
|
23
|
+
view.setInt16(44 + i * 2, clipped < 0 ? clipped * 32768 : clipped * 32767, true);
|
|
24
|
+
}
|
|
25
|
+
return bytes;
|
|
26
|
+
}
|
package/src/server.ts
CHANGED
|
@@ -18,6 +18,8 @@ import { spawn, spawnSync } from "node:child_process";
|
|
|
18
18
|
import { readFileSync } from "node:fs";
|
|
19
19
|
import { Readable } from "node:stream";
|
|
20
20
|
import { pipeline as pipeStream } from "node:stream/promises";
|
|
21
|
+
import { SharedTranslations, type SharedEvent } from "./shared-translation.ts";
|
|
22
|
+
import { TranslationPasses } from "./translation-passes.ts";
|
|
21
23
|
import { LiveVoice, LIVE_VOICE_LANGUAGES, LIVE_VOICE_MODEL, type VoiceRequest } from "./live-voice.ts";
|
|
22
24
|
import { Connections, type Kind } from "./connections.ts";
|
|
23
25
|
import {
|
|
@@ -1599,6 +1601,8 @@ async function readBytes(request: IncomingMessage, limit: number): Promise<Uint8
|
|
|
1599
1601
|
|
|
1600
1602
|
export interface HandlerOptions {
|
|
1601
1603
|
web: string | null;
|
|
1604
|
+
/** Branded clients on this same backend, keyed by an explicitly configured host. */
|
|
1605
|
+
webSites?: ReadonlyMap<string, import("./web-sites.ts").WebSite>;
|
|
1602
1606
|
media: boolean;
|
|
1603
1607
|
version: string;
|
|
1604
1608
|
/** The key from the share link, or null to serve to anyone who can connect. */
|
|
@@ -1760,6 +1764,8 @@ export interface HandlerOptions {
|
|
|
1760
1764
|
/** Speech to text: a line said out loud, heard here. Needs the optional model. */
|
|
1761
1765
|
speech?: Speech;
|
|
1762
1766
|
liveVoice?: LiveVoice;
|
|
1767
|
+
translationPasses?: TranslationPasses;
|
|
1768
|
+
sharedTranslations?: SharedTranslations;
|
|
1763
1769
|
/** Translation: texts in another language, by a model here. The same optional library. */
|
|
1764
1770
|
translator?: Translator;
|
|
1765
1771
|
/** The transcript store: what was heard, kept under the media's identity. Where the accounts are. */
|
|
@@ -1843,6 +1849,9 @@ export function createHandler(engine: Engine, options: HandlerOptions) {
|
|
|
1843
1849
|
return async function handle(request: IncomingMessage, response: ServerResponse): Promise<void> {
|
|
1844
1850
|
const url = new URL(request.url ?? "/", "http://localhost");
|
|
1845
1851
|
const path = url.pathname;
|
|
1852
|
+
const webSite = options.webSites?.get((request.headers.host ?? "").toLowerCase());
|
|
1853
|
+
const web = webSite?.web ?? options.web;
|
|
1854
|
+
const eventSite = webSite?.site ?? options.invites?.site;
|
|
1846
1855
|
const key = options.key ?? null;
|
|
1847
1856
|
const behindProxy = options.behindProxy ?? false;
|
|
1848
1857
|
const listenKey = options.listenKey ?? null;
|
|
@@ -1934,7 +1943,7 @@ export function createHandler(engine: Engine, options: HandlerOptions) {
|
|
|
1934
1943
|
...(options.layouts ? { layouts: options.layouts } : {}),
|
|
1935
1944
|
...(options.rooms ? { rooms: options.rooms } : {}),
|
|
1936
1945
|
...(options.tickets ? { tickets: options.tickets } : {}),
|
|
1937
|
-
...(
|
|
1946
|
+
...(eventSite ? { site: eventSite } : {}),
|
|
1938
1947
|
...(options.invites?.email ? { email: options.invites.email } : {}),
|
|
1939
1948
|
})) return;
|
|
1940
1949
|
|
|
@@ -2852,6 +2861,75 @@ export function createHandler(engine: Engine, options: HandlerOptions) {
|
|
|
2852
2861
|
return;
|
|
2853
2862
|
}
|
|
2854
2863
|
|
|
2864
|
+
if (path === "/api/v1/speech/shared" && request.method === "POST") {
|
|
2865
|
+
try {
|
|
2866
|
+
const service = options.sharedTranslations;
|
|
2867
|
+
const who = await options.accounts?.whoIs(tokenFrom(request.headers));
|
|
2868
|
+
if (!who) throw new SpeechError("Sign in to translate live audio.", 401);
|
|
2869
|
+
if (!service) throw new SpeechError("Shared translation is unavailable.", 503);
|
|
2870
|
+
const caller = callerOf(request.headers, request.socket.remoteAddress, options.behindProxy ?? false);
|
|
2871
|
+
if (!guard.check(`shared-join:${who.id}`, { allowed: 10, windowMs: 60_000 }).ok || !guard.check(`shared-ip:${caller}`, { allowed: 30, windowMs: 60_000 }).ok) throw new SpeechError("Too many live translation joins.", 429);
|
|
2872
|
+
if (!request.headers["content-type"]?.startsWith("application/json")) throw new SpeechError("Send a live source as JSON.", 415);
|
|
2873
|
+
let body: { source?: unknown; language?: unknown };
|
|
2874
|
+
try { body = JSON.parse(await readBody(request, 4096)); } catch { throw new SpeechError("Choose a live source and language.", 400); }
|
|
2875
|
+
if (typeof body?.source !== "string" || typeof body.language !== "string") throw new SpeechError("Choose a live source and language.", 400);
|
|
2876
|
+
const queued: SharedEvent[] = [];
|
|
2877
|
+
let finished = false;
|
|
2878
|
+
const send = (event: SharedEvent): void => {
|
|
2879
|
+
if (!response.headersSent) { queued.push(event); return; }
|
|
2880
|
+
if (response.destroyed || response.writableLength > 512_000) throw new Error("slow listener");
|
|
2881
|
+
response.write(`data: ${JSON.stringify(event)}\n\n`);
|
|
2882
|
+
};
|
|
2883
|
+
const leave = await service.join(body.source, body.language, who.id, send, () => { finished = true; if (response.headersSent) response.end(); });
|
|
2884
|
+
if (response.destroyed) { leave(); return; }
|
|
2885
|
+
response.writeHead(200, { ...CORS, "content-type": "text/event-stream", "cache-control": "no-store", "x-accel-buffering": "no" });
|
|
2886
|
+
for (const event of queued) send(event);
|
|
2887
|
+
if (finished) { leave(); response.end(); return; }
|
|
2888
|
+
const heartbeat = setInterval(() => { if (!response.destroyed) response.write(": listening\n\n"); }, 10_000);
|
|
2889
|
+
response.once("close", () => { clearInterval(heartbeat); leave(); });
|
|
2890
|
+
} catch (error) {
|
|
2891
|
+
if (response.headersSent) response.end();
|
|
2892
|
+
else json(response, error instanceof SpeechError ? error.status : 502, { error: error instanceof SpeechError ? error.message : "Shared translation is unavailable." });
|
|
2893
|
+
}
|
|
2894
|
+
return;
|
|
2895
|
+
}
|
|
2896
|
+
|
|
2897
|
+
if (path === "/api/v1/translation-passes" || path.startsWith("/api/v1/translation-passes/")) {
|
|
2898
|
+
response.setHeader("cache-control", "no-store");
|
|
2899
|
+
try {
|
|
2900
|
+
const caller = callerOf(request.headers, request.socket.remoteAddress, options.behindProxy ?? false);
|
|
2901
|
+
if (!guard.check(`translation-pass-ip:${caller}`, { allowed: 120, windowMs: 60_000 }).ok) throw new SpeechError("Too many purchase requests.", 429);
|
|
2902
|
+
const service = options.translationPasses;
|
|
2903
|
+
if (!service || !options.accounts) {
|
|
2904
|
+
if (path === "/api/v1/translation-passes" && request.method === "GET") json(response, 200, { required: false, available: false, balanceMicros: 0, expires: null, plans: [], coins: [], orders: [] });
|
|
2905
|
+
else json(response, 503, { error: "Translation checkout is unavailable." });
|
|
2906
|
+
return;
|
|
2907
|
+
}
|
|
2908
|
+
const who = await options.accounts.whoIs(tokenFrom(request.headers));
|
|
2909
|
+
if (path === "/api/v1/translation-passes" && request.method === "GET") {
|
|
2910
|
+
json(response, 200, await service.access(who?.id)); return;
|
|
2911
|
+
}
|
|
2912
|
+
if (!who) throw new SpeechError("Sign in to buy translated audio.", 401);
|
|
2913
|
+
if (!guard.check(`translation-pass-account:${who.id}`, { allowed: 30, windowMs: 60_000 }).ok) throw new SpeechError("Too many purchase requests.", 429);
|
|
2914
|
+
if (path === "/api/v1/translation-passes/checkout" && request.method === "POST") {
|
|
2915
|
+
if (!request.headers["content-type"]?.startsWith("application/json")) throw new SpeechError("Send a purchase as JSON.", 415);
|
|
2916
|
+
if (!guard.check(`translation-checkout-ip:${caller}`, { allowed: 10, windowMs: 3600_000 }).ok) throw new SpeechError("Too many new checkouts. Resume your pending purchase.", 429);
|
|
2917
|
+
let body: Record<string, unknown>;
|
|
2918
|
+
try { body = JSON.parse(await readBody(request, 1024)); } catch { throw new SpeechError("Choose a translation pass.", 400); }
|
|
2919
|
+
if (!body || typeof body !== "object" || typeof body["plan"] !== "string" || typeof body["coin"] !== "string" || typeof body["requestKey"] !== "string") throw new SpeechError("Choose a translation pass and payment currency.", 400);
|
|
2920
|
+
json(response, 200, await service.checkout(who.id, body["plan"], body["coin"], body["requestKey"])); return;
|
|
2921
|
+
}
|
|
2922
|
+
const order = path.match(/^\/api\/v1\/translation-passes\/orders\/([^/]+)$/);
|
|
2923
|
+
if (order && request.method === "GET") {
|
|
2924
|
+
json(response, 200, await service.check(who.id, order[1]!)); return;
|
|
2925
|
+
}
|
|
2926
|
+
json(response, 405, { error: "GET your balance or purchase status; POST a checkout." });
|
|
2927
|
+
} catch (error) {
|
|
2928
|
+
json(response, error instanceof SpeechError ? error.status : 502, { error: error instanceof SpeechError ? error.message : "Translation purchases are temporarily unavailable." });
|
|
2929
|
+
}
|
|
2930
|
+
return;
|
|
2931
|
+
}
|
|
2932
|
+
|
|
2855
2933
|
/*
|
|
2856
2934
|
* Speech to text: a WAV in, the words out. Signed in only -- the ear
|
|
2857
2935
|
* costs CPU and a trollbox line needs a name anyway -- and, given a
|
|
@@ -5358,8 +5436,8 @@ export function createHandler(engine: Engine, options: HandlerOptions) {
|
|
|
5358
5436
|
return;
|
|
5359
5437
|
}
|
|
5360
5438
|
|
|
5361
|
-
if (
|
|
5362
|
-
const direct = safeJoin(
|
|
5439
|
+
if (web !== null) {
|
|
5440
|
+
const direct = safeJoin(web, path);
|
|
5363
5441
|
if (direct === null) {
|
|
5364
5442
|
json(response, 400, { error: "bad path" });
|
|
5365
5443
|
return;
|
|
@@ -5368,7 +5446,7 @@ export function createHandler(engine: Engine, options: HandlerOptions) {
|
|
|
5368
5446
|
if (isFile(direct)) file = direct;
|
|
5369
5447
|
else if (isFile(join(direct, "index.html"))) file = join(direct, "index.html");
|
|
5370
5448
|
// A single-page app: any unknown path is the shell, and the client routes.
|
|
5371
|
-
else if (isFile(join(
|
|
5449
|
+
else if (isFile(join(web, "index.html"))) file = join(web, "index.html");
|
|
5372
5450
|
if (file !== null) {
|
|
5373
5451
|
if (file.endsWith("index.html") && path.startsWith("/live/") && options.events) {
|
|
5374
5452
|
try {
|
|
@@ -5381,7 +5459,7 @@ export function createHandler(engine: Engine, options: HandlerOptions) {
|
|
|
5381
5459
|
if (event && (await options.events.canAccess(event, account?.id) || invited)) {
|
|
5382
5460
|
const shell = readIfPossible(file);
|
|
5383
5461
|
if (shell !== null) {
|
|
5384
|
-
html(response, 200, eventDocument(shell, event,
|
|
5462
|
+
html(response, 200, eventDocument(shell, event, eventSite ?? "https://backtoschool.help"));
|
|
5385
5463
|
return;
|
|
5386
5464
|
}
|
|
5387
5465
|
}
|
|
@@ -5391,7 +5469,7 @@ export function createHandler(engine: Engine, options: HandlerOptions) {
|
|
|
5391
5469
|
const subject = joinSubject(url, options);
|
|
5392
5470
|
const shell = subject ? readIfPossible(file) : null;
|
|
5393
5471
|
if (subject && shell !== null) {
|
|
5394
|
-
html(response, 200, joinDocument(shell, subject,
|
|
5472
|
+
html(response, 200, joinDocument(shell, subject, eventSite ?? "https://nixamp.com"));
|
|
5395
5473
|
return;
|
|
5396
5474
|
}
|
|
5397
5475
|
}
|
|
@@ -5832,6 +5910,8 @@ export async function serve(argv: string[], version = "0.1.0"): Promise<void> {
|
|
|
5832
5910
|
if (!options.noJingle) playJingle(tools);
|
|
5833
5911
|
|
|
5834
5912
|
const web = options.web !== null ? resolve(options.web) : defaultWebDir();
|
|
5913
|
+
const { webSitesFromEnv } = await import("./web-sites.ts");
|
|
5914
|
+
const webSites = webSitesFromEnv();
|
|
5835
5915
|
// The same keys this port used last time, so a link somebody was given
|
|
5836
5916
|
// still works after a restart -- and a server is restarted to pick up a new
|
|
5837
5917
|
// version, which is to say often. `--new-key` mints a fresh pair and forgets
|
|
@@ -6001,8 +6081,14 @@ export async function serve(argv: string[], version = "0.1.0"): Promise<void> {
|
|
|
6001
6081
|
// the first person to speak does not wait for the model to arrive; a box
|
|
6002
6082
|
// without the optional model never says it is ready, and answers 503.
|
|
6003
6083
|
const speech = accounts && process.env["NIXAMP_STT"] !== "off" ? new Speech() : undefined;
|
|
6084
|
+
// Account servers meter paid providers by default, including when checkout
|
|
6085
|
+
// is temporarily unconfigured. Self-hosters can explicitly sponsor usage.
|
|
6086
|
+
const translationPasses = accounts && pool && process.env["NIXAMP_TRANSLATION_BILLING"] !== "off"
|
|
6087
|
+
? new TranslationPasses({ db: pool, key: process.env["COINPAY_X402_KEY"] ?? "", site: nixampSite }) : undefined;
|
|
6088
|
+
if (translationPasses) void translationPasses.access().catch(() => console.error("nixamp: translation checkout could not warm; it will retry on demand"));
|
|
6004
6089
|
const liveVoice = accounts && process.env["NIXAMP_DUBBING"] !== "off" ? new LiveVoice({
|
|
6005
6090
|
...(pool ? { db: pool } : {}),
|
|
6091
|
+
...(translationPasses ? { billing: translationPasses } : {}),
|
|
6006
6092
|
dailyChars: Number(process.env["NIXAMP_DUB_DAILY_CHARS"] ?? 200_000),
|
|
6007
6093
|
dailyAudioSeconds: Number(process.env["NIXAMP_DUB_DAILY_AUDIO_SECONDS"] ?? 86_400),
|
|
6008
6094
|
userDailyChars: Number(process.env["NIXAMP_DUB_USER_DAILY_CHARS"] ?? 120_000),
|
|
@@ -6018,6 +6104,7 @@ export async function serve(argv: string[], version = "0.1.0"): Promise<void> {
|
|
|
6018
6104
|
// names pairs to load at boot (en-de,en-sv) so the first line does not
|
|
6019
6105
|
// wait; NIXAMP_MT=off turns it off.
|
|
6020
6106
|
const translator = accounts && process.env["NIXAMP_MT"] !== "off" ? new Translator() : undefined;
|
|
6107
|
+
const sharedTranslations = liveVoice && translator ? new SharedTranslations({ voice: liveVoice, translator, ...(translationPasses ? { billing: translationPasses } : {}) }) : undefined;
|
|
6021
6108
|
if (translator) {
|
|
6022
6109
|
const warm = (process.env["NIXAMP_MT_WARM"] ?? "").split(",").map((one) => one.trim()).filter(Boolean);
|
|
6023
6110
|
if (warm.length > 0) {
|
|
@@ -6414,6 +6501,7 @@ export async function serve(argv: string[], version = "0.1.0"): Promise<void> {
|
|
|
6414
6501
|
|
|
6415
6502
|
const server = createServer(engine, {
|
|
6416
6503
|
web,
|
|
6504
|
+
webSites,
|
|
6417
6505
|
media: options.media,
|
|
6418
6506
|
owner,
|
|
6419
6507
|
channels,
|
|
@@ -6500,6 +6588,8 @@ export async function serve(argv: string[], version = "0.1.0"): Promise<void> {
|
|
|
6500
6588
|
...(trollbox ? { trollbox } : {}),
|
|
6501
6589
|
...(speech ? { speech } : {}),
|
|
6502
6590
|
...(liveVoice ? { liveVoice } : {}),
|
|
6591
|
+
...(translationPasses ? { translationPasses } : {}),
|
|
6592
|
+
...(sharedTranslations ? { sharedTranslations } : {}),
|
|
6503
6593
|
...(translator ? { translator } : {}),
|
|
6504
6594
|
...(transcripts ? { transcripts } : {}),
|
|
6505
6595
|
...(translations ? { translations } : {}),
|
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
/** One live source/language pipeline, paid access for every listening account.
|
|
2
|
+
* Browsers receive the same PCM, never upload a replacement for a public feed. */
|
|
3
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
4
|
+
import { lookup } from "node:dns/promises";
|
|
5
|
+
import { BlockList, isIP } from "node:net";
|
|
6
|
+
import http from "node:http";
|
|
7
|
+
import https from "node:https";
|
|
8
|
+
import { spawn } from "node:child_process";
|
|
9
|
+
import { Interpreter, type Caption } from "./live-interpreter.ts";
|
|
10
|
+
import { LiveVoice, LIVE_VOICE_LANGUAGES } from "./live-voice.ts";
|
|
11
|
+
import { SpeechError } from "./speech.ts";
|
|
12
|
+
import type { Translator } from "./translate.ts";
|
|
13
|
+
import type { TranslationMeter, TranslationUsage } from "./translation-passes.ts";
|
|
14
|
+
|
|
15
|
+
const blocked = new BlockList();
|
|
16
|
+
for (const [ip, bits] of [["0.0.0.0",8],["10.0.0.0",8],["100.64.0.0",10],["127.0.0.0",8],["169.254.0.0",16],["172.16.0.0",12],["192.0.0.0",24],["192.0.2.0",24],["192.168.0.0",16],["198.18.0.0",15],["198.51.100.0",24],["203.0.113.0",24],["224.0.0.0",4],["240.0.0.0",4]] as const) blocked.addSubnet(ip, bits, "ipv4");
|
|
17
|
+
const v6global = new BlockList(); v6global.addSubnet("2000::", 3, "ipv6");
|
|
18
|
+
blocked.addSubnet("2001:db8::", 32, "ipv6"); blocked.addSubnet("2002::", 16, "ipv6"); blocked.addSubnet("2001::", 32, "ipv6");
|
|
19
|
+
export function publicStreamAddress(address: string): boolean {
|
|
20
|
+
const family = isIP(address);
|
|
21
|
+
return family === 4 ? !blocked.check(address, "ipv4") : family === 6 && v6global.check(address, "ipv6") && !blocked.check(address, "ipv6");
|
|
22
|
+
}
|
|
23
|
+
export function sharedSource(raw: string): URL {
|
|
24
|
+
let url: URL;
|
|
25
|
+
try { url = new URL(raw); } catch { throw new SpeechError("Choose a live Nixamp channel.", 400); }
|
|
26
|
+
if (!["http:", "https:"].includes(url.protocol) || url.username || url.password || url.hash || !/^\/api\/channels\/[\w-]{1,80}$/.test(url.pathname) || [...url.searchParams.keys()].some(key => key !== "k")) throw new SpeechError("Choose a live Nixamp channel.", 400);
|
|
27
|
+
if (url.toString().length > 2048) throw new SpeechError("Invalid live source.", 400);
|
|
28
|
+
url.searchParams.sort(); return url;
|
|
29
|
+
}
|
|
30
|
+
export interface LiveInput { stop(): void; }
|
|
31
|
+
export type OpenLiveInput = (source: URL, pcm: (bytes: Buffer) => void, ended: () => void, signal: AbortSignal) => Promise<LiveInput>;
|
|
32
|
+
|
|
33
|
+
/** Resolve once and pin that public address; no redirects or ffmpeg URL fetches. */
|
|
34
|
+
export function liveInput(ffmpeg: string[] = ["ffmpeg"]): OpenLiveInput {
|
|
35
|
+
return async (source, pcm, ended, signal) => {
|
|
36
|
+
const hostname = source.hostname.replace(/^\[|\]$/g, "");
|
|
37
|
+
const addresses = await lookup(hostname, { all: true });
|
|
38
|
+
if (!addresses.length || addresses.some(one => !publicStreamAddress(one.address))) throw new SpeechError("This live source is not publicly reachable.", 400);
|
|
39
|
+
signal.throwIfAborted();
|
|
40
|
+
const selected = addresses[0]!;
|
|
41
|
+
const child = spawn(ffmpeg[0]!, [...ffmpeg.slice(1), "-v", "error", "-nostats", "-protocol_whitelist", "pipe", "-readrate", "1", "-analyzeduration", "500000", "-probesize", "262144", "-i", "pipe:0", "-vn", "-ac", "1", "-ar", "16000", "-f", "s16le", "pipe:1"], { stdio: ["pipe", "pipe", "ignore"] });
|
|
42
|
+
let stopped = false;
|
|
43
|
+
const request = (source.protocol === "https:" ? https : http).get(source, {
|
|
44
|
+
signal, headers: { "user-agent": "Nixamp shared translated audio", accept: "audio/*,video/*,application/octet-stream" },
|
|
45
|
+
lookup: (_name, options, callback) => { if (typeof options === "object" && options.all) callback(null, [selected]); else callback(null, selected.address, selected.family); },
|
|
46
|
+
});
|
|
47
|
+
const stop = (): void => { if (stopped) return; stopped = true; request.destroy(); child.stdin.destroy(); child.kill("SIGKILL"); signal.removeEventListener("abort", stop); };
|
|
48
|
+
const end = (): void => { if (!stopped) { stop(); ended(); } };
|
|
49
|
+
signal.addEventListener("abort", stop, { once: true });
|
|
50
|
+
child.stdout.on("data", pcm); child.on("error", end); child.on("close", end); child.stdin.on("error", end);
|
|
51
|
+
request.on("response", response => {
|
|
52
|
+
if (response.statusCode !== 200) { response.destroy(); end(); return; }
|
|
53
|
+
response.on("error", end); response.on("end", end); response.pipe(child.stdin);
|
|
54
|
+
});
|
|
55
|
+
request.on("error", end);
|
|
56
|
+
request.setTimeout(15_000, end);
|
|
57
|
+
return { stop };
|
|
58
|
+
};
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
export type SharedEvent = { type: "status"; listeners: number } | { type: "line"; id: string; line: Caption } | { type: "audio"; id: string; data: string } | { type: "end"; id: string } | { type: "error"; error: string };
|
|
62
|
+
type Member = { by: string; send: (event: SharedEvent) => void; close: () => void };
|
|
63
|
+
interface Feed { id: string; source: URL; language: string; members: Set<Member>; controller: AbortController; input?: LiveInput; interpreter: Interpreter; meter: TranslationMeter; reservations: Map<string, string[]>; queue: Caption[]; speaking: boolean; previous: Buffer; pending: Buffer; until: number; heartbeat?: ReturnType<typeof setInterval>; }
|
|
64
|
+
|
|
65
|
+
export class SharedTranslations {
|
|
66
|
+
private readonly feeds = new Map<string, Feed>();
|
|
67
|
+
constructor(private readonly options: { voice: LiveVoice; translator: Pick<Translator, "translate">; billing?: TranslationMeter; open?: OpenLiveInput }) {}
|
|
68
|
+
get size(): number { return this.feeds.size; }
|
|
69
|
+
|
|
70
|
+
async join(raw: string, language: string, by: string, send: Member["send"], close: () => void): Promise<() => void> {
|
|
71
|
+
if (!this.options.voice.available()) throw new SpeechError("Translated audio is unavailable.", 503);
|
|
72
|
+
const source = sharedSource(raw);
|
|
73
|
+
if (!LIVE_VOICE_LANGUAGES.has(language)) throw new SpeechError("Choose a supported audio language.", 400);
|
|
74
|
+
await this.options.billing?.require(by);
|
|
75
|
+
const connections = [...this.feeds.values()].reduce((total, feed) => total + [...feed.members].filter(member => member.by === by).length, 0);
|
|
76
|
+
if (connections >= 2) throw new SpeechError("Translated audio is already open in two players for this account.", 429);
|
|
77
|
+
const id = `dub-${createHash("sha256").update(`${source}|${language}`).digest("hex").slice(0, 40)}`;
|
|
78
|
+
let feed = this.feeds.get(id);
|
|
79
|
+
if (!feed) {
|
|
80
|
+
if (this.feeds.size >= 4) throw new SpeechError("Live translation is busy. Try again shortly.", 429);
|
|
81
|
+
feed = this.make(id, source, language); this.feeds.set(id, feed);
|
|
82
|
+
const current = feed;
|
|
83
|
+
current.heartbeat = setInterval(() => void this.checkMembers(current), 10_000); current.heartbeat.unref?.();
|
|
84
|
+
}
|
|
85
|
+
if (feed.members.size >= 1000) throw new SpeechError("This translated stream is full. Try again shortly.", 429);
|
|
86
|
+
const member = { by, send, close }; feed.members.add(member);
|
|
87
|
+
this.broadcast(feed, { type: "status", listeners: feed.members.size });
|
|
88
|
+
if (feed.members.size === 1 && !feed.input) void this.start(feed);
|
|
89
|
+
const current = feed;
|
|
90
|
+
return () => this.leave(current, member);
|
|
91
|
+
}
|
|
92
|
+
private leave(feed: Feed, member: Member): void {
|
|
93
|
+
if (!feed.members.delete(member)) return;
|
|
94
|
+
if (!feed.members.size) {
|
|
95
|
+
if (feed.heartbeat) clearInterval(feed.heartbeat);
|
|
96
|
+
feed.controller.abort(); feed.input?.stop(); feed.interpreter.reset(); feed.queue = [];
|
|
97
|
+
this.feeds.delete(feed.id);
|
|
98
|
+
} else this.broadcast(feed, { type: "status", listeners: feed.members.size });
|
|
99
|
+
}
|
|
100
|
+
private async checkMembers(feed: Feed): Promise<void> {
|
|
101
|
+
const accounts = [...new Set([...feed.members].map(member => member.by))];
|
|
102
|
+
let funded = new Set(accounts), message = "Your translation balance is used up or expired. Buy another pass to continue.";
|
|
103
|
+
try {
|
|
104
|
+
if (this.options.billing?.eligible) funded = new Set(await this.options.billing.eligible(accounts));
|
|
105
|
+
else if (this.options.billing) {
|
|
106
|
+
const results = await Promise.allSettled(accounts.map(by => this.options.billing!.require(by)));
|
|
107
|
+
funded = new Set(accounts.filter((_by, index) => results[index]?.status === "fulfilled"));
|
|
108
|
+
}
|
|
109
|
+
} catch { funded.clear(); message = "Your translation balance could not be checked. Try again shortly."; }
|
|
110
|
+
for (const member of [...feed.members]) if (accounts.includes(member.by) && !funded.has(member.by)) {
|
|
111
|
+
try { member.send({ type: "error", error: message }); } catch { /* disconnected */ }
|
|
112
|
+
finally { this.leave(feed, member); member.close(); }
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
private broadcast(feed: Feed, event: SharedEvent): void {
|
|
116
|
+
for (const member of [...feed.members]) {
|
|
117
|
+
try { member.send(event); } catch { this.leave(feed, member); member.close(); }
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
private stop(feed: Feed, error: string): void {
|
|
121
|
+
this.broadcast(feed, { type: "error", error });
|
|
122
|
+
for (const member of [...feed.members]) { this.leave(feed, member); member.close(); }
|
|
123
|
+
}
|
|
124
|
+
private make(id: string, source: URL, language: string): Feed {
|
|
125
|
+
const feed = { id, source, language, members: new Set<Member>(), controller: new AbortController(), reservations: new Map<string, string[]>(), queue: [], speaking: false, previous: Buffer.alloc(0), pending: Buffer.alloc(0), until: Date.now() } as unknown as Feed;
|
|
126
|
+
// Every active account pays the same published access rate. API work is
|
|
127
|
+
// done once; additional listeners do not trigger recognition or synthesis.
|
|
128
|
+
feed.meter = {
|
|
129
|
+
require: async () => { if (!feed.members.size) throw new SpeechError("No listeners.", 410); },
|
|
130
|
+
reserve: async (_by: string, kind: TranslationUsage, units: number) => {
|
|
131
|
+
const ids: string[] = [];
|
|
132
|
+
const accounts = [...new Set([...feed.members].map(member => member.by))];
|
|
133
|
+
const accepted = new Set<string>();
|
|
134
|
+
const billing = this.options.billing;
|
|
135
|
+
if (billing?.reserveMany) {
|
|
136
|
+
for (const charged of await billing.reserveMany(accounts, kind, units)) { ids.push(charged.id); accepted.add(charged.by); }
|
|
137
|
+
} else {
|
|
138
|
+
const results = await Promise.allSettled(accounts.map(async account => ({ by: account, id: await billing?.reserve(account, kind, units) })));
|
|
139
|
+
let failure: unknown;
|
|
140
|
+
for (const result of results) {
|
|
141
|
+
if (result.status === "fulfilled") { accepted.add(result.value.by); if (result.value.id) ids.push(result.value.id); }
|
|
142
|
+
else if (!(result.reason instanceof SpeechError) || result.reason.status !== 402) failure = result.reason;
|
|
143
|
+
}
|
|
144
|
+
if (failure) { await Promise.all(ids.map(id => billing!.refund(id))); throw failure; }
|
|
145
|
+
}
|
|
146
|
+
for (const member of [...feed.members]) if (accounts.includes(member.by) && !accepted.has(member.by)) {
|
|
147
|
+
try { member.send({ type: "error", error: "Your translation balance is used up or expired. Buy another pass to continue." }); }
|
|
148
|
+
catch { /* disconnected */ } finally { this.leave(feed, member); member.close(); }
|
|
149
|
+
}
|
|
150
|
+
if (!feed.members.size) { await Promise.all(ids.map(id => this.options.billing!.refund(id))); throw new SpeechError("No funded listeners.", 402); }
|
|
151
|
+
const reservation = randomUUID(); feed.reservations.set(reservation, ids); return reservation;
|
|
152
|
+
},
|
|
153
|
+
commit: async id => { const ids = feed.reservations.get(id) ?? []; if (this.options.billing?.commitMany) await this.options.billing.commitMany(ids); else await Promise.all(ids.map(id => this.options.billing!.commit(id))); feed.reservations.delete(id); },
|
|
154
|
+
refund: async id => { const ids = feed.reservations.get(id) ?? []; if (this.options.billing?.refundMany) await this.options.billing.refundMany(ids); else await Promise.all(ids.map(id => this.options.billing!.refund(id))); feed.reservations.delete(id); },
|
|
155
|
+
};
|
|
156
|
+
let voices: Awaited<ReturnType<LiveVoice["voices"]>> = [];
|
|
157
|
+
feed.interpreter = new Interpreter({
|
|
158
|
+
language: () => language, speakers: () => true, voices: () => voices, channel: () => id,
|
|
159
|
+
status: text => { if (!feed.controller.signal.aborted && !feed.members.size) this.stop(feed, text); },
|
|
160
|
+
failed: () => this.stop(feed, "Live translation stopped. Enable it again to retry."),
|
|
161
|
+
lines: lines => {
|
|
162
|
+
feed.queue.push(...lines);
|
|
163
|
+
if (feed.queue.length > 24) { this.stop(feed, "Live translation fell behind. Enable it again to retry."); return; }
|
|
164
|
+
if (!feed.speaking) void this.speak(feed);
|
|
165
|
+
},
|
|
166
|
+
fetcher: (async (url, init) => {
|
|
167
|
+
feed.controller.signal.throwIfAborted();
|
|
168
|
+
const signal = AbortSignal.any([feed.controller.signal, ...(init?.signal ? [init.signal] : [])]);
|
|
169
|
+
if (String(url).includes("/speakers")) {
|
|
170
|
+
voices = await this.options.voice.voices();
|
|
171
|
+
return Response.json(await this.options.voice.hear(new Uint8Array(init?.body as Uint8Array), id, signal, feed.meter));
|
|
172
|
+
}
|
|
173
|
+
const body = JSON.parse(String(init?.body));
|
|
174
|
+
return Response.json(await this.options.translator.translate(body.texts, body.from, body.to, { by: id, deadline: Date.now() + 10_000 }));
|
|
175
|
+
}) as typeof fetch,
|
|
176
|
+
});
|
|
177
|
+
return feed;
|
|
178
|
+
}
|
|
179
|
+
private async start(feed: Feed): Promise<void> {
|
|
180
|
+
try {
|
|
181
|
+
const input = await (this.options.open ?? liveInput())(feed.source, bytes => {
|
|
182
|
+
if (feed.controller.signal.aborted) return;
|
|
183
|
+
feed.pending = Buffer.concat([feed.pending, bytes]);
|
|
184
|
+
while (feed.pending.length >= 64_000) {
|
|
185
|
+
const fresh = feed.pending.subarray(0, 64_000); feed.pending = feed.pending.subarray(64_000);
|
|
186
|
+
const pcm = Buffer.concat([feed.previous, fresh]); feed.previous = pcm.subarray(-128_000);
|
|
187
|
+
feed.until += 2000;
|
|
188
|
+
if (Math.abs(Date.now() - feed.until) > 2000) feed.until = Date.now();
|
|
189
|
+
const samples = new Float32Array(pcm.length / 2);
|
|
190
|
+
for (let index = 0; index < samples.length; index++) samples[index] = pcm.readInt16LE(index * 2) / 32768;
|
|
191
|
+
feed.interpreter.push({ samples, at: feed.until - samples.length / 16, until: feed.until, freshAt: feed.until - 2000 });
|
|
192
|
+
}
|
|
193
|
+
}, () => this.stop(feed, "The live source stopped. Enable translated audio again when playback resumes."), feed.controller.signal);
|
|
194
|
+
if (feed.controller.signal.aborted) input.stop(); else feed.input = input;
|
|
195
|
+
} catch (error) { this.stop(feed, error instanceof SpeechError ? error.message : "The live source could not be opened."); }
|
|
196
|
+
}
|
|
197
|
+
private async speak(feed: Feed): Promise<void> {
|
|
198
|
+
feed.speaking = true;
|
|
199
|
+
try {
|
|
200
|
+
while (feed.queue.length && !feed.controller.signal.aborted) {
|
|
201
|
+
const line = feed.queue.shift()!;
|
|
202
|
+
if (Date.now() - line.until > 12_000) throw new SpeechError("Live translation fell behind. Enable it again to retry.", 503);
|
|
203
|
+
const response = await this.options.voice.stream({ text: line.text, language: feed.language, channel: feed.id, speaker: line.speaker, voice: feed.interpreter.tracker.speakers.get(line.speaker ?? "")?.voice }, feed.id, feed.controller.signal, feed.meter);
|
|
204
|
+
const id = randomUUID();
|
|
205
|
+
this.broadcast(feed, { type: "line", id, line });
|
|
206
|
+
const reader = response.body!.getReader();
|
|
207
|
+
try {
|
|
208
|
+
while (true) {
|
|
209
|
+
const { done, value } = await reader.read(); if (done) break;
|
|
210
|
+
for (let at = 0; at < value.length; at += 16_384) this.broadcast(feed, { type: "audio", id, data: Buffer.from(value.subarray(at, at + 16_384)).toString("base64") });
|
|
211
|
+
}
|
|
212
|
+
} finally { reader.releaseLock(); }
|
|
213
|
+
this.broadcast(feed, { type: "end", id });
|
|
214
|
+
}
|
|
215
|
+
} catch (error) { if (!feed.controller.signal.aborted) this.stop(feed, error instanceof SpeechError ? error.message : "Translated audio was interrupted."); }
|
|
216
|
+
finally { feed.speaking = false; }
|
|
217
|
+
}
|
|
218
|
+
}
|