nixamp 0.26.7 → 0.27.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/live-voice.ts CHANGED
@@ -4,6 +4,7 @@ import { SpeechError, decodeWav, encodeWav, quietSamples } from "./speech.ts";
4
4
  import { speakerTurns, type ScribeResult, type SpeakerTranscript } from "./speaker-turns.ts";
5
5
  import type { VoiceProfile } from "./voice-profile.ts";
6
6
  import { Guard } from "./guard.ts";
7
+ import type { TranslationMeter } from "./translation-passes.ts";
7
8
  import type { Queryable } from "./follows.ts";
8
9
 
9
10
  export const LIVE_VOICE_MODEL = "eleven_flash_v2_5";
@@ -28,6 +29,7 @@ export class LiveVoice {
28
29
  private readonly grants = new Map<string, { by: string; channel: string; expires: number; remaining: number }>();
29
30
  private readonly activeBy = new Map<string, number>();
30
31
  private readonly db?: Queryable;
32
+ private readonly billing?: TranslationMeter;
31
33
  private schema: Promise<unknown> | null = null;
32
34
  private readonly dailyChars: number;
33
35
  private cleanupAt = 0;
@@ -36,7 +38,7 @@ export class LiveVoice {
36
38
  private readonly userDailyChars: number;
37
39
  private readonly userDailyAudioSeconds: number;
38
40
 
39
- constructor(options: { apiKey?: string; fetcher?: typeof fetch; now?: () => number; charsPerMinute?: number; dailyChars?: number; dailyAudioSeconds?: number; userDailyChars?: number; userDailyAudioSeconds?: number; db?: Queryable } = {}) {
41
+ constructor(options: { apiKey?: string; fetcher?: typeof fetch; now?: () => number; charsPerMinute?: number; dailyChars?: number; dailyAudioSeconds?: number; userDailyChars?: number; userDailyAudioSeconds?: number; db?: Queryable; billing?: TranslationMeter } = {}) {
40
42
  this.key = options.apiKey ?? process.env["ELEVENLABS_API_KEY"] ?? "";
41
43
  this.fetcher = options.fetcher ?? fetch;
42
44
  this.now = options.now ?? Date.now;
@@ -47,6 +49,7 @@ export class LiveVoice {
47
49
  this.userDailyAudioSeconds = budget(options.userDailyAudioSeconds, 43_200);
48
50
  this.requests = new Guard(this.now);
49
51
  this.db = options.db;
52
+ this.billing = options.billing;
50
53
  }
51
54
 
52
55
  available(): boolean { return this.key !== ""; }
@@ -54,8 +57,9 @@ export class LiveVoice {
54
57
  /** Optional diarization, billed only while a signed-in listener requests it.
55
58
  * Rolling audio is bounded to 15 seconds, including overlap. Every second
56
59
  * submitted (also repeated context) consumes the persistent provider budget. */
57
- async hear(bytes: Uint8Array, by: string, signal?: AbortSignal): Promise<SpeakerTranscript> {
60
+ async hear(bytes: Uint8Array, by: string, signal?: AbortSignal, meter = this.billing): Promise<SpeakerTranscript> {
58
61
  if (!this.available()) throw new SpeechError("speaker voices are unavailable", 503);
62
+ await meter?.require(by);
59
63
  const wav = decodeWav(bytes);
60
64
  const seconds = wav.samples.length / wav.rate;
61
65
  if (wav.rate !== 16000 || wav.channels !== 1 || seconds < 0.2 || seconds > 15.1) throw new SpeechError("send up to 15 seconds of mono 16 kHz WAV", 400);
@@ -64,6 +68,8 @@ export class LiveVoice {
64
68
  if (quietSamples(wav.samples)) return { language: "", seconds, turns: [] };
65
69
  if (this.hearing.has(by) || this.hearing.size >= 4) throw new SpeechError("speaker transcription is busy", 429);
66
70
  this.hearing.add(by);
71
+ let reservation: string | undefined;
72
+ let accepted = false;
67
73
  try {
68
74
  const billed = Math.ceil(seconds);
69
75
  await this.reserve(`scribe:user:${by}`, billed, 300, 60_000);
@@ -80,12 +86,18 @@ export class LiveVoice {
80
86
  form.set("tag_audio_events", "false");
81
87
  form.set("timestamps_granularity", "word");
82
88
  // No language_code: preserve the source language, including Spanish.
89
+ reservation = await meter?.reserve(by, "transcription", wav.samples.length);
83
90
  const answer = await this.fetcher("https://api.elevenlabs.io/v1/speech-to-text", {
84
91
  method: "POST", headers: { "xi-api-key": this.key }, body: form,
85
92
  signal: signal ? AbortSignal.any([signal, AbortSignal.timeout(10_000)]) : AbortSignal.timeout(10_000),
86
93
  });
87
94
  if (!answer.ok) throw new SpeechError("speaker transcription could not run; check provider quota and permissions", answer.status === 429 ? 429 : 502);
95
+ accepted = true;
96
+ if (reservation) await meter!.commit(reservation);
88
97
  return speakerTurns(await answer.json() as ScribeResult, wav);
98
+ } catch (error) {
99
+ if (reservation && !accepted) await meter!.refund(reservation);
100
+ throw error;
89
101
  } finally { this.hearing.delete(by); }
90
102
  }
91
103
 
@@ -110,6 +122,7 @@ export class LiveVoice {
110
122
  async grant(by: string, channel: string): Promise<{ token: string; expires: number }> {
111
123
  if (!/^[\w-]{1,80}$/.test(channel)) throw new SpeechError("choose a playback session or live channel", 400);
112
124
  if (!this.requests.check(`grant:${by}`, { allowed: 10, windowMs: 60_000 }).ok) throw new SpeechError("too many audio authorization requests", 429);
125
+ await this.billing?.require(by);
113
126
  for (const [token, grant] of this.grants) if (grant.expires <= this.now()) this.grants.delete(token);
114
127
  if (this.grants.size >= 5000) throw new SpeechError("audio authorization is busy", 503);
115
128
  const token = `nxd_${randomBytes(32).toString("base64url")}`;
@@ -185,8 +198,9 @@ export class LiveVoice {
185
198
  }
186
199
 
187
200
  /** Stream the first request immediately; concurrent listeners share its cached result. */
188
- async stream(ask: VoiceRequest, by: string, signal?: AbortSignal): Promise<Response> {
201
+ async stream(ask: VoiceRequest, by: string, signal?: AbortSignal, meter = this.billing): Promise<Response> {
189
202
  this.checkRequest(by);
203
+ await meter?.require(by);
190
204
  const text = typeof ask.text === "string" ? ask.text.trim() : "";
191
205
  if (!text || text.length > 600) throw new SpeechError("translated audio needs a caption of 1–600 characters", 400);
192
206
  if (!LIVE_VOICE_LANGUAGES.has(ask.language)) throw new SpeechError("this language is not supported by Flash voices", 400);
@@ -202,9 +216,14 @@ export class LiveVoice {
202
216
  const id = createHash("sha256").update(JSON.stringify([LIVE_VOICE_MODEL, voice.id, ask.language, text])).digest("hex");
203
217
  for (const [key, item] of this.cache) if (item.until < this.now()) this.cache.delete(key);
204
218
  const cached = this.cache.get(id);
205
- if (cached) return new Response(new Uint8Array(cached.bytes), { headers: HEADERS });
206
219
  const pending = this.pending.get(id);
207
- if (pending) return new Response(new Uint8Array(await pending), { headers: HEADERS });
220
+ if (cached || pending) {
221
+ const bytes = cached?.bytes ?? await pending!;
222
+ signal?.throwIfAborted();
223
+ const paid = await meter?.reserve(by, "voice", text.length);
224
+ if (paid) await meter!.commit(paid);
225
+ return new Response(new Uint8Array(bytes), { headers: HEADERS });
226
+ }
208
227
  if (this.pending.size >= 4) throw new SpeechError("translated audio is busy; waiting for the next caption", 429);
209
228
  if ((this.activeBy.get(by) ?? 0) >= 2) throw new SpeechError("two voice requests are already active for this account", 429);
210
229
  // Register before the first provider await, so identical requests cannot both bill.
@@ -215,9 +234,12 @@ export class LiveVoice {
215
234
  this.activeBy.set(by, (this.activeBy.get(by) ?? 0) + 1);
216
235
  const release = (): void => { this.pending.delete(id); this.activeBy.set(by, Math.max(0, (this.activeBy.get(by) ?? 1) - 1)); if (!this.activeBy.get(by)) this.activeBy.delete(by); };
217
236
  void finished.catch(() => undefined);
237
+ let reservation: string | undefined;
238
+ let accepted = false;
218
239
  try {
219
240
  await this.charge(by, ask.channel ?? "direct", text.length);
220
241
  signal?.throwIfAborted();
242
+ reservation = await meter?.reserve(by, "voice", text.length);
221
243
  const response = await this.fetcher(`https://api.elevenlabs.io/v1/text-to-speech/${voice.id}/stream?output_format=pcm_16000`, {
222
244
  method: "POST",
223
245
  headers: { "xi-api-key": this.key, "content-type": "application/json" },
@@ -225,6 +247,9 @@ export class LiveVoice {
225
247
  signal: signal ? AbortSignal.any([signal, AbortSignal.timeout(10_000)]) : AbortSignal.timeout(10_000),
226
248
  });
227
249
  if (!response.ok || !response.body) throw new SpeechError(response.status === 429 ? "ElevenLabs audio quota is temporarily exhausted" : "ElevenLabs could not generate audio; check the server key and quota", response.status === 429 ? 429 : 502);
250
+ // Once the provider accepts, aborting playback cannot refund heard audio.
251
+ accepted = true;
252
+ if (reservation) await meter!.commit(reservation);
228
253
  const [play, keep] = response.body.tee();
229
254
  void (async () => {
230
255
  const reader = keep.getReader();
@@ -251,7 +276,9 @@ export class LiveVoice {
251
276
  })();
252
277
  return new Response(play, { headers: HEADERS });
253
278
  } catch (error) {
254
- fail(error); release(); throw error;
279
+ fail(error); release();
280
+ if (reservation && !accepted) await meter!.refund(reservation);
281
+ throw error;
255
282
  }
256
283
  }
257
284
  }
package/src/pcm-wav.ts ADDED
@@ -0,0 +1,26 @@
1
+ /** 16-bit mono PCM WAV bytes from samples in [-1, 1]. */
2
+ export function encodeWav(samples: Float32Array, rate = 16000): Uint8Array {
3
+ const bytes = new Uint8Array(44 + samples.length * 2);
4
+ const view = new DataView(bytes.buffer);
5
+ const ascii = (at: number, text: string): void => {
6
+ for (let i = 0; i < text.length; i++) bytes[at + i] = text.charCodeAt(i);
7
+ };
8
+ ascii(0, "RIFF");
9
+ view.setUint32(4, 36 + samples.length * 2, true);
10
+ ascii(8, "WAVE");
11
+ ascii(12, "fmt ");
12
+ view.setUint32(16, 16, true);
13
+ view.setUint16(20, 1, true);
14
+ view.setUint16(22, 1, true);
15
+ view.setUint32(24, rate, true);
16
+ view.setUint32(28, rate * 2, true);
17
+ view.setUint16(32, 2, true);
18
+ view.setUint16(34, 16, true);
19
+ ascii(36, "data");
20
+ view.setUint32(40, samples.length * 2, true);
21
+ for (let i = 0; i < samples.length; i++) {
22
+ const clipped = Math.max(-1, Math.min(1, samples[i] as number));
23
+ view.setInt16(44 + i * 2, clipped < 0 ? clipped * 32768 : clipped * 32767, true);
24
+ }
25
+ return bytes;
26
+ }
package/src/server.ts CHANGED
@@ -18,6 +18,8 @@ import { spawn, spawnSync } from "node:child_process";
18
18
  import { readFileSync } from "node:fs";
19
19
  import { Readable } from "node:stream";
20
20
  import { pipeline as pipeStream } from "node:stream/promises";
21
+ import { SharedTranslations, type SharedEvent } from "./shared-translation.ts";
22
+ import { TranslationPasses } from "./translation-passes.ts";
21
23
  import { LiveVoice, LIVE_VOICE_LANGUAGES, LIVE_VOICE_MODEL, type VoiceRequest } from "./live-voice.ts";
22
24
  import { Connections, type Kind } from "./connections.ts";
23
25
  import {
@@ -1599,6 +1601,8 @@ async function readBytes(request: IncomingMessage, limit: number): Promise<Uint8
1599
1601
 
1600
1602
  export interface HandlerOptions {
1601
1603
  web: string | null;
1604
+ /** Branded clients on this same backend, keyed by an explicitly configured host. */
1605
+ webSites?: ReadonlyMap<string, import("./web-sites.ts").WebSite>;
1602
1606
  media: boolean;
1603
1607
  version: string;
1604
1608
  /** The key from the share link, or null to serve to anyone who can connect. */
@@ -1760,6 +1764,8 @@ export interface HandlerOptions {
1760
1764
  /** Speech to text: a line said out loud, heard here. Needs the optional model. */
1761
1765
  speech?: Speech;
1762
1766
  liveVoice?: LiveVoice;
1767
+ translationPasses?: TranslationPasses;
1768
+ sharedTranslations?: SharedTranslations;
1763
1769
  /** Translation: texts in another language, by a model here. The same optional library. */
1764
1770
  translator?: Translator;
1765
1771
  /** The transcript store: what was heard, kept under the media's identity. Where the accounts are. */
@@ -1843,6 +1849,9 @@ export function createHandler(engine: Engine, options: HandlerOptions) {
1843
1849
  return async function handle(request: IncomingMessage, response: ServerResponse): Promise<void> {
1844
1850
  const url = new URL(request.url ?? "/", "http://localhost");
1845
1851
  const path = url.pathname;
1852
+ const webSite = options.webSites?.get((request.headers.host ?? "").toLowerCase());
1853
+ const web = webSite?.web ?? options.web;
1854
+ const eventSite = webSite?.site ?? options.invites?.site;
1846
1855
  const key = options.key ?? null;
1847
1856
  const behindProxy = options.behindProxy ?? false;
1848
1857
  const listenKey = options.listenKey ?? null;
@@ -1934,7 +1943,7 @@ export function createHandler(engine: Engine, options: HandlerOptions) {
1934
1943
  ...(options.layouts ? { layouts: options.layouts } : {}),
1935
1944
  ...(options.rooms ? { rooms: options.rooms } : {}),
1936
1945
  ...(options.tickets ? { tickets: options.tickets } : {}),
1937
- ...(options.invites?.site ? { site: options.invites.site } : {}),
1946
+ ...(eventSite ? { site: eventSite } : {}),
1938
1947
  ...(options.invites?.email ? { email: options.invites.email } : {}),
1939
1948
  })) return;
1940
1949
 
@@ -2852,6 +2861,75 @@ export function createHandler(engine: Engine, options: HandlerOptions) {
2852
2861
  return;
2853
2862
  }
2854
2863
 
2864
+ if (path === "/api/v1/speech/shared" && request.method === "POST") {
2865
+ try {
2866
+ const service = options.sharedTranslations;
2867
+ const who = await options.accounts?.whoIs(tokenFrom(request.headers));
2868
+ if (!who) throw new SpeechError("Sign in to translate live audio.", 401);
2869
+ if (!service) throw new SpeechError("Shared translation is unavailable.", 503);
2870
+ const caller = callerOf(request.headers, request.socket.remoteAddress, options.behindProxy ?? false);
2871
+ if (!guard.check(`shared-join:${who.id}`, { allowed: 10, windowMs: 60_000 }).ok || !guard.check(`shared-ip:${caller}`, { allowed: 30, windowMs: 60_000 }).ok) throw new SpeechError("Too many live translation joins.", 429);
2872
+ if (!request.headers["content-type"]?.startsWith("application/json")) throw new SpeechError("Send a live source as JSON.", 415);
2873
+ let body: { source?: unknown; language?: unknown };
2874
+ try { body = JSON.parse(await readBody(request, 4096)); } catch { throw new SpeechError("Choose a live source and language.", 400); }
2875
+ if (typeof body?.source !== "string" || typeof body.language !== "string") throw new SpeechError("Choose a live source and language.", 400);
2876
+ const queued: SharedEvent[] = [];
2877
+ let finished = false;
2878
+ const send = (event: SharedEvent): void => {
2879
+ if (!response.headersSent) { queued.push(event); return; }
2880
+ if (response.destroyed || response.writableLength > 512_000) throw new Error("slow listener");
2881
+ response.write(`data: ${JSON.stringify(event)}\n\n`);
2882
+ };
2883
+ const leave = await service.join(body.source, body.language, who.id, send, () => { finished = true; if (response.headersSent) response.end(); });
2884
+ if (response.destroyed) { leave(); return; }
2885
+ response.writeHead(200, { ...CORS, "content-type": "text/event-stream", "cache-control": "no-store", "x-accel-buffering": "no" });
2886
+ for (const event of queued) send(event);
2887
+ if (finished) { leave(); response.end(); return; }
2888
+ const heartbeat = setInterval(() => { if (!response.destroyed) response.write(": listening\n\n"); }, 10_000);
2889
+ response.once("close", () => { clearInterval(heartbeat); leave(); });
2890
+ } catch (error) {
2891
+ if (response.headersSent) response.end();
2892
+ else json(response, error instanceof SpeechError ? error.status : 502, { error: error instanceof SpeechError ? error.message : "Shared translation is unavailable." });
2893
+ }
2894
+ return;
2895
+ }
2896
+
2897
+ if (path === "/api/v1/translation-passes" || path.startsWith("/api/v1/translation-passes/")) {
2898
+ response.setHeader("cache-control", "no-store");
2899
+ try {
2900
+ const caller = callerOf(request.headers, request.socket.remoteAddress, options.behindProxy ?? false);
2901
+ if (!guard.check(`translation-pass-ip:${caller}`, { allowed: 120, windowMs: 60_000 }).ok) throw new SpeechError("Too many purchase requests.", 429);
2902
+ const service = options.translationPasses;
2903
+ if (!service || !options.accounts) {
2904
+ if (path === "/api/v1/translation-passes" && request.method === "GET") json(response, 200, { required: false, available: false, balanceMicros: 0, expires: null, plans: [], coins: [], orders: [] });
2905
+ else json(response, 503, { error: "Translation checkout is unavailable." });
2906
+ return;
2907
+ }
2908
+ const who = await options.accounts.whoIs(tokenFrom(request.headers));
2909
+ if (path === "/api/v1/translation-passes" && request.method === "GET") {
2910
+ json(response, 200, await service.access(who?.id)); return;
2911
+ }
2912
+ if (!who) throw new SpeechError("Sign in to buy translated audio.", 401);
2913
+ if (!guard.check(`translation-pass-account:${who.id}`, { allowed: 30, windowMs: 60_000 }).ok) throw new SpeechError("Too many purchase requests.", 429);
2914
+ if (path === "/api/v1/translation-passes/checkout" && request.method === "POST") {
2915
+ if (!request.headers["content-type"]?.startsWith("application/json")) throw new SpeechError("Send a purchase as JSON.", 415);
2916
+ if (!guard.check(`translation-checkout-ip:${caller}`, { allowed: 10, windowMs: 3600_000 }).ok) throw new SpeechError("Too many new checkouts. Resume your pending purchase.", 429);
2917
+ let body: Record<string, unknown>;
2918
+ try { body = JSON.parse(await readBody(request, 1024)); } catch { throw new SpeechError("Choose a translation pass.", 400); }
2919
+ if (!body || typeof body !== "object" || typeof body["plan"] !== "string" || typeof body["coin"] !== "string" || typeof body["requestKey"] !== "string") throw new SpeechError("Choose a translation pass and payment currency.", 400);
2920
+ json(response, 200, await service.checkout(who.id, body["plan"], body["coin"], body["requestKey"])); return;
2921
+ }
2922
+ const order = path.match(/^\/api\/v1\/translation-passes\/orders\/([^/]+)$/);
2923
+ if (order && request.method === "GET") {
2924
+ json(response, 200, await service.check(who.id, order[1]!)); return;
2925
+ }
2926
+ json(response, 405, { error: "GET your balance or purchase status; POST a checkout." });
2927
+ } catch (error) {
2928
+ json(response, error instanceof SpeechError ? error.status : 502, { error: error instanceof SpeechError ? error.message : "Translation purchases are temporarily unavailable." });
2929
+ }
2930
+ return;
2931
+ }
2932
+
2855
2933
  /*
2856
2934
  * Speech to text: a WAV in, the words out. Signed in only -- the ear
2857
2935
  * costs CPU and a trollbox line needs a name anyway -- and, given a
@@ -5358,8 +5436,8 @@ export function createHandler(engine: Engine, options: HandlerOptions) {
5358
5436
  return;
5359
5437
  }
5360
5438
 
5361
- if (options.web !== null) {
5362
- const direct = safeJoin(options.web, path);
5439
+ if (web !== null) {
5440
+ const direct = safeJoin(web, path);
5363
5441
  if (direct === null) {
5364
5442
  json(response, 400, { error: "bad path" });
5365
5443
  return;
@@ -5368,7 +5446,7 @@ export function createHandler(engine: Engine, options: HandlerOptions) {
5368
5446
  if (isFile(direct)) file = direct;
5369
5447
  else if (isFile(join(direct, "index.html"))) file = join(direct, "index.html");
5370
5448
  // A single-page app: any unknown path is the shell, and the client routes.
5371
- else if (isFile(join(options.web, "index.html"))) file = join(options.web, "index.html");
5449
+ else if (isFile(join(web, "index.html"))) file = join(web, "index.html");
5372
5450
  if (file !== null) {
5373
5451
  if (file.endsWith("index.html") && path.startsWith("/live/") && options.events) {
5374
5452
  try {
@@ -5381,7 +5459,7 @@ export function createHandler(engine: Engine, options: HandlerOptions) {
5381
5459
  if (event && (await options.events.canAccess(event, account?.id) || invited)) {
5382
5460
  const shell = readIfPossible(file);
5383
5461
  if (shell !== null) {
5384
- html(response, 200, eventDocument(shell, event, options.invites?.site ?? "https://backtoschool.help"));
5462
+ html(response, 200, eventDocument(shell, event, eventSite ?? "https://backtoschool.help"));
5385
5463
  return;
5386
5464
  }
5387
5465
  }
@@ -5391,7 +5469,7 @@ export function createHandler(engine: Engine, options: HandlerOptions) {
5391
5469
  const subject = joinSubject(url, options);
5392
5470
  const shell = subject ? readIfPossible(file) : null;
5393
5471
  if (subject && shell !== null) {
5394
- html(response, 200, joinDocument(shell, subject, options.invites?.site ?? "https://nixamp.com"));
5472
+ html(response, 200, joinDocument(shell, subject, eventSite ?? "https://nixamp.com"));
5395
5473
  return;
5396
5474
  }
5397
5475
  }
@@ -5832,6 +5910,8 @@ export async function serve(argv: string[], version = "0.1.0"): Promise<void> {
5832
5910
  if (!options.noJingle) playJingle(tools);
5833
5911
 
5834
5912
  const web = options.web !== null ? resolve(options.web) : defaultWebDir();
5913
+ const { webSitesFromEnv } = await import("./web-sites.ts");
5914
+ const webSites = webSitesFromEnv();
5835
5915
  // The same keys this port used last time, so a link somebody was given
5836
5916
  // still works after a restart -- and a server is restarted to pick up a new
5837
5917
  // version, which is to say often. `--new-key` mints a fresh pair and forgets
@@ -6001,8 +6081,14 @@ export async function serve(argv: string[], version = "0.1.0"): Promise<void> {
6001
6081
  // the first person to speak does not wait for the model to arrive; a box
6002
6082
  // without the optional model never says it is ready, and answers 503.
6003
6083
  const speech = accounts && process.env["NIXAMP_STT"] !== "off" ? new Speech() : undefined;
6084
+ // Account servers meter paid providers by default, including when checkout
6085
+ // is temporarily unconfigured. Self-hosters can explicitly sponsor usage.
6086
+ const translationPasses = accounts && pool && process.env["NIXAMP_TRANSLATION_BILLING"] !== "off"
6087
+ ? new TranslationPasses({ db: pool, key: process.env["COINPAY_X402_KEY"] ?? "", site: nixampSite }) : undefined;
6088
+ if (translationPasses) void translationPasses.access().catch(() => console.error("nixamp: translation checkout could not warm; it will retry on demand"));
6004
6089
  const liveVoice = accounts && process.env["NIXAMP_DUBBING"] !== "off" ? new LiveVoice({
6005
6090
  ...(pool ? { db: pool } : {}),
6091
+ ...(translationPasses ? { billing: translationPasses } : {}),
6006
6092
  dailyChars: Number(process.env["NIXAMP_DUB_DAILY_CHARS"] ?? 200_000),
6007
6093
  dailyAudioSeconds: Number(process.env["NIXAMP_DUB_DAILY_AUDIO_SECONDS"] ?? 86_400),
6008
6094
  userDailyChars: Number(process.env["NIXAMP_DUB_USER_DAILY_CHARS"] ?? 120_000),
@@ -6018,6 +6104,7 @@ export async function serve(argv: string[], version = "0.1.0"): Promise<void> {
6018
6104
  // names pairs to load at boot (en-de,en-sv) so the first line does not
6019
6105
  // wait; NIXAMP_MT=off turns it off.
6020
6106
  const translator = accounts && process.env["NIXAMP_MT"] !== "off" ? new Translator() : undefined;
6107
+ const sharedTranslations = liveVoice && translator ? new SharedTranslations({ voice: liveVoice, translator, ...(translationPasses ? { billing: translationPasses } : {}) }) : undefined;
6021
6108
  if (translator) {
6022
6109
  const warm = (process.env["NIXAMP_MT_WARM"] ?? "").split(",").map((one) => one.trim()).filter(Boolean);
6023
6110
  if (warm.length > 0) {
@@ -6414,6 +6501,7 @@ export async function serve(argv: string[], version = "0.1.0"): Promise<void> {
6414
6501
 
6415
6502
  const server = createServer(engine, {
6416
6503
  web,
6504
+ webSites,
6417
6505
  media: options.media,
6418
6506
  owner,
6419
6507
  channels,
@@ -6500,6 +6588,8 @@ export async function serve(argv: string[], version = "0.1.0"): Promise<void> {
6500
6588
  ...(trollbox ? { trollbox } : {}),
6501
6589
  ...(speech ? { speech } : {}),
6502
6590
  ...(liveVoice ? { liveVoice } : {}),
6591
+ ...(translationPasses ? { translationPasses } : {}),
6592
+ ...(sharedTranslations ? { sharedTranslations } : {}),
6503
6593
  ...(translator ? { translator } : {}),
6504
6594
  ...(transcripts ? { transcripts } : {}),
6505
6595
  ...(translations ? { translations } : {}),
@@ -0,0 +1,218 @@
1
+ /** One live source/language pipeline, paid access for every listening account.
2
+ * Browsers receive the same PCM, never upload a replacement for a public feed. */
3
+ import { createHash, randomUUID } from "node:crypto";
4
+ import { lookup } from "node:dns/promises";
5
+ import { BlockList, isIP } from "node:net";
6
+ import http from "node:http";
7
+ import https from "node:https";
8
+ import { spawn } from "node:child_process";
9
+ import { Interpreter, type Caption } from "./live-interpreter.ts";
10
+ import { LiveVoice, LIVE_VOICE_LANGUAGES } from "./live-voice.ts";
11
+ import { SpeechError } from "./speech.ts";
12
+ import type { Translator } from "./translate.ts";
13
+ import type { TranslationMeter, TranslationUsage } from "./translation-passes.ts";
14
+
15
+ const blocked = new BlockList();
16
+ for (const [ip, bits] of [["0.0.0.0",8],["10.0.0.0",8],["100.64.0.0",10],["127.0.0.0",8],["169.254.0.0",16],["172.16.0.0",12],["192.0.0.0",24],["192.0.2.0",24],["192.168.0.0",16],["198.18.0.0",15],["198.51.100.0",24],["203.0.113.0",24],["224.0.0.0",4],["240.0.0.0",4]] as const) blocked.addSubnet(ip, bits, "ipv4");
17
+ const v6global = new BlockList(); v6global.addSubnet("2000::", 3, "ipv6");
18
+ blocked.addSubnet("2001:db8::", 32, "ipv6"); blocked.addSubnet("2002::", 16, "ipv6"); blocked.addSubnet("2001::", 32, "ipv6");
19
+ export function publicStreamAddress(address: string): boolean {
20
+ const family = isIP(address);
21
+ return family === 4 ? !blocked.check(address, "ipv4") : family === 6 && v6global.check(address, "ipv6") && !blocked.check(address, "ipv6");
22
+ }
23
+ export function sharedSource(raw: string): URL {
24
+ let url: URL;
25
+ try { url = new URL(raw); } catch { throw new SpeechError("Choose a live Nixamp channel.", 400); }
26
+ if (!["http:", "https:"].includes(url.protocol) || url.username || url.password || url.hash || !/^\/api\/channels\/[\w-]{1,80}$/.test(url.pathname) || [...url.searchParams.keys()].some(key => key !== "k")) throw new SpeechError("Choose a live Nixamp channel.", 400);
27
+ if (url.toString().length > 2048) throw new SpeechError("Invalid live source.", 400);
28
+ url.searchParams.sort(); return url;
29
+ }
30
+ export interface LiveInput { stop(): void; }
31
+ export type OpenLiveInput = (source: URL, pcm: (bytes: Buffer) => void, ended: () => void, signal: AbortSignal) => Promise<LiveInput>;
32
+
33
+ /** Resolve once and pin that public address; no redirects or ffmpeg URL fetches. */
34
+ export function liveInput(ffmpeg: string[] = ["ffmpeg"]): OpenLiveInput {
35
+ return async (source, pcm, ended, signal) => {
36
+ const hostname = source.hostname.replace(/^\[|\]$/g, "");
37
+ const addresses = await lookup(hostname, { all: true });
38
+ if (!addresses.length || addresses.some(one => !publicStreamAddress(one.address))) throw new SpeechError("This live source is not publicly reachable.", 400);
39
+ signal.throwIfAborted();
40
+ const selected = addresses[0]!;
41
+ const child = spawn(ffmpeg[0]!, [...ffmpeg.slice(1), "-v", "error", "-nostats", "-protocol_whitelist", "pipe", "-readrate", "1", "-analyzeduration", "500000", "-probesize", "262144", "-i", "pipe:0", "-vn", "-ac", "1", "-ar", "16000", "-f", "s16le", "pipe:1"], { stdio: ["pipe", "pipe", "ignore"] });
42
+ let stopped = false;
43
+ const request = (source.protocol === "https:" ? https : http).get(source, {
44
+ signal, headers: { "user-agent": "Nixamp shared translated audio", accept: "audio/*,video/*,application/octet-stream" },
45
+ lookup: (_name, options, callback) => { if (typeof options === "object" && options.all) callback(null, [selected]); else callback(null, selected.address, selected.family); },
46
+ });
47
+ const stop = (): void => { if (stopped) return; stopped = true; request.destroy(); child.stdin.destroy(); child.kill("SIGKILL"); signal.removeEventListener("abort", stop); };
48
+ const end = (): void => { if (!stopped) { stop(); ended(); } };
49
+ signal.addEventListener("abort", stop, { once: true });
50
+ child.stdout.on("data", pcm); child.on("error", end); child.on("close", end); child.stdin.on("error", end);
51
+ request.on("response", response => {
52
+ if (response.statusCode !== 200) { response.destroy(); end(); return; }
53
+ response.on("error", end); response.on("end", end); response.pipe(child.stdin);
54
+ });
55
+ request.on("error", end);
56
+ request.setTimeout(15_000, end);
57
+ return { stop };
58
+ };
59
+ }
60
+
61
+ export type SharedEvent = { type: "status"; listeners: number } | { type: "line"; id: string; line: Caption } | { type: "audio"; id: string; data: string } | { type: "end"; id: string } | { type: "error"; error: string };
62
+ type Member = { by: string; send: (event: SharedEvent) => void; close: () => void };
63
+ interface Feed { id: string; source: URL; language: string; members: Set<Member>; controller: AbortController; input?: LiveInput; interpreter: Interpreter; meter: TranslationMeter; reservations: Map<string, string[]>; queue: Caption[]; speaking: boolean; previous: Buffer; pending: Buffer; until: number; heartbeat?: ReturnType<typeof setInterval>; }
64
+
65
+ export class SharedTranslations {
66
+ private readonly feeds = new Map<string, Feed>();
67
+ constructor(private readonly options: { voice: LiveVoice; translator: Pick<Translator, "translate">; billing?: TranslationMeter; open?: OpenLiveInput }) {}
68
+ get size(): number { return this.feeds.size; }
69
+
70
+ async join(raw: string, language: string, by: string, send: Member["send"], close: () => void): Promise<() => void> {
71
+ if (!this.options.voice.available()) throw new SpeechError("Translated audio is unavailable.", 503);
72
+ const source = sharedSource(raw);
73
+ if (!LIVE_VOICE_LANGUAGES.has(language)) throw new SpeechError("Choose a supported audio language.", 400);
74
+ await this.options.billing?.require(by);
75
+ const connections = [...this.feeds.values()].reduce((total, feed) => total + [...feed.members].filter(member => member.by === by).length, 0);
76
+ if (connections >= 2) throw new SpeechError("Translated audio is already open in two players for this account.", 429);
77
+ const id = `dub-${createHash("sha256").update(`${source}|${language}`).digest("hex").slice(0, 40)}`;
78
+ let feed = this.feeds.get(id);
79
+ if (!feed) {
80
+ if (this.feeds.size >= 4) throw new SpeechError("Live translation is busy. Try again shortly.", 429);
81
+ feed = this.make(id, source, language); this.feeds.set(id, feed);
82
+ const current = feed;
83
+ current.heartbeat = setInterval(() => void this.checkMembers(current), 10_000); current.heartbeat.unref?.();
84
+ }
85
+ if (feed.members.size >= 1000) throw new SpeechError("This translated stream is full. Try again shortly.", 429);
86
+ const member = { by, send, close }; feed.members.add(member);
87
+ this.broadcast(feed, { type: "status", listeners: feed.members.size });
88
+ if (feed.members.size === 1 && !feed.input) void this.start(feed);
89
+ const current = feed;
90
+ return () => this.leave(current, member);
91
+ }
92
+ private leave(feed: Feed, member: Member): void {
93
+ if (!feed.members.delete(member)) return;
94
+ if (!feed.members.size) {
95
+ if (feed.heartbeat) clearInterval(feed.heartbeat);
96
+ feed.controller.abort(); feed.input?.stop(); feed.interpreter.reset(); feed.queue = [];
97
+ this.feeds.delete(feed.id);
98
+ } else this.broadcast(feed, { type: "status", listeners: feed.members.size });
99
+ }
100
+ private async checkMembers(feed: Feed): Promise<void> {
101
+ const accounts = [...new Set([...feed.members].map(member => member.by))];
102
+ let funded = new Set(accounts), message = "Your translation balance is used up or expired. Buy another pass to continue.";
103
+ try {
104
+ if (this.options.billing?.eligible) funded = new Set(await this.options.billing.eligible(accounts));
105
+ else if (this.options.billing) {
106
+ const results = await Promise.allSettled(accounts.map(by => this.options.billing!.require(by)));
107
+ funded = new Set(accounts.filter((_by, index) => results[index]?.status === "fulfilled"));
108
+ }
109
+ } catch { funded.clear(); message = "Your translation balance could not be checked. Try again shortly."; }
110
+ for (const member of [...feed.members]) if (accounts.includes(member.by) && !funded.has(member.by)) {
111
+ try { member.send({ type: "error", error: message }); } catch { /* disconnected */ }
112
+ finally { this.leave(feed, member); member.close(); }
113
+ }
114
+ }
115
+ private broadcast(feed: Feed, event: SharedEvent): void {
116
+ for (const member of [...feed.members]) {
117
+ try { member.send(event); } catch { this.leave(feed, member); member.close(); }
118
+ }
119
+ }
120
+ private stop(feed: Feed, error: string): void {
121
+ this.broadcast(feed, { type: "error", error });
122
+ for (const member of [...feed.members]) { this.leave(feed, member); member.close(); }
123
+ }
124
+ private make(id: string, source: URL, language: string): Feed {
125
+ const feed = { id, source, language, members: new Set<Member>(), controller: new AbortController(), reservations: new Map<string, string[]>(), queue: [], speaking: false, previous: Buffer.alloc(0), pending: Buffer.alloc(0), until: Date.now() } as unknown as Feed;
126
+ // Every active account pays the same published access rate. API work is
127
+ // done once; additional listeners do not trigger recognition or synthesis.
128
+ feed.meter = {
129
+ require: async () => { if (!feed.members.size) throw new SpeechError("No listeners.", 410); },
130
+ reserve: async (_by: string, kind: TranslationUsage, units: number) => {
131
+ const ids: string[] = [];
132
+ const accounts = [...new Set([...feed.members].map(member => member.by))];
133
+ const accepted = new Set<string>();
134
+ const billing = this.options.billing;
135
+ if (billing?.reserveMany) {
136
+ for (const charged of await billing.reserveMany(accounts, kind, units)) { ids.push(charged.id); accepted.add(charged.by); }
137
+ } else {
138
+ const results = await Promise.allSettled(accounts.map(async account => ({ by: account, id: await billing?.reserve(account, kind, units) })));
139
+ let failure: unknown;
140
+ for (const result of results) {
141
+ if (result.status === "fulfilled") { accepted.add(result.value.by); if (result.value.id) ids.push(result.value.id); }
142
+ else if (!(result.reason instanceof SpeechError) || result.reason.status !== 402) failure = result.reason;
143
+ }
144
+ if (failure) { await Promise.all(ids.map(id => billing!.refund(id))); throw failure; }
145
+ }
146
+ for (const member of [...feed.members]) if (accounts.includes(member.by) && !accepted.has(member.by)) {
147
+ try { member.send({ type: "error", error: "Your translation balance is used up or expired. Buy another pass to continue." }); }
148
+ catch { /* disconnected */ } finally { this.leave(feed, member); member.close(); }
149
+ }
150
+ if (!feed.members.size) { await Promise.all(ids.map(id => this.options.billing!.refund(id))); throw new SpeechError("No funded listeners.", 402); }
151
+ const reservation = randomUUID(); feed.reservations.set(reservation, ids); return reservation;
152
+ },
153
+ commit: async id => { const ids = feed.reservations.get(id) ?? []; if (this.options.billing?.commitMany) await this.options.billing.commitMany(ids); else await Promise.all(ids.map(id => this.options.billing!.commit(id))); feed.reservations.delete(id); },
154
+ refund: async id => { const ids = feed.reservations.get(id) ?? []; if (this.options.billing?.refundMany) await this.options.billing.refundMany(ids); else await Promise.all(ids.map(id => this.options.billing!.refund(id))); feed.reservations.delete(id); },
155
+ };
156
+ let voices: Awaited<ReturnType<LiveVoice["voices"]>> = [];
157
+ feed.interpreter = new Interpreter({
158
+ language: () => language, speakers: () => true, voices: () => voices, channel: () => id,
159
+ status: text => { if (!feed.controller.signal.aborted && !feed.members.size) this.stop(feed, text); },
160
+ failed: () => this.stop(feed, "Live translation stopped. Enable it again to retry."),
161
+ lines: lines => {
162
+ feed.queue.push(...lines);
163
+ if (feed.queue.length > 24) { this.stop(feed, "Live translation fell behind. Enable it again to retry."); return; }
164
+ if (!feed.speaking) void this.speak(feed);
165
+ },
166
+ fetcher: (async (url, init) => {
167
+ feed.controller.signal.throwIfAborted();
168
+ const signal = AbortSignal.any([feed.controller.signal, ...(init?.signal ? [init.signal] : [])]);
169
+ if (String(url).includes("/speakers")) {
170
+ voices = await this.options.voice.voices();
171
+ return Response.json(await this.options.voice.hear(new Uint8Array(init?.body as Uint8Array), id, signal, feed.meter));
172
+ }
173
+ const body = JSON.parse(String(init?.body));
174
+ return Response.json(await this.options.translator.translate(body.texts, body.from, body.to, { by: id, deadline: Date.now() + 10_000 }));
175
+ }) as typeof fetch,
176
+ });
177
+ return feed;
178
+ }
179
+ private async start(feed: Feed): Promise<void> {
180
+ try {
181
+ const input = await (this.options.open ?? liveInput())(feed.source, bytes => {
182
+ if (feed.controller.signal.aborted) return;
183
+ feed.pending = Buffer.concat([feed.pending, bytes]);
184
+ while (feed.pending.length >= 64_000) {
185
+ const fresh = feed.pending.subarray(0, 64_000); feed.pending = feed.pending.subarray(64_000);
186
+ const pcm = Buffer.concat([feed.previous, fresh]); feed.previous = pcm.subarray(-128_000);
187
+ feed.until += 2000;
188
+ if (Math.abs(Date.now() - feed.until) > 2000) feed.until = Date.now();
189
+ const samples = new Float32Array(pcm.length / 2);
190
+ for (let index = 0; index < samples.length; index++) samples[index] = pcm.readInt16LE(index * 2) / 32768;
191
+ feed.interpreter.push({ samples, at: feed.until - samples.length / 16, until: feed.until, freshAt: feed.until - 2000 });
192
+ }
193
+ }, () => this.stop(feed, "The live source stopped. Enable translated audio again when playback resumes."), feed.controller.signal);
194
+ if (feed.controller.signal.aborted) input.stop(); else feed.input = input;
195
+ } catch (error) { this.stop(feed, error instanceof SpeechError ? error.message : "The live source could not be opened."); }
196
+ }
197
+ private async speak(feed: Feed): Promise<void> {
198
+ feed.speaking = true;
199
+ try {
200
+ while (feed.queue.length && !feed.controller.signal.aborted) {
201
+ const line = feed.queue.shift()!;
202
+ if (Date.now() - line.until > 12_000) throw new SpeechError("Live translation fell behind. Enable it again to retry.", 503);
203
+ const response = await this.options.voice.stream({ text: line.text, language: feed.language, channel: feed.id, speaker: line.speaker, voice: feed.interpreter.tracker.speakers.get(line.speaker ?? "")?.voice }, feed.id, feed.controller.signal, feed.meter);
204
+ const id = randomUUID();
205
+ this.broadcast(feed, { type: "line", id, line });
206
+ const reader = response.body!.getReader();
207
+ try {
208
+ while (true) {
209
+ const { done, value } = await reader.read(); if (done) break;
210
+ for (let at = 0; at < value.length; at += 16_384) this.broadcast(feed, { type: "audio", id, data: Buffer.from(value.subarray(at, at + 16_384)).toString("base64") });
211
+ }
212
+ } finally { reader.releaseLock(); }
213
+ this.broadcast(feed, { type: "end", id });
214
+ }
215
+ } catch (error) { if (!feed.controller.signal.aborted) this.stop(feed, error instanceof SpeechError ? error.message : "Translated audio was interrupted."); }
216
+ finally { feed.speaking = false; }
217
+ }
218
+ }