nixamp 0.28.44 → 0.29.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (89) hide show
  1. package/dist/audio.js +3 -0
  2. package/dist/channels.d.ts +14 -0
  3. package/dist/channels.js +16 -3
  4. package/dist/decoder-policy.d.ts +20 -0
  5. package/dist/decoder-policy.js +34 -0
  6. package/dist/destinations.d.ts +5 -0
  7. package/dist/destinations.js +82 -0
  8. package/dist/library.d.ts +2 -0
  9. package/dist/library.js +4 -1
  10. package/dist/links.d.ts +4 -0
  11. package/dist/links.js +20 -2
  12. package/dist/live-interpreter.d.ts +1 -72
  13. package/dist/live-interpreter.js +1 -269
  14. package/dist/live-recovery.d.ts +1 -6
  15. package/dist/live-recovery.js +1 -11
  16. package/dist/live-voice.d.ts +6 -76
  17. package/dist/live-voice.js +6 -352
  18. package/dist/owner.js +4 -0
  19. package/dist/pcm-wav.d.ts +1 -2
  20. package/dist/pcm-wav.js +1 -27
  21. package/dist/playlist-input.d.ts +46 -0
  22. package/dist/playlist-input.js +158 -54
  23. package/dist/publish-choice.d.ts +21 -0
  24. package/dist/publish-choice.js +17 -0
  25. package/dist/server.js +66 -11
  26. package/dist/share.d.ts +1 -1
  27. package/dist/share.js +5 -4
  28. package/dist/shared-translation.d.ts +6 -53
  29. package/dist/shared-translation.js +7 -336
  30. package/dist/speaker-turns.d.ts +1 -32
  31. package/dist/speaker-turns.js +1 -30
  32. package/dist/speech.d.ts +3 -5
  33. package/dist/speech.js +4 -7
  34. package/dist/translation-passes.d.ts +5 -125
  35. package/dist/translation-passes.js +5 -294
  36. package/dist/upgrade-allowance.d.ts +1 -23
  37. package/dist/upgrade-allowance.js +1 -73
  38. package/dist/voice-profile.d.ts +1 -7
  39. package/dist/voice-profile.js +1 -53
  40. package/dist/voice-provider.d.ts +1 -9
  41. package/dist/voice-provider.js +1 -27
  42. package/package.json +2 -1
  43. package/src/audio.ts +3 -0
  44. package/src/channels.ts +26 -4
  45. package/src/decoder-policy.ts +33 -0
  46. package/src/destinations.ts +78 -0
  47. package/src/library.ts +6 -1
  48. package/src/links.ts +23 -4
  49. package/src/live-interpreter.ts +1 -213
  50. package/src/live-recovery.ts +1 -10
  51. package/src/live-voice.ts +6 -304
  52. package/src/owner.ts +3 -0
  53. package/src/pcm-wav.ts +1 -26
  54. package/src/playlist-input.ts +172 -50
  55. package/src/publish-choice.ts +35 -0
  56. package/src/server.ts +58 -11
  57. package/src/share.ts +5 -4
  58. package/src/shared-translation.ts +7 -222
  59. package/src/speaker-turns.ts +1 -34
  60. package/src/speech.ts +4 -5
  61. package/src/translation-passes.ts +6 -292
  62. package/src/upgrade-allowance.ts +1 -72
  63. package/src/voice-profile.ts +1 -46
  64. package/src/voice-provider.ts +1 -22
  65. package/web/dist/assets/{ar-CLNm1oA7.js → ar-D6_ZFDan.js} +1 -1
  66. package/web/dist/assets/{cs-Cptn8RbB.js → cs-DEPGOGEV.js} +1 -1
  67. package/web/dist/assets/{da-CRbNCFOu.js → da-CQw7SPC9.js} +1 -1
  68. package/web/dist/assets/{de-D4DnC5NQ.js → de-OWnobbah.js} +1 -1
  69. package/web/dist/assets/{es-DAHnD9bs.js → es-qMpyH9bM.js} +1 -1
  70. package/web/dist/assets/{fi-Csbl41Vz.js → fi-DpqqwJTD.js} +1 -1
  71. package/web/dist/assets/{fr-jZYQJed8.js → fr-BiO_ceen.js} +1 -1
  72. package/web/dist/assets/{hi-DjT9u0v_.js → hi-D-q5wZuY.js} +1 -1
  73. package/web/dist/assets/{hls-3VKVEQE3-BjAVnvGY.js → hls-3VKVEQE3-D2W4K8VR.js} +1 -1
  74. package/web/dist/assets/{hu-CVrU8gC4.js → hu-FIjHyhVC.js} +1 -1
  75. package/web/dist/assets/{id-B_gN24X5.js → id-gmB6lu56.js} +1 -1
  76. package/web/dist/assets/{index-BueBHHcv.css → index-Bo3utcDw.css} +1 -1
  77. package/web/dist/assets/index-UmGvzuun.js +22 -0
  78. package/web/dist/assets/{it-BUC1dMpV.js → it-Dxfo713A.js} +1 -1
  79. package/web/dist/assets/{mpegts-CnqmXTu2.js → mpegts-BT4XGr_E.js} +1 -1
  80. package/web/dist/assets/{mpegts-LO6RVLD6-B4QgG3Xb.js → mpegts-LO6RVLD6-CpZnlq0W.js} +1 -1
  81. package/web/dist/assets/{nl-CGi-aIrz.js → nl-mk5eN3Io.js} +1 -1
  82. package/web/dist/assets/{ru-YIbS63cK.js → ru-B2psU0kN.js} +1 -1
  83. package/web/dist/assets/{sv-CuJUpOCk.js → sv-Cg6waoNZ.js} +1 -1
  84. package/web/dist/assets/{uk-DLfG8YhV.js → uk-BcFkukLd.js} +1 -1
  85. package/web/dist/assets/{vi-nv2siY2T.js → vi-CnqIC1Sz.js} +1 -1
  86. package/web/dist/assets/{zh-P5x8w1OW.js → zh-CvnXIFvN.js} +1 -1
  87. package/web/dist/index.html +7 -12
  88. package/web/dist/sw.js +23 -23
  89. package/web/dist/assets/index-c3uxZS8_.js +0 -3
package/src/live-voice.ts CHANGED
@@ -1,306 +1,8 @@
1
- /** ElevenLabs Flash for live captions. The key stays on the account server. */
2
- import { createHash, randomBytes } from "node:crypto";
3
- import { SpeechError, decodeWav, encodeWav, quietSamples } from "./speech.ts";
4
- import { speakerTurns, type ScribeResult, type SpeakerTranscript } from "./speaker-turns.ts";
5
- import type { VoiceProfile } from "./voice-profile.ts";
6
- import { Guard } from "./guard.ts";
7
- import type { TranslationMeter } from "./translation-passes.ts";
8
- import type { Queryable } from "./follows.ts";
9
- import { voiceProviderFailure } from "./voice-provider.ts";
1
+ /** ElevenLabs Flash/Scribe for live captions, from @profullstack/live-translate.
2
+ * nixamp keeps its `nxd_` grant tokens: captions.ts and older pages expect them. */
3
+ import { LiveVoice as Base } from "@profullstack/live-translate";
4
+ export { LIVE_VOICE_MODEL, LIVE_VOICE_RATE, LIVE_VOICE_LANGUAGES, type LiveVoiceChoice, type VoiceRequest } from "@profullstack/live-translate";
10
5
 
11
- export const LIVE_VOICE_MODEL = "eleven_flash_v2_5";
12
- export const LIVE_VOICE_RATE = 16_000;
13
- export const LIVE_VOICE_LANGUAGES = new Set("en ja zh de hi fr ko pt it es id nl tr fil pl sv bg ro ar cs el fi hr ms sk da ta uk ru hu no vi".split(" "));
14
- export interface LiveVoiceChoice { id: string; name: string; gender: string; language: string; }
15
- export interface VoiceRequest { text: string; language: string; voice?: string; profile?: VoiceProfile; channel?: string; speaker?: string; }
16
- const HEADERS = { "content-type": "audio/pcm", "cache-control": "no-store", "x-audio-sample-rate": String(LIVE_VOICE_RATE) };
17
- const budget = (value: number | undefined, fallback: number): number => Number.isFinite(value) && value! >= 0 ? Math.floor(value!) : fallback;
18
-
19
- export class LiveVoice {
20
- private readonly key: string;
21
- private readonly fetcher: typeof fetch;
22
- private readonly now: () => number;
23
- private catalog: Promise<LiveVoiceChoice[]> | null = null;
24
- private catalogUntil = 0;
25
- private readonly cache = new Map<string, { until: number; bytes: Uint8Array }>();
26
- private readonly pending = new Map<string, Promise<Uint8Array>>();
27
- private readonly usage = new Map<string, { minute: number; chars: number }>();
28
- private readonly charsPerMinute: number;
29
- private readonly requests: Guard;
30
- private readonly grants = new Map<string, { by: string; channel: string; expires: number; remaining: number }>();
31
- private readonly activeBy = new Map<string, number>();
32
- private readonly db?: Queryable;
33
- private readonly billing?: TranslationMeter;
34
- private schema: Promise<unknown> | null = null;
35
- private readonly dailyChars: number;
36
- private cleanupAt = 0;
37
- private readonly hearing = new Set<string>();
38
- private readonly dailyAudioSeconds: number;
39
- private readonly userDailyChars: number;
40
- private readonly userDailyAudioSeconds: number;
41
- private providerPause: { until: number; error: SpeechError } | null = null;
42
-
43
- private checkProvider(): void {
44
- if (this.providerPause && this.providerPause.until > this.now()) throw this.providerPause.error;
45
- }
46
-
47
- private async providerFailed(response: Response, operation: string): Promise<never> {
48
- const failure = await voiceProviderFailure(response);
49
- const until = this.now() + failure.cooldownMs;
50
- // Another in-flight response must not shorten an existing billing pause.
51
- if (!this.providerPause || this.providerPause.until < until) this.providerPause = { until, error: failure.error };
52
- console.error(JSON.stringify({ event: "voice_provider_failure", provider: "elevenlabs", operation, status: response.status, code: failure.code, cooldownMs: failure.cooldownMs }));
53
- throw failure.error;
54
- }
55
-
56
- constructor(options: { apiKey?: string; fetcher?: typeof fetch; now?: () => number; charsPerMinute?: number; dailyChars?: number; dailyAudioSeconds?: number; userDailyChars?: number; userDailyAudioSeconds?: number; db?: Queryable; billing?: TranslationMeter } = {}) {
57
- this.key = options.apiKey ?? process.env["ELEVENLABS_API_KEY"] ?? "";
58
- this.fetcher = options.fetcher ?? fetch;
59
- this.now = options.now ?? Date.now;
60
- this.charsPerMinute = budget(options.charsPerMinute, 3000);
61
- this.dailyChars = budget(options.dailyChars, 200_000);
62
- this.dailyAudioSeconds = budget(options.dailyAudioSeconds, 86_400);
63
- this.userDailyChars = budget(options.userDailyChars, 120_000);
64
- this.userDailyAudioSeconds = budget(options.userDailyAudioSeconds, 43_200);
65
- this.requests = new Guard(this.now);
66
- this.db = options.db;
67
- this.billing = options.billing;
68
- }
69
-
70
- available(): boolean { return this.key !== ""; }
71
-
72
- /** Optional diarization, billed only while a signed-in listener requests it.
73
- * Rolling audio is bounded to 15 seconds, including overlap. Every second
74
- * submitted (also repeated context) consumes the persistent provider budget. */
75
- async hear(bytes: Uint8Array, by: string, signal?: AbortSignal, meter = this.billing, resource = ""): Promise<SpeakerTranscript> {
76
- if (!this.available()) throw new SpeechError("speaker voices are unavailable", 503);
77
- await meter?.require(by, resource);
78
- this.checkProvider();
79
- const wav = decodeWav(bytes);
80
- const seconds = wav.samples.length / wav.rate;
81
- if (wav.rate !== 16000 || wav.channels !== 1 || seconds < 0.2 || seconds > 15.1) throw new SpeechError("send up to 15 seconds of mono 16 kHz WAV", 400);
82
- if (wav.samples.some(sample => !Number.isFinite(sample))) throw new SpeechError("invalid audio samples", 400);
83
- if (!this.requests.check(`hear:${by}`, { allowed: 36, windowMs: 60_000 }).ok) throw new SpeechError("too many speaker transcription requests", 429);
84
- if (quietSamples(wav.samples)) return { language: "", seconds, turns: [] };
85
- if (this.hearing.has(by) || this.hearing.size >= 4) throw new SpeechError("speaker transcription is busy", 429);
86
- this.hearing.add(by);
87
- let reservation: string | undefined;
88
- let accepted = false;
89
- try {
90
- const billed = Math.ceil(seconds);
91
- await this.reserve(`scribe:user:${by}`, billed, 300, 60_000);
92
- await this.reserve("scribe:server", billed, 600, 60_000);
93
- await this.reserve(`scribe:user:${by}`, billed, this.userDailyAudioSeconds, 86_400_000);
94
- await this.reserve("scribe:server", billed, this.dailyAudioSeconds, 86_400_000);
95
- signal?.throwIfAborted();
96
- this.checkProvider();
97
- const form = new FormData();
98
- // Canonical PCM prevents a crafted container from billing more audio
99
- // than the duration we validated, and removes uploaded metadata.
100
- form.set("file", new Blob([new Uint8Array(encodeWav(wav.samples))], { type: "audio/wav" }), "listening.wav");
101
- form.set("model_id", "scribe_v2");
102
- form.set("diarize", "true");
103
- form.set("tag_audio_events", "false");
104
- form.set("timestamps_granularity", "word");
105
- // No language_code: preserve the source language, including Spanish.
106
- reservation = await meter?.reserve(by, "transcription", wav.samples.length, resource);
107
- const answer = await this.fetcher("https://api.elevenlabs.io/v1/speech-to-text", {
108
- method: "POST", headers: { "xi-api-key": this.key }, body: form,
109
- signal: signal ? AbortSignal.any([signal, AbortSignal.timeout(10_000)]) : AbortSignal.timeout(10_000),
110
- });
111
- if (!answer.ok) await this.providerFailed(answer, "transcription");
112
- accepted = true;
113
- console.info(JSON.stringify({ event: "voice_provider_usage", provider: "elevenlabs", operation: "transcription", account: createHash("sha256").update(by).digest("hex").slice(0, 16), audioSeconds: seconds }));
114
- if (reservation) await meter!.commit(reservation);
115
- return speakerTurns(await answer.json() as ScribeResult, wav);
116
- } catch (error) {
117
- if (reservation && !accepted) await meter!.refund(reservation);
118
- throw error;
119
- } finally { this.hearing.delete(by); }
120
- }
121
-
122
- async voices(): Promise<LiveVoiceChoice[]> {
123
- if (!this.available()) throw new SpeechError("translated audio needs ELEVENLABS_API_KEY on the account server", 503);
124
- this.checkProvider();
125
- if (!this.catalog || this.now() >= this.catalogUntil) {
126
- this.catalogUntil = this.now() + 3600_000;
127
- this.catalog = this.fetcher("https://api.elevenlabs.io/v2/voices?page_size=100&voice_type=default", {
128
- headers: { "xi-api-key": this.key }, signal: AbortSignal.timeout(8000),
129
- }).then(async response => {
130
- if (!response.ok) await this.providerFailed(response, "voices");
131
- const body = await response.json() as { voices: { voice_id: string; name: string; labels?: Record<string, string> }[] };
132
- return body.voices.filter(voice => /^[a-zA-Z0-9_-]+$/.test(voice.voice_id)).map(voice => ({
133
- id: voice.voice_id, name: voice.name, gender: voice.labels?.["gender"] ?? "neutral", language: voice.labels?.["language"] ?? "",
134
- }));
135
- }).catch(error => { this.catalog = null; throw error; });
136
- }
137
- return this.catalog;
138
- }
139
-
140
- /** A 90-second capability for one channel, never the viewer's account credential. */
141
- async grant(by: string, channel: string): Promise<{ token: string; expires: number }> {
142
- if (!/^[\w-]{1,80}$/.test(channel)) throw new SpeechError("choose a playback session or live channel", 400);
143
- if (!this.requests.check(`grant:${by}`, { allowed: 10, windowMs: 60_000 }).ok) throw new SpeechError("too many audio authorization requests", 429);
144
- await this.billing?.require(by, channel);
145
- for (const [token, grant] of this.grants) if (grant.expires <= this.now()) this.grants.delete(token);
146
- if (this.grants.size >= 5000) throw new SpeechError("audio authorization is busy", 503);
147
- const token = `nxd_${randomBytes(32).toString("base64url")}`;
148
- const expires = this.now() + 90_000;
149
- if (this.db) {
150
- await this.ensure();
151
- await this.db.query("INSERT INTO live_voice_grants (token_hash, by_account, channel, expires_at, remaining) VALUES ($1, $2, $3, $4, 2000)", [createHash("sha256").update(token).digest("hex"), by, channel, new Date(expires)]);
152
- } else this.grants.set(token, { by, channel, expires, remaining: 2000 });
153
- return { token, expires };
154
- }
155
-
156
- async authorize(token: string, channel: string, chars: number): Promise<string> {
157
- if (!/^nxd_[A-Za-z0-9_-]{43}$/.test(token)) throw new SpeechError("sign in to enable translated audio", 401);
158
- if (!Number.isFinite(chars) || chars < 1 || chars > 600) throw new SpeechError("invalid caption length", 400);
159
- if (this.db) {
160
- await this.ensure();
161
- const result = await this.db.query(`UPDATE live_voice_grants SET remaining = remaining - $3
162
- WHERE token_hash = $1 AND channel = $2 AND expires_at > $4 AND remaining >= $3 RETURNING by_account`, [createHash("sha256").update(token).digest("hex"), channel, chars, new Date(this.now())]);
163
- if (!result.rows.length) throw new SpeechError("audio authorization expired or reached its limit; enable translated audio again", 401);
164
- return String(result.rows[0]?.["by_account"]);
165
- }
166
- const grant = this.grants.get(token);
167
- if (!grant || grant.expires <= this.now() || grant.channel !== channel) throw new SpeechError("sign in to renew translated audio", 401);
168
- if (grant.remaining < chars) throw new SpeechError("this audio authorization reached its character limit", 429);
169
- grant.remaining -= chars;
170
- return grant.by;
171
- }
172
-
173
- private async ensure(): Promise<void> {
174
- if (!this.db) return;
175
- this.schema ??= (async () => {
176
- await this.db!.query("CREATE TABLE IF NOT EXISTS live_voice_usage (bucket TEXT PRIMARY KEY, chars BIGINT NOT NULL, expires_at TIMESTAMPTZ NOT NULL)");
177
- await this.db!.query("CREATE TABLE IF NOT EXISTS live_voice_grants (token_hash TEXT PRIMARY KEY, by_account TEXT NOT NULL, channel TEXT NOT NULL, expires_at TIMESTAMPTZ NOT NULL, remaining INTEGER NOT NULL)");
178
- })().catch(error => { this.schema = null; throw error; });
179
- await this.schema;
180
- if (this.now() >= this.cleanupAt) {
181
- this.cleanupAt = this.now() + 60_000;
182
- // Keep only live windows; accounting survives restarts and is shared by replicas.
183
- await this.db.query("DELETE FROM live_voice_usage WHERE expires_at <= $1", [new Date(this.now())]);
184
- await this.db.query("DELETE FROM live_voice_grants WHERE expires_at <= $1", [new Date(this.now())]);
185
- }
186
- }
187
-
188
- private async reserve(bucket: string, chars: number, limit: number, windowMs: number): Promise<void> {
189
- if (chars > limit) throw new SpeechError("translated audio reached its character budget; try again later", 429);
190
- const window = Math.floor(this.now() / windowMs);
191
- const key = `${bucket}:${windowMs}:${window}`;
192
- if (this.db) {
193
- await this.ensure();
194
- const result = await this.db.query(`INSERT INTO live_voice_usage (bucket, chars, expires_at) VALUES ($1, $2, $4)
195
- ON CONFLICT (bucket) DO UPDATE SET chars = live_voice_usage.chars + EXCLUDED.chars
196
- WHERE live_voice_usage.chars + EXCLUDED.chars <= $3 RETURNING chars`, [key, chars, limit, new Date((window + 1) * windowMs)]);
197
- if (!result.rows.length) throw new SpeechError("translated audio reached its character budget; try again later", 429);
198
- return;
199
- }
200
- // Local/single-process installations without a database. Account servers pass their pool.
201
- const record = this.usage.get(key) ?? { minute: (window + 1) * windowMs, chars: 0 };
202
- if (record.chars + chars > limit) throw new SpeechError("translated audio reached its character budget; try again later", 429);
203
- record.chars += chars; this.usage.set(key, record);
204
- }
205
-
206
- private async charge(by: string, channel: string, chars: number): Promise<void> {
207
- for (const [key, record] of this.usage) if (record.minute <= this.now()) this.usage.delete(key);
208
- await this.reserve(`user:${by}`, chars, this.charsPerMinute, 60_000);
209
- await this.reserve(`channel:${channel}`, chars, 6000, 60_000);
210
- await this.reserve("server", chars, 12_000, 60_000);
211
- await this.reserve(`user:${by}`, chars, this.userDailyChars, 86400_000);
212
- await this.reserve("server", chars, this.dailyChars, 86400_000);
213
- }
214
-
215
- private checkRequest(by: string): void {
216
- if (!this.requests.check(`audio:${by}`, { allowed: 60, windowMs: 60_000 }).ok) throw new SpeechError("too many translated audio requests", 429);
217
- }
218
-
219
- /** Stream the first request immediately; concurrent listeners share its cached result. */
220
- async stream(ask: VoiceRequest, by: string, signal?: AbortSignal, meter = this.billing): Promise<Response> {
221
- this.checkRequest(by);
222
- await meter?.require(by, ask.channel);
223
- const text = typeof ask.text === "string" ? ask.text.trim() : "";
224
- if (!text || text.length > 600) throw new SpeechError("translated audio needs a caption of 1–600 characters", 400);
225
- if (!LIVE_VOICE_LANGUAGES.has(ask.language)) throw new SpeechError("this language is not supported by Flash voices", 400);
226
- const voices = await this.voices();
227
- signal?.throwIfAborted();
228
- // The browser chooses unique voices for speakers. Legacy callers get a
229
- // stable stock voice; pitch is not used to infer gender.
230
- const seed = createHash("sha256").update(`${ask.channel ?? ""}|${ask.speaker ?? ""}`).digest().readUInt32BE(0);
231
- const voice = ask.voice && ask.voice !== "auto"
232
- ? voices.find(voice => voice.id === ask.voice)
233
- : voices[seed % voices.length];
234
- if (!voice) throw new SpeechError("choose an available voice", 400);
235
- const id = createHash("sha256").update(JSON.stringify([LIVE_VOICE_MODEL, voice.id, ask.language, text])).digest("hex");
236
- for (const [key, item] of this.cache) if (item.until < this.now()) this.cache.delete(key);
237
- const cached = this.cache.get(id);
238
- const pending = this.pending.get(id);
239
- if (cached || pending) {
240
- const bytes = cached?.bytes ?? await pending!;
241
- signal?.throwIfAborted();
242
- const paid = await meter?.reserve(by, "voice", text.length, ask.channel);
243
- if (paid) await meter!.commit(paid);
244
- return new Response(new Uint8Array(bytes), { headers: HEADERS });
245
- }
246
- if (this.pending.size >= 4) throw new SpeechError("translated audio is busy; waiting for the next caption", 429);
247
- if ((this.activeBy.get(by) ?? 0) >= 2) throw new SpeechError("two voice requests are already active for this account", 429);
248
- // Register before the first provider await, so identical requests cannot both bill.
249
- let done!: (bytes: Uint8Array) => void;
250
- let fail!: (error: unknown) => void;
251
- const finished = new Promise<Uint8Array>((resolve, reject) => { done = resolve; fail = reject; });
252
- this.pending.set(id, finished);
253
- this.activeBy.set(by, (this.activeBy.get(by) ?? 0) + 1);
254
- const release = (): void => { this.pending.delete(id); this.activeBy.set(by, Math.max(0, (this.activeBy.get(by) ?? 1) - 1)); if (!this.activeBy.get(by)) this.activeBy.delete(by); };
255
- void finished.catch(() => undefined);
256
- let reservation: string | undefined;
257
- let accepted = false;
258
- try {
259
- await this.charge(by, ask.channel ?? "direct", text.length);
260
- this.checkProvider();
261
- signal?.throwIfAborted();
262
- reservation = await meter?.reserve(by, "voice", text.length, ask.channel);
263
- const response = await this.fetcher(`https://api.elevenlabs.io/v1/text-to-speech/${voice.id}/stream?output_format=pcm_16000`, {
264
- method: "POST",
265
- headers: { "xi-api-key": this.key, "content-type": "application/json" },
266
- body: JSON.stringify({ text, model_id: LIVE_VOICE_MODEL, language_code: ask.language }),
267
- signal: signal ? AbortSignal.any([signal, AbortSignal.timeout(10_000)]) : AbortSignal.timeout(10_000),
268
- });
269
- if (!response.ok) await this.providerFailed(response, "synthesis");
270
- if (!response.body) throw new SpeechError("ElevenLabs returned no audio", 502);
271
- // Once the provider accepts, aborting playback cannot refund heard audio.
272
- accepted = true;
273
- console.info(JSON.stringify({ event: "voice_provider_usage", provider: "elevenlabs", operation: "synthesis", account: createHash("sha256").update(by).digest("hex").slice(0, 16), characters: text.length }));
274
- if (reservation) await meter!.commit(reservation);
275
- const [play, keep] = response.body.tee();
276
- void (async () => {
277
- const reader = keep.getReader();
278
- const chunks: Uint8Array[] = [];
279
- let size = 0;
280
- try {
281
- while (true) {
282
- const { done: ended, value } = await reader.read();
283
- if (ended) break;
284
- size += value.length;
285
- if (size > 2 * 1024 * 1024) { await reader.cancel(); throw new SpeechError("voice response was too long", 502); }
286
- chunks.push(value);
287
- }
288
- if (!size || size % 2) throw new SpeechError("voice response contained incomplete audio", 502);
289
- const bytes = new Uint8Array(size);
290
- let at = 0;
291
- for (const chunk of chunks) { bytes.set(chunk, at); at += chunk.length; }
292
- this.cache.set(id, { until: this.now() + 60_000, bytes });
293
- // At most a few MB for recent lines, never a recording archive.
294
- while (this.cache.size > 32) this.cache.delete(this.cache.keys().next().value as string);
295
- done(bytes);
296
- } catch (error) { fail(error); }
297
- finally { reader.releaseLock(); release(); }
298
- })();
299
- return new Response(play, { headers: HEADERS });
300
- } catch (error) {
301
- fail(error); release();
302
- if (reservation && !accepted) await meter!.refund(reservation);
303
- throw error;
304
- }
305
- }
6
+ export class LiveVoice extends Base {
7
+ constructor(options: ConstructorParameters<typeof Base>[0] = {}) { super({ tokenPrefix: "nxd_", ...options }); }
306
8
  }
package/src/owner.ts CHANGED
@@ -130,6 +130,9 @@ export function needsAdmin(path: string, method = "GET"): boolean {
130
130
  // Going live with a file on this server is the same act as going live
131
131
  // with a catalog entry, and the same question.
132
132
  if (/^\/api\/tracks\/\d+\/live$/.test(path)) return true;
133
+ // And so is a folder: without this the gate never ran for it, and a
134
+ // listen link alone could put any folder on the air, uncapped.
135
+ if (path === "/api/folders/live") return true;
133
136
  return false;
134
137
  }
135
138
 
package/src/pcm-wav.ts CHANGED
@@ -1,26 +1 @@
1
- /** 16-bit mono PCM WAV bytes from samples in [-1, 1]. */
2
- export function encodeWav(samples: Float32Array, rate = 16000): Uint8Array {
3
- const bytes = new Uint8Array(44 + samples.length * 2);
4
- const view = new DataView(bytes.buffer);
5
- const ascii = (at: number, text: string): void => {
6
- for (let i = 0; i < text.length; i++) bytes[at + i] = text.charCodeAt(i);
7
- };
8
- ascii(0, "RIFF");
9
- view.setUint32(4, 36 + samples.length * 2, true);
10
- ascii(8, "WAVE");
11
- ascii(12, "fmt ");
12
- view.setUint32(16, 16, true);
13
- view.setUint16(20, 1, true);
14
- view.setUint16(22, 1, true);
15
- view.setUint32(24, rate, true);
16
- view.setUint32(28, rate * 2, true);
17
- view.setUint16(32, 2, true);
18
- view.setUint16(34, 16, true);
19
- ascii(36, "data");
20
- view.setUint32(40, samples.length * 2, true);
21
- for (let i = 0; i < samples.length; i++) {
22
- const clipped = Math.max(-1, Math.min(1, samples[i] as number));
23
- view.setInt16(44 + i * 2, clipped < 0 ? clipped * 32768 : clipped * 32767, true);
24
- }
25
- return bytes;
26
- }
1
+ export { encodeWav } from "@profullstack/live-translate/core";
@@ -1,4 +1,5 @@
1
1
  import { spawn } from "node:child_process";
2
+ import { protocolArgs } from "./decoder-policy.ts";
2
3
  import type { Readable } from "node:stream";
3
4
  import { codecsOf } from "./audio.ts";
4
5
 
@@ -11,24 +12,72 @@ export interface PlaylistInput {
11
12
  video: boolean;
12
13
  width?: number;
13
14
  height?: number;
15
+ /**
16
+ * Back to the top when the end is reached, instead of finishing: a
17
+ * station, which plays until somebody stops it.
18
+ */
19
+ repeat?: boolean;
20
+ /**
21
+ * The order to play a list of this length in, asked at the top of every
22
+ * pass: a station-wide shuffle is one order for everybody listening, drawn
23
+ * again each time round. Unset is the list's own order.
24
+ */
25
+ order?: (length: number) => number[];
26
+ /** Read at every entry boundary, so an edit to the list lands on the next track. */
27
+ current?: () => string[];
28
+ /** Entries already known not to play, passed over without being tried. */
29
+ excluded?: (file: string) => boolean;
30
+ /**
31
+ * Pass over an entry that will not play, instead of failing the stream.
32
+ * For a station, where the next track is what a listener should hear. A
33
+ * folder live leaves it unset: there a missing entry is usually a file
34
+ * still arriving, and the channel redials and waits for it.
35
+ */
36
+ passOver?: boolean;
37
+ /** Told when an entry is passed over because it would not play. */
38
+ onFailure?: (event: { at: number; file: string; reason: string }) => void;
14
39
  }
15
40
 
41
+ /**
42
+ * How many entries in a row may fail before the list is given up on. A
43
+ * corrupt file is skipped; a list where nothing plays is an error, said once,
44
+ * not a loop.
45
+ */
46
+ export const MAX_FAILURES_IN_A_ROW = 5;
47
+
16
48
  /** Decode each file independently; feed one persistent muxer a fixed stream
17
49
  * profile and continuous timestamps. A concat demuxer cannot decode a list
18
50
  * whose input codecs differ. Only one entry is encoded at a time, with pipe
19
51
  * backpressure bounding read-ahead independently of the directory's size. */
20
52
  export class PlaylistInputStream {
21
- private entries: { at: number; start: number; position: number }[] = [];
53
+ private entries: { at: number; file: string; start: number; position: number }[] = [];
22
54
  error: Error | null = null;
23
55
  complete = false;
24
56
  constructor(private readonly options: PlaylistInput) {}
25
57
 
58
+ /** Every entry that has started, in the order it started, with where it sits in the stream. */
59
+ get history(): ReadonlyArray<{ at: number; file: string; start: number; position: number }> {
60
+ return this.entries;
61
+ }
62
+
26
63
  place(seconds: number): { at: number; position: number } {
27
64
  const entry = this.entries.findLast(one => seconds >= one.start) ?? this.entries[0];
28
65
  return entry ? { at: entry.at, position: Math.max(0, seconds - entry.start + entry.position) }
29
66
  : { at: this.options.at, position: this.options.position };
30
67
  }
31
68
 
69
+ /** The decoder for the entry that is on, so skip() can end it early. */
70
+ private stopCurrent: (() => void) | null = null;
71
+ private skipping = false;
72
+
73
+ /** End the entry that is on and go to the next. False when nothing is on. */
74
+ skip(): boolean {
75
+ if (!this.stopCurrent) return false;
76
+ this.skipping = true;
77
+ this.stopCurrent();
78
+ return true;
79
+ }
80
+
32
81
  async *open(signal: AbortSignal): AsyncGenerator<Buffer> {
33
82
  const o = this.options;
34
83
  const [command, ...prefix] = o.ffmpeg as [string, ...string[]];
@@ -36,59 +85,132 @@ export class PlaylistInputStream {
36
85
  const width = Math.max(2, Math.floor((o.width || 1280) * ratio / 2) * 2);
37
86
  const height = Math.max(2, Math.floor((o.height || 720) * ratio / 2) * 2);
38
87
  let offset = 0;
88
+ let failuresInARow = 0;
89
+ let first = true;
39
90
  try {
40
- for (let at = o.at; at < o.files.length && !signal.aborted; at++) {
41
- const file = o.files[at]!;
42
- const codecs = await codecsOf({ ffmpeg: o.ffmpeg, ffprobe: o.ffprobe, play: null }, file, [], AbortSignal.any([signal, AbortSignal.timeout(10_000)]));
43
- if (signal.aborted) return;
44
- if (!codecs.audio && !codecs.video) throw new Error(`Cannot decode playlist entry ${at + 1}`);
45
- const position = at === o.at ? o.position : 0;
46
- const duration = (codecs.duration ?? 0) - position;
47
- if (duration <= 0) throw new Error(`No playable duration for playlist entry ${at + 1}`);
48
- this.entries.push({ at, start: offset, position });
49
- const hasVideo = Boolean(codecs.video);
50
- const extra = o.video && !hasVideo
51
- ? ["-f", "lavfi", "-i", `color=c=black:s=${width}x${height}:r=30`]
52
- : o.video && !codecs.audio ? ["-f", "lavfi", "-i", "anullsrc=r=48000:cl=stereo"] : [];
53
- const args = [
54
- ...prefix, "-hide_banner", "-loglevel", "error", "-nostdin", "-filter_threads", "2",
55
- "-threads", "2", ...(position > 0 ? ["-ss", String(position)] : []), "-i", file,
56
- ...extra, "-t", String(duration),
57
- ...(o.video ? [
58
- "-map", hasVideo ? "0:v:0" : "1:v:0", "-map", codecs.audio ? "0:a:0" : "1:a:0",
59
- "-vf", `fps=30,scale=${width}:${height}:force_original_aspect_ratio=decrease,pad=${width}:${height}:(ow-iw)/2:(oh-ih)/2,setsar=1,setpts=PTS-STARTPTS`,
60
- "-c:v", "libx264", "-threads", "2", "-preset", "veryfast", "-tune", "zerolatency",
61
- "-crf", "23", "-pix_fmt", "yuv420p", "-g", "60", "-keyint_min", "60", "-sc_threshold", "0",
62
- ] : ["-map", "0:a:0", "-vn"]),
63
- "-af", "aresample=48000,asetpts=PTS-STARTPTS", "-c:a", "aac", "-b:a", "160k", "-ac", "2",
64
- "-mpegts_flags", "+initial_discontinuity", "-output_ts_offset", String(offset), "-mpegts_copyts", "1", "-muxdelay", "0", "-muxpreload", "0",
65
- "-f", "mpegts", "pipe:1",
66
- ];
67
- const child = spawn(command, args, { stdio: ["ignore", "pipe", "pipe"] });
68
- let tail = "";
69
- child.stderr.on("data", chunk => { tail = (tail + chunk.toString()).slice(-1500); });
70
- const closed = new Promise<number | null>((resolve) => {
71
- child.once("error", error => { tail = error.message; resolve(-1); });
72
- child.once("close", code => resolve(code));
73
- });
74
- const abort = (): void => { child.kill("SIGKILL"); };
75
- signal.addEventListener("abort", abort, { once: true });
76
- try {
77
- for await (const chunk of child.stdout as Readable) {
91
+ const orderFor = (list: string[]): number[] =>
92
+ o.order ? o.order(list.length).filter((one) => one >= 0 && one < list.length) : list.map((_, index) => index);
93
+ let files = o.current?.() ?? o.files;
94
+ if (files.length === 0) throw new Error("The playlist is empty");
95
+ let order = orderFor(files);
96
+ let step = Math.min(o.at, order.length);
97
+ let played = 0;
98
+ let last = "";
99
+ for (;;) {
100
+ if (signal.aborted) break;
101
+ // Asked at every boundary, so an edit lands on the next track. The
102
+ // list goes on after the file that just played, where the new list
103
+ // has it; a shuffled list draws a new order.
104
+ if (o.current) {
105
+ const fresh = o.current();
106
+ if (fresh.length !== files.length || fresh.some((one, index) => one !== files[index])) {
107
+ if (fresh.length === 0) throw new Error("The playlist is empty");
108
+ files = fresh;
109
+ order = orderFor(files);
110
+ const where = last ? files.indexOf(last) : -1;
111
+ step = o.order ? 0 : where >= 0 ? where + 1 : Math.min(step, files.length);
112
+ }
113
+ }
114
+ if (step >= order.length) {
115
+ if (!o.repeat) break;
116
+ // A pass where everything was passed over is a list with nothing
117
+ // to play, and repeating it would be the loop this exists to avoid.
118
+ if (played === 0) throw new Error("Nothing in this playlist would play");
119
+ played = 0;
120
+ order = orderFor(files);
121
+ step = 0;
122
+ continue;
123
+ }
124
+ {
125
+ const at = order[step]!;
126
+ step += 1;
127
+ const file = files[at]!;
128
+ last = file;
129
+ if (o.excluded?.(file)) continue;
130
+ const position = first && step - 1 === o.at ? o.position : 0;
131
+ first = false;
132
+ let reached = 0;
133
+ try {
134
+ const codecs = await codecsOf({ ffmpeg: o.ffmpeg, ffprobe: o.ffprobe, play: null }, file, [], AbortSignal.any([signal, AbortSignal.timeout(10_000)]));
78
135
  if (signal.aborted) return;
79
- yield chunk as Buffer;
136
+ if (!codecs.audio && !codecs.video) throw new Error(`Cannot decode playlist entry ${at + 1}`);
137
+ const duration = (codecs.duration ?? 0) - position;
138
+ if (duration <= 0) throw new Error(`No playable duration for playlist entry ${at + 1}`);
139
+ this.entries.push({ at, file, start: offset, position });
140
+ const hasVideo = Boolean(codecs.video);
141
+ const extra = o.video && !hasVideo
142
+ ? ["-f", "lavfi", "-i", `color=c=black:s=${width}x${height}:r=30`]
143
+ : o.video && !codecs.audio ? ["-f", "lavfi", "-i", "anullsrc=r=48000:cl=stereo"] : [];
144
+ const args = [
145
+ ...prefix, "-hide_banner", "-loglevel", "error", "-nostdin", "-filter_threads", "2",
146
+ // How far this entry got, so an entry cut short -- skipped, or
147
+ // broken halfway -- moves the clock by what was played and the
148
+ // next one starts where this one stopped, not after a gap.
149
+ "-progress", "pipe:3", "-stats_period", "0.5",
150
+ "-threads", "2", ...(position > 0 ? ["-ss", String(position)] : []), ...protocolArgs(file), "-i", file,
151
+ ...extra, "-t", String(duration),
152
+ ...(o.video ? [
153
+ "-map", hasVideo ? "0:v:0" : "1:v:0", "-map", codecs.audio ? "0:a:0" : "1:a:0",
154
+ "-vf", `fps=30,scale=${width}:${height}:force_original_aspect_ratio=decrease,pad=${width}:${height}:(ow-iw)/2:(oh-ih)/2,setsar=1,setpts=PTS-STARTPTS`,
155
+ "-c:v", "libx264", "-threads", "2", "-preset", "veryfast", "-tune", "zerolatency",
156
+ "-crf", "23", "-pix_fmt", "yuv420p", "-g", "60", "-keyint_min", "60", "-sc_threshold", "0",
157
+ ] : ["-map", "0:a:0", "-vn"]),
158
+ "-af", "aresample=48000,asetpts=PTS-STARTPTS", "-c:a", "aac", "-b:a", "160k", "-ac", "2",
159
+ "-mpegts_flags", "+initial_discontinuity", "-output_ts_offset", String(offset), "-mpegts_copyts", "1", "-muxdelay", "0", "-muxpreload", "0",
160
+ "-f", "mpegts", "pipe:1",
161
+ ];
162
+ const child = spawn(command, args, { stdio: ["ignore", "pipe", "pipe", "pipe"] });
163
+ let tail = "";
164
+ child.stderr!.on("data", chunk => { tail = (tail + chunk.toString()).slice(-1500); });
165
+ (child.stdio[3] as Readable | null)?.on("data", (chunk: Buffer) => {
166
+ const found = /out_time_us=(\d+)/g;
167
+ let match: RegExpExecArray | null;
168
+ while ((match = found.exec(chunk.toString())) !== null) reached = Math.max(reached, Number(match[1]) / 1e6);
169
+ });
170
+ const closed = new Promise<number | null>((resolve) => {
171
+ child.once("error", error => { tail = error.message; resolve(-1); });
172
+ child.once("close", code => resolve(code));
173
+ });
174
+ const abort = (): void => { child.kill("SIGKILL"); };
175
+ signal.addEventListener("abort", abort, { once: true });
176
+ this.stopCurrent = abort;
177
+ let skipped = false;
178
+ try {
179
+ for await (const chunk of child.stdout as Readable) {
180
+ if (signal.aborted) return;
181
+ yield chunk as Buffer;
182
+ }
183
+ const code = await closed;
184
+ if (signal.aborted) return;
185
+ skipped = this.skipping;
186
+ if (code !== 0 && !skipped) throw new Error(`Playlist entry ${at + 1} failed: ${tail.trim()}`);
187
+ } finally {
188
+ this.stopCurrent = null;
189
+ this.skipping = false;
190
+ signal.removeEventListener("abort", abort);
191
+ abort();
192
+ await closed;
193
+ }
194
+ failuresInARow = 0;
195
+ played += 1;
196
+ // Match the frame grid used above so rounding cannot accumulate a
197
+ // backwards timestamp over hundreds of short files.
198
+ const length = skipped ? Math.min(duration, reached) : duration;
199
+ offset += o.video ? Math.ceil(length * 30) / 30 : length;
200
+ } catch (error) {
201
+ if (signal.aborted || !o.passOver) throw error;
202
+ // Passed over, not dialled again: the next entry is what a
203
+ // listener should hear, and a file that will not decode now will
204
+ // not decode in two seconds either.
205
+ const reason = error instanceof Error ? error.message : String(error);
206
+ o.onFailure?.({ at, file, reason });
207
+ if (reached > 0) offset += o.video ? Math.ceil(reached * 30) / 30 : reached;
208
+ failuresInARow += 1;
209
+ if (failuresInARow >= Math.min(MAX_FAILURES_IN_A_ROW, Math.max(1, files.length))) {
210
+ throw new Error(`Nothing in this playlist would play: ${reason}`);
211
+ }
80
212
  }
81
- const code = await closed;
82
- if (signal.aborted) return;
83
- if (code !== 0) throw new Error(`Playlist entry ${at + 1} failed: ${tail.trim()}`);
84
- } finally {
85
- signal.removeEventListener("abort", abort);
86
- abort();
87
- await closed;
88
213
  }
89
- // Match the frame grid used above so rounding cannot accumulate a
90
- // backwards timestamp over hundreds of short files.
91
- offset += o.video ? Math.ceil(duration * 30) / 30 : duration;
92
214
  }
93
215
  this.complete = !signal.aborted;
94
216
  } catch (error) {