nixamp 0.23.6 → 0.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +91 -3
- package/dist/captions.d.ts +57 -10
- package/dist/captions.js +253 -29
- package/dist/channels.d.ts +31 -0
- package/dist/channels.js +99 -4
- package/dist/main.js +63 -19
- package/dist/mcp.d.ts +7 -1
- package/dist/mcp.js +159 -24
- package/dist/outro.d.ts +48 -0
- package/dist/outro.js +225 -0
- package/dist/rtmp-in.js +6 -1
- package/dist/server.d.ts +11 -0
- package/dist/server.js +362 -8
- package/dist/speech.d.ts +21 -3
- package/dist/speech.js +65 -8
- package/dist/transcribe.d.ts +70 -0
- package/dist/transcribe.js +337 -41
- package/dist/transcript-client.d.ts +81 -0
- package/dist/transcript-client.js +76 -0
- package/dist/transcript.d.ts +7 -2
- package/dist/transcript.js +73 -5
- package/dist/transcripts.d.ts +135 -0
- package/dist/transcripts.js +384 -0
- package/dist/translate-cli.d.ts +16 -0
- package/dist/translate-cli.js +97 -0
- package/dist/translate-jobs.d.ts +39 -0
- package/dist/translate-jobs.js +120 -0
- package/dist/translate.d.ts +78 -0
- package/dist/translate.js +241 -0
- package/dist/warm.d.ts +1 -0
- package/dist/warm.js +40 -0
- package/package.json +1 -1
- package/src/captions.ts +276 -29
- package/src/channels.ts +107 -4
- package/src/main.ts +63 -19
- package/src/mcp.ts +156 -21
- package/src/outro.ts +245 -0
- package/src/rtmp-in.ts +5 -2
- package/src/server.ts +361 -8
- package/src/speech.ts +103 -13
- package/src/transcribe.ts +374 -41
- package/src/transcript-client.ts +147 -0
- package/src/transcript.ts +76 -4
- package/src/transcripts.ts +462 -0
- package/src/translate-cli.ts +111 -0
- package/src/translate-jobs.ts +132 -0
- package/src/translate.ts +276 -0
- package/src/warm.ts +43 -0
- package/web/dist/assets/{hls-3VKVEQE3-CqYm-w53.js → hls-3VKVEQE3-Dtl-3mpW.js} +1 -1
- package/web/dist/assets/index-B0h4Nexr.js +1 -0
- package/web/dist/assets/index-BTGV3Pi5.css +1 -0
- package/web/dist/assets/{mpegts-BXa-l6Og.js → mpegts-BJC48bFV.js} +1 -1
- package/web/dist/assets/{mpegts-LO6RVLD6-CkxdQRL_.js → mpegts-LO6RVLD6-CUIAB9k3.js} +1 -1
- package/web/dist/index.html +12 -8
- package/web/dist/sw.js +6 -6
- package/web/dist/assets/index-46pwGn5-.css +0 -1
- package/web/dist/assets/index-Qo7uZEAJ.js +0 -1
package/src/captions.ts
CHANGED
|
@@ -20,10 +20,21 @@
|
|
|
20
20
|
* A quiet window -- the gap between songs, a picture with no talking -- is
|
|
21
21
|
* never sent. Most of a music channel is that, and hearing it costs the
|
|
22
22
|
* same as hearing speech.
|
|
23
|
+
*
|
|
24
|
+
* What is heard is kept (see transcripts.ts): every line goes to nixamp.com
|
|
25
|
+
* under the identity of what the channel is playing, as seconds into it.
|
|
26
|
+
* When a captioner starts it asks for what is already known, and a window
|
|
27
|
+
* whose moment the store has been through is read out of it instead of
|
|
28
|
+
* heard. A film captioned once is captioned by nobody again; a live is
|
|
29
|
+
* kept as the broadcast it was. And a viewer may ask for the lines in
|
|
30
|
+
* another language: each heard line is translated once, on nixamp.com,
|
|
31
|
+
* handed to whoever wanted that language, and kept beside the original.
|
|
23
32
|
*/
|
|
24
33
|
import { spawn } from "node:child_process";
|
|
25
34
|
import type { Listener } from "./channels.ts";
|
|
26
35
|
import { RATE } from "./speech.ts";
|
|
36
|
+
import { fetchTranscript, keepLines, translateTexts } from "./transcript-client.ts";
|
|
37
|
+
import { covered, lineAt, transcriptIdOf, type TranscriptLine } from "./transcripts.ts";
|
|
27
38
|
|
|
28
39
|
export interface CaptionLine {
|
|
29
40
|
/** The channel's id. */
|
|
@@ -32,6 +43,10 @@ export interface CaptionLine {
|
|
|
32
43
|
at: number;
|
|
33
44
|
until: number;
|
|
34
45
|
text: string;
|
|
46
|
+
/** The language of the words, as heard or as translated into; absent when nobody said. */
|
|
47
|
+
language?: string;
|
|
48
|
+
/** What was heard, when this line is a translation of it. */
|
|
49
|
+
original?: string;
|
|
35
50
|
}
|
|
36
51
|
|
|
37
52
|
/** What turns a channel's bytes into 16 kHz mono 16-bit PCM. ffmpeg, or a test's stand-in. */
|
|
@@ -40,6 +55,22 @@ export interface Decoder {
|
|
|
40
55
|
end(): void;
|
|
41
56
|
}
|
|
42
57
|
|
|
58
|
+
/**
|
|
59
|
+
* What a channel is playing, for the store: its identity, and how the
|
|
60
|
+
* sound a new listener gets maps onto seconds of it.
|
|
61
|
+
*/
|
|
62
|
+
export interface ChannelMedia {
|
|
63
|
+
/** The media identity, as transcripts.ts spells one. */
|
|
64
|
+
media: string;
|
|
65
|
+
title: string;
|
|
66
|
+
/** Seconds into the media at this moment, for a film; absent for anything live. */
|
|
67
|
+
position?: number;
|
|
68
|
+
/** When the channel began, wall clock, ms. A live's seconds count from here. */
|
|
69
|
+
startedAt: number;
|
|
70
|
+
/** How many seconds behind the live edge a new listener's sound starts. */
|
|
71
|
+
backlog: number;
|
|
72
|
+
}
|
|
73
|
+
|
|
43
74
|
export interface CaptionsOptions {
|
|
44
75
|
/** A listener on a channel, or null when there is no such channel. */
|
|
45
76
|
listen: (id: string, listener: Listener) => (() => void) | null;
|
|
@@ -49,10 +80,14 @@ export interface CaptionsOptions {
|
|
|
49
80
|
fetcher?: typeof fetch;
|
|
50
81
|
/** How bytes become PCM. The default spawns ffmpeg; the tests hand in something quieter. */
|
|
51
82
|
decoder?: (onPcm: (pcm: Buffer) => void, onEnd: () => void) => Decoder;
|
|
83
|
+
/** What a channel is playing, for the store. Null, or absent, means the lines are not kept. */
|
|
84
|
+
mediaOf?: (id: string) => ChannelMedia | null;
|
|
52
85
|
now?: () => number;
|
|
53
86
|
onEvent?: (message: string) => void;
|
|
54
87
|
windowMs?: number;
|
|
55
88
|
idleMs?: number;
|
|
89
|
+
/** How often heard lines go to the store. */
|
|
90
|
+
flushMs?: number;
|
|
56
91
|
}
|
|
57
92
|
|
|
58
93
|
export const WINDOW_MS = 5000;
|
|
@@ -63,6 +98,10 @@ export const IDLE_MS = 60_000;
|
|
|
63
98
|
export const QUIET = 0.004;
|
|
64
99
|
/** Windows waiting on the ear at once. Past this the sound is dropped, not queued: late words are worse than none. */
|
|
65
100
|
export const IN_FLIGHT = 2;
|
|
101
|
+
/** Heard lines wait this long, at most, before they are kept. */
|
|
102
|
+
export const FLUSH_MS = 20_000;
|
|
103
|
+
/** Or this many. */
|
|
104
|
+
export const FLUSH_LINES = 12;
|
|
66
105
|
|
|
67
106
|
/**
|
|
68
107
|
* The ffmpeg arguments: whatever arrives on stdin, as PCM on stdout, with
|
|
@@ -148,13 +187,52 @@ export function wavAround(pcm: Buffer, rate = RATE): Buffer {
|
|
|
148
187
|
return Buffer.concat([header, pcm]);
|
|
149
188
|
}
|
|
150
189
|
|
|
190
|
+
/**
|
|
191
|
+
* Seconds into the media that a window covers. A film is paced in real
|
|
192
|
+
* time from a known position, so a new listener's first byte is that
|
|
193
|
+
* position less the backlog it was handed, and every window after it is
|
|
194
|
+
* one window further on. Anything live counts from when the channel
|
|
195
|
+
* began, in wall-clock time.
|
|
196
|
+
*/
|
|
197
|
+
export function mediaSpan(media: ChannelMedia, windowIndex: number, windowSeconds: number, at: number, until: number): { start: number; end: number } {
|
|
198
|
+
if (typeof media.position === "number") {
|
|
199
|
+
const join = Math.max(0, media.position - media.backlog);
|
|
200
|
+
return { start: round(join + windowIndex * windowSeconds), end: round(join + (windowIndex + 1) * windowSeconds) };
|
|
201
|
+
}
|
|
202
|
+
return { start: round(Math.max(0, (at - media.startedAt) / 1000)), end: round(Math.max(0, (until - media.startedAt) / 1000)) };
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
function round(seconds: number): number {
|
|
206
|
+
return Math.round(seconds * 1000) / 1000;
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
export { lineAt };
|
|
210
|
+
|
|
151
211
|
type Subscriber = (line: CaptionLine) => void;
|
|
152
212
|
|
|
213
|
+
export interface CaptionStatus {
|
|
214
|
+
on: boolean;
|
|
215
|
+
lines: number;
|
|
216
|
+
error: string;
|
|
217
|
+
/** The language heard, when known. */
|
|
218
|
+
language: string;
|
|
219
|
+
/** How many lines the store already had for this media when the captioner began. */
|
|
220
|
+
known: number;
|
|
221
|
+
/** The languages lines are being given in besides the original. */
|
|
222
|
+
languages: string[];
|
|
223
|
+
}
|
|
224
|
+
|
|
153
225
|
class Captioner {
|
|
226
|
+
/** The lines as heard, oldest first. */
|
|
154
227
|
readonly lines: CaptionLine[] = [];
|
|
155
|
-
|
|
228
|
+
/** The lines in each other language somebody asked for. */
|
|
229
|
+
readonly linesBy = new Map<string, CaptionLine[]>();
|
|
230
|
+
readonly subscribers = new Map<Subscriber, string>();
|
|
156
231
|
/** The last thing that went wrong, for whoever asks why there are no lines. */
|
|
157
232
|
error = "";
|
|
233
|
+
/** What the ear says the sound is in, or the store said it was; "" until one of them has. */
|
|
234
|
+
language = "";
|
|
235
|
+
private model = "";
|
|
158
236
|
private decoder: Decoder | null = null;
|
|
159
237
|
private detach: (() => void) | null = null;
|
|
160
238
|
private idle: ReturnType<typeof setTimeout> | null = null;
|
|
@@ -163,17 +241,41 @@ class Captioner {
|
|
|
163
241
|
private inFlight = 0;
|
|
164
242
|
private stopped = false;
|
|
165
243
|
private complainedAt = 0;
|
|
244
|
+
private windows = 0;
|
|
245
|
+
/** What the channel is playing, when the store is to be told. */
|
|
246
|
+
private readonly media: ChannelMedia | null;
|
|
247
|
+
private readonly transcriptId: string | null;
|
|
248
|
+
/** What the store had, by language ("" for the original), and which stored moments have been read out. */
|
|
249
|
+
private readonly known = new Map<string, TranscriptLine[]>();
|
|
250
|
+
private readonly readOut = new Set<number>();
|
|
251
|
+
private readonly asked = new Set<string>();
|
|
252
|
+
/** Lines heard or translated here and not yet kept, by language. */
|
|
253
|
+
private readonly unsaved = new Map<string, TranscriptLine[]>();
|
|
254
|
+
private flush: ReturnType<typeof setTimeout> | null = null;
|
|
255
|
+
/** Translations in order, per language: a slow one must not overtake the next. */
|
|
256
|
+
private readonly chains = new Map<string, Promise<void>>();
|
|
166
257
|
|
|
167
258
|
constructor(
|
|
168
259
|
readonly id: string,
|
|
169
260
|
private readonly options: CaptionsOptions,
|
|
170
261
|
private readonly onStop: () => void,
|
|
171
|
-
) {
|
|
262
|
+
) {
|
|
263
|
+
this.media = options.mediaOf?.(id) ?? null;
|
|
264
|
+
this.transcriptId = this.media ? transcriptIdOf(this.media.media) : null;
|
|
265
|
+
}
|
|
172
266
|
|
|
173
267
|
private get windowBytes(): number {
|
|
174
268
|
return Math.round(((this.options.windowMs ?? WINDOW_MS) / 1000) * RATE) * 2;
|
|
175
269
|
}
|
|
176
270
|
|
|
271
|
+
private get windowSeconds(): number {
|
|
272
|
+
return (this.options.windowMs ?? WINDOW_MS) / 1000;
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
private now(): number {
|
|
276
|
+
return (this.options.now ?? Date.now)();
|
|
277
|
+
}
|
|
278
|
+
|
|
177
279
|
start(): boolean {
|
|
178
280
|
const make = this.options.decoder ?? ((onPcm, onEnd) => ffmpegDecoder(this.options.ffmpeg, onPcm, onEnd));
|
|
179
281
|
this.decoder = make((pcm) => this.onPcm(pcm), () => this.stop());
|
|
@@ -185,9 +287,29 @@ class Captioner {
|
|
|
185
287
|
this.stop();
|
|
186
288
|
return false;
|
|
187
289
|
}
|
|
290
|
+
void this.consult("");
|
|
188
291
|
return true;
|
|
189
292
|
}
|
|
190
293
|
|
|
294
|
+
/** Ask the store what it already knows of this media in a language, once. */
|
|
295
|
+
private async consult(language: string): Promise<void> {
|
|
296
|
+
if (!this.transcriptId || this.asked.has(language)) return;
|
|
297
|
+
this.asked.add(language);
|
|
298
|
+
const session = this.options.session();
|
|
299
|
+
if (session === null) return;
|
|
300
|
+
const got = await fetchTranscript(session, this.transcriptId, language, this.options.fetcher ?? fetch);
|
|
301
|
+
if (this.stopped) return;
|
|
302
|
+
if (!got.ok) {
|
|
303
|
+
if (got.status !== 404) this.complain(`the store did not answer: ${got.error}`);
|
|
304
|
+
return;
|
|
305
|
+
}
|
|
306
|
+
this.known.set(language, got.body.lines);
|
|
307
|
+
if (language === "" && this.language === "" && got.body.language) this.language = got.body.language;
|
|
308
|
+
if (got.body.lines.length > 0) {
|
|
309
|
+
this.options.onEvent?.(`captions for "${this.id}": the store knows ${got.body.lines.length} lines of this${language ? ` in ${language}` : ""}`);
|
|
310
|
+
}
|
|
311
|
+
}
|
|
312
|
+
|
|
191
313
|
private onPcm(pcm: Buffer): void {
|
|
192
314
|
if (this.stopped) return;
|
|
193
315
|
this.pending.push(pcm);
|
|
@@ -199,13 +321,29 @@ class Captioner {
|
|
|
199
321
|
const rest = all.subarray(size);
|
|
200
322
|
this.pending = rest.length > 0 ? [Buffer.from(rest)] : [];
|
|
201
323
|
this.pendingBytes = rest.length;
|
|
202
|
-
const until =
|
|
203
|
-
|
|
324
|
+
const until = this.now();
|
|
325
|
+
const index = this.windows;
|
|
326
|
+
this.windows += 1;
|
|
327
|
+
void this.hear(Buffer.from(window), until - (this.options.windowMs ?? WINDOW_MS), until, index);
|
|
204
328
|
}
|
|
205
329
|
}
|
|
206
330
|
|
|
207
|
-
private async hear(pcm: Buffer, at: number, until: number): Promise<void> {
|
|
331
|
+
private async hear(pcm: Buffer, at: number, until: number, index: number): Promise<void> {
|
|
208
332
|
if (isQuiet(pcm)) return;
|
|
333
|
+
const span = this.media ? mediaSpan(this.media, index, this.windowSeconds, at, until) : null;
|
|
334
|
+
if (span) {
|
|
335
|
+
const stored = covered(this.known.get("") ?? [], span.start, span.end);
|
|
336
|
+
if (stored.length > 0) {
|
|
337
|
+
// The store has been through this moment: read it out, and let the ear rest.
|
|
338
|
+
for (const line of stored) {
|
|
339
|
+
if (this.readOut.has(line.start)) continue;
|
|
340
|
+
this.readOut.add(line.start);
|
|
341
|
+
const lineAt = at + (line.start - span.start) * 1000;
|
|
342
|
+
this.emit({ channel: this.id, at: lineAt, until: lineAt + (line.end - line.start) * 1000, text: line.text, ...(this.language ? { language: this.language } : {}) }, line.start);
|
|
343
|
+
}
|
|
344
|
+
return;
|
|
345
|
+
}
|
|
346
|
+
}
|
|
209
347
|
if (this.inFlight >= IN_FLIGHT) return;
|
|
210
348
|
const session = this.options.session();
|
|
211
349
|
if (session === null) {
|
|
@@ -215,12 +353,14 @@ class Captioner {
|
|
|
215
353
|
this.inFlight += 1;
|
|
216
354
|
try {
|
|
217
355
|
const wav = wavAround(pcm);
|
|
218
|
-
const
|
|
356
|
+
const url = new URL(`${session.site.replace(/\/+$/, "")}/api/v1/speech/transcribe`);
|
|
357
|
+
if (this.language) url.searchParams.set("language", this.language);
|
|
358
|
+
const response = await (this.options.fetcher ?? fetch)(url.toString(), {
|
|
219
359
|
method: "POST",
|
|
220
360
|
headers: { authorization: `Bearer ${session.token}`, "content-type": "audio/wav" },
|
|
221
361
|
body: new Blob([wav.buffer.slice(wav.byteOffset, wav.byteOffset + wav.byteLength) as ArrayBuffer]),
|
|
222
362
|
});
|
|
223
|
-
const body = (await response.json().catch(() => ({}))) as { text?: string; error?: string };
|
|
363
|
+
const body = (await response.json().catch(() => ({}))) as { text?: string; language?: string; model?: string; error?: string };
|
|
224
364
|
if (!response.ok) {
|
|
225
365
|
this.complain(body.error ?? `nixamp.com answered ${response.status}`);
|
|
226
366
|
return;
|
|
@@ -228,16 +368,11 @@ class Captioner {
|
|
|
228
368
|
const text = (body.text ?? "").trim();
|
|
229
369
|
if (text === "" || this.stopped) return;
|
|
230
370
|
this.error = "";
|
|
231
|
-
|
|
232
|
-
this.
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
subscriber(line);
|
|
237
|
-
} catch {
|
|
238
|
-
// A listener that throws is not this channel's problem.
|
|
239
|
-
}
|
|
240
|
-
}
|
|
371
|
+
if (body.language && this.language === "") this.language = body.language;
|
|
372
|
+
if (body.model) this.model = body.model;
|
|
373
|
+
const line: CaptionLine = { channel: this.id, at, until, text, ...(this.language ? { language: this.language } : {}) };
|
|
374
|
+
this.emit(line, span?.start ?? null);
|
|
375
|
+
if (span) this.keep("", { start: span.start, end: span.end, text });
|
|
241
376
|
} catch (error) {
|
|
242
377
|
this.complain(`could not reach the ear: ${(error as Error).message}`);
|
|
243
378
|
} finally {
|
|
@@ -245,17 +380,112 @@ class Captioner {
|
|
|
245
380
|
}
|
|
246
381
|
}
|
|
247
382
|
|
|
383
|
+
/** A line as heard, to whoever wants the original, and translated to whoever wants another language. */
|
|
384
|
+
private emit(line: CaptionLine, mediaStart: number | null): void {
|
|
385
|
+
this.lines.push(line);
|
|
386
|
+
while (this.lines.length > KEEP) this.lines.shift();
|
|
387
|
+
for (const [subscriber, language] of this.subscribers) {
|
|
388
|
+
if (language === "" || language === this.language) this.tell(subscriber, line);
|
|
389
|
+
}
|
|
390
|
+
for (const language of this.wanted()) this.translated(language, line, mediaStart);
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
private tell(subscriber: Subscriber, line: CaptionLine): void {
|
|
394
|
+
try {
|
|
395
|
+
subscriber(line);
|
|
396
|
+
} catch {
|
|
397
|
+
// A listener that throws is not this channel's problem.
|
|
398
|
+
}
|
|
399
|
+
}
|
|
400
|
+
|
|
401
|
+
/** The languages somebody wants besides the one being heard. */
|
|
402
|
+
private wanted(): Set<string> {
|
|
403
|
+
const languages = new Set<string>();
|
|
404
|
+
for (const language of this.subscribers.values()) if (language !== "" && language !== this.language) languages.add(language);
|
|
405
|
+
return languages;
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
/** The line in another language: from the store when it has been through this moment, from nixamp.com otherwise. */
|
|
409
|
+
private translated(language: string, line: CaptionLine, mediaStart: number | null): void {
|
|
410
|
+
const chain = (this.chains.get(language) ?? Promise.resolve()).then(async () => {
|
|
411
|
+
if (this.stopped) return;
|
|
412
|
+
let text = "";
|
|
413
|
+
const stored = mediaStart === null ? null : lineAt(this.known.get(language) ?? [], mediaStart);
|
|
414
|
+
if (stored) {
|
|
415
|
+
text = stored.text;
|
|
416
|
+
} else {
|
|
417
|
+
const session = this.options.session();
|
|
418
|
+
if (session === null) return;
|
|
419
|
+
const got = await translateTexts(session, [line.text], this.language, language, this.options.fetcher ?? fetch);
|
|
420
|
+
if (!got.ok) {
|
|
421
|
+
this.complain(`could not translate to ${language}: ${got.error}`);
|
|
422
|
+
return;
|
|
423
|
+
}
|
|
424
|
+
text = (got.body.texts[0] ?? "").trim();
|
|
425
|
+
if (text === "") return;
|
|
426
|
+
if (mediaStart !== null) this.keep(language, { start: mediaStart, end: round(mediaStart + (line.until - line.at) / 1000), text });
|
|
427
|
+
}
|
|
428
|
+
if (this.stopped) return;
|
|
429
|
+
const said: CaptionLine = { ...line, text, language, original: line.text };
|
|
430
|
+
const lines = this.linesBy.get(language) ?? [];
|
|
431
|
+
lines.push(said);
|
|
432
|
+
while (lines.length > KEEP) lines.shift();
|
|
433
|
+
this.linesBy.set(language, lines);
|
|
434
|
+
for (const [subscriber, wanted] of this.subscribers) if (wanted === language) this.tell(subscriber, said);
|
|
435
|
+
});
|
|
436
|
+
this.chains.set(language, chain.catch(() => undefined));
|
|
437
|
+
}
|
|
438
|
+
|
|
439
|
+
/** A line for the store, kept with the others of its language until the next flush. */
|
|
440
|
+
private keep(language: string, line: TranscriptLine): void {
|
|
441
|
+
if (!this.media) return;
|
|
442
|
+
const lines = this.unsaved.get(language) ?? [];
|
|
443
|
+
lines.push(line);
|
|
444
|
+
this.unsaved.set(language, lines);
|
|
445
|
+
if (lines.length >= FLUSH_LINES) {
|
|
446
|
+
void this.flushNow();
|
|
447
|
+
return;
|
|
448
|
+
}
|
|
449
|
+
if (this.flush === null) {
|
|
450
|
+
this.flush = setTimeout(() => void this.flushNow(), this.options.flushMs ?? FLUSH_MS);
|
|
451
|
+
this.flush.unref?.();
|
|
452
|
+
}
|
|
453
|
+
}
|
|
454
|
+
|
|
455
|
+
/** Everything not yet kept, to the store. Never throws; a store that is away costs nothing but the keeping. */
|
|
456
|
+
private async flushNow(): Promise<void> {
|
|
457
|
+
if (this.flush) clearTimeout(this.flush);
|
|
458
|
+
this.flush = null;
|
|
459
|
+
if (!this.media || !this.transcriptId) return;
|
|
460
|
+
const session = this.options.session();
|
|
461
|
+
if (session === null) return;
|
|
462
|
+
const batches = [...this.unsaved.entries()].filter(([, lines]) => lines.length > 0);
|
|
463
|
+
this.unsaved.clear();
|
|
464
|
+
for (const [language, lines] of batches) {
|
|
465
|
+
const got = await keepLines(session, this.transcriptId, {
|
|
466
|
+
media: this.media.media,
|
|
467
|
+
title: this.media.title,
|
|
468
|
+
language: language === "" ? this.language : language,
|
|
469
|
+
...(language === "" ? {} : { translatedFrom: this.language }),
|
|
470
|
+
...(this.model ? { model: this.model } : {}),
|
|
471
|
+
lines,
|
|
472
|
+
}, this.options.fetcher ?? fetch);
|
|
473
|
+
if (!got.ok) this.complain(`the store did not keep ${lines.length} lines: ${got.error}`);
|
|
474
|
+
}
|
|
475
|
+
}
|
|
476
|
+
|
|
248
477
|
/** Said once a minute at most: a broken ear would otherwise say so twelve times a minute. */
|
|
249
478
|
private complain(message: string): void {
|
|
250
479
|
this.error = message;
|
|
251
|
-
const now =
|
|
480
|
+
const now = this.now();
|
|
252
481
|
if (now - this.complainedAt < 60_000) return;
|
|
253
482
|
this.complainedAt = now;
|
|
254
483
|
this.options.onEvent?.(`captions for "${this.id}": ${message}`);
|
|
255
484
|
}
|
|
256
485
|
|
|
257
|
-
subscribe(subscriber: Subscriber): () => void {
|
|
258
|
-
this.subscribers.
|
|
486
|
+
subscribe(subscriber: Subscriber, language = ""): () => void {
|
|
487
|
+
this.subscribers.set(subscriber, language);
|
|
488
|
+
if (language !== "") void this.consult(language);
|
|
259
489
|
if (this.idle) clearTimeout(this.idle);
|
|
260
490
|
this.idle = null;
|
|
261
491
|
return () => {
|
|
@@ -269,6 +499,22 @@ class Captioner {
|
|
|
269
499
|
};
|
|
270
500
|
}
|
|
271
501
|
|
|
502
|
+
recent(after: number, language = ""): CaptionLine[] {
|
|
503
|
+
const lines = language === "" || language === this.language ? this.lines : (this.linesBy.get(language) ?? []);
|
|
504
|
+
return after > 0 ? lines.filter((line) => line.at > after) : [...lines];
|
|
505
|
+
}
|
|
506
|
+
|
|
507
|
+
status(): CaptionStatus {
|
|
508
|
+
return {
|
|
509
|
+
on: true,
|
|
510
|
+
lines: this.lines.length,
|
|
511
|
+
error: this.error,
|
|
512
|
+
language: this.language,
|
|
513
|
+
known: this.known.get("")?.length ?? 0,
|
|
514
|
+
languages: [...this.linesBy.keys()],
|
|
515
|
+
};
|
|
516
|
+
}
|
|
517
|
+
|
|
272
518
|
stop(): void {
|
|
273
519
|
if (this.stopped) return;
|
|
274
520
|
this.stopped = true;
|
|
@@ -281,6 +527,7 @@ class Captioner {
|
|
|
281
527
|
this.pending = [];
|
|
282
528
|
this.pendingBytes = 0;
|
|
283
529
|
this.subscribers.clear();
|
|
530
|
+
void this.flushNow();
|
|
284
531
|
this.onStop();
|
|
285
532
|
}
|
|
286
533
|
}
|
|
@@ -299,9 +546,10 @@ export class Captions {
|
|
|
299
546
|
* Lines for a channel as they are heard, starting the captioner if it is
|
|
300
547
|
* not running. Null when there is no such channel. The returned function
|
|
301
548
|
* is how to stop listening; the captioner itself stops a minute after the
|
|
302
|
-
* last listener does.
|
|
549
|
+
* last listener does. A language asks for the lines translated into it;
|
|
550
|
+
* "" is the original.
|
|
303
551
|
*/
|
|
304
|
-
subscribe(id: string, subscriber: Subscriber): (() => void) | null {
|
|
552
|
+
subscribe(id: string, subscriber: Subscriber, language = ""): (() => void) | null {
|
|
305
553
|
let captioner = this.running.get(id);
|
|
306
554
|
if (!captioner) {
|
|
307
555
|
const made = new Captioner(id, this.options, () => {
|
|
@@ -312,20 +560,19 @@ export class Captions {
|
|
|
312
560
|
this.options.onEvent?.(`captions for "${id}": started`);
|
|
313
561
|
captioner = made;
|
|
314
562
|
}
|
|
315
|
-
return captioner.subscribe(subscriber);
|
|
563
|
+
return captioner.subscribe(subscriber, language);
|
|
316
564
|
}
|
|
317
565
|
|
|
318
|
-
/** The recent lines of a channel, oldest first, after a moment when given. Empty when nobody has asked for them. */
|
|
319
|
-
recent(id: string, after = 0): CaptionLine[] {
|
|
566
|
+
/** The recent lines of a channel, oldest first, after a moment when given, in a language when asked. Empty when nobody has asked for them. */
|
|
567
|
+
recent(id: string, after = 0, language = ""): CaptionLine[] {
|
|
320
568
|
const captioner = this.running.get(id);
|
|
321
|
-
|
|
322
|
-
return after > 0 ? captioner.lines.filter((line) => line.at > after) : [...captioner.lines];
|
|
569
|
+
return captioner ? captioner.recent(after, language) : [];
|
|
323
570
|
}
|
|
324
571
|
|
|
325
572
|
/** Whether a channel is being captioned, and what last went wrong if the lines are not coming. */
|
|
326
|
-
status(id: string):
|
|
573
|
+
status(id: string): CaptionStatus {
|
|
327
574
|
const captioner = this.running.get(id);
|
|
328
|
-
return captioner ? { on:
|
|
575
|
+
return captioner ? captioner.status() : { on: false, lines: 0, error: "", language: "", known: 0, languages: [] };
|
|
329
576
|
}
|
|
330
577
|
|
|
331
578
|
stopAll(): void {
|
package/src/channels.ts
CHANGED
|
@@ -95,6 +95,12 @@ export interface ChannelInfo {
|
|
|
95
95
|
* plain URL is read this way, and only when a policy asks for it.
|
|
96
96
|
*/
|
|
97
97
|
teed?: boolean;
|
|
98
|
+
/**
|
|
99
|
+
* When its show ended, wall clock, while the outro plays. A channel with
|
|
100
|
+
* this set is still on the air -- the picture says the stream has ended
|
|
101
|
+
* -- and closes on its own an hour later. Absent while the show is on.
|
|
102
|
+
*/
|
|
103
|
+
ended?: number;
|
|
98
104
|
/**
|
|
99
105
|
* A picture of it from somewhere else: the thumbnail the site offered
|
|
100
106
|
* for a pasted link, the logo a catalog gave a channel. A channel with
|
|
@@ -156,6 +162,13 @@ export const GIVE_UP = 5;
|
|
|
156
162
|
export const STALL = 30_000;
|
|
157
163
|
/** How long an on-demand channel stays up with nobody watching. */
|
|
158
164
|
export const IDLE = 60_000;
|
|
165
|
+
/** How long a channel plays its outro after the show, before it closes. */
|
|
166
|
+
export const OUTRO_MS = 60 * 60 * 1000;
|
|
167
|
+
/** The encode the outro is copied through: what a channel sends on the wire. */
|
|
168
|
+
const OUTRO_ENCODE: Record<"audio" | "video", string[]> = {
|
|
169
|
+
video: ["-c", "copy", "-f", "mp4", "-movflags", "frag_keyframe+empty_moov+default_base_moof"],
|
|
170
|
+
audio: ["-c", "copy", "-f", "mp3"],
|
|
171
|
+
};
|
|
159
172
|
/** How much of what ffmpeg said to keep, for the last line when it dies. */
|
|
160
173
|
const TAIL = 2000;
|
|
161
174
|
/**
|
|
@@ -241,6 +254,16 @@ export interface ChannelOptions {
|
|
|
241
254
|
ffmpeg: string[];
|
|
242
255
|
onStart?: (info: ChannelInfo) => void;
|
|
243
256
|
onEnd?: (info: ChannelInfo) => void;
|
|
257
|
+
/**
|
|
258
|
+
* The outro: the clip a channel plays once its show is over, by kind,
|
|
259
|
+
* encoded as the wire wants it (see outro.ts). Null, or absent, means a
|
|
260
|
+
* show that ends closes its channel as it always did.
|
|
261
|
+
*/
|
|
262
|
+
outro?: (kind: "audio" | "video") => Promise<string | null>;
|
|
263
|
+
/** How long the outro plays before the channel closes. An hour. */
|
|
264
|
+
outroMs?: number;
|
|
265
|
+
/** Said once, when a show ends and the outro begins. */
|
|
266
|
+
onOutro?: (info: ChannelInfo) => void;
|
|
244
267
|
/** How long an on-demand channel outlives its last viewer. Tests shorten it. */
|
|
245
268
|
idleMs?: number;
|
|
246
269
|
/** Unsent bytes a listener may hold before it is dropped. Tests shrink it. */
|
|
@@ -272,6 +295,11 @@ export class Channel {
|
|
|
272
295
|
/** Fires when a pulled source has said nothing for STALL. */
|
|
273
296
|
private watchdog: ReturnType<typeof setTimeout> | null = null;
|
|
274
297
|
private stall = STALL;
|
|
298
|
+
/** Whether the outro is what is playing now: the show is over. */
|
|
299
|
+
private outroOn = false;
|
|
300
|
+
private outroTimer: ReturnType<typeof setTimeout> | null = null;
|
|
301
|
+
/** How the show was dialled, so a restart after the outro is the show again. */
|
|
302
|
+
private dialed: { source: string; encode: string[]; paced: boolean; stall: number; input: string[]; audio: string; resume: PullResume } | null = null;
|
|
275
303
|
private stderr = "";
|
|
276
304
|
/**
|
|
277
305
|
* The last few seconds, for whoever joins next.
|
|
@@ -368,6 +396,7 @@ export class Channel {
|
|
|
368
396
|
resume: PullResume = { live: true, position: 0 },
|
|
369
397
|
): void {
|
|
370
398
|
this.stall = stall;
|
|
399
|
+
if (!this.outroOn) this.dialed = { source, encode, paced, stall, input: [...input], audio, resume };
|
|
371
400
|
if (this.info.kind === "video") this.fragments = new Fragments();
|
|
372
401
|
const [command, ...prefix] = this.options.ffmpeg as [string, ...string[]];
|
|
373
402
|
this.info.live = resume.live;
|
|
@@ -382,9 +411,12 @@ export class Channel {
|
|
|
382
411
|
this.info.playlistAt = at;
|
|
383
412
|
}
|
|
384
413
|
let current = list ? (list[at] as string) : source;
|
|
414
|
+
// The last entry is the end of the show, not a way back to the first:
|
|
415
|
+
// a list that played through is over, and says so with the outro.
|
|
385
416
|
this.advance = list && list.length > 1
|
|
386
417
|
? () => {
|
|
387
|
-
|
|
418
|
+
if (at + 1 >= list.length) return false;
|
|
419
|
+
at += 1;
|
|
388
420
|
this.info.playlistAt = at;
|
|
389
421
|
current = list[at] as string;
|
|
390
422
|
return true;
|
|
@@ -414,7 +446,8 @@ export class Channel {
|
|
|
414
446
|
// policy set after the channel started applies at its next restart.
|
|
415
447
|
this.throughAbort?.abort();
|
|
416
448
|
this.throughAbort = null;
|
|
417
|
-
|
|
449
|
+
// The outro is a file read by ffmpeg itself: a pipe cannot loop.
|
|
450
|
+
const through = this.outroOn ? null : this.options.through?.(this.info, from, input, audio) ?? null;
|
|
418
451
|
this.info.teed = through !== null;
|
|
419
452
|
const child = spawn(
|
|
420
453
|
command,
|
|
@@ -509,7 +542,43 @@ export class Channel {
|
|
|
509
542
|
|
|
510
543
|
this.redial = dial;
|
|
511
544
|
dial();
|
|
512
|
-
this.options.onStart?.(this.info);
|
|
545
|
+
if (!this.outroOn) this.options.onStart?.(this.info);
|
|
546
|
+
}
|
|
547
|
+
|
|
548
|
+
/**
|
|
549
|
+
* The show is over: the outro, then the end. A publisher's channel is
|
|
550
|
+
* told this when the publisher goes, so whoever joins in the next hour
|
|
551
|
+
* is shown that the stream has ended rather than nothing at all. Without
|
|
552
|
+
* an outro to play it is the same as close().
|
|
553
|
+
*/
|
|
554
|
+
finish(): void {
|
|
555
|
+
if (this.closing || this.outroOn) return;
|
|
556
|
+
void this.endShow();
|
|
557
|
+
}
|
|
558
|
+
|
|
559
|
+
private async endShow(): Promise<void> {
|
|
560
|
+
const kind = this.info.kind ?? "audio";
|
|
561
|
+
const old = this.child;
|
|
562
|
+
this.child = null;
|
|
563
|
+
if (this.watchdog) clearTimeout(this.watchdog);
|
|
564
|
+
this.watchdog = null;
|
|
565
|
+
if (this.timer) clearTimeout(this.timer);
|
|
566
|
+
this.timer = null;
|
|
567
|
+
old?.kill("SIGKILL");
|
|
568
|
+
const clip = this.options.outro ? await this.options.outro(kind).catch(() => null) : null;
|
|
569
|
+
if (this.closing) return;
|
|
570
|
+
if (!clip) {
|
|
571
|
+
this.close();
|
|
572
|
+
return;
|
|
573
|
+
}
|
|
574
|
+
this.outroOn = true;
|
|
575
|
+
this.info.ended = Date.now();
|
|
576
|
+
this.info.error = undefined;
|
|
577
|
+
this.startOver();
|
|
578
|
+
this.options.onOutro?.(this.info);
|
|
579
|
+
this.pull(clip, OUTRO_ENCODE[kind], true, this.stall, ["-stream_loop", "-1"], "", { live: true, position: 0 });
|
|
580
|
+
this.outroTimer = setTimeout(() => this.close(), this.options.outroMs ?? OUTRO_MS);
|
|
581
|
+
this.outroTimer.unref?.();
|
|
513
582
|
}
|
|
514
583
|
|
|
515
584
|
/**
|
|
@@ -522,8 +591,26 @@ export class Channel {
|
|
|
522
591
|
* strikes a dead CDN ran up an hour ago.
|
|
523
592
|
*/
|
|
524
593
|
restart(): boolean {
|
|
594
|
+
if (this.closing) return false;
|
|
595
|
+
// Started over from the outro: the show itself, from its beginning.
|
|
596
|
+
if (this.outroOn && this.dialed) {
|
|
597
|
+
this.outroOn = false;
|
|
598
|
+
delete this.info.ended;
|
|
599
|
+
if (this.outroTimer) clearTimeout(this.outroTimer);
|
|
600
|
+
this.outroTimer = null;
|
|
601
|
+
const old = this.child;
|
|
602
|
+
this.child = null;
|
|
603
|
+
old?.kill("SIGKILL");
|
|
604
|
+
this.failures = 0;
|
|
605
|
+
this.info.error = undefined;
|
|
606
|
+
this.info.playlistAt = 0;
|
|
607
|
+
this.startOver();
|
|
608
|
+
const show = this.dialed;
|
|
609
|
+
this.pull(show.source, show.encode, show.paced, show.stall, show.input, show.audio, { ...show.resume, position: 0 });
|
|
610
|
+
return true;
|
|
611
|
+
}
|
|
525
612
|
const dial = this.redial;
|
|
526
|
-
if (!dial
|
|
613
|
+
if (!dial) return false;
|
|
527
614
|
if (this.timer) clearTimeout(this.timer);
|
|
528
615
|
this.timer = null;
|
|
529
616
|
if (this.watchdog) clearTimeout(this.watchdog);
|
|
@@ -613,6 +700,11 @@ export class Channel {
|
|
|
613
700
|
*/
|
|
614
701
|
private dropped(sent: boolean): void {
|
|
615
702
|
if (this.closing || !this.redial) return;
|
|
703
|
+
// An outro that stopped is over; there is nothing after it.
|
|
704
|
+
if (this.outroOn) {
|
|
705
|
+
this.close();
|
|
706
|
+
return;
|
|
707
|
+
}
|
|
616
708
|
this.child = null;
|
|
617
709
|
if (this.watchdog) clearTimeout(this.watchdog);
|
|
618
710
|
this.watchdog = null;
|
|
@@ -628,6 +720,15 @@ export class Channel {
|
|
|
628
720
|
// gave nothing is skipped the same way, counted as the failure it was,
|
|
629
721
|
// so a list of dead links gives up rather than cycling for ever.
|
|
630
722
|
const moved = this.advance?.() ?? false;
|
|
723
|
+
// A show that ended: a film or a podcast that played to its end, a
|
|
724
|
+
// list whose last entry did. That is not a source that dropped, and it
|
|
725
|
+
// is not dialled again from the top; it is over, and the outro says
|
|
726
|
+
// so. A live feed that stopped is a feed that dropped, and is redialled.
|
|
727
|
+
const list = (this.info.playlist?.length ?? 0) > 0;
|
|
728
|
+
if (sent && !moved && (this.info.live === false || list)) {
|
|
729
|
+
void this.endShow();
|
|
730
|
+
return;
|
|
731
|
+
}
|
|
631
732
|
const ended = moved && sent;
|
|
632
733
|
if (ended) this.info.error = undefined;
|
|
633
734
|
else this.info.redials = (this.info.redials ?? 0) + 1;
|
|
@@ -914,6 +1015,8 @@ export class Channel {
|
|
|
914
1015
|
this.watchdog = null;
|
|
915
1016
|
if (this.idle) clearTimeout(this.idle);
|
|
916
1017
|
this.idle = null;
|
|
1018
|
+
if (this.outroTimer) clearTimeout(this.outroTimer);
|
|
1019
|
+
this.outroTimer = null;
|
|
917
1020
|
const said = lastLine(this.stderr);
|
|
918
1021
|
if (said && !this.info.error) this.info.error = said;
|
|
919
1022
|
const child = this.child;
|