nixamp 0.23.7 → 0.24.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +79 -3
- package/dist/captions.d.ts +57 -10
- package/dist/captions.js +253 -29
- package/dist/main.js +63 -19
- package/dist/mcp.d.ts +7 -1
- package/dist/mcp.js +159 -24
- package/dist/server.d.ts +11 -0
- package/dist/server.js +353 -8
- package/dist/speech.d.ts +21 -3
- package/dist/speech.js +65 -8
- package/dist/transcribe.d.ts +70 -0
- package/dist/transcribe.js +337 -41
- package/dist/transcript-client.d.ts +81 -0
- package/dist/transcript-client.js +76 -0
- package/dist/transcript.d.ts +7 -2
- package/dist/transcript.js +73 -5
- package/dist/transcripts.d.ts +135 -0
- package/dist/transcripts.js +384 -0
- package/dist/translate-cli.d.ts +16 -0
- package/dist/translate-cli.js +97 -0
- package/dist/translate-jobs.d.ts +39 -0
- package/dist/translate-jobs.js +120 -0
- package/dist/translate.d.ts +86 -0
- package/dist/translate.js +256 -0
- package/dist/warm.d.ts +1 -0
- package/dist/warm.js +40 -0
- package/package.json +1 -1
- package/src/captions.ts +276 -29
- package/src/main.ts +63 -19
- package/src/mcp.ts +156 -21
- package/src/server.ts +352 -8
- package/src/speech.ts +103 -13
- package/src/transcribe.ts +374 -41
- package/src/transcript-client.ts +147 -0
- package/src/transcript.ts +76 -4
- package/src/transcripts.ts +462 -0
- package/src/translate-cli.ts +111 -0
- package/src/translate-jobs.ts +132 -0
- package/src/translate.ts +292 -0
- package/src/warm.ts +43 -0
- package/web/dist/assets/{hls-3VKVEQE3-CI1U7kbP.js → hls-3VKVEQE3-Dtl-3mpW.js} +1 -1
- package/web/dist/assets/index-B0h4Nexr.js +1 -0
- package/web/dist/assets/index-BTGV3Pi5.css +1 -0
- package/web/dist/assets/{mpegts-CPOYjgRP.js → mpegts-BJC48bFV.js} +1 -1
- package/web/dist/assets/{mpegts-LO6RVLD6-CE8YPjx1.js → mpegts-LO6RVLD6-CUIAB9k3.js} +1 -1
- package/web/dist/index.html +12 -8
- package/web/dist/sw.js +6 -6
- package/web/dist/assets/index-46pwGn5-.css +0 -1
- package/web/dist/assets/index-5H3sUGHu.js +0 -1
package/README.md
CHANGED
|
@@ -140,7 +140,12 @@ DATABASE_URL=postgres://user:pass@host/nixamp NIXAMP_JWT_SECRET=… nixamp serve
|
|
|
140
140
|
Accounts live where the directory lives and nowhere else: a nixamp on a laptop
|
|
141
141
|
has nobody to be an account of.
|
|
142
142
|
|
|
143
|
-
##
|
|
143
|
+
## Parties, and signing in with nixamp
|
|
144
|
+
|
|
145
|
+
The web app uses **Join party** for joining a stream or a party on a connected
|
|
146
|
+
site. In the **Parties** panel, enter an **Invite code** and select **Join party**
|
|
147
|
+
to open its room. A listed party's **Join party** link opens the film on the
|
|
148
|
+
site hosting it; **Open room** opens its nixamp room.
|
|
144
149
|
|
|
145
150
|
A watch party lives on the site that has the film. bittorrented.com has them:
|
|
146
151
|
a six-character code, a host, and everybody at the same second. nixamp has
|
|
@@ -192,11 +197,15 @@ nixamp party sync ABC123 --at 930 where playback is (hosts only)
|
|
|
192
197
|
and an agent reaches the same five actions over the Model Context Protocol:
|
|
193
198
|
|
|
194
199
|
```
|
|
195
|
-
nixamp mcp a stdio MCP server:
|
|
200
|
+
nixamp mcp a stdio MCP server: the parties, the rooms, the transcripts
|
|
196
201
|
```
|
|
197
202
|
|
|
198
203
|
It acts as whoever the machine is signed in as, so `nixamp login` comes first.
|
|
199
|
-
The film never crosses over: what nixamp carries is the room.
|
|
204
|
+
The film never crosses over: what nixamp carries is the room. The same tools
|
|
205
|
+
are at `https://nixamp.com/mcp` over HTTP, with a nixamp token
|
|
206
|
+
(`nixamp token create`) as the bearer, for an agent with no nixamp installed;
|
|
207
|
+
`/.well-known/oauth-protected-resource` says where the authorization server
|
|
208
|
+
is.
|
|
200
209
|
|
|
201
210
|
## BackToSchool.help
|
|
202
211
|
|
|
@@ -405,6 +414,73 @@ nixamp.com instead. `NIXAMP_STT_MODEL` picks another Whisper
|
|
|
405
414
|
takes twice as long), `NIXAMP_STT_CACHE` says where its files are kept, and
|
|
406
415
|
`NIXAMP_STT=off` leaves the ear out of a deployment altogether.
|
|
407
416
|
|
|
417
|
+
The ear tells which language it heard: one pass over the first thirty
|
|
418
|
+
seconds, the way whisper.cpp does it, before the words are read. Without
|
|
419
|
+
that a Swedish channel came back as three English words repeated to the end
|
|
420
|
+
of the window. A captioner learns the language from its first line and says
|
|
421
|
+
it on every ask after that.
|
|
422
|
+
|
|
423
|
+
### Kept: written down once, for everybody
|
|
424
|
+
|
|
425
|
+
What the ear hears is kept on nixamp.com under the identity of what was
|
|
426
|
+
playing, not of the channel that happened to play it: a file by its
|
|
427
|
+
fingerprint (its size and a megabyte at each end), a link by its address, a
|
|
428
|
+
live as the one broadcast it was. Lines are seconds into the media. The next
|
|
429
|
+
captioner to meet the same film reads the lines out of the store instead of
|
|
430
|
+
hearing them, whichever server it is on; what it hears beyond them is added.
|
|
431
|
+
|
|
432
|
+
```
|
|
433
|
+
nixamp transcribe FILE the whole film, a minute at a time, kept when it is done
|
|
434
|
+
nixamp transcribe FILE --srt > film.srt as subtitles; --vtt, --txt, --json
|
|
435
|
+
nixamp transcribe FILE --out DIR a subtitle file per language in DIR
|
|
436
|
+
nixamp transcript --kept MEDIA_OR_ID what nixamp.com keeps, for a file, a link or a past live
|
|
437
|
+
nixamp transcript --list everything this account has had written down
|
|
438
|
+
```
|
|
439
|
+
|
|
440
|
+
```
|
|
441
|
+
GET /api/v1/transcripts what you have had written down
|
|
442
|
+
GET /api/v1/transcripts/ID the transcript; ?format=srt|vtt|txt, ?language=de
|
|
443
|
+
POST /api/v1/transcripts/ID/lines keep lines: {media, language, lines: [{start, end, text}], complete?}
|
|
444
|
+
DELETE /api/v1/transcripts/ID forget it (whoever kept it)
|
|
445
|
+
```
|
|
446
|
+
|
|
447
|
+
ID is the sha256 of the media identity, or the identity itself
|
|
448
|
+
(`file:v1:<hash>`, `url:<address>`, `live:<server>/<channel>@<started>`).
|
|
449
|
+
Signed in to read and to keep, like the ear. A whole-file pass marks the row
|
|
450
|
+
complete and replaces the pieces a captioner left; a live grows as it goes
|
|
451
|
+
and a page that asks for it reads what there is so far. An agent has
|
|
452
|
+
`transcript_get` and `transcripts_list`, and `transcribe_audio` keeps a film
|
|
453
|
+
the same way.
|
|
454
|
+
|
|
455
|
+
### In another language
|
|
456
|
+
|
|
457
|
+
Ask for a language and the lines come translated, by an open-source model on
|
|
458
|
+
nixamp.com's own CPU (Helsinki-NLP's OPUS-MT pairs, through Transformers.js):
|
|
459
|
+
German and Swedish among the languages, and anything with a model from or
|
|
460
|
+
into English; a pair with no model of its own goes through English. A
|
|
461
|
+
translation is made once and kept beside the original.
|
|
462
|
+
|
|
463
|
+
```
|
|
464
|
+
GET /api/channels/ID/captions?language=sv a live's lines in Swedish, each translated as it is heard
|
|
465
|
+
GET /api/v1/transcripts/ID?language=de a kept transcript in German; 202 with progress while a long one is made
|
|
466
|
+
GET /api/v1/translate the languages, and what each can be turned into here
|
|
467
|
+
POST /api/v1/translate {texts, from, to} -> {texts}
|
|
468
|
+
```
|
|
469
|
+
|
|
470
|
+
```
|
|
471
|
+
nixamp transcript --channel ID --language sv a live, in Swedish, as it speaks
|
|
472
|
+
nixamp transcribe FILE --translate de,sv a film in German and Swedish too
|
|
473
|
+
nixamp translate --to sv "Hello there" a line; or lines on stdin
|
|
474
|
+
nixamp translate --languages what nixamp.com can do
|
|
475
|
+
```
|
|
476
|
+
|
|
477
|
+
The page has the same choice beside the Captions switch, remembered per
|
|
478
|
+
device; a translated line is marked with its language and shows what was
|
|
479
|
+
heard under the pointer. An agent has `translate_text`. `NIXAMP_MT_WARM`
|
|
480
|
+
names pairs to load at boot (`en-de,en-sv`), `NIXAMP_MT=off` leaves
|
|
481
|
+
translation out, and the Docker image bakes the ear and the German and
|
|
482
|
+
Swedish pairs in so a deploy never downloads them again.
|
|
483
|
+
|
|
408
484
|
## Several streams at once
|
|
409
485
|
|
|
410
486
|
A channel is one publisher and everybody listening to them. Two or three devices
|
package/dist/captions.d.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { Listener } from "./channels.ts";
|
|
2
|
+
import { lineAt } from "./transcripts.ts";
|
|
2
3
|
export interface CaptionLine {
|
|
3
4
|
/** The channel's id. */
|
|
4
5
|
channel: string;
|
|
@@ -6,12 +7,31 @@ export interface CaptionLine {
|
|
|
6
7
|
at: number;
|
|
7
8
|
until: number;
|
|
8
9
|
text: string;
|
|
10
|
+
/** The language of the words, as heard or as translated into; absent when nobody said. */
|
|
11
|
+
language?: string;
|
|
12
|
+
/** What was heard, when this line is a translation of it. */
|
|
13
|
+
original?: string;
|
|
9
14
|
}
|
|
10
15
|
/** What turns a channel's bytes into 16 kHz mono 16-bit PCM. ffmpeg, or a test's stand-in. */
|
|
11
16
|
export interface Decoder {
|
|
12
17
|
write(chunk: Buffer): boolean;
|
|
13
18
|
end(): void;
|
|
14
19
|
}
|
|
20
|
+
/**
|
|
21
|
+
* What a channel is playing, for the store: its identity, and how the
|
|
22
|
+
* sound a new listener gets maps onto seconds of it.
|
|
23
|
+
*/
|
|
24
|
+
export interface ChannelMedia {
|
|
25
|
+
/** The media identity, as transcripts.ts spells one. */
|
|
26
|
+
media: string;
|
|
27
|
+
title: string;
|
|
28
|
+
/** Seconds into the media at this moment, for a film; absent for anything live. */
|
|
29
|
+
position?: number;
|
|
30
|
+
/** When the channel began, wall clock, ms. A live's seconds count from here. */
|
|
31
|
+
startedAt: number;
|
|
32
|
+
/** How many seconds behind the live edge a new listener's sound starts. */
|
|
33
|
+
backlog: number;
|
|
34
|
+
}
|
|
15
35
|
export interface CaptionsOptions {
|
|
16
36
|
/** A listener on a channel, or null when there is no such channel. */
|
|
17
37
|
listen: (id: string, listener: Listener) => (() => void) | null;
|
|
@@ -24,10 +44,14 @@ export interface CaptionsOptions {
|
|
|
24
44
|
fetcher?: typeof fetch;
|
|
25
45
|
/** How bytes become PCM. The default spawns ffmpeg; the tests hand in something quieter. */
|
|
26
46
|
decoder?: (onPcm: (pcm: Buffer) => void, onEnd: () => void) => Decoder;
|
|
47
|
+
/** What a channel is playing, for the store. Null, or absent, means the lines are not kept. */
|
|
48
|
+
mediaOf?: (id: string) => ChannelMedia | null;
|
|
27
49
|
now?: () => number;
|
|
28
50
|
onEvent?: (message: string) => void;
|
|
29
51
|
windowMs?: number;
|
|
30
52
|
idleMs?: number;
|
|
53
|
+
/** How often heard lines go to the store. */
|
|
54
|
+
flushMs?: number;
|
|
31
55
|
}
|
|
32
56
|
export declare const WINDOW_MS = 5000;
|
|
33
57
|
/** Lines kept per channel for whoever arrives late. */
|
|
@@ -37,6 +61,10 @@ export declare const IDLE_MS = 60000;
|
|
|
37
61
|
export declare const QUIET = 0.004;
|
|
38
62
|
/** Windows waiting on the ear at once. Past this the sound is dropped, not queued: late words are worse than none. */
|
|
39
63
|
export declare const IN_FLIGHT = 2;
|
|
64
|
+
/** Heard lines wait this long, at most, before they are kept. */
|
|
65
|
+
export declare const FLUSH_MS = 20000;
|
|
66
|
+
/** Or this many. */
|
|
67
|
+
export declare const FLUSH_LINES = 12;
|
|
40
68
|
/**
|
|
41
69
|
* The ffmpeg arguments: whatever arrives on stdin, as PCM on stdout, with
|
|
42
70
|
* a short probe so the first line is not long in coming. Never `-fflags
|
|
@@ -48,7 +76,30 @@ export declare function decoderArgs(): string[];
|
|
|
48
76
|
export declare function isQuiet(pcm: Buffer, threshold?: number): boolean;
|
|
49
77
|
/** A WAV around 16-bit mono PCM, without copying it through floats. */
|
|
50
78
|
export declare function wavAround(pcm: Buffer, rate?: number): Buffer;
|
|
79
|
+
/**
|
|
80
|
+
* Seconds into the media that a window covers. A film is paced in real
|
|
81
|
+
* time from a known position, so a new listener's first byte is that
|
|
82
|
+
* position less the backlog it was handed, and every window after it is
|
|
83
|
+
* one window further on. Anything live counts from when the channel
|
|
84
|
+
* began, in wall-clock time.
|
|
85
|
+
*/
|
|
86
|
+
export declare function mediaSpan(media: ChannelMedia, windowIndex: number, windowSeconds: number, at: number, until: number): {
|
|
87
|
+
start: number;
|
|
88
|
+
end: number;
|
|
89
|
+
};
|
|
90
|
+
export { lineAt };
|
|
51
91
|
type Subscriber = (line: CaptionLine) => void;
|
|
92
|
+
export interface CaptionStatus {
|
|
93
|
+
on: boolean;
|
|
94
|
+
lines: number;
|
|
95
|
+
error: string;
|
|
96
|
+
/** The language heard, when known. */
|
|
97
|
+
language: string;
|
|
98
|
+
/** How many lines the store already had for this media when the captioner began. */
|
|
99
|
+
known: number;
|
|
100
|
+
/** The languages lines are being given in besides the original. */
|
|
101
|
+
languages: string[];
|
|
102
|
+
}
|
|
52
103
|
export declare class Captions {
|
|
53
104
|
private readonly options;
|
|
54
105
|
private readonly running;
|
|
@@ -59,17 +110,13 @@ export declare class Captions {
|
|
|
59
110
|
* Lines for a channel as they are heard, starting the captioner if it is
|
|
60
111
|
* not running. Null when there is no such channel. The returned function
|
|
61
112
|
* is how to stop listening; the captioner itself stops a minute after the
|
|
62
|
-
* last listener does.
|
|
113
|
+
* last listener does. A language asks for the lines translated into it;
|
|
114
|
+
* "" is the original.
|
|
63
115
|
*/
|
|
64
|
-
subscribe(id: string, subscriber: Subscriber): (() => void) | null;
|
|
65
|
-
/** The recent lines of a channel, oldest first, after a moment when given. Empty when nobody has asked for them. */
|
|
66
|
-
recent(id: string, after?: number): CaptionLine[];
|
|
116
|
+
subscribe(id: string, subscriber: Subscriber, language?: string): (() => void) | null;
|
|
117
|
+
/** The recent lines of a channel, oldest first, after a moment when given, in a language when asked. Empty when nobody has asked for them. */
|
|
118
|
+
recent(id: string, after?: number, language?: string): CaptionLine[];
|
|
67
119
|
/** Whether a channel is being captioned, and what last went wrong if the lines are not coming. */
|
|
68
|
-
status(id: string):
|
|
69
|
-
on: boolean;
|
|
70
|
-
lines: number;
|
|
71
|
-
error: string;
|
|
72
|
-
};
|
|
120
|
+
status(id: string): CaptionStatus;
|
|
73
121
|
stopAll(): void;
|
|
74
122
|
}
|
|
75
|
-
export {};
|
package/dist/captions.js
CHANGED
|
@@ -20,9 +20,20 @@
|
|
|
20
20
|
* A quiet window -- the gap between songs, a picture with no talking -- is
|
|
21
21
|
* never sent. Most of a music channel is that, and hearing it costs the
|
|
22
22
|
* same as hearing speech.
|
|
23
|
+
*
|
|
24
|
+
* What is heard is kept (see transcripts.ts): every line goes to nixamp.com
|
|
25
|
+
* under the identity of what the channel is playing, as seconds into it.
|
|
26
|
+
* When a captioner starts it asks for what is already known, and a window
|
|
27
|
+
* whose moment the store has been through is read out of it instead of
|
|
28
|
+
* heard. A film captioned once is captioned by nobody again; a live is
|
|
29
|
+
* kept as the broadcast it was. And a viewer may ask for the lines in
|
|
30
|
+
* another language: each heard line is translated once, on nixamp.com,
|
|
31
|
+
* handed to whoever wanted that language, and kept beside the original.
|
|
23
32
|
*/
|
|
24
33
|
import { spawn } from "node:child_process";
|
|
25
34
|
import { RATE } from "./speech.js";
|
|
35
|
+
import { fetchTranscript, keepLines, translateTexts } from "./transcript-client.js";
|
|
36
|
+
import { covered, lineAt, transcriptIdOf } from "./transcripts.js";
|
|
26
37
|
export const WINDOW_MS = 5000;
|
|
27
38
|
/** Lines kept per channel for whoever arrives late. */
|
|
28
39
|
export const KEEP = 200;
|
|
@@ -31,6 +42,10 @@ export const IDLE_MS = 60_000;
|
|
|
31
42
|
export const QUIET = 0.004;
|
|
32
43
|
/** Windows waiting on the ear at once. Past this the sound is dropped, not queued: late words are worse than none. */
|
|
33
44
|
export const IN_FLIGHT = 2;
|
|
45
|
+
/** Heard lines wait this long, at most, before they are kept. */
|
|
46
|
+
export const FLUSH_MS = 20_000;
|
|
47
|
+
/** Or this many. */
|
|
48
|
+
export const FLUSH_LINES = 12;
|
|
34
49
|
/**
|
|
35
50
|
* The ffmpeg arguments: whatever arrives on stdin, as PCM on stdout, with
|
|
36
51
|
* a short probe so the first line is not long in coming. Never `-fflags
|
|
@@ -116,14 +131,38 @@ export function wavAround(pcm, rate = RATE) {
|
|
|
116
131
|
header.writeUInt32LE(pcm.length, 40);
|
|
117
132
|
return Buffer.concat([header, pcm]);
|
|
118
133
|
}
|
|
134
|
+
/**
|
|
135
|
+
* Seconds into the media that a window covers. A film is paced in real
|
|
136
|
+
* time from a known position, so a new listener's first byte is that
|
|
137
|
+
* position less the backlog it was handed, and every window after it is
|
|
138
|
+
* one window further on. Anything live counts from when the channel
|
|
139
|
+
* began, in wall-clock time.
|
|
140
|
+
*/
|
|
141
|
+
export function mediaSpan(media, windowIndex, windowSeconds, at, until) {
|
|
142
|
+
if (typeof media.position === "number") {
|
|
143
|
+
const join = Math.max(0, media.position - media.backlog);
|
|
144
|
+
return { start: round(join + windowIndex * windowSeconds), end: round(join + (windowIndex + 1) * windowSeconds) };
|
|
145
|
+
}
|
|
146
|
+
return { start: round(Math.max(0, (at - media.startedAt) / 1000)), end: round(Math.max(0, (until - media.startedAt) / 1000)) };
|
|
147
|
+
}
|
|
148
|
+
function round(seconds) {
|
|
149
|
+
return Math.round(seconds * 1000) / 1000;
|
|
150
|
+
}
|
|
151
|
+
export { lineAt };
|
|
119
152
|
class Captioner {
|
|
120
153
|
id;
|
|
121
154
|
options;
|
|
122
155
|
onStop;
|
|
156
|
+
/** The lines as heard, oldest first. */
|
|
123
157
|
lines = [];
|
|
124
|
-
|
|
158
|
+
/** The lines in each other language somebody asked for. */
|
|
159
|
+
linesBy = new Map();
|
|
160
|
+
subscribers = new Map();
|
|
125
161
|
/** The last thing that went wrong, for whoever asks why there are no lines. */
|
|
126
162
|
error = "";
|
|
163
|
+
/** What the ear says the sound is in, or the store said it was; "" until one of them has. */
|
|
164
|
+
language = "";
|
|
165
|
+
model = "";
|
|
127
166
|
decoder = null;
|
|
128
167
|
detach = null;
|
|
129
168
|
idle = null;
|
|
@@ -132,14 +171,35 @@ class Captioner {
|
|
|
132
171
|
inFlight = 0;
|
|
133
172
|
stopped = false;
|
|
134
173
|
complainedAt = 0;
|
|
174
|
+
windows = 0;
|
|
175
|
+
/** What the channel is playing, when the store is to be told. */
|
|
176
|
+
media;
|
|
177
|
+
transcriptId;
|
|
178
|
+
/** What the store had, by language ("" for the original), and which stored moments have been read out. */
|
|
179
|
+
known = new Map();
|
|
180
|
+
readOut = new Set();
|
|
181
|
+
asked = new Set();
|
|
182
|
+
/** Lines heard or translated here and not yet kept, by language. */
|
|
183
|
+
unsaved = new Map();
|
|
184
|
+
flush = null;
|
|
185
|
+
/** Translations in order, per language: a slow one must not overtake the next. */
|
|
186
|
+
chains = new Map();
|
|
135
187
|
constructor(id, options, onStop) {
|
|
136
188
|
this.id = id;
|
|
137
189
|
this.options = options;
|
|
138
190
|
this.onStop = onStop;
|
|
191
|
+
this.media = options.mediaOf?.(id) ?? null;
|
|
192
|
+
this.transcriptId = this.media ? transcriptIdOf(this.media.media) : null;
|
|
139
193
|
}
|
|
140
194
|
get windowBytes() {
|
|
141
195
|
return Math.round(((this.options.windowMs ?? WINDOW_MS) / 1000) * RATE) * 2;
|
|
142
196
|
}
|
|
197
|
+
get windowSeconds() {
|
|
198
|
+
return (this.options.windowMs ?? WINDOW_MS) / 1000;
|
|
199
|
+
}
|
|
200
|
+
now() {
|
|
201
|
+
return (this.options.now ?? Date.now)();
|
|
202
|
+
}
|
|
143
203
|
start() {
|
|
144
204
|
const make = this.options.decoder ?? ((onPcm, onEnd) => ffmpegDecoder(this.options.ffmpeg, onPcm, onEnd));
|
|
145
205
|
this.decoder = make((pcm) => this.onPcm(pcm), () => this.stop());
|
|
@@ -151,8 +211,32 @@ class Captioner {
|
|
|
151
211
|
this.stop();
|
|
152
212
|
return false;
|
|
153
213
|
}
|
|
214
|
+
void this.consult("");
|
|
154
215
|
return true;
|
|
155
216
|
}
|
|
217
|
+
/** Ask the store what it already knows of this media in a language, once. */
|
|
218
|
+
async consult(language) {
|
|
219
|
+
if (!this.transcriptId || this.asked.has(language))
|
|
220
|
+
return;
|
|
221
|
+
this.asked.add(language);
|
|
222
|
+
const session = this.options.session();
|
|
223
|
+
if (session === null)
|
|
224
|
+
return;
|
|
225
|
+
const got = await fetchTranscript(session, this.transcriptId, language, this.options.fetcher ?? fetch);
|
|
226
|
+
if (this.stopped)
|
|
227
|
+
return;
|
|
228
|
+
if (!got.ok) {
|
|
229
|
+
if (got.status !== 404)
|
|
230
|
+
this.complain(`the store did not answer: ${got.error}`);
|
|
231
|
+
return;
|
|
232
|
+
}
|
|
233
|
+
this.known.set(language, got.body.lines);
|
|
234
|
+
if (language === "" && this.language === "" && got.body.language)
|
|
235
|
+
this.language = got.body.language;
|
|
236
|
+
if (got.body.lines.length > 0) {
|
|
237
|
+
this.options.onEvent?.(`captions for "${this.id}": the store knows ${got.body.lines.length} lines of this${language ? ` in ${language}` : ""}`);
|
|
238
|
+
}
|
|
239
|
+
}
|
|
156
240
|
onPcm(pcm) {
|
|
157
241
|
if (this.stopped)
|
|
158
242
|
return;
|
|
@@ -165,13 +249,30 @@ class Captioner {
|
|
|
165
249
|
const rest = all.subarray(size);
|
|
166
250
|
this.pending = rest.length > 0 ? [Buffer.from(rest)] : [];
|
|
167
251
|
this.pendingBytes = rest.length;
|
|
168
|
-
const until =
|
|
169
|
-
|
|
252
|
+
const until = this.now();
|
|
253
|
+
const index = this.windows;
|
|
254
|
+
this.windows += 1;
|
|
255
|
+
void this.hear(Buffer.from(window), until - (this.options.windowMs ?? WINDOW_MS), until, index);
|
|
170
256
|
}
|
|
171
257
|
}
|
|
172
|
-
async hear(pcm, at, until) {
|
|
258
|
+
async hear(pcm, at, until, index) {
|
|
173
259
|
if (isQuiet(pcm))
|
|
174
260
|
return;
|
|
261
|
+
const span = this.media ? mediaSpan(this.media, index, this.windowSeconds, at, until) : null;
|
|
262
|
+
if (span) {
|
|
263
|
+
const stored = covered(this.known.get("") ?? [], span.start, span.end);
|
|
264
|
+
if (stored.length > 0) {
|
|
265
|
+
// The store has been through this moment: read it out, and let the ear rest.
|
|
266
|
+
for (const line of stored) {
|
|
267
|
+
if (this.readOut.has(line.start))
|
|
268
|
+
continue;
|
|
269
|
+
this.readOut.add(line.start);
|
|
270
|
+
const lineAt = at + (line.start - span.start) * 1000;
|
|
271
|
+
this.emit({ channel: this.id, at: lineAt, until: lineAt + (line.end - line.start) * 1000, text: line.text, ...(this.language ? { language: this.language } : {}) }, line.start);
|
|
272
|
+
}
|
|
273
|
+
return;
|
|
274
|
+
}
|
|
275
|
+
}
|
|
175
276
|
if (this.inFlight >= IN_FLIGHT)
|
|
176
277
|
return;
|
|
177
278
|
const session = this.options.session();
|
|
@@ -182,7 +283,10 @@ class Captioner {
|
|
|
182
283
|
this.inFlight += 1;
|
|
183
284
|
try {
|
|
184
285
|
const wav = wavAround(pcm);
|
|
185
|
-
const
|
|
286
|
+
const url = new URL(`${session.site.replace(/\/+$/, "")}/api/v1/speech/transcribe`);
|
|
287
|
+
if (this.language)
|
|
288
|
+
url.searchParams.set("language", this.language);
|
|
289
|
+
const response = await (this.options.fetcher ?? fetch)(url.toString(), {
|
|
186
290
|
method: "POST",
|
|
187
291
|
headers: { authorization: `Bearer ${session.token}`, "content-type": "audio/wav" },
|
|
188
292
|
body: new Blob([wav.buffer.slice(wav.byteOffset, wav.byteOffset + wav.byteLength)]),
|
|
@@ -196,18 +300,14 @@ class Captioner {
|
|
|
196
300
|
if (text === "" || this.stopped)
|
|
197
301
|
return;
|
|
198
302
|
this.error = "";
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
this.
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
}
|
|
207
|
-
catch {
|
|
208
|
-
// A listener that throws is not this channel's problem.
|
|
209
|
-
}
|
|
210
|
-
}
|
|
303
|
+
if (body.language && this.language === "")
|
|
304
|
+
this.language = body.language;
|
|
305
|
+
if (body.model)
|
|
306
|
+
this.model = body.model;
|
|
307
|
+
const line = { channel: this.id, at, until, text, ...(this.language ? { language: this.language } : {}) };
|
|
308
|
+
this.emit(line, span?.start ?? null);
|
|
309
|
+
if (span)
|
|
310
|
+
this.keep("", { start: span.start, end: span.end, text });
|
|
211
311
|
}
|
|
212
312
|
catch (error) {
|
|
213
313
|
this.complain(`could not reach the ear: ${error.message}`);
|
|
@@ -216,17 +316,127 @@ class Captioner {
|
|
|
216
316
|
this.inFlight -= 1;
|
|
217
317
|
}
|
|
218
318
|
}
|
|
319
|
+
/** A line as heard, to whoever wants the original, and translated to whoever wants another language. */
|
|
320
|
+
emit(line, mediaStart) {
|
|
321
|
+
this.lines.push(line);
|
|
322
|
+
while (this.lines.length > KEEP)
|
|
323
|
+
this.lines.shift();
|
|
324
|
+
for (const [subscriber, language] of this.subscribers) {
|
|
325
|
+
if (language === "" || language === this.language)
|
|
326
|
+
this.tell(subscriber, line);
|
|
327
|
+
}
|
|
328
|
+
for (const language of this.wanted())
|
|
329
|
+
this.translated(language, line, mediaStart);
|
|
330
|
+
}
|
|
331
|
+
tell(subscriber, line) {
|
|
332
|
+
try {
|
|
333
|
+
subscriber(line);
|
|
334
|
+
}
|
|
335
|
+
catch {
|
|
336
|
+
// A listener that throws is not this channel's problem.
|
|
337
|
+
}
|
|
338
|
+
}
|
|
339
|
+
/** The languages somebody wants besides the one being heard. */
|
|
340
|
+
wanted() {
|
|
341
|
+
const languages = new Set();
|
|
342
|
+
for (const language of this.subscribers.values())
|
|
343
|
+
if (language !== "" && language !== this.language)
|
|
344
|
+
languages.add(language);
|
|
345
|
+
return languages;
|
|
346
|
+
}
|
|
347
|
+
/** The line in another language: from the store when it has been through this moment, from nixamp.com otherwise. */
|
|
348
|
+
translated(language, line, mediaStart) {
|
|
349
|
+
const chain = (this.chains.get(language) ?? Promise.resolve()).then(async () => {
|
|
350
|
+
if (this.stopped)
|
|
351
|
+
return;
|
|
352
|
+
let text = "";
|
|
353
|
+
const stored = mediaStart === null ? null : lineAt(this.known.get(language) ?? [], mediaStart);
|
|
354
|
+
if (stored) {
|
|
355
|
+
text = stored.text;
|
|
356
|
+
}
|
|
357
|
+
else {
|
|
358
|
+
const session = this.options.session();
|
|
359
|
+
if (session === null)
|
|
360
|
+
return;
|
|
361
|
+
const got = await translateTexts(session, [line.text], this.language, language, this.options.fetcher ?? fetch);
|
|
362
|
+
if (!got.ok) {
|
|
363
|
+
this.complain(`could not translate to ${language}: ${got.error}`);
|
|
364
|
+
return;
|
|
365
|
+
}
|
|
366
|
+
text = (got.body.texts[0] ?? "").trim();
|
|
367
|
+
if (text === "")
|
|
368
|
+
return;
|
|
369
|
+
if (mediaStart !== null)
|
|
370
|
+
this.keep(language, { start: mediaStart, end: round(mediaStart + (line.until - line.at) / 1000), text });
|
|
371
|
+
}
|
|
372
|
+
if (this.stopped)
|
|
373
|
+
return;
|
|
374
|
+
const said = { ...line, text, language, original: line.text };
|
|
375
|
+
const lines = this.linesBy.get(language) ?? [];
|
|
376
|
+
lines.push(said);
|
|
377
|
+
while (lines.length > KEEP)
|
|
378
|
+
lines.shift();
|
|
379
|
+
this.linesBy.set(language, lines);
|
|
380
|
+
for (const [subscriber, wanted] of this.subscribers)
|
|
381
|
+
if (wanted === language)
|
|
382
|
+
this.tell(subscriber, said);
|
|
383
|
+
});
|
|
384
|
+
this.chains.set(language, chain.catch(() => undefined));
|
|
385
|
+
}
|
|
386
|
+
/** A line for the store, kept with the others of its language until the next flush. */
|
|
387
|
+
keep(language, line) {
|
|
388
|
+
if (!this.media)
|
|
389
|
+
return;
|
|
390
|
+
const lines = this.unsaved.get(language) ?? [];
|
|
391
|
+
lines.push(line);
|
|
392
|
+
this.unsaved.set(language, lines);
|
|
393
|
+
if (lines.length >= FLUSH_LINES) {
|
|
394
|
+
void this.flushNow();
|
|
395
|
+
return;
|
|
396
|
+
}
|
|
397
|
+
if (this.flush === null) {
|
|
398
|
+
this.flush = setTimeout(() => void this.flushNow(), this.options.flushMs ?? FLUSH_MS);
|
|
399
|
+
this.flush.unref?.();
|
|
400
|
+
}
|
|
401
|
+
}
|
|
402
|
+
/** Everything not yet kept, to the store. Never throws; a store that is away costs nothing but the keeping. */
|
|
403
|
+
async flushNow() {
|
|
404
|
+
if (this.flush)
|
|
405
|
+
clearTimeout(this.flush);
|
|
406
|
+
this.flush = null;
|
|
407
|
+
if (!this.media || !this.transcriptId)
|
|
408
|
+
return;
|
|
409
|
+
const session = this.options.session();
|
|
410
|
+
if (session === null)
|
|
411
|
+
return;
|
|
412
|
+
const batches = [...this.unsaved.entries()].filter(([, lines]) => lines.length > 0);
|
|
413
|
+
this.unsaved.clear();
|
|
414
|
+
for (const [language, lines] of batches) {
|
|
415
|
+
const got = await keepLines(session, this.transcriptId, {
|
|
416
|
+
media: this.media.media,
|
|
417
|
+
title: this.media.title,
|
|
418
|
+
language: language === "" ? this.language : language,
|
|
419
|
+
...(language === "" ? {} : { translatedFrom: this.language }),
|
|
420
|
+
...(this.model ? { model: this.model } : {}),
|
|
421
|
+
lines,
|
|
422
|
+
}, this.options.fetcher ?? fetch);
|
|
423
|
+
if (!got.ok)
|
|
424
|
+
this.complain(`the store did not keep ${lines.length} lines: ${got.error}`);
|
|
425
|
+
}
|
|
426
|
+
}
|
|
219
427
|
/** Said once a minute at most: a broken ear would otherwise say so twelve times a minute. */
|
|
220
428
|
complain(message) {
|
|
221
429
|
this.error = message;
|
|
222
|
-
const now =
|
|
430
|
+
const now = this.now();
|
|
223
431
|
if (now - this.complainedAt < 60_000)
|
|
224
432
|
return;
|
|
225
433
|
this.complainedAt = now;
|
|
226
434
|
this.options.onEvent?.(`captions for "${this.id}": ${message}`);
|
|
227
435
|
}
|
|
228
|
-
subscribe(subscriber) {
|
|
229
|
-
this.subscribers.
|
|
436
|
+
subscribe(subscriber, language = "") {
|
|
437
|
+
this.subscribers.set(subscriber, language);
|
|
438
|
+
if (language !== "")
|
|
439
|
+
void this.consult(language);
|
|
230
440
|
if (this.idle)
|
|
231
441
|
clearTimeout(this.idle);
|
|
232
442
|
this.idle = null;
|
|
@@ -241,6 +451,20 @@ class Captioner {
|
|
|
241
451
|
}
|
|
242
452
|
};
|
|
243
453
|
}
|
|
454
|
+
recent(after, language = "") {
|
|
455
|
+
const lines = language === "" || language === this.language ? this.lines : (this.linesBy.get(language) ?? []);
|
|
456
|
+
return after > 0 ? lines.filter((line) => line.at > after) : [...lines];
|
|
457
|
+
}
|
|
458
|
+
status() {
|
|
459
|
+
return {
|
|
460
|
+
on: true,
|
|
461
|
+
lines: this.lines.length,
|
|
462
|
+
error: this.error,
|
|
463
|
+
language: this.language,
|
|
464
|
+
known: this.known.get("")?.length ?? 0,
|
|
465
|
+
languages: [...this.linesBy.keys()],
|
|
466
|
+
};
|
|
467
|
+
}
|
|
244
468
|
stop() {
|
|
245
469
|
if (this.stopped)
|
|
246
470
|
return;
|
|
@@ -255,6 +479,7 @@ class Captioner {
|
|
|
255
479
|
this.pending = [];
|
|
256
480
|
this.pendingBytes = 0;
|
|
257
481
|
this.subscribers.clear();
|
|
482
|
+
void this.flushNow();
|
|
258
483
|
this.onStop();
|
|
259
484
|
}
|
|
260
485
|
}
|
|
@@ -272,9 +497,10 @@ export class Captions {
|
|
|
272
497
|
* Lines for a channel as they are heard, starting the captioner if it is
|
|
273
498
|
* not running. Null when there is no such channel. The returned function
|
|
274
499
|
* is how to stop listening; the captioner itself stops a minute after the
|
|
275
|
-
* last listener does.
|
|
500
|
+
* last listener does. A language asks for the lines translated into it;
|
|
501
|
+
* "" is the original.
|
|
276
502
|
*/
|
|
277
|
-
subscribe(id, subscriber) {
|
|
503
|
+
subscribe(id, subscriber, language = "") {
|
|
278
504
|
let captioner = this.running.get(id);
|
|
279
505
|
if (!captioner) {
|
|
280
506
|
const made = new Captioner(id, this.options, () => {
|
|
@@ -287,19 +513,17 @@ export class Captions {
|
|
|
287
513
|
this.options.onEvent?.(`captions for "${id}": started`);
|
|
288
514
|
captioner = made;
|
|
289
515
|
}
|
|
290
|
-
return captioner.subscribe(subscriber);
|
|
516
|
+
return captioner.subscribe(subscriber, language);
|
|
291
517
|
}
|
|
292
|
-
/** The recent lines of a channel, oldest first, after a moment when given. Empty when nobody has asked for them. */
|
|
293
|
-
recent(id, after = 0) {
|
|
518
|
+
/** The recent lines of a channel, oldest first, after a moment when given, in a language when asked. Empty when nobody has asked for them. */
|
|
519
|
+
recent(id, after = 0, language = "") {
|
|
294
520
|
const captioner = this.running.get(id);
|
|
295
|
-
|
|
296
|
-
return [];
|
|
297
|
-
return after > 0 ? captioner.lines.filter((line) => line.at > after) : [...captioner.lines];
|
|
521
|
+
return captioner ? captioner.recent(after, language) : [];
|
|
298
522
|
}
|
|
299
523
|
/** Whether a channel is being captioned, and what last went wrong if the lines are not coming. */
|
|
300
524
|
status(id) {
|
|
301
525
|
const captioner = this.running.get(id);
|
|
302
|
-
return captioner ? { on:
|
|
526
|
+
return captioner ? captioner.status() : { on: false, lines: 0, error: "", language: "", known: 0, languages: [] };
|
|
303
527
|
}
|
|
304
528
|
stopAll() {
|
|
305
529
|
for (const captioner of [...this.running.values()])
|