@tongue-core/ui 0.1.0-rc.7 → 0.1.0-rc.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +24 -0
- package/dist/player/assets.d.ts +26 -2
- package/dist/player/assets.js +84 -9
- package/dist/player/audio.d.ts +2 -1
- package/dist/player/audio.js +13 -3
- package/dist/player/frames.d.ts +4 -0
- package/dist/player/frames.js +12 -0
- package/dist/player/index.d.ts +1 -1
- package/dist/player/needs.d.ts +7 -1
- package/dist/player/needs.js +52 -8
- package/dist/player/player.d.ts +9 -3
- package/dist/player/player.js +87 -20
- package/dist/player/pool.d.ts +3 -0
- package/dist/player/pool.js +11 -2
- package/dist/player/protocol.d.ts +15 -0
- package/dist/player/protocol.js +12 -1
- package/dist/player/source.d.ts +3 -2
- package/dist/player/source.js +17 -5
- package/dist/player/video.d.ts +4 -0
- package/dist/player/video.js +90 -9
- package/dist/player/worker.js +36 -8
- package/dist/timeline/connect.js +1 -0
- package/dist/timeline/timeline.d.ts +1 -0
- package/package.json +4 -4
package/dist/player/player.js
CHANGED
|
@@ -7,6 +7,7 @@ import { Cancelled, WorkerPool } from "./pool.js";
|
|
|
7
7
|
import { bufferedAhead, clampFrame, startBuffer, wantedFrames } from "./schedule.js";
|
|
8
8
|
import { openFilm } from "./source.js";
|
|
9
9
|
import { Transport } from "./transport.js";
|
|
10
|
+
import { CANCELLED_PREVIEW } from "./protocol.js";
|
|
10
11
|
/** Which workers use a file: audio the soundtrack's, proxy segments the video worker, a video's
|
|
11
12
|
* facts both (frames lay out with them; the decoder reads its proxy from them), the rest frames. */
|
|
12
13
|
function fileFor(path, audio) {
|
|
@@ -82,8 +83,23 @@ export function createPlayer(options) {
|
|
|
82
83
|
emit("error", { message: text });
|
|
83
84
|
}
|
|
84
85
|
const bounds = (s) => ({ first: range?.from ?? 0, last: Math.min(s.length - 1, range?.to ?? s.length - 1) });
|
|
85
|
-
const frameReady = (s) => (frame) => s.needs.pathsAt(frame).every(s.loader.isReady);
|
|
86
|
-
const audioReady = (s) => s.needs.
|
|
86
|
+
const frameReady = (s) => (frame) => !s.blocked.has(frame) && s.needs.pathsAt(frame).every(s.loader.isReady);
|
|
87
|
+
const audioReady = (s) => s.needs.audioPathsIn(transport.playhead, transport.playhead + s.fps * 5).every(s.loader.isReady);
|
|
88
|
+
pool.onMediaNeed = (paths) => {
|
|
89
|
+
const s = session;
|
|
90
|
+
if (!s)
|
|
91
|
+
return;
|
|
92
|
+
for (const path of paths) {
|
|
93
|
+
s.demand.set(path, transport.playhead);
|
|
94
|
+
if (s.loader.status(path) === null)
|
|
95
|
+
s.loader.failMissing(path);
|
|
96
|
+
if (s.loader.status(path) === "failed")
|
|
97
|
+
pool.mediaFailed(path, s.loader.error(path) ?? "Media failed");
|
|
98
|
+
if (s.loader.isReady(path))
|
|
99
|
+
deliverTo(s, path);
|
|
100
|
+
}
|
|
101
|
+
s.loader.want(paths, 0);
|
|
102
|
+
};
|
|
87
103
|
function requestTick() {
|
|
88
104
|
if (tickQueued || disposed)
|
|
89
105
|
return;
|
|
@@ -131,7 +147,7 @@ export function createPlayer(options) {
|
|
|
131
147
|
const playing = transport.state !== "paused";
|
|
132
148
|
const wants = wantedFrames({
|
|
133
149
|
playhead: transport.playhead, playing, moving: !playing && moving(), first, last, fps: s.fps, scales, ahead,
|
|
134
|
-
cachedScale: (f) => cache.
|
|
150
|
+
cachedScale: (f) => { const value = cache.get(f); return value && (value.exact || !s.bounded) ? value.scale : null; },
|
|
135
151
|
rendering: (f, scale) => pool.wants(f, scale),
|
|
136
152
|
ready: frameReady(s),
|
|
137
153
|
});
|
|
@@ -145,22 +161,37 @@ export function createPlayer(options) {
|
|
|
145
161
|
if (session !== s)
|
|
146
162
|
return result.pixels.bitmap.close();
|
|
147
163
|
cache.put(result.frame, result.pixels.bitmap, result.scale, result.width * result.height * 4, !result.standIn);
|
|
164
|
+
if (s.bounded)
|
|
165
|
+
readinessDirty = true;
|
|
148
166
|
requestTick();
|
|
149
167
|
}, (e) => {
|
|
150
|
-
if (!(e instanceof Cancelled) && session === s)
|
|
168
|
+
if (!(e instanceof Cancelled) && session === s) {
|
|
169
|
+
s.blocked.add(want.frame);
|
|
151
170
|
report(message(e));
|
|
171
|
+
}
|
|
152
172
|
});
|
|
153
173
|
});
|
|
154
174
|
// Files: what the next few seconds need, now; the rest of the film was asked for at load.
|
|
155
|
-
const window = Math.round(s.fps * Math.max(2, ahead * 2));
|
|
156
|
-
s.
|
|
175
|
+
const window = Math.round(s.fps * Math.max(2, s.bounded ? options.mediaAhead ?? 15 : ahead * 2));
|
|
176
|
+
const paths = [...s.needs.pathsIn(Math.max(0, transport.playhead - s.fps), transport.playhead + window), ...s.needs.audioPathsIn(transport.playhead, transport.playhead + window + s.fps * 5)];
|
|
177
|
+
if (s.bounded) {
|
|
178
|
+
for (const [path, frame] of s.demand)
|
|
179
|
+
if (Math.abs(frame - transport.playhead) > s.fps * 5)
|
|
180
|
+
s.demand.delete(path);
|
|
181
|
+
paths.push(...s.demand.keys());
|
|
182
|
+
paths.push(...s.requested.keys());
|
|
183
|
+
s.loader.retain(paths);
|
|
184
|
+
}
|
|
185
|
+
s.loader.want(paths, 0);
|
|
157
186
|
if (playing)
|
|
158
187
|
soundtrack?.prepare(transport.playhead);
|
|
159
188
|
}
|
|
160
189
|
/** What is ready now (cheap: one pass over the spans). */
|
|
161
190
|
function readinessOf(s) {
|
|
162
|
-
const failed = s.
|
|
163
|
-
|
|
191
|
+
const failed = s.loader.failures();
|
|
192
|
+
// Dynamic scene video can choose arbitrary source times. Report cached exact frames
|
|
193
|
+
// conservatively until those dependencies have actually been evaluated.
|
|
194
|
+
return { ranges: s.dynamicVideo ? cache.exactRanges() : s.needs.readyRanges(s.loader.isReady), progress: s.loader.progress(s.bounded), failed, mediaBytes: s.loader.heldBytes };
|
|
164
195
|
}
|
|
165
196
|
/** Reports readiness while files arrive, a few times a second at most. */
|
|
166
197
|
function reportReadiness(s, force = false) {
|
|
@@ -183,8 +214,8 @@ export function createPlayer(options) {
|
|
|
183
214
|
const audible = clockKind === "audio";
|
|
184
215
|
const result = transport.tick({
|
|
185
216
|
now: now(), first, last,
|
|
186
|
-
showable: (f) => cache.has(f, scales.play),
|
|
187
|
-
canStart: (f) => bufferedAhead((g) => cache.
|
|
217
|
+
showable: (f) => cache.has(f, scales.play) && (!s.bounded || cache.get(f)?.exact === true),
|
|
218
|
+
canStart: (f) => bufferedAhead((g) => { const value = cache.get(g); return value && (value.exact || !s.bounded) ? value.scale : null; }, f, last, scales.play) >= startBuffer(f, last, s.fps) && (!audible || (audioReady(s) && (soundtrack?.ready(f) ?? true))),
|
|
188
219
|
});
|
|
189
220
|
if (result.started && audible)
|
|
190
221
|
soundtrack?.start(result.frame, now());
|
|
@@ -218,8 +249,11 @@ export function createPlayer(options) {
|
|
|
218
249
|
if (mine !== generation)
|
|
219
250
|
throw new Cancelled();
|
|
220
251
|
const doc = opened.doc;
|
|
221
|
-
const
|
|
222
|
-
const
|
|
252
|
+
const bounded = opened.assets.some((file) => file.url !== undefined);
|
|
253
|
+
const needs = FilmNeeds.of(doc, opened.assets.map((a) => a.path), { demandVideo: bounded });
|
|
254
|
+
const loader = new AssetLoader(opened.assets, opened.fetchAsset, 6, bounded ? options.mediaCacheBytes ?? 64 * 1024 * 1024 : Infinity);
|
|
255
|
+
loader.onEvict = (path) => pool.forget(path);
|
|
256
|
+
loader.onCancel = (path) => pool.mediaFailed(path, CANCELLED_PREVIEW);
|
|
223
257
|
const previous = session;
|
|
224
258
|
const carried = previous ? loader.carryOver(previous.loader) : [];
|
|
225
259
|
const info = await pool.open(opened.manifest, opened.files).catch(async (e) => {
|
|
@@ -241,13 +275,21 @@ export function createPlayer(options) {
|
|
|
241
275
|
transport.pause();
|
|
242
276
|
transport = new Transport(doc.film.fps);
|
|
243
277
|
soundtrack?.dispose();
|
|
244
|
-
soundtrack = new Soundtrack({ fps: doc.film.fps, length: doc.film.length,
|
|
278
|
+
soundtrack = new Soundtrack({ fps: doc.film.fps, length: doc.film.length, ahead: bounded ? Math.max(1, Math.ceil((options.mediaAhead ?? 15) / 5)) : 3, mix: async (from, to, signal) => {
|
|
279
|
+
const s = session;
|
|
280
|
+
if (!s)
|
|
281
|
+
throw new Cancelled();
|
|
282
|
+
await filesFor(s, s.needs.audioPathsIn(from, to), 1, signal);
|
|
283
|
+
return pool.mix(from, to, 5);
|
|
284
|
+
} });
|
|
245
285
|
cache.clear();
|
|
246
286
|
shownKey = "";
|
|
247
287
|
shown = null;
|
|
248
288
|
if (range && (range.from >= doc.film.length || range.to >= doc.film.length))
|
|
249
289
|
range = null;
|
|
250
|
-
const
|
|
290
|
+
const videoScenes = new Set(doc.scenes.filter((scene) => scene.assets.some((asset) => asset.kind === "video")).map((scene) => scene.id));
|
|
291
|
+
const dynamicVideo = bounded && doc.tracks.some((track) => track.clips.some((clip) => clip.kind === "scene" && videoScenes.has(clip.scene)));
|
|
292
|
+
const s = { generation: mine, source: opened, needs, loader, bounded, dynamicVideo, blocked: new Set(), demand: new Map(), requested: new Map(), film: { doc, build: opened.build, info }, fps: doc.film.fps, length: doc.film.length };
|
|
251
293
|
session = s;
|
|
252
294
|
const deliver = (path) => deliverTo(s, path);
|
|
253
295
|
for (const path of carried)
|
|
@@ -255,8 +297,12 @@ export function createPlayer(options) {
|
|
|
255
297
|
loader.onChange((path) => {
|
|
256
298
|
if (session !== s)
|
|
257
299
|
return;
|
|
258
|
-
if (loader.isReady(path))
|
|
300
|
+
if (loader.isReady(path)) {
|
|
301
|
+
s.blocked.clear();
|
|
259
302
|
deliver(path);
|
|
303
|
+
}
|
|
304
|
+
if (loader.status(path) === "failed")
|
|
305
|
+
pool.mediaFailed(path, loader.error(path) ?? "Media failed");
|
|
260
306
|
readinessDirty = true;
|
|
261
307
|
requestTick();
|
|
262
308
|
});
|
|
@@ -268,8 +314,9 @@ export function createPlayer(options) {
|
|
|
268
314
|
transport.play(0, s.length - 1);
|
|
269
315
|
// The playhead's files first, then the rest of the film in the order it needs them.
|
|
270
316
|
loader.want(needs.pathsAt(transport.playhead), 0);
|
|
271
|
-
|
|
272
|
-
|
|
317
|
+
if (!bounded)
|
|
318
|
+
for (const [path, at] of needs.firstNeeded())
|
|
319
|
+
loader.want([path], 10 + at / s.fps);
|
|
273
320
|
reportReadiness(s, true);
|
|
274
321
|
emit("load", s.film);
|
|
275
322
|
requestTick();
|
|
@@ -287,7 +334,7 @@ export function createPlayer(options) {
|
|
|
287
334
|
if (data)
|
|
288
335
|
pool.file(path, data, fileFor(path, audio));
|
|
289
336
|
// Chunks mixed before an audio file arrived are silent where it plays: mix them again.
|
|
290
|
-
if (audio.has(path))
|
|
337
|
+
if (audio.has(path) && !s.bounded)
|
|
291
338
|
soundtrack?.reset();
|
|
292
339
|
}
|
|
293
340
|
function resize(s) {
|
|
@@ -302,6 +349,8 @@ export function createPlayer(options) {
|
|
|
302
349
|
/** Resolves once every file of `paths` is loaded; rejects with the first that failed, or with
|
|
303
350
|
* Cancelled when `signal` aborts first. */
|
|
304
351
|
function filesFor(s, paths, priority, signal) {
|
|
352
|
+
for (const path of paths)
|
|
353
|
+
s.requested.set(path, (s.requested.get(path) ?? 0) + 1);
|
|
305
354
|
s.loader.want(paths, priority);
|
|
306
355
|
const failed = () => paths.find((p) => s.loader.status(p) === "failed");
|
|
307
356
|
const done = () => failed() !== undefined || paths.every(s.loader.isReady);
|
|
@@ -329,9 +378,18 @@ export function createPlayer(options) {
|
|
|
329
378
|
settle(resolve, reject);
|
|
330
379
|
});
|
|
331
380
|
signal?.addEventListener("abort", abort, { once: true });
|
|
381
|
+
}).finally(() => {
|
|
382
|
+
for (const path of paths) {
|
|
383
|
+
const count = s.requested.get(path) ?? 0;
|
|
384
|
+
if (count <= 1)
|
|
385
|
+
s.requested.delete(path);
|
|
386
|
+
else
|
|
387
|
+
s.requested.set(path, count - 1);
|
|
388
|
+
}
|
|
332
389
|
});
|
|
333
390
|
}
|
|
334
391
|
const player = {
|
|
392
|
+
retry() { reportedErrors.clear(); session?.blocked.clear(); session?.loader.retry(); requestTick(); },
|
|
335
393
|
load,
|
|
336
394
|
reload: () => {
|
|
337
395
|
if (!lastSource)
|
|
@@ -345,7 +403,10 @@ export function createPlayer(options) {
|
|
|
345
403
|
return;
|
|
346
404
|
}
|
|
347
405
|
const { first, last } = bounds(s);
|
|
348
|
-
|
|
406
|
+
if (s.bounded && Math.round(frame) !== transport.playhead)
|
|
407
|
+
soundtrack?.reset();
|
|
408
|
+
else
|
|
409
|
+
soundtrack?.stop();
|
|
349
410
|
if (Math.round(frame) !== transport.playhead) {
|
|
350
411
|
movedBefore = movedAt;
|
|
351
412
|
movedAt = performance.now();
|
|
@@ -405,6 +466,10 @@ export function createPlayer(options) {
|
|
|
405
466
|
const result = await pool.frame(f, Math.min(1, width / s.film.doc.film.width), { output: "bitmap", priority: 1000, signal });
|
|
406
467
|
if (result.pixels.kind !== "bitmap")
|
|
407
468
|
throw new Error("The worker sent bytes for a preview.");
|
|
469
|
+
if (result.standIn) {
|
|
470
|
+
result.pixels.bitmap.close();
|
|
471
|
+
throw new Error("The requested video preview is unavailable.");
|
|
472
|
+
}
|
|
408
473
|
return result.pixels.bitmap;
|
|
409
474
|
},
|
|
410
475
|
async pixels(frame, scale = 1) {
|
|
@@ -416,6 +481,8 @@ export function createPlayer(options) {
|
|
|
416
481
|
const result = await pool.frame(f, scale, { output: "rgba", priority: 100 });
|
|
417
482
|
if (result.pixels.kind !== "rgba")
|
|
418
483
|
throw new Error("The worker sent a bitmap for pixels.");
|
|
484
|
+
if (result.standIn)
|
|
485
|
+
throw new Error("The requested frame contains unavailable video.");
|
|
419
486
|
return { width: result.width, height: result.height, rgba: result.pixels.rgba };
|
|
420
487
|
},
|
|
421
488
|
async mix(from, to) {
|
|
@@ -425,7 +492,7 @@ export function createPlayer(options) {
|
|
|
425
492
|
const a = clampFrame(Math.min(from, to), 0, s.length), b = clampFrame(Math.max(from, to), 0, s.length);
|
|
426
493
|
if (b <= a)
|
|
427
494
|
return { sampleRate: 48000, left: new Float32Array(0), right: new Float32Array(0) };
|
|
428
|
-
await filesFor(s, s.needs.
|
|
495
|
+
await filesFor(s, s.needs.audioPathsIn(from, to), 1);
|
|
429
496
|
return pool.mix(a, b, 50);
|
|
430
497
|
},
|
|
431
498
|
on(event, listener) {
|
package/dist/player/pool.d.ts
CHANGED
|
@@ -46,6 +46,7 @@ export declare class WorkerPool {
|
|
|
46
46
|
private opening;
|
|
47
47
|
/** Called when a worker reports a failure that isn't a request's (a file it couldn't read). */
|
|
48
48
|
onProblem: (message: string) => void;
|
|
49
|
+
onMediaNeed: (paths: ReadonlyArray<string>) => void;
|
|
49
50
|
constructor(options: PoolOptions);
|
|
50
51
|
get size(): number;
|
|
51
52
|
/** Opens a build in every worker (dropping whatever was asked of the last one); resolves once all have. */
|
|
@@ -53,6 +54,8 @@ export declare class WorkerPool {
|
|
|
53
54
|
/** Sends an imported file to the workers it is for: audio to the soundtrack's worker only
|
|
54
55
|
* (decoded there), proxy segments to the video worker. */
|
|
55
56
|
file(path: string, bytes: Uint8Array, to: FileFor): void;
|
|
57
|
+
forget(path: string): void;
|
|
58
|
+
mediaFailed(path: string, message: string): void;
|
|
56
59
|
/** Draws a frame. Rejects with Cancelled if it is dropped (or its signal aborts) before a worker takes it. */
|
|
57
60
|
frame(frame: number, scale: number, request: FrameRequest): Promise<FrameResult>;
|
|
58
61
|
/** Mixes the soundtrack between two frames (on the soundtrack's worker). */
|
package/dist/player/pool.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { parseFromWorker } from "./protocol.js";
|
|
1
|
+
import { parseFromWorker, parseMediaNeed } from "./protocol.js";
|
|
2
2
|
export class Cancelled extends Error {
|
|
3
3
|
name = "Cancelled";
|
|
4
4
|
constructor() {
|
|
@@ -13,11 +13,14 @@ export class WorkerPool {
|
|
|
13
13
|
opening = null;
|
|
14
14
|
/** Called when a worker reports a failure that isn't a request's (a file it couldn't read). */
|
|
15
15
|
onProblem = () => { };
|
|
16
|
+
onMediaNeed = () => { };
|
|
16
17
|
constructor(options) {
|
|
17
18
|
const videoWorker = new Worker(options.workerUrl, { type: "module" });
|
|
18
19
|
const videoChannel = new MessageChannel();
|
|
19
20
|
videoWorker.postMessage({ role: "video" }, [videoChannel.port2]);
|
|
20
21
|
this.video = { worker: videoWorker, port: videoChannel.port1 };
|
|
22
|
+
this.video.port.onmessage = (event) => { const paths = parseMediaNeed(event.data); if (paths)
|
|
23
|
+
this.onMediaNeed(paths); };
|
|
21
24
|
this.slots = Array.from({ length: Math.max(1, options.workers) }, () => {
|
|
22
25
|
const worker = new Worker(options.workerUrl, { type: "module" });
|
|
23
26
|
const channel = new MessageChannel();
|
|
@@ -70,6 +73,12 @@ export class WorkerPool {
|
|
|
70
73
|
for (const slot of this.slots)
|
|
71
74
|
this.post(slot, { kind: "file", path, bytes, audio: false });
|
|
72
75
|
}
|
|
76
|
+
forget(path) {
|
|
77
|
+
this.video.port.postMessage({ kind: "forget", path });
|
|
78
|
+
for (const slot of this.slots)
|
|
79
|
+
this.post(slot, { kind: "forget", path });
|
|
80
|
+
}
|
|
81
|
+
mediaFailed(path, message) { this.video.port.postMessage({ kind: "failure", path, message }); }
|
|
73
82
|
/** Draws a frame. Rejects with Cancelled if it is dropped (or its signal aborts) before a worker takes it. */
|
|
74
83
|
frame(frame, scale, request) {
|
|
75
84
|
return new Promise((resolve, reject) => this.enqueue({ kind: "frame", frame, scale, output: request.output, playback: request.playback ?? false, resolve }, request.priority, reject, request.signal));
|
|
@@ -184,7 +193,7 @@ export class WorkerPool {
|
|
|
184
193
|
break;
|
|
185
194
|
}
|
|
186
195
|
if (slot.current?.id === msg.id) {
|
|
187
|
-
slot.current.reject(new Error(msg.message));
|
|
196
|
+
slot.current.reject(msg.cancelled ? new Cancelled() : new Error(msg.message));
|
|
188
197
|
slot.current = null;
|
|
189
198
|
}
|
|
190
199
|
break;
|
|
@@ -4,6 +4,9 @@ export type Role = "frames" | "video";
|
|
|
4
4
|
export declare function parseRole(data: unknown): Role | null;
|
|
5
5
|
export type FrameOutput = "bitmap" | "rgba";
|
|
6
6
|
export type ToWorker = {
|
|
7
|
+
readonly kind: "forget";
|
|
8
|
+
readonly path: string;
|
|
9
|
+
} | {
|
|
7
10
|
readonly kind: "open";
|
|
8
11
|
readonly manifest: string;
|
|
9
12
|
readonly files: Readonly<Record<string, string>>;
|
|
@@ -58,12 +61,23 @@ export type FromWorker = {
|
|
|
58
61
|
readonly kind: "error";
|
|
59
62
|
readonly id: number | null;
|
|
60
63
|
readonly message: string;
|
|
64
|
+
readonly cancelled?: boolean;
|
|
61
65
|
};
|
|
66
|
+
export declare const CANCELLED_PREVIEW = "The preview was superseded by a seek.";
|
|
67
|
+
export declare class MediaCancelled extends Error {
|
|
68
|
+
}
|
|
62
69
|
export declare function parseToWorker(data: unknown): ToWorker | null;
|
|
63
70
|
/** A plan summary (summarizePlan in @tongue-core/film), checked field by field. */
|
|
64
71
|
export declare function parsePlanSummary(v: unknown): PlanSummary | null;
|
|
65
72
|
export declare function parseFromWorker(data: unknown): FromWorker | null;
|
|
66
73
|
export type ToVideo = {
|
|
74
|
+
readonly kind: "failure";
|
|
75
|
+
readonly path: string;
|
|
76
|
+
readonly message: string;
|
|
77
|
+
} | {
|
|
78
|
+
readonly kind: "forget";
|
|
79
|
+
readonly path: string;
|
|
80
|
+
} | {
|
|
67
81
|
readonly kind: "file";
|
|
68
82
|
readonly path: string;
|
|
69
83
|
readonly bytes: Uint8Array;
|
|
@@ -75,6 +89,7 @@ export type ToVideo = {
|
|
|
75
89
|
readonly kind: "client";
|
|
76
90
|
};
|
|
77
91
|
export declare function parseToVideo(data: unknown): ToVideo | null;
|
|
92
|
+
export declare function parseMediaNeed(data: unknown): ReadonlyArray<string> | null;
|
|
78
93
|
/** A frame of a video, by the video's name and its own frame numbering. */
|
|
79
94
|
export interface VideoFrameWant {
|
|
80
95
|
readonly video: string;
|
package/dist/player/protocol.js
CHANGED
|
@@ -2,6 +2,9 @@
|
|
|
2
2
|
export function parseRole(data) {
|
|
3
3
|
return isRecord(data) && (data["role"] === "frames" || data["role"] === "video") ? data["role"] : null;
|
|
4
4
|
}
|
|
5
|
+
export const CANCELLED_PREVIEW = "The preview was superseded by a seek.";
|
|
6
|
+
export class MediaCancelled extends Error {
|
|
7
|
+
}
|
|
5
8
|
const isRecord = (v) => typeof v === "object" && v !== null && !Array.isArray(v);
|
|
6
9
|
const isNumber = (v) => typeof v === "number" && Number.isFinite(v);
|
|
7
10
|
const isString = (v) => typeof v === "string";
|
|
@@ -22,6 +25,7 @@ export function parseToWorker(data) {
|
|
|
22
25
|
if (!isRecord(data))
|
|
23
26
|
return null;
|
|
24
27
|
switch (data["kind"]) {
|
|
28
|
+
case "forget": return isString(data["path"]) ? { kind: "forget", path: data["path"] } : null;
|
|
25
29
|
case "open": {
|
|
26
30
|
const manifest = data["manifest"], files = textFiles(data["files"]);
|
|
27
31
|
return isString(manifest) && files ? { kind: "open", manifest, files } : null;
|
|
@@ -131,7 +135,7 @@ export function parseFromWorker(data) {
|
|
|
131
135
|
}
|
|
132
136
|
case "error": {
|
|
133
137
|
const id = data["id"], message = data["message"];
|
|
134
|
-
return { kind: "error", id: isNumber(id) ? id : null, message: isString(message) ? message : "unknown error" };
|
|
138
|
+
return { kind: "error", id: isNumber(id) ? id : null, message: isString(message) ? message : "unknown error", ...(data["cancelled"] === true ? { cancelled: true } : {}) };
|
|
135
139
|
}
|
|
136
140
|
default:
|
|
137
141
|
return null;
|
|
@@ -141,6 +145,8 @@ export function parseToVideo(data) {
|
|
|
141
145
|
if (!isRecord(data))
|
|
142
146
|
return null;
|
|
143
147
|
switch (data["kind"]) {
|
|
148
|
+
case "failure": return isString(data["path"]) && isString(data["message"]) ? { kind: "failure", path: data["path"], message: data["message"] } : null;
|
|
149
|
+
case "forget": return isString(data["path"]) ? { kind: "forget", path: data["path"] } : null;
|
|
144
150
|
case "file": {
|
|
145
151
|
const path = data["path"], bytes = data["bytes"];
|
|
146
152
|
return isString(path) && bytes instanceof Uint8Array ? { kind: "file", path, bytes } : null;
|
|
@@ -153,6 +159,11 @@ export function parseToVideo(data) {
|
|
|
153
159
|
return null;
|
|
154
160
|
}
|
|
155
161
|
}
|
|
162
|
+
export function parseMediaNeed(data) {
|
|
163
|
+
if (!isRecord(data) || data["kind"] !== "need")
|
|
164
|
+
return null;
|
|
165
|
+
return listOf(data["paths"], (value) => isString(value) ? value : null);
|
|
166
|
+
}
|
|
156
167
|
function parseWant(v) {
|
|
157
168
|
if (!isRecord(v))
|
|
158
169
|
return null;
|
package/dist/player/source.d.ts
CHANGED
|
@@ -1,14 +1,15 @@
|
|
|
1
1
|
import { type BuildDoc, type FilmDoc } from "@tongue-core/project/doc";
|
|
2
2
|
import type { AssetFileInfo, FetchBytes } from "./assets.js";
|
|
3
|
+
import { type MediaRequestOptions } from "@tongue-core/project/media";
|
|
3
4
|
/** A build served as files: its folder's URL. */
|
|
4
|
-
export interface BuildSource {
|
|
5
|
+
export interface BuildSource extends MediaRequestOptions {
|
|
5
6
|
/** The build's folder (where build.json is), absolute or relative to the page. */
|
|
6
7
|
readonly build: string | URL;
|
|
7
8
|
/** Another manifest in the build, such as one scene on its own. Default build.json. */
|
|
8
9
|
readonly manifest?: string;
|
|
9
10
|
}
|
|
10
11
|
/** A build read a file at a time, from wherever it is kept: a Git commit, memory, an archive. */
|
|
11
|
-
export interface FileSource {
|
|
12
|
+
export interface FileSource extends MediaRequestOptions {
|
|
12
13
|
/** One file of the build, by its path in the build as build.json and film.json give it:
|
|
13
14
|
* `build.json`, `film.json`, `scenes/title.5ebf1e9a05d5.js`, `assets/fonts/inter.ttf`.
|
|
14
15
|
* Rejects when the file can't be read. */
|
package/dist/player/source.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { parseBuildDoc, parseFilmDoc } from "@tongue-core/project/doc";
|
|
2
|
-
import { Effect } from "effect";
|
|
2
|
+
import { Cause, Effect, Exit } from "effect";
|
|
3
|
+
import { readMedia } from "@tongue-core/project/media";
|
|
3
4
|
export class SourceError extends Error {
|
|
4
5
|
name = "SourceError";
|
|
5
6
|
}
|
|
@@ -22,7 +23,7 @@ async function fetchOk(url, signal) {
|
|
|
22
23
|
const message = (e) => (e instanceof Error ? e.message : String(e));
|
|
23
24
|
const failure = (e) => new SourceError(message(e));
|
|
24
25
|
/** Opens a build: its manifest and film.json, then every scene bundle at once. */
|
|
25
|
-
async function open(manifest, reader) {
|
|
26
|
+
async function open(manifest, reader, options = {}) {
|
|
26
27
|
const buildText = await reader.text(manifest);
|
|
27
28
|
const build = await Effect.runPromise(parseBuildDoc(buildText)).catch((e) => { throw failure(e); });
|
|
28
29
|
const filmText = await reader.text(build.film);
|
|
@@ -31,7 +32,18 @@ async function open(manifest, reader) {
|
|
|
31
32
|
const files = { [manifest]: buildText, [build.film]: filmText };
|
|
32
33
|
for (const [path, code] of bundles)
|
|
33
34
|
files[path] = code;
|
|
34
|
-
|
|
35
|
+
const local = reader.assets(build.assets.dir);
|
|
36
|
+
const locations = new Map(build.assets.files.map((file) => [file.path, file.url]));
|
|
37
|
+
const fetchAsset = async (path, signal) => {
|
|
38
|
+
const url = locations.get(path);
|
|
39
|
+
if (url === undefined)
|
|
40
|
+
return local(path, signal);
|
|
41
|
+
const result = await Effect.runPromiseExit(readMedia(path, url, async (response) => new Uint8Array(await response.arrayBuffer()), options), { signal });
|
|
42
|
+
if (Exit.isSuccess(result))
|
|
43
|
+
return result.value;
|
|
44
|
+
throw Cause.squash(result.cause);
|
|
45
|
+
};
|
|
46
|
+
return { manifest, build, doc, files, assets: build.assets.files, fetchAsset };
|
|
35
47
|
}
|
|
36
48
|
/** Opens a build served as files: every path resolved against its folder's URL. */
|
|
37
49
|
export async function openSource(source, page, signal) {
|
|
@@ -42,7 +54,7 @@ export async function openSource(source, page, signal) {
|
|
|
42
54
|
const assetsBase = buildBase(dir, base.href);
|
|
43
55
|
return async (path, abort) => new Uint8Array(await (await fetchOk(new URL(path, assetsBase), abort)).arrayBuffer());
|
|
44
56
|
},
|
|
45
|
-
});
|
|
57
|
+
}, source);
|
|
46
58
|
return { ...opened, base };
|
|
47
59
|
}
|
|
48
60
|
/** A file in one of the build's folders: "assets" and "fonts/inter.ttf" → "assets/fonts/inter.ttf". */
|
|
@@ -59,7 +71,7 @@ export async function openFiles(source, signal) {
|
|
|
59
71
|
return open(source.manifest ?? "build.json", {
|
|
60
72
|
text: async (path) => utf8.decode(await bytes(path, signal)),
|
|
61
73
|
assets: (dir) => (path, abort) => bytes(inFolder(dir, path), abort),
|
|
62
|
-
});
|
|
74
|
+
}, source);
|
|
63
75
|
}
|
|
64
76
|
/** Opens either kind of source; a served build's folder is resolved against `page`. */
|
|
65
77
|
export function openFilm(source, page, signal) {
|
package/dist/player/video.d.ts
CHANGED
|
@@ -10,11 +10,15 @@ export declare class VideoError extends Error {
|
|
|
10
10
|
}
|
|
11
11
|
/** Every video's proxy, by name. Files arrive from the player: info.json first, then segments. */
|
|
12
12
|
export declare class ProxyVideos {
|
|
13
|
+
private readonly need;
|
|
14
|
+
constructor(need?: (path: string) => void);
|
|
13
15
|
private readonly videos;
|
|
14
16
|
private readonly early;
|
|
15
17
|
/** Takes a file the player sent: a video's info.json (with its proxy facts) or a segment. */
|
|
16
18
|
file(path: string, bytes: Uint8Array, proxyOf: (bytes: Uint8Array) => ProxyInfo | null): void;
|
|
17
19
|
has(name: string): boolean;
|
|
20
|
+
forget(path: string): void;
|
|
21
|
+
fail(path: string, message: string): void;
|
|
18
22
|
frame(name: string, index: number): Promise<DecodedFrame>;
|
|
19
23
|
reset(): void;
|
|
20
24
|
}
|
package/dist/player/video.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { demuxMp4, encodeQoi, sampleData, writeYuvHeader, YUV_HEADER, yuvImageLayout, yuvSize } from "@tongue-core/engine";
|
|
2
|
+
import { CANCELLED_PREVIEW } from "./protocol.js";
|
|
2
3
|
/** Bytes of decoded frames kept per video (about a dozen 720p frames). */
|
|
3
4
|
const RING_BYTES = 48 * 1024 * 1024;
|
|
4
5
|
/** Decode on from the last frame, rather than from a keyframe, when the next is this close ahead. */
|
|
@@ -8,7 +9,7 @@ const FORWARD = 45;
|
|
|
8
9
|
* the frame itself is kept: the ones before it would only delay it. */
|
|
9
10
|
const KEEP_BEHIND = 8;
|
|
10
11
|
/** How long to wait for a segment the player hasn't sent yet, and for a frame to come out. */
|
|
11
|
-
const SEGMENT_WAIT_MS =
|
|
12
|
+
const SEGMENT_WAIT_MS = 120_000;
|
|
12
13
|
const OUTPUT_WAIT_MS = 40;
|
|
13
14
|
export class VideoError extends Error {
|
|
14
15
|
name = "VideoError";
|
|
@@ -34,6 +35,8 @@ async function imageOf(frame) {
|
|
|
34
35
|
class ProxyVideo {
|
|
35
36
|
name;
|
|
36
37
|
info;
|
|
38
|
+
need;
|
|
39
|
+
failures = new Map();
|
|
37
40
|
segments = new Map();
|
|
38
41
|
demuxed = new Map();
|
|
39
42
|
arrivals = new Map();
|
|
@@ -48,18 +51,49 @@ class ProxyVideo {
|
|
|
48
51
|
waiting = new Map();
|
|
49
52
|
queue = Promise.resolve();
|
|
50
53
|
total;
|
|
51
|
-
constructor(name, info) {
|
|
54
|
+
constructor(name, info, need) {
|
|
52
55
|
this.name = name;
|
|
53
56
|
this.info = info;
|
|
57
|
+
this.need = need;
|
|
54
58
|
const last = info.segments[info.segments.length - 1];
|
|
55
59
|
this.total = last ? last.first + last.frames : 0;
|
|
56
60
|
}
|
|
57
61
|
addSegment(index, bytes) {
|
|
62
|
+
this.failures.delete(index);
|
|
58
63
|
this.segments.set(index, bytes);
|
|
59
64
|
for (const arrived of this.arrivals.get(index) ?? [])
|
|
60
65
|
arrived();
|
|
61
66
|
this.arrivals.delete(index);
|
|
62
67
|
}
|
|
68
|
+
fail(index, message) {
|
|
69
|
+
this.failures.set(index, message);
|
|
70
|
+
for (const arrived of this.arrivals.get(index) ?? [])
|
|
71
|
+
arrived();
|
|
72
|
+
this.arrivals.delete(index);
|
|
73
|
+
}
|
|
74
|
+
forget(index) {
|
|
75
|
+
this.segments.delete(index);
|
|
76
|
+
this.demuxed.delete(index);
|
|
77
|
+
const segment = this.info.segments[index];
|
|
78
|
+
if (segment)
|
|
79
|
+
for (const [frame, data] of this.ring)
|
|
80
|
+
if (frame >= segment.first && frame < segment.first + segment.frames) {
|
|
81
|
+
this.ring.delete(frame);
|
|
82
|
+
this.ringBytes -= data.image.length;
|
|
83
|
+
}
|
|
84
|
+
if (this.segments.size === 0)
|
|
85
|
+
this.closeDecoder();
|
|
86
|
+
}
|
|
87
|
+
get bytes() { return this.ringBytes; }
|
|
88
|
+
trim(bytes) {
|
|
89
|
+
while (this.ringBytes > bytes && this.ring.size > 0) {
|
|
90
|
+
const first = this.ring.entries().next();
|
|
91
|
+
if (first.done)
|
|
92
|
+
break;
|
|
93
|
+
this.ring.delete(first.value[0]);
|
|
94
|
+
this.ringBytes -= first.value[1].image.length;
|
|
95
|
+
}
|
|
96
|
+
}
|
|
63
97
|
/** A frame by the video's own numbering (clamped to the proxy's frames). Requests run one at a time. */
|
|
64
98
|
frame(index) {
|
|
65
99
|
const job = this.queue.then(() => this.decodeTo(Math.max(0, Math.min(this.total - 1, Math.round(index)))));
|
|
@@ -67,9 +101,20 @@ class ProxyVideo {
|
|
|
67
101
|
return job;
|
|
68
102
|
}
|
|
69
103
|
close() {
|
|
104
|
+
this.closeDecoder();
|
|
105
|
+
for (const [index, listeners] of this.arrivals) {
|
|
106
|
+
this.failures.set(index, CANCELLED_PREVIEW);
|
|
107
|
+
for (const arrived of listeners)
|
|
108
|
+
arrived();
|
|
109
|
+
}
|
|
110
|
+
this.arrivals.clear();
|
|
111
|
+
}
|
|
112
|
+
closeDecoder() {
|
|
70
113
|
if (this.decoder && this.decoder.state !== "closed")
|
|
71
114
|
this.decoder.close();
|
|
72
115
|
this.decoder = null;
|
|
116
|
+
this.next = -1;
|
|
117
|
+
this.keep.clear();
|
|
73
118
|
for (const list of this.waiting.values())
|
|
74
119
|
for (const w of list)
|
|
75
120
|
w(null);
|
|
@@ -91,14 +136,23 @@ class ProxyVideo {
|
|
|
91
136
|
let bytes = this.segments.get(k);
|
|
92
137
|
if (!bytes) {
|
|
93
138
|
await new Promise((resolve, reject) => {
|
|
94
|
-
const
|
|
139
|
+
const arrived = () => { clearTimeout(timer); resolve(); };
|
|
140
|
+
const timer = setTimeout(() => {
|
|
141
|
+
const pending = this.arrivals.get(k)?.filter((listener) => listener !== arrived) ?? [];
|
|
142
|
+
if (pending.length === 0)
|
|
143
|
+
this.arrivals.delete(k);
|
|
144
|
+
else
|
|
145
|
+
this.arrivals.set(k, pending);
|
|
146
|
+
reject(new VideoError(`video "${this.name}": proxy segment ${k} never arrived`));
|
|
147
|
+
}, SEGMENT_WAIT_MS);
|
|
95
148
|
const list = this.arrivals.get(k) ?? [];
|
|
96
|
-
list.push(
|
|
149
|
+
list.push(arrived);
|
|
97
150
|
this.arrivals.set(k, list);
|
|
151
|
+
this.need(`video/${this.name}/proxy/${String(k).padStart(5, "0")}.mp4`);
|
|
98
152
|
});
|
|
99
153
|
bytes = this.segments.get(k);
|
|
100
154
|
if (!bytes)
|
|
101
|
-
throw new VideoError(`video "${this.name}": proxy segment ${k} is missing`);
|
|
155
|
+
throw new VideoError(this.failures.get(k) ?? `video "${this.name}": proxy segment ${k} is missing`);
|
|
102
156
|
}
|
|
103
157
|
let video = this.demuxed.get(k);
|
|
104
158
|
if (!video) {
|
|
@@ -205,12 +259,16 @@ class ProxyVideo {
|
|
|
205
259
|
const decoded = await result;
|
|
206
260
|
clearTimeout(timer);
|
|
207
261
|
if (!decoded)
|
|
208
|
-
throw new VideoError(`video "${this.name}": frame ${index} didn't decode`);
|
|
262
|
+
throw new VideoError(decoder.state === "closed" ? CANCELLED_PREVIEW : `video "${this.name}": frame ${index} didn't decode`);
|
|
209
263
|
return decoded;
|
|
210
264
|
}
|
|
211
265
|
}
|
|
212
266
|
/** Every video's proxy, by name. Files arrive from the player: info.json first, then segments. */
|
|
213
267
|
export class ProxyVideos {
|
|
268
|
+
need;
|
|
269
|
+
constructor(need = () => { }) {
|
|
270
|
+
this.need = need;
|
|
271
|
+
}
|
|
214
272
|
videos = new Map();
|
|
215
273
|
early = new Map();
|
|
216
274
|
/** Takes a file the player sent: a video's info.json (with its proxy facts) or a segment. */
|
|
@@ -221,7 +279,7 @@ export class ProxyVideos {
|
|
|
221
279
|
if (!proxy)
|
|
222
280
|
return;
|
|
223
281
|
this.videos.get(info[1])?.close();
|
|
224
|
-
const video = new ProxyVideo(info[1], proxy);
|
|
282
|
+
const video = new ProxyVideo(info[1], proxy, this.need);
|
|
225
283
|
this.videos.set(info[1], video);
|
|
226
284
|
for (const [k, segment] of this.early.get(info[1]) ?? [])
|
|
227
285
|
video.addSegment(k, segment);
|
|
@@ -244,11 +302,34 @@ export class ProxyVideos {
|
|
|
244
302
|
has(name) {
|
|
245
303
|
return this.videos.has(name);
|
|
246
304
|
}
|
|
247
|
-
|
|
305
|
+
forget(path) {
|
|
306
|
+
const match = /^video\/([^/]+)\/proxy\/(\d+)\.mp4$/.exec(path);
|
|
307
|
+
if (match?.[1] !== undefined) {
|
|
308
|
+
this.videos.get(match[1])?.forget(Number(match[2]));
|
|
309
|
+
this.early.get(match[1])?.delete(Number(match[2]));
|
|
310
|
+
}
|
|
311
|
+
}
|
|
312
|
+
fail(path, message) {
|
|
313
|
+
const match = /^video\/([^/]+)\/proxy\/(\d+)\.mp4$/.exec(path);
|
|
314
|
+
if (match?.[1] !== undefined)
|
|
315
|
+
this.videos.get(match[1])?.fail(Number(match[2]), message);
|
|
316
|
+
}
|
|
317
|
+
async frame(name, index) {
|
|
248
318
|
const video = this.videos.get(name);
|
|
249
319
|
if (!video)
|
|
250
320
|
return Promise.reject(new VideoError(`video "${name}" has no proxy here`));
|
|
251
|
-
|
|
321
|
+
const frame = await video.frame(index);
|
|
322
|
+
let bytes = [...this.videos.values()].reduce((total, item) => total + item.bytes, 0);
|
|
323
|
+
for (const item of this.videos.values()) {
|
|
324
|
+
if (bytes <= 64 * 1024 * 1024)
|
|
325
|
+
break;
|
|
326
|
+
if (item === video)
|
|
327
|
+
continue;
|
|
328
|
+
const held = item.bytes;
|
|
329
|
+
item.trim(0);
|
|
330
|
+
bytes -= held;
|
|
331
|
+
}
|
|
332
|
+
return frame;
|
|
252
333
|
}
|
|
253
334
|
reset() {
|
|
254
335
|
for (const v of this.videos.values())
|