@tongue-core/ui 0.1.0-rc.1 → 0.1.0-rc.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +41 -2
- package/dist/player/assets.d.ts +29 -2
- package/dist/player/assets.js +89 -9
- package/dist/player/audio.d.ts +30 -2
- package/dist/player/audio.js +41 -17
- package/dist/player/frames.d.ts +12 -2
- package/dist/player/frames.js +28 -6
- package/dist/player/index.d.ts +2 -2
- package/dist/player/index.js +5 -4
- package/dist/player/needs.d.ts +7 -1
- package/dist/player/needs.js +52 -8
- package/dist/player/player.d.ts +13 -6
- package/dist/player/player.js +110 -27
- package/dist/player/pool.d.ts +8 -0
- package/dist/player/pool.js +22 -4
- package/dist/player/protocol.d.ts +32 -1
- package/dist/player/protocol.js +22 -6
- package/dist/player/schedule.d.ts +17 -4
- package/dist/player/schedule.js +15 -4
- package/dist/player/source.d.ts +24 -4
- package/dist/player/source.js +53 -13
- package/dist/player/video.d.ts +11 -1
- package/dist/player/video.js +130 -21
- package/dist/player/worker.js +42 -11
- package/dist/react/timeline.d.ts +2 -0
- package/dist/react/timeline.js +1 -1
- package/dist/timeline/connect.js +1 -0
- package/dist/timeline/index.d.ts +1 -0
- package/dist/timeline/index.js +1 -0
- package/dist/timeline/model.d.ts +2 -0
- package/dist/timeline/model.js +1 -1
- package/dist/timeline/narration.d.ts +18 -0
- package/dist/timeline/narration.js +42 -0
- package/dist/timeline/timeline.d.ts +1 -0
- package/package.json +5 -5
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Backpine
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
CHANGED
|
@@ -38,13 +38,52 @@ player.fit(canvas.clientWidth); // render a
|
|
|
38
38
|
const bitmap = await player.preview(900, 160); // a thumbnail, when the workers are free
|
|
39
39
|
```
|
|
40
40
|
|
|
41
|
+
The picture keeps the film's shape in whatever box the page gives the canvas: unless the page
|
|
42
|
+
sets its own `object-fit`, the player sets `contain`, so a box of another shape letterboxes the
|
|
43
|
+
frame instead of stretching it.
|
|
44
|
+
|
|
41
45
|
A build is static files: `build.json`, `film.json`, `scenes/*.js`, `assets/…` (`tongue build`
|
|
42
|
-
writes one). The player fetches the
|
|
43
|
-
|
|
46
|
+
writes one). The player fetches them from the build's folder or, given `read` instead of `build`,
|
|
47
|
+
reads them a file at a time from wherever the build is kept (a Git commit, memory, an archive):
|
|
48
|
+
|
|
49
|
+
```ts
|
|
50
|
+
// Each path as the build gives it: "build.json", "scenes/title.5ebf1e9a05d5.js", "assets/fonts/inter.ttf"…
|
|
51
|
+
await player.load({ read: (path, signal) => commit.bytes(path, signal) });
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
Either way it reads the manifests and bundles, then the asset files frames need, nearest the
|
|
55
|
+
playhead first, each checked against its hash. **A frame is drawn only once every file it needs
|
|
44
56
|
is there**; until then the last frame stays up. Video plays from the proxy segments import makes
|
|
45
57
|
(H.264, decoded with WebCodecs in a worker), sound from the soundtrack mixed a few seconds at a time.
|
|
46
58
|
Frames, video frames and sound are byte-for-byte what Bun makes from the same build.
|
|
47
59
|
|
|
60
|
+
Version 2 builds can keep their media outside the build. The player reads each remote file from
|
|
61
|
+
the URL in its manifest, verifies its hash, and bypasses `read` for that file. A Git-backed studio
|
|
62
|
+
therefore needs only code and manifests in Git; originals are never fetched by the browser.
|
|
63
|
+
Remote builds fetch a bounded window of proxy video and segmented audio (15 seconds ahead by
|
|
64
|
+
default). Scenes request their actual video segments when drawn. Old compressed data and decoded
|
|
65
|
+
audio segments are evicted, with stale transfers cancelled after seeks; a queued request the window
|
|
66
|
+
leaves behind is cancelled too, so no decoder waits on a segment nobody will fetch. A video's
|
|
67
|
+
decodes wait for their segments before taking their turn (playback is never held up behind a
|
|
68
|
+
thumbnail on its way), and a jump skips playback's decodes still queued for where it was. `mediaCacheBytes`
|
|
69
|
+
(default 64 MiB) and `mediaAhead` configure the window; files required by active requests remain
|
|
70
|
+
pinned until those requests finish. Decoded video frames have a separate shared byte budget.
|
|
71
|
+
|
|
72
|
+
```ts
|
|
73
|
+
await player.load({
|
|
74
|
+
read: (path, signal) => commit.bytes(path, signal),
|
|
75
|
+
request: (url, signal, headers) => fetch(url, { signal, headers }),
|
|
76
|
+
});
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
The optional `request` hook supplies authentication or URL refresh. Transient downloads retry
|
|
80
|
+
through the shared media transport. `readiness.failed` reports permanent failures; `player.retry()`
|
|
81
|
+
retries them after the host restores access. `readiness.mediaBytes` reports retained compressed
|
|
82
|
+
data. Failure records also expose `reason`, `status` and `retryable` when known. For streaming
|
|
83
|
+
builds, progress counts describe the current fetch window. Readiness ranges for dynamic scene
|
|
84
|
+
video conservatively report already rendered exact frames, since a scene can choose arbitrary
|
|
85
|
+
source times. Local packaged builds preserve their existing background preloading behavior.
|
|
86
|
+
|
|
48
87
|
## A timeline
|
|
49
88
|
|
|
50
89
|
```ts
|
package/dist/player/assets.d.ts
CHANGED
|
@@ -1,4 +1,12 @@
|
|
|
1
|
+
import { MediaFailure } from "@tongue-core/project/media";
|
|
2
|
+
export interface AssetFailureInfo {
|
|
3
|
+
readonly message: string;
|
|
4
|
+
readonly reason?: MediaFailure["reason"];
|
|
5
|
+
readonly status?: number;
|
|
6
|
+
readonly retryable?: boolean;
|
|
7
|
+
}
|
|
1
8
|
export interface AssetFileInfo {
|
|
9
|
+
readonly url?: string;
|
|
2
10
|
/** Relative to the build's assets folder: fonts/inter.ttf. */
|
|
3
11
|
readonly path: string;
|
|
4
12
|
/** Its size, when the build says (0 when unknown). */
|
|
@@ -8,6 +16,8 @@ export interface AssetFileInfo {
|
|
|
8
16
|
}
|
|
9
17
|
export type AssetStatus = "waiting" | "loading" | "ready" | "failed";
|
|
10
18
|
export interface AssetProgress {
|
|
19
|
+
/** Counts describe the current fetch window when streaming. */
|
|
20
|
+
readonly streaming?: boolean;
|
|
11
21
|
readonly files: number;
|
|
12
22
|
readonly ready: number;
|
|
13
23
|
readonly failed: number;
|
|
@@ -19,20 +29,37 @@ export type FetchBytes = (path: string, signal: AbortSignal) => Promise<Uint8Arr
|
|
|
19
29
|
export declare class AssetLoader {
|
|
20
30
|
private readonly fetchBytes;
|
|
21
31
|
private readonly concurrency;
|
|
32
|
+
private readonly maxBytes;
|
|
22
33
|
private readonly entries;
|
|
23
34
|
private readonly listeners;
|
|
24
35
|
private readonly abort;
|
|
25
36
|
private active;
|
|
26
|
-
|
|
37
|
+
private clock;
|
|
38
|
+
private readonly retained;
|
|
39
|
+
onEvict: (path: string) => void;
|
|
40
|
+
/** A request for `path` was withdrawn before it was answered — its transfer stopped, or it never
|
|
41
|
+
* started — so whoever waits for the file hears it won't come (and asks again if it still wants it). */
|
|
42
|
+
onCancel: (path: string) => void;
|
|
43
|
+
constructor(files: ReadonlyArray<AssetFileInfo>, fetchBytes: FetchBytes, concurrency?: number, maxBytes?: number);
|
|
27
44
|
/** Asks for files at a priority: lower is sooner. Files the build doesn't list are ignored. */
|
|
28
45
|
want(paths: Iterable<string>, priority: number): void;
|
|
29
46
|
status(path: string): AssetStatus | null;
|
|
30
47
|
readonly isReady: (path: string) => boolean;
|
|
31
48
|
/** A ready file's bytes. */
|
|
32
49
|
data(path: string): Uint8Array | null;
|
|
50
|
+
/** Changes the fetch window. Requests outside it are withdrawn, under way or still queued; old data
|
|
51
|
+
* is evicted under the byte budget. */
|
|
52
|
+
retain(paths: Iterable<string>): void;
|
|
53
|
+
get heldBytes(): number;
|
|
54
|
+
private evict;
|
|
33
55
|
/** Why a file failed. */
|
|
34
56
|
error(path: string): string | null;
|
|
35
|
-
|
|
57
|
+
failure(path: string): AssetFailureInfo;
|
|
58
|
+
failMissing(path: string): void;
|
|
59
|
+
failures(): ReadonlyArray<{
|
|
60
|
+
readonly path: string;
|
|
61
|
+
} & AssetFailureInfo>;
|
|
62
|
+
progress(windowOnly?: boolean): AssetProgress;
|
|
36
63
|
/** Takes over files another loader (the last build's) already has, where the hashes match: a
|
|
37
64
|
* rebuild fetches only what changed. Returns the paths taken over. */
|
|
38
65
|
carryOver(previous: AssetLoader): ReadonlyArray<string>;
|
package/dist/player/assets.js
CHANGED
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
// time, each checked against the hash its build gives it. Every file has a status, and the loader
|
|
4
4
|
// says when one changes: that is what readiness, progress bars and "can this frame be drawn
|
|
5
5
|
// exactly" are made of. Nothing here knows about frames.
|
|
6
|
+
import { MediaFailure } from "@tongue-core/project/media";
|
|
6
7
|
async function sha256(bytes) {
|
|
7
8
|
const copy = new Uint8Array(bytes.length);
|
|
8
9
|
copy.set(bytes);
|
|
@@ -12,20 +13,30 @@ async function sha256(bytes) {
|
|
|
12
13
|
export class AssetLoader {
|
|
13
14
|
fetchBytes;
|
|
14
15
|
concurrency;
|
|
16
|
+
maxBytes;
|
|
15
17
|
entries = new Map();
|
|
16
18
|
listeners = new Set();
|
|
17
19
|
abort = new AbortController();
|
|
18
20
|
active = 0;
|
|
19
|
-
|
|
21
|
+
clock = 0;
|
|
22
|
+
retained = new Set();
|
|
23
|
+
onEvict = () => { };
|
|
24
|
+
/** A request for `path` was withdrawn before it was answered — its transfer stopped, or it never
|
|
25
|
+
* started — so whoever waits for the file hears it won't come (and asks again if it still wants it). */
|
|
26
|
+
onCancel = () => { };
|
|
27
|
+
constructor(files, fetchBytes, concurrency = 6, maxBytes = Infinity) {
|
|
20
28
|
this.fetchBytes = fetchBytes;
|
|
21
29
|
this.concurrency = concurrency;
|
|
30
|
+
this.maxBytes = maxBytes;
|
|
22
31
|
for (const info of files)
|
|
23
|
-
this.entries.set(info.path, { info, status: "waiting", priority: Number.POSITIVE_INFINITY, data: null, error: null });
|
|
32
|
+
this.entries.set(info.path, { info, status: "waiting", priority: Number.POSITIVE_INFINITY, data: null, error: null, abort: null, used: 0, failure: null });
|
|
24
33
|
}
|
|
25
34
|
/** Asks for files at a priority: lower is sooner. Files the build doesn't list are ignored. */
|
|
26
35
|
want(paths, priority) {
|
|
27
36
|
for (const path of paths) {
|
|
28
37
|
const e = this.entries.get(path);
|
|
38
|
+
if (e)
|
|
39
|
+
e.used = ++this.clock;
|
|
29
40
|
if (e && priority < e.priority)
|
|
30
41
|
e.priority = priority;
|
|
31
42
|
}
|
|
@@ -37,15 +48,67 @@ export class AssetLoader {
|
|
|
37
48
|
isReady = (path) => this.entries.get(path)?.status === "ready";
|
|
38
49
|
/** A ready file's bytes. */
|
|
39
50
|
data(path) {
|
|
40
|
-
|
|
51
|
+
const entry = this.entries.get(path);
|
|
52
|
+
if (entry)
|
|
53
|
+
entry.used = ++this.clock;
|
|
54
|
+
return entry?.data ?? null;
|
|
55
|
+
}
|
|
56
|
+
/** Changes the fetch window. Requests outside it are withdrawn, under way or still queued; old data
|
|
57
|
+
* is evicted under the byte budget. */
|
|
58
|
+
retain(paths) {
|
|
59
|
+
this.retained.clear();
|
|
60
|
+
for (const path of paths)
|
|
61
|
+
this.retained.add(path);
|
|
62
|
+
for (const [path, entry] of this.entries) {
|
|
63
|
+
if (this.retained.has(path))
|
|
64
|
+
continue;
|
|
65
|
+
const queued = entry.status === "waiting" && entry.priority < Number.POSITIVE_INFINITY;
|
|
66
|
+
const transfer = entry.abort !== null && !entry.abort.signal.aborted ? entry.abort : null;
|
|
67
|
+
entry.priority = Number.POSITIVE_INFINITY;
|
|
68
|
+
transfer?.abort();
|
|
69
|
+
if (queued || transfer)
|
|
70
|
+
this.onCancel(path);
|
|
71
|
+
}
|
|
72
|
+
this.evict();
|
|
73
|
+
}
|
|
74
|
+
get heldBytes() { let bytes = 0; for (const entry of this.entries.values())
|
|
75
|
+
bytes += entry.data?.length ?? 0; return bytes; }
|
|
76
|
+
evict() {
|
|
77
|
+
let bytes = this.heldBytes;
|
|
78
|
+
const candidates = [...this.entries.values()].filter((entry) => entry.data && !this.retained.has(entry.info.path)).sort((a, b) => a.used - b.used);
|
|
79
|
+
for (const entry of candidates) {
|
|
80
|
+
if (bytes <= this.maxBytes)
|
|
81
|
+
break;
|
|
82
|
+
bytes -= entry.data?.length ?? 0;
|
|
83
|
+
entry.data = null;
|
|
84
|
+
entry.status = "waiting";
|
|
85
|
+
entry.priority = Infinity;
|
|
86
|
+
this.onEvict(entry.info.path);
|
|
87
|
+
this.changed(entry.info.path);
|
|
88
|
+
}
|
|
41
89
|
}
|
|
42
90
|
/** Why a file failed. */
|
|
43
91
|
error(path) {
|
|
44
92
|
return this.entries.get(path)?.error ?? null;
|
|
45
93
|
}
|
|
46
|
-
|
|
47
|
-
|
|
94
|
+
failure(path) {
|
|
95
|
+
const entry = this.entries.get(path), failure = entry?.failure;
|
|
96
|
+
return { message: entry?.error ?? "failed", ...(failure ? { reason: failure.reason, retryable: failure.retryable, ...(failure.status === undefined ? {} : { status: failure.status }) } : {}) };
|
|
97
|
+
}
|
|
98
|
+
failMissing(path) {
|
|
99
|
+
const failure = new MediaFailure({ asset: path, source: "build.json", reason: "invalid", retryable: false, message: `The build does not list required preview file ${path}. Rebuild or repair its prepared-media manifest.` });
|
|
100
|
+
this.entries.set(path, { info: { path, bytes: 0, hash: null }, status: "failed", priority: Infinity, data: null, error: failure.message, abort: null, used: ++this.clock, failure });
|
|
101
|
+
this.changed(path);
|
|
102
|
+
}
|
|
103
|
+
failures() {
|
|
104
|
+
return [...this.entries.values()].filter((entry) => entry.status === "failed").map((entry) => ({ path: entry.info.path, ...this.failure(entry.info.path) }));
|
|
105
|
+
}
|
|
106
|
+
progress(windowOnly = false) {
|
|
107
|
+
let ready = 0, failed = 0, bytes = 0, bytesReady = 0, files = 0;
|
|
48
108
|
for (const e of this.entries.values()) {
|
|
109
|
+
if (windowOnly && !this.retained.has(e.info.path))
|
|
110
|
+
continue;
|
|
111
|
+
files++;
|
|
49
112
|
bytes += e.info.bytes;
|
|
50
113
|
if (e.status === "ready") {
|
|
51
114
|
ready++;
|
|
@@ -54,7 +117,7 @@ export class AssetLoader {
|
|
|
54
117
|
if (e.status === "failed")
|
|
55
118
|
failed++;
|
|
56
119
|
}
|
|
57
|
-
return { files
|
|
120
|
+
return { files, ready, failed, bytes, bytesReady, ...(windowOnly ? { streaming: true } : {}) };
|
|
58
121
|
}
|
|
59
122
|
/** Takes over files another loader (the last build's) already has, where the hashes match: a
|
|
60
123
|
* rebuild fetches only what changed. Returns the paths taken over. */
|
|
@@ -73,9 +136,10 @@ export class AssetLoader {
|
|
|
73
136
|
/** Files that failed go back in the queue. */
|
|
74
137
|
retry() {
|
|
75
138
|
for (const e of this.entries.values())
|
|
76
|
-
if (e.status === "failed") {
|
|
139
|
+
if (e.status === "failed" && e.failure?.reason !== "invalid") {
|
|
77
140
|
e.status = "waiting";
|
|
78
141
|
e.error = null;
|
|
142
|
+
e.failure = null;
|
|
79
143
|
}
|
|
80
144
|
this.pump();
|
|
81
145
|
}
|
|
@@ -85,6 +149,8 @@ export class AssetLoader {
|
|
|
85
149
|
}
|
|
86
150
|
dispose() {
|
|
87
151
|
this.abort.abort();
|
|
152
|
+
for (const entry of this.entries.values())
|
|
153
|
+
entry.abort?.abort();
|
|
88
154
|
this.listeners.clear();
|
|
89
155
|
}
|
|
90
156
|
changed(path) {
|
|
@@ -115,19 +181,33 @@ export class AssetLoader {
|
|
|
115
181
|
}
|
|
116
182
|
}
|
|
117
183
|
async load(e) {
|
|
184
|
+
const abort = new AbortController();
|
|
185
|
+
e.abort = abort;
|
|
118
186
|
try {
|
|
119
|
-
const data = await this.fetchBytes(e.info.path, this.abort.signal);
|
|
187
|
+
const data = await this.fetchBytes(e.info.path, AbortSignal.any([this.abort.signal, abort.signal]));
|
|
188
|
+
const integrity = (message) => new MediaFailure({ asset: e.info.path, source: e.info.url ?? e.info.path, reason: "integrity", retryable: false, message });
|
|
189
|
+
if (e.info.bytes > 0 && data.length !== e.info.bytes)
|
|
190
|
+
throw integrity(`its size differs from the build (${data.length} bytes, expected ${e.info.bytes})`);
|
|
120
191
|
if (e.info.hash !== null && (await sha256(data)) !== e.info.hash)
|
|
121
|
-
throw
|
|
192
|
+
throw integrity("its contents don't match the build (the hash differs)");
|
|
122
193
|
e.data = data;
|
|
123
194
|
e.status = "ready";
|
|
124
195
|
}
|
|
125
196
|
catch (error) {
|
|
126
197
|
if (this.abort.signal.aborted)
|
|
127
198
|
return;
|
|
199
|
+
if (abort.signal.aborted) {
|
|
200
|
+
e.status = "waiting";
|
|
201
|
+
return;
|
|
202
|
+
}
|
|
128
203
|
e.status = "failed";
|
|
204
|
+
e.failure = error instanceof MediaFailure ? error : null;
|
|
129
205
|
e.error = error instanceof Error ? error.message : String(error);
|
|
130
206
|
}
|
|
207
|
+
finally {
|
|
208
|
+
e.abort = null;
|
|
209
|
+
}
|
|
131
210
|
this.changed(e.info.path);
|
|
211
|
+
this.evict();
|
|
132
212
|
}
|
|
133
213
|
}
|
package/dist/player/audio.d.ts
CHANGED
|
@@ -4,17 +4,45 @@ export interface SoundtrackOptions {
|
|
|
4
4
|
/** The film's length in frames. */
|
|
5
5
|
readonly length: number;
|
|
6
6
|
/** Mixes frames [from, to). */
|
|
7
|
-
readonly mix: (from: number, to: number) => Promise<MixResult>;
|
|
7
|
+
readonly mix: (from: number, to: number, signal: AbortSignal) => Promise<MixResult>;
|
|
8
8
|
/** Frames per chunk (default 5 seconds). */
|
|
9
9
|
readonly chunk?: number;
|
|
10
10
|
/** Chunks kept mixed ahead of the playhead (default 3). */
|
|
11
11
|
readonly ahead?: number;
|
|
12
12
|
}
|
|
13
|
+
/** A chunk to put on the audio clock: when it starts, and how far into it (the chunk playback started in starts part way). */
|
|
14
|
+
export interface DueChunk {
|
|
15
|
+
readonly chunk: number;
|
|
16
|
+
readonly when: number;
|
|
17
|
+
readonly offset: number;
|
|
18
|
+
}
|
|
19
|
+
export interface DueOptions {
|
|
20
|
+
/** The first chunk not yet on the clock. */
|
|
21
|
+
readonly next: number;
|
|
22
|
+
/** Frames per chunk, frames a second, and the film's length in frames. */
|
|
23
|
+
readonly size: number;
|
|
24
|
+
readonly fps: number;
|
|
25
|
+
readonly length: number;
|
|
26
|
+
/** Playback started at this frame, at this time on the audio clock. */
|
|
27
|
+
readonly startFrame: number;
|
|
28
|
+
readonly startTime: number;
|
|
29
|
+
/** The audio clock now, and how far ahead of it (seconds) sound may be put. */
|
|
30
|
+
readonly now: number;
|
|
31
|
+
readonly horizon: number;
|
|
32
|
+
readonly mixed: (chunk: number) => boolean;
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* The chunks to put on the audio clock now, in order from `next`: each one mixed, starting no further
|
|
36
|
+
* than `horizon` seconds ahead of the clock. It stops at the first chunk still being mixed (it is on
|
|
37
|
+
* its way) or too far ahead (a later tick takes it), and at the end of the film. Pure.
|
|
38
|
+
*/
|
|
39
|
+
export declare function dueChunks(o: DueOptions): ReadonlyArray<DueChunk>;
|
|
13
40
|
export declare class Soundtrack {
|
|
14
41
|
private readonly options;
|
|
15
42
|
private context;
|
|
16
43
|
private readonly chunks;
|
|
17
44
|
private readonly mixing;
|
|
45
|
+
private readonly controllers;
|
|
18
46
|
private playing;
|
|
19
47
|
private readonly size;
|
|
20
48
|
private readonly ahead;
|
|
@@ -33,7 +61,7 @@ export declare class Soundtrack {
|
|
|
33
61
|
prepare(frame: number): void;
|
|
34
62
|
/** Starts sound at `frame`, at audio time `time` (from clock()). */
|
|
35
63
|
start(frame: number, time: number): void;
|
|
36
|
-
/** Schedules the chunks that are mixed and due; call it every tick while playing. */
|
|
64
|
+
/** Schedules the chunks that are mixed and due (`dueChunks`); call it every tick while playing. */
|
|
37
65
|
schedule(): void;
|
|
38
66
|
stop(): void;
|
|
39
67
|
/** The film changed: forget every chunk. */
|
package/dist/player/audio.js
CHANGED
|
@@ -2,11 +2,32 @@
|
|
|
2
2
|
const KEPT = 12;
|
|
3
3
|
/** Seconds between asking to start and the first sample, so the start is never late. */
|
|
4
4
|
const LEAD = 0.05;
|
|
5
|
+
/** How far ahead of the audio clock sound is put on it, in chunks: later chunks wait for later ticks
|
|
6
|
+
* (and are mixed only as they near). */
|
|
7
|
+
const HORIZON_CHUNKS = 1.5;
|
|
8
|
+
/**
|
|
9
|
+
* The chunks to put on the audio clock now, in order from `next`: each one mixed, starting no further
|
|
10
|
+
* than `horizon` seconds ahead of the clock. It stops at the first chunk still being mixed (it is on
|
|
11
|
+
* its way) or too far ahead (a later tick takes it), and at the end of the film. Pure.
|
|
12
|
+
*/
|
|
13
|
+
export function dueChunks(o) {
|
|
14
|
+
const due = [];
|
|
15
|
+
for (let chunk = o.next; chunk * o.size < o.length && o.mixed(chunk); chunk++) {
|
|
16
|
+
// Seconds after the start frame this chunk begins (negative for the chunk the start is in).
|
|
17
|
+
const at = (chunk * o.size - o.startFrame) / o.fps;
|
|
18
|
+
const when = o.startTime + Math.max(0, at);
|
|
19
|
+
if (when - o.now > o.horizon)
|
|
20
|
+
break;
|
|
21
|
+
due.push({ chunk, when, offset: Math.max(0, -at) });
|
|
22
|
+
}
|
|
23
|
+
return due;
|
|
24
|
+
}
|
|
5
25
|
export class Soundtrack {
|
|
6
26
|
options;
|
|
7
27
|
context = null;
|
|
8
28
|
chunks = new Map();
|
|
9
29
|
mixing = new Map();
|
|
30
|
+
controllers = new Map();
|
|
10
31
|
playing = null;
|
|
11
32
|
size;
|
|
12
33
|
ahead;
|
|
@@ -58,33 +79,27 @@ export class Soundtrack {
|
|
|
58
79
|
this.playing = { frame, time: time + LEAD, sources: [], next: this.chunkOf(frame) };
|
|
59
80
|
this.schedule();
|
|
60
81
|
}
|
|
61
|
-
/** Schedules the chunks that are mixed and due; call it every tick while playing. */
|
|
82
|
+
/** Schedules the chunks that are mixed and due (`dueChunks`); call it every tick while playing. */
|
|
62
83
|
schedule() {
|
|
63
84
|
const p = this.playing, ctx = this.context, gain = this.gain;
|
|
64
85
|
if (!p || !ctx || !gain)
|
|
65
86
|
return;
|
|
66
87
|
this.prepare(p.next * this.size);
|
|
67
|
-
|
|
68
|
-
|
|
88
|
+
const due = dueChunks({
|
|
89
|
+
next: p.next, size: this.size, fps: this.options.fps, length: this.options.length, startFrame: p.frame, startTime: p.time,
|
|
90
|
+
now: ctx.currentTime, horizon: (this.size / this.options.fps) * HORIZON_CHUNKS, mixed: (chunk) => this.chunks.has(chunk),
|
|
91
|
+
});
|
|
92
|
+
for (const { chunk, when, offset } of due) {
|
|
93
|
+
const buffer = this.chunks.get(chunk);
|
|
69
94
|
if (!buffer)
|
|
70
95
|
return;
|
|
71
|
-
const chunkStart = p.next * this.size;
|
|
72
|
-
// Seconds after the start frame this chunk begins (negative for the chunk the start is in).
|
|
73
|
-
const at = (chunkStart - p.frame) / this.options.fps;
|
|
74
96
|
const source = ctx.createBufferSource();
|
|
75
97
|
source.buffer = buffer;
|
|
76
98
|
source.connect(gain);
|
|
77
|
-
const when = p.time + Math.max(0, at);
|
|
78
|
-
const offset = Math.max(0, -at);
|
|
79
99
|
if (offset < buffer.duration)
|
|
80
100
|
source.start(Math.max(when, ctx.currentTime), offset);
|
|
81
101
|
p.sources.push(source);
|
|
82
|
-
p.next
|
|
83
|
-
if (p.next * this.size >= this.options.length)
|
|
84
|
-
return;
|
|
85
|
-
// Far enough ahead: the rest is scheduled on later ticks.
|
|
86
|
-
if (when - ctx.currentTime > (this.size / this.options.fps) * 1.5)
|
|
87
|
-
return;
|
|
102
|
+
p.next = chunk + 1;
|
|
88
103
|
}
|
|
89
104
|
}
|
|
90
105
|
stop() {
|
|
@@ -104,6 +119,9 @@ export class Soundtrack {
|
|
|
104
119
|
this.stop();
|
|
105
120
|
this.chunks.clear();
|
|
106
121
|
this.mixing.clear();
|
|
122
|
+
for (const controller of this.controllers.values())
|
|
123
|
+
controller.abort();
|
|
124
|
+
this.controllers.clear();
|
|
107
125
|
this.generation++;
|
|
108
126
|
}
|
|
109
127
|
dispose() {
|
|
@@ -127,10 +145,13 @@ export class Soundtrack {
|
|
|
127
145
|
return pending;
|
|
128
146
|
const generation = this.generation;
|
|
129
147
|
const from = k * this.size, to = Math.min(this.options.length, from + this.size);
|
|
130
|
-
const
|
|
131
|
-
|
|
148
|
+
const controller = new AbortController();
|
|
149
|
+
this.controllers.set(k, controller);
|
|
150
|
+
const job = this.options.mix(from, to, controller.signal).then((mixed) => {
|
|
132
151
|
if (generation !== this.generation)
|
|
133
152
|
return null;
|
|
153
|
+
this.mixing.delete(k);
|
|
154
|
+
this.controllers.delete(k);
|
|
134
155
|
const buffer = new AudioBuffer({ numberOfChannels: 2, length: Math.max(1, mixed.left.length), sampleRate: mixed.sampleRate });
|
|
135
156
|
buffer.copyToChannel(own(mixed.left), 0);
|
|
136
157
|
buffer.copyToChannel(own(mixed.right), 1);
|
|
@@ -143,7 +164,10 @@ export class Soundtrack {
|
|
|
143
164
|
}
|
|
144
165
|
return buffer;
|
|
145
166
|
}, () => {
|
|
146
|
-
this.
|
|
167
|
+
if (generation === this.generation) {
|
|
168
|
+
this.mixing.delete(k);
|
|
169
|
+
this.controllers.delete(k);
|
|
170
|
+
}
|
|
147
171
|
return null;
|
|
148
172
|
});
|
|
149
173
|
this.mixing.set(k, job);
|
package/dist/player/frames.d.ts
CHANGED
|
@@ -8,11 +8,15 @@ export interface Cached<T> {
|
|
|
8
8
|
/** False when something stood in for part of it (a video's poster for its frames). */
|
|
9
9
|
readonly exact: boolean;
|
|
10
10
|
}
|
|
11
|
+
/** What eviction keeps: the frames around the focus (paused: someone may step either way), or the
|
|
12
|
+
* frames ahead of it (playing: what has been played is the first to go). */
|
|
13
|
+
export type Keep = "around" | "ahead";
|
|
11
14
|
export declare class FrameCache<T extends Closable> {
|
|
12
15
|
private readonly budget;
|
|
13
16
|
private readonly frames;
|
|
14
17
|
private total;
|
|
15
18
|
private focus;
|
|
19
|
+
private keep;
|
|
16
20
|
constructor(budget: number);
|
|
17
21
|
/** The best scale `frame` is cached at, or null. */
|
|
18
22
|
scaleOf(frame: number): number | null;
|
|
@@ -22,13 +26,19 @@ export declare class FrameCache<T extends Closable> {
|
|
|
22
26
|
/** Keeps the better of what is there and what arrived (sharper; exact over stood-in), closing
|
|
23
27
|
* the other, then keeps to the budget. */
|
|
24
28
|
put(frame: number, value: T, scale: number, bytes: number, exact?: boolean): void;
|
|
25
|
-
/** Where the playhead is
|
|
26
|
-
setFocus(frame: number): void;
|
|
29
|
+
/** Where the playhead is, and what eviction keeps around it. */
|
|
30
|
+
setFocus(frame: number, keep?: Keep): void;
|
|
31
|
+
/** How many frames of `bytes` each the budget holds (at least one: the focus is never evicted). */
|
|
32
|
+
capacity(bytes: number): number;
|
|
27
33
|
/** The cached frame nearest `frame`, at most `radius` away (the earlier one on a tie). */
|
|
28
34
|
nearest(frame: number, radius: number): number | null;
|
|
29
35
|
clear(): void;
|
|
30
36
|
get bytes(): number;
|
|
31
37
|
get size(): number;
|
|
38
|
+
exactRanges(): ReadonlyArray<{
|
|
39
|
+
readonly from: number;
|
|
40
|
+
readonly to: number;
|
|
41
|
+
}>;
|
|
32
42
|
private remove;
|
|
33
43
|
private trim;
|
|
34
44
|
}
|
package/dist/player/frames.js
CHANGED
|
@@ -1,11 +1,14 @@
|
|
|
1
1
|
// Rendered frames kept for showing again, under a budget of bytes. A frame is kept at the best
|
|
2
|
-
// scale it was rendered at; when the budget is exceeded,
|
|
3
|
-
// playhead)
|
|
2
|
+
// scale it was rendered at; when the budget is exceeded, frames go by how little they are worth to
|
|
3
|
+
// the focus (the playhead): paused, the furthest from it on either side; playing, every frame
|
|
4
|
+
// behind it first (they won't be shown again), then the furthest ahead. Whatever leaves is closed,
|
|
5
|
+
// so an ImageBitmap's memory is released at once.
|
|
4
6
|
export class FrameCache {
|
|
5
7
|
budget;
|
|
6
8
|
frames = new Map();
|
|
7
9
|
total = 0;
|
|
8
10
|
focus = 0;
|
|
11
|
+
keep = "around";
|
|
9
12
|
constructor(budget) {
|
|
10
13
|
this.budget = budget;
|
|
11
14
|
}
|
|
@@ -35,9 +38,14 @@ export class FrameCache {
|
|
|
35
38
|
this.total += bytes;
|
|
36
39
|
this.trim();
|
|
37
40
|
}
|
|
38
|
-
/** Where the playhead is
|
|
39
|
-
setFocus(frame) {
|
|
41
|
+
/** Where the playhead is, and what eviction keeps around it. */
|
|
42
|
+
setFocus(frame, keep = "around") {
|
|
40
43
|
this.focus = frame;
|
|
44
|
+
this.keep = keep;
|
|
45
|
+
}
|
|
46
|
+
/** How many frames of `bytes` each the budget holds (at least one: the focus is never evicted). */
|
|
47
|
+
capacity(bytes) {
|
|
48
|
+
return Math.max(1, Math.floor(this.budget / bytes));
|
|
41
49
|
}
|
|
42
50
|
/** The cached frame nearest `frame`, at most `radius` away (the earlier one on a tie). */
|
|
43
51
|
nearest(frame, radius) {
|
|
@@ -61,6 +69,18 @@ export class FrameCache {
|
|
|
61
69
|
get size() {
|
|
62
70
|
return this.frames.size;
|
|
63
71
|
}
|
|
72
|
+
exactRanges() {
|
|
73
|
+
const frames = [...this.frames].filter(([, value]) => value.exact).map(([frame]) => frame).sort((a, b) => a - b);
|
|
74
|
+
const ranges = [];
|
|
75
|
+
for (const frame of frames) {
|
|
76
|
+
const last = ranges[ranges.length - 1];
|
|
77
|
+
if (last?.to === frame)
|
|
78
|
+
last.to++;
|
|
79
|
+
else
|
|
80
|
+
ranges.push({ from: frame, to: frame + 1 });
|
|
81
|
+
}
|
|
82
|
+
return ranges;
|
|
83
|
+
}
|
|
64
84
|
remove(frame, entry) {
|
|
65
85
|
entry.value.close();
|
|
66
86
|
this.frames.delete(frame);
|
|
@@ -69,8 +89,10 @@ export class FrameCache {
|
|
|
69
89
|
trim() {
|
|
70
90
|
if (this.total <= this.budget)
|
|
71
91
|
return;
|
|
72
|
-
const
|
|
73
|
-
|
|
92
|
+
const behind = (frame) => (this.keep === "ahead" && frame < this.focus ? 1 : 0);
|
|
93
|
+
// Least worth first: playing, every frame behind the focus before any ahead; then the furthest.
|
|
94
|
+
const byWorth = [...this.frames.keys()].sort((a, b) => behind(b) - behind(a) || Math.abs(b - this.focus) - Math.abs(a - this.focus));
|
|
95
|
+
for (const frame of byWorth) {
|
|
74
96
|
if (this.total <= this.budget)
|
|
75
97
|
break;
|
|
76
98
|
if (frame === this.focus)
|
package/dist/player/index.d.ts
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
export { createPlayer, type Player, type PlayerOptions, type PlayerEvents, type PlayerEvent, type Readiness, type Shown, type LoadedFilm, type Pixels, type Mix } from "./player.js";
|
|
2
2
|
export { Cancelled, type FrameResult } from "./pool.js";
|
|
3
|
-
export { openSource, buildBase, SourceError, type BuildSource, type OpenedSource } from "./source.js";
|
|
3
|
+
export { openFilm, openSource, openFiles, buildBase, SourceError, type FilmSource, type BuildSource, type FileSource, type OpenedBuild, type OpenedSource } from "./source.js";
|
|
4
4
|
export { FilmNeeds, type Span, type Range } from "./needs.js";
|
|
5
|
-
export { AssetLoader, type AssetFileInfo, type AssetStatus, type AssetProgress, type FetchBytes } from "./assets.js";
|
|
5
|
+
export { AssetLoader, type AssetFileInfo, type AssetStatus, type AssetProgress, type AssetFailureInfo, type FetchBytes } from "./assets.js";
|
|
6
6
|
export { FrameCache, type Cached, type Closable } from "./frames.js";
|
|
7
7
|
export { Transport, type PlayState, type TickInput, type TickResult } from "./transport.js";
|
|
8
8
|
export { wantedFrames, bufferedAhead, startBuffer, clockFrame, clampFrame, type Scales, type Want, type Situation } from "./schedule.js";
|
package/dist/player/index.js
CHANGED
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
// @tongue-core/ui/player: a film's player, without React or Effect. It opens a build from wherever it is
|
|
2
|
-
// served (build.json, film.json, the scene
|
|
3
|
-
//
|
|
4
|
-
// @tongue-core/ui/player/worker; bundle it and
|
|
2
|
+
// served, or reads it a file at a time from wherever it is kept (build.json, film.json, the scene
|
|
3
|
+
// bundles, then the asset files frames need), draws exact frames with the engine in Web Workers, and
|
|
4
|
+
// plays them with the soundtrack. The worker script is @tongue-core/ui/player/worker; bundle it and
|
|
5
|
+
// pass its URL.
|
|
5
6
|
export { createPlayer } from "./player.js";
|
|
6
7
|
export { Cancelled } from "./pool.js";
|
|
7
|
-
export { openSource, buildBase, SourceError } from "./source.js";
|
|
8
|
+
export { openFilm, openSource, openFiles, buildBase, SourceError } from "./source.js";
|
|
8
9
|
export { FilmNeeds } from "./needs.js";
|
|
9
10
|
export { AssetLoader } from "./assets.js";
|
|
10
11
|
export { FrameCache } from "./frames.js";
|
package/dist/player/needs.d.ts
CHANGED
|
@@ -20,12 +20,18 @@ export declare class FilmNeeds {
|
|
|
20
20
|
/** What the soundtrack needs. */
|
|
21
21
|
readonly audio: ReadonlyArray<string>;
|
|
22
22
|
readonly length: number;
|
|
23
|
+
private readonly audioSegments;
|
|
24
|
+
private readonly audioClips;
|
|
25
|
+
private readonly fps;
|
|
23
26
|
private constructor();
|
|
24
27
|
/** A film's needs from its film.json, and the files its build has (which videos have proxies,
|
|
25
28
|
* and how many segments). */
|
|
26
|
-
static of(doc: FilmDoc, files?: ReadonlyArray<string
|
|
29
|
+
static of(doc: FilmDoc, files?: ReadonlyArray<string>, options?: {
|
|
30
|
+
readonly demandVideo?: boolean;
|
|
31
|
+
}): FilmNeeds;
|
|
27
32
|
/** A film whose needs aren't known per clip: every frame needs every file. */
|
|
28
33
|
static everything(paths: ReadonlyArray<string>, length: number): FilmNeeds;
|
|
34
|
+
audioPathsIn(from: number, to: number): ReadonlyArray<string>;
|
|
29
35
|
/** The spans on screen at a frame, over every track. */
|
|
30
36
|
spansAt(frame: number): ReadonlyArray<Span>;
|
|
31
37
|
/** The spans on screen anywhere in [from, to). */
|