@tongue-core/ui 0.1.0-rc.7 → 0.1.0-rc.9

package/README.md

@@ -57,6 +57,30 @@ is there**; until then the last frame stays up. Video plays from the proxy segme
 (H.264, decoded with WebCodecs in a worker), sound from the soundtrack mixed a few seconds at a time.
 Frames, video frames and sound are byte-for-byte what Bun makes from the same build.
 
+Version 2 builds can keep their media outside the build. The player reads each remote file from
+the URL in its manifest, verifies its hash, and bypasses `read` for that file. A Git-backed studio
+therefore needs only code and manifests in Git; originals are never fetched by the browser.
+Remote builds fetch a bounded window of proxy video and segmented audio (15 seconds ahead by
+default). Scenes request their actual video segments when drawn. Old compressed data and decoded
+audio segments are evicted, with stale transfers cancelled after seeks. `mediaCacheBytes`
+(default 64 MiB) and `mediaAhead` configure the window; files required by active requests remain
+pinned until those requests finish. Decoded video frames have a separate shared byte budget.
+
+```ts
+await player.load({
+  read: (path, signal) => commit.bytes(path, signal),
+  request: (url, signal, headers) => fetch(url, { signal, headers }),
+});
+```
+
+The optional `request` hook supplies authentication or URL refresh. Transient downloads retry
+through the shared media transport. `readiness.failed` reports permanent failures; `player.retry()`
+retries them after the host restores access. `readiness.mediaBytes` reports retained compressed
+data. Failure records also expose `reason`, `status` and `retryable` when known. For streaming
+builds, progress counts describe the current fetch window. Readiness ranges for dynamic scene
+video conservatively report already rendered exact frames, since a scene can choose arbitrary
+source times. Local packaged builds preserve their existing background preloading behavior.
+
 ## A timeline
 
 ```ts

package/dist/player/assets.d.ts

@@ -1,4 +1,12 @@
+import { MediaFailure } from "@tongue-core/project/media";
+export interface AssetFailureInfo {
+    readonly message: string;
+    readonly reason?: MediaFailure["reason"];
+    readonly status?: number;
+    readonly retryable?: boolean;
+}
 export interface AssetFileInfo {
+    readonly url?: string;
     /** Relative to the build's assets folder: fonts/inter.ttf. */
     readonly path: string;
     /** Its size, when the build says (0 when unknown). */
@@ -8,6 +16,8 @@ export interface AssetFileInfo {
 }
 export type AssetStatus = "waiting" | "loading" | "ready" | "failed";
 export interface AssetProgress {
+    /** Counts describe the current fetch window when streaming. */
+    readonly streaming?: boolean;
     readonly files: number;
     readonly ready: number;
     readonly failed: number;
@@ -19,20 +29,34 @@ export type FetchBytes = (path: string, signal: AbortSignal) => Promise<Uint8Arr
 export declare class AssetLoader {
     private readonly fetchBytes;
     private readonly concurrency;
+    private readonly maxBytes;
     private readonly entries;
     private readonly listeners;
     private readonly abort;
     private active;
-    constructor(files: ReadonlyArray<AssetFileInfo>, fetchBytes: FetchBytes, concurrency?: number);
+    private clock;
+    private readonly retained;
+    onEvict: (path: string) => void;
+    onCancel: (path: string) => void;
+    constructor(files: ReadonlyArray<AssetFileInfo>, fetchBytes: FetchBytes, concurrency?: number, maxBytes?: number);
     /** Asks for files at a priority: lower is sooner. Files the build doesn't list are ignored. */
     want(paths: Iterable<string>, priority: number): void;
     status(path: string): AssetStatus | null;
     readonly isReady: (path: string) => boolean;
     /** A ready file's bytes. */
     data(path: string): Uint8Array | null;
+    /** Changes the fetch window. Obsolete transfers stop; old data is evicted under the byte budget. */
+    retain(paths: Iterable<string>): void;
+    get heldBytes(): number;
+    private evict;
     /** Why a file failed. */
     error(path: string): string | null;
-    progress(): AssetProgress;
+    failure(path: string): AssetFailureInfo;
+    failMissing(path: string): void;
+    failures(): ReadonlyArray<{
+        readonly path: string;
+    } & AssetFailureInfo>;
+    progress(windowOnly?: boolean): AssetProgress;
     /** Takes over files another loader (the last build's) already has, where the hashes match: a
      *  rebuild fetches only what changed. Returns the paths taken over. */
     carryOver(previous: AssetLoader): ReadonlyArray<string>;

package/dist/player/assets.js

@@ -3,6 +3,7 @@
 // time, each checked against the hash its build gives it. Every file has a status, and the loader
 // says when one changes: that is what readiness, progress bars and "can this frame be drawn
 // exactly" are made of. Nothing here knows about frames.
+import { MediaFailure } from "@tongue-core/project/media";
 async function sha256(bytes) {
     const copy = new Uint8Array(bytes.length);
     copy.set(bytes);
@@ -12,20 +13,28 @@ async function sha256(bytes) {
 export class AssetLoader {
     fetchBytes;
     concurrency;
+    maxBytes;
     entries = new Map();
     listeners = new Set();
     abort = new AbortController();
     active = 0;
-    constructor(files, fetchBytes, concurrency = 6) {
+    clock = 0;
+    retained = new Set();
+    onEvict = () => { };
+    onCancel = () => { };
+    constructor(files, fetchBytes, concurrency = 6, maxBytes = Infinity) {
         this.fetchBytes = fetchBytes;
         this.concurrency = concurrency;
+        this.maxBytes = maxBytes;
         for (const info of files)
-            this.entries.set(info.path, { info, status: "waiting", priority: Number.POSITIVE_INFINITY, data: null, error: null });
+            this.entries.set(info.path, { info, status: "waiting", priority: Number.POSITIVE_INFINITY, data: null, error: null, abort: null, used: 0, failure: null });
     }
     /** Asks for files at a priority: lower is sooner. Files the build doesn't list are ignored. */
     want(paths, priority) {
         for (const path of paths) {
             const e = this.entries.get(path);
+            if (e)
+                e.used = ++this.clock;
             if (e && priority < e.priority)
                 e.priority = priority;
         }
@@ -37,15 +46,64 @@ export class AssetLoader {
     isReady = (path) => this.entries.get(path)?.status === "ready";
     /** A ready file's bytes. */
     data(path) {
-        return this.entries.get(path)?.data ?? null;
+        const entry = this.entries.get(path);
+        if (entry)
+            entry.used = ++this.clock;
+        return entry?.data ?? null;
+    }
+    /** Changes the fetch window. Obsolete transfers stop; old data is evicted under the byte budget. */
+    retain(paths) {
+        this.retained.clear();
+        for (const path of paths)
+            this.retained.add(path);
+        for (const [path, entry] of this.entries)
+            if (!this.retained.has(path)) {
+                entry.priority = Infinity;
+                if (entry.abort && !entry.abort.signal.aborted) {
+                    entry.abort.abort();
+                    this.onCancel(path);
+                }
+            }
+        this.evict();
+    }
+    get heldBytes() { let bytes = 0; for (const entry of this.entries.values())
+        bytes += entry.data?.length ?? 0; return bytes; }
+    evict() {
+        let bytes = this.heldBytes;
+        const candidates = [...this.entries.values()].filter((entry) => entry.data && !this.retained.has(entry.info.path)).sort((a, b) => a.used - b.used);
+        for (const entry of candidates) {
+            if (bytes <= this.maxBytes)
+                break;
+            bytes -= entry.data?.length ?? 0;
+            entry.data = null;
+            entry.status = "waiting";
+            entry.priority = Infinity;
+            this.onEvict(entry.info.path);
+            this.changed(entry.info.path);
+        }
     }
     /** Why a file failed. */
     error(path) {
         return this.entries.get(path)?.error ?? null;
     }
-    progress() {
-        let ready = 0, failed = 0, bytes = 0, bytesReady = 0;
+    failure(path) {
+        const entry = this.entries.get(path), failure = entry?.failure;
+        return { message: entry?.error ?? "failed", ...(failure ? { reason: failure.reason, retryable: failure.retryable, ...(failure.status === undefined ? {} : { status: failure.status }) } : {}) };
+    }
+    failMissing(path) {
+        const failure = new MediaFailure({ asset: path, source: "build.json", reason: "invalid", retryable: false, message: `The build does not list required preview file ${path}. Rebuild or repair its prepared-media manifest.` });
+        this.entries.set(path, { info: { path, bytes: 0, hash: null }, status: "failed", priority: Infinity, data: null, error: failure.message, abort: null, used: ++this.clock, failure });
+        this.changed(path);
+    }
+    failures() {
+        return [...this.entries.values()].filter((entry) => entry.status === "failed").map((entry) => ({ path: entry.info.path, ...this.failure(entry.info.path) }));
+    }
+    progress(windowOnly = false) {
+        let ready = 0, failed = 0, bytes = 0, bytesReady = 0, files = 0;
         for (const e of this.entries.values()) {
+            if (windowOnly && !this.retained.has(e.info.path))
+                continue;
+            files++;
             bytes += e.info.bytes;
             if (e.status === "ready") {
                 ready++;
@@ -54,7 +112,7 @@ export class AssetLoader {
             if (e.status === "failed")
                 failed++;
         }
-        return { files: this.entries.size, ready, failed, bytes, bytesReady };
+        return { files, ready, failed, bytes, bytesReady, ...(windowOnly ? { streaming: true } : {}) };
     }
     /** Takes over files another loader (the last build's) already has, where the hashes match: a
      *  rebuild fetches only what changed. Returns the paths taken over. */
@@ -73,9 +131,10 @@ export class AssetLoader {
     /** Files that failed go back in the queue. */
     retry() {
         for (const e of this.entries.values())
-            if (e.status === "failed") {
+            if (e.status === "failed" && e.failure?.reason !== "invalid") {
                 e.status = "waiting";
                 e.error = null;
+                e.failure = null;
             }
         this.pump();
     }
@@ -85,6 +144,8 @@ export class AssetLoader {
     }
     dispose() {
         this.abort.abort();
+        for (const entry of this.entries.values())
+            entry.abort?.abort();
         this.listeners.clear();
     }
     changed(path) {
@@ -115,19 +176,33 @@ export class AssetLoader {
         }
     }
     async load(e) {
+        const abort = new AbortController();
+        e.abort = abort;
         try {
-            const data = await this.fetchBytes(e.info.path, this.abort.signal);
+            const data = await this.fetchBytes(e.info.path, AbortSignal.any([this.abort.signal, abort.signal]));
+            const integrity = (message) => new MediaFailure({ asset: e.info.path, source: e.info.url ?? e.info.path, reason: "integrity", retryable: false, message });
+            if (e.info.bytes > 0 && data.length !== e.info.bytes)
+                throw integrity(`its size differs from the build (${data.length} bytes, expected ${e.info.bytes})`);
             if (e.info.hash !== null && (await sha256(data)) !== e.info.hash)
-                throw new Error("its contents don't match the build (the hash differs)");
+                throw integrity("its contents don't match the build (the hash differs)");
             e.data = data;
             e.status = "ready";
         }
         catch (error) {
             if (this.abort.signal.aborted)
                 return;
+            if (abort.signal.aborted) {
+                e.status = "waiting";
+                return;
+            }
             e.status = "failed";
+            e.failure = error instanceof MediaFailure ? error : null;
             e.error = error instanceof Error ? error.message : String(error);
         }
+        finally {
+            e.abort = null;
+        }
         this.changed(e.info.path);
+        this.evict();
     }
 }

package/dist/player/audio.d.ts

@@ -4,7 +4,7 @@ export interface SoundtrackOptions {
     /** The film's length in frames. */
     readonly length: number;
     /** Mixes frames [from, to). */
-    readonly mix: (from: number, to: number) => Promise<MixResult>;
+    readonly mix: (from: number, to: number, signal: AbortSignal) => Promise<MixResult>;
     /** Frames per chunk (default 5 seconds). */
     readonly chunk?: number;
     /** Chunks kept mixed ahead of the playhead (default 3). */
@@ -15,6 +15,7 @@ export declare class Soundtrack {
     private context;
     private readonly chunks;
     private readonly mixing;
+    private readonly controllers;
     private playing;
     private readonly size;
     private readonly ahead;

package/dist/player/audio.js

@@ -7,6 +7,7 @@ export class Soundtrack {
     context = null;
     chunks = new Map();
     mixing = new Map();
+    controllers = new Map();
     playing = null;
     size;
     ahead;
@@ -104,6 +105,9 @@ export class Soundtrack {
         this.stop();
         this.chunks.clear();
         this.mixing.clear();
+        for (const controller of this.controllers.values())
+            controller.abort();
+        this.controllers.clear();
         this.generation++;
     }
     dispose() {
@@ -127,10 +131,13 @@ export class Soundtrack {
             return pending;
         const generation = this.generation;
         const from = k * this.size, to = Math.min(this.options.length, from + this.size);
-        const job = this.options.mix(from, to).then((mixed) => {
-            this.mixing.delete(k);
+        const controller = new AbortController();
+        this.controllers.set(k, controller);
+        const job = this.options.mix(from, to, controller.signal).then((mixed) => {
             if (generation !== this.generation)
                 return null;
+            this.mixing.delete(k);
+            this.controllers.delete(k);
             const buffer = new AudioBuffer({ numberOfChannels: 2, length: Math.max(1, mixed.left.length), sampleRate: mixed.sampleRate });
             buffer.copyToChannel(own(mixed.left), 0);
             buffer.copyToChannel(own(mixed.right), 1);
@@ -143,7 +150,10 @@ export class Soundtrack {
             }
             return buffer;
         }, () => {
-            this.mixing.delete(k);
+            if (generation === this.generation) {
+                this.mixing.delete(k);
+                this.controllers.delete(k);
+            }
             return null;
         });
         this.mixing.set(k, job);

package/dist/player/frames.d.ts

@@ -29,6 +29,10 @@ export declare class FrameCache<T extends Closable> {
     clear(): void;
     get bytes(): number;
     get size(): number;
+    exactRanges(): ReadonlyArray<{
+        readonly from: number;
+        readonly to: number;
+    }>;
     private remove;
     private trim;
 }

package/dist/player/frames.js

@@ -61,6 +61,18 @@ export class FrameCache {
     get size() {
         return this.frames.size;
     }
+    exactRanges() {
+        const frames = [...this.frames].filter(([, value]) => value.exact).map(([frame]) => frame).sort((a, b) => a - b);
+        const ranges = [];
+        for (const frame of frames) {
+            const last = ranges[ranges.length - 1];
+            if (last?.to === frame)
+                last.to++;
+            else
+                ranges.push({ from: frame, to: frame + 1 });
+        }
+        return ranges;
+    }
     remove(frame, entry) {
         entry.value.close();
         this.frames.delete(frame);

package/dist/player/index.d.ts

@@ -2,7 +2,7 @@ export { createPlayer, type Player, type PlayerOptions, type PlayerEvents, type
 export { Cancelled, type FrameResult } from "./pool.js";
 export { openFilm, openSource, openFiles, buildBase, SourceError, type FilmSource, type BuildSource, type FileSource, type OpenedBuild, type OpenedSource } from "./source.js";
 export { FilmNeeds, type Span, type Range } from "./needs.js";
-export { AssetLoader, type AssetFileInfo, type AssetStatus, type AssetProgress, type FetchBytes } from "./assets.js";
+export { AssetLoader, type AssetFileInfo, type AssetStatus, type AssetProgress, type AssetFailureInfo, type FetchBytes } from "./assets.js";
 export { FrameCache, type Cached, type Closable } from "./frames.js";
 export { Transport, type PlayState, type TickInput, type TickResult } from "./transport.js";
 export { wantedFrames, bufferedAhead, startBuffer, clockFrame, clampFrame, type Scales, type Want, type Situation } from "./schedule.js";

package/dist/player/needs.d.ts

@@ -20,12 +20,18 @@ export declare class FilmNeeds {
     /** What the soundtrack needs. */
     readonly audio: ReadonlyArray<string>;
     readonly length: number;
+    private readonly audioSegments;
+    private readonly audioClips;
+    private readonly fps;
     private constructor();
     /** A film's needs from its film.json, and the files its build has (which videos have proxies,
      *  and how many segments). */
-    static of(doc: FilmDoc, files?: ReadonlyArray<string>): FilmNeeds;
+    static of(doc: FilmDoc, files?: ReadonlyArray<string>, options?: {
+        readonly demandVideo?: boolean;
+    }): FilmNeeds;
     /** A film whose needs aren't known per clip: every frame needs every file. */
     static everything(paths: ReadonlyArray<string>, length: number): FilmNeeds;
+    audioPathsIn(from: number, to: number): ReadonlyArray<string>;
     /** The spans on screen at a frame, over every track. */
     spansAt(frame: number): ReadonlyArray<Span>;
     /** The spans on screen anywhere in [from, to). */

package/dist/player/needs.js

@@ -52,21 +52,27 @@ export class FilmNeeds {
     all;
     audio;
     length;
+    audioSegments;
+    audioClips;
+    fps;
     constructor(
     /** Visual tracks: what frames need. */
     lanes, 
     /** Every file the film loads: frames' and the soundtrack's. */
     all, 
     /** What the soundtrack needs. */
-    audio, length) {
+    audio, length, audioSegments = new Map(), audioClips = [], fps = 30) {
         this.lanes = lanes;
         this.all = all;
         this.audio = audio;
         this.length = length;
+        this.audioSegments = audioSegments;
+        this.audioClips = audioClips;
+        this.fps = fps;
     }
     /** A film's needs from its film.json, and the files its build has (which videos have proxies,
      *  and how many segments). */
-    static of(doc, files = []) {
+    static of(doc, files = [], options = {}) {
         const assets = clipAssets(doc);
         // Proxy segments by video, from the build's files: video/<name>/proxy/<k>.mp4.
         const segmentsOf = new Map();
@@ -79,13 +85,31 @@ export class FilmNeeds {
         const allSegments = (name) => Array.from({ length: segmentsOf.get(name) ?? 0 }, (_, k) => videoProxyPath({ name }, k));
         const lanes = [];
         const audio = new Set();
+        const audioSegments = new Map();
+        const audioClips = [];
+        for (const path of files) {
+            const name = /^audio\/([^/]+)\/\d+\.wav$/.exec(path)?.[1];
+            if (name !== undefined)
+                audioSegments.set(name, (audioSegments.get(name) ?? 0) + 1);
+        }
         for (const track of doc.tracks) {
             if (track.kind !== "visual") {
-                for (const clip of track.clips)
+                for (const clip of track.clips) {
+                    if (clip.kind === "audio")
+                        audioClips.push(clip);
                     for (const a of assets.get(clip.id) ?? [])
-                        if (a.kind === "audio")
-                            for (const path of importedFiles(a))
-                                audio.add(path);
+                        if (a.kind === "audio") {
+                            const count = audioSegments.get(a.name) ?? 0;
+                            if (count > 0) {
+                                audio.add(`audio/${a.name}/info.json`);
+                                for (let k = 0; k < count; k++)
+                                    audio.add(`audio/${a.name}/${String(k).padStart(5, "0")}.wav`);
+                            }
+                            else
+                                for (const path of importedFiles(a))
+                                    audio.add(path);
+                        }
+                }
                 continue;
             }
             const holds = new Map(track.transitions.map((t) => [t.from, t.frames]));
@@ -98,19 +122,39 @@ export class FilmNeeds {
                 if (clip.kind === "video" && segments > 0)
                     spans.push(...videoSpans(clip, to, base, segments, doc.film.fps));
                 else {
-                    const shown = clip.kind === "scene" ? refs.filter((a) => a.kind === "video").flatMap((a) => allSegments(a.name)) : [];
+                    const shown = clip.kind === "scene" && !options.demandVideo ? refs.filter((asset) => asset.kind === "video").flatMap((asset) => allSegments(asset.name)) : [];
                     spans.push({ clip: clip.id, from: clip.start, to, paths: [...base, ...shown] });
                 }
             }
             lanes.push(lane(spans));
         }
         const all = new Set([...lanes.flatMap((l) => l.spans.flatMap((s) => s.paths)), ...audio]);
-        return new FilmNeeds(lanes, [...all].sort(), [...audio].sort(), doc.film.length);
+        return new FilmNeeds(lanes, [...new Set([...all, ...[...segmentsOf.keys()].flatMap(allSegments)])].sort(), [...audio].sort(), doc.film.length, audioSegments, audioClips, doc.film.fps);
     }
     /** A film whose needs aren't known per clip: every frame needs every file. */
     static everything(paths, length) {
         return new FilmNeeds([lane([{ clip: "film", from: 0, to: length, paths }])], [...paths].sort(), [], length);
     }
+    audioPathsIn(from, to) {
+        if (this.audioSegments.size === 0)
+            return this.audio;
+        const paths = new Set();
+        for (const clip of this.audioClips) {
+            if (clip.end <= from || clip.start >= to)
+                continue;
+            const count = this.audioSegments.get(clip.asset) ?? 0;
+            if (count === 0) {
+                paths.add(`audio/${clip.asset}.wav`);
+                continue;
+            }
+            paths.add(`audio/${clip.asset}/info.json`);
+            const first = Math.max(0, Math.floor((clip.in + Math.max(0, from - clip.start) / this.fps * clip.speed - 0.5 * clip.speed) / 4));
+            const last = Math.min(count - 1, Math.floor((clip.in + Math.min(clip.duration, to - clip.start) / this.fps * clip.speed + 0.5 * clip.speed) / 4));
+            for (let k = first; k <= last; k++)
+                paths.add(`audio/${clip.asset}/${String(k).padStart(5, "0")}.wav`);
+        }
+        return [...paths];
+    }
     /** The spans on screen at a frame, over every track. */
     spansAt(frame) {
         return this.spansIn(frame, frame + 1);

package/dist/player/player.d.ts

@@ -1,11 +1,15 @@
 import type { PlanSummary } from "@tongue-core/film";
 import type { BuildDoc, FilmDoc } from "@tongue-core/project/doc";
-import { type AssetProgress } from "./assets.js";
+import { type AssetFailureInfo, type AssetProgress } from "./assets.js";
 import { type Range } from "./needs.js";
 import { type Scales } from "./schedule.js";
 import { type FilmSource } from "./source.js";
 import { type PlayState } from "./transport.js";
 export interface PlayerOptions {
+    /** Compressed media retained by the page; workers discard evicted segments too. Default 64 MiB. */
+    readonly mediaCacheBytes?: number;
+    /** Seconds of preview media to fetch ahead. Default 15. */
+    readonly mediaAhead?: number;
     /** Where frames are drawn. Its pixel size follows the film's size at the still scale. */
     readonly canvas: HTMLCanvasElement;
     /** The worker script: @tongue-core/ui/player/worker, bundled. */
@@ -20,14 +24,14 @@ export interface PlayerOptions {
     readonly ahead?: number;
 }
 export interface Readiness {
+    readonly mediaBytes?: number;
     /** Stretches of the film whose frames can all be drawn now. */
     readonly ranges: ReadonlyArray<Range>;
     readonly progress: AssetProgress;
     /** Files that couldn't be loaded, and why. */
     readonly failed: ReadonlyArray<{
         readonly path: string;
-        readonly message: string;
-    }>;
+    } & AssetFailureInfo>;
 }
 /** What is on the canvas: which frame, how sharp, and whether it is the playhead's, sharp. */
 export interface Shown {
@@ -77,6 +81,8 @@ export interface Player {
     load(source: FilmSource): Promise<LoadedFilm>;
     /** Loads the current build again. */
     reload(): Promise<LoadedFilm>;
+    /** Retries failed media after the host has restored access. */
+    retry(): void;
     seek(frame: number): void;
     play(): void;
     pause(): void;