@tongue-core/engine 0.1.0-rc.8 → 0.1.0-rc.9

package/dist/assets/codecs/types.d.ts

@@ -6,6 +6,8 @@ export interface RgbaImage {
 }
 /** Planar float audio in -1..1. Every channel has the same length. */
 export interface PcmAudio {
+    /** Source sample represented by channels[0][0], for a window decoded from segmented audio. */
+    readonly offset?: number;
     readonly sampleRate: number;
     readonly channels: ReadonlyArray<Float32Array>;
 }

package/dist/assets/store.d.ts

@@ -21,6 +21,7 @@ export interface LoadedVideo {
     readonly poster: LoadedImage | null;
 }
 export interface LoadedAssets {
+    readonly audioWindow?: (name: string, fromSeconds: number, toSeconds: number) => PcmAudio | null;
     readonly fonts: FontLibrary;
     readonly images: ReadonlyMap<string, LoadedImage>;
     readonly audio: ReadonlyMap<string, PcmAudio>;
@@ -65,6 +66,9 @@ export declare function loadAssets(assets: ReadonlyArray<Asset> | undefined, fet
  *  need): the maps a renderer reads at draw time, filled as files come. Drawing a frame whose
  *  files aren't here yet throws, as with any missing asset; nothing is drawn half loaded. */
 export declare class AssetShelf implements LoadedAssets {
+    private readonly pendingPosters;
+    private readonly audioInfo;
+    private readonly audioParts;
     readonly fonts: FontLibrary;
     readonly images: Map<string, LoadedImage>;
     readonly audio: Map<string, PcmAudio>;
@@ -75,7 +79,16 @@ export declare class AssetShelf implements LoadedAssets {
     add(asset: Asset, path: string, bytes: Uint8Array, audio?: boolean): void;
     /** Whether the file at `path` is on the shelf. */
     has(path: string): boolean;
+    remove(path: string): void;
+    readonly audioWindow: (name: string, fromSeconds: number, toSeconds: number) => PcmAudio | null;
 }
+export declare function parseAudioInfo(bytes: Uint8Array): {
+    readonly sampleRate: number;
+    readonly segments: ReadonlyArray<{
+        readonly first: number;
+        readonly samples: number;
+    }>;
+};
 /** A frame needs an image that isn't here: a host loading on demand fetches `asset` and draws again. */
 export declare class ImageNotLoaded extends AssetError {
     readonly asset: ImageAsset;

package/dist/assets/store.js

@@ -82,6 +82,9 @@ export async function loadAssets(assets = declaredAssets(), fetch, options = {})
  *  need): the maps a renderer reads at draw time, filled as files come. Drawing a frame whose
  *  files aren't here yet throws, as with any missing asset; nothing is drawn half loaded. */
 export class AssetShelf {
+    pendingPosters = new Map();
+    audioInfo = new Map();
+    audioParts = new Map();
     fonts = new FontLibrary();
     images = new Map();
     audio = new Map();
@@ -97,18 +100,33 @@ export class AssetShelf {
             case "image":
                 this.images.set(asset.name, { qoi: bytes, ...qoiSize(bytes) });
                 break;
-            case "audio":
-                if (audio)
+            case "audio": {
+                if (!audio)
+                    break;
+                if (path.endsWith("/info.json"))
+                    this.audioInfo.set(asset.name, parseAudioInfo(bytes));
+                else if (path.includes(`/audio/${asset.name}/`) || path.startsWith(`audio/${asset.name}/`)) {
+                    const index = Number(/\/(\d+)\.wav$/.exec(path)?.[1]);
+                    if (!Number.isInteger(index))
+                        throw new AssetError(`Invalid audio segment ${path}.`);
+                    const parts = this.audioParts.get(asset.name) ?? new Map();
+                    parts.set(index, readWav(bytes));
+                    this.audioParts.set(asset.name, parts);
+                }
+                else
                     this.audio.set(asset.name, readWav(bytes));
                 break;
+            }
             case "video": {
                 const known = this.videos.get(asset.name);
-                if (path === videoInfoPath(asset))
-                    this.videos.set(asset.name, { info: parseVideoInfo(bytes, asset.name), poster: known?.poster ?? null });
+                if (path === videoInfoPath(asset)) {
+                    this.videos.set(asset.name, { info: parseVideoInfo(bytes, asset.name), poster: known?.poster ?? this.pendingPosters.get(asset.name) ?? null });
+                    this.pendingPosters.delete(asset.name);
+                }
                 else if (path === videoPosterPath(asset) && known)
                     this.videos.set(asset.name, { info: known.info, poster: { qoi: bytes, ...qoiSize(bytes) } });
                 else if (path === videoPosterPath(asset))
-                    throw new AssetError(`video "${asset.name}": its poster came before its info.json.`);
+                    this.pendingPosters.set(asset.name, { qoi: bytes, ...qoiSize(bytes) });
                 break;
             }
         }
@@ -118,6 +136,56 @@ export class AssetShelf {
     has(path) {
         return this.held.has(path);
     }
+    remove(path) {
+        this.held.delete(path);
+        const image = /^images\/([^/]+)\.qoi$/.exec(path)?.[1];
+        if (image !== undefined)
+            this.images.delete(image);
+        const poster = /^video\/([^/]+)\/poster\.qoi$/.exec(path)?.[1];
+        const video = poster === undefined ? undefined : this.videos.get(poster);
+        if (poster !== undefined)
+            this.pendingPosters.delete(poster);
+        if (poster !== undefined && video)
+            this.videos.set(poster, { ...video, poster: null });
+        const audio = /^audio\/([^/]+)\/(\d+)\.wav$/.exec(path);
+        if (audio?.[1] !== undefined)
+            this.audioParts.get(audio[1])?.delete(Number(audio[2]));
+    }
+    audioWindow = (name, fromSeconds, toSeconds) => {
+        const info = this.audioInfo.get(name);
+        if (!info)
+            return this.audio.get(name) ?? null;
+        const first = Math.max(0, Math.floor(fromSeconds * info.sampleRate));
+        const end = Math.ceil(toSeconds * info.sampleRate);
+        const channels = [new Float32Array(Math.max(0, end - first)), new Float32Array(Math.max(0, end - first))];
+        for (let k = 0; k < info.segments.length; k++) {
+            const segment = info.segments[k];
+            if (!segment || segment.first >= end || segment.first + segment.samples <= first)
+                continue;
+            const part = this.audioParts.get(name)?.get(k);
+            if (!part)
+                throw new AssetError(`audio "${name}": segment ${k} is not loaded.`);
+            if (part.sampleRate !== info.sampleRate || part.channels.length !== 2 || part.channels.some((channel) => channel.length !== segment.samples))
+                throw new AssetError(`audio "${name}": segment ${k} has a different rate, channel count or length from its metadata.`);
+            const a = Math.max(first, segment.first), b = Math.min(end, segment.first + segment.samples);
+            channels.forEach((channel, c) => channel.set(part.channels[c]?.subarray(a - segment.first, b - segment.first) ?? new Float32Array(b - a), a - first));
+        }
+        return { sampleRate: info.sampleRate, channels, offset: first };
+    };
+}
+export function parseAudioInfo(bytes) {
+    const value = JSON.parse(new TextDecoder().decode(bytes));
+    if (!isRecord(value) || value["sampleRate"] !== 48000 || value["channels"] !== 2 || !Array.isArray(value["segments"]))
+        throw new AssetError("Invalid segmented audio metadata.");
+    let end = 0;
+    const segments = value["segments"].map((entry) => {
+        if (!isRecord(entry) || entry["first"] !== end || typeof entry["samples"] !== "number" || !Number.isInteger(entry["samples"]) || entry["samples"] <= 0)
+            throw new AssetError("Audio segments must be contiguous and have a positive sample count.");
+        const segment = { first: end, samples: entry["samples"] };
+        end += segment.samples;
+        return segment;
+    });
+    return { sampleRate: 48000, segments };
 }
 /** A frame needs an image that isn't here: a host loading on demand fetches `asset` and draws again. */
 export class ImageNotLoaded extends AssetError {

package/dist/audio/mix.js

@@ -94,9 +94,10 @@ function channelAt(pcm, channel, fromSeconds, length) {
         return out;
     const ratio = pcm.sampleRate / SAMPLE_RATE;
     const offset = fromSeconds * pcm.sampleRate;
+    const windowOffset = pcm.offset ?? 0;
     for (let i = 0; i < length; i++) {
         const pos = offset + i * ratio;
-        const j = Math.floor(pos), f = pos - j;
+        const index = Math.floor(pos), j = index - windowOffset, f = pos - index;
         out[i] = (src[j] ?? 0) * (1 - f) + (src[j + 1] ?? 0) * f;
     }
     return out;

package/dist/render/renderer.d.ts

@@ -65,6 +65,8 @@ export declare class FilmRenderer {
     /** Puts an image in the raster now (decoded, mip-mapped), so the first frame that shows it doesn't
      *  wait for that: what a player does with an image as it arrives. */
     preloadImage(name: string): void;
+    /** Releases the raster's decoded copy when a host evicts a remote image. */
+    forgetImage(name: string): void;
     /** Keeps only the images `out` draws in the raster and returns their names. Decoded images are
      *  the raster's biggest cost, so a host with little memory (a Worker) calls this before drawing,
      *  and drops the files of everything else. A later frame that shows one loads it again. */

package/dist/render/renderer.js

@@ -172,6 +172,12 @@ export class FilmRenderer {
         this.backend.loadImage(id, image.qoi);
         this.loaded.set(id, "image");
     }
+    /** Releases the raster's decoded copy when a host evicts a remote image. */
+    forgetImage(name) {
+        const id = this.imageIds.get(`${IMAGE_KEY}${name}`);
+        if (id !== undefined && this.loaded.delete(id))
+            this.backend.loadImage(id, NOTHING);
+    }
     /** Keeps only the images `out` draws in the raster and returns their names. Decoded images are
      *  the raster's biggest cost, so a host with little memory (a Worker) calls this before drawing,
      *  and drops the files of everything else. A later frame that shows one loads it again. */

package/dist/render/sounds.d.ts

@@ -9,4 +9,7 @@ export declare function collectSounds(plan: FilmPlan, range?: {
     readonly to: number;
 }): ReadonlyArray<SoundEvent>;
 /** The film's audio clips with their decoded audio (clips whose audio isn't loaded are left out). */
-export declare function audioClips(plan: FilmPlan, assets: LoadedAssets): ReadonlyArray<ClipAudio>;
+export declare function audioClips(plan: FilmPlan, assets: LoadedAssets, range?: {
+    readonly from: number;
+    readonly to: number;
+}): ReadonlyArray<ClipAudio>;

package/dist/render/sounds.js

@@ -34,12 +34,16 @@ export function collectSounds(plan, range) {
     return [...seen.values()].sort((a, b) => a.at - b.at);
 }
 /** The film's audio clips with their decoded audio (clips whose audio isn't loaded are left out). */
-export function audioClips(plan, assets) {
+export function audioClips(plan, assets, range) {
     return plan.tracks.filter((t) => t.kind === "audio").flatMap((t) => t.clips).flatMap((clip) => {
         const s = clip.source;
         if (s.kind !== "audio")
             return [];
-        const pcm = assets.audio.get(s.asset.name);
+        if (range && (clip.end <= range.from || clip.start >= range.to))
+            return [];
+        const from = range ? s.in + Math.max(0, (range.from - clip.start) / plan.fps) * s.speed : s.in;
+        const to = range ? s.in + Math.min(clip.duration / plan.fps, (range.to - clip.start) / plan.fps) * s.speed : s.out;
+        const pcm = assets.audioWindow?.(s.asset.name, Math.max(s.in, from - 0.2 * s.speed), to + 0.2 * s.speed) ?? assets.audio.get(s.asset.name);
         return pcm ? [{ pcm, start: clip.start, end: clip.end, in: s.in, speed: s.speed, gain: s.gain, fadeIn: s.fadeIn, fadeOut: s.fadeOut }] : [];
     });
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tongue-core/engine",
-  "version": "0.1.0-rc.8",
+  "version": "0.1.0-rc.9",
   "description": "Frame to pixels for Tongue films: evaluate, layout, paint and rasterize, deterministic everywhere.",
   "type": "module",
   "license": "MIT",
@@ -26,7 +26,7 @@
     "README.md"
   ],
   "dependencies": {
-    "@tongue-core/film": "0.1.0-rc.8"
+    "@tongue-core/film": "0.1.0-rc.9"
   },
   "publishConfig": {
     "access": "public"