@clipkit/runtime 1.0.0

package/dist/runtime.d.ts

@@ -0,0 +1,216 @@
+import type { Source } from '@clipkit/protocol';
+import { type ExportOptions, type FrameProducer } from './encoder/exporter.js';
+export interface RuntimeInitOptions {
+    /**
+     * Preferred backend. Defaults to 'auto' (try WebGPU, fall back to WebGL2).
+     * Use 'webgpu' or 'webgl2' to force a specific backend, e.g. for testing.
+     */
+    backend?: 'auto' | 'webgpu' | 'webgl2';
+}
+export declare class ClipkitRuntime implements FrameProducer {
+    readonly canvas: HTMLCanvasElement | OffscreenCanvas;
+    private backend;
+    private currentSource;
+    /** When true, frame() clears to a transparent background if the Source has no
+     *  background_color — set during alpha (transparent) frame export. */
+    private transparentBackground;
+    private images;
+    private videos;
+    private audioBuffers;
+    private fontAtlases;
+    private svgRasters;
+    private maskedTexts;
+    private groupTargets;
+    /**
+     * Eviction epoch for the groupTargets pool — bumped once per frame() and once
+     * per renderFinalFrame(), threaded into RenderContext.frameIndex so pool
+     * entries can be stamped last-touched for frame-boundary LRU eviction. NOT a
+     * clean 1:1 output-frame ordinal: it advances per motion-blur SAMPLE (each
+     * export sample funnels through frame()) and the renderFinalFrame→frame()
+     * fallback bumps twice. Only ever compared for equality at eviction time, so
+     * gaps/jumps are harmless — do not treat it as a frame number.
+     */
+    private frameIndex;
+    /**
+     * Soft cap (bytes) on the offscreen-FBO pool (groupTargets). After each render
+     * pass's endFrame(), entries NOT touched this frame are evicted LRU-first
+     * until total pooled bytes (logical w*h*4 per target) drop under this. SOFT:
+     * targets touched this frame are never evicted, so a single frame whose
+     * working set exceeds the cap stays above it (graceful — never thrashes).
+     * Default 512 MiB: a glassy element peaks at ~3 full-surface 1080p targets
+     * (::bd + ::fx + ::fx-scratch ≈ 25 MB), so 512 MiB holds ~18-20
+     * concurrently-active glass elements — well above any real frame's live set —
+     * while bounding the cross-timeline tail of inactive-element keys that is the
+     * unbounded-growth / cloud-BLACK cause. Logical bytes ≈ real VRAM only when
+     * pixelRatio=1 (the cloud Job, renderResolution='source'); high-DPI preview
+     * under-accounts by pixelRatio² (harmless — just a looser bound off-cloud).
+     * Override via setGroupTargetPoolCap(); ≤ 0 disables eviction (legacy).
+     */
+    private groupTargetPoolCapBytes;
+    private luts;
+    /** Scratch targets for renderFinalFrame's blur accumulation. */
+    private blurTargets;
+    private initialized;
+    private disposed;
+    constructor(canvas: HTMLCanvasElement | OffscreenCanvas);
+    init(options?: RuntimeInitOptions): Promise<boolean>;
+    /** Which graphics API this runtime is using. Only valid after init() resolves true. */
+    get api(): 'webgpu' | 'webgl2' | null;
+    /** Set the current source. Does NOT trigger asset loading — call preload(). */
+    load(source: Source): void;
+    /**
+     * Load every external asset referenced by the current source. Images
+     * and videos are downloaded; fonts are requested via FontFace API.
+     * Idempotent — already-loaded URLs are skipped.
+     */
+    preload(source?: Source): Promise<void>;
+    private preloadAudio;
+    /**
+     * Decode a video's embedded audio track. decodeAudioData accepts MP4
+     * containers directly (it extracts the first audio track), so this is
+     * the same loadAudio path audio elements use. Failures are expected
+     * (video without audio) and logged at debug. Workers skip — the
+     * main-thread AudioScheduler owns preview audio.
+     */
+    private preloadVideoAudio;
+    private preloadLut;
+    private preloadImage;
+    /**
+     * Preload an arbitrary image URL into the shared image cache (used by
+     * image elements AND material normal maps, §4.8 Phase 2). Normal maps
+     * upload as plain rgba8unorm and are sampled linearly — no premultiply
+     * or sRGB transform corrupts the encoded normals.
+     */
+    /** Average color of each loaded equirect environment image (§4.8). */
+    private envAvg;
+    /**
+     * Preload an equirect environment image AND compute its average color
+     * (downscale to 1×1) for the roughness-blurred reflection fallback.
+     */
+    private preloadEnvImage;
+    private preloadImageUrl;
+    private preloadVideo;
+    /** Whether a decodable video asset exists for this URL. */
+    hasVideoAsset(url: string): boolean;
+    /**
+     * Walk the element tree and collect every ACTIVE video element with
+     * the LOCAL clock value it should be evaluated at. Groups translate
+     * the clock (child times are group-relative) and, with time_remap
+     * (§5.8.4), WARP it — so nested videos inside speed-ramped/frozen/
+     * reversed groups decode the right frames. Fixes the former flat
+     * scan, which never updated videos nested in groups at all.
+     */
+    private collectActiveVideos;
+    /**
+     * Decode + upload the exact video frame each active frameSource-backed
+     * video element needs at composition time `time`. Await this before
+     * the synchronous `frame(time)` pass in contexts that use the
+     * WebCodecs path (the playback worker). Element-backed and externally
+     * pumped assets are untouched.
+     */
+    prepareVideoFrames(time: number): Promise<void>;
+    /**
+     * Feed one decoded video frame for a source URL from outside the
+     * runtime. This is the worker-context video path: the host decodes
+     * with an HTMLVideoElement on the main thread, transfers ImageBitmaps
+     * here, and the video renderer samples the resulting texture exactly
+     * like an element-backed asset.
+     *
+     * The bitmap is uploaded immediately and then closed. If the media
+     * dimensions change (different bitmap size), the texture is recreated.
+     */
+    pushExternalVideoFrame(url: string, bitmap: ImageBitmap): void;
+    /** Render one frame at the given time. Synchronous. */
+    frame(time: number): void;
+    /**
+     * Whole-frame bloom post (§4.8): extract pixels above the threshold,
+     * Gaussian-blur them, and add them back over the scene. Bright regions
+     * (specular highlights, emissive surfaces, bright media) bleed light;
+     * the amount is driven by each region's own brightness. Draws the final
+     * composite to the CURRENT target (the surface). Reuses blur slots 3/4.
+     */
+    private applyBloom;
+    /**
+     * Render one frame WITH source-level motion blur (§2.1) — the
+     * preview's "final quality" path, used when the playhead is parked.
+     * Renders N sub-frame samples into an offscreen layer and keeps a
+     * GPU running average (avg_k = avg_{k-1}·k/(k+1) + sample·1/(k+1)),
+     * then composites the result to the canvas. Within a few 8-bit
+     * rounding steps of the exporter's exact float average — preview
+     * parity, not the normative path. Falls back to frame() when the
+     * source has no motion blur. Video textures are decoded once at the
+     * frame time (call prepareVideoFrames first), not per sample —
+     * WebGPU executes the whole frame's draws at submit, so per-sample
+     * uploads would all show the last sample's pixels anyway.
+     */
+    renderFinalFrame(time: number, sampleOverride?: number): void;
+    /** Resolve when all submitted GPU work has executed (see Backend.finish). */
+    gpuFinish(): Promise<void>;
+    /**
+     * Set the soft cap (bytes) for the offscreen-FBO pool. ≤ 0 disables eviction
+     * (unbounded — the legacy behavior). Lets the cloud render Job tighten the cap
+     * on memory-constrained SwiftShader instances.
+     */
+    setGroupTargetPoolCap(bytes: number): void;
+    /**
+     * Frame-boundary LRU eviction of the groupTargets offscreen-FBO pool. MUST be
+     * called only AFTER backend.endFrame() (post queue.submit on WebGPU) so
+     * destroyRenderTarget()'s synchronous gpuTexture.destroy() can never race a
+     * not-yet-submitted command encoder. Entries touched THIS frame (lastTouched
+     * === this.frameIndex) are never candidates; stale entries are dropped
+     * least-recently-touched first until total pooled bytes (w*h*4) fall under the
+     * cap. Safe because every pooled target is fully rewritten (create-or-reuse →
+     * pushTarget+clear+draw) each frame it is used, so evicting an unused entry
+     * only costs a realloc next time. No double-destroy: delete() removes the
+     * entry, so the size-change path (group.ts/scene.ts) and dispose() never see
+     * it again. MUST stay STATELESS across calls (no cached candidate/untouched
+     * set) — frameIndex can advance >1 per output frame (per motion-blur sample +
+     * the renderFinalFrame→frame() fallback).
+     */
+    private evictGroupTargets;
+    private acquireBlurTarget;
+    /**
+     * Render one frame at the given time, awaiting any per-element async
+     * work (video seeking) first. Used by the exporter to render frames
+     * deterministically.
+     */
+    renderAsync(source: Source, time: number): Promise<void>;
+    /**
+     * Export to MP4. Renders frames serially through the same backend used
+     * for preview — the canvas visibly resizes during export. For an off-
+     * screen export, construct a separate ClipkitRuntime around an
+     * OffscreenCanvas at the desired dimensions.
+     */
+    /**
+     * Render each frame and stream its PNG (with alpha when `alpha`) to `onFrame`.
+     * The transparent export path — the PNGs are assembled by a server-side ffmpeg
+     * (ProRes 4444 / VP9-alpha). WebCodecs can't encode alpha, so this is the only
+     * route to transparent output.
+     */
+    /**
+     * Mix all of the loaded Source's audio (audio elements + video-embedded
+     * tracks) into one AudioBuffer for export. Shared by both export paths —
+     * the opaque MP4 path (export → WebCodecs muxer) and the transparent frame
+     * path (exportFrames → WAV → ffmpeg) — so they never diverge. Returns null
+     * when the comp has no audio (or the mix fails); the caller exports silently.
+     */
+    private mixExportAudio;
+    exportFrames(options: {
+        framerate?: number;
+        alpha?: boolean;
+        onFrame: (index: number, pngBase64: string) => void | Promise<void>;
+        /**
+         * Receives the mixed audio as a 16-bit PCM WAV (raw bytes) once, before
+         * the frames stream. Not called when the comp has no audio. The frame
+         * path has no muxer, so audio rides as a separate WAV the server-side
+         * ffmpeg maps in as a second input.
+         */
+        onAudio?: (wav: ArrayBuffer) => void | Promise<void>;
+        onProgress?: (progress: number) => void;
+    }): Promise<number>;
+    export(options?: ExportOptions): Promise<Blob>;
+    /** Release all GPU resources. After this the runtime is unusable. */
+    dispose(): void;
+    private makeContext;
+}
+//# sourceMappingURL=runtime.d.ts.map

package/dist/runtime.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"runtime.d.ts","sourceRoot":"","sources":["../src/runtime.ts"],"names":[],"mappings":"AAYA,OAAO,KAAK,EAAyB,MAAM,EAA2C,MAAM,mBAAmB,CAAC;AAqBhH,OAAO,EAA4C,KAAK,aAAa,EAAE,KAAK,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAIzH,MAAM,WAAW,kBAAkB;IACjC;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,GAAG,QAAQ,GAAG,QAAQ,CAAC;CACxC;AAED,qBAAa,cAAe,YAAW,aAAa;IAClD,QAAQ,CAAC,MAAM,EAAE,iBAAiB,GAAG,eAAe,CAAC;IAErD,OAAO,CAAC,OAAO,CAAW;IAC1B,OAAO,CAAC,aAAa,CAAuB;IAC5C;0EACsE;IACtE,OAAO,CAAC,qBAAqB,CAAS;IACtC,OAAO,CAAC,MAAM,CAAgC;IAC9C,OAAO,CAAC,MAAM,CAAgC;IAC9C,OAAO,CAAC,YAAY,CAAiC;IACrD,OAAO,CAAC,WAAW,CAA+B;IAClD,OAAO,CAAC,UAAU,CAAoC;IACtD,OAAO,CAAC,WAAW,CAAqC;IACxD,OAAO,CAAC,YAAY,CAAqC;IACzD;;;;;;;;OAQG;IACH,OAAO,CAAC,UAAU,CAAK;IACvB;;;;;;;;;;;;;;OAcG;IACH,OAAO,CAAC,uBAAuB,CAAqB;IACpD,OAAO,CAAC,IAAI,CAA8B;IAC1C,gEAAgE;IAChE,OAAO,CAAC,WAAW,CAAiE;IACpF,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,QAAQ,CAAS;gBAEb,MAAM,EAAE,iBAAiB,GAAG,eAAe;IAIjD,IAAI,CAAC,OAAO,GAAE,kBAAuB,GAAG,OAAO,CAAC,OAAO,CAAC;IA4B9D,uFAAuF;IACvF,IAAI,GAAG,IAAI,QAAQ,GAAG,QAAQ,GAAG,IAAI,CAEpC;IAED,+EAA+E;IAC/E,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI;IAQ1B;;;;OAIG;IACG,OAAO,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;YAgJ/B,YAAY;IAc1B;;;;;;OAMG;YACW,iBAAiB;YAejB,UAAU;YAgBV,YAAY;IAI1B;;;;;OAKG;IACH,sEAAsE;IACtE,OAAO,CAAC,MAAM,CAA+C;IAE7D;;;OAGG;YACW,eAAe;YAiBf,eAAe;YAqBf,YAAY;IA8D1B,2DAA2D;IAC3D,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO;IAKnC;;;;;;;OAOG;IACH,OAAO,CAAC,mBAAmB;IA0B3B;;;;;;OAMG;IACG,kBAAkB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAwCrD;;;;;;;;;OASG;IACH,sBAAsB,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,GAAG,IAAI;IA4B9D,uDAAuD;IACvD,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI;IAkCzB;;;;;;OAMG;IACH,OAAO,CAAC,UAAU;IAwBlB;;;;;;;;;;;;OAYG;IACH,gBAAgB,CAAC,IAAI,EAAE,MAAM,EAAE,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI;IAuE7D,6EAA6E;IACvE,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC;IAKhC;;;;OAIG;IACH,qBAAqB,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI;IAI1C;;;;;;;;;;;;;;OAcG;IACH,OAAO,CAAC,iBAAiB;IA0BzB,OAAO,CAAC,iBAAiB;IAWzB;;;;OAIG;IACG,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAmC9D;;;;;OAKG;IACH;;;;;OAKG;IACH;;;;;;OAMG;YACW,cAAc;IAYtB,YAAY,CAAC,OAAO,EAAE;QAC1B,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,KAAK,CAAC,EAAE,OAAO,CAAC;QAChB,OAAO,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;QACpE;;;;;WAKG;QACH,OAAO,CAAC,EAAE,CAAC,GAAG,EAAE,WAAW,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;QACrD,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,IAAI,CAAC;KACzC,GAAG,OAAO,CAAC,MAAM,CAAC;IAsBb,MAAM,CAAC,OAAO,GAAE,aAAkB,GAAG,OAAO,CAAC,IAAI,CAAC;IAkDxD,qEAAqE;IACrE,OAAO,IAAI,IAAI;IAmBf,OAAO,CAAC,WAAW;CAwDpB"}