@vosjs/render-core 0.1.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 vosso
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,55 @@
+# @vosso/render-core
+
+The orchestration math and Node-side media plumbing for deterministic vos
+renders. Pixels never leave the browser — every frame is encoded in-page via
+WebCodecs + [mediabunny](https://mediabunny.dev). This package owns everything
+around that: how a timeline is sharded into parallel chunks, and how the encoded
+chunks are stitched back into one file without re-encoding.
+
+Pure Node, no browser and no WebGL. Consumed in-source by
+[`@vosso/vos-plugin`](../vos-plugin) (bundled via tsup `noExternal`) and by the
+vosso render worker's finalize path. MIT.
+
+## Why sharding is correct here
+
+A vos render is a pure `seek(t)` function, so a timeline splits cleanly into
+independent frame ranges. Each range is rendered concurrently as its own page,
+encoded with **pinned** encoder params and chunk-local timestamps, then the
+packets are stream-copied into a single container — a plain demux/remux, no
+re-encode, no quality loss. Timestamp offsets come from the plan (frames ÷ fps),
+never from measured durations, so rounding can't drift.
+
+```
+planChunks(total, fps, policy) ──▶ [ {startFrame, endFrame}, … ]
+        │  render each range concurrently (browser, pinned encoder)
+        ▼
+concatEncodedVideo([chunk₀, chunk₁, …]) ──▶ one file, packet-identical to a single-pass render
+```
+
+Audio stays out of the chunks by design and is mixed **once** at finalize, so AAC
+priming seams never exist.
+
+## Exports
+
+| Export                                       | Purpose                                                                                                                                                            |
+| -------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
+| `planChunks(totalFrames, fps, policy)`       | Split a timeline into balanced frame ranges (floor `DEFAULT_MIN_FRAMES_PER_CHUNK` per chunk). Pure math.                                                           |
+| `concatEncodedVideo(chunks, options)`        | Stream-copy encoded chunks into one video (mediabunny demux/mux, no re-encode).                                                                                    |
+| `countVideoPackets(bytes)`                   | Count video packets in an encoded file — used by parity checks.                                                                                                    |
+| `buildFinalizeConcatPage(options)`           | Build the in-browser finalize page: fetch chunk parts, concat, mux produced audio, upload the result. Runs in a browser page (its memory, not the 128 MB isolate). |
+| `audioProducerCode()` / `dataHasAudio(data)` | Page-JS mirror of the client audio exporter — produces the mixed audio buffer at finalize from the lowered Voila data.                                             |
+
+## Contract: the concat mirror
+
+`concat.ts` and the concat inside `buildFinalizeConcatPage` must stay
+**packet-identical** — the finalize page runs in a browser, the standalone concat
+runs in Node, and cloud exports fan-in through the page while the CLI uses the Node
+path. `vos-plugin`'s `verify-finalize-page` asserts they produce identical output;
+keep the two in sync when touching either.
+
+## Development
+
+```bash
+pnpm --filter @vosso/render-core test       # vitest
+pnpm --filter @vosso/render-core typecheck
+```

package/dist/index.d.ts

@@ -0,0 +1,394 @@
+import { VideoCodec } from 'mediabunny';
+
+/**
+ * Chunk planner — timeline sharding for deterministic renders.
+ *
+ * Because vos evaluation is a pure function of time (seek(t) — the whole
+ * point of the sampler tween backend), any frame range can render in
+ * isolation. The planner splits [0, totalFrames) into balanced, contiguous
+ * ranges; each chunk encodes independently (so it starts on a keyframe by
+ * construction) with chunk-LOCAL timestamps starting at 0, and the finalize
+ * step (concat.ts) offsets packets back onto the global timeline.
+ *
+ * Pure math — no I/O — so the policy stays unit-testable and reusable by
+ * every harness (CLI, local server, render service).
+ */
+interface ChunkPlanPolicy {
+    /** Upper bound on simultaneous chunk renders (browser pages/sessions). */
+    maxParallel: number;
+    /**
+     * Below this many frames per chunk, extra parallelism costs more in
+     * per-chunk fixed overhead (page load, module import, first seek) than it
+     * saves — stop splitting. (Remotion's floor is 5; ours is higher because
+     * chunk startup includes CDN module imports.)
+     */
+    minFramesPerChunk?: number;
+}
+declare const DEFAULT_MIN_FRAMES_PER_CHUNK = 24;
+interface RenderChunk {
+    /** 0-based chunk index; also the concat order. */
+    index: number;
+    /** First frame of the chunk (inclusive, global frame numbering). */
+    startFrame: number;
+    /** End frame (exclusive). */
+    endFrame: number;
+    frameCount: number;
+    /** Global start time in seconds (startFrame / fps). */
+    startTime: number;
+    /** Exact chunk duration in seconds (frameCount / fps). */
+    duration: number;
+}
+/**
+ * Split `totalFrames` into at most `maxParallel` balanced contiguous chunks.
+ * Sizes differ by at most one frame; chunk boundaries are exact frame
+ * indices so no frame is rendered twice or skipped.
+ */
+declare function planChunks(totalFrames: number, fps: number, policy: ChunkPlanPolicy): RenderChunk[];
+
+/**
+ * Finalize concat — stream-copy chunk videos into one file, no re-encode.
+ *
+ * Each chunk is an independently encoded file whose timestamps start at 0
+ * (chunk-local). Concat demuxes every chunk's video packets and re-muxes
+ * them into one output, offsetting timestamps by the chunk's global start
+ * time. Because chunks were encoded with identical codec/params (the caller's
+ * responsibility — one encoder config per render) and every independently
+ * encoded file begins with a keyframe, packet-level concatenation is valid
+ * with zero quality loss and near-zero CPU: pure container work, runs in
+ * plain Node (mediabunny demux/mux is pure JS; no WebCodecs needed).
+ *
+ * Audio is deliberately absent here: per the rendering plan, audio renders
+ * ONCE centrally and gets muxed against the concatenated video, so encoder
+ * priming seams never exist.
+ */
+
+interface ConcatChunk {
+    /** Encoded chunk file bytes (WebM or MP4, matching `format`). */
+    data: Uint8Array;
+    /**
+     * The chunk's exact intended duration in seconds (frameCount / fps, from
+     * the chunk plan). Used as the timestamp offset for the NEXT chunk —
+     * derived from the plan, not from packet rounding, so drift can't
+     * accumulate across chunks.
+     */
+    duration: number;
+}
+interface ConcatOptions {
+    format: 'webm' | 'mp4';
+    /** Stamped into the output track metadata (players use it as a hint). */
+    frameRate?: number;
+}
+interface ConcatResult {
+    bytes: Uint8Array;
+    /** Total video packets written. */
+    packetCount: number;
+    /** Codec copied through (from the first chunk). */
+    codec: VideoCodec;
+}
+interface MuxExportOptions extends ConcatOptions {
+    /**
+     * Video parts in plan order. An async iterable lets the caller feed parts
+     * ONE at a time from storage instead of materializing all of them first
+     * (the worker finalize's memory shape).
+     */
+    video: AsyncIterable<ConcatChunk> | Iterable<ConcatChunk>;
+    /**
+     * Optional encoded audio file (any container mediabunny reads — the audio
+     * mix page produces Opus in WebM). Its packets are STREAM-COPIED into the
+     * output's audio track: no decode, no WebCodecs, pure container work, so
+     * this runs in a Worker isolate exactly like the video concat.
+     */
+    audio?: Uint8Array;
+}
+/**
+ * Mux a chunked export's final artifact: stream-copy the video parts with
+ * plan-derived timestamp offsets, and stream-copy the pre-encoded audio
+ * track when one is provided. `packetCount` counts VIDEO packets only —
+ * that is the frame-parity contract callers assert against the plan.
+ */
+declare function muxEncodedExport(options: MuxExportOptions): Promise<ConcatResult>;
+/**
+ * Concatenate independently encoded chunk videos into one stream-copied file.
+ * Chunks must share codec and encoder params; the first packet of every
+ * chunk must be a keyframe (violations throw — better a loud failure at
+ * finalize than a corrupt artifact). Thin wrapper over muxEncodedExport,
+ * kept as the CLI's finalize entry point.
+ */
+declare function concatEncodedVideo(chunks: ConcatChunk[], options: ConcatOptions): Promise<ConcatResult>;
+/**
+ * Count video packets in an encoded file — cheap integrity check used by
+ * parity verification (chunked and single-flight renders of the same
+ * composition must contain the same number of frames).
+ */
+declare function countVideoPackets(data: Uint8Array): Promise<number>;
+
+/**
+ * Finalize page — an HTML page that stream-copy concatenates encoded chunk
+ * videos IN A BROWSER and PUTs the result to an upload URL.
+ *
+ * Why a page and not the Worker: concat needs every chunk in memory plus the
+ * output buffer, which busts a 128MB isolate on real exports; a browser page
+ * (the same Browser Run substrate that rendered the chunks) has gigabytes,
+ * mediabunny already proven in it, and gives the later audio-mix step a home
+ * (OfflineAudioContext exists only in browsers).
+ *
+ * The in-page algorithm MIRRORS ./concat.ts (concatEncodedVideo) — same
+ * packet walk, same plan-derived timestamp offsets, same keyframe guard.
+ * KEEP THEM IN SYNC; scripts/verify-finalize-page.ts (vos-plugin) asserts the
+ * two implementations produce packet-identical output.
+ *
+ * Failure contract: any error (fetch, codec mismatch, upload) lands in
+ * `__renderComplete = { success: false, error }` — there is no base64
+ * fallback here (finals can be huge); the caller fails the job.
+ */
+interface FinalizePart {
+    /** Where the page fetches this chunk's encoded bytes (CORS-accessible). */
+    url: string;
+    /** The chunk's PLANNED duration in seconds (frameCount / fps) — the
+     * timestamp offset source, never demuxed durations (rounding drifts). */
+    duration: number;
+}
+interface FinalizeConcatPageOptions {
+    parts: FinalizePart[];
+    format: 'webm' | 'mp4';
+    /** Stamped into the output track metadata. */
+    frameRate: number;
+    /** The finished file is PUT here (Content-Type set from `format`). */
+    uploadUrl: string;
+    /**
+     * Audio-once-at-finalize (the rendering plan's audio model — chunks are
+     * video-only by construction, so encoder priming seams never exist):
+     * `producerCode` defines `window.__vosAudioProducer__({ data, duration,
+     * sampleRate }) => AudioBuffer | null` (the studio clips' plan rides the
+     * code itself, baked by `audioProducerCode({ plan })`, since this
+     * page passes none), evaluated before the concat; a
+     * returned buffer is muxed as the output's audio track (AAC for mp4 with
+     * an Opus fallback, Opus for webm). Null/absent = video-only.
+     */
+    audio?: {
+        producerCode: string;
+        data: unknown;
+        /** Total OUTPUT duration in seconds (the mix length). */
+        duration: number;
+    };
+    /**
+     * Pre-encoded audio: the `audio` ingest part produced by the audio
+     * mix page. Its packets are STREAM-COPIED into the output — no decode, no
+     * OfflineAudioContext, no producer — so a fallback finalize on this page
+     * carries none of the audio-production memory bill. Mutually exclusive
+     * with `audio`.
+     */
+    audioPart?: {
+        url: string;
+    };
+}
+declare function buildFinalizeConcatPage(options: FinalizeConcatPageOptions): string;
+
+/**
+ * The take audio producer — page JavaScript implementing the export audio mix
+ * for the take data schema, as a source string for injection into render
+ * pages (the engine's `capture.audioProducerCode` seam for single-flight
+ * captures, the audio mix page, and the finalize page for chunked exports).
+ *
+ * MIRRORS the client exporter's audio path (the platform's
+ * decodeAudio → spliceAudio → mixExportAudio) — the same
+ * algorithm, so a cloud export sounds identical to a client export of the
+ * same composition. KEEP THEM IN SYNC when the client mixer changes.
+ *
+ * The producer is `window.__vosAudioProducer__({ data, plan, duration,
+ * sampleRate })`. Inputs read from `data` (all optional):
+ *   videoSrc   recording URL (requires hasAudio; the VOICE on legacy takes,
+ *              the SYSTEM track on split takes)
+ *   micSrc     mic sidecar URL (the mic/system split) — the voice; when present the
+ *              recording's own track mixes as system audio under sysGain
+ *   hasAudio   whether the recording carries audio
+ *   segments   [{ in, out, rate? }] SOURCE spans — mic is spliced/resampled
+ *              so trims and speed changes stay lip-synced (pitch shifts with
+ *              rate, matching preservesPitch=false preview playback)
+ *   micGain    voice master fader (default 1)
+ *   sysGain    system-audio master fader (default 1; split takes only)
+ *
+ * The music/SFX clips no longer ride `data`: they live on the
+ * studio stack entry (`vosso.studio`), and the host builds an ENGINE audio
+ * plan from them ahead of the page (studio-core's `studioAudioPlan`), the
+ * shape `@vosjs/core/audio`'s `mixAudio` renders. The page imports that
+ * mixer from the CDN and renders the plan to ONE buffer beside the
+ * recording's tracks. `plan` reaches the producer either as a call argument
+ * (a page's CONFIG) or baked into the code as `window.__vosAudioPlan__`
+ * (the engine's capture template calls the producer with `{ data, duration,
+ * sampleRate }` only, so single-flight captures bake it); no plan, no clips.
+ */
+/**
+ * The `@vosjs/core/audio` build a render page imports for `mixAudio`.
+ * Pinned to the version the api installs (`coreAudioCdn.test.ts` there
+ * holds it to the installed package): render-core has no engine dependency.
+ */
+declare const CORE_AUDIO_CDN_URL = "https://esm.sh/@vosjs/core@0.23.1/audio?target=es2022";
+/**
+ * A sampled audio plan, structurally `@vosjs/core/audio`'s `AudioPlan` (and
+ * studio-core's `StudioAudioPlan`), typed here so render-core depends on
+ * neither.
+ */
+interface AudioPlanJson {
+    duration: number;
+    step: number;
+    tracks: {
+        id: string;
+        src: string;
+        loop: boolean;
+        points: {
+            t: number;
+            on: boolean;
+            pos: number;
+            gain: number;
+        }[];
+    }[];
+}
+interface AudioProducerCodeOptions {
+    /**
+     * Bake the plan into the code (`window.__vosAudioPlan__`) for callers
+     * that cannot pass one at call time (the engine's capture template).
+     */
+    plan?: AudioPlanJson | null;
+    /** Override the mixer import (tests, a self-hosted engine). */
+    coreAudioUrl?: string;
+}
+/**
+ * The studio entry's own data out of a stack in either shape: the STORED
+ * config's `stack` array (`[{ id, data, … }]`) or the lowered record keyed
+ * by entry id (`{ 'vosso.studio': data }`). Null when there is none.
+ */
+declare function studioEntryData(stack: unknown): Record<string, unknown> | null;
+declare function audioProducerCode(options?: AudioProducerCodeOptions): string;
+/**
+ * Does this composition carry anything the audio producer could mix? The
+ * voice reads off `data`; the clips off the studio stack entry (`stack` in
+ * either shape `studioEntryData` reads) or an already-built plan.
+ */
+declare function dataHasAudio(data: unknown, stack?: unknown, plan?: AudioPlanJson | null): boolean;
+
+/**
+ * Audio mix page — an HTML page that renders a chunked export's audio track
+ * ALONE and PUTs it to the ingest route as the `audio` part.
+ *
+ * Why a page: the mix needs OfflineAudioContext + WebCodecs AudioEncoder,
+ * which exist only in browsers. Why ALONE: the finalize page used to produce
+ * audio AND concat video in one context, and the combined memory bill
+ * (whole source recording + full-length PCM + all parts + output buffer)
+ * is what the fleet kills at peak (job 288257ae). This page's
+ * live set stays ~tens of MB by construction:
+ *
+ *   - the source recording is never fetched whole: `__vosStreamSplice__`
+ *     (installed here, consumed by the shared audio producer) streams-decodes
+ *     ONLY the needed source spans through mediabunny's UrlSource, whose
+ *     read cache is bounded (~8MiB) — the asset route serves Range/206.
+ *   - the spliced voice buffer is capped at the REQUESTED output duration,
+ *     which is also the duration fix (a duration-capped render must not carry the
+ *     full take's audio).
+ *   - the encoded result is ~1MB of Opus in WebM.
+ *
+ * The whole-file decodeAudioData path stays as the in-page fallback rung
+ * (unsupported codec, no Range support): the producer falls back on any
+ * stream-splice failure, so a mix is never LOST to the optimization.
+ *
+ * Runs CONCURRENTLY with the chunk phase — it depends only on the source
+ * recording and the doc, never on a part.
+ *
+ * Failure contract: mirrors the finalize page — any error lands in
+ * `__renderComplete = { success: false, error }`; stage markers ride
+ * `__finalizeStage` (heap-stamped) so a death localizes itself.
+ */
+
+interface AudioMixPageOptions {
+    /** The vos's resolved ctx.data (asset URLs absolute, render token baked). */
+    data: unknown;
+    /**
+     * The studio clips as an engine audio plan, built by the host from
+     * the stored config's `vosso.studio` stack entry; null when the take has
+     * no music/SFX. Rides the page's CONFIG, once.
+     */
+    plan?: AudioPlanJson | null;
+    /** Requested OUTPUT duration in seconds — the mix length, the trim bound. */
+    duration: number;
+    /** The finished audio file is PUT here (ingest `?part=audio`). */
+    uploadUrl: string;
+}
+declare function buildAudioMixPage(options: AudioMixPageOptions): string;
+
+/**
+ * Image diff page — computes the RMS difference between two images in a
+ * browser page. Workers can't decode images (no canvas/Image APIs), so the
+ * golden-frame canary hands both images to a page and reads one number back.
+ *
+ * Same completion contract as every render page: `__renderComplete` with
+ * `{ success, rms, width, height }` or `{ success: false, error }`.
+ * RMS is over RGB (alpha ignored) in 0–255 units: WebP re-encode jitter of
+ * the same frame lands well under 5; a real engine/fleet drift (missing
+ * layer, changed shader, font fallback) lands far above 10.
+ */
+interface ImageDiffPageOptions {
+    /** data: URLs — the canary inlines both images (they are ~tens of KB). */
+    candidateUrl: string;
+    goldenUrl: string;
+}
+declare function buildImageDiffPage(options: ImageDiffPageOptions): string;
+
+/**
+ * Digest page — the fleet's eyes for a take. One bare page decodes the
+ * recording through a <video> and does what the CLI's digest page does
+ * locally: per-source-second motion bins (64×36 luma diff, changed-pixel
+ * fraction), the scene instants those bins reveal, one FOOTAGE frame plus a
+ * crop per moment, and a contact sheet. Every image PUTs itself to the ingest
+ * route as a `digest-*` part (the page's bytes never marshal through the
+ * browser session), then a manifest with the bins and every image's size.
+ *
+ * Same completion contract as every render page: `__renderComplete` with
+ * `{ success, uploaded: true }` or `{ success: false, error }`.
+ *
+ * The bins must be byte-comparable with the CLI's (`MOTION_DELTA` 24, one
+ * seek per source second at i + 0.5, 64×36): they feed the speed planner.
+ */
+interface DigestShot {
+    /** Part stem; the page writes `<name>.full.png` and, with a box, `<name>.crop.png`. */
+    name: string;
+    /** Source instant, seconds. */
+    t: number;
+    /** Crop box in FRAME px, or null for a full-only moment. */
+    box: {
+        x: number;
+        y: number;
+        w: number;
+        h: number;
+    } | null;
+    /** A short label for the contact sheet. */
+    label: string;
+}
+interface DigestPageOptions {
+    /** The recording, reachable from the page (a `?rt=` asset URL). */
+    videoUrl: string;
+    /** Source duration, seconds. */
+    durationS: number;
+    /** The region of the frame the doc renders (a window take's crop, or all). */
+    region: {
+        x: number;
+        y: number;
+        w: number;
+        h: number;
+    };
+    shots: DigestShot[];
+    /** Long edges (px) of the emitted images. */
+    fullMax: number;
+    cropMax: number;
+    /** Changed-pixel luma threshold for the bins. */
+    motionDelta: number;
+    /** Scene rule: a bin at/above `motion` after one at/below `quiet`. */
+    scene: {
+        motion: number;
+        quiet: number;
+    };
+    /** `…/api/render/ingest/{jobId}?token=…` — parts append `&part=digest-<name>`. */
+    uploadUrl: string;
+}
+declare function buildDigestPage(options: DigestPageOptions): string;
+
+export { type AudioMixPageOptions, type AudioPlanJson, type AudioProducerCodeOptions, CORE_AUDIO_CDN_URL, type ChunkPlanPolicy, type ConcatChunk, type ConcatOptions, type ConcatResult, DEFAULT_MIN_FRAMES_PER_CHUNK, type DigestPageOptions, type DigestShot, type FinalizeConcatPageOptions, type FinalizePart, type ImageDiffPageOptions, type MuxExportOptions, type RenderChunk, audioProducerCode, buildAudioMixPage, buildDigestPage, buildFinalizeConcatPage, buildImageDiffPage, concatEncodedVideo, countVideoPackets, dataHasAudio, muxEncodedExport, planChunks, studioEntryData };