@sailfish-ai/recorder 1.12.5 → 1.12.7

package/dist/canvas/mediaRecorderCapture.js

@@ -0,0 +1,257 @@
+/**
+ * A41 — MediaRecorder fallback video capture (SDK half), for browsers that have
+ * NO WebCodecs (`VideoEncoder` / `VideoFrame`) but DO have `MediaRecorder` +
+ * `HTMLCanvasElement.captureStream` (notably Safari < 26). Without this, such a
+ * canvas would degrade to keyframe-only; here we capture continuous video via
+ * MediaRecorder instead.
+ *
+ * Wire contract (the player agent implements the exact mirror):
+ *   - ONE `video-init` SFCV frame, header
+ *       { kind:"video-init", codec:"mediarecorder", mimeType:<recorder.mimeType>, w, h }
+ *     (no binary payload).
+ *   - `video-chunk` SFCV frames, header { kind:"video-chunk", codec:"mediarecorder", seq }
+ *     each carrying ONE `MediaRecorder.ondataavailable` Blob's bytes as the
+ *     payload. The FIRST webm chunk holds the init segment, so ORDER MUST be
+ *     preserved (seq is monotonic, assigned in dataavailable order).
+ *
+ * Differences from VideoCapture (WebCodecs):
+ *   - No same-task readback hazard: MediaRecorder pulls frames off the
+ *     captureStream track itself, so there is no afterRender pixel grab and no
+ *     guardrail-#6 concern. The manager still calls afterRender for dirtiness
+ *     bookkeeping, but it is a no-op here.
+ *   - Chunk bytes are read from the Blob asynchronously (blob.arrayBuffer());
+ *     emission is chained so seq order is preserved even if reads resolve out
+ *     of order.
+ *
+ * SSR rule: nothing here touches window/document at module scope; capability
+ * detection and all DOM/Media access happen inside methods called from the
+ * manager (which itself only runs in the browser).
+ */
+const TIMESLICE_MS = 1_000; // ~1s cadence so chunks flow steadily
+const CAPTURE_FPS = 30;
+/** webm mime candidates, preferred first. Safari < 26 generally supports
+ * `video/mp4` via MediaRecorder; we still probe webm first for player parity
+ * (the player decodes whatever mimeType we report). */
+const MIME_CANDIDATES = [
+    "video/webm;codecs=vp9",
+    "video/webm;codecs=vp8",
+    "video/webm",
+    "video/mp4",
+];
+/**
+ * Feature detection (contract): MediaRecorder fallback is usable ONLY when
+ * `MediaRecorder`, `HTMLCanvasElement.prototype.captureStream`, and
+ * `MediaRecorder.isTypeSupported` for at least one candidate mime are present.
+ * Callers prefer WebCodecs and reach here only when WebCodecs is ABSENT.
+ */
+export function mediaRecorderSupported() {
+    if (typeof MediaRecorder === "undefined")
+        return false;
+    if (typeof HTMLCanvasElement === "undefined")
+        return false;
+    if (typeof HTMLCanvasElement.prototype.captureStream !== "function")
+        return false;
+    if (typeof MediaRecorder.isTypeSupported !== "function")
+        return false;
+    return pickMime() !== null;
+}
+/** First candidate mime MediaRecorder claims to support, or null. */
+export function pickMime() {
+    if (typeof MediaRecorder === "undefined" || typeof MediaRecorder.isTypeSupported !== "function") {
+        return null;
+    }
+    for (const m of MIME_CANDIDATES) {
+        try {
+            if (MediaRecorder.isTypeSupported(m))
+                return m;
+        }
+        catch { /* probe next */ }
+    }
+    return null;
+}
+export class MediaRecorderCapture {
+    canvas;
+    out;
+    recorder = null;
+    stream = null;
+    stopped = false;
+    started = false;
+    seq = 0;
+    /** chain Blob -> bytes emission so seq order is preserved across async reads */
+    emitChain = Promise.resolve();
+    constructor(canvas, out) {
+        this.canvas = canvas;
+        this.out = out;
+    }
+    static supported() {
+        return mediaRecorderSupported();
+    }
+    /** Resolves true once the recorder started; false -> keyframes-only fallback. */
+    async start() {
+        if (this.stopped)
+            return false;
+        const w = this.canvas.width;
+        const h = this.canvas.height;
+        if (!w || !h) {
+            this.out.note("zero-size canvas — MediaRecorder capture unavailable");
+            return false;
+        }
+        const mime = pickMime();
+        if (!mime) {
+            this.out.note("no MediaRecorder-supported mime — keyframes only");
+            return false;
+        }
+        let stream;
+        try {
+            stream = this.canvas.captureStream(CAPTURE_FPS);
+        }
+        catch (e) {
+            this.out.note(`captureStream failed: ${e.message}`);
+            return false;
+        }
+        let recorder;
+        try {
+            recorder = new MediaRecorder(stream, { mimeType: mime });
+        }
+        catch (e) {
+            this.out.note(`MediaRecorder construct failed: ${e.message}`);
+            try {
+                stopStream(stream);
+            }
+            catch { /* best effort */ }
+            return false;
+        }
+        this.stream = stream;
+        this.recorder = recorder;
+        recorder.ondataavailable = (e) => this.onData(e.data);
+        recorder.onerror = (e) => {
+            this.out.note(`MediaRecorder error: ${String(e.error ?? e)}`);
+        };
+        try {
+            // timeslice => periodic dataavailable (~1s) so chunks stream live.
+            recorder.start(TIMESLICE_MS);
+        }
+        catch (e) {
+            this.out.note(`MediaRecorder start failed: ${e.message}`);
+            this.teardownStream();
+            return false;
+        }
+        this.started = true;
+        // The mimeType the recorder actually negotiated may differ from the
+        // requested one (e.g. codecs appended); report the recorder's own value so
+        // the player decodes against the truth.
+        this.out.onInit(recorder.mimeType || mime, w, h);
+        return true;
+    }
+    /** A non-empty Blob from dataavailable. Read bytes async, emit in seq order. */
+    onData(blob) {
+        if (!blob || blob.size === 0)
+            return;
+        const seq = this.seq++;
+        this.emitChain = this.emitChain.then(async () => {
+            try {
+                const buf = await blob.arrayBuffer();
+                if (buf.byteLength > 0)
+                    this.out.onChunk(seq, buf);
+            }
+            catch (e) {
+                this.out.note(`chunk read failed: ${e.message}`);
+            }
+        });
+    }
+    /**
+     * The manager's microtask pump calls this after every render for dirtiness
+     * bookkeeping. MediaRecorder pulls frames off the captureStream track on its
+     * own, so this is intentionally a no-op (kept for VideoCapture parity). The
+     * `_t` mirrors VideoCapture.afterRender's signature.
+     */
+    afterRender(_t) { }
+    /** Interaction boost is a no-op: the captureStream fps is fixed and the
+     * recorder cannot be told to key on demand. Kept for VideoCapture parity. */
+    onInteraction(_tMs) { }
+    /**
+     * Non-terminal drain (visibilitychange:hidden): force the recorder to emit a
+     * chunk for everything buffered so far WITHOUT stopping, so capture resumes if
+     * the tab returns. Mirrors VideoCapture.flush(). No-op once stopped.
+     */
+    async flush() {
+        if (this.stopped || !this.recorder)
+            return;
+        try {
+            if (this.recorder.state === "recording")
+                this.recorder.requestData();
+        }
+        catch { /* best effort */ }
+        // let the synchronous-ish dataavailable enqueue, then settle the chain
+        await Promise.resolve();
+        await this.emitChain.catch(() => { });
+    }
+    /**
+     * Terminal stop: flush the final chunk (requestData then stop fire a last
+     * dataavailable), settle the emission chain so the final bytes reach onChunk,
+     * then release the recorder + stream tracks. Idempotent.
+     */
+    async stop() {
+        if (this.stopped)
+            return;
+        this.stopped = true;
+        const recorder = this.recorder;
+        if (recorder && this.started) {
+            const lastData = new Promise((resolve) => {
+                const done = () => resolve();
+                recorder.addEventListener("stop", done, { once: true });
+                // safety: never hang the manager's stop on a wedged recorder
+                setTimeout(done, 3_000);
+            });
+            try {
+                if (recorder.state !== "inactive") {
+                    recorder.requestData(); // flush the tail timeslice
+                    recorder.stop(); // fires a final dataavailable then 'stop'
+                }
+            }
+            catch { /* already inactive */ }
+            await lastData;
+        }
+        // settle any in-flight blob reads so their chunks are emitted before teardown
+        await this.emitChain.catch(() => { });
+        this.teardownStream();
+    }
+    /**
+     * Immediate teardown WITHOUT flushing (A22/A6/A10 parity): the canvas is gone
+     * or must not be recorded, so drop the recorder and stop the stream tracks
+     * without emitting the tail. Idempotent.
+     */
+    abort() {
+        if (this.stopped)
+            return;
+        this.stopped = true;
+        try {
+            if (this.recorder && this.recorder.state !== "inactive")
+                this.recorder.stop();
+        }
+        catch { /* best effort */ }
+        this.teardownStream();
+    }
+    teardownStream() {
+        if (this.recorder) {
+            this.recorder.ondataavailable = null;
+            this.recorder.onerror = null;
+            this.recorder = null;
+        }
+        if (this.stream) {
+            stopStream(this.stream);
+            this.stream = null;
+        }
+    }
+}
+function stopStream(stream) {
+    try {
+        for (const track of stream.getTracks()) {
+            try {
+                track.stop();
+            }
+            catch { /* best effort */ }
+        }
+    }
+    catch { /* getTracks unavailable */ }
+}

package/dist/canvas/objectLog.js

@@ -0,0 +1,150 @@
+/**
+ * Cross-context canvas object logging — the global creation log for
+ * CanvasGradient / CanvasPattern / Path2D objects.
+ *
+ * Gradients, patterns, and Path2Ds are OPAQUE: their stops/segments cannot be
+ * read back after construction, so they can only be captured by observing the
+ * factory call live. They may also be created on one context (or no context —
+ * Konva builds gradients on a hidden dummy context) and used on another, so
+ * creation is logged GLOBALLY here, and each recorder lazily emits its own
+ * copy into whichever command stream first references the object
+ * (patch2d.ts serializeFactoryObject).
+ *
+ * ROOT-CAUSE GUARDRAIL (Phase 0.5 E2E, found on /2d-objects): app scene code
+ * typically constructs these objects ONCE at startup — often before the SDK's
+ * settings fetch + session start complete and startCanvasRecording() installs
+ * the full capture hooks. Anything created in that gap is permanently
+ * unserializable ({$:"unsupported"}). This module is therefore dependency-free
+ * and cheap so index.tsx can install it SYNCHRONOUSLY at startRecording()
+ * entry (Phase 1 critical interceptors) whenever canvas recording is opted
+ * in — long before any network roundtrip.
+ *
+ * SSR rule (veritas/jsts-frontend/CLAUDE.md): no window/document/navigator or
+ * canvas globals at module scope; all installation happens inside the
+ * exported install functions, each guarded with typeof checks.
+ */
+/** factory-created object -> its construction + method-call history */
+export const objectLogs = new WeakMap();
+/**
+ * A26 — per-object recorded-calls cap. Gradients/patterns/Path2Ds are normally
+ * built with a handful of stops/segments at construction; a few apps, though,
+ * reuse one long-lived object and keep appending to it for the whole session
+ * (e.g. a Path2D that accumulates points), growing `calls[]` without bound and
+ * leaking memory. We cap the recorded history: the first MAX_OBJECT_CALLS calls
+ * are kept (covering every realistic construction), and anything beyond is
+ * dropped with a one-time marker. Replay of the common case is unaffected.
+ */
+export const MAX_OBJECT_CALLS = 4_096;
+/** Append a method call to an object's log, honoring the per-object cap. */
+export function pushObjectCall(log, method, args) {
+    if (!log)
+        return;
+    if (log.calls.length >= MAX_OBJECT_CALLS) {
+        log.callsCapped = true;
+        return;
+    }
+    log.calls.push({ method, args });
+}
+export const OBJECT_FACTORIES = [
+    "createLinearGradient",
+    "createRadialGradient",
+    "createConicGradient",
+    "createPattern",
+];
+export const PATH_OPS = [
+    "addPath", "arc", "arcTo", "bezierCurveTo", "closePath", "ellipse",
+    "lineTo", "moveTo", "quadraticCurveTo", "rect", "roundRect",
+];
+let nativePath2D = null;
+/** The unwrapped Path2D constructor (set once Path2D logging installs). */
+export function getNativePath2D() {
+    return nativePath2D;
+}
+/** prototypes whose factory methods are already wrapped */
+const patchedFactoryProtos = new WeakSet();
+let sharedLoggingInstalled = false;
+/**
+ * Wrap the gradient/pattern factory methods of one 2D-context prototype
+ * (CanvasRenderingContext2D on the main thread, OffscreenCanvasRendering-
+ * Context2D inside workers) so every created object lands in `objectLogs`.
+ * Idempotent per prototype.
+ */
+export function installFactoryLoggingOnProto(protoObj) {
+    if (patchedFactoryProtos.has(protoObj))
+        return;
+    patchedFactoryProtos.add(protoObj);
+    const proto = protoObj;
+    for (const name of OBJECT_FACTORIES) {
+        const desc = Object.getOwnPropertyDescriptor(protoObj, name);
+        if (!desc || typeof desc.value !== "function")
+            continue;
+        const orig = desc.value;
+        proto[name] = function (...args) {
+            const result = orig.apply(this, args);
+            // log unconditionally — the object may be used on a DIFFERENT
+            // context than the one that created it (Konva's dummy context).
+            if (result && typeof result === "object") {
+                objectLogs.set(result, { factory: name, args, calls: [] });
+            }
+            return result;
+        };
+    }
+}
+/**
+ * Wrap the realm-global object types: CanvasGradient.addColorStop appends to
+ * the creating factory's log; Path2D construction + segment ops are logged
+ * per instance (replayed via __create '__path2d' + __objcall when a path is
+ * first used on a recorded context). Idempotent.
+ */
+export function installSharedObjectLogging() {
+    if (sharedLoggingInstalled)
+        return;
+    sharedLoggingInstalled = true;
+    // gradient stops: append to the object's global log (cross-context safe)
+    if (typeof CanvasGradient !== "undefined") {
+        const orig = CanvasGradient.prototype.addColorStop;
+        CanvasGradient.prototype.addColorStop = function (...args) {
+            pushObjectCall(objectLogs.get(this), "addColorStop", args);
+            return orig.apply(this, args);
+        };
+    }
+    if (typeof Path2D !== "undefined" && !nativePath2D) {
+        nativePath2D = Path2D;
+        for (const m of PATH_OPS) {
+            const orig = Path2D.prototype[m];
+            if (typeof orig !== "function")
+                continue;
+            Path2D.prototype[m] = function (...args) {
+                pushObjectCall(objectLogs.get(this), m, args);
+                return orig.apply(this, args);
+            };
+        }
+        const native = nativePath2D;
+        const Wrapped = function (arg) {
+            const instance = arg === undefined ? new native() : new native(arg);
+            objectLogs.set(instance, {
+                factory: "__path2d",
+                args: [typeof arg === "string" ? arg : null],
+                calls: arg instanceof native ? [...(objectLogs.get(arg)?.calls ?? [])] : [],
+            });
+            return instance;
+        };
+        Wrapped.prototype = native.prototype;
+        globalThis.Path2D = Wrapped;
+    }
+}
+/**
+ * Early hook for the SDK entry path: arm all object logging on the main
+ * thread BEFORE app scene code constructs its gradients/patterns/paths.
+ * Safe to call multiple times; no-ops outside a browser realm.
+ */
+export function installEarlyObjectLogging() {
+    if (typeof CanvasRenderingContext2D !== "undefined") {
+        installFactoryLoggingOnProto(CanvasRenderingContext2D.prototype);
+    }
+    if (typeof OffscreenCanvasRenderingContext2D !== "undefined") {
+        // main-thread OffscreenCanvas 2D contexts share the same object types
+        installFactoryLoggingOnProto(OffscreenCanvasRenderingContext2D.prototype);
+    }
+    installSharedObjectLogging();
+}