@sailfish-ai/recorder 1.12.5 → 1.12.6

package/dist/canvas/patchGL.js

@@ -0,0 +1,371 @@
+/**
+ * WebGL1/2 command capture — port of the validated bench core
+ * (test-applications/canvas-recorder-bench/src/capture/patchGL.ts;
+ * evanw/webgl-recorder lineage, binary-encoded). Creator calls intern their
+ * results via the same __create meta-op the 2D path uses, so the shared codec
+ * + replayer machinery handles GL streams unchanged.
+ *
+ * Replay fidelity is "deterministic on the same GPU/driver" — pixel keyframes
+ * remain the bit-exact truth, exactly like the 2D path.
+ *
+ * SDK adaptations: SSR-safe installation (no top-level globals), untracked-
+ * context notifier for contexts created before install, DiscardSink
+ * short-circuit, attachGLRecorderToContext for mid-stream attach.
+ */
+import { DIRTY_OP, DiscardSink } from "./protocol";
+import { BufferSink, serializeTypedArray } from "./protocol";
+import { deflate } from "./codec";
+import { ignore2DContext, isTaintError } from "./patch2d";
+/** methods whose return value is an interned object */
+const CREATOR_RE = /^create[A-Z]/;
+const OTHER_CREATORS = new Set(["getUniformLocation", "getExtension"]);
+/** pure readbacks — never recorded, never replayed */
+const PURE_RE = /^(get(?!Extension$|UniformLocation$)|is[A-Z]|check|read|finish$|flush$)/;
+let installed = false;
+const glRecorders = new WeakMap();
+// ── GL texture-source capture ───────────────────────────────────────────────
+// GL texture uploads (texImage2D/texSubImage2D) from a STATIC image source
+// (HTMLImageElement / ImageBitmap / ImageData) are captured as imagedata assets
+// so textured content REPLAYS (and becomes click-to-code pickable) instead of
+// rendering blank. We deliberately capture only STATIC sources (dedup'd by
+// identity → captured once) and bound the count, so a dynamic video/canvas
+// texture uploaded per-frame can't blow up bandwidth (those stay uncaptured —
+// the prior behavior). Rasterization needs a DOM canvas, so element sources are
+// skipped under SSR/worker (ImageData, which has raw pixels, still captures).
+/** Max captured texture assets per recorder — bounds bandwidth for an app that
+ * uploads many distinct (e.g. per-frame ImageData) textures. Past this, capture
+ * degrades to "unsupported" (blank on replay), never unbounded. */
+const MAX_GL_TEX_ASSETS = 64;
+/** [audit B5] Max RAW (uncompressed RGBA) bytes of a single capturable texture.
+ * The captured asset is deflated and shipped as ONE SFCV frame, which the
+ * transport drops if it exceeds the 8 MiB per-frame cap (canvasManager
+ * MAX_FRAME_BYTES). Pixel data is often incompressible (photos/noise), so the
+ * deflated size can approach the raw size — gate on raw bytes with headroom for
+ * the deflate stream + frame envelope so a too-large texture is reported as
+ * `texture-too-large` UP FRONT (the player shows an "8 MB limit" badge) instead
+ * of the op referencing an asset that is then silently dropped at send → a blank
+ * texture with no explanation. 7 MiB raw ≈ a 1290² RGBA texture. */
+const MAX_GL_TEX_RAW_BYTES = 7 * 1024 * 1024;
+/** Raw RGBA byte size of a static texture source from its dimensions (no
+ * rasterization — cheap enough to gate before capture). */
+function texSourceRawBytes(src) {
+    if (typeof ImageData !== "undefined" && src instanceof ImageData) {
+        return src.width * src.height * 4;
+    }
+    const s = src;
+    const w = Number(s.naturalWidth || s.width) || 0;
+    const h = Number(s.naturalHeight || s.height) || 0;
+    return w * h * 4;
+}
+let _texScratch = null;
+function texScratchCtx(w, h) {
+    if (typeof document === "undefined")
+        return null; // worker / SSR
+    if (!_texScratch) {
+        _texScratch = document.createElement("canvas");
+        const c0 = _texScratch.getContext("2d");
+        if (c0)
+            ignore2DContext(c0); // our own scratch — never record it
+    }
+    if (_texScratch.width !== w || _texScratch.height !== h) {
+        _texScratch.width = w;
+        _texScratch.height = h;
+    }
+    const ctx = _texScratch.getContext("2d");
+    if (ctx)
+        ignore2DContext(ctx);
+    return ctx;
+}
+/** A static, capturable texture source. Dynamic sources (video / canvas /
+ * OffscreenCanvas) are intentionally excluded (they change per-frame). */
+function isStaticTexSource(o) {
+    return ((typeof ImageData !== "undefined" && o instanceof ImageData) ||
+        (typeof HTMLImageElement !== "undefined" && o instanceof HTMLImageElement) ||
+        (typeof ImageBitmap !== "undefined" && o instanceof ImageBitmap));
+}
+/** Rasterize an element source to ImageData (ImageData passes through). Returns
+ * null when it can't be read for a NON-error reason (no DOM, zero-size). A tainted
+ * / cross-origin source whose `getImageData` THROWS is propagated to the caller
+ * ([audit B13] so it can emit a taint signal), not swallowed. */
+function texSourceToImageData(src) {
+    if (typeof ImageData !== "undefined" && src instanceof ImageData)
+        return src;
+    const s = src;
+    const w = Number(s.naturalWidth || s.width) || 0;
+    const h = Number(s.naturalHeight || s.height) || 0;
+    if (!w || !h)
+        return null;
+    const ctx = texScratchCtx(w, h);
+    if (!ctx)
+        return null;
+    ctx.clearRect(0, 0, w, h);
+    ctx.drawImage(src, 0, 0);
+    return ctx.getImageData(0, 0, w, h); // throws on a tainted source → caller handles
+}
+/**
+ * Draw-site stamp routing (Phase 3b): when stamped app code calls
+ * `__sfds(glctx, id).drawArrays(...)`, canvasManager's __sfDrawSite calls this
+ * with the GL context; if it has a recorder, the id rides the next op.
+ * Returns true when handled (ctx is a recorded GL context).
+ */
+export function noteGLPendingSiteId(ctx, id) {
+    const rec = glRecorders.get(ctx);
+    if (!rec)
+        return false;
+    rec.pendingSiteId = id;
+    return true;
+}
+let untrackedNotify = null;
+export function setUntrackedGLContextNotifier(fn) {
+    untrackedNotify = fn;
+}
+let glAutoConfig = null;
+const glAutoRecorders = new Map();
+export function setGLAutoAttach(config) {
+    glAutoConfig = config;
+    if (config)
+        glAutoRecorders.clear();
+}
+export function pruneUnboundGLRecorders() {
+    for (const [canvas, rec] of glAutoRecorders) {
+        if (!rec.bound) {
+            glRecorders.delete(rec.rawCtx);
+            glAutoRecorders.delete(canvas);
+        }
+    }
+}
+export class GLRecorder {
+    now;
+    cpuMs = 0;
+    bound = false;
+    rawCtx = null;
+    /** ids for GL objects, scoped to this recorder's stream */
+    refs = new WeakMap();
+    nextId = 1_000_000; // distinct range from 2D object ids (cosmetic)
+    sink;
+    // GL texture-source capture (asset ids in a distinct 2M+ range; dedup static
+    // sources by identity; bounded count).
+    // [audit B6] In-flight texture-encode promises — a Set with auto-removal so it
+    // stays bounded to actually-pending work (see ContextRecorder.trackEncode).
+    pendingEncodes = new Set();
+    assetBySource = new WeakMap();
+    nextAssetId = 2_000_000;
+    assetCount = 0;
+    constructor(sink, now) {
+        this.now = now;
+        this.sink = sink;
+    }
+    /** Await pending texture-asset encodes so they ship before the stream stops
+     * (mirrors ContextRecorder.settle). The manager awaits this at teardown. */
+    async settle() {
+        // [audit B12] Loop-drain: if an encode is registered while we await (it
+        // shouldn't, now that teardown detaches first — but defensively), await it too
+        // so no asset is left un-shipped.
+        while (this.pendingEncodes.size)
+            await Promise.all([...this.pendingEncodes]);
+    }
+    /** [audit B6] Register an in-flight encode + auto-remove on completion so the
+     * set never grows unbounded. (then/finally swallow avoids unhandled rejection.) */
+    trackEncode(p) {
+        this.pendingEncodes.add(p);
+        p.then(() => { }, () => { }).finally(() => this.pendingEncodes.delete(p));
+    }
+    /** Capture a static GL texture source as an imagedata asset (dedup'd by source
+     * identity, bounded). Returns the asset ref to serialize; an unreadable /
+     * tainted / over-cap source returns `unsupported` (uncaptured → blank replay,
+     * the prior behavior). */
+    captureTexSource(src, t) {
+        const existing = this.assetBySource.get(src);
+        if (existing !== undefined)
+            return { $: "imagedata", id: existing };
+        if (this.assetCount >= MAX_GL_TEX_ASSETS)
+            return { $: "unsupported", kind: "texture-cap" };
+        // [audit B5] Reject before rasterizing: a texture whose deflated asset would
+        // exceed the 8 MiB transport frame cap is reported as `texture-too-large` so
+        // the player shows a badge, rather than emitting an asset ref that gets
+        // silently dropped at send → an unexplained blank texture. (Not added to the
+        // dedup map / asset count, so re-uploads of the same source report the same.)
+        if (texSourceRawBytes(src) > MAX_GL_TEX_RAW_BYTES) {
+            return { $: "unsupported", kind: "texture-too-large" };
+        }
+        let img;
+        try {
+            img = texSourceToImageData(src);
+        }
+        catch (e) {
+            // [audit B13] A cross-origin/tainted texture source throws on readback. Emit
+            // a taint signal so the player paints its A40 taint placeholder/badge,
+            // instead of silently replaying a blank textured draw with no explanation
+            // (the 2D image path already signals taint this way).
+            if (isTaintError(e)) {
+                this.sink.onTaint?.(t);
+                return { $: "unsupported", kind: "texture-tainted" };
+            }
+            return { $: "unsupported", kind: "texture" };
+        }
+        if (!img)
+            return { $: "unsupported", kind: src.constructor?.name ?? "texture" };
+        const id = this.nextAssetId++;
+        this.assetCount++;
+        this.assetBySource.set(src, id);
+        const w = img.width, h = img.height;
+        // Copy the pixels now — the source may be reused/mutated after this call.
+        const bytes = new Uint8Array(img.data.buffer.slice(img.data.byteOffset, img.data.byteOffset + img.data.byteLength));
+        this.trackEncode(deflate(bytes)
+            .then((z) => {
+            this.sink.onAssetBytes(id, z.length, t);
+            this.sink.onAssetData?.({ id, format: "imagedata", w, h, t }, z);
+        })
+            .catch((e) => this.sink.note(`GL texture asset deflate failed: ${e.message}`)));
+        return { $: "imagedata", id };
+    }
+    /** Bind the strategy's sink, draining anything buffered since getContext. */
+    swapSink(sink) {
+        const old = this.sink;
+        this.sink = sink;
+        this.bound = true;
+        if (old instanceof BufferSink) {
+            for (const op of old.ops)
+                sink.onOp(op);
+            for (const a of old.assetEvents)
+                sink.onAssetBytes(a.id, a.bytes, a.t);
+            for (const d of old.assetData)
+                sink.onAssetData?.(d.info, d.payload); // GL texture assets
+            for (const n of old.notes)
+                sink.note(n);
+        }
+    }
+    /** draw-site stamp id for the next op (Phase 3b); see ContextRecorder.pendingSiteId */
+    pendingSiteId;
+    recordCall(op, args, result) {
+        // Video-mode dirtiness only: zero serialize/encode cost.
+        if (this.sink instanceof DiscardSink) {
+            this.sink.onOp(DIRTY_OP);
+            this.pendingSiteId = undefined;
+            return;
+        }
+        const t0 = performance.now();
+        const t = this.now();
+        const sfId = this.pendingSiteId;
+        this.pendingSiteId = undefined;
+        if ((CREATOR_RE.test(op) || OTHER_CREATORS.has(op)) && result && typeof result === "object") {
+            const id = this.nextId++;
+            this.refs.set(result, id);
+            this.sink.onOp({
+                t, op: "__create", create: true,
+                args: [op, id, ...args.map((a) => this.arg(a, t))],
+            });
+        }
+        else {
+            const rec = { t, op, args: args.map((a) => this.arg(a, t)) };
+            if (sfId)
+                rec.sfId = sfId;
+            this.sink.onOp(rec);
+        }
+        this.cpuMs += performance.now() - t0;
+    }
+    arg(a, _t) {
+        if (a === null || a === undefined)
+            return null;
+        switch (typeof a) {
+            case "number": return Number.isFinite(a) ? a : 0;
+            case "string": return a;
+            case "boolean": return a;
+            case "object": break;
+            default: return { $: "unsupported", kind: typeof a };
+        }
+        const obj = a;
+        const ref = this.refs.get(obj);
+        if (ref !== undefined)
+            return { $: "obj", id: ref };
+        if (ArrayBuffer.isView(obj) || obj instanceof ArrayBuffer) {
+            return serializeTypedArray(obj);
+        }
+        if (Array.isArray(obj))
+            return { $: "arr", v: obj.map((n) => Number(n) || 0) };
+        // GL texture source (texImage2D/texSubImage2D) — capture static image /
+        // ImageData sources as imagedata assets so textured content replays + is
+        // pickable. Dynamic (video/canvas) sources fall through to unsupported.
+        if (isStaticTexSource(obj))
+            return this.captureTexSource(obj, _t);
+        return { $: "unsupported", kind: obj.constructor?.name ?? "object" };
+    }
+}
+export function installGLHooks() {
+    if (installed)
+        return;
+    installed = true;
+    const protos = [];
+    if (typeof WebGLRenderingContext !== "undefined")
+        protos.push(WebGLRenderingContext.prototype);
+    if (typeof WebGL2RenderingContext !== "undefined")
+        protos.push(WebGL2RenderingContext.prototype);
+    for (const proto of protos) {
+        for (const name of Object.getOwnPropertyNames(proto)) {
+            const desc = Object.getOwnPropertyDescriptor(proto, name);
+            if (!desc || typeof desc.value !== "function" || name === "constructor")
+                continue;
+            if (PURE_RE.test(name) && !OTHER_CREATORS.has(name))
+                continue;
+            const orig = desc.value;
+            proto[name] = function (...args) {
+                const result = orig.apply(this, args);
+                const rec = glRecorders.get(this);
+                if (rec)
+                    rec.recordCall(name, args, result);
+                else if (untrackedNotify)
+                    untrackedNotify(this);
+                return result;
+            };
+        }
+    }
+    // auto-attach on context creation (chained over any earlier getContext patch)
+    if (typeof HTMLCanvasElement === "undefined")
+        return;
+    const origGetContext = HTMLCanvasElement.prototype.getContext;
+    HTMLCanvasElement.prototype.getContext = function (type, attrs) {
+        const result = origGetContext.call(this, type, attrs);
+        if ((type === "webgl" || type === "webgl2" || type === "experimental-webgl") &&
+            result && glAutoConfig && !glRecorders.has(result)) {
+            const rec = new GLRecorder(new BufferSink(), glAutoConfig.now);
+            rec.rawCtx = result;
+            glRecorders.set(result, rec);
+            glAutoRecorders.set(this, rec);
+        }
+        return result;
+    };
+}
+export function attachGLRecorder(canvas, contextType, sink, now) {
+    const auto = glAutoRecorders.get(canvas);
+    if (auto) {
+        auto.swapSink(sink);
+        return auto;
+    }
+    const ctx = canvas.getContext(contextType);
+    if (!ctx)
+        return null;
+    const rec = new GLRecorder(sink, now);
+    rec.rawCtx = ctx;
+    rec.bound = true;
+    glRecorders.set(ctx, rec);
+    return rec;
+}
+/** Mid-stream attach to a known GL context (created before hooks installed). */
+export function attachGLRecorderToContext(ctx, sink, now) {
+    const rec = new GLRecorder(sink, now);
+    rec.rawCtx = ctx;
+    rec.bound = true;
+    glRecorders.set(ctx, rec);
+    return rec;
+}
+export function detachGLRecorder(canvas, contextType) {
+    try {
+        const ctx = canvas.getContext(contextType);
+        if (ctx)
+            glRecorders.delete(ctx);
+    }
+    catch { /* gone */ }
+}
+export function detachGLRecorderFromContext(ctx) {
+    glRecorders.delete(ctx);
+}