@sailfish-ai/recorder 1.12.5 → 1.12.6

package/dist/canvas/encodeWorker.js

@@ -0,0 +1,284 @@
+/**
+ * Inline-blob encode workers (webp image encoder + WebCodecs video encoder)
+ * and their promise/event clients. Ports of the bench workers
+ * (test-applications/canvas-recorder-bench/src/workers/{imgEncoder,videoEncoder}.worker.ts)
+ * as TEMPLATE-STRING sources so the vite lib build emits NO separate worker
+ * files — same precedent as WS_WORKER_CODE in src/websocket.tsx. Unlike the
+ * WS worker, this code may use modern JS: every browser with WebCodecs /
+ * OffscreenCanvas supports ES2020.
+ *
+ * SDK adaptation to the bench video worker: encoded chunk DATA is posted
+ * LIVE (transferred) instead of accumulated until stop, and the decoder
+ * config is posted once as `decoder-config` — the manager streams video-chunk
+ * / video-init SFCV frames as they are produced.
+ *
+ * SSR rule: workers are created lazily inside functions, never at module scope.
+ */
+// ─── Image encoder: ImageBitmap -> webp bytes ────────────────────────────────
+const IMG_ENCODER_CODE = `
+self.onmessage = async (e) => {
+  const { id, bitmap, quality, wantBytes } = e.data;
+  try {
+    const off = new OffscreenCanvas(bitmap.width, bitmap.height);
+    const ctx = off.getContext('2d');
+    ctx.drawImage(bitmap, 0, 0);
+    bitmap.close();
+    const blob = await off.convertToBlob({ type: 'image/webp', quality });
+    if (wantBytes) {
+      const buf = await blob.arrayBuffer();
+      self.postMessage({ id, size: buf.byteLength, bytes: buf }, [buf]);
+    } else {
+      self.postMessage({ id, size: blob.size });
+    }
+  } catch (err) {
+    self.postMessage({ id, size: 0, error: String(err) });
+  }
+};
+`;
+// ─── Video encoder: transferred VideoFrames -> encoded chunks (live) ─────────
+const VIDEO_ENCODER_CODE = `
+let encoder = null;
+let decoderConfig = null;
+let sentConfig = false;
+
+async function pickCodec(codecs, width, height, bitrate) {
+  for (const codec of codecs) {
+    try {
+      const support = await VideoEncoder.isConfigSupported({
+        codec, width, height, bitrate, framerate: 30,
+        ...(codec.startsWith('avc1') ? { avc: { format: 'avc' } } : {}),
+      });
+      if (support.supported) return codec;
+    } catch (e) { /* try next */ }
+  }
+  return null;
+}
+
+self.onmessage = async (e) => {
+  const msg = e.data;
+  if (msg.type === 'config') {
+    const codec = await pickCodec(msg.codecs, msg.width, msg.height, msg.bitrate);
+    if (!codec) {
+      self.postMessage({ type: 'config-failed' });
+      return;
+    }
+    encoder = new VideoEncoder({
+      output: (chunk, meta) => {
+        if (meta && meta.decoderConfig && !decoderConfig) {
+          const desc = meta.decoderConfig.description;
+          decoderConfig = {
+            codec: meta.decoderConfig.codec,
+            description: desc
+              ? (desc instanceof ArrayBuffer
+                  ? desc.slice(0)
+                  : desc.buffer.slice(desc.byteOffset, desc.byteOffset + desc.byteLength))
+              : null,
+          };
+        }
+        if (decoderConfig && !sentConfig) {
+          sentConfig = true;
+          self.postMessage({
+            type: 'decoder-config',
+            codec: decoderConfig.codec,
+            description: decoderConfig.description,
+          });
+        }
+        const data = new ArrayBuffer(chunk.byteLength);
+        chunk.copyTo(data);
+        self.postMessage({
+          type: 'chunk',
+          data,
+          timestampUs: chunk.timestamp,
+          key: chunk.type === 'key',
+        }, [data]);
+      },
+      error: (err) => self.postMessage({ type: 'error', message: String(err) }),
+    });
+    encoder.configure({
+      codec,
+      width: msg.width,
+      height: msg.height,
+      bitrate: msg.bitrate,
+      framerate: 30,
+      latencyMode: 'realtime',
+      ...(codec.startsWith('avc1') ? { avc: { format: 'avc' } } : {}),
+    });
+    self.postMessage({ type: 'configured', codec });
+    return;
+  }
+  if (msg.type === 'frame') {
+    try {
+      if (encoder) encoder.encode(msg.frame, { keyFrame: msg.key });
+    } catch (err) {
+      self.postMessage({ type: 'error', message: String(err) });
+    } finally {
+      msg.frame.close();
+    }
+    return;
+  }
+  if (msg.type === 'flush') {
+    // Non-terminal drain (visibilitychange:hidden): push queued frames out as
+    // chunks WITHOUT closing the encoder, so capture can resume if the tab
+    // returns. Chunks stream through the existing output handler above.
+    try {
+      if (encoder) await encoder.flush();
+    } catch (e) { /* flush may fail when nothing was encoded */ }
+    self.postMessage({ type: 'flushed' });
+    return;
+  }
+  if (msg.type === 'stop') {
+    try {
+      if (encoder) await encoder.flush();
+    } catch (e) { /* flush may fail when nothing was encoded */ }
+    self.postMessage({ type: 'done' });
+    if (encoder) encoder.close();
+    encoder = null;
+  }
+};
+`;
+function createBlobWorker(code) {
+    const blob = new Blob([code], { type: "application/javascript" });
+    const url = URL.createObjectURL(blob);
+    const w = new Worker(url);
+    URL.revokeObjectURL(url); // Worker keeps the blob reference; safe to revoke
+    return w;
+}
+let imgWorker = null;
+let nextId = 1;
+const pendingEncodes = new Map();
+function getImgWorker() {
+    if (!imgWorker) {
+        imgWorker = createBlobWorker(IMG_ENCODER_CODE);
+        imgWorker.onmessage = (e) => {
+            const resolve = pendingEncodes.get(e.data.id);
+            if (resolve) {
+                pendingEncodes.delete(e.data.id);
+                resolve({ size: e.data.size, bytes: e.data.bytes });
+            }
+        };
+    }
+    return imgWorker;
+}
+/** Encode a bitmap to webp in the worker. TRANSFERS (detaches) the bitmap. */
+export function encodeBitmap(bitmap, quality, wantBytes = false) {
+    const id = nextId++;
+    return new Promise((resolve) => {
+        pendingEncodes.set(id, resolve);
+        getImgWorker().postMessage({ id, bitmap, quality, wantBytes }, [bitmap]);
+    });
+}
+/** Tear down the shared image-encoder worker (manager stop path). */
+export function terminateImgEncoder() {
+    if (imgWorker) {
+        imgWorker.terminate();
+        imgWorker = null;
+    }
+    // resolve any stragglers so awaiting callers do not hang
+    for (const [id, resolve] of pendingEncodes) {
+        pendingEncodes.delete(id);
+        resolve({ size: 0 });
+    }
+}
+export class VideoEncoderClient {
+    events;
+    worker;
+    configured = false;
+    doneResolve = null;
+    flushResolve = null;
+    configResolve = null;
+    constructor(events) {
+        this.events = events;
+        this.worker = createBlobWorker(VIDEO_ENCODER_CODE);
+        this.worker.onmessage = (e) => {
+            const m = e.data;
+            switch (m.type) {
+                case "configured":
+                    this.configured = true;
+                    this.configResolve?.(true);
+                    this.configResolve = null;
+                    break;
+                case "config-failed":
+                    this.configResolve?.(false);
+                    this.configResolve = null;
+                    break;
+                case "decoder-config":
+                    this.events.onDecoderConfig(m.codec, m.description ?? null);
+                    break;
+                case "chunk":
+                    this.events.onChunk(m.data, m.timestampUs, m.key);
+                    break;
+                case "error":
+                    this.events.onError(String(m.message));
+                    break;
+                case "done":
+                    this.doneResolve?.();
+                    this.doneResolve = null;
+                    break;
+                case "flushed":
+                    this.flushResolve?.();
+                    this.flushResolve = null;
+                    break;
+            }
+        };
+    }
+    get isConfigured() {
+        return this.configured;
+    }
+    configure(cfg) {
+        return new Promise((resolve) => {
+            this.configResolve = resolve;
+            this.worker.postMessage({ type: "config", ...cfg });
+        });
+    }
+    /** TRANSFERS (closes) the frame. */
+    encodeFrame(frame, key) {
+        this.worker.postMessage({ type: "frame", frame, key }, [frame]);
+    }
+    /** Non-terminal drain: flush queued frames out as chunks WITHOUT closing the
+     * encoder, so capture can resume. Resolves once the worker acks (or times
+     * out). Safe to call repeatedly. */
+    flush(timeoutMs = 3000) {
+        return new Promise((resolve) => {
+            const tid = setTimeout(() => {
+                this.flushResolve = null;
+                resolve();
+            }, timeoutMs);
+            this.flushResolve = () => {
+                clearTimeout(tid);
+                resolve();
+            };
+            try {
+                this.worker.postMessage({ type: "flush" });
+            }
+            catch {
+                clearTimeout(tid);
+                this.flushResolve = null;
+                resolve();
+            }
+        });
+    }
+    /** Flush the encoder; resolves once all chunks have been delivered. */
+    stop(timeoutMs = 3000) {
+        return new Promise((resolve) => {
+            const tid = setTimeout(() => {
+                this.doneResolve = null;
+                resolve();
+            }, timeoutMs);
+            this.doneResolve = () => {
+                clearTimeout(tid);
+                resolve();
+            };
+            try {
+                this.worker.postMessage({ type: "stop" });
+            }
+            catch {
+                clearTimeout(tid);
+                this.doneResolve = null;
+                resolve();
+            }
+        });
+    }
+    terminate() {
+        this.worker.terminate();
+    }
+}

package/dist/canvas/governor.js

@@ -0,0 +1,79 @@
+/**
+ * Phase 2 live-demotion governor (contract §5): per-canvas counters that
+ * decide when a command stream is too heavy and the canvas should demote to
+ * video capture.
+ *
+ *  - POST-deflate command bytes: demote when the trailing 3s window carries
+ *    more than 512 KB/s (i.e. > 1536 KB total in the window).
+ *  - dynamic VIDEO-source drawImage: demote when more than 2 video-element
+ *    draws land within any trailing 1s window.
+ *  - ops per frame: demote when a single frame (ops sharing one timestamp)
+ *    exceeds 5000 ops.
+ */
+export const DEFAULT_GOVERNOR_THRESHOLDS = {
+    maxBytesPerSec: 512 * 1024,
+    sustainMs: 3_000,
+    maxVideoDrawsPerSec: 2,
+    maxOpsPerFrame: 5_000,
+};
+export class CanvasGovernor {
+    now;
+    thresholds;
+    byteSamples = [];
+    videoDraws = [];
+    frameT = NaN;
+    frameOps = 0;
+    opsExceeded = false;
+    constructor(now, thresholds) {
+        this.now = now;
+        this.thresholds = { ...DEFAULT_GOVERNOR_THRESHOLDS, ...thresholds };
+    }
+    /** Record post-deflate bytes for a flushed command batch. */
+    addBytes(n) {
+        const t = this.now();
+        this.byteSamples.push({ t, n });
+        this.pruneBytes(t);
+    }
+    /** Record one dynamic video-source drawImage capture. */
+    addVideoDraw() {
+        const t = this.now();
+        this.videoDraws.push(t);
+        this.pruneVideo(t);
+    }
+    /** Record one recorded op (tMs = the op's frame timestamp). */
+    addOp(tMs) {
+        if (tMs !== this.frameT) {
+            this.frameT = tMs;
+            this.frameOps = 0;
+        }
+        if (++this.frameOps > this.thresholds.maxOpsPerFrame)
+            this.opsExceeded = true;
+    }
+    /** Returns the demotion reason once a threshold trips, else null. */
+    shouldDemote() {
+        if (this.opsExceeded)
+            return "ops";
+        const t = this.now();
+        this.pruneVideo(t);
+        if (this.videoDraws.length > this.thresholds.maxVideoDrawsPerSec)
+            return "video-source";
+        this.pruneBytes(t);
+        let total = 0;
+        for (const s of this.byteSamples)
+            total += s.n;
+        const budget = (this.thresholds.maxBytesPerSec * this.thresholds.sustainMs) / 1000;
+        if (total > budget)
+            return "bytes";
+        return null;
+    }
+    pruneBytes(t) {
+        const cutoff = t - this.thresholds.sustainMs;
+        while (this.byteSamples.length > 0 && this.byteSamples[0].t < cutoff)
+            this.byteSamples.shift();
+    }
+    pruneVideo(t) {
+        const cutoff = t - 1000;
+        while (this.videoDraws.length > 0 && this.videoDraws[0] < cutoff)
+            this.videoDraws.shift();
+    }
+}

package/dist/canvas/keyframes.js

@@ -0,0 +1,183 @@
+/**
+ * Periodic pixel keyframes captured same-task — port of the bench hybrid
+ * strategy's takeKeyframe (test-applications/canvas-recorder-bench/src/
+ * capture/hybrid.ts). Keyframes are the bit-exact ground truth and the seek
+ * anchor; for 2D they also enable replay-from-keyframe (state snapshot rides
+ * the SFCV keyframe meta).
+ */
+import { REDACTION_COLOR } from "./canvasPrivacy";
+import { encodeBitmap } from "./encodeWorker";
+import { ignore2DContext } from "./patch2d";
+const KEYFRAME_INTERVAL_MS = 4_000;
+/** min spacing for interaction-triggered keyframes */
+const INTERACTION_KEYFRAME_GAP_MS = 250; // tight anchors near interactions — GPU streams replay from keyframes
+export class KeyframeCapturer {
+    canvas;
+    snapshotState;
+    emit;
+    note;
+    getMaskedRegions;
+    scratch = null;
+    scratchCtx = null;
+    lastKeyframe = -Infinity;
+    /** has the canvas actually rendered since the last keyframe? */
+    dirtySinceKeyframe = false;
+    interactionRequested = false;
+    pending = [];
+    stopped = false;
+    constructor(canvas, snapshotState, emit, note, 
+    /**
+     * Masked-region source (maskCanvasDraws / redact-mode text). Each device-space
+     * bbox is filled with REDACTION_COLOR on the scratch copy BEFORE the keyframe
+     * is encoded, so the real pixels of masked content never ship — replay then
+     * draws the baked redaction (recolored shapes / asterisks / bars) from the
+     * command stream on top of the blanked keyframe. Defaults to none (GL/other
+     * sources have no per-op masking).
+     */
+    getMaskedRegions = () => []) {
+        this.canvas = canvas;
+        this.snapshotState = snapshotState;
+        this.emit = emit;
+        this.note = note;
+        this.getMaskedRegions = getMaskedRegions;
+    }
+    onInteraction(tMs) {
+        // BUGFIX (Phase 0.5 E2E, found on /webgl1 — guardrail #6 third strike):
+        // this runs in the POINTER-EVENT task. Copying pixels here reads a
+        // cleared buffer on preserveDrawingBuffer:false GL/WebGPU canvases
+        // (blank ~1KB keyframes), and on 2D it raced the app's click-handler
+        // redraw (the earlier tMs-1 backdating workaround). Instead, REQUEST a
+        // keyframe: the next afterRender pump runs in the render task, so the
+        // copy is buffer-valid and its pixels already include the interaction's
+        // redraw ops — the keyframe invariant holds with no backdating. Canvases
+        // that don't redraw on interaction take no keyframe (dirty-gating kept:
+        // afterRender only runs after draws).
+        if (tMs - this.lastKeyframe >= INTERACTION_KEYFRAME_GAP_MS) {
+            this.interactionRequested = true;
+        }
+    }
+    /**
+     * A27 — one-shot keyframe for a canvas that was already painted BEFORE the
+     * SDK loaded and may never redraw (a static pre-existing canvas). The normal
+     * keyframe pacing is dirty-gated (afterRender only runs after a draw), so a
+     * canvas that is painted once and then idle would never emit a keyframe and
+     * would replay blank. This forces a single ground-truth keyframe of the
+     * canvas's current pixels, independent of the dirty flag or the interval.
+     *
+     * Unlike afterRender this is NOT same-task with a render — but a STATIC
+     * pre-existing 2D canvas's committed pixels persist outside any rendering
+     * task (the guardrail-#6 hazard only affects pdb:false WebGL / task-scoped
+     * WebGPU textures, which are not what A27 targets). takeKeyframe's
+     * drawImage(canvas) copy throws on a zero-size canvas (caught -> no emit) and
+     * a masked/tainted canvas never reaches here (the manager only builds a
+     * controller for unmasked canvases, and a tainted readback simply fails the
+     * encode). Returns true if a keyframe copy was kicked off.
+     */
+    initialKeyframe(tMs) {
+        if (this.stopped)
+            return false;
+        if (!this.canvas.width || !this.canvas.height)
+            return false; // zero-size — skip
+        const before = this.pending.length;
+        this.takeKeyframe(tMs);
+        return this.pending.length > before;
+    }
+    /** Same-task keyframes: GPU-canvas-safe and never captures mid-draw. */
+    afterRender(tMs) {
+        this.dirtySinceKeyframe = true;
+        if (this.interactionRequested) {
+            this.interactionRequested = false;
+            if (tMs - this.lastKeyframe >= INTERACTION_KEYFRAME_GAP_MS) {
+                this.takeKeyframe(tMs);
+                return;
+            }
+        }
+        if (tMs - this.lastKeyframe >= KEYFRAME_INTERVAL_MS)
+            this.takeKeyframe(tMs);
+    }
+    takeKeyframe(t) {
+        if (this.stopped)
+            return;
+        this.lastKeyframe = t;
+        this.dirtySinceKeyframe = false;
+        const state = this.snapshotState();
+        // Keyframe pixels MUST be copied synchronously in the rendering task
+        // (guardrail #6): createImageBitmap(canvas) defers its copy past the task
+        // boundary, where WebGPU textures have expired and pdb:false WebGL
+        // buffers are cleared — producing blank keyframes. drawImage to a scratch
+        // canvas is the in-task copy; the bitmap is then safely taken from it.
+        if (!this.scratch) {
+            this.scratch = document.createElement("canvas");
+            this.scratchCtx = this.scratch.getContext("2d");
+            if (this.scratchCtx)
+                ignore2DContext(this.scratchCtx); // our own context — never record it
+        }
+        const scratch = this.scratch;
+        const scratchCtx = this.scratchCtx;
+        if (!scratchCtx)
+            return;
+        if (scratch.width !== this.canvas.width || scratch.height !== this.canvas.height) {
+            scratch.width = this.canvas.width;
+            scratch.height = this.canvas.height;
+        }
+        else {
+            scratchCtx.clearRect(0, 0, scratch.width, scratch.height);
+        }
+        try {
+            scratchCtx.drawImage(this.canvas, 0, 0);
+        }
+        catch {
+            return; // zero-size canvas
+        }
+        // Region-blank: paint over the bbox of every masked draw so the keyframe
+        // ships NO real pixels of masked content. Regions are device-space (the
+        // scratch is canvas-sized), so reset the transform before filling.
+        const regions = this.getMaskedRegions();
+        if (regions.length > 0) {
+            scratchCtx.save();
+            scratchCtx.setTransform(1, 0, 0, 1, 0, 0);
+            scratchCtx.fillStyle = REDACTION_COLOR;
+            for (const r of regions)
+                scratchCtx.fillRect(r.x, r.y, r.w, r.h);
+            scratchCtx.restore();
+        }
+        const w = scratch.width;
+        const h = scratch.height;
+        const p = createImageBitmap(scratch)
+            .then(async (bitmap) => {
+            // encodeBitmap TRANSFERS the bitmap to the worker (detaching it) —
+            // it is consumed here, never reused.
+            const r = await encodeBitmap(bitmap, 0.9, true);
+            if (r.bytes && r.bytes.byteLength > 0) {
+                this.emit({ tMs: t, w, h, state, bytes: r.bytes });
+            }
+        })
+            .catch((e) => this.note(`keyframe encode failed: ${e.message}`));
+        this.pending.push(p);
+    }
+    /** Final keyframe (if dirty) + settle outstanding encodes. */
+    async stop(finalT) {
+        if (this.dirtySinceKeyframe)
+            this.takeKeyframe(finalT);
+        this.stopped = true;
+        await Promise.all(this.pending);
+    }
+    /**
+     * A22 / A6: immediate disposal with NO final keyframe — used when the canvas
+     * was removed from the DOM or masked mid-session (its pixels are gone or must
+     * not be recorded). Releases the scratch keyframe canvas so it can be GC'd.
+     */
+    dispose() {
+        this.stopped = true;
+        // Drop the scratch canvas (sizing it to 0 frees its backing store promptly).
+        if (this.scratch) {
+            try {
+                this.scratch.width = 0;
+                this.scratch.height = 0;
+            }
+            catch { /* already detached */ }
+        }
+        this.scratch = null;
+        this.scratchCtx = null;
+    }
+}