@engenty/wizards-runtime 0.0.0-stage → 0.7.5

package/apps/runtime/dist/widgets/render.js

@@ -0,0 +1,329 @@
+import { spawn } from "node:child_process";
+import { readFile, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { env } from "../env.js";
+import { extFor } from "../files/storage.js";
+import { ffmpeg, hasFfmpeg, probeMedia, withTempDir } from "../media/ffmpeg.js";
+import { newContext, offlineContext } from "../render/chromium.js";
+export { hasFfmpeg } from "../media/ffmpeg.js";
+const FPS = 25;
+const MAX_SECONDS = 60;
+/** Exports set `wizard.mode` to "export" so the widget hides its controls and does not autoplay. */
+function forExport(html) {
+    return html.replace('<script type="application/json" id="wizard-payload">', '<script>window.__WIZARD_EXPORT=true</script><script type="application/json" id="wizard-payload">');
+}
+async function withWidget(html, size, scale, use) {
+    const context = await offlineContext({
+        viewport: { width: size.width, height: size.height },
+        deviceScaleFactor: scale,
+    });
+    const errors = [];
+    try {
+        const page = await context.newPage();
+        page.on("pageerror", (err) => errors.push(err.message));
+        page.on("console", (msg) => {
+            if (msg.type() === "error") {
+                errors.push(msg.text());
+            }
+        });
+        await page.setContent(forExport(html), { waitUntil: "load", timeout: 30_000 });
+        // A widget calls wizard.ready() after its first frame; one that never does gets 3 seconds.
+        await page.evaluate(() => Promise.race([window.wizard?.__ready, new Promise((r) => setTimeout(r, 3000))]));
+        return await use(page, errors);
+    }
+    finally {
+        await context.close();
+    }
+}
+async function timelineOf(page) {
+    return page.evaluate(() => {
+        const t = window.wizard?.__timeline?.();
+        return t ? { duration: Number(t.duration), poster: Number(t.poster ?? 0) } : null;
+    });
+}
+async function seek(page, seconds) {
+    await page.evaluate(async (t) => {
+        // A film's seek waits for its clips to reach the frame; a drawn animation returns at once.
+        await window.wizard.__timeline().seek(t);
+        await new Promise((r) => requestAnimationFrame(() => r(null)));
+    }, seconds);
+}
+/** Loads the widget once: does it run, and does it animate? */
+export async function probeWidget(html, size) {
+    return withWidget(html, size, 1, async (page, errors) => {
+        const timeline = await timelineOf(page);
+        if (timeline) {
+            await seek(page, timeline.poster);
+        }
+        const png = await page.screenshot({ type: "png" });
+        return {
+            duration: timeline ? Math.min(timeline.duration, MAX_SECONDS) : null,
+            errors: [...new Set(errors)].slice(0, 10),
+            png,
+        };
+    });
+}
+export async function widgetPng(html, size) {
+    return withWidget(html, size, 2, async (page) => {
+        const timeline = await timelineOf(page);
+        if (timeline) {
+            await seek(page, timeline.poster);
+        }
+        return page.screenshot({ type: "png", fullPage: true });
+    });
+}
+export async function widgetPdf(html, size) {
+    return withWidget(html, size, 1, async (page) => {
+        const timeline = await timelineOf(page);
+        if (timeline) {
+            await seek(page, timeline.poster);
+        }
+        const height = await page.evaluate(() => document.documentElement.scrollHeight);
+        return page.pdf({
+            width: `${size.width}px`,
+            height: `${Math.max(size.height, height)}px`,
+            printBackground: true,
+            pageRanges: "1",
+        });
+    });
+}
+/** Steps through the timeline frame by frame and encodes the frames as H.264 (no sound). */
+async function encodeFrames(page, seconds, out, onProgress) {
+    const encoder = spawn(env.ffmpegPath, [
+        "-y",
+        "-loglevel",
+        "error",
+        "-f",
+        "image2pipe",
+        "-framerate",
+        String(FPS),
+        "-i",
+        "-",
+        "-c:v",
+        "libx264",
+        "-preset",
+        "veryfast",
+        "-crf",
+        "20",
+        "-pix_fmt",
+        "yuv420p",
+        "-movflags",
+        "+faststart",
+        out,
+    ], { stdio: ["pipe", "ignore", "pipe"] });
+    let stderr = "";
+    encoder.stderr.on("data", (d) => {
+        stderr += String(d);
+    });
+    const done = new Promise((resolve, reject) => {
+        encoder.on("error", reject);
+        encoder.on("exit", (code) => code === 0 ? resolve() : reject(new Error(`ffmpeg: ${stderr.slice(0, 300)}`)));
+    });
+    const frames = Math.max(1, Math.round(seconds * FPS));
+    for (let i = 0; i < frames; i++) {
+        await seek(page, (i * seconds) / frames);
+        const jpeg = await page.screenshot({ type: "jpeg", quality: 92 });
+        if (!encoder.stdin.write(jpeg)) {
+            await new Promise((r) => encoder.stdin.once("drain", r));
+        }
+        if (i % FPS === 0) {
+            onProgress?.(i / frames);
+        }
+    }
+    encoder.stdin.end();
+    await done;
+}
+/** Steps through the widget's timeline frame by frame and encodes the frames as H.264. */
+export async function widgetMp4(html, size) {
+    if (!(await hasFfmpeg())) {
+        return null;
+    }
+    const even = { width: size.width - (size.width % 2), height: size.height - (size.height % 2) };
+    return withTempDir(async (dir) => {
+        const out = join(dir, "out.mp4");
+        return withWidget(html, even, 1, async (page) => {
+            const timeline = await timelineOf(page);
+            if (!timeline) {
+                return null;
+            }
+            await encodeFrames(page, Math.min(timeline.duration, MAX_SECONDS), out);
+            return new Uint8Array(await readFile(out));
+        });
+    });
+}
+// --- films: a timeline of clips and stills, with sound ---------------------------------
+/** Where a film's page finds this run's clips, images and voice-over while it is rendered. */
+export const FILM_MEDIA_ORIGIN = "http://wizard.media";
+/** A film names at most this many sound tracks: voice-over, music, and every cut of a recording. */
+const MAX_TRACKS = 48;
+/**
+ * Renders a film widget: its page draws every frame from this run's clips and images, and names
+ * the sound that belongs to it (voice-over, the clips' own sound). Clips are re-encoded with a
+ * key frame every few frames first, so the page reaches any frame quickly and exactly.
+ */
+export async function renderFilm(html, size, media, onProgress) {
+    if (!(await hasFfmpeg())) {
+        return null;
+    }
+    const even = { width: size.width - (size.width % 2), height: size.height - (size.height % 2) };
+    return withTempDir(async (dir) => {
+        // url → the file as it came (for its sound) and what the page is served.
+        const files = new Map();
+        let n = 0;
+        for (const [url, file] of media) {
+            const path = join(dir, `media-${n++}.${extFor(file.mime)}`);
+            await writeFile(path, file.bytes);
+            let served = file;
+            if (file.mime.startsWith("video/")) {
+                const seekable = join(dir, `seek-${n}.mp4`);
+                const { code } = await ffmpeg([
+                    "-y",
+                    "-loglevel",
+                    "error",
+                    "-i",
+                    path,
+                    "-an",
+                    "-c:v",
+                    "libx264",
+                    "-preset",
+                    "veryfast",
+                    "-crf",
+                    "20",
+                    "-g",
+                    "5",
+                    "-pix_fmt",
+                    "yuv420p",
+                    "-movflags",
+                    "+faststart",
+                    seekable,
+                ]);
+                if (code === 0) {
+                    served = { bytes: new Uint8Array(await readFile(seekable)), mime: "video/mp4" };
+                }
+            }
+            files.set(url, { path, served });
+        }
+        const context = await newContext({ viewport: even, deviceScaleFactor: 1 });
+        const errors = [];
+        try {
+            await context.route("**/*", async (route) => {
+                const request = route.request();
+                const file = files.get(request.url().split(/[?#]/)[0])?.served;
+                if (!file) {
+                    await route.abort("blockedbyclient");
+                    return;
+                }
+                // Players ask for byte ranges and only seek freely when they get them.
+                const total = file.bytes.byteLength;
+                const range = request.headers().range?.match(/^bytes=(\d+)-(\d*)$/);
+                const from = range ? Math.min(Number(range[1]), total - 1) : 0;
+                const to = range?.[2] ? Math.min(Number(range[2]), total - 1) : total - 1;
+                await route.fulfill({
+                    status: range ? 206 : 200,
+                    contentType: file.mime,
+                    headers: {
+                        "accept-ranges": "bytes",
+                        ...(range ? { "content-range": `bytes ${from}-${to}/${total}` } : {}),
+                    },
+                    body: Buffer.from(file.bytes.subarray(from, to + 1)),
+                });
+            });
+            const page = await context.newPage();
+            page.on("pageerror", (err) => errors.push(err.message));
+            await page.setContent(forExport(html), { waitUntil: "load", timeout: 30_000 });
+            // A film is ready once its clips are loaded; that takes longer than a drawn widget.
+            await page.evaluate(() => Promise.race([window.wizard?.__ready, new Promise((r) => setTimeout(r, 45_000))]));
+            const timeline = await page.evaluate(() => {
+                const t = window.wizard?.__timeline?.();
+                return t
+                    ? {
+                        duration: Number(t.duration),
+                        poster: Number(t.poster ?? 0),
+                        audio: (Array.isArray(t.audio) ? t.audio : []),
+                    }
+                    : null;
+            });
+            if (!timeline) {
+                return null;
+            }
+            const seconds = Math.min(timeline.duration, MAX_SECONDS);
+            const silent = join(dir, "silent.mp4");
+            await encodeFrames(page, seconds, silent, onProgress);
+            await seek(page, Math.min(timeline.poster, seconds));
+            const png = await page.screenshot({ type: "png" });
+            const tracks = [];
+            // Sounds from the wizard's workspace (wizard.url) arrive as data URLs.
+            const inline = new Map();
+            for (const track of timeline.audio.slice(0, MAX_TRACKS)) {
+                const data = String(track.src).match(/^data:(audio\/[\w.+-]+);base64,(.+)$/);
+                if (data && !inline.has(track.src)) {
+                    const path = join(dir, `sound-${inline.size}.${extFor(data[1])}`);
+                    await writeFile(path, Buffer.from(data[2], "base64"));
+                    inline.set(track.src, path);
+                }
+            }
+            for (const track of timeline.audio.slice(0, MAX_TRACKS)) {
+                const path = inline.get(track.src) ?? files.get(String(track.src).split(/[?#]/)[0])?.path;
+                if (path && track.start < seconds && (await probeMedia(path)).audio) {
+                    tracks.push({ ...track, path });
+                }
+            }
+            let out = silent;
+            if (tracks.length) {
+                const mixed = join(dir, "film.mp4");
+                const chains = tracks.map((track, i) => {
+                    const length = Math.min(track.duration ?? seconds, seconds - track.start);
+                    const volume = Math.max(0, Math.min(2, track.volume ?? 1));
+                    const delay = Math.round(Math.max(0, track.start) * 1000);
+                    const cut = track.from !== undefined;
+                    const from = Math.max(0, Number(track.from) || 0);
+                    // A cut out of a recording gets short fades, so the hard cut does not click.
+                    const fade = cut ? Math.min(0.03, length / 4) : 0.25;
+                    const fadeIn = cut ? `afade=t=in:d=${fade.toFixed(3)},` : "";
+                    return `[${i + 1}:a]atrim=${from.toFixed(3)}:${(from + length).toFixed(3)},asetpts=PTS-STARTPTS,${fadeIn}afade=t=out:st=${Math.max(0, length - fade).toFixed(3)}:d=${fade.toFixed(3)},adelay=${delay}:all=1,volume=${volume}[a${i}]`;
+                });
+                const mix = `${tracks.map((_, i) => `[a${i}]`).join("")}amix=inputs=${tracks.length}:normalize=0:duration=longest[mix]`;
+                const { code, stderr } = await ffmpeg([
+                    "-y",
+                    "-loglevel",
+                    "error",
+                    "-i",
+                    silent,
+                    ...tracks.flatMap((track) => ["-i", track.path]),
+                    "-filter_complex",
+                    [...chains, mix].join(";"),
+                    "-map",
+                    "0:v",
+                    "-map",
+                    "[mix]",
+                    "-c:v",
+                    "copy",
+                    "-c:a",
+                    "aac",
+                    "-b:a",
+                    "160k",
+                    "-t",
+                    seconds.toFixed(3),
+                    "-movflags",
+                    "+faststart",
+                    mixed,
+                ]);
+                if (code === 0) {
+                    out = mixed;
+                }
+                else {
+                    errors.push(`The sound could not be mixed: ${stderr.slice(-200)}`);
+                }
+            }
+            return {
+                mp4: new Uint8Array(await readFile(out)),
+                png,
+                duration: seconds,
+                errors: [...new Set(errors)].slice(0, 10),
+            };
+        }
+        finally {
+            await context.close();
+        }
+    });
+}

package/apps/runtime/dist/widgets/runtime.js

@@ -0,0 +1,108 @@
+/**
+ * `window.wizard` — the whole contract between a widget and the platform. Injected ahead of the
+ * widget's own scripts; reads its payload from <script type="application/json" id="wizard-payload">.
+ *
+ *   wizard.data            the run's data (keys from the widget step's `data` map)
+ *   wizard.brand           { name, accent, logo, colors, facts, theme } — logo as a data URL or
+ *                          null, colors [{ name, value }], facts { key: value }, theme { accent,
+ *                          second, background, display, body, dark } as the run page uses it
+ *   wizard.state           what the person did last time (ticks, an open tab), kept with the run
+ *   wizard.save(state)     keeps `state` with the run (small JSON); the next visit reads it
+ *   wizard.share(text)     hands text to the device's share sheet (or copies it)
+ *   wizard.ai              "generated" | "edited" when the run's media in `data` was made or
+ *                          changed by a model, else null — a film labels itself with it
+ *   wizard.mode            "view" in the browser, "export" while a PNG/PDF/MP4 is rendered
+ *   wizard.file(path)      text of a workspace file (JSON, CSV, SVG, …)
+ *   wizard.json(path)      a workspace JSON file, parsed
+ *   wizard.url(path)       a workspace file as a data URL (images, fonts, audio)
+ *   wizard.ready()         call once the first frame is drawn; exports wait for it
+ *   wizard.timeline({ duration, seek, poster?, audio? })
+ *                          an animation of `duration` seconds; seek(t) must draw time t —
+ *                          synchronously, or as a promise that resolves once it is drawn (a
+ *                          film waits for its clips). Enables the MP4 export; `poster` is the
+ *                          second the PNG/PDF show (default 0). A film names its sound in
+ *                          `audio`: [{ src, start, duration?, from?, volume? }] — `from` is the
+ *                          second of the source a cut plays from.
+ */
+export const WIDGET_RUNTIME = `(function () {
+  var el = document.getElementById("wizard-payload");
+  var payload = el ? JSON.parse(el.textContent) : { data: {}, brand: {}, files: {}, base: "" };
+  var files = payload.files || {};
+  var resolveReady;
+  var readyPromise = new Promise(function (r) { resolveReady = r; });
+  var timeline = null;
+  function find(path) {
+    var p = String(path || "").replace(/^\\.?\\/+/, "");
+    return files[p] || files[payload.base + p] || null;
+  }
+  function utf8ToB64(text) {
+    var bytes = new TextEncoder().encode(text);
+    var bin = "";
+    for (var i = 0; i < bytes.length; i++) bin += String.fromCharCode(bytes[i]);
+    return btoa(bin);
+  }
+  function b64ToUtf8(b64) {
+    var bin = atob(b64);
+    var bytes = new Uint8Array(bin.length);
+    for (var i = 0; i < bin.length; i++) bytes[i] = bin.charCodeAt(i);
+    return new TextDecoder().decode(bytes);
+  }
+  window.wizard = {
+    data: payload.data || {},
+    brand: payload.brand || {},
+    ai: payload.ai || null,
+    mode: window.__WIZARD_EXPORT ? "export" : "view",
+    file: function (path) {
+      var f = find(path);
+      if (!f) return undefined;
+      return f.text != null ? f.text : b64ToUtf8(f.b64);
+    },
+    json: function (path) {
+      var t = this.file(path);
+      return t == null ? undefined : JSON.parse(t);
+    },
+    url: function (path) {
+      var f = find(path);
+      if (!f) return "";
+      return "data:" + f.mime + ";base64," + (f.b64 != null ? f.b64 : utf8ToB64(f.text));
+    },
+    state: window.__WIZARD_STATE || {},
+    save: function (state) {
+      this.state = state;
+      if (parent !== window) parent.postMessage({ wizard: "state", state: state }, "*");
+    },
+    share: function (text) {
+      if (parent !== window) parent.postMessage({ wizard: "share", text: String(text) }, "*");
+    },
+    ready: function () { resolveReady(); },
+    timeline: function (t) {
+      if (t && typeof t.seek === "function" && t.duration > 0) timeline = t;
+    },
+    __ready: readyPromise,
+    __timeline: function () { return timeline; }
+  };
+  // A sandboxed frame can start at 0×0 and get its real size later without a "resize" event;
+  // widgets that fit themselves to the window would stay at scale 0. Fire it for them.
+  var lastW = innerWidth, lastH = innerHeight;
+  addEventListener("resize", function () { lastW = innerWidth; lastH = innerHeight; });
+  // A widget laid out as part of the page tells the page how tall its content is.
+  var lastTall = 0;
+  function tell() {
+    var tall = Math.ceil(document.documentElement.scrollHeight);
+    if (parent !== window && tall !== lastTall) {
+      lastTall = tall;
+      parent.postMessage({ wizard: "height", height: tall }, "*");
+    }
+  }
+  addEventListener("load", tell);
+  if (window.ResizeObserver) {
+    new ResizeObserver(function () {
+      if (innerWidth !== lastW || innerHeight !== lastH) {
+        lastW = innerWidth;
+        lastH = innerHeight;
+        dispatchEvent(new Event("resize"));
+      }
+      tell();
+    }).observe(document.documentElement);
+  }
+})();`;

package/apps/runtime/dist/widgets/script.js

@@ -0,0 +1,30 @@
+import { offlineContext } from "../render/chromium.js";
+/**
+ * Runs a model-written script over workspace files inside Chromium with no network: reshaping
+ * downloaded data (simplify outlines, merge files, precompute paths) without passing it through
+ * the model. `files` maps paths to text; the script's return value is the result.
+ */
+export async function runScript(script, files, timeoutMs = 20_000) {
+    const context = await offlineContext();
+    try {
+        const page = await context.newPage();
+        await page.setContent("<!doctype html><html><body></body></html>");
+        const run = page.evaluate(async ({ script, files }) => {
+            const fn = new Function("files", `return (async () => {\n${script}\n})();`);
+            return await fn(files);
+        }, { script, files });
+        let timer;
+        const timeout = new Promise((_, reject) => {
+            timer = setTimeout(() => reject(new Error(`The script ran longer than ${timeoutMs / 1000} seconds.`)), timeoutMs);
+        });
+        try {
+            return await Promise.race([run, timeout]);
+        }
+        finally {
+            clearTimeout(timer);
+        }
+    }
+    finally {
+        await context.close();
+    }
+}