@engenty/wizards-runtime 0.0.0-stage → 0.7.5

package/apps/runtime/dist/render/downloads.js

@@ -0,0 +1,299 @@
+import { formatsFor, widgetSize } from "@engenty-wizards/shared/definition";
+import { listAsMarkdown } from "@engenty-wizards/shared/store";
+import { and, eq } from "drizzle-orm";
+import { zipSync } from "fflate";
+import { db, schema } from "../db/client.js";
+import { extFor, inlineAssetRefs, loadAsset, loadAssetText, saveAsset } from "../files/storage.js";
+import { widgetMp4, widgetPdf, widgetPng } from "../widgets/render.js";
+import { allTables, firstTable, htmlToDocx, htmlToMarkdown, htmlToPdf, htmlToPng, jsonToMarkdown, MIME, markdownToHtml, tablesToXlsx, tableToCsv, } from "./convert.js";
+import { guardHtml } from "./guard.js";
+import { shapedCsvRows, shapedMarkdown, shapedTables } from "./shaped.js";
+export function slug(s) {
+    return (s
+        .toLowerCase()
+        .normalize("NFKD")
+        .replace(/[̀-ͯ]/g, "")
+        .replace(/ß/g, "ss")
+        .replace(/[^a-z0-9]+/g, "-")
+        .replace(/^-|-$/g, "")
+        .slice(0, 60) || "result");
+}
+/**
+ * Renders of an asset (PDF, PNG, MP4 …) are kept next to it: a shared link opened a hundred
+ * times renders once. A regenerated output is a new asset, so its renders start fresh.
+ */
+async function cachedRender(source, format, produce) {
+    const name = `${source.id}.${format}`;
+    const hit = await db.query.asset.findFirst({
+        where: and(eq(schema.asset.kind, "render"), eq(schema.asset.name, name)),
+    });
+    if (hit) {
+        const found = await loadAsset(hit.id);
+        if (found) {
+            return new Uint8Array(found.data);
+        }
+    }
+    const data = await produce();
+    if (data) {
+        const row = await db.query.asset.findFirst({ where: eq(schema.asset.id, source.id) });
+        await saveAsset({
+            runId: row?.runId ?? null,
+            stepId: row?.stepId ?? null,
+            kind: "render",
+            mime: MIME[format],
+            name,
+            data,
+        });
+    }
+    return data;
+}
+/** The HTML a step produced, ready to show or save: images inlined, network and frames locked. */
+export async function stepHtml(output) {
+    const asset = output.assets?.find((a) => a.mime === "text/html");
+    if (!asset) {
+        return null;
+    }
+    return guardHtml(await inlineAssetRefs(await loadAssetText(asset.id)));
+}
+/** One step's output in one format. Conversions happen on demand. */
+export async function renderDownload(step, output, format, baseName) {
+    if (!formatsFor(step).includes(format)) {
+        return null;
+    }
+    const name = slug(baseName);
+    const file = (ext) => `${name}.${ext}`;
+    if ((step.type === "widget" && step.video) || step.type === "film") {
+        // A film was rendered when its step ran: the video and its poster are the files.
+        const asset = output.assets?.find((a) => a.kind === (format === "mp4" ? "video" : "poster"));
+        const found = asset ? await loadAsset(asset.id) : null;
+        return found
+            ? { data: new Uint8Array(found.data), mime: found.row.mime, filename: file(format) }
+            : null;
+    }
+    if (step.type === "surface") {
+        return {
+            data: JSON.stringify(output.surface ?? {}, null, 2),
+            mime: MIME.json,
+            filename: file("json"),
+        };
+    }
+    if (step.type === "widget") {
+        const source = output.assets?.find((a) => a.mime === "text/html");
+        if (format === "json") {
+            return {
+                data: JSON.stringify(output.json ?? {}, null, 2),
+                mime: MIME.json,
+                filename: file("json"),
+            };
+        }
+        if (!source) {
+            return null;
+        }
+        const html = await loadAssetText(source.id);
+        const size = widgetSize(step);
+        const data = format === "html"
+            ? html
+            : await cachedRender(source, format, async () => {
+                // Pictures the widget took from the web are assets: the renderer gets them inline.
+                const inlined = await inlineAssetRefs(html);
+                switch (format) {
+                    case "png":
+                        return widgetPng(inlined, size);
+                    case "pdf":
+                        return widgetPdf(inlined, size);
+                    case "mp4":
+                        return output.widget?.duration ? widgetMp4(inlined, size) : null;
+                    default:
+                        return null;
+                }
+            });
+        return data ? { data, mime: MIME[format], filename: file(format) } : null;
+    }
+    if (step.type === "generate") {
+        const asset = output.assets?.[0];
+        if (!asset) {
+            return null;
+        }
+        if (format === "zip") {
+            const zip = await zipAssets(output.assets ?? []);
+            return zip ? { data: zip, mime: MIME.zip, filename: file("zip") } : null;
+        }
+        if (step.asset === "image" || step.asset === "video" || step.asset === "voice") {
+            const found = await loadAsset(asset.id);
+            return found
+                ? {
+                    data: new Uint8Array(found.data),
+                    mime: found.row.mime,
+                    filename: file(extFor(found.row.mime)),
+                }
+                : null;
+        }
+        const html = await inlineAssetRefs(await loadAssetText(asset.id));
+        const render = (produce) => cachedRender(asset, format, produce);
+        switch (format) {
+            case "html":
+                return { data: guardHtml(html), mime: MIME.html, filename: file("html") };
+            case "pdf":
+                return { data: await render(() => htmlToPdf(html)), mime: MIME.pdf, filename: file("pdf") };
+            case "png":
+                return { data: await render(() => htmlToPng(html)), mime: MIME.png, filename: file("png") };
+            case "docx":
+                return {
+                    data: await render(() => htmlToDocx(html)),
+                    mime: MIME.docx,
+                    filename: file("docx"),
+                };
+            case "md":
+                return { data: htmlToMarkdown(html), mime: MIME.md, filename: file("md") };
+            default:
+                return null;
+        }
+    }
+    if (step.type === "agent") {
+        if (format === "zip") {
+            const zip = await zipAssets(output.assets?.filter((a) => a.kind === "file") ?? []);
+            return zip ? { data: zip, mime: MIME.zip, filename: file("zip") } : null;
+        }
+        const text = output.text ?? "";
+        if (step.output.format === "json") {
+            // A step with a schema: its lists by their labels, the tree as headings.
+            const shape = step.output.schema;
+            const data = (output.json ?? {});
+            const rows = shape ? shapedCsvRows(shape, data) : firstTable(output.json);
+            const tables = shape ? shapedTables(shape, data) : allTables(output.json);
+            const md = shape
+                ? `# ${baseName}\n\n${shapedMarkdown(shape, data)}`
+                : jsonAsMarkdown(output.json);
+            switch (format) {
+                case "json":
+                    return {
+                        data: JSON.stringify(output.json, null, 2),
+                        mime: MIME.json,
+                        filename: file("json"),
+                    };
+                case "csv":
+                    return rows ? { data: tableToCsv(rows), mime: MIME.csv, filename: file("csv") } : null;
+                case "xlsx":
+                    return tables.length
+                        ? { data: await tablesToXlsx(tables), mime: MIME.xlsx, filename: file("xlsx") }
+                        : null;
+                case "md":
+                    return { data: md, mime: MIME.md, filename: file("md") };
+                case "pdf":
+                    return {
+                        data: await htmlToPdf(markdownToHtml(md, baseName)),
+                        mime: MIME.pdf,
+                        filename: file("pdf"),
+                    };
+                default:
+                    return null;
+            }
+        }
+        switch (format) {
+            case "md":
+                return { data: text, mime: MIME.md, filename: file("md") };
+            case "txt":
+                return { data: text, mime: MIME.txt, filename: file("txt") };
+            case "html":
+                return { data: markdownToHtml(text, baseName), mime: MIME.html, filename: file("html") };
+            case "pdf":
+                return {
+                    data: await htmlToPdf(markdownToHtml(text, baseName)),
+                    mime: MIME.pdf,
+                    filename: file("pdf"),
+                };
+            case "docx":
+                return {
+                    data: await htmlToDocx(markdownToHtml(text, baseName)),
+                    mime: MIME.docx,
+                    filename: file("docx"),
+                };
+            default:
+                return null;
+        }
+    }
+    return null;
+}
+/** A structured result as readable Markdown: text fields as paragraphs, tables as tables. */
+/** The files a step collected, as one archive. Two files of the same name both stay. */
+async function zipAssets(assets) {
+    const entries = {};
+    for (const asset of assets) {
+        const found = await loadAsset(asset.id);
+        if (!found) {
+            continue;
+        }
+        let name = asset.name;
+        for (let n = 2; name in entries; n++) {
+            name = asset.name.replace(/(\.[^.]+)?$/, (ext) => `-${n}${ext}`);
+        }
+        entries[name] = new Uint8Array(found.data);
+    }
+    return Object.keys(entries).length ? zipSync(entries, { level: 6 }) : null;
+}
+/**
+ * A stored list in one format. Column labels head the spreadsheet; ids stay the JSON keys.
+ * `files` reads a kept file by its path — a zip holds the files the rows name, and the list.
+ */
+export async function listDownload(def, rows, format, baseName, files) {
+    const name = slug(baseName);
+    if (format === "zip") {
+        const column = def.check?.file;
+        if (!column || !files) {
+            return null;
+        }
+        const entries = {};
+        for (const row of rows) {
+            const path = String(row.cells[column] ?? "").trim();
+            const data = path && !(path in entries) ? await files(path) : null;
+            if (data) {
+                entries[path] = data;
+            }
+        }
+        const sheet = await listDownload(def, rows, "xlsx", baseName);
+        if (sheet) {
+            entries[sheet.filename] = sheet.data;
+        }
+        return { data: zipSync(entries, { level: 6 }), mime: MIME.zip, filename: `${name}.zip` };
+    }
+    const cells = rows.map((r) => r.cells);
+    const headers = Object.fromEntries(def.columns.map((c) => [c.id, c.name]));
+    const labelled = cells.map((row) => Object.fromEntries(def.columns.map((c) => [c.name, row[c.id] ?? null])));
+    switch (format) {
+        case "xlsx":
+            return {
+                data: await tablesToXlsx([{ name: def.title, rows: cells, headers }]),
+                mime: MIME.xlsx,
+                filename: `${name}.xlsx`,
+            };
+        case "csv":
+            return { data: tableToCsv(labelled), mime: MIME.csv, filename: `${name}.csv` };
+        case "json":
+            return { data: JSON.stringify(cells, null, 2), mime: MIME.json, filename: `${name}.json` };
+        case "md":
+            return {
+                data: `# ${def.title}\n\n${listAsMarkdown(def, rows, 5000)}`,
+                mime: MIME.md,
+                filename: `${name}.md`,
+            };
+        default:
+            return null;
+    }
+}
+export function jsonAsMarkdown(value) {
+    if (!value || typeof value !== "object" || Array.isArray(value)) {
+        return jsonToMarkdown(value);
+    }
+    return Object.entries(value)
+        .map(([key, v]) => {
+        const heading = `### ${key}`;
+        if (typeof v === "string" || typeof v === "number") {
+            return `${heading}\n${v}`;
+        }
+        if (Array.isArray(v) && v.every((x) => typeof x !== "object")) {
+            return `${heading}\n${v.map((x) => `- ${x}`).join("\n")}`;
+        }
+        return `${heading}\n${jsonToMarkdown(v)}`;
+    })
+        .join("\n\n");
+}

package/apps/runtime/dist/render/guard.js

@@ -0,0 +1,25 @@
+/**
+ * Every generated HTML document (documents, dashboards, widgets) gets this before anyone sees
+ * it: scripts may run, but the page can load nothing and send nothing — everything it shows is
+ * inlined. Links open in a new tab instead of inside the preview frame.
+ */
+export const HTML_CSP = "default-src 'none'; script-src 'unsafe-inline' 'unsafe-eval'; style-src 'unsafe-inline'; img-src data: blob:; media-src data: blob:; font-src data:; connect-src data: blob:; worker-src blob:";
+/** The response header for a generated document opened on its own: an opaque origin, scripts allowed. */
+export const HTML_RESPONSE_CSP = `sandbox allow-scripts allow-popups allow-popups-to-escape-sandbox; ${HTML_CSP}`;
+/**
+ * `media`: one more origin images, video and audio may come from — a film's page gets this
+ * run's clips from the renderer that way; they are too large to inline.
+ */
+export function guardHtml(html, media) {
+    const csp = media
+        ? HTML_CSP.replace("img-src", `img-src ${media}`).replace("media-src", `media-src ${media}`)
+        : HTML_CSP;
+    const head = `<meta http-equiv="Content-Security-Policy" content="${csp}"><base target="_blank">`;
+    if (/<head[^>]*>/i.test(html)) {
+        return html.replace(/<head[^>]*>/i, (m) => `${m}${head}`);
+    }
+    if (/<html[^>]*>/i.test(html)) {
+        return html.replace(/<html[^>]*>/i, (m) => `${m}<head>${head}</head>`);
+    }
+    return `<!doctype html><html><head>${head}</head><body>${html}</body></html>`;
+}

package/apps/runtime/dist/render/shaped.js

@@ -0,0 +1,137 @@
+import { labelOf, titleKey, } from "@engenty-wizards/shared/shape";
+const isEntry = (v) => Boolean(v) && typeof v === "object" && !Array.isArray(v);
+/** A value that fits in a cell or a line: no lists of entries, no objects. */
+const flat = (s) => s.kind !== "object" && s.kind !== "table" && !(s.kind === "list" && s.of.kind === "object");
+/** An entry's title: its first short text. */
+function titleOf(shape, entry) {
+    const key = titleKey(shape);
+    return key ? String(entry[key] ?? "") : "";
+}
+/** A flat value as words. */
+function words(s, value, root, data) {
+    if (value === null || value === undefined) {
+        return "";
+    }
+    switch (s.kind) {
+        case "yesno":
+            return value ? (s.yes ?? "✓") : (s.no ?? "–");
+        case "number":
+            return `${value}${s.unit ? ` ${s.unit}` : ""}`;
+        case "ref": {
+            const target = root.fields[s.to];
+            const list = Array.isArray(data[s.to]) ? data[s.to] : [];
+            const hit = list.find((e) => isEntry(e) && e.id === value);
+            return target?.kind === "list" && target.of.kind === "object" && isEntry(hit)
+                ? titleOf(target.of, hit) || String(value)
+                : String(value);
+        }
+        case "list":
+            return Array.isArray(value) ? value.map((v) => words(s.of, v, root, data)).join(", ") : "";
+        default:
+            return typeof value === "object" ? JSON.stringify(value) : String(value);
+    }
+}
+/** One row of a sheet: flat values by label; a list of entries as their titles. */
+function rowOf(shape, entry, root, data) {
+    const row = {};
+    for (const [k, f] of Object.entries(shape.fields)) {
+        const v = entry[k];
+        if (f.kind === "list" && f.of.kind === "object") {
+            const of = f.of;
+            row[labelOf(k, f)] = Array.isArray(v)
+                ? v
+                    .map((e) => (isEntry(e) ? titleOf(of, e) : ""))
+                    .filter(Boolean)
+                    .join("; ")
+                : "";
+        }
+        else if (f.kind === "number" && typeof v === "number") {
+            row[labelOf(k, f)] = v;
+        }
+        else if (flat(f)) {
+            row[labelOf(k, f)] = words(f, v, root, data);
+        }
+    }
+    return row;
+}
+/** One sheet per top-level list, named by its label. */
+export function shapedTables(shape, data) {
+    const out = [];
+    for (const [k, f] of Object.entries(shape.fields)) {
+        const value = Array.isArray(data[k]) ? data[k] : [];
+        if (f.kind !== "list" || !value.length) {
+            continue;
+        }
+        const of = f.of;
+        const rows = of.kind === "object"
+            ? value.filter(isEntry).map((e) => rowOf(of, e, shape, data))
+            : value.map((v) => ({ [labelOf(k, f)]: words(of, v, shape, data) }));
+        out.push({ name: labelOf(k, f), rows });
+    }
+    return out;
+}
+/** The rows of the first top-level list whose entries are flat: what a CSV holds. */
+export function shapedCsvRows(shape, data) {
+    for (const [k, f] of Object.entries(shape.fields)) {
+        if (f.kind === "list" &&
+            f.of.kind === "object" &&
+            Object.values(f.of.fields).every(flat) &&
+            Array.isArray(data[k])) {
+            const of = f.of;
+            return data[k].filter(isEntry).map((e) => rowOf(of, e, shape, data));
+        }
+    }
+    return null;
+}
+const cellText = (v) => String(v ?? "")
+    .replace(/\|/g, "\\|")
+    .replace(/\n/g, " ");
+/** The tree as Markdown: a heading per part and per entry with lists of its own, tables for flat entries. */
+export function shapedMarkdown(shape, data) {
+    return objectMarkdown(shape, data, 2, shape, data);
+}
+const h = (n) => "#".repeat(Math.min(6, n));
+function objectMarkdown(shape, value, depth, root, data) {
+    const parts = [];
+    const short = [];
+    // An entry under its title as a heading says neither the title nor its id again.
+    const skip = shape === root ? [] : ["id", titleKey(shape)];
+    for (const [k, f] of Object.entries(shape.fields)) {
+        const v = value[k];
+        const label = labelOf(k, f);
+        if (v === undefined || v === null || v === "" || skip.includes(k)) {
+            continue;
+        }
+        if (f.kind === "text" && f.long) {
+            parts.push(`${h(depth)} ${label}\n\n${String(v)}`);
+        }
+        else if (f.kind === "list" && Array.isArray(v)) {
+            parts.push(`${h(depth)} ${label}\n\n${listMarkdown(f, v, depth + 1, root, data)}`);
+        }
+        else if (flat(f)) {
+            short.push(`**${label}:** ${words(f, v, root, data)}`);
+        }
+    }
+    return [short.join("  \n"), ...parts].filter(Boolean).join("\n\n");
+}
+function listMarkdown(f, value, depth, root, data) {
+    const of = f.of;
+    if (of.kind !== "object") {
+        return value.map((v) => `- ${words(of, v, root, data)}`).join("\n");
+    }
+    const entries = value.filter(isEntry);
+    if (Object.values(of.fields).every(flat)) {
+        const cols = Object.entries(of.fields);
+        return [
+            `| ${cols.map(([c, s]) => cellText(labelOf(c, s))).join(" | ")} |`,
+            `| ${cols.map(() => "---").join(" | ")} |`,
+            ...entries.map((e) => `| ${cols.map(([c, s]) => cellText(words(s, e[c], root, data))).join(" | ")} |`),
+        ].join("\n");
+    }
+    return entries
+        .map((e, i) => {
+        const title = titleOf(of, e) || `${of.label ?? f.label ?? ""} ${i + 1}`.trim();
+        return `${h(depth)} ${title}\n\n${objectMarkdown(of, e, depth + 1, root, data)}`;
+    })
+        .join("\n\n");
+}