@engenty/wizards-runtime 0.0.0-stage → 0.7.5

package/apps/runtime/dist/features.js

@@ -0,0 +1,123 @@
+import { mkdirSync, readFileSync, statSync, writeFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { declaredByPlugin, defaultFeatures, FEATURES, featureOn, parseFeatures, resolveFeatures, } from "@engenty-wizards/shared/features";
+import { accountOverview } from "./auth/account.js";
+import { isCheckout, layout, packageRoot } from "./cli/home.js";
+import { env } from "./env.js";
+import { managed, tenantInfo } from "./manage.js";
+import { loadedPlugins } from "./plugins/registry.js";
+import { currentTenantOrNull } from "./tenants/tenant.js";
+/**
+ * Which feature switches are on for this runtime (packages/shared/src/features.ts), laid over
+ * each other, the last winning:
+ *
+ *   1. the defaults the code declares: the core's (packages/shared/src/feature-flags.ts) and the
+ *      loaded plugins' (their manifests' `features`, under the plugin's id)
+ *   2. the `FEATURES` environment variable (`teamPlans=on,site=off`): the cloud image, compose, dev
+ *   3. what the Manage-App says — for the team of a cloud runtime, for the linked account of a
+ *      local one; a plan's switches and what an admin set for the team or for everyone
+ *   4. the `.features` file of this install (`~/.engenty/wizards/.features`; a checkout reads
+ *      the repo root's, a cloud runtime its data folder's) — this machine's word is the last
+ *
+ * The file is read on every call, so a change counts at once. Nothing here is documented
+ * outside the code.
+ */
+/** The `.features` file this runtime reads: the install's, the checkout's root, or the data folder's. */
+export function featuresFile() {
+    if (managed) {
+        return join(env.dataDir, ".features");
+    }
+    return isCheckout() ? join(packageRoot, ".features") : join(layout().home, ".features");
+}
+let fileCache = null;
+/** What the `.features` file says, read again when it changed. */
+export function fileFeatures(file = featuresFile()) {
+    let mtime = -1;
+    try {
+        mtime = statSync(file).mtimeMs;
+    }
+    catch {
+        fileCache = null;
+        return {};
+    }
+    if (fileCache && fileCache.file === file && fileCache.mtime === mtime) {
+        return fileCache.value;
+    }
+    const value = parseFeatures(readFileSync(file, "utf8"));
+    fileCache = { file, mtime, value };
+    return value;
+}
+const envFeatures = () => parseFeatures(process.env.FEATURES ?? "");
+/** Every switch something declared: the core's, then each plugin's under its id. */
+export function declaredFeatures(sources = loadedPlugins().map((p) => p.source)) {
+    const out = { ...FEATURES };
+    for (const source of sources) {
+        Object.assign(out, declaredByPlugin(source.id, source.manifest.features));
+    }
+    return out;
+}
+/** The switches, with what the Manage-App said for the team or account laid in. */
+export function features(remote) {
+    return resolveFeatures(defaultFeatures(declaredFeatures()), envFeatures(), remote, fileFeatures());
+}
+export function feature(name, remote) {
+    return featureOn(features(remote), name);
+}
+let accountCache = null;
+/**
+ * What the Manage-App says for the tenant at hand: the team's switches on a cloud runtime
+ * (cached a minute by `tenantInfo`), the linked account's on a local one (cached a minute
+ * here). Null when nothing is linked or the Manage-App does not answer.
+ */
+export async function remoteFeatures() {
+    if (managed) {
+        const id = currentTenantOrNull();
+        if (!id) {
+            return null;
+        }
+        return tenantInfo(id)
+            .then((info) => info.features ?? null)
+            .catch(() => null);
+    }
+    if (accountCache && Date.now() - accountCache.at < 60_000) {
+        return accountCache.value;
+    }
+    const overview = await accountOverview().catch(() => null);
+    accountCache = { at: Date.now(), value: overview?.tenant.features ?? null };
+    return accountCache.value;
+}
+/** The switches for the tenant at hand, with what the Manage-App says fetched. */
+export async function currentFeatures() {
+    return features(await remoteFeatures());
+}
+/**
+ * Writes one switch into the `.features` file as `name=on` or `name=off`, on the line that
+ * names it when there is one (commented out or not), else at the end; `null` turns the line
+ * back into the template's `# name=on|off`. Other lines and comments stay as they are. For the
+ * `wizards features` command.
+ */
+export function writeFeature(name, on, file = featuresFile()) {
+    let lines = [];
+    try {
+        lines = readFileSync(file, "utf8").replace(/\n+$/, "").split("\n");
+    }
+    catch {
+        lines = [];
+    }
+    const names = (line) => {
+        const m = /^\s*#?\s*([A-Za-z0-9.]+)\s*=\s*\S+\s*(?:#.*)?$/.exec(line);
+        return m?.[1] === name;
+    };
+    const written = on === null ? `# ${name}=on|off` : `${name}=${on ? "on" : "off"}`;
+    const at = lines.findIndex(names);
+    if (at === -1) {
+        lines.push(written);
+    }
+    else {
+        lines[at] = written;
+    }
+    mkdirSync(dirname(file), { recursive: true });
+    writeFileSync(file, `${lines.join("\n")}\n`);
+    fileCache = null;
+    return fileFeatures(file);
+}

package/apps/runtime/dist/files/blobs.js

@@ -0,0 +1,22 @@
+import { createHash } from "node:crypto";
+import { objects } from "./objects.js";
+/** Content-addressed file store for workspaces: a blob never changes, so snapshots are just lists of hashes. */
+const blobKey = (hash) => `blobs/${hash.slice(0, 2)}/${hash}`;
+export async function putBlob(data) {
+    const hash = createHash("sha256").update(data).digest("hex");
+    const key = blobKey(hash);
+    if (!(await objects.has(key))) {
+        await objects.put(key, data);
+    }
+    return hash;
+}
+export async function getBlob(hash) {
+    if (!/^[a-f0-9]{64}$/.test(hash)) {
+        throw new Error("bad blob hash");
+    }
+    const data = await objects.get(blobKey(hash));
+    if (!data) {
+        throw new Error("blob missing");
+    }
+    return data;
+}

package/apps/runtime/dist/files/objects.js

@@ -0,0 +1,86 @@
+import { randomBytes } from "node:crypto";
+import { existsSync } from "node:fs";
+import { mkdir, readFile, rename, rm, writeFile } from "node:fs/promises";
+import { dirname, join } from "node:path";
+import { AwsClient } from "aws4fetch";
+import { env } from "../env.js";
+import { currentTenant } from "../tenants/tenant.js";
+function fileStore(root) {
+    const at = (key) => join(root, key);
+    return {
+        async put(key, data) {
+            const path = at(key);
+            await mkdir(dirname(path), { recursive: true });
+            // Two writers of the same key (the same blob, at the same moment) each need their own.
+            const tmp = `${path}.${process.pid}.${randomBytes(6).toString("hex")}.tmp`;
+            await writeFile(tmp, data);
+            await rename(tmp, path);
+        },
+        get: (key) => readFile(at(key)).catch(() => null),
+        has: async (key) => existsSync(at(key)),
+        remove: (key) => rm(at(key), { force: true }),
+        removePrefix: (prefix) => rm(at(prefix), { recursive: true, force: true }),
+    };
+}
+function s3Store() {
+    const aws = new AwsClient({
+        accessKeyId: env.s3.accessKeyId,
+        secretAccessKey: env.s3.secretAccessKey,
+        region: env.s3.region,
+        service: "s3",
+    });
+    const base = `${env.s3.endpoint}/${env.s3.bucket}`;
+    const url = (key) => `${base}/${key.split("/").map(encodeURIComponent).join("/")}`;
+    const check = (res, what) => {
+        if (!res.ok) {
+            throw new Error(`Object store ${what} failed: ${res.status}`);
+        }
+    };
+    return {
+        async put(key, data) {
+            check(await aws.fetch(url(key), { method: "PUT", body: data }), "put");
+        },
+        async get(key) {
+            const res = await aws.fetch(url(key));
+            if (res.status === 404) {
+                return null;
+            }
+            check(res, "get");
+            return Buffer.from(await res.arrayBuffer());
+        },
+        async has(key) {
+            return (await aws.fetch(url(key), { method: "HEAD" })).ok;
+        },
+        async remove(key) {
+            check(await aws.fetch(url(key), { method: "DELETE" }), "delete");
+        },
+        async removePrefix(prefix) {
+            for (let token = ""; token !== null;) {
+                const list = new URL(base);
+                list.searchParams.set("list-type", "2");
+                list.searchParams.set("prefix", prefix);
+                if (token) {
+                    list.searchParams.set("continuation-token", token);
+                }
+                const res = await aws.fetch(list.toString());
+                check(res, "list");
+                const xml = await res.text();
+                const keys = [...xml.matchAll(/<Key>([^<]+)<\/Key>/g)].map((m) => m[1]);
+                await Promise.all(keys.map((key) => aws.fetch(url(key), { method: "DELETE" })));
+                token = xml.match(/<NextContinuationToken>([^<]+)</)?.[1] ?? null;
+            }
+        },
+    };
+}
+const store = env.s3.endpoint && env.s3.bucket ? s3Store() : fileStore(join(env.dataDir, "objects"));
+const tenantKey = (key) => `t/${currentTenant()}/${key}`;
+export const objects = {
+    put: (key, data) => store.put(tenantKey(key), data),
+    get: (key) => store.get(tenantKey(key)),
+    has: (key) => store.has(tenantKey(key)),
+    remove: (key) => store.remove(tenantKey(key)),
+};
+/** Deletes every file of a tenant. */
+export function removeTenantObjects(tenantId) {
+    return store.removePrefix(`t/${tenantId}/`);
+}

package/apps/runtime/dist/files/storage.js

@@ -0,0 +1,120 @@
+import { rm } from "node:fs/promises";
+import { join } from "node:path";
+import { eq } from "drizzle-orm";
+import { nanoid } from "nanoid";
+import { db, schema } from "../db/client.js";
+import { env } from "../env.js";
+import { currentTenant } from "../tenants/tenant.js";
+import { objects } from "./objects.js";
+const assetKey = (file) => `assets/${file}`;
+const EXT = {
+    "image/png": "png",
+    "image/jpeg": "jpg",
+    "image/webp": "webp",
+    "image/gif": "gif",
+    "image/avif": "avif",
+    "image/svg+xml": "svg",
+    "video/mp4": "mp4",
+    "video/webm": "webm",
+    "video/quicktime": "mov",
+    "audio/webm": "weba",
+    "audio/mp4": "m4a",
+    "audio/mpeg": "mp3",
+    "audio/ogg": "ogg",
+    "audio/wav": "wav",
+    "text/html": "html",
+    "text/markdown": "md",
+    "text/plain": "txt",
+    "application/json": "json",
+    "application/pdf": "pdf",
+};
+export function extFor(mime) {
+    return EXT[mime] ?? "bin";
+}
+export async function saveAsset(input) {
+    const id = nanoid(16);
+    const bytes = typeof input.data === "string" ? Buffer.from(input.data, "utf8") : Buffer.from(input.data);
+    const file = `${id}.${extFor(input.mime)}`;
+    await objects.put(assetKey(file), bytes);
+    await db.insert(schema.asset).values({
+        id,
+        tenantId: currentTenant(),
+        runId: input.runId ?? null,
+        stepId: input.stepId ?? null,
+        kind: input.kind,
+        mime: input.mime,
+        name: input.name,
+        path: file,
+        size: bytes.byteLength,
+    });
+    return {
+        id,
+        kind: input.kind,
+        mime: input.mime,
+        name: input.name,
+        ...(input.ai ? { ai: input.ai } : {}),
+    };
+}
+export async function loadAsset(id) {
+    const row = await db.query.asset.findFirst({ where: eq(schema.asset.id, id) });
+    if (!row) {
+        return null;
+    }
+    const data = await objects.get(assetKey(row.path));
+    return data ? { row, data } : null;
+}
+export async function loadAssetText(id) {
+    const found = await loadAsset(id);
+    return found ? found.data.toString("utf8") : "";
+}
+/**
+ * Generated HTML points at images as `asset://ID`; previews and downloads get them
+ * inlined as data URIs, so every file stands on its own. Only the tenant's assets resolve.
+ */
+export async function inlineAssetRefs(html) {
+    const ids = [...new Set([...html.matchAll(/asset:\/\/([A-Za-z0-9_-]{6,40})/g)].map((m) => m[1]))];
+    let out = html;
+    for (const id of ids) {
+        const found = await loadAsset(id);
+        const uri = found?.row.mime.startsWith("image/")
+            ? `data:${found.row.mime};base64,${found.data.toString("base64")}`
+            : "";
+        out = out.split(`asset://${id}`).join(uri);
+    }
+    return out;
+}
+/** Gives an asset new content under the id it has: what points to it keeps pointing to it. */
+export async function replaceAsset(id, input) {
+    const row = await db.query.asset.findFirst({ where: eq(schema.asset.id, id) });
+    if (!row) {
+        return;
+    }
+    const bytes = Buffer.from(input.data);
+    const file = `${id}-${nanoid(6)}.${extFor(input.mime)}`;
+    await objects.put(assetKey(file), bytes);
+    await db
+        .update(schema.asset)
+        .set({ mime: input.mime, name: input.name, path: file, size: bytes.byteLength })
+        .where(eq(schema.asset.id, id));
+    await objects.remove(assetKey(row.path));
+}
+/** Deletes one asset: its file and its row. */
+export async function removeAsset(id) {
+    const row = await db.query.asset.findFirst({ where: eq(schema.asset.id, id) });
+    if (row) {
+        await objects.remove(assetKey(row.path));
+        await db.delete(schema.asset).where(eq(schema.asset.id, id));
+    }
+}
+/** Deletes every file a run made (uploads, results, renders, film folders) and their rows. */
+export async function removeAssetFiles(runId) {
+    const rows = await db.query.asset.findMany({ where: eq(schema.asset.runId, runId) });
+    for (const row of rows) {
+        await objects.remove(assetKey(row.path));
+    }
+    await db.delete(schema.asset).where(eq(schema.asset.runId, runId));
+    // The folders its films were made in (../film/project.ts).
+    if (/^[A-Za-z0-9_-]+$/.test(runId)) {
+        await rm(join(env.dataDir, "films", runId), { recursive: true, force: true });
+    }
+}

package/apps/runtime/dist/files/web-images.js

@@ -0,0 +1,101 @@
+import { safeFetch } from "../tools/net-guard.js";
+/** Pictures one step may take from the web, and how large one may be. */
+const MAX_IMAGES = 40;
+const MAX_BYTES = 8_000_000;
+const TIMEOUT_MS = 10_000;
+const PARALLEL = 4;
+const IMAGE_EXT = /\.(?:jpe?g|png|webp|gif|avif|svg)(?:[?#]|$)/i;
+/** A key whose value is likely a picture: the extension of its URL may be hidden behind a token. */
+const IMAGE_KEY = /image|photo|picture|thumb|cover|logo|avatar|img|poster|banner/i;
+/** Fetches one picture; null when it is no image, too large, unreachable or private. */
+export async function fetchWebImage(url, signal) {
+    const timeout = AbortSignal.timeout(TIMEOUT_MS);
+    try {
+        const res = await safeFetch(url, {
+            signal: signal ? AbortSignal.any([signal, timeout]) : timeout,
+            headers: { accept: "image/*" },
+        });
+        const mime = (res.headers.get("content-type") ?? "").split(";")[0].trim().toLowerCase();
+        const length = Number(res.headers.get("content-length") ?? 0);
+        if (!res.ok || !mime.startsWith("image/") || length > MAX_BYTES) {
+            await res.body?.cancel();
+            return null;
+        }
+        const data = Buffer.from(await res.arrayBuffer());
+        return data.byteLength && data.byteLength <= MAX_BYTES ? { mime, data } : null;
+    }
+    catch {
+        return null;
+    }
+}
+/** Takes the pictures behind `urls` into the run, a few at a time. */
+async function snapshot(urls, save, stepId, signal) {
+    const refs = new Map();
+    const list = [...new Set(urls)].slice(0, MAX_IMAGES);
+    let next = 0;
+    await Promise.all(Array.from({ length: Math.min(PARALLEL, list.length) }, async () => {
+        while (next < list.length && !signal?.aborted) {
+            const url = list[next++];
+            const image = await fetchWebImage(url, signal);
+            if (!image) {
+                continue;
+            }
+            const ref = await save({
+                stepId,
+                kind: "web-image",
+                mime: image.mime,
+                name: new URL(url).pathname.split("/").pop()?.slice(0, 120) || "image",
+                data: image.data,
+            });
+            refs.set(url, `asset://${ref.id}`);
+        }
+    }));
+    return { refs };
+}
+const IMG_SRC = /(<img\b[^>]*?\ssrc\s*=\s*)(["'])(https?:\/\/[^"']+)\2/gi;
+/** The pictures a generated page loads from the web, kept as assets; its <img> tags then point at them. */
+export async function snapshotHtmlImages(html, save, stepId, signal) {
+    const urls = [...html.matchAll(IMG_SRC)].map((m) => m[3]);
+    if (!urls.length) {
+        return html;
+    }
+    const { refs } = await snapshot(urls, save, stepId, signal);
+    return html.replace(IMG_SRC, (m, head, q, url) => {
+        const ref = refs.get(url);
+        return ref ? `${head}${q}${ref}${q}` : m;
+    });
+}
+function isImageUrl(value, key) {
+    if (!/^https?:\/\/\S+$/.test(value) || value.length > 2000) {
+        return false;
+    }
+    return IMAGE_EXT.test(new URL(value).pathname + new URL(value).search) || IMAGE_KEY.test(key);
+}
+function walk(value, key, visit) {
+    if (typeof value === "string") {
+        return isImageUrl(value, key) ? visit(value, key) : value;
+    }
+    if (Array.isArray(value)) {
+        return value.map((v) => walk(v, key, visit));
+    }
+    if (value && typeof value === "object") {
+        return Object.fromEntries(Object.entries(value).map(([k, v]) => [k, walk(v, k, visit)]));
+    }
+    return value;
+}
+/**
+ * The pictures a widget's data points to on the web (a listing's photos, a product's cover),
+ * kept as assets; the data then carries asset://ID where the page expects the URL.
+ */
+export async function snapshotDataImages(data, save, stepId, signal) {
+    const urls = [];
+    walk(data, "", (url) => {
+        urls.push(url);
+        return url;
+    });
+    if (!urls.length) {
+        return data;
+    }
+    const { refs } = await snapshot(urls, save, stepId, signal);
+    return walk(data, "", (url) => refs.get(url) ?? url);
+}