@johnhenry/sitefile 0.0.0-stage → 0.0.0

package/lib/src/plugins/cache.js

@@ -0,0 +1,226 @@
+import { createHash } from 'node:crypto';
+import { mkdir, readFile, readdir, rename, rm, stat, utimes, writeFile } from 'node:fs/promises';
+import path from 'node:path';
+import { sha256 } from '../digest.js';
+/**
+ * Content-addressed cache of TRANSFORM results (see docs/PLUGINS.md, "Caching").
+ *
+ *   <dir>/entries/<key>.json   manifest: which blobs make up the result
+ *   <dir>/blobs/<hex>          result bytes, named by their SHA-256
+ *
+ * The key is the SHA-256 of a canonical JSON document describing everything the
+ * result can depend on. Reads verify every blob's digest and size; a damaged
+ * entry is removed and counts as a miss, so the next put rewrites it.
+ */
+/** Bump to invalidate every existing entry when the key or entry format changes. */
+export const CACHE_VERSION = 1;
+/** JSON with sorted object keys, so equal values always serialize equally. */
+export function canonicalJson(value) {
+    const norm = (v) => {
+        if (v === null || typeof v !== 'object')
+            return typeof v === 'bigint' ? v.toString() : v;
+        if (v instanceof Date)
+            return v.toISOString();
+        if (Array.isArray(v))
+            return v.map(norm);
+        const out = {};
+        for (const k of Object.keys(v).sort()) {
+            const x = v[k];
+            if (x !== undefined)
+                out[k] = norm(x);
+        }
+        return out;
+    };
+    return JSON.stringify(norm(value)) ?? 'null';
+}
+export function cacheKey(parts) {
+    return createHash('sha256').update(canonicalJson({ cacheVersion: CACHE_VERSION, ...parts })).digest('hex');
+}
+const HEX64 = /^[0-9a-f]{64}$/;
+export class TransformCache {
+    dir;
+    remote;
+    stats = { hits: 0, misses: 0, writes: 0, corrupt: 0 };
+    constructor(dir, remote) {
+        this.dir = dir;
+        this.remote = remote;
+    }
+    entryPath(key) {
+        return path.join(this.dir, 'entries', `${key}.json`);
+    }
+    blobPath(hex) {
+        return path.join(this.dir, 'blobs', hex);
+    }
+    async get(key) {
+        const file = this.entryPath(key);
+        let raw;
+        try {
+            raw = await readFile(file, 'utf8');
+        }
+        catch {
+            const fetched = await this.remote?.fetch(key, (hex) => this.blobPath(hex));
+            if (!fetched) {
+                this.stats.misses++;
+                return undefined;
+            }
+            const m = { version: CACHE_VERSION, kind: fetched.kind, files: fetched.files };
+            raw = JSON.stringify(m);
+            await this.writeAtomic(file, raw, false).catch(() => { });
+        }
+        const loaded = await this.load(raw);
+        if (!loaded) {
+            this.stats.corrupt++;
+            this.stats.misses++;
+            await rm(file, { force: true });
+            return undefined;
+        }
+        this.stats.hits++;
+        const now = new Date();
+        await utimes(file, now, now).catch(() => { });
+        this.remote?.used(key, loaded.entry);
+        return loaded.result;
+    }
+    async load(raw) {
+        let m;
+        try {
+            m = JSON.parse(raw);
+        }
+        catch {
+            return undefined;
+        }
+        if (m?.version !== CACHE_VERSION || (m.kind !== 'file' && m.kind !== 'files') || !Array.isArray(m.files))
+            return undefined;
+        if (m.kind === 'file' && m.files.length !== 1)
+            return undefined;
+        const loaded = [];
+        for (const f of m.files) {
+            if (!Array.isArray(f) || typeof f[0] !== 'string' || typeof f[1] !== 'string' || !HEX64.test(f[1]) || typeof f[2] !== 'number')
+                return undefined;
+            let bytes;
+            try {
+                bytes = await readFile(this.blobPath(f[1]));
+            }
+            catch {
+                return undefined;
+            }
+            if (bytes.byteLength !== f[2] || sha256(bytes) !== `sha256:${f[1]}`) {
+                await rm(this.blobPath(f[1]), { force: true });
+                return undefined;
+            }
+            loaded.push([f[0], new Uint8Array(bytes)]);
+        }
+        const result = m.kind === 'file' ? { kind: 'file', bytes: loaded[0][1] } : { kind: 'files', files: loaded };
+        return { result, entry: { kind: m.kind, files: m.files.map((f) => [f[0], f[1], f[2]]) } };
+    }
+    async put(key, result) {
+        try {
+            const pairs = result.kind === 'file' ? [['', result.bytes]] : result.files;
+            const files = [];
+            for (const [p, bytes] of pairs) {
+                const hex = sha256(bytes).slice('sha256:'.length);
+                await this.writeAtomic(this.blobPath(hex), bytes, true);
+                files.push([p, hex, bytes.byteLength]);
+            }
+            const manifest = { version: CACHE_VERSION, kind: result.kind, files };
+            await this.writeAtomic(this.entryPath(key), JSON.stringify(manifest), false);
+            this.stats.writes++;
+            this.remote?.used(key, { kind: result.kind, files });
+        }
+        catch {
+            // A cache that cannot be written (read-only home, full disk) must not fail the build.
+        }
+    }
+    async writeAtomic(file, data, keepIfSameSize) {
+        if (keepIfSameSize) {
+            const size = typeof data === 'string' ? Buffer.byteLength(data) : data.byteLength;
+            const st = await stat(file).catch(() => undefined);
+            if (st?.isFile() && st.size === size) {
+                // Reused: refresh its mtime so the size limit's grace period covers it until the entry lands.
+                const now = new Date();
+                await utimes(file, now, now).catch(() => { });
+                return;
+            }
+        }
+        await mkdir(path.dirname(file), { recursive: true });
+        const tmp = `${file}.${process.pid}.${Math.random().toString(36).slice(2)}.tmp`;
+        await writeFile(tmp, data);
+        await rename(tmp, file);
+    }
+    async info() {
+        const entries = (await readdir(path.join(this.dir, 'entries')).catch(() => [])).filter((n) => n.endsWith('.json')).length;
+        let blobs = 0;
+        let bytes = 0;
+        for (const n of await readdir(path.join(this.dir, 'blobs')).catch(() => [])) {
+            const st = await stat(path.join(this.dir, 'blobs', n)).catch(() => undefined);
+            if (st?.isFile()) {
+                blobs++;
+                bytes += st.size;
+            }
+        }
+        return { dir: this.dir, entries, blobs, bytes };
+    }
+    /**
+     * Remove entries not used for `olderThanMs` (0 removes all), then every blob
+     * no remaining entry references. Returns what was removed.
+     */
+    async prune(olderThanMs = 0, now = Date.now()) {
+        let removedEntries = 0;
+        const live = new Set();
+        for (const n of await readdir(path.join(this.dir, 'entries')).catch(() => [])) {
+            const file = path.join(this.dir, 'entries', n);
+            const st = await stat(file).catch(() => undefined);
+            if (!st)
+                continue;
+            if (olderThanMs === 0 || now - st.mtimeMs >= olderThanMs || !n.endsWith('.json')) {
+                await rm(file, { force: true });
+                removedEntries++;
+                continue;
+            }
+            try {
+                for (const f of JSON.parse(await readFile(file, 'utf8')).files)
+                    live.add(f[1]);
+            }
+            catch {
+                await rm(file, { force: true });
+                removedEntries++;
+            }
+        }
+        let removedBlobs = 0;
+        let bytes = 0;
+        for (const n of await readdir(path.join(this.dir, 'blobs')).catch(() => [])) {
+            if (live.has(n))
+                continue;
+            const file = path.join(this.dir, 'blobs', n);
+            const st = await stat(file).catch(() => undefined);
+            await rm(file, { force: true });
+            removedBlobs++;
+            bytes += st?.size ?? 0;
+        }
+        return { entries: removedEntries, blobs: removedBlobs, bytes };
+    }
+}
+/** Digest of every regular file under `dir` (a plugin can read all of them, so any may change its output). */
+export async function digestFolder(dir, limits = { files: 2000, bytes: 128 * 1024 * 1024 }) {
+    const h = createHash('sha256');
+    let files = 0;
+    let bytes = 0;
+    const walk = async (rel) => {
+        const entries = (await readdir(path.join(dir, rel), { withFileTypes: true })).sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0));
+        for (const e of entries) {
+            const r = rel ? `${rel}/${e.name}` : e.name;
+            if (e.isDirectory()) {
+                if (!(await walk(r)))
+                    return false;
+            }
+            else if (e.isFile()) {
+                const data = await readFile(path.join(dir, r));
+                if (++files > limits.files || (bytes += data.byteLength) > limits.bytes)
+                    return false;
+                h.update(`${r}\0${sha256(data)}\n`);
+            }
+        }
+        return true;
+    };
+    return (await walk('')) ? h.digest('hex') : undefined;
+}
+//# sourceMappingURL=cache.js.map

package/lib/src/plugins/exec.d.ts

@@ -0,0 +1,51 @@
+import { type SourceSpan } from '../errors.js';
+import type { PluginResult } from './host.js';
+/**
+ * The exec adapter: runs an external program that the user allow-listed with
+ * `--allow-exec`. There is no shell: the program is started with an argv array
+ * (`shell: false`), in a fresh temporary directory, with an almost empty
+ * environment and a closed stdin. Node's permission model cannot restrict a
+ * native binary, so the tool runs with the user's privileges: the allow-list is
+ * the control. See docs/PLUGINS.md.
+ */
+export interface ToolPolicy {
+    /** Per-call time limit (default 60 s). */
+    readonly timeoutMs: number;
+    /** Largest output file accepted (default 64 MiB). */
+    readonly maxOutputBytes: number;
+}
+export declare const DEFAULT_TOOL_POLICY: ToolPolicy;
+/** An allow-listed program, resolved to an absolute real path. */
+export interface ResolvedTool {
+    readonly name: string;
+    readonly path: string;
+    readonly size: number;
+    readonly mtimeMs: number;
+}
+/** The environment given to every tool: nothing but a fixed PATH (tools are started by absolute path). */
+export declare const TOOL_ENV: Readonly<Record<string, string>>;
+/**
+ * Resolve `--allow-exec name[=path]`. A bare name is looked up on PATH; an explicit
+ * path must be absolute. The result is the real path (symlinks followed), so what
+ * is printed, keyed and run is one file.
+ */
+export declare function resolveTool(name: string, explicit: string | undefined, searchPath: string | undefined, span?: SourceSpan): Promise<ResolvedTool>;
+/** What the runner needs from one TRANSFORM call. */
+export interface ToolCall {
+    /** argv template with ${…} already interpolated; `{input}`, `{output}` and `{option.key}` are replaced here. */
+    readonly args: readonly string[];
+    readonly dest: string;
+    readonly options: Readonly<Record<string, string>>;
+    readonly input?: {
+        readonly name: string;
+        readonly bytes: Uint8Array;
+    };
+}
+/**
+ * The argv after placeholder substitution. `{input}` and `{output}` become
+ * *relative* names inside the call's temp dir (the tool's cwd), so filter syntaxes
+ * with `:` or `,` need no escaping. Values are inserted verbatim into one argv entry;
+ * they are never re-scanned, and never split.
+ */
+export declare function substituteArgs(args: readonly string[], inputFile: string | undefined, outputFile: string, options: Readonly<Record<string, string>>): string[];
+export declare function runTool(tool: ResolvedTool, call: ToolCall, policy: ToolPolicy, log: (line: string) => void, span?: SourceSpan): Promise<PluginResult>;

package/lib/src/plugins/exec.js

@@ -0,0 +1,144 @@
+import { spawn } from 'node:child_process';
+import { constants } from 'node:fs';
+import { access, lstat, mkdtemp, readFile, realpath, rm, stat, writeFile } from 'node:fs/promises';
+import os from 'node:os';
+import path from 'node:path';
+import { SitefileError } from '../errors.js';
+import { TOOL_PLACEHOLDER_RE } from '../language/validate.js';
+export const DEFAULT_TOOL_POLICY = { timeoutMs: 60_000, maxOutputBytes: 64 * 1024 * 1024 };
+/** The environment given to every tool: nothing but a fixed PATH (tools are started by absolute path). */
+export const TOOL_ENV = { PATH: '/usr/bin:/bin' };
+const FLAG_HINT = (name) => `--allow-exec ${name}`;
+/**
+ * Resolve `--allow-exec name[=path]`. A bare name is looked up on PATH; an explicit
+ * path must be absolute. The result is the real path (symlinks followed), so what
+ * is printed, keyed and run is one file.
+ */
+export async function resolveTool(name, explicit, searchPath, span) {
+    const candidates = [];
+    if (explicit !== undefined) {
+        if (!path.isAbsolute(explicit)) {
+            throw new SitefileError('E_USAGE', `--allow-exec ${name}=${explicit}: the path must be absolute`, span);
+        }
+        candidates.push(explicit);
+    }
+    else {
+        for (const dir of (searchPath ?? '').split(path.delimiter))
+            if (dir && path.isAbsolute(dir))
+                candidates.push(path.join(dir, name));
+    }
+    for (const c of candidates) {
+        try {
+            const real = await realpath(c);
+            const st = await stat(real);
+            if (!st.isFile())
+                continue;
+            await access(real, constants.X_OK);
+            return { name, path: real, size: st.size, mtimeMs: st.mtimeMs };
+        }
+        catch {
+            // try the next PATH entry
+        }
+    }
+    throw new SitefileError('E_NOT_FOUND', explicit !== undefined ? `TOOL ${name}: ${explicit} is not an executable file` : `TOOL ${name}: no executable '${name}' found on PATH (or pass ${FLAG_HINT(name)}=/absolute/path)`, span);
+}
+const MAX_LOG_LINES = 20;
+const MAX_LINE = 300;
+/** A file extension safe to put in a temp file name (tools such as ffmpeg pick formats from it). */
+function safeExt(name) {
+    const ext = path.extname(name);
+    return /^\.[A-Za-z0-9]{1,12}$/.test(ext) ? ext : '';
+}
+/**
+ * The argv after placeholder substitution. `{input}` and `{output}` become
+ * *relative* names inside the call's temp dir (the tool's cwd), so filter syntaxes
+ * with `:` or `,` need no escaping. Values are inserted verbatim into one argv entry;
+ * they are never re-scanned, and never split.
+ */
+export function substituteArgs(args, inputFile, outputFile, options) {
+    return args.map((a) => a.replace(TOOL_PLACEHOLDER_RE, (_m, name) => {
+        if (name === 'input')
+            return inputFile ?? '';
+        if (name === 'output')
+            return outputFile;
+        return options[name.slice('option.'.length)] ?? '';
+    }));
+}
+export async function runTool(tool, call, policy, log, span) {
+    const label = `TOOL ${tool.name}`;
+    const dir = await mkdtemp(path.join(os.tmpdir(), 'sitefile-tool-'));
+    try {
+        let inputFile;
+        if (call.input) {
+            inputFile = `input${safeExt(call.input.name)}`;
+            await writeFile(path.join(dir, inputFile), call.input.bytes, { mode: 0o600 });
+        }
+        const outputFile = `output${safeExt(call.dest)}`;
+        const argv = substituteArgs(call.args, inputFile, outputFile, call.options);
+        const { code, signal, stderr, stdout, timedOut, oversized, spawnError } = await new Promise((resolve) => {
+            const child = spawn(tool.path, argv, { cwd: dir, env: { ...TOOL_ENV }, shell: false, stdio: ['ignore', 'pipe', 'pipe'], windowsHide: true });
+            let stdout = '';
+            let stderr = '';
+            let timedOut = false;
+            let oversized = false;
+            let spawnError;
+            // Keep only the tail of each stream; a chatty tool must not exhaust memory.
+            const keep = (cur, chunk) => (cur + chunk).slice(-16 * 1024);
+            child.stdout.setEncoding('utf8').on('data', (c) => (stdout = keep(stdout, c)));
+            child.stderr.setEncoding('utf8').on('data', (c) => (stderr = keep(stderr, c)));
+            const timer = setTimeout(() => {
+                timedOut = true;
+                child.kill('SIGKILL');
+            }, policy.timeoutMs);
+            // Stop a tool that is filling the disk with an oversized output.
+            const watch = setInterval(() => {
+                stat(path.join(dir, outputFile)).then((st) => {
+                    if (st.size > policy.maxOutputBytes && !oversized) {
+                        oversized = true;
+                        child.kill('SIGKILL');
+                    }
+                }, () => { });
+            }, 100);
+            child.on('error', (e) => {
+                spawnError = e.message;
+            });
+            child.on('close', (code, signal) => {
+                clearTimeout(timer);
+                clearInterval(watch);
+                resolve({ code, signal, stderr, stdout, timedOut, oversized, spawnError });
+            });
+        });
+        const tail = (s) => s.split('\n').map((l) => l.trimEnd()).filter(Boolean).slice(-MAX_LOG_LINES).map((l) => (l.length > MAX_LINE ? `${l.slice(0, MAX_LINE)}…` : l));
+        const fail = (why) => {
+            const lines = tail(stderr || stdout).slice(-8);
+            throw new SitefileError('E_PLUGIN', `${label}: ${why}${lines.length ? `\n${lines.join('\n')}` : ''}`, span);
+        };
+        if (spawnError)
+            fail(`could not start ${tool.path}: ${spawnError}`);
+        if (timedOut)
+            fail(`timed out after ${policy.timeoutMs} ms`);
+        if (oversized)
+            fail(`output is larger than ${policy.maxOutputBytes} bytes`);
+        if (signal)
+            fail(`stopped by ${signal}`);
+        if (code !== 0)
+            fail(`exited with code ${code}`);
+        for (const line of tail(stdout))
+            log(`[tool ${tool.name}] ${line}`);
+        for (const line of tail(stderr))
+            log(`[tool ${tool.name}] ${line}`);
+        const out = path.join(dir, outputFile);
+        const st = await lstat(out).catch(() => undefined);
+        if (!st)
+            fail(`did not write {output} (${outputFile})`);
+        if (!st.isFile())
+            fail(`{output} is not a regular file`);
+        if (st.size > policy.maxOutputBytes)
+            fail(`output is larger than ${policy.maxOutputBytes} bytes`);
+        return { kind: 'file', bytes: new Uint8Array(await readFile(out)) };
+    }
+    finally {
+        await rm(dir, { recursive: true, force: true });
+    }
+}
+//# sourceMappingURL=exec.js.map

package/lib/src/plugins/host.d.ts

@@ -0,0 +1,75 @@
+import { type SourceSpan } from '../errors.js';
+/**
+ * Plugins run one Node process each, under Node's permission model:
+ *
+ *   node --permission --allow-fs-read=<plugin folder> --input-type=module -e <runner>
+ *
+ * so a plugin can read files in its own folder and nothing else: no writes,
+ * child processes, workers, native addons, WASI or network, and an empty
+ * environment (no secrets). Requests and results travel over stdin/stdout.
+ *
+ * Node documents the permission model as a "seat belt" for trusted code, not a
+ * defence against deliberately malicious code: plugins are code you choose to
+ * run. Pin them with INTEGRITY.
+ */
+export interface PluginPolicy {
+    /** Per-call time limit (default 60 s); the process is killed when it passes. */
+    readonly timeoutMs: number;
+    /** Largest result accepted from one call (default 64 MiB). */
+    readonly maxOutputBytes: number;
+}
+export declare const DEFAULT_PLUGIN_POLICY: PluginPolicy;
+/** A directory TRANSFORM runs at most this many wasm or tool calls at a time (a JS plugin, one). */
+export declare const TRANSFORM_JOBS: number;
+/** What a plugin receives (see docs/PLUGINS.md). */
+export interface PluginRequest {
+    readonly apiVersion: 1;
+    readonly dest: string;
+    readonly options: Readonly<Record<string, string>>;
+    readonly input?: {
+        readonly path: string;
+        readonly name: string;
+        readonly bytes: Uint8Array;
+        readonly text?: string;
+    };
+    readonly data?: unknown;
+    readonly item?: unknown;
+}
+/** A plugin's result, normalized: one file, or several under the destination. */
+export type PluginResult = {
+    readonly kind: 'file';
+    readonly bytes: Uint8Array;
+} | {
+    readonly kind: 'files';
+    readonly files: ReadonlyArray<readonly [string, Uint8Array]>;
+};
+/** Counters a build reports (see BuildResult.transformStats). */
+export interface HostStats {
+    /** Plugin processes started. */
+    spawned: number;
+    /** transform() calls sent to a plugin process. */
+    calls: number;
+}
+export declare class PluginHost {
+    private readonly policy;
+    private readonly log;
+    private readonly registered;
+    private readonly procs;
+    readonly stats: HostStats;
+    constructor(policy?: PluginPolicy, log?: (line: string) => void);
+    /**
+     * `file` is the plugin's absolute real path (already sandbox- and integrity-checked).
+     * For a `.wasm` plugin, pass the checked module bytes. Runners start lazily.
+     */
+    register(name: string, file: string, span?: SourceSpan, wasm?: Uint8Array): void;
+    /** Whether a registered plugin is a WebAssembly module. */
+    isWasm(name: string): boolean;
+    /** A registered wasm plugin's module bytes. */
+    wasmBytes(name: string): Uint8Array | undefined;
+    call(name: string, request: PluginRequest, span?: SourceSpan): Promise<PluginResult>;
+    /** The folder a registered plugin lives in. */
+    folderOf(name: string): string | undefined;
+    /** Whether results of this plugin may be cached (false after it exported `cache = false`). */
+    cacheable(name: string): boolean;
+    close(): void;
+}