@johnhenry/sitefile 0.0.0-stage → 0.0.1

package/lib/src/tree.js

@@ -0,0 +1,134 @@
+import { SitefileError } from './errors.js';
+import { sha256 } from './digest.js';
+import { foldCase, nfc } from './paths.js';
+/**
+ * A path and its ancestors, as [path, foldCase(path)] pairs from the
+ * shortest ancestor to the path itself. Folding is per segment, so prefix
+ * lengths never depend on how folding changes a string's length.
+ */
+function prefixes(path) {
+    const out = [];
+    let raw = '';
+    let folded = '';
+    for (const seg of path.slice(1).split('/')) {
+        raw += '/' + seg;
+        folded += '/' + foldCase(seg);
+        out.push([raw, folded]);
+    }
+    return out;
+}
+export function makeFile(bytes, origin, mode = 0o644) {
+    return { bytes, mode: mode & 0o111 ? 0o755 : 0o644, digest: sha256(bytes), origin };
+}
+/**
+ * Immutable snapshot of a filesystem tree. Keys are normalized absolute
+ * virtual paths of files; directories are implicit. Every operation returns
+ * a new Tree.
+ */
+export class Tree {
+    files;
+    dirs;
+    folded;
+    static empty = new Tree(new Map(), new Set(), new Map());
+    constructor(files, dirs, 
+    /** foldCase(path) -> the one spelling of it in this tree (files and directories). */
+    folded) {
+        this.files = files;
+        this.dirs = dirs;
+        this.folded = folded;
+    }
+    get size() {
+        return this.files.size;
+    }
+    get(path) {
+        return this.files.get(path);
+    }
+    isFile(path) {
+        return this.files.has(path);
+    }
+    /** True for `/` and any implicit directory. */
+    isDirectory(path) {
+        return path === '/' || this.dirs.has(path);
+    }
+    /** Sorted [path, node] pairs. */
+    entries() {
+        return [...this.files.entries()].sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0));
+    }
+    /**
+     * Add files. Paths are normalized to NFC. Writing identical content (digest
+     * and mode) is a no-op that keeps the first owner. Different content fails
+     * with E_CONFLICT unless the policy is 'overwrite'. A file can never replace
+     * a directory or vice versa. A path (or one of its directories) that equals
+     * an existing one under simple case folding but is spelled differently
+     * fails with E_PATH_COLLISION whatever the policy: case-insensitive file
+     * systems cannot hold both.
+     */
+    withFiles(additions, policy, span) {
+        const files = new Map(this.files);
+        const dirs = new Set(this.dirs);
+        const folded = new Map(this.folded);
+        for (const [rawPath, node] of additions) {
+            const path = nfc(rawPath);
+            if (path === '/' || dirs.has(path)) {
+                throw new SitefileError('E_CONFLICT', `cannot write file '${path}': it is a directory`, span);
+            }
+            const chain = prefixes(path);
+            const ancestors = chain.slice(0, -1);
+            for (const [a] of ancestors) {
+                const f = files.get(a);
+                if (f) {
+                    throw new SitefileError('E_CONFLICT', `cannot write '${path}': '${a}' is a file (written by ${f.origin})`, span);
+                }
+            }
+            for (const [p, key] of chain) {
+                const prior = folded.get(key);
+                if (prior && prior.path !== p) {
+                    const by = `${prior.dir ? 'created' : 'written'} by ${prior.origin}`;
+                    throw new SitefileError('E_PATH_COLLISION', p === path
+                        ? `'${path}' differs only in case from '${prior.path}' (${by}); case-insensitive file systems cannot hold both`
+                        : `cannot write '${path}': '${p}' differs only in case from '${prior.path}' (${by}); case-insensitive file systems cannot hold both`, span);
+                }
+            }
+            const existing = files.get(path);
+            if (existing) {
+                if (existing.digest === node.digest && existing.mode === node.mode)
+                    continue;
+                if (policy === 'error') {
+                    throw new SitefileError('E_CONFLICT', `'${path}' was already written with different content by ${existing.origin}; use --overwrite to replace it`, span);
+                }
+            }
+            files.set(path, node);
+            folded.set(chain[chain.length - 1][1], { path, origin: node.origin, dir: false });
+            for (const [a, key] of ancestors) {
+                if (dirs.has(a))
+                    continue;
+                dirs.add(a);
+                folded.set(key, { path: a, origin: node.origin, dir: true });
+            }
+        }
+        return new Tree(files, dirs, folded);
+    }
+    /** The files under a directory, re-rooted at `/`. */
+    subtree(dir) {
+        if (dir === '/')
+            return this;
+        const prefix = dir + '/';
+        const out = [];
+        for (const [p, n] of this.files)
+            if (p.startsWith(prefix))
+                out.push([p.slice(dir.length), n]);
+        return Tree.empty.withFiles(out, 'error');
+    }
+    /** The files of this tree placed under `dir`. */
+    mountedAt(dir) {
+        return this.entries().map(([p, n]) => [dir === '/' ? p : dir + p, n]);
+    }
+    /** Content digest of the whole tree (paths, modes and file digests). */
+    digest() {
+        return sha256(this.entries().map(([p, n]) => `${p}\0${n.mode.toString(8)}\0${n.digest}\n`).join(''));
+    }
+    static fromFiles(files) {
+        return Tree.empty.withFiles(files, 'error');
+    }
+}
+//# sourceMappingURL=tree.js.map

package/lib/src/values.d.ts

@@ -0,0 +1,7 @@
+import { type SourceSpan } from './errors.js';
+import type { Operand, ValueType } from './language/ast.js';
+export type Scalar = string | number | boolean;
+/** Convert a raw string to a typed scalar, or fail with E_TYPE. */
+export declare function coerce(raw: string, type: ValueType, span: SourceSpan | undefined, what: string): Scalar;
+/** Interpolate an instruction operand with ARG/LET bindings. */
+export declare function evaluateOperand(op: Operand, env: ReadonlyMap<string, Scalar>): string;

package/lib/src/values.js

@@ -0,0 +1,37 @@
+import { SitefileError } from './errors.js';
+const NUMBER_RE = /^-?(?:\d+)(?:\.\d+)?(?:[eE][+-]?\d+)?$/;
+/** Convert a raw string to a typed scalar, or fail with E_TYPE. */
+export function coerce(raw, type, span, what) {
+    switch (type) {
+        case 'string':
+            return raw;
+        case 'number': {
+            if (!NUMBER_RE.test(raw.trim()))
+                throw new SitefileError('E_TYPE', `${what} expects a number, got '${raw}'`, span);
+            return Number(raw);
+        }
+        case 'boolean': {
+            if (raw === 'true')
+                return true;
+            if (raw === 'false')
+                return false;
+            throw new SitefileError('E_TYPE', `${what} expects true or false, got '${raw}'`, span);
+        }
+    }
+}
+/** Interpolate an instruction operand with ARG/LET bindings. */
+export function evaluateOperand(op, env) {
+    let out = '';
+    for (const part of op.parts) {
+        if (part.kind === 'text') {
+            out += part.value;
+            continue;
+        }
+        const v = env.get(part.name);
+        if (v === undefined)
+            throw new SitefileError('E_UNDEFINED', `undefined variable '${part.name}'`, part.span);
+        out += String(v);
+    }
+    return out;
+}
+//# sourceMappingURL=values.js.map

package/lib/src/vendor.d.ts

@@ -0,0 +1,19 @@
+import { type BuildOptions } from './executor.js';
+export interface VendorResult {
+    /** Real path of the vendor directory written. */
+    readonly dir: string;
+    readonly resources: number;
+    readonly images: number;
+    readonly blobs: number;
+    /** Total size of the blobs. */
+    readonly bytes: number;
+    readonly lockfileWritten: boolean;
+    readonly warnings: readonly string[];
+}
+/**
+ * `sitefile vendor` (issue #96): fetch every remote IMPORT and the BASE image exactly as a build would
+ * (same network policy, INTEGRITY and lockfile checks, and Sitefile.lock is updated the same way), then
+ * replace the vendor directory (`options.vendorDir`, default `vendor/sitefile`) with their bytes under
+ * content-addressed names and a manifest. Always fetches: an existing vendor directory is not read.
+ */
+export declare function vendor(options: BuildOptions): Promise<VendorResult>;

package/lib/src/vendor.js

@@ -0,0 +1,145 @@
+import { realpath } from 'node:fs/promises';
+import path from 'node:path';
+import { resolveBase } from './base-image.js';
+import { sha256 } from './digest.js';
+import { SitefileError } from './errors.js';
+import { fetchGitImport, fetchRemoteImport, gitSelectionBytes, loadProgram, readImportedSitefile, rootSitefileSource, warnUnusedSecrets } from './executor.js';
+import { redactError, redactSecrets } from './secrets.js';
+import { parseManifest } from './oci.js';
+import { parseRef, withDefaultRegistry } from './ref.js';
+import { importedStageRefs, plan } from './planner.js';
+import { Lockfile } from './resources/lockfile.js';
+import { DEFAULT_FETCH_POLICY } from './resources/types.js';
+import { DEFAULT_VENDOR_DIR, serializeVendorManifest, vendorDirRel, writeVendorDir } from './resources/vendor.js';
+import { DEFAULT_ARCHIVE_LIMITS } from './resources/types.js';
+import { Store } from './store.js';
+/**
+ * `sitefile vendor` (issue #96): fetch every remote IMPORT and the BASE image exactly as a build would
+ * (same network policy, INTEGRITY and lockfile checks, and Sitefile.lock is updated the same way), then
+ * replace the vendor directory (`options.vendorDir`, default `vendor/sitefile`) with their bytes under
+ * content-addressed names and a manifest. Always fetches: an existing vendor directory is not read.
+ */
+export async function vendor(options) {
+    const secrets = options.secrets;
+    if (!secrets?.size)
+        return vendorUnredacted(options);
+    const onWarning = options.onWarning;
+    try {
+        const r = await vendorUnredacted({ ...options, ...(onWarning ? { onWarning: (m) => onWarning(redactSecrets(m, secrets)) } : {}) });
+        return { ...r, warnings: r.warnings.map((w) => redactSecrets(w, secrets)) };
+    }
+    catch (e) {
+        throw redactError(e, secrets);
+    }
+}
+async function vendorUnredacted(options) {
+    const { program, sitefilePath, label } = await loadProgram(options);
+    const p = plan(program, options.buildArgs ?? {});
+    const warnings = [];
+    const warn = (m) => {
+        warnings.push(m);
+        options.onWarning?.(m);
+    };
+    p.warnings.forEach(warn);
+    const dir = vendorDirRel(options.vendorDir ?? DEFAULT_VENDOR_DIR);
+    const contextReal = await realpath(options.contextDir).catch((e) => {
+        throw new SitefileError('E_NOT_FOUND', `build context '${options.contextDir}' does not exist`, undefined, { cause: e });
+    });
+    const policy = { ...DEFAULT_FETCH_POLICY, ...options.fetchPolicy };
+    const lockPath = options.lockfile === false ? undefined : (options.lockfile ?? path.join(path.dirname(sitefilePath), 'Sitefile.lock'));
+    const lock = lockPath ? await Lockfile.load(lockPath) : undefined;
+    const frozen = !!options.frozenLockfile;
+    const update = !!options.updateLock;
+    warnUnusedSecrets(p, options.secrets, warn);
+    const env = { policy, lock, label, warn, frozen, update, archiveLimits: { ...DEFAULT_ARCHIVE_LIMITS, ...options.archiveLimits }, ...(options.secrets ? { secrets: options.secrets } : {}) };
+    const blobs = new Map();
+    const resources = {};
+    const git = {};
+    /** A Git import's content, vendored (a Sitefile imported from Git is read from it too). */
+    const vendorGit = async (spec, loc, e) => {
+        const got = await fetchGitImport(spec, loc, e);
+        const bytes = gitSelectionBytes(got.selection);
+        const digest = sha256(bytes);
+        blobs.set(digest, bytes);
+        git[loc.url] = { commit: got.commit, kind: got.selection.kind, integrity: digest, size: bytes.byteLength, ...(got.selection.kind === 'file' ? { mode: got.selection.mode === 0o755 ? '755' : '644' } : {}) };
+        return got;
+    };
+    /** Every remote input of `sp` (a Sitefile read from `source`), then those of the Sitefiles it imports AS sitefile. */
+    const vendorImports = async (sp, source) => {
+        const e = { ...env, label: source.label };
+        for (const spec of sp.imports) {
+            if (spec.as === 'sitefile') {
+                const sub = await readImportedSitefile(spec, source, importedStageRefs(sp.stages), (s, loc) => vendorGit(s, loc, e));
+                sub.plan.warnings.forEach(warn);
+                await vendorImports(sub.plan, sub.source);
+                nestedImages.push(...sub.plan.stages);
+                continue;
+            }
+            if (spec.location.kind === 'git') {
+                await vendorGit(spec, spec.location, e);
+                continue;
+            }
+            if (spec.location.kind !== 'remote')
+                continue;
+            const url = spec.location.url;
+            const { bytes, finalUrl } = await fetchRemoteImport(spec, url, e);
+            const digest = sha256(bytes);
+            blobs.set(digest, bytes);
+            resources[url] = { integrity: digest, size: bytes.byteLength, resolved: finalUrl };
+        }
+    };
+    /** The stages of imported Sitefiles: their registry FROM images are vendored after this Sitefile's. */
+    const nestedImages = [];
+    await vendorImports(p, await rootSitefileSource(options, label, sitefilePath));
+    const images = {};
+    const platform = options.platform ?? 'linux/amd64';
+    let store;
+    /** Resolve an image as a build would and vendor what an offline pull of it needs. */
+    const vendorImage = async (input, lead, enforcePlatform, span) => {
+        store ??= options.store ?? (await Store.open());
+        const s = store;
+        const base = await resolveBase(input, {
+            store: s,
+            platform,
+            lock,
+            frozen,
+            update,
+            registry: { ...options.registry, allowPrivateNetwork: options.registry?.allowPrivateNetwork ?? policy.allowPrivateNetwork, network: options.registry?.network ?? policy.network },
+            warn,
+            lead,
+            enforcePlatform,
+            ...(span ? { span } : {}),
+        });
+        // resolveBase has validated the reference; this is its lockfile key.
+        const ref = withDefaultRegistry(parseRef(input));
+        const registry = ref.registry;
+        const lockKey = `${registry}/${ref.name}:${ref.tag ?? 'latest'}`;
+        const root = ref.digest ?? lock?.getImage(lockKey)?.digest ?? base.manifestDigest;
+        const add = async (digest, size) => {
+            const bytes = await s.readBlob(digest, size);
+            blobs.set(digest, bytes);
+            return bytes;
+        };
+        await add(root);
+        const m = parseManifest(await add(base.manifestDigest));
+        for (const d of [m.config, ...m.layers])
+            await add(d.digest, d.size);
+        const key = ref.digest !== undefined ? `${registry}/${ref.name}${ref.tag !== undefined ? `:${ref.tag}` : ''}@${ref.digest}` : lockKey;
+        images[key] = { digest: root, platform, manifest: base.manifestDigest };
+    };
+    // FROM image stage bases with a registry (local-store images are not vendored), then the BASE.
+    for (const st of [...p.stages, ...nestedImages])
+        if (st.base.kind === 'image' && !st.base.local)
+            await vendorImage(st.base.ref, `${st.base.lead} image`, false, st.base.span);
+    const baseInput = options.baseImage ?? (p.exportFormat === 'oci' ? p.exportBase : undefined);
+    if (baseInput !== undefined)
+        await vendorImage(baseInput, 'BASE', true);
+    const text = serializeVendorManifest({ version: 1, resources, images, git });
+    const written = await writeVendorDir(contextReal, dir, text, blobs);
+    const lockfileWritten = lock && !frozen ? await lock.save() : false;
+    let bytes = 0;
+    for (const b of blobs.values())
+        bytes += b.byteLength;
+    return { dir: written, resources: Object.keys(resources).length + Object.keys(git).length, images: Object.keys(images).length, blobs: blobs.size, bytes, lockfileWritten, warnings };
+}
+//# sourceMappingURL=vendor.js.map

package/lib/src/watch.d.ts

@@ -0,0 +1,44 @@
+/** Quiet period after the last change before a rebuild starts. */
+export declare const DEBOUNCE_MS = 100;
+/**
+ * Runs `build` once changes settle (no new change for `debounceMs`). Changes that
+ * arrive while a build runs are collected and coalesced into one follow-up build.
+ * `build` receives the paths reported since the previous build started.
+ */
+export declare class RebuildScheduler {
+    private readonly build;
+    private readonly debounceMs;
+    private timer;
+    private running;
+    private changes;
+    private closed;
+    constructor(build: (changed: readonly string[]) => Promise<void>, debounceMs?: number);
+    /** Record a change; (re)starts the debounce unless a build is running. */
+    notify(changed: string): void;
+    /** Build now (the initial build), then any follow-up. Resolves when idle. */
+    run(): Promise<void>;
+    /** True while a build runs or one is scheduled. */
+    get busy(): boolean;
+    close(): void;
+}
+/** `p` is `dir` or inside it. */
+export declare function isWithin(p: string, dir: string): boolean;
+/**
+ * Paths a watcher never reacts to: anything under `.git`, the exporter's temp and
+ * backup siblings (`.<name>.sitefile-tmp-*`, `.<name>.sitefile-old-*`), and any of
+ * `ignored` (the output, the store, Sitefile.lock), checked at call time so the
+ * set may grow between builds.
+ */
+export declare function isIgnoredChange(abs: string, roots: readonly string[], ignored: Iterable<string>): boolean;
+/** realpath of `p`, else of its parent joined with its name, else `p` resolved. */
+export declare function resolveReal(p: string): Promise<string>;
+export interface WatchSpec {
+    /** Directories watched recursively (the build context). Absolute, real paths. */
+    readonly dirs: readonly string[];
+    /** Single files (the Sitefile, --build-arg-file files) outside `dirs`; their directory is watched for their name. */
+    readonly files: readonly string[];
+    /** Called with the absolute path of each change (the directory itself when the OS gives no name). */
+    readonly onChange: (abs: string) => void;
+}
+/** Start fs.watch watchers for `spec`; returns a function that closes them. */
+export declare function watchPaths(spec: WatchSpec): () => void;

package/lib/src/watch.js

@@ -0,0 +1,137 @@
+import { watch as fsWatch, promises as fs } from 'node:fs';
+import path from 'node:path';
+import { TEMP_PREFIX_PATTERN } from './exporter.js';
+/** Quiet period after the last change before a rebuild starts. */
+export const DEBOUNCE_MS = 100;
+/**
+ * Runs `build` once changes settle (no new change for `debounceMs`). Changes that
+ * arrive while a build runs are collected and coalesced into one follow-up build.
+ * `build` receives the paths reported since the previous build started.
+ */
+export class RebuildScheduler {
+    build;
+    debounceMs;
+    timer;
+    running;
+    changes = new Set();
+    closed = false;
+    constructor(build, debounceMs = DEBOUNCE_MS) {
+        this.build = build;
+        this.debounceMs = debounceMs;
+    }
+    /** Record a change; (re)starts the debounce unless a build is running. */
+    notify(changed) {
+        if (this.closed)
+            return;
+        this.changes.add(changed);
+        if (this.running)
+            return; // picked up when the running build finishes
+        clearTimeout(this.timer);
+        this.timer = setTimeout(() => void this.run(), this.debounceMs);
+    }
+    /** Build now (the initial build), then any follow-up. Resolves when idle. */
+    run() {
+        if (this.running)
+            return this.running;
+        clearTimeout(this.timer);
+        this.timer = undefined;
+        const batch = [...this.changes];
+        this.changes.clear();
+        this.running = this.build(batch)
+            .catch((e) => void process.stderr.write(`internal error: ${e?.stack ?? String(e)}\n`))
+            .finally(() => {
+            this.running = undefined;
+            if (this.changes.size && !this.closed) {
+                clearTimeout(this.timer);
+                this.timer = setTimeout(() => void this.run(), this.debounceMs);
+            }
+        });
+        return this.running;
+    }
+    /** True while a build runs or one is scheduled. */
+    get busy() {
+        return this.running !== undefined || this.timer !== undefined;
+    }
+    close() {
+        this.closed = true;
+        clearTimeout(this.timer);
+        this.timer = undefined;
+    }
+}
+/** `p` is `dir` or inside it. */
+export function isWithin(p, dir) {
+    const rel = path.relative(dir, p);
+    return rel === '' || (!rel.startsWith('..') && !path.isAbsolute(rel));
+}
+/**
+ * Paths a watcher never reacts to: anything under `.git`, the exporter's temp and
+ * backup siblings (`.<name>.sitefile-tmp-*`, `.<name>.sitefile-old-*`), and any of
+ * `ignored` (the output, the store, Sitefile.lock), checked at call time so the
+ * set may grow between builds.
+ */
+export function isIgnoredChange(abs, roots, ignored) {
+    for (const q of ignored)
+        if (isWithin(abs, q))
+            return true;
+    const root = roots.find((r) => isWithin(abs, r));
+    if (root === undefined)
+        return false;
+    return path
+        .relative(root, abs)
+        .split(path.sep)
+        .some((seg) => seg === '.git' || TEMP_PREFIX_PATTERN.test(seg));
+}
+/** realpath of `p`, else of its parent joined with its name, else `p` resolved. */
+export async function resolveReal(p) {
+    const abs = path.resolve(p);
+    try {
+        return await fs.realpath(abs);
+    }
+    catch {
+        try {
+            return path.join(await fs.realpath(path.dirname(abs)), path.basename(abs));
+        }
+        catch {
+            return abs;
+        }
+    }
+}
+/** Start fs.watch watchers for `spec`; returns a function that closes them. */
+export function watchPaths(spec) {
+    const watchers = [];
+    const add = (dir, recursive, names) => {
+        const w = fsWatch(dir, { recursive, persistent: true }, (_event, filename) => {
+            const name = filename == null ? undefined : filename.toString();
+            if (names && (name === undefined || !names.has(path.basename(name))))
+                return;
+            spec.onChange(name === undefined ? dir : path.join(dir, name));
+        });
+        // A watched directory that disappears (or a platform error) must not crash the build loop.
+        w.on('error', () => undefined);
+        watchers.push(w);
+    };
+    for (const d of spec.dirs)
+        add(d, true);
+    const byDir = new Map();
+    for (const f of spec.files) {
+        if (spec.dirs.some((d) => isWithin(f, d)))
+            continue;
+        const dir = path.dirname(f);
+        const set = byDir.get(dir) ?? new Set();
+        set.add(path.basename(f));
+        byDir.set(dir, set);
+    }
+    for (const [dir, names] of byDir) {
+        try {
+            add(dir, false, names);
+        }
+        catch {
+            // the directory of a missing file: the build reports the missing file
+        }
+    }
+    return () => {
+        for (const w of watchers)
+            w.close();
+    };
+}
+//# sourceMappingURL=watch.js.map

package/lib/src/wbn.d.ts

@@ -0,0 +1,26 @@
+import type { Tree } from './tree.js';
+export declare function mimeTypeFor(path: string): string;
+/** An absolute https URL ending in `/`, without credentials, query or fragment. Returns its normalized form. */
+export declare function validateBaseUrl(raw: string | undefined): string;
+export declare function writeWebBundle(tree: Tree, baseUrl: string | undefined): Uint8Array;
+/**
+ * The responses section and its URLs (tree paths, percent-encoded by the caller), as `sitefile
+ * serve` would answer them:
+ *
+ * - Each file is a 200 with `content-type` and the `/_headers` blocks whose pattern matches its
+ *   URL path (`headersFor`; a `content-type` there replaces the inferred one). A directory URL
+ *   (`/docs/` for `/docs/index.html`) matches with its trailing slash, so it gets its own
+ *   response when its headers differ from the file URL's.
+ * - An exact-path `/_redirects` rule with a 3xx status becomes a response with that `:status`, a
+ *   `location` of the rule's target as written, and an empty body: at its path when no file
+ *   would be served there, and over the file when forced (`301!`). Like `serve`, the first rule
+ *   matching the path decides; when that is a splat or a 200/404 rewrite, the bundle leaves the
+ *   path to `/_redirects`. Splat rules and rewrites are not baked in.
+ *
+ * `/_headers` and `/_redirects` stay in the bundle as files. Responses are deduplicated (both URLs
+ * of an index page share one when their headers agree; identical redirects share one).
+ */
+export declare function bundleResponses(tree: Tree): {
+    responses: Uint8Array[];
+    urls: Array<[string, number]>;
+};