@johnhenry/sitefile 0.0.0-stage → 0.0.0

package/lib/src/planner.js

@@ -0,0 +1,410 @@
+import { SitefileError } from './errors.js';
+import { importedStage, isFragment, isImageBase, isInline, stageNames } from './language/ast.js';
+import { normalizeLocal, normalizeVirtual } from './paths.js';
+import { parseRef } from './ref.js';
+import { checkPattern } from './fingerprint.js';
+import { checkBaseUrl, checkHeaderName, checkHeaderValue, checkRedirectTarget, checkSitePath } from './site.js';
+import { coerce, evaluateOperand } from './values.js';
+import { isGitUri, parseGitUri } from './resources/git.js';
+import { parseFragmentBody } from './language/validate.js';
+const URL_RE = /^([A-Za-z][A-Za-z0-9+.-]*):\/\//;
+const IMPORT_REF_RE = /^([A-Za-z_][A-Za-z0-9_]*):(.*)$/s;
+/**
+ * Evaluate a program. `argSource` names where a build arg came from in messages (`--build-arg NAME` for the
+ * CLI's; an imported Sitefile's come from the `ARG` clauses of its IMPORT).
+ */
+export function plan(program, buildArgs = {}, argSource = (n) => `--build-arg ${n}`) {
+    const vars = new Map();
+    const fragments = new Map();
+    const imports = new Map();
+    const stages = [];
+    const plugins = [];
+    const tools = [];
+    const redirects = [];
+    const headers = [];
+    const warnings = [];
+    const declaredArgs = new Set();
+    const allStageNames = stageNames(program.instructions);
+    // validate() guarantees a FROM precedes every operation.
+    const operations = () => stages[stages.length - 1].operations;
+    let exportPath;
+    let exportFormat = 'directory';
+    let exportBase;
+    let exportRoot;
+    let exportCompression;
+    const bind = (name, raw, type, span, what) => {
+        vars.set(name, coerce(raw, type, span, what));
+    };
+    for (const ins of program.instructions) {
+        switch (ins.kind) {
+            case 'FROM':
+                stages.push({
+                    ...(ins.alias !== undefined && { name: ins.alias }),
+                    base: ins.base === 'scratch'
+                        ? { kind: 'scratch' }
+                        : isImageBase(ins.base)
+                            ? imageBase(ins.base, vars, ins.keyword === 'STAGE' ? `STAGE ${ins.alias} FROM` : 'FROM')
+                            : stageBase(ins.base, vars, stages, allStageNames, imports, ins.keyword === 'STAGE' ? `STAGE ${ins.alias} FROM` : 'FROM'),
+                    operations: [],
+                    span: ins.span,
+                });
+                break;
+            case 'ARG': {
+                declaredArgs.add(ins.name);
+                const given = buildArgs[ins.name];
+                if (given !== undefined)
+                    bind(ins.name, given, ins.type, ins.span, argSource(ins.name));
+                else if (ins.defaultValue)
+                    bind(ins.name, evaluateOperand(ins.defaultValue, vars), ins.type, ins.defaultValue.span, `ARG ${ins.name}`);
+                else
+                    throw new SitefileError('E_MISSING_ARG', `ARG ${ins.name} has no default; pass --build-arg ${ins.name}=<value>`, ins.span);
+                break;
+            }
+            case 'LET':
+                if (isFragment(ins)) {
+                    fragments.set(ins.name, { name: ins.name, text: ins.body.text, nodes: parseFragmentBody(ins.body, program.file), span: ins.span });
+                    break;
+                }
+                bind(ins.name, evaluateOperand(ins.value, vars), ins.type, ins.value.span, `LET ${ins.name}`);
+                break;
+            case 'IMPORT': {
+                const uri = evaluateOperand(ins.uri, vars);
+                const spec = {
+                    name: ins.name,
+                    uri,
+                    location: classifyUri(uri, ins.uri.span),
+                    ...(ins.as ? { as: ins.as } : {}),
+                    ...(ins.integrity ? { integrity: ins.integrity } : {}),
+                    ...(ins.sort ? { sort: { key: ins.sort.key, descending: ins.sort.descending } } : {}),
+                    ...(ins.schema ? { schema: schemaSpec(ins.name, ins.schema.path, vars, ins.schema.span) } : {}),
+                    ...(ins.drafts ? { drafts: draftsValue(ins.name, ins.drafts.value, vars) } : {}),
+                    ...(ins.secrets ? { secrets: ins.secrets.map((s) => ({ id: s.id, header: s.header, bearer: s.bearer })) } : {}),
+                    ...(ins.args ? { args: ins.args.map((a) => ({ name: a.name, value: evaluateOperand(a.value, vars), span: a.span })) } : {}),
+                    span: ins.span,
+                };
+                if (spec.as === 'sitefile' && spec.location.kind === 'remote') {
+                    throw new SitefileError('E_PROTOCOL', `IMPORT ${ins.name}: AS sitefile reads a context directory or a git+https:// repository, not a URL`, ins.uri.span);
+                }
+                if (spec.location.kind === 'local' && ins.secrets) {
+                    throw new SitefileError('E_VALIDATION', `IMPORT ${ins.name}: SECRET applies to remote imports (https:// or git+https://), not local paths`, ins.secrets[0].span);
+                }
+                if (spec.location.kind === 'git' && spec.integrity) {
+                    throw new SitefileError('E_VALIDATION', `IMPORT ${ins.name}: INTEGRITY does not apply to Git imports; pin a commit instead (#<40 hex>[:path])`, ins.span);
+                }
+                if (spec.location.kind === 'local' && spec.integrity) {
+                    warnings.push(`${program.file}:${ins.span.line}: INTEGRITY on a local import is checked but not recorded in the lockfile`);
+                }
+                imports.set(ins.name, spec);
+                break;
+            }
+            case 'COPY': {
+                const dest = normalizeVirtual(evaluateOperand(ins.dest, vars), ins.dest.span, 'COPY destination');
+                if (isInline(ins.source) && dest.trailingSlash) {
+                    throw new SitefileError('E_PATH', `COPY of an inline body needs a file destination, not a directory`, ins.dest.span);
+                }
+                operations().push({
+                    kind: 'COPY',
+                    source: sourceRef(ins.source, vars, imports, ins.from),
+                    dest: dest.path,
+                    destIsDir: dest.trailingSlash,
+                    overwrite: ins.overwrite,
+                    ...(ins.exclude ? { exclude: ins.exclude } : {}),
+                    ...(ins.chmod !== undefined ? { chmod: ins.chmod } : {}),
+                    ...(ins.parents ? { parents: true } : {}),
+                    span: ins.span,
+                });
+                break;
+            }
+            case 'RENDER': {
+                if (ins.each) {
+                    operations().push({
+                        kind: 'RENDER',
+                        template: sourceRef(ins.template, vars, imports, ins.from),
+                        dest: '',
+                        each: {
+                            as: ins.each.as,
+                            list: ins.each.list,
+                            dest: ins.dest,
+                            ...(ins.each.by ? { by: ins.each.by.key } : {}),
+                            ...(ins.each.page
+                                ? { page: { size: pageSize(ins.each.page.size, vars), ...(ins.each.page.first ? { first: ins.each.page.first } : {}) } }
+                                : {}),
+                        },
+                        escape: ins.escape,
+                        overwrite: ins.overwrite,
+                        ...(ins.inlineSvg ? { inlineSvg: true } : {}),
+                        span: ins.span,
+                    });
+                    break;
+                }
+                const raw = evaluateOperand(ins.dest, vars);
+                const dest = normalizeVirtual(raw, ins.dest.span, 'RENDER destination');
+                if (dest.trailingSlash) {
+                    throw new SitefileError('E_PATH', `RENDER destination '${raw}' must be a file path, not a directory`, ins.dest.span);
+                }
+                operations().push({
+                    kind: 'RENDER',
+                    template: sourceRef(ins.template, vars, imports, ins.from),
+                    dest: dest.path,
+                    escape: ins.escape,
+                    overwrite: ins.overwrite,
+                    ...(ins.inlineSvg ? { inlineSvg: true } : {}),
+                    span: ins.span,
+                });
+                break;
+            }
+            case 'PLUGIN': {
+                const raw = evaluateOperand(ins.uri, vars);
+                if (URL_RE.test(raw)) {
+                    throw new SitefileError('E_PROTOCOL', `PLUGIN ${ins.name}: plugins must be local files in the build context, not URLs`, ins.uri.span);
+                }
+                const rel = normalizeLocal(raw, ins.uri.span).path;
+                if (!/\.(mjs|wasm)$/.test(rel)) {
+                    throw new SitefileError('E_VALIDATION', `PLUGIN ${ins.name}: '${raw}' must be an ES module (.mjs) or a WebAssembly module (.wasm)`, ins.uri.span);
+                }
+                plugins.push({ name: ins.name, path: rel, ...(ins.integrity ? { integrity: ins.integrity } : {}), span: ins.span });
+                break;
+            }
+            case 'TOOL':
+                tools.push({ name: ins.name, span: ins.span });
+                break;
+            case 'TRANSFORM':
+                operations().push({
+                    kind: 'TRANSFORM',
+                    ...(ins.source ? { source: sourceRef(ins.source, vars, imports, ins.from) } : {}),
+                    plugin: ins.plugin,
+                    dest: ins.dest,
+                    options: ins.options.map((o) => ({ key: o.key, value: o.value })),
+                    ...(ins.each ? { each: { as: ins.each.as, list: ins.each.list } } : {}),
+                    ...(ins.args ? { args: ins.args } : {}),
+                    ...(ins.include ? { include: ins.include } : {}),
+                    ...(ins.exclude ? { exclude: ins.exclude } : {}),
+                    overwrite: ins.overwrite,
+                    span: ins.span,
+                });
+                break;
+            case 'SITEMAP': {
+                const base = checkBaseUrl(evaluateOperand(ins.base, vars), 'SITEMAP BASE', ins.base.span);
+                operations().push({ kind: 'SITEMAP', base, dest: fileDest(ins.dest, vars, '/sitemap.xml', 'SITEMAP'), overwrite: ins.overwrite, span: ins.span });
+                break;
+            }
+            case 'FINGERPRINT': {
+                const patterns = ins.patterns.map((o) => checkPattern(evaluateOperand(o, vars), o.span));
+                const manifest = ins.manifest ? fileDest(ins.manifest, vars, '', 'FINGERPRINT MANIFEST') : undefined;
+                // `root` is the EXPORT path, filled in below (EXPORT is the last instruction).
+                operations().push({ kind: 'FINGERPRINT', patterns, ...(manifest !== undefined ? { manifest } : {}), root: '/', overwrite: ins.overwrite, span: ins.span });
+                break;
+            }
+            case 'CHECKLINKS':
+                // `root` and `redirects` are filled in below, once every line is planned.
+                operations().push({ kind: 'CHECKLINKS', warn: ins.warn, external: ins.external, root: '/', redirects: [], span: ins.span });
+                break;
+            case 'FEED': {
+                const base = checkBaseUrl(evaluateOperand(ins.base, vars), 'FEED BASE', ins.base.span);
+                operations().push({
+                    kind: 'FEED',
+                    list: ins.list,
+                    base,
+                    link: ins.link,
+                    dest: fileDest(ins.dest, vars, '/feed.xml', 'FEED'),
+                    title: ins.title ? evaluateOperand(ins.title, vars) : base,
+                    overwrite: ins.overwrite,
+                    span: ins.span,
+                });
+                break;
+            }
+            case 'REDIRECT':
+                redirects.push({
+                    from: checkSitePath(evaluateOperand(ins.from, vars), 'REDIRECT source', ins.from.span),
+                    to: checkRedirectTarget(evaluateOperand(ins.to, vars), ins.to.span),
+                    status: ins.status,
+                    overwrite: ins.overwrite,
+                    span: ins.span,
+                });
+                break;
+            case 'HEADER': {
+                const name = checkHeaderName(evaluateOperand(ins.name, vars), ins.name.span);
+                headers.push({
+                    pattern: checkSitePath(evaluateOperand(ins.pattern, vars), 'HEADER path pattern', ins.pattern.span),
+                    name,
+                    value: checkHeaderValue(ins.value.map((v) => evaluateOperand(v, vars)).join(' '), name, ins.value[0].span),
+                    overwrite: ins.overwrite,
+                    span: ins.span,
+                });
+                break;
+            }
+            case 'EXPORT':
+                exportPath = normalizeVirtual(evaluateOperand(ins.path, vars), ins.path.span, 'EXPORT path').path;
+                exportFormat = ins.format;
+                exportBase = ins.base ? evaluateOperand(ins.base, vars) : undefined;
+                if (ins.at && ins.format !== 'oci')
+                    throw new SitefileError('E_VALIDATION', 'EXPORT ... AT applies only to AS oci', ins.at.span);
+                exportRoot = ins.at ? normalizeVirtual(evaluateOperand(ins.at, vars), ins.at.span, 'EXPORT AT path').path : undefined;
+                if (ins.compression && ins.format !== 'oci')
+                    throw new SitefileError('E_VALIDATION', 'EXPORT ... COMPRESSION applies only to AS oci', ins.compression.span);
+                exportCompression = ins.compression?.value;
+                break;
+        }
+    }
+    for (const st of stages) {
+        st.operations.forEach((op, i) => {
+            if (op.kind === 'FINGERPRINT')
+                st.operations[i] = { ...op, root: exportPath ?? '/' };
+            if (op.kind === 'CHECKLINKS')
+                st.operations[i] = { ...op, root: exportPath ?? '/', redirects: redirects.map((r) => r.from) };
+        });
+    }
+    // An unnamed stage can be neither a FROM base nor a COPY --from source; only the last one is useful (EXPORT reads it).
+    for (const st of stages.slice(0, -1)) {
+        if (st.name === undefined) {
+            warnings.push(`${program.file}:${st.span.line}: unnamed stage is not the last one, so its output can never be used; name it with FROM … AS <name>`);
+        }
+    }
+    for (const name of Object.keys(buildArgs)) {
+        if (!declaredArgs.has(name))
+            warnings.push(`--build-arg ${name} does not match any ARG and was ignored`);
+    }
+    if (stages.length === 0 || exportPath === undefined) {
+        // validate() guarantees both; this keeps the types honest.
+        throw new SitefileError('E_VALIDATION', 'a Sitefile needs FROM and EXPORT');
+    }
+    return { file: program.file, stages, vars, fragments, imports: [...imports.values()], plugins, tools, exportPath, exportFormat, exportBase, exportRoot, exportCompression, redirects, headers, warnings };
+}
+/**
+ * `--target <name>`: keep the stages up to and including the named one, and export that stage's whole tree
+ * (`/`) instead of the EXPORT path. The EXPORT's format and its options (AS, BASE, AT, COMPRESSION) still apply;
+ * the `/_redirects` and `/_headers` files EXPORT writes are not (they belong to the EXPORT path). An unknown
+ * name is a usage error that lists the named stages.
+ */
+export function targetPlan(p, name) {
+    const index = p.stages.findIndex((s) => s.name === name);
+    if (index === -1) {
+        const names = p.stages.flatMap((s) => (s.name === undefined ? [] : [s.name]));
+        throw new SitefileError('E_USAGE', names.length
+            ? `--target '${name}' does not name a stage (stages: ${names.join(', ')})`
+            : `--target '${name}' does not name a stage (the Sitefile has no named stages; name one with FROM … AS <name>)`);
+    }
+    return { ...p, stages: p.stages.slice(0, index + 1), exportPath: '/', redirects: [], headers: [], target: name };
+}
+/** `IMPORT … SCHEMA <path>`: a local `.json`, `.yaml` or `.yml` file in the build context. */
+function schemaSpec(name, op, vars, span) {
+    const uri = evaluateOperand(op, vars);
+    if (URL_RE.test(uri)) {
+        throw new SitefileError('E_PROTOCOL', `IMPORT ${name}: SCHEMA must be a local file in the build context, not a URL`, op.span);
+    }
+    const rel = normalizeLocal(uri, op.span).path;
+    if (!/\.(json|ya?ml)$/i.test(rel)) {
+        throw new SitefileError('E_VALIDATION', `IMPORT ${name}: SCHEMA '${uri}' must be a .json, .yaml or .yml file`, op.span);
+    }
+    return { uri, path: rel, span };
+}
+/** `IMPORT … DRAFTS <value>`: `true` keeps drafts, `false` leaves them out. */
+function draftsValue(name, op, vars) {
+    const raw = evaluateOperand(op, vars);
+    if (raw === 'true' || raw === 'false')
+        return raw === 'true';
+    throw new SitefileError('E_VALIDATION', `IMPORT ${name}: DRAFTS must be true or false, got '${raw}'`, op.span);
+}
+/** `RENDER … PAGE <n>`: a whole number of items per page, at least 1. */
+function pageSize(op, vars) {
+    const raw = evaluateOperand(op, vars);
+    if (!/^[1-9][0-9]{0,8}$/.test(raw)) {
+        throw new SitefileError('E_VALIDATION', `RENDER … PAGE: '${raw}' is not a page size (a whole number from 1 to 999999999)`, op.span);
+    }
+    return Number(raw);
+}
+/** `TO <dest>` of SITEMAP/FEED or `MANIFEST <dest>` of FINGERPRINT (a file path), or the default. */
+function fileDest(op, vars, fallback, what) {
+    if (!op)
+        return fallback;
+    const raw = evaluateOperand(op, vars);
+    const dest = normalizeVirtual(raw, op.span, `${what} destination`);
+    if (dest.trailingSlash)
+        throw new SitefileError('E_PATH', `${what} destination '${raw}' must be a file path, not a directory`, op.span);
+    return dest.path;
+}
+function classifyUri(uri, span) {
+    if (isGitUri(uri))
+        return { kind: 'git', ...parseGitUri(uri, span) };
+    const m = URL_RE.exec(uri);
+    if (m) {
+        const scheme = m[1].toLowerCase();
+        if (scheme === 'https' || scheme === 'http')
+            return { kind: 'remote', url: uri };
+        if (scheme === 'file') {
+            throw new SitefileError('E_PROTOCOL', `file: URLs are not allowed; use a path relative to the build context`, span);
+        }
+        throw new SitefileError('E_PROTOCOL', `unsupported protocol '${scheme}:' (supported: https, git+https, relative paths)`, span);
+    }
+    return { kind: 'local', path: normalizeLocal(uri, span).path };
+}
+/** `image <ref> [AT <dir>]`: the reference must parse; a reference without a registry host names a local-store image. */
+function imageBase(b, vars, lead) {
+    const ref = evaluateOperand(b.ref, vars);
+    let local;
+    try {
+        local = parseRef(ref).registry === undefined;
+    }
+    catch (e) {
+        if (e instanceof SitefileError)
+            throw new SitefileError('E_VALIDATION', `${lead} image: ${e.detail}`, b.ref.span, { cause: e });
+        throw e;
+    }
+    const at = b.at ? normalizeVirtual(evaluateOperand(b.at, vars), b.at.span, 'image directory').path : undefined;
+    return { kind: 'image', ref, local, ...(at !== undefined ? { at } : {}), lead, span: b.ref.span };
+}
+/** A bare identifier naming an earlier stage is a stage; anything else is a context-relative directory. */
+function stageBase(op, vars, earlier, all, imports, lead = 'FROM') {
+    const raw = evaluateOperand(op, vars);
+    if (earlier.some((s) => s.name === raw))
+        return { kind: 'stage', name: raw };
+    // `<import>+<stage>`: a stage of a Sitefile imported AS sitefile on an earlier line.
+    const imported = importedStage(raw);
+    if (imported && imports.get(imported.import)?.as === 'sitefile')
+        return { kind: 'stage', name: raw };
+    if (all.has(raw)) {
+        throw new SitefileError('E_UNDEFINED', `${lead} ${raw}: stage '${raw}' is not defined before this line (write ./${raw} for a local directory)`, op.span);
+    }
+    return { kind: 'local', path: normalizeLocal(raw, op.span).path };
+}
+function sourceRef(op, vars, imports, from) {
+    if (isInline(op))
+        return { kind: 'inline', text: op.text, delimiter: op.delimiter, bodyLine: op.bodyLine, span: op.span };
+    const raw = evaluateOperand(op, vars);
+    if (from)
+        return { kind: 'stage', name: from.name, path: normalizeVirtual(raw, op.span, `path inside stage '${from.name}'`).path, span: op.span };
+    if (URL_RE.test(raw)) {
+        throw new SitefileError('E_VALIDATION', `cannot read '${raw}' directly; IMPORT it under a name, then COPY name:`, op.span);
+    }
+    const m = IMPORT_REF_RE.exec(raw);
+    if (m) {
+        const name = m[1];
+        const inner = m[2];
+        if (!imports.has(name)) {
+            throw new SitefileError('E_UNDEFINED', `'${name}:' does not name an IMPORT; write './${raw}' for a local file`, op.span);
+        }
+        const path = inner === '' ? '/' : normalizeVirtual(inner, op.span, `path inside import '${name}'`).path;
+        return { kind: 'import', name, path, span: op.span };
+    }
+    return { kind: 'local', path: normalizeLocal(raw, op.span).path, span: op.span };
+}
+/** Every reference to a stage of an imported Sitefile, in source order (stage bases, then `--from` sources, per stage). */
+export function importedStageRefs(stages) {
+    const out = [];
+    const add = (name, span) => {
+        const s = importedStage(name);
+        if (s)
+            out.push({ name, import: s.import, stage: s.stage, span });
+    };
+    for (const st of stages) {
+        if (st.base.kind === 'stage')
+            add(st.base.name, st.span);
+        for (const op of st.operations) {
+            const src = op.kind === 'COPY' ? op.source : op.kind === 'RENDER' ? op.template : op.kind === 'TRANSFORM' ? op.source : undefined;
+            if (src?.kind === 'stage')
+                add(src.name, op.span);
+        }
+    }
+    return out;
+}
+//# sourceMappingURL=planner.js.map

package/lib/src/plugins/cache.d.ts

@@ -0,0 +1,70 @@
+import type { PluginResult } from './host.js';
+/**
+ * Content-addressed cache of TRANSFORM results (see docs/PLUGINS.md, "Caching").
+ *
+ *   <dir>/entries/<key>.json   manifest: which blobs make up the result
+ *   <dir>/blobs/<hex>          result bytes, named by their SHA-256
+ *
+ * The key is the SHA-256 of a canonical JSON document describing everything the
+ * result can depend on. Reads verify every blob's digest and size; a damaged
+ * entry is removed and counts as a miss, so the next put rewrites it.
+ */
+/** Bump to invalidate every existing entry when the key or entry format changes. */
+export declare const CACHE_VERSION = 1;
+/** What an entry records: the result kind and `[path, blob hex, size]` per file (a `file` result has one, with path ''). */
+export interface CacheEntry {
+    readonly kind: 'file' | 'files';
+    readonly files: ReadonlyArray<readonly [string, string, number]>;
+}
+/**
+ * A second-level cache consulted on a local miss (`--cache-from`, see src/registry-cache.ts), and told
+ * about every entry this build used or wrote (`--cache-to`).
+ */
+export interface RemoteCache {
+    /** Fetch the entry for `key` and its blobs (written verified to `blobPath(hex)`), or undefined. Never throws. */
+    fetch(key: string, blobPath: (hex: string) => string): Promise<CacheEntry | undefined>;
+    used(key: string, entry: CacheEntry): void;
+}
+export interface CacheStats {
+    hits: number;
+    misses: number;
+    writes: number;
+    /** Entries found damaged (and discarded). */
+    corrupt: number;
+}
+export interface CacheInfo {
+    readonly dir: string;
+    readonly entries: number;
+    readonly blobs: number;
+    readonly bytes: number;
+}
+/** JSON with sorted object keys, so equal values always serialize equally. */
+export declare function canonicalJson(value: unknown): string;
+export declare function cacheKey(parts: object): string;
+export declare class TransformCache {
+    readonly dir: string;
+    private readonly remote?;
+    readonly stats: CacheStats;
+    constructor(dir: string, remote?: RemoteCache | undefined);
+    private entryPath;
+    private blobPath;
+    get(key: string): Promise<PluginResult | undefined>;
+    private load;
+    put(key: string, result: PluginResult): Promise<void>;
+    private writeAtomic;
+    info(): Promise<CacheInfo>;
+    /**
+     * Remove entries not used for `olderThanMs` (0 removes all), then every blob
+     * no remaining entry references. Returns what was removed.
+     */
+    prune(olderThanMs?: number, now?: number): Promise<{
+        entries: number;
+        blobs: number;
+        bytes: number;
+    }>;
+}
+/** Digest of every regular file under `dir` (a plugin can read all of them, so any may change its output). */
+export declare function digestFolder(dir: string, limits?: {
+    files: number;
+    bytes: number;
+}): Promise<string | undefined>;