@johnhenry/sitefile 0.0.0-stage → 0.0.0

package/lib/src/language/validate.js

@@ -0,0 +1,367 @@
+import { SitefileError } from '../errors.js';
+import { FEED_ITEM, IDENT_RE, importedStage, isFragment, isImageBase, isInline, literalText, stageNames } from './ast.js';
+import { parseSitefile } from './parser.js';
+import { fragmentNotValue, freeRefs, parseRenderTemplate } from '../render.js';
+/** A fragment's template: a heredoc body or partial file without its final newline (it is inserted inline). */
+export function fragmentText(text) {
+    return text.replace(/\r?\n$/, '');
+}
+/** Parse the template of an html LET (errors point into the Sitefile). */
+export function parseFragmentBody(body, file) {
+    return body.kind === 'line' ? parseRenderTemplate(body.text, file, body.span.line, body.span.column) : parseRenderTemplate(fragmentText(body.text), file, body.bodyLine);
+}
+/** Import kinds that can produce a list (and so can be sorted). */
+const SORTABLE = new Set(['markdown', 'json', 'yaml', 'csv']);
+const IMPORT_REF = /^([A-Za-z_][A-Za-z0-9_]*):/;
+export function operandsOf(ins) {
+    switch (ins.kind) {
+        case 'FROM':
+            return ins.base === 'scratch' ? [] : isImageBase(ins.base) ? [ins.base.ref, ...(ins.base.at ? [ins.base.at] : [])] : [ins.base];
+        case 'ARG':
+            return ins.defaultValue ? [ins.defaultValue] : [];
+        case 'LET':
+            return isFragment(ins) ? [] : [ins.value];
+        case 'IMPORT':
+            return [ins.uri, ...(ins.schema ? [ins.schema.path] : []), ...(ins.drafts ? [ins.drafts.value] : []), ...(ins.args ?? []).map((a) => a.value)];
+        case 'COPY':
+            return isInline(ins.source) ? [ins.dest] : [ins.source, ins.dest];
+        case 'RENDER': {
+            const page = ins.each?.page;
+            const extra = page ? [page.size, ...(page.first ? [page.first] : [])] : [];
+            return [...(isInline(ins.template) ? [] : [ins.template]), ins.dest, ...extra];
+        }
+        case 'PLUGIN':
+            return [ins.uri];
+        case 'TRANSFORM': {
+            const ops = ins.source && !isInline(ins.source) ? [ins.source] : [];
+            return [...ops, ins.dest, ...ins.options.map((o) => o.value), ...(ins.args ?? [])];
+        }
+        case 'TOOL':
+            return [];
+        case 'SITEMAP':
+            return [ins.base, ...(ins.dest ? [ins.dest] : [])];
+        case 'FEED':
+            return [ins.base, ins.link, ...(ins.dest ? [ins.dest] : []), ...(ins.title ? [ins.title] : [])];
+        case 'REDIRECT':
+            return [ins.from, ins.to];
+        case 'HEADER':
+            return [ins.pattern, ins.name, ...ins.value];
+        case 'FINGERPRINT':
+            return [...ins.patterns, ...(ins.manifest ? [ins.manifest] : [])];
+        case 'CHECKLINKS':
+            return [];
+        case 'EXPORT':
+            return [ins.path];
+    }
+}
+/** The only placeholders the host replaces inside TOOL arguments; any other `{…}` stays literal (e.g. ffmpeg's `%{pts}`). */
+export const TOOL_PLACEHOLDER_RE = /\{(input|output|option\.[A-Za-z_][A-Za-z0-9_-]*)\}/g;
+function checkToolArgs(ins) {
+    if (!ins.args || ins.args.length === 0) {
+        throw new SitefileError('E_VALIDATION', `TRANSFORM … WITH ${ins.plugin}: a TOOL needs ARGS (the argument list, e.g. ARGS -i {input} {output})`, ins.span);
+    }
+    let usesOutput = false;
+    for (const arg of ins.args) {
+        for (const part of arg.parts) {
+            if (part.kind !== 'text')
+                continue;
+            for (const m of part.value.matchAll(TOOL_PLACEHOLDER_RE)) {
+                const name = m[1];
+                if (name === 'output')
+                    usesOutput = true;
+                else if (name === 'input') {
+                    if (!ins.source)
+                        throw new SitefileError('E_VALIDATION', `{input} needs a TRANSFORM source file`, arg.span);
+                }
+                else if (!ins.options.some((o) => `option.${o.key}` === name)) {
+                    throw new SitefileError('E_UNDEFINED', `{${name}}: no OPTIONS ${name.slice(7)}=… on this TRANSFORM`, arg.span);
+                }
+            }
+        }
+    }
+    if (!usesOutput)
+        throw new SitefileError('E_VALIDATION', `TRANSFORM … WITH ${ins.plugin}: ARGS must contain {output}, the file the tool writes`, ins.span);
+}
+/** Check semantic rules (see docs/LANGUAGE.md, "v0.1 implementation decisions"). */
+export function validate(instructions, file) {
+    if (instructions.length === 0)
+        throw new SitefileError('E_VALIDATION', 'Sitefile has no instructions', { file, line: 1, column: 1 });
+    const vars = new Set();
+    const imports = new Set();
+    // Stage names: every alias in the file, those already declared, and those whose stage is complete.
+    const allStages = stageNames(instructions);
+    const declaredStages = new Set();
+    const finishedStages = new Set();
+    let currentStage;
+    const plugins = new Set();
+    const tools = new Set();
+    /** html LETs and `IMPORT … AS html`: usable only inside RENDER templates. */
+    const fragments = new Set();
+    /** IMPORTs `AS sitefile`: `<name>+<stage>` names a stage of the imported Sitefile. */
+    const sitefiles = new Set();
+    /** One namespace for ARG/LET, IMPORT, stage and PLUGIN names. */
+    const taken = (name) => vars.has(name) || imports.has(name) || declaredStages.has(name) || plugins.has(name) || tools.has(name) || fragments.has(name);
+    // Where each ARG, LET and IMPORT name is first declared (a fragment may only use names declared before it).
+    const declaredAt = new Map();
+    for (const ins of instructions) {
+        if ((ins.kind === 'ARG' || ins.kind === 'LET' || ins.kind === 'IMPORT') && !declaredAt.has(ins.name))
+            declaredAt.set(ins.name, ins.span.line);
+    }
+    /** Not a value a template can read. */
+    const notValue = (name) => declaredStages.has(name) || plugins.has(name) || tools.has(name);
+    let sawFrom = false;
+    let sawExport = false;
+    const out = [];
+    for (const ins of instructions) {
+        // The first instruction other than ARG, with no FROM yet, starts an implicit unnamed `FROM scratch`
+        // stage. It has no source line of its own, so it takes that instruction's span.
+        // An IMPORT … AS sitefile may come first too, so a file can start FROM one of its stages.
+        if (!sawFrom && ins.kind !== 'FROM' && ins.kind !== 'ARG' && !(ins.kind === 'IMPORT' && ins.as === 'sitefile')) {
+            out.push({ kind: 'FROM', span: ins.span, base: 'scratch' });
+            sawFrom = true;
+        }
+        out.push(ins);
+        if (sawExport)
+            throw new SitefileError('E_VALIDATION', 'EXPORT must be the last instruction', ins.span);
+        // RENDER … FOR item IN list: the destination may use ${item} / ${item.key}.
+        // RENDER/TRANSFORM … FOR item IN list: the destination (and TRANSFORM options) may use ${item.key}.
+        // FEED … LINK <path>: the link may use ${item.key} for the current element.
+        const loopVar = ins.kind === 'RENDER' || ins.kind === 'TRANSFORM' ? ins.each?.as : ins.kind === 'FEED' ? FEED_ITEM : undefined;
+        for (const op of operandsOf(ins)) {
+            const isDest = (ins.kind === 'FEED' && op === ins.link) ||
+                (ins.kind === 'RENDER' && (op === ins.dest || op === ins.each?.page?.first)) ||
+                (ins.kind === 'TRANSFORM' && (op === ins.dest || ins.options.some((o) => o.value === op) || (ins.args?.includes(op) ?? false)));
+            for (const part of op.parts) {
+                if (part.kind !== 'var')
+                    continue;
+                if (isDest && part.name === loopVar)
+                    continue;
+                if (part.path.length > 0) {
+                    throw new SitefileError('E_SYNTAX', `dotted access \${${[part.name, ...part.path].join('.')}} is only allowed in RENDER templates and for a RENDER … FOR loop variable`, part.span);
+                }
+                if (fragments.has(part.name)) {
+                    throw new SitefileError('E_TYPE', `'${part.name}' is an html fragment; html values can only be used in RENDER templates`, part.span);
+                }
+                if (vars.has(part.name))
+                    continue;
+                const hint = imports.has(part.name) ? ` ('${part.name}' is an import; use it in a RENDER template)` : '';
+                throw new SitefileError('E_UNDEFINED', `undefined variable \${${part.name}}: declare it with ARG or LET on an earlier line${hint}`, part.span);
+            }
+        }
+        if (ins.kind === 'FROM') {
+            if (currentStage !== undefined)
+                finishedStages.add(currentStage);
+            if (ins.base !== 'scratch' && !isImageBase(ins.base)) {
+                // A bare word naming a stage must name an earlier one; './name' always means a directory.
+                const text = literalText(ins.base);
+                const imported = text === undefined ? undefined : importedStage(text);
+                // `<import>+<stage>` of an IMPORT … AS sitefile: whether the stage exists is checked when that Sitefile is read.
+                if (!(imported && sitefiles.has(imported.import)) && text !== undefined && IDENT_RE.test(text) && allStages.has(text) && !finishedStages.has(text)) {
+                    throw new SitefileError('E_UNDEFINED', `${ins.keyword === 'STAGE' ? `STAGE ${ins.alias} FROM` : 'FROM'} ${text}: stage '${text}' is not defined before this line (write ./${text} for a local directory)`, ins.base.span);
+                }
+            }
+            if (ins.alias !== undefined) {
+                if (taken(ins.alias)) {
+                    throw new SitefileError('E_DUPLICATE', `'${ins.alias}' is already defined`, ins.span);
+                }
+                declaredStages.add(ins.alias);
+            }
+            currentStage = ins.alias;
+            sawFrom = true;
+            continue;
+        }
+        switch (ins.kind) {
+            case 'ARG':
+            case 'LET':
+                if (taken(ins.name)) {
+                    throw new SitefileError('E_DUPLICATE', `'${ins.name}' is already defined`, ins.span);
+                }
+                if (isFragment(ins)) {
+                    checkFragment(ins, file, { vars, imports, fragments, notValue, declaredAt });
+                    fragments.add(ins.name);
+                }
+                else {
+                    vars.add(ins.name);
+                }
+                break;
+            case 'IMPORT':
+                if (taken(ins.name)) {
+                    throw new SitefileError('E_DUPLICATE', `'${ins.name}' is already defined`, ins.span);
+                }
+                if (ins.sort && !SORTABLE.has(ins.as ?? '')) {
+                    throw new SitefileError('E_VALIDATION', `IMPORT ${ins.name}: SORT BY applies to lists: AS markdown, json, yaml or csv`, ins.sort.span);
+                }
+                if (ins.drafts && !SORTABLE.has(ins.as ?? '')) {
+                    throw new SitefileError('E_VALIDATION', `IMPORT ${ins.name}: DRAFTS applies to lists: AS markdown, json, yaml or csv`, ins.drafts.span);
+                }
+                if (ins.schema && !SORTABLE.has(ins.as ?? '')) {
+                    throw new SitefileError('E_VALIDATION', `IMPORT ${ins.name}: SCHEMA applies to documents and lists: AS markdown, json, yaml or csv`, ins.schema.span);
+                }
+                // `AS html`: a file-based fragment, usable only in templates (still an import for `name:` sources).
+                if (ins.as === 'html')
+                    fragments.add(ins.name);
+                if (ins.args && ins.as !== 'sitefile') {
+                    throw new SitefileError('E_VALIDATION', `IMPORT ${ins.name}: ARG sets a build arg of an imported Sitefile, so it needs AS sitefile`, ins.args[0].span);
+                }
+                if (ins.as === 'sitefile') {
+                    if (ins.integrity) {
+                        throw new SitefileError('E_VALIDATION', `IMPORT ${ins.name}: INTEGRITY does not apply to AS sitefile (pin a Git import by its commit, #<40 hex>[:path])`, ins.span);
+                    }
+                    sitefiles.add(ins.name);
+                }
+                imports.add(ins.name);
+                break;
+            case 'COPY':
+                checkSource(ins.source, imports, ins.from, finishedStages, allStages, sitefiles);
+                break;
+            case 'RENDER':
+                checkSource(ins.template, imports, ins.from, finishedStages, allStages, sitefiles);
+                if (ins.each) {
+                    const { as, list, span } = ins.each;
+                    if (fragments.has(list))
+                        throw notAList(as, list, span);
+                    if (!imports.has(list)) {
+                        throw new SitefileError('E_UNDEFINED', `FOR ${as} IN ${list}: '${list}' is not an IMPORT declared on an earlier line`, span);
+                    }
+                    notSitefile(list, `FOR ${as} IN ${list}`, span, sitefiles);
+                    if (taken(as) || allStages.has(as)) {
+                        throw new SitefileError('E_DUPLICATE', `FOR ${as}: '${as}' is already defined; pick another loop variable name`, span);
+                    }
+                }
+                break;
+            case 'PLUGIN':
+                if (taken(ins.name))
+                    throw new SitefileError('E_DUPLICATE', `'${ins.name}' is already defined`, ins.span);
+                plugins.add(ins.name);
+                break;
+            case 'TOOL':
+                if (taken(ins.name))
+                    throw new SitefileError('E_DUPLICATE', `'${ins.name}' is already defined`, ins.span);
+                tools.add(ins.name);
+                break;
+            case 'TRANSFORM':
+                if (tools.has(ins.plugin))
+                    checkToolArgs(ins);
+                else if (!plugins.has(ins.plugin)) {
+                    throw new SitefileError('E_UNDEFINED', `TRANSFORM … WITH ${ins.plugin}: no PLUGIN or TOOL ${ins.plugin} is declared on an earlier line`, ins.pluginSpan);
+                }
+                else if (ins.args) {
+                    throw new SitefileError('E_VALIDATION', `ARGS applies to a TOOL; ${ins.plugin} is a PLUGIN (pass values with OPTIONS)`, ins.span);
+                }
+                // With no source and no FOR the plugin gets only its OPTIONS (e.g. a badge or a generated file).
+                if (ins.source)
+                    checkSource(ins.source, imports, ins.from, finishedStages, allStages, sitefiles);
+                if (ins.each) {
+                    const { as, list, span } = ins.each;
+                    if (fragments.has(list))
+                        throw notAList(as, list, span);
+                    if (!imports.has(list)) {
+                        throw new SitefileError('E_UNDEFINED', `FOR ${as} IN ${list}: '${list}' is not an IMPORT declared on an earlier line`, span);
+                    }
+                    notSitefile(list, `FOR ${as} IN ${list}`, span, sitefiles);
+                    if (taken(as))
+                        throw new SitefileError('E_DUPLICATE', `FOR ${as}: '${as}' is already defined; pick another loop variable name`, span);
+                }
+                break;
+            case 'FEED':
+                if (fragments.has(ins.list)) {
+                    throw new SitefileError('E_TYPE', `FEED ${ins.list}: '${ins.list}' is an html fragment, not a list`, ins.listSpan);
+                }
+                if (!imports.has(ins.list)) {
+                    throw new SitefileError('E_UNDEFINED', `FEED ${ins.list}: '${ins.list}' is not an IMPORT declared on an earlier line`, ins.listSpan);
+                }
+                notSitefile(ins.list, `FEED ${ins.list}`, ins.listSpan, sitefiles);
+                break;
+            case 'SITEMAP':
+            case 'REDIRECT':
+            case 'HEADER':
+            case 'FINGERPRINT':
+            case 'CHECKLINKS':
+                break;
+            case 'EXPORT':
+                sawExport = true;
+                break;
+        }
+    }
+    const last = instructions[instructions.length - 1];
+    if (!sawExport)
+        throw new SitefileError('E_VALIDATION', 'missing EXPORT: the last instruction must be EXPORT', last.span);
+    return { file, instructions: out };
+}
+function notAList(as, list, span) {
+    return new SitefileError('E_TYPE', `FOR ${as} IN ${list}: '${list}' is an html fragment, not a list`, span);
+}
+/**
+ * An html LET is checked where it is declared: its template syntax, and every name it reads that
+ * is declared before it. A name declared later is an error (define before use); a name declared
+ * nowhere is left to the place it is used (a `RENDER … FOR` loop variable, say).
+ */
+function checkFragment(ins, file, s) {
+    for (const r of freeRefs(parseFragmentBody(ins.body, file))) {
+        if (r.name === ins.name)
+            throw new SitefileError('E_TEMPLATE', `html fragment '${ins.name}' uses itself`, r.span);
+        if (s.fragments.has(r.name)) {
+            if (r.use !== 'var' || r.path.length > 0)
+                throw fragmentNotValue(r.name, r.span);
+        }
+        else if (s.vars.has(r.name)) {
+            if ((r.use === 'var' || r.use === 'text') && r.path.length > 0)
+                throw new SitefileError('E_TYPE', `'${r.name}' is an ARG/LET value; dotted access only works on JSON imports`, r.span);
+        }
+        else if (s.imports.has(r.name)) {
+            // Its kind (and so whether a template can read it) is known once it is resolved.
+        }
+        else if (s.notValue(r.name)) {
+            throw new SitefileError('E_UNDEFINED', `'${r.name}' is not an ARG, LET or IMPORT`, r.span);
+        }
+        else {
+            const line = s.declaredAt.get(r.name);
+            if (line !== undefined) {
+                throw new SitefileError('E_UNDEFINED', `'${r.name}' is declared on line ${line}, after LET ${ins.name}: html; a fragment can only use names declared before it`, r.span);
+            }
+        }
+    }
+}
+/** A FOR or FEED list must be a value import, not a Sitefile imported AS sitefile. */
+function notSitefile(name, what, span, sitefiles) {
+    if (sitefiles.has(name))
+        throw new SitefileError('E_TYPE', `${what}: '${name}' is a Sitefile import, not a list`, span);
+}
+function checkSource(op, imports, from, finished, all, sitefiles) {
+    if (isInline(op))
+        return; // content is in the Sitefile itself
+    const imported = from ? importedStage(from.name) : undefined;
+    if (from && imported) {
+        // `--from=<import>+<stage>`: a stage of an imported Sitefile (whether it exists is checked when that Sitefile is read).
+        if (!sitefiles.has(imported.import)) {
+            throw new SitefileError('E_UNDEFINED', `--from=${from.name}: '${imported.import}' is not an IMPORT … AS sitefile declared on an earlier line`, from.span);
+        }
+        return;
+    }
+    if (from) {
+        // The source is a path inside the stage, not an import ref or local path.
+        if (!finished.has(from.name)) {
+            const why = all.has(from.name) ? 'is not complete before this line (a stage cannot read itself or a later stage)' : 'is not defined';
+            throw new SitefileError('E_UNDEFINED', `--from=${from.name}: stage '${from.name}' ${why}`, from.span);
+        }
+        return;
+    }
+    const text = literalText(op);
+    if (text === undefined)
+        return;
+    if (text.includes('://')) {
+        throw new SitefileError('E_VALIDATION', 'URL sources are not allowed here: use IMPORT then COPY', op.span);
+    }
+    const m = IMPORT_REF.exec(text);
+    if (m && !imports.has(m[1])) {
+        throw new SitefileError('E_UNDEFINED', `unknown import '${m[1]}' (declare it with IMPORT on an earlier line, or write ./${text} for a local path)`, op.span);
+    }
+    if (m && sitefiles.has(m[1])) {
+        throw new SitefileError('E_TYPE', `'${m[1]}' is a Sitefile import; read one of its stages with --from=${m[1]}+<stage>`, op.span);
+    }
+}
+/** Parse and validate a Sitefile. */
+export function parseProgram(source, file = 'Sitefile') {
+    return validate(parseSitefile(source, file), file);
+}
+//# sourceMappingURL=validate.js.map

package/lib/src/links.d.ts

@@ -0,0 +1,79 @@
+/** One URL found in a file. */
+export interface Reference {
+    /** Offsets of the URL as written in the scanned text (`raw` is `text.slice(start, end)`). */
+    readonly start: number;
+    readonly end: number;
+    /** The URL as written. */
+    readonly raw: string;
+    /** The URL with character references (HTML) or escapes (CSS) decoded and ASCII whitespace trimmed. */
+    readonly url: string;
+    /** 1-based line of `start`. */
+    readonly line: number;
+    /** Written as CSS (a CSS file, a <style> element or a `style` attribute), so CSS escapes apply. */
+    readonly css: boolean;
+}
+/** Decoded text, and for each of its offsets (and its end) the offset in the source it came from. */
+interface Mapped {
+    readonly text: string;
+    readonly map: readonly number[];
+}
+/** Decode the character references this scanner knows (see the top of the file); others stay as written. */
+export declare function decodeHtml(s: string): Mapped;
+export declare const decodeHtmlText: (s: string) => string;
+/** Decode CSS escapes: `\` + newline is removed, `\` + 1-6 hex digits (and one following whitespace) is a code point, `\` + anything else is that character. */
+export declare function decodeCss(s: string): Mapped;
+export declare const decodeCssText: (s: string) => string;
+/** The URLs a CSS file references, in order. */
+export declare function scanCss(text: string): Reference[];
+/** The URLs an HTML file references, in order (including those of its CSS). */
+export declare function scanHtml(text: string): Reference[];
+/** Whether a path is read by {@link scanFile}, and how. */
+export declare function referenceKind(path: string): 'html' | 'css' | undefined;
+/** A file's text and references; undefined when it is not HTML or CSS, or not valid UTF-8. */
+export declare function scanFile(path: string, bytes: Uint8Array): {
+    text: string;
+    refs: Reference[];
+} | undefined;
+/** What a reference points at. */
+export type Target = 
+/** Nothing to check: empty, a fragment or query of the same document, or a non-http(s) scheme (mailto:, data:, …). */
+{
+    readonly kind: 'none';
+}
+/** An http(s) URL or a protocol-relative `//host/…`. */
+ | {
+    readonly kind: 'external';
+    readonly url: string;
+}
+/**
+ * A file of the site: `path` is absolute, normalized and NFC (`..` stops at the root), percent-decoded;
+ * `dir` when the URL ends with `/` (or `.`/`..`), so it names a directory.
+ */
+ | {
+    readonly kind: 'internal';
+    readonly path: string;
+    readonly dir: boolean;
+};
+/** Percent-decode one path segment; a segment whose decoding is not valid UTF-8 stays as written. */
+export declare function percentDecode(seg: string): string;
+/**
+ * Resolve a reference's URL against the path of the file that contains it. `root` is the directory the
+ * site is served from (the EXPORT path): an absolute URL path (`/img/a.png`) names a file below it, and
+ * `..` never climbs above it (as in a browser), for references in files below it; elsewhere `..` stops at `/`.
+ */
+export declare function resolveReference(url: string, from: string, root?: string): Target;
+/** Percent-encode a path, keeping `/` and the URL-unreserved characters `A-Z a-z 0-9 - . _ ~`. */
+export declare function encodePath(p: string): string;
+/** A relative URL from the file `from` to the file `to` (both absolute tree paths), percent-encoded. */
+export declare function relativeUrl(from: string, to: string): string;
+/**
+ * The reference `raw` (written in `from`, resolved to `oldPath`) pointing at `newPath` instead, where only the
+ * file name changed by the insertion of `.<hash>`. The hash is inserted into the name as written, before the
+ * dot of its extension (or after the name), so the directories, the query, the fragment and the way it is
+ * written (escapes, character references, percent-encoding) are kept. When the name can't be edited that way
+ * (its extension's dot is percent-encoded, say), the path becomes a relative URL from `from` to `newPath`.
+ */
+export declare function renameReference(raw: string, css: boolean, from: string, oldPath: string, newPath: string): string;
+/** `text` with each reference whose target `rename` maps to a new path rewritten; undefined when nothing changed. */
+export declare function rewriteReferences(text: string, refs: readonly Reference[], from: string, root: string, rename: (path: string) => string | undefined): string | undefined;
+export {};