@johnhenry/sitefile 0.0.0-stage → 0.0.0

package/lib/src/language/scanner.d.ts

@@ -0,0 +1,110 @@
+import { SitefileError, type ErrorCode, type SourceSpan } from '../errors.js';
+import type { InlineBody, Operand } from './ast.js';
+export interface Pos {
+    readonly line: number;
+    readonly column: number;
+}
+/** One instruction's text after continuations are joined, with a source position per character. */
+export interface LogicalLine {
+    readonly text: string;
+    readonly positions: readonly Pos[];
+    /** Position just after the last character (for "expected X" errors at end of line). */
+    readonly end: Pos;
+    /** A heredoc body that followed this line (COPY, RENDER, TRANSFORM and html LET only). */
+    readonly heredoc?: Heredoc;
+}
+export interface Heredoc {
+    /** Index in `text` of the `<<` marker. */
+    readonly at: number;
+    /** Length of the marker text (`<<-NAME`). */
+    readonly length: number;
+    readonly delimiter: string;
+    readonly text: string;
+    readonly bodyLine: number;
+}
+/** A comment line outside heredoc bodies: `text` runs from its `#` to the end of the line. */
+export interface CommentLine {
+    readonly line: number;
+    /** Column of the `#`. */
+    readonly column: number;
+    readonly text: string;
+}
+/**
+ * Whether a logical line declares an html fragment, `LET name: html …` (the type ASCII
+ * case-insensitive), whose value may be a heredoc. Other LET lines are not scanned for `<<`.
+ */
+export declare function isHtmlLet(text: string): boolean;
+/** The language version this implementation reads (`# sitefile=1`; a file without the directive is this version). */
+export declare const SYNTAX_VERSION = "1";
+/**
+ * The version directive `# sitefile=N` (name case-insensitive, spaces or tabs around `=`), matched on a
+ * comment line from its `#`. Group 1 is everything after `=`.
+ * Other comments, `# sitefile-lint: …` included, are ordinary comments here.
+ */
+export declare const VERSION_DIRECTIVE: RegExp;
+/**
+ * Split source into logical lines. Blank lines and comment lines (first non-blank
+ * char `#`) are skipped, also inside continuations. A line ending in an odd number
+ * of `\` continues; pieces are trimmed at the join and joined with one space.
+ * A `# sitefile=N` comment is the syntax version directive: it must come before
+ * the first instruction, at most once, and name a supported version.
+ * `comments`, when given, receives every comment line (for `# sitefile-lint:`).
+ */
+export declare function logicalLines(source: string, file?: string, comments?: CommentLine[]): LogicalLine[];
+/** Reads tokens from one logical line. All methods skip leading whitespace. */
+export declare class Cursor {
+    private readonly line;
+    readonly file: string;
+    private i;
+    private inlineUsed;
+    private readonly text;
+    constructor(line: LogicalLine, file: string);
+    spanAt(i: number): SourceSpan;
+    /** Span of the next non-whitespace character (or end of line). */
+    get span(): SourceSpan;
+    error(code: ErrorCode, detail: string, at?: number | SourceSpan): SitefileError;
+    skipWs(): void;
+    atEnd(): boolean;
+    private wordEnd;
+    /** Consume raw text up to the next whitespace ('' at end of line). */
+    word(): string;
+    /** Consume `kw` (case-insensitive) if it is the next whole bare word. */
+    keyword(kw: string): boolean;
+    /** Whether `kw` (case-insensitive) is the next whole bare word, without consuming it. */
+    peekKeyword(kw: string): boolean;
+    /** The current position, for {@link reset} (one-token lookahead). */
+    mark(): number;
+    reset(mark: number): void;
+    expectKeyword(kw: string): void;
+    /** Consume `ch` if it is the next non-whitespace character. */
+    eat(ch: string): boolean;
+    /**
+     * Read an identifier. It must be followed by whitespace, end of line, or one
+     * of the characters in `stops`.
+     */
+    ident(what: string, stops?: string): string;
+    /** Read `--name` or `--name=value` if the next word starts with `--`. */
+    flag(): {
+        name: string;
+        value: string | undefined;
+        span: SourceSpan;
+    } | undefined;
+    expectEnd(): void;
+    /**
+     * Consume the rest of the line (from the next non-whitespace character, trailing whitespace
+     * dropped): its text, its span, and whether it continues onto another physical line.
+     */
+    rest(): {
+        text: string;
+        span: SourceSpan;
+        multiline: boolean;
+    };
+    /** Consume this line's heredoc marker if it is the next word. */
+    inline(): InlineBody | undefined;
+    /** Fail if the line has a heredoc that was not used as the source. */
+    expectInlineUsed(where: string): void;
+    /** Read a bare, "double-quoted" or 'single-quoted' operand. */
+    operand(what: string): Operand;
+    /** At `${`: read `${NAME}`. */
+    private interpolation;
+}

package/lib/src/language/scanner.js

@@ -0,0 +1,449 @@
+import { SitefileError } from '../errors.js';
+/** Instructions whose source may be a heredoc (and `LET name: html`, see {@link isHtmlLet}). */
+const HEREDOC_INSTRUCTIONS = new Set(['COPY', 'RENDER', 'TRANSFORM']);
+/**
+ * Whether a logical line declares an html fragment, `LET name: html …` (the type ASCII
+ * case-insensitive), whose value may be a heredoc. Other LET lines are not scanned for `<<`.
+ */
+export function isHtmlLet(text) {
+    const m = /^(\S+)/.exec(text);
+    if (!m || m[1].toUpperCase() !== 'LET')
+        return false;
+    let i = m[1].length;
+    const skip = () => {
+        while (i < text.length && WS.test(text[i]))
+            i++;
+    };
+    skip();
+    if (!IDENT_START.test(text[i] ?? ''))
+        return false;
+    while (IDENT_CHAR.test(text[i] ?? ''))
+        i++;
+    skip();
+    if (text[i] !== ':')
+        return false;
+    i++;
+    skip();
+    if (text.slice(i, i + 4).toLowerCase() !== 'html')
+        return false;
+    const next = text[i + 4];
+    return next === undefined || next === '=' || WS.test(next);
+}
+/** `<<NAME` / `<<-NAME` markers that start a word outside quotes. */
+function heredocMarkers(text) {
+    const out = [];
+    let quote;
+    for (let i = 0; i < text.length; i++) {
+        const ch = text[i];
+        if (quote) {
+            if (ch === '\\' && quote === '"')
+                i++;
+            else if (ch === quote)
+                quote = undefined;
+            continue;
+        }
+        if (ch === '"' || ch === "'") {
+            quote = ch;
+            continue;
+        }
+        if (ch === '\\') {
+            i++;
+            continue;
+        }
+        if (ch !== '<' || (i > 0 && !WS.test(text[i - 1])))
+            continue;
+        const m = /^<<(-?)([A-Za-z_][A-Za-z0-9_]*)(?=\s|$)/.exec(text.slice(i));
+        if (m)
+            out.push({ at: i, length: m[0].length, strip: m[1] === '-', delimiter: m[2] });
+    }
+    return out;
+}
+/** The language version this implementation reads (`# sitefile=1`; a file without the directive is this version). */
+export const SYNTAX_VERSION = '1';
+/**
+ * The version directive `# sitefile=N` (name case-insensitive, spaces or tabs around `=`), matched on a
+ * comment line from its `#`. Group 1 is everything after `=`.
+ * Other comments, `# sitefile-lint: …` included, are ordinary comments here.
+ */
+export const VERSION_DIRECTIVE = /^#[ \t]*[Ss][Ii][Tt][Ee][Ff][Ii][Ll][Ee][ \t]*=([\s\S]*)$/;
+const IDENT_START = /[A-Za-z_]/;
+const IDENT_CHAR = /[A-Za-z0-9_]/;
+const WS = /\s/;
+function trailingBackslashes(s) {
+    let n = 0;
+    while (n < s.length && s[s.length - 1 - n] === '\\')
+        n++;
+    return n;
+}
+/**
+ * Split source into logical lines. Blank lines and comment lines (first non-blank
+ * char `#`) are skipped, also inside continuations. A line ending in an odd number
+ * of `\` continues; pieces are trimmed at the join and joined with one space.
+ * A `# sitefile=N` comment is the syntax version directive: it must come before
+ * the first instruction, at most once, and name a supported version.
+ * `comments`, when given, receives every comment line (for `# sitefile-lint:`).
+ */
+export function logicalLines(source, file = 'Sitefile', comments) {
+    const out = [];
+    let sawVersion = false;
+    /** A comment line: check a `# sitefile=N` directive (only allowed before the first instruction). */
+    const directive = (raw, line, late) => {
+        const hash = raw.indexOf('#');
+        const m = VERSION_DIRECTIVE.exec(raw.slice(hash));
+        if (!m)
+            return;
+        const at = { file, line, column: hash + 1 };
+        if (late)
+            throw new SitefileError('E_SYNTAX', "the '# sitefile=' directive must come before the first instruction", at);
+        if (sawVersion)
+            throw new SitefileError('E_SYNTAX', "'# sitefile=' given twice", at);
+        sawVersion = true;
+        const lead = m[1].length - m[1].trimStart().length;
+        const version = m[1].trim();
+        const vat = { file, line, column: hash + m[0].length - m[1].length + lead + 1 };
+        if (!/^[1-9][0-9]*$/.test(version)) {
+            throw new SitefileError('E_SYNTAX', `invalid syntax version '${version}' in '# sitefile=' (expected a whole number such as ${SYNTAX_VERSION})`, vat);
+        }
+        if (version !== SYNTAX_VERSION) {
+            throw new SitefileError('E_UNSUPPORTED', `this Sitefile uses syntax version ${version}; this sitefile supports version ${SYNTAX_VERSION} (upgrade sitefile)`, vat);
+        }
+    };
+    let text = '';
+    let positions = [];
+    let end = { line: 1, column: 1 };
+    let continuing = false;
+    // Set while reading a heredoc body: lines are taken verbatim until the delimiter.
+    let heredocBody;
+    const physical = source.split(/\r?\n/);
+    // A trailing newline is not an extra (empty) line of a heredoc body.
+    if (physical.length > 1 && physical[physical.length - 1] === '')
+        physical.pop();
+    physical.forEach((raw, index) => {
+        const line = index + 1;
+        if (heredocBody) {
+            const candidate = heredocBody.marker.strip ? raw.replace(/^\t+/, '') : raw;
+            if (candidate === heredocBody.marker.delimiter) {
+                const heredoc = {
+                    at: heredocBody.marker.at,
+                    length: heredocBody.marker.length,
+                    delimiter: heredocBody.marker.delimiter,
+                    text: heredocBody.lines.map((l) => l + '\n').join(''),
+                    bodyLine: heredocBody.start,
+                };
+                out[out.length - 1] = { ...heredocBody.owner, heredoc };
+                heredocBody = undefined;
+            }
+            else {
+                heredocBody.lines.push(candidate);
+            }
+            return;
+        }
+        const trimmed = raw.trim();
+        if (trimmed.startsWith('#')) {
+            comments?.push({ line, column: raw.indexOf('#') + 1, text: raw.slice(raw.indexOf('#')) });
+            directive(raw, line, out.length > 0 || continuing);
+            return;
+        }
+        if (trimmed === '')
+            return;
+        const cont = trailingBackslashes(raw) % 2 === 1;
+        const body = cont ? raw.slice(0, -1) : raw;
+        let start = 0;
+        if (continuing)
+            while (start < body.length && WS.test(body[start]))
+                start++;
+        let stop = body.length;
+        if (cont)
+            while (stop > start && WS.test(body[stop - 1]))
+                stop--;
+        if (continuing && text !== '' && stop > start) {
+            text += ' ';
+            positions.push(end);
+        }
+        for (let c = start; c < stop; c++) {
+            text += body[c];
+            positions.push({ line, column: c + 1 });
+        }
+        if (stop > start || !continuing)
+            end = { line, column: stop + 1 };
+        if (cont) {
+            continuing = true;
+        }
+        else {
+            const logical = { text, positions, end };
+            out.push(logical);
+            const keyword = /^\S+/.exec(text)?.[0]?.toUpperCase() ?? '';
+            if (HEREDOC_INSTRUCTIONS.has(keyword) || isHtmlLet(text)) {
+                const markers = heredocMarkers(text);
+                if (markers.length > 1) {
+                    const p = positions[markers[1].at];
+                    throw new SitefileError('E_SYNTAX', 'only one inline (<<) body is allowed per instruction', { file, ...p });
+                }
+                if (markers[0])
+                    heredocBody = { marker: markers[0], lines: [], start: line + 1, owner: logical };
+            }
+            text = '';
+            positions = [];
+            continuing = false;
+        }
+    });
+    if (heredocBody) {
+        const p = heredocBody.owner.positions[heredocBody.marker.at];
+        throw new SitefileError('E_SYNTAX', `inline body is missing its closing '${heredocBody.marker.delimiter}' line`, { file, ...p });
+    }
+    if (continuing && text.trim() !== '')
+        out.push({ text, positions, end });
+    return out;
+}
+/** Reads tokens from one logical line. All methods skip leading whitespace. */
+export class Cursor {
+    line;
+    file;
+    i = 0;
+    inlineUsed = false;
+    text;
+    constructor(line, file) {
+        this.line = line;
+        this.file = file;
+        this.text = line.text;
+    }
+    spanAt(i) {
+        const p = this.line.positions[i] ?? this.line.end;
+        return { file: this.file, line: p.line, column: p.column };
+    }
+    /** Span of the next non-whitespace character (or end of line). */
+    get span() {
+        this.skipWs();
+        return this.spanAt(this.i);
+    }
+    error(code, detail, at = this.i) {
+        return new SitefileError(code, detail, typeof at === 'number' ? this.spanAt(at) : at);
+    }
+    skipWs() {
+        while (this.i < this.text.length && WS.test(this.text[this.i]))
+            this.i++;
+    }
+    atEnd() {
+        this.skipWs();
+        return this.i >= this.text.length;
+    }
+    wordEnd() {
+        let j = this.i;
+        while (j < this.text.length && !WS.test(this.text[j]))
+            j++;
+        return j;
+    }
+    /** Consume raw text up to the next whitespace ('' at end of line). */
+    word() {
+        this.skipWs();
+        const j = this.wordEnd();
+        const w = this.text.slice(this.i, j);
+        this.i = j;
+        return w;
+    }
+    /** Consume `kw` (case-insensitive) if it is the next whole bare word. */
+    keyword(kw) {
+        this.skipWs();
+        const j = this.wordEnd();
+        if (this.text.slice(this.i, j).toLowerCase() !== kw.toLowerCase())
+            return false;
+        this.i = j;
+        return true;
+    }
+    /** Whether `kw` (case-insensitive) is the next whole bare word, without consuming it. */
+    peekKeyword(kw) {
+        const at = this.i;
+        const found = this.keyword(kw);
+        this.i = at;
+        return found;
+    }
+    /** The current position, for {@link reset} (one-token lookahead). */
+    mark() {
+        return this.i;
+    }
+    reset(mark) {
+        this.i = mark;
+    }
+    expectKeyword(kw) {
+        const at = this.span;
+        if (!this.keyword(kw))
+            throw this.error('E_SYNTAX', `expected ${kw.toUpperCase()}`, at);
+    }
+    /** Consume `ch` if it is the next non-whitespace character. */
+    eat(ch) {
+        this.skipWs();
+        if (this.text[this.i] !== ch)
+            return false;
+        this.i++;
+        return true;
+    }
+    /**
+     * Read an identifier. It must be followed by whitespace, end of line, or one
+     * of the characters in `stops`.
+     */
+    ident(what, stops = '') {
+        this.skipWs();
+        const start = this.i;
+        if (!IDENT_START.test(this.text[this.i] ?? ''))
+            throw this.error('E_SYNTAX', `expected ${what}`);
+        while (IDENT_CHAR.test(this.text[this.i] ?? ''))
+            this.i++;
+        const next = this.text[this.i];
+        if (next !== undefined && !WS.test(next) && !stops.includes(next)) {
+            throw this.error('E_SYNTAX', `invalid ${what}: unexpected '${next}'`);
+        }
+        return this.text.slice(start, this.i);
+    }
+    /** Read `--name` or `--name=value` if the next word starts with `--`. */
+    flag() {
+        this.skipWs();
+        if (!this.text.startsWith('--', this.i))
+            return undefined;
+        const span = this.spanAt(this.i);
+        const w = this.word().slice(2);
+        const eq = w.indexOf('=');
+        return eq < 0 ? { name: w, value: undefined, span } : { name: w.slice(0, eq), value: w.slice(eq + 1), span };
+    }
+    expectEnd() {
+        if (this.atEnd())
+            return;
+        const at = this.i;
+        throw this.error('E_SYNTAX', `unexpected '${this.word()}'`, at);
+    }
+    /**
+     * Consume the rest of the line (from the next non-whitespace character, trailing whitespace
+     * dropped): its text, its span, and whether it continues onto another physical line.
+     */
+    rest() {
+        this.skipWs();
+        const start = this.i;
+        let stop = this.text.length;
+        while (stop > start && WS.test(this.text[stop - 1]))
+            stop--;
+        this.i = this.text.length;
+        const first = this.line.positions[start];
+        const last = this.line.positions[stop - 1];
+        return { text: this.text.slice(start, stop), span: this.spanAt(start), multiline: first !== undefined && last !== undefined && last.line !== first.line };
+    }
+    /** Consume this line's heredoc marker if it is the next word. */
+    inline() {
+        this.skipWs();
+        const h = this.line.heredoc;
+        if (!h || h.at !== this.i)
+            return undefined;
+        const span = this.spanAt(this.i);
+        this.i += h.length;
+        this.inlineUsed = true;
+        return { kind: 'inline', delimiter: h.delimiter, text: h.text, span, bodyLine: h.bodyLine };
+    }
+    /** Fail if the line has a heredoc that was not used as the source. */
+    expectInlineUsed(where) {
+        const h = this.line.heredoc;
+        if (h && !this.inlineUsed) {
+            throw this.error('E_SYNTAX', `an inline <<${h.delimiter} body can only be ${where}`, h.at);
+        }
+    }
+    /** Read a bare, "double-quoted" or 'single-quoted' operand. */
+    operand(what) {
+        this.skipWs();
+        const start = this.i;
+        const t = this.text;
+        if (start >= t.length)
+            throw this.error('E_SYNTAX', `expected ${what}`);
+        const parts = [];
+        const addText = (s) => {
+            const last = parts[parts.length - 1];
+            if (last?.kind === 'text')
+                parts[parts.length - 1] = { kind: 'text', value: last.value + s };
+            else
+                parts.push({ kind: 'text', value: s });
+        };
+        const quote = t[start];
+        if (quote === "'") {
+            const close = t.indexOf("'", start + 1);
+            if (close < 0)
+                throw this.error('E_SYNTAX', 'unterminated single-quoted string', start);
+            if (close > start + 1)
+                addText(t.slice(start + 1, close));
+            this.i = close + 1;
+        }
+        else if (quote === '"') {
+            this.i++;
+            for (;;) {
+                const ch = t[this.i];
+                if (ch === undefined)
+                    throw this.error('E_SYNTAX', 'unterminated double-quoted string', start);
+                if (ch === '"') {
+                    this.i++;
+                    break;
+                }
+                if (ch === '\\') {
+                    const next = t[this.i + 1];
+                    if (next === undefined)
+                        throw this.error('E_SYNTAX', 'unterminated double-quoted string', start);
+                    addText(next === 'n' ? '\n' : next === 't' ? '\t' : next);
+                    this.i += 2;
+                }
+                else if (ch === '$' && t[this.i + 1] === '{') {
+                    parts.push(this.interpolation());
+                }
+                else {
+                    addText(ch);
+                    this.i++;
+                }
+            }
+        }
+        else {
+            while (this.i < t.length && !WS.test(t[this.i])) {
+                const ch = t[this.i];
+                if (ch === '\\') {
+                    const next = t[this.i + 1];
+                    if (next === undefined)
+                        throw this.error('E_SYNTAX', 'dangling escape at end of line');
+                    addText(next);
+                    this.i += 2;
+                }
+                else if (ch === '$' && t[this.i + 1] === '{') {
+                    parts.push(this.interpolation());
+                }
+                else {
+                    addText(ch);
+                    this.i++;
+                }
+            }
+            return { parts, span: this.spanAt(start) };
+        }
+        const after = t[this.i];
+        if (after !== undefined && !WS.test(after))
+            throw this.error('E_SYNTAX', 'unexpected character after closing quote');
+        return { parts, span: this.spanAt(start) };
+    }
+    /** At `${`: read `${NAME}`. */
+    interpolation() {
+        const at = this.i;
+        const t = this.text;
+        this.i += 2;
+        if (!IDENT_START.test(t[this.i] ?? ''))
+            throw this.error('E_SYNTAX', 'expected a name after ${', at);
+        const nameStart = this.i;
+        while (IDENT_CHAR.test(t[this.i] ?? ''))
+            this.i++;
+        const name = t.slice(nameStart, this.i);
+        // Dotted paths are parsed here; the validator allows them only for a RENDER … FOR loop variable.
+        const path = [];
+        while (t[this.i] === '.') {
+            this.i++;
+            const segStart = this.i;
+            while (/[A-Za-z0-9_-]/.test(t[this.i] ?? ''))
+                this.i++;
+            if (this.i === segStart)
+                throw this.error('E_SYNTAX', `expected a key after '.' in \${${name}…}`, at);
+            path.push(t.slice(segStart, this.i));
+        }
+        if (t[this.i] !== '}')
+            throw this.error('E_SYNTAX', 'unterminated ${...}: expected }', at);
+        this.i++;
+        return { kind: 'var', name, path, span: this.spanAt(at) };
+    }
+}
+//# sourceMappingURL=scanner.js.map

package/lib/src/language/validate.d.ts

@@ -0,0 +1,13 @@
+import { type FragmentBody, type Instruction, type Operand, type Program } from './ast.js';
+import { type TemplateNode } from '../render.js';
+/** A fragment's template: a heredoc body or partial file without its final newline (it is inserted inline). */
+export declare function fragmentText(text: string): string;
+/** Parse the template of an html LET (errors point into the Sitefile). */
+export declare function parseFragmentBody(body: FragmentBody, file: string): TemplateNode[];
+export declare function operandsOf(ins: Instruction): Operand[];
+/** The only placeholders the host replaces inside TOOL arguments; any other `{…}` stays literal (e.g. ffmpeg's `%{pts}`). */
+export declare const TOOL_PLACEHOLDER_RE: RegExp;
+/** Check semantic rules (see docs/LANGUAGE.md, "v0.1 implementation decisions"). */
+export declare function validate(instructions: readonly Instruction[], file: string): Program;
+/** Parse and validate a Sitefile. */
+export declare function parseProgram(source: string, file?: string): Program;