@johnhenry/sitefile 0.0.0-stage → 0.0.1

package/lib/src/schema.js

@@ -0,0 +1,169 @@
+// Frontmatter and data schemas: `IMPORT posts FROM ./posts AS markdown SCHEMA ./post.schema.yaml`.
+//
+// A schema is a small, deterministic subset of JSON Schema, written as a YAML or JSON mapping:
+//
+//   required: [title, date]          # keys every document must have (null counts as missing)
+//   properties:                      # key -> rule; keys not listed are not checked
+//     title: string                  # a type name ...
+//     tags: { type: list, items: string }
+//     status: { enum: [draft, published] }
+//
+// Types: string, number, boolean, date (YYYY-MM-DD or an RFC 3339 date-time, as FEED reads it) and list
+// (`items` gives the type of its elements). `enum` lists the allowed strings, numbers or booleans (for a list,
+// the allowed elements). In a CSV import every cell is a string, so `number` also accepts a numeric string and
+// `boolean` the strings `true` and `false`, enum entries match their text, and an empty cell counts as missing. docs/LANGUAGE.md has the rules.
+import { SitefileError } from './errors.js';
+import { parseFeedDate } from './site.js';
+const TYPES = ['string', 'number', 'boolean', 'date', 'list'];
+const ITEM_TYPES = ['string', 'number', 'boolean', 'date'];
+const NUMERIC = /^-?\d+(\.\d+)?([eE][+-]?\d+)?$/;
+/** What a value of each type looks like, for messages. */
+const EXPECTED = {
+    string: 'a string',
+    number: 'a number',
+    boolean: 'true or false',
+    date: 'a date (YYYY-MM-DD or an RFC 3339 date-time)',
+    list: 'a list',
+};
+const isObject = (v) => v !== null && typeof v === 'object' && !Array.isArray(v);
+/** Compile a parsed schema file. Errors are E_SCHEMA at `span` (the SCHEMA clause), prefixed with `what`. */
+export function compileSchema(value, label, what, span) {
+    const fail = (m) => {
+        throw new SitefileError('E_SCHEMA', `${what}: schema '${label}' ${m}`, span);
+    };
+    if (!isObject(value))
+        fail('must be a mapping (with required and/or properties)');
+    const root = value;
+    for (const k of Object.keys(root)) {
+        if (k !== 'required' && k !== 'properties')
+            fail(`has an unknown key '${k}' (expected required or properties)`);
+    }
+    const required = [];
+    if (root['required'] !== undefined) {
+        const r = root['required'];
+        if (!Array.isArray(r) || !r.every((k) => typeof k === 'string' && k !== ''))
+            fail("needs 'required' to be a list of key names");
+        for (const k of r)
+            if (!required.includes(k))
+                required.push(k);
+    }
+    const properties = [];
+    if (root['properties'] !== undefined) {
+        const p = root['properties'];
+        if (!isObject(p))
+            fail("needs 'properties' to be a mapping of key names to rules");
+        for (const [key, rule] of Object.entries(p)) {
+            const at = `property '${key}':`;
+            const typeName = (t, allowed, kind) => {
+                if (typeof t !== 'string' || !allowed.includes(t)) {
+                    const shown = typeof t === 'string' ? ` '${t}'` : '';
+                    fail(`${at} unknown ${kind}${shown} (expected ${allowed.slice(0, -1).join(', ')} or ${allowed[allowed.length - 1]})`);
+                }
+                return t;
+            };
+            if (typeof rule === 'string') {
+                properties.push({ key, type: typeName(rule, TYPES, 'type') });
+                continue;
+            }
+            if (!isObject(rule))
+                fail(`${at} expected a type name or a mapping with type, items or enum`);
+            const r = rule;
+            for (const k of Object.keys(r)) {
+                if (k !== 'type' && k !== 'items' && k !== 'enum')
+                    fail(`${at} unknown key '${k}' (expected type, items or enum)`);
+            }
+            const type = r['type'] !== undefined ? typeName(r['type'], TYPES, 'type') : undefined;
+            let items;
+            if (r['items'] !== undefined) {
+                if (type !== 'list')
+                    fail(`${at} items needs type list`);
+                items = typeName(r['items'], ITEM_TYPES, 'items type');
+            }
+            let values;
+            if (r['enum'] !== undefined) {
+                const e = r['enum'];
+                if (!Array.isArray(e) || e.length === 0 || !e.every((x) => typeof x === 'string' || typeof x === 'number' || typeof x === 'boolean')) {
+                    fail(`${at} enum must be a non-empty list of strings, numbers or booleans`);
+                }
+                values = e;
+            }
+            properties.push({ key, ...(type ? { type } : {}), ...(items ? { items } : {}), ...(values ? { enum: values } : {}) });
+        }
+    }
+    return { label, required, properties };
+}
+/** How a value is shown in a mismatch message. */
+function describe(v) {
+    if (v === null)
+        return 'null';
+    if (typeof v === 'string')
+        return `string '${v}'`;
+    if (typeof v === 'number')
+        return `number ${String(v)}`;
+    if (typeof v === 'boolean')
+        return `boolean ${String(v)}`;
+    if (Array.isArray(v))
+        return 'a list';
+    return 'an object';
+}
+function hasType(v, t, csv) {
+    switch (t) {
+        case 'string':
+            return typeof v === 'string';
+        case 'number':
+            return typeof v === 'number' || (csv && typeof v === 'string' && NUMERIC.test(v.trim()));
+        case 'boolean':
+            return typeof v === 'boolean' || (csv && (v === 'true' || v === 'false'));
+        case 'date':
+            return typeof v === 'string' && parseFeedDate(v) !== undefined;
+        case 'list':
+            return Array.isArray(v);
+    }
+}
+function inEnum(v, values, csv) {
+    return values.some((e) => e === v || (csv && typeof v === 'string' && v === String(e)));
+}
+/**
+ * Check an import's value against a schema: each element of a list, or a single document. Every element must be
+ * an object (E_TYPE otherwise). The first mismatch fails with E_SCHEMA at `span`, naming the document (its `path`,
+ * or `#n` for the n-th element), the key, what was expected and what was found.
+ */
+export function checkSchema(value, schema, what, csv, span) {
+    const docs = Array.isArray(value) ? value : [value];
+    docs.forEach((doc, i) => {
+        if (!isObject(doc)) {
+            throw new SitefileError('E_TYPE', `${what}: SCHEMA checks documents (objects), but element ${i + 1} is ${describe(doc)}`, span);
+        }
+        const name = typeof doc['path'] === 'string' ? doc['path'] : `#${i + 1}`;
+        const fail = (m) => {
+            throw new SitefileError('E_SCHEMA', `${what}: document '${name}' ${m}`, span);
+        };
+        // A CSV cell is always there: an empty one counts as missing.
+        const missing = (v) => v === undefined || v === null || (csv && v === '');
+        for (const key of schema.required) {
+            if (missing(Object.hasOwn(doc, key) ? doc[key] : undefined))
+                fail(`has no '${key}' (required by schema '${schema.label}')`);
+        }
+        for (const rule of schema.properties) {
+            const v = Object.hasOwn(doc, rule.key) ? doc[rule.key] : undefined;
+            if (missing(v))
+                continue;
+            const enumText = (values) => values.map((e) => String(e)).join(', ');
+            if (rule.type !== undefined && !hasType(v, rule.type, csv))
+                fail(`has '${rule.key}' ${describe(v)}; it should be ${EXPECTED[rule.type]}`);
+            if (rule.type === 'list') {
+                v.forEach((item, j) => {
+                    if (rule.items !== undefined && !hasType(item, rule.items, csv)) {
+                        fail(`has '${rule.key}' item ${j + 1} ${describe(item)}; it should be ${EXPECTED[rule.items]}`);
+                    }
+                    if (rule.enum !== undefined && !inEnum(item, rule.enum, csv))
+                        fail(`has '${rule.key}' item ${j + 1} ${describe(item)}; it should be one of ${enumText(rule.enum)}`);
+                });
+            }
+            else if (rule.enum !== undefined && !inEnum(v, rule.enum, csv)) {
+                fail(`has '${rule.key}' ${describe(v)}; it should be one of ${enumText(rule.enum)}`);
+            }
+        }
+    });
+}
+//# sourceMappingURL=schema.js.map

package/lib/src/secrets.d.ts

@@ -0,0 +1,21 @@
+export type Secrets = ReadonlyMap<string, string>;
+/**
+ * Parse `--secret` flags: `id=<id>,env=<VAR>` (an environment variable) or `id=<id>,src=<file>` (a
+ * file, relative to the working directory; one trailing newline is dropped). Values are checked to
+ * be usable as an HTTP header value. Messages never include a value.
+ */
+export declare function loadSecrets(flags: readonly string[], env?: NodeJS.ProcessEnv): Promise<Map<string, string>>;
+/** Replace every occurrence of a secret value (longest first) with `***`. */
+export declare function redactSecrets(text: string, secrets: Secrets | undefined): string;
+/** A SitefileError whose message has the secrets redacted (the same error when nothing changes). */
+export declare function redactError(e: unknown, secrets: Secrets | undefined): unknown;
+/** The request headers of an import's SECRET clauses; a secret that was not given is E_SECRET. */
+export declare function secretHeaders(spec: {
+    readonly name: string;
+    readonly secrets?: readonly {
+        readonly id: string;
+        readonly header: string;
+        readonly bearer: boolean;
+    }[];
+    readonly span: import('./errors.js').SourceSpan;
+}, secrets: Secrets | undefined): Record<string, string> | undefined;

package/lib/src/secrets.js

@@ -0,0 +1,103 @@
+import { readFile } from 'node:fs/promises';
+import { SitefileError, isSitefileError } from './errors.js';
+/**
+ * Build secrets (issue #85), modelled on Docker's `--secret id=…,env=…|src=…`: values that remote
+ * IMPORTs send as HTTP request headers (`IMPORT … SECRET <id> AS HEADER <Name>` / `AS BEARER`).
+ *
+ * A secret value lives only in memory: it is sent to the IMPORT URL's own origin (never after a
+ * redirect to another one) and never becomes part of the output, a cache or memo key, Sitefile.lock,
+ * a vendor manifest, provenance, a --profile trace, a warning or an error message. As a last line of
+ * defence, warnings and errors of a build that has secrets are passed through {@link redactSecrets}.
+ */
+const ID_RE = /^[A-Za-z_][A-Za-z0-9_]*$/;
+/** Visible ASCII, space and tab: what every HTTP client sends unchanged in a header value. */
+const HEADER_VALUE_RE = /^[\t\x20-\x7e]+$/;
+function usage(message) {
+    return new SitefileError('E_USAGE', message);
+}
+/**
+ * Parse `--secret` flags: `id=<id>,env=<VAR>` (an environment variable) or `id=<id>,src=<file>` (a
+ * file, relative to the working directory; one trailing newline is dropped). Values are checked to
+ * be usable as an HTTP header value. Messages never include a value.
+ */
+export async function loadSecrets(flags, env = process.env) {
+    const out = new Map();
+    for (const flag of flags) {
+        const fields = new Map();
+        for (const part of flag.split(',')) {
+            const eq = part.indexOf('=');
+            const key = eq < 0 ? part : part.slice(0, eq);
+            if (eq < 0 || (key !== 'id' && key !== 'env' && key !== 'src')) {
+                throw usage('--secret expects id=<id>,env=<VAR> or id=<id>,src=<file>'); // never echo the flag: it might be the secret itself
+            }
+            if (fields.has(key))
+                throw usage(`--secret: '${key}' given twice`);
+            fields.set(key, part.slice(eq + 1));
+        }
+        const id = fields.get('id');
+        if (id === undefined || !ID_RE.test(id))
+            throw usage(`--secret needs id=<id> (letters, digits and _, not starting with a digit)`);
+        if (out.has(id))
+            throw usage(`--secret id=${id} given twice`);
+        const name = fields.get('env');
+        const src = fields.get('src');
+        if ((name === undefined) === (src === undefined))
+            throw usage(`--secret id=${id}: give exactly one of env=<VAR> or src=<file>`);
+        let value;
+        if (name !== undefined) {
+            const v = env[name];
+            if (v === undefined)
+                throw new SitefileError('E_SECRET', `--secret id=${id}: environment variable ${name} is not set`);
+            value = v;
+        }
+        else {
+            let bytes;
+            try {
+                bytes = await readFile(src);
+            }
+            catch (e) {
+                throw new SitefileError('E_SECRET', `--secret id=${id}: cannot read '${src}'`, undefined, { cause: isSitefileError(e) ? e : undefined });
+            }
+            value = bytes.toString('latin1').replace(/\r?\n$/, '');
+        }
+        if (value === '')
+            throw new SitefileError('E_SECRET', `--secret id=${id} is empty`);
+        if (!HEADER_VALUE_RE.test(value)) {
+            throw new SitefileError('E_SECRET', `--secret id=${id} cannot be sent in an HTTP header (only printable ASCII, spaces and tabs)`);
+        }
+        out.set(id, value);
+    }
+    return out;
+}
+/** Replace every occurrence of a secret value (longest first) with `***`. */
+export function redactSecrets(text, secrets) {
+    if (!secrets || secrets.size === 0)
+        return text;
+    let out = text;
+    for (const v of [...secrets.values()].sort((a, b) => b.length - a.length))
+        if (v.length > 0)
+            out = out.split(v).join('***');
+    return out;
+}
+/** A SitefileError whose message has the secrets redacted (the same error when nothing changes). */
+export function redactError(e, secrets) {
+    if (!isSitefileError(e) || !secrets || secrets.size === 0)
+        return e;
+    const detail = redactSecrets(e.detail, secrets);
+    return detail === e.detail ? e : new SitefileError(e.code, detail, e.span, e.needs ? { needs: e.needs } : undefined);
+}
+/** The request headers of an import's SECRET clauses; a secret that was not given is E_SECRET. */
+export function secretHeaders(spec, secrets) {
+    if (!spec.secrets?.length)
+        return undefined;
+    const headers = {};
+    for (const s of spec.secrets) {
+        const v = secrets?.get(s.id);
+        if (v === undefined) {
+            throw new SitefileError('E_SECRET', `IMPORT ${spec.name}: secret '${s.id}' was not given; pass --secret id=${s.id},env=<VAR> or --secret id=${s.id},src=<file>`, spec.span);
+        }
+        headers[s.header.toLowerCase()] = s.bearer ? `Bearer ${v}` : v;
+    }
+    return headers;
+}
+//# sourceMappingURL=secrets.js.map

package/lib/src/serve-rules.d.ts

@@ -0,0 +1,29 @@
+/** A `_redirects` rule `serve` understands. */
+export interface ServeRedirect {
+    /** Decoded path without a trailing slash (`/` for the root); for a splat rule, the prefix before `/*` (`''` for `/*`). */
+    readonly from: string;
+    readonly splat: boolean;
+    readonly to: string;
+    readonly status: number;
+    /** `301!`: applies even when a file exists at the path. */
+    readonly force: boolean;
+}
+export interface ServeHeaders {
+    readonly pattern: string;
+    /** Lowercase names. */
+    readonly headers: ReadonlyArray<readonly [string, string]>;
+}
+/**
+ * Parse `_redirects`: `from to [status[!]]` per line, `#` comments. Supported: exact paths and a
+ * trailing `/*` with `:splat` in the target; statuses 301, 302, 303, 307, 308 (redirect), 200
+ * (rewrite) and 404. Other lines (placeholders, conditions, query matching) are ignored.
+ */
+export declare function parseRedirectRules(text: string): ServeRedirect[];
+/** Parse `_headers`: a path pattern line (`*` matches anything), then indented `Name: value` lines. */
+export declare function parseHeaderRules(text: string): ServeHeaders[];
+/** Glob match where `*` matches any run of characters (including `/`). */
+export declare function globMatch(pattern: string, s: string): boolean;
+/** Every matching block's headers; a name set by several blocks gets its values joined with `, `. */
+export declare function headersFor(rules: readonly ServeHeaders[], path: string): Record<string, string>;
+/** The remainder a rule matches for `path` (`''` for the prefix itself), or undefined. */
+export declare function matchRedirect(r: ServeRedirect, path: string): string | undefined;

package/lib/src/serve-rules.js

@@ -0,0 +1,123 @@
+// `_redirects` and `_headers` as `sitefile serve` reads them (docs/CLI.md, "serve").
+// Ported as the rules section of rust/src/server.rs.
+import { safeSegments } from './server.js';
+const HEADER_TOKEN = /^[!#$%&'*+\-.^_`|~0-9A-Za-z]+$/;
+/** Headers the server owns; `_headers` cannot set them. */
+const PROTECTED_HEADERS = new Set(['content-length', 'transfer-encoding', 'connection', 'keep-alive', 'etag', 'location']);
+const SERVE_STATUSES = new Set([200, 301, 302, 303, 307, 308, 404]);
+/**
+ * Parse `_redirects`: `from to [status[!]]` per line, `#` comments. Supported: exact paths and a
+ * trailing `/*` with `:splat` in the target; statuses 301, 302, 303, 307, 308 (redirect), 200
+ * (rewrite) and 404. Other lines (placeholders, conditions, query matching) are ignored.
+ */
+export function parseRedirectRules(text) {
+    const out = [];
+    for (const line of text.split('\n')) {
+        const t = line.trim();
+        if (t === '' || t.startsWith('#'))
+            continue;
+        const fields = t.split(/[ \t]+/);
+        if (fields.length < 2 || fields.length > 3)
+            continue;
+        const from = fields[0];
+        const to = fields[1];
+        const st = fields[2] ?? '301';
+        const force = st.endsWith('!');
+        const code = force ? st.slice(0, -1) : st;
+        if (!/^[0-9]{3}$/.test(code) || !SERVE_STATUSES.has(Number(code)))
+            continue;
+        const status = Number(code);
+        if (!/^[\x21-\x7e]+$/.test(to) || !(to.startsWith('/') || /^https?:\/\/[^/]/i.test(to)))
+            continue;
+        if ((status === 200 || status === 404) && !to.startsWith('/'))
+            continue;
+        if (!from.startsWith('/'))
+            continue;
+        const splat = from === '/*' || from.endsWith('/*');
+        const prefix = splat ? from.slice(0, -2) : from;
+        if (prefix.includes('*') || prefix.split('/').some((seg) => seg.startsWith(':')))
+            continue;
+        const segs = safeSegments(prefix === '' ? '/' : prefix);
+        if (!segs)
+            continue;
+        const decoded = '/' + segs.join('/');
+        out.push({ from: splat && decoded === '/' ? '' : decoded, splat, to, status, force });
+    }
+    return out;
+}
+/** Parse `_headers`: a path pattern line (`*` matches anything), then indented `Name: value` lines. */
+export function parseHeaderRules(text) {
+    const out = [];
+    let cur;
+    for (const raw of text.split('\n')) {
+        const line = raw.endsWith('\r') ? raw.slice(0, -1) : raw;
+        const t = line.trim();
+        if (t === '' || t.startsWith('#'))
+            continue;
+        if (line[0] === ' ' || line[0] === '\t') {
+            const i = t.indexOf(':');
+            if (!cur || i <= 0)
+                continue;
+            const name = t.slice(0, i).trim().toLowerCase();
+            const value = t.slice(i + 1).trim();
+            if (!HEADER_TOKEN.test(name) || !/^[\t\x20-\x7e]*$/.test(value) || PROTECTED_HEADERS.has(name))
+                continue;
+            cur.headers.push([name, value]);
+            continue;
+        }
+        cur = t.startsWith('/') ? { pattern: t, headers: [] } : undefined;
+        if (cur)
+            out.push(cur);
+    }
+    return out;
+}
+/** Glob match where `*` matches any run of characters (including `/`). */
+export function globMatch(pattern, s) {
+    let p = 0;
+    let i = 0;
+    let star = -1;
+    let mark = 0;
+    while (i < s.length) {
+        if (p < pattern.length && pattern[p] !== '*' && pattern[p] === s[i]) {
+            p++;
+            i++;
+        }
+        else if (p < pattern.length && pattern[p] === '*') {
+            star = p++;
+            mark = i;
+        }
+        else if (star >= 0) {
+            p = star + 1;
+            i = ++mark;
+        }
+        else {
+            return false;
+        }
+    }
+    while (pattern[p] === '*')
+        p++;
+    return p === pattern.length;
+}
+/** Every matching block's headers; a name set by several blocks gets its values joined with `, `. */
+export function headersFor(rules, path) {
+    // No prototype: a header named `__proto__` is an ordinary name (as in the Rust port).
+    const out = Object.create(null);
+    for (const r of rules) {
+        if (!globMatch(r.pattern, path))
+            continue;
+        for (const [k, v] of r.headers)
+            out[k] = out[k] === undefined ? v : `${out[k]}, ${v}`;
+    }
+    return out;
+}
+/** The remainder a rule matches for `path` (`''` for the prefix itself), or undefined. */
+export function matchRedirect(r, path) {
+    if (!r.splat)
+        return path === r.from ? '' : undefined;
+    if (r.from === '')
+        return path.slice(1);
+    if (path === r.from)
+        return '';
+    return path.startsWith(r.from + '/') ? path.slice(r.from.length + 1) : undefined;
+}
+//# sourceMappingURL=serve-rules.js.map

package/lib/src/server.d.ts

@@ -0,0 +1,45 @@
+import http from 'node:http';
+import type { Tree } from './tree.js';
+export interface SiteServerOptions {
+    /** Fall back to `/index.html` for unknown extension-less paths (single-page apps). */
+    readonly spa?: boolean;
+    /** One access-log line per request: `METHOD /path STATUS BYTES MSms`. */
+    readonly log?: (line: string) => void;
+    /** Sent as `x-sitefile-server` so `servers` can tell its own servers from a reused port. */
+    readonly serverId?: string;
+    /**
+     * `serve --watch`: answer {@link RELOAD_PATH} with an event stream and add
+     * {@link RELOAD_SCRIPT} to HTML responses. Never set otherwise, and never written to exports.
+     */
+    readonly liveReload?: LiveReload;
+}
+/** Reserved for the live-reload event stream while `serve --watch` runs. */
+export declare const RELOAD_PATH = "/__sitefile/reload";
+/** Added to served HTML (before the last `</body>`, else at the end) under `serve --watch`. */
+export declare const RELOAD_SCRIPT = "<script>new EventSource(\"/__sitefile/reload\").onmessage=function(){location.reload()}</script>";
+/** Open live-reload streams; {@link LiveReload.reload} tells every page to reload. */
+export declare class LiveReload {
+    private readonly clients;
+    /** Hold `res` open as a text/event-stream until the client goes away. */
+    attach(res: http.ServerResponse, headers: Record<string, string>): void;
+    reload(): void;
+    get size(): number;
+}
+/** `html` with {@link RELOAD_SCRIPT} inserted before the last `</body` (ASCII case-insensitive), else appended. */
+export declare function injectReloadScript(html: Uint8Array): Uint8Array;
+/** Decode a request path into safe segments, or undefined when it is malformed or unsafe. */
+export declare function safeSegments(rawPath: string): string[] | undefined;
+/**
+ * An HTTP server for an immutable Tree. GET and HEAD only; nothing outside the tree is
+ * ever reachable because lookups are exact map keys built from validated segments.
+ * Honours `/404.html`, `/_redirects` and `/_headers` (which it never serves), and answers
+ * `/about` with `/about.html` when there is no such file or directory.
+ */
+export declare function createSiteServer(site: Tree | (() => Tree), options?: SiteServerOptions): http.Server;
+/** Listen on host:port (port 0 = any free port); EADDRINUSE and friends become E_SERVER. */
+export declare function listen(server: http.Server, host: string, port: number): Promise<{
+    host: string;
+    port: number;
+}>;
+export declare function closeServer(server: http.Server): Promise<void>;
+export declare function formatUrl(host: string, port: number): string;