@johnhenry/sitefile 0.0.0-stage → 0.0.0

package/lib/src/resources/archive.js

@@ -0,0 +1,306 @@
+import { gunzip as gunzipCb, inflateRaw as inflateRawCb } from 'node:zlib';
+import { promisify } from 'node:util';
+import { SitefileError } from '../errors.js';
+import { normalizeVirtual } from '../paths.js';
+import { makeFile, Tree } from '../tree.js';
+import { DEFAULT_ARCHIVE_LIMITS } from './types.js';
+import { isWebBundle, readWebBundle, webBundleToTree } from './wbn-reader.js';
+/** Below this many output bytes the ratio limit is not enforced (tar block padding compresses extremely well). */
+const RATIO_FLOOR = 1024 * 1024;
+const gunzipAsync = promisify(gunzipCb);
+const inflateRawAsync = promisify(inflateRawCb);
+/** Tar framing per entry, worst case: header + padding + a pax/GNU long-name header and its padding. */
+const TAR_OVERHEAD_PER_ENTRY = 4 * 512;
+/**
+ * Extract a tar, tar.gz/tgz, zip or Web Bundle (b1/b2) archive into a Tree. The format is detected
+ * from magic bytes, never from the name. Nothing is inflated beyond the limits,
+ * and unsafe entries (absolute or `..` paths, backslashes, NUL, symlinks, hard
+ * links, devices) are rejected, not skipped. A single top-level directory is
+ * kept as-is.
+ */
+export async function extractArchive(bytes, origin, limits = DEFAULT_ARCHIVE_LIMITS, span) {
+    if (isWebBundle(bytes))
+        return webBundleToTree(readWebBundle(bytes, origin, limits, span), origin, span);
+    const files = new Map();
+    const dirs = new Set();
+    let total = 0;
+    const add = (name, kind, data, mode) => {
+        const path = archivePath(name, span);
+        if (path === '/')
+            return;
+        if (files.has(path) || (kind === 'file' && dirs.has(path)) || (kind === 'dir' && files.has(path))) {
+            throw new SitefileError('E_CONFLICT', `${origin}: duplicate archive entry '${name}'`, span);
+        }
+        if (kind === 'dir') {
+            dirs.add(path);
+            return;
+        }
+        if (files.size + 1 > limits.maxEntries) {
+            throw new SitefileError('E_ARCHIVE_LIMIT', `${origin}: archive has more than ${limits.maxEntries} entries`, span);
+        }
+        total += data.byteLength;
+        if (total > limits.maxTotalBytes) {
+            throw new SitefileError('E_ARCHIVE_LIMIT', `${origin}: archive expands beyond ${limits.maxTotalBytes} bytes`, span);
+        }
+        files.set(path, makeFile(data, `${origin}:${name}`, mode));
+    };
+    if (isGzip(bytes)) {
+        const tar = await gunzip(bytes, origin, limits, span);
+        if (!isTar(tar))
+            throw new SitefileError('E_TYPE', `${origin}: gzip content is not a tar archive`, span);
+        readTar(tar, origin, limits, span, add);
+    }
+    else if (isZip(bytes)) {
+        await readZip(bytes, origin, limits, span, add);
+    }
+    else if (isTar(bytes)) {
+        readTar(bytes, origin, limits, span, add);
+    }
+    else {
+        throw new SitefileError('E_TYPE', `${origin}: not a recognized archive (expected tar, tar.gz, zip or web bundle)`, span);
+    }
+    return Tree.fromFiles(files);
+}
+const isGzip = (b) => b.length > 2 && b[0] === 0x1f && b[1] === 0x8b;
+const isZip = (b) => b.length >= 4 && b[0] === 0x50 && b[1] === 0x4b && ((b[2] === 3 && b[3] === 4) || (b[2] === 5 && b[3] === 6));
+const isTar = (b) => b.length >= 512 &&
+    (ascii(b, 257, 262) === 'ustar' ||
+        (tarChecksumOk(b, 0) && b.subarray(0, 512).some((x) => x !== 0)) ||
+        (b.length === 1024 && b.every((x) => x === 0))); // an empty tar
+function ascii(b, from, to) {
+    return Buffer.from(b.subarray(from, to)).toString('latin1');
+}
+/** Validate an entry name and return its normalized virtual path. */
+function archivePath(name, span) {
+    const bad = (why) => new SitefileError('E_ARCHIVE_UNSAFE', `unsafe archive entry '${name.replace(/\0/g, '\\0')}': ${why}`, span);
+    if (name.includes('\0'))
+        throw bad('NUL byte');
+    if (name.includes('\\'))
+        throw bad('backslash in path');
+    if (name.startsWith('/') || /^[A-Za-z]:/.test(name))
+        throw bad('absolute path');
+    if (name.split('/').includes('..'))
+        throw bad("'..' segment");
+    return normalizeVirtual('/' + name, span, 'archive entry').path;
+}
+// ------------------------------------------------------------------ gzip
+async function gunzip(bytes, origin, limits, span) {
+    const byRatio = Math.max(bytes.length * limits.maxRatio, RATIO_FLOOR);
+    // The inflated stream is tar: file data plus headers and padding for every entry.
+    const byTotal = limits.maxTotalBytes + (limits.maxEntries + 2) * TAR_OVERHEAD_PER_ENTRY;
+    try {
+        // Async so a large archive doesn't stall other imports' sockets and timers.
+        return await gunzipAsync(bytes, { maxOutputLength: Math.min(byRatio, byTotal) });
+    }
+    catch (e) {
+        throw inflateError(e, origin, byRatio <= byTotal, limits, span);
+    }
+}
+function inflateError(e, origin, ratio, limits, span) {
+    if (e.code === 'ERR_BUFFER_TOO_LARGE') {
+        return ratio
+            ? new SitefileError('E_ARCHIVE_LIMIT', `${origin}: compression ratio exceeds ${limits.maxRatio}:1`, span)
+            : new SitefileError('E_ARCHIVE_LIMIT', `${origin}: archive expands beyond ${limits.maxTotalBytes} bytes`, span);
+    }
+    return new SitefileError('E_ARCHIVE', `${origin}: corrupt compressed data`, span, { cause: e });
+}
+// ------------------------------------------------------------------- tar
+function tarChecksumOk(b, off) {
+    let sum = 0;
+    for (let i = 0; i < 512; i++)
+        sum += i >= 148 && i < 156 ? 32 : b[off + i];
+    const stored = parseInt(ascii(b, off + 148, off + 156).replace(/[\0 ]/g, ''), 8);
+    return stored === sum;
+}
+function tarNumber(b, from, to) {
+    if (b[from] & 0x80) {
+        let n = b[from] & 0x7f; // GNU base-256
+        for (let i = from + 1; i < to; i++)
+            n = n * 256 + b[i];
+        return n;
+    }
+    const s = ascii(b, from, to).replace(/[\0 ]/g, '');
+    return s === '' ? 0 : parseInt(s, 8);
+}
+function cString(b, from, to) {
+    let end = from;
+    while (end < to && b[end] !== 0)
+        end++;
+    return Buffer.from(b.subarray(from, end)).toString('utf8');
+}
+/** Supports ustar prefixes, GNU `L` long names and pax `path`/`size` records. */
+function readTar(b, origin, limits, span, add) {
+    const bad = (why) => new SitefileError('E_ARCHIVE', `${origin}: ${why}`, span);
+    let off = 0;
+    let nextName;
+    let nextSize;
+    let entries = 0;
+    while (off + 512 <= b.length) {
+        const h = off;
+        if (b.subarray(h, h + 512).every((x) => x === 0))
+            return; // end-of-archive marker
+        if (!tarChecksumOk(b, h))
+            throw bad('invalid tar header checksum');
+        const type = String.fromCharCode(b[h + 156]);
+        const size = nextSize ?? tarNumber(b, h + 124, h + 136);
+        if (!Number.isSafeInteger(size) || size < 0 || size > limits.maxTotalBytes) {
+            throw new SitefileError('E_ARCHIVE_LIMIT', `${origin}: archive entry larger than ${limits.maxTotalBytes} bytes`, span);
+        }
+        if (h + 512 + size > b.length)
+            throw bad('truncated tar archive');
+        const data = b.subarray(h + 512, h + 512 + size);
+        off = h + 512 + Math.ceil(size / 512) * 512;
+        if (type === 'L') {
+            nextName = cString(data, 0, data.length);
+            continue;
+        }
+        if (type === 'x') {
+            ({ path: nextName, size: nextSize } = parsePax(data, bad));
+            continue;
+        }
+        if (type === 'g' || type === 'K')
+            continue; // global pax header / GNU long link name: unused
+        if (++entries > limits.maxEntries) {
+            throw new SitefileError('E_ARCHIVE_LIMIT', `${origin}: archive has more than ${limits.maxEntries} entries`, span);
+        }
+        let name = nextName;
+        if (name === undefined) {
+            name = cString(b, h, h + 100);
+            // Only POSIX ustar ('ustar\0') has a name prefix; old GNU ('ustar ') keeps atime/ctime there.
+            const prefix = ascii(b, h + 257, h + 263) === 'ustar\0' ? cString(b, h + 345, h + 500) : '';
+            if (prefix)
+                name = `${prefix}/${name}`;
+        }
+        nextName = nextSize = undefined;
+        if (type === '0' || type === '\0' || type === '7')
+            add(name, 'file', data, tarNumber(b, h + 100, h + 108));
+        else if (type === '5')
+            add(name, 'dir', data, 0o755);
+        else {
+            const what = type === '1' ? 'hard link' : type === '2' ? 'symlink' : 'special file';
+            throw new SitefileError('E_ARCHIVE_UNSAFE', `unsafe archive entry '${name}': ${what} entries are not allowed`, span);
+        }
+    }
+}
+function parsePax(data, bad) {
+    const out = { path: undefined, size: undefined };
+    const buf = Buffer.from(data);
+    let pos = 0;
+    while (pos < buf.length) {
+        const sp = buf.indexOf(0x20, pos);
+        // No length prefix left: stop (without this a record missing its space looped forever).
+        if (sp < 0)
+            break;
+        const len = parseInt(buf.toString('latin1', pos, sp), 10);
+        if (!(len > sp - pos) || pos + len > buf.length)
+            break;
+        const kv = buf.toString('utf8', sp + 1, pos + len - 1); // drop trailing newline
+        const eq = kv.indexOf('=');
+        if (eq <= 0)
+            throw bad('malformed pax header record');
+        const key = kv.slice(0, eq);
+        const value = kv.slice(eq + 1);
+        if (key === 'path')
+            out.path = value;
+        if (key === 'size') {
+            if (!/^\d+$/.test(value))
+                throw bad(`malformed pax size '${value}'`);
+            out.size = Number(value);
+        }
+        pos += len;
+    }
+    return out;
+}
+async function readZip(b, origin, limits, span, add) {
+    const bad = (why) => new SitefileError('E_ARCHIVE', `${origin}: ${why}`, span);
+    const v = new DataView(b.buffer, b.byteOffset, b.byteLength);
+    let eocd = -1;
+    for (let i = b.length - 22; i >= Math.max(0, b.length - 22 - 0xffff); i--) {
+        if (v.getUint32(i, true) === 0x06054b50) {
+            eocd = i;
+            break;
+        }
+    }
+    if (eocd < 0)
+        throw bad('zip end-of-central-directory record not found');
+    const count = v.getUint16(eocd + 10, true);
+    let p = v.getUint32(eocd + 16, true);
+    if (count === 0xffff || p === 0xffffffff)
+        throw bad('zip64 archives are not supported');
+    if (count > limits.maxEntries) {
+        throw new SitefileError('E_ARCHIVE_LIMIT', `${origin}: archive has more than ${limits.maxEntries} entries`, span);
+    }
+    const list = [];
+    let declared = 0;
+    for (let i = 0; i < count; i++) {
+        if (p + 46 > b.length || v.getUint32(p, true) !== 0x02014b50)
+            throw bad('corrupt zip central directory');
+        const nameLen = v.getUint16(p + 28, true);
+        const advance = 46 + nameLen + v.getUint16(p + 30, true) + v.getUint16(p + 32, true);
+        if (p + advance > b.length)
+            throw bad('corrupt zip central directory');
+        const unix = v.getUint16(p + 4, true) >> 8 === 3;
+        const attr = v.getUint32(p + 38, true) >>> 16;
+        const fileType = attr & 0o170000;
+        const name = Buffer.from(b.subarray(p + 46, p + 46 + nameLen)).toString('utf8');
+        const e = {
+            name,
+            method: v.getUint16(p + 10, true),
+            csize: v.getUint32(p + 20, true),
+            usize: v.getUint32(p + 24, true),
+            local: v.getUint32(p + 42, true),
+            mode: unix ? attr & 0o7777 : 0o644,
+            // Some tools mark directories only by mode, without a trailing slash.
+            isDir: name.endsWith('/') || (unix && fileType === 0o040000),
+        };
+        if (v.getUint16(p + 8, true) & 1)
+            throw bad(`encrypted zip entry '${e.name}' is not supported`);
+        if (e.csize === 0xffffffff || e.usize === 0xffffffff)
+            throw bad('zip64 archives are not supported');
+        if (unix && fileType !== 0 && fileType !== 0o100000 && fileType !== 0o040000) {
+            throw new SitefileError('E_ARCHIVE_UNSAFE', `unsafe archive entry '${e.name}': symlink or special file entries are not allowed`, span);
+        }
+        declared += e.usize;
+        list.push(e);
+        p += advance;
+    }
+    if (declared > limits.maxTotalBytes) {
+        throw new SitefileError('E_ARCHIVE_LIMIT', `${origin}: archive expands beyond ${limits.maxTotalBytes} bytes`, span);
+    }
+    if (declared > Math.max(b.length * limits.maxRatio, RATIO_FLOOR)) {
+        throw new SitefileError('E_ARCHIVE_LIMIT', `${origin}: compression ratio exceeds ${limits.maxRatio}:1`, span);
+    }
+    for (const e of list) {
+        if (e.isDir) {
+            add(e.name, 'dir', new Uint8Array(0), 0o755);
+            continue;
+        }
+        const h = e.local;
+        if (h + 30 > b.length || v.getUint32(h, true) !== 0x04034b50)
+            throw bad(`corrupt zip local header for '${e.name}'`);
+        const start = h + 30 + v.getUint16(h + 26, true) + v.getUint16(h + 28, true);
+        if (start + e.csize > b.length)
+            throw bad(`truncated zip entry '${e.name}'`);
+        const raw = b.subarray(start, start + e.csize);
+        let data;
+        if (e.method === 0) {
+            data = raw;
+        }
+        else if (e.method === 8) {
+            try {
+                // Headers can lie: never inflate more than the declared size.
+                data = await inflateRawAsync(raw, { maxOutputLength: Math.max(e.usize, 1) });
+            }
+            catch (err) {
+                throw inflateError(err, origin, false, limits, span);
+            }
+        }
+        else {
+            throw bad(`zip entry '${e.name}' uses unsupported compression method ${e.method}`);
+        }
+        if (data.length !== e.usize)
+            throw bad(`zip entry '${e.name}' size does not match its header`);
+        add(e.name, 'file', data, e.mode);
+    }
+}
+//# sourceMappingURL=archive.js.map

package/lib/src/resources/bundle-data.d.ts

@@ -0,0 +1,16 @@
+import type { BundleEntry } from './wbn-reader.js';
+/**
+ * A Web Bundle as template data (`IMPORT b FROM ./x.wbn AS wbn`):
+ *
+ *   { version, primaryUrl, entries: [ { url, path, status, contentType,
+ *     headers, size, content? } ] }
+ *
+ * `path` is the URL's pathname; `content` is set only for textual responses
+ * (text/*, JSON, XML, JavaScript, SVG) whose body is valid UTF-8. Binary
+ * bodies are not exposed as data: use `AS tree` and COPY to place them.
+ */
+export declare function bundleValue(bundle: {
+    version: string;
+    primaryUrl: string | undefined;
+    entries: readonly BundleEntry[];
+}): Record<string, unknown>;

package/lib/src/resources/bundle-data.js

@@ -0,0 +1,50 @@
+/**
+ * A Web Bundle as template data (`IMPORT b FROM ./x.wbn AS wbn`):
+ *
+ *   { version, primaryUrl, entries: [ { url, path, status, contentType,
+ *     headers, size, content? } ] }
+ *
+ * `path` is the URL's pathname; `content` is set only for textual responses
+ * (text/*, JSON, XML, JavaScript, SVG) whose body is valid UTF-8. Binary
+ * bodies are not exposed as data: use `AS tree` and COPY to place them.
+ */
+export function bundleValue(bundle) {
+    return {
+        version: bundle.version,
+        primaryUrl: bundle.primaryUrl ?? null,
+        entries: bundle.entries.map((e) => {
+            const contentType = e.headers['content-type'] ?? '';
+            const content = isTextual(contentType) ? utf8(e.body) : undefined;
+            return {
+                url: e.url,
+                path: pathOf(e.url),
+                status: e.status,
+                contentType,
+                headers: { ...e.headers },
+                size: e.body.byteLength,
+                ...(content !== undefined ? { content } : {}),
+            };
+        }),
+    };
+}
+const TEXTUAL = /^(text\/|application\/(json|xml|javascript|ecmascript|manifest\+json|[a-z0-9.+-]*\+(json|xml))|image\/svg\+xml)/i;
+function isTextual(contentType) {
+    return TEXTUAL.test(contentType.trim());
+}
+function utf8(bytes) {
+    try {
+        return new TextDecoder('utf-8', { fatal: true }).decode(bytes);
+    }
+    catch {
+        return undefined;
+    }
+}
+function pathOf(url) {
+    try {
+        return new URL(url).pathname;
+    }
+    catch {
+        return url;
+    }
+}
+//# sourceMappingURL=bundle-data.js.map

package/lib/src/resources/convert.d.ts

@@ -0,0 +1,12 @@
+import { type SourceSpan } from '../errors.js';
+import { type ArchiveLimits, type Resource } from './types.js';
+export interface BytesInfo {
+    readonly uri: string;
+    readonly resolvedUrl?: string;
+    readonly basename: string;
+    readonly mode?: number;
+}
+/** Turn fetched or read bytes into a resource of the requested kind. */
+export declare function bytesToResource(bytes: Uint8Array, kind: 'file' | 'text' | 'json' | 'tree' | 'markdown' | 'yaml' | 'csv' | 'wbn' | 'html', info: BytesInfo, span?: SourceSpan, archiveLimits?: ArchiveLimits): Promise<Resource>;
+/** Throw E_INTEGRITY unless sha256(bytes) equals `expected` (`sha256:<hex>`). */
+export declare function verifyIntegrity(bytes: Uint8Array, expected: string, uri: string, span?: SourceSpan): void;

package/lib/src/resources/convert.js

@@ -0,0 +1,63 @@
+import { SitefileError } from '../errors.js';
+import { sha256 } from '../digest.js';
+import { extractArchive } from './archive.js';
+import { markdownDocument } from './documents.js';
+import { bundleValue } from './bundle-data.js';
+import { readWebBundle } from './wbn-reader.js';
+import { parseYaml } from '../yaml.js';
+import { parseCsvRecords } from '../csv.js';
+import { DEFAULT_ARCHIVE_LIMITS } from './types.js';
+function decodeUtf8(bytes, uri, span) {
+    try {
+        return new TextDecoder('utf-8', { fatal: true }).decode(bytes);
+    }
+    catch (e) {
+        throw new SitefileError('E_ENCODING', `${uri} is not valid UTF-8`, span, { cause: e });
+    }
+}
+/** Turn fetched or read bytes into a resource of the requested kind. */
+export async function bytesToResource(bytes, kind, info, span, archiveLimits = DEFAULT_ARCHIVE_LIMITS) {
+    const digest = sha256(bytes);
+    const provenance = info.resolvedUrl === undefined ? { uri: info.uri, digest } : { uri: info.uri, resolvedUrl: info.resolvedUrl, digest };
+    if (kind === 'tree') {
+        // The provenance digest is that of the archive bytes, which is what pins and the lockfile cover.
+        return { kind: 'tree', tree: await extractArchive(bytes, info.uri, archiveLimits, span), provenance };
+    }
+    if (kind === 'markdown') {
+        return { kind: 'value', format: 'markdown', value: markdownDocument(bytes, '/' + info.basename, info.uri, span), provenance };
+    }
+    if (kind === 'wbn') {
+        return { kind: 'value', format: 'wbn', value: bundleValue(readWebBundle(bytes, info.uri, archiveLimits, span)), provenance };
+    }
+    if (kind === 'file') {
+        const mode = (info.mode ?? 0o644) & 0o111 ? 0o755 : 0o644;
+        return { kind: 'file', bytes, mode, basename: info.basename, provenance };
+    }
+    const text = decodeUtf8(bytes, info.uri, span);
+    if (kind === 'text')
+        return { kind: 'value', format: 'text', value: text, provenance };
+    // A file-based html fragment: its template is parsed where a RENDER uses it.
+    if (kind === 'html')
+        return { kind: 'value', format: 'html', value: text, provenance };
+    if (kind === 'yaml')
+        return { kind: 'value', format: 'yaml', value: parseYaml(text, info.uri), provenance };
+    if (kind === 'csv')
+        return { kind: 'value', format: 'csv', value: parseCsvRecords(text, info.uri), provenance };
+    let value;
+    try {
+        // TextDecoder already strips a leading BOM.
+        value = JSON.parse(text);
+    }
+    catch (e) {
+        throw new SitefileError('E_TYPE', `${info.uri} is not valid JSON: ${e.message}`, span, { cause: e });
+    }
+    return { kind: 'value', format: 'json', value, provenance };
+}
+/** Throw E_INTEGRITY unless sha256(bytes) equals `expected` (`sha256:<hex>`). */
+export function verifyIntegrity(bytes, expected, uri, span) {
+    const actual = sha256(bytes);
+    if (actual !== expected.toLowerCase()) {
+        throw new SitefileError('E_INTEGRITY', `integrity mismatch for ${uri}: expected ${expected}, got ${actual}`, span);
+    }
+}
+//# sourceMappingURL=convert.js.map

package/lib/src/resources/documents.d.ts

@@ -0,0 +1,35 @@
+import { type SourceSpan } from '../errors.js';
+import type { Tree } from '../tree.js';
+import type { DocumentMemo } from '../memo.js';
+/**
+ * A Markdown document as templates see it: its frontmatter fields plus
+ *
+ * - `slug`: the file name without `.md` and without a leading `YYYY-MM-DD-`
+ *   (frontmatter `slug` wins)
+ * - `date`: that leading date, if the frontmatter has no `date`
+ * - `path`: the file's path inside the import (`/2026-01-02-hello.md`)
+ * - `content`: the body rendered to HTML (insert it with `${post.content|raw}`)
+ * - `excerpt`: plain text of the first paragraph (up to ~200 characters)
+ *
+ * `path`, `content` and `excerpt` are always computed; frontmatter keys with
+ * those names are ignored.
+ */
+export type MarkdownDocument = Record<string, unknown>;
+export declare function markdownDocument(bytes: Uint8Array, path: string, label: string, span?: SourceSpan): MarkdownDocument;
+/** Every `.md` / `.markdown` file in a tree, as documents, ordered by path. */
+export declare function markdownCollection(tree: Tree, labelPrefix: string, span?: SourceSpan, memo?: DocumentMemo): MarkdownDocument[];
+/**
+ * Every `.json` (AS json) or `.yaml`/`.yml` (AS yaml) file in a tree, as a list
+ * ordered by path. An object becomes `{ slug, date?, path, ...fields }` (slug and
+ * date from the file name as for Markdown, overridable by the data); any other
+ * value becomes `{ slug, path, value }`.
+ */
+export declare function dataCollection(tree: Tree, kind: 'json' | 'yaml', labelPrefix: string, span?: SourceSpan, memo?: DocumentMemo): MarkdownDocument[];
+/**
+ * Sort documents by a field. Numbers and numeric strings compare numerically,
+ * everything else as strings (ISO dates sort correctly). Documents missing the field go last in
+ * either direction; ties keep path order.
+ */
+export declare function sortDocuments(docs: readonly MarkdownDocument[], key: string, descending: boolean): MarkdownDocument[];
+/** SORT BY's order of two field values: numbers and numeric strings (such as CSV cells) numerically, else as strings. */
+export declare function compareValues(a: unknown, b: unknown): number;

package/lib/src/resources/documents.js

@@ -0,0 +1,122 @@
+import { posix } from 'node:path';
+import { SitefileError } from '../errors.js';
+import { parseFrontmatter } from '../frontmatter.js';
+import { renderMarkdown } from '../markdown.js';
+import { parseYaml } from '../yaml.js';
+import { decodeUtf8 } from '../render.js';
+const DATED = /^(\d{4}-\d{2}-\d{2})-(.+)$/;
+const NUMERIC = /^-?\d+(\.\d+)?([eE][+-]?\d+)?$/;
+const SAFE_SLUG = /^[A-Za-z0-9][A-Za-z0-9._~-]*$/;
+export function markdownDocument(bytes, path, label, span) {
+    const text = decodeUtf8(bytes, `'${label}'`, span);
+    const { data, body } = parseFrontmatter(text, label);
+    const { html, excerpt } = renderMarkdown(body);
+    const stem = posix.basename(path).replace(/\.(md|markdown)$/i, '');
+    const dated = DATED.exec(stem);
+    const doc = {
+        slug: dated ? dated[2] : stem,
+        ...(dated ? { date: dated[1] } : {}),
+        ...data,
+        path,
+        content: html,
+        excerpt,
+    };
+    const slug = doc['slug'];
+    if (typeof slug !== 'string' || !SAFE_SLUG.test(slug)) {
+        throw new SitefileError('E_VALIDATION', `'${label}': slug ${JSON.stringify(slug)} must be letters, digits, '.', '_', '~' or '-' (it becomes part of a URL)`, span);
+    }
+    return doc;
+}
+/** Every `.md` / `.markdown` file in a tree, as documents, ordered by path. */
+export function markdownCollection(tree, labelPrefix, span, memo) {
+    const docs = [];
+    for (const [p, node] of tree.entries()) {
+        if (!/\.(md|markdown)$/i.test(p))
+            continue;
+        const label = `${labelPrefix}${p}`;
+        const make = () => markdownDocument(node.bytes, p, label, span);
+        // Incremental rebuilds: a document is a function of its bytes, path and label.
+        docs.push(memo ? memo(['md', node.digest, p, label], make) : make());
+    }
+    return docs;
+}
+/**
+ * Every `.json` (AS json) or `.yaml`/`.yml` (AS yaml) file in a tree, as a list
+ * ordered by path. An object becomes `{ slug, date?, path, ...fields }` (slug and
+ * date from the file name as for Markdown, overridable by the data); any other
+ * value becomes `{ slug, path, value }`.
+ */
+export function dataCollection(tree, kind, labelPrefix, span, memo) {
+    const ext = kind === 'json' ? /\.json$/i : /\.ya?ml$/i;
+    const docs = [];
+    for (const [p, node] of tree.entries()) {
+        if (!ext.test(p))
+            continue;
+        const label = `${labelPrefix}${p}`;
+        if (memo) {
+            docs.push(memo([kind, node.digest, p, label], () => dataDocument(node.bytes, kind, ext, p, label, span)));
+            continue;
+        }
+        docs.push(dataDocument(node.bytes, kind, ext, p, label, span));
+    }
+    return docs;
+}
+function dataDocument(bytes, kind, ext, p, label, span) {
+    const text = decodeUtf8(bytes, `'${label}'`, span);
+    let value;
+    if (kind === 'yaml')
+        value = parseYaml(text, label);
+    else {
+        try {
+            value = JSON.parse(text);
+        }
+        catch (e) {
+            throw new SitefileError('E_TYPE', `${label} is not valid JSON: ${e.message}`, span, { cause: e });
+        }
+    }
+    const stem = posix.basename(p).replace(ext, '');
+    const dated = DATED.exec(stem);
+    const base = { slug: dated ? dated[2] : stem, ...(dated ? { date: dated[1] } : {}) };
+    const fields = value !== null && typeof value === 'object' && !Array.isArray(value) ? value : { value };
+    const doc = { ...base, ...fields, path: p };
+    const slug = doc['slug'];
+    if (typeof slug !== 'string' || !SAFE_SLUG.test(slug)) {
+        throw new SitefileError('E_VALIDATION', `'${label}': slug ${JSON.stringify(slug)} must be letters, digits, '.', '_', '~' or '-'`, span);
+    }
+    return doc;
+}
+/**
+ * Sort documents by a field. Numbers and numeric strings compare numerically,
+ * everything else as strings (ISO dates sort correctly). Documents missing the field go last in
+ * either direction; ties keep path order.
+ */
+export function sortDocuments(docs, key, descending) {
+    const cmp = compareValues;
+    return [...docs].sort((a, b) => {
+        const va = a[key];
+        const vb = b[key];
+        const ma = va === undefined || va === null;
+        const mb = vb === undefined || vb === null;
+        if (ma || mb)
+            return ma === mb ? byPath(a, b) : ma ? 1 : -1;
+        const c = cmp(va, vb);
+        return c === 0 ? byPath(a, b) : descending ? -c : c;
+    });
+}
+/** SORT BY's order of two field values: numbers and numeric strings (such as CSV cells) numerically, else as strings. */
+export function compareValues(a, b) {
+    const num = (v) => typeof v === 'number' ? v : typeof v === 'string' && NUMERIC.test(v.trim()) ? Number(v) : undefined;
+    const na = num(a);
+    const nb = num(b);
+    if (na !== undefined && nb !== undefined)
+        return na - nb;
+    const x = String(a);
+    const y = String(b);
+    return x < y ? -1 : x > y ? 1 : 0;
+}
+function byPath(a, b) {
+    const x = String(a['path']);
+    const y = String(b['path']);
+    return x < y ? -1 : x > y ? 1 : 0;
+}
+//# sourceMappingURL=documents.js.map