canopycms 0.0.68-int.98 → 0.1.0

package/dist/build/asset-refs.js

@@ -0,0 +1,267 @@
+/**
+ * Collect the asset URLs a static build references, so a release can make them exist first.
+ *
+ * The public `/assets/t/` path serves only what is stored, and widths are chosen by site code at
+ * render time, so build output is the one place every final URL appears. The adopter contract
+ * that follows: every `/assets/t/` URL the site can request must appear as text in its build
+ * output (ARCHITECTURE.md, "Why transform by URL directive and materialize at release?").
+ */
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { z } from 'zod';
+import { ASSET_PREFIXES } from '../assets/asset-prefixes.js';
+import { canonicalizeTransformPath } from '../assets/transform-directives.js';
+import { atomicWriteFile } from '../utils/atomic-write.js';
+import { extractInlineFlight } from './inline-flight.js';
+/** @internal Written into the scanned directory, so a manifest the adopter builds afterwards covers it. */
+export const ASSET_REFS_FILENAME = 'canopy-asset-refs.json';
+/**
+ * Text outputs a URL can hide in: pages, RSC payloads, data, scripts, styles, feeds, generated
+ * markdown. A file with no extension (a route handler's output) is scanned unless it is binary.
+ */
+const SCANNED_EXTENSIONS = new Set([
+    '',
+    '.html',
+    '.htm',
+    '.txt',
+    '.rsc',
+    '.json',
+    '.js',
+    '.mjs',
+    '.cjs',
+    '.css',
+    '.xml',
+    '.svg',
+    '.md',
+    '.webmanifest',
+]);
+/**
+ * Anchored on the path, never on an origin or a mount prefix, so whatever precedes `/assets/` —
+ * an absolute origin, a basePath, the editor's authenticated prefix — is dropped. The directive
+ * and filename classes stop at anything that ends a URL in HTML, JSON, JS or CSS (quotes,
+ * whitespace, a backslash escape, parens, a query; the filename also at a srcset comma) or at any
+ * non-ASCII character (typographic quotes and dashes in prose; no stored key has one), and exclude
+ * `{`, `}` and `$`, so a URL template in a script never reads as a URL. A non-ASCII character
+ * inside a directive segment therefore ends the match and the URL is not seen. The hash class
+ * admits upper case only so that such a URL is reported rather than skipped.
+ */
+const URL_STOP = String.raw `\s"'\x60<>()\\/?#&{}$\u0080-\uffff`;
+// eslint-disable-next-line security/detect-non-literal-regexp -- built from constants above
+const TRANSFORM_URL_RE = new RegExp(String.raw `/${ASSET_PREFIXES.transform}/([^${URL_STOP}]+)/([a-fA-F0-9]{32})/([^${URL_STOP},]+)`, 'g');
+// eslint-disable-next-line security/detect-non-literal-regexp -- built from constants above
+const STATIC_URL_RE = new RegExp(String.raw `/${ASSET_PREFIXES.public}/([a-fA-F0-9]{32})/([^${URL_STOP},]+)`, 'g');
+const STATIC_FILENAME_RE = /^[a-z0-9-]+\.[a-z0-9]{1,10}$/;
+const HASH32_RE = /^[a-f0-9]{32}$/;
+const assetRefEntrySchema = z
+    .object({
+    key: z.string().min(1),
+    routes: z.array(z.string()),
+    files: z.array(z.string()),
+})
+    .strict();
+const assetRefsFileSchema = z
+    .object({
+    version: z.literal(1),
+    transforms: z.array(assetRefEntrySchema),
+    statics: z.array(assetRefEntrySchema),
+})
+    .strict();
+/** The output references URLs that cannot be served from the store; nothing was written. */
+export class AssetRefsError extends Error {
+    constructor(problems) {
+        super(`${problems.length} asset URL(s) in the build output cannot be served from the store:\n` +
+            problems.map((p) => `  ${p.file}: ${p.url}\n    ${p.error}`).join('\n'));
+        this.problems = problems;
+        this.name = 'AssetRefsError';
+    }
+}
+/** Output-relative POSIX paths of every file under `root` with a scanned extension, sorted. */
+async function listScannedFiles(root) {
+    const entries = await fs.readdir(root, { recursive: true, withFileTypes: true });
+    return entries
+        .filter((entry) => entry.isFile() && SCANNED_EXTENSIONS.has(path.extname(entry.name)))
+        .map((entry) => path.relative(root, path.join(entry.parentPath, entry.name)))
+        .map((rel) => rel.split(path.sep).join('/'))
+        .filter((rel) => rel !== ASSET_REFS_FILENAME)
+        .sort();
+}
+const HTML_FILE_RE = /\.html?$/;
+/** The route a static-export HTML file serves (`about/index.html` and `about.html` are `/about`). */
+function routeForFile(file) {
+    if (!HTML_FILE_RE.test(file))
+        return undefined;
+    const withoutExt = file.replace(HTML_FILE_RE, '');
+    const route = withoutExt === 'index' ? '' : withoutExt.replace(/\/index$/, '');
+    return `/${route}`;
+}
+const ENCODED_ASSETS = /%2[fF]assets%2[fF]/g;
+const RUN_STOP = /[\s"'`<>&]/;
+/** Each `%XX` escape of an ASCII character, decoded on its own; anything else stays as written. */
+function decodeAsciiEscapes(text) {
+    return text.replace(/%([0-7][0-9a-fA-F])/g, (_, hex) => String.fromCharCode(parseInt(hex, 16)));
+}
+/** One malformed or non-UTF-8 escape elsewhere in the run must not hide the URL inside it. */
+function decodeOrKeep(run) {
+    try {
+        return decodeURIComponent(run);
+    }
+    catch {
+        return decodeAsciiEscapes(run);
+    }
+}
+/**
+ * Decode, whole, each percent-encoded run holding an encoded `/assets/` (an image optimizer's
+ * `?url=`), so an encoded `%20` or `%3F` after the URL ends it just as a literal one would. Runs are
+ * found by expanding outward from each match, so the scan stays linear on long text with no stop
+ * character.
+ */
+function decodeEncodedAssetRuns(text) {
+    let out = '';
+    let last = 0;
+    for (const match of text.matchAll(ENCODED_ASSETS)) {
+        if (match.index < last)
+            continue;
+        let start = match.index;
+        while (start > last && !RUN_STOP.test(text[start - 1]))
+            start--;
+        let end = match.index + match[0].length;
+        while (end < text.length && !RUN_STOP.test(text[end]))
+            end++;
+        out += text.slice(last, start) + decodeOrKeep(text.slice(start, end));
+        last = end;
+    }
+    return out + text.slice(last);
+}
+/** JSON and JS may escape `/` as a backslash-slash or a `u002F` unicode escape. */
+function decodeUrlEscapes(text) {
+    return decodeEncodedAssetRuns(text.replace(/\\u002[fF]/g, '/').replace(/\\\//g, '/'));
+}
+/**
+ * A slug and ext end in `[a-z0-9]`, so punctuation after a URL in prose is not part of it. Trimmed
+ * wherever it occurs: an RSC payload carries prose as JSON strings, where a sentence-final URL is
+ * followed by a quote exactly as an attribute value is.
+ */
+function trimTrailingPunctuation(url, filename) {
+    const trimmed = filename.replace(/[.:;!?]+$/, '');
+    return [url.slice(0, url.length - (filename.length - trimmed.length)), trimmed];
+}
+/** A NUL in the first 8 KiB marks a file binary, close to git's own test; only that block is read. */
+async function startsBinary(filePath) {
+    const handle = await fs.open(filePath, 'r');
+    try {
+        const block = Buffer.alloc(8192);
+        const { bytesRead } = await handle.read(block, 0, block.length, 0);
+        return block.subarray(0, bytesRead).includes(0);
+    }
+    finally {
+        await handle.close();
+    }
+}
+class RefCollector {
+    constructor() {
+        this.entries = new Map();
+    }
+    add(key, file) {
+        const entry = this.entries.get(key) ?? { routes: new Set(), files: new Set() };
+        entry.files.add(file);
+        const route = routeForFile(file);
+        if (route !== undefined)
+            entry.routes.add(route);
+        this.entries.set(key, entry);
+    }
+    toEntries() {
+        return [...this.entries.keys()].sort().map((key) => {
+            const entry = this.entries.get(key);
+            return {
+                key,
+                routes: [...(entry?.routes ?? [])].sort(),
+                files: [...(entry?.files ?? [])].sort(),
+            };
+        });
+    }
+}
+/**
+ * Scan `outDir` for `/assets/t/...` transform URLs and `/assets/{hash32}/...` static URLs and
+ * write their stored keys to `outDir/canopy-asset-refs.json`, sorted so the same output always
+ * yields the same file. Throws `AssetRefsError`, writing nothing, if any transform URL is not in
+ * canonical form or any static URL is malformed: only canonical keys are stored, and the public
+ * path is to compute nothing on a miss.
+ */
+export async function collectAssetRefs(outDir) {
+    const root = path.resolve(outDir);
+    const files = await listScannedFiles(root);
+    const transforms = new RefCollector();
+    const statics = new RefCollector();
+    const problems = [];
+    const scan = (file, raw) => {
+        const text = decodeUrlEscapes(raw);
+        for (const match of text.matchAll(TRANSFORM_URL_RE)) {
+            const [url, filename] = trimTrailingPunctuation(match[0], match[3]);
+            const [, encodedDirectives, hash32] = match;
+            // A loader that encodes the directive segment alone is still requesting this key.
+            const directives = decodeAsciiEscapes(encodedDirectives);
+            const canonical = canonicalizeTransformPath([directives, hash32, filename], 'any');
+            if (!canonical.ok) {
+                problems.push({ file, url, error: canonical.error });
+            }
+            else if (!canonical.isCanonical) {
+                problems.push({
+                    file,
+                    url,
+                    error: `Not canonical; assetUrl would write /${ASSET_PREFIXES.transform}/${canonical.canonicalPath}`,
+                });
+            }
+            else {
+                transforms.add(`${ASSET_PREFIXES.transform}/${canonical.canonicalPath}`, file);
+            }
+        }
+        for (const match of text.matchAll(STATIC_URL_RE)) {
+            const [url, filename] = trimTrailingPunctuation(match[0], match[2]);
+            const hash32 = match[1];
+            if (!HASH32_RE.test(hash32) || !STATIC_FILENAME_RE.test(filename)) {
+                problems.push({ file, url, error: 'Not a stored static asset path' });
+            }
+            else {
+                statics.add(`${ASSET_PREFIXES.public}/${hash32}/${filename}`, file);
+            }
+        }
+    };
+    let scannedFiles = 0;
+    for (const file of files) {
+        const filePath = path.join(root, file);
+        if (path.extname(file) === '' && (await startsBinary(filePath)))
+            continue;
+        scannedFiles++;
+        const text = await fs.readFile(filePath, 'utf-8');
+        if (!HTML_FILE_RE.test(file)) {
+            scan(file, text);
+            continue;
+        }
+        // A page's inline RSC payload is split across scripts at arbitrary bytes (inline-flight.ts).
+        const page = extractInlineFlight(text);
+        scan(file, page.markup);
+        for (const flight of page.texts)
+            scan(file, flight);
+        for (const { script, error } of page.problems)
+            problems.push({ file, url: script, error });
+    }
+    if (problems.length > 0)
+        throw new AssetRefsError(problems);
+    const refs = {
+        version: 1,
+        transforms: transforms.toEntries(),
+        statics: statics.toEntries(),
+    };
+    const filePath = path.join(root, ASSET_REFS_FILENAME);
+    await atomicWriteFile(filePath, `${JSON.stringify(refs, null, 2)}\n`);
+    return { refs, filePath, scannedFiles };
+}
+/** Read and validate a refs file `collectAssetRefs` wrote. */
+export async function readAssetRefsFile(filePath) {
+    const raw = JSON.parse(await fs.readFile(filePath, 'utf-8'));
+    const parsed = assetRefsFileSchema.safeParse(raw);
+    if (!parsed.success) {
+        throw new Error(`${filePath} is not a valid asset refs file: ${parsed.error.message}`);
+    }
+    return parsed.data;
+}

package/dist/build/generate-ai-content.js

@@ -181,6 +181,11 @@ export async function generateAIContentFiles(options) {
     else {
         const schemaCache = new BranchSchemaCache(config.mode);
         const cached = await schemaCache.getSchema(branchRoot, entrySchemaRegistry, contentRootName);
+        // A branch workspace resolves degraded (see BranchSchemaCache); a build command fails instead.
+        if (cached.issues.length > 0) {
+            throw new Error(`AI content generation: the schema in ${branchRoot} does not match this code's entry ` +
+                `schema registry:\n${cached.issues.map((issue) => `  - ${issue.message}`).join('\n')}`);
+        }
         flatSchema = cached.flatSchema;
     }
     // Fail the build on abandoned create-scaffolds (schema-invalid entries) rather than baking them

package/dist/build/inline-flight.d.ts

@@ -0,0 +1,37 @@
+/**
+ * Reassemble the RSC (flight) stream Next's App Router inlines into a page, so a URL the stream
+ * splits across two scripts is scanned whole.
+ *
+ * Next writes each chunk of the stream as its own `<script>`, and React Flight cuts chunks at
+ * arbitrary bytes (it fills fixed-size views and enqueues the partial one at every flush), so a URL
+ * can straddle two scripts and the page text holds only its fragments. The format is the same in Next 15.5.21 and 16.1.7
+ * (`createInlinedDataReadableStream` in `dist/server/app-render/use-flight-response.js`, read back
+ * by `nextServerDataCallback` in `dist/client/app-index.js`):
+ *
+ * - `(self.__next_f=self.__next_f||[]).push([0])` starts the stream, followed in the same script by
+ *   `;self.__next_f.push([2,formState])` when there is form state;
+ * - each chunk is then `self.__next_f.push([1,text])`, or `[3,base64]` for a chunk that is not
+ *   valid UTF-8, whose bytes join the same stream;
+ * - every argument is `JSON.stringify` output passed through `htmlEscapeJsonString`, which only
+ *   rewrites `& < > U+2028 U+2029` as `\uXXXX` escapes, so it parses as JSON and never holds
+ *   `</script`.
+ *
+ * Every push on a page joins one stream: Next inlines one flight stream per page.
+ */
+export interface InlineFlight {
+    /** The page with each parsed flight script replaced by a space, so no fragment is scanned. */
+    markup: string;
+    /** The stream's text, joined in document order, then each form state as JSON. */
+    texts: string[];
+    /** Scripts that name `__next_f` but are not flight pushes; their data cannot be scanned. */
+    problems: {
+        script: string;
+        error: string;
+    }[];
+}
+/**
+ * Split `html` into its markup and its inline flight stream. Only a script whose body names
+ * `__next_f` is flight; one that does but does not parse stays in the markup and is reported,
+ * because skipping it could drop a reference.
+ */
+export declare function extractInlineFlight(html: string): InlineFlight;

package/dist/build/inline-flight.js

@@ -0,0 +1,142 @@
+/**
+ * Reassemble the RSC (flight) stream Next's App Router inlines into a page, so a URL the stream
+ * splits across two scripts is scanned whole.
+ *
+ * Next writes each chunk of the stream as its own `<script>`, and React Flight cuts chunks at
+ * arbitrary bytes (it fills fixed-size views and enqueues the partial one at every flush), so a URL
+ * can straddle two scripts and the page text holds only its fragments. The format is the same in Next 15.5.21 and 16.1.7
+ * (`createInlinedDataReadableStream` in `dist/server/app-render/use-flight-response.js`, read back
+ * by `nextServerDataCallback` in `dist/client/app-index.js`):
+ *
+ * - `(self.__next_f=self.__next_f||[]).push([0])` starts the stream, followed in the same script by
+ *   `;self.__next_f.push([2,formState])` when there is form state;
+ * - each chunk is then `self.__next_f.push([1,text])`, or `[3,base64]` for a chunk that is not
+ *   valid UTF-8, whose bytes join the same stream;
+ * - every argument is `JSON.stringify` output passed through `htmlEscapeJsonString`, which only
+ *   rewrites `& < > U+2028 U+2029` as `\uXXXX` escapes, so it parses as JSON and never holds
+ *   `</script`.
+ *
+ * Every push on a page joins one stream: Next inlines one flight stream per page.
+ */
+import { getErrorMessage } from '../utils/error.js';
+const PUSH_CALL_RE = /(?:\(self\.__next_f=self\.__next_f\|\|\[\]\)|self\.__next_f)\.push\(/y;
+const BASE64_RE = /^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/;
+/** The index just past the JSON array that opens at `start`; `JSON.parse` checks the rest. */
+function jsonArrayEnd(body, start) {
+    if (body[start] !== '[')
+        throw new Error(`expected a JSON array at ${start}`);
+    let depth = 0;
+    let inString = false;
+    for (let i = start; i < body.length; i++) {
+        const c = body[i];
+        if (inString) {
+            if (c === '\\')
+                i++;
+            else if (c === '"')
+                inString = false;
+        }
+        else if (c === '"') {
+            inString = true;
+        }
+        else if (c === '[' || c === '{') {
+            depth++;
+        }
+        else if (c === ']' || c === '}') {
+            depth--;
+            if (depth === 0)
+                return i + 1;
+        }
+    }
+    throw new Error('unterminated JSON array');
+}
+/** The arguments of a script body that is nothing but `;`-joined `self.__next_f.push(...)` calls. */
+function parsePushArguments(body) {
+    const args = [];
+    let pos = 0;
+    for (;;) {
+        PUSH_CALL_RE.lastIndex = pos;
+        if (!PUSH_CALL_RE.test(body))
+            throw new Error(`expected self.__next_f.push( at ${pos}`);
+        const end = jsonArrayEnd(body, PUSH_CALL_RE.lastIndex);
+        args.push(JSON.parse(body.slice(PUSH_CALL_RE.lastIndex, end)));
+        if (body[end] !== ')')
+            throw new Error(`expected ) at ${end}`);
+        pos = end + 1;
+        if (pos === body.length)
+            return args;
+        if (body[pos] !== ';')
+            throw new Error(`expected ; or the end of the script at ${pos}`);
+        pos++;
+    }
+}
+/** A push's contribution: bytes of the stream, form state as JSON, or nothing (the bootstrap). */
+function readSegment(arg) {
+    if (!Array.isArray(arg))
+        throw new Error('argument is not an array');
+    const [kind, payload] = arg;
+    if (kind === 0 && arg.length === 1)
+        return {};
+    if (kind === 1 && arg.length === 2 && typeof payload === 'string') {
+        return { bytes: Buffer.from(payload, 'utf8') };
+    }
+    if (kind === 2 && arg.length === 2)
+        return { formState: JSON.stringify(payload) };
+    if (kind === 3 && arg.length === 2 && typeof payload === 'string' && BASE64_RE.test(payload)) {
+        return { bytes: Buffer.from(payload, 'base64') };
+    }
+    throw new Error(`unknown segment ${JSON.stringify(arg).slice(0, 40)}`);
+}
+/** Each script element's span and body, in document order; one pass, however the page ends. */
+function* scriptElements(html) {
+    const openRe = /<script\b[^<>]*>/gi;
+    const closeRe = /<\/script\s*>/gi;
+    for (let open = openRe.exec(html); open; open = openRe.exec(html)) {
+        closeRe.lastIndex = openRe.lastIndex;
+        const close = closeRe.exec(html);
+        if (!close)
+            return;
+        yield {
+            start: open.index,
+            end: closeRe.lastIndex,
+            body: html.slice(openRe.lastIndex, close.index),
+        };
+        openRe.lastIndex = closeRe.lastIndex;
+    }
+}
+/**
+ * Split `html` into its markup and its inline flight stream. Only a script whose body names
+ * `__next_f` is flight; one that does but does not parse stays in the markup and is reported,
+ * because skipping it could drop a reference.
+ */
+export function extractInlineFlight(html) {
+    const stream = [];
+    const formStates = [];
+    const problems = [];
+    let markup = '';
+    let last = 0;
+    for (const { start, end, body } of scriptElements(html)) {
+        if (!body.includes('__next_f'))
+            continue;
+        let segments;
+        try {
+            segments = parsePushArguments(body).map(readSegment);
+        }
+        catch (err) {
+            problems.push({
+                script: `<script> at offset ${start}`,
+                error: `Not a Next inline flight script (${getErrorMessage(err)}); its RSC payload cannot be scanned`,
+            });
+            continue;
+        }
+        for (const { bytes, formState } of segments) {
+            if (bytes)
+                stream.push(bytes);
+            if (formState !== undefined)
+                formStates.push(formState);
+        }
+        markup += `${html.slice(last, start)} `;
+        last = end;
+    }
+    const texts = stream.length > 0 ? [Buffer.concat(stream).toString('utf8'), ...formStates] : formStates;
+    return { markup: markup + html.slice(last), texts, problems };
+}

package/dist/cli/asset-refs.d.ts

@@ -0,0 +1,52 @@
+/**
+ * CLI commands: `canopycms collect-asset-refs <outDir>` and
+ * `canopycms materialize-assets --refs <file>`, the two release steps that make every image URL a
+ * build references exist in the store before that build is served. The logic lives in
+ * `build/asset-refs.ts` and `assets/materialize.ts`; this file picks the store and reports.
+ */
+import type { AssetStore } from '../assets/types.js';
+/** @returns the process exit code. */
+export declare function collectAssetRefsCLI(options: {
+    outDir: string | undefined;
+}): Promise<number>;
+/**
+ * @internal Exported for tests. `materialize-assets` exit codes: a release gates on them, so `1`
+ * means the run did not finish, `2` that it finished and some keys cannot be produced, and `3`
+ * that the store refused or kept failing, so nothing is known about the keys it could not check.
+ */
+export declare const MATERIALIZE_EXIT_CODES: {
+    readonly ok: 0;
+    /** Bad flags, an unreadable refs file, a config or sharp failure, any uncaught throw. */
+    readonly error: 1;
+    /** Content failures only; `--allow-failures` turns this into `ok`. */
+    readonly contentFailures: 2;
+    /** Any store failure, whatever `--allow-failures` says. */
+    readonly storeFailures: 3;
+};
+export interface MaterializeAssetsCLIOptions {
+    /** The project root; read only when the store comes from `canopycms.config.ts`. */
+    projectDir?: string;
+    refsPath: string | undefined;
+    reportPath?: string;
+    allowFailures: boolean;
+    concurrency?: string;
+    transformConcurrency?: string;
+    /** With `region`, builds the S3 store directly and never loads the site config. */
+    bucket?: string;
+    region?: string;
+    /** Accept a non-S3 store resolved from the config. */
+    allowLocal?: boolean;
+    /** `MaterializeOptions.outputPrefix`; an invalid one exits 1 before anything is read. */
+    outputPrefix?: string;
+    /** @internal Test seam: skips resolving the store. */
+    store?: AssetStore;
+}
+/**
+ * `--allow-failures` tolerates content failures only — a reference to an asset the store can no
+ * longer produce — so one deleted image does not block a release. A store failure (access
+ * denied, an outage that outlasted the retries) still fails the run, since it says nothing about
+ * which images are actually fine.
+ *
+ * @returns a `MATERIALIZE_EXIT_CODES` value; a throw is caught and reported as `error`.
+ */
+export declare function materializeAssetsCLI(options: MaterializeAssetsCLIOptions): Promise<number>;