canopycms 0.0.67 → 0.0.68-int.100

package/dist/build/asset-refs.js

@@ -0,0 +1,251 @@
+/**
+ * Collect the asset URLs a static build references, so a release can make them exist first.
+ *
+ * The public `/assets/t/` path serves only what is stored, and widths are chosen by site code at
+ * render time, so build output is the one place every final URL appears. The adopter contract
+ * that follows: every `/assets/t/` URL the site can request must appear as text in its build
+ * output (ARCHITECTURE.md, "Why transform by URL directive and materialize at release?").
+ */
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { z } from 'zod';
+import { ASSET_PREFIXES } from '../assets/asset-prefixes.js';
+import { canonicalizeTransformPath } from '../assets/transform-directives.js';
+import { atomicWriteFile } from '../utils/atomic-write.js';
+/** @internal Written into the scanned directory, so a manifest the adopter builds afterwards covers it. */
+export const ASSET_REFS_FILENAME = 'canopy-asset-refs.json';
+/**
+ * Text outputs a URL can hide in: pages, RSC payloads, data, scripts, styles, feeds, generated
+ * markdown. A file with no extension (a route handler's output) is scanned unless it is binary.
+ */
+const SCANNED_EXTENSIONS = new Set([
+    '',
+    '.html',
+    '.htm',
+    '.txt',
+    '.rsc',
+    '.json',
+    '.js',
+    '.mjs',
+    '.cjs',
+    '.css',
+    '.xml',
+    '.svg',
+    '.md',
+    '.webmanifest',
+]);
+/**
+ * Anchored on the path, never on an origin or a mount prefix, so whatever precedes `/assets/` —
+ * an absolute origin, a basePath, the editor's authenticated prefix — is dropped. The directive
+ * and filename classes stop at anything that ends a URL in HTML, JSON, JS or CSS (quotes,
+ * whitespace, a backslash escape, parens, a query; the filename also at a srcset comma) or at any
+ * non-ASCII character (typographic quotes and dashes in prose; no stored key has one), and exclude
+ * `{`, `}` and `$`, so a URL template in a script never reads as a URL. A non-ASCII character
+ * inside a directive segment therefore ends the match and the URL is not seen. The hash class
+ * admits upper case only so that such a URL is reported rather than skipped.
+ */
+const URL_STOP = String.raw `\s"'\x60<>()\\/?#&{}$\u0080-\uffff`;
+// eslint-disable-next-line security/detect-non-literal-regexp -- built from constants above
+const TRANSFORM_URL_RE = new RegExp(String.raw `/${ASSET_PREFIXES.transform}/([^${URL_STOP}]+)/([a-fA-F0-9]{32})/([^${URL_STOP},]+)`, 'g');
+// eslint-disable-next-line security/detect-non-literal-regexp -- built from constants above
+const STATIC_URL_RE = new RegExp(String.raw `/${ASSET_PREFIXES.public}/([a-fA-F0-9]{32})/([^${URL_STOP},]+)`, 'g');
+const STATIC_FILENAME_RE = /^[a-z0-9-]+\.[a-z0-9]{1,10}$/;
+const HASH32_RE = /^[a-f0-9]{32}$/;
+const assetRefEntrySchema = z
+    .object({
+    key: z.string().min(1),
+    routes: z.array(z.string()),
+    files: z.array(z.string()),
+})
+    .strict();
+const assetRefsFileSchema = z
+    .object({
+    version: z.literal(1),
+    transforms: z.array(assetRefEntrySchema),
+    statics: z.array(assetRefEntrySchema),
+})
+    .strict();
+/** The output references URLs that cannot be served from the store; nothing was written. */
+export class AssetRefsError extends Error {
+    constructor(problems) {
+        super(`${problems.length} asset URL(s) in the build output cannot be served from the store:\n` +
+            problems.map((p) => `  ${p.file}: ${p.url}\n    ${p.error}`).join('\n'));
+        this.problems = problems;
+        this.name = 'AssetRefsError';
+    }
+}
+/** Output-relative POSIX paths of every file under `root` with a scanned extension, sorted. */
+async function listScannedFiles(root) {
+    const entries = await fs.readdir(root, { recursive: true, withFileTypes: true });
+    return entries
+        .filter((entry) => entry.isFile() && SCANNED_EXTENSIONS.has(path.extname(entry.name)))
+        .map((entry) => path.relative(root, path.join(entry.parentPath, entry.name)))
+        .map((rel) => rel.split(path.sep).join('/'))
+        .filter((rel) => rel !== ASSET_REFS_FILENAME)
+        .sort();
+}
+/** The route a static-export HTML file serves (`about/index.html` and `about.html` are `/about`). */
+function routeForFile(file) {
+    if (!/\.html?$/.test(file))
+        return undefined;
+    const withoutExt = file.replace(/\.html?$/, '');
+    const route = withoutExt === 'index' ? '' : withoutExt.replace(/\/index$/, '');
+    return `/${route}`;
+}
+const ENCODED_ASSETS = /%2[fF]assets%2[fF]/g;
+const RUN_STOP = /[\s"'`<>&]/;
+/** Each `%XX` escape of an ASCII character, decoded on its own; anything else stays as written. */
+function decodeAsciiEscapes(text) {
+    return text.replace(/%([0-7][0-9a-fA-F])/g, (_, hex) => String.fromCharCode(parseInt(hex, 16)));
+}
+/** One malformed or non-UTF-8 escape elsewhere in the run must not hide the URL inside it. */
+function decodeOrKeep(run) {
+    try {
+        return decodeURIComponent(run);
+    }
+    catch {
+        return decodeAsciiEscapes(run);
+    }
+}
+/**
+ * Decode, whole, each percent-encoded run holding an encoded `/assets/` (an image optimizer's
+ * `?url=`), so an encoded `%20` or `%3F` after the URL ends it just as a literal one would. Runs are
+ * found by expanding outward from each match, so the scan stays linear on long text with no stop
+ * character.
+ */
+function decodeEncodedAssetRuns(text) {
+    let out = '';
+    let last = 0;
+    for (const match of text.matchAll(ENCODED_ASSETS)) {
+        if (match.index < last)
+            continue;
+        let start = match.index;
+        while (start > last && !RUN_STOP.test(text[start - 1]))
+            start--;
+        let end = match.index + match[0].length;
+        while (end < text.length && !RUN_STOP.test(text[end]))
+            end++;
+        out += text.slice(last, start) + decodeOrKeep(text.slice(start, end));
+        last = end;
+    }
+    return out + text.slice(last);
+}
+/** JSON and JS may escape `/` as a backslash-slash or a `u002F` unicode escape. */
+function decodeUrlEscapes(text) {
+    return decodeEncodedAssetRuns(text.replace(/\\u002[fF]/g, '/').replace(/\\\//g, '/'));
+}
+/**
+ * A slug and ext end in `[a-z0-9]`, so punctuation after a URL in prose is not part of it. Trimmed
+ * wherever it occurs: an RSC payload carries prose as JSON strings, where a sentence-final URL is
+ * followed by a quote exactly as an attribute value is.
+ */
+function trimTrailingPunctuation(url, filename) {
+    const trimmed = filename.replace(/[.:;!?]+$/, '');
+    return [url.slice(0, url.length - (filename.length - trimmed.length)), trimmed];
+}
+/** A NUL in the first 8 KiB marks a file binary, close to git's own test; only that block is read. */
+async function startsBinary(filePath) {
+    const handle = await fs.open(filePath, 'r');
+    try {
+        const block = Buffer.alloc(8192);
+        const { bytesRead } = await handle.read(block, 0, block.length, 0);
+        return block.subarray(0, bytesRead).includes(0);
+    }
+    finally {
+        await handle.close();
+    }
+}
+class RefCollector {
+    constructor() {
+        this.entries = new Map();
+    }
+    add(key, file) {
+        const entry = this.entries.get(key) ?? { routes: new Set(), files: new Set() };
+        entry.files.add(file);
+        const route = routeForFile(file);
+        if (route !== undefined)
+            entry.routes.add(route);
+        this.entries.set(key, entry);
+    }
+    toEntries() {
+        return [...this.entries.keys()].sort().map((key) => {
+            const entry = this.entries.get(key);
+            return {
+                key,
+                routes: [...(entry?.routes ?? [])].sort(),
+                files: [...(entry?.files ?? [])].sort(),
+            };
+        });
+    }
+}
+/**
+ * Scan `outDir` for `/assets/t/...` transform URLs and `/assets/{hash32}/...` static URLs and
+ * write their stored keys to `outDir/canopy-asset-refs.json`, sorted so the same output always
+ * yields the same file. Throws `AssetRefsError`, writing nothing, if any transform URL is not in
+ * canonical form or any static URL is malformed: only canonical keys are stored, and the public
+ * path is to compute nothing on a miss.
+ */
+export async function collectAssetRefs(outDir) {
+    const root = path.resolve(outDir);
+    const files = await listScannedFiles(root);
+    const transforms = new RefCollector();
+    const statics = new RefCollector();
+    const problems = [];
+    let scannedFiles = 0;
+    for (const file of files) {
+        const filePath = path.join(root, file);
+        if (path.extname(file) === '' && (await startsBinary(filePath)))
+            continue;
+        scannedFiles++;
+        const text = decodeUrlEscapes(await fs.readFile(filePath, 'utf-8'));
+        for (const match of text.matchAll(TRANSFORM_URL_RE)) {
+            const [url, filename] = trimTrailingPunctuation(match[0], match[3]);
+            const [, encodedDirectives, hash32] = match;
+            // A loader that encodes the directive segment alone is still requesting this key.
+            const directives = decodeAsciiEscapes(encodedDirectives);
+            const canonical = canonicalizeTransformPath([directives, hash32, filename], 'any');
+            if (!canonical.ok) {
+                problems.push({ file, url, error: canonical.error });
+            }
+            else if (!canonical.isCanonical) {
+                problems.push({
+                    file,
+                    url,
+                    error: `Not canonical; assetUrl would write /${ASSET_PREFIXES.transform}/${canonical.canonicalPath}`,
+                });
+            }
+            else {
+                transforms.add(`${ASSET_PREFIXES.transform}/${canonical.canonicalPath}`, file);
+            }
+        }
+        for (const match of text.matchAll(STATIC_URL_RE)) {
+            const [url, filename] = trimTrailingPunctuation(match[0], match[2]);
+            const hash32 = match[1];
+            if (!HASH32_RE.test(hash32) || !STATIC_FILENAME_RE.test(filename)) {
+                problems.push({ file, url, error: 'Not a stored static asset path' });
+            }
+            else {
+                statics.add(`${ASSET_PREFIXES.public}/${hash32}/${filename}`, file);
+            }
+        }
+    }
+    if (problems.length > 0)
+        throw new AssetRefsError(problems);
+    const refs = {
+        version: 1,
+        transforms: transforms.toEntries(),
+        statics: statics.toEntries(),
+    };
+    const filePath = path.join(root, ASSET_REFS_FILENAME);
+    await atomicWriteFile(filePath, `${JSON.stringify(refs, null, 2)}\n`);
+    return { refs, filePath, scannedFiles };
+}
+/** Read and validate a refs file `collectAssetRefs` wrote. */
+export async function readAssetRefsFile(filePath) {
+    const raw = JSON.parse(await fs.readFile(filePath, 'utf-8'));
+    const parsed = assetRefsFileSchema.safeParse(raw);
+    if (!parsed.success) {
+        throw new Error(`${filePath} is not a valid asset refs file: ${parsed.error.message}`);
+    }
+    return parsed.data;
+}

package/dist/build-identity.d.ts

@@ -0,0 +1,3 @@
+import type { BuildIdentity } from './types.js';
+export type { BuildIdentity };
+export declare function getBuildIdentity(env?: NodeJS.ProcessEnv): BuildIdentity;

package/dist/build-identity.js

@@ -0,0 +1,13 @@
+import { CANOPYCMS_VERSION } from './version.js';
+/**
+ * Set by the image build (Dockerfile.cms `ARG`), not by the infrastructure: a Lambda env var
+ * would name the infrastructure's commit and hide an image/infrastructure skew.
+ */
+const SOURCE_REVISION_ENV = 'CANOPY_SOURCE_SHA';
+export function getBuildIdentity(env = process.env) {
+    // The Dockerfile sets the variable to the empty string when the build arg is not passed.
+    const sourceRevision = env[SOURCE_REVISION_ENV]?.trim();
+    return sourceRevision
+        ? { canopycmsVersion: CANOPYCMS_VERSION, sourceRevision }
+        : { canopycmsVersion: CANOPYCMS_VERSION };
+}

package/dist/cli/asset-refs.d.ts

@@ -0,0 +1,30 @@
+/**
+ * CLI commands: `canopycms collect-asset-refs <outDir>` and
+ * `canopycms materialize-assets --refs <file>`, the two release steps that make every image URL a
+ * build references exist in the store before that build is served. The logic lives in
+ * `build/asset-refs.ts` and `assets/materialize.ts`; this file loads config and reports.
+ */
+import type { AssetStore } from '../assets/types.js';
+/** @returns the process exit code. */
+export declare function collectAssetRefsCLI(options: {
+    outDir: string | undefined;
+}): Promise<number>;
+export interface MaterializeAssetsCLIOptions {
+    projectDir: string;
+    refsPath: string | undefined;
+    reportPath?: string;
+    allowFailures: boolean;
+    concurrency?: string;
+    transformConcurrency?: string;
+    /** @internal Test seam: skips loading the project's config. */
+    store?: AssetStore;
+}
+/**
+ * Exits non-zero on any failure. `--allow-failures` tolerates content failures only — a
+ * reference to an asset the store can no longer produce — so one deleted image does not block a
+ * release. A store failure (access denied, an outage that outlasted the retries) still fails the
+ * run, since it says nothing about which images are actually fine.
+ *
+ * @returns the process exit code.
+ */
+export declare function materializeAssetsCLI(options: MaterializeAssetsCLIOptions): Promise<number>;

package/dist/cli/asset-refs.js

@@ -0,0 +1,168 @@
+/**
+ * CLI commands: `canopycms collect-asset-refs <outDir>` and
+ * `canopycms materialize-assets --refs <file>`, the two release steps that make every image URL a
+ * build references exist in the store before that build is served. The logic lives in
+ * `build/asset-refs.ts` and `assets/materialize.ts`; this file loads config and reports.
+ */
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { createJiti } from 'jiti';
+import { z } from 'zod';
+import { createAssetStore } from '../assets/factory.js';
+import { materializeAssets, SharpUnavailableError, } from '../assets/materialize.js';
+import { AssetRefsError, collectAssetRefs, readAssetRefsFile } from '../build/asset-refs.js';
+import { mediaSchema } from '../config/schemas/media.js';
+import { operatingStrategy } from '../operating-mode/index.js';
+import { atomicWriteFile } from '../utils/atomic-write.js';
+import { isNotFoundError } from '../utils/error.js';
+/** @returns the process exit code. */
+export async function collectAssetRefsCLI(options) {
+    if (!options.outDir) {
+        console.error('Usage: canopycms collect-asset-refs <outDir>');
+        return 1;
+    }
+    const outDir = path.resolve(options.outDir);
+    try {
+        if (!(await fs.stat(outDir)).isDirectory()) {
+            console.error(`Not a directory: ${outDir}`);
+            return 1;
+        }
+    }
+    catch (err) {
+        if (!isNotFoundError(err))
+            throw err;
+        console.error(`No build output at ${outDir}. Run this after the static build.`);
+        return 1;
+    }
+    try {
+        const { refs, filePath, scannedFiles } = await collectAssetRefs(outDir);
+        console.log(`canopycms collect-asset-refs: ${refs.transforms.length} transform and ` +
+            `${refs.statics.length} static asset reference(s) in ${scannedFiles} file(s), ` +
+            `written to ${filePath}`);
+        return 0;
+    }
+    catch (err) {
+        if (!(err instanceof AssetRefsError))
+            throw err;
+        console.error(`canopycms collect-asset-refs: ${err.message}`);
+        console.error('Build these URLs with assetUrl/assetSrcSet, which always write the canonical form; ' +
+            'a hand-written or edited URL has no stored object behind it.');
+        return 1;
+    }
+}
+const serverConfigSchema = z.object({
+    mode: z.enum(['prod', 'dev']),
+    media: mediaSchema.optional(),
+});
+/**
+ * The store the site's `media` config names, resolved as the running CMS resolves it when its
+ * working directory is the project: dev mode with no `media` falls back to the dev workspace's
+ * local store, prod never does.
+ */
+async function loadConfiguredAssetStore(projectDir) {
+    const configPath = path.join(projectDir, 'canopycms.config.ts');
+    const jiti = createJiti(import.meta.url);
+    const mod = (await jiti.import(configPath));
+    const configExport = mod.default ?? mod.config ?? mod;
+    const server = typeof configExport === 'object' && configExport !== null && 'server' in configExport
+        ? configExport.server
+        : configExport;
+    const parsed = serverConfigSchema.safeParse(server);
+    if (!parsed.success) {
+        throw new Error(`Invalid CanopyCMS config at ${configPath}: ${parsed.error.message}`);
+    }
+    const { mode, media } = parsed.data;
+    const devAssetsDir = mode === 'dev'
+        ? path.join(operatingStrategy(mode).getWorkspaceRoot(projectDir), 'assets')
+        : undefined;
+    // A relative `media.directory` is relative to the project, wherever the command was run from.
+    const resolvedMedia = media?.adapter === 'local' && media.directory
+        ? { ...media, directory: path.resolve(projectDir, media.directory) }
+        : media;
+    const store = createAssetStore(resolvedMedia, { devAssetsDir });
+    if (!store) {
+        throw new Error(`No asset store is configured in ${configPath}: set \`media\` (an s3 adapter in prod).`);
+    }
+    return store;
+}
+function describeReferences(result) {
+    if (result.routes.length > 0)
+        return `pages: ${result.routes.join(', ')}`;
+    return `files: ${result.files.join(', ')}`;
+}
+function printReport(report) {
+    const { summary } = report;
+    console.log(`canopycms materialize-assets: ${summary.total} key(s): ${summary.existed} existed, ` +
+        `${summary.created} created, ${summary.failed} failed`);
+    for (const result of report.results) {
+        if (result.status !== 'failed')
+            continue;
+        console.error(`  FAILED (${result.failure}) /${result.key}`);
+        console.error(`    ${result.error}`);
+        console.error(`    ${describeReferences(result)}`);
+    }
+}
+/** `undefined` when the flag is absent; `null`, after printing why, when it is not a positive integer. */
+function parsePositiveInteger(flag, value) {
+    if (value === undefined)
+        return undefined;
+    const parsed = Number(value);
+    if (Number.isInteger(parsed) && parsed >= 1)
+        return parsed;
+    console.error(`${flag} must be a positive integer, got "${value}"`);
+    return null;
+}
+/**
+ * Exits non-zero on any failure. `--allow-failures` tolerates content failures only — a
+ * reference to an asset the store can no longer produce — so one deleted image does not block a
+ * release. A store failure (access denied, an outage that outlasted the retries) still fails the
+ * run, since it says nothing about which images are actually fine.
+ *
+ * @returns the process exit code.
+ */
+export async function materializeAssetsCLI(options) {
+    if (!options.refsPath) {
+        console.error('Usage: canopycms materialize-assets --refs <file> [--report <file>]');
+        return 1;
+    }
+    const concurrency = parsePositiveInteger('--concurrency', options.concurrency);
+    const transformConcurrency = parsePositiveInteger('--transform-concurrency', options.transformConcurrency);
+    if (concurrency === null || transformConcurrency === null)
+        return 1;
+    const refs = await readAssetRefsFile(path.resolve(options.refsPath));
+    const store = options.store ?? (await loadConfiguredAssetStore(options.projectDir));
+    let report;
+    try {
+        report = await materializeAssets({
+            store,
+            targets: refs.transforms,
+            statics: refs.statics,
+            concurrency,
+            transformConcurrency,
+        });
+    }
+    catch (err) {
+        if (!(err instanceof SharpUnavailableError))
+            throw err;
+        console.error(`canopycms materialize-assets: ${err.message}`);
+        return 1;
+    }
+    if (options.reportPath) {
+        await atomicWriteFile(path.resolve(options.reportPath), `${JSON.stringify(report, null, 2)}\n`);
+    }
+    printReport(report);
+    const { contentFailures, storeFailures } = report.summary;
+    if (storeFailures > 0) {
+        if (options.allowFailures) {
+            console.error('--allow-failures does not cover store failures; fix access or rerun.');
+        }
+        return 1;
+    }
+    if (contentFailures === 0)
+        return 0;
+    if (!options.allowFailures)
+        return 1;
+    console.warn(`\nWARNING: --allow-failures is set. ${contentFailures} image URL(s) above will be MISSING ` +
+        `from the released site. Fix the pages listed and rerun.\n`);
+    return 0;
+}

package/dist/cli/cli.d.ts

@@ -3,7 +3,8 @@
  * CanopyCMS CLI entrypoint.
  *
  * Routes commands to their implementations:
- *   init, init-deploy, init-github-app, worker, generate-ai-content, sync, migrate
+ *   init, init-deploy, init-github-app, worker, generate-ai-content, collect-asset-refs,
+ *   materialize-assets, sync, migrate
  *
  * Command implementations live in separate files (init.ts, sync.ts, etc.)
  * and are dynamically imported to keep startup fast.