canopycms 0.12.0 → 0.13.0

package/dist/cli/backfill-item-keys.js

@@ -0,0 +1,320 @@
+/**
+ * CLI command: npx canopycms backfill-item-keys [--check] [--force] [--app-dir <dir>]
+ *
+ * Writes the `_key` of every object-list item and block (`item-keys.ts`) into the site's content,
+ * so an editor's first save of an entry adds no key lines and a merge sees only real edits.
+ *
+ * Each entry goes through the editor's own save path, with nothing re-derived:
+ * `ContentStore.read` keys the items exactly as the editor's form holds them, and
+ * `ContentStore.write` writes that data back with the read's version, through the serialiser,
+ * content-write lock and version check a save uses, after the content API's own
+ * `entryDataAsSaved`. So a written file is byte for byte what an editor save of the unedited entry
+ * leaves, a second run finds every item keyed and writes nothing, and the next editor save adds no
+ * lines of its own, a CRLF markdown file aside (safeinsights/canopycms#1234).
+ *
+ * A file is written only when its items lack keys. The write then also carries whatever else an
+ * unedited save normalises (a one-line JSON file comes out pretty-printed, a stray frontmatter key
+ * named as the body goes), and those files are listed apart for review. Out of scope, as for the editor: list fields the schema does not
+ * describe, `.collection.json`, and entries of a type this code's registry lacks.
+ */
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { diffArrays } from 'diff';
+import { simpleGit } from 'simple-git';
+import { flattenSchema } from '../config/index.js';
+import { contentIndexGenerationPath } from '../content-index-generation.js';
+import { listCollectionEntries, parseEntryParts, } from '../content-listing.js';
+import { entryDataAsSaved } from '../api/content.js';
+import { ContentStore } from '../content-store.js';
+import { ITEM_KEY, withItemKeys } from '../item-keys.js';
+import { isValidSchema, resolveSchema } from '../schema/index.js';
+import { serializeEntry } from '../utils/content-serialize.js';
+import { getErrorMessage, isNodeError } from '../utils/error.js';
+import { filePathExists } from '../utils/fs.js';
+import { CANOPY_META_DIR } from '../utils/git.js';
+import { isPlainRecord } from '../validation/field-traversal.js';
+import { importSiteEntrySchemaRegistry, importSiteServerConfig } from './site-config.js';
+/**
+ * Keys every entry's items under `root`, or with `check`, reports which files a run would write.
+ * @internal Exported for tests.
+ */
+export async function backfillItemKeys(options) {
+    const { root, flatSchema, contentRootName, check } = options;
+    const store = new ContentStore(root, flatSchema, { contentRootName });
+    const result = { changed: [], failed: [], unavailable: [], malformed: [] };
+    const collections = flatSchema.filter((item) => item.type === 'collection' && !!item.entries);
+    for (const collection of collections) {
+        const entryTypes = collection.entries ?? [];
+        let listed;
+        try {
+            listed = await listCollectionEntries(root, collection, (file) => result.malformed.push(path.posix.join(file.collectionPath, file.filename)));
+        }
+        catch (err) {
+            result.failed.push({ path: `${collection.logicalPath}/`, error: getErrorMessage(err) });
+            continue;
+        }
+        for (const item of listed) {
+            const entryType = entryTypes.find((type) => type.name === item.entryType);
+            if (!entryType || entryType.unavailable) {
+                result.unavailable.push(item.physicalPath);
+                continue;
+            }
+            try {
+                const changed = await backfillEntry(store, root, item, entryType.schema, check);
+                if (changed)
+                    result.changed.push({ path: item.physicalPath, ...changed });
+            }
+            catch (err) {
+                result.failed.push({ path: item.physicalPath, error: getErrorMessage(err) });
+            }
+        }
+    }
+    const byPath = (a, b) => a.path.localeCompare(b.path);
+    result.changed.sort(byPath);
+    result.failed.sort(byPath);
+    result.unavailable.sort();
+    result.malformed.sort();
+    return result;
+}
+/** The entry's counts when its items lack keys, after writing it unless `check`; else null. */
+async function backfillEntry(store, root, item, fields, check) {
+    const doc = await store.read(item.collectionPath, item.slug, { resolveReferences: false });
+    // The store finds a slug's file by name alone, so a slug another file in the collection also
+    // carries (another entry type's, or a stray copy) addresses only one of them.
+    if (doc.absolutePath !== path.resolve(root, item.physicalPath)) {
+        throw new Error(`Another file in its collection has the slug "${item.slug}", so neither a save nor the ` +
+            'backfill can address this one. Rename or remove one of them.');
+    }
+    // After the read: its version was taken before its own read of the file, so a change landing
+    // between the two fails the write's version check rather than writing over it.
+    const before = await fs.readFile(doc.absolutePath, 'utf8');
+    const asWritten = parseEntryParts(before, doc.format).data;
+    const keyed = withItemKeys(fields, asWritten, 'content');
+    if (keyed === asWritten)
+        return null;
+    const entry = savedEntry(doc, fields);
+    let after;
+    if (check) {
+        after = serializeEntry(entry, before, fields);
+    }
+    else {
+        if (doc.version === undefined)
+            throw new Error('The read carried no version to write against.');
+        await store.write(item.collectionPath, item.slug, { ...entry, expectedVersion: doc.version }, item.entryType);
+        after = await fs.readFile(doc.absolutePath, 'utf8');
+    }
+    return {
+        itemsKeyed: countAddedKeys(keyed, asWritten),
+        otherLines: countOtherLines(before, after),
+    };
+}
+/** What a save of the entry, opened and saved unedited, hands the serialiser. */
+function savedEntry(doc, fields) {
+    const data = entryDataAsSaved(fields, doc.format, doc.data);
+    if (doc.format === 'json')
+        return { format: 'json', data };
+    if (doc.format === 'yaml')
+        return { format: 'yaml', data };
+    return { format: doc.format, data, body: doc.body };
+}
+/**
+ * The `_key`s `keyed` holds that `asWritten` lacks or holds differently. `withItemKeys` returns
+ * every subtree it left alone as the same object, so the walk skips those.
+ */
+function countAddedKeys(keyed, asWritten) {
+    if (keyed === asWritten)
+        return 0;
+    if (Array.isArray(keyed)) {
+        const was = Array.isArray(asWritten) ? asWritten : [];
+        return keyed.reduce((n, value, i) => n + countAddedKeys(value, was[i]), 0);
+    }
+    if (!isPlainRecord(keyed))
+        return 0;
+    const was = isPlainRecord(asWritten) ? asWritten : {};
+    let count = ITEM_KEY in keyed && keyed[ITEM_KEY] !== was[ITEM_KEY] ? 1 : 0;
+    for (const [name, value] of Object.entries(keyed)) {
+        if (name !== ITEM_KEY)
+            count += countAddedKeys(value, was[name]);
+    }
+    return count;
+}
+/** A `_key` line as YAML (frontmatter included) or JSON spells it, its trailing comma trimmed. */
+function isKeyLine(line) {
+    const text = line.trim().replace(/^-\s/, '').trimStart();
+    for (const name of ['_key:', '"_key":']) {
+        if (text.startsWith(name))
+            return /^\s*\S+$/.test(text.slice(name.length));
+    }
+    return false;
+}
+/**
+ * Lines changed between `before` and `after` other than added `_key` lines, a changed line ending
+ * included. A trailing comma is ignored, since a JSON key appended to an object adds one to the
+ * line above it.
+ */
+function countOtherLines(before, after) {
+    const lines = (text) => text.split('\n').map((line) => line.replace(/,(\r?)$/, '$1'));
+    let other = 0;
+    for (const change of diffArrays(lines(before), lines(after))) {
+        if (change.removed)
+            other += change.value.length;
+        else if (change.added)
+            other += change.value.filter((line) => !isKeyLine(line)).length;
+    }
+    return other;
+}
+/**
+ * Exit codes. `--check`: 0 when no file needs keys, 1 when some would change. A run: 0 when done,
+ * 1 when it would not write: uncommitted content changes, or no git to ask. Both: 2 when the
+ * site's config or schema could not be loaded, a collection listed, or an entry read or written.
+ * @internal Exported for tests.
+ */
+export const BACKFILL_EXIT_CODES = { ok: 0, changesOrRefused: 1, loadError: 2 };
+/** Runs the command and returns its exit code. */
+export async function backfillItemKeysCLI(options) {
+    const { projectDir, appDir = 'app', check = false, force = false } = options;
+    console.log('\nCanopyCMS backfill-item-keys\n');
+    let contentRootName;
+    let registry;
+    try {
+        const { server } = await importSiteServerConfig(projectDir);
+        const contentRoot = isPlainRecord(server) ? (server.contentRoot ?? 'content') : null;
+        if (typeof contentRoot !== 'string') {
+            throw new Error('canopycms.config.ts does not export a config made by defineCanopyConfig().');
+        }
+        contentRootName = contentRoot;
+        registry = (await importSiteEntrySchemaRegistry(projectDir, appDir)).registry;
+    }
+    catch (err) {
+        console.error(`Could not load the site: ${getErrorMessage(err)}`);
+        return BACKFILL_EXIT_CODES.loadError;
+    }
+    let flatSchema;
+    try {
+        const contentDir = path.join(projectDir, contentRootName);
+        const resolved = await resolveSchema(contentDir, registry, {
+            unknownSchema: 'degrade',
+        });
+        if (!isValidSchema(resolved.schema)) {
+            throw new Error(`No .collection.json files describe ${contentRootName}/.`);
+        }
+        for (const issue of resolved.issues)
+            console.warn(`  Warning: ${issue.message}`);
+        flatSchema = flattenSchema(resolved.schema, contentRootName);
+    }
+    catch (err) {
+        console.error(`Could not load the schema: ${getErrorMessage(err)}`);
+        return BACKFILL_EXIT_CODES.loadError;
+    }
+    if (!check && !force) {
+        const refusal = await uncommittedContent(projectDir, contentRootName);
+        if (refusal) {
+            console.error(refusal);
+            return BACKFILL_EXIT_CODES.changesOrRefused;
+        }
+    }
+    const metaDir = path.join(projectDir, CANOPY_META_DIR);
+    const metaDirExisted = await filePathExists(metaDir);
+    let result;
+    try {
+        result = await backfillItemKeys({ root: projectDir, flatSchema, contentRootName, check });
+    }
+    catch (err) {
+        console.error(`Could not finish: ${getErrorMessage(err)}` +
+            (check ? '' : '\nFiles keyed before the error stay written; finish with --force.'));
+        return BACKFILL_EXIT_CODES.loadError;
+    }
+    finally {
+        // The marker tells other processes to rebuild their ID index, and the backfill moves no file,
+        // so no index goes stale. An adopter's .gitignore need not cover `.canopy-meta/`, so at a
+        // project root that had none, remove what this run created there.
+        if (!check && !metaDirExisted)
+            await removeOwnMetaDir(projectDir, metaDir);
+    }
+    printResult(result, check);
+    if (result.failed.length > 0)
+        return BACKFILL_EXIT_CODES.loadError;
+    if (check && result.changed.length > 0)
+        return BACKFILL_EXIT_CODES.changesOrRefused;
+    return BACKFILL_EXIT_CODES.ok;
+}
+/** Why a run must not write: the content tree has uncommitted changes, or no git to ask. */
+async function uncommittedContent(projectDir, contentRootName) {
+    let status;
+    try {
+        status = await simpleGit({ baseDir: projectDir }).raw([
+            'status',
+            '--porcelain',
+            '--untracked-files=all',
+            '--',
+            contentRootName,
+        ]);
+    }
+    catch (err) {
+        return (`Could not ask git whether ${contentRootName}/ has uncommitted changes ` +
+            `(${getErrorMessage(err)}). Run inside the site's git checkout, or pass --force.`);
+    }
+    const dirty = status.split('\n').filter((line) => line.trim() !== '');
+    if (dirty.length === 0)
+        return null;
+    const shown = dirty.slice(0, 10).map((line) => `    ${line}`);
+    if (dirty.length > shown.length)
+        shown.push(`    … and ${dirty.length - shown.length} more`);
+    return (`${contentRootName}/ has uncommitted changes, so the backfill's own changes could not be ` +
+        `reviewed apart from them:\n${shown.join('\n')}\n` +
+        'Commit or stash them first, or pass --force.');
+}
+async function removeOwnMetaDir(projectDir, metaDir) {
+    try {
+        await fs.rm(contentIndexGenerationPath(projectDir), { force: true });
+        await fs.rmdir(metaDir);
+    }
+    catch (err) {
+        if (isNodeError(err) && err.code === 'ENOENT')
+            return;
+        console.warn(`  Left ${CANOPY_META_DIR}/ in place: ${getErrorMessage(err)}`);
+    }
+}
+function printResult(result, check) {
+    const plural = (n, word, words = `${word}s`) => `${n} ${n === 1 ? word : words}`;
+    const keysOnly = result.changed.filter((file) => file.otherLines === 0);
+    const normalised = result.changed.filter((file) => file.otherLines > 0);
+    const verb = check ? 'Would key' : 'Keyed';
+    if (keysOnly.length > 0) {
+        console.log(`  ${verb} items in ${plural(keysOnly.length, 'file')}, adding only _key lines:`);
+        for (const file of keysOnly)
+            console.log(`    ${file.path}  ${plural(file.itemsKeyed, 'item')}`);
+    }
+    if (normalised.length > 0) {
+        console.log(`\n  ${verb} items in ${plural(normalised.length, 'file')} that an unedited save also ` +
+            'normalises in other lines. Review these before committing:');
+        for (const file of normalised) {
+            console.log(`    ${file.path}  ${plural(file.itemsKeyed, 'item')}, ` +
+                `${plural(file.otherLines, 'other line')}`);
+        }
+    }
+    if (result.unavailable.length > 0) {
+        console.warn(`\n  Skipped ${plural(result.unavailable.length, 'entry', 'entries')} whose entry type the schema ` +
+            'registry does not have.');
+    }
+    if (result.malformed.length > 0) {
+        console.warn(`\n  Skipped ${plural(result.malformed.length, 'file')} that look like entries but whose ` +
+            `names don't parse:\n${result.malformed.map((file) => `    ${file}`).join('\n')}`);
+    }
+    if (result.failed.length > 0) {
+        console.error(`\n  Could not ${check ? 'check' : 'key'}:`);
+        for (const file of result.failed)
+            console.error(`    ${file.path}: ${file.error}`);
+    }
+    const skippedAny = result.failed.length + result.unavailable.length + result.malformed.length > 0;
+    if (result.changed.length === 0) {
+        console.log(`  Every ${skippedAny ? 'other ' : ''}entry's items already have their keys.`);
+    }
+    else if (check) {
+        console.log(`\n  Run \`canopycms backfill-item-keys\` to write them.`);
+    }
+    else {
+        console.log('\n  Commit these changes on their own, so the commit holds nothing else.');
+    }
+    console.log('');
+}

package/dist/cli/cli.d.ts

@@ -4,7 +4,7 @@
  *
  * Routes commands to their implementations:
  *   init, init-deploy, init-github-app, worker, generate-ai-content, collect-asset-refs,
- *   materialize-assets, sync, migrate
+ *   materialize-assets, sync, migrate, backfill-item-keys
  *
  * Command implementations live in separate files (init.ts, sync.ts, etc.)
  * and are dynamically imported to keep startup fast.