canopycms 0.0.68-int.98 → 0.1.0

package/dist/ai/generate.js

@@ -12,26 +12,40 @@ import { isDataOnlyFormat } from '../utils/format.js';
 import { extractEntryTypeFromFilename, extractIdFromFilename, } from '../content-id-index.js';
 import { hasTraversalSequence } from '../paths/index.js';
 import { getErrorMessage, isNodeError } from '../utils/error.js';
-import { entryToMarkdown } from './json-to-markdown.js';
+import { buildResolvedReference } from '../entry-schema.js';
+import { entryToMarkdown, frontmatterTitle } from './json-to-markdown.js';
+import { createReferenceRendering, createReferenceTargetResolver, decideReferenceLinks, maskUnexportedTargets, resolveReferenceFields, } from './references.js';
 import { resolveEntryLinksInText } from '../entry-link-resolver.js';
+import { computeEntryUrl } from '../utils/entry-url.js';
+import { DEFAULT_SEO_FIELD_NAMES, isNoindexEntry } from '../static/seo.js';
 /**
  * Generate all AI content from the content store.
  *
  * Walks the schema tree, reads entries, converts to markdown,
  * and produces per-entry files, per-collection all.md files,
  * bundle files, and a manifest.
+ *
+ * A reference or body `entry:` link may only show a target that is itself exported, so every
+ * entry is read and filtered first, which settles the exported set. Only then are body links
+ * resolved for output, references to anything outside the set masked, entry transforms run, and markdown
+ * rendered, in rounds that repeat while a render fails (see the loop below).
  */
 export async function generateAIContent(options) {
     const { store, flatSchema, contentRoot, config, entryLinkUrl, generatedAt, buildId } = options;
+    assertValidNoindexConfig(config);
+    assertValidReferenceLink(config);
     const files = new Map();
     const idIndex = await store.idIndex();
+    const read = {
+        store,
+        contentRoot,
+        config,
+        idIndex,
+        entryLinkUrl,
+        resolveTarget: createReferenceTargetResolver(store),
+    };
     const collections = flatSchema.filter((item) => item.type === 'collection');
-    // Track all entries for bundle filtering
-    const allEntries = [];
-    // Track manifest collections (tree structure)
-    const manifestCollections = [];
-    // Track root-level entries (entries in the content root, not in a subcollection)
-    const rootEntries = [];
+    const collectionNodes = [];
     for (const collection of collections) {
         // Skip the content root itself — we process its children
         if (collection.logicalPath === contentRoot)
@@ -42,23 +56,97 @@ export async function generateAIContent(options) {
         // (subcollections are handled recursively via their parent)
         if (collection.parentPath && collection.parentPath !== contentRoot)
             continue;
-        const collectionResult = await processCollection(store, collection, flatSchema, contentRoot, config, idIndex, entryLinkUrl);
-        allEntries.push(...collectionResult.entries);
-        for (const [filePath, content] of collectionResult.files) {
-            files.set(filePath, content);
-        }
-        manifestCollections.push(collectionResult.manifestCollection);
+        collectionNodes.push(await collectCollection(read, collection, flatSchema));
     }
-    // Process root-level entries (entries in content root, not in any subcollection)
+    // Root-level entries (entries in content root, not in any subcollection)
     const rootCollection = collections.find((c) => c.logicalPath === contentRoot);
-    if (rootCollection?.entries) {
-        const rootResult = await processRootEntries(store, rootCollection, contentRoot, config, idIndex, entryLinkUrl);
-        allEntries.push(...rootResult.entries);
-        for (const [filePath, content] of rootResult.files) {
-            files.set(filePath, content);
+    const rootEntries = rootCollection?.entries ? await collectEntries(read, rootCollection, '') : [];
+    // Collection entries first, then root entries: the order bundles list them in
+    const allPending = [...collectionNodes.flatMap(subtreeEntries), ...rootEntries];
+    // Content id -> output file of every exported entry: the export set, and where each
+    // reference's markdown copy lives
+    const exported = new Map();
+    for (const pending of allPending) {
+        if (pending.contentId !== null)
+            exported.set(pending.contentId, pending.filePath);
+    }
+    const pendingById = new Map();
+    for (const pending of allPending) {
+        if (pending.contentId !== null)
+            pendingById.set(pending.contentId, pending);
+    }
+    // The target as `resolveReferenceTarget` would give it, built from this export's own read, so
+    // an entry outside the export set is never offered and a target cannot change mid-run
+    const subjectOf = (id) => {
+        const pending = pendingById.get(id);
+        if (!pending || !exported.has(id))
+            return undefined;
+        const { slug } = pending.entry;
+        const target = buildResolvedReference(structuredClone(pending.storedData), {
+            id,
+            slug,
+            collection: pending.collectionPath,
+            urlPath: computeEntryUrl(pending.collectionPath, slug, contentRoot),
+        });
+        return { target, body: pending.body };
+    };
+    const linkFor = config?.referenceLink
+        ? await decideReferenceLinks(config.referenceLink, allPending, subjectOf, entryLinkUrl)
+        : undefined;
+    // A body `entry:` link takes the same page URL a reference to its target renders with
+    const bodyLinkUrl = linkFor
+        ? (location) => {
+            const decided = linkFor(location.id);
+            if (typeof decided === 'string')
+                return decided;
+            return entryLinkUrl
+                ? entryLinkUrl(location)
+                : computeEntryUrl(location.collection, location.slug, contentRoot);
         }
-        rootEntries.push(...rootResult.manifestEntries);
+        : entryLinkUrl;
+    const references = createReferenceRendering(idIndex, flatSchema, entryLinkUrl, { mountPath: config?.mountPath ?? '/ai', files: exported }, linkFor);
+    // An entry whose render throws is not in the export after all, so references and body links to
+    // it are withheld like any other left-out target, and the entries that changed run their
+    // transforms and render again. That can fail another entry, so this repeats until a round fails
+    // none; each round removes at least one entry, so it ends.
+    let round = allPending;
+    for (;;) {
+        for (const pending of round) {
+            if (pending.body !== undefined) {
+                const linked = new Set();
+                pending.entry.body = resolveEntryLinksInText(pending.body, idIndex, contentRoot, bodyLinkUrl, {
+                    isLinkable: (id) => {
+                        linked.add(id);
+                        return exported.has(id) && linkFor?.(id) !== false;
+                    },
+                });
+                pending.linkedIds = [...linked];
+            }
+            maskUnexportedTargets(pending.entry, exported);
+            // Fold in adopter-supplied markdown (e.g. a colocated sibling artifact), on data as masked
+            pending.entry.appendedSections = undefined;
+            await runEntryTransform(pending.entry, pending.absolutePath, pending.contentId, config);
+        }
+        const failed = renderAll(round, config, references);
+        if (failed.length === 0)
+            break;
+        const failedIds = new Set();
+        for (const pending of failed) {
+            if (pending.contentId === null)
+                continue;
+            exported.delete(pending.contentId);
+            failedIds.add(pending.contentId);
+        }
+        round = allPending.filter((pending) => {
+            if (!isRendered(pending))
+                return false;
+            const masked = maskUnexportedTargets(pending.entry, exported);
+            return masked || pending.linkedIds.some((id) => failedIds.has(id));
+        });
     }
+    const manifestCollections = collectionNodes.map((node) => emitCollection(node, files));
+    const manifestRootEntries = emitEntries(rootEntries, files);
+    const allEntries = allPending.filter(isRendered);
     const manifestBundles = [];
     if (config?.bundles) {
         for (const bundle of config.bundles) {
@@ -66,11 +154,9 @@ export async function generateAIContent(options) {
             if (/[/\\]|\.\./.test(bundle.name)) {
                 throw new Error(`Invalid bundle name "${bundle.name}": must not contain slashes or ".."`);
             }
-            const matchingEntries = allEntries.filter((entry) => matchesBundleFilter(entry, bundle.filter, contentRoot));
+            const matchingEntries = allEntries.filter((pending) => matchesBundleFilter(pending.entry, bundle.filter, contentRoot));
             if (matchingEntries.length > 0) {
-                const bundleContent = matchingEntries
-                    .map((e) => entryToMarkdown(e, config))
-                    .join('\n---\n\n');
+                const bundleContent = matchingEntries.map((p) => p.markdown).join('\n---\n\n');
                 const bundlePath = `bundles/${bundle.name}.md`;
                 files.set(bundlePath, bundleContent);
                 manifestBundles.push({
@@ -89,18 +175,53 @@ export async function generateAIContent(options) {
     const manifest = {
         ...(buildId ? { buildId } : {}),
         ...(generatedAt || !buildId ? { generated: generatedAt || new Date().toISOString() } : {}),
-        entries: rootEntries,
+        entries: manifestRootEntries,
         collections: manifestCollections,
         bundles: manifestBundles,
     };
     files.set('manifest.json', JSON.stringify(manifest, null, 2));
     return { manifest, files };
 }
-async function processCollection(store, collection, flatSchema, contentRoot, config, idIndex, entryLinkUrl) {
-    const files = new Map();
-    const entries = [];
-    const cleanPath = stripContentRoot(collection.logicalPath, contentRoot);
-    const manifestEntries = [];
+/** Render each entry's markdown, returning those whose render threw (left without markdown). */
+function renderAll(entries, config, references) {
+    const failed = [];
+    for (const pending of entries) {
+        try {
+            pending.markdown = entryToMarkdown(pending.entry, config, references);
+        }
+        catch (err) {
+            pending.markdown = undefined;
+            failed.push(pending);
+            console.warn(`AI content: skipping entry "${pending.entry.slug}" in ${pending.collectionPath}:`, getErrorMessage(err));
+        }
+    }
+    return failed;
+}
+function isRendered(pending) {
+    return pending.markdown !== undefined;
+}
+/** A collection's direct entries, then each subcollection's, in the order all.md lists them. */
+function subtreeEntries(node) {
+    return [...node.entries, ...node.subcollections.flatMap(subtreeEntries)];
+}
+async function collectCollection(read, collection, flatSchema) {
+    const cleanPath = stripContentRoot(collection.logicalPath, read.contentRoot);
+    const entries = await collectEntries(read, collection, cleanPath);
+    const subcollections = [];
+    for (const sub of flatSchema) {
+        if (sub.type !== 'collection' || sub.parentPath !== collection.logicalPath)
+            continue;
+        if (isCollectionExcluded(sub.logicalPath, read.contentRoot, read.config))
+            continue;
+        subcollections.push(await collectCollection(read, sub, flatSchema));
+    }
+    return { collection, cleanPath, entries, subcollections };
+}
+/** Read, resolve and filter the entries directly in `collection` (not its subcollections). */
+async function collectEntries(read, collection, cleanPath) {
+    const { store, contentRoot, config, idIndex, entryLinkUrl, resolveTarget } = read;
+    const skipNoindex = config?.exclude?.noindex !== false;
+    const pending = [];
     const listed = await store.getCollectionEntryPaths(collection.logicalPath);
     // Filter to only entries in this exact collection (not subcollections)
     const directEntries = listed.filter((e) => e.collection === collection.logicalPath);
@@ -111,109 +232,116 @@ async function processCollection(store, collection, flatSchema, contentRoot, con
         if (config?.exclude?.entryTypes?.includes(entryTypeName))
             continue;
         const entryTypeConfig = findEntryType(collection, entryTypeName);
-        if (!entryTypeConfig)
+        if (!entryTypeConfig || entryTypeConfig.unavailable)
             continue;
         try {
             const doc = await store.read(listEntry.collection, listEntry.slug, {
                 resolveReferences: false,
             });
+            // AI content advertises an entry, so `noindex` leaves it out here as it does the sitemap.
+            if (skipNoindex && isNoindexEntry(doc.data, config?.seo))
+                continue;
+            const storedData = doc.data;
+            doc.data = await resolveReferenceFields(doc.data, entryTypeConfig.schema, resolveTarget);
             const aiEntry = docToAIEntry(doc, listEntry.slug, entryTypeName, entryTypeConfig, cleanPath);
-            if (aiEntry.body && idIndex) {
-                aiEntry.body = resolveEntryLinksInText(aiEntry.body, idIndex, contentRoot, entryLinkUrl);
-            }
+            const body = aiEntry.body;
+            // `where` sees every link resolved; the export's own rendering re-resolves from `body`
+            if (body)
+                aiEntry.body = resolveEntryLinksInText(body, idIndex, contentRoot, entryLinkUrl);
             if (config?.exclude?.where?.(aiEntry))
                 continue;
-            // Fold in adopter-supplied markdown (e.g. a colocated sibling artifact), once per entry
-            await runEntryTransform(aiEntry, doc.absolutePath, extractIdFromFilename(path.basename(listEntry.relativePath)), config);
-            entries.push(aiEntry);
-            const entryFilePath = `${cleanPath}/${listEntry.slug}.md`;
-            const entryMarkdown = entryToMarkdown(aiEntry, config);
-            files.set(entryFilePath, entryMarkdown);
-            manifestEntries.push({
-                slug: listEntry.slug,
-                title: aiEntry.data.title ? String(aiEntry.data.title) : undefined,
-                file: entryFilePath,
+            pending.push({
+                entry: aiEntry,
+                body,
+                storedData,
+                linkedIds: [],
+                contentId: extractIdFromFilename(path.basename(listEntry.relativePath)),
+                absolutePath: doc.absolutePath,
+                collectionPath: collection.logicalPath,
+                filePath: cleanPath ? `${cleanPath}/${listEntry.slug}.md` : `${listEntry.slug}.md`,
             });
         }
         catch (err) {
             console.warn(`AI content: skipping entry "${listEntry.slug}" in ${collection.logicalPath}:`, getErrorMessage(err));
-            continue;
-        }
-    }
-    const subcollections = flatSchema.filter((item) => item.type === 'collection' && item.parentPath === collection.logicalPath);
-    const manifestSubcollections = [];
-    for (const sub of subcollections) {
-        if (isCollectionExcluded(sub.logicalPath, contentRoot, config))
-            continue;
-        const subResult = await processCollection(store, sub, flatSchema, contentRoot, config, idIndex, entryLinkUrl);
-        entries.push(...subResult.entries);
-        for (const [filePath, content] of subResult.files) {
-            files.set(filePath, content);
         }
-        manifestSubcollections.push(subResult.manifestCollection);
     }
-    // Write all.md for this collection (includes direct entries + subcollection entries)
-    if (entries.length > 0) {
-        const allContent = entries.map((e) => entryToMarkdown(e, config)).join('\n---\n\n');
-        const allPath = `${cleanPath}/all.md`;
-        files.set(allPath, allContent);
+    return pending;
+}
+/** Write each rendered entry's file and return its manifest rows. */
+function emitEntries(entries, files) {
+    return entries.filter(isRendered).map((pending) => {
+        files.set(pending.filePath, pending.markdown);
+        return {
+            slug: pending.entry.slug,
+            title: frontmatterTitle(pending.entry.data),
+            file: pending.filePath,
+        };
+    });
+}
+/** Write a collection's entry files and all.md, recursively, and return its manifest node. */
+function emitCollection(node, files) {
+    const manifestEntries = emitEntries(node.entries, files);
+    const manifestSubcollections = node.subcollections.map((sub) => emitCollection(sub, files));
+    // all.md covers direct entries + subcollection entries
+    const rendered = subtreeEntries(node).filter(isRendered);
+    const allPath = `${node.cleanPath}/all.md`;
+    if (rendered.length > 0) {
+        files.set(allPath, rendered.map((p) => p.markdown).join('\n---\n\n'));
     }
-    const manifestCollection = {
-        name: collection.name,
-        label: collection.label,
-        description: collection.description,
-        path: cleanPath,
-        allFile: entries.length > 0 ? `${cleanPath}/all.md` : undefined,
-        entryCount: entries.length,
+    return {
+        name: node.collection.name,
+        label: node.collection.label,
+        description: node.collection.description,
+        path: node.cleanPath,
+        allFile: rendered.length > 0 ? allPath : undefined,
+        entryCount: rendered.length,
         entries: manifestEntries,
         subcollections: manifestSubcollections.length > 0 ? manifestSubcollections : undefined,
     };
-    return { entries, files, manifestCollection };
 }
-async function processRootEntries(store, rootCollection, contentRoot, config, idIndex, entryLinkUrl) {
-    const files = new Map();
-    const entries = [];
-    const manifestEntries = [];
-    const listed = await store.getCollectionEntryPaths(rootCollection.logicalPath);
-    // Only direct entries in root (not in subcollections)
-    const directEntries = listed.filter((e) => e.collection === rootCollection.logicalPath);
-    for (const listEntry of directEntries) {
-        const entryTypeName = extractEntryTypeFromFilename(path.basename(listEntry.relativePath));
-        if (!entryTypeName)
-            continue;
-        if (config?.exclude?.entryTypes?.includes(entryTypeName))
-            continue;
-        const entryTypeConfig = findEntryType(rootCollection, entryTypeName);
-        if (!entryTypeConfig)
-            continue;
-        try {
-            const doc = await store.read(listEntry.collection, listEntry.slug, {
-                resolveReferences: false,
-            });
-            const aiEntry = docToAIEntry(doc, listEntry.slug, entryTypeName, entryTypeConfig, '');
-            if (aiEntry.body && idIndex) {
-                aiEntry.body = resolveEntryLinksInText(aiEntry.body, idIndex, contentRoot, entryLinkUrl);
-            }
-            if (config?.exclude?.where?.(aiEntry))
-                continue;
-            // Fold in adopter-supplied markdown (e.g. a colocated sibling artifact), once per entry
-            await runEntryTransform(aiEntry, doc.absolutePath, extractIdFromFilename(path.basename(listEntry.relativePath)), config);
-            entries.push(aiEntry);
-            const entryFilePath = `${listEntry.slug}.md`;
-            const entryMarkdown = entryToMarkdown(aiEntry, config);
-            files.set(entryFilePath, entryMarkdown);
-            manifestEntries.push({
-                slug: listEntry.slug,
-                title: aiEntry.data.title ? String(aiEntry.data.title) : undefined,
-                file: entryFilePath,
-            });
-        }
-        catch (err) {
-            console.warn(`AI content: skipping root entry "${listEntry.slug}":`, getErrorMessage(err));
-            continue;
-        }
+/**
+ * Refuse a `referenceLink` that is not a function, which would otherwise fail every call and
+ * withhold every link it governs, one warning per target.
+ */
+function assertValidReferenceLink(config) {
+    const referenceLink = config?.referenceLink;
+    if (referenceLink !== undefined && typeof referenceLink !== 'function') {
+        throw new Error(`AI content config: referenceLink must be a function, got ${typeof referenceLink}`);
+    }
+}
+/**
+ * Refuse an `exclude.noindex` or `seo` the noindex gate would misread. A non-boolean
+ * `noindex: 'false'` would leave the gate on while the adopter believes it is off, and a
+ * misshapen `seo` reads the flag from a field no entry sets, so every noindex entry ships.
+ */
+function assertValidNoindexConfig(config) {
+    const noindex = config?.exclude?.noindex;
+    if (noindex !== undefined && typeof noindex !== 'boolean') {
+        throw new Error(`AI content config: exclude.noindex must be a boolean, got ${typeof noindex}`);
+    }
+    const seo = config?.seo;
+    if (seo === undefined)
+        return;
+    if (typeof seo !== 'object' || seo === null || Array.isArray(seo)) {
+        throw new Error('AI content config: seo must be an object of { group?, fields? }');
+    }
+    const { group, fields } = seo;
+    if (group !== undefined && typeof group !== 'string') {
+        throw new Error('AI content config: seo.group must be a string');
+    }
+    if (fields === undefined)
+        return;
+    if (typeof fields !== 'object' ||
+        fields === null ||
+        Array.isArray(fields) ||
+        Object.values(fields).some((name) => typeof name !== 'string')) {
+        throw new Error('AI content config: seo.fields must map SEO fields to field-name strings');
+    }
+    // An unknown key (`noIndex`) would leave the gate reading the default field name
+    const unknown = Object.keys(fields).filter((key) => !(key in DEFAULT_SEO_FIELD_NAMES));
+    if (unknown.length > 0) {
+        throw new Error(`AI content config: unknown seo.fields key(s): ${unknown.join(', ')}`);
     }
-    return { entries, files, manifestEntries };
 }
 function stripContentRoot(logicalPath, contentRoot) {
     if (logicalPath.startsWith(contentRoot + '/')) {
@@ -282,8 +410,9 @@ function makeReadSibling(dir) {
 /**
  * Run the configured entry transform (if any), caching its returned markdown on
  * `entry.appendedSections`. The transform receives the entry's content ID and a directory-bound
- * `readSibling`. Runs once per entry; the cached result is reused across the per-entry file, the
- * collection `all.md`, and any bundle that includes this entry. A throwing transform is logged and
+ * `readSibling`. Runs for every entry in the first round, then again in any later round whose
+ * masking changed the entry, always on the data as masked; the cached result is reused across the per-entry file, the collection `all.md`, and any bundle that
+ * includes this entry. A throwing transform is logged and
  * skipped — the entry still renders without the appended section (distinct from an unreadable
  * entry, which is skipped entirely upstream).
  */

package/dist/ai/index.d.ts

@@ -9,6 +9,6 @@ export { createAIContentHandler } from './handler.js';
 export { generateAIContent } from './generate.js';
 /** See `to-plain-text.ts`'s module doc for the pipeline and why it differs from the AI-content path above. */
 export { toPlainText } from './to-plain-text.js';
-export type { AIContentConfig, ExcludeConfig, BundleConfig, BundleFilter, FieldTransformFn, FieldTransforms, ComponentProps, ComponentTransformFn, ComponentTransforms, BodyTransformFn, BodyTransforms, EntryTransformFn, EntryTransforms, EntryTransformContext, AIManifest, AIManifestCollection, AIManifestEntry, AIManifestBundle, AIEntry, AIEntryMeta, } from './types.js';
+export type { AIContentConfig, ExcludeConfig, BundleConfig, BundleFilter, FieldTransformFn, FieldTransforms, ComponentProps, ComponentTransformFn, ComponentTransforms, BodyTransformFn, BodyTransforms, EntryTransformFn, EntryTransforms, EntryTransformContext, AIManifest, AIManifestCollection, AIManifestEntry, AIManifestBundle, AIEntry, AIEntryMeta, AIExcludedReference, AIReferenceLinkContext, AIReferenceLinkFn, AIReferenceValue, AIResolvedReference, AIUnavailableReference, } from './types.js';
 export type { AIContentHandlerOptions } from './handler.js';
 export type { GenerateOptions, GenerateResult } from './generate.js';

package/dist/ai/json-to-markdown.d.ts

@@ -5,10 +5,30 @@
  * Handles all CanopyCMS field types including nested objects and blocks.
  */
 import type { AIEntry, AIContentConfig } from './types.js';
+/**
+ * How a resolved reference becomes link text and a link target. generate.ts supplies one that
+ * knows every target's schema and the adopter's `entryLinkUrl`; the default needs neither.
+ */
+export interface ReferenceRendering {
+    /** Display title of a resolved target, used unless the field's `displayField` holds text. */
+    title(target: Record<string, unknown>): string;
+    /**
+     * Where the page link points: undefined renders the title as plain text, and `false` also
+     * drops the markdown copy, for a target the site says has no page.
+     */
+    url(target: Record<string, unknown>): string | false | undefined;
+    /** URL of the target's own file in this export, linked after the page as its markdown copy. */
+    markdownUrl?(target: Record<string, unknown>): string | undefined;
+}
+/**
+ * The entry's `title` as frontmatter carries it. Only a scalar qualifies: a `reference` field
+ * named `title` holds an object and renders as a field instead.
+ */
+export declare function frontmatterTitle(data: Record<string, unknown>): string | undefined;
 /**
  * Convert an entry to clean markdown suitable for AI consumption.
  *
  * For MD/MDX entries: renders frontmatter fields as metadata, appends body verbatim.
  * For JSON entries: full schema-driven conversion of all fields.
  */
-export declare function entryToMarkdown(entry: AIEntry, config?: AIContentConfig): string;
+export declare function entryToMarkdown(entry: AIEntry, config?: AIContentConfig, references?: ReferenceRendering): string;