@actuate-media/cms-core 0.65.0 → 0.67.0

package/dist/api/routes/seo-public.js

@@ -3,6 +3,10 @@ import { safeFetch, SsrfBlockedError } from '../../security/safe-fetch.js';
 import { getActuateConfig } from '../../config/runtime.js';
 import { guardedDb } from '../guarded-db.js';
 import { computeLightSeoScore } from './dashboard.js';
+import { resolveDocumentUrl } from '../../seo/page-meta.js';
+import { resolveRobotsDirectives, resolveSeoEnvironment } from '../../seo/robots.js';
+import { buildAiBotRobotsLines } from '../../seo/ai-bots.js';
+import { getSeoOverrides, resolveSeoModuleSettings, } from '../../seo/config-store.js';
 import { json, errorResponse, internalError, escapeXml, SITEMAP_PAGE_SIZE, renderOgSvg, safeFindMany, requireAuth, WRITE_ROLES, ADMIN_ROLES, requireRole, enforceDocumentReadAccess, } from '../route-helpers.js';
 /**
  * Load the runtime config with DB-stored SEO overrides applied. Used by every
@@ -25,6 +29,34 @@ export async function loadEffectiveConfig() {
         return base;
     }
 }
+/**
+ * Whether a published document resolves to `noindex` and therefore must be kept
+ * OUT of the sitemap. Listing a noindexed URL in the sitemap ("index me") while
+ * its page emits a `noindex` meta tag ("don't") sends crawlers conflicting
+ * signals and wastes crawl budget — Google explicitly flags this. Mirrors the
+ * robots inputs used by `composePageMeta` so the sitemap and the rendered meta
+ * tag always agree.
+ */
+function isDocumentNoIndex(data, collectionSeo, siteSeo, environment) {
+    const pageSettings = (data.pageSettings ?? {});
+    const robots = resolveRobotsDirectives({
+        pageData: {
+            robotsPolicy: pageSettings.robotsPolicy ?? data.robotsPolicy,
+            noIndex: pageSettings.noIndex ??
+                data.noIndex ??
+                // Legacy lowercase field still written by some editors / counted by the
+                // sitemap status endpoint.
+                data.noindex ??
+                collectionSeo?.defaultRobots?.noIndex,
+        },
+        settings: {
+            defaultNoIndex: siteSeo?.robots?.defaultNoIndex,
+            noIndexNonProduction: siteSeo?.robots?.noIndexNonProduction,
+        },
+        environment,
+    });
+    return robots.noIndex;
+}
 export function siteUrlFromRequest(request, cfg) {
     const seo = (cfg ?? getActuateConfig())?.seo;
     if (seo?.siteUrl)
@@ -46,10 +78,11 @@ export function registerSeoPublicRoutes(router) {
     // and the dynamic /og.png OG-image endpoint. These are unauthenticated by
     // design — search engines and social crawlers will fetch them.
     // ---------------------------------------------------------------------------
-    function sitemapEligibleCollections(cfg) {
+    function sitemapEligibleCollections(cfg, opts) {
         const resolved = cfg ?? getActuateConfig();
         if (!resolved)
             return [];
+        const includePosts = opts?.includePosts ?? true;
         const excluded = new Set(resolved.seo?.sitemap?.excludeCollections ?? []);
         const out = [];
         for (const [slug, col] of Object.entries(resolved.collections ?? {})) {
@@ -58,6 +91,10 @@ export function registerSeoPublicRoutes(router) {
             // no `type` and must never produce indexable sitemap URLs.
             if (col.type !== 'page' && col.type !== 'post')
                 continue;
+            // Honor the "Include posts in sitemap" crawl setting (Settings → SEO →
+            // Technical). When off, post-type collections are dropped entirely.
+            if (col.type === 'post' && !includePosts)
+                continue;
             if (col.admin?.hidden)
                 continue;
             if (excluded.has(slug))
@@ -74,7 +111,27 @@ export function registerSeoPublicRoutes(router) {
             if (cfg?.seo?.sitemap?.disabled)
                 return errorResponse('Sitemap disabled', 404);
             const base = siteUrlFromRequest(request, cfg);
-            const cols = sitemapEligibleCollections(cfg);
+            const moduleSettings = resolveSeoModuleSettings(await getSeoOverrides(db()));
+            // Master switch: when the whole public site is opted out of indexing, the
+            // sitemap (a positive "please index these" signal) must be empty so it
+            // doesn't contradict the `Disallow: /` we emit in robots.txt.
+            if (moduleSettings.indexEntireSite === false) {
+                const emptyIndex = [
+                    '<?xml version="1.0" encoding="UTF-8"?>',
+                    '<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">',
+                    '</sitemapindex>',
+                ].join('\n');
+                return new Response(emptyIndex, {
+                    status: 200,
+                    headers: {
+                        'Content-Type': 'application/xml; charset=utf-8',
+                        'Cache-Control': 'public, max-age=300, s-maxage=600',
+                    },
+                });
+            }
+            const cols = sitemapEligibleCollections(cfg, {
+                includePosts: moduleSettings.includePostsInSitemap,
+            });
             // Sitemap index points at /sitemaps/:slug.xml for each collection. A
             // collection with more than SITEMAP_PAGE_SIZE published docs is split
             // across `?page=N` files so we never silently truncate (the old code
@@ -82,10 +139,22 @@ export function registerSeoPublicRoutes(router) {
             const now = new Date().toISOString();
             const perCollection = await Promise.all(cols.map(async (c) => {
                 let count = 0;
+                // Report the newest published change in the collection as the child
+                // sitemap's <lastmod> instead of "now". A lastmod that changes on
+                // every fetch trains crawlers to ignore it; an accurate one is the
+                // single most useful signal we can give them.
+                let lastmod = now;
                 try {
                     count = await db().document.count({
                         where: { collection: c.slug, deletedAt: null, status: 'PUBLISHED' },
                     });
+                    const newest = await db().document.findFirst({
+                        where: { collection: c.slug, deletedAt: null, status: 'PUBLISHED' },
+                        select: { updatedAt: true },
+                        orderBy: { updatedAt: 'desc' },
+                    });
+                    if (newest?.updatedAt)
+                        lastmod = newest.updatedAt.toISOString();
                 }
                 catch {
                     count = 0;
@@ -99,7 +168,7 @@ export function registerSeoPublicRoutes(router) {
                     entries.push([
                         '  <sitemap>',
                         `    <loc>${escapeXml(loc)}</loc>`,
-                        `    <lastmod>${now}</lastmod>`,
+                        `    <lastmod>${lastmod}</lastmod>`,
                         '  </sitemap>',
                     ].join('\n'));
                 }
@@ -148,32 +217,115 @@ export function registerSeoPublicRoutes(router) {
                 return errorResponse('Collection excluded from sitemap', 404);
             if (collection.seo?.excludeFromSitemap)
                 return errorResponse('Collection excluded from sitemap', 404);
+            const moduleSettings = resolveSeoModuleSettings(await getSeoOverrides(db()));
+            // Master switch: whole public site opted out of indexing → empty urlset
+            // (mirrors the empty sitemap index + `Disallow: /` in robots.txt).
+            if (moduleSettings.indexEntireSite === false) {
+                const emptyUrlset = [
+                    '<?xml version="1.0" encoding="UTF-8"?>',
+                    '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">',
+                    '</urlset>',
+                ].join('\n');
+                return new Response(emptyUrlset, {
+                    status: 200,
+                    headers: {
+                        'Content-Type': 'application/xml; charset=utf-8',
+                        'Cache-Control': 'public, max-age=300, s-maxage=600',
+                    },
+                });
+            }
+            // Honor the "Include posts in sitemap" crawl setting consistently with the
+            // sitemap index, so a post-type sitemap can't be reached when posts are
+            // excluded there.
+            if (collection.type === 'post' && !moduleSettings.includePostsInSitemap)
+                return errorResponse('Collection excluded from sitemap', 404);
             const base = siteUrlFromRequest(request, cfg);
+            const environment = resolveSeoEnvironment();
             const pageParam = Number(new URL(request.url).searchParams.get('page') ?? '0');
             const page = Number.isFinite(pageParam) && pageParam > 0 ? Math.floor(pageParam) : 0;
             const docs = await db().document.findMany({
                 where: { collection: rawSlug, deletedAt: null, status: 'PUBLISHED' },
-                select: { slug: true, updatedAt: true, data: true },
+                select: { slug: true, updatedAt: true, data: true, locale: true, localeGroupId: true },
                 orderBy: { updatedAt: 'desc' },
                 skip: page * SITEMAP_PAGE_SIZE,
                 take: SITEMAP_PAGE_SIZE,
             });
-            const prefix = (collection.urlPrefix ?? '').replace(/^\/|\/$/g, '');
+            // hreflang: when the site runs i18n with more than one locale, emit
+            // `<xhtml:link rel="alternate">` annotations so search engines serve the
+            // right language variant. Each translation shares a `localeGroupId`; its
+            // own `data.path`/slug yields the localized URL via resolveDocumentUrl, so
+            // this works regardless of the locale routing strategy. Skipped entirely
+            // (zero extra queries) for single-locale sites.
+            const i18nLocales = cfg?.i18n?.locales ?? [];
+            const i18nEnabled = i18nLocales.length > 1;
+            const defaultLocale = cfg?.i18n?.defaultLocale;
+            const alternatesByGroup = new Map();
+            if (i18nEnabled) {
+                const groupIds = [
+                    ...new Set(docs.map((d) => d.localeGroupId).filter((g) => !!g)),
+                ];
+                if (groupIds.length > 0) {
+                    const siblings = await db().document.findMany({
+                        where: {
+                            collection: rawSlug,
+                            deletedAt: null,
+                            status: 'PUBLISHED',
+                            localeGroupId: { in: groupIds },
+                        },
+                        select: { slug: true, data: true, locale: true, localeGroupId: true },
+                    });
+                    for (const s of siblings) {
+                        if (!s.localeGroupId)
+                            continue;
+                        const sData = s.data || {};
+                        const loc = resolveDocumentUrl({
+                            siteUrl: base,
+                            urlPrefix: collection.urlPrefix,
+                            slug: s.slug,
+                            data: sData,
+                        });
+                        const list = alternatesByGroup.get(s.localeGroupId) ?? [];
+                        list.push({ locale: s.locale, loc });
+                        alternatesByGroup.set(s.localeGroupId, list);
+                    }
+                }
+            }
             const defaultPriority = collection.seo?.sitemapPriority ??
                 cfg?.seo?.sitemap?.defaultPriority ??
                 (collection.type === 'page' ? 0.8 : 0.6);
             const changefreq = collection.seo?.sitemapChangeFreq ?? cfg?.seo?.sitemap?.defaultChangeFreq ?? 'weekly';
             const urls = [];
+            let hreflangUsed = false;
+            const renderAlternates = (groupId) => {
+                if (!i18nEnabled || !groupId)
+                    return [];
+                const alts = alternatesByGroup.get(groupId);
+                if (!alts || alts.length < 2)
+                    return [];
+                hreflangUsed = true;
+                const lines = alts.map((a) => `    <xhtml:link rel="alternate" hreflang="${escapeXml(a.locale ?? defaultLocale ?? 'und')}" href="${escapeXml(a.loc)}" />`);
+                const fallback = alts.find((a) => a.locale === defaultLocale) ?? alts[0];
+                if (fallback) {
+                    lines.push(`    <xhtml:link rel="alternate" hreflang="x-default" href="${escapeXml(fallback.loc)}" />`);
+                }
+                return lines;
+            };
             // Archive page first (e.g. /blog) for post-type collections — only on
             // the first page so it isn't duplicated across paginated sitemap files.
             if (page === 0 && collection.seo?.archivePath && collection.type === 'post') {
                 const archive = collection.seo.archivePath.startsWith('http')
                     ? collection.seo.archivePath
                     : `${base}${collection.seo.archivePath}`;
+                // The archive reflects its newest post, so derive lastmod from the most
+                // recently updated published doc rather than the current time.
+                const archiveLastmod = docs.reduce((newest, d) => {
+                    const u = d.updatedAt instanceof Date ? d.updatedAt : null;
+                    return u && (!newest || u > newest) ? u : newest;
+                }, null) ?? new Date();
                 urls.push([
                     '  <url>',
                     `    <loc>${escapeXml(archive)}</loc>`,
-                    `    <lastmod>${new Date().toISOString()}</lastmod>`,
+                    `    <lastmod>${archiveLastmod.toISOString()}</lastmod>`,
                     `    <changefreq>${changefreq}</changefreq>`,
                     '    <priority>0.6</priority>',
                     '  </url>',
@@ -182,21 +334,37 @@ export function registerSeoPublicRoutes(router) {
             for (const d of docs) {
                 const data = d.data || {};
                 const slug = d.slug ?? data.slug;
-                if (!slug)
+                // An explicit `data.path` can stand on its own (e.g. the homepage at
+                // "/"), so only require a slug when there's no canonical path set.
+                const hasExplicitPath = typeof data.path === 'string' && data.path.trim() !== '';
+                if (!slug && !hasExplicitPath)
                     continue;
-                const loc = prefix ? `${base}/${prefix}/${slug}` : `${base}/${slug}`;
+                // Never list a noindexed URL — it conflicts with the page's own robots
+                // meta tag and wastes crawl budget.
+                if (isDocumentNoIndex(data, collection.seo, cfg?.seo, environment))
+                    continue;
+                const loc = resolveDocumentUrl({
+                    siteUrl: base,
+                    urlPrefix: collection.urlPrefix,
+                    slug,
+                    data,
+                });
                 urls.push([
                     '  <url>',
                     `    <loc>${escapeXml(loc)}</loc>`,
                     `    <lastmod>${d.updatedAt?.toISOString() ?? new Date().toISOString()}</lastmod>`,
                     `    <changefreq>${changefreq}</changefreq>`,
                     `    <priority>${defaultPriority.toFixed(1)}</priority>`,
+                    ...renderAlternates(d.localeGroupId),
                     '  </url>',
                 ].join('\n'));
             }
+            const urlsetOpen = hreflangUsed
+                ? '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:xhtml="http://www.w3.org/1999/xhtml">'
+                : '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">';
             const xml = [
                 '<?xml version="1.0" encoding="UTF-8"?>',
-                '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">',
+                urlsetOpen,
                 urls.join('\n'),
                 '</urlset>',
             ].join('\n');
@@ -212,43 +380,165 @@ export function registerSeoPublicRoutes(router) {
             return internalError(err, 'sitemaps/:slug.xml');
         }
     });
+    // ---------------------------------------------------------------------------
+    // Content feeds: RSS 2.0 (/feed.xml) + JSON Feed 1.1 (/feed.json).
+    //
+    // Newest published items across post-type collections (falling back to pages
+    // when a site has no post collections), excluding per-document noindex. Unlike
+    // the sitemap these are a subscription/distribution surface, not a crawl
+    // directive, so they are not gated by `indexEntireSite`.
+    // ---------------------------------------------------------------------------
+    const FEED_LIMIT = 50;
+    async function gatherFeedItems(cfg, base) {
+        const { resolveDocumentUrl } = await import('../../seo/page-meta.js');
+        const environment = resolveSeoEnvironment();
+        const siteSeo = cfg?.seo;
+        const all = sitemapEligibleCollections(cfg);
+        const posts = all.filter((c) => c.type === 'post');
+        const cols = posts.length > 0 ? posts : all;
+        const collected = [];
+        for (const c of cols) {
+            let docs = [];
+            try {
+                docs = await db().document.findMany({
+                    where: { collection: c.slug, deletedAt: null, status: 'PUBLISHED' },
+                    select: { slug: true, title: true, data: true, publishedAt: true, updatedAt: true },
+                    orderBy: { publishedAt: 'desc' },
+                    take: FEED_LIMIT,
+                });
+            }
+            catch {
+                docs = [];
+            }
+            for (const d of docs) {
+                const data = d.data || {};
+                if (isDocumentNoIndex(data, c.seo, siteSeo, environment))
+                    continue;
+                const url = resolveDocumentUrl({
+                    siteUrl: base,
+                    urlPrefix: c.urlPrefix,
+                    slug: d.slug,
+                    data,
+                });
+                const title = (typeof data.metaTitle === 'string' && data.metaTitle) ||
+                    d.title ||
+                    (typeof data.title === 'string' ? data.title : '') ||
+                    'Untitled';
+                const description = (typeof data.keyTakeaway === 'string' && data.keyTakeaway) ||
+                    (typeof data.metaDescription === 'string' && data.metaDescription) ||
+                    (typeof data.excerpt === 'string' ? data.excerpt : '') ||
+                    undefined;
+                const date = (d.publishedAt ?? d.updatedAt)?.toISOString();
+                const modified = d.updatedAt?.toISOString();
+                // Only advertise a modification time when it is genuinely newer than
+                // publication, so unchanged posts don't look perpetually "updated".
+                const dateModified = modified && date && modified > date ? modified : undefined;
+                collected.push({ url, title, description, date, dateModified });
+            }
+        }
+        // Merge collections and keep the globally newest items.
+        collected.sort((a, b) => (b.date ?? '').localeCompare(a.date ?? ''));
+        return collected.slice(0, FEED_LIMIT);
+    }
+    router.get('/feed.xml', async (request) => {
+        try {
+            const cfg = await loadEffectiveConfig();
+            const base = siteUrlFromRequest(request, cfg);
+            const items = await gatherFeedItems(cfg, base);
+            const { buildRssFeed } = await import('../../seo/feeds.js');
+            const xml = buildRssFeed({
+                title: cfg?.seo?.siteName || 'Site',
+                homeUrl: base || '/',
+                feedUrl: `${base}/feed.xml`,
+                description: cfg?.seo?.defaultMetaDescription,
+                updated: items[0]?.date,
+                language: cfg?.i18n?.defaultLocale,
+            }, items);
+            return new Response(xml, {
+                status: 200,
+                headers: {
+                    'Content-Type': 'application/rss+xml; charset=utf-8',
+                    'Cache-Control': 'public, max-age=300, s-maxage=600',
+                },
+            });
+        }
+        catch (err) {
+            return internalError(err, 'feed.xml');
+        }
+    });
+    router.get('/feed.json', async (request) => {
+        try {
+            const cfg = await loadEffectiveConfig();
+            const base = siteUrlFromRequest(request, cfg);
+            const items = await gatherFeedItems(cfg, base);
+            const { buildJsonFeed } = await import('../../seo/feeds.js');
+            const body = buildJsonFeed({
+                title: cfg?.seo?.siteName || 'Site',
+                homeUrl: base || '/',
+                feedUrl: `${base}/feed.json`,
+                description: cfg?.seo?.defaultMetaDescription,
+                updated: items[0]?.date,
+                language: cfg?.i18n?.defaultLocale,
+            }, items);
+            return new Response(body, {
+                status: 200,
+                headers: {
+                    'Content-Type': 'application/feed+json; charset=utf-8',
+                    'Cache-Control': 'public, max-age=300, s-maxage=600',
+                },
+            });
+        }
+        catch (err) {
+            return internalError(err, 'feed.json');
+        }
+    });
     router.get('/robots.txt', async (request) => {
         try {
             const cfg = await loadEffectiveConfig();
             const seo = cfg?.seo;
             if (seo?.robots?.disabled)
                 return errorResponse('robots.txt disabled', 404);
-            // Admin-edited custom robots.txt body (Technical tab) wins when set —
-            // the editor is the single source of truth for operators who want full
-            // control over directives.
+            // Read SEO module settings up-front: they drive the custom-body override,
+            // the `robotsEnabled` kill-switch, and the `indexEntireSite` master toggle.
+            let settings;
             try {
-                const { getSeoOverrides, resolveSeoModuleSettings } = await import('../../seo/config-store.js');
-                const settings = resolveSeoModuleSettings(await getSeoOverrides(db()));
-                if (settings.robotsEnabled === false)
-                    return errorResponse('robots.txt disabled', 404);
-                const custom = settings.robotsTxt?.trim();
-                if (custom) {
-                    return new Response(settings.robotsTxt, {
-                        status: 200,
-                        headers: {
-                            'Content-Type': 'text/plain; charset=utf-8',
-                            'Cache-Control': 'public, max-age=3600, s-maxage=3600',
-                        },
-                    });
-                }
+                settings = resolveSeoModuleSettings(await getSeoOverrides(db()));
             }
             catch {
-                // Fall through to generated defaults on any override-read failure.
+                // Fall back to defaults (indexable) on any override-read failure.
+                settings = resolveSeoModuleSettings(null);
+            }
+            if (settings.robotsEnabled === false)
+                return errorResponse('robots.txt disabled', 404);
+            // Admin-edited custom robots.txt body (Technical tab) wins when set —
+            // the editor is the single source of truth for operators who want full
+            // control over directives.
+            const custom = settings.robotsTxt?.trim();
+            if (custom) {
+                return new Response(settings.robotsTxt, {
+                    status: 200,
+                    headers: {
+                        'Content-Type': 'text/plain; charset=utf-8',
+                        'Cache-Control': 'public, max-age=3600, s-maxage=3600',
+                    },
+                });
             }
             const base = siteUrlFromRequest(request, cfg);
-            const sitemapUrl = seo?.sitemap?.disabled ? undefined : `${base}/api/cms/sitemap.xml`;
-            const lines = [
-                'User-agent: *',
-                'Allow: /',
-                'Disallow: /admin',
-                'Disallow: /api/',
-                '',
-            ];
+            // `indexEntireSite` is the public-site master switch. When OFF we emit a
+            // blanket `Disallow: /` for all crawlers (which already covers /admin and
+            // /api) and stop advertising the sitemap — the operator has opted the
+            // whole public site out of indexing. This only ever tightens crawling; it
+            // never affects CMS admin/API protection, which is enforced by auth + the
+            // admin's own `noindex` meta tag regardless of this file.
+            const indexable = settings.indexEntireSite !== false;
+            // Advertise the canonical public sitemap URL (`/sitemap.xml`), not the
+            // internal `/api/cms/sitemap.xml` mount. Consumer sites proxy the root
+            // path to the CMS route; the rest of the SEO subsystem (search-engine
+            // pings, index status, llms.txt) already standardizes on `/sitemap.xml`.
+            const sitemapUrl = !indexable || seo?.sitemap?.disabled ? undefined : `${base}/sitemap.xml`;
+            const lines = indexable
+                ? ['User-agent: *', 'Allow: /', 'Disallow: /admin', 'Disallow: /api/', '']
+                : ['User-agent: *', 'Disallow: /', ''];
             for (const rule of seo?.robots?.additionalRules ?? []) {
                 lines.push(`User-agent: ${rule.userAgent}`);
                 for (const allow of rule.allow ?? [])
@@ -257,22 +547,12 @@ export function registerSeoPublicRoutes(router) {
                     lines.push(`Disallow: ${dis}`);
                 lines.push('');
             }
-            if (seo?.robots?.blockAIBots) {
-                const bots = [
-                    'GPTBot',
-                    'ChatGPT-User',
-                    'ClaudeBot',
-                    'Claude-Web',
-                    'anthropic-ai',
-                    'Bytespider',
-                    'CCBot',
-                    'Google-Extended',
-                ];
-                for (const bot of bots) {
-                    lines.push(`User-agent: ${bot}`);
-                    lines.push('Disallow: /');
-                    lines.push('');
-                }
+            // Nuanced AI-crawler policy (training vs retrieval), falling back to the
+            // legacy `blockAIBots` boolean. Only emitted when the site is indexable —
+            // when the master switch is off, the blanket `Disallow: /` above already
+            // covers every crawler.
+            if (indexable) {
+                lines.push(...buildAiBotRobotsLines(seo?.robots?.aiPolicy, seo?.robots?.blockAIBots));
             }
             if (sitemapUrl)
                 lines.push(`Sitemap: ${sitemapUrl}`, '');
@@ -288,6 +568,29 @@ export function registerSeoPublicRoutes(router) {
             return internalError(err, 'robots.txt');
         }
     });
+    // IndexNow key verification file. IndexNow (Bing, Yandex, Seznam, Naver, …)
+    // requires the submitting key to be retrievable from the same host; we serve
+    // it here and advertise this path as `keyLocation` when submitting. The key
+    // comes from the `INDEXNOW_KEY` env var — when unset, IndexNow is disabled and
+    // this returns 404. The key is a public verification token by design (it's
+    // embedded in every submission), so serving it is not a secret leak.
+    router.get('/indexnow-key.txt', async () => {
+        try {
+            const key = process.env.INDEXNOW_KEY?.trim();
+            if (!key)
+                return errorResponse('IndexNow not configured', 404);
+            return new Response(key, {
+                status: 200,
+                headers: {
+                    'Content-Type': 'text/plain; charset=utf-8',
+                    'Cache-Control': 'public, max-age=3600, s-maxage=3600',
+                },
+            });
+        }
+        catch (err) {
+            return internalError(err, 'indexnow-key.txt');
+        }
+    });
     router.get('/og.png', async (request) => {
         try {
             const cfg = await loadEffectiveConfig();
@@ -341,7 +644,7 @@ export function registerSeoPublicRoutes(router) {
                 url: `/${doc.collection}/${doc.slug || ''}`,
                 title: doc.title || data.title || '',
                 description: data.metaDescription || data.seoDescription || '',
-                datePublished: doc.createdAt?.toISOString(),
+                datePublished: (doc.publishedAt ?? doc.createdAt)?.toISOString(),
                 dateModified: doc.updatedAt?.toISOString(),
                 type: doc.collection === 'posts' ? 'post' : 'page',
                 keywords: data.tags || data.keywords,
@@ -398,11 +701,11 @@ export function registerSeoPublicRoutes(router) {
                 twitterTitle: data.twitterTitle,
                 twitterDescription: data.twitterDescription,
                 twitterImage: data.twitterImage,
-                publishedTime: doc.createdAt?.toISOString(),
+                publishedTime: (doc.publishedAt ?? doc.createdAt)?.toISOString(),
                 modifiedTime: doc.updatedAt?.toISOString(),
                 noIndex: robots.noIndex,
                 noFollow: robots.noFollow,
-                canonical: data.canonical,
+                canonical: data.canonical || data.canonicalUrl,
             });
             return new Response(JSON.stringify(tags), {
                 status: 200,
@@ -687,10 +990,16 @@ export function registerSeoPublicRoutes(router) {
         }, userId);
     }
     /** Build the default robots.txt body (mirrors the public /robots.txt route). */
-    function buildDefaultRobots(request, cfg) {
+    function buildDefaultRobots(request, cfg, settings) {
         const base = siteUrlFromRequest(request, cfg);
-        const sitemapUrl = cfg?.seo?.sitemap?.disabled ? undefined : `${base}/api/cms/sitemap.xml`;
-        const lines = ['User-agent: *', 'Allow: /', 'Disallow: /admin', 'Disallow: /api/', ''];
+        const indexable = settings.indexEntireSite !== false;
+        const sitemapUrl = !indexable || cfg?.seo?.sitemap?.disabled ? undefined : `${base}/sitemap.xml`;
+        const lines = indexable
+            ? ['User-agent: *', 'Allow: /', 'Disallow: /admin', 'Disallow: /api/', '']
+            : ['User-agent: *', 'Disallow: /', ''];
+        if (indexable) {
+            lines.push(...buildAiBotRobotsLines(cfg?.seo?.robots?.aiPolicy, cfg?.seo?.robots?.blockAIBots));
+        }
         if (sitemapUrl)
             lines.push(`Sitemap: ${sitemapUrl}`, '');
         return lines.join('\n');
@@ -707,7 +1016,7 @@ export function registerSeoPublicRoutes(router) {
             const cfg = await loadEffectiveConfig();
             const settings = resolveSeoModuleSettings(await getSeoOverrides(db()));
             const base = siteUrlFromRequest(request, cfg);
-            const cols = sitemapEligibleCollections(cfg);
+            const cols = sitemapEligibleCollections(cfg, { includePosts: settings.includePostsInSitemap });
             let pagesIncluded = 0;
             let pagesExcluded = 0;
             for (const col of cols) {
@@ -736,7 +1045,7 @@ export function registerSeoPublicRoutes(router) {
                 warnings.push('No published pages are eligible for the sitemap.');
             return json({
                 data: {
-                    sitemapUrl: `${base}/api/cms/sitemap.xml`,
+                    sitemapUrl: `${base}/sitemap.xml`,
                     lastGeneratedAt: settings.sitemapLastGeneratedAt,
                     pagesIncluded,
                     pagesExcluded,
@@ -764,7 +1073,7 @@ export function registerSeoPublicRoutes(router) {
             const cfg = await loadEffectiveConfig();
             const settings = resolveSeoModuleSettings(await getSeoOverrides(db()));
             const base = siteUrlFromRequest(request, cfg);
-            const cols = sitemapEligibleCollections(cfg);
+            const cols = sitemapEligibleCollections(cfg, { includePosts: settings.includePostsInSitemap });
             let pagesIncluded = 0;
             for (const col of cols) {
                 pagesIncluded += await db().document.count({
@@ -773,7 +1082,7 @@ export function registerSeoPublicRoutes(router) {
             }
             return json({
                 data: {
-                    sitemapUrl: `${base}/api/cms/sitemap.xml`,
+                    sitemapUrl: `${base}/sitemap.xml`,
                     lastGeneratedAt: now,
                     pagesIncluded,
                     pagesExcluded: 0,
@@ -887,7 +1196,7 @@ export function registerSeoPublicRoutes(router) {
             const cfg = await loadEffectiveConfig();
             const settings = resolveSeoModuleSettings(await getSeoOverrides(db()));
             const custom = settings.robotsTxt?.trim();
-            const body = custom ? settings.robotsTxt : buildDefaultRobots(request, cfg);
+            const body = custom ? settings.robotsTxt : buildDefaultRobots(request, cfg, settings);
             return json({ data: { body, generated: !custom } });
         }
         catch (err) {