@koehler8/cms 1.0.0-beta.37 → 1.0.0-beta.39

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@koehler8/cms",
-  "version": "1.0.0-beta.37",
+  "version": "1.0.0-beta.39",
   "description": "Vertex CMS — lightweight, config-driven Vue 3 framework with theming, extensions, and SSG support",
   "license": "MIT",
   "type": "module",

package/scripts/ssg-build/findEmptyRenders.mjs

@@ -0,0 +1,42 @@
+import fsp from 'node:fs/promises';
+import path from 'node:path';
+
+// Detect a page whose root container never mounted any real markup — just
+// Vue's SSR placeholder comment for "nothing rendered here". This is the
+// signature of a rare, silent failure in the onServerPrefetch data-fetch
+// path: @vue/server-renderer's renderComponentVNode gates a component's
+// render on `Promise.all(prefetches).catch(NOOP)` (see server-renderer.cjs.js
+// in any consuming site's node_modules) — if that promise ever rejects, the
+// renderer swallows the error entirely and renders anyway, using whatever
+// default/empty state the component held. No warning, no non-zero exit —
+// just well-formed, blank HTML on disk. A real page from this framework is
+// never legitimately empty (Home.vue always renders at least the skip-link +
+// <main>, DraftGate renders a dialog, 404 renders NotFound), so any match
+// here is unconditionally a bug. Kept in its own module (no side effects on
+// import) so it's unit-testable without tripping index.mjs's top-level
+// main() invocation — see bin/cms-ssg-build.js.
+const EMPTY_ROOT_RE = /<div id="[^"]+"[^>]*>\s*<!---->\s*<\/div>/;
+
+export async function findEmptyRenders(dir) {
+  const empties = [];
+  let entries;
+  try {
+    entries = await fsp.readdir(dir, { recursive: true });
+  } catch {
+    return empties;
+  }
+  for (const entry of entries) {
+    if (!entry.endsWith('index.html')) continue;
+    const full = path.join(dir, entry);
+    let html;
+    try {
+      html = await fsp.readFile(full, 'utf8');
+    } catch {
+      continue;
+    }
+    if (EMPTY_ROOT_RE.test(html)) {
+      empties.push(full);
+    }
+  }
+  return empties;
+}

package/scripts/ssg-build/index.mjs

@@ -26,6 +26,7 @@ import fs from 'node:fs';
 import fsp from 'node:fs/promises';
 import path from 'node:path';
 import os from 'node:os';
+import { findEmptyRenders } from './findEmptyRenders.mjs';
 
 const CWD = process.cwd();
 const TAG = '[cms-ssg-build]';
@@ -157,6 +158,14 @@ async function main() {
     console.log(`${TAG} ~${estRoutes} routes ≤ shard size ${opts.shardSize}: running a single vite-ssg build (no sharding needed).`);
     const r = await runViteSsg({ heap: opts.heap, passthrough: opts.passthrough });
     process.stdout.write(r.out);
+    if (r.code === 0) {
+      const empties = await findEmptyRenders(path.join(CWD, 'dist'));
+      if (empties.length) {
+        console.error(`${TAG} build succeeded but ${empties.length} page(s) rendered with an empty body — a known intermittent SSR race (see findEmptyRenders in this file). Re-run the build:`);
+        for (const f of empties) console.error(`${TAG}   - ${f}`);
+        process.exit(1);
+      }
+    }
     process.exit(r.code);
   }
 
@@ -175,7 +184,20 @@ async function main() {
     const started = Date.now();
     process.stdout.write(`${TAG} shard ${s.k + 1}/${N} rendering…\n`);
     const r = await runViteSsg({ shard: `${s.k}/${N}`, outDir: s.outDir, heap: opts.heap, passthrough: opts.passthrough });
-    process.stdout.write(`${TAG} shard ${s.k + 1}/${N} ${r.code === 0 ? 'ok' : 'FAILED'} (${((Date.now() - started) / 1000).toFixed(1)}s)\n`);
+    let empties = [];
+    if (r.code === 0) {
+      empties = await findEmptyRenders(path.join(CWD, s.outDir));
+    }
+    const ok = r.code === 0 && empties.length === 0;
+    process.stdout.write(`${TAG} shard ${s.k + 1}/${N} ${ok ? 'ok' : 'FAILED'} (${((Date.now() - started) / 1000).toFixed(1)}s)\n`);
+    if (empties.length) {
+      return {
+        ...r,
+        code: 1,
+        out: `${r.out}\n${TAG} ${empties.length} page(s) rendered with an empty body — a known intermittent SSR race (see findEmptyRenders in this file):\n${empties.map((f) => `  - ${f}`).join('\n')}\n`,
+        k: s.k,
+      };
+    }
     return { ...r, k: s.k };
   });
 

package/src/utils/robotsGenerator.js

@@ -15,6 +15,16 @@
  *   - "Disallow: <prefix>" for each entry in site.draftPaths
  *   - "Disallow: <path>" for each page with draft === true
  *
+ * site.robots.allowAiCrawlers (opt-in, default off): appends a dedicated block
+ * per known AI/LLM crawler (GPTBot, ClaudeBot, PerplexityBot, CCBot, etc.),
+ * each carrying the exact same Disallow list as the wildcard block — i.e.
+ * identical effective access, just declared explicitly per bot. Several sites
+ * were hand-editing this list directly into public/robots.txt; since that file
+ * is regenerated on every build, the hand edits were silently reverted on the
+ * next `build:ssg`/`generate:public-assets` run. This option makes the
+ * customization durable. Skipped entirely when site.draft === true — a
+ * pre-launch site shouldn't explicitly invite crawlers.
+ *
  * If sitemapUrl is provided, appends a "Sitemap: <url>" line.
  */
 
@@ -22,18 +32,27 @@ import { normalizeDraftPath } from './draftMode.js';
 
 const FRAMEWORK_DEFAULTS = ['/admin'];
 
-export function buildRobotsTxt(siteConfig, sitemapUrl = '') {
-  const lines = ['User-agent: *'];
-
-  for (const p of FRAMEWORK_DEFAULTS) {
-    lines.push(`Disallow: ${p}`);
-  }
+// Order/grouping mirrors the hand-rolled lists sites had converged on before
+// this became a config option.
+const AI_CRAWLER_GROUPS = [
+  { comment: 'OpenAI / ChatGPT', agents: ['GPTBot', 'OAI-SearchBot', 'ChatGPT-User'] },
+  { comment: 'Anthropic / Claude', agents: ['anthropic-ai', 'ClaudeBot'] },
+  { comment: 'Perplexity', agents: ['PerplexityBot'] },
+  { comment: 'Common Crawl (used by many LLM training datasets)', agents: ['CCBot'] },
+  { comment: 'Google', agents: ['Googlebot', 'Googlebot-Extended'] },
+  { comment: 'Microsoft / Bing', agents: ['Bingbot'] },
+  { comment: 'Meta', agents: ['FacebookBot'] },
+  { comment: 'Diffbot', agents: ['Diffbot'] },
+];
 
+export function buildRobotsTxt(siteConfig, sitemapUrl = '') {
   const site = siteConfig?.site || {};
   const pages = siteConfig?.pages || {};
 
+  const disallows = [...FRAMEWORK_DEFAULTS];
+
   if (site.draft === true) {
-    lines.push('Disallow: /');
+    disallows.push('/');
   } else {
     const additional = new Set();
 
@@ -51,7 +70,25 @@ export function buildRobotsTxt(siteConfig, sitemapUrl = '') {
     }
 
     for (const p of Array.from(additional).sort()) {
-      lines.push(`Disallow: ${p}`);
+      disallows.push(p);
+    }
+  }
+
+  const lines = ['User-agent: *'];
+  for (const d of disallows) {
+    lines.push(`Disallow: ${d}`);
+  }
+
+  if (site.robots?.allowAiCrawlers === true && site.draft !== true) {
+    for (const group of AI_CRAWLER_GROUPS) {
+      lines.push('');
+      lines.push(`# ${group.comment}`);
+      for (const agent of group.agents) {
+        lines.push(`User-agent: ${agent}`);
+        for (const d of disallows) {
+          lines.push(`Disallow: ${d}`);
+        }
+      }
     }
   }