@stack-mode/site-manager-kit 1.44.1 → 1.46.0

package/README.md

@@ -194,6 +194,18 @@ export default function Robots() {
 }
 ```
 
+robots.txt also keeps out the AI crawlers the site blocks in Site Manager (Search Settings > Indexing): AI search tools,
+AI training crawlers, or both.
+
+`app/llms.txt/route.ts` (optional), the site's pages with their titles and descriptions for AI tools that read
+`/llms.txt`. It answers 404 while llms.txt is off for the site (Search Settings > Tools):
+
+```ts
+import { llmsRoute } from "site-manager-kit/llms-route";
+
+export const GET = llmsRoute();
+```
+
 `src/proxy.ts` (or `proxy.ts` at the root when there's no `src` folder):
 
 ```ts
@@ -497,7 +509,7 @@ It reads `BLUEPRINT_URL` and `BLUEPRINT_API_KEY` from `.env.local` (or `--env <f
 - **Wired**: the site reads it from Site Manager.
 - **Not wired**: Site Manager has it, but editing it changes nothing on the website (a key the site never asks for, a section that's on but never read, a Business Info field the code ignores). Fails the audit.
 - **Missing in Site Manager**: the site uses it, but Site Manager has no such field (a key not synced yet, a form key with no form). Fails the audit.
-- **Hard-coded**: the code keeps its own copy of a Site Manager value, with file and line. Reported, never failing, since a fallback can be on purpose.
+- **Built in** (status `hard_coded`): the code keeps its own copy of a Site Manager value, with file and line. Items the site also reads from Site Manager carry `fallback: true` and show as **Has a fallback**. Reported, never failing, since a fallback can be on purpose.
 - **Unknown**: the audit couldn't tell (the copy keys file didn't load, a section it doesn't know yet).
 
 Exit codes: 0 wired, 1 something Not wired or missing, 2 it couldn't run. Folders it shouldn't scan (an old admin, scripts) go in `package.json`:

package/audit/wiring-audit.d.mts

@@ -2,7 +2,8 @@
 
 export type AuditStatus = "wired" | "not_wired" | "missing" | "hard_coded" | "unknown" | "info";
 export type Where = { file: string; line: number };
-export type AuditItem = { area: string; status: AuditStatus; name: string; detail: string; fix: string | null; where: Where[]; more: number };
+/** `fallback`: a Built in (hard_coded) item the site reads from Site Manager, keeping its own copy only as a backup. */
+export type AuditItem = { area: string; status: AuditStatus; name: string; detail: string; fix: string | null; where: Where[]; more: number; fallback?: boolean };
 export type SiteFile = { rel: string; text: string; lines: string[]; usesKit: boolean };
 export type SiteFiles = { files: SiteFile[]; publicFiles: string[] };
 export type ApiAnswer = { status: number; data: any; meta: any; code: string | null };

package/audit/wiring-audit.mjs

@@ -14,7 +14,7 @@ export const STATUS = {
   wired: "Wired",
   not_wired: "Not wired",
   missing: "Missing in Site Manager",
-  hard_coded: "Hard-coded",
+  hard_coded: "Built in",
   unknown: "Unknown",
   info: "Note",
 };
@@ -279,7 +279,9 @@ export async function uploadResult(base, key, result, kitVersion, fetchImpl = fe
  */
 export async function audit({ site: siteFiles, api, declaredResult, keysFile, hasCopyKeysRoute }) {
   const items = [];
-  const add = (area, status, name, detail, fix = null, where = []) => items.push({ area, status, name, detail, fix, where: where.slice(0, MAX_WHERE), more: Math.max(0, where.length - MAX_WHERE) });
+  // `extra.fallback`: a Built in item the site does read from Site Manager, keeping its own copy only as a backup.
+  const add = (area, status, name, detail, fix = null, where = [], extra = {}) =>
+    items.push({ area, status, name, detail, fix, where: where.slice(0, MAX_WHERE), more: Math.max(0, where.length - MAX_WHERE), ...extra });
   const { files, publicFiles } = siteFiles;
 
   const siteRes = await api("/site");
@@ -347,7 +349,7 @@ export async function audit({ site: siteFiles, api, declaredResult, keysFile, ha
     const read = readsField(fact.read);
     const where = fact.pattern ? findAll(files, fact.pattern) : [];
     if (!read) add("Business Info", "not_wired", fact.name, `Set in Site Manager but the site never reads it${where.length ? "; the site shows its own copy instead" : ""}.`, "Read it from getSite().business (or the site's settings helper) and delete the built-in copy.", where);
-    else if (where.length) add("Business Info", "hard_coded", fact.name, "Read from Site Manager, but the code also keeps its own copy (a fallback or a second place that shows it). If the value is cleared or changed in Site Manager, these places can show the old one.", "Show the Site Manager value everywhere and drop the built-in copy, or keep it only as a fallback that's never shown when Site Manager has a value.", where);
+    else if (where.length) add("Business Info", "hard_coded", fact.name, "Read from Site Manager, but the code also keeps its own copy (a fallback or a second place that shows it). If the value is cleared or changed in Site Manager, these places can show the old one.", "Show the Site Manager value everywhere and drop the built-in copy, or keep it only as a fallback that's never shown when Site Manager has a value.", where, { fallback: true });
     else add("Business Info", "wired", fact.name, "Read from Site Manager.");
   }
 
@@ -419,7 +421,7 @@ export async function audit({ site: siteFiles, api, declaredResult, keysFile, ha
     const reads = readsModule(files, { names: l.reader.filter((n) => !n.startsWith("/")), paths: l.reader.filter((n) => n.startsWith("/")) });
     for (const name of l.names.filter((n) => n && n.trim().length >= 3)) {
       const where = findAll(files, literalPattern(name));
-      if (where.length) add(l.area, "hard_coded", name, `${reads ? "Read from Site Manager, but the" : "The"} code also has its own copy of this name${reads ? "" : " and never reads the list"}, so a rename, a new entry or a new order in Site Manager may not show.`, "Read the list (names and order) from Site Manager and drop the built-in list.", where);
+      if (where.length) add(l.area, "hard_coded", name, `${reads ? "Read from Site Manager, but the" : "The"} code also has its own copy of this name${reads ? "" : " and never reads the list"}, so a rename, a new entry or a new order in Site Manager may not show.`, "Read the list (names and order) from Site Manager and drop the built-in list.", where, reads ? { fallback: true } : {});
       else add(l.area, reads ? "wired" : "not_wired", name, reads ? "Read from Site Manager." : "In Site Manager, but the site never reads this list.", reads ? null : "Read the list from Site Manager.");
     }
   }
@@ -482,9 +484,9 @@ export function renderReport(result, meta) {
   out.push(`# Wiring audit: ${result.site.name}`, "");
   out.push(`Run ${meta.when} against ${meta.host} (site ${meta.siteId}), site kit ${meta.kitVersion}. Read only.`, "");
   out.push(failed ? "**Result: not wired.** Fix the Not wired and Missing in Site Manager items below." : "**Result: wired.** Nothing is Not wired or missing in Site Manager.", "");
-  out.push("| Wired | Not wired | Missing in Site Manager | Hard-coded | Unknown |", "| --- | --- | --- | --- | --- |");
+  out.push("| Wired | Not wired | Missing in Site Manager | Built in | Unknown |", "| --- | --- | --- | --- | --- |");
   out.push(`| ${counts.wired} | ${counts.not_wired} | ${counts.missing} | ${counts.hard_coded} | ${counts.unknown} |`, "");
-  out.push("Not wired: Site Manager has it, but editing it changes nothing on the website. Missing in Site Manager: the website uses it, but Site Manager has no such field. Hard-coded: the code keeps its own copy of a Site Manager value. Unknown: the audit couldn't tell.", "");
+  out.push("Not wired: Site Manager has it, but editing it changes nothing on the website. Missing in Site Manager: the website uses it, but Site Manager has no such field. Built in: the code keeps its own copy of a Site Manager value (Has a fallback: it reads Site Manager and keeps the copy as a backup). Unknown: the audit couldn't tell.", "");
   const order = ["not_wired", "missing", "hard_coded", "unknown", "info", "wired"];
   const areas = [...new Set(result.items.map((i) => i.area))];
   for (const area of areas) {
@@ -499,7 +501,7 @@ export function renderReport(result, meta) {
     rows.sort((a, b) => order.indexOf(a.status) - order.indexOf(b.status));
     for (const i of rows) {
       const where = i.where.map((w) => `${w.file}:${w.line}`).join("<br>") + (i.more ? `<br>and ${i.more} more` : "");
-      out.push(`| ${STATUS[i.status]} | ${cell(i.name)} | ${cell(i.detail)} | ${cell(i.fix ?? "")} | ${where} |`);
+      out.push(`| ${i.fallback ? "Has a fallback" : STATUS[i.status]} | ${cell(i.name)} | ${cell(i.detail)} | ${cell(i.fix ?? "")} | ${where} |`);
     }
     if (wiredHidden) out.push("", `Plus ${wiredHidden} wired (run with --all to list them).`);
     out.push("");
@@ -592,7 +594,7 @@ export async function main(argv) {
   writeFileSync(out, renderReport(result, meta));
   writeFileSync(out.replace(/\.md$/, "") + ".json", JSON.stringify({ ...meta, ...result }, null, 2));
   const { counts, failed } = summarize(result.items);
-  console.log(`Wiring audit for ${result.site.name}: ${counts.wired} wired, ${counts.not_wired} not wired, ${counts.missing} missing in Site Manager, ${counts.hard_coded} hard-coded, ${counts.unknown} unknown.`);
+  console.log(`Wiring audit for ${result.site.name}: ${counts.wired} wired, ${counts.not_wired} not wired, ${counts.missing} missing in Site Manager, ${counts.hard_coded} built in, ${counts.unknown} unknown.`);
   console.log(`Report: ${path.relative(process.cwd(), out) || out}`);
   if (args.upload) {
     const up = await uploadResult(env.BLUEPRINT_URL, env.BLUEPRINT_AUDIT_KEY, result, kitVersion);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@stack-mode/site-manager-kit",
-  "version": "1.44.1",
+  "version": "1.46.0",
   "description": "Reads a site's content from Site Manager and builds its metadata, sitemap, robots, redirects, forms and refreshes. For Next.js App Router sites on Vercel.",
   "license": "UNLICENSED",
   "repository": {
@@ -28,6 +28,7 @@
     "./attribution": "./src/attribution.tsx",
     "./phone": "./src/phone-input.tsx",
     "./rss-route": "./src/rss-route.ts",
+    "./llms-route": "./src/llms-route.ts",
     "./visual": "./src/visual.ts",
     "./visual-tags": "./src/visual-tags.ts",
     "./visual-accents": "./src/visual-accents.ts",

package/src/llms-route.ts

@@ -0,0 +1,45 @@
+import { BlueprintError, bp, tagsFor } from "./client";
+
+export type LlmsData = {
+  name: string;
+  summary: string | null;
+  sections: { title: string; pages: { url: string; title: string; description: string | null }[] }[];
+};
+
+const line = (s: string) => s.replace(/\s+/g, " ").trim();
+const linkText = (s: string) => line(s).replace(/[[\]]/g, "");
+
+/** Pure: the llms.txt text (llmstxt.org): the business, its summary, then each section's pages as links. */
+export function buildLlmsTxt(d: LlmsData): string {
+  const out = [`# ${line(d.name)}`, ""];
+  if (d.summary) out.push(`> ${line(d.summary)}`, "");
+  for (const s of d.sections) {
+    if (!s.pages.length) continue;
+    out.push(`## ${line(s.title)}`, "");
+    for (const p of s.pages) out.push(`- [${linkText(p.title)}](${p.url})${p.description ? `: ${line(p.description)}` : ""}`);
+    out.push("");
+  }
+  return out.join("\n");
+}
+
+/**
+ * /llms.txt, from the site's pages in Site Manager. Put in app/llms.txt/route.ts:
+ *
+ *   import { llmsRoute } from "site-manager-kit/llms-route";
+ *   export const GET = llmsRoute();
+ *
+ * Answers 404 when llms.txt is off for the site in Site Manager, or the Search section is.
+ */
+export function llmsRoute() {
+  return async function GET(): Promise<Response> {
+    let data: LlmsData | null;
+    try {
+      data = await bp<LlmsData>("/llms", { tags: tagsFor("seo") });
+    } catch (e) {
+      if (e instanceof BlueprintError && (e.code === "module_off" || e.code === "feature_off")) return new Response("Not found", { status: 404 });
+      throw e;
+    }
+    if (!data) return new Response("Not found", { status: 404 });
+    return new Response(buildLlmsTxt(data), { headers: { "content-type": "text/plain; charset=utf-8", "cache-control": "public, max-age=0, s-maxage=3600, stale-while-revalidate=86400" } });
+  };
+}

package/src/robots.ts

@@ -10,7 +10,11 @@ import type { Robots } from "./types";
 export function buildRobots(r: Robots | null, websiteUrl: string | null): MetadataRoute.Robots {
   if (!r || !r.indexable) return { rules: [{ userAgent: "*", disallow: "/" }] };
   return {
-    rules: [{ userAgent: "*", allow: "/", ...(r.disallow.length ? { disallow: r.disallow } : {}) }],
+    rules: [
+      { userAgent: "*", allow: "/", ...(r.disallow.length ? { disallow: r.disallow } : {}) },
+      // A crawler named in its own group ignores the "*" group, so the AI crawlers the site keeps out are listed apart.
+      ...(r.blockedBots?.length ? [{ userAgent: r.blockedBots, disallow: "/" }] : []),
+    ],
     ...(websiteUrl ? { sitemap: `${websiteUrl.replace(/\/+$/, "")}/sitemap.xml` } : {}),
   };
 }

package/src/types.ts

@@ -312,7 +312,8 @@ export type PageMeta = {
   maxImagePreview?: "large" | "standard" | "none";
 };
 export type SitemapEntry = { url: string; lastModified: string };
-export type Robots = { indexable: boolean; disallow: string[] };
+/** blockedBots: AI crawlers the site keeps out, by user agent (Site Manager 2026-11, optional for older servers). */
+export type Robots = { indexable: boolean; disallow: string[]; blockedBots?: string[] };
 export type Redirect = { from: string; to: string; status: 301 | 302 | 308 };
 
 // Real estate (Site Manager's Listings and Team sections).

package/src/version.ts

@@ -1 +1 @@
-export const KIT_VERSION = "1.44.1";
+export const KIT_VERSION = "1.46.0";