human-machine-swapper 1.1.0 → 1.2.1

package/README.md

@@ -22,6 +22,11 @@ and one custom element.
 npx human-machine-swapper init
 ```
 
+> Only here for the `llms.txt` and not the pill? [`npm create llms-txt`](https://www.npmjs.com/package/create-llms-txt)
+> writes the file from your routes and offers the pill at the end. Same code, different
+> front door.
+
+
 Detects your framework, writes a starter `llms.txt` if you have none, declares it, loads the
 component **pinned to a version with an integrity hash**, mounts the element, and adds the
 `:not(:defined)` guard below. Idempotent, never overwrites an existing `llms.txt`, and

package/bin/cli.mjs

@@ -6,6 +6,7 @@ import { fileURLToPath } from "node:url";
 
 import { detect, readSiteFacts } from "../src/init/detect.mjs";
 import { buildLlmsTxt, countTodos } from "../src/init/llms.mjs";
+import { findPages } from "../src/init/pages.mjs";
 import { apply, buildScriptTag, isJsxLayout, ELEMENT_TAG, LINK_TAG, CSS_GUARD } from "../src/init/apply.mjs";
 
 /**
@@ -117,7 +118,8 @@ Then make sure the site serves an llms.txt. Full docs: ${DOCS}`);
   }
 
   const facts = readSiteFacts(args.dir);
-  const llmsTxt = buildLlmsTxt(facts, { framework: framework.name });
+  const pages = findPages(args.dir, framework);
+  const llmsTxt = buildLlmsTxt(facts, { framework: framework.name, pages });
   const scriptTag = buildScriptTag({
     version: PKG.version,
     integrity: integrityOfSelf(),
@@ -125,7 +127,8 @@ Then make sure the site serves an llms.txt. Full docs: ${DOCS}`);
     jsx: isJsxLayout(framework.layout),
   });
 
-  console.log(`\n${C.b}${framework.name}${C.x}${args.dryRun ? `  ${C.y}(dry run, nothing written)${C.x}` : ""}\n`);
+  const pageNote = pages.length ? `  ${C.dim}${pages.length} page${pages.length === 1 ? "" : "s"} read${C.x}` : "";
+  console.log(`\n${C.b}${framework.name}${C.x}${pageNote}${args.dryRun ? `  ${C.y}(dry run, nothing written)${C.x}` : ""}\n`);
 
   const { actions, wrote } = apply({ dir: args.dir, framework, llmsTxt, scriptTag, dryRun: args.dryRun });
   report(actions);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "human-machine-swapper",
-  "version": "1.1.0",
+  "version": "1.2.1",
   "description": "A web component that adds a Human / Machine pill to any website. Machine shows the site's llms.txt, the plain-text reading written for language models, rendered in place. One file, about 6 KB gzipped, no dependencies and no build step.",
   "keywords": [
     "llms.txt",
@@ -34,6 +34,7 @@
   "jsdelivr": "src/human-machine-swapper.js",
   "exports": {
     ".": "./src/human-machine-swapper.js",
+    "./init": "./src/init/index.mjs",
     "./package.json": "./package.json"
   },
   "files": [

package/src/init/apply.mjs

@@ -73,18 +73,27 @@ function insertBefore(source, marker, snippet) {
   const at = source.indexOf(marker);
   if (at === -1) return null;
   const lineStart = source.lastIndexOf("\n", at) + 1;
-  // The marker is a CLOSING tag, so its own indentation is the parent's. Children sit one
-  // level in, and every line of a multi-line snippet gets the same treatment or the
-  // continuation lines land wherever they happened to be authored.
-  const indent = source.slice(lineStart, at).match(/^\s*/)[0] + "  ";
-  // Keep the snippet's OWN relative indentation: a multi-line tag authors its attributes
-  // one level in and its closing bracket flush, and flattening that puts `></script>`
+  const before = source.slice(lineStart, at);
+  const indent = before.match(/^\s*/)[0];
+
+  // Indent every line of the snippet one level in from the closing tag, keeping the
+  // snippet's own relative indentation so a multi-line tag's `></script>` does not end up
   // deeper than the tag it closes.
   const body = snippet
     .split("\n")
-    .map((l) => (l.trim() ? indent + l : l))
+    .map((l) => (l.trim() ? indent + "  " + l : l))
     .join("\n");
-  return source.slice(0, lineStart) + body + "\n" + source.slice(lineStart);
+
+  // Two shapes, and getting this wrong writes broken HTML into somebody's repo.
+  //
+  // When the closing tag starts its own line, insert a line above it. When it does NOT -
+  // `<head><meta /><title>x</title></head>` all on one line - inserting "before the line"
+  // puts the tags OUTSIDE the element entirely, above <head>. That is what happened to the
+  // first Astro site this ran on. Split the line instead and keep the content inside.
+  if (before.trim() === "") {
+    return source.slice(0, lineStart) + body + "\n" + source.slice(lineStart);
+  }
+  return source.slice(0, at) + "\n" + body + "\n" + indent + source.slice(at);
 }
 
 /** Already installed? Then there is nothing to do and we say so instead of doubling up. */

package/src/init/index.mjs

@@ -0,0 +1,11 @@
+/**
+ * The init toolkit, exported so a second front door can reuse it rather than fork it.
+ *
+ * `create-llms-txt` is the same machinery pointed at a different question: people search
+ * for "how do I add llms.txt", not for "human machine toggle". One codebase, two entry
+ * points, so the two can never drift into disagreeing about how to install this.
+ */
+export { detect, readSiteFacts } from "./detect.mjs";
+export { findPages } from "./pages.mjs";
+export { buildLlmsTxt, countTodos } from "./llms.mjs";
+export { apply, buildScriptTag, isJsxLayout, LINK_TAG, ELEMENT_TAG, CSS_GUARD } from "./apply.mjs";

package/src/init/llms.mjs

@@ -12,7 +12,7 @@
 
 const TODO = "TODO";
 
-export function buildLlmsTxt(facts = {}, { framework } = {}) {
+export function buildLlmsTxt(facts = {}, { framework, pages } = {}) {
   const name = facts.name || "This site";
   const summary = facts.description || `${TODO}: one sentence on what this site is.`;
 
@@ -30,7 +30,15 @@ export function buildLlmsTxt(facts = {}, { framework } = {}) {
   }
   lines.push("");
 
-  if (facts.headings && facts.headings.length) {
+  // The pages are the point. A file built from a README says what the project calls
+  // itself; one built from the routes says what is ON the site, which is what a model
+  // needs to answer "where do I find X".
+  if (pages && pages.length) {
+    lines.push("## Pages");
+    lines.push("");
+    for (const p of pages) lines.push(`- [${p.title}](${p.route})`);
+    lines.push("");
+  } else if (facts.headings && facts.headings.length) {
     lines.push("## Sections");
     lines.push("");
     for (const h of facts.headings) lines.push(`- ${h}`);

package/src/init/pages.mjs

@@ -0,0 +1,185 @@
+import { readdirSync, readFileSync, statSync } from "node:fs";
+import { basename, extname, join, relative, sep } from "node:path";
+
+/**
+ * Finds the pages a site actually has, so the generated llms.txt can list them.
+ *
+ * This is what makes a generated file worth keeping. An llms.txt built from a README says
+ * what the project calls itself; one built from the routes says what is *on the site*,
+ * which is the thing a model needs in order to answer "where do I find X".
+ *
+ * Read-only and best-effort throughout. A site with an unusual layout gets fewer pages
+ * listed, never a crash and never an invented route.
+ */
+
+const CODE = new Set([".js", ".jsx", ".ts", ".tsx", ".astro", ".vue", ".svelte"]);
+const PROSE = new Set([".md", ".mdx", ".markdown"]);
+const SKIP_DIRS = new Set([
+  "node_modules", ".git", ".next", ".nuxt", ".svelte-kit", "dist", "build", "out",
+  "coverage", ".vercel", ".astro", "public", "static", "assets",
+]);
+
+/** Hard ceiling. An llms.txt listing 400 routes is noise, not a machine reading. */
+const MAX_PAGES = 60;
+
+function walk(dir, root = dir, out = [], depth = 0) {
+  if (depth > 8) return out;
+  let entries;
+  try {
+    entries = readdirSync(dir, { withFileTypes: true });
+  } catch {
+    return out;
+  }
+  for (const e of entries) {
+    if (e.name.startsWith(".") && e.name !== ".") continue;
+    const abs = join(dir, e.name);
+    if (e.isDirectory()) {
+      if (SKIP_DIRS.has(e.name)) continue;
+      walk(abs, root, out, depth + 1);
+    } else {
+      out.push(relative(root, abs));
+    }
+  }
+  return out;
+}
+
+/** `app/blog/[slug]/page.tsx` -> `/blog/[slug]`, `app/page.tsx` -> `/` */
+function routeFromNextApp(rel) {
+  const parts = rel.split(sep);
+  parts.pop(); // page.tsx
+  const segs = parts
+    .slice(1) // drop app/ or src/
+    .filter((p) => p !== "app")
+    // (marketing) and @slots are organisational, not URL segments
+    .filter((p) => !(p.startsWith("(") && p.endsWith(")")) && !p.startsWith("@"));
+  return "/" + segs.join("/");
+}
+
+function routeFromFile(rel, baseDir) {
+  let p = rel.replace(new RegExp(`^${baseDir}\\${sep}?`), "");
+  p = p.replace(new RegExp(`\\${extname(p)}$`), "");
+  p = p.replace(/(^|\/)index$/, "");
+  p = p.replace(/\+page$/, "").replace(/\/$/, "");
+  return "/" + p.split(sep).filter(Boolean).join("/");
+}
+
+/**
+ * A page's title, from whatever the file happens to declare. Checked in order of how
+ * likely each is to be the real human-facing title.
+ */
+function titleOf(abs, route) {
+  let text = "";
+  try {
+    text = readFileSync(abs, "utf8").slice(0, 8000);
+  } catch {
+    return titleiseRoute(route);
+  }
+
+  const frontmatter = text.match(/^---\r?\n([\s\S]*?)\r?\n---/);
+  if (frontmatter) {
+    const t = frontmatter[1].match(/^title:\s*["']?(.+?)["']?\s*$/m);
+    if (t) return clean(t[1]);
+  }
+  // Next metadata: `title: "..."` inside an exported metadata object.
+  const meta = text.match(/export\s+const\s+metadata[\s\S]{0,400}?title:\s*["'`](.+?)["'`]/);
+  if (meta) return clean(meta[1]);
+
+  const h1 = text.match(/<h1[^>]*>([^<]{2,120})<\/h1>/i);
+  if (h1) return clean(h1[1]);
+
+  const md = text.match(/^#\s+(.{2,120})$/m);
+  if (md) return clean(md[1]);
+
+  return titleiseRoute(route);
+}
+
+const clean = (s) => s.replace(/\s+/g, " ").trim();
+
+function titleiseRoute(route) {
+  const last = route.split("/").filter(Boolean).pop();
+  if (!last) return "Home";
+  return last
+    .replace(/[[\]]/g, "")
+    .split(/[-_]/)
+    .map((w) => w.charAt(0).toUpperCase() + w.slice(1))
+    .join(" ");
+}
+
+/** Dynamic routes are a template, not a page; listing `/blog/[slug]` helps nobody. */
+const isDynamic = (route) => /\[.+\]|:\w+/.test(route);
+
+/**
+ * @returns {{route:string,title:string}[]} deduped, sorted, dynamic routes dropped
+ */
+export function findPages(dir, framework) {
+  if (!framework) return [];
+  const files = walk(dir);
+  const found = [];
+
+  const add = (route, abs) => {
+    if (!route || isDynamic(route)) return;
+    found.push({ route: route === "" ? "/" : route, abs });
+  };
+
+  for (const rel of files) {
+    const ext = extname(rel);
+    const abs = join(dir, rel);
+
+    switch (framework.id) {
+      case "next-app":
+        if (/(^|\/)page\.(js|jsx|ts|tsx)$/.test(rel.split(sep).join("/")) && /^(src\/)?app\//.test(rel.split(sep).join("/"))) {
+          add(routeFromNextApp(rel), abs);
+        }
+        break;
+      case "next-pages":
+        if (CODE.has(ext) && /^(src\/)?pages\//.test(rel.split(sep).join("/"))) {
+          const name = basename(rel);
+          if (name.startsWith("_") || rel.split(sep).join("/").includes("pages/api/")) break;
+          add(routeFromFile(rel.replace(/^src\//, ""), "pages"), abs);
+        }
+        break;
+      case "astro":
+        if ((CODE.has(ext) || PROSE.has(ext)) && rel.split(sep).join("/").startsWith("src/pages/")) {
+          add(routeFromFile(rel.replace(/^src\//, ""), "pages"), abs);
+        }
+        break;
+      case "sveltekit":
+        if (/\+page\.(svelte|md)$/.test(rel) && rel.split(sep).join("/").startsWith("src/routes/")) {
+          add(routeFromFile(rel.replace(/^src\//, ""), "routes"), abs);
+        }
+        break;
+      case "nuxt":
+        if (ext === ".vue" && rel.split(sep).join("/").startsWith("pages/")) {
+          add(routeFromFile(rel, "pages"), abs);
+        }
+        break;
+      case "html":
+        if (ext === ".html") add(routeFromFile(rel, ""), abs);
+        break;
+      default:
+        break;
+    }
+  }
+
+  // Markdown content collections are pages too on most of these stacks.
+  if (framework.id !== "html") {
+    for (const rel of files) {
+      const unix = rel.split(sep).join("/");
+      if (!PROSE.has(extname(rel))) continue;
+      if (!/^(src\/)?content\//.test(unix)) continue;
+      add("/" + unix.replace(/^(src\/)?content\//, "").replace(/\.[^.]+$/, ""), join(dir, rel));
+    }
+  }
+
+  const seen = new Set();
+  const pages = [];
+  for (const f of found) {
+    if (seen.has(f.route)) continue;
+    seen.add(f.route);
+    pages.push({ route: f.route, title: titleOf(f.abs, f.route) });
+    if (pages.length >= MAX_PAGES) break;
+  }
+
+  pages.sort((a, b) => (a.route === "/" ? -1 : b.route === "/" ? 1 : a.route.localeCompare(b.route)));
+  return pages;
+}