aeo-ready 1.4.1 → 1.6.0

package/CHANGELOG.md

@@ -1,5 +1,13 @@
 # Changelog
 
+## 1.5.0
+
+- Smart auto-fix engine — "Fix now?" actually fixes things
+- Auto-creates agents.txt, sitemap.md, patches robots.txt for AI bots, scaffolds llms.txt
+- Deduplicates overlapping failures across all 5 benchmarks into unified fix actions
+- Prints actionable manual instructions for server-config and platform-specific issues
+- Requires `--dir` for file-based fixes (won't blindly write to CWD)
+
 ## 1.4.1
 
 - Fix false "afdocs failed" message — `afdocs check` exits 1 when it finds failures, which is expected

package/README.md

@@ -88,7 +88,12 @@ With --dir: agentic-seo 92/100 (A)
   Fix now? [y/N]
 ```
 
-Say `y` and it runs `npx agentic-seo init` to scaffold missing files (llms.txt, AGENTS.md, skill.md), then `npx afdocs` for Fern issues. Non-interactive in CI (`--json` or non-TTY).
+Say `y` and aeo-ready analyzes failures across all 5 benchmarks, deduplicates overlapping issues, and fixes what it can:
+
+- **Auto-fixes** (with `--dir`): patches robots.txt for AI bots, creates agents.txt, generates sitemap.md, scaffolds llms.txt/AGENTS.md, adds missing pages to llms.txt
+- **Manual instructions**: prints actionable steps for server config (content negotiation, .md URLs, Vary header) and platform-specific issues
+
+Non-interactive in CI (`--json` or non-TTY).
 
 ## CI Mode
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "aeo-ready",
-  "version": "1.4.1",
+  "version": "1.6.0",
   "description": "AEO benchmark aggregator. One scan, every score. Collects agentic-seo, Cloudflare, Fern, Vercel, and AgentGrade in one report.",
   "type": "module",
   "bin": {
@@ -39,6 +39,7 @@
     "src/history/",
     "src/index.js",
     "src/scan.js",
+    "src/fix.js",
     "skills/",
     "README.md",
     "CHANGELOG.md"

package/src/benchmark/agentic-seo.js

@@ -12,21 +12,31 @@ const KNOWN_FILES = [
   "skill.md",
   "agent-permissions.json",
   "agents.json",
+  "agents.txt",
+  "openapi.json",
   "sitemap.xml",
   ".well-known/ai-plugin.json",
+  ".well-known/agent-card.json",
+  ".well-known/mcp/server-card.json",
+  ".well-known/agent-skills/index.json",
 ];
 
-async function fetchText(url) {
+async function fetchText(url, headers) {
   try {
-    const res = await fetch(url, { redirect: "follow" });
+    const res = await fetch(url, { redirect: "follow", headers });
     if (!res.ok) return null;
     return await res.text();
-  } catch (err) {
-    console.warn(`Warning: failed to fetch ${url}: ${err.message}`);
+  } catch {
     return null;
   }
 }
 
+function saveToDisk(tempDir, relativePath, content) {
+  const filePath = join(tempDir, relativePath);
+  mkdirSync(dirname(filePath), { recursive: true });
+  writeFileSync(filePath, content);
+}
+
 function parseSitemapUrls(xml, baseUrl) {
   const urls = [];
   const matches = xml.matchAll(/<loc>([^<]+)<\/loc>/g);
@@ -37,25 +47,71 @@ function parseSitemapUrls(xml, baseUrl) {
   return urls;
 }
 
-function urlToFilePath(url, baseUrl) {
+function urlToFilePath(url) {
   let path = new URL(url).pathname;
   if (path.endsWith("/")) path += "index.html";
   else if (!path.includes(".")) path += ".html";
   return path.replace(/^\//, "");
 }
 
-async function fetchSiteToDir(baseUrl) {
-  const tempDir = mkdtempSync(join(tmpdir(), "aeo-"));
+async function discoverSkillFiles(baseUrl, tempDir) {
+  const indexContent = await fetchText(
+    `${baseUrl}/.well-known/agent-skills/index.json`,
+  );
+  if (!indexContent) return;
 
-  for (const file of KNOWN_FILES) {
-    const content = await fetchText(`${baseUrl}/${file}`);
-    if (content) {
-      const filePath = join(tempDir, file);
-      mkdirSync(dirname(filePath), { recursive: true });
-      writeFileSync(filePath, content);
+  let index;
+  try {
+    index = JSON.parse(indexContent);
+  } catch {
+    return;
+  }
+
+  const skills = index.skills || index;
+  if (!Array.isArray(skills)) return;
+
+  const fetches = [];
+  for (const skill of skills) {
+    const dir = skill.path || skill.id;
+    if (!dir) continue;
+
+    const base = `.well-known/agent-skills/${dir}`;
+    fetches.push(
+      fetchText(`${baseUrl}/${base}/SKILL.md`).then((content) => {
+        if (content) saveToDisk(tempDir, `${base}/SKILL.md`, content);
+      }),
+    );
+
+    const refs = skill.references || [];
+    for (const ref of refs) {
+      const refPath = typeof ref === "string" ? ref : ref.path || ref.file;
+      if (!refPath) continue;
+      fetches.push(
+        fetchText(`${baseUrl}/${base}/references/${refPath}`).then(
+          (content) => {
+            if (content)
+              saveToDisk(tempDir, `${base}/references/${refPath}`, content);
+          },
+        ),
+      );
     }
   }
 
+  await Promise.all(fetches);
+}
+
+async function fetchSiteToDir(baseUrl) {
+  const tempDir = mkdtempSync(join(tmpdir(), "aeo-"));
+
+  await Promise.all(
+    KNOWN_FILES.map(async (file) => {
+      const content = await fetchText(`${baseUrl}/${file}`);
+      if (content) saveToDisk(tempDir, file, content);
+    }),
+  );
+
+  await discoverSkillFiles(baseUrl, tempDir);
+
   const sitemap = await fetchText(`${baseUrl}/sitemap.xml`);
   if (!sitemap) return tempDir;
 
@@ -63,25 +119,27 @@ async function fetchSiteToDir(baseUrl) {
 
   await Promise.all(
     urls.map(async (url) => {
-      const path = urlToFilePath(url, baseUrl);
+      const path = urlToFilePath(url);
       if (KNOWN_FILES.includes(path)) return;
 
-      const html = await fetchText(url);
-      if (html) {
-        const htmlPath = join(tempDir, path);
-        mkdirSync(dirname(htmlPath), { recursive: true });
-        writeFileSync(htmlPath, html);
-      }
-
-      const mdUrl = url
-        .replace(/\.html$/, ".md")
-        .replace(/\/?$/, (m) => (m === "/" ? "/index.md" : ".md"));
-      if (mdUrl !== url) {
-        const md = await fetchText(mdUrl);
-        if (md) {
-          const mdPath = join(tempDir, path.replace(/\.html$/, ".md"));
-          mkdirSync(dirname(mdPath), { recursive: true });
-          writeFileSync(mdPath, md);
+      const [html, md] = await Promise.all([
+        fetchText(url),
+        fetchText(url, { Accept: "text/markdown" }),
+      ]);
+
+      if (html) saveToDisk(tempDir, path, html);
+
+      if (md && md !== html) {
+        saveToDisk(tempDir, path.replace(/\.html$/, ".md"), md);
+      } else {
+        const mdUrl = url
+          .replace(/\.html$/, ".md")
+          .replace(/\/?$/, (m) => (m === "/" ? "/index.md" : ".md"));
+        if (mdUrl !== url) {
+          const mdFallback = await fetchText(mdUrl);
+          if (mdFallback) {
+            saveToDisk(tempDir, path.replace(/\.html$/, ".md"), mdFallback);
+          }
         }
       }
     }),

package/src/benchmark/index.js

@@ -8,24 +8,23 @@ import { runAgentgrade } from "./agentgrade.js";
 const REFERENCE_SCORES = {
   agenticSeo: {
     Stripe: 17,
-    Vercel: 48,
-    Supabase: 52,
-    Cloudflare: 55,
-    Average: 25,
+    Cloudflare: 20,
+    Supabase: 20,
+    Average: 19,
   },
   cloudflare: {
-    Stripe: 2,
-    Vercel: 4,
-    Supabase: 3,
-    Cloudflare: 5,
+    Supabase: 4,
+    Cloudflare: 3,
+    Vercel: 2,
+    Stripe: 1,
     Average: 2,
   },
   fern: {
-    Stripe: 85,
-    Vercel: 60,
-    Supabase: 78,
-    Anthropic: 72,
-    Average: 45,
+    Cloudflare: 85,
+    Stripe: 84,
+    Supabase: 82,
+    Vercel: 75,
+    Average: 55,
   },
 };
 

package/src/fix.js

@@ -0,0 +1,307 @@
+import chalk from "chalk";
+import { existsSync, readFileSync, writeFileSync, mkdirSync } from "fs";
+import { join } from "path";
+import { execFileSync } from "child_process";
+
+const AI_BOTS = [
+  "GPTBot",
+  "OAI-SearchBot",
+  "ClaudeBot",
+  "Claude-User",
+  "Claude-SearchBot",
+  "Google-Extended",
+  "CCBot",
+  "PerplexityBot",
+  "Meta-ExternalAgent",
+];
+
+const FIX_ACTIONS = [
+  {
+    key: "robots-ai-rules",
+    label: "Add AI bot rules to robots.txt",
+    auto: true,
+    match: (id) => /^robots\.txt$|robots.*blocked/i.test(id),
+    apply: patchRobotsTxt,
+  },
+  {
+    key: "llms-txt-bootstrap",
+    label: "Scaffold llms.txt and AGENTS.md",
+    auto: true,
+    match: () => false,
+    apply: bootstrapLlmsTxt,
+  },
+  {
+    key: "agents-txt",
+    label: "Create agents.txt",
+    auto: true,
+    match: (id) => /agents\.txt/i.test(id),
+    apply: createAgentsTxt,
+  },
+  {
+    key: "sitemap-md",
+    label: "Generate sitemap.md from sitemap.xml",
+    auto: true,
+    match: (id) => /^sitemap\.md$/i.test(id),
+    apply: generateSitemapMd,
+  },
+  {
+    key: "llms-txt-coverage",
+    label: "Add missing pages to llms.txt",
+    auto: true,
+    match: (id) => id === "llms-txt-coverage",
+    apply: patchLlmsTxtCoverage,
+  },
+  {
+    key: "llms-txt-directive",
+    label:
+      'Add <link rel="llms-txt" href="/llms.txt"> to your HTML <head> template',
+    auto: false,
+    match: (id) =>
+      /llms-txt-directive|llms.*linked.*html|llms-full.*linked/i.test(id),
+  },
+  {
+    key: "content-negotiation",
+    label:
+      "Configure server to return markdown for Accept: text/markdown requests",
+    auto: false,
+    match: (id) =>
+      /content.negotiation|agent ua.*markdown|accept.*markdown|accept.*text.*returns|accept.*json.*returns|preferred content.type/i.test(
+        id,
+      ),
+  },
+  {
+    key: "md-url-support",
+    label: "Serve markdown at .md URLs (e.g. /docs/page.md)",
+    auto: false,
+    match: (id) => /markdown.url.support|\.md.*url.*markdown/i.test(id),
+  },
+  {
+    key: "vary-header",
+    label: "Add Vary: Accept response header for content negotiation caching",
+    auto: false,
+    match: (id) => /^vary/i.test(id),
+  },
+  {
+    key: "content-start",
+    label: "Move nav/chrome below main content so agents find content earlier",
+    auto: false,
+    match: (id) => /content.start.position/i.test(id),
+  },
+  {
+    key: "markdown-parity",
+    label:
+      "Ensure markdown versions match HTML content (check tabbed/accordion sections)",
+    auto: false,
+    match: (id) => /markdown.content.parity/i.test(id),
+  },
+  {
+    key: "redirect",
+    label: "Avoid cross-host redirects — agents may not follow them",
+    auto: false,
+    match: (id) => /redirect behavior/i.test(id),
+  },
+  {
+    key: "json-ld",
+    label: "Add Organization JSON-LD to <head> — see schema.org/Organization",
+    auto: false,
+    match: (id) => /organization.*json.ld|json.ld.*organization/i.test(id),
+  },
+  {
+    key: "identity",
+    label:
+      "Identity protocols (WebFinger, DID, A2A Agent Card) — see agentgrade.com",
+    auto: false,
+    match: (id) =>
+      /webfinger|did document|nostr|at protocol|agent card|webmcp/i.test(id),
+  },
+];
+
+function collectFailedChecks(benchmarks) {
+  const failed = [];
+  for (const key of ["cloudflare", "fern", "vercel", "agentgrade"]) {
+    const b = benchmarks[key];
+    if (!b?.available || !b.checks) continue;
+    for (const check of b.checks) {
+      if (check.status === "fail" || check.status === "warn") {
+        failed.push({ ...check, benchmark: key });
+      }
+    }
+  }
+  return failed;
+}
+
+function dedup(failedChecks) {
+  const triggered = new Map();
+  for (const check of failedChecks) {
+    for (const action of FIX_ACTIONS) {
+      if (triggered.has(action.key)) continue;
+      if (action.match(check.id)) {
+        triggered.set(action.key, action);
+        break;
+      }
+    }
+  }
+  return triggered;
+}
+
+export async function runFixes(result, dir) {
+  const failed = collectFailedChecks(result.benchmarks);
+  if (failed.length === 0) {
+    console.log(chalk.green("\n  All checks passed!\n"));
+    return;
+  }
+
+  const triggered = dedup(failed);
+
+  if (!triggered.has("llms-txt-bootstrap") && dir) {
+    const llms = join(dir, "llms.txt");
+    const agents = join(dir, "AGENTS.md");
+    if (!existsSync(llms) || !existsSync(agents)) {
+      triggered.set("llms-txt-bootstrap", FIX_ACTIONS[1]);
+    }
+  }
+
+  const fixed = [];
+  const skipped = [];
+  const manual = [];
+
+  for (const [, action] of triggered) {
+    if (!action.auto) {
+      manual.push(action.label);
+      continue;
+    }
+    if (!dir) {
+      skipped.push(action.label);
+      continue;
+    }
+    const r = await action.apply(dir, result);
+    if (r) fixed.push(r);
+  }
+
+  console.log("");
+  if (fixed.length > 0) {
+    console.log(
+      chalk.bold(
+        `  Fixed ${fixed.length} issue${fixed.length > 1 ? "s" : ""}:\n`,
+      ),
+    );
+    for (const f of fixed) {
+      console.log(`    ${chalk.green("✓")} ${f}`);
+    }
+    console.log("");
+  }
+
+  if (skipped.length > 0) {
+    console.log(
+      chalk.bold("  Skipped") + chalk.dim(" (run with --dir to auto-fix):\n"),
+    );
+    for (const s of skipped) {
+      console.log(`    ${chalk.yellow("-")} ${s}`);
+    }
+    console.log("");
+  }
+
+  if (manual.length > 0) {
+    console.log(chalk.bold("  Manual fixes needed:\n"));
+    manual.forEach((m, i) => {
+      console.log(`    ${chalk.dim(`${i + 1}.`)} ${m}`);
+    });
+    console.log("");
+  }
+
+  const rescan = `npx aeo-ready scan ${result.url}${dir ? ` --dir ${dir}` : ""}`;
+  if (fixed.length > 0) {
+    console.log(chalk.dim("  Deploy, then re-scan: ") + rescan + "\n");
+  } else {
+    console.log(chalk.dim("  Re-scan to verify: ") + rescan + "\n");
+  }
+}
+
+function patchRobotsTxt(dir) {
+  const file = join(dir, "robots.txt");
+  let content = "";
+  if (existsSync(file)) {
+    content = readFileSync(file, "utf8");
+  }
+
+  const missing = AI_BOTS.filter(
+    (bot) => !content.toLowerCase().includes(bot.toLowerCase()),
+  );
+  if (missing.length === 0) return null;
+
+  const block = missing
+    .map((bot) => `User-agent: ${bot}\nAllow: /`)
+    .join("\n\n");
+  const sep =
+    content.length > 0 && !content.endsWith("\n")
+      ? "\n\n"
+      : content.length > 0
+        ? "\n"
+        : "";
+  writeFileSync(file, content + sep + block + "\n");
+  return `Added ${missing.length} AI bot rules to robots.txt`;
+}
+
+function bootstrapLlmsTxt(dir) {
+  const llms = join(dir, "llms.txt");
+  const agents = join(dir, "AGENTS.md");
+  if (existsSync(llms) && existsSync(agents)) return null;
+
+  try {
+    execFileSync("npx", ["agentic-seo", "init", dir], {
+      timeout: 30000,
+      stdio: ["pipe", "pipe", "pipe"],
+    });
+    return "Scaffolded llms.txt and AGENTS.md via agentic-seo";
+  } catch (err) {
+    console.log(
+      chalk.red(`    agentic-seo init failed: ${err.message?.slice(0, 80)}`),
+    );
+    return null;
+  }
+}
+
+function createAgentsTxt(dir) {
+  const file = join(dir, "agents.txt");
+  if (existsSync(file)) return null;
+
+  writeFileSync(file, "# agents.txt\nUser-agent: *\nAllow: /\n");
+  return "Created agents.txt with default permissions";
+}
+
+function generateSitemapMd(dir) {
+  const file = join(dir, "sitemap.md");
+  if (existsSync(file)) return null;
+
+  const xmlFile = join(dir, "sitemap.xml");
+  if (!existsSync(xmlFile)) return null;
+
+  const xml = readFileSync(xmlFile, "utf8");
+  const urls = [...xml.matchAll(/<loc>([^<]+)<\/loc>/g)].map((m) => m[1]);
+  if (urls.length === 0) return null;
+
+  const lines = ["# Sitemap\n", ...urls.map((u) => `- [${u}](${u})`)];
+  writeFileSync(file, lines.join("\n") + "\n");
+  return `Generated sitemap.md with ${urls.length} URLs`;
+}
+
+function patchLlmsTxtCoverage(dir, result) {
+  const llmsFile = join(dir, "llms.txt");
+  const xmlFile = join(dir, "sitemap.xml");
+  if (!existsSync(llmsFile) || !existsSync(xmlFile)) return null;
+
+  const llms = readFileSync(llmsFile, "utf8");
+  const xml = readFileSync(xmlFile, "utf8");
+  const sitemapUrls = [...xml.matchAll(/<loc>([^<]+)<\/loc>/g)].map(
+    (m) => m[1],
+  );
+
+  const missing = sitemapUrls.filter((u) => !llms.includes(u));
+  if (missing.length === 0) return null;
+
+  const block =
+    "\n## Additional Pages\n\n" +
+    missing.map((u) => `- [${u}](${u})`).join("\n");
+  writeFileSync(llmsFile, llms.trimEnd() + "\n" + block + "\n");
+  return `Added ${missing.length} missing pages to llms.txt`;
+}

package/src/scan.js

@@ -1,8 +1,8 @@
 import chalk from "chalk";
 import { createInterface } from "readline";
-import { execFileSync } from "child_process";
 import { runAllBenchmarks, printBenchmarks } from "./benchmark/index.js";
 import { saveResult } from "./history/index.js";
+import { runFixes } from "./fix.js";
 
 export async function scan(opts) {
   const { url, dir, json } = opts;
@@ -144,39 +144,5 @@ function ask(question) {
 async function promptFix(result, dir) {
   const answer = await ask(chalk.bold("  Fix now? ") + chalk.dim("[y/N] "));
   if (answer !== "y" && answer !== "yes") return;
-
-  console.log("");
-
-  const targetDir = dir || ".";
-
-  console.log(chalk.dim(`  Running: npx agentic-seo init ${targetDir}\n`));
-  try {
-    execFileSync("npx", ["agentic-seo", "init", targetDir], {
-      stdio: "inherit",
-    });
-  } catch (err) {
-    console.log(chalk.red(`\n  agentic-seo init failed: ${err.message}\n`));
-  }
-
-  const fernFails =
-    result.benchmarks.fern?.checks?.filter(
-      (c) => c.status === "fail" || c.status === "warn",
-    ) || [];
-  if (fernFails.length > 0) {
-    console.log(chalk.dim(`\n  Running: npx afdocs check ${result.url}\n`));
-    try {
-      execFileSync("npx", ["afdocs", "check", result.url], {
-        stdio: "inherit",
-      });
-    } catch (err) {
-      if (err.status !== 1) {
-        console.log(chalk.red(`\n  afdocs failed: ${err.message}\n`));
-      }
-    }
-  }
-
-  console.log(
-    chalk.dim("\n  Re-scan to verify: ") +
-      `npx aeo-ready scan ${result.url}${dir ? ` --dir ${dir}` : ""}\n`,
-  );
+  await runFixes(result, dir);
 }