@rankcli/agent-runtime 0.0.36 → 0.0.37

package/dist/index.d.mts

@@ -3106,14 +3106,9 @@ declare function servedDir(framework: Framework | undefined, files: string[], ap
 
 interface ReplacementConcern {
     /** Machine-readable, for logs and the PR body. */
-    code: 'shrunk' | 'lost_closing_tag' | 'lost_script' | 'lost_stylesheet' | 'lost_verification' | 'lost_import' | 'invalid_json' | 'invalid_xml' | 'invalid_json_ld' | 'fewer_sitemap_urls' | 'lastmod_moved_backwards';
+    code: 'shrunk' | 'lost_closing_tag' | 'lost_script' | 'lost_stylesheet' | 'lost_verification' | 'lost_import' | 'invalid_json' | 'invalid_xml' | 'invalid_json_ld' | 'fewer_sitemap_urls' | 'lastmod_moved_backwards' | 'placeholder_content' | 'canonical_in_shared_shell';
     detail: string;
 }
-/**
- * Everything wrong with replacing `original` with `replacement` at `path`. Empty means safe
- * to commit. `original` is null for a file being created, where there is nothing to lose and
- * only the content's own validity matters.
- */
 declare function assessReplacement(path: string, original: string | null, replacement: string): ReplacementConcern[];
 /** One line naming why a file was dropped, for a log line or a PR body. */
 declare function describeConcerns(concerns: ReplacementConcern[]): string;

package/dist/index.d.ts

@@ -3106,14 +3106,9 @@ declare function servedDir(framework: Framework | undefined, files: string[], ap
 
 interface ReplacementConcern {
     /** Machine-readable, for logs and the PR body. */
-    code: 'shrunk' | 'lost_closing_tag' | 'lost_script' | 'lost_stylesheet' | 'lost_verification' | 'lost_import' | 'invalid_json' | 'invalid_xml' | 'invalid_json_ld' | 'fewer_sitemap_urls' | 'lastmod_moved_backwards';
+    code: 'shrunk' | 'lost_closing_tag' | 'lost_script' | 'lost_stylesheet' | 'lost_verification' | 'lost_import' | 'invalid_json' | 'invalid_xml' | 'invalid_json_ld' | 'fewer_sitemap_urls' | 'lastmod_moved_backwards' | 'placeholder_content' | 'canonical_in_shared_shell';
     detail: string;
 }
-/**
- * Everything wrong with replacing `original` with `replacement` at `path`. Empty means safe
- * to commit. `original` is null for a file being created, where there is nothing to lose and
- * only the content's own validity matters.
- */
 declare function assessReplacement(path: string, original: string | null, replacement: string): ReplacementConcern[];
 /** One line naming why a file was dropped, for a log line or a PR body. */
 declare function describeConcerns(concerns: ReplacementConcern[]): string;

package/dist/index.js

@@ -10401,6 +10401,121 @@ async function checkInternalRedirects(internalLinks, batchSize = 10) {
   return { redirectLinks };
 }
 
+// src/audit/checks/crawler-access.ts
+init_http();
+var CRAWLERS = [
+  {
+    name: "Googlebot",
+    userAgent: "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
+    matters: "Google Search indexing",
+    critical: true
+  },
+  {
+    name: "bingbot",
+    userAgent: "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)",
+    matters: "Bing indexing, and Copilot answers",
+    critical: true
+  },
+  {
+    name: "GPTBot",
+    userAgent: "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.1; +https://openai.com/gptbot)",
+    matters: "ChatGPT being able to read and cite the page",
+    critical: false
+  },
+  {
+    name: "ClaudeBot",
+    userAgent: "Mozilla/5.0 (compatible; ClaudeBot/1.0; +claudebot@anthropic.com)",
+    matters: "Claude being able to read and cite the page",
+    critical: false
+  },
+  {
+    name: "PerplexityBot",
+    userAgent: "Mozilla/5.0 (compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot)",
+    matters: "Perplexity being able to read and cite the page",
+    critical: false
+  }
+];
+var BROWSER_UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36";
+function isRefusal(status) {
+  return status === 401 || status === 403 || status === 429 || status === 503;
+}
+async function statusFor(url, userAgent) {
+  try {
+    const response = await httpGet(url, {
+      headers: { "User-Agent": userAgent },
+      timeout: 15e3,
+      // We want the status itself, not an exception on 4xx/5xx.
+      validateStatus: () => true
+    });
+    return response.status;
+  } catch {
+    return null;
+  }
+}
+async function analyzeCrawlerAccess(url) {
+  const issues = [];
+  const baselineStatus = await statusFor(url, BROWSER_UA);
+  const data = { baselineStatus, crawlers: [] };
+  if (baselineStatus === null || baselineStatus >= 400) {
+    return { issues, data };
+  }
+  const results = await Promise.all(
+    CRAWLERS.map(async (crawler) => ({
+      crawler,
+      status: await statusFor(url, crawler.userAgent)
+    }))
+  );
+  const blocked = results.filter(({ status }) => status !== null && isRefusal(status));
+  data.crawlers = results.map(({ crawler, status }) => ({
+    name: crawler.name,
+    status,
+    blocked: status !== null && isRefusal(status)
+  }));
+  if (blocked.length === 0) return { issues, data };
+  const search = blocked.filter(({ crawler }) => crawler.critical);
+  const ai = blocked.filter(({ crawler }) => !crawler.critical);
+  if (search.length > 0) {
+    const names = search.map(({ crawler }) => crawler.name).join(", ");
+    issues.push({
+      code: "SEARCH_CRAWLER_BLOCKED_AT_EDGE",
+      severity: "error",
+      category: "crawlability",
+      title: `${names} refused at the edge, not by robots.txt`,
+      description: `A browser request to this URL returned ${baselineStatus}, but ${names} received ${search.map(({ status }) => status).join(", ")}. Something in front of the origin - a CDN, WAF or bot rule - is turning the crawler away, and robots.txt cannot show you this.`,
+      impact: "Pages a search crawler cannot fetch fall out of the index. This is the failure mode behind Cloudflare\u2019s 2026-09-15 default, where blocking AI training also blocked Googlebot because it is classed as a mixed-use crawler.",
+      howToFix: `Check your CDN or WAF bot rules rather than robots.txt.
+
+On Cloudflare: the zone's Security > Bots settings, and any "Block AI Bots" or AI Crawler Control toggle. Because Googlebot crawls for both Search and AI training, blocking Training applies to all of its functions on ad-supported pages - allow Search crawlers explicitly, or opt out of the default.
+
+Confirm with Search Console's URL Inspection, which fetches as the verified Googlebot.`,
+      affectedUrls: [url],
+      details: {
+        baselineStatus,
+        blocked: search.map(({ crawler, status }) => ({ crawler: crawler.name, status })),
+        note: "Observed with the crawler\u2019s user-agent. Real crawlers are also verified by IP, so a site may treat them differently."
+      }
+    });
+  }
+  if (ai.length > 0) {
+    const names = ai.map(({ crawler }) => crawler.name).join(", ");
+    issues.push({
+      code: "AI_CRAWLER_BLOCKED_AT_EDGE",
+      severity: "warning",
+      category: "ai-readiness",
+      title: `${names} refused at the edge, not by robots.txt`,
+      description: `A browser request returned ${baselineStatus}, but ${names} received ${ai.map(({ status }) => status).join(", ")}. The block is in front of the origin, so allowing these agents in robots.txt will not change it.`,
+      impact: "An assistant that cannot fetch the page cannot quote it. This is invisible to a robots.txt check, which is where most AI-visibility tools stop.",
+      howToFix: `If blocking them is deliberate, nothing to do. If not, the rule is in your CDN or WAF, not robots.txt - on Cloudflare see Security > Bots and any AI Crawler Control setting.`,
+      affectedUrls: [url],
+      details: {
+        baselineStatus,
+        blocked: ai.map(({ crawler, status }) => ({ crawler: crawler.name, status }))
+      }
+    });
+  }
+  return { issues, data };
+}
+
 // src/audit/checks/ai-readiness.ts
 init_http();
 var cheerio13 = __toESM(require("cheerio"));
@@ -17708,6 +17823,7 @@ async function runFullAudit(options) {
   asyncChecks.push(safeAsyncIssues("Certificate", async () => (await checkCertificate(url)).issues));
   asyncChecks.push(safeAsync("Redirects", () => analyzeRedirects(url)));
   asyncChecks.push(safeAsync("Robots Validation", () => validateRobotsTxt(url)));
+  asyncChecks.push(safeAsyncIssues("Crawler Access", async () => (await analyzeCrawlerAccess(url)).issues));
   asyncChecks.push(safeAsync("Canonical Domain", () => analyzeCanonicalDomain(url)));
   asyncChecks.push(safeAsyncIssues("Framework Detection", async () => {
     const frameworkInfo = detectFramework({ html, headers });
@@ -33095,9 +33211,37 @@ function jsonLdConcerns(html) {
   }
   return bad;
 }
+var PLACEHOLDERS = [
+  { pattern: /\+?1[-\s.]?\(?800\)?[-\s.]?555[-\s.]?\d{4}/i, what: "a reserved 555 phone number" },
+  { pattern: /\b555[-\s.]?01\d{2}\b/, what: "a reserved 555 phone number" },
+  { pattern: /\bAnytown\b/i, what: '"Anytown"' },
+  { pattern: /\b123 Main (Street|St)\b/i, what: '"123 Main Street"' },
+  { pattern: /your_[a-z_]+/i, what: 'an unfilled "your_..." placeholder' },
+  { pattern: /\b(yourdomain|your-domain|example)\.(com|org|net)\b/i, what: "a placeholder domain" },
+  { pattern: /\blorem ipsum\b/i, what: "lorem ipsum" },
+  { pattern: /\bXXX-XXX-XXXX\b/i, what: "an unfilled phone template" }
+];
+function isSharedShell(html) {
+  return /<div[^>]+id=["'](root|app|__next|__nuxt)["']/i.test(html);
+}
 function assessReplacement(path3, original, replacement) {
   const concerns = [];
   const add = (code, detail) => concerns.push({ code, detail });
+  for (const { pattern, what } of PLACEHOLDERS) {
+    const hit = replacement.match(pattern);
+    if (hit && !(original && pattern.test(original))) {
+      add("placeholder_content", `contains ${what} (${hit[0]})`);
+    }
+  }
+  if (/\.html?$/i.test(path3) && isSharedShell(replacement)) {
+    const addsCanonical = /<link[^>]+rel=["']canonical["']/i.test(replacement) && !(original && /<link[^>]+rel=["']canonical["']/i.test(original));
+    if (addsCanonical) {
+      add(
+        "canonical_in_shared_shell",
+        `${path3} mounts a single-page app, so a canonical here applies to every route`
+      );
+    }
+  }
   if (/\.json$/i.test(path3) || /\.webmanifest$/i.test(path3)) {
     try {
       JSON.parse(replacement);

package/dist/index.mjs

@@ -6862,6 +6862,120 @@ async function checkInternalRedirects(internalLinks, batchSize = 10) {
   return { redirectLinks };
 }
 
+// src/audit/checks/crawler-access.ts
+var CRAWLERS = [
+  {
+    name: "Googlebot",
+    userAgent: "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
+    matters: "Google Search indexing",
+    critical: true
+  },
+  {
+    name: "bingbot",
+    userAgent: "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)",
+    matters: "Bing indexing, and Copilot answers",
+    critical: true
+  },
+  {
+    name: "GPTBot",
+    userAgent: "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.1; +https://openai.com/gptbot)",
+    matters: "ChatGPT being able to read and cite the page",
+    critical: false
+  },
+  {
+    name: "ClaudeBot",
+    userAgent: "Mozilla/5.0 (compatible; ClaudeBot/1.0; +claudebot@anthropic.com)",
+    matters: "Claude being able to read and cite the page",
+    critical: false
+  },
+  {
+    name: "PerplexityBot",
+    userAgent: "Mozilla/5.0 (compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot)",
+    matters: "Perplexity being able to read and cite the page",
+    critical: false
+  }
+];
+var BROWSER_UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36";
+function isRefusal(status) {
+  return status === 401 || status === 403 || status === 429 || status === 503;
+}
+async function statusFor(url, userAgent) {
+  try {
+    const response = await httpGet(url, {
+      headers: { "User-Agent": userAgent },
+      timeout: 15e3,
+      // We want the status itself, not an exception on 4xx/5xx.
+      validateStatus: () => true
+    });
+    return response.status;
+  } catch {
+    return null;
+  }
+}
+async function analyzeCrawlerAccess(url) {
+  const issues = [];
+  const baselineStatus = await statusFor(url, BROWSER_UA);
+  const data = { baselineStatus, crawlers: [] };
+  if (baselineStatus === null || baselineStatus >= 400) {
+    return { issues, data };
+  }
+  const results = await Promise.all(
+    CRAWLERS.map(async (crawler) => ({
+      crawler,
+      status: await statusFor(url, crawler.userAgent)
+    }))
+  );
+  const blocked = results.filter(({ status }) => status !== null && isRefusal(status));
+  data.crawlers = results.map(({ crawler, status }) => ({
+    name: crawler.name,
+    status,
+    blocked: status !== null && isRefusal(status)
+  }));
+  if (blocked.length === 0) return { issues, data };
+  const search = blocked.filter(({ crawler }) => crawler.critical);
+  const ai = blocked.filter(({ crawler }) => !crawler.critical);
+  if (search.length > 0) {
+    const names = search.map(({ crawler }) => crawler.name).join(", ");
+    issues.push({
+      code: "SEARCH_CRAWLER_BLOCKED_AT_EDGE",
+      severity: "error",
+      category: "crawlability",
+      title: `${names} refused at the edge, not by robots.txt`,
+      description: `A browser request to this URL returned ${baselineStatus}, but ${names} received ${search.map(({ status }) => status).join(", ")}. Something in front of the origin - a CDN, WAF or bot rule - is turning the crawler away, and robots.txt cannot show you this.`,
+      impact: "Pages a search crawler cannot fetch fall out of the index. This is the failure mode behind Cloudflare\u2019s 2026-09-15 default, where blocking AI training also blocked Googlebot because it is classed as a mixed-use crawler.",
+      howToFix: `Check your CDN or WAF bot rules rather than robots.txt.
+
+On Cloudflare: the zone's Security > Bots settings, and any "Block AI Bots" or AI Crawler Control toggle. Because Googlebot crawls for both Search and AI training, blocking Training applies to all of its functions on ad-supported pages - allow Search crawlers explicitly, or opt out of the default.
+
+Confirm with Search Console's URL Inspection, which fetches as the verified Googlebot.`,
+      affectedUrls: [url],
+      details: {
+        baselineStatus,
+        blocked: search.map(({ crawler, status }) => ({ crawler: crawler.name, status })),
+        note: "Observed with the crawler\u2019s user-agent. Real crawlers are also verified by IP, so a site may treat them differently."
+      }
+    });
+  }
+  if (ai.length > 0) {
+    const names = ai.map(({ crawler }) => crawler.name).join(", ");
+    issues.push({
+      code: "AI_CRAWLER_BLOCKED_AT_EDGE",
+      severity: "warning",
+      category: "ai-readiness",
+      title: `${names} refused at the edge, not by robots.txt`,
+      description: `A browser request returned ${baselineStatus}, but ${names} received ${ai.map(({ status }) => status).join(", ")}. The block is in front of the origin, so allowing these agents in robots.txt will not change it.`,
+      impact: "An assistant that cannot fetch the page cannot quote it. This is invisible to a robots.txt check, which is where most AI-visibility tools stop.",
+      howToFix: `If blocking them is deliberate, nothing to do. If not, the rule is in your CDN or WAF, not robots.txt - on Cloudflare see Security > Bots and any AI Crawler Control setting.`,
+      affectedUrls: [url],
+      details: {
+        baselineStatus,
+        blocked: ai.map(({ crawler, status }) => ({ crawler: crawler.name, status }))
+      }
+    });
+  }
+  return { issues, data };
+}
+
 // src/audit/checks/ai-readiness.ts
 import * as cheerio13 from "cheerio";
 
@@ -14151,6 +14265,7 @@ async function runFullAudit(options) {
   asyncChecks.push(safeAsyncIssues("Certificate", async () => (await checkCertificate(url)).issues));
   asyncChecks.push(safeAsync("Redirects", () => analyzeRedirects(url)));
   asyncChecks.push(safeAsync("Robots Validation", () => validateRobotsTxt(url)));
+  asyncChecks.push(safeAsyncIssues("Crawler Access", async () => (await analyzeCrawlerAccess(url)).issues));
   asyncChecks.push(safeAsync("Canonical Domain", () => analyzeCanonicalDomain(url)));
   asyncChecks.push(safeAsyncIssues("Framework Detection", async () => {
     const frameworkInfo = detectFramework({ html, headers });
@@ -29523,9 +29638,37 @@ function jsonLdConcerns(html) {
   }
   return bad;
 }
+var PLACEHOLDERS = [
+  { pattern: /\+?1[-\s.]?\(?800\)?[-\s.]?555[-\s.]?\d{4}/i, what: "a reserved 555 phone number" },
+  { pattern: /\b555[-\s.]?01\d{2}\b/, what: "a reserved 555 phone number" },
+  { pattern: /\bAnytown\b/i, what: '"Anytown"' },
+  { pattern: /\b123 Main (Street|St)\b/i, what: '"123 Main Street"' },
+  { pattern: /your_[a-z_]+/i, what: 'an unfilled "your_..." placeholder' },
+  { pattern: /\b(yourdomain|your-domain|example)\.(com|org|net)\b/i, what: "a placeholder domain" },
+  { pattern: /\blorem ipsum\b/i, what: "lorem ipsum" },
+  { pattern: /\bXXX-XXX-XXXX\b/i, what: "an unfilled phone template" }
+];
+function isSharedShell(html) {
+  return /<div[^>]+id=["'](root|app|__next|__nuxt)["']/i.test(html);
+}
 function assessReplacement(path3, original, replacement) {
   const concerns = [];
   const add = (code, detail) => concerns.push({ code, detail });
+  for (const { pattern, what } of PLACEHOLDERS) {
+    const hit = replacement.match(pattern);
+    if (hit && !(original && pattern.test(original))) {
+      add("placeholder_content", `contains ${what} (${hit[0]})`);
+    }
+  }
+  if (/\.html?$/i.test(path3) && isSharedShell(replacement)) {
+    const addsCanonical = /<link[^>]+rel=["']canonical["']/i.test(replacement) && !(original && /<link[^>]+rel=["']canonical["']/i.test(original));
+    if (addsCanonical) {
+      add(
+        "canonical_in_shared_shell",
+        `${path3} mounts a single-page app, so a canonical here applies to every route`
+      );
+    }
+  }
   if (/\.json$/i.test(path3) || /\.webmanifest$/i.test(path3)) {
     try {
       JSON.parse(replacement);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rankcli/agent-runtime",
-  "version": "0.0.36",
+  "version": "0.0.37",
   "description": "RankCLI agent runtime - executes SEO audits and fixes with AI",
   "homepage": "https://rankcli.dev",
   "main": "dist/index.js",
@@ -28,7 +28,8 @@
     "test": "vitest run",
     "test:watch": "vitest",
     "test:coverage": "vitest run --coverage",
-    "benchmark:detection": "node scripts/benchmark-detection.mjs"
+    "benchmark:detection": "node scripts/benchmark-detection.mjs",
+    "typecheck": "tsc --noEmit"
   },
   "dependencies": {
     "@anthropic-ai/sdk": "^0.52.0",