geo-ai-search-optimization 2.0.0 → 2.2.0

package/src/config.js

@@ -0,0 +1,105 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+
+const CONFIG_FILE_NAMES = [".georc.json", "geo.config.json"];
+
+async function findConfigFile(startDir) {
+  let dir = path.resolve(startDir || ".");
+
+  for (let i = 0; i < 10; i++) {
+    for (const name of CONFIG_FILE_NAMES) {
+      const candidate = path.join(dir, name);
+      try {
+        await fs.access(candidate);
+        return candidate;
+      } catch {
+        // continue
+      }
+    }
+
+    const parent = path.dirname(dir);
+    if (parent === dir) break;
+    dir = parent;
+  }
+
+  return null;
+}
+
+export async function loadConfig(options = {}) {
+  const configPath = options.configPath || await findConfigFile(options.startDir || ".");
+
+  if (!configPath) {
+    return { _source: null, _found: false };
+  }
+
+  try {
+    const content = await fs.readFile(configPath, "utf8");
+    const config = JSON.parse(content);
+    return { ...config, _source: configPath, _found: true };
+  } catch (err) {
+    throw new Error(`Failed to read config at ${configPath}: ${err.message}`);
+  }
+}
+
+export async function initConfig(options = {}) {
+  const targetDir = path.resolve(options.targetDir || ".");
+  const outputPath = path.join(targetDir, ".georc.json");
+
+  try {
+    await fs.access(outputPath);
+    if (!options.overwrite) {
+      throw new Error(`Config already exists at ${outputPath}. Use --overwrite to replace.`);
+    }
+  } catch (err) {
+    if (err.message.includes("already exists")) throw err;
+  }
+
+  const config = {
+    $schema: "https://geo-ai-search-optimization.dev/schema/georc.json",
+    site: {
+      name: options.siteName || null,
+      url: options.siteUrl || null
+    },
+    audit: {
+      minScore: options.minScore || 40,
+      maxFileSize: 1_000_000,
+      maxExamples: 5
+    },
+    ci: {
+      minScore: options.minScore || 40,
+      failOnRegression: false,
+      baselineDir: ".geo-data"
+    },
+    crawlers: {
+      strategy: "open"
+    },
+    output: {
+      format: "markdown",
+      dataDir: ".geo-data"
+    },
+    plugins: []
+  };
+
+  await fs.mkdir(targetDir, { recursive: true });
+  await fs.writeFile(outputPath, `${JSON.stringify(config, null, 2)}\n`, "utf8");
+
+  return { outputPath, config };
+}
+
+export function mergeConfigWithOptions(config, cliOptions) {
+  const merged = { ...cliOptions };
+
+  if (config._found) {
+    if (config.site?.url && !merged.siteUrl) merged.siteUrl = config.site.url;
+    if (config.site?.name && !merged.siteName) merged.siteName = config.site.name;
+    if (config.audit?.minScore !== undefined && merged.minScore === undefined) merged.minScore = config.audit.minScore;
+    if (config.audit?.maxFileSize !== undefined && merged.maxFileSize === undefined) merged.maxFileSize = config.audit.maxFileSize;
+    if (config.audit?.maxExamples !== undefined && merged.maxExamples === undefined) merged.maxExamples = config.audit.maxExamples;
+    if (config.ci?.failOnRegression && merged.failOnRegression === undefined) merged.failOnRegression = config.ci.failOnRegression;
+    if (config.output?.format && !merged.format) merged.format = config.output.format;
+    if (config.output?.dataDir && !merged.dataDir) merged.dataDir = config.output.dataDir;
+    if (config.crawlers?.strategy && !merged.strategy) merged.strategy = config.crawlers.strategy;
+  }
+
+  return merged;
+}

package/src/crawlers.js

@@ -0,0 +1,286 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+import { writeScanOutput } from "./scan.js";
+
+const KNOWN_AI_CRAWLERS = [
+  { name: "GPTBot", owner: "OpenAI", engine: "ChatGPT", purpose: "Training & search" },
+  { name: "ChatGPT-User", owner: "OpenAI", engine: "ChatGPT", purpose: "Live browsing" },
+  { name: "OAI-SearchBot", owner: "OpenAI", engine: "ChatGPT Search", purpose: "Search results" },
+  { name: "Google-Extended", owner: "Google", engine: "Gemini / Bard", purpose: "Training" },
+  { name: "Googlebot", owner: "Google", engine: "Google AI Overviews", purpose: "Indexing & AIO" },
+  { name: "anthropic-ai", owner: "Anthropic", engine: "Claude", purpose: "Training" },
+  { name: "ClaudeBot", owner: "Anthropic", engine: "Claude", purpose: "Web access" },
+  { name: "PerplexityBot", owner: "Perplexity", engine: "Perplexity", purpose: "Search results" },
+  { name: "Bytespider", owner: "ByteDance", engine: "Doubao / TikTok", purpose: "Training" },
+  { name: "CCBot", owner: "Common Crawl", engine: "Multiple LLMs", purpose: "Training data" },
+  { name: "cohere-ai", owner: "Cohere", engine: "Cohere", purpose: "Training" },
+  { name: "meta-externalagent", owner: "Meta", engine: "Meta AI", purpose: "Training" },
+  { name: "Applebot-Extended", owner: "Apple", engine: "Apple Intelligence", purpose: "Training" },
+  { name: "FacebookBot", owner: "Meta", engine: "Meta AI", purpose: "Training & search" },
+  { name: "Amazonbot", owner: "Amazon", engine: "Alexa / Rufus", purpose: "Search & shopping" }
+];
+
+function parseRobotsTxt(content) {
+  const lines = content.split("\n").map((l) => l.trim());
+  const blocks = [];
+  let current = null;
+
+  for (const line of lines) {
+    if (!line || line.startsWith("#")) continue;
+
+    const uaMatch = line.match(/^user-agent:\s*(.+)$/i);
+    if (uaMatch) {
+      current = { userAgent: uaMatch[1].trim(), rules: [] };
+      blocks.push(current);
+      continue;
+    }
+
+    if (!current) continue;
+
+    const allowMatch = line.match(/^allow:\s*(.+)$/i);
+    if (allowMatch) {
+      current.rules.push({ type: "allow", path: allowMatch[1].trim() });
+      continue;
+    }
+
+    const disallowMatch = line.match(/^disallow:\s*(.+)$/i);
+    if (disallowMatch) {
+      current.rules.push({ type: "disallow", path: disallowMatch[1].trim() });
+      continue;
+    }
+
+    const crawlDelayMatch = line.match(/^crawl-delay:\s*(\d+)$/i);
+    if (crawlDelayMatch) {
+      current.rules.push({ type: "crawl-delay", value: Number.parseInt(crawlDelayMatch[1], 10) });
+    }
+  }
+
+  return blocks;
+}
+
+function analyzeCrawlerAccess(blocks) {
+  const results = [];
+
+  for (const crawler of KNOWN_AI_CRAWLERS) {
+    const specificBlock = blocks.find(
+      (b) => b.userAgent.toLowerCase() === crawler.name.toLowerCase()
+    );
+    const wildcardBlock = blocks.find((b) => b.userAgent === "*");
+    const activeBlock = specificBlock || wildcardBlock;
+
+    let status = "allowed";
+    let reason = "No specific rules found; defaults to allowed.";
+    let matchedAgent = null;
+
+    if (specificBlock) {
+      matchedAgent = specificBlock.userAgent;
+      const hasDisallow = specificBlock.rules.some(
+        (r) => r.type === "disallow" && (r.path === "/" || r.path === "")
+      );
+      const hasAllow = specificBlock.rules.some(
+        (r) => r.type === "allow" && r.path === "/"
+      );
+
+      if (hasDisallow && !hasAllow) {
+        status = "blocked";
+        reason = `Explicitly blocked via User-agent: ${matchedAgent}`;
+      } else if (hasDisallow && hasAllow) {
+        status = "partial";
+        reason = `Mixed rules for User-agent: ${matchedAgent}`;
+      } else {
+        status = "allowed";
+        reason = `Explicitly allowed via User-agent: ${matchedAgent}`;
+      }
+    } else if (wildcardBlock) {
+      matchedAgent = "*";
+      const hasDisallow = wildcardBlock.rules.some(
+        (r) => r.type === "disallow" && r.path === "/"
+      );
+      if (hasDisallow) {
+        status = "blocked";
+        reason = "Blocked by wildcard User-agent: * Disallow: /";
+      }
+    }
+
+    const partialPaths = activeBlock
+      ? activeBlock.rules.filter((r) => r.type === "disallow" && r.path !== "/" && r.path !== "").map((r) => r.path)
+      : [];
+
+    results.push({
+      crawler: crawler.name,
+      owner: crawler.owner,
+      engine: crawler.engine,
+      purpose: crawler.purpose,
+      status,
+      reason,
+      matchedAgent,
+      blockedPaths: partialPaths
+    });
+  }
+
+  return results;
+}
+
+function computeCrawlerScore(crawlerResults) {
+  const total = crawlerResults.length;
+  if (total === 0) return 100;
+
+  let points = 0;
+  const keyEngines = ["GPTBot", "ChatGPT-User", "Googlebot", "PerplexityBot", "ClaudeBot"];
+
+  for (const result of crawlerResults) {
+    const isKey = keyEngines.includes(result.crawler);
+    const weight = isKey ? 2 : 1;
+
+    if (result.status === "allowed") points += 10 * weight;
+    else if (result.status === "partial") points += 5 * weight;
+  }
+
+  const maxPoints = crawlerResults.reduce((sum, r) => {
+    const isKey = keyEngines.includes(r.crawler);
+    return sum + 10 * (isKey ? 2 : 1);
+  }, 0);
+
+  return Math.round((points / maxPoints) * 100);
+}
+
+function generateOptimalRobotsTxt(crawlerResults, options = {}) {
+  const strategy = options.strategy || "open";
+  const lines = ["# AI Crawler Rules", `# Strategy: ${strategy}`, `# Generated by geo-ai-search-optimization`, ""];
+
+  if (strategy === "open") {
+    lines.push("# Allow all AI crawlers for maximum AI visibility");
+    for (const crawler of KNOWN_AI_CRAWLERS) {
+      lines.push(`User-agent: ${crawler.name}`);
+      lines.push("Allow: /");
+      lines.push("");
+    }
+  } else if (strategy === "selective") {
+    const allowList = options.allow || ["GPTBot", "ChatGPT-User", "Googlebot", "PerplexityBot", "ClaudeBot"];
+    lines.push("# Allow key AI crawlers, block training-only bots");
+    for (const crawler of KNOWN_AI_CRAWLERS) {
+      const isAllowed = allowList.some((a) => a.toLowerCase() === crawler.name.toLowerCase());
+      lines.push(`User-agent: ${crawler.name}`);
+      lines.push(isAllowed ? "Allow: /" : "Disallow: /");
+      lines.push("");
+    }
+  } else if (strategy === "block-all") {
+    lines.push("# Block all known AI crawlers");
+    for (const crawler of KNOWN_AI_CRAWLERS) {
+      lines.push(`User-agent: ${crawler.name}`);
+      lines.push("Disallow: /");
+      lines.push("");
+    }
+  }
+
+  return lines.join("\n");
+}
+
+async function fetchRobotsTxt(url) {
+  const parsedUrl = new URL(url);
+  const robotsUrl = `${parsedUrl.protocol}//${parsedUrl.host}/robots.txt`;
+  const response = await fetch(robotsUrl, {
+    redirect: "follow",
+    headers: { "user-agent": "geo-ai-search-optimization/2.2.0" },
+    signal: AbortSignal.timeout(10_000)
+  });
+  if (!response.ok) {
+    return { found: false, url: robotsUrl, status: response.status, content: "" };
+  }
+  const content = await response.text();
+  return { found: true, url: robotsUrl, status: response.status, content };
+}
+
+export async function analyzeCrawlers(input, options = {}) {
+  let robotsContent = "";
+  let source = "";
+
+  if (/^https?:\/\//i.test(input)) {
+    const result = await fetchRobotsTxt(input);
+    robotsContent = result.content;
+    source = result.url;
+    if (!result.found) {
+      return {
+        kind: "geo-crawlers",
+        source,
+        found: false,
+        status: result.status,
+        crawlers: KNOWN_AI_CRAWLERS.map((c) => ({
+          ...c, status: "allowed", reason: "No robots.txt found; all crawlers allowed by default.",
+          matchedAgent: null, blockedPaths: []
+        })),
+        score: 100,
+        summary: "No robots.txt found. All AI crawlers have unrestricted access by default.",
+        recommendation: generateOptimalRobotsTxt([], options)
+      };
+    }
+  } else {
+    const filePath = path.resolve(input);
+    robotsContent = await fs.readFile(filePath, "utf8");
+    source = filePath;
+  }
+
+  const blocks = parseRobotsTxt(robotsContent);
+  const crawlers = analyzeCrawlerAccess(blocks);
+  const score = computeCrawlerScore(crawlers);
+  const blocked = crawlers.filter((c) => c.status === "blocked");
+  const partial = crawlers.filter((c) => c.status === "partial");
+
+  let summary;
+  if (blocked.length === 0 && partial.length === 0) {
+    summary = "All known AI crawlers have full access. Maximum AI visibility.";
+  } else if (blocked.length > 0) {
+    summary = `${blocked.length} AI crawler(s) blocked: ${blocked.map((c) => c.crawler).join(", ")}. This may reduce visibility in ${blocked.map((c) => c.engine).join(", ")}.`;
+  } else {
+    summary = `${partial.length} AI crawler(s) have partial access. Some content may not be indexed.`;
+  }
+
+  return {
+    kind: "geo-crawlers",
+    source,
+    found: true,
+    totalBlocks: blocks.length,
+    crawlers,
+    score,
+    blocked: blocked.map((c) => c.crawler),
+    partial: partial.map((c) => c.crawler),
+    summary,
+    recommendation: generateOptimalRobotsTxt(crawlers, options)
+  };
+}
+
+export function renderCrawlersMarkdown(report) {
+  const lines = [
+    "# AI Crawler Access Analysis",
+    "",
+    `- Source: \`${report.source}\``,
+    `- robots.txt found: \`${report.found}\``,
+    `- AI Visibility Score: \`${report.score}/100\``,
+    `- Summary: ${report.summary}`,
+    "",
+    "## Crawler Status Matrix",
+    "",
+    "| Crawler | Owner | Engine | Status | Reason |",
+    "|---------|-------|--------|--------|--------|"
+  ];
+
+  for (const c of report.crawlers) {
+    const statusIcon = c.status === "allowed" ? "✅" : c.status === "blocked" ? "🚫" : "⚠️";
+    lines.push(`| ${c.crawler} | ${c.owner} | ${c.engine} | ${statusIcon} ${c.status} | ${c.reason} |`);
+  }
+
+  if (report.blocked && report.blocked.length > 0) {
+    lines.push("", "## Blocked Crawlers", "");
+    for (const name of report.blocked) {
+      const c = report.crawlers.find((cr) => cr.crawler === name);
+      lines.push(`- **${name}** (${c.engine}): ${c.reason}`);
+    }
+  }
+
+  lines.push("", "## Recommended robots.txt AI Section", "", "```", report.recommendation.trim(), "```", "");
+  return lines.join("\n");
+}
+
+export async function writeCrawlersOutput(outputPath, content) {
+  return writeScanOutput(outputPath, content);
+}

package/src/diagnose.js

@@ -0,0 +1,221 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+import { writeScanOutput } from "./scan.js";
+import { fullPageAudit } from "./full-page-audit.js";
+import { fullAudit } from "./full-audit.js";
+
+/**
+ * Smart diagnose: auto-detects input type and runs the right analyses.
+ * - URL → full-page-audit (12 dimensions)
+ * - Directory → full-audit (project + infra)
+ * - HTML/MD file → full-page-audit (12 dimensions)
+ */
+
+async function detectInputType(input) {
+  if (/^https?:\/\//i.test(input)) return "url";
+
+  const resolved = path.resolve(input);
+  try {
+    const stat = await fs.stat(resolved);
+    if (stat.isDirectory()) return "directory";
+    if (stat.isFile()) return "file";
+  } catch {
+    // Not found
+  }
+
+  return "unknown";
+}
+
+function buildDiagnosisSummary(type, result) {
+  if (type === "directory") {
+    const score = result.overallScore;
+    const issues = result.actionPlan?.length || 0;
+    return `Project diagnosis complete. GEO Score: ${score}/100. ${issues} action item(s).`;
+  }
+
+  const score = result.compositeScore;
+  const weakDims = Object.entries(result.dimensions)
+    .filter(([, d]) => d.score < 40)
+    .map(([k]) => k);
+
+  if (weakDims.length === 0) {
+    return `Page diagnosis complete. Composite: ${score}/100. All dimensions healthy.`;
+  }
+
+  return `Page diagnosis complete. Composite: ${score}/100. Weak: ${weakDims.join(", ")}.`;
+}
+
+function buildPrioritizedActions(type, result) {
+  const actions = [];
+
+  if (type === "directory") {
+    // From full-audit action plan
+    for (const task of (result.actionPlan || []).slice(0, 10)) {
+      actions.push({
+        priority: task.priority,
+        action: task.action,
+        owner: task.owner || "Engineering",
+        source: task.source || "project-audit"
+      });
+    }
+    return actions;
+  }
+
+  // Page-level: sort recommendations by dimension weakness
+  const dimOrder = Object.entries(result.dimensions)
+    .sort((a, b) => a[1].score - b[1].score);
+
+  for (const [dim, data] of dimOrder) {
+    if (data.score >= 80) continue;
+
+    const detail = result.details?.[dim];
+    const recs = detail?.recommendations || [];
+    for (const rec of recs.slice(0, 2)) {
+      const recText = typeof rec === "string" ? rec : rec.rec || rec.action || JSON.stringify(rec);
+      actions.push({
+        priority: data.score < 30 ? "P0" : data.score < 50 ? "P1" : "P2",
+        action: recText,
+        owner: dim === "security" ? "Engineering" : dim === "eeat" || dim === "citability" ? "Content" : "SEO",
+        source: dim
+      });
+    }
+  }
+
+  return actions.slice(0, 15);
+}
+
+function buildQuickWins(type, result) {
+  const wins = [];
+
+  if (type !== "directory") {
+    const d = result.dimensions;
+    if (d.socialMeta?.score < 50) wins.push("Add Open Graph and Twitter Card meta tags for better social sharing and AI discoverability.");
+    if (d.headingStructure?.score < 40) wins.push("Add question-style H2 headings (e.g., 'What is...?', 'How to...?') for AI answer extraction.");
+    if (d.freshness?.score < 40) wins.push("Add datePublished and dateModified to your JSON-LD structured data.");
+    if (d.schema?.score < 40) wins.push("Add JSON-LD structured data. Run: geo-ai-search-optimization auto-fix <url>");
+    if (d.readability?.score < 40) wins.push("Simplify sentence structure. Target 15-20 word average sentence length.");
+  } else {
+    if (result.scores?.crawlers < 50) wins.push("Review robots.txt AI crawler rules. Run: geo-ai-search-optimization crawlers <url>");
+    if (result.scores?.llmsValidation < 50) wins.push("Create or fix llms.txt. Run: geo-ai-search-optimization init-llms");
+    if (result.scores?.base < 40) wins.push("Fix foundational GEO signals. Run: geo-ai-search-optimization audit <path>");
+  }
+
+  return wins.slice(0, 5);
+}
+
+export async function diagnose(input, options = {}) {
+  const type = await detectInputType(input);
+
+  if (type === "unknown") {
+    throw new Error(`Cannot diagnose: "${input}" is not a valid URL, file, or directory.`);
+  }
+
+  let result;
+  let analysisType;
+
+  if (type === "directory") {
+    result = await fullAudit(input, options);
+    analysisType = "project";
+  } else {
+    result = await fullPageAudit(input, options);
+    analysisType = "page";
+  }
+
+  const summary = buildDiagnosisSummary(type, result);
+  const actions = buildPrioritizedActions(type, result);
+  const quickWins = buildQuickWins(type, result);
+  const score = type === "directory" ? result.overallScore : result.compositeScore;
+
+  return {
+    kind: "geo-diagnose",
+    input,
+    inputType: type,
+    analysisType,
+    score,
+    scoreLabel: score >= 80 ? "Strong" : score >= 60 ? "Moderate" : score >= 40 ? "Weak" : "Critical",
+    summary,
+    quickWins,
+    actions,
+    nextCommands: buildNextCommands(type, result),
+    result, // Full underlying result for JSON consumers
+  };
+}
+
+function buildNextCommands(type, result) {
+  const cmds = [];
+
+  if (type !== "directory") {
+    cmds.push({ cmd: "auto-fix", args: "<url>", reason: "Generate ready-to-use fix code" });
+    cmds.push({ cmd: "full-page-audit", args: "<url> --save", reason: "Save snapshot for trend tracking" });
+    cmds.push({ cmd: "page-trend", args: "<url>", reason: "View score history" });
+  } else {
+    cmds.push({ cmd: "full-audit", args: "<path> --sample-urls <url1,url2>", reason: "Add page sampling for deeper analysis" });
+    cmds.push({ cmd: "trend", args: "", reason: "View project score trend" });
+  }
+
+  cmds.push({ cmd: "benchmark", args: "<url> --competitors <url1,url2>", reason: "Compare against competitors" });
+
+  return cmds;
+}
+
+export function renderDiagnoseMarkdown(report) {
+  const lines = [
+    "# GEO Diagnosis",
+    "",
+    `- Input: \`${report.input}\``,
+    `- Type: \`${report.inputType}\` → \`${report.analysisType}\` analysis`,
+    `- **Score: \`${report.score}/100\` (${report.scoreLabel})**`,
+    `- Summary: ${report.summary}`,
+    ""
+  ];
+
+  if (report.analysisType === "page" && report.result.dimensions) {
+    lines.push("## Dimension Scores", "");
+    const dimLabels = {
+      base: "Base", citability: "Citability", eeat: "E-E-A-T", readability: "Readability",
+      headingStructure: "Headings", internalLinks: "Links", socialMeta: "Social",
+      platformReady: "Platforms", schema: "Schema", freshness: "Freshness",
+      security: "Security", topics: "Topics"
+    };
+
+    const sorted = Object.entries(report.result.dimensions)
+      .sort((a, b) => a[1].score - b[1].score);
+
+    for (const [key, dim] of sorted) {
+      const icon = dim.score >= 70 ? "🟢" : dim.score >= 40 ? "🟡" : "🔴";
+      lines.push(`- ${icon} **${dimLabels[key] || key}**: ${dim.score}/100`);
+    }
+    lines.push("");
+  }
+
+  if (report.quickWins.length > 0) {
+    lines.push("## Quick Wins", "");
+    for (const win of report.quickWins) {
+      lines.push(`- ⚡ ${win}`);
+    }
+    lines.push("");
+  }
+
+  if (report.actions.length > 0) {
+    lines.push("## Prioritized Actions", "");
+    for (const action of report.actions) {
+      const badge = action.priority === "P0" ? "🔴" : action.priority === "P1" ? "🟠" : "🟡";
+      lines.push(`- ${badge} **${action.priority}** [${action.source}] ${action.action}`);
+    }
+    lines.push("");
+  }
+
+  if (report.nextCommands.length > 0) {
+    lines.push("## Suggested Next Commands", "");
+    for (const cmd of report.nextCommands) {
+      lines.push(`- \`geo-ai-search-optimization ${cmd.cmd} ${cmd.args}\` — ${cmd.reason}`);
+    }
+    lines.push("");
+  }
+
+  return lines.join("\n");
+}
+
+export async function writeDiagnoseOutput(outputPath, content) {
+  return writeScanOutput(outputPath, content);
+}