geo-ai-search-optimization 2.0.0 → 2.2.0

package/src/topics.js

@@ -0,0 +1,275 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+import { writeScanOutput } from "./scan.js";
+
+const STOP_WORDS = new Set([
+  "a", "an", "the", "and", "or", "but", "in", "on", "at", "to", "for", "of", "with",
+  "by", "from", "as", "is", "are", "was", "were", "be", "been", "being", "have", "has",
+  "had", "do", "does", "did", "will", "would", "could", "should", "may", "might", "can",
+  "shall", "it", "its", "this", "that", "these", "those", "i", "we", "you", "he", "she",
+  "they", "them", "their", "my", "your", "our", "his", "her", "not", "no", "if", "so",
+  "up", "out", "just", "also", "more", "most", "very", "than", "then", "only", "all",
+  "about", "into", "over", "after", "before", "between", "through", "during", "each",
+  "any", "some", "such", "what", "which", "who", "how", "when", "where", "there",
+  "here", "both", "other", "new", "one", "two", "first", "last", "get", "use", "make",
+  "like", "well", "back", "even", "still", "way", "take", "come", "see", "know", "need"
+]);
+
+function stripHtml(text) {
+  return text
+    .replace(/<script[\s\S]*?<\/script>/gi, " ")
+    .replace(/<style[\s\S]*?<\/style>/gi, " ")
+    .replace(/<[^>]+>/g, " ")
+    .replace(/&[a-z0-9#]+;/gi, " ")
+    .replace(/[^\w\s-]/g, " ")
+    .replace(/\s+/g, " ")
+    .trim();
+}
+
+function extractPlainText(content) {
+  if (/<html|<head|<body/i.test(content)) return stripHtml(content);
+  return content.replace(/^---[\s\S]*?---/, "").replace(/[^\w\s-]/g, " ").replace(/\s+/g, " ").trim();
+}
+
+function tokenize(text) {
+  return text.toLowerCase().split(/\s+/).filter((w) => w.length > 2 && !STOP_WORDS.has(w) && !/^\d+$/.test(w));
+}
+
+function computeTermFrequency(tokens) {
+  const freq = {};
+  for (const token of tokens) {
+    freq[token] = (freq[token] || 0) + 1;
+  }
+  return freq;
+}
+
+function extractNgrams(tokens, n) {
+  const ngrams = {};
+  for (let i = 0; i <= tokens.length - n; i++) {
+    const gram = tokens.slice(i, i + n).join(" ");
+    // Skip if any token is a stop word
+    if (n > 1 && tokens.slice(i, i + n).some((t) => STOP_WORDS.has(t))) continue;
+    ngrams[gram] = (ngrams[gram] || 0) + 1;
+  }
+  return ngrams;
+}
+
+function computeTfIdf(termFreq, totalTokens) {
+  if (totalTokens === 0) return [];
+  // Simplified single-document TF-IDF using log normalization
+  const results = [];
+  for (const [term, count] of Object.entries(termFreq)) {
+    const tf = count / totalTokens;
+    // Approximate IDF using inverse document frequency heuristic
+    // More frequent = less interesting; rare terms get boosted
+    const idf = Math.log(1 + totalTokens / (count * 10));
+    const tfidf = tf * idf;
+    results.push({ term, count, tf: Math.round(tf * 10000) / 10000, tfidf: Math.round(tfidf * 10000) / 10000 });
+  }
+  return results.sort((a, b) => b.tfidf - a.tfidf);
+}
+
+function extractHeadingTopics(content) {
+  const headings = [];
+  const mdMatches = content.matchAll(/^#{1,6}\s+(.+)$/gm);
+  for (const match of mdMatches) headings.push(match[1].trim());
+  const htmlMatches = content.matchAll(/<h[1-6][^>]*>([\s\S]*?)<\/h[1-6]>/gim);
+  for (const match of htmlMatches) headings.push(match[1].replace(/<[^>]+>/g, "").trim());
+  return headings;
+}
+
+function identifyTopicClusters(keywords, bigrams) {
+  const clusters = {};
+
+  // Group by shared root or co-occurrence
+  for (const kw of keywords.slice(0, 30)) {
+    const root = kw.term.length > 5 ? kw.term.slice(0, Math.ceil(kw.term.length * 0.7)) : kw.term;
+    const related = keywords.filter((k) =>
+      k.term !== kw.term && (k.term.startsWith(root) || kw.term.startsWith(k.term.slice(0, Math.ceil(k.term.length * 0.7))))
+    );
+
+    if (related.length > 0) {
+      const clusterKey = kw.term;
+      if (!clusters[clusterKey]) {
+        clusters[clusterKey] = {
+          primary: kw.term,
+          related: related.map((r) => r.term).slice(0, 5),
+          totalFreq: kw.count + related.reduce((sum, r) => sum + r.count, 0)
+        };
+      }
+    }
+  }
+
+  // Add bigram-based topics
+  for (const bg of bigrams.slice(0, 15)) {
+    clusters[bg.term] = clusters[bg.term] || {
+      primary: bg.term,
+      related: [],
+      totalFreq: bg.count
+    };
+  }
+
+  return Object.values(clusters)
+    .sort((a, b) => b.totalFreq - a.totalFreq)
+    .slice(0, 10);
+}
+
+function buildRecommendations(keywords, bigrams, headingTopics, clusters) {
+  const recs = [];
+
+  if (keywords.length < 5) {
+    recs.push("Content lacks distinct topic signals. Add more domain-specific terminology.");
+  }
+
+  // Check if top keywords appear in headings
+  const headingText = headingTopics.join(" ").toLowerCase();
+  const topTerms = keywords.slice(0, 5);
+  const missingInHeadings = topTerms.filter((kw) => !headingText.includes(kw.term));
+  if (missingInHeadings.length > 0) {
+    recs.push(`Top keywords not in headings: ${missingInHeadings.map((k) => `"${k.term}"`).join(", ")}. Consider using them in H2/H3.`);
+  }
+
+  if (clusters.length < 3) {
+    recs.push("Content covers few topic clusters. Broaden to cover related subtopics for topical authority.");
+  }
+
+  if (bigrams.length > 0 && bigrams[0].count > 5) {
+    recs.push(`Dominant phrase: "${bigrams[0].term}" (${bigrams[0].count}x). Ensure this aligns with your target topic.`);
+  }
+
+  return recs;
+}
+
+async function fetchContent(url) {
+  const response = await fetch(url, {
+    redirect: "follow",
+    headers: { "user-agent": "geo-ai-search-optimization/2.2.0" },
+    signal: AbortSignal.timeout(10_000)
+  });
+  if (!response.ok) throw new Error(`Failed to fetch: ${url} (status ${response.status})`);
+  return response.text();
+}
+
+export async function analyzeTopics(input, options = {}) {
+  let rawContent;
+  let source;
+
+  if (/^https?:\/\//i.test(input)) {
+    rawContent = await fetchContent(input);
+    source = input;
+  } else {
+    const filePath = path.resolve(input);
+    rawContent = await fs.readFile(filePath, "utf8");
+    source = filePath;
+  }
+
+  const plainText = extractPlainText(rawContent);
+  const tokens = tokenize(plainText);
+  const termFreq = computeTermFrequency(tokens);
+  const tfidf = computeTfIdf(termFreq, tokens.length);
+  const bigrams = Object.entries(extractNgrams(tokens, 2))
+    .map(([term, count]) => ({ term, count }))
+    .filter((bg) => bg.count >= 2)
+    .sort((a, b) => b.count - a.count);
+  const trigrams = Object.entries(extractNgrams(tokens, 3))
+    .map(([term, count]) => ({ term, count }))
+    .filter((tg) => tg.count >= 2)
+    .sort((a, b) => b.count - a.count);
+
+  const headingTopics = extractHeadingTopics(rawContent);
+  const clusters = identifyTopicClusters(tfidf, bigrams);
+
+  const keywords = tfidf.slice(0, 20);
+  const recommendations = buildRecommendations(keywords, bigrams, headingTopics, clusters);
+
+  // Topic coverage score
+  let score = 0;
+  score += Math.min(keywords.length * 3, 30);
+  score += Math.min(bigrams.length * 3, 20);
+  score += Math.min(clusters.length * 8, 25);
+  score += Math.min(headingTopics.length * 3, 15);
+  score += tokens.length >= 200 ? 10 : (tokens.length >= 100 ? 5 : 0);
+  score = Math.max(0, Math.min(100, score));
+
+  return {
+    kind: "geo-topics",
+    source,
+    wordCount: tokens.length,
+    uniqueTerms: Object.keys(termFreq).length,
+    keywords: keywords.map((k) => ({ term: k.term, count: k.count, tfidf: k.tfidf })),
+    bigrams: bigrams.slice(0, 15),
+    trigrams: trigrams.slice(0, 10),
+    headingTopics,
+    topicClusters: clusters,
+    score,
+    scoreLabel: score >= 70 ? "Rich" : score >= 40 ? "Moderate" : "Thin",
+    recommendations,
+    summary: `${keywords.length} keywords extracted. Top: ${keywords.slice(0, 5).map((k) => `"${k.term}"`).join(", ")}. ${clusters.length} topic cluster(s) identified. Score: ${score}/100.`
+  };
+}
+
+export function renderTopicsMarkdown(report) {
+  const lines = [
+    "# Topic & Keyword Analysis",
+    "",
+    `- Source: \`${report.source}\``,
+    `- Total tokens: \`${report.wordCount}\``,
+    `- Unique terms: \`${report.uniqueTerms}\``,
+    `- Summary: ${report.summary}`,
+    "",
+    "## Top Keywords (TF-IDF)",
+    "",
+    "| Rank | Keyword | Count | TF-IDF |",
+    "|------|---------|-------|--------|"
+  ];
+
+  for (let i = 0; i < report.keywords.length; i++) {
+    const kw = report.keywords[i];
+    lines.push(`| ${i + 1} | ${kw.term} | ${kw.count} | ${kw.tfidf} |`);
+  }
+
+  if (report.bigrams.length > 0) {
+    lines.push("", "## Top Bigrams", "");
+    for (const bg of report.bigrams.slice(0, 10)) {
+      lines.push(`- "${bg.term}" (${bg.count}x)`);
+    }
+  }
+
+  if (report.trigrams.length > 0) {
+    lines.push("", "## Top Trigrams", "");
+    for (const tg of report.trigrams.slice(0, 5)) {
+      lines.push(`- "${tg.term}" (${tg.count}x)`);
+    }
+  }
+
+  if (report.topicClusters.length > 0) {
+    lines.push("", "## Topic Clusters", "");
+    for (const cluster of report.topicClusters) {
+      const related = cluster.related.length > 0 ? ` → ${cluster.related.join(", ")}` : "";
+      lines.push(`- **${cluster.primary}** (freq: ${cluster.totalFreq})${related}`);
+    }
+  }
+
+  if (report.headingTopics.length > 0) {
+    lines.push("", "## Heading Topics", "");
+    for (const h of report.headingTopics) {
+      lines.push(`- ${h}`);
+    }
+  }
+
+  lines.push("", "## Recommendations", "");
+  if (report.recommendations.length === 0) {
+    lines.push("- Topic coverage looks good.");
+  } else {
+    for (const rec of report.recommendations) {
+      lines.push(`- ${rec}`);
+    }
+  }
+  lines.push("");
+
+  return lines.join("\n");
+}
+
+export async function writeTopicsOutput(outputPath, content) {
+  return writeScanOutput(outputPath, content);
+}

package/src/url-onboarding.js

@@ -26,7 +26,7 @@ async function fetchText(url, options = {}) {
     const response = await fetch(url, {
       redirect: "follow",
       headers: {
-        "user-agent": "geo-ai-search-optimization/1.0.6"
+        "user-agent": "geo-ai-search-optimization/2.2.0"
       }
     });
 

package/src/validate-llms.js

@@ -0,0 +1,307 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+import { writeScanOutput } from "./scan.js";
+
+function parseLlmsTxt(content) {
+  const lines = content.split("\n");
+  const sections = [];
+  let currentSection = null;
+  let title = null;
+  let description = null;
+  let urls = [];
+
+  for (const line of lines) {
+    const trimmed = line.trim();
+
+    if (trimmed.startsWith("# ") && !title) {
+      title = trimmed.slice(2).trim();
+      continue;
+    }
+
+    if (trimmed.startsWith("> ") && !description) {
+      description = trimmed.slice(2).trim();
+      continue;
+    }
+
+    if (trimmed.startsWith("## ")) {
+      if (currentSection) sections.push(currentSection);
+      currentSection = { heading: trimmed.slice(3).trim(), lines: [], urls: [] };
+      continue;
+    }
+
+    if (currentSection && trimmed) {
+      currentSection.lines.push(trimmed);
+      const urlMatch = trimmed.match(/https?:\/\/[^\s)>]+/g);
+      if (urlMatch) {
+        urls.push(...urlMatch);
+        currentSection.urls.push(...urlMatch);
+      }
+    }
+  }
+
+  if (currentSection) sections.push(currentSection);
+
+  return { title, description, sections, urls };
+}
+
+const REQUIRED_SECTIONS = ["About", "Priority URLs"];
+const RECOMMENDED_SECTIONS = ["Recommended Sources", "AI Assistant Guidance", "Update Policy"];
+
+function validateStructure(parsed) {
+  const issues = [];
+  const warnings = [];
+  const sectionNames = parsed.sections.map((s) => s.heading.toLowerCase());
+
+  if (!parsed.title) {
+    issues.push({ severity: "error", message: "Missing title (first # heading)" });
+  }
+
+  if (!parsed.description) {
+    warnings.push({ severity: "warning", message: "Missing description (> blockquote after title)" });
+  }
+
+  for (const required of REQUIRED_SECTIONS) {
+    if (!sectionNames.includes(required.toLowerCase())) {
+      issues.push({ severity: "error", message: `Missing required section: ## ${required}` });
+    }
+  }
+
+  for (const recommended of RECOMMENDED_SECTIONS) {
+    if (!sectionNames.includes(recommended.toLowerCase())) {
+      warnings.push({ severity: "warning", message: `Missing recommended section: ## ${recommended}` });
+    }
+  }
+
+  const aboutSection = parsed.sections.find((s) => s.heading.toLowerCase() === "about");
+  if (aboutSection && aboutSection.lines.length === 0) {
+    issues.push({ severity: "error", message: "About section is empty" });
+  }
+
+  const prioritySection = parsed.sections.find((s) => s.heading.toLowerCase() === "priority urls");
+  if (prioritySection && prioritySection.urls.length === 0) {
+    issues.push({ severity: "error", message: "Priority URLs section contains no URLs" });
+  }
+
+  return { issues, warnings };
+}
+
+function validateUrls(urls) {
+  const results = [];
+
+  for (const url of urls) {
+    try {
+      const parsed = new URL(url);
+      const isHttps = parsed.protocol === "https:";
+      results.push({
+        url,
+        valid: true,
+        https: isHttps,
+        warning: isHttps ? null : "URL uses HTTP instead of HTTPS"
+      });
+    } catch {
+      results.push({ url, valid: false, https: false, warning: "Invalid URL format" });
+    }
+  }
+
+  return results;
+}
+
+function validateContent(parsed) {
+  const issues = [];
+  const lines = parsed.sections.flatMap((s) => s.lines);
+  const fullText = lines.join(" ");
+
+  const placeholders = fullText.match(/\b(example\.com|your[- ]site|your[- ]brand|your[- ]organization|replace this|todo)\b/gi);
+  if (placeholders && placeholders.length > 0) {
+    issues.push({
+      severity: "warning",
+      message: `Found ${placeholders.length} placeholder(s): ${[...new Set(placeholders)].join(", ")}`
+    });
+  }
+
+  if (fullText.length < 100) {
+    issues.push({ severity: "warning", message: "Content is very short (< 100 characters). Consider adding more detail." });
+  }
+
+  const hasCanonical = /canonical:\s*https?:\/\//i.test(fullText) ||
+    parsed.sections.some((s) => s.lines.some((l) => /^canonical:/i.test(l)));
+  if (!hasCanonical && !lines.some((l) => /canonical/i.test(l))) {
+    issues.push({ severity: "warning", message: "No canonical URL declared. Consider adding one." });
+  }
+
+  return issues;
+}
+
+function computeScore(structureResult, urlResults, contentIssues) {
+  let score = 100;
+
+  const errors = structureResult.issues.filter((i) => i.severity === "error");
+  const warnings = [...structureResult.warnings, ...contentIssues.filter((i) => i.severity === "warning")];
+  const invalidUrls = urlResults.filter((u) => !u.valid);
+  const httpUrls = urlResults.filter((u) => u.valid && !u.https);
+
+  score -= errors.length * 15;
+  score -= warnings.length * 5;
+  score -= invalidUrls.length * 10;
+  score -= httpUrls.length * 3;
+
+  return Math.max(0, Math.min(100, score));
+}
+
+function getScoreLabel(score) {
+  if (score >= 90) return "Excellent";
+  if (score >= 70) return "Good";
+  if (score >= 50) return "Needs improvement";
+  return "Poor";
+}
+
+async function fetchLlmsTxt(url) {
+  const parsedUrl = new URL(url);
+  const llmsUrl = `${parsedUrl.protocol}//${parsedUrl.host}/llms.txt`;
+  const response = await fetch(llmsUrl, {
+    redirect: "follow",
+    headers: { "user-agent": "geo-ai-search-optimization/2.2.0" },
+    signal: AbortSignal.timeout(10_000)
+  });
+  if (!response.ok) return { found: false, url: llmsUrl, status: response.status, content: "" };
+  const content = await response.text();
+  return { found: true, url: llmsUrl, status: response.status, content };
+}
+
+export async function validateLlmsTxt(input, options = {}) {
+  let content;
+  let source;
+
+  if (/^https?:\/\//i.test(input)) {
+    const result = await fetchLlmsTxt(input);
+    if (!result.found) {
+      return {
+        kind: "geo-validate-llms",
+        source: result.url,
+        found: false,
+        score: 0,
+        scoreLabel: "Not found",
+        summary: `No llms.txt found at ${result.url}. Create one with: geo-ai-search-optimization init-llms`,
+        parsed: null,
+        issues: [{ severity: "error", message: "llms.txt not found" }],
+        warnings: [],
+        urlValidation: [],
+        recommendations: ["Create an llms.txt file. Run: geo-ai-search-optimization init-llms --site-name 'Your Site' --site-url 'https://yoursite.com'"]
+      };
+    }
+    content = result.content;
+    source = result.url;
+  } else {
+    const filePath = path.resolve(input);
+    content = await fs.readFile(filePath, "utf8");
+    source = filePath;
+  }
+
+  const parsed = parseLlmsTxt(content);
+  const structureResult = validateStructure(parsed);
+  const urlResults = validateUrls(parsed.urls);
+  const contentIssues = validateContent(parsed);
+  const score = computeScore(structureResult, urlResults, contentIssues);
+  const allIssues = [...structureResult.issues, ...contentIssues.filter((i) => i.severity === "error")];
+  const allWarnings = [...structureResult.warnings, ...contentIssues.filter((i) => i.severity === "warning")];
+
+  const recommendations = [];
+  if (allIssues.length > 0) {
+    recommendations.push(`Fix ${allIssues.length} error(s): ${allIssues.map((i) => i.message).join("; ")}`);
+  }
+  if (allWarnings.length > 0) {
+    recommendations.push(`Address ${allWarnings.length} warning(s) to improve completeness.`);
+  }
+  const invalidUrls = urlResults.filter((u) => !u.valid);
+  if (invalidUrls.length > 0) {
+    recommendations.push(`Fix ${invalidUrls.length} invalid URL(s).`);
+  }
+
+  return {
+    kind: "geo-validate-llms",
+    source,
+    found: true,
+    score,
+    scoreLabel: getScoreLabel(score),
+    summary: `llms.txt validation: ${score}/100 (${getScoreLabel(score)}). ${allIssues.length} error(s), ${allWarnings.length} warning(s).`,
+    parsed: {
+      title: parsed.title,
+      description: parsed.description,
+      sectionCount: parsed.sections.length,
+      sections: parsed.sections.map((s) => s.heading),
+      urlCount: parsed.urls.length
+    },
+    issues: allIssues,
+    warnings: allWarnings,
+    urlValidation: urlResults,
+    recommendations
+  };
+}
+
+export function renderValidateLlmsMarkdown(report) {
+  const lines = [
+    "# llms.txt Validation Report",
+    "",
+    `- Source: \`${report.source}\``,
+    `- Found: \`${report.found}\``,
+    `- Score: \`${report.score}/100\` (${report.scoreLabel})`,
+    `- Summary: ${report.summary}`,
+    ""
+  ];
+
+  if (!report.found) {
+    lines.push("## Action Required", "", "- Create an llms.txt file for your site.", "");
+    return lines.join("\n");
+  }
+
+  if (report.parsed) {
+    lines.push(
+      "## Structure",
+      "",
+      `- Title: \`${report.parsed.title || "missing"}\``,
+      `- Description: \`${report.parsed.description || "missing"}\``,
+      `- Sections: \`${report.parsed.sectionCount}\` (${report.parsed.sections.join(", ")})`,
+      `- URLs found: \`${report.parsed.urlCount}\``,
+      ""
+    );
+  }
+
+  if (report.issues.length > 0) {
+    lines.push("## Errors", "");
+    for (const issue of report.issues) {
+      lines.push(`- ❌ ${issue.message}`);
+    }
+    lines.push("");
+  }
+
+  if (report.warnings.length > 0) {
+    lines.push("## Warnings", "");
+    for (const warning of report.warnings) {
+      lines.push(`- ⚠️ ${warning.message}`);
+    }
+    lines.push("");
+  }
+
+  if (report.urlValidation.length > 0) {
+    lines.push("## URL Validation", "");
+    for (const url of report.urlValidation) {
+      const icon = url.valid ? (url.https ? "✅" : "⚠️") : "❌";
+      lines.push(`- ${icon} \`${url.url}\`${url.warning ? ` — ${url.warning}` : ""}`);
+    }
+    lines.push("");
+  }
+
+  if (report.recommendations.length > 0) {
+    lines.push("## Recommendations", "");
+    for (const rec of report.recommendations) {
+      lines.push(`- ${rec}`);
+    }
+    lines.push("");
+  }
+
+  return lines.join("\n");
+}
+
+export async function writeValidateLlmsOutput(outputPath, content) {
+  return writeScanOutput(outputPath, content);
+}