geo-ai-search-optimization 2.0.0 → 2.2.0

package/src/citability.js

@@ -0,0 +1,311 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+import { writeScanOutput } from "./scan.js";
+
+function stripHtml(text) {
+  return text
+    .replace(/<script[\s\S]*?<\/script>/gi, " ")
+    .replace(/<style[\s\S]*?<\/style>/gi, " ")
+    .replace(/<[^>]+>/g, " ")
+    .replace(/&[a-z0-9#]+;/gi, " ")
+    .replace(/\s+/g, " ")
+    .trim();
+}
+
+function extractPlainText(content) {
+  if (/<html|<head|<body/i.test(content)) {
+    return stripHtml(content);
+  }
+  return content.replace(/^---[\s\S]*?---/, "").trim();
+}
+
+function splitSentences(text) {
+  return text
+    .split(/(?<=[.!?。！？])\s+/)
+    .map((s) => s.trim())
+    .filter((s) => s.length > 10);
+}
+
+function splitParagraphs(text) {
+  return text
+    .split(/\n\s*\n/)
+    .map((p) => p.replace(/\s+/g, " ").trim())
+    .filter((p) => p.length > 20);
+}
+
+const STAT_PATTERN = /\b\d+(\.\d+)?\s*(%|percent|million|billion|thousand|x|times|fold)\b/i;
+const DATE_PATTERN = /\b(20[12]\d|19\d\d)[-/]\d{1,2}[-/]\d{1,2}\b|\b(January|February|March|April|May|June|July|August|September|October|November|December)\s+\d{1,2},?\s+20[12]\d\b/i;
+const NUMBER_FACT_PATTERN = /\b\d{2,}(\.\d+)?\b/;
+const CLAIM_VERBS = /\b(is|are|was|were|shows?|demonstrates?|indicates?|proves?|reveals?|found|according to|reported|measured|calculated|estimated|averaged)\b/i;
+const COMPARISON_PATTERN = /\b(more than|less than|greater|fewer|higher|lower|faster|slower|better|worse|compared to|versus|outperform|exceed)\b/i;
+
+function analyzeClaims(sentences) {
+  let factualClaims = 0;
+  let statClaims = 0;
+  let datedClaims = 0;
+  let comparisonClaims = 0;
+
+  for (const sentence of sentences) {
+    const hasNumber = NUMBER_FACT_PATTERN.test(sentence);
+    const hasClaimVerb = CLAIM_VERBS.test(sentence);
+    const hasStat = STAT_PATTERN.test(sentence);
+    const hasDate = DATE_PATTERN.test(sentence);
+    const hasComparison = COMPARISON_PATTERN.test(sentence);
+
+    if (hasNumber && hasClaimVerb) factualClaims++;
+    if (hasStat) statClaims++;
+    if (hasDate) datedClaims++;
+    if (hasComparison) comparisonClaims++;
+  }
+
+  const total = sentences.length || 1;
+  return {
+    total: sentences.length,
+    factualClaims,
+    statClaims,
+    datedClaims,
+    comparisonClaims,
+    claimDensity: Math.round((factualClaims / total) * 100),
+    statDensity: Math.round((statClaims / total) * 100)
+  };
+}
+
+const ENTITY_PATTERN = /\b[A-Z][a-z]+(?:\s+[A-Z][a-z]+)+\b/g;
+const TECH_TERM_PATTERN = /\b[A-Z]{2,}(?:\s+[A-Z]{2,})*\b/g;
+const BRAND_PATTERN = /\b[A-Z][a-zA-Z]*(?:\.(?:com|io|ai|org|net))\b/g;
+
+function analyzeEntities(text) {
+  const namedEntities = new Set((text.match(ENTITY_PATTERN) || []).map((e) => e.trim()));
+  const techTerms = new Set((text.match(TECH_TERM_PATTERN) || []).filter((t) => t.length > 1));
+  const brands = new Set((text.match(BRAND_PATTERN) || []).map((b) => b.trim()));
+  const wordCount = text.split(/\s+/).length || 1;
+
+  return {
+    namedEntities: [...namedEntities].slice(0, 20),
+    techTerms: [...techTerms].slice(0, 20),
+    brands: [...brands].slice(0, 10),
+    entityDensity: Math.round(((namedEntities.size + techTerms.size) / wordCount) * 1000) / 10
+  };
+}
+
+function analyzeQuotability(sentences) {
+  const quotable = [];
+
+  for (const sentence of sentences) {
+    const wordCount = sentence.split(/\s+/).length;
+    if (wordCount < 8 || wordCount > 40) continue;
+
+    let score = 0;
+    if (STAT_PATTERN.test(sentence)) score += 3;
+    if (CLAIM_VERBS.test(sentence)) score += 2;
+    if (NUMBER_FACT_PATTERN.test(sentence)) score += 2;
+    if (COMPARISON_PATTERN.test(sentence)) score += 2;
+    if (wordCount >= 12 && wordCount <= 25) score += 1;
+
+    if (score >= 3) {
+      quotable.push({ sentence: sentence.slice(0, 200), score, wordCount });
+    }
+  }
+
+  return quotable.sort((a, b) => b.score - a.score).slice(0, 10);
+}
+
+function analyzeStructure(content) {
+  const hasList = /<[ou]l\b/i.test(content) || /^[-*]\s/m.test(content);
+  const hasTable = /<table\b/i.test(content) || /\|.*\|.*\|/m.test(content);
+  const hasDefinition = /<dl\b/i.test(content) || /\b(is defined as|refers to|means)\b/i.test(content);
+  const hasHeading = /<h[1-6]\b/i.test(content) || /^#{1,6}\s/m.test(content);
+  const headingCount = ((content.match(/<h[1-6]\b/gi) || []).length) +
+    ((content.match(/^#{1,6}\s/gm) || []).length);
+
+  let structureScore = 0;
+  if (hasList) structureScore += 20;
+  if (hasTable) structureScore += 20;
+  if (hasDefinition) structureScore += 15;
+  if (hasHeading && headingCount >= 3) structureScore += 25;
+  else if (hasHeading) structureScore += 15;
+  if (headingCount >= 5) structureScore += 20;
+
+  return {
+    hasList,
+    hasTable,
+    hasDefinition,
+    hasHeading,
+    headingCount,
+    structureScore: Math.min(structureScore, 100)
+  };
+}
+
+function computeCitabilityScore(claims, entities, quotable, structure, wordCount) {
+  const w = {
+    claimDensity: 25,
+    entityRichness: 15,
+    quotability: 25,
+    structure: 20,
+    length: 15
+  };
+
+  const claimScore = Math.min(claims.claimDensity * 2, 100);
+  const entityScore = Math.min(entities.entityDensity * 10, 100);
+  const quotabilityScore = Math.min(quotable.length * 10, 100);
+  const lengthScore = wordCount >= 300 ? (wordCount >= 800 ? 100 : 60) : 20;
+
+  const weighted =
+    (claimScore * w.claimDensity +
+      entityScore * w.entityRichness +
+      quotabilityScore * w.quotability +
+      structure.structureScore * w.structure +
+      lengthScore * w.length) / 100;
+
+  return Math.round(Math.min(weighted, 100));
+}
+
+function getScoreLabel(score) {
+  if (score >= 80) return "Highly citable";
+  if (score >= 60) return "Moderately citable";
+  if (score >= 40) return "Low citability";
+  return "Poor citability";
+}
+
+function buildRecommendations(claims, entities, quotable, structure, wordCount) {
+  const recs = [];
+
+  if (claims.claimDensity < 15) {
+    recs.push("Add more factual claims with specific numbers, statistics, or data points to increase claim density.");
+  }
+  if (claims.statDensity < 5) {
+    recs.push("Include statistics (percentages, measurements, benchmarks) to make claims more citable.");
+  }
+  if (entities.entityDensity < 1) {
+    recs.push("Reference more named entities, brands, or technical terms to help AI identify the topic.");
+  }
+  if (quotable.length < 3) {
+    recs.push("Write more self-contained, quotable sentences (12-25 words) with clear factual assertions.");
+  }
+  if (!structure.hasList) {
+    recs.push("Add bulleted or numbered lists to make information scannable and extractable.");
+  }
+  if (!structure.hasTable) {
+    recs.push("Add comparison tables or data tables for easy extraction by AI systems.");
+  }
+  if (structure.headingCount < 3) {
+    recs.push("Add more descriptive headings to organize content into clearly defined sections.");
+  }
+  if (wordCount < 300) {
+    recs.push("Expand content to at least 300 words to provide sufficient depth for AI citation.");
+  }
+
+  return recs;
+}
+
+async function fetchContent(url) {
+  const response = await fetch(url, {
+    redirect: "follow",
+    headers: { "user-agent": "geo-ai-search-optimization/2.2.0" },
+    signal: AbortSignal.timeout(10_000)
+  });
+  if (!response.ok) throw new Error(`Failed to fetch: ${url} (status ${response.status})`);
+  return response.text();
+}
+
+export async function analyzeCitability(input, options = {}) {
+  let rawContent;
+  let source;
+
+  if (/^https?:\/\//i.test(input)) {
+    rawContent = await fetchContent(input);
+    source = input;
+  } else {
+    const filePath = path.resolve(input);
+    rawContent = await fs.readFile(filePath, "utf8");
+    source = filePath;
+  }
+
+  const plainText = extractPlainText(rawContent);
+  const sentences = splitSentences(plainText);
+  const paragraphs = splitParagraphs(plainText);
+  const wordCount = plainText.split(/\s+/).length;
+
+  const claims = analyzeClaims(sentences);
+  const entities = analyzeEntities(plainText);
+  const quotable = analyzeQuotability(sentences);
+  const structure = analyzeStructure(rawContent);
+  const score = computeCitabilityScore(claims, entities, quotable, structure, wordCount);
+  const recommendations = buildRecommendations(claims, entities, quotable, structure, wordCount);
+
+  return {
+    kind: "geo-citability",
+    source,
+    wordCount,
+    sentenceCount: sentences.length,
+    paragraphCount: paragraphs.length,
+    score,
+    scoreLabel: getScoreLabel(score),
+    claims,
+    entities,
+    quotableSentences: quotable,
+    structure,
+    recommendations,
+    summary: `Citability score: ${score}/100 (${getScoreLabel(score)}). ${recommendations[0] || "Content is well-structured for AI citation."}`
+  };
+}
+
+export function renderCitabilityMarkdown(report) {
+  const lines = [
+    "# Citability Analysis",
+    "",
+    `- Source: \`${report.source}\``,
+    `- Citability Score: \`${report.score}/100\` (${report.scoreLabel})`,
+    `- Word Count: \`${report.wordCount}\``,
+    `- Sentences: \`${report.sentenceCount}\``,
+    `- Summary: ${report.summary}`,
+    "",
+    "## Claim Analysis",
+    "",
+    `- Factual claims: \`${report.claims.factualClaims}/${report.claims.total}\` (${report.claims.claimDensity}%)`,
+    `- Statistical claims: \`${report.claims.statClaims}\` (${report.claims.statDensity}%)`,
+    `- Dated claims: \`${report.claims.datedClaims}\``,
+    `- Comparison claims: \`${report.claims.comparisonClaims}\``,
+    "",
+    "## Entity Analysis",
+    "",
+    `- Entity density: \`${report.entities.entityDensity}%\``,
+    `- Named entities: ${report.entities.namedEntities.slice(0, 10).map((e) => `\`${e}\``).join(", ") || "none detected"}`,
+    `- Technical terms: ${report.entities.techTerms.slice(0, 10).map((t) => `\`${t}\``).join(", ") || "none detected"}`,
+    "",
+    "## Structure",
+    "",
+    `- Lists: \`${report.structure.hasList}\``,
+    `- Tables: \`${report.structure.hasTable}\``,
+    `- Definitions: \`${report.structure.hasDefinition}\``,
+    `- Headings: \`${report.structure.headingCount}\``,
+    `- Structure score: \`${report.structure.structureScore}/100\``,
+    "",
+    "## Top Quotable Sentences",
+    ""
+  ];
+
+  if (report.quotableSentences.length === 0) {
+    lines.push("- No highly quotable sentences detected.");
+  } else {
+    for (const q of report.quotableSentences) {
+      lines.push(`- (score ${q.score}) ${q.sentence}`);
+    }
+  }
+
+  lines.push("", "## Recommendations", "");
+  if (report.recommendations.length === 0) {
+    lines.push("- Content is well-optimized for AI citability.");
+  } else {
+    for (const rec of report.recommendations) {
+      lines.push(`- ${rec}`);
+    }
+  }
+
+  lines.push("");
+  return lines.join("\n");
+}
+
+export async function writeCitabilityOutput(outputPath, content) {
+  return writeScanOutput(outputPath, content);
+}

package/src/citation-check.js

@@ -25,7 +25,7 @@ async function fetchWithTimeout(url, timeoutMs = 10000) {
     const response = await fetch(url, {
       signal: controller.signal,
       headers: {
-        "user-agent": "geo-ai-search-optimization/1.5.0"
+        "user-agent": "geo-ai-search-optimization/2.2.0"
       },
       redirect: "follow"
     });