geo-ai-search-optimization 2.3.0 → 2.5.0

package/src/content-freshness.js

@@ -0,0 +1,411 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+import { fetchText } from "./fetch-utils.js";
+import { writeScanOutput } from "./scan.js";
+
+const CURRENT_YEAR = new Date().getFullYear();
+
+const FRESH_SIGNALS = [
+  { pattern: /\brecently\b/gi, label: "recently" },
+  { pattern: /\bthis year\b/gi, label: "this year" },
+  { pattern: /\blatest\b/gi, label: "latest" },
+  { pattern: /\bupdated\b/gi, label: "updated" },
+  { pattern: /\bnew\b/gi, label: "new" },
+  { pattern: /\bcurrent\b/gi, label: "current" }
+];
+
+const STALE_SIGNALS = [
+  { pattern: /\blast year\b/gi, label: "last year" },
+  { pattern: /\bpreviously\b/gi, label: "previously" },
+  { pattern: /\bformerly\b/gi, label: "formerly" },
+  { pattern: /\bold\b/gi, label: "old" },
+  { pattern: /\blegacy\b/gi, label: "legacy" },
+  { pattern: /\bdeprecated\b/gi, label: "deprecated" }
+];
+
+const EVERGREEN_PATTERNS = [
+  /\bhow to\b/i,
+  /\bwhat is\b/i,
+  /\bdefinition\b/i,
+  /\bguide\b/i,
+  /\btutorial\b/i,
+  /\bprinciples?\b/i,
+  /\bfundamentals?\b/i,
+  /\bbest practices?\b/i,
+  /\boverview\b/i,
+  /\bintroduction\b/i
+];
+
+// Reuse date extraction logic inline (freshness.js does not export extractDates/parseDate)
+const DATE_PATTERNS = [
+  { pattern: /\b(20\d{2}-[01]\d-[0-3]\d)/g, format: "ISO" },
+  { pattern: /\b(January|February|March|April|May|June|July|August|September|October|November|December)\s+(\d{1,2}),?\s+(20\d{2})/gi, format: "US" },
+  { pattern: /"datePublished"\s*:\s*"([^"]+)"/g, format: "schema-published" },
+  { pattern: /"dateModified"\s*:\s*"([^"]+)"/g, format: "schema-modified" },
+  { pattern: /<time[^>]+datetime=["']([^"']+)["']/gi, format: "html-time" },
+  { pattern: /name=["'](?:date|article:published_time|article:modified_time)["'][^>]+content=["']([^"']+)["']/gi, format: "meta" },
+  { pattern: /content=["']([^"']+)["'][^>]+name=["'](?:date|article:published_time|article:modified_time)["']/gi, format: "meta-reverse" },
+  { pattern: /\b(?:Published|Updated|Posted|Modified|Created|Reviewed|Last updated|Last reviewed)[:\s]+(\d{4}-\d{2}-\d{2}|\w+\s+\d{1,2},?\s+\d{4})/gi, format: "visible-label" }
+];
+
+const MONTH_MAP = {
+  january: 0, february: 1, march: 2, april: 3, may: 4, june: 5,
+  july: 6, august: 7, september: 8, october: 9, november: 10, december: 11
+};
+
+function parseDate(dateStr) {
+  const isoMatch = dateStr.match(/(\d{4})-(\d{2})-(\d{2})/);
+  if (isoMatch) {
+    const d = new Date(isoMatch[0]);
+    if (!Number.isNaN(d.getTime())) return d;
+  }
+  const usMatch = dateStr.match(/(January|February|March|April|May|June|July|August|September|October|November|December)\s+(\d{1,2}),?\s+(\d{4})/i);
+  if (usMatch) {
+    const month = MONTH_MAP[usMatch[1].toLowerCase()];
+    if (month !== undefined) {
+      return new Date(Number.parseInt(usMatch[3]), month, Number.parseInt(usMatch[2]));
+    }
+  }
+  const d = new Date(dateStr);
+  if (!Number.isNaN(d.getTime()) && d.getFullYear() >= 2000) return d;
+  return null;
+}
+
+function extractDates(content) {
+  const found = [];
+  const seen = new Set();
+  for (const { pattern, format } of DATE_PATTERNS) {
+    let match;
+    const regex = new RegExp(pattern.source, pattern.flags);
+    while ((match = regex.exec(content)) !== null) {
+      const raw = match[1] || match[0];
+      const parsed = parseDate(raw);
+      if (parsed && !seen.has(parsed.toISOString().slice(0, 10))) {
+        seen.add(parsed.toISOString().slice(0, 10));
+        found.push({ raw: raw.trim().slice(0, 60), date: parsed.toISOString().slice(0, 10), source: format, timestamp: parsed.getTime() });
+      }
+    }
+  }
+  return found.sort((a, b) => b.timestamp - a.timestamp);
+}
+
+function resolvePublishedAndModified(dates) {
+  if (dates.length === 0) return { publishedDate: null, modifiedDate: null };
+  const published = dates.find((d) => d.source.includes("published") || d.source === "visible-label") || dates[dates.length - 1];
+  const modified = dates.find((d) => d.source.includes("modified")) || dates[0];
+  return {
+    publishedDate: published.date,
+    modifiedDate: modified.date
+  };
+}
+
+function countWords(text) {
+  const stripped = text.replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim();
+  return stripped ? stripped.split(/\s+/).length : 0;
+}
+
+function getLines(content) {
+  return content.replace(/<[^>]+>/g, " ").split(/\n/);
+}
+
+function detectTemporalSignals(content) {
+  const lines = getLines(content);
+  const signals = [];
+
+  for (const group of [{ list: FRESH_SIGNALS, type: "fresh" }, { list: STALE_SIGNALS, type: "stale" }]) {
+    for (const { pattern, label } of group.list) {
+      const regex = new RegExp(pattern.source, pattern.flags);
+      for (let i = 0; i < lines.length; i++) {
+        let match;
+        while ((match = regex.exec(lines[i])) !== null) {
+          signals.push({ phrase: match[0], type: group.type, line: i + 1 });
+        }
+      }
+    }
+  }
+
+  // Year references
+  const yearRegex = /\b(20\d{2})\b/g;
+  for (let i = 0; i < lines.length; i++) {
+    let match;
+    const regex = new RegExp(yearRegex.source, yearRegex.flags);
+    while ((match = regex.exec(lines[i])) !== null) {
+      const year = Number.parseInt(match[1]);
+      if (year >= 2000 && year <= CURRENT_YEAR + 1) {
+        const type = year >= CURRENT_YEAR ? "fresh" : year >= CURRENT_YEAR - 1 ? "neutral" : "stale";
+        signals.push({ phrase: match[0], type, line: i + 1 });
+      }
+    }
+  }
+
+  return signals;
+}
+
+function detectStaleReferences(content) {
+  const stale = [];
+  const yearRegex = /\b(20\d{2})\b/g;
+  let match;
+  while ((match = yearRegex.exec(content)) !== null) {
+    const year = Number.parseInt(match[1]);
+    if (year <= CURRENT_YEAR - 3) {
+      stale.push({ reference: match[0], reason: `Year ${year} is ${CURRENT_YEAR - year} years old` });
+    }
+  }
+
+  const deprecatedTerms = [
+    { pattern: /\bInternet Explorer\b/gi, reason: "Internet Explorer is discontinued" },
+    { pattern: /\bFlash Player\b/gi, reason: "Adobe Flash Player is deprecated" },
+    { pattern: /\bAngularJS\b/gi, reason: "AngularJS (1.x) is end-of-life" },
+    { pattern: /\bPython 2\b/gi, reason: "Python 2 is end-of-life" }
+  ];
+
+  for (const { pattern, reason } of deprecatedTerms) {
+    const regex = new RegExp(pattern.source, pattern.flags);
+    if (regex.test(content)) {
+      stale.push({ reference: pattern.source.replace(/\\b/g, ""), reason });
+    }
+  }
+
+  return stale;
+}
+
+function computeEvergreenScore(content) {
+  let hits = 0;
+  for (const pattern of EVERGREEN_PATTERNS) {
+    if (pattern.test(content)) hits++;
+  }
+  return Math.min(Math.round((hits / EVERGREEN_PATTERNS.length) * 100), 100);
+}
+
+function computeUpdateFrequencySignal(publishedDate, modifiedDate) {
+  if (!publishedDate || !modifiedDate) return "unknown";
+  if (publishedDate === modifiedDate) return "never-updated";
+  const pubMs = new Date(publishedDate).getTime();
+  const modMs = new Date(modifiedDate).getTime();
+  const gapDays = Math.floor((modMs - pubMs) / (1000 * 60 * 60 * 24));
+  if (gapDays <= 7) return "frequent";
+  if (gapDays <= 30) return "monthly";
+  if (gapDays <= 90) return "quarterly";
+  if (gapDays <= 365) return "yearly";
+  return "infrequent";
+}
+
+function computeScore(ageDays, datesFound, staleReferences, temporalSignals, publishedDate, modifiedDate, evergreenScore) {
+  let score = 100;
+
+  // Content age: -1 per 30 days since last modification (max -30)
+  if (ageDays !== null) {
+    const agePenalty = Math.min(Math.floor(ageDays / 30), 30);
+    score -= agePenalty;
+  }
+
+  // No dates found: -20
+  if (datesFound === 0) {
+    score -= 20;
+  }
+
+  // Stale references: -5 each (max -15)
+  const staleYearRefs = staleReferences.filter((r) => /^Year \d+/.test(r.reason) || /^\d{4}$/.test(r.reference));
+  const stalePenalty = Math.min(staleYearRefs.length * 5, 15);
+  score -= stalePenalty;
+
+  // No temporal signals: -10
+  if (temporalSignals.length === 0) {
+    score -= 10;
+  }
+
+  // datePublished == dateModified (never updated): -10
+  if (publishedDate && modifiedDate && publishedDate === modifiedDate) {
+    score -= 10;
+  }
+
+  // Evergreen bonus: +10 if content uses timeless language patterns
+  if (evergreenScore >= 30) {
+    score += 10;
+  }
+
+  return Math.max(0, Math.min(100, score));
+}
+
+function scoreLabel(score) {
+  if (score >= 80) return "Fresh";
+  if (score >= 60) return "Aging";
+  if (score >= 40) return "Stale";
+  return "Outdated";
+}
+
+function buildRecommendations(report) {
+  const recs = [];
+
+  if (!report.publishedDate && !report.modifiedDate) {
+    recs.push("Add datePublished and dateModified to your structured data (JSON-LD) for AI freshness signals.");
+  }
+
+  if (report.updateFrequencySignal === "never-updated") {
+    recs.push("Content has never been updated (datePublished equals dateModified). Update dateModified when content changes.");
+  }
+
+  if (report.staleReferences.length > 0) {
+    recs.push(`Found ${report.staleReferences.length} stale reference(s). Review and update outdated year mentions or deprecated technology references.`);
+  }
+
+  const freshCount = report.temporalSignals.filter((s) => s.type === "fresh").length;
+  const staleCount = report.temporalSignals.filter((s) => s.type === "stale").length;
+
+  if (freshCount === 0) {
+    recs.push("Add temporal freshness language (e.g., 'recently', 'latest', current year) to signal content currency.");
+  }
+
+  if (staleCount > freshCount) {
+    recs.push("Content has more stale temporal signals than fresh ones. Consider updating language to reflect current state.");
+  }
+
+  if (report.evergreenScore < 20) {
+    recs.push("Content lacks evergreen patterns. Consider adding timeless structural elements (guides, tutorials, definitions).");
+  }
+
+  if (report.ageDays !== null && report.ageDays > 180) {
+    recs.push("Content is over 6 months old. Review and refresh to maintain AI citation relevance.");
+  }
+
+  return recs;
+}
+
+export async function analyzeContentFreshness(input, options = {}) {
+  let content;
+  let source;
+
+  if (/^https?:\/\//i.test(input)) {
+    content = await fetchText(input);
+    source = input;
+  } else {
+    const filePath = path.resolve(input);
+    content = await fs.readFile(filePath, "utf8");
+    source = filePath;
+  }
+
+  const wordCount = countWords(content);
+  const dates = extractDates(content);
+  const { publishedDate, modifiedDate } = resolvePublishedAndModified(dates);
+
+  const now = new Date();
+  let ageDays = null;
+  if (modifiedDate) {
+    ageDays = Math.floor((now - new Date(modifiedDate)) / (1000 * 60 * 60 * 24));
+  } else if (publishedDate) {
+    ageDays = Math.floor((now - new Date(publishedDate)) / (1000 * 60 * 60 * 24));
+  }
+
+  const temporalSignals = detectTemporalSignals(content);
+  const staleReferences = detectStaleReferences(content);
+  const evergreenScore = computeEvergreenScore(content);
+  const updateFrequencySignal = computeUpdateFrequencySignal(publishedDate, modifiedDate);
+
+  const score = computeScore(ageDays, dates.length, staleReferences, temporalSignals, publishedDate, modifiedDate, evergreenScore);
+  const label = scoreLabel(score);
+
+  const report = {
+    kind: "geo-content-freshness",
+    source,
+    wordCount,
+    publishedDate,
+    modifiedDate,
+    ageDays,
+    temporalSignals,
+    staleReferences,
+    evergreenScore,
+    updateFrequencySignal,
+    score,
+    scoreLabel: label,
+    recommendations: [],
+    summary: ""
+  };
+
+  report.recommendations = buildRecommendations(report);
+  report.summary = ageDays !== null
+    ? `Content freshness: ${score}/100 (${label}). Last modified: ${modifiedDate || publishedDate} (${ageDays} days ago). ${temporalSignals.length} temporal signals, ${staleReferences.length} stale references.`
+    : `Content freshness: ${score}/100 (${label}). No dates detected. ${temporalSignals.length} temporal signals, ${staleReferences.length} stale references.`;
+
+  return report;
+}
+
+export function renderContentFreshnessMarkdown(report) {
+  const lines = [
+    "# Content Freshness Analysis",
+    "",
+    `- Source: \`${report.source}\``,
+    `- Score: \`${report.score}/100\` (${report.scoreLabel})`,
+    `- Word count: \`${report.wordCount}\``,
+    `- Summary: ${report.summary}`,
+    ""
+  ];
+
+  lines.push(
+    "## Dates",
+    "",
+    `- Published: \`${report.publishedDate || "not detected"}\``,
+    `- Modified: \`${report.modifiedDate || "not detected"}\``,
+    `- Age (days): \`${report.ageDays ?? "N/A"}\``,
+    `- Update frequency: \`${report.updateFrequencySignal}\``,
+    ""
+  );
+
+  lines.push(
+    "## Evergreen Score",
+    "",
+    `- Evergreen score: \`${report.evergreenScore}/100\``,
+    ""
+  );
+
+  if (report.temporalSignals.length > 0) {
+    lines.push("## Temporal Signals", "");
+    const freshSignals = report.temporalSignals.filter((s) => s.type === "fresh");
+    const staleSignals = report.temporalSignals.filter((s) => s.type === "stale");
+    const neutralSignals = report.temporalSignals.filter((s) => s.type === "neutral");
+    if (freshSignals.length > 0) {
+      lines.push(`### Fresh (${freshSignals.length})`, "");
+      for (const s of freshSignals.slice(0, 10)) {
+        lines.push(`- "${s.phrase}" (line ${s.line})`);
+      }
+      lines.push("");
+    }
+    if (staleSignals.length > 0) {
+      lines.push(`### Stale (${staleSignals.length})`, "");
+      for (const s of staleSignals.slice(0, 10)) {
+        lines.push(`- "${s.phrase}" (line ${s.line})`);
+      }
+      lines.push("");
+    }
+    if (neutralSignals.length > 0) {
+      lines.push(`### Neutral (${neutralSignals.length})`, "");
+      for (const s of neutralSignals.slice(0, 10)) {
+        lines.push(`- "${s.phrase}" (line ${s.line})`);
+      }
+      lines.push("");
+    }
+  }
+
+  if (report.staleReferences.length > 0) {
+    lines.push("## Stale References", "");
+    for (const ref of report.staleReferences.slice(0, 10)) {
+      lines.push(`- \`${ref.reference}\`: ${ref.reason}`);
+    }
+    lines.push("");
+  }
+
+  lines.push("## Recommendations", "");
+  if (report.recommendations.length === 0) {
+    lines.push("- Content freshness signals are well-configured.");
+  } else {
+    for (const rec of report.recommendations) {
+      lines.push(`- ${rec}`);
+    }
+  }
+  lines.push("");
+
+  return lines.join("\n");
+}
+
+export async function writeContentFreshnessOutput(outputPath, content) {
+  return writeScanOutput(outputPath, content);
+}

package/src/crawlers.js

@@ -1,5 +1,6 @@
 import fs from "node:fs/promises";
 import path from "node:path";
+import { fetchResponse } from "./fetch-utils.js";
 import { writeScanOutput } from "./scan.js";
 
 const KNOWN_AI_CRAWLERS = [
@@ -43,9 +44,10 @@ function parseRobotsTxt(content) {
       continue;
     }
 
-    const disallowMatch = line.match(/^disallow:\s*(.+)$/i);
+    const disallowMatch = line.match(/^disallow:\s*(.*)$/i);
     if (disallowMatch) {
-      current.rules.push({ type: "disallow", path: disallowMatch[1].trim() });
+      const disallowPath = disallowMatch[1].trim();
+      current.rules.push({ type: "disallow", path: disallowPath });
       continue;
     }
 
@@ -75,7 +77,7 @@ function analyzeCrawlerAccess(blocks) {
     if (specificBlock) {
       matchedAgent = specificBlock.userAgent;
       const hasDisallow = specificBlock.rules.some(
-        (r) => r.type === "disallow" && (r.path === "/" || r.path === "")
+        (r) => r.type === "disallow" && r.path === "/"
       );
       const hasAllow = specificBlock.rules.some(
         (r) => r.type === "allow" && r.path === "/"
@@ -96,9 +98,15 @@ function analyzeCrawlerAccess(blocks) {
       const hasDisallow = wildcardBlock.rules.some(
         (r) => r.type === "disallow" && r.path === "/"
       );
-      if (hasDisallow) {
+      const hasAllow = wildcardBlock.rules.some(
+        (r) => r.type === "allow" && r.path === "/"
+      );
+      if (hasDisallow && !hasAllow) {
         status = "blocked";
         reason = "Blocked by wildcard User-agent: * Disallow: /";
+      } else if (hasDisallow && hasAllow) {
+        status = "partial";
+        reason = "Mixed rules for wildcard User-agent: *";
       }
     }
 
@@ -179,16 +187,11 @@ function generateOptimalRobotsTxt(crawlerResults, options = {}) {
 async function fetchRobotsTxt(url) {
   const parsedUrl = new URL(url);
   const robotsUrl = `${parsedUrl.protocol}//${parsedUrl.host}/robots.txt`;
-  const response = await fetch(robotsUrl, {
-    redirect: "follow",
-    headers: { "user-agent": "geo-ai-search-optimization/2.2.0" },
-    signal: AbortSignal.timeout(10_000)
-  });
-  if (!response.ok) {
-    return { found: false, url: robotsUrl, status: response.status, content: "" };
+  const result = await fetchResponse(robotsUrl);
+  if (!result.ok) {
+    return { found: false, url: robotsUrl, status: result.status, content: "" };
   }
-  const content = await response.text();
-  return { found: true, url: robotsUrl, status: response.status, content };
+  return { found: true, url: robotsUrl, status: result.status, content: result.text };
 }
 
 export async function analyzeCrawlers(input, options = {}) {

package/src/deep-benchmark.js

@@ -125,8 +125,9 @@ export async function deepBenchmark(ownUrl, competitorUrls, options = {}) {
   }));
 
   const ownComposite = ownResult.ok ? ownResult.data.compositeScore : 0;
-  const avgCompComposite = successful.length > 1
-    ? Math.round(successful.filter((r) => r.url !== ownUrl).reduce((sum, r) => sum + r.data.compositeScore, 0) / (successful.length - 1))
+  const successfulCompetitors = successful.filter((r) => r.url !== ownUrl);
+  const avgCompComposite = successfulCompetitors.length > 0
+    ? Math.round(successfulCompetitors.reduce((sum, r) => sum + r.data.compositeScore, 0) / successfulCompetitors.length)
     : null;
 
   return {

package/src/eeat.js

@@ -1,5 +1,6 @@
 import fs from "node:fs/promises";
 import path from "node:path";
+import { fetchText } from "./fetch-utils.js";
 import { writeScanOutput } from "./scan.js";
 
 function stripHtml(text) {
@@ -125,22 +126,12 @@ function buildRecommendations(experience, expertise, authority, trust, authorSig
   return recs;
 }
 
-async function fetchContent(url) {
-  const response = await fetch(url, {
-    redirect: "follow",
-    headers: { "user-agent": "geo-ai-search-optimization/2.2.0" },
-    signal: AbortSignal.timeout(10_000)
-  });
-  if (!response.ok) throw new Error(`Failed to fetch: ${url} (status ${response.status})`);
-  return response.text();
-}
-
 export async function analyzeEeat(input, options = {}) {
   let rawContent;
   let source;
 
   if (/^https?:\/\//i.test(input)) {
-    rawContent = await fetchContent(input);
+    rawContent = await fetchText(input);
     source = input;
   } else {
     const filePath = path.resolve(input);