@rankcli/agent-runtime 0.0.14 → 0.0.15

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -239,7 +239,7 @@ async function crawlUrl(params) {
239
239
  async function extractMeta(params) {
240
240
  const { html, url } = params;
241
241
  try {
242
- const $ = cheerio62.load(html);
242
+ const $ = cheerio63.load(html);
243
243
  const meta = {
244
244
  title: $("title").text().trim() || void 0,
245
245
  description: $('meta[name="description"]').attr("content")?.trim(),
@@ -284,7 +284,7 @@ async function extractMeta(params) {
284
284
  async function analyzeHeadings(params) {
285
285
  const { html } = params;
286
286
  try {
287
- const $ = cheerio62.load(html);
287
+ const $ = cheerio63.load(html);
288
288
  const headings = [];
289
289
  $("h1, h2, h3, h4, h5, h6").each((_, el) => {
290
290
  const tag = el.tagName.toLowerCase();
@@ -305,7 +305,7 @@ async function analyzeHeadings(params) {
305
305
  async function extractImages2(params) {
306
306
  const { html } = params;
307
307
  try {
308
- const $ = cheerio62.load(html);
308
+ const $ = cheerio63.load(html);
309
309
  const images = [];
310
310
  $("img").each((_, el) => {
311
311
  images.push({
@@ -327,7 +327,7 @@ async function extractImages2(params) {
327
327
  async function extractLinks3(params) {
328
328
  const { html, baseUrl } = params;
329
329
  try {
330
- const $ = cheerio62.load(html);
330
+ const $ = cheerio63.load(html);
331
331
  const links = [];
332
332
  const baseHostname = new URL(baseUrl).hostname;
333
333
  $("a[href]").each((_, el) => {
@@ -362,7 +362,7 @@ async function extractLinks3(params) {
362
362
  async function extractSchema2(params) {
363
363
  const { html } = params;
364
364
  try {
365
- const $ = cheerio62.load(html);
365
+ const $ = cheerio63.load(html);
366
366
  const schemas = [];
367
367
  $('script[type="application/ld+json"]').each((_, el) => {
368
368
  try {
@@ -431,7 +431,7 @@ async function checkSitemap2(params) {
431
431
  timeout: 1e4
432
432
  });
433
433
  if (response.status === 200 && response.data.includes("<?xml")) {
434
- const $ = cheerio62.load(response.data, { xmlMode: true });
434
+ const $ = cheerio63.load(response.data, { xmlMode: true });
435
435
  const urlCount = $("url").length || $("sitemap").length;
436
436
  return {
437
437
  success: true,
@@ -460,12 +460,12 @@ async function checkSitemap2(params) {
460
460
  };
461
461
  }
462
462
  }
463
- var cheerio62;
463
+ var cheerio63;
464
464
  var init_crawl = __esm({
465
465
  "src/tools/crawl.ts"() {
466
466
  "use strict";
467
467
  init_http();
468
- cheerio62 = __toESM(require("cheerio"));
468
+ cheerio63 = __toESM(require("cheerio"));
469
469
  }
470
470
  });
471
471
 
@@ -862,7 +862,7 @@ function analyzeRobotsTxtForAI(robotsTxt) {
862
862
  return { allowed, blocked, recommendations };
863
863
  }
864
864
  function detectRenderingMode(html) {
865
- const $ = cheerio63.load(html);
865
+ const $ = cheerio64.load(html);
866
866
  const signals = [];
867
867
  const hasNextData = html.includes("__NEXT_DATA__");
868
868
  const hasNuxtData = html.includes("__NUXT__");
@@ -903,7 +903,7 @@ function detectRenderingMode(html) {
903
903
  };
904
904
  }
905
905
  function analyzeContentStructure(html) {
906
- const $ = cheerio63.load(html);
906
+ const $ = cheerio64.load(html);
907
907
  const jsonLdScripts = $('script[type="application/ld+json"]');
908
908
  let hasStructuredData = jsonLdScripts.length > 0;
909
909
  let hasFAQSchema = false;
@@ -953,7 +953,7 @@ function analyzeContentStructure(html) {
953
953
  };
954
954
  }
955
955
  function analyzeCitationReadiness(html) {
956
- const $ = cheerio63.load(html);
956
+ const $ = cheerio64.load(html);
957
957
  const bodyText = $("body").text();
958
958
  const hasCitations = $('cite, blockquote, [class*="citation"], [class*="reference"]').length > 0 || /\[\d+\]|\(\d{4}\)/.test(bodyText);
959
959
  const externalLinks = $('a[href^="http"]').filter((_, el) => {
@@ -986,7 +986,7 @@ function analyzeCitationReadiness(html) {
986
986
  };
987
987
  }
988
988
  function analyzeEntityExtraction(html) {
989
- const $ = cheerio63.load(html);
989
+ const $ = cheerio64.load(html);
990
990
  const bodyText = $("body").text();
991
991
  const definedTerms = [];
992
992
  $("dfn, abbr[title]").each((_, el) => {
@@ -1022,7 +1022,7 @@ function analyzeEntityExtraction(html) {
1022
1022
  };
1023
1023
  }
1024
1024
  function calculateLLMSignals(structure, citation, entity, html) {
1025
- const $ = cheerio63.load(html);
1025
+ const $ = cheerio64.load(html);
1026
1026
  let contentClarity = 0;
1027
1027
  if (structure.headingHierarchy === "good") contentClarity += 30;
1028
1028
  else if (structure.headingHierarchy === "needs-work") contentClarity += 15;
@@ -1328,11 +1328,11 @@ Allow: /
1328
1328
  Sitemap: ${siteUrl}/sitemap.xml
1329
1329
  `;
1330
1330
  }
1331
- var cheerio63, AI_CRAWLERS, AI_CRAWLERS_INFO;
1331
+ var cheerio64, AI_CRAWLERS, AI_CRAWLERS_INFO;
1332
1332
  var init_geo_analyzer = __esm({
1333
1333
  "src/analyzers/geo-analyzer.ts"() {
1334
1334
  "use strict";
1335
- cheerio63 = __toESM(require("cheerio"));
1335
+ cheerio64 = __toESM(require("cheerio"));
1336
1336
  AI_CRAWLERS = {
1337
1337
  // OpenAI
1338
1338
  GPTBot: { userAgent: "GPTBot", company: "OpenAI", purpose: "ChatGPT training & browsing" },
@@ -1691,7 +1691,7 @@ function generateCWVIssues(lcp, fidInp, cls, ttfb, url) {
1691
1691
  return issues;
1692
1692
  }
1693
1693
  function analyzeCoreWebVitals(html, url, headers) {
1694
- const $ = cheerio64.load(html);
1694
+ const $ = cheerio65.load(html);
1695
1695
  const lcp = analyzeLCP(html, $);
1696
1696
  const fidInp = analyzeFID_INP(html, $);
1697
1697
  const cls = analyzeCLS(html, $);
@@ -1736,11 +1736,11 @@ function analyzeCoreWebVitals(html, url, headers) {
1736
1736
  issues
1737
1737
  };
1738
1738
  }
1739
- var cheerio64;
1739
+ var cheerio65;
1740
1740
  var init_core_web_vitals_analyzer = __esm({
1741
1741
  "src/analyzers/core-web-vitals-analyzer.ts"() {
1742
1742
  "use strict";
1743
- cheerio64 = __toESM(require("cheerio"));
1743
+ cheerio65 = __toESM(require("cheerio"));
1744
1744
  }
1745
1745
  });
1746
1746
 
@@ -2075,7 +2075,7 @@ function validateSchema(data, type) {
2075
2075
  };
2076
2076
  }
2077
2077
  function analyzeStructuredData2(html, url) {
2078
- const $ = cheerio65.load(html);
2078
+ const $ = cheerio66.load(html);
2079
2079
  const schemas = [];
2080
2080
  const issues = [];
2081
2081
  const recommendations = [];
@@ -2339,11 +2339,11 @@ function generateSchemaTemplate(pageType, options) {
2339
2339
  }
2340
2340
  return JSON.stringify(result, null, 2);
2341
2341
  }
2342
- var cheerio65, SCHEMA_REQUIREMENTS;
2342
+ var cheerio66, SCHEMA_REQUIREMENTS;
2343
2343
  var init_structured_data_analyzer = __esm({
2344
2344
  "src/analyzers/structured-data-analyzer.ts"() {
2345
2345
  "use strict";
2346
- cheerio65 = __toESM(require("cheerio"));
2346
+ cheerio66 = __toESM(require("cheerio"));
2347
2347
  SCHEMA_REQUIREMENTS = {
2348
2348
  "Organization": {
2349
2349
  required: ["name", "url"],
@@ -2459,7 +2459,7 @@ function getImageFormat2(src) {
2459
2459
  }
2460
2460
  }
2461
2461
  function analyzeImages3(html, url) {
2462
- const $ = cheerio66.load(html);
2462
+ const $ = cheerio67.load(html);
2463
2463
  const issues = [];
2464
2464
  const recommendations = [];
2465
2465
  let score = 100;
@@ -2677,11 +2677,11 @@ function generateResponsiveImage(options) {
2677
2677
  />
2678
2678
  </picture>`;
2679
2679
  }
2680
- var cheerio66, MODERN_FORMATS3, LEGACY_FORMATS2, MAX_RECOMMENDED_DIMENSION;
2680
+ var cheerio67, MODERN_FORMATS3, LEGACY_FORMATS2, MAX_RECOMMENDED_DIMENSION;
2681
2681
  var init_image_optimization_analyzer = __esm({
2682
2682
  "src/analyzers/image-optimization-analyzer.ts"() {
2683
2683
  "use strict";
2684
- cheerio66 = __toESM(require("cheerio"));
2684
+ cheerio67 = __toESM(require("cheerio"));
2685
2685
  MODERN_FORMATS3 = ["webp", "avif"];
2686
2686
  LEGACY_FORMATS2 = ["jpg", "jpeg", "png", "gif", "bmp"];
2687
2687
  MAX_RECOMMENDED_DIMENSION = 2e3;
@@ -2722,7 +2722,7 @@ function analyzeAnchorText2(text) {
2722
2722
  };
2723
2723
  }
2724
2724
  function analyzeInternalLinking(html, url) {
2725
- const $ = cheerio67.load(html);
2725
+ const $ = cheerio68.load(html);
2726
2726
  const issues = [];
2727
2727
  const recommendations = [];
2728
2728
  let score = 100;
@@ -2961,11 +2961,11 @@ function suggestInternalLinks(content, availablePages) {
2961
2961
  }
2962
2962
  return suggestions.sort((a, b) => b.relevance - a.relevance).filter((s, i, arr) => arr.findIndex((x) => x.suggestedUrl === s.suggestedUrl) === i).slice(0, 10);
2963
2963
  }
2964
- var cheerio67, GENERIC_ANCHOR_TEXTS;
2964
+ var cheerio68, GENERIC_ANCHOR_TEXTS;
2965
2965
  var init_internal_linking_analyzer = __esm({
2966
2966
  "src/analyzers/internal-linking-analyzer.ts"() {
2967
2967
  "use strict";
2968
- cheerio67 = __toESM(require("cheerio"));
2968
+ cheerio68 = __toESM(require("cheerio"));
2969
2969
  GENERIC_ANCHOR_TEXTS = [
2970
2970
  "click here",
2971
2971
  "read more",
@@ -3159,7 +3159,7 @@ function analyzeMobileSpecific($) {
3159
3159
  };
3160
3160
  }
3161
3161
  function analyzeMobileSEO(html, url) {
3162
- const $ = cheerio68.load(html);
3162
+ const $ = cheerio69.load(html);
3163
3163
  const issues = [];
3164
3164
  const recommendations = [];
3165
3165
  let score = 100;
@@ -3290,11 +3290,11 @@ function analyzeMobileSEO(html, url) {
3290
3290
  recommendations
3291
3291
  };
3292
3292
  }
3293
- var cheerio68;
3293
+ var cheerio69;
3294
3294
  var init_mobile_seo_analyzer = __esm({
3295
3295
  "src/analyzers/mobile-seo-analyzer.ts"() {
3296
3296
  "use strict";
3297
- cheerio68 = __toESM(require("cheerio"));
3297
+ cheerio69 = __toESM(require("cheerio"));
3298
3298
  }
3299
3299
  });
3300
3300
 
@@ -4443,6 +4443,24 @@ var ISSUE_DEFINITIONS = {
4443
4443
  impact: "Your content will not be used for Bard/Gemini AI training (regular search unaffected).",
4444
4444
  howToFix: 'Remove "User-agent: Google-Extended Disallow: /" if you want Google AI visibility.'
4445
4445
  },
4446
+ CLOUDFLARE_AI_CRAWLER_GATE_AMBIGUOUS: {
4447
+ code: "CLOUDFLARE_AI_CRAWLER_GATE_AMBIGUOUS",
4448
+ severity: "warning",
4449
+ category: "ai-readiness",
4450
+ title: "No explicit AI crawler rules on a Cloudflare-fronted site",
4451
+ description: 'This site appears to be served through Cloudflare but robots.txt has no explicit Allow/Disallow rules for AI crawlers. Cloudflare blocks "mixed-use" AI crawlers by default on ad-hosting zones starting September 15, 2026, and is rolling out Pay Per Crawl / Pay Per Use gating beyond that.',
4452
+ impact: "Without an explicit rule, whether AI crawlers (and future citation opportunities in ChatGPT, Claude, Perplexity, etc.) can reach this site now depends on Cloudflare account-level bot-management defaults, not on this codebase \u2014 a silent, invisible-to-git failure mode.",
4453
+ howToFix: "Add explicit User-agent rules for GPTBot, Claude-Web/ClaudeBot, PerplexityBot, and Google-Extended in robots.txt, and confirm the matching allow/block posture in the Cloudflare dashboard under Bot Management \u2192 AI Crawl Control. If you'd rather charge than block, Cloudflare's Pay Per Crawl (AI Crawl Control \u2192 Payments tab) lets you set a per-crawl price instead of a flat allow/deny \u2014 worth a look if you get meaningful AI-crawler traffic."
4454
+ },
4455
+ NO_AGENT_EXPERIENCE_SURFACE: {
4456
+ code: "NO_AGENT_EXPERIENCE_SURFACE",
4457
+ severity: "notice",
4458
+ category: "ai-readiness",
4459
+ title: "No agent-facing discovery surface found",
4460
+ description: "None of the emerging AI-agent discovery conventions were found: llms-full.txt, SKILL.md, a discoverable MCP server, or an OpenAPI spec. This is a genuinely new, low-adoption category as of 2026 \u2014 most sites have none of these yet \u2014 so this is an opportunity, not a compliance failure.",
4461
+ impact: "AI agents (not just chatbots answering questions, but agents that browse and act on a user's behalf) increasingly prefer sites that expose machine-readable capabilities over ones that require scraping and guessing at HTML structure. Being an early, discoverable site in this space is a low-competition differentiator right now.",
4462
+ howToFix: "Start with whichever fits your site: llms-full.txt (a single Markdown dump of your key content \u2014 measured as fetched roughly 2x more than plain llms.txt), a SKILL.md capability manifest, or an OpenAPI spec at /openapi.json if you already have an API. A discoverable MCP server is the highest-effort, highest-payoff option if your product has one to expose."
4463
+ },
4446
4464
  HIGH_JS_RENDERING_RATIO: {
4447
4465
  code: "HIGH_JS_RENDERING_RATIO",
4448
4466
  severity: "warning",
@@ -7389,6 +7407,62 @@ async function checkInternalRedirects(internalLinks, batchSize = 10) {
7389
7407
  // src/audit/checks/ai-readiness.ts
7390
7408
  init_http();
7391
7409
  var cheerio13 = __toESM(require("cheerio"));
7410
+
7411
+ // src/audit/checks/agent-experience.ts
7412
+ init_http();
7413
+ var SIGNAL_PATHS = [
7414
+ { path: "/llms-full.txt", description: "Full-site content dump for AI agents \u2014 the fuller sibling of llms.txt, measured as fetched roughly 2x more often" },
7415
+ { path: "/skill.md", description: "SKILL.md capability manifest (Anthropic Agent Skills convention)" },
7416
+ { path: "/mcp", description: "MCP (Model Context Protocol) server, discoverable at a conventional path" },
7417
+ { path: "/.well-known/mcp.json", description: "MCP server descriptor at the .well-known convention" },
7418
+ { path: "/openapi.json", description: "OpenAPI spec \u2014 lets an agent discover and call your API directly instead of scraping HTML" },
7419
+ { path: "/.well-known/ai-plugin.json", description: "AI plugin manifest (older but still-referenced convention for agent tool discovery)" }
7420
+ ];
7421
+ async function fetchBody(baseUrl, path3) {
7422
+ try {
7423
+ const url = new URL(path3, baseUrl).href;
7424
+ const response = await httpGet(url, {
7425
+ timeout: 8e3,
7426
+ validateStatus: () => true
7427
+ });
7428
+ return { status: response.status, body: String(response.data ?? "") };
7429
+ } catch {
7430
+ return null;
7431
+ }
7432
+ }
7433
+ async function pathExists(baseUrl, path3, fallbackBody) {
7434
+ const result = await fetchBody(baseUrl, path3);
7435
+ if (!result) return false;
7436
+ if (result.status === 404) return false;
7437
+ if (result.status < 200 || result.status >= 400) return false;
7438
+ if (fallbackBody != null && result.body === fallbackBody) return false;
7439
+ return true;
7440
+ }
7441
+ async function checkAgentExperience(baseUrl) {
7442
+ const issues = [];
7443
+ const probePath = `/__rankcli_ax_probe_${Math.random().toString(36).slice(2)}`;
7444
+ const baseline = await fetchBody(baseUrl, probePath);
7445
+ const fallbackBody = baseline && baseline.status >= 200 && baseline.status < 400 ? baseline.body : null;
7446
+ const signals = await Promise.all(
7447
+ SIGNAL_PATHS.map(async ({ path: path3, description }) => ({
7448
+ path: path3,
7449
+ description,
7450
+ present: await pathExists(baseUrl, path3, fallbackBody)
7451
+ }))
7452
+ );
7453
+ const presentCount = signals.filter((s) => s.present).length;
7454
+ const score = Math.round(presentCount / signals.length * 100);
7455
+ if (presentCount === 0) {
7456
+ issues.push({
7457
+ ...ISSUE_DEFINITIONS.NO_AGENT_EXPERIENCE_SURFACE,
7458
+ affectedUrls: [baseUrl],
7459
+ details: { checkedPaths: SIGNAL_PATHS.map((s) => s.path) }
7460
+ });
7461
+ }
7462
+ return { issues, data: { signals, score } };
7463
+ }
7464
+
7465
+ // src/audit/checks/ai-readiness.ts
7392
7466
  var AI_BOTS = {
7393
7467
  GPTBot: "GPTBot",
7394
7468
  "ChatGPT-User": "ChatGPT-User",
@@ -7609,6 +7683,32 @@ function checkJSRenderingRatio(html, url) {
7609
7683
  data: { ratio, staticWordCount }
7610
7684
  };
7611
7685
  }
7686
+ async function checkCloudflareAICrawlerGate(baseUrl, botBlocking) {
7687
+ const issues = [];
7688
+ let behindCloudflare = false;
7689
+ try {
7690
+ const response = await httpGet(baseUrl, {
7691
+ timeout: 1e4,
7692
+ validateStatus: () => true
7693
+ });
7694
+ const server = response.headers["server"] || "";
7695
+ behindCloudflare = server.toLowerCase().includes("cloudflare") || "cf-ray" in response.headers;
7696
+ } catch {
7697
+ behindCloudflare = false;
7698
+ }
7699
+ const hasExplicitAIRules = botBlocking.robotsExists && botBlocking.blockedBots.length > 0;
7700
+ const ambiguous = behindCloudflare && !hasExplicitAIRules;
7701
+ if (ambiguous) {
7702
+ issues.push({
7703
+ ...ISSUE_DEFINITIONS.CLOUDFLARE_AI_CRAWLER_GATE_AMBIGUOUS,
7704
+ affectedUrls: [baseUrl]
7705
+ });
7706
+ }
7707
+ return {
7708
+ issues,
7709
+ data: { behindCloudflare, hasExplicitAIRules, ambiguous }
7710
+ };
7711
+ }
7612
7712
  async function runAIReadinessChecks(baseUrl, html) {
7613
7713
  const allIssues = [];
7614
7714
  const llmsResult = await checkLlmsTxt(baseUrl);
@@ -7617,12 +7717,18 @@ async function runAIReadinessChecks(baseUrl, html) {
7617
7717
  allIssues.push(...botResult.issues);
7618
7718
  const jsResult = checkJSRenderingRatio(html, baseUrl);
7619
7719
  allIssues.push(...jsResult.issues);
7720
+ const cfGateResult = await checkCloudflareAICrawlerGate(baseUrl, botResult.data);
7721
+ allIssues.push(...cfGateResult.issues);
7722
+ const agentExperienceResult = await checkAgentExperience(baseUrl);
7723
+ allIssues.push(...agentExperienceResult.issues);
7620
7724
  return {
7621
7725
  issues: allIssues,
7622
7726
  data: {
7623
7727
  llmsTxt: llmsResult.data,
7624
7728
  botBlocking: botResult.data,
7625
- jsRenderingRatio: jsResult.data.ratio
7729
+ jsRenderingRatio: jsResult.data.ratio,
7730
+ cloudflareAIGate: cfGateResult.data,
7731
+ agentExperience: agentExperienceResult.data
7626
7732
  }
7627
7733
  };
7628
7734
  }
@@ -16077,8 +16183,94 @@ function analyzeAIContentStructure(html, url) {
16077
16183
  };
16078
16184
  }
16079
16185
 
16080
- // src/audit/checks/citation-quality.ts
16186
+ // src/audit/checks/rag-chunk-readiness.ts
16081
16187
  var cheerio44 = __toESM(require("cheerio"));
16188
+ var MIN_GOOD_WORDS = 150;
16189
+ var MAX_GOOD_WORDS = 450;
16190
+ var TOKENS_PER_WORD = 1.33;
16191
+ var DANGLING_OPENERS = /^(this|it|these|that|they|such|those|the former|the latter)\b/i;
16192
+ function classifySize(wordCount) {
16193
+ if (wordCount < MIN_GOOD_WORDS) return "too-short";
16194
+ if (wordCount > MAX_GOOD_WORDS) return "too-long";
16195
+ return "good";
16196
+ }
16197
+ function extractSections($) {
16198
+ const elements = $("h1, h2, h3, p, li, blockquote").toArray();
16199
+ const sections = [];
16200
+ let current = null;
16201
+ for (const el of elements) {
16202
+ const tag = el.tagName?.toLowerCase();
16203
+ if (tag === "h1" || tag === "h2" || tag === "h3") {
16204
+ if (current) sections.push(current);
16205
+ current = { heading: $(el).text().trim(), headingLevel: parseInt(tag.slice(1), 10), text: "" };
16206
+ } else if (current) {
16207
+ current.text += " " + $(el).text();
16208
+ }
16209
+ }
16210
+ if (current) sections.push(current);
16211
+ return sections;
16212
+ }
16213
+ function analyzeRAGChunkReadiness(html, url) {
16214
+ const issues = [];
16215
+ const $ = cheerio44.load(html);
16216
+ $("nav, footer, aside, script, style, noscript, header").remove();
16217
+ const rawSections = extractSections($);
16218
+ const sections = rawSections.map((s) => {
16219
+ const words = s.text.trim().split(/\s+/).filter(Boolean);
16220
+ const wordCount = words.length;
16221
+ const estimatedTokens = Math.round(wordCount * TOKENS_PER_WORD);
16222
+ const firstSentence = s.text.trim().split(/[.!?]/)[0] || "";
16223
+ return {
16224
+ heading: s.heading,
16225
+ headingLevel: s.headingLevel,
16226
+ wordCount,
16227
+ estimatedTokens,
16228
+ sizeQuality: classifySize(wordCount),
16229
+ startsWithDanglingReference: DANGLING_OPENERS.test(firstSentence.trim())
16230
+ };
16231
+ });
16232
+ const totalSections = sections.length;
16233
+ const wellSizedSectionCount = sections.filter((s) => s.sizeQuality === "good").length;
16234
+ const danglingReferenceCount = sections.filter((s) => s.startsWithDanglingReference).length;
16235
+ const chunkReadinessScore = totalSections === 0 ? 0 : Math.round(
16236
+ (wellSizedSectionCount / totalSections * 0.7 + (totalSections - danglingReferenceCount) / totalSections * 0.3) * 100
16237
+ );
16238
+ if (totalSections >= 3 && wellSizedSectionCount / totalSections < 0.5) {
16239
+ const tooLong = sections.filter((s) => s.sizeQuality === "too-long").length;
16240
+ const tooShort = sections.filter((s) => s.sizeQuality === "too-short").length;
16241
+ issues.push({
16242
+ code: "RAG_CHUNK_SIZE_MISMATCH",
16243
+ severity: "notice",
16244
+ category: "ai-readiness",
16245
+ title: "Most sections are poorly sized for AI retrieval chunking",
16246
+ description: `${wellSizedSectionCount} of ${totalSections} sections fall in a well-sized range for RAG retrieval (roughly 150-450 words); ${tooLong} are too long and likely to get split mid-thought, ${tooShort} are too short and likely to get merged with unrelated neighbors.`,
16247
+ impact: "When an AI answer engine retrieves a chunk of this page in response to a query, oversized sections risk being cut off mid-point and undersized ones risk losing standalone context \u2014 both reduce the odds the retrieved chunk reads coherently in an AI-generated answer.",
16248
+ howToFix: "Break up long sections (450+ words) with an additional H2/H3 subheading roughly every 300-400 words. Merge very short sections (under 150 words) into a neighboring section or expand them with enough context to stand alone.",
16249
+ affectedUrls: [url],
16250
+ details: { wellSizedSectionCount, totalSections, tooLong, tooShort }
16251
+ });
16252
+ }
16253
+ if (totalSections >= 3 && danglingReferenceCount / totalSections > 0.3) {
16254
+ issues.push({
16255
+ code: "RAG_CHUNK_DANGLING_REFERENCES",
16256
+ severity: "notice",
16257
+ category: "ai-readiness",
16258
+ title: "Several sections open with a reference to prior context",
16259
+ description: `${danglingReferenceCount} of ${totalSections} sections start with a pronoun or demonstrative ("This...", "It...", "These...") that depends on the previous section to make sense.`,
16260
+ impact: "A RAG system that retrieves one of these sections on its own \u2014 which is exactly how retrieval works, one chunk at a time \u2014 surfaces a sentence whose subject is undefined, reading as broken or confusing in an AI-generated answer.",
16261
+ howToFix: 'Open each section by naming its actual subject instead of referring back to the previous one \u2014 e.g. "This approach reduces cost" becomes "Caching reduces cost."',
16262
+ affectedUrls: [url],
16263
+ details: { danglingReferenceCount, totalSections }
16264
+ });
16265
+ }
16266
+ return {
16267
+ issues,
16268
+ data: { sections, totalSections, wellSizedSectionCount, danglingReferenceCount, chunkReadinessScore }
16269
+ };
16270
+ }
16271
+
16272
+ // src/audit/checks/citation-quality.ts
16273
+ var cheerio45 = __toESM(require("cheerio"));
16082
16274
  var REPUTABLE_SOURCES = {
16083
16275
  academic: [
16084
16276
  "scholar.google.com",
@@ -16145,7 +16337,7 @@ var REPUTABLE_SOURCES = {
16145
16337
  };
16146
16338
  function analyzeCitationQuality(html, url) {
16147
16339
  const issues = [];
16148
- const $ = cheerio44.load(html);
16340
+ const $ = cheerio45.load(html);
16149
16341
  const parsedUrl = new URL(url);
16150
16342
  const currentDomain = parsedUrl.hostname;
16151
16343
  const $content = $("body").clone();
@@ -16374,12 +16566,12 @@ function analyzeCitationQuality(html, url) {
16374
16566
  }
16375
16567
 
16376
16568
  // src/audit/checks/answer-conciseness.ts
16377
- var cheerio45 = __toESM(require("cheerio"));
16569
+ var cheerio46 = __toESM(require("cheerio"));
16378
16570
  var IDEAL_ANSWER_LENGTH = { min: 40, max: 150 };
16379
16571
  var MAX_FIRST_SENTENCE = 200;
16380
16572
  function analyzeAnswerConciseness(html, url) {
16381
16573
  const issues = [];
16382
- const $ = cheerio45.load(html);
16574
+ const $ = cheerio46.load(html);
16383
16575
  $("nav, footer, aside, script, style, noscript, header").remove();
16384
16576
  const headings = $("h1, h2, h3, h4, h5, h6");
16385
16577
  let totalHeadings = 0;
@@ -16570,7 +16762,7 @@ function analyzeAnswerConciseness(html, url) {
16570
16762
  }
16571
16763
 
16572
16764
  // src/audit/checks/brand-mention-optimization.ts
16573
- var cheerio46 = __toESM(require("cheerio"));
16765
+ var cheerio47 = __toESM(require("cheerio"));
16574
16766
  var BRAND_DEFINITION_PATTERNS = [
16575
16767
  /(?:we are|we're|is a|is the|is an)\s+(?:leading|premier|top|best|trusted|innovative|professional)/i,
16576
16768
  /(?:our mission|our vision|we help|we provide|we offer|we specialize)/i,
@@ -16608,7 +16800,7 @@ var REVIEW_PLATFORMS = [
16608
16800
  ];
16609
16801
  function analyzeBrandMentionOptimization(html, url) {
16610
16802
  const issues = [];
16611
- const $ = cheerio46.load(html);
16803
+ const $ = cheerio47.load(html);
16612
16804
  const parsedUrl = new URL(url);
16613
16805
  const $content = $("body").clone();
16614
16806
  $content.find("nav, footer, script, style, noscript").remove();
@@ -16800,7 +16992,7 @@ function analyzeBrandMentionOptimization(html, url) {
16800
16992
  }
16801
16993
 
16802
16994
  // src/audit/checks/ai-citation-worthiness.ts
16803
- var cheerio47 = __toESM(require("cheerio"));
16995
+ var cheerio48 = __toESM(require("cheerio"));
16804
16996
  var ORIGINAL_DATA_PATTERNS = [
16805
16997
  /(?:our data|our research|our study|our analysis|we found|we discovered)/i,
16806
16998
  /(?:survey of|surveyed|interviewed|analyzed)\s+\d+/i,
@@ -16847,7 +17039,7 @@ var STOCK_IMAGE_DOMAINS = [
16847
17039
  ];
16848
17040
  function analyzeAICitationWorthiness(html, url) {
16849
17041
  const issues = [];
16850
- const $ = cheerio47.load(html);
17042
+ const $ = cheerio48.load(html);
16851
17043
  const $content = $("body").clone();
16852
17044
  $content.find("nav, footer, script, style, noscript, aside").remove();
16853
17045
  const bodyText = $content.text();
@@ -17072,7 +17264,7 @@ function analyzeAICitationWorthiness(html, url) {
17072
17264
  }
17073
17265
 
17074
17266
  // src/audit/checks/review-ecosystem.ts
17075
- var cheerio48 = __toESM(require("cheerio"));
17267
+ var cheerio49 = __toESM(require("cheerio"));
17076
17268
  var REVIEW_PLATFORMS2 = {
17077
17269
  general: [
17078
17270
  { name: "Google Business", domain: "google.com/maps", aliases: ["goo.gl/maps", "maps.google"] },
@@ -17132,7 +17324,7 @@ var TRUST_BADGE_PATTERNS2 = [
17132
17324
  ];
17133
17325
  function analyzeReviewEcosystem(html, url) {
17134
17326
  const issues = [];
17135
- const $ = cheerio48.load(html);
17327
+ const $ = cheerio49.load(html);
17136
17328
  const allLinks = $("a[href]").map((_, a) => $(a).attr("href") || "").get();
17137
17329
  const allLinksLower = allLinks.map((l) => l.toLowerCase());
17138
17330
  const linkedPlatforms = [];
@@ -17816,11 +18008,11 @@ function analyzeStructure(content) {
17816
18008
  }
17817
18009
 
17818
18010
  // src/audit/checks/html-compliance.ts
17819
- var cheerio49 = __toESM(require("cheerio"));
18011
+ var cheerio50 = __toESM(require("cheerio"));
17820
18012
  init_http();
17821
18013
  async function analyzeHtmlCompliance(html, url, headers) {
17822
18014
  const issues = [];
17823
- const $ = cheerio49.load(html);
18015
+ const $ = cheerio50.load(html);
17824
18016
  const parsedUrl = new URL(url);
17825
18017
  const doctypeMatch = html.match(/<!DOCTYPE\s+([^>]+)>/i);
17826
18018
  const hasDoctype = doctypeMatch !== null;
@@ -18386,7 +18578,7 @@ function getAssetType(url) {
18386
18578
  }
18387
18579
 
18388
18580
  // src/audit/checks/dom-size.ts
18389
- var cheerio50 = __toESM(require("cheerio"));
18581
+ var cheerio51 = __toESM(require("cheerio"));
18390
18582
  var THRESHOLDS = {
18391
18583
  totalElements: {
18392
18584
  warning: 1500,
@@ -18403,7 +18595,7 @@ var THRESHOLDS = {
18403
18595
  };
18404
18596
  function analyzeDomSize(html, url) {
18405
18597
  const issues = [];
18406
- const $ = cheerio50.load(html);
18598
+ const $ = cheerio51.load(html);
18407
18599
  const allElements = $("*");
18408
18600
  const totalElements = allElements.length;
18409
18601
  let maxDepth = 0;
@@ -18559,10 +18751,10 @@ function getDomReductionSuggestions(breakdown) {
18559
18751
  }
18560
18752
 
18561
18753
  // src/audit/checks/image-dimensions.ts
18562
- var cheerio51 = __toESM(require("cheerio"));
18754
+ var cheerio52 = __toESM(require("cheerio"));
18563
18755
  function analyzeImageDimensions(html, url) {
18564
18756
  const issues = [];
18565
- const $ = cheerio51.load(html);
18757
+ const $ = cheerio52.load(html);
18566
18758
  const images = $("img");
18567
18759
  const totalImages = images.length;
18568
18760
  let withDimensions = 0;
@@ -18673,7 +18865,7 @@ function truncateSrc(src) {
18673
18865
  }
18674
18866
 
18675
18867
  // src/audit/checks/color-contrast.ts
18676
- var cheerio52 = __toESM(require("cheerio"));
18868
+ var cheerio53 = __toESM(require("cheerio"));
18677
18869
  var KNOWN_LOW_CONTRAST_PAIRS = [
18678
18870
  { fg: "#999999", bg: "#ffffff", ratio: 2.85 },
18679
18871
  { fg: "#888888", bg: "#ffffff", ratio: 3.54 },
@@ -18708,7 +18900,7 @@ var NAMED_COLORS = {
18708
18900
  };
18709
18901
  function analyzeColorContrast(html, url) {
18710
18902
  const issues = [];
18711
- const $ = cheerio52.load(html);
18903
+ const $ = cheerio53.load(html);
18712
18904
  const potentialIssues = [];
18713
18905
  let elementsAnalyzed = 0;
18714
18906
  let passedChecks = 0;
@@ -18907,7 +19099,7 @@ function calculateContrastRatio(fg, bg) {
18907
19099
 
18908
19100
  // src/audit/checks/asset-minification.ts
18909
19101
  init_http();
18910
- var cheerio53 = __toESM(require("cheerio"));
19102
+ var cheerio54 = __toESM(require("cheerio"));
18911
19103
  function isMinified(content, type) {
18912
19104
  const lines = content.split("\n");
18913
19105
  const totalLines = lines.length;
@@ -18935,7 +19127,7 @@ function isMinified(content, type) {
18935
19127
  };
18936
19128
  }
18937
19129
  function extractAssetUrls(html, baseUrl) {
18938
- const $ = cheerio53.load(html);
19130
+ const $ = cheerio54.load(html);
18939
19131
  const base = new URL(baseUrl);
18940
19132
  const css = [];
18941
19133
  const js = [];
@@ -19081,10 +19273,10 @@ async function analyzeAssetMinification(html, url) {
19081
19273
  }
19082
19274
 
19083
19275
  // src/audit/checks/page-resources.ts
19084
- var cheerio54 = __toESM(require("cheerio"));
19276
+ var cheerio55 = __toESM(require("cheerio"));
19085
19277
  function analyzePageResources(html, url) {
19086
19278
  const issues = [];
19087
- const $ = cheerio54.load(html);
19279
+ const $ = cheerio55.load(html);
19088
19280
  const baseUrl = new URL(url);
19089
19281
  const baseHostname = baseUrl.hostname;
19090
19282
  const stylesheets = [];
@@ -19285,7 +19477,7 @@ function analyzePageResources(html, url) {
19285
19477
 
19286
19478
  // src/audit/checks/responsive-css.ts
19287
19479
  init_http();
19288
- var cheerio55 = __toESM(require("cheerio"));
19480
+ var cheerio56 = __toESM(require("cheerio"));
19289
19481
  function extractMediaQueries(css) {
19290
19482
  const mediaQueries = [];
19291
19483
  const regex = /@media\s*([^{]+)/g;
@@ -19335,7 +19527,7 @@ function classifyBreakpoints(mediaQueries) {
19335
19527
  }
19336
19528
  async function analyzeResponsiveCss(html, url) {
19337
19529
  const issues = [];
19338
- const $ = cheerio55.load(html);
19530
+ const $ = cheerio56.load(html);
19339
19531
  const baseUrl = new URL(url);
19340
19532
  const viewportMeta = $('meta[name="viewport"]').attr("content");
19341
19533
  const hasViewport = !!viewportMeta;
@@ -20073,10 +20265,10 @@ var urlSafetyDatabase = {
20073
20265
  };
20074
20266
 
20075
20267
  // src/audit/checks/tracking-verification.ts
20076
- var cheerio56 = __toESM(require("cheerio"));
20268
+ var cheerio57 = __toESM(require("cheerio"));
20077
20269
  function analyzeTrackingVerification(html, url) {
20078
20270
  const issues = [];
20079
- const $ = cheerio56.load(html);
20271
+ const $ = cheerio57.load(html);
20080
20272
  const ga4Match = html.match(/gtag\s*\(\s*['"]config['"]\s*,\s*['"](G-[A-Z0-9]+)['"]/i) || html.match(/googletagmanager\.com\/gtag\/js\?id=(G-[A-Z0-9]+)/i);
20081
20273
  const hasGa4 = !!ga4Match;
20082
20274
  const ga4MeasurementId = ga4Match ? ga4Match[1] : void 0;
@@ -20210,10 +20402,10 @@ async function runFullAudit(options) {
20210
20402
  const runExtendedChecks = tier !== "free";
20211
20403
  const parsedUrl = new URL(url);
20212
20404
  const domain = parsedUrl.hostname;
20213
- console.log(`
20405
+ console.error(`
20214
20406
  \u{1F50D} Running comprehensive SEO audit on ${url}...
20215
20407
  `);
20216
- console.log("\u{1F4CB} Phase 1: Crawlability checks + page fetch (parallel)...");
20408
+ console.error("\u{1F4CB} Phase 1: Crawlability checks + page fetch (parallel)...");
20217
20409
  const [crawlabilityResult, fetchResult] = await Promise.all([
20218
20410
  runCrawlabilityChecks(url).catch((err) => {
20219
20411
  console.error("Crawlability check failed:", err);
@@ -20248,7 +20440,7 @@ async function runFullAudit(options) {
20248
20440
  }
20249
20441
  const html = fetchResult.data;
20250
20442
  const headers = fetchResult.headers;
20251
- console.log(`\u{1F4DD} Phase 2: Running synchronous HTML checks (tier: ${tier}, limit: ${checksLimit})...`);
20443
+ console.error(`\u{1F4DD} Phase 2: Running synchronous HTML checks (tier: ${tier}, limit: ${checksLimit})...`);
20252
20444
  const onPageResult = analyzeOnPage(html, url);
20253
20445
  const structuredDataResult = analyzeStructuredData(html, url);
20254
20446
  const mobileResult = analyzeMobile(html, url);
@@ -20288,6 +20480,7 @@ async function runFullAudit(options) {
20288
20480
  const entityResult = runPremiumChecks ? analyzeEntitySEO(html, url) : { issues: [], data: {} };
20289
20481
  const qdfFreshnessResult = runPremiumChecks ? analyzeFreshnessSignals(html, url) : { issues: [], data: {} };
20290
20482
  const aiContentStructureResult = runPremiumChecks ? analyzeAIContentStructure(html, url) : { issues: [], data: {} };
20483
+ const ragChunkReadinessResult = runPremiumChecks ? analyzeRAGChunkReadiness(html, url) : { issues: [], data: {} };
20291
20484
  const citationQualityResult = runPremiumChecks ? analyzeCitationQuality(html, url) : { issues: [], data: {} };
20292
20485
  const answerConcisenessResult = runPremiumChecks ? analyzeAnswerConciseness(html, url) : { issues: [], data: {} };
20293
20486
  const brandMentionResult = runPremiumChecks ? analyzeBrandMentionOptimization(html, url) : { issues: [], data: {} };
@@ -20326,6 +20519,7 @@ async function runFullAudit(options) {
20326
20519
  ...entityResult.issues,
20327
20520
  ...qdfFreshnessResult.issues,
20328
20521
  ...aiContentStructureResult.issues,
20522
+ ...ragChunkReadinessResult.issues,
20329
20523
  ...citationQualityResult.issues,
20330
20524
  ...answerConcisenessResult.issues,
20331
20525
  ...brandMentionResult.issues,
@@ -20349,7 +20543,7 @@ async function runFullAudit(options) {
20349
20543
  };
20350
20544
  }
20351
20545
  }
20352
- console.log("\u{1F517} Phase 3: Running async checks (parallel)...");
20546
+ console.error("\u{1F517} Phase 3: Running async checks (parallel)...");
20353
20547
  const safeAsync = async (name, fn, timeoutMs = 1e4) => {
20354
20548
  try {
20355
20549
  const resultPromise = fn();
@@ -20462,7 +20656,7 @@ async function runFullAudit(options) {
20462
20656
  loadTime: perfData.loadTime,
20463
20657
  issues: allIssues.map((i) => i.code)
20464
20658
  });
20465
- console.log("\n\u2705 Audit complete!\n");
20659
+ console.error("\n\u2705 Audit complete!\n");
20466
20660
  return createReport(url, domain, allIssues, pages);
20467
20661
  }
20468
20662
  function createReport(url, domain, issues, pages) {
@@ -20623,10 +20817,10 @@ function groupIssuesByCategory(issues) {
20623
20817
  }
20624
20818
 
20625
20819
  // src/audit/checks/duplicate-content.ts
20626
- var cheerio57 = __toESM(require("cheerio"));
20820
+ var cheerio58 = __toESM(require("cheerio"));
20627
20821
  var import_crypto = require("crypto");
20628
20822
  function extractContentHash(html, url) {
20629
- const $ = cheerio57.load(html);
20823
+ const $ = cheerio58.load(html);
20630
20824
  $("script, style, nav, header, footer, aside, .nav, .header, .footer, .sidebar").remove();
20631
20825
  const title = $("title").text().trim();
20632
20826
  const bodyText = $("body").text().replace(/\s+/g, " ").trim();
@@ -21854,7 +22048,7 @@ var PRIORITY_WEIGHTS = {
21854
22048
  };
21855
22049
 
21856
22050
  // src/keywords/engine.ts
21857
- var cheerio59 = __toESM(require("cheerio"));
22051
+ var cheerio60 = __toESM(require("cheerio"));
21858
22052
  init_http();
21859
22053
 
21860
22054
  // src/keywords/prioritizer.ts
@@ -22151,7 +22345,7 @@ function enrichKeywordsWithEstimates(keywords) {
22151
22345
 
22152
22346
  // src/keywords/sources/free-sources.ts
22153
22347
  init_http();
22154
- var cheerio58 = __toESM(require("cheerio"));
22348
+ var cheerio59 = __toESM(require("cheerio"));
22155
22349
  var USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36";
22156
22350
  async function getPeopleAlsoAsk(query) {
22157
22351
  try {
@@ -22160,7 +22354,7 @@ async function getPeopleAlsoAsk(query) {
22160
22354
  headers: { "User-Agent": USER_AGENT },
22161
22355
  timeout: 1e4
22162
22356
  });
22163
- const $ = cheerio58.load(response.data);
22357
+ const $ = cheerio59.load(response.data);
22164
22358
  const questions = [];
22165
22359
  $("[data-sgrd]").each((_, el) => {
22166
22360
  const text = $(el).text().trim();
@@ -22186,7 +22380,7 @@ async function getRelatedSearches(query) {
22186
22380
  headers: { "User-Agent": USER_AGENT },
22187
22381
  timeout: 1e4
22188
22382
  });
22189
- const $ = cheerio58.load(response.data);
22383
+ const $ = cheerio59.load(response.data);
22190
22384
  const related = [];
22191
22385
  $("div[data-ved] a").each((_, el) => {
22192
22386
  const href = $(el).attr("href");
@@ -22276,7 +22470,7 @@ async function analyzeCompetitorTitles(query) {
22276
22470
  headers: { "User-Agent": USER_AGENT },
22277
22471
  timeout: 1e4
22278
22472
  });
22279
- const $ = cheerio58.load(response.data);
22473
+ const $ = cheerio59.load(response.data);
22280
22474
  const titles = [];
22281
22475
  const keywords = /* @__PURE__ */ new Set();
22282
22476
  $("h3").each((_, el) => {
@@ -22609,7 +22803,7 @@ async function fetchPageMeta(url) {
22609
22803
  const response = await httpGet(url, {
22610
22804
  timeout: 1e4
22611
22805
  });
22612
- const $ = cheerio59.load(response.data);
22806
+ const $ = cheerio60.load(response.data);
22613
22807
  return {
22614
22808
  url,
22615
22809
  title: $("title").text().trim() || void 0,
@@ -22740,7 +22934,7 @@ async function extractSeedKeywords(url) {
22740
22934
  const response = await httpGet(url, {
22741
22935
  timeout: 1e4
22742
22936
  });
22743
- const $ = cheerio59.load(response.data);
22937
+ const $ = cheerio60.load(response.data);
22744
22938
  const seeds = /* @__PURE__ */ new Set();
22745
22939
  const title = $("title").text().toLowerCase();
22746
22940
  const titleWords = title.split(/[\s\-|:]+/).filter((w) => w.length > 3);
@@ -22762,7 +22956,7 @@ async function extractSeedKeywords(url) {
22762
22956
  }
22763
22957
 
22764
22958
  // src/keywords/site-crawler.ts
22765
- var cheerio60 = __toESM(require("cheerio"));
22959
+ var cheerio61 = __toESM(require("cheerio"));
22766
22960
  init_http();
22767
22961
  var EXCLUDED_PATHS = [
22768
22962
  "/cdn-cgi/",
@@ -22875,7 +23069,7 @@ async function crawlPage(url, timeout) {
22875
23069
  validateStatus: (status) => status === 200
22876
23070
  });
22877
23071
  const html = response.data;
22878
- const $ = cheerio60.load(html);
23072
+ const $ = cheerio61.load(html);
22879
23073
  $('script, style, noscript, iframe, nav, footer, header, aside, [role="navigation"]').remove();
22880
23074
  const title = $("title").text().trim();
22881
23075
  const description = $('meta[name="description"]').attr("content")?.trim() || "";
@@ -25533,7 +25727,7 @@ function getDateRange(days = 28) {
25533
25727
 
25534
25728
  // src/keywords/sources/competitor-analysis.ts
25535
25729
  init_http();
25536
- var cheerio61 = __toESM(require("cheerio"));
25730
+ var cheerio62 = __toESM(require("cheerio"));
25537
25731
  var USER_AGENT2 = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36";
25538
25732
  async function discoverCompetitorKeywords(yourDomain, seedKeywords, competitors) {
25539
25733
  const yourKeywords = /* @__PURE__ */ new Set();
@@ -25630,7 +25824,7 @@ async function analyzeSERP(query) {
25630
25824
  headers: { "User-Agent": USER_AGENT2 },
25631
25825
  timeout: 1e4
25632
25826
  });
25633
- const $ = cheerio61.load(response.data);
25827
+ const $ = cheerio62.load(response.data);
25634
25828
  const results = [];
25635
25829
  const relatedSearches = [];
25636
25830
  const peopleAlsoAsk = [];