@rankcli/agent-runtime 0.0.13 → 0.0.15

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rankcli/agent-runtime",
-  "version": "0.0.13",
+  "version": "0.0.15",
   "description": "RankCLI agent runtime - executes SEO audits and fixes with AI",
   "homepage": "https://rankcli.dev",
   "main": "dist/index.js",
@@ -27,6 +27,7 @@
   },
   "dependencies": {
     "@anthropic-ai/sdk": "^0.52.0",
+    "@supabase/supabase-js": "^2.98.0",
     "cheerio": "1.0.0-rc.12",
     "openai": "^6.16.0",
     "yaml": "^2.3.0"

package/src/audit/checks/agent-experience.ts

@@ -0,0 +1,108 @@
+/**
+ * Agent Experience (AX) readiness
+ *
+ * Classic GEO asks "can an AI crawler read this page and cite it in an
+ * answer?" — that's what ai-readiness.ts covers. AX asks a different
+ * question: "can an AI AGENT (not just a chatbot) discover what this site
+ * can DO and act on it?" — a distinct, newer discipline (named tooling in
+ * this space: AgentGrade and similar agent-readiness scanners, 2026).
+ *
+ * This checks for the machine-readable discovery surface an agent looks
+ * for before it can act: llms-full.txt (the fuller, more-fetched sibling
+ * of llms.txt), a SKILL.md capability manifest, a discoverable MCP server,
+ * and an OpenAPI spec. None of these are required — most sites won't have
+ * any of them yet, this is genuinely emerging — so absence is reported as
+ * a low-severity opportunity, not a failure.
+ */
+
+import { httpGet } from '../../utils/http.js';
+import type { AuditIssue } from '../types.js';
+import { ISSUE_DEFINITIONS } from '../types.js';
+
+export interface AgentExperienceSignal {
+  path: string;
+  present: boolean;
+  description: string;
+}
+
+export interface AgentExperienceData {
+  signals: AgentExperienceSignal[];
+  score: number; // 0-100, percentage of signals present
+}
+
+const SIGNAL_PATHS: Array<{ path: string; description: string }> = [
+  { path: '/llms-full.txt', description: 'Full-site content dump for AI agents — the fuller sibling of llms.txt, measured as fetched roughly 2x more often' },
+  { path: '/skill.md', description: 'SKILL.md capability manifest (Anthropic Agent Skills convention)' },
+  { path: '/mcp', description: 'MCP (Model Context Protocol) server, discoverable at a conventional path' },
+  { path: '/.well-known/mcp.json', description: 'MCP server descriptor at the .well-known convention' },
+  { path: '/openapi.json', description: 'OpenAPI spec — lets an agent discover and call your API directly instead of scraping HTML' },
+  { path: '/.well-known/ai-plugin.json', description: 'AI plugin manifest (older but still-referenced convention for agent tool discovery)' },
+];
+
+async function fetchBody(baseUrl: string, path: string): Promise<{ status: number; body: string } | null> {
+  try {
+    const url = new URL(path, baseUrl).href;
+    const response = await httpGet<string>(url, {
+      timeout: 8000,
+      validateStatus: () => true,
+    });
+    return { status: response.status, body: String(response.data ?? '') };
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Many sites — anything client-side-routed (React/Vue/etc. SPAs in
+ * particular) — serve their app shell with HTTP 200 for EVERY unmatched
+ * path rather than a real 404. A naive "status 200 = found" check would
+ * report every single signal as present on such a site, which is worse
+ * than not checking at all — actively misleading rather than merely
+ * incomplete. Fetching one definitely-nonexistent path first and
+ * comparing every signal's response against it catches this: identical
+ * bytes back means "this is the fallback shell," not a real resource,
+ * regardless of what status code came back with it.
+ */
+async function pathExists(baseUrl: string, path: string, fallbackBody: string | null): Promise<boolean> {
+  const result = await fetchBody(baseUrl, path);
+  if (!result) return false;
+  if (result.status === 404) return false;
+  if (result.status < 200 || result.status >= 400) return false;
+  if (fallbackBody != null && result.body === fallbackBody) return false;
+  return true;
+}
+
+export async function checkAgentExperience(
+  baseUrl: string
+): Promise<{ issues: AuditIssue[]; data: AgentExperienceData }> {
+  const issues: AuditIssue[] = [];
+
+  const probePath = `/__rankcli_ax_probe_${Math.random().toString(36).slice(2)}`;
+  const baseline = await fetchBody(baseUrl, probePath);
+  const fallbackBody = baseline && baseline.status >= 200 && baseline.status < 400 ? baseline.body : null;
+
+  const signals = await Promise.all(
+    SIGNAL_PATHS.map(async ({ path, description }) => ({
+      path,
+      description,
+      present: await pathExists(baseUrl, path, fallbackBody),
+    }))
+  );
+
+  const presentCount = signals.filter((s) => s.present).length;
+  const score = Math.round((presentCount / signals.length) * 100);
+
+  // Only worth flagging when a site has essentially no agent-facing
+  // discovery surface at all — this is a forward-looking opportunity
+  // check, not a compliance requirement, so it stays low-severity and
+  // only fires below a meaningful threshold.
+  if (presentCount === 0) {
+    issues.push({
+      ...ISSUE_DEFINITIONS.NO_AGENT_EXPERIENCE_SURFACE,
+      affectedUrls: [baseUrl],
+      details: { checkedPaths: SIGNAL_PATHS.map((s) => s.path) },
+    });
+  }
+
+  return { issues, data: { signals, score } };
+}

package/src/audit/checks/ai-readiness.ts

@@ -5,6 +5,7 @@ import { httpGet } from '../../utils/http.js';
 import * as cheerio from 'cheerio';
 import type { AuditIssue } from '../types.js';
 import { ISSUE_DEFINITIONS } from '../types.js';
+import { checkAgentExperience, type AgentExperienceData } from './agent-experience.js';
 
 // Known AI bot user agents
 const AI_BOTS = {
@@ -34,10 +35,18 @@ export interface AIBotBlockingResult {
   allBlocked: boolean;
 }
 
+export interface CloudflareAICrawlerGateResult {
+  behindCloudflare: boolean;
+  hasExplicitAIRules: boolean;
+  ambiguous: boolean; // behind Cloudflare with no explicit AI crawler rules
+}
+
 export interface AIReadinessData {
   llmsTxt: LlmsTxtResult;
   botBlocking: AIBotBlockingResult;
   jsRenderingRatio: number; // 0-100%
+  cloudflareAIGate: CloudflareAICrawlerGateResult;
+  agentExperience: AgentExperienceData;
 }
 
 /**
@@ -307,6 +316,54 @@ export function checkJSRenderingRatio(
   };
 }
 
+/**
+ * Check whether a Cloudflare-fronted site has explicit AI crawler rules.
+ *
+ * Cloudflare blocks "mixed-use" AI crawlers by default on ad-hosting zones
+ * starting September 15, 2026, and is expanding Pay Per Crawl into a
+ * broader Pay Per Use gating model. A site with no explicit robots.txt
+ * stance on AI crawlers is now relying on whatever Cloudflare's
+ * account-level bot-management default happens to be — invisible from
+ * the codebase, and liable to change without a corresponding commit.
+ */
+export async function checkCloudflareAICrawlerGate(
+  baseUrl: string,
+  botBlocking: AIBotBlockingResult
+): Promise<{ issues: AuditIssue[]; data: CloudflareAICrawlerGateResult }> {
+  const issues: AuditIssue[] = [];
+
+  let behindCloudflare = false;
+  try {
+    const response = await httpGet<string>(baseUrl, {
+      timeout: 10000,
+      validateStatus: () => true,
+    });
+    const server = response.headers['server'] || '';
+    behindCloudflare = server.toLowerCase().includes('cloudflare') || 'cf-ray' in response.headers;
+  } catch {
+    behindCloudflare = false;
+  }
+
+  // "Explicit" means robots.txt exists and names at least one AI crawler
+  // specifically, rather than relying purely on a bare wildcard rule or
+  // the absence of a file.
+  const hasExplicitAIRules = botBlocking.robotsExists && botBlocking.blockedBots.length > 0;
+
+  const ambiguous = behindCloudflare && !hasExplicitAIRules;
+
+  if (ambiguous) {
+    issues.push({
+      ...ISSUE_DEFINITIONS.CLOUDFLARE_AI_CRAWLER_GATE_AMBIGUOUS,
+      affectedUrls: [baseUrl],
+    });
+  }
+
+  return {
+    issues,
+    data: { behindCloudflare, hasExplicitAIRules, ambiguous },
+  };
+}
+
 /**
  * Run all AI readiness checks
  */
@@ -328,12 +385,22 @@ export async function runAIReadinessChecks(
   const jsResult = checkJSRenderingRatio(html, baseUrl);
   allIssues.push(...jsResult.issues);
 
+  // Check Cloudflare AI-crawler-access gating posture
+  const cfGateResult = await checkCloudflareAICrawlerGate(baseUrl, botResult.data);
+  allIssues.push(...cfGateResult.issues);
+
+  // Check agent-facing discovery surface (llms-full.txt, SKILL.md, MCP, OpenAPI)
+  const agentExperienceResult = await checkAgentExperience(baseUrl);
+  allIssues.push(...agentExperienceResult.issues);
+
   return {
     issues: allIssues,
     data: {
       llmsTxt: llmsResult.data,
       botBlocking: botResult.data,
       jsRenderingRatio: jsResult.data.ratio,
+      cloudflareAIGate: cfGateResult.data,
+      agentExperience: agentExperienceResult.data,
     },
   };
 }

package/src/audit/checks/client-rendering.ts

@@ -312,17 +312,17 @@ function generateRecommendations(
 
     if (framework === 'React') {
       recommendations.push(
-        'Quick fix: Add react-snap to pre-render pages at build time (npm install -D react-snap, add "postbuild": "react-snap" to scripts)'
+        'Quick fix: Use Vike (vike.dev) for SSR/SSG - works with Vite, minimal config needed'
       );
       recommendations.push(
-        'Alternative: Use Vike (vite-plugin-ssr) for SSR/SSG without changing frameworks'
+        'Alternative: Migrate to Next.js or Remix for built-in SSR/SSG support'
       );
     } else if (framework === 'Vue') {
       recommendations.push(
-        'Quick fix: Add prerender-spa-plugin to pre-render pages at build time'
+        'Quick fix: Use Vike (vike.dev) for SSR/SSG - works with Vite, minimal config needed'
       );
       recommendations.push(
-        'Alternative: Use Vike (vite-plugin-ssr) for SSR/SSG without changing frameworks'
+        'Alternative: Migrate to Nuxt for built-in SSR/SSG support'
       );
     } else if (framework === 'Angular Universal') {
       recommendations.push(
@@ -330,7 +330,7 @@ function generateRecommendations(
       );
     } else {
       recommendations.push(
-        'Quick fix: Use a pre-rendering tool like react-snap or prerender-spa-plugin'
+        'Quick fix: Use Vike (vike.dev) for SSR/SSG with any Vite-based framework'
       );
     }
 
@@ -368,10 +368,10 @@ export function analyzeClientRendering(
   // Critical: Client-side only rendering
   if (analysis.renderingMethod === 'csr' && analysis.confidence !== 'low') {
     const howToFixByFramework = analysis.frameworkDetected === 'React'
-      ? 'Add react-snap to pre-render pages: npm install -D react-snap, then add "postbuild": "react-snap" to package.json scripts. No code changes needed.'
+      ? 'Use Vike (vike.dev) to add SSR/SSG to your Vite React app: npm install vike vike-react, then follow the setup guide. Alternatively, migrate to Next.js or Remix.'
       : analysis.frameworkDetected === 'Vue'
-      ? 'Add prerender-spa-plugin to pre-render pages at build time. Alternatively, use Vike for SSR/SSG.'
-      : 'Pre-render your pages using react-snap, prerender-spa-plugin, or similar build-time tools.';
+      ? 'Use Vike (vike.dev) to add SSR/SSG to your Vite Vue app: npm install vike vike-vue, then follow the setup guide. Alternatively, migrate to Nuxt.'
+      : 'Use Vike (vike.dev) for SSR/SSG with Vite-based frameworks, or use a framework with built-in SSR support (Next.js, Nuxt, Remix, SvelteKit).';
 
     issues.push({
       code: 'CLIENT_SIDE_RENDERING',
@@ -431,8 +431,8 @@ export function analyzeClientRendering(
         'Single Page Applications without SSR have slower time-to-content for search crawlers.',
       howToFix:
         analysis.frameworkDetected === 'React'
-          ? 'Quick fix: Add react-snap (npm install -D react-snap) to pre-render at build time. Alternative: Use Vike for SSR/SSG.'
-          : 'Quick fix: Add prerender-spa-plugin to pre-render at build time. Alternative: Use Vike for SSR/SSG.',
+          ? 'Add SSR/SSG using Vike (vike.dev): npm install vike vike-react. Alternative: migrate to Next.js or Remix.'
+          : 'Add SSR/SSG using Vike (vike.dev): npm install vike vike-vue. Alternative: migrate to Nuxt.',
       affectedUrls: [url],
       details: {
         framework: analysis.frameworkDetected,

package/src/audit/checks/rag-chunk-readiness.test.ts

@@ -0,0 +1,159 @@
+import { describe, it, expect } from 'vitest';
+import { analyzeRAGChunkReadiness } from './rag-chunk-readiness.js';
+
+const testUrl = 'https://example.com';
+
+function words(n: number, prefix = 'word'): string {
+  return Array.from({ length: n }, (_, i) => `${prefix}${i}`).join(' ');
+}
+
+describe('analyzeRAGChunkReadiness', () => {
+  it('returns zero sections and score 0 for a page with no headings', () => {
+    const html = `<html><body><p>${words(200)}</p></body></html>`;
+    const { data } = analyzeRAGChunkReadiness(html, testUrl);
+    expect(data.totalSections).toBe(0);
+    expect(data.chunkReadinessScore).toBe(0);
+  });
+
+  it('classifies a well-sized section (150-450 words) as good', () => {
+    const html = `<html><body><h2>Topic</h2><p>${words(300)}</p></body></html>`;
+    const { data } = analyzeRAGChunkReadiness(html, testUrl);
+    expect(data.totalSections).toBe(1);
+    expect(data.sections[0].wordCount).toBe(300);
+    expect(data.sections[0].sizeQuality).toBe('good');
+  });
+
+  it('classifies a very short section as too-short', () => {
+    const html = `<html><body><h2>Topic</h2><p>${words(20)}</p></body></html>`;
+    const { data } = analyzeRAGChunkReadiness(html, testUrl);
+    expect(data.sections[0].sizeQuality).toBe('too-short');
+  });
+
+  it('classifies a very long section as too-long', () => {
+    const html = `<html><body><h2>Topic</h2><p>${words(800)}</p></body></html>`;
+    const { data } = analyzeRAGChunkReadiness(html, testUrl);
+    expect(data.sections[0].sizeQuality).toBe('too-long');
+  });
+
+  it('splits content into sections at each h1/h2/h3, regardless of nesting depth', () => {
+    // Deliberately nested in wrapper divs — a sibling-only DOM walk would
+    // fail to associate the paragraphs with their heading here; document
+    // order should still get this right.
+    const html = `
+      <html><body>
+        <div class="section-wrapper">
+          <div class="inner"><h2>First</h2></div>
+          <div class="content"><p>${words(200, 'a')}</p></div>
+        </div>
+        <div class="section-wrapper">
+          <div class="inner"><h2>Second</h2></div>
+          <div class="content"><p>${words(200, 'b')}</p></div>
+        </div>
+      </body></html>
+    `;
+    const { data } = analyzeRAGChunkReadiness(html, testUrl);
+    expect(data.totalSections).toBe(2);
+    expect(data.sections[0].heading).toBe('First');
+    expect(data.sections[0].wordCount).toBe(200);
+    expect(data.sections[1].heading).toBe('Second');
+    expect(data.sections[1].wordCount).toBe(200);
+  });
+
+  it('accumulates multiple paragraphs and list items under one heading', () => {
+    const html = `
+      <html><body>
+        <h2>Topic</h2>
+        <p>${words(100, 'a')}</p>
+        <p>${words(100, 'b')}</p>
+        <ul><li>${words(50, 'c')}</li><li>${words(50, 'd')}</li></ul>
+      </body></html>
+    `;
+    const { data } = analyzeRAGChunkReadiness(html, testUrl);
+    expect(data.totalSections).toBe(1);
+    expect(data.sections[0].wordCount).toBe(300);
+  });
+
+  it('content before the first heading is not attributed to any section', () => {
+    const html = `
+      <html><body>
+        <p>${words(500)}</p>
+        <h2>Topic</h2>
+        <p>${words(200)}</p>
+      </body></html>
+    `;
+    const { data } = analyzeRAGChunkReadiness(html, testUrl);
+    expect(data.totalSections).toBe(1);
+    expect(data.sections[0].wordCount).toBe(200);
+  });
+
+  it('detects a dangling-reference opener', () => {
+    const html = `<html><body><h2>Topic</h2><p>This approach reduces cost significantly. ${words(150)}</p></body></html>`;
+    const { data } = analyzeRAGChunkReadiness(html, testUrl);
+    expect(data.sections[0].startsWithDanglingReference).toBe(true);
+  });
+
+  it('does not flag a section that names its subject', () => {
+    const html = `<html><body><h2>Topic</h2><p>Caching reduces cost significantly. ${words(150)}</p></body></html>`;
+    const { data } = analyzeRAGChunkReadiness(html, testUrl);
+    expect(data.sections[0].startsWithDanglingReference).toBe(false);
+  });
+
+  it('raises RAG_CHUNK_SIZE_MISMATCH when most sections are poorly sized', () => {
+    const html = `
+      <html><body>
+        <h2>One</h2><p>${words(20)}</p>
+        <h2>Two</h2><p>${words(900)}</p>
+        <h2>Three</h2><p>${words(15)}</p>
+      </body></html>
+    `;
+    const { issues } = analyzeRAGChunkReadiness(html, testUrl);
+    expect(issues.some((i) => i.code === 'RAG_CHUNK_SIZE_MISMATCH')).toBe(true);
+  });
+
+  it('does not raise RAG_CHUNK_SIZE_MISMATCH when most sections are well-sized', () => {
+    const html = `
+      <html><body>
+        <h2>One</h2><p>${words(200)}</p>
+        <h2>Two</h2><p>${words(250)}</p>
+        <h2>Three</h2><p>${words(300)}</p>
+      </body></html>
+    `;
+    const { issues } = analyzeRAGChunkReadiness(html, testUrl);
+    expect(issues.some((i) => i.code === 'RAG_CHUNK_SIZE_MISMATCH')).toBe(false);
+  });
+
+  it('raises RAG_CHUNK_DANGLING_REFERENCES when over 30% of sections open with one', () => {
+    const html = `
+      <html><body>
+        <h2>One</h2><p>This reduces cost. ${words(150)}</p>
+        <h2>Two</h2><p>It also improves speed. ${words(150)}</p>
+        <h2>Three</h2><p>Caching is the mechanism. ${words(150)}</p>
+      </body></html>
+    `;
+    const { issues } = analyzeRAGChunkReadiness(html, testUrl);
+    expect(issues.some((i) => i.code === 'RAG_CHUNK_DANGLING_REFERENCES')).toBe(true);
+  });
+
+  it('does not raise issues for a page with too few sections to judge (under 3)', () => {
+    const html = `<html><body><h2>One</h2><p>${words(20)}</p><h2>Two</h2><p>${words(900)}</p></body></html>`;
+    const { issues } = analyzeRAGChunkReadiness(html, testUrl);
+    expect(issues.length).toBe(0);
+  });
+
+  it('computes a chunkReadinessScore of 100 when all sections are good and self-contained', () => {
+    const html = `
+      <html><body>
+        <h2>One</h2><p>Caching reduces latency. ${words(200)}</p>
+        <h2>Two</h2><p>Compression reduces bandwidth. ${words(200)}</p>
+      </body></html>
+    `;
+    const { data } = analyzeRAGChunkReadiness(html, testUrl);
+    expect(data.chunkReadinessScore).toBe(100);
+  });
+
+  it('estimates tokens at roughly 1.33 tokens per word', () => {
+    const html = `<html><body><h2>Topic</h2><p>${words(300)}</p></body></html>`;
+    const { data } = analyzeRAGChunkReadiness(html, testUrl);
+    expect(data.sections[0].estimatedTokens).toBe(399); // round(300 * 1.33)
+  });
+});

package/src/audit/checks/rag-chunk-readiness.ts

@@ -0,0 +1,163 @@
+/**
+ * RAG Chunk Readiness
+ *
+ * How AI answer engines actually retrieve content: RAG (Retrieval-Augmented
+ * Generation) systems chunk a page into pieces — the 2026 convergence
+ * benchmark is roughly 400-512 tokens per chunk, 10-25% overlap — then
+ * retrieve whichever chunk(s) best match a query, not the whole page. A page
+ * whose own heading-delimited sections are naturally chunk-sized, with clear
+ * headings and paragraphs that don't lean on the previous section's context
+ * to make sense, is less likely to get split mid-thought by a RAG chunker —
+ * preserving the semantic boundaries the author actually wrote, which is
+ * what the RAG-chunking literature credits with better retrieval recall.
+ *
+ * This is a genuinely different angle from ai-content-structure.ts (tables/
+ * lists/Q&A formatting): that's about FORMAT, this is about SECTION SIZE
+ * and SELF-CONTAINMENT. Novel enough that no dedicated site-auditing tool
+ * for it was found as of this writing — RAG-chunking guidance that exists
+ * targets people building RAG systems, not people writing web content.
+ */
+
+import * as cheerio from 'cheerio';
+import type { AuditIssue } from '../types.js';
+
+export interface ChunkSection {
+  heading: string;
+  headingLevel: number;
+  wordCount: number;
+  estimatedTokens: number;
+  sizeQuality: 'too-short' | 'good' | 'too-long';
+  startsWithDanglingReference: boolean;
+}
+
+export interface RAGChunkReadinessData {
+  sections: ChunkSection[];
+  totalSections: number;
+  wellSizedSectionCount: number;
+  danglingReferenceCount: number;
+  chunkReadinessScore: number; // 0-100
+}
+
+// 400-512 tokens is the 2026 RAG-chunking convergence benchmark. English
+// prose runs roughly 1.3 words per token (~0.75 tokens/word), so that's
+// roughly 300-385 words — widened to 150-450 here since this measures how
+// naturally an AUTHOR sectioned their content, not what a chunker itself
+// targets; a section in this band is unlikely to need mid-thought splitting
+// either way, which is the property that actually matters.
+const MIN_GOOD_WORDS = 150;
+const MAX_GOOD_WORDS = 450;
+const TOKENS_PER_WORD = 1.33;
+
+// A section opening on a bare pronoun/demonstrative reference ("This
+// approach...", "It also...") reads fine in place but loses its antecedent
+// the moment a RAG system retrieves it standalone — a real self-containment
+// signal, not a grammar nitpick.
+const DANGLING_OPENERS = /^(this|it|these|that|they|such|those|the former|the latter)\b/i;
+
+function classifySize(wordCount: number): ChunkSection['sizeQuality'] {
+  if (wordCount < MIN_GOOD_WORDS) return 'too-short';
+  if (wordCount > MAX_GOOD_WORDS) return 'too-long';
+  return 'good';
+}
+
+/**
+ * Splits content into heading-delimited sections by walking all
+ * heading/content elements in DOCUMENT ORDER (a single multi-tag cheerio
+ * selector), rather than via sibling traversal — sibling-only walks break
+ * on the wrapper-div-heavy DOM nesting common in modern frameworks, where
+ * a heading and "its" paragraphs aren't actually direct siblings. Document
+ * order is robust to arbitrary nesting depth.
+ */
+function extractSections($: cheerio.CheerioAPI): Array<{ heading: string; headingLevel: number; text: string }> {
+  const elements = $('h1, h2, h3, p, li, blockquote').toArray();
+  const sections: Array<{ heading: string; headingLevel: number; text: string }> = [];
+  let current: { heading: string; headingLevel: number; text: string } | null = null;
+
+  for (const el of elements) {
+    const tag = (el as { tagName?: string }).tagName?.toLowerCase();
+    if (tag === 'h1' || tag === 'h2' || tag === 'h3') {
+      if (current) sections.push(current);
+      current = { heading: $(el).text().trim(), headingLevel: parseInt(tag.slice(1), 10), text: '' };
+    } else if (current) {
+      current.text += ' ' + $(el).text();
+    }
+    // Content before the first heading has no section to attribute it to —
+    // intentionally skipped rather than inventing a synthetic "intro" chunk.
+  }
+  if (current) sections.push(current);
+  return sections;
+}
+
+export function analyzeRAGChunkReadiness(
+  html: string,
+  url: string
+): { issues: AuditIssue[]; data: RAGChunkReadinessData } {
+  const issues: AuditIssue[] = [];
+  const $ = cheerio.load(html);
+  $('nav, footer, aside, script, style, noscript, header').remove();
+
+  const rawSections = extractSections($);
+
+  const sections: ChunkSection[] = rawSections.map((s) => {
+    const words = s.text.trim().split(/\s+/).filter(Boolean);
+    const wordCount = words.length;
+    const estimatedTokens = Math.round(wordCount * TOKENS_PER_WORD);
+    const firstSentence = s.text.trim().split(/[.!?]/)[0] || '';
+    return {
+      heading: s.heading,
+      headingLevel: s.headingLevel,
+      wordCount,
+      estimatedTokens,
+      sizeQuality: classifySize(wordCount),
+      startsWithDanglingReference: DANGLING_OPENERS.test(firstSentence.trim()),
+    };
+  });
+
+  const totalSections = sections.length;
+  const wellSizedSectionCount = sections.filter((s) => s.sizeQuality === 'good').length;
+  const danglingReferenceCount = sections.filter((s) => s.startsWithDanglingReference).length;
+
+  const chunkReadinessScore =
+    totalSections === 0
+      ? 0
+      : Math.round(
+          ((wellSizedSectionCount / totalSections) * 0.7 +
+            ((totalSections - danglingReferenceCount) / totalSections) * 0.3) *
+            100
+        );
+
+  if (totalSections >= 3 && wellSizedSectionCount / totalSections < 0.5) {
+    const tooLong = sections.filter((s) => s.sizeQuality === 'too-long').length;
+    const tooShort = sections.filter((s) => s.sizeQuality === 'too-short').length;
+    issues.push({
+      code: 'RAG_CHUNK_SIZE_MISMATCH',
+      severity: 'notice',
+      category: 'ai-readiness',
+      title: 'Most sections are poorly sized for AI retrieval chunking',
+      description: `${wellSizedSectionCount} of ${totalSections} sections fall in a well-sized range for RAG retrieval (roughly 150-450 words); ${tooLong} are too long and likely to get split mid-thought, ${tooShort} are too short and likely to get merged with unrelated neighbors.`,
+      impact: 'When an AI answer engine retrieves a chunk of this page in response to a query, oversized sections risk being cut off mid-point and undersized ones risk losing standalone context — both reduce the odds the retrieved chunk reads coherently in an AI-generated answer.',
+      howToFix: 'Break up long sections (450+ words) with an additional H2/H3 subheading roughly every 300-400 words. Merge very short sections (under 150 words) into a neighboring section or expand them with enough context to stand alone.',
+      affectedUrls: [url],
+      details: { wellSizedSectionCount, totalSections, tooLong, tooShort },
+    });
+  }
+
+  if (totalSections >= 3 && danglingReferenceCount / totalSections > 0.3) {
+    issues.push({
+      code: 'RAG_CHUNK_DANGLING_REFERENCES',
+      severity: 'notice',
+      category: 'ai-readiness',
+      title: 'Several sections open with a reference to prior context',
+      description: `${danglingReferenceCount} of ${totalSections} sections start with a pronoun or demonstrative ("This...", "It...", "These...") that depends on the previous section to make sense.`,
+      impact: 'A RAG system that retrieves one of these sections on its own — which is exactly how retrieval works, one chunk at a time — surfaces a sentence whose subject is undefined, reading as broken or confusing in an AI-generated answer.',
+      howToFix: 'Open each section by naming its actual subject instead of referring back to the previous one — e.g. "This approach reduces cost" becomes "Caching reduces cost."',
+      affectedUrls: [url],
+      details: { danglingReferenceCount, totalSections },
+    });
+  }
+
+  return {
+    issues,
+    data: { sections, totalSections, wellSizedSectionCount, danglingReferenceCount, chunkReadinessScore },
+  };
+}

package/src/audit/checks/security-headers.ts

@@ -59,7 +59,15 @@ export async function analyzeSecurityHeaders(url: string): Promise<{ issues: Aud
     });
 
     const headers = response.headers;
-    const isHttps = url.startsWith('https://');
+    // Use URL parsing for robust protocol detection (handles case variations like Https://)
+    let isHttps = false;
+    try {
+      const parsedUrl = new URL(url);
+      isHttps = parsedUrl.protocol === 'https:';
+    } catch {
+      // If URL parsing fails, fall back to startsWith check
+      isHttps = url.toLowerCase().startsWith('https://');
+    }
 
     // Extract security headers (case-insensitive)
     const getHeader = (name: string): string | null => {
@@ -151,10 +159,18 @@ export async function analyzeSecurityHeaders(url: string): Promise<{ issues: Aud
       },
     };
   } catch (error) {
+    // Use URL parsing for robust protocol detection in error case too
+    let isHttps = false;
+    try {
+      const parsedUrl = new URL(url);
+      isHttps = parsedUrl.protocol === 'https:';
+    } catch {
+      isHttps = url.toLowerCase().startsWith('https://');
+    }
     return {
       issues,
       data: {
-        https: url.startsWith('https://'),
+        https: isHttps,
         headers: {
           hsts: null,
           csp: null,