@readme/cli 0.0.26 → 0.0.29

package/src/utils/llms.test.js

@@ -0,0 +1,257 @@
+import { test } from 'node:test'
+import assert from 'node:assert/strict'
+import { parseLlmsTxt, analyzeLlmsTxt } from './llms.js'
+
+const BASE = 'https://docs.example.com/llms.txt'
+
+// ---------------------------------------------------------------------------
+// parseLlmsTxt — root-relative resolution
+// ---------------------------------------------------------------------------
+
+test('parseLlmsTxt: root-relative links resolve from the origin by default', () => {
+  const parsed = parseLlmsTxt(
+    `# Example\n\n## Guides\n- [Overview](/overview.md)\n- [Install](/docs/install.md)\n`,
+    'https://example.com/docs/llms.txt',
+  )
+
+  const urls = parsed.sections[0].items.map((item) => item.url)
+  assert.deepEqual(urls, [
+    'https://example.com/overview.md',
+    'https://example.com/docs/install.md',
+  ])
+})
+
+test('parseLlmsTxt: explicit llms-dir policy resolves nested root-relative page links from the llms.txt directory', () => {
+  const parsed = parseLlmsTxt(
+    `# Example\n\n## Guides\n- [Overview](/overview.md)\n- [API](/api/reference.md)\n- [Absolute](https://example.com/reference/openapi.md)\n`,
+    'https://example.com/docs/llms.txt',
+    { rootRelativeResolution: 'llms-dir' },
+  )
+
+  const urls = parsed.sections[0].items.map((item) => item.url)
+  assert.deepEqual(urls, [
+    'https://example.com/docs/overview.md',
+    'https://example.com/docs/api/reference.md',
+    'https://example.com/reference/openapi.md',
+  ])
+})
+
+test('parseLlmsTxt: llms-dir policy preserves links already scoped to the llms.txt directory', () => {
+  const parsed = parseLlmsTxt(
+    `# Example\n\n## Guides\n- [Already scoped](/docs/reference.md)\n- [Deep already scoped](/docs/api/reference.md?version=1#auth)\n`,
+    'https://example.com/docs/llms.txt',
+    { rootRelativeResolution: 'llms-dir' },
+  )
+
+  const urls = parsed.sections[0].items.map((item) => item.url)
+  assert.deepEqual(urls, [
+    'https://example.com/docs/reference.md',
+    'https://example.com/docs/api/reference.md?version=1#auth',
+  ])
+})
+
+test('parseLlmsTxt: llms-dir policy does not rewrite explicit nested llms.txt links', () => {
+  const parsed = parseLlmsTxt(
+    `# Example\n\n## Guides\n- [API llms](/api/llms.txt)\n- [Docs API llms](/docs/api/llms.txt)\n`,
+    'https://example.com/docs/llms.txt',
+    { rootRelativeResolution: 'llms-dir' },
+  )
+
+  const urls = parsed.sections[0].items.map((item) => item.url)
+  assert.deepEqual(urls, [
+    'https://example.com/api/llms.txt',
+    'https://example.com/docs/api/llms.txt',
+  ])
+})
+
+test('parseLlmsTxt: llms-dir policy preserves root-scoped non-page files', () => {
+  const parsed = parseLlmsTxt(
+    `# Example\n\n## Guides\n- [Overview](/overview.md)\n- [OpenAPI](/reference/openapi.yaml)\n- [Image](/assets/logo.png)\n`,
+    'https://example.com/docs/llms.txt',
+    { rootRelativeResolution: 'llms-dir' },
+  )
+
+  const urls = parsed.sections[0].items.map((item) => item.url)
+  assert.deepEqual(urls, [
+    'https://example.com/docs/overview.md',
+    'https://example.com/reference/openapi.yaml',
+    'https://example.com/assets/logo.png',
+  ])
+})
+
+test('parseLlmsTxt: llms-dir policy handles query and hash page links', () => {
+  const parsed = parseLlmsTxt(
+    `# Example\n\n## Guides\n- [Overview](/overview.md?tab=js#install)\n- [No extension](/get-started?tab=js#install)\n- [Hash only](#local)\n`,
+    'https://example.com/docs/llms.txt',
+    { rootRelativeResolution: 'llms-dir' },
+  )
+
+  const urls = parsed.sections[0].items.map((item) => item.url)
+  assert.deepEqual(urls, [
+    'https://example.com/docs/overview.md?tab=js#install',
+    'https://example.com/docs/get-started?tab=js#install',
+  ])
+})
+
+test('parseLlmsTxt: malformed and unsupported links are ignored', () => {
+  const parsed = parseLlmsTxt(
+    `# Example\n\n## Guides\n- [Mail](mailto:test@example.com)\n- [JS](javascript:alert(1))\n- [OK](/ok.md)\n`,
+    'https://example.com/docs/llms.txt',
+    { rootRelativeResolution: 'llms-dir' },
+  )
+
+  const urls = parsed.sections[0].items.map((item) => item.url)
+  assert.deepEqual(urls, ['https://example.com/docs/ok.md'])
+})
+
+// ---------------------------------------------------------------------------
+// parseLlmsTxt — standard H2 sections
+// ---------------------------------------------------------------------------
+
+test('parseLlmsTxt: parses H1 title and H2 sections', () => {
+  const body = [
+    '# My Docs',
+    '',
+    '## Getting Started',
+    '- [Overview](https://docs.example.com/overview)',
+    '- [Quickstart](https://docs.example.com/quickstart)',
+    '',
+    '## Reference',
+    '- [API](https://docs.example.com/api)',
+  ].join('\n')
+
+  const result = parseLlmsTxt(body, BASE)
+  assert.equal(result.title, 'My Docs')
+  assert.equal(result.sections.length, 2)
+  assert.equal(result.sections[0].title, 'Getting Started')
+  assert.equal(result.sections[0].items.length, 2)
+  assert.equal(result.sections[1].title, 'Reference')
+  assert.equal(result.h3Fallback, undefined)
+})
+
+test('parseLlmsTxt: items with no H2 fall into implicit Resources section', () => {
+  const body = [
+    '# My Docs',
+    '- [Overview](https://docs.example.com/overview)',
+  ].join('\n')
+
+  const result = parseLlmsTxt(body, BASE)
+  assert.equal(result.sections.length, 1)
+  assert.equal(result.sections[0].title, 'Resources')
+})
+
+// ---------------------------------------------------------------------------
+// parseLlmsTxt — H3 fallback
+// ---------------------------------------------------------------------------
+
+// Generates a Couchbase-style body: one H2 bucket, N H3 sub-sections each
+// with `itemsPerSection` pages. With itemsPerSection >= 68, the single H2
+// section exceeds MAX_SECTION_ITEMS (200) and triggers the H3 fallback.
+function makeCouchbaseBody(sections, itemsPerSection) {
+  const lines = ['# Couchbase', '', '> Official Couchbase docs.', '', '## Docs', '']
+  for (const title of sections) {
+    lines.push(`### ${title}`)
+    for (let i = 0; i < itemsPerSection; i++) {
+      const slug = title.toLowerCase().replace(/[^a-z0-9]+/g, '-')
+      lines.push(`- [Page ${i}](https://docs.couchbase.com/${slug}/page-${i})`)
+    }
+    lines.push('')
+  }
+  return lines.join('\n')
+}
+
+const CB_SECTIONS = ['.NET Analytics SDK (1.0)', '.NET Entity Framework (1.0)', '.NET SDK (3.9)']
+
+test('parseLlmsTxt: falls back to H3 when single H2 section is oversized', () => {
+  const manyItems = Array.from({ length: 201 }, (_, i) =>
+    `- [Page ${i}](https://docs.example.com/page-${i})`
+  ).join('\n')
+
+  const body = [
+    '# Big Site',
+    '',
+    '## Docs',
+    manyItems,
+    '',
+    '### Section A',
+    '- [A1](https://docs.example.com/a1)',
+    '- [A2](https://docs.example.com/a2)',
+    '',
+    '### Section B',
+    '- [B1](https://docs.example.com/b1)',
+  ].join('\n')
+
+  const result = parseLlmsTxt(body, BASE)
+  assert.equal(result.h3Fallback, true)
+  assert.ok(result.sections.length > 1, `expected >1 section, got ${result.sections.length}`)
+
+  const titles = result.sections.map((s) => s.title)
+  assert.ok(titles.includes('Section A'))
+  assert.ok(titles.includes('Section B'))
+})
+
+test('parseLlmsTxt: Couchbase-style H3 structure produces one section per SDK', () => {
+  const body = makeCouchbaseBody(CB_SECTIONS, 70)
+  const result = parseLlmsTxt(body, 'https://docs.couchbase.com/llms.txt')
+
+  assert.equal(result.h3Fallback, true)
+  assert.equal(result.title, 'Couchbase')
+  assert.equal(result.sections.length, 3)
+  assert.equal(result.sections[0].title, '.NET Analytics SDK (1.0)')
+  assert.equal(result.sections[1].title, '.NET Entity Framework (1.0)')
+  assert.equal(result.sections[2].title, '.NET SDK (3.9)')
+  assert.equal(result.sections[0].items.length, 70)
+})
+
+test('parseLlmsTxt: does NOT use H3 fallback when H2 sections are usable', () => {
+  const body = [
+    '# Site',
+    '',
+    '## Section A',
+    '- [A1](https://docs.example.com/a1)',
+    '- [A2](https://docs.example.com/a2)',
+    '',
+    '## Section B',
+    '- [B1](https://docs.example.com/b1)',
+    '',
+    '### Sub B',
+    '- [B2](https://docs.example.com/b2)',
+  ].join('\n')
+
+  const result = parseLlmsTxt(body, BASE)
+  assert.equal(result.h3Fallback, undefined)
+  assert.equal(result.sections.length, 2)
+  assert.equal(result.sections[0].title, 'Section A')
+  assert.equal(result.sections[1].title, 'Section B')
+})
+
+test('parseLlmsTxt: does NOT use H3 fallback when no H3 headings exist', () => {
+  const manyItems = Array.from({ length: 201 }, (_, i) =>
+    `- [Page ${i}](https://docs.example.com/page-${i})`
+  ).join('\n')
+
+  const body = ['# Big Site', '', '## Docs', manyItems].join('\n')
+
+  const result = parseLlmsTxt(body, BASE)
+  assert.equal(result.h3Fallback, undefined)
+  assert.equal(result.sections.length, 1)
+  assert.equal(result.sections[0].title, 'Docs')
+})
+
+// ---------------------------------------------------------------------------
+// analyzeLlmsTxt
+// ---------------------------------------------------------------------------
+
+test('analyzeLlmsTxt: marks usable when it has link items', () => {
+  const body = '## Section\n- [Page](https://docs.example.com/page)'
+  const result = analyzeLlmsTxt(body, BASE)
+  assert.equal(result.usable, true)
+  assert.equal(result.reason, null)
+})
+
+test('analyzeLlmsTxt: marks not usable when no link items', () => {
+  const body = '# Just a title\n\nSome prose with no links.'
+  const result = analyzeLlmsTxt(body, BASE)
+  assert.equal(result.usable, false)
+  assert.ok(result.reason)
+})

package/src/utils/marketplace.js

@@ -0,0 +1,208 @@
+/**
+ * Talks to the readmeio/marketplace repo: lists the available MDX components,
+ * fetches a single component's files (the .mdx source + its readme.md), and
+ * caches everything on disk so we're not hitting GitHub on every command.
+ *
+ * Components are read at https://github.com/readmeio/marketplace/tree/main/components
+ * and copied into the user's local `components/` folder. The marketplace
+ * `.mdx` files are the canonical source — they hold the JSX export plus an
+ * example invocation at the bottom.
+ */
+
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+
+const REPO = 'readmeio/marketplace';
+const BRANCH = 'main';
+const COMPONENTS_PATH = 'components';
+const CACHE_TTL_MS = 24 * 60 * 60 * 1000; // 24h
+
+const CACHE_DIR = path.join(os.homedir(), '.cache', 'readme-cli', 'marketplace');
+
+function ensureCacheDir() {
+  fs.mkdirSync(CACHE_DIR, { recursive: true });
+}
+
+function cachePath(name) {
+  return path.join(CACHE_DIR, name);
+}
+
+function readCache(name, maxAgeMs = CACHE_TTL_MS) {
+  const file = cachePath(name);
+  try {
+    const stat = fs.statSync(file);
+    if (Date.now() - stat.mtimeMs > maxAgeMs) return null;
+    return fs.readFileSync(file, 'utf-8');
+  } catch {
+    return null;
+  }
+}
+
+function writeCache(name, contents) {
+  ensureCacheDir();
+  fs.writeFileSync(cachePath(name), contents, 'utf-8');
+}
+
+async function fetchText(url) {
+  const res = await fetch(url, {
+    headers: { 'User-Agent': '@readme/cli', Accept: 'application/vnd.github.v3+json' },
+  });
+  if (!res.ok) {
+    throw new Error(`GET ${url} → ${res.status} ${res.statusText}`);
+  }
+  return res.text();
+}
+
+/**
+ * Returns the list of marketplace components as an array of:
+ *   { name, mdxUrl, readmeUrl, htmlUrl }
+ * Hidden entries (e.g. `.ExampleComponent`) are filtered out.
+ */
+export async function listMarketplaceComponents({ force = false } = {}) {
+  const cacheKey = 'index.json';
+  if (!force) {
+    const cached = readCache(cacheKey);
+    if (cached) {
+      try { return JSON.parse(cached); } catch { /* fall through */ }
+    }
+  }
+
+  const url = `https://api.github.com/repos/${REPO}/contents/${COMPONENTS_PATH}?ref=${BRANCH}`;
+  const body = await fetchText(url);
+  const entries = JSON.parse(body);
+
+  const components = entries
+    .filter((e) => e.type === 'dir' && !e.name.startsWith('.'))
+    .map((e) => ({
+      name: e.name,
+      htmlUrl: `https://github.com/${REPO}/tree/${BRANCH}/${COMPONENTS_PATH}/${e.name}`,
+      mdxUrl: `https://raw.githubusercontent.com/${REPO}/${BRANCH}/${COMPONENTS_PATH}/${e.name}/${e.name}.mdx`,
+      readmeUrl: `https://raw.githubusercontent.com/${REPO}/${BRANCH}/${COMPONENTS_PATH}/${e.name}/readme.md`,
+    }));
+
+  writeCache(cacheKey, JSON.stringify(components, null, 2));
+  return components;
+}
+
+/**
+ * Fetch the .mdx + readme.md for a single component. Returns:
+ *   { name, mdx, readme, htmlUrl }
+ * `readme` may be null if the component has no readme.md.
+ */
+export async function fetchMarketplaceComponent(name, { force = false } = {}) {
+  const components = await listMarketplaceComponents({ force });
+  const match = components.find((c) => c.name.toLowerCase() === name.toLowerCase());
+  if (!match) {
+    throw new Error(`Component "${name}" not found in marketplace.`);
+  }
+
+  const mdxCacheKey = `${match.name}.mdx`;
+  const readmeCacheKey = `${match.name}.readme.md`;
+
+  let mdx = !force && readCache(mdxCacheKey);
+  if (!mdx) {
+    mdx = await fetchText(match.mdxUrl);
+    writeCache(mdxCacheKey, mdx);
+  }
+
+  let readme = !force && readCache(readmeCacheKey);
+  if (!readme) {
+    try {
+      readme = await fetchText(match.readmeUrl);
+      writeCache(readmeCacheKey, readme);
+    } catch {
+      readme = null;
+    }
+  }
+
+  return { name: match.name, mdx, readme, htmlUrl: match.htmlUrl };
+}
+
+/**
+ * Pull the one-line summary out of a component's readme.md. We look for the
+ * first non-empty paragraph after the `## Overview` heading, falling back to
+ * the first non-heading line. Returns "" if nothing usable is found.
+ */
+export function summarizeReadme(readme) {
+  if (!readme) return '';
+  const lines = readme.split('\n');
+  let inOverview = false;
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i].trim();
+    if (/^##\s+overview/i.test(line)) { inOverview = true; continue; }
+    if (inOverview && line && !line.startsWith('#') && !line.startsWith('![')) {
+      return line.replace(/[*_`]/g, '').slice(0, 140);
+    }
+  }
+  // Fallback: first non-heading, non-image line.
+  for (const line of lines) {
+    const t = line.trim();
+    if (t && !t.startsWith('#') && !t.startsWith('![') && !t.startsWith('<')) {
+      return t.replace(/[*_`]/g, '').slice(0, 140);
+    }
+  }
+  return '';
+}
+
+/**
+ * Pull the first example invocation block out of a component's mdx. Components
+ * conventionally end with a bare `<ComponentName ... />` line below the React
+ * export — return that. Returns the full mdx as a fallback.
+ */
+export function extractExample(mdx, name) {
+  const re = new RegExp(`^<${name}[\\s\\S]*?(?:/>|</${name}>)\\s*$`, 'm');
+  const match = mdx.match(re);
+  return match ? match[0] : '';
+}
+
+/**
+ * Returns the absolute path to the local `components/` folder for a repo
+ * (relative to gitRoot). Does not create it.
+ */
+export function localComponentsDir(gitRoot) {
+  return path.join(gitRoot, 'components');
+}
+
+/**
+ * Read a previously-cached readme for a component without hitting the network.
+ * Returns null if there's nothing cached. Used so we can show short
+ * descriptions in prompts/lists without slowing down the command on a cold
+ * cache.
+ */
+export function readCachedReadme(name) {
+  return readCache(`${name}.readme.md`, Infinity);
+}
+
+/**
+ * Install a marketplace component into the user's local components/ folder.
+ * Used by both the `components add` subcommand and the auto-install path in
+ * `pretty`. Returns the destination path on success.
+ */
+export async function installComponent(gitRoot, name, { force = false } = {}) {
+  const comp = await fetchMarketplaceComponent(name, { force });
+  const dest = localComponentsDir(gitRoot);
+  fs.mkdirSync(dest, { recursive: true });
+  const file = path.join(dest, `${comp.name}.mdx`);
+  fs.writeFileSync(file, comp.mdx, 'utf-8');
+  return { name: comp.name, file, mdx: comp.mdx };
+}
+
+/**
+ * List components that have been copied into the user's local components/
+ * folder. Returns an array of { name, file, source } where `source` is the
+ * raw mdx contents.
+ */
+export function listLocalComponents(gitRoot) {
+  const dir = localComponentsDir(gitRoot);
+  if (!fs.existsSync(dir)) return [];
+
+  const out = [];
+  for (const entry of fs.readdirSync(dir)) {
+    if (!entry.endsWith('.mdx')) continue;
+    const file = path.join(dir, entry);
+    const source = fs.readFileSync(file, 'utf-8');
+    out.push({ name: entry.replace(/\.mdx$/, ''), file, source });
+  }
+  return out.sort((a, b) => a.name.localeCompare(b.name));
+}

package/src/utils/multi-spinner.js

@@ -0,0 +1,71 @@
+import { createLogUpdate } from 'log-update';
+import cliSpinners from 'cli-spinners';
+import chalk from 'chalk';
+
+/**
+ * Multi-row spinner. Renders `rows` labeled lines, repainted in unison at the
+ * spinner's tick interval. Cleared rows render as blank lines (the live region
+ * keeps a fixed height). Falls back to no-op render + line-buffered `log()`
+ * in non-TTY environments.
+ */
+export class MultiSpinner {
+  constructor({ rows, color = 'blue', stream = process.stderr } = {}) {
+    this.rows = Math.max(0, rows);
+    this.color = color;
+    this.stream = stream;
+    this.isTTY = !!stream.isTTY;
+    this.lines = new Array(this.rows).fill('');
+    this.frame = 0;
+    this.spinner = cliSpinners.dots;
+    this.timer = null;
+    this.update = null;
+  }
+
+  start() {
+    if (!this.isTTY) return;
+    this.update = createLogUpdate(this.stream);
+    this._render();
+    this.timer = setInterval(() => this._render(), this.spinner.interval);
+  }
+
+  setLine(rowIdx, text) {
+    if (!this.isTTY) return;
+    if (rowIdx < 0 || rowIdx >= this.rows) return;
+    this.lines[rowIdx] = text;
+  }
+
+  clearLine(rowIdx) {
+    this.setLine(rowIdx, '');
+  }
+
+  log(text) {
+    if (this.isTTY && this.update) {
+      this.update.persist(text);
+    } else {
+      this.stream.write(text + '\n');
+    }
+  }
+
+  stop() {
+    if (this.timer) {
+      clearInterval(this.timer);
+      this.timer = null;
+    }
+    if (this.update) {
+      this.update.clear();
+      this.update.done();
+      this.update = null;
+    }
+  }
+
+  _render() {
+    if (!this.update) return;
+    const glyph = this.spinner.frames[this.frame % this.spinner.frames.length];
+    const colorize = chalk[this.color] ?? ((s) => s);
+    const out = this.lines
+      .map((text) => (text ? `${colorize(glyph)} ${text}` : ''))
+      .join('\n');
+    this.update(out);
+    this.frame++;
+  }
+}

package/src/utils/promise-pool.js

@@ -0,0 +1,17 @@
+/**
+ * Run `worker` over `items` with at most `concurrency` calls in flight.
+ * Each worker pulls the next item from a shared cursor; finishes when none left.
+ * `worker(item, index, workerIdx)` is responsible for catching its own errors —
+ * an uncaught rejection will propagate up and abort other in-flight workers.
+ */
+export async function runPool(items, concurrency, worker) {
+  let cursor = 0;
+  const next = () => (cursor < items.length ? cursor++ : -1);
+  const lanes = Math.max(1, Math.min(concurrency, items.length));
+  const workers = Array.from({ length: lanes }, (_, workerIdx) => (async () => {
+    for (let i = next(); i !== -1; i = next()) {
+      await worker(items[i], i, workerIdx);
+    }
+  })());
+  await Promise.all(workers);
+}

package/src/utils/url-segs.js

@@ -0,0 +1,124 @@
+/**
+ * URL segment parsing utilities shared across import.js pipeline stages.
+ * Centralised here so the single-extension-strip bug only lives in one place.
+ */
+
+// Extensions that are implementation artifacts, not meaningful path segments.
+const DOC_FILE_EXTENSIONS = ['md', 'mdx', 'html', 'htm', 'txt']
+const DOC_EXT_RE = new RegExp(`(\\.(${DOC_FILE_EXTENSIONS.join('|')}))+$`, 'i')
+
+/**
+ * Strip all chained doc-file extensions (.html.md, .html, .md, .mdx, .htm, .txt)
+ * from a single URL path segment. Handles the double-extension pattern used
+ * by some SSGs (e.g. `.html.md`).
+ */
+export function stripSegmentExtensions(s) {
+  return s.replace(DOC_EXT_RE, '')
+}
+
+/**
+ * Parse a URL's pathname into segments suitable for trie nesting. Strips file
+ * extensions (including double-extensions like `.html.md`) and drops a trailing
+ * `index` segment so `/foo`, `/foo/`, `/foo/index.html`, `/foo/index.html.md`,
+ * and `/foo/index.md` all collapse to the same logical page.
+ * Returns null if the URL can't be parsed.
+ */
+export function urlTrieSegs(url) {
+  try {
+    const segs = new URL(url).pathname
+      .split('/')
+      .filter(Boolean)
+      .map((s) => stripSegmentExtensions(s))
+      .filter(Boolean)
+    if (segs.length > 0 && segs[segs.length - 1].toLowerCase() === 'index') segs.pop()
+    return segs
+  } catch {
+    return null
+  }
+}
+
+/**
+ * Extract URL path segments for slug planning. Strips file extensions
+ * (including double-extensions like `.html.md`) and leading numeric prefixes
+ * (e.g. `01-intro` → `intro`). Also drops a trailing `index` segment so
+ * `/foo/index.html.md` and `/foo/` both collapse to the same slug base.
+ */
+export function extractUrlPathSegments(url) {
+  if (!url) return []
+  try {
+    const segs = new URL(url).pathname
+      .split('/')
+      .filter(Boolean)
+      .map((s) => stripSegmentExtensions(s).replace(/^\d+[-_.]/, ''))
+      .filter(Boolean)
+    if (segs.length > 0 && segs[segs.length - 1].toLowerCase() === 'index') segs.pop()
+    return segs
+  } catch {
+    return []
+  }
+}
+
+/**
+ * Normalise a URL to a canonical path key for deduplication. Strips trailing
+ * slash, strips all chained doc-file extensions from the path tail, drops a
+ * trailing `index`, and lowercases — so `/foo/bar.html.md`, `/foo/bar.html`,
+ * `/foo/bar/index.html`, and `/foo/bar` all resolve to the same key `/foo/bar`.
+ */
+export function normalizePath(url) {
+  try {
+    const u = new URL(url)
+    let p = u.pathname.replace(/\/$/, '').toLowerCase()
+    p = p.replace(DOC_EXT_RE, '')
+    p = p.replace(/\/index$/i, '')
+    return p || '/'
+  } catch {
+    return String(url).toLowerCase()
+  }
+}
+
+/**
+ * Dedupe key for llms.txt page rows only.
+ *
+ * Invariants:
+ * - includes origin so same-path docs on different hosts stay distinct
+ * - collapses doc URL spellings for the same page tail (.md/.mdx/.html/.htm, /index, trailing slash)
+ * - preserves query params because some docs routers encode page state there
+ * - ignores fragments because they identify anchors within a page, not separate import pages
+ */
+export function llmsPageDedupeKey(url) {
+  try {
+    const u = new URL(url)
+    let p = u.pathname.replace(/\/$/, '').toLowerCase()
+    p = p.replace(DOC_EXT_RE, '')
+    p = p.replace(/\/index$/i, '')
+
+    const params = [...u.searchParams.entries()].sort(([ak, av], [bk, bv]) => (ak === bk ? av.localeCompare(bv) : ak.localeCompare(bk)))
+    const query = params.length > 0 ? `?${params.map(([k, v]) => `${encodeURIComponent(k)}=${encodeURIComponent(v)}`).join('&')}` : ''
+
+    return `${u.origin.toLowerCase()}${p || '/'}${query}`
+  } catch {
+    return String(url).toLowerCase()
+  }
+}
+
+export function compareCanonicalUrlPreference(a, b) {
+  const aRank = canonicalUrlPreferenceRank(a)
+  const bRank = canonicalUrlPreferenceRank(b)
+  if (aRank !== bRank) return aRank - bRank
+  return String(a || '').length - String(b || '').length
+}
+
+function canonicalUrlPreferenceRank(url) {
+  try {
+    const pathname = new URL(url).pathname
+    const lower = pathname.toLowerCase()
+    if (/\.md$/.test(lower)) return 0
+    if (/\.txt$/.test(lower)) return 1
+    if (!/\/$/.test(pathname) && !/\/index\.html?$/.test(lower) && !/\.[^/]+$/.test(lower)) return 2
+    if (/\/$/.test(pathname)) return 3
+    if (/\/index\.html?$/.test(lower)) return 4
+    return 5
+  } catch {
+    return 5
+  }
+}