@readme/cli 0.0.27 → 0.0.30

package/src/commands/import.js

@@ -9,7 +9,18 @@ import matter from 'gray-matter'
 import * as styles from '../utils/styles.js'
 import { syncOas } from './oas-sync.js'
 import OASNormalize from 'oas-normalize'
-import { slotOrphansPrompt, iconizeNavPrompt, organizeFromSectionsPrompt, organizeFromScratchPrompt, stripCodeFences } from '../prompts/index.js'
+import {
+  slotOrphansPrompt,
+  slotOrphansOutputSchema,
+  iconizeNavPrompt,
+  iconizeNavOutputSchema,
+  organizeFromSectionsPrompt,
+  organizeFromSectionsOutputSchema,
+  organizeFromScratchPrompt,
+  organizeFromScratchOutputSchema,
+} from '../prompts/index.js'
+import { analyzeLlmsTxt } from '../utils/llms.js'
+import { urlTrieSegs, extractUrlPathSegments, normalizePath, stripSegmentExtensions, compareCanonicalUrlPreference, llmsPageDedupeKey } from '../utils/url-segs.js'
 
 export const command = 'import'
 export const order = 7
@@ -17,12 +28,21 @@ export const description = 'Import content from a URL and package it as a ReadMe
 export const hidden = true
 export const skipBootstrap = true
 
+export const DEFAULT_MODEL = 'claude-sonnet-5'
+
 export function args(cmd) {
-  cmd.requiredOption('--source <url-or-file>', 'URL to import from, or path to a local OpenAPI spec (.json/.yaml/.yml)')
+  cmd.requiredOption(
+    '--source <url-or-file...>',
+    'One or more URLs to import from (space-separated or repeated), or a single local OpenAPI spec (.json/.yaml/.yml)',
+  )
   cmd.option('-o, --output <path>', 'Output zip path (defaults to <basename>-readme.zip in cwd)')
-  cmd.addOption(new Option('-m, --model <name>', 'Claude model alias: haiku, sonnet, opus').choices(['haiku', 'sonnet', 'opus']).default('sonnet'))
+  cmd.addOption(new Option('-m, --model <name>', 'Claude model alias: haiku, sonnet, opus').choices(['haiku', 'sonnet', 'opus']).default(DEFAULT_MODEL))
   cmd.option('--firecrawl-key <key>', 'Firecrawl API key (or set FIRECRAWL_API_KEY env var) — enables JS-rendered sidebar scraping')
   cmd.option('--skip-api-reference', 'Drop pages routed to the API Reference / reference dir. Use when uploading the OAS spec separately.')
+  cmd.option(
+    '--separate-changelog',
+    'Move changelog / release-notes pages into a top-level changelog/ directory instead of docs/Changelog/. Note: changelog/ is outside the git-format schema, so it will not lint or sync to ReadMe.',
+  )
   // Internal dev-only flag: skip the zip, keep staging, and boot the dev server
   // against it for quick visual previews. Hidden from --help.
   cmd.addOption(new Option('--test').hideHelp())
@@ -32,22 +52,59 @@ export function args(cmd) {
   cmd.addOption(new Option('--debug').hideHelp())
 }
 
+// Docs locations probed when the user hands us a bare homepage. Each is tried
+// as a subdomain swap (docs.example.com) and, unless subdomain-only, a path
+// prefix (example.com/docs/). Order is preference order — earlier wins ties.
+const WELL_KNOWN_DOC_ROUTES = ['docs', 'doc', 'developers', 'developer', 'documentation', 'documentations', 'platform']
+// Routes probed only as a subdomain (platform.example.com). Their path form
+// (example.com/platform/) is usually a marketing/product page, not docs.
+const SUBDOMAIN_ONLY_DOC_ROUTES = new Set(['platform'])
+// Max concurrent HTTP probes when checking whether a section's landing URL
+// (e.g. /docs/cli) really exists before importing it as a real overview page.
+const LANDING_PROBE_CAP = 8
+// Well-known spec locations probed on every import. Tried under the
+// post-adoption origin, the pre-adoption origin when the docs-route probe
+// switched hosts, and (first three only) under a path-kind docs base.
+const WELL_KNOWN_OAS_PATHS = [
+  'openapi.json',
+  'openapi.yaml',
+  'openapi.yml',
+  'swagger.json',
+  'swagger.yaml',
+  '.well-known/openapi.json',
+  '.well-known/openapi.yaml',
+  'api/openapi.json',
+  'docs/openapi.json',
+  'v3/api-docs',
+  'api-docs',
+  'swagger/v1/swagger.json',
+]
+// Per-source cap on candidate specs. Candidates are sorted by discovery-source
+// confidence first, so the cap trims the least trusted ones.
+const MAX_OAS_CANDIDATES_PER_SOURCE = 10
+const OAS_SOURCE_CONFIDENCE = ['llms', 'mintlify', 'probe', 'html', 'sitemap', 'link-sweep']
+
 /**
  * Run the importer programmatically. Mirrors the CLI command but throws on
  * fatal errors instead of calling `process.exit`, and returns a result object
  * on success.
  *
  * @param {object} options
- * @param {string} options.source             URL to import from, or path to a local OAS spec.
+ * @param {string | string[]} options.source  One or more URLs to import from, or a single path to a local OAS spec.
+ *                                            Multiple URLs are merged into one zip; same-titled categories merge their pages.
  * @param {string} [options.output]           Output zip path. Defaults to `<basename>-readme.zip` in cwd.
  * @param {string} [options.model]            Claude model alias: 'haiku' | 'sonnet' | 'opus'. Defaults to 'sonnet'.
  * @param {string} [options.firecrawlKey]     Firecrawl API key (falls back to FIRECRAWL_API_KEY env var).
  * @param {boolean} [options.skipApiReference] Drop pages routed to the API Reference dir.
+ * @param {boolean} [options.separateChangelog] Move changelog pages into a top-level `changelog/` dir instead of `docs/Changelog/`.
  * @param {boolean} [options.test]            Skip the zip, keep staging, and boot the dev server.
  * @param {boolean} [options.debug]           Dump intermediate pipeline artifacts to a tmp dir.
  * @returns {Promise<{ source: 'url' | 'oas', outputZip?: string, stagingDir?: string, fileCount: number, duration: number, phases: Array<{ label: string, ms: number }> }>}
  */
 export async function importDocs(options) {
+  const model = options.model || DEFAULT_MODEL
+  options = { ...options, model }
+
   const startedAt = Date.now()
   const phases = []
   const timePhase = async (label, fn) => {
@@ -59,40 +116,285 @@ export async function importDocs(options) {
 
   const debugSnapshots = options.debug ? {} : null
 
-  // Dispatch: http(s) URL → docs-site scrape flow; anything else → local OAS.
-  if (!/^https?:\/\//i.test(options.source)) {
-    return runOasImport(options.source, options, startedAt, phases, timePhase)
-  }
+  // Normalize source → array; preserve back-compat with single-string callers.
+  const rawSources = Array.isArray(options.source) ? options.source : [options.source]
+  if (rawSources.length === 0) throw new Error('At least one --source is required')
 
-  let sourceUrl
-  try {
-    sourceUrl = new URL(options.source)
-  } catch {
-    throw new Error(`Invalid --source URL: ${options.source}`)
+  // Dispatch: any non-URL entry triggers the local-OAS path, which is
+  // strictly single-source.
+  const hasNonUrl = rawSources.some((s) => !/^https?:\/\//i.test(s))
+  if (hasNonUrl) {
+    if (rawSources.length > 1) {
+      throw new Error('OAS imports accept a single --source; URL imports can accept multiple.')
+    }
+    return runOasImport(rawSources[0], options, startedAt, phases, timePhase)
   }
 
-  const outputZip = path.resolve(options.output || path.join(process.cwd(), `${sourceUrl.hostname}-readme.zip`))
+  const sourceUrls = rawSources.map((s) => {
+    try {
+      return new URL(s)
+    } catch {
+      throw new Error(`Invalid --source URL: ${s}`)
+    }
+  })
+
+  const hostnameJoined = sourceUrls.map((u) => u.hostname).join('-')
+  const outputZip = path.resolve(options.output || path.join(process.cwd(), `${hostnameJoined}-readme.zip`))
 
   console.log()
-  styles.info(`Importing from ${styles.bold(sourceUrl.toString())}`)
+  if (sourceUrls.length === 1) {
+    styles.info(`Importing from ${styles.bold(sourceUrls[0].toString())}`)
+  } else {
+    styles.info(`Importing from ${styles.bold(String(sourceUrls.length))} sources:`)
+    for (const u of sourceUrls) styles.info(styles.dim(`  · ${u.toString()}`))
+  }
   if (!options.test) styles.info(`Output: ${styles.bold(outputZip)}`)
   console.log()
 
-  // Build the list of llms.txt URLs to probe, walking up the supplied path
-  // from most-specific to root. For `https://mintlify.com/docs/quickstart`
-  // we try `/docs/quickstart/llms.txt`, then `/docs/llms.txt`, then root.
-  // This catches sites that scope llms.txt to a docs subpath.
-  const llmsCandidates = buildLlmsCandidates(sourceUrl)
-  styles.info(`Checking for llms.txt (${llmsCandidates.length} candidate${llmsCandidates.length === 1 ? '' : 's'})...`)
+  // Per-source pipeline: discovery + scrape + organize, in parallel.
+  // Each call returns an `organized` object with the standard shape
+  // `{ title, categories: [...] }`; we merge them before staging.
+  const perSourceOrganized = await Promise.all(
+    sourceUrls.map((sourceUrl) => produceOrganizedForSource(sourceUrl, options, timePhase, debugSnapshots)),
+  )
 
-  const { llms, llmsUrl } = await timePhase('fetch llms.txt', async () => {
-    for (const candidate of llmsCandidates) {
-      const res = await fetchLlmsTxt(candidate)
-      if (res.ok) return { llms: res, llmsUrl: candidate }
-      styles.info(styles.dim(`  ${candidate} → ${res.status ? `HTTP ${res.status}` : res.error || 'failed'}`))
+  const organized = sourceUrls.length === 1 ? perSourceOrganized[0] : mergeOrganized(perSourceOrganized)
+
+  // Union the candidate OpenAPI-spec URLs across all sources, deduped by URL
+  // (mergeOrganized intentionally drops non-category fields, so collect from
+  // the per-source results directly).
+  const oasJsonUrls = []
+  const seenOasUrls = new Set()
+  for (const o of perSourceOrganized) {
+    for (const item of o.oasJsonUrls || []) {
+      if (seenOasUrls.has(item.url)) continue
+      seenOasUrls.add(item.url)
+      oasJsonUrls.push(item)
     }
-    return { llms: null, llmsUrl: null }
-  })
+  }
+
+  const llmsPaths = new Set()
+  for (const o of perSourceOrganized) {
+    for (const p of o.llmsPaths || []) llmsPaths.add(p)
+  }
+
+  if (debugSnapshots) {
+    debugSnapshots['05-organized.json'] = organized
+    const debugDir = path.join(os.tmpdir(), `readme-import-debug-${hostnameJoined}-${Date.now()}`)
+    fs.mkdirSync(debugDir, { recursive: true })
+    // Probe-hit candidates carry prefetched spec bytes — summarize instead of
+    // dumping megabytes of serialized Buffer into the snapshot.
+    const bufferSummary = (key, value) =>
+      value && value.type === 'Buffer' && Array.isArray(value.data) ? `<Buffer ${value.data.length} bytes>` : value
+    for (const [name, data] of Object.entries(debugSnapshots)) {
+      fs.writeFileSync(path.join(debugDir, name), JSON.stringify(data, bufferSummary, 2))
+    }
+    styles.info(`${styles.dim(`Debug snapshots → ${debugDir}`)}`)
+  }
+  console.log()
+
+  console.log(`  ${styles.bold(organized.title || '(untitled)')}`)
+  for (const cat of organized.categories || []) {
+    console.log()
+    const iconLabel = cat.icon ? `${styles.brand(`[${cat.icon}]`)} ` : ''
+    console.log(`  ${iconLabel}${styles.bold(cat.title)}`)
+    printPagesTree(cat.pages || [], 2)
+  }
+
+  const stagingDir = fs.mkdtempSync(path.join(os.tmpdir(), 'readme-import-'))
+
+  let result
+  try {
+    styles.info(`Staging frontmatter stubs in ${styles.bold(stagingDir)}...`)
+    const stageStart = Date.now()
+    const staged = await timePhase('stage stubs', async () =>
+      stageOrganized(organized, stagingDir, { skipApiReference: !!options.skipApiReference, llmsPaths }),
+    )
+
+    // Changelog pages stage into docs/Changelog/ (a schema-valid docs
+    // category). With --separate-changelog, relocate them afterwards into a
+    // top-level changelog/ dir — done as a post-staging move so the staged
+    // tree is a valid git-format layout up to this point.
+    if (options.separateChangelog) {
+      const movedChangelog = relocateChangelogDir(stagingDir)
+      if (movedChangelog > 0) {
+        styles.info(
+          `Moved ${styles.bold(String(movedChangelog))} changelog page${movedChangelog === 1 ? '' : 's'} into ${styles.bold('changelog/')} (--separate-changelog).`,
+        )
+      }
+    }
+
+    const landingTitle =
+      organized.title ||
+      (sourceUrls.length === 1 ? sourceUrls[0].hostname : sourceUrls.map((u) => u.hostname).join(' + '))
+    ensureDocsLandingPage(stagingDir, landingTitle)
+    styles.ok(
+      `Staged ${styles.bold(String(staged.fileCount))} stub${staged.fileCount === 1 ? '' : 's'} across ${styles.bold(String(staged.dirCount))} director${staged.dirCount === 1 ? 'y' : 'ies'} in ${styles.bold(formatDuration(Date.now() - stageStart))}.`,
+    )
+    if (staged.skippedApiRef > 0) {
+      styles.info(`Skipped ${styles.bold(String(staged.skippedApiRef))} API reference page${staged.skippedApiRef === 1 ? '' : 's'} (--skip-api-reference)`)
+    }
+
+    // Download captured candidate specs into <stagingDir>/oas/. Raw bytes only —
+    // validation + OpenAPI detection + upload all happen in the runner. Staged
+    // here (not in docs/ or reference/) so it rides the zip but is invisible to
+    // the docs build. Non-fatal: a failed download is logged and skipped.
+    if (oasJsonUrls.length > 0) {
+      const oasCount = await timePhase('download oas specs', () => downloadOasSpecs(oasJsonUrls, stagingDir))
+      if (oasCount > 0) {
+        styles.ok(`Downloaded ${styles.bold(String(oasCount))} candidate OpenAPI spec${oasCount === 1 ? '' : 's'} into ${styles.bold('oas/')}.`)
+      }
+    }
+    console.log()
+
+    if (options.test) {
+      styles.ok(`Done in ${styles.bold(formatDuration(Date.now() - startedAt))}! Staged ${styles.bold(String(staged.fileCount))} files at ${styles.bold(stagingDir)}`)
+      console.log()
+      styles.info('Starting the dev server for preview...')
+      console.log()
+      await runDevPreview(stagingDir)
+      return { source: 'url', stagingDir, fileCount: staged.fileCount, duration: Date.now() - startedAt, phases }
+    }
+
+    if (staged.fileCount === 0) {
+      styles.warning('Staging directory is empty — skipping zip.')
+      return { source: 'url', fileCount: 0, duration: Date.now() - startedAt, phases }
+    }
+
+    styles.info(`Packaging ${styles.bold(String(staged.fileCount))} files into ${styles.bold(outputZip)}...`)
+    await timePhase('zip', () => createZip(stagingDir, outputZip))
+
+    console.log()
+    styles.ok(`Done in ${styles.bold(formatDuration(Date.now() - startedAt))}! Your ReadMe import is ready at ${styles.bold(outputZip)}`)
+    console.log(styles.dim(`  ⏱  ${phases.map((p) => `${p.label} ${formatDuration(p.ms)}`).join(' · ')}`))
+    result = { source: 'url', outputZip, fileCount: staged.fileCount, duration: Date.now() - startedAt, phases }
+  } finally {
+    if (!options.test) {
+      fs.rmSync(stagingDir, { recursive: true, force: true })
+    }
+  }
+
+  return result
+}
+
+/**
+ * Run the discovery + scrape + organize pipeline for ONE source URL and
+ * return its `organized` object (`{ title, categories: [...] }`). Pure
+ * per-source work — no staging, no zipping. `importDocs` runs this in
+ * parallel across multiple sources and merges the results.
+ *
+ * `debugSnapshots`, when provided, gets per-source keys suffixed with
+ * the hostname so parallel runs don't clobber each other.
+ */
+async function produceOrganizedForSource(sourceUrl, options, timePhase, debugSnapshots) {
+  const redirected = await timePhase('follow source redirects', () => resolveRedirectedSourceUrl(sourceUrl))
+  if (redirected) {
+    styles.info(`${styles.bold(sourceUrl.toString())} redirects to ${styles.bold(redirected.toString())} — rebasing import onto the final URL.`)
+    sourceUrl = redirected
+  }
+
+  // Probe well-known docs routes (docs./developer. subdomains and /docs/,
+  // … path prefixes) and adopt the first that resolves as the real base, so everything downstream runs
+  const preAdoptionOrigin = sourceUrl.origin
+  const docsBase = await timePhase('probe well-known docs routes', () => resolveDocsBaseUrl(sourceUrl))
+  if (docsBase) {
+    styles.info(
+      `${styles.bold(sourceUrl.toString())} looks like a homepage — switching to docs route ${styles.bold(docsBase.url.toString())}${styles.dim(docsBase.hasLlms ? ' (llms.txt found)' : ' (no llms.txt; using for scrape fallback)')}.`,
+    )
+    sourceUrl = docsBase.url
+  }
+
+  // Fired here so the spec-path probe fully overlaps llms/scrape/organize;
+  // awaited just before the candidates are attached to `organized`.
+  const oasProbePromise = probeWellKnownOasPaths(sourceUrl, docsBase, preAdoptionOrigin).catch(() => [])
+
+  // Discover llms.txt files by BFS walking parents AND children. Seed the
+  // frontier with the walk-up paths (source → root). Any "hit" (file exists
+  // with at least one link row) expands the frontier: parent path + distinct
+  // first-child segments extracted from its URL list. Invalid hits (fail the
+  // usability ratio) still expand — they're stepping stones — only valid
+  // hits contribute content to the merged result. See
+  // mydocs.com/section1/section2 → also tries /section1/, /, and every
+  // sibling section2 we discover along the way.
+  styles.info(`Discovering llms.txt (walking up and down from ${styles.bold(sourceUrl.toString())})...`)
+
+  const discovery = await timePhase('discover llms.txt', () => discoverLlmsTxt(sourceUrl))
+  for (const hit of discovery.hits) {
+    if (hit.usable) {
+      styles.info(styles.dim(`  ${hit.llmsUrl} → valid (${hit.stats.linkItems} links)`))
+    } else {
+      styles.info(styles.dim(`  ${hit.llmsUrl} → invalid: ${hit.reason} (kept as stepping stone)`))
+    }
+  }
+  for (const miss of discovery.misses) {
+    styles.info(styles.dim(`  ${miss.url} → ${miss.status ? `HTTP ${miss.status}` : miss.error || 'no llms.txt'}`))
+  }
+  let llms = mergeValidHits(discovery.hits)
+  let llmsUrl = llms?.llmsUrl || null
+  const skippedLlms = discovery.skipped
+
+  // Capture spec-shaped links (.json/.yaml/.yml — candidate OpenAPI specs)
+  // BEFORE anything else sees the parsed tree. They're removed in place here,
+  // downloaded into the staged `oas/` dir by importDocs, and validated +
+  // uploaded by the runner. Stub generation never sees them.
+  const oasCandidates = []
+  const seenOasCandidateUrls = new Set()
+  const addOasCandidates = (items) => {
+    for (const item of items || []) {
+      if (!item || !item.url) continue
+      const url = item.url.split('#')[0]
+      if (seenOasCandidateUrls.has(url)) continue
+      seenOasCandidateUrls.add(url)
+      oasCandidates.push({ ...item, url })
+    }
+  }
+  if (llms) {
+    const llmsSpecUrls = extractOasSpecUrlsFromParsed(llms.parsed)
+    if (llmsSpecUrls.length > 0) {
+      styles.info(styles.dim(`Captured ${llmsSpecUrls.length} spec link${llmsSpecUrls.length === 1 ? '' : 's'} as candidate OpenAPI spec${llmsSpecUrls.length === 1 ? '' : 's'} → oas/.`))
+      addOasCandidates(llmsSpecUrls)
+    }
+  }
+
+  // Drop the remaining asset/meta items (llms-full.txt, .xml, …) from
+  // the merged parsed result up-front, so every downstream consumer — knownUrls
+  // AND the "use llms.txt sections directly" organize path — sees a clean URL
+  // list. Spec-shaped items were already pulled out above.
+  if (llms) {
+    const dropped = dropAssetItemsFromParsed(llms.parsed)
+    if (dropped > 0) {
+      styles.info(styles.dim(`Dropped ${dropped} asset/meta URL${dropped === 1 ? '' : 's'} (.yaml/.xml/.toml, llms*.txt) from llms.txt items.`))
+    }
+  }
+
+  if (llms) {
+    const narrowed = narrowToDocsSubtreeIfNeeded(llms, sourceUrl, discovery.hits)
+    if (narrowed) {
+      styles.info(
+        styles.dim(
+          `Narrowed merged result to /${narrowed.segment}/ subtree (kept ${narrowed.kept}, dropped ${narrowed.dropped} out-of-scope page${narrowed.dropped === 1 ? '' : 's'}).`,
+        ),
+      )
+    }
+  }
+
+  // Fall back to scrape if the merged llms.txt covers too few endpoints to
+  // be useful as a structural backbone. Replaces the old per-file
+  // MIN_LINK_ROWS=10 check (now relaxed in utils/llms.js) with a global
+  // post-merge floor — many tiny per-product llms.txt files can still pass
+  // as long as their union clears the bar.
+  const MIN_MERGED_LINK_COUNT = 10
+  if (llms) {
+    const mergedLinkCount = llms.parsed.sections.reduce((n, s) => n + s.items.length, 0)
+    if (mergedLinkCount < MIN_MERGED_LINK_COUNT) {
+      styles.warning(
+        `Merged llms.txt has only ${styles.bold(String(mergedLinkCount))} endpoint${mergedLinkCount === 1 ? '' : 's'} (< ${MIN_MERGED_LINK_COUNT}) — falling back to scrape.`,
+      )
+      skippedLlms.push({ url: llmsUrl, reason: `merged total only ${mergedLinkCount} link${mergedLinkCount === 1 ? '' : 's'}` })
+      llms = null
+      llmsUrl = null
+    }
+  }
   console.log()
 
   let sitemapUrl = null
@@ -105,7 +407,7 @@ export async function importDocs(options) {
     const sitemapCandidates = buildSitemapCandidates(sourceUrl)
     const scopePrefix = deriveSitemapScope(sourceUrl)
     styles.info(
-      `No llms.txt — checking for sitemap.xml (${sitemapCandidates.length} candidate${sitemapCandidates.length === 1 ? '' : 's'})${scopePrefix ? `, scoped to ${styles.bold(scopePrefix)}` : ''}...`,
+      `${skippedLlms.length > 0 ? 'No usable llms.txt' : 'No llms.txt'} — checking for sitemap.xml (${sitemapCandidates.length} candidate${sitemapCandidates.length === 1 ? '' : 's'})${scopePrefix ? `, scoped to ${styles.bold(scopePrefix)}` : ''}...`,
     )
     const sitemapResult = await timePhase('fetch sitemap.xml', async () => {
       for (const candidate of sitemapCandidates) {
@@ -119,25 +421,85 @@ export async function importDocs(options) {
     console.log()
     if (sitemapResult) {
       sitemapUrl = sitemapResult.url
-      sitemapKnownUrls = sitemapUrlsToKnownUrls(sitemapResult.urls)
+      const sitemapOasSink = []
+      sitemapKnownUrls = sitemapUrlsToKnownUrls(sitemapResult.urls, sitemapOasSink)
+      addOasCandidates(sitemapOasSink)
       styles.ok(`Found sitemap.xml at ${styles.bold(sitemapUrl)} — ${styles.bold(String(sitemapKnownUrls.length))} URL${sitemapKnownUrls.length === 1 ? '' : 's'} in scope.`)
     } else {
       styles.warning(`No llms.txt or sitemap.xml found — falling back to sidebar discovery via scrape.`)
     }
   } else {
-    styles.info(styles.dim(`Using ${llmsUrl}.`))
+    const s = llms.stats
+    if (llms.sourceFiles.length === 1) {
+      styles.info(styles.dim(`Using ${llmsUrl} (${s.conforming}/${s.total} lines conforming, ratio ${s.ratio.toFixed(2)}).`))
+    } else {
+      styles.info(styles.dim(`Merged ${llms.sourceFiles.length} llms.txt files (root: ${llmsUrl}; aggregate ratio ${s.ratio.toFixed(2)}).`))
+    }
+    if (llms.parsed.h3Fallback) {
+      styles.info(styles.dim(`H2 sections oversized — re-parsed using H3 (###) headings as section boundaries.`))
+    }
+  }
+
+  // Pre-extract changelog pages so AI / URL clustering / section-direct paths
+  // never see them as candidates for organization. We re-attach them as a
+  // dedicated Changelog category once `organized` is finalized below — that
+  // guarantees they route to docs/Changelog/ (or to changelog/ with
+  // --separate-changelog) regardless of which organize path ran. Done here,
+  // before knownUrls is derived, so every downstream consumer operates on the
+  // already-changelog-free view.
+  let extractedChangelog = []
+  if (llms) {
+    extractedChangelog = extractChangelogFromSections(llms.parsed.sections)
+  } else if (sitemapKnownUrls.length > 0) {
+    const { extracted, kept } = partitionChangelogFromKnownUrls(sitemapKnownUrls)
+    extractedChangelog = extracted
+    sitemapKnownUrls = kept
+  }
+  if (extractedChangelog.length > 0) {
+    styles.info(
+      styles.dim(
+        `Pre-extracted ${extractedChangelog.length} changelog page${extractedChangelog.length === 1 ? '' : 's'} — bypassing organization, attaching as Changelog category at the end.`,
+      ),
+    )
+  }
+
+  // Dedupe items across llms.parsed.sections by URL (first occurrence wins).
+  // Some llms.txt files cross-reference the same page under multiple headings —
+  // feeding duplicate URLs into the organize step causes the same page to appear
+  // in multiple sidebar sections regardless of which path (direct, icons, full
+  // reorg) runs downstream.
+  if (llms) {
+    const seenItemUrls = new Set()
+    let deduped = 0
+    for (const section of llms.parsed.sections) {
+      const before = section.items.length
+      section.items = section.items.filter((item) => {
+        const key = normalizePath(item.url)
+        if (seenItemUrls.has(key)) return false
+        seenItemUrls.add(key)
+        return true
+      })
+      deduped += before - section.items.length
+    }
+    if (deduped > 0) {
+      styles.info(styles.dim(`Deduped ${deduped} cross-section duplicate URL${deduped === 1 ? '' : 's'} from llms.txt sections.`))
+    }
   }
 
+  const dbgSuffix = `-${sourceUrl.hostname}`
   if (debugSnapshots) {
-    debugSnapshots['01-llms-parsed.json'] = { llmsUrl, parsed: llms ? llms.parsed : null }
-    debugSnapshots['01b-sitemap.json'] = { sitemapUrl, urls: sitemapKnownUrls }
+    debugSnapshots[`01-llms-parsed${dbgSuffix}.json`] = { llmsUrl, parsed: llms ? llms.parsed : null, skipped: skippedLlms }
+    debugSnapshots[`01b-sitemap${dbgSuffix}.json`] = { sitemapUrl, urls: sitemapKnownUrls }
+    debugSnapshots[`01c-extracted-changelog${dbgSuffix}.json`] = extractedChangelog
   }
 
   let knownUrls = []
   if (llms) {
     const totalItems = llms.parsed.sections.reduce((n, s) => n + s.items.length, 0)
+    const srcCount = llms.sourceFiles.length
+    const srcLabel = srcCount === 1 ? 'llms.txt' : `${srcCount} llms.txt files (merged)`
     styles.ok(
-      `Found llms.txt — ${styles.bold(String(totalItems))} page${totalItems === 1 ? '' : 's'} across ${styles.bold(String(llms.parsed.sections.length))} section${llms.parsed.sections.length === 1 ? '' : 's'}${llms.parsed.title ? ` (${llms.parsed.title})` : ''}.`,
+      `Found ${srcLabel} — ${styles.bold(String(totalItems))} page${totalItems === 1 ? '' : 's'} across ${styles.bold(String(llms.parsed.sections.length))} section${llms.parsed.sections.length === 1 ? '' : 's'}${llms.parsed.title ? ` (${llms.parsed.title})` : ''}.`,
     )
 
     const rawKnownUrls = llms.parsed.sections.flatMap((s) => s.items.map((i) => ({ title: i.text, url: i.url, description: i.description })))
@@ -145,13 +507,13 @@ export async function importDocs(options) {
     // Dedupe llms.txt entries by pathname. Some sites (zod.dev, fumadocs) list
     // every in-page anchor as its own llms.txt row (`/v4?id=wrapping-up`,
     // `/v4?id=metadata`, …) even though they all live on one rendered page.
-    // We prefer the "cleanest" URL per path — the shortest one, which is
-    // usually the one without a query string or hash.
+    // Prefer direct markdown source forms over shorter rendered URLs because
+    // they are better import sources for downstream content fetching.
     const byKnownPath = new Map()
     for (const p of rawKnownUrls) {
       const key = normalizePath(p.url)
       const prev = byKnownPath.get(key)
-      if (!prev || p.url.length < prev.url.length) byKnownPath.set(key, p)
+      if (!prev || compareCanonicalUrlPreference(p.url, prev.url) < 0) byKnownPath.set(key, p)
     }
     knownUrls = Array.from(byKnownPath.values())
     const dropped = rawKnownUrls.length - knownUrls.length
@@ -170,58 +532,117 @@ export async function importDocs(options) {
   // HTML parsing, so try it before falling back to generic nav scraping.
   styles.info(`Probing for Mintlify config (docs.json, mint.json)...`)
   const mintlifyStart = Date.now()
-  const mintlifyNav = await timePhase('mintlify probe', () => tryMintlifyNav(sourceUrl.toString(), knownUrls, firecrawlKey))
+  const mintlifyOasSink = []
+  const mintlifyNav = await timePhase('mintlify probe', () => tryMintlifyNav(sourceUrl.toString(), knownUrls, firecrawlKey, mintlifyOasSink))
+  addOasCandidates(mintlifyOasSink)
   if (debugSnapshots) {
-    debugSnapshots['02a-mintlify-nav.json'] = mintlifyNav ? JSON.parse(JSON.stringify(mintlifyNav)) : null
+    debugSnapshots[`02a-mintlify-nav${dbgSuffix}.json`] = mintlifyNav ? JSON.parse(JSON.stringify(mintlifyNav)) : null
   }
   if (mintlifyNav) {
-    const pageCount = mintlifyNav.categories.reduce((n, c) => n + c.pages.length, 0)
+    const pageCount = mintlifyNav.categories.reduce((n, c) => n + countUrlPagesDeep(c.pages), 0)
     styles.ok(
       `Found Mintlify config at ${styles.bold(mintlifyNav.source)} in ${styles.bold(formatDuration(Date.now() - mintlifyStart))} — ${styles.bold(String(mintlifyNav.categories.length))} categor${mintlifyNav.categories.length === 1 ? 'y' : 'ies'}, ${styles.bold(String(pageCount))} pages.`,
     )
   }
   console.log()
 
+  // Fern sites ship the canonical sidebar in RSC flight chunks on every
+  // page — deterministic structure where the Firecrawl scrape usually fails
+  // acceptance and llms.txt clustering invents categories.
+  let fernNav = null
+  if (!mintlifyNav) {
+    styles.info(`Probing for Fern nav tree...`)
+    const fernStart = Date.now()
+    fernNav = await timePhase('fern probe', () => tryFernNav(sourceUrl.toString(), knownUrls))
+    if (fernNav) {
+      const pageCount = fernNav.categories.reduce((n, c) => n + countUrlPagesDeep(c.pages), 0)
+      styles.ok(
+        `Found Fern nav tree in ${styles.bold(formatDuration(Date.now() - fernStart))} — ${styles.bold(String(fernNav.categories.length))} categor${fernNav.categories.length === 1 ? 'y' : 'ies'}, ${styles.bold(String(pageCount))} page${pageCount === 1 ? '' : 's'}.`,
+      )
+    }
+    console.log()
+  }
+  if (debugSnapshots) {
+    debugSnapshots[`02a2-fern-nav${dbgSuffix}.json`] = fernNav ? JSON.parse(JSON.stringify(fernNav)) : null
+  }
+
+  // Archbee sites embed the canonical document tree in Next.js page data.
+  // Their llms.txt export can be a flat, shuffled list, so prefer the tree
+  // when present.
+  let archbeeNav = null
+  if (!mintlifyNav && !fernNav) {
+    styles.info(`Probing for Archbee document tree...`)
+    const archbeeStart = Date.now()
+    const archbeeOasSink = []
+    archbeeNav = await timePhase('archbee probe', () => tryArchbeeNav(sourceUrl.toString(), knownUrls, firecrawlKey, archbeeOasSink))
+    addOasCandidates(archbeeOasSink)
+    if (archbeeNav) {
+      const pageCount = archbeeNav.categories.reduce((n, c) => n + countUrlPagesDeep(c.pages), 0)
+      styles.ok(
+        `Found Archbee document tree in ${styles.bold(formatDuration(Date.now() - archbeeStart))} — ${styles.bold(String(pageCount))} page${pageCount === 1 ? '' : 's'}.`,
+      )
+    }
+    console.log()
+  }
+
   let scraped
   let scrapeStart = Date.now()
+  const scrapeDiagnostics = {}
   if (mintlifyNav) {
     scraped = { title: mintlifyNav.title, categories: mintlifyNav.categories }
+  } else if (fernNav) {
+    scraped = { title: fernNav.title, categories: fernNav.categories }
+  } else if (archbeeNav) {
+    scraped = { title: archbeeNav.title, categories: archbeeNav.categories }
   } else {
     styles.info(`Scraping sidebar nav from ${styles.bold(sourceUrl.toString())}${firecrawlKey ? ' ' + styles.dim('(via Firecrawl)') : ''}...`)
     scrapeStart = Date.now()
-    scraped = await timePhase('scrape nav', () => scrapeNavFromSite(sourceUrl.toString(), knownUrls, firecrawlKey))
+    const scrapeOasSink = []
+    scraped = await timePhase('scrape nav', () => scrapeNavFromSite(sourceUrl.toString(), knownUrls, firecrawlKey, scrapeDiagnostics, scrapeOasSink))
+    addOasCandidates(scrapeOasSink)
+    if (!scraped) scrapeDiagnostics.reason ??= 'unknown'
   }
   if (debugSnapshots) {
-    debugSnapshots['02-scraped-raw.json'] = scraped ? JSON.parse(JSON.stringify(scraped)) : null
+    debugSnapshots[`02-scraped-raw${dbgSuffix}.json`] = scraped ? JSON.parse(JSON.stringify(scraped)) : null
   }
-  // Prefer llms.txt when it has strong multi-section structure and the
-  // scrape was a thin snapshot (common on big multi-tab docs — Stripe, AWS,
-  // Twilio — where each page only renders its own tab's sidebar). Without
-  // this override the 4-category scrape wins over a 25-section llms.txt and
-  // hundreds of real pages end up smeared into orphan buckets.
+  // If the sidebar scrape covers less than 75% of the llms.txt URLs, the
+  // scrape is too thin to trust as the import's spine. Common on multi-tab
+  // docs (Stripe, AWS, Twilio, Xata) where each page only renders its own
+  // tab's sidebar — the visible categories would otherwise absorb hundreds
+  // of orphan URLs via prefix-matching and produce a misleading tree (e.g.
+  // every /docs/* URL dumped under a single "Overview > Xata Documentation"
+  // node because that's the one /docs page the scrape saw). Discard the
+  // scrape and fall through to the llms.txt path, which uses URL-based
+  // clustering when multiple files were merged.
+  // API-reference pages are excluded from the denominator: sidebars routinely
+  // omit generated endpoint stubs, and those pages are swept into reference/
+  // regardless of nav quality, so they say nothing about the nav's fitness as
+  // the import's spine.
+  let scrapeDiscardedForCoverage = false
   if (scraped && llms && knownUrls.length > 0) {
-    const scrapedPages = scraped.categories.reduce((n, c) => n + c.pages.length, 0)
-    const coverage = scrapedPages / knownUrls.length
-    const llmsUsable = usableSections(llms.parsed.sections)
-    if (llmsUsable.length >= 5 && coverage < 0.5) {
+    const scrapedPages = scraped.categories.reduce((n, c) => n + countUrlPagesDeep(c.pages), 0)
+    const nonReferenceKnown = knownUrls.filter((p) => !urlIsApiReference(p.url))
+    const coverage = nonReferenceKnown.length > 0 ? scrapedPages / nonReferenceKnown.length : 1
+    if (coverage < 0.75) {
       styles.info(
-        `Scrape covered ${styles.bold(Math.round(coverage * 100) + '%')} of llms.txt pages; preferring llms.txt's ${styles.bold(String(llmsUsable.length))} sections for structure.`,
+        `Scrape covered ${styles.bold(Math.round(coverage * 100) + '%')} of llms.txt pages (need ≥75%${nonReferenceKnown.length < knownUrls.length ? `, ${knownUrls.length - nonReferenceKnown.length} api-reference pages excluded` : ''}) — discarding scrape and organizing from llms.txt.`,
       )
       scraped = null
+      scrapeDiscardedForCoverage = true
     }
   }
 
   if (scraped) {
-    const directMatches = scraped.categories.reduce((n, c) => n + c.pages.length, 0)
+    const directMatches = scraped.categories.reduce((n, c) => n + countUrlPagesDeep(c.pages), 0)
     if (knownUrls.length > 0) {
       const slotted = slotOrphansByPath(scraped, knownUrls)
       if (debugSnapshots) {
-        debugSnapshots['03-after-slot-by-path.json'] = {
+        debugSnapshots[`03-after-slot-by-path${dbgSuffix}.json`] = {
           scraped: JSON.parse(JSON.stringify(scraped)),
           unslottedOrphans: slotted,
         }
       }
-      const totalMatched = scraped.categories.reduce((n, c) => n + c.pages.length, 0)
+      const totalMatched = scraped.categories.reduce((n, c) => n + countUrlPagesDeep(c.pages), 0)
       styles.ok(
         `Scraped nav in ${styles.bold(formatDuration(Date.now() - scrapeStart))} — ${styles.bold(String(scraped.categories.length))} categor${scraped.categories.length === 1 ? 'y' : 'ies'}, ${styles.bold(String(directMatches))} direct matches + ${styles.bold(String(totalMatched - directMatches))} slotted by path = ${styles.bold(String(totalMatched))}/${knownUrls.length}.`,
       )
@@ -235,40 +656,85 @@ export async function importDocs(options) {
         styles.info(`Moved ${styles.bold(String(moved))} page${moved === 1 ? '' : 's'} into ${styles.bold('API Reference')} based on URL path.`)
       }
 
+      // Same idea for changelog / release-notes pages — pull any whose URL
+      // carries a changelog-style segment into one "Changelog" category. It
+      // routes to docs/Changelog/ unless --separate-changelog relocates it.
+      const movedChangelog = reclassifyChangelogPages(scraped)
+      if (movedChangelog > 0) {
+        styles.info(`Moved ${styles.bold(String(movedChangelog))} page${movedChangelog === 1 ? '' : 's'} into ${styles.bold('Changelog')} based on URL path.`)
+      }
+
       if (slotted.length > 0) {
-        // Mintlify-style docs put API endpoints in a separate tab rooted at
-        // /api-reference/* (or /api/*, /reference/*). Collapse remaining such
-        // pages into a single "API Reference" category (absorbing the flat
-        // category the sweep pass just built, if any) and nest it by resource
-        // segment so routeCategory() maps the whole thing to ReadMe's
-        // `reference/` top-level dir.
-        const apiResult = collectApiReferencePages(slotted, scraped)
-        const otherOrphans = apiResult.nonApiOrphans
-        if (apiResult.category) scraped.categories.push(apiResult.category)
-
-        const buckets = bucketOrphansByPathType(otherOrphans, scraped)
-        for (const b of buckets) scraped.categories.push(b)
-        if (debugSnapshots) {
-          debugSnapshots['04-after-orphan-buckets.json'] = {
-            apiReferenceCollected: apiResult.category
-              ? {
+        // When orphans dwarf direct matches, the sidebar scrape was too thin
+        // to trust as the import's spine — keeping it would produce a small
+        // "real" tree plus a soup of bucketed-by-URL-type orphan categories.
+        // Discard the scrape and cluster every page (scrape + orphans) by
+        // its top URL segment instead, so each `/docs/`, `/sdk/`, `/rest-api/`
+        // becomes its own category. `nestByUrlHierarchy` (later) handles the
+        // empty-parent nesting within each category.
+
+        // Trip when orphans are at least 2× the direct matches: the scrape
+        // accounts for less than a third of the known pages, so its category
+        // labels aren't a trustworthy spine for the remainder.
+        const orphansDwarfDirect = slotted.length >= directMatches * 2
+        const scrapeAllPages = scraped.categories.flatMap((c) => collectUrlPagesDeep(c.pages))
+        let reclustered = null
+        if (orphansDwarfDirect) {
+          reclustered = clusterByUrlPath([...scrapeAllPages, ...slotted])
+        }
+        styles.info(
+          styles.dim(
+            `  orphan triage: ${slotted.length} orphan${slotted.length === 1 ? '' : 's'} vs ${directMatches} direct match${directMatches === 1 ? '' : 'es'} (ratio ${directMatches === 0 ? '∞' : (slotted.length / directMatches).toFixed(2)}) — gate ${orphansDwarfDirect ? 'tripped' : 'NOT tripped'} (need ≥2.00); URL re-cluster ${reclustered ? `→ ${reclustered.length} categor${reclustered.length === 1 ? 'y' : 'ies'}` : 'skipped'}`,
+          ),
+        )
+        if (reclustered) {
+          scraped.categories = reclustered.map((c) => ({ title: c.title, pages: c.pages }))
+          styles.info(
+            `Scrape covered only ${styles.bold(String(directMatches))}/${styles.bold(String(knownUrls.length))} pages — discarded as too thin and re-clustered ${styles.bold(String(scrapeAllPages.length + slotted.length))} pages by URL path into ${styles.bold(String(reclustered.length))} categor${reclustered.length === 1 ? 'y' : 'ies'}: ${reclustered.map((c) => styles.bold(c.title)).join(', ')}.`,
+          )
+          if (debugSnapshots) {
+            debugSnapshots[`04-after-orphan-buckets${dbgSuffix}.json`] = {
+              mode: 'reclustered-by-url-path',
+              scraped: JSON.parse(JSON.stringify(scraped)),
+            }
+          }
+        } else {
+          // Mintlify-style docs put API endpoints in a separate tab rooted at
+          // /api-reference/* (or /api/*, /reference/*). Collapse remaining such
+          // pages into a single "API Reference" category (absorbing the flat
+          // category the sweep pass just built, if any) and nest it by resource
+          // segment so routeCategory() maps the whole thing to ReadMe's
+          // `reference/` top-level dir.
+          const linkSweepOasSink = []
+          const apiResult = collectApiReferencePages(slotted, scraped, linkSweepOasSink)
+          const otherOrphans = apiResult.nonApiOrphans
+          if (apiResult.category) scraped.categories.push(apiResult.category)
+
+          const buckets = bucketOrphansByPathType(otherOrphans, scraped, linkSweepOasSink)
+          for (const b of buckets) scraped.categories.push(b)
+          addOasCandidates(linkSweepOasSink)
+          if (debugSnapshots) {
+            debugSnapshots[`04-after-orphan-buckets${dbgSuffix}.json`] = {
+              apiReferenceCollected: apiResult.category
+                ? {
                   pageCount: apiResult.category.pages.length,
                   mergedFromScraped: apiResult.mergedScrapedTitles,
                 }
-              : null,
-            buckets,
-            scraped: JSON.parse(JSON.stringify(scraped)),
+                : null,
+              buckets,
+              scraped: JSON.parse(JSON.stringify(scraped)),
+            }
+          }
+          const parts = []
+          if (apiResult.category) {
+            parts.push(`${styles.bold(String(apiResult.category.pages.length))} in ${styles.bold('API Reference')}`)
+          }
+          for (const b of buckets) {
+            parts.push(`${styles.bold(String(b.pages.length))} in ${styles.bold(b.title)}`)
+          }
+          if (parts.length > 0) {
+            styles.info(`${styles.bold(String(slotted.length))} orphan page${slotted.length === 1 ? '' : 's'} bucketed by URL type: ${parts.join(', ')}.`)
           }
-        }
-        const parts = []
-        if (apiResult.category) {
-          parts.push(`${styles.bold(String(apiResult.category.pages.length))} in ${styles.bold('API Reference')}`)
-        }
-        for (const b of buckets) {
-          parts.push(`${styles.bold(String(b.pages.length))} in ${styles.bold(b.title)}`)
-        }
-        if (parts.length > 0) {
-          styles.info(`${styles.bold(String(slotted.length))} orphan page${slotted.length === 1 ? '' : 's'} bucketed by URL type: ${parts.join(', ')}.`)
         }
       }
     } else {
@@ -298,9 +764,31 @@ export async function importDocs(options) {
   } else if (!llms && knownUrls.length === 0) {
     throw new Error(`No llms.txt or sitemap.xml and the sidebar scrape found no usable structure — can't import ${sourceUrl.toString()}.`)
   } else if (llms) {
-    styles.warning(`Couldn't extract a useful nav — falling back to llms.txt-based organization.`)
+    if (!scrapeDiscardedForCoverage) {
+      const d = scrapeDiagnostics || {}
+      if (d.reason === 'no-categories-round0') {
+        styles.warning(`Sidebar scrape found no nav structure on the round-0 fetch — falling back to llms.txt organization.`)
+      } else if (d.reason === 'below-threshold') {
+        styles.warning(
+          `Sidebar scrape below acceptance threshold (got ${d.categories} categor${d.categories === 1 ? 'y' : 'ies'}, ${d.matched} matched pages — need ${d.need}) — falling back to llms.txt organization.`,
+        )
+      } else {
+        styles.warning(`Sidebar scrape returned no usable nav — falling back to llms.txt organization.`)
+      }
+    }
   } else {
-    styles.warning(`Couldn't extract a useful nav — falling back to sitemap URL clustering.`)
+    if (!scrapeDiscardedForCoverage) {
+      const d = scrapeDiagnostics || {}
+      if (d.reason === 'no-categories-round0') {
+        styles.warning(`Sidebar scrape found no nav structure on the round-0 fetch — falling back to sitemap URL clustering.`)
+      } else if (d.reason === 'below-threshold') {
+        styles.warning(
+          `Sidebar scrape below acceptance threshold (got ${d.categories} categor${d.categories === 1 ? 'y' : 'ies'}, ${d.matched} matched pages — need ${d.need}) — falling back to sitemap URL clustering.`,
+        )
+      } else {
+        styles.warning(`Sidebar scrape returned no usable nav — falling back to sitemap URL clustering.`)
+      }
+    }
   }
   console.log()
 
@@ -314,9 +802,60 @@ export async function importDocs(options) {
       categories: scraped.categories.map((c) => ({ title: c.title, icon: null, pages: c.pages })),
     }
   } else if (llms) {
-    const fastPath = sectionsLookUsable(llms.parsed.sections)
-    styles.info(`Organizing with Claude (${styles.bold(options.model)}, ${fastPath ? 'fast path: icons only' : 'full reorg'})...`)
-    organized = await timePhase('claude organize', () => organizeWithClaude(llms.parsed, options.model))
+    if (llms.sourceFiles.length > 1) {
+      // Multi-file merge: H2 section headings across the merged files are
+      // heterogeneous and don't form a coherent spine (e.g. elevenlabs.io's
+      // root file uses `Docs`/`Products` while /docs/llms.txt uses
+      // `API Docs` — concatenating them produces meaningless top-level
+      // categories like `docs/Docs`, `docs/Products`, `docs/API Docs`).
+      // Throw away the section grouping and re-cluster by URL path instead.
+      // Per-page metadata (title, url, description) is still carried over.
+      const flatPages = llms.parsed.sections.flatMap((s) =>
+        s.items.map((it) => ({
+          title: it.text,
+          url: it.url,
+          ...(it.description ? { description: it.description } : {}),
+        })),
+      )
+      const clustered = clusterByUrlPath(flatPages) || [{ title: 'Documentation', pages: flatPages }]
+      styles.info(
+        `Multi-file llms.txt merge — clustered ${styles.bold(String(flatPages.length))} pages by URL path into ${styles.bold(String(clustered.length))} categor${clustered.length === 1 ? 'y' : 'ies'} (H2 sections discarded).`,
+      )
+      organized = {
+        title: llms.parsed.title || null,
+        categories: clustered.map((c) => ({ title: c.title, icon: null, pages: c.pages })),
+      }
+    } else {
+      // Single-file mode: H2 structure IS the author's intended organization,
+      // so trust it. Use sections directly when usable; otherwise run Claude.
+      const usable = usableSections(llms.parsed.sections)
+      const avgPages = usable.length > 0 ? usable.reduce((n, s) => n + s.items.length, 0) / usable.length : 0
+      if (sectionsLookUsable(llms.parsed.sections) && avgPages < 50) {
+        styles.info(
+          `Using llms.txt sections directly — ${styles.bold(String(usable.length))} sections, ${styles.bold(avgPages.toFixed(1))} avg pages/section (≤50). Skipping Claude.`,
+        )
+        organized = {
+          title: llms.parsed.title || null,
+          categories: usable.map((s) => ({
+            title: s.title,
+            icon: null,
+            pages: s.items.map((it) => ({
+              title: it.text,
+              url: it.url,
+              ...(it.description ? { description: it.description } : {}),
+            })),
+          })),
+        }
+      } else {
+        const fastPath = sectionsLookUsable(llms.parsed.sections)
+        styles.info(`Organizing with Claude (${styles.bold(options.model)}, ${fastPath ? 'fast path: icons only' : 'full reorg'})...`)
+        organized = await timePhase('claude organize', () => organizeWithClaude(llms.parsed, options.model))
+        const movedRef = reclassifyReferencePages(organized)
+        if (movedRef > 0) {
+          styles.info(`Moved ${styles.bold(String(movedRef))} page${movedRef === 1 ? '' : 's'} into ${styles.bold('API Reference')} based on URL path.`)
+        }
+      }
+    }
   } else {
     // Sitemap-only fallback: synthesize categories by clustering the URL
     // paths. clusterByUrlPath returns null when the URLs don't split cleanly,
@@ -327,75 +866,86 @@ export async function importDocs(options) {
       categories: clustered.map((c) => ({ title: c.title, icon: null, pages: c.pages })),
     }
   }
-  
+
+  // Re-attach the pre-extracted changelog pages as their own Changelog
+  // category. Done before nestByUrlHierarchy so the injected pages get nested
+  // alongside the rest of the tree for consistency. Merges into a pre-existing
+  // Changelog category (scrape path's reclassifier may have already built one).
+  injectChangelogCategory(organized, extractedChangelog)
+
+  // Import section landing pages (e.g. /docs/cli) that llms.txt models as a
+  // bare heading + child links and so never lists as a page of their own.
+  // Runs before nesting so the injected page lands at its trie node and
+  // nestByUrlHierarchy renders it as the real parent of its children.
+  await injectSectionLandingPages(organized, sourceUrl)
+
+  // A Fern tree is the authored sidebar: re-nesting by URL invents parent
+  // folders the source site renders flat (a section landing absorbing its
+  // path-descendant siblings, e.g. vapi's /server-url under Webhooks). Only
+  // the sweep-synthesized categories still nest — their pages were moved in
+  // flat and have no authored structure.
+  const sweepCategories = new Set(['API Reference', 'Changelog'])
   for (const cat of organized.categories || []) {
+    if (fernNav && !sweepCategories.has(cat.title)) continue
     cat.pages = nestByUrlHierarchy(cat.pages)
   }
 
   styles.ok(`Organized in ${styles.bold(formatDuration(Date.now() - organizeStart))}.`)
-  if (debugSnapshots) {
-    debugSnapshots['05-organized.json'] = organized
-    const debugDir = path.join(os.tmpdir(), `readme-import-debug-${sourceUrl.hostname}-${Date.now()}`)
-    fs.mkdirSync(debugDir, { recursive: true })
-    for (const [name, data] of Object.entries(debugSnapshots)) {
-      fs.writeFileSync(path.join(debugDir, name), JSON.stringify(data, null, 2))
-    }
-    styles.info(`${styles.dim(`Debug snapshots → ${debugDir}`)}`)
-  }
-  console.log()
-
-  console.log(`  ${styles.bold(organized.title || '(untitled)')}`)
-  for (const cat of organized.categories || []) {
-    console.log()
-    const iconLabel = cat.icon ? `${styles.brand(`[${cat.icon}]`)} ` : ''
-    console.log(`  ${iconLabel}${styles.bold(cat.title)}`)
-    printPagesTree(cat.pages || [], 2)
-  }
 
-  const stagingDir = fs.mkdtempSync(path.join(os.tmpdir(), 'readme-import-'))
-
-  let result
-  try {
-    styles.info(`Staging frontmatter stubs in ${styles.bold(stagingDir)}...`)
-    const stageStart = Date.now()
-    const staged = await timePhase('stage stubs', async () => stageOrganized(organized, stagingDir, { skipApiReference: !!options.skipApiReference }))
-    ensureDocsLandingPage(stagingDir, organized.title || sourceUrl.hostname)
-    styles.ok(
-      `Staged ${styles.bold(String(staged.fileCount))} stub${staged.fileCount === 1 ? '' : 's'} across ${styles.bold(String(staged.dirCount))} director${staged.dirCount === 1 ? 'y' : 'ies'} in ${styles.bold(formatDuration(Date.now() - stageStart))}.`,
+  addOasCandidates(await oasProbePromise)
+  oasCandidates.sort((a, b) => OAS_SOURCE_CONFIDENCE.indexOf(a.source) - OAS_SOURCE_CONFIDENCE.indexOf(b.source))
+  const cappedOasCandidates = oasCandidates.slice(0, MAX_OAS_CANDIDATES_PER_SOURCE)
+  if (oasCandidates.length > 0) {
+    const bySource = new Map()
+    for (const c of cappedOasCandidates) bySource.set(c.source, (bySource.get(c.source) || 0) + 1)
+    const summary = Array.from(bySource, ([source, n]) => `${n} ${source}`).join(', ')
+    const overCap = oasCandidates.length - cappedOasCandidates.length
+    styles.info(
+      styles.dim(`OAS candidates: ${summary}${overCap > 0 ? `; dropped ${overCap} over the cap of ${MAX_OAS_CANDIDATES_PER_SOURCE}` : ''}.`),
     )
-    if (staged.skippedApiRef > 0) {
-      styles.info(`Skipped ${styles.bold(String(staged.skippedApiRef))} API reference page${staged.skippedApiRef === 1 ? '' : 's'} (--skip-api-reference)`)
-    }
-    console.log()
-
-    if (options.test) {
-      styles.ok(`Done in ${styles.bold(formatDuration(Date.now() - startedAt))}! Staged ${styles.bold(String(staged.fileCount))} files at ${styles.bold(stagingDir)}`)
-      console.log()
-      styles.info('Starting the dev server for preview...')
-      console.log()
-      await runDevPreview(stagingDir)
-      return { source: 'url', stagingDir, fileCount: staged.fileCount, duration: Date.now() - startedAt, phases }
-    }
+  }
+  organized.oasJsonUrls = cappedOasCandidates
 
-    if (staged.fileCount === 0) {
-      styles.warning('Staging directory is empty — skipping zip.')
-      return { source: 'url', fileCount: 0, duration: Date.now() - startedAt, phases }
+  // Record which page URLs came from llms.txt
+  const llmsPaths = new Set()
+  if (llms) {
+    for (const p of [...knownUrls, ...extractedChangelog]) {
+      if (p.url) llmsPaths.add(normalizePath(p.url))
     }
+  }
+  organized.llmsPaths = llmsPaths
+  return organized
+}
 
-    styles.info(`Packaging ${styles.bold(String(staged.fileCount))} files into ${styles.bold(outputZip)}...`)
-    await timePhase('zip', () => createZip(stagingDir, outputZip))
-
-    console.log()
-    styles.ok(`Done in ${styles.bold(formatDuration(Date.now() - startedAt))}! Your ReadMe import is ready at ${styles.bold(outputZip)}`)
-    console.log(styles.dim(`  ⏱  ${phases.map((p) => `${p.label} ${formatDuration(p.ms)}`).join(' · ')}`))
-    result = { source: 'url', outputZip, fileCount: staged.fileCount, duration: Date.now() - startedAt, phases }
-  } finally {
-    if (!options.test) {
-      fs.rmSync(stagingDir, { recursive: true, force: true })
+/**
+ * Merge per-source `organized` results into one. Same-titled categories MERGE
+ * their page lists (intentional — when a user bundles related sources into
+ * one project, they probably want a coherent sidebar; `ensureUniqueSlugs` later
+ * dedupes any slug collisions globally). First non-null title wins; first
+ * non-null icon per merged category wins. Page order within a merged category
+ * is the concatenation of input sources in argument order.
+ */
+function mergeOrganized(perSource) {
+  const title = perSource.map((o) => o.title).find(Boolean) || null
+
+  const byTitle = new Map()
+  const order = []
+  for (const o of perSource) {
+    for (const cat of o.categories || []) {
+      const key = cat.title
+      const existing = byTitle.get(key)
+      if (existing) {
+        existing.pages.push(...(cat.pages || []))
+        if (!existing.icon && cat.icon) existing.icon = cat.icon
+      } else {
+        const copy = { ...cat, pages: [...(cat.pages || [])] }
+        byTitle.set(key, copy)
+        order.push(copy)
+      }
     }
   }
 
-  return result
+  return { title, categories: order }
 }
 
 /**
@@ -798,7 +1348,7 @@ function createZip(sourceDir, outputZip) {
  * Returns { source, title, categories } or null if no Mintlify config is
  * found or parseable.
  */
-async function tryMintlifyNav(sourceUrl, knownPages, firecrawlKey) {
+async function tryMintlifyNav(sourceUrl, knownPages, firecrawlKey, oasSink) {
   const origin = new URL(sourceUrl).origin
   const fetchHtml = firecrawlKey ? makeFirecrawlFetcher(firecrawlKey) : fetchHtmlDirect
 
@@ -810,7 +1360,9 @@ async function tryMintlifyNav(sourceUrl, knownPages, firecrawlKey) {
     const body = await fetchHtml(configUrl)
     if (!body) continue
     const config = extractMintlifyConfig(body)
-    if (!config || !config.navigation) continue
+    if (!config) continue
+    if (oasSink) oasSink.push(...extractMintlifyOasCandidates(config, origin))
+    if (!config.navigation) continue
     const parsed = parseMintlifyConfig(config, origin, byPath)
     if (parsed.categories.length > 0) {
       return { source: configUrl, title: parsed.title, categories: parsed.categories }
@@ -829,7 +1381,7 @@ async function tryMintlifyNav(sourceUrl, knownPages, firecrawlKey) {
 function extractMintlifyConfig(body) {
   try {
     return JSON.parse(body)
-  } catch {}
+  } catch { }
   // Pull out the first balanced `{ ... }` that contains a "navigation" key.
   const navIdx = body.indexOf('"navigation"')
   if (navIdx === -1) return null
@@ -840,17 +1392,62 @@ function extractMintlifyConfig(body) {
       try {
         const parsed = JSON.parse(candidate)
         if (parsed && parsed.navigation) return parsed
-      } catch {}
+      } catch { }
     }
     start = body.lastIndexOf('{', start - 1)
   }
   return null
 }
 
-function parseMintlifyConfig(config, origin, byPath) {
-  const title = config.name || null
-  const categories = []
-
+/**
+ * Collect candidate spec URLs from a Mintlify config's `openapi` fields —
+ * `string | string[] | { source }` at the top level and on any navigation
+ * node. Relative values resolve against the origin. Pushed before the
+ * navigation guard so a config with `openapi` but unusable nav still yields
+ * specs.
+ */
+function extractMintlifyOasCandidates(config, origin) {
+  const candidates = []
+  const seen = new Set()
+  const addUrl = (raw) => {
+    if (typeof raw !== 'string' || !raw.trim()) return
+    let abs
+    try {
+      abs = new URL(raw, origin)
+    } catch {
+      return
+    }
+    if (abs.protocol !== 'http:' && abs.protocol !== 'https:') return
+    const href = abs.toString()
+    if (seen.has(href)) return
+    seen.add(href)
+    candidates.push({ url: href, text: null, source: 'mintlify' })
+  }
+  const addValue = (value) => {
+    if (typeof value === 'string') addUrl(value)
+    else if (Array.isArray(value)) for (const v of value) addValue(v)
+    else if (value && typeof value === 'object') addUrl(value.source)
+  }
+  const NAV_KEYS = ['navigation', 'tabs', 'groups', 'anchors', 'versions', 'languages', 'pages', 'dropdowns']
+  const walk = (node) => {
+    if (Array.isArray(node)) {
+      for (const n of node) walk(n)
+      return
+    }
+    if (!node || typeof node !== 'object') return
+    if (node.openapi !== undefined) addValue(node.openapi)
+    for (const key of NAV_KEYS) {
+      if (node[key] !== undefined) walk(node[key])
+    }
+  }
+  walk(config)
+  return candidates
+}
+
+function parseMintlifyConfig(config, origin, byPath) {
+  const title = config.name || null
+  const categories = []
+
   const slugToTitle = (slug) => {
     const base = String(slug).split('/').pop() || slug
     return base
@@ -927,6 +1524,327 @@ function parseMintlifyConfig(config, origin, byPath) {
   return { title, categories }
 }
 
+/**
+ * Archbee exposes its canonical docs tree in Next.js page data as
+ * `_docSpace.publicDocsTree`. That tree preserves author order and nesting,
+ * while Archbee's llms.txt export may be flat and shuffled.
+ *
+ * Returns { title, categories } or null if the page is not Archbee or the
+ * embedded tree is unavailable.
+ */
+async function tryArchbeeNav(sourceUrl, knownPages, firecrawlKey, oasSink) {
+  const origin = new URL(sourceUrl).origin
+  const fetchHtml = firecrawlKey ? makeFirecrawlFetcher(firecrawlKey) : fetchHtmlDirect
+  const html = await fetchHtml(toBrowsableUrl(sourceUrl))
+  if (html && oasSink) {
+    oasSink.push(...extractOasUrlsFromHtml(html, sourceUrl).map((url) => ({ url, text: null, source: 'html' })))
+  }
+  if (!html || !/publicDocsTree|Archbee/i.test(html)) return null
+
+  const nextData = extractNextData(html)
+  const docSpace = nextData?.props?.pageProps?._docSpace
+  const tree = docSpace?.publicDocsTree
+  if (!Array.isArray(tree) || tree.length === 0) return null
+
+  const byPath = new Map()
+  for (const p of knownPages) byPath.set(normalizePath(p.url), p)
+
+  const pageFromNode = (node) => {
+    if (!node || typeof node !== 'object') return null
+    if (node.isHidden === true) return null
+
+    const title = String(node.name || node.title || node.urlKey || '').trim()
+    const children = Array.isArray(node.children) ? node.children.map(pageFromNode).filter(Boolean) : []
+    if (!title && children.length === 0) return null
+
+    const url = archbeeNodeUrl(origin, node.urlKey)
+    const known = url ? byPath.get(normalizePath(url)) : null
+    const page = {
+      title: known?.title || title || 'Untitled',
+      url,
+      ...(known?.description ? { description: known.description } : {}),
+      ...(children.length > 0 ? { pages: children } : {}),
+    }
+
+    // Root docs pages have no URL path segments, but should stay in their
+    // authored position when the later URL-trie nesting pass runs.
+    if (url && extractUrlPathSegments(url).length === 0) {
+      page._virtualPathSegs = [kebabCase(page.title || 'home') || 'home']
+    }
+
+    return page
+  }
+
+  const pages = tree.map(pageFromNode).filter(Boolean)
+  if (pages.length === 0) return null
+
+  return {
+    title: docSpace.hostingTitle || docSpace.name || null,
+    categories: [{ title: 'Documentation', pages }],
+  }
+}
+
+function extractNextData(html) {
+  const m = String(html).match(/<script\b(?=[^>]*\bid=["']__NEXT_DATA__["'])[^>]*>([\s\S]*?)<\/script>/i)
+  if (!m) return null
+  try {
+    return JSON.parse(m[1])
+  } catch {
+    return null
+  }
+}
+
+function archbeeNodeUrl(origin, urlKey) {
+  if (urlKey === undefined || urlKey === null) return null
+  const key = String(urlKey || '').trim()
+  if (!key || key === '/') return `${origin}/`
+  if (/^https?:\/\//i.test(key)) return key
+  return `${origin}/${key.replace(/^\/+/, '')}`
+}
+
+// Max page fetches while filling in non-active Fern tab trees (one per tab).
+const FERN_TAB_FETCH_CAP = 8
+
+const FERN_SECTION_NODE_TYPES = new Set(['section', 'apiReference', 'apiPackage'])
+const FERN_SKIP_NODE_TYPES = new Set(['link', 'changelog', 'tab'])
+
+/**
+ * Fern-powered docs sites embed the complete nav tree in Next.js RSC flight
+ * chunks (`self.__next_f.push([1,"..."])`) on every page — titles, slugs,
+ * nesting, and order exactly as the live sidebar renders them. Only the
+ * active tab's tree is populated in a given page's payload; other tabs are
+ * listed with a `pointsTo` entry path, so each one costs a single extra
+ * fetch.
+ *
+ * Always fetches directly (never Firecrawl): the flight chunks are only
+ * guaranteed present in the raw SSR HTML, not in a rendered/processed DOM.
+ * A dead entry URL (moved slug) is retried once from a known page, since
+ * Fern soft-404s stale deep links.
+ *
+ * Returns { title, categories } or null if the site is not Fern or the
+ * payload yields no usable tree. Any parse failure returns null so the
+ * probe chain falls through to the existing routes.
+ */
+async function tryFernNav(sourceUrl, knownPages) {
+  try {
+    const origin = new URL(sourceUrl).origin
+    const looksFern = (html) => /<aside[^>]*id="fern-sidebar"/.test(html) || /<meta[^>]+name="generator"[^>]+content="[^"]*buildwithfern/i.test(html)
+    let entryUrl = toBrowsableUrl(sourceUrl)
+    let entryHtml = await fetchHtmlDirect(entryUrl)
+    if (!looksFern(entryHtml)) {
+      // A stale entry deep link 404s on Fern (the not-found boundary renders
+      // with a truncated tree), so retry once from a known live page.
+      const retry = knownPages.find((p) => normalizePath(p.url) !== normalizePath(sourceUrl))
+      if (!retry) return null
+      entryUrl = toBrowsableUrl(retry.url)
+      entryHtml = await fetchHtmlDirect(entryUrl)
+      if (!looksFern(entryHtml)) return null
+    }
+
+    const byPath = new Map()
+    for (const p of knownPages) byPath.set(normalizePath(p.url), p)
+
+    const ctx = { origin, byPath, seenPaths: new Set() }
+    const categories = []
+    const collect = (html, tabTitle) => {
+      const blob = decodeFernFlightBlob(html)
+      addFernCategories(categories, extractFernNavNodes(blob), { ...ctx, tabTitle })
+      return blob
+    }
+
+    const entryBlob = collect(entryHtml, null)
+    const entryPath = normalizePath(entryUrl)
+    const seenTabTargets = new Set()
+    let tabFetches = 0
+    for (const tab of extractFernTabs(entryBlob)) {
+      if (seenTabTargets.has(tab.pointsTo)) continue
+      seenTabTargets.add(tab.pointsTo)
+      const tabUrl = `${origin}/${tab.pointsTo}`
+      if (normalizePath(tabUrl) === entryPath) continue
+      if (tabFetches >= FERN_TAB_FETCH_CAP) break
+      tabFetches++
+      const tabHtml = await fetchHtmlDirect(tabUrl)
+      if (!tabHtml) continue
+      collect(tabHtml, tab.title)
+    }
+
+    const pageCount = categories.reduce((n, c) => n + countUrlPagesDeep(c.pages), 0)
+    if (pageCount === 0) return null
+    return { title: null, categories }
+  } catch {
+    return null
+  }
+}
+
+function decodeFernFlightBlob(html) {
+  const chunks = []
+  const chunkRe = /self\.__next_f\.push\(\[1,"((?:[^"\\]|\\.)*)"\]\)/g
+  let m
+  while ((m = chunkRe.exec(html)) !== null) {
+    try {
+      chunks.push(JSON.parse(`"${m[1]}"`))
+    } catch { }
+  }
+  return chunks.join('')
+}
+
+function extractBalancedObject(str, start) {
+  let depth = 0
+  let inString = false
+  for (let i = start; i < str.length; i++) {
+    const ch = str[i]
+    if (inString) {
+      if (ch === '\\') i++
+      else if (ch === '"') inString = false
+    } else if (ch === '"') inString = true
+    else if (ch === '{') depth++
+    else if (ch === '}') {
+      depth--
+      if (depth === 0) return str.slice(start, i + 1)
+    }
+  }
+  return null
+}
+
+/**
+ * Pull top-level nav container/section objects out of a decoded flight blob.
+ * The blob as a whole is not JSON — nav nodes are embedded mid-stream — so
+ * each marker hit is balanced-brace extracted and parsed on its own. Matches
+ * that fall inside an already-extracted object are its children and skipped.
+ */
+function extractFernNavNodes(blob) {
+  const nodes = []
+  const markerRe = /\{"type":"(?:sidebarRoot|sidebarGroup|section|apiReference|apiPackage)"/g
+  let consumedUpTo = -1
+  let m
+  while ((m = markerRe.exec(blob)) !== null) {
+    if (m.index < consumedUpTo) continue
+    const raw = extractBalancedObject(blob, m.index)
+    if (!raw) continue
+    consumedUpTo = m.index + raw.length
+    try {
+      nodes.push(JSON.parse(raw))
+    } catch { }
+  }
+  return nodes
+}
+
+function extractFernTabs(blob) {
+  const tabs = []
+  const tabRe = /\{"type":"tab",/g
+  let m
+  while ((m = tabRe.exec(blob)) !== null) {
+    const raw = extractBalancedObject(blob, m.index)
+    if (!raw) continue
+    let tab
+    try {
+      tab = JSON.parse(raw)
+    } catch {
+      continue
+    }
+    const pointsTo = fernField(tab.pointsTo)
+    if (!pointsTo) continue
+    tabs.push({ title: fernField(tab.title), pointsTo })
+  }
+  return tabs
+}
+
+// Absent fields serialize as the literal string "$undefined" in flight payloads.
+function fernField(value) {
+  return typeof value === 'string' && value !== '$undefined' && value !== '' ? value : null
+}
+
+function fernPageFromNode(node, ctx) {
+  if (!node || typeof node !== 'object' || node.hidden === true) return null
+  if (FERN_SKIP_NODE_TYPES.has(node.type)) return null
+  const children = Array.isArray(node.children) ? node.children.map((c) => fernPageFromNode(c, ctx)).filter(Boolean) : []
+  if (FERN_SECTION_NODE_TYPES.has(node.type)) {
+    // A section owns a page only when overviewPageId is set — it then renders
+    // at the section's own slug. pointsTo is a navigation alias to the
+    // section's first descendant page, never a page of the section itself.
+    const landing = fernField(node.overviewPageId) ? fernField(node.slug) : null
+    const title = fernField(node.title) || 'Untitled'
+    if (!landing) {
+      if (children.length === 0) return null
+      return { title, url: null, _emptyParent: true, _virtualPathSegs: [kebabCase(title) || 'group'], pages: children }
+    }
+    const landingUrl = `${ctx.origin}/${landing}`
+    const known = ctx.byPath.get(normalizePath(landingUrl))
+    return {
+      title,
+      url: known?.url || landingUrl,
+      ...(children.length > 0 ? { pages: children } : {}),
+    }
+  }
+  const slug = fernField(node.slug)
+  if (slug) {
+    const url = `${ctx.origin}/${slug}`
+    const known = ctx.byPath.get(normalizePath(url))
+    return {
+      // The payload title is the sidebar label as rendered; llms.txt titles
+      // come from page headings, which can differ from the nav.
+      title: fernField(node.title) || known?.title || slug,
+      url: known?.url || url,
+      ...(known?.description ? { description: known.description } : {}),
+      ...(children.length > 0 ? { pages: children } : {}),
+    }
+  }
+  if (children.length === 1) return children[0]
+  if (children.length > 1) return { title: fernField(node.title) || 'Untitled', url: null, pages: children }
+  return null
+}
+
+/**
+ * Top-level sections become categories; untitled ones adopt the tab title so
+ * two tabs' unnamed roots don't merge into one "Untitled" category with
+ * colliding group slugs. Loose top-level pages (tabs without sections, e.g. a
+ * two-page MCP tab) collapse into one category named after the tab. Pages
+ * already claimed by an earlier category are dropped — the
+ * payload repeats trees across fetches, and multi-product sites nest the
+ * same API groups under both a product tab and the API-reference tab.
+ */
+function addFernCategories(categories, nodes, ctx) {
+  const roots = []
+  for (const node of nodes) {
+    if (node.type === 'sidebarRoot' || node.type === 'sidebarGroup') roots.push(...(node.children || []))
+    else roots.push(node)
+  }
+  const loosePages = []
+  for (const root of roots) {
+    if (FERN_SECTION_NODE_TYPES.has(root.type)) {
+      const section = fernPageFromNode(root, ctx)
+      if (!section) continue
+      if (!fernField(root.title) && ctx.tabTitle) section.title = ctx.tabTitle
+      const pages = section.pages || []
+      if (section.url && !pages.some((p) => p.url && normalizePath(p.url) === normalizePath(section.url))) {
+        pages.unshift({ title: section.title, url: section.url })
+      }
+      const deduped = dedupeFernPages(pages, ctx.seenPaths)
+      if (deduped.length > 0) categories.push({ title: section.title, pages: deduped })
+    } else {
+      const page = fernPageFromNode(root, ctx)
+      if (page) loosePages.push(page)
+    }
+  }
+  const deduped = dedupeFernPages(loosePages, ctx.seenPaths)
+  if (deduped.length > 0) categories.push({ title: ctx.tabTitle || 'Documentation', pages: deduped })
+}
+
+function dedupeFernPages(pages, seenPaths) {
+  const out = []
+  for (const p of pages) {
+    const urlSeen = p.url ? seenPaths.has(normalizePath(p.url)) : false
+    if (p.url && !urlSeen) seenPaths.add(normalizePath(p.url))
+    const kids = p.pages ? dedupeFernPages(p.pages, seenPaths) : []
+    if (kids.length === 0 && (urlSeen || !p.url)) continue
+    const entry = { ...p, url: urlSeen ? null : p.url }
+    if (kids.length > 0) entry.pages = kids
+    else delete entry.pages
+    out.push(entry)
+  }
+  return out
+}
+
 /**
  * Score a parsed nav tree for "sidebar-likeness". A real docs sidebar has
  * multiple section headers (hierarchy) and tens of links; secondary navs
@@ -961,6 +1879,63 @@ function isMonotonicAlpha(titles) {
   return true
 }
 
+/**
+ * Scan a rendered docs page for spec URLs embedded by the common API-reference
+ * renderers — Swagger UI, Redoc, Scalar, Stoplight Elements. Returns absolute
+ * http(s) URLs resolved against the page URL. Pure string scanning on HTML we
+ * already fetched — zero extra requests.
+ */
+function extractOasUrlsFromHtml(html, pageUrl) {
+  const doc = String(html)
+  const raws = []
+
+  // Swagger UI: `SwaggerUIBundle({ url: "…" })` or `urls: [{ url: "…" }, …]`.
+  // Scanning a bounded window after each init call handles both shapes
+  // without catastrophic backtracking.
+  const swaggerInitRe = /SwaggerUI(?:Bundle)?\s*\(/g
+  let m
+  while ((m = swaggerInitRe.exec(doc)) !== null) {
+    const windowText = doc.slice(m.index, m.index + 3000)
+    const urlRe = /["']?\burl["']?\s*:\s*["']([^"']+)["']/g
+    let u
+    while ((u = urlRe.exec(windowText)) !== null) raws.push(u[1])
+  }
+
+  const redocAttrRe = /<redoc\b[^>]*?\bspec-url\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'>]+))/gi
+  while ((m = redocAttrRe.exec(doc)) !== null) raws.push(m[1] ?? m[2] ?? m[3])
+  const redocInitRe = /Redoc\.init\s*\(\s*["']([^"']+)["']/g
+  while ((m = redocInitRe.exec(doc)) !== null) raws.push(m[1])
+
+  // Scalar: <script id="api-reference" data-url="…"> — attr-order agnostic.
+  const scriptTagRe = /<script\b[^>]*>/gi
+  while ((m = scriptTagRe.exec(doc)) !== null) {
+    const tag = m[0]
+    if (!/\bid\s*=\s*["']api-reference["']/i.test(tag)) continue
+    const u = /\bdata-url\s*=\s*["']([^"']+)["']/i.exec(tag)
+    if (u) raws.push(u[1])
+  }
+
+  const stoplightRe = /apiDescriptionUrl\s*=\s*"([^"]+)"/g
+  while ((m = stoplightRe.exec(doc)) !== null) raws.push(m[1])
+
+  const out = []
+  const seen = new Set()
+  for (const raw of raws) {
+    let abs
+    try {
+      abs = new URL(raw, pageUrl)
+    } catch {
+      continue
+    }
+    if (abs.protocol !== 'http:' && abs.protocol !== 'https:') continue
+    const href = abs.toString()
+    if (seen.has(href)) continue
+    seen.add(href)
+    out.push(href)
+  }
+  return out
+}
+
 /**
  * Fetch the source URL, find the `<nav>` or `<aside>` that contains the most
  * links matching our known llms.txt URLs, and extract its heading/link
@@ -970,7 +1945,7 @@ function isMonotonicAlpha(titles) {
  * renders its sidebar server-side as <nav>/<aside> with <h*> section headers.
  * Returns null if coverage is too low to be useful.
  */
-async function scrapeNavFromSite(sourceUrl, knownPages, firecrawlKey) {
+async function scrapeNavFromSite(sourceUrl, knownPages, firecrawlKey, diagnostics = {}, oasSink) {
   // Index known pages by normalized pathname so we can match nav hrefs against them.
   const byPath = new Map()
   for (const p of knownPages) byPath.set(normalizePath(p.url), p)
@@ -994,6 +1969,9 @@ async function scrapeNavFromSite(sourceUrl, knownPages, firecrawlKey) {
 
     const html = await fetchHtml(url)
     if (!html) return 0
+    if (oasSink) {
+      oasSink.push(...extractOasUrlsFromHtml(html, url).map((u) => ({ url: u, text: null, source: 'html' })))
+    }
 
     const base = new URL(url)
     let best = { score: -Infinity, count: 0, tree: null }
@@ -1106,7 +2084,10 @@ async function scrapeNavFromSite(sourceUrl, knownPages, firecrawlKey) {
   const r0Start = Date.now()
   await visit(sourceUrl)
   const r0Ms = Date.now() - r0Start
-  if (categoryOrder.length === 0) return null
+  if (categoryOrder.length === 0) {
+    diagnostics.reason = 'no-categories-round0'
+    return null
+  }
 
   // Round 1 (parallel): visit pages so each branch has a chance to expose
   // its sub-items. Sidebars on most docs sites auto-expand the current
@@ -1123,9 +2104,9 @@ async function scrapeNavFromSite(sourceUrl, knownPages, firecrawlKey) {
   const r1Urls = isDiscovery
     ? flattenTree(categoryOrder).slice(0, MAX_DISCOVERY_FETCHES)
     : categoryOrder
-        .map((c) => c.pages[0])
-        .filter(Boolean)
-        .map((p) => toBrowsableUrl(p.url))
+      .map((c) => c.pages[0])
+      .filter(Boolean)
+      .map((p) => toBrowsableUrl(p.url))
   const r1Start = Date.now()
   // Firecrawl standard-plan concurrency is 10; 5 leaves headroom for retries.
   // Native HTTP can run hotter since we're hitting our own loopback.
@@ -1135,14 +2116,44 @@ async function scrapeNavFromSite(sourceUrl, knownPages, firecrawlKey) {
     styles.dim(`  ⏱  scrape breakdown: round0=${formatDuration(r0Ms)} round1=${formatDuration(r1Ms)} (${r1Urls.length} ${isDiscovery ? 'discovery' : 'category rep'} fetches)`),
   )
 
+  // Prune login/account chrome now that crawling is done. On Stoplight sites
+  // the "Sign in" → /auth link is the only seed on the bare landing page, so we
+  // had to keep it discoverable to reach the real sidebar — but it's not a doc.
+  const pruneAuthPages = (pages) => {
+    let removed = 0
+    for (let i = pages.length - 1; i >= 0; i--) {
+      const page = pages[i]
+      if (page.pages && page.pages.length > 0) removed += pruneAuthPages(page.pages)
+      if (isAuthChromeUrl(page.url) && (!page.pages || page.pages.length === 0)) {
+        matched.delete(normalizePath(page.url))
+        pages.splice(i, 1)
+        removed++
+      }
+    }
+    return removed
+  }
+  for (const cat of categoryOrder) pruneAuthPages(cat.pages)
+
   // Accept thresholds — looser in discovery mode (no llms.txt) where even a
   // single flat "Overview" bucket is better than nothing, stricter when we
   // have llms.txt to compare against.
   const categories = categoryOrder.filter((c) => c.pages.length > 0)
   if (isDiscovery) {
-    if (categories.length < 1 || matched.size < 5) return null
+    if (categories.length < 1 || matched.size < 5) {
+      diagnostics.reason = 'below-threshold'
+      diagnostics.categories = categories.length
+      diagnostics.matched = matched.size
+      diagnostics.need = '≥1 category, ≥5 matched pages'
+      return null
+    }
   } else {
-    if (categories.length < 2 || matched.size < 10) return null
+    if (categories.length < 2 || matched.size < 10) {
+      diagnostics.reason = 'below-threshold'
+      diagnostics.categories = categories.length
+      diagnostics.matched = matched.size
+      diagnostics.need = '≥2 categories, ≥10 matched pages'
+      return null
+    }
   }
   return { title: null, categories }
 }
@@ -1167,12 +2178,17 @@ function extractSidebarContainers(html) {
   const SIDEBAR_ATTR_RE =
     /\b(?:id|class|aria-label|data-testid)=(?:"[^"]*sidebar[^"]*"|'[^']*sidebar[^']*'|"[^"]*navigation[^"]*"|'[^']*navigation[^']*')|\brole=(?:"navigation"|'navigation')/i
 
+  // Layout-wrapper modifiers like `Page--with-sidebar` / `has-sidebar` describe
+  // an element that CONTAINS a sidebar, not one that IS a sidebar.
+  const WRAPPER_MODIFIER_RE = /(?:^|[\s"'-])(?:with|has)-sidebar(?=[\s"']|$)/i
+
   const out = []
   for (const tag of SIDEBAR_TAGS) {
     const openRe = new RegExp(`<${tag}\\b([^>]*)>`, 'gi')
     let m
     while ((m = openRe.exec(html)) !== null) {
       if (!SIDEBAR_ATTR_RE.test(m[1])) continue
+      if (WRAPPER_MODIFIER_RE.test(m[1])) continue
       const inner = extractBalancedTag(html, tag, m.index + m[0].length)
       if (inner != null && inner.length > 100) out.push(inner)
     }
@@ -1322,7 +2338,7 @@ function parseNavBlock(blockHtml, base, byPath) {
  *
  * Returns { category, nonApiOrphans, mergedScrapedTitles }.
  */
-function collectApiReferencePages(orphans, scraped) {
+function collectApiReferencePages(orphans, scraped, oasSink) {
   const API_PREFIX_RE = /^\/(api[-_]?reference|api|reference)(\/|$)/i
   const isApiUrl = (url) => {
     try {
@@ -1338,9 +2354,12 @@ function collectApiReferencePages(orphans, scraped) {
   const seenUrls = new Set()
   const push = (p) => {
     if (!p || !p.url) return
-    // Skip non-page assets like /api-reference/openapi.json that some
-    // llms.txt files list alongside real pages.
-    if (/\.(json|yaml|yml)$/i.test(p.url)) return
+    // Non-page assets like /api-reference/openapi.json that some llms.txt
+    // files list alongside real pages become spec candidates, not doc stubs.
+    if (/\.(json|ya?ml)$/i.test(p.url)) {
+      if (oasSink) oasSink.push({ url: p.url, text: p.title || null, source: 'link-sweep' })
+      return
+    }
     if (seenUrls.has(p.url)) return
     seenUrls.add(p.url)
     apiPages.push(p)
@@ -1383,7 +2402,7 @@ function collectApiReferencePages(orphans, scraped) {
     let segs = []
     try {
       segs = new URL(p.url).pathname.split('/').filter(Boolean)
-    } catch {}
+    } catch { }
     // segs[0] is the api-prefix itself; segs[1] (if any) is the resource.
     const resource = segs.length >= 3 ? segs[1] : null
     if (!resource) {
@@ -1430,7 +2449,7 @@ function collectApiReferencePages(orphans, scraped) {
  * Uses distinctive titles — just "Docs" or "Reference" — rather than the raw
  * segment name. Falls back to "Other" when the path has no usable type.
  */
-function bucketOrphansByPathType(orphans, scraped) {
+function bucketOrphansByPathType(orphans, scraped, oasSink) {
   const TYPE_TITLES = {
     reference: 'API Reference',
     api: 'API Reference',
@@ -1439,9 +2458,9 @@ function bucketOrphansByPathType(orphans, scraped) {
     endpoints: 'API Reference',
     endpoint: 'API Reference',
     changelog: 'Changelog',
-    release: 'Release Notes',
-    releases: 'Release Notes',
-    'release-notes': 'Release Notes',
+    release: 'Changelog',
+    releases: 'Changelog',
+    'release-notes': 'Changelog',
     recipes: 'Recipes',
     recipe: 'Recipes',
     guides: 'Guides',
@@ -1478,9 +2497,12 @@ function bucketOrphansByPathType(orphans, scraped) {
       continue
     }
     const pathname = url.pathname
-    // Skip OAS spec endpoints — `/api-reference/openapi.json` etc. aren't
-    // documentation pages and shouldn't become stubs.
-    if (/\.(json|ya?ml)$/i.test(pathname)) continue
+    // OAS spec endpoints — `/api-reference/openapi.json` etc. aren't
+    // documentation pages; capture them as spec candidates instead of stubs.
+    if (/\.(json|ya?ml)$/i.test(pathname)) {
+      if (oasSink) oasSink.push({ url: p.url, text: p.title || null, source: 'link-sweep' })
+      continue
+    }
 
     const segs = pathname.split('/').filter(Boolean)
     let type = null
@@ -1534,43 +2556,42 @@ function titleCase(s) {
 
 /**
  * Walk all scraped pages (including nested sub-pages) and move any whose URL
- * contains a strong reference segment (`/api-reference/`, `/endpoints/`, etc.)
- * into a single "API Reference" category. Some docs sites (e.g. greenflash.ai)
- * spotlight a handful of endpoints under "Developers" in the sidebar while the
- * bulk of endpoints live under a separate API Reference section — we favor
- * the URL-path signal over the sidebar placement so all reference pages land
- * together in `reference/` after staging.
+ * carries one of `segmentRe`'s path segments into a single category,
+ * flattening sub-pages as it goes. Shared backbone for the API Reference and
+ * Changelog sweeps: both favor the URL-path signal over sidebar placement, so
+ * e.g. `/api-reference/*` or `/changelog/*` pages land together in their own
+ * category even when a site's sidebar spotlights a few of them elsewhere.
  *
- * Returns the number of pages relocated.
+ * `categoryRe` matches an existing category title to merge into (so we don't
+ * create a duplicate); when none matches, a fresh `{ title: defaultTitle }`
+ * category is appended. Returns the number of pages relocated.
  */
-function reclassifyReferencePages(scraped) {
-  const REFERENCE_SEGMENT = /^(api[-_]?reference|endpoints?)$/i
+function reclassifyPagesByUrlSegment(scraped, { segmentRe, categoryRe, defaultTitle }) {
   const normTitle = (t) =>
     String(t || '')
       .replace(INVISIBLE_CHARS, '')
       .trim()
       .toLowerCase()
 
-  const looksLikeRefUrl = (url) => {
+  const urlHasSegment = (url) => {
     try {
       const segs = new URL(url).pathname.split('/').filter(Boolean)
-      return segs.some((s) => REFERENCE_SEGMENT.test(s))
+      return segs.some((s) => segmentRe.test(s))
     } catch {
       return false
     }
   }
 
-  // Find (or create) the canonical API Reference category. Prefer an existing
-  // one with a reference-shaped title so we don't end up with duplicates.
-  let refCat = scraped.categories.find((c) => /^(api[ -]?reference|reference|api|endpoints?)$/i.test(normTitle(c.title).replace(/\s+/g, ' ')))
-  const existedBefore = Boolean(refCat)
+  // Find (or create) the canonical destination category. Prefer an existing
+  // one with a matching title so we don't end up with duplicates.
+  let destCat = scraped.categories.find((c) => categoryRe.test(normTitle(c.title).replace(/\s+/g, ' ')))
 
   const collected = []
   const filterPages = (pages) => {
     const kept = []
     for (const p of pages || []) {
-      if (looksLikeRefUrl(p.url)) {
-        // Flatten sub-pages when relocating — API Reference is a flat list.
+      if (urlHasSegment(p.url)) {
+        // Flatten sub-pages when relocating — these sections are flat lists.
         collectFlat(p, collected)
         continue
       }
@@ -1580,32 +2601,156 @@ function reclassifyReferencePages(scraped) {
     return kept
   }
 
-  // Never pull pages out of the reference category itself.
+  // Never pull pages out of the destination category itself.
   for (const cat of scraped.categories) {
-    if (cat === refCat) continue
+    if (cat === destCat) continue
     cat.pages = filterPages(cat.pages)
   }
 
   if (collected.length === 0) return 0
 
-  if (!refCat) {
-    refCat = { title: 'API Reference', pages: [] }
-    scraped.categories.push(refCat)
+  if (!destCat) {
+    destCat = { title: defaultTitle, pages: [] }
+    scraped.categories.push(destCat)
   }
-  // Dedupe against anything already in the reference category.
-  const seen = new Set(refCat.pages.map((p) => normalizePath(p.url)))
+  // Dedupe against anything already in the destination category.
+  const seen = new Set(destCat.pages.map((p) => normalizePath(p.url)))
   for (const p of collected) {
     const key = normalizePath(p.url)
     if (seen.has(key)) continue
     seen.add(key)
-    refCat.pages.push(p)
+    destCat.pages.push(p)
+  }
+
+  // Drop now-empty categories (other than the one we may have just created).
+  scraped.categories = scraped.categories.filter((c) => c === destCat || (c.pages && c.pages.length > 0))
+
+  return collected.length
+}
+
+/**
+ * Sweep `/api-reference/*` and `/endpoints/*` pages into one "API Reference"
+ * category. Some docs sites (e.g. greenflash.ai) spotlight a handful of
+ * endpoints under "Developers" in the sidebar while the bulk of endpoints
+ * live under a separate API Reference section — this lands them all in
+ * `reference/` after staging. Returns the number of pages relocated.
+ */
+const API_REFERENCE_URL_SEGMENT_RE = /^(api[-_]?reference|endpoints?)$/i
+
+function urlIsApiReference(url) {
+  try {
+    const segs = new URL(url).pathname.split('/').filter(Boolean)
+    return segs.some((s) => API_REFERENCE_URL_SEGMENT_RE.test(s))
+  } catch {
+    return false
+  }
+}
+
+function reclassifyReferencePages(scraped) {
+  return reclassifyPagesByUrlSegment(scraped, {
+    segmentRe: API_REFERENCE_URL_SEGMENT_RE,
+    categoryRe: /^(api[ -]?reference|reference|api|endpoints?)$/i,
+    defaultTitle: 'API Reference',
+  })
+}
+
+// Anchored half covers ambiguous bare keywords (release, releases, versioned
+// release-2026-2) — those need to be the whole segment so we don't sweep
+// `/release-pipeline/` etc. Unanchored half covers the unambiguous keywords
+// (changelog, release-notes, whats-new), which we catch as substrings so
+// slugs like `/docs/changelog-javascript-agent` and `/docs/ios-sdk-changelog`
+// get pulled in too.
+const CHANGELOG_URL_SEGMENT_RE = /(?:^(?:releases?|release[-_]?v?\d[\w.-]*)$)|(?:changelog|change[-_]?log|release[-_]?notes?|whats?[-_]?new)/i
+const CHANGELOG_CATEGORY_TITLE_RE = /^(changelog|change ?log|release ?notes?|releases?|what'?s ?new)$/i
+
+function urlIsChangelog(url) {
+  try {
+    const segs = new URL(url).pathname.split('/').filter(Boolean)
+    return segs.some((s) => CHANGELOG_URL_SEGMENT_RE.test(s))
+  } catch {
+    return false
+  }
+}
+
+/**
+ * Pull changelog items OUT of llms.parsed.sections in place. Done before the
+ * organize pipeline runs so AI / clustering never sees them — they get
+ * re-attached as a dedicated Changelog category at the end via
+ * injectChangelogCategory. Sections that go empty afterwards are filtered out
+ * naturally by `usableSections` downstream.
+ *
+ * Returns the extracted items in `{ title, url, description? }` shape.
+ */
+function extractChangelogFromSections(sections) {
+  const extracted = []
+  for (const section of sections || []) {
+    const kept = []
+    for (const item of section.items || []) {
+      if (urlIsChangelog(item.url)) {
+        extracted.push({
+          title: item.text,
+          url: item.url,
+          ...(item.description ? { description: item.description } : {}),
+        })
+      } else {
+        kept.push(item)
+      }
+    }
+    section.items = kept
+  }
+  return extracted
+}
+
+/**
+ * Split a flat known-URL list into changelog vs. the rest. Used for the
+ * sitemap-only fallback, where there's no llms.txt section structure to mutate.
+ */
+function partitionChangelogFromKnownUrls(knownUrls) {
+  const extracted = []
+  const kept = []
+  for (const p of knownUrls || []) {
+    if (urlIsChangelog(p.url)) extracted.push(p)
+    else kept.push(p)
   }
+  return { extracted, kept }
+}
 
-  // Drop now-empty categories (other than the reference one we may have just created).
-  scraped.categories = scraped.categories.filter((c) => c === refCat || (c.pages && c.pages.length > 0))
+/**
+ * Attach pre-extracted changelog items as a dedicated Changelog category on
+ * the final organized tree. Merges into an existing Changelog-titled category
+ * if one is already present (e.g. the scrape path's reclassifier already built
+ * one); otherwise creates a fresh one. Dedupe is by normalized URL path.
+ */
+function injectChangelogCategory(organized, items) {
+  if (!items || items.length === 0) return
+  organized.categories = organized.categories || []
+  let dest = organized.categories.find((c) => CHANGELOG_CATEGORY_TITLE_RE.test(String(c.title || '').trim()))
+  if (!dest) {
+    dest = { title: 'Changelog', icon: null, pages: [] }
+    organized.categories.push(dest)
+  }
+  const seen = new Set((dest.pages || []).map((p) => normalizePath(p.url)))
+  for (const item of items) {
+    const key = normalizePath(item.url)
+    if (seen.has(key)) continue
+    seen.add(key)
+    dest.pages.push(item)
+  }
+}
 
-  // If the category existed before but the relocation was a no-op, surface 0.
-  return existedBefore ? collected.length : collected.length
+/**
+ * Sweep `/changelog/*`, `/release-notes/*` and `/releases/*` pages into one
+ * "Changelog" category, wherever the site's sidebar happened to place them.
+ * That category routes to `docs/Changelog/`; `--separate-changelog` later
+ * relocates it to a top-level `changelog/` dir. Returns the number of pages
+ * relocated.
+ */
+function reclassifyChangelogPages(scraped) {
+  return reclassifyPagesByUrlSegment(scraped, {
+    segmentRe: CHANGELOG_URL_SEGMENT_RE,
+    categoryRe: CHANGELOG_CATEGORY_TITLE_RE,
+    defaultTitle: 'Changelog',
+  })
 }
 
 function collectFlat(page, out) {
@@ -1619,71 +2764,168 @@ function collectFlat(page, out) {
   }
 }
 
+function countUrlPagesDeep(pages) {
+  let n = 0
+  for (const p of pages || []) {
+    if (p.url) n++
+    if (p.pages && p.pages.length > 0) n += countUrlPagesDeep(p.pages)
+  }
+  return n
+}
+
+function collectUrlPagesDeep(pages, out = []) {
+  for (const p of pages || []) {
+    if (p.url) out.push(p)
+    if (p.pages && p.pages.length > 0) collectUrlPagesDeep(p.pages, out)
+  }
+  return out
+}
+
+// urlTrieSegs is imported from ../utils/url-segs.js
+
 /**
- * Re-parent URL-bearing siblings by URL path so a page like `/foo/bar`
- * becomes a child of `/foo` when both appear in the same list.
+ * Re-parent URL-bearing siblings by URL path so descendants nest under their
+ * URL prefix even when no page exists at every intermediate segment. We build
+ * a trie from each page's URL path and render it back out as a page tree:
+ *
+ *   - The shared top-level prefix is elided (the category folder is its
+ *     implicit container — e.g. /docs/ai/evals collapses into "Pydantic Evals").
+ *   - Below the elision point, every segment becomes an entry: a real page
+ *     where one exists, otherwise an empty-parent stub (folder-icon, no URL).
+ *
+ * `anchorSegs` is the parent page's URL segments during recursion; when set,
+ * we walk the trie to that exact depth instead of eliding to the natural
+ * branch point, so a missing intermediate like `b` in `/a → /a/b/c, /a/b/d`
+ * still renders as an empty parent under `/a`.
  *
  * @example
  *   nestByUrlHierarchy([
- *     { title: 'CLI',      url: 'https://x.com/getting-started/cli' },
- *     { title: 'Overview', url: 'https://x.com/getting-started/cli/overview' },
+ *     { title: 'Hello',  url: 'https://x.com/main/sub1/sub2/sub3/hello' },
+ *     { title: 'World',  url: 'https://x.com/main/sub1/sub2/sub3/world' },
+ *     { title: 'Sub 1',  url: 'https://x.com/main/sub1' },
  *   ])
  *   // [
- *   //   { title: 'CLI', url: '…/cli', pages: [
- *   //     { title: 'Overview', url: '…/cli/overview' },
+ *   //   { title: 'Sub 1', url: '…/main/sub1', pages: [
+ *   //     { title: 'Sub2', _emptyParent: true, pages: [
+ *   //       { title: 'Sub3', _emptyParent: true, pages: [
+ *   //         { title: 'Hello', url: '…/main/sub1/sub2/sub3/hello' },
+ *   //         { title: 'World', url: '…/main/sub1/sub2/sub3/world' },
+ *   //       ]},
+ *   //     ]},
  *   //   ]},
  *   // ]
  */
-function nestByUrlHierarchy(pages) {
+function nestByUrlHierarchy(pages, anchorSegs = null) {
   if (!pages || pages.length === 0) return pages
 
-  // Recurse first so group-only / already-nested subtrees get re-nested too.
-  for (const p of pages) {
-    if (p.pages && p.pages.length > 0) p.pages = nestByUrlHierarchy(p.pages)
-  }
-
-  // Need at least two URL-bearing siblings to form a parent/child pair.
-  const byPath = new Map()
+  // Recurse first so subtrees that came in pre-nested (e.g. from the scraper)
+  // get re-nested against their parent's URL as anchor.
   for (const p of pages) {
-    if (!p.url) continue
-    const k = normalizePath(p.url)
-    if (!byPath.has(k)) byPath.set(k, p)
+    if (p.pages && p.pages.length > 0) {
+      let nextAnchor = anchorSegs
+      if (p._virtualPathSegs) nextAnchor = p._virtualPathSegs
+      else if (p.url) {
+        const s = urlTrieSegs(p.url)
+        if (s) nextAnchor = s
+      }
+      p.pages = nestByUrlHierarchy(p.pages, nextAnchor)
+    }
   }
-  if (byPath.size < 2) return pages
 
-  const result = []
+  const urlEntries = []
+  const groupOnly = []
   for (const p of pages) {
     if (!p.url) {
-      result.push(p)
+      groupOnly.push(p)
       continue
     }
-    let segs
-    try {
-      segs = new URL(p.url).pathname.split('/').filter(Boolean)
-    } catch {
-      result.push(p)
-      continue
-    }
-    let parent = null
-    for (let depth = segs.length - 1; depth >= 1; depth--) {
-      const key = ('/' + segs.slice(0, depth).join('/'))
-        .toLowerCase()
-        .replace(/\.(md|mdx|html?)$/i, '')
-        .replace(/\/$/, '')
-      const candidate = byPath.get(key)
-      if (candidate && candidate !== p) {
-        parent = candidate
-        break
+    const segs = urlTrieSegs(p.url)
+    // Zero-segment URLs (origin-only links like `https://build.example.com/`)
+    // have no path to nest by — if we inserted them into the trie they'd land
+    // at the root node and the container walk would render them as the
+    // top-level entry with every other page nested beneath them. Treat them
+    // like group-only pages so they just sit alongside the real tree.
+    if (segs && segs.length > 0) urlEntries.push({ page: p, segs })
+    else groupOnly.push(p)
+  }
+
+  if (urlEntries.length === 0) return pages
+  if (urlEntries.length === 1 && groupOnly.length === 0) return pages
+
+  // Build the URL trie. Each node tracks its own segment, the page at that
+  // exact path (if any), insertion-ordered children, and the segment path
+  // from root (used as the slug source for empty parents).
+  const root = { segment: '', page: null, children: new Map(), segs: [] }
+  for (const { page, segs } of urlEntries) {
+    let node = root
+    for (let i = 0; i < segs.length; i++) {
+      const segLower = segs[i].toLowerCase()
+      if (!node.children.has(segLower)) {
+        node.children.set(segLower, { segment: segs[i], page: null, children: new Map(), segs: segs.slice(0, i + 1) })
       }
+      node = node.children.get(segLower)
     }
-    if (parent) {
-      if (!parent.pages) parent.pages = []
-      parent.pages.push(p)
+    if (!node.page) node.page = page
+  }
+
+  // Find the trie node that represents the "container" for this level.
+  //   - With an anchor: that's the parent page's exact depth; we don't elide
+  //     past it, so intermediate segments below the anchor become empties.
+  //   - Without: walk down while there's only one no-page child (the shared
+  //     prefix). Stop at the first node that either has a page or branches.
+  let container = root
+  if (anchorSegs && anchorSegs.length > 0) {
+    for (let i = 0; i < anchorSegs.length; i++) {
+      const seg = anchorSegs[i].toLowerCase()
+      if (container.children.has(seg)) container = container.children.get(seg)
+      else break
+    }
+  } else {
+    while (container.children.size === 1 && !container.page) {
+      container = container.children.values().next().value
+    }
+  }
+
+  function nodeToPage(node) {
+    let outPage
+    if (node.page) {
+      outPage = node.page
     } else {
-      result.push(p)
+      const rawSeg = node.segment
+      const cleanedSeg = stripSegmentExtensions(rawSeg).replace(/^\d+[-_.]/, '') || rawSeg
+      outPage = {
+        title: titleCase(cleanedSeg),
+        _emptyParent: true,
+        _virtualPathSegs: node.segs,
+        pages: [],
+      }
+    }
+    if (node.children.size > 0) {
+      const childPages = []
+      for (const c of node.children.values()) childPages.push(nodeToPage(c))
+      // Preserve any pre-existing nested children whose URLs aren't already
+      // represented in the trie (group-only containers from the scraper, etc.).
+      const existing = outPage.pages || []
+      const trieUrls = new Set()
+      const collect = (p) => {
+        if (p.url) trieUrls.add(normalizePath(p.url))
+        for (const c of p.pages || []) collect(c)
+      }
+      for (const cp of childPages) collect(cp)
+      const preservedExisting = existing.filter((e) => !e.url || !trieUrls.has(normalizePath(e.url)))
+      outPage.pages = [...childPages, ...preservedExisting]
     }
+    return outPage
   }
-  return result
+
+  const trieResult = []
+  if (container.page) {
+    trieResult.push(nodeToPage(container))
+  } else {
+    for (const c of container.children.values()) trieResult.push(nodeToPage(c))
+  }
+
+  return [...trieResult, ...groupOnly]
 }
 
 /**
@@ -1697,10 +2939,10 @@ function slotOrphansByPath(scraped, knownPages) {
   const matched = new Set()
   const pathToCategory = new Map() // normalizedPath → category
   for (const cat of scraped.categories) {
-    for (const p of cat.pages) {
+    for (const p of collectUrlPagesDeep(cat.pages)) {
       const norm = normalizePath(p.url)
       matched.add(norm)
-      pathToCategory.set(norm, cat)
+      if (!pathToCategory.has(norm)) pathToCategory.set(norm, cat)
     }
   }
 
@@ -1800,6 +3042,84 @@ async function fetchHtmlDirect(url) {
   }
 }
 
+/**
+ * Download a candidate spec URL as raw bytes. Returns a Buffer, or null on
+ * non-OK / network error. No parsing or validation — the runner is the OAS gate.
+ */
+async function fetchSpecBytes(url) {
+  try {
+    const res = await fetch(url, {
+      redirect: 'follow',
+      headers: {
+        'User-Agent': 'readme-cli-import',
+        Accept: 'application/json, application/x-yaml, application/yaml, text/yaml, text/plain',
+      },
+    })
+    if (!res.ok) return null
+    return Buffer.from(await res.arrayBuffer())
+  } catch {
+    return null
+  }
+}
+
+// Extension for an extensionless candidate (e.g. /v3/api-docs), sniffed from
+// the body. Defaults to .json — the runner tolerates a mislabeled extension.
+function sniffSpecExtension(text) {
+  try {
+    JSON.parse(text)
+    return '.json'
+  } catch {
+    return OAS_TOP_LEVEL_KEY_RE.test(String(text).slice(0, 64 * 1024)) ? '.yaml' : '.json'
+  }
+}
+
+/**
+ * Download the captured candidate-spec URLs into `<stagingDir>/oas/`. Probe
+ * hits carry their prefetched bytes in `item.body`; everything else is
+ * fetched here. Bodies that look like HTML are skipped so the runner's
+ * user-visible candidate count stays honest — all other validation happens in
+ * the runner. Filenames come from the URL basename, keeping a .json/.yaml/.yml
+ * extension when present and sniffing one otherwise (collisions get a numeric
+ * suffix); the runner de-dups by content, so identical bytes under different
+ * names are handled there. Returns the count of files written.
+ */
+async function downloadOasSpecs(oasJsonUrls, stagingDir) {
+  const oasDir = path.join(stagingDir, 'oas')
+  fs.mkdirSync(oasDir, { recursive: true })
+  const usedNames = new Set()
+  let written = 0
+  for (const item of oasJsonUrls) {
+    const buf = item.body || (await fetchSpecBytes(item.url))
+    if (!buf) {
+      styles.info(styles.dim(`  oas: skipped ${item.url} (download failed)`))
+      continue
+    }
+    const text = buf.toString('utf8')
+    if (/^\s*</.test(text)) {
+      styles.info(styles.dim(`  oas: skipped ${item.url} (looks like HTML)`))
+      continue
+    }
+    let base
+    try {
+      base = path.basename(new URL(item.url).pathname) || 'openapi'
+    } catch {
+      base = 'openapi'
+    }
+    base = base.replace(/[^a-zA-Z0-9._-]/g, '-')
+    if (!/\.(json|ya?ml)$/i.test(base)) base += sniffSpecExtension(text)
+    let name = base
+    let n = 2
+    while (usedNames.has(name)) {
+      name = base.replace(/\.(json|ya?ml)$/i, `-${n}$&`)
+      n++
+    }
+    usedNames.add(name)
+    fs.writeFileSync(path.join(oasDir, name), buf)
+    written++
+  }
+  return written
+}
+
 /**
  * Firecrawl-backed HTML loader. Firecrawl runs a real browser, waits for
  * hydration, and returns the rendered DOM — which is what we need for sites
@@ -1900,14 +3220,29 @@ function urlNamespace(url) {
 function clusterByUrlPath(pages) {
   if (!pages || pages.length < 3) return null
 
-  const parts = pages.map((p) => {
+  // Pages with no path segments (e.g. an origin-only URL like
+  // `https://build.example.com/`) can't participate in segment-based
+  // clustering. A handful are fine — we'll fold them into the first cluster
+  // as a place to land — but if more than 20% of the input is rootless,
+  // the source isn't really a hierarchical doc set and clustering won't
+  // produce a meaningful tree. Bail in that case so the caller falls back.
+  const noSegPages = []
+  const clusterable = []
+  const parts = []
+  for (const p of pages) {
+    let segs = []
     try {
-      return new URL(p.url).pathname.split('/').filter(Boolean)
-    } catch {
-      return []
+      segs = new URL(p.url).pathname.split('/').filter(Boolean)
+    } catch { }
+    if (segs.length === 0) {
+      noSegPages.push(p)
+    } else {
+      clusterable.push(p)
+      parts.push(segs)
     }
-  })
-  if (parts.some((pp) => pp.length === 0)) return null
+  }
+  if (noSegPages.length / pages.length > 0.2) return null
+  if (clusterable.length < 3) return null
 
   // Longest common prefix depth.
   let commonDepth = 0
@@ -1920,13 +3255,13 @@ function clusterByUrlPath(pages) {
   // The segment right after the common base is the category key.
   const keyIdx = commonDepth
   const byKey = new Map()
-  for (let i = 0; i < pages.length; i++) {
+  for (let i = 0; i < clusterable.length; i++) {
     const key = parts[i][keyIdx]
     // Skip pages that have no segment at the cluster index (they're AT the
     // common base — those would become their own "index"-like category).
     if (!key) continue
     if (!byKey.has(key)) byKey.set(key, [])
-    byKey.get(key).push(pages[i])
+    byKey.get(key).push(clusterable[i])
   }
 
   // Reject weak clusterings: need at least 2 groups AND at least one group
@@ -1937,7 +3272,7 @@ function clusterByUrlPath(pages) {
 
   // Preserve first-appearance order so the sidebar reflects source order.
   const firstSeen = new Map()
-  pages.forEach((p, i) => {
+  clusterable.forEach((p, i) => {
     const key = parts[i][keyIdx]
     if (key && !firstSeen.has(key)) firstSeen.set(key, i)
   })
@@ -1948,28 +3283,173 @@ function clusterByUrlPath(pages) {
     pages: byKey.get(key),
   }))
 
-  // A cluster with exactly one top-level page is NOT a real category — it's
-  // a parent page with children wrapped in a pseudo-category label. Categories
-  // are grouping labels with no content of their own; parent pages have
-  // content AND children. Collect those singletons into a shared
-  // "Documentation" bucket so they're siblings at the top level, each with
-  // their own sub-tree intact.
-  const multipageClusters = rawClusters.filter((c) => c.pages.length >= 2)
-  const singletonPages = rawClusters.filter((c) => c.pages.length === 1).flatMap((c) => c.pages)
-
-  const out = []
-  if (singletonPages.length > 0) {
-    out.push({ title: 'Documentation', pages: singletonPages })
+  // Classify each cluster as either a real category or an "Other
+  // Documentation" misfit. A real category needs at least 2 pages — a
+  // "category" with a single file in it just adds a level of nesting
+  // without adding structure. Pydantic's `/docs/logfire/get-started` is
+  // the motivating example: as the only `/docs/logfire/*` URL it
+  // shouldn't get its own top-level "Logfire" folder. Lone pages land in
+  // Other Documentation where they sit alongside other low-volume material.
+  const realCategories = []
+  const misfitPages = []
+  for (const cluster of rawClusters) {
+    if (cluster.pages.length >= 2) {
+      realCategories.push(cluster)
+      continue
+    }
+    misfitPages.push(cluster.pages[0])
   }
-  out.push(...multipageClusters)
 
-  // If we didn't actually produce any multi-page cluster, clustering added no
-  // value — every page was a singleton and we'd just have renamed Overview.
-  // Tell the caller to stick with the original flat shape.
-  if (multipageClusters.length === 0) return null
+  // If clustering produced no real categories, it added no value — tell the
+  // caller to fall back rather than hand back a lone misfit bucket.
+  if (realCategories.length === 0) return null
+
+  const out = [...realCategories]
+
+  // "Other Documentation" goes last so it never displaces a real category.
+  // It absorbs both standalone-page singletons and zero-segment URLs (those
+  // can't participate in segment-based clustering at all but the >20% guard
+  // above already proved they're a minority).
+  const otherDocs = [...misfitPages, ...noSegPages]
+  if (otherDocs.length > 0) {
+    out.push({ title: 'Other Documentation', pages: otherDocs })
+  }
   return out
 }
 
+/**
+ * Import section landing pages that the source models structurally rather than
+ * as a link. In llms.txt a section like the Vercel CLI is a bare `## CLI`
+ * heading followed by 68 `/docs/cli/<cmd>` links — the importer turns the
+ * heading into a category but `/docs/cli` itself is never listed, so its real,
+ * content-bearing overview page is dropped. ReadMe then redirects the
+ * content-less category to its first child (a broken landing page).
+ *
+ * For each category we compute the common URL prefix of its children (the
+ * section's landing URL), HTTP-probe it, and — when it really exists and stays
+ * in scope — prepend it as a real page so nestByUrlHierarchy renders it as the
+ * parent holding the children. Mutates `organized` in place.
+ */
+async function injectSectionLandingPages(organized, sourceUrl) {
+  const cats = organized?.categories || []
+  if (cats.length === 0) return
+
+  // Dedup set: every page URL already in the tree, by normalized path.
+  const known = new Set()
+  for (const c of cats) {
+    for (const p of collectUrlPagesDeep(c.pages || [])) known.add(normalizePath(p.url))
+  }
+
+  // Base path of the import itself — landing candidates must be strictly
+  // deeper than this so we never probe `/docs` (the base) as a "section".
+  const baseDepth = (urlTrieSegs(sourceUrl) || []).length
+  const origin = new URL(sourceUrl).origin
+
+  // Build one landing candidate per eligible category (cheap gate first).
+  const candidates = []
+  for (const cat of cats) {
+    const kids = collectUrlPagesDeep(cat.pages || [])
+    if (kids.length < 2) continue
+
+    // Longest common path prefix across the children — same loop as
+    // clusterByUrlPath's common-depth scan, over urlTrieSegs.
+    const parts = kids.map((p) => urlTrieSegs(p.url)).filter(Boolean)
+    if (parts.length < 2) continue
+    let commonDepth = 0
+    while (commonDepth < parts[0].length) {
+      const seg = parts[0][commonDepth]
+      if (!parts.every((pp) => pp[commonDepth] === seg)) break
+      commonDepth++
+    }
+    const commonSegs = parts[0].slice(0, commonDepth)
+
+    // Must be strictly deeper than the import base, and not already a page.
+    if (commonSegs.length <= baseDepth) continue
+    const landingUrl = `${origin}/${commonSegs.join('/')}`
+    if (known.has(normalizePath(landingUrl))) continue
+
+    candidates.push({ cat, landingUrl })
+  }
+  if (candidates.length === 0) return
+
+  // Probe surviving candidates in parallel. Accept only a real, in-scope page
+  // that doesn't redirect onto an already-known child.
+  const accepted = []
+  await visitAllInParallel(
+    candidates,
+    async ({ cat, landingUrl }) => {
+      try {
+        const res = await fetch(landingUrl, {
+          redirect: 'follow',
+          headers: { 'User-Agent': 'readme-cli-import' },
+        })
+        if (!res.ok) return
+        const candPath = new URL(landingUrl).pathname.toLowerCase()
+        const finalPath = new URL(res.url).pathname.toLowerCase()
+        if (!finalPath.startsWith(candPath)) return
+        if (known.has(normalizePath(res.url))) return
+
+        // Check `<meta http-equiv="refresh">` navigates the browser elsewhere. 
+        const html = await res.text()
+        const metaRefresh = parseMetaRefreshTarget(html, res.url)
+        if (metaRefresh && normalizePath(metaRefresh) !== normalizePath(res.url)) return
+        const canonical = parseHtmlCanonical(html, res.url)
+        if (canonical && known.has(normalizePath(canonical))) return
+
+        accepted.push({ cat, landingUrl })
+      } catch {
+        // Network error / unparseable — treat as "no landing page".
+      }
+    },
+    LANDING_PROBE_CAP,
+  )
+
+  for (const { cat, landingUrl } of accepted) {
+    // Don't set x-from-llms: the page isn't in llmsPaths, so the runner fetches
+    // its rendered HTML via the content cascade (there's no .md mirror).
+    cat.pages.unshift({ title: cat.title, url: landingUrl })
+    known.add(normalizePath(landingUrl))
+  }
+  if (accepted.length > 0) {
+    styles.info(`Imported ${styles.bold(String(accepted.length))} section landing page(s).`)
+  }
+}
+
+/**
+ * Parse a client-side redirect target from an HTML page. Return pages with 
+ * `<meta http-equiv="refresh" content="0;url=/target/">` that navigates
+ * the browser elsewhere.
+ */
+function parseMetaRefreshTarget(html, baseUrl) {
+  if (!html) return null
+  const tag = /<meta[^>]+http-equiv=["']?refresh["']?[^>]*>/i.exec(html)
+  if (!tag) return null
+  const m = /content=["'][^"']*url=([^"'>\s]+)/i.exec(tag[0])
+  if (!m) return null
+  try {
+    return new URL(decodeEntities(m[1].trim()), baseUrl).toString()
+  } catch {
+    return null
+  }
+}
+
+/**
+ * Parse the `<link rel="canonical" href="...">` target from an HTML page,
+ * resolved absolute against `baseUrl`. Returns null when absent or unparseable.
+ */
+function parseHtmlCanonical(html, baseUrl) {
+  if (!html) return null
+  const tag = /<link[^>]+rel=["']?canonical["']?[^>]*>/i.exec(html)
+  if (!tag) return null
+  const m = /href=["']([^"']+)["']/i.exec(tag[0])
+  if (!m) return null
+  try {
+    return new URL(decodeEntities(m[1].trim()), baseUrl).toString()
+  } catch {
+    return null
+  }
+}
+
 /**
  * Used by discovery-mode scraping (no llms.txt) to decide whether a nav
  * link is worth importing as a doc page. Filters out cross-origin links,
@@ -1990,6 +3470,20 @@ function isDiscoverableLink(abs, base) {
   return true
 }
 
+// Leading-segment auth routes: /auth, /login, /sign-in, /logout, /sso, /oauth…
+// These are login/account chrome (Stoplight's "Sign in" → /auth, etc.), not
+// docs. NOT filtered at link-discovery time — on some Stoplight sites the
+// bare landing page only exposes the "Sign in" link, which we still need as a
+// crawl seed to reach the real sidebar. Pruned from the tree post-crawl.
+const AUTH_ROUTE_RE = /^\/(?:auth|login|log-?in|sign-?in|sign-?up|signup|logout|log-?out|sign-?out|signout|register|sso|oauth2?)(?:\/|$)/i
+function isAuthChromeUrl(url) {
+  try {
+    return AUTH_ROUTE_RE.test(new URL(url).pathname)
+  } catch {
+    return false
+  }
+}
+
 // Zero-width spaces, direction marks, word joiner, BOM — some docs sites
 // inject these into sidebar headings (docs.greenflash.ai prefixes
 // "API Reference" with U+200B), which otherwise defeats downstream
@@ -2029,20 +3523,7 @@ function decodeEntities(s) {
     .replace(/&rdquo;/g, '”')
 }
 
-/**
- * Reduce a URL to a comparable pathname: lowercase host, strip trailing slash
- * and common suffixes (.md, .html) so `/foo/bar.md` and `/foo/bar` match.
- */
-function normalizePath(url) {
-  try {
-    const u = new URL(url)
-    let p = u.pathname.replace(/\/$/, '').toLowerCase()
-    p = p.replace(/\.(md|mdx|html?)$/i, '')
-    return p
-  } catch {
-    return String(url).toLowerCase()
-  }
-}
+// normalizePath is imported from ../utils/url-segs.js
 
 /**
  * Given a set of scraped categories + orphan pages that didn't match the nav
@@ -2056,7 +3537,8 @@ async function slotOrphansWithClaude(scraped, orphans, model) {
     categories: scraped.categories,
     orphans,
   })
-  const raw = await runJsonQuery({ systemPrompt, userPrompt, model })
+  const result = await runJsonQuery({ systemPrompt, userPrompt, model, schema: slotOrphansOutputSchema })
+  const raw = result.assignments
   if (!Array.isArray(raw)) {
     // If the model didn't cooperate, return all orphans unassigned.
     return orphans
@@ -2086,8 +3568,8 @@ async function iconizeScrapedNav(scraped, _unused, model, siteTitle) {
   const { systemPrompt, userPrompt } = iconizeNavPrompt({
     categories: scraped.categories,
   })
-  const icons = await runJsonQuery({ systemPrompt, userPrompt, model })
-  const iconArr = Array.isArray(icons) ? icons : []
+  const result = await runJsonQuery({ systemPrompt, userPrompt, model, schema: iconizeNavOutputSchema })
+  const iconArr = Array.isArray(result.icons) ? result.icons : []
   return {
     title: siteTitle || null,
     categories: scraped.categories.map((c, i) => ({
@@ -2099,31 +3581,22 @@ async function iconizeScrapedNav(scraped, _unused, model, siteTitle) {
 }
 
 /**
- * Sections are "usable" when the llms.txt already did the hard grouping work
- * for us — meaningful titles, not too many/few, each populated. When usable we
- * take a fast path that only asks Claude for icons + title polish instead of
- * re-bucketing every page, which is the slow part of a full reorg.
- */
-// Sections named like these are catch-all buckets — even in richly-structured
-// llms.txt files (e.g. Stripe's "Docs" section is where pages go that don't
-// fit into a named product tab), so always drop them rather than promote the
-// grab-bag contents to a top-level sidebar category.
-const GENERIC_SECTION_RE =
-  /^(resources?|english|root url|pages?|docs?|documentation|content|available languages.*|site|sitemap|index|home|optional|instructions?(\s|:).*|miscellaneous|misc|other)$/i
-
-/**
- * Return the subset of llms.txt sections that carry real structural signal —
- * drop catch-all buckets ("Docs", "Resources", "Optional"), empty sections,
- * and oversized ones (site-dumps masquerading as sections).
+ * Return the subset of llms.txt sections worth sending to the fast path —
+ * drop only structurally bad ones (empty, or oversized site-dumps). Heading
+ * names are preserved verbatim: a heading like "Docs" or "Documentation"
+ * might be a catch-all on one site (bad) and a real top-level category on
+ * another (fine), and we can't tell cheaply, so we don't try.
  */
 function usableSections(sections) {
   if (!sections) return []
-  return sections.filter((s) => s.title && !GENERIC_SECTION_RE.test(s.title.trim()) && s.items && s.items.length > 0 && s.items.length <= 200)
+  return sections.filter((s) => s.title && s.items && s.items.length > 0 && s.items.length <= 200)
 }
 
 function sectionsLookUsable(sections) {
-  if (!sections || sections.length > 40) return false
-  return usableSections(sections).length >= 3
+  const usable = usableSections(sections)
+  if (usable.length < 3) return false
+  const avg = usable.reduce((n, s) => n + s.items.length, 0) / usable.length
+  return avg >= 5 && avg <= 100
 }
 
 async function organizeWithClaude(parsed, model) {
@@ -2139,16 +3612,17 @@ async function organizeWithClaude(parsed, model) {
  * is O(sections), not O(pages), so this is usually ~5-15s vs. a full reorg.
  */
 async function organizeFromSections(parsed, model) {
-  // Drop generic/empty/oversized sections so they don't pollute the sidebar
-  // (e.g. Stripe's llms.txt has a "Docs" catch-all and a 0-item "Instructions
-  // for Large Language Model Agents" section — neither is structural signal).
+  // Drop empty/oversized sections (e.g. a 0-item "Instructions for Large
+  // Language Model Agents" preamble, or a 500-row site-dump) — heading
+  // names are preserved as-is.
   const sections = usableSections(parsed.sections)
 
   const { systemPrompt, userPrompt } = organizeFromSectionsPrompt({
     siteTitle: parsed.title,
     sections,
   })
-  const raw = await runJsonQuery({ systemPrompt, userPrompt, model })
+  const result = await runJsonQuery({ systemPrompt, userPrompt, model, schema: organizeFromSectionsOutputSchema })
+  const raw = result.sections
   if (!Array.isArray(raw)) {
     throw new Error('Fast-path expected a JSON array of {title, icon} entries.')
   }
@@ -2183,18 +3657,22 @@ async function organizeFromScratch(parsed, model) {
     siteTitle: parsed.title,
     items,
   })
-  const raw = await runJsonQuery({ systemPrompt, userPrompt, model })
+  const raw = await runJsonQuery({ systemPrompt, userPrompt, model, schema: organizeFromScratchOutputSchema })
 
   // Rehydrate pages from the id references Claude returned.
   const expandedCategories = []
   const usedIds = new Set()
+  const usedUrls = new Set()
   for (const cat of raw.categories || []) {
     const pages = []
     for (const id of cat.pageIds || []) {
       const item = items[id]
       if (!item) continue // ignore out-of-range ids
       if (usedIds.has(id)) continue // ignore dupes
+      const normUrl = normalizePath(item.url)
+      if (usedUrls.has(normUrl)) continue // guard against same URL at different input indexes
       usedIds.add(id)
+      usedUrls.add(normUrl)
       pages.push({
         title: item.title,
         url: item.url,
@@ -2224,11 +3702,13 @@ async function organizeFromScratch(parsed, model) {
 }
 
 /**
- * Shared Claude call for "send a prompt, parse JSON back". Logs the prompts so
- * we can see what went in, runs a heartbeat so silent model latency doesn't
- * look like a hang, and strips stray code fences if the model adds them.
+ * Shared Claude call for "send a prompt, get schema-validated JSON back".
+ * Logs the prompts so we can see what went in, and runs a heartbeat so silent
+ * model latency doesn't look like a hang. The SDK enforces `schema` and
+ * returns parsed data on the result message's `structured_output` field, so
+ * there's no text extraction or JSON.parse on this path.
  */
-async function runJsonQuery({ systemPrompt, userPrompt, model }) {
+async function runJsonQuery({ systemPrompt, userPrompt, model, schema }) {
   console.log()
   console.log(styles.dim('─ system prompt ─'))
   console.log(styles.dim(systemPrompt))
@@ -2242,24 +3722,28 @@ async function runJsonQuery({ systemPrompt, userPrompt, model }) {
   console.log()
 
   const heartbeat = setInterval(() => process.stdout.write(styles.dim('.')), 1000)
-  let text = ''
+  let structured
   try {
     for await (const message of query({
       prompt: userPrompt,
       options: {
         systemPrompt,
         allowedTools: [],
+        outputFormat: { type: 'json_schema', schema },
         ...(model ? { model } : {}),
       },
     })) {
-      if (message.type === 'assistant' && message.message?.content) {
-        for (const block of message.message.content) {
-          if (block.type === 'text' && block.text) text += block.text
+      if (message.type === 'result') {
+        if (message.subtype === 'error_max_structured_output_retries') {
+          throw new Error(
+            'Claude could not produce output matching the schema after retries. ' +
+            'Likely hit the model\'s output limit — try --model sonnet.',
+          )
         }
-      } else if (message.type === 'result') {
         if (message.subtype && message.subtype !== 'success') {
           throw new Error(`Claude failed: ${message.subtype}${message.error?.message ? ' — ' + message.error.message : ''}`)
         }
+        structured = message.structured_output
         break
       }
     }
@@ -2268,18 +3752,10 @@ async function runJsonQuery({ systemPrompt, userPrompt, model }) {
     process.stdout.write('\n')
   }
 
-  const stripped = stripCodeFences(text)
-
-  try {
-    return JSON.parse(stripped)
-  } catch (e) {
-    throw new Error(
-      `Claude returned invalid JSON: ${e.message}\n` +
-        `Output length: ${stripped.length} chars. Likely hit the model's output limit — try --model sonnet.\n\n` +
-        `First 500 chars:\n${stripped.slice(0, 500)}\n\n` +
-        `Last 500 chars:\n${stripped.slice(-500)}`,
-    )
+  if (!structured || typeof structured !== 'object') {
+    throw new Error('Claude returned no structured output')
   }
+  return structured
 }
 
 /**
@@ -2294,85 +3770,725 @@ async function runJsonQuery({ systemPrompt, userPrompt, model }) {
  *
  * Returns deduped URLs in probe order.
  */
-function buildLlmsCandidates(sourceUrl) {
+// URL patterns we strip from llms.txt items — they show up alongside real
+// pages but shouldn't be staged as docs:
+//   - .yaml specs, package.json-ish assets, sitemaps, etc. (machine-readable)
+//   - llms.txt, llms-full.txt, llms-ctx.txt, … (other llms.txt variants
+//     that point back at the same index/dump, not real pages)
+//
+// `.json`/`.yaml`/`.yml` spec links are captured up-front by
+// extractOasSpecUrlsFromParsed (candidate OpenAPI specs we download into oas/
+// and upload separately) and removed from the parsed tree before this drop
+// runs, so spec-shaped items never reach here — `.yaml` below is a backstop.
+const ASSET_EXT_RE = /\.(ya?ml|xml|toml)$/i
+const LLMS_TXT_RE = /(?:^|\/)llms[^/]*\.txt$/i
+function isAssetOrMetaUrl(url) {
+  try {
+    const pn = new URL(url).pathname
+    return ASSET_EXT_RE.test(pn) || LLMS_TXT_RE.test(pn)
+  } catch {
+    return false
+  }
+}
+
+// A llms.txt item whose URL points at a spec-shaped file (.json/.yaml/.yml).
+// These are pulled out of the parsed tree before organization (so stub
+// generation never sees them) and downloaded into the staged `oas/` dir; the
+// runner validates each as OpenAPI and uploads the valid ones.
+function isOasSpecUrl(url) {
+  try {
+    return /\.(json|ya?ml)$/i.test(new URL(url).pathname)
+  } catch {
+    return false
+  }
+}
+
+// Immediate-child segments of root that signal a docs subtree on a
+// non-docs hostname. We deliberately keep the set tiny — over-matching
+// here (e.g. including `api`, `learn`) would over-filter sites whose
+// "docs" subtree lives at the apex.
+const DOCS_LIKE_ROOT_SEGMENTS = new Set(['docs', 'doc', 'documentation', 'document'])
+
+/**
+ * Narrow a merged llms.txt result to a docs subtree when a root-level
+ * llms.txt pulled non-doc URLs (marketing, pricing, blog, etc.) into the
+ * union.
+ *
+ * Sites that publish a root llms.txt alongside per-product subtree files
+ * often mix marketing/pricing/blog links into the root file. The BFS merge
+ * happily unions all of it, and without this pass we'd stage stubs for
+ * those pages too.
+ *
+ * elevenlabs.io is the canonical example: its root `elevenlabs.io/llms.txt`
+ * lists marketing pages alongside docs, and per-product files live at
+ * `elevenlabs.io/docs/<product>/llms.txt`. After the BFS we have docs URLs
+ * + pricing/blog URLs in the same merged result. With this pass we keep
+ * only URLs under `/docs/`.
+ *
+ * Conditions (all must hold), chosen so we only narrow when the risk is
+ * real and there's a clear scope to narrow to:
+ *   - Hostname doesn't itself contain "docs" — `docs.example.com` is
+ *     already a docs-only host; nothing to narrow.
+ *   - A root-level llms.txt was actually probed and hit (valid OR
+ *     invalid). Either way it's what dragged in the wider URL space via
+ *     the BFS expansion, so it's the trigger we care about.
+ *   - Multiple llms.txt files contributed to the merge. A single-file
+ *     merge is by definition scoped to that one file's intent.
+ *   - The merged URL set has a docs-shaped first-segment child of root
+ *     (`/docs/`, `/documentation/`, …). That child is our scope target.
+ *
+ * Mutates `llms.parsed.sections` in place. Returns `{ segment, kept,
+ * dropped }` when narrowing happened, or `null` when conditions weren't
+ * met.
+ */
+function narrowToDocsSubtreeIfNeeded(llms, sourceUrl, hits) {
+  if (!llms || llms.sourceFiles.length < 2) return null
+  if (sourceUrl.hostname.toLowerCase().includes('docs')) return null
+  const rootHit = hits.some((h) => h.path === '/' || h.path === '')
+  if (!rootHit) return null
+
+  // Identify docs-shaped first segments of root that actually appear in
+  // the merged URLs. Walking just the first path segment matches the
+  // user's spec ("we only look at immediate children in this case").
+  const firstSegs = new Set()
+  for (const section of llms.parsed.sections) {
+    for (const item of section.items) {
+      try {
+        const seg = new URL(item.url).pathname.split('/').filter(Boolean)[0]
+        if (seg) firstSegs.add(seg.toLowerCase())
+      } catch {
+        // ignore unparseable URLs
+      }
+    }
+  }
+  let docsSegment = null
+  for (const seg of firstSegs) {
+    if (DOCS_LIKE_ROOT_SEGMENTS.has(seg)) {
+      docsSegment = seg
+      break
+    }
+  }
+  if (!docsSegment) return null
+
+  const prefix = '/' + docsSegment
+  let kept = 0
+  let dropped = 0
+  const keptSections = []
+  for (const section of llms.parsed.sections) {
+    const items = []
+    for (const item of section.items) {
+      try {
+        const pn = new URL(item.url).pathname.toLowerCase()
+        if (pn === prefix || pn.startsWith(prefix + '/')) {
+          items.push(item)
+          kept++
+        } else {
+          dropped++
+        }
+      } catch {
+        dropped++
+      }
+    }
+    if (items.length > 0) {
+      section.items = items
+      keptSections.push(section)
+    }
+  }
+  llms.parsed.sections = keptSections
+  return { segment: docsSegment, kept, dropped }
+}
+
+/**
+ * Pull spec-shaped items (.json/.yaml/.yml) out of a parsed llms.txt structure
+ * IN PLACE, returning the removed items as `[{ url, text, source }]`. Sections
+ * emptied by the removal are pruned too — same shape as
+ * dropAssetItemsFromParsed. Run BEFORE dropAssetItemsFromParsed / knownUrls /
+ * organize so the captured specs never enter the stub tree (skeleton has zero
+ * visibility of OAS files). The caller downloads the returned URLs into the
+ * staged `oas/` dir.
+ */
+function extractOasSpecUrlsFromParsed(parsed) {
+  const captured = []
+  const keptSections = []
+  for (const section of parsed.sections) {
+    const keptItems = []
+    for (const item of section.items) {
+      if (isOasSpecUrl(item.url)) {
+        captured.push({ url: item.url, text: item.text || null, source: 'llms' })
+        continue
+      }
+      keptItems.push(item)
+    }
+    if (keptItems.length > 0) {
+      section.items = keptItems
+      keptSections.push(section)
+    }
+  }
+  parsed.sections = keptSections
+  return captured
+}
+
+/**
+ * Drop asset/meta items in place from a parsed llms.txt structure. Sections
+ * that end up empty after the drop are removed too. Returns the count of
+ * items dropped.
+ */
+function dropAssetItemsFromParsed(parsed) {
+  let dropped = 0
+  const keptSections = []
+  for (const section of parsed.sections) {
+    const keptItems = []
+    for (const item of section.items) {
+      if (isAssetOrMetaUrl(item.url)) {
+        dropped++
+        continue
+      }
+      keptItems.push(item)
+    }
+    if (keptItems.length > 0) {
+      section.items = keptItems
+      keptSections.push(section)
+    }
+  }
+  parsed.sections = keptSections
+  return dropped
+}
+
+
+/**
+ * Build the ordered list of well-known docs locations to probe. For each route
+ * we emit a subdomain candidate (stripping a leading `www.` first) and, unless
+ * the route is subdomain-only, a path candidate. Base URLs end in `/` so
+ * `${url.href}llms.txt` is well-formed.
+ */
+function buildWellKnownDocRoutes(sourceUrl) {
   const out = []
   const seen = new Set()
-  const add = (url) => {
-    if (!seen.has(url)) {
-      seen.add(url)
-      out.push(url)
-    }
+  const add = (kind, href) => {
+    if (seen.has(href)) return
+    seen.add(href)
+    out.push({ kind, url: new URL(href) })
   }
 
-  const origin = sourceUrl.origin
-  const segs = sourceUrl.pathname.split('/').filter(Boolean)
-  for (let i = segs.length; i >= 0; i--) {
-    const prefix = segs.slice(0, i).join('/')
-    add(`${origin}${prefix ? '/' + prefix : ''}/llms.txt`)
+  const apexHost = sourceUrl.hostname.replace(/^www\./i, '')
+  for (const route of WELL_KNOWN_DOC_ROUTES) {
+    const subHost = `${route}.${apexHost}`
+    if (subHost !== sourceUrl.hostname) add('subdomain', `${sourceUrl.protocol}//${subHost}/`)
+    if (!SUBDOMAIN_ONLY_DOC_ROUTES.has(route)) add('path', `${sourceUrl.origin}/${route}/`)
   }
   return out
 }
 
+// Content gate for spec candidates — many sites 200 with an HTML shell for
+// any path, so a body only counts as a spec if it parses as JSON with a
+// top-level `openapi`/`swagger` version string, or matches the YAML form of
+// that key. (Pattern mirrors ai-cli-runner/app/lib/oas-sniff.ts — copied,
+// never imported.)
+const OAS_TOP_LEVEL_KEY_RE = /^\s*['"]?(openapi|swagger)['"]?\s*:/m
+function sniffOasContent(text) {
+  const body = String(text)
+  if (/^\s*</.test(body)) return false
+  try {
+    const obj = JSON.parse(body)
+    if (!obj || typeof obj !== 'object') return false
+    return typeof obj.openapi === 'string' || typeof obj.swagger === 'string'
+  } catch {
+    return OAS_TOP_LEVEL_KEY_RE.test(body.slice(0, 64 * 1024))
+  }
+}
+
+function buildWellKnownOasProbeUrls(sourceUrl, docsBase, preAdoptionOrigin) {
+  const urls = []
+  const seen = new Set()
+  const add = (href) => {
+    if (seen.has(href)) return
+    seen.add(href)
+    urls.push(href)
+  }
+  const origins = [sourceUrl.origin]
+  if (preAdoptionOrigin && preAdoptionOrigin !== sourceUrl.origin) origins.push(preAdoptionOrigin)
+  for (const origin of origins) {
+    for (const p of WELL_KNOWN_OAS_PATHS) add(`${origin}/${p}`)
+  }
+  if (docsBase && docsBase.kind === 'path') {
+    const base = docsBase.url.href.replace(/\/+$/, '')
+    for (const p of ['openapi.json', 'openapi.yaml', 'swagger.json']) add(`${base}/${p}`)
+  }
+  return urls
+}
+
 /**
- * Best-effort fetch of a site's /llms.txt. Returns { ok, status, error, parsed }
- * where parsed is { title, sections: [{ title, items: [{ text, url, description }] }] }.
+ * Probe well-known spec paths on the source origins. A hit is an OK response
+ * whose body sniffs as a spec and whose final URL stayed on a probed origin
+ * (redirects to a marketing apex don't count). Returns ALL hits — a site can
+ * serve several specs — with the fetched bytes attached so downloadOasSpecs
+ * never re-downloads them.
  */
-async function fetchLlmsTxt(llmsUrl) {
+async function probeWellKnownOasPaths(sourceUrl, docsBase, preAdoptionOrigin) {
+  const probeUrls = buildWellKnownOasProbeUrls(sourceUrl, docsBase, preAdoptionOrigin)
+  const allowedOrigins = new Set([sourceUrl.origin])
+  if (preAdoptionOrigin) allowedOrigins.add(preAdoptionOrigin)
+  const hits = await Promise.all(
+    probeUrls.map(async (url) => {
+      try {
+        const res = await fetch(url, { redirect: 'follow', headers: { 'User-Agent': 'readme-cli-import' } })
+        if (!res.ok) return null
+        let finalOrigin
+        try {
+          finalOrigin = new URL(res.url || url).origin
+        } catch {
+          return null
+        }
+        if (!allowedOrigins.has(finalOrigin)) return null
+        const body = Buffer.from(await res.arrayBuffer())
+        if (!sniffOasContent(body.toString('utf8'))) return null
+        return { url, text: null, source: 'probe', body }
+      } catch {
+        return null
+      }
+    }),
+  )
+  return hits.filter(Boolean)
+}
+
+/**
+ * Follow redirects on the source URL itself and return the final URL when the
+ * site has moved (different origin or path), or null to keep the original.
+ * Trailing-slash and query/hash-only redirects don't count as a move.
+ */
+async function resolveRedirectedSourceUrl(sourceUrl) {
   try {
-    const res = await fetch(llmsUrl, {
-      redirect: 'follow',
-      headers: { 'User-Agent': 'readme-cli-import' },
-    })
-    if (!res.ok) return { ok: false, status: res.status }
-    const text = await res.text()
-    return { ok: true, status: res.status, parsed: parseLlmsTxt(text) }
-  } catch (e) {
-    return { ok: false, error: e.message }
+    const res = await fetch(sourceUrl.href, { redirect: 'follow', headers: { 'User-Agent': 'readme-cli-import' } })
+    if (!res.ok || !res.url) return null
+    const final = new URL(res.url)
+    final.search = ''
+    final.hash = ''
+    const norm = (u) => u.origin + u.pathname.replace(/\/+$/, '')
+    if (norm(final) === norm(sourceUrl)) return null
+    return final
+  } catch {
+    return null
   }
 }
 
 /**
- * Parse the llms.txt format. `##` headings become sections;
- * `- [text](url): description` bullets become items. Items before any `##`
- * land in an implicit "Resources" section.
+ * For a bare homepage, probe well-known docs routes and return the best base to
+ * adopt, or null to leave the source untouched. A route serving a usable
+ * llms.txt always beats one that merely resolves. Returns `{ url, kind, hasLlms }`.
  */
-function parseLlmsTxt(body) {
-  const lines = body.split(/\r?\n/)
-  let title = null
-  const sections = []
-  let current = null
-
-  const itemRe = /^\s*-\s*\[([^\]]*)\]\((https?:\/\/[^)\s]+)\)(?:\s*[:—–-]\s*(.+))?/
+async function resolveDocsBaseUrl(sourceUrl) {
+  if (sourceUrl.pathname && sourceUrl.pathname !== '/') return null
+  if (WELL_KNOWN_DOC_ROUTES.includes(sourceUrl.hostname.split('.')[0].toLowerCase())) return null
+
+  const candidates = buildWellKnownDocRoutes(sourceUrl)
+  if (candidates.length === 0) return null
+
+  styles.info(`Probing ${styles.bold(String(candidates.length))} well-known docs routes from ${styles.bold(sourceUrl.toString())}...`)
+
+  const llmsHits = await Promise.all(candidates.map(async (c) => ({ c, res: await fetchLlmsTxt(`${c.url.href}llms.txt`) })))
+  for (const { c, res } of llmsHits) {
+    const url = `${c.url.href}llms.txt`
+    if (res.ok && res.usable && (res.stats?.linkItems ?? 0) > 0) {
+      if (inCandidateScope(c, res.finalUrl)) {
+        styles.info(styles.dim(`  ${url} → valid llms.txt (${res.stats.linkItems} links)`))
+        return { url: c.url, kind: c.kind, hasLlms: true }
+      }
+      styles.info(styles.dim(`  ${url} → llms.txt redirects to ${new URL(res.finalUrl).host} (alias) — skipping`))
+    } else if (res.ok) {
+      styles.info(styles.dim(`  ${url} → no usable llms.txt`))
+    } else {
+      styles.info(styles.dim(`  ${url} → ${res.status ? `HTTP ${res.status}` : res.error || 'no llms.txt'}`))
+    }
+  }
 
-  for (const line of lines) {
-    const h1 = line.match(/^#\s+(.+)$/)
-    if (h1 && !title) {
-      title = h1[1].trim()
+  const soft404Title = candidates.some((c) => c.kind === 'path') ? await fetchSoft404Title(sourceUrl.origin) : null
+  const existence = await Promise.all(
+    llmsHits.map(async ({ c, res }) => {
+      if (res.error && c.kind === 'subdomain') return { c, exists: false }
+      return { c, ...(await docsRouteResolves(c, soft404Title)) }
+    }),
+  )
+  for (const { c, exists, soft404 } of existence) {
+    if (soft404) {
+      styles.info(styles.dim(`  ${c.url.href} → indistinguishable from a nonexistent path (SPA catch-all) — skipping`))
       continue
     }
+    if (exists) {
+      styles.info(styles.dim(`  ${c.url.href} → resolves (no llms.txt)`))
+      return { url: c.url, kind: c.kind, hasLlms: false }
+    }
+  }
 
-    const h2 = line.match(/^##\s+(.+)$/)
-    if (h2) {
-      current = { title: h2[1].trim(), items: [] }
-      sections.push(current)
-      continue
+  styles.info(styles.dim(`  no well-known docs route matched — keeping ${sourceUrl.toString()}`))
+  return null
+}
+
+/**
+ * Is `finalUrl` (where a fetch landed after following redirects) still within
+ * the candidate's scope? Subdomain candidates must keep their host; path
+ * candidates must keep their path prefix. A cross-scope redirect means the
+ * route is just an alias to somewhere else (usually the marketing apex).
+ */
+function inCandidateScope(candidate, finalUrl) {
+  let final
+  try {
+    final = new URL(finalUrl || candidate.url.href)
+  } catch {
+    return false
+  }
+  if (candidate.kind === 'subdomain') return final.hostname === candidate.url.hostname
+  const strip = (p) => p.toLowerCase().replace(/\/+$/, '')
+  const finalPath = strip(final.pathname)
+  const candidatePath = strip(candidate.url.pathname)
+  return finalPath === candidatePath || finalPath.startsWith(`${candidatePath}/`)
+}
+
+function extractHtmlTitle(html) {
+  const match = /<title[^>]*>([^<]*)<\/title>/i.exec(html)
+  if (!match) return null
+  const title = match[1].replace(/\s+/g, ' ').trim().toLowerCase()
+  return title || null
+}
+
+/**
+ * Fetch a guaranteed-nonexistent path and capture its page title. On SPA
+ * catch-all sites every unknown path returns 200 with the same shell page;
+ * that title becomes the soft-404 baseline. Returns null when the site 404s
+ * properly (no baseline needed) or on any fetch failure.
+ */
+async function fetchSoft404Title(origin) {
+  const probeUrl = `${origin}/readme-cli-nonexistent-${Math.random().toString(36).slice(2)}`
+  try {
+    const res = await fetch(probeUrl, { redirect: 'follow', headers: { 'User-Agent': 'readme-cli-import' } })
+    if (!res.ok) return null
+    return extractHtmlTitle(await res.text())
+  } catch {
+    return null
+  }
+}
+
+/**
+ * Does a well-known docs route exist and stay in scope? Follows redirects and
+ * rejects anything that lands outside the candidate (a `docs.` subdomain that
+ * bounces to marketing, a `/docs/` that 302s home). When a soft-404 baseline
+ * title is known, a path candidate serving that same title is a catch-all
+ * response, not real docs. DNS failures → `{ exists: false }`.
+ */
+async function docsRouteResolves(candidate, soft404Title) {
+  try {
+    const res = await fetch(candidate.url.href, { redirect: 'follow', headers: { 'User-Agent': 'readme-cli-import' } })
+    if (!res.ok || !inCandidateScope(candidate, res.url)) return { exists: false }
+    if (candidate.kind === 'path' && soft404Title) {
+      const title = extractHtmlTitle(await res.text())
+      if (title === soft404Title) return { exists: false, soft404: true }
+    }
+    return { exists: true }
+  } catch {
+    return { exists: false }
+  }
+}
+
+// Hard cap on total probes during BFS discovery — protects against
+// pathological sites where every URL returns a "200 + a few link rows"
+// response and the frontier would otherwise blow up.
+const LLMS_PROBE_CAP = 30
+// Per-node cap on child segments to expand. Root-level llms.txt files can
+// list hundreds of URLs; without this a single root hit could schedule
+// dozens of unrelated probes (`/blog`, `/pricing`, …).
+const LLMS_CHILDREN_PER_NODE = 12
+// Budget for the explicit-link phase of discovery (phase 2 in discoverLlmsTxt).
+// Separate from LLMS_PROBE_CAP so following authored `llms.txt` links — a
+// strong, deliberate signal, e.g. docs.snowflake.com's root index listing ~40
+// per-section files — isn't starved by the speculative slug-walk's budget.
+const EXPLICIT_LLMS_PROBE_CAP = 200
+
+// Matches ONLY the canonical index filename, not sibling dumps like
+// `llms-full.txt` (concatenated page content, not a link index) — those would
+// add no link rows and just waste a fetch.
+const LLMS_INDEX_RE = /\/llms\.txt$/i
+
+/**
+ * Convert a path like `/docs/quickstart` into its llms.txt URL.
+ */
+function pathToLlmsUrl(origin, path) {
+  if (!path || path === '/') return `${origin}/llms.txt`
+  return `${origin}${path}/llms.txt`
+}
+
+function llmsParseOptionsForUrl(llmsUrl) {
+  return { rootRelativeResolution: llmsPathFromUrl(llmsUrl) === '/' ? 'origin' : 'llms-dir' }
+}
+
+/**
+ * Derive the path a fetched llms.txt "lives at" (its pathname minus the
+ * trailing `/llms.txt`), so explicit-link hits carry the same `path` shape
+ * the slug-walk hits do — mergeValidHits sorts by it (deepest wins) and
+ * narrowToDocsSubtreeIfNeeded reads it.
+ */
+function llmsPathFromUrl(llmsUrl) {
+  const pn = new URL(llmsUrl).pathname.replace(/\/llms\.txt$/i, '')
+  return pn || '/'
+}
+
+/**
+ * Pull explicit child-index links out of a parsed llms.txt — item rows whose
+ * URL literally points at another `llms.txt` (e.g. Snowflake's root file
+ * listing `…/data-integration/llms.txt`). Restricted to the same origin as the
+ * source: cross-origin llms.txt would drag in unrelated third-party docs, and
+ * the slug-walk's path math assumes one origin. Returns deduped URL strings.
+ */
+function extractNestedLlmsUrls(parsed, originScope) {
+  const out = new Set()
+  for (const section of parsed.sections) {
+    for (const item of section.items) {
+      try {
+        const u = new URL(item.url)
+        if (u.origin === originScope && LLMS_INDEX_RE.test(u.pathname)) out.add(u.toString())
+      } catch {
+        // ignore unparseable URLs
+      }
     }
+  }
+  return [...out]
+}
+
+/**
+ * Given a current path + a parsed llms.txt, return the next paths to probe:
+ * the parent directory (UP) and distinct first-segment-children extracted
+ * from the URL list (DOWN). Children are scoped to URLs whose pathname
+ * actually starts with the current path's prefix — out-of-scope URLs (e.g.
+ * a `/docs/llms.txt` listing `/blog/foo`) are ignored.
+ */
+function expandLlmsFrontier(currentPath, parsed) {
+  const next = []
+  const prefix = currentPath === '/' || currentPath === '' ? '/' : currentPath + '/'
 
-    const item = line.match(itemRe)
-    if (item) {
-      if (!current) {
-        current = { title: 'Resources', items: [] }
-        sections.push(current)
+  if (currentPath && currentPath !== '/') {
+    const parent = currentPath.replace(/\/[^/]+$/, '') || '/'
+    next.push(parent)
+  }
+
+  const childSegs = new Set()
+  for (const section of parsed.sections) {
+    for (const item of section.items) {
+      try {
+        const u = new URL(item.url)
+        const pn = u.pathname.toLowerCase()
+        if (!pn.startsWith(prefix.toLowerCase())) continue
+        const rest = pn.slice(prefix.length).split('/').filter(Boolean)
+        if (rest.length > 0) childSegs.add(rest[0])
+      } catch {
+        // ignore unparseable URLs
       }
-      current.items.push({
-        text: item[1].trim(),
-        url: item[2].replace(/[.,;]+$/, ''),
-        description: item[3] ? item[3].trim() : null,
-      })
+      if (childSegs.size >= LLMS_CHILDREN_PER_NODE) break
+    }
+    if (childSegs.size >= LLMS_CHILDREN_PER_NODE) break
+  }
+  for (const seg of childSegs) {
+    next.push(currentPath === '/' || !currentPath ? `/${seg}` : `${currentPath}/${seg}`)
+  }
+  return next
+}
+
+/**
+ * BFS-discover all llms.txt files reachable from sourceUrl by walking UP
+ * the path and DOWN into child segments. Invalid hits (file exists but
+ * fails the usability ratio) are kept as "stepping stones" — they still
+ * expand the frontier so we can find their valid siblings/children — but
+ * don't contribute content to the merged result. Returns:
+ *   { hits, misses, skipped }
+ * where `hits` includes both valid (usable: true) and invalid hits.
+ */
+async function discoverLlmsTxt(sourceUrl) {
+  const tried = new Set()
+  const hits = []
+  const misses = []
+  const skipped = []
+
+  const startSegs = sourceUrl.pathname.split('/').filter(Boolean)
+  const initial = []
+  for (let i = startSegs.length; i >= 0; i--) {
+    initial.push('/' + startSegs.slice(0, i).join('/'))
+  }
+
+  let frontier = initial
+  while (frontier.length > 0 && tried.size < LLMS_PROBE_CAP) {
+    const ring = []
+    for (const p of frontier) {
+      const norm = p === '' ? '/' : p
+      if (tried.has(norm)) continue
+      tried.add(norm)
+      ring.push(norm)
+      if (tried.size >= LLMS_PROBE_CAP) break
+    }
+    if (ring.length === 0) break
+
+    const results = await Promise.all(
+      ring.map(async (path) => {
+        const llmsUrl = pathToLlmsUrl(sourceUrl.origin, path)
+        const res = await fetchLlmsTxt(llmsUrl, llmsParseOptionsForUrl(llmsUrl))
+        return { path, llmsUrl, res }
+      }),
+    )
+
+    const next = []
+    for (const { path, llmsUrl, res } of results) {
+      if (!res.ok) {
+        misses.push({ url: llmsUrl, status: res.status, error: res.error })
+        continue
+      }
+      // Treat as a "hit" only if there's at least one link row — that's the
+      // minimum signal that this is an llms.txt-shaped file (and not just a
+      // 200 response serving the site's HTML or a stray text file). Hits
+      // expand the frontier regardless of whether they pass usability.
+      const linkItems = res.stats?.linkItems ?? 0
+      if (linkItems === 0) {
+        misses.push({ url: llmsUrl, status: 'no link items' })
+        continue
+      }
+      hits.push({ ...res, llmsUrl, path })
+      if (!res.usable) skipped.push({ url: llmsUrl, reason: res.reason })
+      for (const np of expandLlmsFrontier(path, res.parsed)) {
+        if (!tried.has(np)) next.push(np)
+      }
+    }
+    frontier = next
+  }
+
+  // Phase 2 — follow EXPLICIT llms.txt links authored inside the files phase 1
+  // discovered, recursively. No slug-guessing here: we only fetch URLs that
+  // literally point at an llms.txt (a root index listing per-section files,
+  // like docs.snowflake.com). Each new file's own explicit links are followed
+  // in turn until none remain. Dedupes against every URL phase 1 touched
+  // (hits + misses) and against itself; phase 1's `tried` set is keyed by path,
+  // so we track full llms.txt URLs separately here and leave phase 1 untouched.
+  const triedUrls = new Set([...hits.map((h) => h.llmsUrl), ...misses.map((m) => m.url)])
+  let explicitFrontier = []
+  for (const hit of hits) {
+    for (const u of extractNestedLlmsUrls(hit.parsed, sourceUrl.origin)) {
+      if (!triedUrls.has(u)) explicitFrontier.push(u)
+    }
+  }
+
+  let explicitProbes = 0
+  while (explicitFrontier.length > 0 && explicitProbes < EXPLICIT_LLMS_PROBE_CAP) {
+    const ring = []
+    for (const u of explicitFrontier) {
+      if (triedUrls.has(u)) continue
+      triedUrls.add(u)
+      ring.push(u)
+      explicitProbes++
+      if (explicitProbes >= EXPLICIT_LLMS_PROBE_CAP) break
+    }
+    if (ring.length === 0) break
+
+    const results = await Promise.all(ring.map(async (llmsUrl) => ({ llmsUrl, res: await fetchLlmsTxt(llmsUrl, llmsParseOptionsForUrl(llmsUrl)) })))
+
+    const next = []
+    for (const { llmsUrl, res } of results) {
+      if (!res.ok) {
+        misses.push({ url: llmsUrl, status: res.status, error: res.error })
+        continue
+      }
+      const linkItems = res.stats?.linkItems ?? 0
+      if (linkItems === 0) {
+        misses.push({ url: llmsUrl, status: 'no link items' })
+        continue
+      }
+      hits.push({ ...res, llmsUrl, path: llmsPathFromUrl(llmsUrl) })
+      if (!res.usable) skipped.push({ url: llmsUrl, reason: res.reason })
+      for (const nested of extractNestedLlmsUrls(res.parsed, sourceUrl.origin)) {
+        if (!triedUrls.has(nested)) next.push(nested)
+      }
+    }
+    explicitFrontier = next
+  }
+
+  return { hits, misses, skipped }
+}
+
+/**
+ * Merge content from all VALID hits into one llms-result-shaped object that
+ * downstream code can consume the same as a single-file result. Pages are
+ * deduped by normalized pathname, with the deepest-path hit winning — the
+ * `/docs/conversational-ai/` file's section assignment beats `/docs/`'s for
+ * any URL under that subtree. Returns null when there are no valid hits.
+ */
+function mergeValidHits(hits) {
+  const valid = hits.filter((h) => h.usable)
+  if (valid.length === 0) return null
+
+  const deepestFirst = [...valid].sort((a, b) => b.path.length - a.path.length)
+  const shallowestFirst = [...valid].sort((a, b) => a.path.length - b.path.length)
+
+  const claimed = new Map()
+  const sections = []
+  for (const hit of deepestFirst) {
+    for (const section of hit.parsed.sections) {
+      const items = []
+      for (const item of section.items) {
+        const key = llmsPageDedupeKey(item.url)
+        const existing = claimed.get(key)
+        if (existing) {
+          if (compareCanonicalUrlPreference(item.url, existing.url) < 0) existing.url = item.url
+          continue
+        }
+        claimed.set(key, item)
+        items.push(item)
+      }
+      if (items.length > 0) sections.push({ title: section.title, items })
     }
   }
 
-  return { title, sections }
+  // Prefer the root-most file's H1 title — usually the canonical site name.
+  const title = shallowestFirst.find((h) => h.parsed.title)?.parsed.title || null
+
+  const stats = valid.reduce(
+    (acc, h) => ({
+      conforming: acc.conforming + (h.stats?.conforming || 0),
+      nonConforming: acc.nonConforming + (h.stats?.nonConforming || 0),
+      total: acc.total + (h.stats?.total || 0),
+      linkItems: acc.linkItems + (h.stats?.linkItems || 0),
+    }),
+    { conforming: 0, nonConforming: 0, total: 0, linkItems: 0 },
+  )
+  stats.ratio = stats.total === 0 ? 0 : stats.conforming / stats.total
+
+  return {
+    parsed: { title, sections },
+    usable: true,
+    reason: null,
+    stats,
+    llmsUrl: shallowestFirst[0].llmsUrl,
+    sourceFiles: shallowestFirst.map((h) => h.llmsUrl),
+  }
+}
+
+/**
+ * Best-effort fetch of a site's /llms.txt plus a simple structural usability check.
+ */
+async function fetchLlmsTxt(llmsUrl, options = {}) {
+  try {
+    const res = await fetch(llmsUrl, {
+      redirect: 'follow',
+      headers: { 'User-Agent': 'readme-cli-import' },
+    })
+    if (!res.ok) return { ok: false, status: res.status }
+    const text = await res.text()
+    const finalUrl = res.url || String(llmsUrl)
+    const analysis = analyzeLlmsTxt(text, finalUrl, finalUrl === String(llmsUrl) ? options : llmsParseOptionsForUrl(finalUrl))
+    return {
+      ok: true,
+      status: res.status,
+      finalUrl: res.url,
+      parsed: analysis.parsed,
+      usable: analysis.usable,
+      reason: analysis.reason,
+      stats: analysis.stats,
+    }
+  } catch (e) {
+    return { ok: false, error: e.message }
+  }
 }
 
 // Path segments that mark a docs scope. Used to find the right "base" for
@@ -2526,9 +4642,10 @@ function urlInScope(urlStr, sitemapUrl, scopePrefix) {
  * Convert a flat sitemap URL list into the `{title, url, description}` shape
  * the rest of the pipeline expects for knownUrls. Title is derived from the
  * last non-empty path segment, with file extensions stripped. Index-style
- * URLs that resolve to the scope root itself are dropped.
+ * URLs that resolve to the scope root itself are dropped. Spec-shaped URLs
+ * (.json/.yaml/.yml) go to `oasSink` instead of becoming pages.
  */
-function sitemapUrlsToKnownUrls(urls) {
+function sitemapUrlsToKnownUrls(urls, oasSink) {
   const out = []
   const seen = new Set()
   for (const url of urls) {
@@ -2541,6 +4658,10 @@ function sitemapUrlsToKnownUrls(urls) {
     const norm = normalizePath(url)
     if (seen.has(norm)) continue
     seen.add(norm)
+    if (/\.(json|ya?ml)$/i.test(pathname)) {
+      if (oasSink) oasSink.push({ url, text: null, source: 'sitemap' })
+      continue
+    }
     const segs = pathname.split('/').filter(Boolean)
     const last = segs[segs.length - 1] || ''
     const slug = last.replace(/\.[a-z0-9]+$/i, '')
@@ -2554,13 +4675,6 @@ function sitemapUrlsToKnownUrls(urls) {
   return out
 }
 
-/**
- * Write the organized hierarchy to disk as git-format markdown stubs — just
- * frontmatter, no body yet. docs/ pages go under docs/<Category>/<slug>.md;
- * reference/recipes/custom_pages/custom_blocks get their own top-level dir
- * without a category subfolder (the git-format schema doesn't nest them).
- * Writes _order.yaml per directory so sidebar order matches input order.
- */
 /**
  * Recursively print the page tree. Sub-pages are indented under their parent
  * with no leading bullet character, to show them as children of the parent.
@@ -2577,34 +4691,112 @@ function printPagesTree(pages, indentLevel) {
   }
 }
 
+/**
+ * Collapse redundant single-folder layers in a category/page subtree, in place.
+ *
+ * When a container (category or page) has exactly one child and that child is a
+ * pure folder node — no `x-import-url` gitto/readme has an issue with unique slug
+ * generation.
+ * This is a temporary workaround.
+ * TODO: Remove after fixing on the Gitto/ReadMe side.
+ */
+function collapseRedundantLayers(container) {
+  for (const child of container.pages || []) collapseRedundantLayers(child)
+  while ((container.pages || []).length === 1) {
+    const only = container.pages[0]
+    if (only.url) break // real page (gets x-import), just has children
+    if (!only.pages || only.pages.length === 0) break // nothing to lift up
+    container.pages = only.pages
+  }
+}
+
+// Some sites serve regular guides under an /api/docs/* path (e.g. /api/docs/guides/*,
+// /api/docs/pricing). Treat a URL as real docs when it has both `api` and `docs` segments 
+// and the segment right after `docs` is not itself a reference/api slug.
+function isDocsUnderApiPath(url) {
+  try {
+    const segs = new URL(url).pathname.split('/').filter(Boolean).map((s) => s.toLowerCase())
+    const docsIdx = segs.findIndex((s) => s === 'docs' || s === 'doc')
+    if (!segs.includes('api') || docsIdx === -1) return false
+    const afterDocs = segs[docsIdx + 1]
+    return !afterDocs || !/^(api|api[-_]?reference|reference)$/.test(afterDocs)
+  } catch {
+    return false
+  }
+}
+
+// Prune a page tree, keeping only URL pages whose url passes `predicate` plus
+// empty-parent nodes that still have surviving descendants. Nesting preserved.
+function filterUrlPagesTree(pages, predicate) {
+  const out = []
+  for (const p of pages || []) {
+    const keptKids = filterUrlPagesTree(p.pages, predicate)
+    if (p.url) {
+      if (predicate(p.url)) out.push({ ...p, pages: keptKids })
+    } else if (keptKids.length > 0) {
+      out.push({ ...p, pages: keptKids })
+    }
+  }
+  return out
+}
+
+/**
+ * Write the organized hierarchy to disk as git-format markdown stubs — just
+ * frontmatter, no body yet. docs/ pages go under docs/<Category>/<slug>.md;
+ * reference/recipes/custom_pages/custom_blocks get their own top-level dir
+ * without a category subfolder (the git-format schema doesn't nest them).
+ * Writes _order.yaml per directory so sidebar order matches input order.
+ */
 function stageOrganized(organized, stagingDir, opts = {}) {
   const pickIcon = makeIconPicker()
   const byDir = new Map()
   const subDirsByTopDir = new Map()
   const counts = { fileCount: 0, skippedApiRef: 0 }
   const skipApiReference = !!opts.skipApiReference
+  const llmsPaths = opts.llmsPaths || new Set()
 
-  const eligibleCategories = (organized.categories || []).filter((cat) => {
+  const eligibleCategories = []
+  for (const cat of organized.categories || []) {
     if (skipApiReference && routeCategory(cat.title).topDir === 'reference') {
-      counts.skippedApiRef += countPagesDeep(cat.pages || [])
-      return false
+      // A reference-routed cluster can mix genuine OAS pages (/api/reference/*)
+      // with real guides served under /api/docs/*. Rescue the docs subtree —
+      // retitled so it routes to docs/ instead of the dropped reference/
+      const rescuedPages = filterUrlPagesTree(cat.pages || [], isDocsUnderApiPath)
+      const totalDeep = collectUrlPagesDeep(cat.pages || []).length
+      const rescuedDeep = collectUrlPagesDeep(rescuedPages).length
+      if (rescuedDeep > 0) {
+        eligibleCategories.push({ ...cat, title: 'API Docs', pages: rescuedPages })
+      }
+      counts.skippedApiRef += totalDeep - rescuedDeep
+      continue
     }
-    return true
-  })
+    eligibleCategories.push(cat)
+  }
+
+  // TODO: Remove after readme/gitto handles better
+  for (const cat of eligibleCategories) collapseRedundantLayers(cat)
 
   // Slug names must be unique
   const slugFor = ensureUniqueSlugs(eligibleCategories)
 
-  const urlWidth = Math.max(...[...slugFor.keys()].map((p) => (p.url || '(group-only)').length))
+  const labelFor = (p) => {
+    if (p.url) return p.url
+    if (p._emptyParent && p._virtualPathSegs) return `(empty) /${p._virtualPathSegs.join('/')}`
+    return '(group-only)'
+  }
+  const urlWidth = Math.max(...[...slugFor.keys()].map((p) => labelFor(p).length))
   for (const [page, slug] of slugFor) {
-    console.log(`${(page.url || '(group-only)').padEnd(urlWidth)} → ${slug}`)
+    console.log(`${labelFor(page).padEnd(urlWidth)} → ${slug}`)
   }
 
   /**
    * Write a page (and its descendants) into `dir`. A page with children gets
-   * its own subfolder named after its slug; the parent page lives at
-   * `<dir>/<slug>.md` while children live at `<dir>/<slug>/<childSlug>.md`.
-   * This matches git-format's on-disk convention for nested sidebars.
+   * its own subfolder named after its slug; its content lives at
+   * `<dir>/<slug>/index.md` while children live at `<dir>/<slug>/<childSlug>.md`.
+   * A leaf page (no children) lives at `<dir>/<slug>.md`. This matches
+   * git-format's on-disk convention: gitto backs a folder's sidebar node with
+   * its `index.md`; a sibling `<slug>.md` next to the folder instead leaves the
+   * folder a contentless placeholder parent and the sibling a stray page.
    */
   function writePage(page, dir, topDir, isSubPage = false) {
     const slug = slugFor.get(page)
@@ -2613,46 +4805,119 @@ function stageOrganized(organized, stagingDir, opts = {}) {
     // no backing page on the source site — they're pure sidebar containers.
     // Skip the stub write but still recurse so their children land in the
     // right subdirectory.
-    const isGroupOnly = !page.url
+    const isEmptyParent = !!page._emptyParent
+    const isGroupOnly = !page.url && !isEmptyParent
+    const children = page.pages || []
+    const hasChildren = children.length > 0
 
     if (!isGroupOnly) {
-      const relFilePath = `${dir}/${slug}.md`
-      // Sub-pages don't get icons per design decision.
-      const frontmatter = buildFrontmatter(topDir, page, slug, pickIcon, { skipIcon: isSubPage })
-      // x-import points at the source URL for this stub. The content-import
-      // step reads it to fetch the page body. x-prefixed custom field is the
-      // git-format convention for metadata the schema doesn't know about.
-      frontmatter['x-import'] = toBrowsableUrl(page.url)
-      // hide pages that need import
-      frontmatter.hidden = 'true'
+      // Parents-with-children store their content INSIDE the folder as
+      // `index.md`; leaf pages stay as `<slug>.md`.
+      const relFilePath = hasChildren ? `${dir}/${slug}/index.md` : `${dir}/${slug}.md`
+      let frontmatter
+      if (isEmptyParent) {
+        // Synthetic parent page: covers a missing URL segment so descendants
+        // nest at the right depth. No source URL to import. Reuse the same
+        // pickIcon heuristic leaf pages use so the parent gets a topical
+        // icon — the sidebar already renders an expand chevron, so a
+        // hardcoded `fa-folder` would just look like a placeholder.
+        frontmatter = { title: page.title, icon: formatIconClass(pickIcon(slug, page.title)) }
+      } else {
+        // Sub-pages don't get icons per design decision.
+        frontmatter = buildFrontmatter(topDir, page, slug, pickIcon, { skipIcon: isSubPage })
+        // x-import points at the source URL for this stub. The content-import
+        // step reads it to fetch the page body. x-prefixed custom field is the
+        // git-format convention for metadata the schema doesn't know about.
+        frontmatter['x-import'] = toBrowsableUrl(page.url)
+        // Pages sourced from llms.txt serve raw markdown at `<url>.md`; flag them
+        // so the runner knows it can append `.md` when fetching the body.
+        if (llmsPaths.has(normalizePath(page.url))) frontmatter['x-from-llms'] = 'true'
+        // hide pages that need import
+        frontmatter.hidden = 'true'
+      }
 
       const absPath = path.join(stagingDir, relFilePath)
       fs.mkdirSync(path.dirname(absPath), { recursive: true })
       fs.writeFileSync(absPath, matter.stringify('', frontmatter))
       counts.fileCount++
 
+      // The sidebar entry is `slug` for both leaves (`<slug>.md`) and parents
+      // (the `<slug>/` folder) — `index` is never listed, it's implicit.
       if (!byDir.has(dir)) byDir.set(dir, [])
       byDir.get(dir).push(slug)
     }
 
-    const children = page.pages || []
-    if (children.length > 0) {
+    if (hasChildren) {
       const subDir = `${dir}/${slug}`
       for (const child of children) writePage(child, subDir, topDir, true)
     }
   }
 
+  // gitto derives a page's slug from the folder BASENAME alone (getSlugFromPath),
+  // so two category folders that share a basename — e.g. docs/Fundamentals and
+  // docs/cockroach/Fundamentals — both slug to `fundamentals` and collide. The
+  // deeper one renders as an unnamed, content-less placeholder parent.
+  // Namespace every NESTED category folder by its ancestor path so its basename
+  // (and thus slug) is globally unique, and back it with an `index.md` so it's a
+  // real titled parent. Top-level categories are matched by title, not slug, so
+  // their folders are left untouched.
+  const usedContainerSlugs = new Set(slugFor.values())
+  const segRename = new Map() // original path prefix -> unique folder name
+  const nestedContainers = new Map() // renamed dir (rel to stagingDir) -> display title
+  const uniqueContainerName = (base) => {
+    let name = base || 'section'
+    if (usedContainerSlugs.has(name)) {
+      let n = 2
+      while (usedContainerSlugs.has(`${name}-${n}`)) n++
+      name = `${name}-${n}`
+    }
+    usedContainerSlugs.add(name)
+    return name
+  }
+  const namespaceSubDir = (topDir, subDir) => {
+    const segs = subDir.split('/').filter(Boolean)
+    const out = []
+    for (let i = 0; i < segs.length; i++) {
+      if (i === 0) {
+        out.push(segs[0])
+        continue
+      }
+      const origPrefix = segs.slice(0, i + 1).join('/')
+      let name = segRename.get(origPrefix)
+      if (!name) {
+        name = uniqueContainerName(kebabCase(segs.slice(0, i + 1).join('-')))
+        segRename.set(origPrefix, name)
+        nestedContainers.set(`${topDir}/${[...out, name].join('/')}`, segs[i])
+      }
+      out.push(name)
+    }
+    return out.join('/')
+  }
+
   for (const cat of eligibleCategories) {
     const { topDir, subDir } = routeCategory(cat.title)
-    const dir = subDir ? `${topDir}/${subDir}` : topDir
-    if (subDir) {
+    const routedSubDir = subDir ? namespaceSubDir(topDir, subDir) : subDir
+    const dir = routedSubDir ? `${topDir}/${routedSubDir}` : topDir
+    if (routedSubDir) {
       if (!subDirsByTopDir.has(topDir)) subDirsByTopDir.set(topDir, [])
-      if (!subDirsByTopDir.get(topDir).includes(subDir)) subDirsByTopDir.get(topDir).push(subDir)
+      if (!subDirsByTopDir.get(topDir).includes(routedSubDir)) subDirsByTopDir.get(topDir).push(routedSubDir)
     }
 
     for (const page of cat.pages || []) writePage(page, dir, topDir, false)
   }
 
+  // Back each namespaced nested-category folder with a titled parent page so
+  // gitto renders it named instead of as an empty placeholder. No x-import —
+  // these are pure nav containers, so the body stays empty and format skips them.
+  for (const [relDir, title] of nestedContainers) {
+    const idxPath = path.join(stagingDir, relDir, 'index.md')
+    if (fs.existsSync(idxPath)) continue
+    fs.mkdirSync(path.dirname(idxPath), { recursive: true })
+    const fm = { title, icon: formatIconClass(pickIcon(path.basename(relDir), title)) }
+    fs.writeFileSync(idxPath, matter.stringify('', fm))
+    counts.fileCount++
+  }
+
   // Per-directory _order.yaml preserves input order in the sidebar.
   for (const [dir, slugs] of byDir) {
     const orderPath = path.join(stagingDir, dir, '_order.yaml')
@@ -2679,6 +4944,85 @@ function countPagesDeep(pages) {
   return n
 }
 
+/**
+ * Post-staging relocation for `--separate-changelog`: move the staged
+ * `docs/Changelog/` category up to a top-level `changelog/` directory.
+ *
+ * Done as a filesystem move *after* staging (rather than routing there
+ * directly) so the staged tree is a valid git-format layout up to this point,
+ * and the one schema-divergent step lives in a single, clearly-named place.
+ * Note `changelog/` is not part of the git-format schema — it won't lint or
+ * sync to ReadMe until git-format adds first-class changelog support.
+ *
+ * Returns the number of changelog pages moved (0 if there were none).
+ */
+function relocateChangelogDir(stagingDir) {
+  const srcDir = path.join(stagingDir, 'docs', 'Changelog')
+  if (!fs.existsSync(srcDir) || !fs.statSync(srcDir).isDirectory()) return 0
+
+  const dstDir = path.join(stagingDir, 'changelog')
+  fs.renameSync(srcDir, dstDir)
+
+  // Collect every real x-import page from the (possibly nested) tree.
+  // Synthetic group-only stubs (no x-import — they exist only to render an
+  // empty parent in the sidebar) are dropped: the flat layout has no concept
+  // of containers, so they'd just be dead frontmatter.
+  const pages = []
+  const walk = (dir, ancestors) => {
+    for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
+      const full = path.join(dir, entry.name)
+      if (entry.isDirectory()) {
+        walk(full, [...ancestors, entry.name])
+        continue
+      }
+      if (!entry.name.endsWith('.md')) continue
+      const parsed = matter(fs.readFileSync(full, 'utf-8'))
+      if (!parsed.data || !parsed.data['x-import']) continue
+      if (entry.name === 'index.md' && ancestors.length > 0) {
+        pages.push({ ancestors: ancestors.slice(0, -1), slug: ancestors.at(-1), content: parsed.content, data: parsed.data })
+        continue
+      }
+      const slug = entry.name.replace(/\.md$/, '')
+      pages.push({ ancestors, slug, content: parsed.content, data: parsed.data })
+    }
+  }
+  walk(dstDir, [])
+
+  // Wipe the nested tree and rewrite it flat. Filename concatenates the
+  // ancestor folder slugs with the leaf slug so siblings share a prefix —
+  // alphabetical order then naturally matches authored reading order, and
+  // names are unique across the (formerly nested) tree.
+  fs.rmSync(dstDir, { recursive: true, force: true })
+  fs.mkdirSync(dstDir, { recursive: true })
+
+  const flatSlugs = []
+  for (const p of pages) {
+    const flatSlug = [...p.ancestors, p.slug].join('-')
+    const frontmatter = { ...p.data }
+    delete frontmatter.icon
+    fs.writeFileSync(path.join(dstDir, `${flatSlug}.md`), matter.stringify(p.content, frontmatter))
+    flatSlugs.push(flatSlug)
+  }
+
+  flatSlugs.sort()
+  fs.writeFileSync(path.join(dstDir, '_order.yaml'), flatSlugs.map((s) => `- ${yamlSafeSlug(s)}`).join('\n') + '\n')
+
+  // docs/_order.yaml lists the docs category subfolders — drop the moved one
+  // so the docs sidebar doesn't point at a folder that's no longer there.
+  const docsOrderPath = path.join(stagingDir, 'docs', '_order.yaml')
+  if (fs.existsSync(docsOrderPath)) {
+    const entries = yamlRequire().load(fs.readFileSync(docsOrderPath, 'utf-8'))
+    const filtered = Array.isArray(entries) ? entries.filter((e) => e !== 'Changelog') : []
+    if (filtered.length > 0) {
+      fs.writeFileSync(docsOrderPath, filtered.map((s) => `- ${yamlSafeSlug(s)}`).join('\n') + '\n')
+    } else {
+      fs.rmSync(docsOrderPath)
+    }
+  }
+
+  return pages.length
+}
+
 /**
  * Map a category title to the git-format top-level directory + optional
  * category subdir. docs/ is the only top dir that takes a subfolder.
@@ -2686,6 +5030,9 @@ function countPagesDeep(pages) {
 function routeCategory(title) {
   const t = (title || '').trim()
   if (/^(api[ -]?reference|reference|api|endpoints?)$/i.test(t)) return { topDir: 'reference', subDir: null }
+  // Changelog-ish categories normalize to one docs/Changelog/ folder so the
+  // --separate-changelog relocation has a single, predictable source dir.
+  if (/^(changelog|change[ -]?log|release[ -]?notes?|releases?|what'?s[ -]?new)$/i.test(t)) return { topDir: 'docs', subDir: 'Changelog' }
   if (/^(recipes?|cookbook|tutorials?|how[ -]?tos?)$/i.test(t)) return { topDir: 'recipes', subDir: null }
   if (/^(custom[ -]?pages?|landing( page)?s?)$/i.test(t)) return { topDir: 'custom_pages', subDir: null }
   if (/^(custom[ -]?blocks?|snippets?|reusable( content)?)$/i.test(t)) return { topDir: 'custom_blocks', subDir: null }
@@ -2722,30 +5069,19 @@ function buildFrontmatter(topDir, page, slug, pickIcon, opts = {}) {
 }
 
 /**
- * Extract URL path segments for slug planning. Strips file extensions and
- * leading numeric prefixes (e.g. `01-intro` → `intro`) the same way the
- * legacy deriveSlug did, so the depth-1 result is byte-for-byte compatible.
- */
-function extractUrlPathSegments(url) {
-  if (!url) return []
-  try {
-    return new URL(url).pathname
-      .split('/')
-      .filter(Boolean)
-      .map((s) => s.replace(/\.(md|mdx|html?)$/i, '').replace(/^\d+[-_.]/, ''))
-      .filter(Boolean)
-  } catch {
-    return []
-  }
-}
+// extractUrlPathSegments is imported from ../utils/url-segs.js
 
 /**
  * Compute a globally-unique slug for every page in the tree.
  *
  * Base slug is the last URL segment (kebab-cased). When two or more pages
  * share a base, we prepend the next-up URL segment to every member of the
- * colliding group and recheck
- * 
+ * colliding group and recheck. The category title is treated as the
+ * outermost segment, so when URL segments alone aren't enough (e.g. a
+ * group-only `/v1` parent that appears in multiple categories), expansion
+ * can still reach for the category to disambiguate before falling back to
+ * a numeric `-N` suffix.
+ *
  * @example
  * ensureUniqueSlugs([
  *   { title: 'Getting Started', pages: [
@@ -2762,15 +5098,47 @@ function extractUrlPathSegments(url) {
  * //   <Overview (Guides)>          => 'guides-overview',            // expanded (collides)
  * // }
  */
-function ensureUniqueSlugs(categories) {
+export function ensureUniqueSlugs(categories) {
   const entries = []
-  const walk = (pages) => {
+  const segmentsFor = (p) => {
+    if (p._virtualPathSegs && p._virtualPathSegs.length > 0) {
+      return p._virtualPathSegs
+        .map((s) => stripSegmentExtensions(s).replace(/^\d+[-_.]/, ''))
+        .filter(Boolean)
+    }
+    return extractUrlPathSegments(p.url)
+  }
+  const normalizedSegmentsFor = (p) => {
+    const semanticSegments = segmentsFor(p).filter((seg) => !isGenericDocsSegment(seg))
+    return semanticSegments.length > 0 ? semanticSegments : ['introduction']
+  }
+  const expansionSegmentsFor = (p, categorySeg) => {
+    const pageSegments = normalizedSegmentsFor(p)
+    if (!categorySeg || isGenericDocsSegment(categorySeg)) return pageSegments
+    return [categorySeg, ...pageSegments]
+  }
+  const walk = (pages, categorySeg) => {
     for (const p of pages || []) {
-      entries.push({ page: p, segments: extractUrlPathSegments(p.url), fallback: p.title, depth: 1 })
-      if (p.pages) walk(p.pages)
+      const segments = expansionSegmentsFor(p, categorySeg)
+      // Synthetic/empty-parent nodes have no content. Never let one win a bare
+      // single-segment slug (e.g. a placeholder `…/vercel-flags/cli` claiming
+      // `cli`), or it squats the slug a real overview page wants and ReadMe
+      // redirects the content-less parent to its first child. Floor their slug
+      // at 2 trailing segments so they namespace to `vercel-flags-cli` instead.
+      entries.push({ page: p, segments, fallback: p.title, depth: p._emptyParent ? 2 : 1 })
+      if (p.pages) walk(p.pages, categorySeg)
     }
   }
-  for (const c of categories || []) walk(c.pages)
+  const genericCategoriesFirst = [...(categories || [])].sort((a, b) => {
+    const aGeneric = isGenericDocsSegment(a?.title || '')
+    const bGeneric = isGenericDocsSegment(b?.title || '')
+    return Number(bGeneric) - Number(aGeneric)
+  })
+  for (const c of genericCategoriesFirst) {
+    const rawTitle = (c?.title || '').trim()
+    const categorySeg = rawTitle ? kebabCase(rawTitle) : ''
+    walk(c.pages, categorySeg)
+  }
 
   const slugFor = (e) => {
     if (e.segments.length === 0) return kebabCase(e.fallback || 'page') || 'page'
@@ -2813,7 +5181,8 @@ function ensureUniqueSlugs(categories) {
       slug = `${base}-${n}`
       styles.error(
         `Slug collision after segment expansion: ${base} — falling back to ${slug} for ${describe(e)}. ` +
-          `Pages were deduped by path before organize, so this indicates the organize step produced duplicates.`,
+        `Pages were deduped by path before organize and category title is already part of the slug, ` +
+        `so this indicates the organize step produced duplicates within a single category.`,
       )
     }
     used.add(slug)
@@ -2822,6 +5191,11 @@ function ensureUniqueSlugs(categories) {
   return result
 }
 
+const GENERIC_DOCS_SEGMENTS = new Set(['doc', 'docs', 'documentation', 'documentations'])
+function isGenericDocsSegment(segment) {
+  return GENERIC_DOCS_SEGMENTS.has(kebabCase(segment))
+}
+
 // Values YAML interprets as non-strings need quoting when used as _order entries.
 const YAML_UNSAFE = /^(?:\d+\.?\d*|true|false|yes|no|on|off|null|~)$/i
 function yamlSafeSlug(slug) {
@@ -2930,6 +5304,37 @@ function makeIconPicker() {
   }
 }
 
+export const __test__ = {
+  discoverLlmsTxt,
+  mergeValidHits,
+  resolveRedirectedSourceUrl,
+  resolveDocsBaseUrl,
+  inCandidateScope,
+  tryFernNav,
+  sniffOasContent,
+  buildWellKnownOasProbeUrls,
+  probeWellKnownOasPaths,
+  extractOasUrlsFromHtml,
+  extractMintlifyOasCandidates,
+  sitemapUrlsToKnownUrls,
+  extractOasSpecUrlsFromParsed,
+  downloadOasSpecs,
+  injectSectionLandingPages,
+  parseMetaRefreshTarget,
+  parseHtmlCanonical,
+  isDiscoverableLink,
+  isAuthChromeUrl,
+  isDocsUnderApiPath,
+  filterUrlPagesTree,
+  stageOrganized,
+  urlIsApiReference,
+  urlIsChangelog,
+  reclassifyReferencePages,
+  extractChangelogFromSections,
+  decodeEntities,
+  stripTags,
+}
+
 function formatDuration(ms) {
   const safe = Math.max(0, ms)
   // Show ms under a second so sub-second work doesn't misleadingly read as "0m 0s".