@readme/cli 0.0.26 → 0.0.29

package/src/commands/import.js

@@ -7,9 +7,20 @@ import { Option } from 'commander'
 import { query } from '@anthropic-ai/claude-agent-sdk'
 import matter from 'gray-matter'
 import * as styles from '../utils/styles.js'
-import { syncOas, extractOperations } from './oas-sync.js'
+import { syncOas } from './oas-sync.js'
 import OASNormalize from 'oas-normalize'
-import { slotOrphansPrompt, iconizeNavPrompt, organizeFromSectionsPrompt, organizeFromScratchPrompt, stripCodeFences } from '../prompts/index.js'
+import {
+  slotOrphansPrompt,
+  slotOrphansOutputSchema,
+  iconizeNavPrompt,
+  iconizeNavOutputSchema,
+  organizeFromSectionsPrompt,
+  organizeFromSectionsOutputSchema,
+  organizeFromScratchPrompt,
+  organizeFromScratchOutputSchema,
+} from '../prompts/index.js'
+import { analyzeLlmsTxt } from '../utils/llms.js'
+import { urlTrieSegs, extractUrlPathSegments, normalizePath, stripSegmentExtensions, compareCanonicalUrlPreference, llmsPageDedupeKey } from '../utils/url-segs.js'
 
 export const command = 'import'
 export const order = 7
@@ -17,12 +28,21 @@ export const description = 'Import content from a URL and package it as a ReadMe
 export const hidden = true
 export const skipBootstrap = true
 
+export const DEFAULT_MODEL = 'claude-sonnet-5'
+
 export function args(cmd) {
-  cmd.requiredOption('--source <url-or-file>', 'URL to import from, or path to a local OpenAPI spec (.json/.yaml/.yml)')
+  cmd.requiredOption(
+    '--source <url-or-file...>',
+    'One or more URLs to import from (space-separated or repeated), or a single local OpenAPI spec (.json/.yaml/.yml)',
+  )
   cmd.option('-o, --output <path>', 'Output zip path (defaults to <basename>-readme.zip in cwd)')
-  cmd.option('--model <name>', 'Claude model alias: haiku, sonnet, opus', 'sonnet')
+  cmd.addOption(new Option('-m, --model <name>', 'Claude model alias: haiku, sonnet, opus').choices(['haiku', 'sonnet', 'opus']).default(DEFAULT_MODEL))
   cmd.option('--firecrawl-key <key>', 'Firecrawl API key (or set FIRECRAWL_API_KEY env var) — enables JS-rendered sidebar scraping')
   cmd.option('--skip-api-reference', 'Drop pages routed to the API Reference / reference dir. Use when uploading the OAS spec separately.')
+  cmd.option(
+    '--separate-changelog',
+    'Move changelog / release-notes pages into a top-level changelog/ directory instead of docs/Changelog/. Note: changelog/ is outside the git-format schema, so it will not lint or sync to ReadMe.',
+  )
   // Internal dev-only flag: skip the zip, keep staging, and boot the dev server
   // against it for quick visual previews. Hidden from --help.
   cmd.addOption(new Option('--test').hideHelp())
@@ -32,22 +52,38 @@ export function args(cmd) {
   cmd.addOption(new Option('--debug').hideHelp())
 }
 
+// Docs locations probed when the user hands us a bare homepage. Each is tried
+// as a subdomain swap (docs.example.com) and, unless subdomain-only, a path
+// prefix (example.com/docs/). Order is preference order — earlier wins ties.
+const WELL_KNOWN_DOC_ROUTES = ['docs', 'doc', 'developers', 'developer', 'documentation', 'documentations', 'platform']
+// Routes probed only as a subdomain (platform.example.com). Their path form
+// (example.com/platform/) is usually a marketing/product page, not docs.
+const SUBDOMAIN_ONLY_DOC_ROUTES = new Set(['platform'])
+// Max concurrent HTTP probes when checking whether a section's landing URL
+// (e.g. /docs/cli) really exists before importing it as a real overview page.
+const LANDING_PROBE_CAP = 8
+
 /**
  * Run the importer programmatically. Mirrors the CLI command but throws on
  * fatal errors instead of calling `process.exit`, and returns a result object
  * on success.
  *
  * @param {object} options
- * @param {string} options.source             URL to import from, or path to a local OAS spec.
+ * @param {string | string[]} options.source  One or more URLs to import from, or a single path to a local OAS spec.
+ *                                            Multiple URLs are merged into one zip; same-titled categories merge their pages.
  * @param {string} [options.output]           Output zip path. Defaults to `<basename>-readme.zip` in cwd.
  * @param {string} [options.model]            Claude model alias: 'haiku' | 'sonnet' | 'opus'. Defaults to 'sonnet'.
  * @param {string} [options.firecrawlKey]     Firecrawl API key (falls back to FIRECRAWL_API_KEY env var).
  * @param {boolean} [options.skipApiReference] Drop pages routed to the API Reference dir.
+ * @param {boolean} [options.separateChangelog] Move changelog pages into a top-level `changelog/` dir instead of `docs/Changelog/`.
  * @param {boolean} [options.test]            Skip the zip, keep staging, and boot the dev server.
  * @param {boolean} [options.debug]           Dump intermediate pipeline artifacts to a tmp dir.
  * @returns {Promise<{ source: 'url' | 'oas', outputZip?: string, stagingDir?: string, fileCount: number, duration: number, phases: Array<{ label: string, ms: number }> }>}
  */
 export async function importDocs(options) {
+  const model = options.model || DEFAULT_MODEL
+  options = { ...options, model }
+
   const startedAt = Date.now()
   const phases = []
   const timePhase = async (label, fn) => {
@@ -59,57 +95,368 @@ export async function importDocs(options) {
 
   const debugSnapshots = options.debug ? {} : null
 
-  // Dispatch: http(s) URL → docs-site scrape flow; anything else → local OAS.
-  if (!/^https?:\/\//i.test(options.source)) {
-    return runOasImport(options.source, options, startedAt, phases, timePhase)
-  }
+  // Normalize source → array; preserve back-compat with single-string callers.
+  const rawSources = Array.isArray(options.source) ? options.source : [options.source]
+  if (rawSources.length === 0) throw new Error('At least one --source is required')
 
-  let sourceUrl
-  try {
-    sourceUrl = new URL(options.source)
-  } catch {
-    throw new Error(`Invalid --source URL: ${options.source}`)
+  // Dispatch: any non-URL entry triggers the local-OAS path, which is
+  // strictly single-source.
+  const hasNonUrl = rawSources.some((s) => !/^https?:\/\//i.test(s))
+  if (hasNonUrl) {
+    if (rawSources.length > 1) {
+      throw new Error('OAS imports accept a single --source; URL imports can accept multiple.')
+    }
+    return runOasImport(rawSources[0], options, startedAt, phases, timePhase)
   }
 
-  const outputZip = path.resolve(options.output || path.join(process.cwd(), `${sourceUrl.hostname}-readme.zip`))
+  const sourceUrls = rawSources.map((s) => {
+    try {
+      return new URL(s)
+    } catch {
+      throw new Error(`Invalid --source URL: ${s}`)
+    }
+  })
+
+  const hostnameJoined = sourceUrls.map((u) => u.hostname).join('-')
+  const outputZip = path.resolve(options.output || path.join(process.cwd(), `${hostnameJoined}-readme.zip`))
 
   console.log()
-  styles.info(`Importing from ${styles.bold(sourceUrl.toString())}`)
+  if (sourceUrls.length === 1) {
+    styles.info(`Importing from ${styles.bold(sourceUrls[0].toString())}`)
+  } else {
+    styles.info(`Importing from ${styles.bold(String(sourceUrls.length))} sources:`)
+    for (const u of sourceUrls) styles.info(styles.dim(`  · ${u.toString()}`))
+  }
   if (!options.test) styles.info(`Output: ${styles.bold(outputZip)}`)
   console.log()
 
-  // Build the list of llms.txt URLs to probe, walking up the supplied path
-  // from most-specific to root. For `https://mintlify.com/docs/quickstart`
-  // we try `/docs/quickstart/llms.txt`, then `/docs/llms.txt`, then root.
-  // This catches sites that scope llms.txt to a docs subpath.
-  const llmsCandidates = buildLlmsCandidates(sourceUrl)
-  styles.info(`Checking for llms.txt (${llmsCandidates.length} candidate${llmsCandidates.length === 1 ? '' : 's'})...`)
+  // Per-source pipeline: discovery + scrape + organize, in parallel.
+  // Each call returns an `organized` object with the standard shape
+  // `{ title, categories: [...] }`; we merge them before staging.
+  const perSourceOrganized = await Promise.all(
+    sourceUrls.map((sourceUrl) => produceOrganizedForSource(sourceUrl, options, timePhase, debugSnapshots)),
+  )
 
-  const { llms, llmsUrl } = await timePhase('fetch llms.txt', async () => {
-    for (const candidate of llmsCandidates) {
-      const res = await fetchLlmsTxt(candidate)
-      if (res.ok) return { llms: res, llmsUrl: candidate }
-      styles.info(styles.dim(`  ${candidate} → ${res.status ? `HTTP ${res.status}` : res.error || 'failed'}`))
+  const organized = sourceUrls.length === 1 ? perSourceOrganized[0] : mergeOrganized(perSourceOrganized)
+
+  // Union the candidate OpenAPI-spec URLs across all sources, deduped by URL
+  // (mergeOrganized intentionally drops non-category fields, so collect from
+  // the per-source results directly).
+  const oasJsonUrls = []
+  const seenOasUrls = new Set()
+  for (const o of perSourceOrganized) {
+    for (const item of o.oasJsonUrls || []) {
+      if (seenOasUrls.has(item.url)) continue
+      seenOasUrls.add(item.url)
+      oasJsonUrls.push(item)
     }
-    return { llms: null, llmsUrl: null }
-  })
+  }
+
+  const llmsPaths = new Set()
+  for (const o of perSourceOrganized) {
+    for (const p of o.llmsPaths || []) llmsPaths.add(p)
+  }
+
+  if (debugSnapshots) {
+    debugSnapshots['05-organized.json'] = organized
+    const debugDir = path.join(os.tmpdir(), `readme-import-debug-${hostnameJoined}-${Date.now()}`)
+    fs.mkdirSync(debugDir, { recursive: true })
+    for (const [name, data] of Object.entries(debugSnapshots)) {
+      fs.writeFileSync(path.join(debugDir, name), JSON.stringify(data, null, 2))
+    }
+    styles.info(`${styles.dim(`Debug snapshots → ${debugDir}`)}`)
+  }
+  console.log()
+
+  console.log(`  ${styles.bold(organized.title || '(untitled)')}`)
+  for (const cat of organized.categories || []) {
+    console.log()
+    const iconLabel = cat.icon ? `${styles.brand(`[${cat.icon}]`)} ` : ''
+    console.log(`  ${iconLabel}${styles.bold(cat.title)}`)
+    printPagesTree(cat.pages || [], 2)
+  }
+
+  const stagingDir = fs.mkdtempSync(path.join(os.tmpdir(), 'readme-import-'))
+
+  let result
+  try {
+    styles.info(`Staging frontmatter stubs in ${styles.bold(stagingDir)}...`)
+    const stageStart = Date.now()
+    const staged = await timePhase('stage stubs', async () =>
+      stageOrganized(organized, stagingDir, { skipApiReference: !!options.skipApiReference, llmsPaths }),
+    )
+
+    // Changelog pages stage into docs/Changelog/ (a schema-valid docs
+    // category). With --separate-changelog, relocate them afterwards into a
+    // top-level changelog/ dir — done as a post-staging move so the staged
+    // tree is a valid git-format layout up to this point.
+    if (options.separateChangelog) {
+      const movedChangelog = relocateChangelogDir(stagingDir)
+      if (movedChangelog > 0) {
+        styles.info(
+          `Moved ${styles.bold(String(movedChangelog))} changelog page${movedChangelog === 1 ? '' : 's'} into ${styles.bold('changelog/')} (--separate-changelog).`,
+        )
+      }
+    }
+
+    const landingTitle =
+      organized.title ||
+      (sourceUrls.length === 1 ? sourceUrls[0].hostname : sourceUrls.map((u) => u.hostname).join(' + '))
+    ensureDocsLandingPage(stagingDir, landingTitle)
+    styles.ok(
+      `Staged ${styles.bold(String(staged.fileCount))} stub${staged.fileCount === 1 ? '' : 's'} across ${styles.bold(String(staged.dirCount))} director${staged.dirCount === 1 ? 'y' : 'ies'} in ${styles.bold(formatDuration(Date.now() - stageStart))}.`,
+    )
+    if (staged.skippedApiRef > 0) {
+      styles.info(`Skipped ${styles.bold(String(staged.skippedApiRef))} API reference page${staged.skippedApiRef === 1 ? '' : 's'} (--skip-api-reference)`)
+    }
+
+    // Download captured `.json` specs into <stagingDir>/oas/. Raw bytes only —
+    // validation + OpenAPI detection + upload all happen in the runner. Staged
+    // here (not in docs/ or reference/) so it rides the zip but is invisible to
+    // the docs build. Non-fatal: a failed download is logged and skipped.
+    if (oasJsonUrls.length > 0) {
+      const oasCount = await timePhase('download oas specs', () => downloadOasSpecs(oasJsonUrls, stagingDir))
+      if (oasCount > 0) {
+        styles.ok(`Downloaded ${styles.bold(String(oasCount))} candidate OpenAPI spec${oasCount === 1 ? '' : 's'} into ${styles.bold('oas/')}.`)
+      }
+    }
+    console.log()
+
+    if (options.test) {
+      styles.ok(`Done in ${styles.bold(formatDuration(Date.now() - startedAt))}! Staged ${styles.bold(String(staged.fileCount))} files at ${styles.bold(stagingDir)}`)
+      console.log()
+      styles.info('Starting the dev server for preview...')
+      console.log()
+      await runDevPreview(stagingDir)
+      return { source: 'url', stagingDir, fileCount: staged.fileCount, duration: Date.now() - startedAt, phases }
+    }
+
+    if (staged.fileCount === 0) {
+      styles.warning('Staging directory is empty — skipping zip.')
+      return { source: 'url', fileCount: 0, duration: Date.now() - startedAt, phases }
+    }
+
+    styles.info(`Packaging ${styles.bold(String(staged.fileCount))} files into ${styles.bold(outputZip)}...`)
+    await timePhase('zip', () => createZip(stagingDir, outputZip))
+
+    console.log()
+    styles.ok(`Done in ${styles.bold(formatDuration(Date.now() - startedAt))}! Your ReadMe import is ready at ${styles.bold(outputZip)}`)
+    console.log(styles.dim(`  ⏱  ${phases.map((p) => `${p.label} ${formatDuration(p.ms)}`).join(' · ')}`))
+    result = { source: 'url', outputZip, fileCount: staged.fileCount, duration: Date.now() - startedAt, phases }
+  } finally {
+    if (!options.test) {
+      fs.rmSync(stagingDir, { recursive: true, force: true })
+    }
+  }
+
+  return result
+}
+
+/**
+ * Run the discovery + scrape + organize pipeline for ONE source URL and
+ * return its `organized` object (`{ title, categories: [...] }`). Pure
+ * per-source work — no staging, no zipping. `importDocs` runs this in
+ * parallel across multiple sources and merges the results.
+ *
+ * `debugSnapshots`, when provided, gets per-source keys suffixed with
+ * the hostname so parallel runs don't clobber each other.
+ */
+async function produceOrganizedForSource(sourceUrl, options, timePhase, debugSnapshots) {
+  const redirected = await timePhase('follow source redirects', () => resolveRedirectedSourceUrl(sourceUrl))
+  if (redirected) {
+    styles.info(`${styles.bold(sourceUrl.toString())} redirects to ${styles.bold(redirected.toString())} — rebasing import onto the final URL.`)
+    sourceUrl = redirected
+  }
+
+  // Probe well-known docs routes (docs./developer. subdomains and /docs/,
+  // … path prefixes) and adopt the first that resolves as the real base, so everything downstream runs
+  const docsBase = await timePhase('probe well-known docs routes', () => resolveDocsBaseUrl(sourceUrl))
+  if (docsBase) {
+    styles.info(
+      `${styles.bold(sourceUrl.toString())} looks like a homepage — switching to docs route ${styles.bold(docsBase.url.toString())}${styles.dim(docsBase.hasLlms ? ' (llms.txt found)' : ' (no llms.txt; using for scrape fallback)')}.`,
+    )
+    sourceUrl = docsBase.url
+  }
+
+  // Discover llms.txt files by BFS walking parents AND children. Seed the
+  // frontier with the walk-up paths (source → root). Any "hit" (file exists
+  // with at least one link row) expands the frontier: parent path + distinct
+  // first-child segments extracted from its URL list. Invalid hits (fail the
+  // usability ratio) still expand — they're stepping stones — only valid
+  // hits contribute content to the merged result. See
+  // mydocs.com/section1/section2 → also tries /section1/, /, and every
+  // sibling section2 we discover along the way.
+  styles.info(`Discovering llms.txt (walking up and down from ${styles.bold(sourceUrl.toString())})...`)
+
+  const discovery = await timePhase('discover llms.txt', () => discoverLlmsTxt(sourceUrl))
+  for (const hit of discovery.hits) {
+    if (hit.usable) {
+      styles.info(styles.dim(`  ${hit.llmsUrl} → valid (${hit.stats.linkItems} links)`))
+    } else {
+      styles.info(styles.dim(`  ${hit.llmsUrl} → invalid: ${hit.reason} (kept as stepping stone)`))
+    }
+  }
+  for (const miss of discovery.misses) {
+    styles.info(styles.dim(`  ${miss.url} → ${miss.status ? `HTTP ${miss.status}` : miss.error || 'no llms.txt'}`))
+  }
+  let llms = mergeValidHits(discovery.hits)
+  let llmsUrl = llms?.llmsUrl || null
+  const skippedLlms = discovery.skipped
+
+  // Capture `.json` links (candidate OpenAPI specs) BEFORE anything else sees
+  // the parsed tree. They're removed in place here, downloaded into the staged
+  // `oas/` dir by importDocs, and validated + uploaded by the runner. Stub
+  // generation never sees them.
+  let oasJsonUrls = []
+  if (llms) {
+    oasJsonUrls = extractOasJsonUrlsFromParsed(llms.parsed)
+    if (oasJsonUrls.length > 0) {
+      styles.info(styles.dim(`Captured ${oasJsonUrls.length} .json link${oasJsonUrls.length === 1 ? '' : 's'} as candidate OpenAPI spec${oasJsonUrls.length === 1 ? '' : 's'} → oas/.`))
+    }
+  }
+
+  // Drop the remaining asset/meta items (llms-full.txt, .yaml specs, …) from
+  // the merged parsed result up-front, so every downstream consumer — knownUrls
+  // AND the "use llms.txt sections directly" organize path — sees a clean URL
+  // list. `.json` items were already pulled out above.
+  if (llms) {
+    const dropped = dropAssetItemsFromParsed(llms.parsed)
+    if (dropped > 0) {
+      styles.info(styles.dim(`Dropped ${dropped} asset/meta URL${dropped === 1 ? '' : 's'} (.yaml/.xml/.toml, llms*.txt) from llms.txt items.`))
+    }
+  }
+
+  if (llms) {
+    const narrowed = narrowToDocsSubtreeIfNeeded(llms, sourceUrl, discovery.hits)
+    if (narrowed) {
+      styles.info(
+        styles.dim(
+          `Narrowed merged result to /${narrowed.segment}/ subtree (kept ${narrowed.kept}, dropped ${narrowed.dropped} out-of-scope page${narrowed.dropped === 1 ? '' : 's'}).`,
+        ),
+      )
+    }
+  }
+
+  // Fall back to scrape if the merged llms.txt covers too few endpoints to
+  // be useful as a structural backbone. Replaces the old per-file
+  // MIN_LINK_ROWS=10 check (now relaxed in utils/llms.js) with a global
+  // post-merge floor — many tiny per-product llms.txt files can still pass
+  // as long as their union clears the bar.
+  const MIN_MERGED_LINK_COUNT = 10
+  if (llms) {
+    const mergedLinkCount = llms.parsed.sections.reduce((n, s) => n + s.items.length, 0)
+    if (mergedLinkCount < MIN_MERGED_LINK_COUNT) {
+      styles.warning(
+        `Merged llms.txt has only ${styles.bold(String(mergedLinkCount))} endpoint${mergedLinkCount === 1 ? '' : 's'} (< ${MIN_MERGED_LINK_COUNT}) — falling back to scrape.`,
+      )
+      skippedLlms.push({ url: llmsUrl, reason: `merged total only ${mergedLinkCount} link${mergedLinkCount === 1 ? '' : 's'}` })
+      llms = null
+      llmsUrl = null
+    }
+  }
   console.log()
 
+  let sitemapUrl = null
+  let sitemapKnownUrls = []
   if (!llms) {
-    styles.warning(`No llms.txt found at any probed path — falling back to sidebar discovery via scrape.`)
+    // Sitemap fallback: gives us a flat URL list to anchor the scrape (slot
+    // orphans by path) even though it lacks llms.txt's section structure.
+    // Scope is narrowed to the docs subtree of the source URL when possible
+    // so we don't pull in marketing/blog pages from a whole-site sitemap.
+    const sitemapCandidates = buildSitemapCandidates(sourceUrl)
+    const scopePrefix = deriveSitemapScope(sourceUrl)
+    styles.info(
+      `${skippedLlms.length > 0 ? 'No usable llms.txt' : 'No llms.txt'} — checking for sitemap.xml (${sitemapCandidates.length} candidate${sitemapCandidates.length === 1 ? '' : 's'})${scopePrefix ? `, scoped to ${styles.bold(scopePrefix)}` : ''}...`,
+    )
+    const sitemapResult = await timePhase('fetch sitemap.xml', async () => {
+      for (const candidate of sitemapCandidates) {
+        const res = await fetchSitemap(candidate, scopePrefix)
+        if (res.ok && res.urls.length > 0) return { url: candidate, urls: res.urls }
+        const reason = res.ok ? '0 urls in scope' : res.status ? `HTTP ${res.status}` : res.error || 'failed'
+        styles.info(styles.dim(`  ${candidate} → ${reason}`))
+      }
+      return null
+    })
+    console.log()
+    if (sitemapResult) {
+      sitemapUrl = sitemapResult.url
+      sitemapKnownUrls = sitemapUrlsToKnownUrls(sitemapResult.urls)
+      styles.ok(`Found sitemap.xml at ${styles.bold(sitemapUrl)} — ${styles.bold(String(sitemapKnownUrls.length))} URL${sitemapKnownUrls.length === 1 ? '' : 's'} in scope.`)
+    } else {
+      styles.warning(`No llms.txt or sitemap.xml found — falling back to sidebar discovery via scrape.`)
+    }
   } else {
-    styles.info(styles.dim(`Using ${llmsUrl}.`))
+    const s = llms.stats
+    if (llms.sourceFiles.length === 1) {
+      styles.info(styles.dim(`Using ${llmsUrl} (${s.conforming}/${s.total} lines conforming, ratio ${s.ratio.toFixed(2)}).`))
+    } else {
+      styles.info(styles.dim(`Merged ${llms.sourceFiles.length} llms.txt files (root: ${llmsUrl}; aggregate ratio ${s.ratio.toFixed(2)}).`))
+    }
+    if (llms.parsed.h3Fallback) {
+      styles.info(styles.dim(`H2 sections oversized — re-parsed using H3 (###) headings as section boundaries.`))
+    }
+  }
+
+  // Pre-extract changelog pages so AI / URL clustering / section-direct paths
+  // never see them as candidates for organization. We re-attach them as a
+  // dedicated Changelog category once `organized` is finalized below — that
+  // guarantees they route to docs/Changelog/ (or to changelog/ with
+  // --separate-changelog) regardless of which organize path ran. Done here,
+  // before knownUrls is derived, so every downstream consumer operates on the
+  // already-changelog-free view.
+  let extractedChangelog = []
+  if (llms) {
+    extractedChangelog = extractChangelogFromSections(llms.parsed.sections)
+  } else if (sitemapKnownUrls.length > 0) {
+    const { extracted, kept } = partitionChangelogFromKnownUrls(sitemapKnownUrls)
+    extractedChangelog = extracted
+    sitemapKnownUrls = kept
+  }
+  if (extractedChangelog.length > 0) {
+    styles.info(
+      styles.dim(
+        `Pre-extracted ${extractedChangelog.length} changelog page${extractedChangelog.length === 1 ? '' : 's'} — bypassing organization, attaching as Changelog category at the end.`,
+      ),
+    )
+  }
+
+  // Dedupe items across llms.parsed.sections by URL (first occurrence wins).
+  // Some llms.txt files cross-reference the same page under multiple headings —
+  // feeding duplicate URLs into the organize step causes the same page to appear
+  // in multiple sidebar sections regardless of which path (direct, icons, full
+  // reorg) runs downstream.
+  if (llms) {
+    const seenItemUrls = new Set()
+    let deduped = 0
+    for (const section of llms.parsed.sections) {
+      const before = section.items.length
+      section.items = section.items.filter((item) => {
+        const key = normalizePath(item.url)
+        if (seenItemUrls.has(key)) return false
+        seenItemUrls.add(key)
+        return true
+      })
+      deduped += before - section.items.length
+    }
+    if (deduped > 0) {
+      styles.info(styles.dim(`Deduped ${deduped} cross-section duplicate URL${deduped === 1 ? '' : 's'} from llms.txt sections.`))
+    }
   }
 
+  const dbgSuffix = `-${sourceUrl.hostname}`
   if (debugSnapshots) {
-    debugSnapshots['01-llms-parsed.json'] = { llmsUrl, parsed: llms ? llms.parsed : null }
+    debugSnapshots[`01-llms-parsed${dbgSuffix}.json`] = { llmsUrl, parsed: llms ? llms.parsed : null, skipped: skippedLlms }
+    debugSnapshots[`01b-sitemap${dbgSuffix}.json`] = { sitemapUrl, urls: sitemapKnownUrls }
+    debugSnapshots[`01c-extracted-changelog${dbgSuffix}.json`] = extractedChangelog
   }
 
   let knownUrls = []
   if (llms) {
     const totalItems = llms.parsed.sections.reduce((n, s) => n + s.items.length, 0)
+    const srcCount = llms.sourceFiles.length
+    const srcLabel = srcCount === 1 ? 'llms.txt' : `${srcCount} llms.txt files (merged)`
     styles.ok(
-      `Found llms.txt — ${styles.bold(String(totalItems))} page${totalItems === 1 ? '' : 's'} across ${styles.bold(String(llms.parsed.sections.length))} section${llms.parsed.sections.length === 1 ? '' : 's'}${llms.parsed.title ? ` (${llms.parsed.title})` : ''}.`,
+      `Found ${srcLabel} — ${styles.bold(String(totalItems))} page${totalItems === 1 ? '' : 's'} across ${styles.bold(String(llms.parsed.sections.length))} section${llms.parsed.sections.length === 1 ? '' : 's'}${llms.parsed.title ? ` (${llms.parsed.title})` : ''}.`,
     )
 
     const rawKnownUrls = llms.parsed.sections.flatMap((s) => s.items.map((i) => ({ title: i.text, url: i.url, description: i.description })))
@@ -117,19 +464,21 @@ export async function importDocs(options) {
     // Dedupe llms.txt entries by pathname. Some sites (zod.dev, fumadocs) list
     // every in-page anchor as its own llms.txt row (`/v4?id=wrapping-up`,
     // `/v4?id=metadata`, …) even though they all live on one rendered page.
-    // We prefer the "cleanest" URL per path — the shortest one, which is
-    // usually the one without a query string or hash.
+    // Prefer direct markdown source forms over shorter rendered URLs because
+    // they are better import sources for downstream content fetching.
     const byKnownPath = new Map()
     for (const p of rawKnownUrls) {
       const key = normalizePath(p.url)
       const prev = byKnownPath.get(key)
-      if (!prev || p.url.length < prev.url.length) byKnownPath.set(key, p)
+      if (!prev || compareCanonicalUrlPreference(p.url, prev.url) < 0) byKnownPath.set(key, p)
     }
     knownUrls = Array.from(byKnownPath.values())
     const dropped = rawKnownUrls.length - knownUrls.length
     if (dropped > 0) {
       styles.info(`${styles.dim(`Collapsed ${dropped} anchor/query duplicates → ${knownUrls.length} unique pages.`)}`)
     }
+  } else if (sitemapKnownUrls.length > 0) {
+    knownUrls = sitemapKnownUrls
   }
 
   console.log()
@@ -142,56 +491,82 @@ export async function importDocs(options) {
   const mintlifyStart = Date.now()
   const mintlifyNav = await timePhase('mintlify probe', () => tryMintlifyNav(sourceUrl.toString(), knownUrls, firecrawlKey))
   if (debugSnapshots) {
-    debugSnapshots['02a-mintlify-nav.json'] = mintlifyNav ? JSON.parse(JSON.stringify(mintlifyNav)) : null
+    debugSnapshots[`02a-mintlify-nav${dbgSuffix}.json`] = mintlifyNav ? JSON.parse(JSON.stringify(mintlifyNav)) : null
   }
   if (mintlifyNav) {
-    const pageCount = mintlifyNav.categories.reduce((n, c) => n + c.pages.length, 0)
+    const pageCount = mintlifyNav.categories.reduce((n, c) => n + countUrlPagesDeep(c.pages), 0)
     styles.ok(
       `Found Mintlify config at ${styles.bold(mintlifyNav.source)} in ${styles.bold(formatDuration(Date.now() - mintlifyStart))} — ${styles.bold(String(mintlifyNav.categories.length))} categor${mintlifyNav.categories.length === 1 ? 'y' : 'ies'}, ${styles.bold(String(pageCount))} pages.`,
     )
   }
   console.log()
 
+  // Archbee sites embed the canonical document tree in Next.js page data.
+  // Their llms.txt export can be a flat, shuffled list, so prefer the tree
+  // when present.
+  let archbeeNav = null
+  if (!mintlifyNav) {
+    styles.info(`Probing for Archbee document tree...`)
+    const archbeeStart = Date.now()
+    archbeeNav = await timePhase('archbee probe', () => tryArchbeeNav(sourceUrl.toString(), knownUrls, firecrawlKey))
+    if (archbeeNav) {
+      const pageCount = archbeeNav.categories.reduce((n, c) => n + countUrlPagesDeep(c.pages), 0)
+      styles.ok(
+        `Found Archbee document tree in ${styles.bold(formatDuration(Date.now() - archbeeStart))} — ${styles.bold(String(pageCount))} page${pageCount === 1 ? '' : 's'}.`,
+      )
+    }
+    console.log()
+  }
+
   let scraped
   let scrapeStart = Date.now()
+  const scrapeDiagnostics = {}
   if (mintlifyNav) {
     scraped = { title: mintlifyNav.title, categories: mintlifyNav.categories }
+  } else if (archbeeNav) {
+    scraped = { title: archbeeNav.title, categories: archbeeNav.categories }
   } else {
     styles.info(`Scraping sidebar nav from ${styles.bold(sourceUrl.toString())}${firecrawlKey ? ' ' + styles.dim('(via Firecrawl)') : ''}...`)
     scrapeStart = Date.now()
-    scraped = await timePhase('scrape nav', () => scrapeNavFromSite(sourceUrl.toString(), knownUrls, firecrawlKey))
+    scraped = await timePhase('scrape nav', () => scrapeNavFromSite(sourceUrl.toString(), knownUrls, firecrawlKey, scrapeDiagnostics))
+    if (!scraped) scrapeDiagnostics.reason ??= 'unknown'
   }
   if (debugSnapshots) {
-    debugSnapshots['02-scraped-raw.json'] = scraped ? JSON.parse(JSON.stringify(scraped)) : null
+    debugSnapshots[`02-scraped-raw${dbgSuffix}.json`] = scraped ? JSON.parse(JSON.stringify(scraped)) : null
   }
-  // Prefer llms.txt when it has strong multi-section structure and the
-  // scrape was a thin snapshot (common on big multi-tab docs — Stripe, AWS,
-  // Twilio — where each page only renders its own tab's sidebar). Without
-  // this override the 4-category scrape wins over a 25-section llms.txt and
-  // hundreds of real pages end up smeared into orphan buckets.
+  // If the sidebar scrape covers less than 75% of the llms.txt URLs, the
+  // scrape is too thin to trust as the import's spine. Common on multi-tab
+  // docs (Stripe, AWS, Twilio, Xata) where each page only renders its own
+  // tab's sidebar — the visible categories would otherwise absorb hundreds
+  // of orphan URLs via prefix-matching and produce a misleading tree (e.g.
+  // every /docs/* URL dumped under a single "Overview > Xata Documentation"
+  // node because that's the one /docs page the scrape saw). Discard the
+  // scrape and fall through to the llms.txt path, which uses URL-based
+  // clustering when multiple files were merged.
+  let scrapeDiscardedForCoverage = false
   if (scraped && llms && knownUrls.length > 0) {
-    const scrapedPages = scraped.categories.reduce((n, c) => n + c.pages.length, 0)
+    const scrapedPages = scraped.categories.reduce((n, c) => n + countUrlPagesDeep(c.pages), 0)
     const coverage = scrapedPages / knownUrls.length
-    const llmsUsable = usableSections(llms.parsed.sections)
-    if (llmsUsable.length >= 5 && coverage < 0.5) {
+    if (coverage < 0.75) {
       styles.info(
-        `Scrape covered ${styles.bold(Math.round(coverage * 100) + '%')} of llms.txt pages; preferring llms.txt's ${styles.bold(String(llmsUsable.length))} sections for structure.`,
+        `Scrape covered ${styles.bold(Math.round(coverage * 100) + '%')} of llms.txt pages (need ≥75%) — discarding scrape and organizing from llms.txt.`,
       )
       scraped = null
+      scrapeDiscardedForCoverage = true
     }
   }
 
   if (scraped) {
-    const directMatches = scraped.categories.reduce((n, c) => n + c.pages.length, 0)
+    const directMatches = scraped.categories.reduce((n, c) => n + countUrlPagesDeep(c.pages), 0)
     if (knownUrls.length > 0) {
       const slotted = slotOrphansByPath(scraped, knownUrls)
       if (debugSnapshots) {
-        debugSnapshots['03-after-slot-by-path.json'] = {
+        debugSnapshots[`03-after-slot-by-path${dbgSuffix}.json`] = {
           scraped: JSON.parse(JSON.stringify(scraped)),
           unslottedOrphans: slotted,
         }
       }
-      const totalMatched = scraped.categories.reduce((n, c) => n + c.pages.length, 0)
+      const totalMatched = scraped.categories.reduce((n, c) => n + countUrlPagesDeep(c.pages), 0)
       styles.ok(
         `Scraped nav in ${styles.bold(formatDuration(Date.now() - scrapeStart))} — ${styles.bold(String(scraped.categories.length))} categor${scraped.categories.length === 1 ? 'y' : 'ies'}, ${styles.bold(String(directMatches))} direct matches + ${styles.bold(String(totalMatched - directMatches))} slotted by path = ${styles.bold(String(totalMatched))}/${knownUrls.length}.`,
       )
@@ -205,40 +580,83 @@ export async function importDocs(options) {
         styles.info(`Moved ${styles.bold(String(moved))} page${moved === 1 ? '' : 's'} into ${styles.bold('API Reference')} based on URL path.`)
       }
 
+      // Same idea for changelog / release-notes pages — pull any whose URL
+      // carries a changelog-style segment into one "Changelog" category. It
+      // routes to docs/Changelog/ unless --separate-changelog relocates it.
+      const movedChangelog = reclassifyChangelogPages(scraped)
+      if (movedChangelog > 0) {
+        styles.info(`Moved ${styles.bold(String(movedChangelog))} page${movedChangelog === 1 ? '' : 's'} into ${styles.bold('Changelog')} based on URL path.`)
+      }
+
       if (slotted.length > 0) {
-        // Mintlify-style docs put API endpoints in a separate tab rooted at
-        // /api-reference/* (or /api/*, /reference/*). Collapse remaining such
-        // pages into a single "API Reference" category (absorbing the flat
-        // category the sweep pass just built, if any) and nest it by resource
-        // segment so routeCategory() maps the whole thing to ReadMe's
-        // `reference/` top-level dir.
-        const apiResult = collectApiReferencePages(slotted, scraped)
-        const otherOrphans = apiResult.nonApiOrphans
-        if (apiResult.category) scraped.categories.push(apiResult.category)
-
-        const buckets = bucketOrphansByPathType(otherOrphans, scraped)
-        for (const b of buckets) scraped.categories.push(b)
-        if (debugSnapshots) {
-          debugSnapshots['04-after-orphan-buckets.json'] = {
-            apiReferenceCollected: apiResult.category
-              ? {
+        // When orphans dwarf direct matches, the sidebar scrape was too thin
+        // to trust as the import's spine — keeping it would produce a small
+        // "real" tree plus a soup of bucketed-by-URL-type orphan categories.
+        // Discard the scrape and cluster every page (scrape + orphans) by
+        // its top URL segment instead, so each `/docs/`, `/sdk/`, `/rest-api/`
+        // becomes its own category. `nestByUrlHierarchy` (later) handles the
+        // empty-parent nesting within each category.
+
+        // Trip when orphans are at least 2× the direct matches: the scrape
+        // accounts for less than a third of the known pages, so its category
+        // labels aren't a trustworthy spine for the remainder.
+        const orphansDwarfDirect = slotted.length >= directMatches * 2
+        const scrapeAllPages = scraped.categories.flatMap((c) => collectUrlPagesDeep(c.pages))
+        let reclustered = null
+        if (orphansDwarfDirect) {
+          reclustered = clusterByUrlPath([...scrapeAllPages, ...slotted])
+        }
+        styles.info(
+          styles.dim(
+            `  orphan triage: ${slotted.length} orphan${slotted.length === 1 ? '' : 's'} vs ${directMatches} direct match${directMatches === 1 ? '' : 'es'} (ratio ${directMatches === 0 ? '∞' : (slotted.length / directMatches).toFixed(2)}) — gate ${orphansDwarfDirect ? 'tripped' : 'NOT tripped'} (need ≥2.00); URL re-cluster ${reclustered ? `→ ${reclustered.length} categor${reclustered.length === 1 ? 'y' : 'ies'}` : 'skipped'}`,
+          ),
+        )
+        if (reclustered) {
+          scraped.categories = reclustered.map((c) => ({ title: c.title, pages: c.pages }))
+          styles.info(
+            `Scrape covered only ${styles.bold(String(directMatches))}/${styles.bold(String(knownUrls.length))} pages — discarded as too thin and re-clustered ${styles.bold(String(scrapeAllPages.length + slotted.length))} pages by URL path into ${styles.bold(String(reclustered.length))} categor${reclustered.length === 1 ? 'y' : 'ies'}: ${reclustered.map((c) => styles.bold(c.title)).join(', ')}.`,
+          )
+          if (debugSnapshots) {
+            debugSnapshots[`04-after-orphan-buckets${dbgSuffix}.json`] = {
+              mode: 'reclustered-by-url-path',
+              scraped: JSON.parse(JSON.stringify(scraped)),
+            }
+          }
+        } else {
+          // Mintlify-style docs put API endpoints in a separate tab rooted at
+          // /api-reference/* (or /api/*, /reference/*). Collapse remaining such
+          // pages into a single "API Reference" category (absorbing the flat
+          // category the sweep pass just built, if any) and nest it by resource
+          // segment so routeCategory() maps the whole thing to ReadMe's
+          // `reference/` top-level dir.
+          const apiResult = collectApiReferencePages(slotted, scraped)
+          const otherOrphans = apiResult.nonApiOrphans
+          if (apiResult.category) scraped.categories.push(apiResult.category)
+
+          const buckets = bucketOrphansByPathType(otherOrphans, scraped)
+          for (const b of buckets) scraped.categories.push(b)
+          if (debugSnapshots) {
+            debugSnapshots[`04-after-orphan-buckets${dbgSuffix}.json`] = {
+              apiReferenceCollected: apiResult.category
+                ? {
                   pageCount: apiResult.category.pages.length,
                   mergedFromScraped: apiResult.mergedScrapedTitles,
                 }
-              : null,
-            buckets,
-            scraped: JSON.parse(JSON.stringify(scraped)),
+                : null,
+              buckets,
+              scraped: JSON.parse(JSON.stringify(scraped)),
+            }
+          }
+          const parts = []
+          if (apiResult.category) {
+            parts.push(`${styles.bold(String(apiResult.category.pages.length))} in ${styles.bold('API Reference')}`)
+          }
+          for (const b of buckets) {
+            parts.push(`${styles.bold(String(b.pages.length))} in ${styles.bold(b.title)}`)
+          }
+          if (parts.length > 0) {
+            styles.info(`${styles.bold(String(slotted.length))} orphan page${slotted.length === 1 ? '' : 's'} bucketed by URL type: ${parts.join(', ')}.`)
           }
-        }
-        const parts = []
-        if (apiResult.category) {
-          parts.push(`${styles.bold(String(apiResult.category.pages.length))} in ${styles.bold('API Reference')}`)
-        }
-        for (const b of buckets) {
-          parts.push(`${styles.bold(String(b.pages.length))} in ${styles.bold(b.title)}`)
-        }
-        if (parts.length > 0) {
-          styles.info(`${styles.bold(String(slotted.length))} orphan page${slotted.length === 1 ? '' : 's'} bucketed by URL type: ${parts.join(', ')}.`)
         }
       }
     } else {
@@ -265,10 +683,34 @@ export async function importDocs(options) {
         )
       }
     }
-  } else if (!llms) {
-    throw new Error(`No llms.txt and the sidebar scrape found no usable structure — can't import ${sourceUrl.toString()}.`)
+  } else if (!llms && knownUrls.length === 0) {
+    throw new Error(`No llms.txt or sitemap.xml and the sidebar scrape found no usable structure — can't import ${sourceUrl.toString()}.`)
+  } else if (llms) {
+    if (!scrapeDiscardedForCoverage) {
+      const d = scrapeDiagnostics || {}
+      if (d.reason === 'no-categories-round0') {
+        styles.warning(`Sidebar scrape found no nav structure on the round-0 fetch — falling back to llms.txt organization.`)
+      } else if (d.reason === 'below-threshold') {
+        styles.warning(
+          `Sidebar scrape below acceptance threshold (got ${d.categories} categor${d.categories === 1 ? 'y' : 'ies'}, ${d.matched} matched pages — need ${d.need}) — falling back to llms.txt organization.`,
+        )
+      } else {
+        styles.warning(`Sidebar scrape returned no usable nav — falling back to llms.txt organization.`)
+      }
+    }
   } else {
-    styles.warning(`Couldn't extract a useful nav — falling back to llms.txt-based organization.`)
+    if (!scrapeDiscardedForCoverage) {
+      const d = scrapeDiagnostics || {}
+      if (d.reason === 'no-categories-round0') {
+        styles.warning(`Sidebar scrape found no nav structure on the round-0 fetch — falling back to sitemap URL clustering.`)
+      } else if (d.reason === 'below-threshold') {
+        styles.warning(
+          `Sidebar scrape below acceptance threshold (got ${d.categories} categor${d.categories === 1 ? 'y' : 'ies'}, ${d.matched} matched pages — need ${d.need}) — falling back to sitemap URL clustering.`,
+        )
+      } else {
+        styles.warning(`Sidebar scrape returned no usable nav — falling back to sitemap URL clustering.`)
+      }
+    }
   }
   console.log()
 
@@ -281,75 +723,131 @@ export async function importDocs(options) {
       title: (llms && llms.parsed.title) || null,
       categories: scraped.categories.map((c) => ({ title: c.title, icon: null, pages: c.pages })),
     }
+  } else if (llms) {
+    if (llms.sourceFiles.length > 1) {
+      // Multi-file merge: H2 section headings across the merged files are
+      // heterogeneous and don't form a coherent spine (e.g. elevenlabs.io's
+      // root file uses `Docs`/`Products` while /docs/llms.txt uses
+      // `API Docs` — concatenating them produces meaningless top-level
+      // categories like `docs/Docs`, `docs/Products`, `docs/API Docs`).
+      // Throw away the section grouping and re-cluster by URL path instead.
+      // Per-page metadata (title, url, description) is still carried over.
+      const flatPages = llms.parsed.sections.flatMap((s) =>
+        s.items.map((it) => ({
+          title: it.text,
+          url: it.url,
+          ...(it.description ? { description: it.description } : {}),
+        })),
+      )
+      const clustered = clusterByUrlPath(flatPages) || [{ title: 'Documentation', pages: flatPages }]
+      styles.info(
+        `Multi-file llms.txt merge — clustered ${styles.bold(String(flatPages.length))} pages by URL path into ${styles.bold(String(clustered.length))} categor${clustered.length === 1 ? 'y' : 'ies'} (H2 sections discarded).`,
+      )
+      organized = {
+        title: llms.parsed.title || null,
+        categories: clustered.map((c) => ({ title: c.title, icon: null, pages: c.pages })),
+      }
+    } else {
+      // Single-file mode: H2 structure IS the author's intended organization,
+      // so trust it. Use sections directly when usable; otherwise run Claude.
+      const usable = usableSections(llms.parsed.sections)
+      const avgPages = usable.length > 0 ? usable.reduce((n, s) => n + s.items.length, 0) / usable.length : 0
+      if (sectionsLookUsable(llms.parsed.sections) && avgPages < 50) {
+        styles.info(
+          `Using llms.txt sections directly — ${styles.bold(String(usable.length))} sections, ${styles.bold(avgPages.toFixed(1))} avg pages/section (≤50). Skipping Claude.`,
+        )
+        organized = {
+          title: llms.parsed.title || null,
+          categories: usable.map((s) => ({
+            title: s.title,
+            icon: null,
+            pages: s.items.map((it) => ({
+              title: it.text,
+              url: it.url,
+              ...(it.description ? { description: it.description } : {}),
+            })),
+          })),
+        }
+      } else {
+        const fastPath = sectionsLookUsable(llms.parsed.sections)
+        styles.info(`Organizing with Claude (${styles.bold(options.model)}, ${fastPath ? 'fast path: icons only' : 'full reorg'})...`)
+        organized = await timePhase('claude organize', () => organizeWithClaude(llms.parsed, options.model))
+        const movedRef = reclassifyReferencePages(organized)
+        if (movedRef > 0) {
+          styles.info(`Moved ${styles.bold(String(movedRef))} page${movedRef === 1 ? '' : 's'} into ${styles.bold('API Reference')} based on URL path.`)
+        }
+      }
+    }
   } else {
-    const fastPath = sectionsLookUsable(llms.parsed.sections)
-    styles.info(`Organizing with Claude (${styles.bold(options.model)}, ${fastPath ? 'fast path: icons only' : 'full reorg'})...`)
-    organized = await timePhase('claude organize', () => organizeWithClaude(llms.parsed, options.model))
-  }
-  styles.ok(`Organized in ${styles.bold(formatDuration(Date.now() - organizeStart))}.`)
-  if (debugSnapshots) {
-    debugSnapshots['05-organized.json'] = organized
-    const debugDir = path.join(os.tmpdir(), `readme-import-debug-${sourceUrl.hostname}-${Date.now()}`)
-    fs.mkdirSync(debugDir, { recursive: true })
-    for (const [name, data] of Object.entries(debugSnapshots)) {
-      fs.writeFileSync(path.join(debugDir, name), JSON.stringify(data, null, 2))
+    // Sitemap-only fallback: synthesize categories by clustering the URL
+    // paths. clusterByUrlPath returns null when the URLs don't split cleanly,
+    // in which case everything lands in a single Documentation bucket.
+    const clustered = clusterByUrlPath(knownUrls) || [{ title: 'Documentation', pages: knownUrls }]
+    organized = {
+      title: null,
+      categories: clustered.map((c) => ({ title: c.title, icon: null, pages: c.pages })),
     }
-    styles.info(`${styles.dim(`Debug snapshots → ${debugDir}`)}`)
   }
-  console.log()
 
-  console.log(`  ${styles.bold(organized.title || '(untitled)')}`)
-  for (const cat of organized.categories || []) {
-    console.log()
-    const iconLabel = cat.icon ? `${styles.brand(`[${cat.icon}]`)} ` : ''
-    console.log(`  ${iconLabel}${styles.bold(cat.title)}`)
-    printPagesTree(cat.pages || [], 2)
-  }
+  // Re-attach the pre-extracted changelog pages as their own Changelog
+  // category. Done before nestByUrlHierarchy so the injected pages get nested
+  // alongside the rest of the tree for consistency. Merges into a pre-existing
+  // Changelog category (scrape path's reclassifier may have already built one).
+  injectChangelogCategory(organized, extractedChangelog)
 
-  const stagingDir = fs.mkdtempSync(path.join(os.tmpdir(), 'readme-import-'))
+  // Import section landing pages (e.g. /docs/cli) that llms.txt models as a
+  // bare heading + child links and so never lists as a page of their own.
+  // Runs before nesting so the injected page lands at its trie node and
+  // nestByUrlHierarchy renders it as the real parent of its children.
+  await injectSectionLandingPages(organized, sourceUrl)
 
-  let result
-  try {
-    styles.info(`Staging frontmatter stubs in ${styles.bold(stagingDir)}...`)
-    const stageStart = Date.now()
-    const staged = await timePhase('stage stubs', async () => stageOrganized(organized, stagingDir, { skipApiReference: !!options.skipApiReference }))
-    ensureDocsLandingPage(stagingDir, organized.title || sourceUrl.hostname)
-    styles.ok(
-      `Staged ${styles.bold(String(staged.fileCount))} stub${staged.fileCount === 1 ? '' : 's'} across ${styles.bold(String(staged.dirCount))} director${staged.dirCount === 1 ? 'y' : 'ies'} in ${styles.bold(formatDuration(Date.now() - stageStart))}.`,
-    )
-    if (staged.skippedApiRef > 0) {
-      styles.info(`Skipped ${styles.bold(String(staged.skippedApiRef))} API reference page${staged.skippedApiRef === 1 ? '' : 's'} (--skip-api-reference)`)
-    }
-    console.log()
+  for (const cat of organized.categories || []) {
+    cat.pages = nestByUrlHierarchy(cat.pages)
+  }
 
-    if (options.test) {
-      styles.ok(`Done in ${styles.bold(formatDuration(Date.now() - startedAt))}! Staged ${styles.bold(String(staged.fileCount))} files at ${styles.bold(stagingDir)}`)
-      console.log()
-      styles.info('Starting the dev server for preview...')
-      console.log()
-      await runDevPreview(stagingDir)
-      return { source: 'url', stagingDir, fileCount: staged.fileCount, duration: Date.now() - startedAt, phases }
-    }
+  styles.ok(`Organized in ${styles.bold(formatDuration(Date.now() - organizeStart))}.`)
+  organized.oasJsonUrls = oasJsonUrls
 
-    if (staged.fileCount === 0) {
-      styles.warning('Staging directory is empty — skipping zip.')
-      return { source: 'url', fileCount: 0, duration: Date.now() - startedAt, phases }
+  // Record which page URLs came from llms.txt
+  const llmsPaths = new Set()
+  if (llms) {
+    for (const p of [...knownUrls, ...extractedChangelog]) {
+      if (p.url) llmsPaths.add(normalizePath(p.url))
     }
+  }
+  organized.llmsPaths = llmsPaths
+  return organized
+}
 
-    styles.info(`Packaging ${styles.bold(String(staged.fileCount))} files into ${styles.bold(outputZip)}...`)
-    await timePhase('zip', () => createZip(stagingDir, outputZip))
-
-    console.log()
-    styles.ok(`Done in ${styles.bold(formatDuration(Date.now() - startedAt))}! Your ReadMe import is ready at ${styles.bold(outputZip)}`)
-    console.log(styles.dim(`  ⏱  ${phases.map((p) => `${p.label} ${formatDuration(p.ms)}`).join(' · ')}`))
-    result = { source: 'url', outputZip, fileCount: staged.fileCount, duration: Date.now() - startedAt, phases }
-  } finally {
-    if (!options.test) {
-      fs.rmSync(stagingDir, { recursive: true, force: true })
+/**
+ * Merge per-source `organized` results into one. Same-titled categories MERGE
+ * their page lists (intentional — when a user bundles related sources into
+ * one project, they probably want a coherent sidebar; `ensureUniqueSlugs` later
+ * dedupes any slug collisions globally). First non-null title wins; first
+ * non-null icon per merged category wins. Page order within a merged category
+ * is the concatenation of input sources in argument order.
+ */
+function mergeOrganized(perSource) {
+  const title = perSource.map((o) => o.title).find(Boolean) || null
+
+  const byTitle = new Map()
+  const order = []
+  for (const o of perSource) {
+    for (const cat of o.categories || []) {
+      const key = cat.title
+      const existing = byTitle.get(key)
+      if (existing) {
+        existing.pages.push(...(cat.pages || []))
+        if (!existing.icon && cat.icon) existing.icon = cat.icon
+      } else {
+        const copy = { ...cat, pages: [...(cat.pages || [])] }
+        byTitle.set(key, copy)
+        order.push(copy)
+      }
     }
   }
 
-  return result
+  return { title, categories: order }
 }
 
 /**
@@ -783,7 +1281,7 @@ async function tryMintlifyNav(sourceUrl, knownPages, firecrawlKey) {
 function extractMintlifyConfig(body) {
   try {
     return JSON.parse(body)
-  } catch {}
+  } catch { }
   // Pull out the first balanced `{ ... }` that contains a "navigation" key.
   const navIdx = body.indexOf('"navigation"')
   if (navIdx === -1) return null
@@ -794,7 +1292,7 @@ function extractMintlifyConfig(body) {
       try {
         const parsed = JSON.parse(candidate)
         if (parsed && parsed.navigation) return parsed
-      } catch {}
+      } catch { }
     }
     start = body.lastIndexOf('{', start - 1)
   }
@@ -882,20 +1380,95 @@ function parseMintlifyConfig(config, origin, byPath) {
 }
 
 /**
- * Score a parsed nav tree for "sidebar-likeness". A real docs sidebar has
- * multiple section headers (hierarchy) and tens of links; secondary navs
- * (sitemaps, footers, search-index result lists) tend to be flat single-
- * category blobs that are either alphabetized or dumped in insertion order.
+ * Archbee exposes its canonical docs tree in Next.js page data as
+ * `_docSpace.publicDocsTree`. That tree preserves author order and nesting,
+ * while Archbee's llms.txt export may be flat and shuffled.
  *
- * Flat single-category blocks are penalized so a slightly smaller hierarchical
- * block wins over a bigger flat one. Without this, greenflash.ai's sidebar
- * order was non-deterministic across runs — sometimes Quickstart first (real
- * sidebar won), sometimes last (a flat alphabetized index block won).
+ * Returns { title, categories } or null if the page is not Archbee or the
+ * embedded tree is unavailable.
  */
-function scoreNavTree(tree) {
-  const count = tree.categories.reduce((n, c) => n + c.pages.length, 0)
-  const cats = tree.categories.length
-  const hierarchyBonus = cats >= 2 ? cats * 5 : -20
+async function tryArchbeeNav(sourceUrl, knownPages, firecrawlKey) {
+  const origin = new URL(sourceUrl).origin
+  const fetchHtml = firecrawlKey ? makeFirecrawlFetcher(firecrawlKey) : fetchHtmlDirect
+  const html = await fetchHtml(toBrowsableUrl(sourceUrl))
+  if (!html || !/publicDocsTree|Archbee/i.test(html)) return null
+
+  const nextData = extractNextData(html)
+  const docSpace = nextData?.props?.pageProps?._docSpace
+  const tree = docSpace?.publicDocsTree
+  if (!Array.isArray(tree) || tree.length === 0) return null
+
+  const byPath = new Map()
+  for (const p of knownPages) byPath.set(normalizePath(p.url), p)
+
+  const pageFromNode = (node) => {
+    if (!node || typeof node !== 'object') return null
+    if (node.isHidden === true) return null
+
+    const title = String(node.name || node.title || node.urlKey || '').trim()
+    const children = Array.isArray(node.children) ? node.children.map(pageFromNode).filter(Boolean) : []
+    if (!title && children.length === 0) return null
+
+    const url = archbeeNodeUrl(origin, node.urlKey)
+    const known = url ? byPath.get(normalizePath(url)) : null
+    const page = {
+      title: known?.title || title || 'Untitled',
+      url,
+      ...(known?.description ? { description: known.description } : {}),
+      ...(children.length > 0 ? { pages: children } : {}),
+    }
+
+    // Root docs pages have no URL path segments, but should stay in their
+    // authored position when the later URL-trie nesting pass runs.
+    if (url && extractUrlPathSegments(url).length === 0) {
+      page._virtualPathSegs = [kebabCase(page.title || 'home') || 'home']
+    }
+
+    return page
+  }
+
+  const pages = tree.map(pageFromNode).filter(Boolean)
+  if (pages.length === 0) return null
+
+  return {
+    title: docSpace.hostingTitle || docSpace.name || null,
+    categories: [{ title: 'Documentation', pages }],
+  }
+}
+
+function extractNextData(html) {
+  const m = String(html).match(/<script\b(?=[^>]*\bid=["']__NEXT_DATA__["'])[^>]*>([\s\S]*?)<\/script>/i)
+  if (!m) return null
+  try {
+    return JSON.parse(m[1])
+  } catch {
+    return null
+  }
+}
+
+function archbeeNodeUrl(origin, urlKey) {
+  if (urlKey === undefined || urlKey === null) return null
+  const key = String(urlKey || '').trim()
+  if (!key || key === '/') return `${origin}/`
+  if (/^https?:\/\//i.test(key)) return key
+  return `${origin}/${key.replace(/^\/+/, '')}`
+}
+
+/**
+ * Score a parsed nav tree for "sidebar-likeness". A real docs sidebar has
+ * multiple section headers (hierarchy) and tens of links; secondary navs
+ * (sitemaps, footers, search-index result lists) tend to be flat single-
+ * category blobs that are either alphabetized or dumped in insertion order.
+ *
+ * Flat single-category blocks are penalized so a slightly smaller hierarchical
+ * block wins over a bigger flat one. Without this, greenflash.ai's sidebar
+ * order was non-deterministic across runs — sometimes Quickstart first (real
+ * sidebar won), sometimes last (a flat alphabetized index block won).
+ */
+function scoreNavTree(tree) {
+  const count = tree.categories.reduce((n, c) => n + c.pages.length, 0)
+  const cats = tree.categories.length
+  const hierarchyBonus = cats >= 2 ? cats * 5 : -20
   // Alphabetical penalty: real sidebars are curated (Overview first, related
   // topics grouped). Auto-generated indexes, search clouds, and footer link
   // lists tend to be strictly alphabetized. When a category's pages come in
@@ -924,7 +1497,7 @@ function isMonotonicAlpha(titles) {
  * renders its sidebar server-side as <nav>/<aside> with <h*> section headers.
  * Returns null if coverage is too low to be useful.
  */
-async function scrapeNavFromSite(sourceUrl, knownPages, firecrawlKey) {
+async function scrapeNavFromSite(sourceUrl, knownPages, firecrawlKey, diagnostics = {}) {
   // Index known pages by normalized pathname so we can match nav hrefs against them.
   const byPath = new Map()
   for (const p of knownPages) byPath.set(normalizePath(p.url), p)
@@ -1060,7 +1633,10 @@ async function scrapeNavFromSite(sourceUrl, knownPages, firecrawlKey) {
   const r0Start = Date.now()
   await visit(sourceUrl)
   const r0Ms = Date.now() - r0Start
-  if (categoryOrder.length === 0) return null
+  if (categoryOrder.length === 0) {
+    diagnostics.reason = 'no-categories-round0'
+    return null
+  }
 
   // Round 1 (parallel): visit pages so each branch has a chance to expose
   // its sub-items. Sidebars on most docs sites auto-expand the current
@@ -1077,9 +1653,9 @@ async function scrapeNavFromSite(sourceUrl, knownPages, firecrawlKey) {
   const r1Urls = isDiscovery
     ? flattenTree(categoryOrder).slice(0, MAX_DISCOVERY_FETCHES)
     : categoryOrder
-        .map((c) => c.pages[0])
-        .filter(Boolean)
-        .map((p) => toBrowsableUrl(p.url))
+      .map((c) => c.pages[0])
+      .filter(Boolean)
+      .map((p) => toBrowsableUrl(p.url))
   const r1Start = Date.now()
   // Firecrawl standard-plan concurrency is 10; 5 leaves headroom for retries.
   // Native HTTP can run hotter since we're hitting our own loopback.
@@ -1089,14 +1665,44 @@ async function scrapeNavFromSite(sourceUrl, knownPages, firecrawlKey) {
     styles.dim(`  ⏱  scrape breakdown: round0=${formatDuration(r0Ms)} round1=${formatDuration(r1Ms)} (${r1Urls.length} ${isDiscovery ? 'discovery' : 'category rep'} fetches)`),
   )
 
+  // Prune login/account chrome now that crawling is done. On Stoplight sites
+  // the "Sign in" → /auth link is the only seed on the bare landing page, so we
+  // had to keep it discoverable to reach the real sidebar — but it's not a doc.
+  const pruneAuthPages = (pages) => {
+    let removed = 0
+    for (let i = pages.length - 1; i >= 0; i--) {
+      const page = pages[i]
+      if (page.pages && page.pages.length > 0) removed += pruneAuthPages(page.pages)
+      if (isAuthChromeUrl(page.url) && (!page.pages || page.pages.length === 0)) {
+        matched.delete(normalizePath(page.url))
+        pages.splice(i, 1)
+        removed++
+      }
+    }
+    return removed
+  }
+  for (const cat of categoryOrder) pruneAuthPages(cat.pages)
+
   // Accept thresholds — looser in discovery mode (no llms.txt) where even a
   // single flat "Overview" bucket is better than nothing, stricter when we
   // have llms.txt to compare against.
   const categories = categoryOrder.filter((c) => c.pages.length > 0)
   if (isDiscovery) {
-    if (categories.length < 1 || matched.size < 5) return null
+    if (categories.length < 1 || matched.size < 5) {
+      diagnostics.reason = 'below-threshold'
+      diagnostics.categories = categories.length
+      diagnostics.matched = matched.size
+      diagnostics.need = '≥1 category, ≥5 matched pages'
+      return null
+    }
   } else {
-    if (categories.length < 2 || matched.size < 10) return null
+    if (categories.length < 2 || matched.size < 10) {
+      diagnostics.reason = 'below-threshold'
+      diagnostics.categories = categories.length
+      diagnostics.matched = matched.size
+      diagnostics.need = '≥2 categories, ≥10 matched pages'
+      return null
+    }
   }
   return { title: null, categories }
 }
@@ -1121,12 +1727,17 @@ function extractSidebarContainers(html) {
   const SIDEBAR_ATTR_RE =
     /\b(?:id|class|aria-label|data-testid)=(?:"[^"]*sidebar[^"]*"|'[^']*sidebar[^']*'|"[^"]*navigation[^"]*"|'[^']*navigation[^']*')|\brole=(?:"navigation"|'navigation')/i
 
+  // Layout-wrapper modifiers like `Page--with-sidebar` / `has-sidebar` describe
+  // an element that CONTAINS a sidebar, not one that IS a sidebar.
+  const WRAPPER_MODIFIER_RE = /(?:^|[\s"'-])(?:with|has)-sidebar(?=[\s"']|$)/i
+
   const out = []
   for (const tag of SIDEBAR_TAGS) {
     const openRe = new RegExp(`<${tag}\\b([^>]*)>`, 'gi')
     let m
     while ((m = openRe.exec(html)) !== null) {
       if (!SIDEBAR_ATTR_RE.test(m[1])) continue
+      if (WRAPPER_MODIFIER_RE.test(m[1])) continue
       const inner = extractBalancedTag(html, tag, m.index + m[0].length)
       if (inner != null && inner.length > 100) out.push(inner)
     }
@@ -1337,7 +1948,7 @@ function collectApiReferencePages(orphans, scraped) {
     let segs = []
     try {
       segs = new URL(p.url).pathname.split('/').filter(Boolean)
-    } catch {}
+    } catch { }
     // segs[0] is the api-prefix itself; segs[1] (if any) is the resource.
     const resource = segs.length >= 3 ? segs[1] : null
     if (!resource) {
@@ -1393,9 +2004,9 @@ function bucketOrphansByPathType(orphans, scraped) {
     endpoints: 'API Reference',
     endpoint: 'API Reference',
     changelog: 'Changelog',
-    release: 'Release Notes',
-    releases: 'Release Notes',
-    'release-notes': 'Release Notes',
+    release: 'Changelog',
+    releases: 'Changelog',
+    'release-notes': 'Changelog',
     recipes: 'Recipes',
     recipe: 'Recipes',
     guides: 'Guides',
@@ -1488,43 +2099,42 @@ function titleCase(s) {
 
 /**
  * Walk all scraped pages (including nested sub-pages) and move any whose URL
- * contains a strong reference segment (`/api-reference/`, `/endpoints/`, etc.)
- * into a single "API Reference" category. Some docs sites (e.g. greenflash.ai)
- * spotlight a handful of endpoints under "Developers" in the sidebar while the
- * bulk of endpoints live under a separate API Reference section — we favor
- * the URL-path signal over the sidebar placement so all reference pages land
- * together in `reference/` after staging.
+ * carries one of `segmentRe`'s path segments into a single category,
+ * flattening sub-pages as it goes. Shared backbone for the API Reference and
+ * Changelog sweeps: both favor the URL-path signal over sidebar placement, so
+ * e.g. `/api-reference/*` or `/changelog/*` pages land together in their own
+ * category even when a site's sidebar spotlights a few of them elsewhere.
  *
- * Returns the number of pages relocated.
+ * `categoryRe` matches an existing category title to merge into (so we don't
+ * create a duplicate); when none matches, a fresh `{ title: defaultTitle }`
+ * category is appended. Returns the number of pages relocated.
  */
-function reclassifyReferencePages(scraped) {
-  const REFERENCE_SEGMENT = /^(api[-_]?reference|endpoints?)$/i
+function reclassifyPagesByUrlSegment(scraped, { segmentRe, categoryRe, defaultTitle }) {
   const normTitle = (t) =>
     String(t || '')
       .replace(INVISIBLE_CHARS, '')
       .trim()
       .toLowerCase()
 
-  const looksLikeRefUrl = (url) => {
+  const urlHasSegment = (url) => {
     try {
       const segs = new URL(url).pathname.split('/').filter(Boolean)
-      return segs.some((s) => REFERENCE_SEGMENT.test(s))
+      return segs.some((s) => segmentRe.test(s))
     } catch {
       return false
     }
   }
 
-  // Find (or create) the canonical API Reference category. Prefer an existing
-  // one with a reference-shaped title so we don't end up with duplicates.
-  let refCat = scraped.categories.find((c) => /^(api[ -]?reference|reference|api|endpoints?)$/i.test(normTitle(c.title).replace(/\s+/g, ' ')))
-  const existedBefore = Boolean(refCat)
+  // Find (or create) the canonical destination category. Prefer an existing
+  // one with a matching title so we don't end up with duplicates.
+  let destCat = scraped.categories.find((c) => categoryRe.test(normTitle(c.title).replace(/\s+/g, ' ')))
 
   const collected = []
   const filterPages = (pages) => {
     const kept = []
     for (const p of pages || []) {
-      if (looksLikeRefUrl(p.url)) {
-        // Flatten sub-pages when relocating — API Reference is a flat list.
+      if (urlHasSegment(p.url)) {
+        // Flatten sub-pages when relocating — these sections are flat lists.
         collectFlat(p, collected)
         continue
       }
@@ -1534,32 +2144,145 @@ function reclassifyReferencePages(scraped) {
     return kept
   }
 
-  // Never pull pages out of the reference category itself.
+  // Never pull pages out of the destination category itself.
   for (const cat of scraped.categories) {
-    if (cat === refCat) continue
+    if (cat === destCat) continue
     cat.pages = filterPages(cat.pages)
   }
 
   if (collected.length === 0) return 0
 
-  if (!refCat) {
-    refCat = { title: 'API Reference', pages: [] }
-    scraped.categories.push(refCat)
+  if (!destCat) {
+    destCat = { title: defaultTitle, pages: [] }
+    scraped.categories.push(destCat)
   }
-  // Dedupe against anything already in the reference category.
-  const seen = new Set(refCat.pages.map((p) => normalizePath(p.url)))
+  // Dedupe against anything already in the destination category.
+  const seen = new Set(destCat.pages.map((p) => normalizePath(p.url)))
   for (const p of collected) {
     const key = normalizePath(p.url)
     if (seen.has(key)) continue
     seen.add(key)
-    refCat.pages.push(p)
+    destCat.pages.push(p)
+  }
+
+  // Drop now-empty categories (other than the one we may have just created).
+  scraped.categories = scraped.categories.filter((c) => c === destCat || (c.pages && c.pages.length > 0))
+
+  return collected.length
+}
+
+/**
+ * Sweep `/api-reference/*` and `/endpoints/*` pages into one "API Reference"
+ * category. Some docs sites (e.g. greenflash.ai) spotlight a handful of
+ * endpoints under "Developers" in the sidebar while the bulk of endpoints
+ * live under a separate API Reference section — this lands them all in
+ * `reference/` after staging. Returns the number of pages relocated.
+ */
+function reclassifyReferencePages(scraped) {
+  return reclassifyPagesByUrlSegment(scraped, {
+    segmentRe: /^(api[-_]?reference|endpoints?)$/i,
+    categoryRe: /^(api[ -]?reference|reference|api|endpoints?)$/i,
+    defaultTitle: 'API Reference',
+  })
+}
+
+// Anchored half covers ambiguous bare keywords (release, releases, versioned
+// release-2026-2) — those need to be the whole segment so we don't sweep
+// `/release-pipeline/` etc. Unanchored half covers the unambiguous keywords
+// (changelog, release-notes, whats-new), which we catch as substrings so
+// slugs like `/docs/changelog-javascript-agent` and `/docs/ios-sdk-changelog`
+// get pulled in too.
+const CHANGELOG_URL_SEGMENT_RE = /(?:^(?:releases?|release[-_]?v?\d[\w.-]*)$)|(?:changelog|change[-_]?log|release[-_]?notes?|whats?[-_]?new)/i
+const CHANGELOG_CATEGORY_TITLE_RE = /^(changelog|change ?log|release ?notes?|releases?|what'?s ?new)$/i
+
+function urlIsChangelog(url) {
+  try {
+    const segs = new URL(url).pathname.split('/').filter(Boolean)
+    return segs.some((s) => CHANGELOG_URL_SEGMENT_RE.test(s))
+  } catch {
+    return false
+  }
+}
+
+/**
+ * Pull changelog items OUT of llms.parsed.sections in place. Done before the
+ * organize pipeline runs so AI / clustering never sees them — they get
+ * re-attached as a dedicated Changelog category at the end via
+ * injectChangelogCategory. Sections that go empty afterwards are filtered out
+ * naturally by `usableSections` downstream.
+ *
+ * Returns the extracted items in `{ title, url, description? }` shape.
+ */
+function extractChangelogFromSections(sections) {
+  const extracted = []
+  for (const section of sections || []) {
+    const kept = []
+    for (const item of section.items || []) {
+      if (urlIsChangelog(item.url)) {
+        extracted.push({
+          title: item.text,
+          url: item.url,
+          ...(item.description ? { description: item.description } : {}),
+        })
+      } else {
+        kept.push(item)
+      }
+    }
+    section.items = kept
+  }
+  return extracted
+}
+
+/**
+ * Split a flat known-URL list into changelog vs. the rest. Used for the
+ * sitemap-only fallback, where there's no llms.txt section structure to mutate.
+ */
+function partitionChangelogFromKnownUrls(knownUrls) {
+  const extracted = []
+  const kept = []
+  for (const p of knownUrls || []) {
+    if (urlIsChangelog(p.url)) extracted.push(p)
+    else kept.push(p)
   }
+  return { extracted, kept }
+}
 
-  // Drop now-empty categories (other than the reference one we may have just created).
-  scraped.categories = scraped.categories.filter((c) => c === refCat || (c.pages && c.pages.length > 0))
+/**
+ * Attach pre-extracted changelog items as a dedicated Changelog category on
+ * the final organized tree. Merges into an existing Changelog-titled category
+ * if one is already present (e.g. the scrape path's reclassifier already built
+ * one); otherwise creates a fresh one. Dedupe is by normalized URL path.
+ */
+function injectChangelogCategory(organized, items) {
+  if (!items || items.length === 0) return
+  organized.categories = organized.categories || []
+  let dest = organized.categories.find((c) => CHANGELOG_CATEGORY_TITLE_RE.test(String(c.title || '').trim()))
+  if (!dest) {
+    dest = { title: 'Changelog', icon: null, pages: [] }
+    organized.categories.push(dest)
+  }
+  const seen = new Set((dest.pages || []).map((p) => normalizePath(p.url)))
+  for (const item of items) {
+    const key = normalizePath(item.url)
+    if (seen.has(key)) continue
+    seen.add(key)
+    dest.pages.push(item)
+  }
+}
 
-  // If the category existed before but the relocation was a no-op, surface 0.
-  return existedBefore ? collected.length : collected.length
+/**
+ * Sweep `/changelog/*`, `/release-notes/*` and `/releases/*` pages into one
+ * "Changelog" category, wherever the site's sidebar happened to place them.
+ * That category routes to `docs/Changelog/`; `--separate-changelog` later
+ * relocates it to a top-level `changelog/` dir. Returns the number of pages
+ * relocated.
+ */
+function reclassifyChangelogPages(scraped) {
+  return reclassifyPagesByUrlSegment(scraped, {
+    segmentRe: CHANGELOG_URL_SEGMENT_RE,
+    categoryRe: CHANGELOG_CATEGORY_TITLE_RE,
+    defaultTitle: 'Changelog',
+  })
 }
 
 function collectFlat(page, out) {
@@ -1573,6 +2296,170 @@ function collectFlat(page, out) {
   }
 }
 
+function countUrlPagesDeep(pages) {
+  let n = 0
+  for (const p of pages || []) {
+    if (p.url) n++
+    if (p.pages && p.pages.length > 0) n += countUrlPagesDeep(p.pages)
+  }
+  return n
+}
+
+function collectUrlPagesDeep(pages, out = []) {
+  for (const p of pages || []) {
+    if (p.url) out.push(p)
+    if (p.pages && p.pages.length > 0) collectUrlPagesDeep(p.pages, out)
+  }
+  return out
+}
+
+// urlTrieSegs is imported from ../utils/url-segs.js
+
+/**
+ * Re-parent URL-bearing siblings by URL path so descendants nest under their
+ * URL prefix even when no page exists at every intermediate segment. We build
+ * a trie from each page's URL path and render it back out as a page tree:
+ *
+ *   - The shared top-level prefix is elided (the category folder is its
+ *     implicit container — e.g. /docs/ai/evals collapses into "Pydantic Evals").
+ *   - Below the elision point, every segment becomes an entry: a real page
+ *     where one exists, otherwise an empty-parent stub (folder-icon, no URL).
+ *
+ * `anchorSegs` is the parent page's URL segments during recursion; when set,
+ * we walk the trie to that exact depth instead of eliding to the natural
+ * branch point, so a missing intermediate like `b` in `/a → /a/b/c, /a/b/d`
+ * still renders as an empty parent under `/a`.
+ *
+ * @example
+ *   nestByUrlHierarchy([
+ *     { title: 'Hello',  url: 'https://x.com/main/sub1/sub2/sub3/hello' },
+ *     { title: 'World',  url: 'https://x.com/main/sub1/sub2/sub3/world' },
+ *     { title: 'Sub 1',  url: 'https://x.com/main/sub1' },
+ *   ])
+ *   // [
+ *   //   { title: 'Sub 1', url: '…/main/sub1', pages: [
+ *   //     { title: 'Sub2', _emptyParent: true, pages: [
+ *   //       { title: 'Sub3', _emptyParent: true, pages: [
+ *   //         { title: 'Hello', url: '…/main/sub1/sub2/sub3/hello' },
+ *   //         { title: 'World', url: '…/main/sub1/sub2/sub3/world' },
+ *   //       ]},
+ *   //     ]},
+ *   //   ]},
+ *   // ]
+ */
+function nestByUrlHierarchy(pages, anchorSegs = null) {
+  if (!pages || pages.length === 0) return pages
+
+  // Recurse first so subtrees that came in pre-nested (e.g. from the scraper)
+  // get re-nested against their parent's URL as anchor.
+  for (const p of pages) {
+    if (p.pages && p.pages.length > 0) {
+      let nextAnchor = anchorSegs
+      if (p._virtualPathSegs) nextAnchor = p._virtualPathSegs
+      else if (p.url) {
+        const s = urlTrieSegs(p.url)
+        if (s) nextAnchor = s
+      }
+      p.pages = nestByUrlHierarchy(p.pages, nextAnchor)
+    }
+  }
+
+  const urlEntries = []
+  const groupOnly = []
+  for (const p of pages) {
+    if (!p.url) {
+      groupOnly.push(p)
+      continue
+    }
+    const segs = urlTrieSegs(p.url)
+    // Zero-segment URLs (origin-only links like `https://build.example.com/`)
+    // have no path to nest by — if we inserted them into the trie they'd land
+    // at the root node and the container walk would render them as the
+    // top-level entry with every other page nested beneath them. Treat them
+    // like group-only pages so they just sit alongside the real tree.
+    if (segs && segs.length > 0) urlEntries.push({ page: p, segs })
+    else groupOnly.push(p)
+  }
+
+  if (urlEntries.length === 0) return pages
+  if (urlEntries.length === 1 && groupOnly.length === 0) return pages
+
+  // Build the URL trie. Each node tracks its own segment, the page at that
+  // exact path (if any), insertion-ordered children, and the segment path
+  // from root (used as the slug source for empty parents).
+  const root = { segment: '', page: null, children: new Map(), segs: [] }
+  for (const { page, segs } of urlEntries) {
+    let node = root
+    for (let i = 0; i < segs.length; i++) {
+      const segLower = segs[i].toLowerCase()
+      if (!node.children.has(segLower)) {
+        node.children.set(segLower, { segment: segs[i], page: null, children: new Map(), segs: segs.slice(0, i + 1) })
+      }
+      node = node.children.get(segLower)
+    }
+    if (!node.page) node.page = page
+  }
+
+  // Find the trie node that represents the "container" for this level.
+  //   - With an anchor: that's the parent page's exact depth; we don't elide
+  //     past it, so intermediate segments below the anchor become empties.
+  //   - Without: walk down while there's only one no-page child (the shared
+  //     prefix). Stop at the first node that either has a page or branches.
+  let container = root
+  if (anchorSegs && anchorSegs.length > 0) {
+    for (let i = 0; i < anchorSegs.length; i++) {
+      const seg = anchorSegs[i].toLowerCase()
+      if (container.children.has(seg)) container = container.children.get(seg)
+      else break
+    }
+  } else {
+    while (container.children.size === 1 && !container.page) {
+      container = container.children.values().next().value
+    }
+  }
+
+  function nodeToPage(node) {
+    let outPage
+    if (node.page) {
+      outPage = node.page
+    } else {
+      const rawSeg = node.segment
+      const cleanedSeg = stripSegmentExtensions(rawSeg).replace(/^\d+[-_.]/, '') || rawSeg
+      outPage = {
+        title: titleCase(cleanedSeg),
+        _emptyParent: true,
+        _virtualPathSegs: node.segs,
+        pages: [],
+      }
+    }
+    if (node.children.size > 0) {
+      const childPages = []
+      for (const c of node.children.values()) childPages.push(nodeToPage(c))
+      // Preserve any pre-existing nested children whose URLs aren't already
+      // represented in the trie (group-only containers from the scraper, etc.).
+      const existing = outPage.pages || []
+      const trieUrls = new Set()
+      const collect = (p) => {
+        if (p.url) trieUrls.add(normalizePath(p.url))
+        for (const c of p.pages || []) collect(c)
+      }
+      for (const cp of childPages) collect(cp)
+      const preservedExisting = existing.filter((e) => !e.url || !trieUrls.has(normalizePath(e.url)))
+      outPage.pages = [...childPages, ...preservedExisting]
+    }
+    return outPage
+  }
+
+  const trieResult = []
+  if (container.page) {
+    trieResult.push(nodeToPage(container))
+  } else {
+    for (const c of container.children.values()) trieResult.push(nodeToPage(c))
+  }
+
+  return [...trieResult, ...groupOnly]
+}
+
 /**
  * The scraped nav only contains top-level items; subcategory pages sit behind
  * `>` chevrons and don't render on a cold fetch. For each llms.txt URL not
@@ -1687,6 +2574,62 @@ async function fetchHtmlDirect(url) {
   }
 }
 
+/**
+ * Download a `.json` URL as raw bytes. Returns a Buffer, or null on non-OK /
+ * network error. No parsing or validation — the runner is the OAS gate.
+ */
+async function fetchJsonSpec(url) {
+  try {
+    const res = await fetch(url, {
+      redirect: 'follow',
+      headers: { 'User-Agent': 'readme-cli-import', Accept: 'application/json' },
+    })
+    if (!res.ok) return null
+    return Buffer.from(await res.arrayBuffer())
+  } catch {
+    return null
+  }
+}
+
+/**
+ * Download the captured candidate-spec URLs into `<stagingDir>/oas/<name>.json`.
+ * Filenames come from the URL basename (collisions get a numeric suffix); the
+ * runner de-dups by content, so identical bytes under different names are
+ * handled there. Returns the count of files written.
+ */
+async function downloadOasSpecs(oasJsonUrls, stagingDir) {
+  const oasDir = path.join(stagingDir, 'oas')
+  fs.mkdirSync(oasDir, { recursive: true })
+  const usedNames = new Set()
+  let written = 0
+  for (const { url } of oasJsonUrls) {
+    const buf = await fetchJsonSpec(url)
+    if (!buf) {
+      styles.info(styles.dim(`  oas: skipped ${url} (download failed)`))
+      continue
+    }
+    // Derive a safe, unique *.json filename from the URL basename.
+    let base
+    try {
+      base = path.basename(new URL(url).pathname) || 'openapi.json'
+    } catch {
+      base = 'openapi.json'
+    }
+    base = base.replace(/[^a-zA-Z0-9._-]/g, '-')
+    if (!/\.json$/i.test(base)) base += '.json'
+    let name = base
+    let n = 2
+    while (usedNames.has(name)) {
+      name = base.replace(/\.json$/i, `-${n}.json`)
+      n++
+    }
+    usedNames.add(name)
+    fs.writeFileSync(path.join(oasDir, name), buf)
+    written++
+  }
+  return written
+}
+
 /**
  * Firecrawl-backed HTML loader. Firecrawl runs a real browser, waits for
  * hydration, and returns the rendered DOM — which is what we need for sites
@@ -1787,14 +2730,29 @@ function urlNamespace(url) {
 function clusterByUrlPath(pages) {
   if (!pages || pages.length < 3) return null
 
-  const parts = pages.map((p) => {
+  // Pages with no path segments (e.g. an origin-only URL like
+  // `https://build.example.com/`) can't participate in segment-based
+  // clustering. A handful are fine — we'll fold them into the first cluster
+  // as a place to land — but if more than 20% of the input is rootless,
+  // the source isn't really a hierarchical doc set and clustering won't
+  // produce a meaningful tree. Bail in that case so the caller falls back.
+  const noSegPages = []
+  const clusterable = []
+  const parts = []
+  for (const p of pages) {
+    let segs = []
     try {
-      return new URL(p.url).pathname.split('/').filter(Boolean)
-    } catch {
-      return []
+      segs = new URL(p.url).pathname.split('/').filter(Boolean)
+    } catch { }
+    if (segs.length === 0) {
+      noSegPages.push(p)
+    } else {
+      clusterable.push(p)
+      parts.push(segs)
     }
-  })
-  if (parts.some((pp) => pp.length === 0)) return null
+  }
+  if (noSegPages.length / pages.length > 0.2) return null
+  if (clusterable.length < 3) return null
 
   // Longest common prefix depth.
   let commonDepth = 0
@@ -1807,13 +2765,13 @@ function clusterByUrlPath(pages) {
   // The segment right after the common base is the category key.
   const keyIdx = commonDepth
   const byKey = new Map()
-  for (let i = 0; i < pages.length; i++) {
+  for (let i = 0; i < clusterable.length; i++) {
     const key = parts[i][keyIdx]
     // Skip pages that have no segment at the cluster index (they're AT the
     // common base — those would become their own "index"-like category).
     if (!key) continue
     if (!byKey.has(key)) byKey.set(key, [])
-    byKey.get(key).push(pages[i])
+    byKey.get(key).push(clusterable[i])
   }
 
   // Reject weak clusterings: need at least 2 groups AND at least one group
@@ -1824,7 +2782,7 @@ function clusterByUrlPath(pages) {
 
   // Preserve first-appearance order so the sidebar reflects source order.
   const firstSeen = new Map()
-  pages.forEach((p, i) => {
+  clusterable.forEach((p, i) => {
     const key = parts[i][keyIdx]
     if (key && !firstSeen.has(key)) firstSeen.set(key, i)
   })
@@ -1835,28 +2793,173 @@ function clusterByUrlPath(pages) {
     pages: byKey.get(key),
   }))
 
-  // A cluster with exactly one top-level page is NOT a real category — it's
-  // a parent page with children wrapped in a pseudo-category label. Categories
-  // are grouping labels with no content of their own; parent pages have
-  // content AND children. Collect those singletons into a shared
-  // "Documentation" bucket so they're siblings at the top level, each with
-  // their own sub-tree intact.
-  const multipageClusters = rawClusters.filter((c) => c.pages.length >= 2)
-  const singletonPages = rawClusters.filter((c) => c.pages.length === 1).flatMap((c) => c.pages)
-
-  const out = []
-  if (singletonPages.length > 0) {
-    out.push({ title: 'Documentation', pages: singletonPages })
+  // Classify each cluster as either a real category or an "Other
+  // Documentation" misfit. A real category needs at least 2 pages — a
+  // "category" with a single file in it just adds a level of nesting
+  // without adding structure. Pydantic's `/docs/logfire/get-started` is
+  // the motivating example: as the only `/docs/logfire/*` URL it
+  // shouldn't get its own top-level "Logfire" folder. Lone pages land in
+  // Other Documentation where they sit alongside other low-volume material.
+  const realCategories = []
+  const misfitPages = []
+  for (const cluster of rawClusters) {
+    if (cluster.pages.length >= 2) {
+      realCategories.push(cluster)
+      continue
+    }
+    misfitPages.push(cluster.pages[0])
   }
-  out.push(...multipageClusters)
 
-  // If we didn't actually produce any multi-page cluster, clustering added no
-  // value — every page was a singleton and we'd just have renamed Overview.
-  // Tell the caller to stick with the original flat shape.
-  if (multipageClusters.length === 0) return null
+  // If clustering produced no real categories, it added no value — tell the
+  // caller to fall back rather than hand back a lone misfit bucket.
+  if (realCategories.length === 0) return null
+
+  const out = [...realCategories]
+
+  // "Other Documentation" goes last so it never displaces a real category.
+  // It absorbs both standalone-page singletons and zero-segment URLs (those
+  // can't participate in segment-based clustering at all but the >20% guard
+  // above already proved they're a minority).
+  const otherDocs = [...misfitPages, ...noSegPages]
+  if (otherDocs.length > 0) {
+    out.push({ title: 'Other Documentation', pages: otherDocs })
+  }
   return out
 }
 
+/**
+ * Import section landing pages that the source models structurally rather than
+ * as a link. In llms.txt a section like the Vercel CLI is a bare `## CLI`
+ * heading followed by 68 `/docs/cli/<cmd>` links — the importer turns the
+ * heading into a category but `/docs/cli` itself is never listed, so its real,
+ * content-bearing overview page is dropped. ReadMe then redirects the
+ * content-less category to its first child (a broken landing page).
+ *
+ * For each category we compute the common URL prefix of its children (the
+ * section's landing URL), HTTP-probe it, and — when it really exists and stays
+ * in scope — prepend it as a real page so nestByUrlHierarchy renders it as the
+ * parent holding the children. Mutates `organized` in place.
+ */
+async function injectSectionLandingPages(organized, sourceUrl) {
+  const cats = organized?.categories || []
+  if (cats.length === 0) return
+
+  // Dedup set: every page URL already in the tree, by normalized path.
+  const known = new Set()
+  for (const c of cats) {
+    for (const p of collectUrlPagesDeep(c.pages || [])) known.add(normalizePath(p.url))
+  }
+
+  // Base path of the import itself — landing candidates must be strictly
+  // deeper than this so we never probe `/docs` (the base) as a "section".
+  const baseDepth = (urlTrieSegs(sourceUrl) || []).length
+  const origin = new URL(sourceUrl).origin
+
+  // Build one landing candidate per eligible category (cheap gate first).
+  const candidates = []
+  for (const cat of cats) {
+    const kids = collectUrlPagesDeep(cat.pages || [])
+    if (kids.length < 2) continue
+
+    // Longest common path prefix across the children — same loop as
+    // clusterByUrlPath's common-depth scan, over urlTrieSegs.
+    const parts = kids.map((p) => urlTrieSegs(p.url)).filter(Boolean)
+    if (parts.length < 2) continue
+    let commonDepth = 0
+    while (commonDepth < parts[0].length) {
+      const seg = parts[0][commonDepth]
+      if (!parts.every((pp) => pp[commonDepth] === seg)) break
+      commonDepth++
+    }
+    const commonSegs = parts[0].slice(0, commonDepth)
+
+    // Must be strictly deeper than the import base, and not already a page.
+    if (commonSegs.length <= baseDepth) continue
+    const landingUrl = `${origin}/${commonSegs.join('/')}`
+    if (known.has(normalizePath(landingUrl))) continue
+
+    candidates.push({ cat, landingUrl })
+  }
+  if (candidates.length === 0) return
+
+  // Probe surviving candidates in parallel. Accept only a real, in-scope page
+  // that doesn't redirect onto an already-known child.
+  const accepted = []
+  await visitAllInParallel(
+    candidates,
+    async ({ cat, landingUrl }) => {
+      try {
+        const res = await fetch(landingUrl, {
+          redirect: 'follow',
+          headers: { 'User-Agent': 'readme-cli-import' },
+        })
+        if (!res.ok) return
+        const candPath = new URL(landingUrl).pathname.toLowerCase()
+        const finalPath = new URL(res.url).pathname.toLowerCase()
+        if (!finalPath.startsWith(candPath)) return
+        if (known.has(normalizePath(res.url))) return
+
+        // Check `<meta http-equiv="refresh">` navigates the browser elsewhere. 
+        const html = await res.text()
+        const metaRefresh = parseMetaRefreshTarget(html, res.url)
+        if (metaRefresh && normalizePath(metaRefresh) !== normalizePath(res.url)) return
+        const canonical = parseHtmlCanonical(html, res.url)
+        if (canonical && known.has(normalizePath(canonical))) return
+
+        accepted.push({ cat, landingUrl })
+      } catch {
+        // Network error / unparseable — treat as "no landing page".
+      }
+    },
+    LANDING_PROBE_CAP,
+  )
+
+  for (const { cat, landingUrl } of accepted) {
+    // Don't set x-from-llms: the page isn't in llmsPaths, so the runner fetches
+    // its rendered HTML via the content cascade (there's no .md mirror).
+    cat.pages.unshift({ title: cat.title, url: landingUrl })
+    known.add(normalizePath(landingUrl))
+  }
+  if (accepted.length > 0) {
+    styles.info(`Imported ${styles.bold(String(accepted.length))} section landing page(s).`)
+  }
+}
+
+/**
+ * Parse a client-side redirect target from an HTML page. Return pages with 
+ * `<meta http-equiv="refresh" content="0;url=/target/">` that navigates
+ * the browser elsewhere.
+ */
+function parseMetaRefreshTarget(html, baseUrl) {
+  if (!html) return null
+  const tag = /<meta[^>]+http-equiv=["']?refresh["']?[^>]*>/i.exec(html)
+  if (!tag) return null
+  const m = /content=["'][^"']*url=([^"'>\s]+)/i.exec(tag[0])
+  if (!m) return null
+  try {
+    return new URL(decodeEntities(m[1].trim()), baseUrl).toString()
+  } catch {
+    return null
+  }
+}
+
+/**
+ * Parse the `<link rel="canonical" href="...">` target from an HTML page,
+ * resolved absolute against `baseUrl`. Returns null when absent or unparseable.
+ */
+function parseHtmlCanonical(html, baseUrl) {
+  if (!html) return null
+  const tag = /<link[^>]+rel=["']?canonical["']?[^>]*>/i.exec(html)
+  if (!tag) return null
+  const m = /href=["']([^"']+)["']/i.exec(tag[0])
+  if (!m) return null
+  try {
+    return new URL(decodeEntities(m[1].trim()), baseUrl).toString()
+  } catch {
+    return null
+  }
+}
+
 /**
  * Used by discovery-mode scraping (no llms.txt) to decide whether a nav
  * link is worth importing as a doc page. Filters out cross-origin links,
@@ -1877,6 +2980,20 @@ function isDiscoverableLink(abs, base) {
   return true
 }
 
+// Leading-segment auth routes: /auth, /login, /sign-in, /logout, /sso, /oauth…
+// These are login/account chrome (Stoplight's "Sign in" → /auth, etc.), not
+// docs. NOT filtered at link-discovery time — on some Stoplight sites the
+// bare landing page only exposes the "Sign in" link, which we still need as a
+// crawl seed to reach the real sidebar. Pruned from the tree post-crawl.
+const AUTH_ROUTE_RE = /^\/(?:auth|login|log-?in|sign-?in|sign-?up|signup|logout|log-?out|sign-?out|signout|register|sso|oauth2?)(?:\/|$)/i
+function isAuthChromeUrl(url) {
+  try {
+    return AUTH_ROUTE_RE.test(new URL(url).pathname)
+  } catch {
+    return false
+  }
+}
+
 // Zero-width spaces, direction marks, word joiner, BOM — some docs sites
 // inject these into sidebar headings (docs.greenflash.ai prefixes
 // "API Reference" with U+200B), which otherwise defeats downstream
@@ -1916,20 +3033,7 @@ function decodeEntities(s) {
     .replace(/&rdquo;/g, '”')
 }
 
-/**
- * Reduce a URL to a comparable pathname: lowercase host, strip trailing slash
- * and common suffixes (.md, .html) so `/foo/bar.md` and `/foo/bar` match.
- */
-function normalizePath(url) {
-  try {
-    const u = new URL(url)
-    let p = u.pathname.replace(/\/$/, '').toLowerCase()
-    p = p.replace(/\.(md|mdx|html?)$/i, '')
-    return p
-  } catch {
-    return String(url).toLowerCase()
-  }
-}
+// normalizePath is imported from ../utils/url-segs.js
 
 /**
  * Given a set of scraped categories + orphan pages that didn't match the nav
@@ -1943,7 +3047,8 @@ async function slotOrphansWithClaude(scraped, orphans, model) {
     categories: scraped.categories,
     orphans,
   })
-  const raw = await runJsonQuery({ systemPrompt, userPrompt, model })
+  const result = await runJsonQuery({ systemPrompt, userPrompt, model, schema: slotOrphansOutputSchema })
+  const raw = result.assignments
   if (!Array.isArray(raw)) {
     // If the model didn't cooperate, return all orphans unassigned.
     return orphans
@@ -1973,8 +3078,8 @@ async function iconizeScrapedNav(scraped, _unused, model, siteTitle) {
   const { systemPrompt, userPrompt } = iconizeNavPrompt({
     categories: scraped.categories,
   })
-  const icons = await runJsonQuery({ systemPrompt, userPrompt, model })
-  const iconArr = Array.isArray(icons) ? icons : []
+  const result = await runJsonQuery({ systemPrompt, userPrompt, model, schema: iconizeNavOutputSchema })
+  const iconArr = Array.isArray(result.icons) ? result.icons : []
   return {
     title: siteTitle || null,
     categories: scraped.categories.map((c, i) => ({
@@ -1986,31 +3091,22 @@ async function iconizeScrapedNav(scraped, _unused, model, siteTitle) {
 }
 
 /**
- * Sections are "usable" when the llms.txt already did the hard grouping work
- * for us — meaningful titles, not too many/few, each populated. When usable we
- * take a fast path that only asks Claude for icons + title polish instead of
- * re-bucketing every page, which is the slow part of a full reorg.
- */
-// Sections named like these are catch-all buckets — even in richly-structured
-// llms.txt files (e.g. Stripe's "Docs" section is where pages go that don't
-// fit into a named product tab), so always drop them rather than promote the
-// grab-bag contents to a top-level sidebar category.
-const GENERIC_SECTION_RE =
-  /^(resources?|english|root url|pages?|docs?|documentation|content|available languages.*|site|sitemap|index|home|optional|instructions?(\s|:).*|miscellaneous|misc|other)$/i
-
-/**
- * Return the subset of llms.txt sections that carry real structural signal —
- * drop catch-all buckets ("Docs", "Resources", "Optional"), empty sections,
- * and oversized ones (site-dumps masquerading as sections).
+ * Return the subset of llms.txt sections worth sending to the fast path —
+ * drop only structurally bad ones (empty, or oversized site-dumps). Heading
+ * names are preserved verbatim: a heading like "Docs" or "Documentation"
+ * might be a catch-all on one site (bad) and a real top-level category on
+ * another (fine), and we can't tell cheaply, so we don't try.
  */
 function usableSections(sections) {
   if (!sections) return []
-  return sections.filter((s) => s.title && !GENERIC_SECTION_RE.test(s.title.trim()) && s.items && s.items.length > 0 && s.items.length <= 200)
+  return sections.filter((s) => s.title && s.items && s.items.length > 0 && s.items.length <= 200)
 }
 
 function sectionsLookUsable(sections) {
-  if (!sections || sections.length > 40) return false
-  return usableSections(sections).length >= 3
+  const usable = usableSections(sections)
+  if (usable.length < 3) return false
+  const avg = usable.reduce((n, s) => n + s.items.length, 0) / usable.length
+  return avg >= 5 && avg <= 100
 }
 
 async function organizeWithClaude(parsed, model) {
@@ -2026,16 +3122,17 @@ async function organizeWithClaude(parsed, model) {
  * is O(sections), not O(pages), so this is usually ~5-15s vs. a full reorg.
  */
 async function organizeFromSections(parsed, model) {
-  // Drop generic/empty/oversized sections so they don't pollute the sidebar
-  // (e.g. Stripe's llms.txt has a "Docs" catch-all and a 0-item "Instructions
-  // for Large Language Model Agents" section — neither is structural signal).
+  // Drop empty/oversized sections (e.g. a 0-item "Instructions for Large
+  // Language Model Agents" preamble, or a 500-row site-dump) — heading
+  // names are preserved as-is.
   const sections = usableSections(parsed.sections)
 
   const { systemPrompt, userPrompt } = organizeFromSectionsPrompt({
     siteTitle: parsed.title,
     sections,
   })
-  const raw = await runJsonQuery({ systemPrompt, userPrompt, model })
+  const result = await runJsonQuery({ systemPrompt, userPrompt, model, schema: organizeFromSectionsOutputSchema })
+  const raw = result.sections
   if (!Array.isArray(raw)) {
     throw new Error('Fast-path expected a JSON array of {title, icon} entries.')
   }
@@ -2070,18 +3167,22 @@ async function organizeFromScratch(parsed, model) {
     siteTitle: parsed.title,
     items,
   })
-  const raw = await runJsonQuery({ systemPrompt, userPrompt, model })
+  const raw = await runJsonQuery({ systemPrompt, userPrompt, model, schema: organizeFromScratchOutputSchema })
 
   // Rehydrate pages from the id references Claude returned.
   const expandedCategories = []
   const usedIds = new Set()
+  const usedUrls = new Set()
   for (const cat of raw.categories || []) {
     const pages = []
     for (const id of cat.pageIds || []) {
       const item = items[id]
       if (!item) continue // ignore out-of-range ids
       if (usedIds.has(id)) continue // ignore dupes
+      const normUrl = normalizePath(item.url)
+      if (usedUrls.has(normUrl)) continue // guard against same URL at different input indexes
       usedIds.add(id)
+      usedUrls.add(normUrl)
       pages.push({
         title: item.title,
         url: item.url,
@@ -2111,11 +3212,13 @@ async function organizeFromScratch(parsed, model) {
 }
 
 /**
- * Shared Claude call for "send a prompt, parse JSON back". Logs the prompts so
- * we can see what went in, runs a heartbeat so silent model latency doesn't
- * look like a hang, and strips stray code fences if the model adds them.
+ * Shared Claude call for "send a prompt, get schema-validated JSON back".
+ * Logs the prompts so we can see what went in, and runs a heartbeat so silent
+ * model latency doesn't look like a hang. The SDK enforces `schema` and
+ * returns parsed data on the result message's `structured_output` field, so
+ * there's no text extraction or JSON.parse on this path.
  */
-async function runJsonQuery({ systemPrompt, userPrompt, model }) {
+async function runJsonQuery({ systemPrompt, userPrompt, model, schema }) {
   console.log()
   console.log(styles.dim('─ system prompt ─'))
   console.log(styles.dim(systemPrompt))
@@ -2129,24 +3232,28 @@ async function runJsonQuery({ systemPrompt, userPrompt, model }) {
   console.log()
 
   const heartbeat = setInterval(() => process.stdout.write(styles.dim('.')), 1000)
-  let text = ''
+  let structured
   try {
     for await (const message of query({
       prompt: userPrompt,
       options: {
         systemPrompt,
         allowedTools: [],
+        outputFormat: { type: 'json_schema', schema },
         ...(model ? { model } : {}),
       },
     })) {
-      if (message.type === 'assistant' && message.message?.content) {
-        for (const block of message.message.content) {
-          if (block.type === 'text' && block.text) text += block.text
+      if (message.type === 'result') {
+        if (message.subtype === 'error_max_structured_output_retries') {
+          throw new Error(
+            'Claude could not produce output matching the schema after retries. ' +
+            'Likely hit the model\'s output limit — try --model sonnet.',
+          )
         }
-      } else if (message.type === 'result') {
         if (message.subtype && message.subtype !== 'success') {
           throw new Error(`Claude failed: ${message.subtype}${message.error?.message ? ' — ' + message.error.message : ''}`)
         }
+        structured = message.structured_output
         break
       }
     }
@@ -2155,18 +3262,10 @@ async function runJsonQuery({ systemPrompt, userPrompt, model }) {
     process.stdout.write('\n')
   }
 
-  const stripped = stripCodeFences(text)
-
-  try {
-    return JSON.parse(stripped)
-  } catch (e) {
-    throw new Error(
-      `Claude returned invalid JSON: ${e.message}\n` +
-        `Output length: ${stripped.length} chars. Likely hit the model's output limit — try --model sonnet.\n\n` +
-        `First 500 chars:\n${stripped.slice(0, 500)}\n\n` +
-        `Last 500 chars:\n${stripped.slice(-500)}`,
-    )
+  if (!structured || typeof structured !== 'object') {
+    throw new Error('Claude returned no structured output')
   }
+  return structured
 }
 
 /**
@@ -2181,30 +3280,593 @@ async function runJsonQuery({ systemPrompt, userPrompt, model }) {
  *
  * Returns deduped URLs in probe order.
  */
-function buildLlmsCandidates(sourceUrl) {
-  const out = []
-  const seen = new Set()
-  const add = (url) => {
-    if (!seen.has(url)) {
-      seen.add(url)
-      out.push(url)
-    }
-  }
-
-  const origin = sourceUrl.origin
-  const segs = sourceUrl.pathname.split('/').filter(Boolean)
-  for (let i = segs.length; i >= 0; i--) {
-    const prefix = segs.slice(0, i).join('/')
-    add(`${origin}${prefix ? '/' + prefix : ''}/llms.txt`)
+// URL patterns we strip from llms.txt items — they show up alongside real
+// pages but shouldn't be staged as docs:
+//   - .yaml specs, package.json-ish assets, sitemaps, etc. (machine-readable)
+//   - llms.txt, llms-full.txt, llms-ctx.txt, … (other llms.txt variants
+//     that point back at the same index/dump, not real pages)
+//
+// `.json` is deliberately NOT in this list: those links are captured up-front
+// by extractOasJsonUrlsFromParsed (they may be OpenAPI specs we want to
+// download into oas/ and upload separately) and removed from the parsed tree
+// before this drop runs, so a `.json` item never reaches here.
+const ASSET_EXT_RE = /\.(ya?ml|xml|toml)$/i
+const LLMS_TXT_RE = /(?:^|\/)llms[^/]*\.txt$/i
+function isAssetOrMetaUrl(url) {
+  try {
+    const pn = new URL(url).pathname
+    return ASSET_EXT_RE.test(pn) || LLMS_TXT_RE.test(pn)
+  } catch {
+    return false
   }
-  return out
+}
+
+// A llms.txt item whose URL points at a `.json` file. These are pulled out of
+// the parsed tree before organization (so stub generation never sees them) and
+// downloaded into the staged `oas/` dir; the runner validates each as OpenAPI
+// and uploads the valid ones. NOTE: only `.json` is captured — `.yaml`/`.yml`
+// specs are intentionally left to the asset drop above.
+function isOasJsonUrl(url) {
+  try {
+    return /\.json$/i.test(new URL(url).pathname)
+  } catch {
+    return false
+  }
+}
+
+// Immediate-child segments of root that signal a docs subtree on a
+// non-docs hostname. We deliberately keep the set tiny — over-matching
+// here (e.g. including `api`, `learn`) would over-filter sites whose
+// "docs" subtree lives at the apex.
+const DOCS_LIKE_ROOT_SEGMENTS = new Set(['docs', 'doc', 'documentation', 'document'])
+
+/**
+ * Narrow a merged llms.txt result to a docs subtree when a root-level
+ * llms.txt pulled non-doc URLs (marketing, pricing, blog, etc.) into the
+ * union.
+ *
+ * Sites that publish a root llms.txt alongside per-product subtree files
+ * often mix marketing/pricing/blog links into the root file. The BFS merge
+ * happily unions all of it, and without this pass we'd stage stubs for
+ * those pages too.
+ *
+ * elevenlabs.io is the canonical example: its root `elevenlabs.io/llms.txt`
+ * lists marketing pages alongside docs, and per-product files live at
+ * `elevenlabs.io/docs/<product>/llms.txt`. After the BFS we have docs URLs
+ * + pricing/blog URLs in the same merged result. With this pass we keep
+ * only URLs under `/docs/`.
+ *
+ * Conditions (all must hold), chosen so we only narrow when the risk is
+ * real and there's a clear scope to narrow to:
+ *   - Hostname doesn't itself contain "docs" — `docs.example.com` is
+ *     already a docs-only host; nothing to narrow.
+ *   - A root-level llms.txt was actually probed and hit (valid OR
+ *     invalid). Either way it's what dragged in the wider URL space via
+ *     the BFS expansion, so it's the trigger we care about.
+ *   - Multiple llms.txt files contributed to the merge. A single-file
+ *     merge is by definition scoped to that one file's intent.
+ *   - The merged URL set has a docs-shaped first-segment child of root
+ *     (`/docs/`, `/documentation/`, …). That child is our scope target.
+ *
+ * Mutates `llms.parsed.sections` in place. Returns `{ segment, kept,
+ * dropped }` when narrowing happened, or `null` when conditions weren't
+ * met.
+ */
+function narrowToDocsSubtreeIfNeeded(llms, sourceUrl, hits) {
+  if (!llms || llms.sourceFiles.length < 2) return null
+  if (sourceUrl.hostname.toLowerCase().includes('docs')) return null
+  const rootHit = hits.some((h) => h.path === '/' || h.path === '')
+  if (!rootHit) return null
+
+  // Identify docs-shaped first segments of root that actually appear in
+  // the merged URLs. Walking just the first path segment matches the
+  // user's spec ("we only look at immediate children in this case").
+  const firstSegs = new Set()
+  for (const section of llms.parsed.sections) {
+    for (const item of section.items) {
+      try {
+        const seg = new URL(item.url).pathname.split('/').filter(Boolean)[0]
+        if (seg) firstSegs.add(seg.toLowerCase())
+      } catch {
+        // ignore unparseable URLs
+      }
+    }
+  }
+  let docsSegment = null
+  for (const seg of firstSegs) {
+    if (DOCS_LIKE_ROOT_SEGMENTS.has(seg)) {
+      docsSegment = seg
+      break
+    }
+  }
+  if (!docsSegment) return null
+
+  const prefix = '/' + docsSegment
+  let kept = 0
+  let dropped = 0
+  const keptSections = []
+  for (const section of llms.parsed.sections) {
+    const items = []
+    for (const item of section.items) {
+      try {
+        const pn = new URL(item.url).pathname.toLowerCase()
+        if (pn === prefix || pn.startsWith(prefix + '/')) {
+          items.push(item)
+          kept++
+        } else {
+          dropped++
+        }
+      } catch {
+        dropped++
+      }
+    }
+    if (items.length > 0) {
+      section.items = items
+      keptSections.push(section)
+    }
+  }
+  llms.parsed.sections = keptSections
+  return { segment: docsSegment, kept, dropped }
+}
+
+/**
+ * Pull `.json` items out of a parsed llms.txt structure IN PLACE, returning the
+ * removed items as `[{ url, text }]`. Sections emptied by the removal are
+ * pruned too — same shape as dropAssetItemsFromParsed. Run BEFORE
+ * dropAssetItemsFromParsed / knownUrls / organize so the captured specs never
+ * enter the stub tree (skeleton has zero visibility of OAS files). The caller
+ * downloads the returned URLs into the staged `oas/` dir.
+ */
+function extractOasJsonUrlsFromParsed(parsed) {
+  const captured = []
+  const keptSections = []
+  for (const section of parsed.sections) {
+    const keptItems = []
+    for (const item of section.items) {
+      if (isOasJsonUrl(item.url)) {
+        captured.push({ url: item.url, text: item.text || null })
+        continue
+      }
+      keptItems.push(item)
+    }
+    if (keptItems.length > 0) {
+      section.items = keptItems
+      keptSections.push(section)
+    }
+  }
+  parsed.sections = keptSections
+  return captured
+}
+
+/**
+ * Drop asset/meta items in place from a parsed llms.txt structure. Sections
+ * that end up empty after the drop are removed too. Returns the count of
+ * items dropped.
+ */
+function dropAssetItemsFromParsed(parsed) {
+  let dropped = 0
+  const keptSections = []
+  for (const section of parsed.sections) {
+    const keptItems = []
+    for (const item of section.items) {
+      if (isAssetOrMetaUrl(item.url)) {
+        dropped++
+        continue
+      }
+      keptItems.push(item)
+    }
+    if (keptItems.length > 0) {
+      section.items = keptItems
+      keptSections.push(section)
+    }
+  }
+  parsed.sections = keptSections
+  return dropped
+}
+
+
+/**
+ * Build the ordered list of well-known docs locations to probe. For each route
+ * we emit a subdomain candidate (stripping a leading `www.` first) and, unless
+ * the route is subdomain-only, a path candidate. Base URLs end in `/` so
+ * `${url.href}llms.txt` is well-formed.
+ */
+function buildWellKnownDocRoutes(sourceUrl) {
+  const out = []
+  const seen = new Set()
+  const add = (kind, href) => {
+    if (seen.has(href)) return
+    seen.add(href)
+    out.push({ kind, url: new URL(href) })
+  }
+
+  const apexHost = sourceUrl.hostname.replace(/^www\./i, '')
+  for (const route of WELL_KNOWN_DOC_ROUTES) {
+    const subHost = `${route}.${apexHost}`
+    if (subHost !== sourceUrl.hostname) add('subdomain', `${sourceUrl.protocol}//${subHost}/`)
+    if (!SUBDOMAIN_ONLY_DOC_ROUTES.has(route)) add('path', `${sourceUrl.origin}/${route}/`)
+  }
+  return out
+}
+
+/**
+ * Follow redirects on the source URL itself and return the final URL when the
+ * site has moved (different origin or path), or null to keep the original.
+ * Trailing-slash and query/hash-only redirects don't count as a move.
+ */
+async function resolveRedirectedSourceUrl(sourceUrl) {
+  try {
+    const res = await fetch(sourceUrl.href, { redirect: 'follow', headers: { 'User-Agent': 'readme-cli-import' } })
+    if (!res.ok || !res.url) return null
+    const final = new URL(res.url)
+    final.search = ''
+    final.hash = ''
+    const norm = (u) => u.origin + u.pathname.replace(/\/+$/, '')
+    if (norm(final) === norm(sourceUrl)) return null
+    return final
+  } catch {
+    return null
+  }
+}
+
+/**
+ * For a bare homepage, probe well-known docs routes and return the best base to
+ * adopt, or null to leave the source untouched. A route serving a usable
+ * llms.txt always beats one that merely resolves. Returns `{ url, kind, hasLlms }`.
+ */
+async function resolveDocsBaseUrl(sourceUrl) {
+  if (sourceUrl.pathname && sourceUrl.pathname !== '/') return null
+  if (WELL_KNOWN_DOC_ROUTES.includes(sourceUrl.hostname.split('.')[0].toLowerCase())) return null
+
+  const candidates = buildWellKnownDocRoutes(sourceUrl)
+  if (candidates.length === 0) return null
+
+  styles.info(`Probing ${styles.bold(String(candidates.length))} well-known docs routes from ${styles.bold(sourceUrl.toString())}...`)
+
+  const llmsHits = await Promise.all(candidates.map(async (c) => ({ c, res: await fetchLlmsTxt(`${c.url.href}llms.txt`) })))
+  for (const { c, res } of llmsHits) {
+    const url = `${c.url.href}llms.txt`
+    if (res.ok && res.usable && (res.stats?.linkItems ?? 0) > 0) {
+      if (inCandidateScope(c, res.finalUrl)) {
+        styles.info(styles.dim(`  ${url} → valid llms.txt (${res.stats.linkItems} links)`))
+        return { url: c.url, kind: c.kind, hasLlms: true }
+      }
+      styles.info(styles.dim(`  ${url} → llms.txt redirects to ${new URL(res.finalUrl).host} (alias) — skipping`))
+    } else if (res.ok) {
+      styles.info(styles.dim(`  ${url} → no usable llms.txt`))
+    } else {
+      styles.info(styles.dim(`  ${url} → ${res.status ? `HTTP ${res.status}` : res.error || 'no llms.txt'}`))
+    }
+  }
+
+  const existence = await Promise.all(
+    llmsHits.map(async ({ c, res }) => {
+      if (res.error && c.kind === 'subdomain') return { c, exists: false }
+      return { c, exists: await docsRouteResolves(c) }
+    }),
+  )
+  for (const { c, exists } of existence) {
+    if (exists) {
+      styles.info(styles.dim(`  ${c.url.href} → resolves (no llms.txt)`))
+      return { url: c.url, kind: c.kind, hasLlms: false }
+    }
+  }
+
+  styles.info(styles.dim(`  no well-known docs route matched — keeping ${sourceUrl.toString()}`))
+  return null
 }
 
 /**
- * Best-effort fetch of a site's /llms.txt. Returns { ok, status, error, parsed }
- * where parsed is { title, sections: [{ title, items: [{ text, url, description }] }] }.
+ * Is `finalUrl` (where a fetch landed after following redirects) still within
+ * the candidate's scope? Subdomain candidates must keep their host; path
+ * candidates must keep their path prefix. A cross-scope redirect means the
+ * route is just an alias to somewhere else (usually the marketing apex).
  */
-async function fetchLlmsTxt(llmsUrl) {
+function inCandidateScope(candidate, finalUrl) {
+  let final
+  try {
+    final = new URL(finalUrl || candidate.url.href)
+  } catch {
+    return false
+  }
+  if (candidate.kind === 'subdomain') return final.hostname === candidate.url.hostname
+  return final.pathname.toLowerCase().startsWith(candidate.url.pathname.toLowerCase())
+}
+
+/**
+ * Does a well-known docs route exist and stay in scope? Follows redirects and
+ * rejects anything that lands outside the candidate (a `docs.` subdomain that
+ * bounces to marketing, a `/docs/` that 302s home). DNS failures → false.
+ */
+async function docsRouteResolves(candidate) {
+  try {
+    const res = await fetch(candidate.url.href, { redirect: 'follow', headers: { 'User-Agent': 'readme-cli-import' } })
+    return res.ok && inCandidateScope(candidate, res.url)
+  } catch {
+    return false
+  }
+}
+
+// Hard cap on total probes during BFS discovery — protects against
+// pathological sites where every URL returns a "200 + a few link rows"
+// response and the frontier would otherwise blow up.
+const LLMS_PROBE_CAP = 30
+// Per-node cap on child segments to expand. Root-level llms.txt files can
+// list hundreds of URLs; without this a single root hit could schedule
+// dozens of unrelated probes (`/blog`, `/pricing`, …).
+const LLMS_CHILDREN_PER_NODE = 12
+// Budget for the explicit-link phase of discovery (phase 2 in discoverLlmsTxt).
+// Separate from LLMS_PROBE_CAP so following authored `llms.txt` links — a
+// strong, deliberate signal, e.g. docs.snowflake.com's root index listing ~40
+// per-section files — isn't starved by the speculative slug-walk's budget.
+const EXPLICIT_LLMS_PROBE_CAP = 200
+
+// Matches ONLY the canonical index filename, not sibling dumps like
+// `llms-full.txt` (concatenated page content, not a link index) — those would
+// add no link rows and just waste a fetch.
+const LLMS_INDEX_RE = /\/llms\.txt$/i
+
+/**
+ * Convert a path like `/docs/quickstart` into its llms.txt URL.
+ */
+function pathToLlmsUrl(origin, path) {
+  if (!path || path === '/') return `${origin}/llms.txt`
+  return `${origin}${path}/llms.txt`
+}
+
+function llmsParseOptionsForUrl(llmsUrl) {
+  return { rootRelativeResolution: llmsPathFromUrl(llmsUrl) === '/' ? 'origin' : 'llms-dir' }
+}
+
+/**
+ * Derive the path a fetched llms.txt "lives at" (its pathname minus the
+ * trailing `/llms.txt`), so explicit-link hits carry the same `path` shape
+ * the slug-walk hits do — mergeValidHits sorts by it (deepest wins) and
+ * narrowToDocsSubtreeIfNeeded reads it.
+ */
+function llmsPathFromUrl(llmsUrl) {
+  const pn = new URL(llmsUrl).pathname.replace(/\/llms\.txt$/i, '')
+  return pn || '/'
+}
+
+/**
+ * Pull explicit child-index links out of a parsed llms.txt — item rows whose
+ * URL literally points at another `llms.txt` (e.g. Snowflake's root file
+ * listing `…/data-integration/llms.txt`). Restricted to the same origin as the
+ * source: cross-origin llms.txt would drag in unrelated third-party docs, and
+ * the slug-walk's path math assumes one origin. Returns deduped URL strings.
+ */
+function extractNestedLlmsUrls(parsed, originScope) {
+  const out = new Set()
+  for (const section of parsed.sections) {
+    for (const item of section.items) {
+      try {
+        const u = new URL(item.url)
+        if (u.origin === originScope && LLMS_INDEX_RE.test(u.pathname)) out.add(u.toString())
+      } catch {
+        // ignore unparseable URLs
+      }
+    }
+  }
+  return [...out]
+}
+
+/**
+ * Given a current path + a parsed llms.txt, return the next paths to probe:
+ * the parent directory (UP) and distinct first-segment-children extracted
+ * from the URL list (DOWN). Children are scoped to URLs whose pathname
+ * actually starts with the current path's prefix — out-of-scope URLs (e.g.
+ * a `/docs/llms.txt` listing `/blog/foo`) are ignored.
+ */
+function expandLlmsFrontier(currentPath, parsed) {
+  const next = []
+  const prefix = currentPath === '/' || currentPath === '' ? '/' : currentPath + '/'
+
+  if (currentPath && currentPath !== '/') {
+    const parent = currentPath.replace(/\/[^/]+$/, '') || '/'
+    next.push(parent)
+  }
+
+  const childSegs = new Set()
+  for (const section of parsed.sections) {
+    for (const item of section.items) {
+      try {
+        const u = new URL(item.url)
+        const pn = u.pathname.toLowerCase()
+        if (!pn.startsWith(prefix.toLowerCase())) continue
+        const rest = pn.slice(prefix.length).split('/').filter(Boolean)
+        if (rest.length > 0) childSegs.add(rest[0])
+      } catch {
+        // ignore unparseable URLs
+      }
+      if (childSegs.size >= LLMS_CHILDREN_PER_NODE) break
+    }
+    if (childSegs.size >= LLMS_CHILDREN_PER_NODE) break
+  }
+  for (const seg of childSegs) {
+    next.push(currentPath === '/' || !currentPath ? `/${seg}` : `${currentPath}/${seg}`)
+  }
+  return next
+}
+
+/**
+ * BFS-discover all llms.txt files reachable from sourceUrl by walking UP
+ * the path and DOWN into child segments. Invalid hits (file exists but
+ * fails the usability ratio) are kept as "stepping stones" — they still
+ * expand the frontier so we can find their valid siblings/children — but
+ * don't contribute content to the merged result. Returns:
+ *   { hits, misses, skipped }
+ * where `hits` includes both valid (usable: true) and invalid hits.
+ */
+async function discoverLlmsTxt(sourceUrl) {
+  const tried = new Set()
+  const hits = []
+  const misses = []
+  const skipped = []
+
+  const startSegs = sourceUrl.pathname.split('/').filter(Boolean)
+  const initial = []
+  for (let i = startSegs.length; i >= 0; i--) {
+    initial.push('/' + startSegs.slice(0, i).join('/'))
+  }
+
+  let frontier = initial
+  while (frontier.length > 0 && tried.size < LLMS_PROBE_CAP) {
+    const ring = []
+    for (const p of frontier) {
+      const norm = p === '' ? '/' : p
+      if (tried.has(norm)) continue
+      tried.add(norm)
+      ring.push(norm)
+      if (tried.size >= LLMS_PROBE_CAP) break
+    }
+    if (ring.length === 0) break
+
+    const results = await Promise.all(
+      ring.map(async (path) => {
+        const llmsUrl = pathToLlmsUrl(sourceUrl.origin, path)
+        const res = await fetchLlmsTxt(llmsUrl, llmsParseOptionsForUrl(llmsUrl))
+        return { path, llmsUrl, res }
+      }),
+    )
+
+    const next = []
+    for (const { path, llmsUrl, res } of results) {
+      if (!res.ok) {
+        misses.push({ url: llmsUrl, status: res.status, error: res.error })
+        continue
+      }
+      // Treat as a "hit" only if there's at least one link row — that's the
+      // minimum signal that this is an llms.txt-shaped file (and not just a
+      // 200 response serving the site's HTML or a stray text file). Hits
+      // expand the frontier regardless of whether they pass usability.
+      const linkItems = res.stats?.linkItems ?? 0
+      if (linkItems === 0) {
+        misses.push({ url: llmsUrl, status: 'no link items' })
+        continue
+      }
+      hits.push({ ...res, llmsUrl, path })
+      if (!res.usable) skipped.push({ url: llmsUrl, reason: res.reason })
+      for (const np of expandLlmsFrontier(path, res.parsed)) {
+        if (!tried.has(np)) next.push(np)
+      }
+    }
+    frontier = next
+  }
+
+  // Phase 2 — follow EXPLICIT llms.txt links authored inside the files phase 1
+  // discovered, recursively. No slug-guessing here: we only fetch URLs that
+  // literally point at an llms.txt (a root index listing per-section files,
+  // like docs.snowflake.com). Each new file's own explicit links are followed
+  // in turn until none remain. Dedupes against every URL phase 1 touched
+  // (hits + misses) and against itself; phase 1's `tried` set is keyed by path,
+  // so we track full llms.txt URLs separately here and leave phase 1 untouched.
+  const triedUrls = new Set([...hits.map((h) => h.llmsUrl), ...misses.map((m) => m.url)])
+  let explicitFrontier = []
+  for (const hit of hits) {
+    for (const u of extractNestedLlmsUrls(hit.parsed, sourceUrl.origin)) {
+      if (!triedUrls.has(u)) explicitFrontier.push(u)
+    }
+  }
+
+  let explicitProbes = 0
+  while (explicitFrontier.length > 0 && explicitProbes < EXPLICIT_LLMS_PROBE_CAP) {
+    const ring = []
+    for (const u of explicitFrontier) {
+      if (triedUrls.has(u)) continue
+      triedUrls.add(u)
+      ring.push(u)
+      explicitProbes++
+      if (explicitProbes >= EXPLICIT_LLMS_PROBE_CAP) break
+    }
+    if (ring.length === 0) break
+
+    const results = await Promise.all(ring.map(async (llmsUrl) => ({ llmsUrl, res: await fetchLlmsTxt(llmsUrl, llmsParseOptionsForUrl(llmsUrl)) })))
+
+    const next = []
+    for (const { llmsUrl, res } of results) {
+      if (!res.ok) {
+        misses.push({ url: llmsUrl, status: res.status, error: res.error })
+        continue
+      }
+      const linkItems = res.stats?.linkItems ?? 0
+      if (linkItems === 0) {
+        misses.push({ url: llmsUrl, status: 'no link items' })
+        continue
+      }
+      hits.push({ ...res, llmsUrl, path: llmsPathFromUrl(llmsUrl) })
+      if (!res.usable) skipped.push({ url: llmsUrl, reason: res.reason })
+      for (const nested of extractNestedLlmsUrls(res.parsed, sourceUrl.origin)) {
+        if (!triedUrls.has(nested)) next.push(nested)
+      }
+    }
+    explicitFrontier = next
+  }
+
+  return { hits, misses, skipped }
+}
+
+/**
+ * Merge content from all VALID hits into one llms-result-shaped object that
+ * downstream code can consume the same as a single-file result. Pages are
+ * deduped by normalized pathname, with the deepest-path hit winning — the
+ * `/docs/conversational-ai/` file's section assignment beats `/docs/`'s for
+ * any URL under that subtree. Returns null when there are no valid hits.
+ */
+function mergeValidHits(hits) {
+  const valid = hits.filter((h) => h.usable)
+  if (valid.length === 0) return null
+
+  const deepestFirst = [...valid].sort((a, b) => b.path.length - a.path.length)
+  const shallowestFirst = [...valid].sort((a, b) => a.path.length - b.path.length)
+
+  const claimed = new Map()
+  const sections = []
+  for (const hit of deepestFirst) {
+    for (const section of hit.parsed.sections) {
+      const items = []
+      for (const item of section.items) {
+        const key = llmsPageDedupeKey(item.url)
+        const existing = claimed.get(key)
+        if (existing) {
+          if (compareCanonicalUrlPreference(item.url, existing.url) < 0) existing.url = item.url
+          continue
+        }
+        claimed.set(key, item)
+        items.push(item)
+      }
+      if (items.length > 0) sections.push({ title: section.title, items })
+    }
+  }
+
+  // Prefer the root-most file's H1 title — usually the canonical site name.
+  const title = shallowestFirst.find((h) => h.parsed.title)?.parsed.title || null
+
+  const stats = valid.reduce(
+    (acc, h) => ({
+      conforming: acc.conforming + (h.stats?.conforming || 0),
+      nonConforming: acc.nonConforming + (h.stats?.nonConforming || 0),
+      total: acc.total + (h.stats?.total || 0),
+      linkItems: acc.linkItems + (h.stats?.linkItems || 0),
+    }),
+    { conforming: 0, nonConforming: 0, total: 0, linkItems: 0 },
+  )
+  stats.ratio = stats.total === 0 ? 0 : stats.conforming / stats.total
+
+  return {
+    parsed: { title, sections },
+    usable: true,
+    reason: null,
+    stats,
+    llmsUrl: shallowestFirst[0].llmsUrl,
+    sourceFiles: shallowestFirst.map((h) => h.llmsUrl),
+  }
+}
+
+/**
+ * Best-effort fetch of a site's /llms.txt plus a simple structural usability check.
+ */
+async function fetchLlmsTxt(llmsUrl, options = {}) {
   try {
     const res = await fetch(llmsUrl, {
       redirect: 'follow',
@@ -2212,63 +3874,201 @@ async function fetchLlmsTxt(llmsUrl) {
     })
     if (!res.ok) return { ok: false, status: res.status }
     const text = await res.text()
-    return { ok: true, status: res.status, parsed: parseLlmsTxt(text) }
+    const finalUrl = res.url || String(llmsUrl)
+    const analysis = analyzeLlmsTxt(text, finalUrl, finalUrl === String(llmsUrl) ? options : llmsParseOptionsForUrl(finalUrl))
+    return {
+      ok: true,
+      status: res.status,
+      finalUrl: res.url,
+      parsed: analysis.parsed,
+      usable: analysis.usable,
+      reason: analysis.reason,
+      stats: analysis.stats,
+    }
   } catch (e) {
     return { ok: false, error: e.message }
   }
 }
 
+// Path segments that mark a docs scope. Used to find the right "base" for
+// filtering sitemap URLs: walking up from the source URL, the first segment
+// that matches becomes the scope prefix. Hits should stay narrow — `blog`
+// or `pricing` would over-include marketing pages.
+const WELL_KNOWN_DOC_SEGMENTS = new Set([
+  'docs',
+  'doc',
+  'documentation',
+  'guides',
+  'guide',
+  'references',
+  'reference',
+  'api',
+  'apis',
+  'api-reference',
+  'api-docs',
+  'learn',
+  'tutorials',
+  'tutorial',
+  'developers',
+  'developer',
+])
+
 /**
- * Parse the llms.txt format. `##` headings become sections;
- * `- [text](url): description` bullets become items. Items before any `##`
- * land in an implicit "Resources" section.
+ * Decide which path prefix to use when filtering sitemap URLs to "the docs
+ * portion of the site." Walk up the source URL's path looking for the first
+ * well-known docs segment (`/docs/`, `/guides/`, `/api-reference/`, …) and
+ * use the prefix up to and including it. Returns null when nothing matches —
+ * in that case we accept any URL on the same origin (no path filter).
  */
-function parseLlmsTxt(body) {
-  const lines = body.split(/\r?\n/)
-  let title = null
-  const sections = []
-  let current = null
-
-  const itemRe = /^\s*-\s*\[([^\]]*)\]\((https?:\/\/[^)\s]+)\)(?:\s*[:—–-]\s*(.+))?/
+function deriveSitemapScope(sourceUrl) {
+  const segs = sourceUrl.pathname.split('/').filter(Boolean)
+  for (let i = segs.length - 1; i >= 0; i--) {
+    if (WELL_KNOWN_DOC_SEGMENTS.has(segs[i].toLowerCase())) {
+      return '/' + segs.slice(0, i + 1).join('/') + '/'
+    }
+  }
+  return null
+}
 
-  for (const line of lines) {
-    const h1 = line.match(/^#\s+(.+)$/)
-    if (h1 && !title) {
-      title = h1[1].trim()
-      continue
+/**
+ * Mirror buildLlmsCandidates but for sitemap.xml: walk up the source path
+ * appending `/sitemap.xml` at each level, ending at origin root.
+ */
+function buildSitemapCandidates(sourceUrl) {
+  const out = []
+  const seen = new Set()
+  const add = (url) => {
+    if (!seen.has(url)) {
+      seen.add(url)
+      out.push(url)
     }
+  }
+  const origin = sourceUrl.origin
+  const segs = sourceUrl.pathname.split('/').filter(Boolean)
+  for (let i = segs.length; i >= 0; i--) {
+    const prefix = segs.slice(0, i).join('/')
+    add(`${origin}${prefix ? '/' + prefix : ''}/sitemap.xml`)
+  }
+  return out
+}
 
-    const h2 = line.match(/^##\s+(.+)$/)
-    if (h2) {
-      current = { title: h2[1].trim(), items: [] }
-      sections.push(current)
-      continue
+/**
+ * Best-effort fetch of a sitemap. Handles both `<urlset>` (a flat list of
+ * page URLs) and `<sitemapindex>` (which references child sitemaps — we
+ * recurse one level, capped at 10 children to avoid runaway fetches).
+ *
+ * `scopePrefix` is a path string ending in `/`, or null. When set, only URLs
+ * whose pathname starts with the prefix and share the sitemap's origin are
+ * returned. Returns { ok, urls } on success.
+ */
+async function fetchSitemap(sitemapUrl, scopePrefix, depth = 0) {
+  try {
+    const res = await fetch(sitemapUrl, {
+      redirect: 'follow',
+      headers: { 'User-Agent': 'readme-cli-import' },
+    })
+    if (!res.ok) return { ok: false, status: res.status }
+    const text = await res.text()
+    const parsed = parseSitemapXml(text)
+    if (!parsed) return { ok: false, error: 'not a sitemap' }
+
+    if (parsed.kind === 'urlset') {
+      const urls = parsed.locs.filter((u) => urlInScope(u, sitemapUrl, scopePrefix))
+      return { ok: true, urls }
     }
 
-    const item = line.match(itemRe)
-    if (item) {
-      if (!current) {
-        current = { title: 'Resources', items: [] }
-        sections.push(current)
-      }
-      current.items.push({
-        text: item[1].trim(),
-        url: item[2].replace(/[.,;]+$/, ''),
-        description: item[3] ? item[3].trim() : null,
-      })
+    if (depth >= 1) return { ok: true, urls: [] }
+    const children = parsed.locs.slice(0, 10)
+    const all = []
+    for (const childUrl of children) {
+      const child = await fetchSitemap(childUrl, scopePrefix, depth + 1)
+      if (child.ok) all.push(...child.urls)
     }
+    return { ok: true, urls: all }
+  } catch (e) {
+    return { ok: false, error: e.message }
   }
+}
 
-  return { title, sections }
+/**
+ * Parse a sitemap with a regex — the format is small and well-defined enough
+ * that a dedicated XML parser would be overkill. Returns
+ * { kind: 'urlset' | 'sitemapindex', locs: string[] } or null when the body
+ * doesn't look like a sitemap (e.g. server returned a 200 HTML 404 page).
+ */
+function parseSitemapXml(body) {
+  const isIndex = /<sitemapindex[\s>]/i.test(body)
+  const isUrlset = /<urlset[\s>]/i.test(body)
+  if (!isIndex && !isUrlset) return null
+  const locs = []
+  const re = /<loc>\s*([^<]+?)\s*<\/loc>/gi
+  let m
+  while ((m = re.exec(body)) !== null) {
+    const url = decodeXmlEntities(m[1].trim())
+    if (url) locs.push(url)
+  }
+  return { kind: isIndex ? 'sitemapindex' : 'urlset', locs }
+}
+
+function decodeXmlEntities(s) {
+  return s
+    .replace(/&amp;/g, '&')
+    .replace(/&lt;/g, '<')
+    .replace(/&gt;/g, '>')
+    .replace(/&quot;/g, '"')
+    .replace(/&apos;/g, "'")
+}
+
+function urlInScope(urlStr, sitemapUrl, scopePrefix) {
+  let u
+  try {
+    u = new URL(urlStr)
+  } catch {
+    return false
+  }
+  let sm
+  try {
+    sm = new URL(sitemapUrl)
+  } catch {
+    return false
+  }
+  if (u.origin !== sm.origin) return false
+  if (scopePrefix && !u.pathname.startsWith(scopePrefix)) return false
+  return true
 }
 
 /**
- * Write the organized hierarchy to disk as git-format markdown stubs — just
- * frontmatter, no body yet. docs/ pages go under docs/<Category>/<slug>.md;
- * reference/recipes/custom_pages/custom_blocks get their own top-level dir
- * without a category subfolder (the git-format schema doesn't nest them).
- * Writes _order.yaml per directory so sidebar order matches input order.
+ * Convert a flat sitemap URL list into the `{title, url, description}` shape
+ * the rest of the pipeline expects for knownUrls. Title is derived from the
+ * last non-empty path segment, with file extensions stripped. Index-style
+ * URLs that resolve to the scope root itself are dropped.
  */
+function sitemapUrlsToKnownUrls(urls) {
+  const out = []
+  const seen = new Set()
+  for (const url of urls) {
+    let pathname
+    try {
+      pathname = new URL(url).pathname
+    } catch {
+      continue
+    }
+    const norm = normalizePath(url)
+    if (seen.has(norm)) continue
+    seen.add(norm)
+    const segs = pathname.split('/').filter(Boolean)
+    const last = segs[segs.length - 1] || ''
+    const slug = last.replace(/\.[a-z0-9]+$/i, '')
+    if (!slug) continue
+    out.push({
+      title: titleCase(slug),
+      url,
+      description: null,
+    })
+  }
+  return out
+}
+
 /**
  * Recursively print the page tree. Sub-pages are indented under their parent
  * with no leading bullet character, to show them as children of the parent.
@@ -2285,13 +4085,103 @@ function printPagesTree(pages, indentLevel) {
   }
 }
 
+/**
+ * Collapse redundant single-folder layers in a category/page subtree, in place.
+ *
+ * When a container (category or page) has exactly one child and that child is a
+ * pure folder node — no `x-import-url` gitto/readme has an issue with unique slug
+ * generation.
+ * This is a temporary workaround.
+ * TODO: Remove after fixing on the Gitto/ReadMe side.
+ */
+function collapseRedundantLayers(container) {
+  for (const child of container.pages || []) collapseRedundantLayers(child)
+  while ((container.pages || []).length === 1) {
+    const only = container.pages[0]
+    if (only.url) break // real page (gets x-import), just has children
+    if (!only.pages || only.pages.length === 0) break // nothing to lift up
+    container.pages = only.pages
+  }
+}
+
+// Some sites serve regular guides under an /api/docs/* path (e.g. /api/docs/guides/*,
+// /api/docs/pricing). Treat a URL as real docs when it has both `api` and `docs` segments 
+// and the segment right after `docs` is not itself a reference/api slug.
+function isDocsUnderApiPath(url) {
+  try {
+    const segs = new URL(url).pathname.split('/').filter(Boolean).map((s) => s.toLowerCase())
+    const docsIdx = segs.findIndex((s) => s === 'docs' || s === 'doc')
+    if (!segs.includes('api') || docsIdx === -1) return false
+    const afterDocs = segs[docsIdx + 1]
+    return !afterDocs || !/^(api|api[-_]?reference|reference)$/.test(afterDocs)
+  } catch {
+    return false
+  }
+}
+
+// Prune a page tree, keeping only URL pages whose url passes `predicate` plus
+// empty-parent nodes that still have surviving descendants. Nesting preserved.
+function filterUrlPagesTree(pages, predicate) {
+  const out = []
+  for (const p of pages || []) {
+    const keptKids = filterUrlPagesTree(p.pages, predicate)
+    if (p.url) {
+      if (predicate(p.url)) out.push({ ...p, pages: keptKids })
+    } else if (keptKids.length > 0) {
+      out.push({ ...p, pages: keptKids })
+    }
+  }
+  return out
+}
+
+/**
+ * Write the organized hierarchy to disk as git-format markdown stubs — just
+ * frontmatter, no body yet. docs/ pages go under docs/<Category>/<slug>.md;
+ * reference/recipes/custom_pages/custom_blocks get their own top-level dir
+ * without a category subfolder (the git-format schema doesn't nest them).
+ * Writes _order.yaml per directory so sidebar order matches input order.
+ */
 function stageOrganized(organized, stagingDir, opts = {}) {
   const pickIcon = makeIconPicker()
-  const usedSlugs = new Set() // cross-dir: duplicates validator is global
   const byDir = new Map()
   const subDirsByTopDir = new Map()
   const counts = { fileCount: 0, skippedApiRef: 0 }
   const skipApiReference = !!opts.skipApiReference
+  const llmsPaths = opts.llmsPaths || new Set()
+
+  const eligibleCategories = []
+  for (const cat of organized.categories || []) {
+    if (skipApiReference && routeCategory(cat.title).topDir === 'reference') {
+      // A reference-routed cluster can mix genuine OAS pages (/api/reference/*)
+      // with real guides served under /api/docs/*. Rescue the docs subtree —
+      // retitled so it routes to docs/ instead of the dropped reference/
+      const rescuedPages = filterUrlPagesTree(cat.pages || [], isDocsUnderApiPath)
+      const totalDeep = collectUrlPagesDeep(cat.pages || []).length
+      const rescuedDeep = collectUrlPagesDeep(rescuedPages).length
+      if (rescuedDeep > 0) {
+        eligibleCategories.push({ ...cat, title: 'API Docs', pages: rescuedPages })
+      }
+      counts.skippedApiRef += totalDeep - rescuedDeep
+      continue
+    }
+    eligibleCategories.push(cat)
+  }
+
+  // TODO: Remove after readme/gitto handles better
+  for (const cat of eligibleCategories) collapseRedundantLayers(cat)
+
+  // Slug names must be unique
+  const slugFor = ensureUniqueSlugs(eligibleCategories)
+
+  const labelFor = (p) => {
+    if (p.url) return p.url
+    if (p._emptyParent && p._virtualPathSegs) return `(empty) /${p._virtualPathSegs.join('/')}`
+    return '(group-only)'
+  }
+  const urlWidth = Math.max(...[...slugFor.keys()].map((p) => labelFor(p).length))
+  for (const [page, slug] of slugFor) {
+    console.log(`${labelFor(page).padEnd(urlWidth)} → ${slug}`)
+  }
 
   /**
    * Write a page (and its descendants) into `dir`. A page with children gets
@@ -2300,25 +4190,38 @@ function stageOrganized(organized, stagingDir, opts = {}) {
    * This matches git-format's on-disk convention for nested sidebars.
    */
   function writePage(page, dir, topDir, isSubPage = false) {
-    const slug = resolveSlug(deriveSlug(page.url, page.title), usedSlugs)
-    usedSlugs.add(slug)
+    const slug = slugFor.get(page)
 
     // Group-only nodes (e.g. a resource sub-group within API Reference) have
     // no backing page on the source site — they're pure sidebar containers.
     // Skip the stub write but still recurse so their children land in the
     // right subdirectory.
-    const isGroupOnly = !page.url
+    const isEmptyParent = !!page._emptyParent
+    const isGroupOnly = !page.url && !isEmptyParent
 
     if (!isGroupOnly) {
       const relFilePath = `${dir}/${slug}.md`
-      // Sub-pages don't get icons per design decision.
-      const frontmatter = buildFrontmatter(topDir, page, slug, pickIcon, { skipIcon: isSubPage })
-      // x-import points at the source URL for this stub. The content-import
-      // step reads it to fetch the page body. x-prefixed custom field is the
-      // git-format convention for metadata the schema doesn't know about.
-      frontmatter['x-import'] = toBrowsableUrl(page.url)
-      // hide pages that need import
-      frontmatter.hidden = true
+      let frontmatter
+      if (isEmptyParent) {
+        // Synthetic parent page: covers a missing URL segment so descendants
+        // nest at the right depth. No source URL to import. Reuse the same
+        // pickIcon heuristic leaf pages use so the parent gets a topical
+        // icon — the sidebar already renders an expand chevron, so a
+        // hardcoded `fa-folder` would just look like a placeholder.
+        frontmatter = { title: page.title, icon: formatIconClass(pickIcon(slug, page.title)) }
+      } else {
+        // Sub-pages don't get icons per design decision.
+        frontmatter = buildFrontmatter(topDir, page, slug, pickIcon, { skipIcon: isSubPage })
+        // x-import points at the source URL for this stub. The content-import
+        // step reads it to fetch the page body. x-prefixed custom field is the
+        // git-format convention for metadata the schema doesn't know about.
+        frontmatter['x-import'] = toBrowsableUrl(page.url)
+        // Pages sourced from llms.txt serve raw markdown at `<url>.md`; flag them
+        // so the runner knows it can append `.md` when fetching the body.
+        if (llmsPaths.has(normalizePath(page.url))) frontmatter['x-from-llms'] = 'true'
+        // hide pages that need import
+        frontmatter.hidden = 'true'
+      }
 
       const absPath = path.join(stagingDir, relFilePath)
       fs.mkdirSync(path.dirname(absPath), { recursive: true })
@@ -2336,12 +4239,8 @@ function stageOrganized(organized, stagingDir, opts = {}) {
     }
   }
 
-  for (const cat of organized.categories || []) {
+  for (const cat of eligibleCategories) {
     const { topDir, subDir } = routeCategory(cat.title)
-    if (skipApiReference && topDir === 'reference') {
-      counts.skippedApiRef += countPagesDeep(cat.pages || [])
-      continue
-    }
     const dir = subDir ? `${topDir}/${subDir}` : topDir
     if (subDir) {
       if (!subDirsByTopDir.has(topDir)) subDirsByTopDir.set(topDir, [])
@@ -2377,6 +4276,81 @@ function countPagesDeep(pages) {
   return n
 }
 
+/**
+ * Post-staging relocation for `--separate-changelog`: move the staged
+ * `docs/Changelog/` category up to a top-level `changelog/` directory.
+ *
+ * Done as a filesystem move *after* staging (rather than routing there
+ * directly) so the staged tree is a valid git-format layout up to this point,
+ * and the one schema-divergent step lives in a single, clearly-named place.
+ * Note `changelog/` is not part of the git-format schema — it won't lint or
+ * sync to ReadMe until git-format adds first-class changelog support.
+ *
+ * Returns the number of changelog pages moved (0 if there were none).
+ */
+function relocateChangelogDir(stagingDir) {
+  const srcDir = path.join(stagingDir, 'docs', 'Changelog')
+  if (!fs.existsSync(srcDir) || !fs.statSync(srcDir).isDirectory()) return 0
+
+  const dstDir = path.join(stagingDir, 'changelog')
+  fs.renameSync(srcDir, dstDir)
+
+  // Collect every real x-import page from the (possibly nested) tree.
+  // Synthetic group-only stubs (no x-import — they exist only to render an
+  // empty parent in the sidebar) are dropped: the flat layout has no concept
+  // of containers, so they'd just be dead frontmatter.
+  const pages = []
+  const walk = (dir, ancestors) => {
+    for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
+      const full = path.join(dir, entry.name)
+      if (entry.isDirectory()) {
+        walk(full, [...ancestors, entry.name])
+        continue
+      }
+      if (!entry.name.endsWith('.md')) continue
+      const parsed = matter(fs.readFileSync(full, 'utf-8'))
+      if (!parsed.data || !parsed.data['x-import']) continue
+      const slug = entry.name.replace(/\.md$/, '')
+      pages.push({ ancestors, slug, content: parsed.content, data: parsed.data })
+    }
+  }
+  walk(dstDir, [])
+
+  // Wipe the nested tree and rewrite it flat. Filename concatenates the
+  // ancestor folder slugs with the leaf slug so siblings share a prefix —
+  // alphabetical order then naturally matches authored reading order, and
+  // names are unique across the (formerly nested) tree.
+  fs.rmSync(dstDir, { recursive: true, force: true })
+  fs.mkdirSync(dstDir, { recursive: true })
+
+  const flatSlugs = []
+  for (const p of pages) {
+    const flatSlug = [...p.ancestors, p.slug].join('-')
+    const frontmatter = { ...p.data }
+    delete frontmatter.icon
+    fs.writeFileSync(path.join(dstDir, `${flatSlug}.md`), matter.stringify(p.content, frontmatter))
+    flatSlugs.push(flatSlug)
+  }
+
+  flatSlugs.sort()
+  fs.writeFileSync(path.join(dstDir, '_order.yaml'), flatSlugs.map((s) => `- ${yamlSafeSlug(s)}`).join('\n') + '\n')
+
+  // docs/_order.yaml lists the docs category subfolders — drop the moved one
+  // so the docs sidebar doesn't point at a folder that's no longer there.
+  const docsOrderPath = path.join(stagingDir, 'docs', '_order.yaml')
+  if (fs.existsSync(docsOrderPath)) {
+    const entries = yamlRequire().load(fs.readFileSync(docsOrderPath, 'utf-8'))
+    const filtered = Array.isArray(entries) ? entries.filter((e) => e !== 'Changelog') : []
+    if (filtered.length > 0) {
+      fs.writeFileSync(docsOrderPath, filtered.map((s) => `- ${yamlSafeSlug(s)}`).join('\n') + '\n')
+    } else {
+      fs.rmSync(docsOrderPath)
+    }
+  }
+
+  return pages.length
+}
+
 /**
  * Map a category title to the git-format top-level directory + optional
  * category subdir. docs/ is the only top dir that takes a subfolder.
@@ -2384,6 +4358,9 @@ function countPagesDeep(pages) {
 function routeCategory(title) {
   const t = (title || '').trim()
   if (/^(api[ -]?reference|reference|api|endpoints?)$/i.test(t)) return { topDir: 'reference', subDir: null }
+  // Changelog-ish categories normalize to one docs/Changelog/ folder so the
+  // --separate-changelog relocation has a single, predictable source dir.
+  if (/^(changelog|change[ -]?log|release[ -]?notes?|releases?|what'?s[ -]?new)$/i.test(t)) return { topDir: 'docs', subDir: 'Changelog' }
   if (/^(recipes?|cookbook|tutorials?|how[ -]?tos?)$/i.test(t)) return { topDir: 'recipes', subDir: null }
   if (/^(custom[ -]?pages?|landing( page)?s?)$/i.test(t)) return { topDir: 'custom_pages', subDir: null }
   if (/^(custom[ -]?blocks?|snippets?|reusable( content)?)$/i.test(t)) return { topDir: 'custom_blocks', subDir: null }
@@ -2420,30 +4397,131 @@ function buildFrontmatter(topDir, page, slug, pickIcon, opts = {}) {
 }
 
 /**
- * Turn a URL's trailing segment into a filename-safe slug. Strips `.md`, kebabs
- * the result, drops any leading numeric prefix (common in imports).
- */
-function deriveSlug(url, fallbackTitle) {
-  let raw = ''
-  try {
-    const segs = new URL(url).pathname.split('/').filter(Boolean)
-    raw = segs[segs.length - 1] || ''
-  } catch {}
-  raw = raw.replace(/\.(md|mdx|html?)$/i, '').replace(/^\d+[-_.]/, '')
-  const slug = kebabCase(raw || fallbackTitle || 'page')
-  return slug || 'page'
-}
+// extractUrlPathSegments is imported from ../utils/url-segs.js
 
 /**
- * If `slug` is already in use anywhere in the staging tree, try `slug-2`,
- * `slug-3`, etc. The duplicates validator flags same-slug collisions across
- * directories, not just within a directory, so uniqueness must be global.
+ * Compute a globally-unique slug for every page in the tree.
+ *
+ * Base slug is the last URL segment (kebab-cased). When two or more pages
+ * share a base, we prepend the next-up URL segment to every member of the
+ * colliding group and recheck. The category title is treated as the
+ * outermost segment, so when URL segments alone aren't enough (e.g. a
+ * group-only `/v1` parent that appears in multiple categories), expansion
+ * can still reach for the category to disambiguate before falling back to
+ * a numeric `-N` suffix.
+ *
+ * @example
+ * ensureUniqueSlugs([
+ *   { title: 'Getting Started', pages: [
+ *     { title: 'Overview', url: 'https://acme.dev/getting-started/overview' },
+ *     { title: 'Install',  url: 'https://acme.dev/getting-started/install' },
+ *   ]},
+ *   { title: 'Guides', pages: [
+ *     { title: 'Overview', url: 'https://acme.dev/guides/overview' },
+ *   ]},
+ * ])
+ * // Map {
+ * //   <Overview (Getting Started)> => 'getting-started-overview',  // expanded (collides)
+ * //   <Install>                    => 'install',                   // unique, no expansion
+ * //   <Overview (Guides)>          => 'guides-overview',            // expanded (collides)
+ * // }
  */
-function resolveSlug(slug, usedSlugs) {
-  if (!usedSlugs.has(slug)) return slug
-  let n = 2
-  while (usedSlugs.has(`${slug}-${n}`)) n++
-  return `${slug}-${n}`
+export function ensureUniqueSlugs(categories) {
+  const entries = []
+  const segmentsFor = (p) => {
+    if (p._virtualPathSegs && p._virtualPathSegs.length > 0) {
+      return p._virtualPathSegs
+        .map((s) => stripSegmentExtensions(s).replace(/^\d+[-_.]/, ''))
+        .filter(Boolean)
+    }
+    return extractUrlPathSegments(p.url)
+  }
+  const normalizedSegmentsFor = (p) => {
+    const semanticSegments = segmentsFor(p).filter((seg) => !isGenericDocsSegment(seg))
+    return semanticSegments.length > 0 ? semanticSegments : ['introduction']
+  }
+  const expansionSegmentsFor = (p, categorySeg) => {
+    const pageSegments = normalizedSegmentsFor(p)
+    if (!categorySeg || isGenericDocsSegment(categorySeg)) return pageSegments
+    return [categorySeg, ...pageSegments]
+  }
+  const walk = (pages, categorySeg) => {
+    for (const p of pages || []) {
+      const segments = expansionSegmentsFor(p, categorySeg)
+      // Synthetic/empty-parent nodes have no content. Never let one win a bare
+      // single-segment slug (e.g. a placeholder `…/vercel-flags/cli` claiming
+      // `cli`), or it squats the slug a real overview page wants and ReadMe
+      // redirects the content-less parent to its first child. Floor their slug
+      // at 2 trailing segments so they namespace to `vercel-flags-cli` instead.
+      entries.push({ page: p, segments, fallback: p.title, depth: p._emptyParent ? 2 : 1 })
+      if (p.pages) walk(p.pages, categorySeg)
+    }
+  }
+  const genericCategoriesFirst = [...(categories || [])].sort((a, b) => {
+    const aGeneric = isGenericDocsSegment(a?.title || '')
+    const bGeneric = isGenericDocsSegment(b?.title || '')
+    return Number(bGeneric) - Number(aGeneric)
+  })
+  for (const c of genericCategoriesFirst) {
+    const rawTitle = (c?.title || '').trim()
+    const categorySeg = rawTitle ? kebabCase(rawTitle) : ''
+    walk(c.pages, categorySeg)
+  }
+
+  const slugFor = (e) => {
+    if (e.segments.length === 0) return kebabCase(e.fallback || 'page') || 'page'
+    const start = Math.max(0, e.segments.length - e.depth)
+    return kebabCase(e.segments.slice(start).join('-')) || 'page'
+  }
+
+  // Iteratively grow any colliding group by one segment until stable. Each
+  // pass only grows pages that still have headroom (depth < segments.length);
+  // a page stuck at full depth stays put while its peers keep expanding.
+  while (true) {
+    const groups = new Map()
+    for (const e of entries) {
+      const s = slugFor(e)
+      if (!groups.has(s)) groups.set(s, [])
+      groups.get(s).push(e)
+    }
+    let grew = false
+    for (const group of groups.values()) {
+      if (group.length < 2) continue
+      for (const e of group) {
+        if (e.depth < e.segments.length) {
+          e.depth++
+          grew = true
+        }
+      }
+    }
+    if (!grew) break
+  }
+
+  const describe = (x) => `"${x.page.title || '(untitled)'}"${x.page.url ? ` <${x.page.url}>` : ' (group-only)'}`
+  const used = new Set()
+  const result = new Map()
+  for (const e of entries) {
+    const base = slugFor(e)
+    let slug = base
+    if (used.has(slug)) {
+      let n = 2
+      while (used.has(`${slug}-${n}`)) n++
+      slug = `${base}-${n}`
+      styles.error(
+        `Slug collision after segment expansion: ${base} — falling back to ${slug} for ${describe(e)}. ` +
+        `Pages were deduped by path before organize and category title is already part of the slug, ` +
+        `so this indicates the organize step produced duplicates within a single category.`,
+      )
+    }
+    used.add(slug)
+    result.set(e.page, slug)
+  }
+  return result
+}
+
+const GENERIC_DOCS_SEGMENTS = new Set(['doc', 'docs', 'documentation', 'documentations'])
+function isGenericDocsSegment(segment) {
+  return GENERIC_DOCS_SEGMENTS.has(kebabCase(segment))
 }
 
 // Values YAML interprets as non-strings need quoting when used as _order entries.
@@ -2554,6 +4632,8 @@ function makeIconPicker() {
   }
 }
 
+export const __test__ = { discoverLlmsTxt, mergeValidHits, resolveRedirectedSourceUrl }
+
 function formatDuration(ms) {
   const safe = Math.max(0, ms)
   // Show ms under a second so sub-second work doesn't misleadingly read as "0m 0s".