@readme/cli 0.0.29 → 0.0.30

package/src/commands/import.js

@@ -62,6 +62,27 @@ const SUBDOMAIN_ONLY_DOC_ROUTES = new Set(['platform'])
 // Max concurrent HTTP probes when checking whether a section's landing URL
 // (e.g. /docs/cli) really exists before importing it as a real overview page.
 const LANDING_PROBE_CAP = 8
+// Well-known spec locations probed on every import. Tried under the
+// post-adoption origin, the pre-adoption origin when the docs-route probe
+// switched hosts, and (first three only) under a path-kind docs base.
+const WELL_KNOWN_OAS_PATHS = [
+  'openapi.json',
+  'openapi.yaml',
+  'openapi.yml',
+  'swagger.json',
+  'swagger.yaml',
+  '.well-known/openapi.json',
+  '.well-known/openapi.yaml',
+  'api/openapi.json',
+  'docs/openapi.json',
+  'v3/api-docs',
+  'api-docs',
+  'swagger/v1/swagger.json',
+]
+// Per-source cap on candidate specs. Candidates are sorted by discovery-source
+// confidence first, so the cap trims the least trusted ones.
+const MAX_OAS_CANDIDATES_PER_SOURCE = 10
+const OAS_SOURCE_CONFIDENCE = ['llms', 'mintlify', 'probe', 'html', 'sitemap', 'link-sweep']
 
 /**
  * Run the importer programmatically. Mirrors the CLI command but throws on
@@ -161,8 +182,12 @@ export async function importDocs(options) {
     debugSnapshots['05-organized.json'] = organized
     const debugDir = path.join(os.tmpdir(), `readme-import-debug-${hostnameJoined}-${Date.now()}`)
     fs.mkdirSync(debugDir, { recursive: true })
+    // Probe-hit candidates carry prefetched spec bytes — summarize instead of
+    // dumping megabytes of serialized Buffer into the snapshot.
+    const bufferSummary = (key, value) =>
+      value && value.type === 'Buffer' && Array.isArray(value.data) ? `<Buffer ${value.data.length} bytes>` : value
     for (const [name, data] of Object.entries(debugSnapshots)) {
-      fs.writeFileSync(path.join(debugDir, name), JSON.stringify(data, null, 2))
+      fs.writeFileSync(path.join(debugDir, name), JSON.stringify(data, bufferSummary, 2))
     }
     styles.info(`${styles.dim(`Debug snapshots → ${debugDir}`)}`)
   }
@@ -210,7 +235,7 @@ export async function importDocs(options) {
       styles.info(`Skipped ${styles.bold(String(staged.skippedApiRef))} API reference page${staged.skippedApiRef === 1 ? '' : 's'} (--skip-api-reference)`)
     }
 
-    // Download captured `.json` specs into <stagingDir>/oas/. Raw bytes only —
+    // Download captured candidate specs into <stagingDir>/oas/. Raw bytes only —
     // validation + OpenAPI detection + upload all happen in the runner. Staged
     // here (not in docs/ or reference/) so it rides the zip but is invisible to
     // the docs build. Non-fatal: a failed download is logged and skipped.
@@ -270,6 +295,7 @@ async function produceOrganizedForSource(sourceUrl, options, timePhase, debugSna
 
   // Probe well-known docs routes (docs./developer. subdomains and /docs/,
   // … path prefixes) and adopt the first that resolves as the real base, so everything downstream runs
+  const preAdoptionOrigin = sourceUrl.origin
   const docsBase = await timePhase('probe well-known docs routes', () => resolveDocsBaseUrl(sourceUrl))
   if (docsBase) {
     styles.info(
@@ -278,6 +304,10 @@ async function produceOrganizedForSource(sourceUrl, options, timePhase, debugSna
     sourceUrl = docsBase.url
   }
 
+  // Fired here so the spec-path probe fully overlaps llms/scrape/organize;
+  // awaited just before the candidates are attached to `organized`.
+  const oasProbePromise = probeWellKnownOasPaths(sourceUrl, docsBase, preAdoptionOrigin).catch(() => [])
+
   // Discover llms.txt files by BFS walking parents AND children. Seed the
   // frontier with the walk-up paths (source → root). Any "hit" (file exists
   // with at least one link row) expands the frontier: parent path + distinct
@@ -303,22 +333,33 @@ async function produceOrganizedForSource(sourceUrl, options, timePhase, debugSna
   let llmsUrl = llms?.llmsUrl || null
   const skippedLlms = discovery.skipped
 
-  // Capture `.json` links (candidate OpenAPI specs) BEFORE anything else sees
-  // the parsed tree. They're removed in place here, downloaded into the staged
-  // `oas/` dir by importDocs, and validated + uploaded by the runner. Stub
-  // generation never sees them.
-  let oasJsonUrls = []
+  // Capture spec-shaped links (.json/.yaml/.yml — candidate OpenAPI specs)
+  // BEFORE anything else sees the parsed tree. They're removed in place here,
+  // downloaded into the staged `oas/` dir by importDocs, and validated +
+  // uploaded by the runner. Stub generation never sees them.
+  const oasCandidates = []
+  const seenOasCandidateUrls = new Set()
+  const addOasCandidates = (items) => {
+    for (const item of items || []) {
+      if (!item || !item.url) continue
+      const url = item.url.split('#')[0]
+      if (seenOasCandidateUrls.has(url)) continue
+      seenOasCandidateUrls.add(url)
+      oasCandidates.push({ ...item, url })
+    }
+  }
   if (llms) {
-    oasJsonUrls = extractOasJsonUrlsFromParsed(llms.parsed)
-    if (oasJsonUrls.length > 0) {
-      styles.info(styles.dim(`Captured ${oasJsonUrls.length} .json link${oasJsonUrls.length === 1 ? '' : 's'} as candidate OpenAPI spec${oasJsonUrls.length === 1 ? '' : 's'} → oas/.`))
+    const llmsSpecUrls = extractOasSpecUrlsFromParsed(llms.parsed)
+    if (llmsSpecUrls.length > 0) {
+      styles.info(styles.dim(`Captured ${llmsSpecUrls.length} spec link${llmsSpecUrls.length === 1 ? '' : 's'} as candidate OpenAPI spec${llmsSpecUrls.length === 1 ? '' : 's'} → oas/.`))
+      addOasCandidates(llmsSpecUrls)
     }
   }
 
-  // Drop the remaining asset/meta items (llms-full.txt, .yaml specs, …) from
+  // Drop the remaining asset/meta items (llms-full.txt, .xml, …) from
   // the merged parsed result up-front, so every downstream consumer — knownUrls
   // AND the "use llms.txt sections directly" organize path — sees a clean URL
-  // list. `.json` items were already pulled out above.
+  // list. Spec-shaped items were already pulled out above.
   if (llms) {
     const dropped = dropAssetItemsFromParsed(llms.parsed)
     if (dropped > 0) {
@@ -380,7 +421,9 @@ async function produceOrganizedForSource(sourceUrl, options, timePhase, debugSna
     console.log()
     if (sitemapResult) {
       sitemapUrl = sitemapResult.url
-      sitemapKnownUrls = sitemapUrlsToKnownUrls(sitemapResult.urls)
+      const sitemapOasSink = []
+      sitemapKnownUrls = sitemapUrlsToKnownUrls(sitemapResult.urls, sitemapOasSink)
+      addOasCandidates(sitemapOasSink)
       styles.ok(`Found sitemap.xml at ${styles.bold(sitemapUrl)} — ${styles.bold(String(sitemapKnownUrls.length))} URL${sitemapKnownUrls.length === 1 ? '' : 's'} in scope.`)
     } else {
       styles.warning(`No llms.txt or sitemap.xml found — falling back to sidebar discovery via scrape.`)
@@ -489,7 +532,9 @@ async function produceOrganizedForSource(sourceUrl, options, timePhase, debugSna
   // HTML parsing, so try it before falling back to generic nav scraping.
   styles.info(`Probing for Mintlify config (docs.json, mint.json)...`)
   const mintlifyStart = Date.now()
-  const mintlifyNav = await timePhase('mintlify probe', () => tryMintlifyNav(sourceUrl.toString(), knownUrls, firecrawlKey))
+  const mintlifyOasSink = []
+  const mintlifyNav = await timePhase('mintlify probe', () => tryMintlifyNav(sourceUrl.toString(), knownUrls, firecrawlKey, mintlifyOasSink))
+  addOasCandidates(mintlifyOasSink)
   if (debugSnapshots) {
     debugSnapshots[`02a-mintlify-nav${dbgSuffix}.json`] = mintlifyNav ? JSON.parse(JSON.stringify(mintlifyNav)) : null
   }
@@ -501,14 +546,36 @@ async function produceOrganizedForSource(sourceUrl, options, timePhase, debugSna
   }
   console.log()
 
+  // Fern sites ship the canonical sidebar in RSC flight chunks on every
+  // page — deterministic structure where the Firecrawl scrape usually fails
+  // acceptance and llms.txt clustering invents categories.
+  let fernNav = null
+  if (!mintlifyNav) {
+    styles.info(`Probing for Fern nav tree...`)
+    const fernStart = Date.now()
+    fernNav = await timePhase('fern probe', () => tryFernNav(sourceUrl.toString(), knownUrls))
+    if (fernNav) {
+      const pageCount = fernNav.categories.reduce((n, c) => n + countUrlPagesDeep(c.pages), 0)
+      styles.ok(
+        `Found Fern nav tree in ${styles.bold(formatDuration(Date.now() - fernStart))} — ${styles.bold(String(fernNav.categories.length))} categor${fernNav.categories.length === 1 ? 'y' : 'ies'}, ${styles.bold(String(pageCount))} page${pageCount === 1 ? '' : 's'}.`,
+      )
+    }
+    console.log()
+  }
+  if (debugSnapshots) {
+    debugSnapshots[`02a2-fern-nav${dbgSuffix}.json`] = fernNav ? JSON.parse(JSON.stringify(fernNav)) : null
+  }
+
   // Archbee sites embed the canonical document tree in Next.js page data.
   // Their llms.txt export can be a flat, shuffled list, so prefer the tree
   // when present.
   let archbeeNav = null
-  if (!mintlifyNav) {
+  if (!mintlifyNav && !fernNav) {
     styles.info(`Probing for Archbee document tree...`)
     const archbeeStart = Date.now()
-    archbeeNav = await timePhase('archbee probe', () => tryArchbeeNav(sourceUrl.toString(), knownUrls, firecrawlKey))
+    const archbeeOasSink = []
+    archbeeNav = await timePhase('archbee probe', () => tryArchbeeNav(sourceUrl.toString(), knownUrls, firecrawlKey, archbeeOasSink))
+    addOasCandidates(archbeeOasSink)
     if (archbeeNav) {
       const pageCount = archbeeNav.categories.reduce((n, c) => n + countUrlPagesDeep(c.pages), 0)
       styles.ok(
@@ -523,12 +590,16 @@ async function produceOrganizedForSource(sourceUrl, options, timePhase, debugSna
   const scrapeDiagnostics = {}
   if (mintlifyNav) {
     scraped = { title: mintlifyNav.title, categories: mintlifyNav.categories }
+  } else if (fernNav) {
+    scraped = { title: fernNav.title, categories: fernNav.categories }
   } else if (archbeeNav) {
     scraped = { title: archbeeNav.title, categories: archbeeNav.categories }
   } else {
     styles.info(`Scraping sidebar nav from ${styles.bold(sourceUrl.toString())}${firecrawlKey ? ' ' + styles.dim('(via Firecrawl)') : ''}...`)
     scrapeStart = Date.now()
-    scraped = await timePhase('scrape nav', () => scrapeNavFromSite(sourceUrl.toString(), knownUrls, firecrawlKey, scrapeDiagnostics))
+    const scrapeOasSink = []
+    scraped = await timePhase('scrape nav', () => scrapeNavFromSite(sourceUrl.toString(), knownUrls, firecrawlKey, scrapeDiagnostics, scrapeOasSink))
+    addOasCandidates(scrapeOasSink)
     if (!scraped) scrapeDiagnostics.reason ??= 'unknown'
   }
   if (debugSnapshots) {
@@ -543,13 +614,18 @@ async function produceOrganizedForSource(sourceUrl, options, timePhase, debugSna
   // node because that's the one /docs page the scrape saw). Discard the
   // scrape and fall through to the llms.txt path, which uses URL-based
   // clustering when multiple files were merged.
+  // API-reference pages are excluded from the denominator: sidebars routinely
+  // omit generated endpoint stubs, and those pages are swept into reference/
+  // regardless of nav quality, so they say nothing about the nav's fitness as
+  // the import's spine.
   let scrapeDiscardedForCoverage = false
   if (scraped && llms && knownUrls.length > 0) {
     const scrapedPages = scraped.categories.reduce((n, c) => n + countUrlPagesDeep(c.pages), 0)
-    const coverage = scrapedPages / knownUrls.length
+    const nonReferenceKnown = knownUrls.filter((p) => !urlIsApiReference(p.url))
+    const coverage = nonReferenceKnown.length > 0 ? scrapedPages / nonReferenceKnown.length : 1
     if (coverage < 0.75) {
       styles.info(
-        `Scrape covered ${styles.bold(Math.round(coverage * 100) + '%')} of llms.txt pages (need ≥75%) — discarding scrape and organizing from llms.txt.`,
+        `Scrape covered ${styles.bold(Math.round(coverage * 100) + '%')} of llms.txt pages (need ≥75%${nonReferenceKnown.length < knownUrls.length ? `, ${knownUrls.length - nonReferenceKnown.length} api-reference pages excluded` : ''}) — discarding scrape and organizing from llms.txt.`,
       )
       scraped = null
       scrapeDiscardedForCoverage = true
@@ -629,12 +705,14 @@ async function produceOrganizedForSource(sourceUrl, options, timePhase, debugSna
           // category the sweep pass just built, if any) and nest it by resource
           // segment so routeCategory() maps the whole thing to ReadMe's
           // `reference/` top-level dir.
-          const apiResult = collectApiReferencePages(slotted, scraped)
+          const linkSweepOasSink = []
+          const apiResult = collectApiReferencePages(slotted, scraped, linkSweepOasSink)
           const otherOrphans = apiResult.nonApiOrphans
           if (apiResult.category) scraped.categories.push(apiResult.category)
 
-          const buckets = bucketOrphansByPathType(otherOrphans, scraped)
+          const buckets = bucketOrphansByPathType(otherOrphans, scraped, linkSweepOasSink)
           for (const b of buckets) scraped.categories.push(b)
+          addOasCandidates(linkSweepOasSink)
           if (debugSnapshots) {
             debugSnapshots[`04-after-orphan-buckets${dbgSuffix}.json`] = {
               apiReferenceCollected: apiResult.category
@@ -801,12 +879,32 @@ async function produceOrganizedForSource(sourceUrl, options, timePhase, debugSna
   // nestByUrlHierarchy renders it as the real parent of its children.
   await injectSectionLandingPages(organized, sourceUrl)
 
+  // A Fern tree is the authored sidebar: re-nesting by URL invents parent
+  // folders the source site renders flat (a section landing absorbing its
+  // path-descendant siblings, e.g. vapi's /server-url under Webhooks). Only
+  // the sweep-synthesized categories still nest — their pages were moved in
+  // flat and have no authored structure.
+  const sweepCategories = new Set(['API Reference', 'Changelog'])
   for (const cat of organized.categories || []) {
+    if (fernNav && !sweepCategories.has(cat.title)) continue
     cat.pages = nestByUrlHierarchy(cat.pages)
   }
 
   styles.ok(`Organized in ${styles.bold(formatDuration(Date.now() - organizeStart))}.`)
-  organized.oasJsonUrls = oasJsonUrls
+
+  addOasCandidates(await oasProbePromise)
+  oasCandidates.sort((a, b) => OAS_SOURCE_CONFIDENCE.indexOf(a.source) - OAS_SOURCE_CONFIDENCE.indexOf(b.source))
+  const cappedOasCandidates = oasCandidates.slice(0, MAX_OAS_CANDIDATES_PER_SOURCE)
+  if (oasCandidates.length > 0) {
+    const bySource = new Map()
+    for (const c of cappedOasCandidates) bySource.set(c.source, (bySource.get(c.source) || 0) + 1)
+    const summary = Array.from(bySource, ([source, n]) => `${n} ${source}`).join(', ')
+    const overCap = oasCandidates.length - cappedOasCandidates.length
+    styles.info(
+      styles.dim(`OAS candidates: ${summary}${overCap > 0 ? `; dropped ${overCap} over the cap of ${MAX_OAS_CANDIDATES_PER_SOURCE}` : ''}.`),
+    )
+  }
+  organized.oasJsonUrls = cappedOasCandidates
 
   // Record which page URLs came from llms.txt
   const llmsPaths = new Set()
@@ -1250,7 +1348,7 @@ function createZip(sourceDir, outputZip) {
  * Returns { source, title, categories } or null if no Mintlify config is
  * found or parseable.
  */
-async function tryMintlifyNav(sourceUrl, knownPages, firecrawlKey) {
+async function tryMintlifyNav(sourceUrl, knownPages, firecrawlKey, oasSink) {
   const origin = new URL(sourceUrl).origin
   const fetchHtml = firecrawlKey ? makeFirecrawlFetcher(firecrawlKey) : fetchHtmlDirect
 
@@ -1262,7 +1360,9 @@ async function tryMintlifyNav(sourceUrl, knownPages, firecrawlKey) {
     const body = await fetchHtml(configUrl)
     if (!body) continue
     const config = extractMintlifyConfig(body)
-    if (!config || !config.navigation) continue
+    if (!config) continue
+    if (oasSink) oasSink.push(...extractMintlifyOasCandidates(config, origin))
+    if (!config.navigation) continue
     const parsed = parseMintlifyConfig(config, origin, byPath)
     if (parsed.categories.length > 0) {
       return { source: configUrl, title: parsed.title, categories: parsed.categories }
@@ -1299,6 +1399,51 @@ function extractMintlifyConfig(body) {
   return null
 }
 
+/**
+ * Collect candidate spec URLs from a Mintlify config's `openapi` fields —
+ * `string | string[] | { source }` at the top level and on any navigation
+ * node. Relative values resolve against the origin. Pushed before the
+ * navigation guard so a config with `openapi` but unusable nav still yields
+ * specs.
+ */
+function extractMintlifyOasCandidates(config, origin) {
+  const candidates = []
+  const seen = new Set()
+  const addUrl = (raw) => {
+    if (typeof raw !== 'string' || !raw.trim()) return
+    let abs
+    try {
+      abs = new URL(raw, origin)
+    } catch {
+      return
+    }
+    if (abs.protocol !== 'http:' && abs.protocol !== 'https:') return
+    const href = abs.toString()
+    if (seen.has(href)) return
+    seen.add(href)
+    candidates.push({ url: href, text: null, source: 'mintlify' })
+  }
+  const addValue = (value) => {
+    if (typeof value === 'string') addUrl(value)
+    else if (Array.isArray(value)) for (const v of value) addValue(v)
+    else if (value && typeof value === 'object') addUrl(value.source)
+  }
+  const NAV_KEYS = ['navigation', 'tabs', 'groups', 'anchors', 'versions', 'languages', 'pages', 'dropdowns']
+  const walk = (node) => {
+    if (Array.isArray(node)) {
+      for (const n of node) walk(n)
+      return
+    }
+    if (!node || typeof node !== 'object') return
+    if (node.openapi !== undefined) addValue(node.openapi)
+    for (const key of NAV_KEYS) {
+      if (node[key] !== undefined) walk(node[key])
+    }
+  }
+  walk(config)
+  return candidates
+}
+
 function parseMintlifyConfig(config, origin, byPath) {
   const title = config.name || null
   const categories = []
@@ -1387,10 +1532,13 @@ function parseMintlifyConfig(config, origin, byPath) {
  * Returns { title, categories } or null if the page is not Archbee or the
  * embedded tree is unavailable.
  */
-async function tryArchbeeNav(sourceUrl, knownPages, firecrawlKey) {
+async function tryArchbeeNav(sourceUrl, knownPages, firecrawlKey, oasSink) {
   const origin = new URL(sourceUrl).origin
   const fetchHtml = firecrawlKey ? makeFirecrawlFetcher(firecrawlKey) : fetchHtmlDirect
   const html = await fetchHtml(toBrowsableUrl(sourceUrl))
+  if (html && oasSink) {
+    oasSink.push(...extractOasUrlsFromHtml(html, sourceUrl).map((url) => ({ url, text: null, source: 'html' })))
+  }
   if (!html || !/publicDocsTree|Archbee/i.test(html)) return null
 
   const nextData = extractNextData(html)
@@ -1454,6 +1602,249 @@ function archbeeNodeUrl(origin, urlKey) {
   return `${origin}/${key.replace(/^\/+/, '')}`
 }
 
+// Max page fetches while filling in non-active Fern tab trees (one per tab).
+const FERN_TAB_FETCH_CAP = 8
+
+const FERN_SECTION_NODE_TYPES = new Set(['section', 'apiReference', 'apiPackage'])
+const FERN_SKIP_NODE_TYPES = new Set(['link', 'changelog', 'tab'])
+
+/**
+ * Fern-powered docs sites embed the complete nav tree in Next.js RSC flight
+ * chunks (`self.__next_f.push([1,"..."])`) on every page — titles, slugs,
+ * nesting, and order exactly as the live sidebar renders them. Only the
+ * active tab's tree is populated in a given page's payload; other tabs are
+ * listed with a `pointsTo` entry path, so each one costs a single extra
+ * fetch.
+ *
+ * Always fetches directly (never Firecrawl): the flight chunks are only
+ * guaranteed present in the raw SSR HTML, not in a rendered/processed DOM.
+ * A dead entry URL (moved slug) is retried once from a known page, since
+ * Fern soft-404s stale deep links.
+ *
+ * Returns { title, categories } or null if the site is not Fern or the
+ * payload yields no usable tree. Any parse failure returns null so the
+ * probe chain falls through to the existing routes.
+ */
+async function tryFernNav(sourceUrl, knownPages) {
+  try {
+    const origin = new URL(sourceUrl).origin
+    const looksFern = (html) => /<aside[^>]*id="fern-sidebar"/.test(html) || /<meta[^>]+name="generator"[^>]+content="[^"]*buildwithfern/i.test(html)
+    let entryUrl = toBrowsableUrl(sourceUrl)
+    let entryHtml = await fetchHtmlDirect(entryUrl)
+    if (!looksFern(entryHtml)) {
+      // A stale entry deep link 404s on Fern (the not-found boundary renders
+      // with a truncated tree), so retry once from a known live page.
+      const retry = knownPages.find((p) => normalizePath(p.url) !== normalizePath(sourceUrl))
+      if (!retry) return null
+      entryUrl = toBrowsableUrl(retry.url)
+      entryHtml = await fetchHtmlDirect(entryUrl)
+      if (!looksFern(entryHtml)) return null
+    }
+
+    const byPath = new Map()
+    for (const p of knownPages) byPath.set(normalizePath(p.url), p)
+
+    const ctx = { origin, byPath, seenPaths: new Set() }
+    const categories = []
+    const collect = (html, tabTitle) => {
+      const blob = decodeFernFlightBlob(html)
+      addFernCategories(categories, extractFernNavNodes(blob), { ...ctx, tabTitle })
+      return blob
+    }
+
+    const entryBlob = collect(entryHtml, null)
+    const entryPath = normalizePath(entryUrl)
+    const seenTabTargets = new Set()
+    let tabFetches = 0
+    for (const tab of extractFernTabs(entryBlob)) {
+      if (seenTabTargets.has(tab.pointsTo)) continue
+      seenTabTargets.add(tab.pointsTo)
+      const tabUrl = `${origin}/${tab.pointsTo}`
+      if (normalizePath(tabUrl) === entryPath) continue
+      if (tabFetches >= FERN_TAB_FETCH_CAP) break
+      tabFetches++
+      const tabHtml = await fetchHtmlDirect(tabUrl)
+      if (!tabHtml) continue
+      collect(tabHtml, tab.title)
+    }
+
+    const pageCount = categories.reduce((n, c) => n + countUrlPagesDeep(c.pages), 0)
+    if (pageCount === 0) return null
+    return { title: null, categories }
+  } catch {
+    return null
+  }
+}
+
+function decodeFernFlightBlob(html) {
+  const chunks = []
+  const chunkRe = /self\.__next_f\.push\(\[1,"((?:[^"\\]|\\.)*)"\]\)/g
+  let m
+  while ((m = chunkRe.exec(html)) !== null) {
+    try {
+      chunks.push(JSON.parse(`"${m[1]}"`))
+    } catch { }
+  }
+  return chunks.join('')
+}
+
+function extractBalancedObject(str, start) {
+  let depth = 0
+  let inString = false
+  for (let i = start; i < str.length; i++) {
+    const ch = str[i]
+    if (inString) {
+      if (ch === '\\') i++
+      else if (ch === '"') inString = false
+    } else if (ch === '"') inString = true
+    else if (ch === '{') depth++
+    else if (ch === '}') {
+      depth--
+      if (depth === 0) return str.slice(start, i + 1)
+    }
+  }
+  return null
+}
+
+/**
+ * Pull top-level nav container/section objects out of a decoded flight blob.
+ * The blob as a whole is not JSON — nav nodes are embedded mid-stream — so
+ * each marker hit is balanced-brace extracted and parsed on its own. Matches
+ * that fall inside an already-extracted object are its children and skipped.
+ */
+function extractFernNavNodes(blob) {
+  const nodes = []
+  const markerRe = /\{"type":"(?:sidebarRoot|sidebarGroup|section|apiReference|apiPackage)"/g
+  let consumedUpTo = -1
+  let m
+  while ((m = markerRe.exec(blob)) !== null) {
+    if (m.index < consumedUpTo) continue
+    const raw = extractBalancedObject(blob, m.index)
+    if (!raw) continue
+    consumedUpTo = m.index + raw.length
+    try {
+      nodes.push(JSON.parse(raw))
+    } catch { }
+  }
+  return nodes
+}
+
+function extractFernTabs(blob) {
+  const tabs = []
+  const tabRe = /\{"type":"tab",/g
+  let m
+  while ((m = tabRe.exec(blob)) !== null) {
+    const raw = extractBalancedObject(blob, m.index)
+    if (!raw) continue
+    let tab
+    try {
+      tab = JSON.parse(raw)
+    } catch {
+      continue
+    }
+    const pointsTo = fernField(tab.pointsTo)
+    if (!pointsTo) continue
+    tabs.push({ title: fernField(tab.title), pointsTo })
+  }
+  return tabs
+}
+
+// Absent fields serialize as the literal string "$undefined" in flight payloads.
+function fernField(value) {
+  return typeof value === 'string' && value !== '$undefined' && value !== '' ? value : null
+}
+
+function fernPageFromNode(node, ctx) {
+  if (!node || typeof node !== 'object' || node.hidden === true) return null
+  if (FERN_SKIP_NODE_TYPES.has(node.type)) return null
+  const children = Array.isArray(node.children) ? node.children.map((c) => fernPageFromNode(c, ctx)).filter(Boolean) : []
+  if (FERN_SECTION_NODE_TYPES.has(node.type)) {
+    // A section owns a page only when overviewPageId is set — it then renders
+    // at the section's own slug. pointsTo is a navigation alias to the
+    // section's first descendant page, never a page of the section itself.
+    const landing = fernField(node.overviewPageId) ? fernField(node.slug) : null
+    const title = fernField(node.title) || 'Untitled'
+    if (!landing) {
+      if (children.length === 0) return null
+      return { title, url: null, _emptyParent: true, _virtualPathSegs: [kebabCase(title) || 'group'], pages: children }
+    }
+    const landingUrl = `${ctx.origin}/${landing}`
+    const known = ctx.byPath.get(normalizePath(landingUrl))
+    return {
+      title,
+      url: known?.url || landingUrl,
+      ...(children.length > 0 ? { pages: children } : {}),
+    }
+  }
+  const slug = fernField(node.slug)
+  if (slug) {
+    const url = `${ctx.origin}/${slug}`
+    const known = ctx.byPath.get(normalizePath(url))
+    return {
+      // The payload title is the sidebar label as rendered; llms.txt titles
+      // come from page headings, which can differ from the nav.
+      title: fernField(node.title) || known?.title || slug,
+      url: known?.url || url,
+      ...(known?.description ? { description: known.description } : {}),
+      ...(children.length > 0 ? { pages: children } : {}),
+    }
+  }
+  if (children.length === 1) return children[0]
+  if (children.length > 1) return { title: fernField(node.title) || 'Untitled', url: null, pages: children }
+  return null
+}
+
+/**
+ * Top-level sections become categories; untitled ones adopt the tab title so
+ * two tabs' unnamed roots don't merge into one "Untitled" category with
+ * colliding group slugs. Loose top-level pages (tabs without sections, e.g. a
+ * two-page MCP tab) collapse into one category named after the tab. Pages
+ * already claimed by an earlier category are dropped — the
+ * payload repeats trees across fetches, and multi-product sites nest the
+ * same API groups under both a product tab and the API-reference tab.
+ */
+function addFernCategories(categories, nodes, ctx) {
+  const roots = []
+  for (const node of nodes) {
+    if (node.type === 'sidebarRoot' || node.type === 'sidebarGroup') roots.push(...(node.children || []))
+    else roots.push(node)
+  }
+  const loosePages = []
+  for (const root of roots) {
+    if (FERN_SECTION_NODE_TYPES.has(root.type)) {
+      const section = fernPageFromNode(root, ctx)
+      if (!section) continue
+      if (!fernField(root.title) && ctx.tabTitle) section.title = ctx.tabTitle
+      const pages = section.pages || []
+      if (section.url && !pages.some((p) => p.url && normalizePath(p.url) === normalizePath(section.url))) {
+        pages.unshift({ title: section.title, url: section.url })
+      }
+      const deduped = dedupeFernPages(pages, ctx.seenPaths)
+      if (deduped.length > 0) categories.push({ title: section.title, pages: deduped })
+    } else {
+      const page = fernPageFromNode(root, ctx)
+      if (page) loosePages.push(page)
+    }
+  }
+  const deduped = dedupeFernPages(loosePages, ctx.seenPaths)
+  if (deduped.length > 0) categories.push({ title: ctx.tabTitle || 'Documentation', pages: deduped })
+}
+
+function dedupeFernPages(pages, seenPaths) {
+  const out = []
+  for (const p of pages) {
+    const urlSeen = p.url ? seenPaths.has(normalizePath(p.url)) : false
+    if (p.url && !urlSeen) seenPaths.add(normalizePath(p.url))
+    const kids = p.pages ? dedupeFernPages(p.pages, seenPaths) : []
+    if (kids.length === 0 && (urlSeen || !p.url)) continue
+    const entry = { ...p, url: urlSeen ? null : p.url }
+    if (kids.length > 0) entry.pages = kids
+    else delete entry.pages
+    out.push(entry)
+  }
+  return out
+}
+
 /**
  * Score a parsed nav tree for "sidebar-likeness". A real docs sidebar has
  * multiple section headers (hierarchy) and tens of links; secondary navs
@@ -1488,6 +1879,63 @@ function isMonotonicAlpha(titles) {
   return true
 }
 
+/**
+ * Scan a rendered docs page for spec URLs embedded by the common API-reference
+ * renderers — Swagger UI, Redoc, Scalar, Stoplight Elements. Returns absolute
+ * http(s) URLs resolved against the page URL. Pure string scanning on HTML we
+ * already fetched — zero extra requests.
+ */
+function extractOasUrlsFromHtml(html, pageUrl) {
+  const doc = String(html)
+  const raws = []
+
+  // Swagger UI: `SwaggerUIBundle({ url: "…" })` or `urls: [{ url: "…" }, …]`.
+  // Scanning a bounded window after each init call handles both shapes
+  // without catastrophic backtracking.
+  const swaggerInitRe = /SwaggerUI(?:Bundle)?\s*\(/g
+  let m
+  while ((m = swaggerInitRe.exec(doc)) !== null) {
+    const windowText = doc.slice(m.index, m.index + 3000)
+    const urlRe = /["']?\burl["']?\s*:\s*["']([^"']+)["']/g
+    let u
+    while ((u = urlRe.exec(windowText)) !== null) raws.push(u[1])
+  }
+
+  const redocAttrRe = /<redoc\b[^>]*?\bspec-url\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'>]+))/gi
+  while ((m = redocAttrRe.exec(doc)) !== null) raws.push(m[1] ?? m[2] ?? m[3])
+  const redocInitRe = /Redoc\.init\s*\(\s*["']([^"']+)["']/g
+  while ((m = redocInitRe.exec(doc)) !== null) raws.push(m[1])
+
+  // Scalar: <script id="api-reference" data-url="…"> — attr-order agnostic.
+  const scriptTagRe = /<script\b[^>]*>/gi
+  while ((m = scriptTagRe.exec(doc)) !== null) {
+    const tag = m[0]
+    if (!/\bid\s*=\s*["']api-reference["']/i.test(tag)) continue
+    const u = /\bdata-url\s*=\s*["']([^"']+)["']/i.exec(tag)
+    if (u) raws.push(u[1])
+  }
+
+  const stoplightRe = /apiDescriptionUrl\s*=\s*"([^"]+)"/g
+  while ((m = stoplightRe.exec(doc)) !== null) raws.push(m[1])
+
+  const out = []
+  const seen = new Set()
+  for (const raw of raws) {
+    let abs
+    try {
+      abs = new URL(raw, pageUrl)
+    } catch {
+      continue
+    }
+    if (abs.protocol !== 'http:' && abs.protocol !== 'https:') continue
+    const href = abs.toString()
+    if (seen.has(href)) continue
+    seen.add(href)
+    out.push(href)
+  }
+  return out
+}
+
 /**
  * Fetch the source URL, find the `<nav>` or `<aside>` that contains the most
  * links matching our known llms.txt URLs, and extract its heading/link
@@ -1497,7 +1945,7 @@ function isMonotonicAlpha(titles) {
  * renders its sidebar server-side as <nav>/<aside> with <h*> section headers.
  * Returns null if coverage is too low to be useful.
  */
-async function scrapeNavFromSite(sourceUrl, knownPages, firecrawlKey, diagnostics = {}) {
+async function scrapeNavFromSite(sourceUrl, knownPages, firecrawlKey, diagnostics = {}, oasSink) {
   // Index known pages by normalized pathname so we can match nav hrefs against them.
   const byPath = new Map()
   for (const p of knownPages) byPath.set(normalizePath(p.url), p)
@@ -1521,6 +1969,9 @@ async function scrapeNavFromSite(sourceUrl, knownPages, firecrawlKey, diagnostic
 
     const html = await fetchHtml(url)
     if (!html) return 0
+    if (oasSink) {
+      oasSink.push(...extractOasUrlsFromHtml(html, url).map((u) => ({ url: u, text: null, source: 'html' })))
+    }
 
     const base = new URL(url)
     let best = { score: -Infinity, count: 0, tree: null }
@@ -1887,7 +2338,7 @@ function parseNavBlock(blockHtml, base, byPath) {
  *
  * Returns { category, nonApiOrphans, mergedScrapedTitles }.
  */
-function collectApiReferencePages(orphans, scraped) {
+function collectApiReferencePages(orphans, scraped, oasSink) {
   const API_PREFIX_RE = /^\/(api[-_]?reference|api|reference)(\/|$)/i
   const isApiUrl = (url) => {
     try {
@@ -1903,9 +2354,12 @@ function collectApiReferencePages(orphans, scraped) {
   const seenUrls = new Set()
   const push = (p) => {
     if (!p || !p.url) return
-    // Skip non-page assets like /api-reference/openapi.json that some
-    // llms.txt files list alongside real pages.
-    if (/\.(json|yaml|yml)$/i.test(p.url)) return
+    // Non-page assets like /api-reference/openapi.json that some llms.txt
+    // files list alongside real pages become spec candidates, not doc stubs.
+    if (/\.(json|ya?ml)$/i.test(p.url)) {
+      if (oasSink) oasSink.push({ url: p.url, text: p.title || null, source: 'link-sweep' })
+      return
+    }
     if (seenUrls.has(p.url)) return
     seenUrls.add(p.url)
     apiPages.push(p)
@@ -1995,7 +2449,7 @@ function collectApiReferencePages(orphans, scraped) {
  * Uses distinctive titles — just "Docs" or "Reference" — rather than the raw
  * segment name. Falls back to "Other" when the path has no usable type.
  */
-function bucketOrphansByPathType(orphans, scraped) {
+function bucketOrphansByPathType(orphans, scraped, oasSink) {
   const TYPE_TITLES = {
     reference: 'API Reference',
     api: 'API Reference',
@@ -2043,9 +2497,12 @@ function bucketOrphansByPathType(orphans, scraped) {
       continue
     }
     const pathname = url.pathname
-    // Skip OAS spec endpoints — `/api-reference/openapi.json` etc. aren't
-    // documentation pages and shouldn't become stubs.
-    if (/\.(json|ya?ml)$/i.test(pathname)) continue
+    // OAS spec endpoints — `/api-reference/openapi.json` etc. aren't
+    // documentation pages; capture them as spec candidates instead of stubs.
+    if (/\.(json|ya?ml)$/i.test(pathname)) {
+      if (oasSink) oasSink.push({ url: p.url, text: p.title || null, source: 'link-sweep' })
+      continue
+    }
 
     const segs = pathname.split('/').filter(Boolean)
     let type = null
@@ -2178,9 +2635,20 @@ function reclassifyPagesByUrlSegment(scraped, { segmentRe, categoryRe, defaultTi
  * live under a separate API Reference section — this lands them all in
  * `reference/` after staging. Returns the number of pages relocated.
  */
+const API_REFERENCE_URL_SEGMENT_RE = /^(api[-_]?reference|endpoints?)$/i
+
+function urlIsApiReference(url) {
+  try {
+    const segs = new URL(url).pathname.split('/').filter(Boolean)
+    return segs.some((s) => API_REFERENCE_URL_SEGMENT_RE.test(s))
+  } catch {
+    return false
+  }
+}
+
 function reclassifyReferencePages(scraped) {
   return reclassifyPagesByUrlSegment(scraped, {
-    segmentRe: /^(api[-_]?reference|endpoints?)$/i,
+    segmentRe: API_REFERENCE_URL_SEGMENT_RE,
     categoryRe: /^(api[ -]?reference|reference|api|endpoints?)$/i,
     defaultTitle: 'API Reference',
   })
@@ -2471,10 +2939,10 @@ function slotOrphansByPath(scraped, knownPages) {
   const matched = new Set()
   const pathToCategory = new Map() // normalizedPath → category
   for (const cat of scraped.categories) {
-    for (const p of cat.pages) {
+    for (const p of collectUrlPagesDeep(cat.pages)) {
       const norm = normalizePath(p.url)
       matched.add(norm)
-      pathToCategory.set(norm, cat)
+      if (!pathToCategory.has(norm)) pathToCategory.set(norm, cat)
     }
   }
 
@@ -2575,14 +3043,17 @@ async function fetchHtmlDirect(url) {
 }
 
 /**
- * Download a `.json` URL as raw bytes. Returns a Buffer, or null on non-OK /
- * network error. No parsing or validation — the runner is the OAS gate.
+ * Download a candidate spec URL as raw bytes. Returns a Buffer, or null on
+ * non-OK / network error. No parsing or validation — the runner is the OAS gate.
  */
-async function fetchJsonSpec(url) {
+async function fetchSpecBytes(url) {
   try {
     const res = await fetch(url, {
       redirect: 'follow',
-      headers: { 'User-Agent': 'readme-cli-import', Accept: 'application/json' },
+      headers: {
+        'User-Agent': 'readme-cli-import',
+        Accept: 'application/json, application/x-yaml, application/yaml, text/yaml, text/plain',
+      },
     })
     if (!res.ok) return null
     return Buffer.from(await res.arrayBuffer())
@@ -2591,36 +3062,55 @@ async function fetchJsonSpec(url) {
   }
 }
 
+// Extension for an extensionless candidate (e.g. /v3/api-docs), sniffed from
+// the body. Defaults to .json — the runner tolerates a mislabeled extension.
+function sniffSpecExtension(text) {
+  try {
+    JSON.parse(text)
+    return '.json'
+  } catch {
+    return OAS_TOP_LEVEL_KEY_RE.test(String(text).slice(0, 64 * 1024)) ? '.yaml' : '.json'
+  }
+}
+
 /**
- * Download the captured candidate-spec URLs into `<stagingDir>/oas/<name>.json`.
- * Filenames come from the URL basename (collisions get a numeric suffix); the
- * runner de-dups by content, so identical bytes under different names are
- * handled there. Returns the count of files written.
+ * Download the captured candidate-spec URLs into `<stagingDir>/oas/`. Probe
+ * hits carry their prefetched bytes in `item.body`; everything else is
+ * fetched here. Bodies that look like HTML are skipped so the runner's
+ * user-visible candidate count stays honest — all other validation happens in
+ * the runner. Filenames come from the URL basename, keeping a .json/.yaml/.yml
+ * extension when present and sniffing one otherwise (collisions get a numeric
+ * suffix); the runner de-dups by content, so identical bytes under different
+ * names are handled there. Returns the count of files written.
  */
 async function downloadOasSpecs(oasJsonUrls, stagingDir) {
   const oasDir = path.join(stagingDir, 'oas')
   fs.mkdirSync(oasDir, { recursive: true })
   const usedNames = new Set()
   let written = 0
-  for (const { url } of oasJsonUrls) {
-    const buf = await fetchJsonSpec(url)
+  for (const item of oasJsonUrls) {
+    const buf = item.body || (await fetchSpecBytes(item.url))
     if (!buf) {
-      styles.info(styles.dim(`  oas: skipped ${url} (download failed)`))
+      styles.info(styles.dim(`  oas: skipped ${item.url} (download failed)`))
+      continue
+    }
+    const text = buf.toString('utf8')
+    if (/^\s*</.test(text)) {
+      styles.info(styles.dim(`  oas: skipped ${item.url} (looks like HTML)`))
       continue
     }
-    // Derive a safe, unique *.json filename from the URL basename.
     let base
     try {
-      base = path.basename(new URL(url).pathname) || 'openapi.json'
+      base = path.basename(new URL(item.url).pathname) || 'openapi'
     } catch {
-      base = 'openapi.json'
+      base = 'openapi'
     }
     base = base.replace(/[^a-zA-Z0-9._-]/g, '-')
-    if (!/\.json$/i.test(base)) base += '.json'
+    if (!/\.(json|ya?ml)$/i.test(base)) base += sniffSpecExtension(text)
     let name = base
     let n = 2
     while (usedNames.has(name)) {
-      name = base.replace(/\.json$/i, `-${n}.json`)
+      name = base.replace(/\.(json|ya?ml)$/i, `-${n}$&`)
       n++
     }
     usedNames.add(name)
@@ -3286,10 +3776,10 @@ async function runJsonQuery({ systemPrompt, userPrompt, model, schema }) {
 //   - llms.txt, llms-full.txt, llms-ctx.txt, … (other llms.txt variants
 //     that point back at the same index/dump, not real pages)
 //
-// `.json` is deliberately NOT in this list: those links are captured up-front
-// by extractOasJsonUrlsFromParsed (they may be OpenAPI specs we want to
-// download into oas/ and upload separately) and removed from the parsed tree
-// before this drop runs, so a `.json` item never reaches here.
+// `.json`/`.yaml`/`.yml` spec links are captured up-front by
+// extractOasSpecUrlsFromParsed (candidate OpenAPI specs we download into oas/
+// and upload separately) and removed from the parsed tree before this drop
+// runs, so spec-shaped items never reach here — `.yaml` below is a backstop.
 const ASSET_EXT_RE = /\.(ya?ml|xml|toml)$/i
 const LLMS_TXT_RE = /(?:^|\/)llms[^/]*\.txt$/i
 function isAssetOrMetaUrl(url) {
@@ -3301,14 +3791,13 @@ function isAssetOrMetaUrl(url) {
   }
 }
 
-// A llms.txt item whose URL points at a `.json` file. These are pulled out of
-// the parsed tree before organization (so stub generation never sees them) and
-// downloaded into the staged `oas/` dir; the runner validates each as OpenAPI
-// and uploads the valid ones. NOTE: only `.json` is captured — `.yaml`/`.yml`
-// specs are intentionally left to the asset drop above.
-function isOasJsonUrl(url) {
+// A llms.txt item whose URL points at a spec-shaped file (.json/.yaml/.yml).
+// These are pulled out of the parsed tree before organization (so stub
+// generation never sees them) and downloaded into the staged `oas/` dir; the
+// runner validates each as OpenAPI and uploads the valid ones.
+function isOasSpecUrl(url) {
   try {
-    return /\.json$/i.test(new URL(url).pathname)
+    return /\.(json|ya?ml)$/i.test(new URL(url).pathname)
   } catch {
     return false
   }
@@ -3410,21 +3899,22 @@ function narrowToDocsSubtreeIfNeeded(llms, sourceUrl, hits) {
 }
 
 /**
- * Pull `.json` items out of a parsed llms.txt structure IN PLACE, returning the
- * removed items as `[{ url, text }]`. Sections emptied by the removal are
- * pruned too — same shape as dropAssetItemsFromParsed. Run BEFORE
- * dropAssetItemsFromParsed / knownUrls / organize so the captured specs never
- * enter the stub tree (skeleton has zero visibility of OAS files). The caller
- * downloads the returned URLs into the staged `oas/` dir.
+ * Pull spec-shaped items (.json/.yaml/.yml) out of a parsed llms.txt structure
+ * IN PLACE, returning the removed items as `[{ url, text, source }]`. Sections
+ * emptied by the removal are pruned too — same shape as
+ * dropAssetItemsFromParsed. Run BEFORE dropAssetItemsFromParsed / knownUrls /
+ * organize so the captured specs never enter the stub tree (skeleton has zero
+ * visibility of OAS files). The caller downloads the returned URLs into the
+ * staged `oas/` dir.
  */
-function extractOasJsonUrlsFromParsed(parsed) {
+function extractOasSpecUrlsFromParsed(parsed) {
   const captured = []
   const keptSections = []
   for (const section of parsed.sections) {
     const keptItems = []
     for (const item of section.items) {
-      if (isOasJsonUrl(item.url)) {
-        captured.push({ url: item.url, text: item.text || null })
+      if (isOasSpecUrl(item.url)) {
+        captured.push({ url: item.url, text: item.text || null, source: 'llms' })
         continue
       }
       keptItems.push(item)
@@ -3489,6 +3979,78 @@ function buildWellKnownDocRoutes(sourceUrl) {
   return out
 }
 
+// Content gate for spec candidates — many sites 200 with an HTML shell for
+// any path, so a body only counts as a spec if it parses as JSON with a
+// top-level `openapi`/`swagger` version string, or matches the YAML form of
+// that key. (Pattern mirrors ai-cli-runner/app/lib/oas-sniff.ts — copied,
+// never imported.)
+const OAS_TOP_LEVEL_KEY_RE = /^\s*['"]?(openapi|swagger)['"]?\s*:/m
+function sniffOasContent(text) {
+  const body = String(text)
+  if (/^\s*</.test(body)) return false
+  try {
+    const obj = JSON.parse(body)
+    if (!obj || typeof obj !== 'object') return false
+    return typeof obj.openapi === 'string' || typeof obj.swagger === 'string'
+  } catch {
+    return OAS_TOP_LEVEL_KEY_RE.test(body.slice(0, 64 * 1024))
+  }
+}
+
+function buildWellKnownOasProbeUrls(sourceUrl, docsBase, preAdoptionOrigin) {
+  const urls = []
+  const seen = new Set()
+  const add = (href) => {
+    if (seen.has(href)) return
+    seen.add(href)
+    urls.push(href)
+  }
+  const origins = [sourceUrl.origin]
+  if (preAdoptionOrigin && preAdoptionOrigin !== sourceUrl.origin) origins.push(preAdoptionOrigin)
+  for (const origin of origins) {
+    for (const p of WELL_KNOWN_OAS_PATHS) add(`${origin}/${p}`)
+  }
+  if (docsBase && docsBase.kind === 'path') {
+    const base = docsBase.url.href.replace(/\/+$/, '')
+    for (const p of ['openapi.json', 'openapi.yaml', 'swagger.json']) add(`${base}/${p}`)
+  }
+  return urls
+}
+
+/**
+ * Probe well-known spec paths on the source origins. A hit is an OK response
+ * whose body sniffs as a spec and whose final URL stayed on a probed origin
+ * (redirects to a marketing apex don't count). Returns ALL hits — a site can
+ * serve several specs — with the fetched bytes attached so downloadOasSpecs
+ * never re-downloads them.
+ */
+async function probeWellKnownOasPaths(sourceUrl, docsBase, preAdoptionOrigin) {
+  const probeUrls = buildWellKnownOasProbeUrls(sourceUrl, docsBase, preAdoptionOrigin)
+  const allowedOrigins = new Set([sourceUrl.origin])
+  if (preAdoptionOrigin) allowedOrigins.add(preAdoptionOrigin)
+  const hits = await Promise.all(
+    probeUrls.map(async (url) => {
+      try {
+        const res = await fetch(url, { redirect: 'follow', headers: { 'User-Agent': 'readme-cli-import' } })
+        if (!res.ok) return null
+        let finalOrigin
+        try {
+          finalOrigin = new URL(res.url || url).origin
+        } catch {
+          return null
+        }
+        if (!allowedOrigins.has(finalOrigin)) return null
+        const body = Buffer.from(await res.arrayBuffer())
+        if (!sniffOasContent(body.toString('utf8'))) return null
+        return { url, text: null, source: 'probe', body }
+      } catch {
+        return null
+      }
+    }),
+  )
+  return hits.filter(Boolean)
+}
+
 /**
  * Follow redirects on the source URL itself and return the final URL when the
  * site has moved (different origin or path), or null to keep the original.
@@ -3539,13 +4101,18 @@ async function resolveDocsBaseUrl(sourceUrl) {
     }
   }
 
+  const soft404Title = candidates.some((c) => c.kind === 'path') ? await fetchSoft404Title(sourceUrl.origin) : null
   const existence = await Promise.all(
     llmsHits.map(async ({ c, res }) => {
       if (res.error && c.kind === 'subdomain') return { c, exists: false }
-      return { c, exists: await docsRouteResolves(c) }
+      return { c, ...(await docsRouteResolves(c, soft404Title)) }
     }),
   )
-  for (const { c, exists } of existence) {
+  for (const { c, exists, soft404 } of existence) {
+    if (soft404) {
+      styles.info(styles.dim(`  ${c.url.href} → indistinguishable from a nonexistent path (SPA catch-all) — skipping`))
+      continue
+    }
     if (exists) {
       styles.info(styles.dim(`  ${c.url.href} → resolves (no llms.txt)`))
       return { url: c.url, kind: c.kind, hasLlms: false }
@@ -3570,20 +4137,54 @@ function inCandidateScope(candidate, finalUrl) {
     return false
   }
   if (candidate.kind === 'subdomain') return final.hostname === candidate.url.hostname
-  return final.pathname.toLowerCase().startsWith(candidate.url.pathname.toLowerCase())
+  const strip = (p) => p.toLowerCase().replace(/\/+$/, '')
+  const finalPath = strip(final.pathname)
+  const candidatePath = strip(candidate.url.pathname)
+  return finalPath === candidatePath || finalPath.startsWith(`${candidatePath}/`)
+}
+
+function extractHtmlTitle(html) {
+  const match = /<title[^>]*>([^<]*)<\/title>/i.exec(html)
+  if (!match) return null
+  const title = match[1].replace(/\s+/g, ' ').trim().toLowerCase()
+  return title || null
+}
+
+/**
+ * Fetch a guaranteed-nonexistent path and capture its page title. On SPA
+ * catch-all sites every unknown path returns 200 with the same shell page;
+ * that title becomes the soft-404 baseline. Returns null when the site 404s
+ * properly (no baseline needed) or on any fetch failure.
+ */
+async function fetchSoft404Title(origin) {
+  const probeUrl = `${origin}/readme-cli-nonexistent-${Math.random().toString(36).slice(2)}`
+  try {
+    const res = await fetch(probeUrl, { redirect: 'follow', headers: { 'User-Agent': 'readme-cli-import' } })
+    if (!res.ok) return null
+    return extractHtmlTitle(await res.text())
+  } catch {
+    return null
+  }
 }
 
 /**
  * Does a well-known docs route exist and stay in scope? Follows redirects and
  * rejects anything that lands outside the candidate (a `docs.` subdomain that
- * bounces to marketing, a `/docs/` that 302s home). DNS failures → false.
+ * bounces to marketing, a `/docs/` that 302s home). When a soft-404 baseline
+ * title is known, a path candidate serving that same title is a catch-all
+ * response, not real docs. DNS failures → `{ exists: false }`.
  */
-async function docsRouteResolves(candidate) {
+async function docsRouteResolves(candidate, soft404Title) {
   try {
     const res = await fetch(candidate.url.href, { redirect: 'follow', headers: { 'User-Agent': 'readme-cli-import' } })
-    return res.ok && inCandidateScope(candidate, res.url)
+    if (!res.ok || !inCandidateScope(candidate, res.url)) return { exists: false }
+    if (candidate.kind === 'path' && soft404Title) {
+      const title = extractHtmlTitle(await res.text())
+      if (title === soft404Title) return { exists: false, soft404: true }
+    }
+    return { exists: true }
   } catch {
-    return false
+    return { exists: false }
   }
 }
 
@@ -4041,9 +4642,10 @@ function urlInScope(urlStr, sitemapUrl, scopePrefix) {
  * Convert a flat sitemap URL list into the `{title, url, description}` shape
  * the rest of the pipeline expects for knownUrls. Title is derived from the
  * last non-empty path segment, with file extensions stripped. Index-style
- * URLs that resolve to the scope root itself are dropped.
+ * URLs that resolve to the scope root itself are dropped. Spec-shaped URLs
+ * (.json/.yaml/.yml) go to `oasSink` instead of becoming pages.
  */
-function sitemapUrlsToKnownUrls(urls) {
+function sitemapUrlsToKnownUrls(urls, oasSink) {
   const out = []
   const seen = new Set()
   for (const url of urls) {
@@ -4056,6 +4658,10 @@ function sitemapUrlsToKnownUrls(urls) {
     const norm = normalizePath(url)
     if (seen.has(norm)) continue
     seen.add(norm)
+    if (/\.(json|ya?ml)$/i.test(pathname)) {
+      if (oasSink) oasSink.push({ url, text: null, source: 'sitemap' })
+      continue
+    }
     const segs = pathname.split('/').filter(Boolean)
     const last = segs[segs.length - 1] || ''
     const slug = last.replace(/\.[a-z0-9]+$/i, '')
@@ -4185,9 +4791,12 @@ function stageOrganized(organized, stagingDir, opts = {}) {
 
   /**
    * Write a page (and its descendants) into `dir`. A page with children gets
-   * its own subfolder named after its slug; the parent page lives at
-   * `<dir>/<slug>.md` while children live at `<dir>/<slug>/<childSlug>.md`.
-   * This matches git-format's on-disk convention for nested sidebars.
+   * its own subfolder named after its slug; its content lives at
+   * `<dir>/<slug>/index.md` while children live at `<dir>/<slug>/<childSlug>.md`.
+   * A leaf page (no children) lives at `<dir>/<slug>.md`. This matches
+   * git-format's on-disk convention: gitto backs a folder's sidebar node with
+   * its `index.md`; a sibling `<slug>.md` next to the folder instead leaves the
+   * folder a contentless placeholder parent and the sibling a stray page.
    */
   function writePage(page, dir, topDir, isSubPage = false) {
     const slug = slugFor.get(page)
@@ -4198,9 +4807,13 @@ function stageOrganized(organized, stagingDir, opts = {}) {
     // right subdirectory.
     const isEmptyParent = !!page._emptyParent
     const isGroupOnly = !page.url && !isEmptyParent
+    const children = page.pages || []
+    const hasChildren = children.length > 0
 
     if (!isGroupOnly) {
-      const relFilePath = `${dir}/${slug}.md`
+      // Parents-with-children store their content INSIDE the folder as
+      // `index.md`; leaf pages stay as `<slug>.md`.
+      const relFilePath = hasChildren ? `${dir}/${slug}/index.md` : `${dir}/${slug}.md`
       let frontmatter
       if (isEmptyParent) {
         // Synthetic parent page: covers a missing URL segment so descendants
@@ -4228,28 +4841,83 @@ function stageOrganized(organized, stagingDir, opts = {}) {
       fs.writeFileSync(absPath, matter.stringify('', frontmatter))
       counts.fileCount++
 
+      // The sidebar entry is `slug` for both leaves (`<slug>.md`) and parents
+      // (the `<slug>/` folder) — `index` is never listed, it's implicit.
       if (!byDir.has(dir)) byDir.set(dir, [])
       byDir.get(dir).push(slug)
     }
 
-    const children = page.pages || []
-    if (children.length > 0) {
+    if (hasChildren) {
       const subDir = `${dir}/${slug}`
       for (const child of children) writePage(child, subDir, topDir, true)
     }
   }
 
+  // gitto derives a page's slug from the folder BASENAME alone (getSlugFromPath),
+  // so two category folders that share a basename — e.g. docs/Fundamentals and
+  // docs/cockroach/Fundamentals — both slug to `fundamentals` and collide. The
+  // deeper one renders as an unnamed, content-less placeholder parent.
+  // Namespace every NESTED category folder by its ancestor path so its basename
+  // (and thus slug) is globally unique, and back it with an `index.md` so it's a
+  // real titled parent. Top-level categories are matched by title, not slug, so
+  // their folders are left untouched.
+  const usedContainerSlugs = new Set(slugFor.values())
+  const segRename = new Map() // original path prefix -> unique folder name
+  const nestedContainers = new Map() // renamed dir (rel to stagingDir) -> display title
+  const uniqueContainerName = (base) => {
+    let name = base || 'section'
+    if (usedContainerSlugs.has(name)) {
+      let n = 2
+      while (usedContainerSlugs.has(`${name}-${n}`)) n++
+      name = `${name}-${n}`
+    }
+    usedContainerSlugs.add(name)
+    return name
+  }
+  const namespaceSubDir = (topDir, subDir) => {
+    const segs = subDir.split('/').filter(Boolean)
+    const out = []
+    for (let i = 0; i < segs.length; i++) {
+      if (i === 0) {
+        out.push(segs[0])
+        continue
+      }
+      const origPrefix = segs.slice(0, i + 1).join('/')
+      let name = segRename.get(origPrefix)
+      if (!name) {
+        name = uniqueContainerName(kebabCase(segs.slice(0, i + 1).join('-')))
+        segRename.set(origPrefix, name)
+        nestedContainers.set(`${topDir}/${[...out, name].join('/')}`, segs[i])
+      }
+      out.push(name)
+    }
+    return out.join('/')
+  }
+
   for (const cat of eligibleCategories) {
     const { topDir, subDir } = routeCategory(cat.title)
-    const dir = subDir ? `${topDir}/${subDir}` : topDir
-    if (subDir) {
+    const routedSubDir = subDir ? namespaceSubDir(topDir, subDir) : subDir
+    const dir = routedSubDir ? `${topDir}/${routedSubDir}` : topDir
+    if (routedSubDir) {
       if (!subDirsByTopDir.has(topDir)) subDirsByTopDir.set(topDir, [])
-      if (!subDirsByTopDir.get(topDir).includes(subDir)) subDirsByTopDir.get(topDir).push(subDir)
+      if (!subDirsByTopDir.get(topDir).includes(routedSubDir)) subDirsByTopDir.get(topDir).push(routedSubDir)
     }
 
     for (const page of cat.pages || []) writePage(page, dir, topDir, false)
   }
 
+  // Back each namespaced nested-category folder with a titled parent page so
+  // gitto renders it named instead of as an empty placeholder. No x-import —
+  // these are pure nav containers, so the body stays empty and format skips them.
+  for (const [relDir, title] of nestedContainers) {
+    const idxPath = path.join(stagingDir, relDir, 'index.md')
+    if (fs.existsSync(idxPath)) continue
+    fs.mkdirSync(path.dirname(idxPath), { recursive: true })
+    const fm = { title, icon: formatIconClass(pickIcon(path.basename(relDir), title)) }
+    fs.writeFileSync(idxPath, matter.stringify('', fm))
+    counts.fileCount++
+  }
+
   // Per-directory _order.yaml preserves input order in the sidebar.
   for (const [dir, slugs] of byDir) {
     const orderPath = path.join(stagingDir, dir, '_order.yaml')
@@ -4310,6 +4978,10 @@ function relocateChangelogDir(stagingDir) {
       if (!entry.name.endsWith('.md')) continue
       const parsed = matter(fs.readFileSync(full, 'utf-8'))
       if (!parsed.data || !parsed.data['x-import']) continue
+      if (entry.name === 'index.md' && ancestors.length > 0) {
+        pages.push({ ancestors: ancestors.slice(0, -1), slug: ancestors.at(-1), content: parsed.content, data: parsed.data })
+        continue
+      }
       const slug = entry.name.replace(/\.md$/, '')
       pages.push({ ancestors, slug, content: parsed.content, data: parsed.data })
     }
@@ -4632,7 +5304,36 @@ function makeIconPicker() {
   }
 }
 
-export const __test__ = { discoverLlmsTxt, mergeValidHits, resolveRedirectedSourceUrl }
+export const __test__ = {
+  discoverLlmsTxt,
+  mergeValidHits,
+  resolveRedirectedSourceUrl,
+  resolveDocsBaseUrl,
+  inCandidateScope,
+  tryFernNav,
+  sniffOasContent,
+  buildWellKnownOasProbeUrls,
+  probeWellKnownOasPaths,
+  extractOasUrlsFromHtml,
+  extractMintlifyOasCandidates,
+  sitemapUrlsToKnownUrls,
+  extractOasSpecUrlsFromParsed,
+  downloadOasSpecs,
+  injectSectionLandingPages,
+  parseMetaRefreshTarget,
+  parseHtmlCanonical,
+  isDiscoverableLink,
+  isAuthChromeUrl,
+  isDocsUnderApiPath,
+  filterUrlPagesTree,
+  stageOrganized,
+  urlIsApiReference,
+  urlIsChangelog,
+  reclassifyReferencePages,
+  extractChangelogFromSections,
+  decodeEntities,
+  stripTags,
+}
 
 function formatDuration(ms) {
   const safe = Math.max(0, ms)