@readme/cli 0.0.26 → 0.0.27
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/package.json +3 -1
- package/src/cli.js +5 -0
- package/src/commands/components.js +288 -0
- package/src/commands/import.js +410 -34
- package/src/commands/lint.js +32 -35
- package/src/commands/pretty.js +314 -77
- package/src/prompts/index.js +86 -11
- package/src/utils/lint.js +33 -33
- package/src/utils/marketplace.js +208 -0
- package/src/utils/multi-spinner.js +71 -0
- package/src/utils/promise-pool.js +17 -0
- package/src/validators/numbering.js +64 -66
package/src/commands/import.js
CHANGED
|
@@ -7,7 +7,7 @@ import { Option } from 'commander'
|
|
|
7
7
|
import { query } from '@anthropic-ai/claude-agent-sdk'
|
|
8
8
|
import matter from 'gray-matter'
|
|
9
9
|
import * as styles from '../utils/styles.js'
|
|
10
|
-
import { syncOas
|
|
10
|
+
import { syncOas } from './oas-sync.js'
|
|
11
11
|
import OASNormalize from 'oas-normalize'
|
|
12
12
|
import { slotOrphansPrompt, iconizeNavPrompt, organizeFromSectionsPrompt, organizeFromScratchPrompt, stripCodeFences } from '../prompts/index.js'
|
|
13
13
|
|
|
@@ -20,7 +20,7 @@ export const skipBootstrap = true
|
|
|
20
20
|
export function args(cmd) {
|
|
21
21
|
cmd.requiredOption('--source <url-or-file>', 'URL to import from, or path to a local OpenAPI spec (.json/.yaml/.yml)')
|
|
22
22
|
cmd.option('-o, --output <path>', 'Output zip path (defaults to <basename>-readme.zip in cwd)')
|
|
23
|
-
cmd.
|
|
23
|
+
cmd.addOption(new Option('-m, --model <name>', 'Claude model alias: haiku, sonnet, opus').choices(['haiku', 'sonnet', 'opus']).default('sonnet'))
|
|
24
24
|
cmd.option('--firecrawl-key <key>', 'Firecrawl API key (or set FIRECRAWL_API_KEY env var) — enables JS-rendered sidebar scraping')
|
|
25
25
|
cmd.option('--skip-api-reference', 'Drop pages routed to the API Reference / reference dir. Use when uploading the OAS spec separately.')
|
|
26
26
|
// Internal dev-only flag: skip the zip, keep staging, and boot the dev server
|
|
@@ -95,14 +95,42 @@ export async function importDocs(options) {
|
|
|
95
95
|
})
|
|
96
96
|
console.log()
|
|
97
97
|
|
|
98
|
+
let sitemapUrl = null
|
|
99
|
+
let sitemapKnownUrls = []
|
|
98
100
|
if (!llms) {
|
|
99
|
-
|
|
101
|
+
// Sitemap fallback: gives us a flat URL list to anchor the scrape (slot
|
|
102
|
+
// orphans by path) even though it lacks llms.txt's section structure.
|
|
103
|
+
// Scope is narrowed to the docs subtree of the source URL when possible
|
|
104
|
+
// so we don't pull in marketing/blog pages from a whole-site sitemap.
|
|
105
|
+
const sitemapCandidates = buildSitemapCandidates(sourceUrl)
|
|
106
|
+
const scopePrefix = deriveSitemapScope(sourceUrl)
|
|
107
|
+
styles.info(
|
|
108
|
+
`No llms.txt — checking for sitemap.xml (${sitemapCandidates.length} candidate${sitemapCandidates.length === 1 ? '' : 's'})${scopePrefix ? `, scoped to ${styles.bold(scopePrefix)}` : ''}...`,
|
|
109
|
+
)
|
|
110
|
+
const sitemapResult = await timePhase('fetch sitemap.xml', async () => {
|
|
111
|
+
for (const candidate of sitemapCandidates) {
|
|
112
|
+
const res = await fetchSitemap(candidate, scopePrefix)
|
|
113
|
+
if (res.ok && res.urls.length > 0) return { url: candidate, urls: res.urls }
|
|
114
|
+
const reason = res.ok ? '0 urls in scope' : res.status ? `HTTP ${res.status}` : res.error || 'failed'
|
|
115
|
+
styles.info(styles.dim(` ${candidate} → ${reason}`))
|
|
116
|
+
}
|
|
117
|
+
return null
|
|
118
|
+
})
|
|
119
|
+
console.log()
|
|
120
|
+
if (sitemapResult) {
|
|
121
|
+
sitemapUrl = sitemapResult.url
|
|
122
|
+
sitemapKnownUrls = sitemapUrlsToKnownUrls(sitemapResult.urls)
|
|
123
|
+
styles.ok(`Found sitemap.xml at ${styles.bold(sitemapUrl)} — ${styles.bold(String(sitemapKnownUrls.length))} URL${sitemapKnownUrls.length === 1 ? '' : 's'} in scope.`)
|
|
124
|
+
} else {
|
|
125
|
+
styles.warning(`No llms.txt or sitemap.xml found — falling back to sidebar discovery via scrape.`)
|
|
126
|
+
}
|
|
100
127
|
} else {
|
|
101
128
|
styles.info(styles.dim(`Using ${llmsUrl}.`))
|
|
102
129
|
}
|
|
103
130
|
|
|
104
131
|
if (debugSnapshots) {
|
|
105
132
|
debugSnapshots['01-llms-parsed.json'] = { llmsUrl, parsed: llms ? llms.parsed : null }
|
|
133
|
+
debugSnapshots['01b-sitemap.json'] = { sitemapUrl, urls: sitemapKnownUrls }
|
|
106
134
|
}
|
|
107
135
|
|
|
108
136
|
let knownUrls = []
|
|
@@ -130,6 +158,8 @@ export async function importDocs(options) {
|
|
|
130
158
|
if (dropped > 0) {
|
|
131
159
|
styles.info(`${styles.dim(`Collapsed ${dropped} anchor/query duplicates → ${knownUrls.length} unique pages.`)}`)
|
|
132
160
|
}
|
|
161
|
+
} else if (sitemapKnownUrls.length > 0) {
|
|
162
|
+
knownUrls = sitemapKnownUrls
|
|
133
163
|
}
|
|
134
164
|
|
|
135
165
|
console.log()
|
|
@@ -265,10 +295,12 @@ export async function importDocs(options) {
|
|
|
265
295
|
)
|
|
266
296
|
}
|
|
267
297
|
}
|
|
268
|
-
} else if (!llms) {
|
|
269
|
-
throw new Error(`No llms.txt and the sidebar scrape found no usable structure — can't import ${sourceUrl.toString()}.`)
|
|
270
|
-
} else {
|
|
298
|
+
} else if (!llms && knownUrls.length === 0) {
|
|
299
|
+
throw new Error(`No llms.txt or sitemap.xml and the sidebar scrape found no usable structure — can't import ${sourceUrl.toString()}.`)
|
|
300
|
+
} else if (llms) {
|
|
271
301
|
styles.warning(`Couldn't extract a useful nav — falling back to llms.txt-based organization.`)
|
|
302
|
+
} else {
|
|
303
|
+
styles.warning(`Couldn't extract a useful nav — falling back to sitemap URL clustering.`)
|
|
272
304
|
}
|
|
273
305
|
console.log()
|
|
274
306
|
|
|
@@ -281,11 +313,25 @@ export async function importDocs(options) {
|
|
|
281
313
|
title: (llms && llms.parsed.title) || null,
|
|
282
314
|
categories: scraped.categories.map((c) => ({ title: c.title, icon: null, pages: c.pages })),
|
|
283
315
|
}
|
|
284
|
-
} else {
|
|
316
|
+
} else if (llms) {
|
|
285
317
|
const fastPath = sectionsLookUsable(llms.parsed.sections)
|
|
286
318
|
styles.info(`Organizing with Claude (${styles.bold(options.model)}, ${fastPath ? 'fast path: icons only' : 'full reorg'})...`)
|
|
287
319
|
organized = await timePhase('claude organize', () => organizeWithClaude(llms.parsed, options.model))
|
|
320
|
+
} else {
|
|
321
|
+
// Sitemap-only fallback: synthesize categories by clustering the URL
|
|
322
|
+
// paths. clusterByUrlPath returns null when the URLs don't split cleanly,
|
|
323
|
+
// in which case everything lands in a single Documentation bucket.
|
|
324
|
+
const clustered = clusterByUrlPath(knownUrls) || [{ title: 'Documentation', pages: knownUrls }]
|
|
325
|
+
organized = {
|
|
326
|
+
title: null,
|
|
327
|
+
categories: clustered.map((c) => ({ title: c.title, icon: null, pages: c.pages })),
|
|
328
|
+
}
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
for (const cat of organized.categories || []) {
|
|
332
|
+
cat.pages = nestByUrlHierarchy(cat.pages)
|
|
288
333
|
}
|
|
334
|
+
|
|
289
335
|
styles.ok(`Organized in ${styles.bold(formatDuration(Date.now() - organizeStart))}.`)
|
|
290
336
|
if (debugSnapshots) {
|
|
291
337
|
debugSnapshots['05-organized.json'] = organized
|
|
@@ -1573,6 +1619,73 @@ function collectFlat(page, out) {
|
|
|
1573
1619
|
}
|
|
1574
1620
|
}
|
|
1575
1621
|
|
|
1622
|
+
/**
|
|
1623
|
+
* Re-parent URL-bearing siblings by URL path so a page like `/foo/bar`
|
|
1624
|
+
* becomes a child of `/foo` when both appear in the same list.
|
|
1625
|
+
*
|
|
1626
|
+
* @example
|
|
1627
|
+
* nestByUrlHierarchy([
|
|
1628
|
+
* { title: 'CLI', url: 'https://x.com/getting-started/cli' },
|
|
1629
|
+
* { title: 'Overview', url: 'https://x.com/getting-started/cli/overview' },
|
|
1630
|
+
* ])
|
|
1631
|
+
* // [
|
|
1632
|
+
* // { title: 'CLI', url: '…/cli', pages: [
|
|
1633
|
+
* // { title: 'Overview', url: '…/cli/overview' },
|
|
1634
|
+
* // ]},
|
|
1635
|
+
* // ]
|
|
1636
|
+
*/
|
|
1637
|
+
function nestByUrlHierarchy(pages) {
|
|
1638
|
+
if (!pages || pages.length === 0) return pages
|
|
1639
|
+
|
|
1640
|
+
// Recurse first so group-only / already-nested subtrees get re-nested too.
|
|
1641
|
+
for (const p of pages) {
|
|
1642
|
+
if (p.pages && p.pages.length > 0) p.pages = nestByUrlHierarchy(p.pages)
|
|
1643
|
+
}
|
|
1644
|
+
|
|
1645
|
+
// Need at least two URL-bearing siblings to form a parent/child pair.
|
|
1646
|
+
const byPath = new Map()
|
|
1647
|
+
for (const p of pages) {
|
|
1648
|
+
if (!p.url) continue
|
|
1649
|
+
const k = normalizePath(p.url)
|
|
1650
|
+
if (!byPath.has(k)) byPath.set(k, p)
|
|
1651
|
+
}
|
|
1652
|
+
if (byPath.size < 2) return pages
|
|
1653
|
+
|
|
1654
|
+
const result = []
|
|
1655
|
+
for (const p of pages) {
|
|
1656
|
+
if (!p.url) {
|
|
1657
|
+
result.push(p)
|
|
1658
|
+
continue
|
|
1659
|
+
}
|
|
1660
|
+
let segs
|
|
1661
|
+
try {
|
|
1662
|
+
segs = new URL(p.url).pathname.split('/').filter(Boolean)
|
|
1663
|
+
} catch {
|
|
1664
|
+
result.push(p)
|
|
1665
|
+
continue
|
|
1666
|
+
}
|
|
1667
|
+
let parent = null
|
|
1668
|
+
for (let depth = segs.length - 1; depth >= 1; depth--) {
|
|
1669
|
+
const key = ('/' + segs.slice(0, depth).join('/'))
|
|
1670
|
+
.toLowerCase()
|
|
1671
|
+
.replace(/\.(md|mdx|html?)$/i, '')
|
|
1672
|
+
.replace(/\/$/, '')
|
|
1673
|
+
const candidate = byPath.get(key)
|
|
1674
|
+
if (candidate && candidate !== p) {
|
|
1675
|
+
parent = candidate
|
|
1676
|
+
break
|
|
1677
|
+
}
|
|
1678
|
+
}
|
|
1679
|
+
if (parent) {
|
|
1680
|
+
if (!parent.pages) parent.pages = []
|
|
1681
|
+
parent.pages.push(p)
|
|
1682
|
+
} else {
|
|
1683
|
+
result.push(p)
|
|
1684
|
+
}
|
|
1685
|
+
}
|
|
1686
|
+
return result
|
|
1687
|
+
}
|
|
1688
|
+
|
|
1576
1689
|
/**
|
|
1577
1690
|
* The scraped nav only contains top-level items; subcategory pages sit behind
|
|
1578
1691
|
* `>` chevrons and don't render on a cold fetch. For each llms.txt URL not
|
|
@@ -2262,6 +2375,185 @@ function parseLlmsTxt(body) {
|
|
|
2262
2375
|
return { title, sections }
|
|
2263
2376
|
}
|
|
2264
2377
|
|
|
2378
|
+
// Path segments that mark a docs scope. Used to find the right "base" for
|
|
2379
|
+
// filtering sitemap URLs: walking up from the source URL, the first segment
|
|
2380
|
+
// that matches becomes the scope prefix. Hits should stay narrow — `blog`
|
|
2381
|
+
// or `pricing` would over-include marketing pages.
|
|
2382
|
+
const WELL_KNOWN_DOC_SEGMENTS = new Set([
|
|
2383
|
+
'docs',
|
|
2384
|
+
'doc',
|
|
2385
|
+
'documentation',
|
|
2386
|
+
'guides',
|
|
2387
|
+
'guide',
|
|
2388
|
+
'references',
|
|
2389
|
+
'reference',
|
|
2390
|
+
'api',
|
|
2391
|
+
'apis',
|
|
2392
|
+
'api-reference',
|
|
2393
|
+
'api-docs',
|
|
2394
|
+
'learn',
|
|
2395
|
+
'tutorials',
|
|
2396
|
+
'tutorial',
|
|
2397
|
+
'developers',
|
|
2398
|
+
'developer',
|
|
2399
|
+
])
|
|
2400
|
+
|
|
2401
|
+
/**
|
|
2402
|
+
* Decide which path prefix to use when filtering sitemap URLs to "the docs
|
|
2403
|
+
* portion of the site." Walk up the source URL's path looking for the first
|
|
2404
|
+
* well-known docs segment (`/docs/`, `/guides/`, `/api-reference/`, …) and
|
|
2405
|
+
* use the prefix up to and including it. Returns null when nothing matches —
|
|
2406
|
+
* in that case we accept any URL on the same origin (no path filter).
|
|
2407
|
+
*/
|
|
2408
|
+
function deriveSitemapScope(sourceUrl) {
|
|
2409
|
+
const segs = sourceUrl.pathname.split('/').filter(Boolean)
|
|
2410
|
+
for (let i = segs.length - 1; i >= 0; i--) {
|
|
2411
|
+
if (WELL_KNOWN_DOC_SEGMENTS.has(segs[i].toLowerCase())) {
|
|
2412
|
+
return '/' + segs.slice(0, i + 1).join('/') + '/'
|
|
2413
|
+
}
|
|
2414
|
+
}
|
|
2415
|
+
return null
|
|
2416
|
+
}
|
|
2417
|
+
|
|
2418
|
+
/**
|
|
2419
|
+
* Mirror buildLlmsCandidates but for sitemap.xml: walk up the source path
|
|
2420
|
+
* appending `/sitemap.xml` at each level, ending at origin root.
|
|
2421
|
+
*/
|
|
2422
|
+
function buildSitemapCandidates(sourceUrl) {
|
|
2423
|
+
const out = []
|
|
2424
|
+
const seen = new Set()
|
|
2425
|
+
const add = (url) => {
|
|
2426
|
+
if (!seen.has(url)) {
|
|
2427
|
+
seen.add(url)
|
|
2428
|
+
out.push(url)
|
|
2429
|
+
}
|
|
2430
|
+
}
|
|
2431
|
+
const origin = sourceUrl.origin
|
|
2432
|
+
const segs = sourceUrl.pathname.split('/').filter(Boolean)
|
|
2433
|
+
for (let i = segs.length; i >= 0; i--) {
|
|
2434
|
+
const prefix = segs.slice(0, i).join('/')
|
|
2435
|
+
add(`${origin}${prefix ? '/' + prefix : ''}/sitemap.xml`)
|
|
2436
|
+
}
|
|
2437
|
+
return out
|
|
2438
|
+
}
|
|
2439
|
+
|
|
2440
|
+
/**
|
|
2441
|
+
* Best-effort fetch of a sitemap. Handles both `<urlset>` (a flat list of
|
|
2442
|
+
* page URLs) and `<sitemapindex>` (which references child sitemaps — we
|
|
2443
|
+
* recurse one level, capped at 10 children to avoid runaway fetches).
|
|
2444
|
+
*
|
|
2445
|
+
* `scopePrefix` is a path string ending in `/`, or null. When set, only URLs
|
|
2446
|
+
* whose pathname starts with the prefix and share the sitemap's origin are
|
|
2447
|
+
* returned. Returns { ok, urls } on success.
|
|
2448
|
+
*/
|
|
2449
|
+
async function fetchSitemap(sitemapUrl, scopePrefix, depth = 0) {
|
|
2450
|
+
try {
|
|
2451
|
+
const res = await fetch(sitemapUrl, {
|
|
2452
|
+
redirect: 'follow',
|
|
2453
|
+
headers: { 'User-Agent': 'readme-cli-import' },
|
|
2454
|
+
})
|
|
2455
|
+
if (!res.ok) return { ok: false, status: res.status }
|
|
2456
|
+
const text = await res.text()
|
|
2457
|
+
const parsed = parseSitemapXml(text)
|
|
2458
|
+
if (!parsed) return { ok: false, error: 'not a sitemap' }
|
|
2459
|
+
|
|
2460
|
+
if (parsed.kind === 'urlset') {
|
|
2461
|
+
const urls = parsed.locs.filter((u) => urlInScope(u, sitemapUrl, scopePrefix))
|
|
2462
|
+
return { ok: true, urls }
|
|
2463
|
+
}
|
|
2464
|
+
|
|
2465
|
+
if (depth >= 1) return { ok: true, urls: [] }
|
|
2466
|
+
const children = parsed.locs.slice(0, 10)
|
|
2467
|
+
const all = []
|
|
2468
|
+
for (const childUrl of children) {
|
|
2469
|
+
const child = await fetchSitemap(childUrl, scopePrefix, depth + 1)
|
|
2470
|
+
if (child.ok) all.push(...child.urls)
|
|
2471
|
+
}
|
|
2472
|
+
return { ok: true, urls: all }
|
|
2473
|
+
} catch (e) {
|
|
2474
|
+
return { ok: false, error: e.message }
|
|
2475
|
+
}
|
|
2476
|
+
}
|
|
2477
|
+
|
|
2478
|
+
/**
|
|
2479
|
+
* Parse a sitemap with a regex — the format is small and well-defined enough
|
|
2480
|
+
* that a dedicated XML parser would be overkill. Returns
|
|
2481
|
+
* { kind: 'urlset' | 'sitemapindex', locs: string[] } or null when the body
|
|
2482
|
+
* doesn't look like a sitemap (e.g. server returned a 200 HTML 404 page).
|
|
2483
|
+
*/
|
|
2484
|
+
function parseSitemapXml(body) {
|
|
2485
|
+
const isIndex = /<sitemapindex[\s>]/i.test(body)
|
|
2486
|
+
const isUrlset = /<urlset[\s>]/i.test(body)
|
|
2487
|
+
if (!isIndex && !isUrlset) return null
|
|
2488
|
+
const locs = []
|
|
2489
|
+
const re = /<loc>\s*([^<]+?)\s*<\/loc>/gi
|
|
2490
|
+
let m
|
|
2491
|
+
while ((m = re.exec(body)) !== null) {
|
|
2492
|
+
const url = decodeXmlEntities(m[1].trim())
|
|
2493
|
+
if (url) locs.push(url)
|
|
2494
|
+
}
|
|
2495
|
+
return { kind: isIndex ? 'sitemapindex' : 'urlset', locs }
|
|
2496
|
+
}
|
|
2497
|
+
|
|
2498
|
+
function decodeXmlEntities(s) {
|
|
2499
|
+
return s
|
|
2500
|
+
.replace(/&/g, '&')
|
|
2501
|
+
.replace(/</g, '<')
|
|
2502
|
+
.replace(/>/g, '>')
|
|
2503
|
+
.replace(/"/g, '"')
|
|
2504
|
+
.replace(/'/g, "'")
|
|
2505
|
+
}
|
|
2506
|
+
|
|
2507
|
+
function urlInScope(urlStr, sitemapUrl, scopePrefix) {
|
|
2508
|
+
let u
|
|
2509
|
+
try {
|
|
2510
|
+
u = new URL(urlStr)
|
|
2511
|
+
} catch {
|
|
2512
|
+
return false
|
|
2513
|
+
}
|
|
2514
|
+
let sm
|
|
2515
|
+
try {
|
|
2516
|
+
sm = new URL(sitemapUrl)
|
|
2517
|
+
} catch {
|
|
2518
|
+
return false
|
|
2519
|
+
}
|
|
2520
|
+
if (u.origin !== sm.origin) return false
|
|
2521
|
+
if (scopePrefix && !u.pathname.startsWith(scopePrefix)) return false
|
|
2522
|
+
return true
|
|
2523
|
+
}
|
|
2524
|
+
|
|
2525
|
+
/**
|
|
2526
|
+
* Convert a flat sitemap URL list into the `{title, url, description}` shape
|
|
2527
|
+
* the rest of the pipeline expects for knownUrls. Title is derived from the
|
|
2528
|
+
* last non-empty path segment, with file extensions stripped. Index-style
|
|
2529
|
+
* URLs that resolve to the scope root itself are dropped.
|
|
2530
|
+
*/
|
|
2531
|
+
function sitemapUrlsToKnownUrls(urls) {
|
|
2532
|
+
const out = []
|
|
2533
|
+
const seen = new Set()
|
|
2534
|
+
for (const url of urls) {
|
|
2535
|
+
let pathname
|
|
2536
|
+
try {
|
|
2537
|
+
pathname = new URL(url).pathname
|
|
2538
|
+
} catch {
|
|
2539
|
+
continue
|
|
2540
|
+
}
|
|
2541
|
+
const norm = normalizePath(url)
|
|
2542
|
+
if (seen.has(norm)) continue
|
|
2543
|
+
seen.add(norm)
|
|
2544
|
+
const segs = pathname.split('/').filter(Boolean)
|
|
2545
|
+
const last = segs[segs.length - 1] || ''
|
|
2546
|
+
const slug = last.replace(/\.[a-z0-9]+$/i, '')
|
|
2547
|
+
if (!slug) continue
|
|
2548
|
+
out.push({
|
|
2549
|
+
title: titleCase(slug),
|
|
2550
|
+
url,
|
|
2551
|
+
description: null,
|
|
2552
|
+
})
|
|
2553
|
+
}
|
|
2554
|
+
return out
|
|
2555
|
+
}
|
|
2556
|
+
|
|
2265
2557
|
/**
|
|
2266
2558
|
* Write the organized hierarchy to disk as git-format markdown stubs — just
|
|
2267
2559
|
* frontmatter, no body yet. docs/ pages go under docs/<Category>/<slug>.md;
|
|
@@ -2287,12 +2579,27 @@ function printPagesTree(pages, indentLevel) {
|
|
|
2287
2579
|
|
|
2288
2580
|
function stageOrganized(organized, stagingDir, opts = {}) {
|
|
2289
2581
|
const pickIcon = makeIconPicker()
|
|
2290
|
-
const usedSlugs = new Set() // cross-dir: duplicates validator is global
|
|
2291
2582
|
const byDir = new Map()
|
|
2292
2583
|
const subDirsByTopDir = new Map()
|
|
2293
2584
|
const counts = { fileCount: 0, skippedApiRef: 0 }
|
|
2294
2585
|
const skipApiReference = !!opts.skipApiReference
|
|
2295
2586
|
|
|
2587
|
+
const eligibleCategories = (organized.categories || []).filter((cat) => {
|
|
2588
|
+
if (skipApiReference && routeCategory(cat.title).topDir === 'reference') {
|
|
2589
|
+
counts.skippedApiRef += countPagesDeep(cat.pages || [])
|
|
2590
|
+
return false
|
|
2591
|
+
}
|
|
2592
|
+
return true
|
|
2593
|
+
})
|
|
2594
|
+
|
|
2595
|
+
// Slug names must be unique
|
|
2596
|
+
const slugFor = ensureUniqueSlugs(eligibleCategories)
|
|
2597
|
+
|
|
2598
|
+
const urlWidth = Math.max(...[...slugFor.keys()].map((p) => (p.url || '(group-only)').length))
|
|
2599
|
+
for (const [page, slug] of slugFor) {
|
|
2600
|
+
console.log(`${(page.url || '(group-only)').padEnd(urlWidth)} → ${slug}`)
|
|
2601
|
+
}
|
|
2602
|
+
|
|
2296
2603
|
/**
|
|
2297
2604
|
* Write a page (and its descendants) into `dir`. A page with children gets
|
|
2298
2605
|
* its own subfolder named after its slug; the parent page lives at
|
|
@@ -2300,8 +2607,7 @@ function stageOrganized(organized, stagingDir, opts = {}) {
|
|
|
2300
2607
|
* This matches git-format's on-disk convention for nested sidebars.
|
|
2301
2608
|
*/
|
|
2302
2609
|
function writePage(page, dir, topDir, isSubPage = false) {
|
|
2303
|
-
const slug =
|
|
2304
|
-
usedSlugs.add(slug)
|
|
2610
|
+
const slug = slugFor.get(page)
|
|
2305
2611
|
|
|
2306
2612
|
// Group-only nodes (e.g. a resource sub-group within API Reference) have
|
|
2307
2613
|
// no backing page on the source site — they're pure sidebar containers.
|
|
@@ -2318,7 +2624,7 @@ function stageOrganized(organized, stagingDir, opts = {}) {
|
|
|
2318
2624
|
// git-format convention for metadata the schema doesn't know about.
|
|
2319
2625
|
frontmatter['x-import'] = toBrowsableUrl(page.url)
|
|
2320
2626
|
// hide pages that need import
|
|
2321
|
-
frontmatter.hidden = true
|
|
2627
|
+
frontmatter.hidden = 'true'
|
|
2322
2628
|
|
|
2323
2629
|
const absPath = path.join(stagingDir, relFilePath)
|
|
2324
2630
|
fs.mkdirSync(path.dirname(absPath), { recursive: true })
|
|
@@ -2336,12 +2642,8 @@ function stageOrganized(organized, stagingDir, opts = {}) {
|
|
|
2336
2642
|
}
|
|
2337
2643
|
}
|
|
2338
2644
|
|
|
2339
|
-
for (const cat of
|
|
2645
|
+
for (const cat of eligibleCategories) {
|
|
2340
2646
|
const { topDir, subDir } = routeCategory(cat.title)
|
|
2341
|
-
if (skipApiReference && topDir === 'reference') {
|
|
2342
|
-
counts.skippedApiRef += countPagesDeep(cat.pages || [])
|
|
2343
|
-
continue
|
|
2344
|
-
}
|
|
2345
2647
|
const dir = subDir ? `${topDir}/${subDir}` : topDir
|
|
2346
2648
|
if (subDir) {
|
|
2347
2649
|
if (!subDirsByTopDir.has(topDir)) subDirsByTopDir.set(topDir, [])
|
|
@@ -2420,30 +2722,104 @@ function buildFrontmatter(topDir, page, slug, pickIcon, opts = {}) {
|
|
|
2420
2722
|
}
|
|
2421
2723
|
|
|
2422
2724
|
/**
|
|
2423
|
-
*
|
|
2424
|
-
*
|
|
2725
|
+
* Extract URL path segments for slug planning. Strips file extensions and
|
|
2726
|
+
* leading numeric prefixes (e.g. `01-intro` → `intro`) the same way the
|
|
2727
|
+
* legacy deriveSlug did, so the depth-1 result is byte-for-byte compatible.
|
|
2425
2728
|
*/
|
|
2426
|
-
function
|
|
2427
|
-
|
|
2729
|
+
function extractUrlPathSegments(url) {
|
|
2730
|
+
if (!url) return []
|
|
2428
2731
|
try {
|
|
2429
|
-
|
|
2430
|
-
|
|
2431
|
-
|
|
2432
|
-
|
|
2433
|
-
|
|
2434
|
-
|
|
2732
|
+
return new URL(url).pathname
|
|
2733
|
+
.split('/')
|
|
2734
|
+
.filter(Boolean)
|
|
2735
|
+
.map((s) => s.replace(/\.(md|mdx|html?)$/i, '').replace(/^\d+[-_.]/, ''))
|
|
2736
|
+
.filter(Boolean)
|
|
2737
|
+
} catch {
|
|
2738
|
+
return []
|
|
2739
|
+
}
|
|
2435
2740
|
}
|
|
2436
2741
|
|
|
2437
2742
|
/**
|
|
2438
|
-
*
|
|
2439
|
-
*
|
|
2440
|
-
*
|
|
2743
|
+
* Compute a globally-unique slug for every page in the tree.
|
|
2744
|
+
*
|
|
2745
|
+
* Base slug is the last URL segment (kebab-cased). When two or more pages
|
|
2746
|
+
* share a base, we prepend the next-up URL segment to every member of the
|
|
2747
|
+
* colliding group and recheck
|
|
2748
|
+
*
|
|
2749
|
+
* @example
|
|
2750
|
+
* ensureUniqueSlugs([
|
|
2751
|
+
* { title: 'Getting Started', pages: [
|
|
2752
|
+
* { title: 'Overview', url: 'https://acme.dev/getting-started/overview' },
|
|
2753
|
+
* { title: 'Install', url: 'https://acme.dev/getting-started/install' },
|
|
2754
|
+
* ]},
|
|
2755
|
+
* { title: 'Guides', pages: [
|
|
2756
|
+
* { title: 'Overview', url: 'https://acme.dev/guides/overview' },
|
|
2757
|
+
* ]},
|
|
2758
|
+
* ])
|
|
2759
|
+
* // Map {
|
|
2760
|
+
* // <Overview (Getting Started)> => 'getting-started-overview', // expanded (collides)
|
|
2761
|
+
* // <Install> => 'install', // unique, no expansion
|
|
2762
|
+
* // <Overview (Guides)> => 'guides-overview', // expanded (collides)
|
|
2763
|
+
* // }
|
|
2441
2764
|
*/
|
|
2442
|
-
function
|
|
2443
|
-
|
|
2444
|
-
|
|
2445
|
-
|
|
2446
|
-
|
|
2765
|
+
function ensureUniqueSlugs(categories) {
|
|
2766
|
+
const entries = []
|
|
2767
|
+
const walk = (pages) => {
|
|
2768
|
+
for (const p of pages || []) {
|
|
2769
|
+
entries.push({ page: p, segments: extractUrlPathSegments(p.url), fallback: p.title, depth: 1 })
|
|
2770
|
+
if (p.pages) walk(p.pages)
|
|
2771
|
+
}
|
|
2772
|
+
}
|
|
2773
|
+
for (const c of categories || []) walk(c.pages)
|
|
2774
|
+
|
|
2775
|
+
const slugFor = (e) => {
|
|
2776
|
+
if (e.segments.length === 0) return kebabCase(e.fallback || 'page') || 'page'
|
|
2777
|
+
const start = Math.max(0, e.segments.length - e.depth)
|
|
2778
|
+
return kebabCase(e.segments.slice(start).join('-')) || 'page'
|
|
2779
|
+
}
|
|
2780
|
+
|
|
2781
|
+
// Iteratively grow any colliding group by one segment until stable. Each
|
|
2782
|
+
// pass only grows pages that still have headroom (depth < segments.length);
|
|
2783
|
+
// a page stuck at full depth stays put while its peers keep expanding.
|
|
2784
|
+
while (true) {
|
|
2785
|
+
const groups = new Map()
|
|
2786
|
+
for (const e of entries) {
|
|
2787
|
+
const s = slugFor(e)
|
|
2788
|
+
if (!groups.has(s)) groups.set(s, [])
|
|
2789
|
+
groups.get(s).push(e)
|
|
2790
|
+
}
|
|
2791
|
+
let grew = false
|
|
2792
|
+
for (const group of groups.values()) {
|
|
2793
|
+
if (group.length < 2) continue
|
|
2794
|
+
for (const e of group) {
|
|
2795
|
+
if (e.depth < e.segments.length) {
|
|
2796
|
+
e.depth++
|
|
2797
|
+
grew = true
|
|
2798
|
+
}
|
|
2799
|
+
}
|
|
2800
|
+
}
|
|
2801
|
+
if (!grew) break
|
|
2802
|
+
}
|
|
2803
|
+
|
|
2804
|
+
const describe = (x) => `"${x.page.title || '(untitled)'}"${x.page.url ? ` <${x.page.url}>` : ' (group-only)'}`
|
|
2805
|
+
const used = new Set()
|
|
2806
|
+
const result = new Map()
|
|
2807
|
+
for (const e of entries) {
|
|
2808
|
+
const base = slugFor(e)
|
|
2809
|
+
let slug = base
|
|
2810
|
+
if (used.has(slug)) {
|
|
2811
|
+
let n = 2
|
|
2812
|
+
while (used.has(`${slug}-${n}`)) n++
|
|
2813
|
+
slug = `${base}-${n}`
|
|
2814
|
+
styles.error(
|
|
2815
|
+
`Slug collision after segment expansion: ${base} — falling back to ${slug} for ${describe(e)}. ` +
|
|
2816
|
+
`Pages were deduped by path before organize, so this indicates the organize step produced duplicates.`,
|
|
2817
|
+
)
|
|
2818
|
+
}
|
|
2819
|
+
used.add(slug)
|
|
2820
|
+
result.set(e.page, slug)
|
|
2821
|
+
}
|
|
2822
|
+
return result
|
|
2447
2823
|
}
|
|
2448
2824
|
|
|
2449
2825
|
// Values YAML interprets as non-strings need quoting when used as _order entries.
|
package/src/commands/lint.js
CHANGED
|
@@ -1,22 +1,22 @@
|
|
|
1
|
-
import { createRequire } from 'node:module'
|
|
2
|
-
import { collectFiles, runValidators } from '../utils/lint.js'
|
|
3
|
-
import { createHumanReporter, createJsonReporter, createGithubReporter } from '../utils/reporter.js'
|
|
4
|
-
import { printHeader, isAgenticCli } from '../utils/eyes.js'
|
|
5
|
-
import * as styles from '../utils/styles.js'
|
|
1
|
+
import { createRequire } from 'node:module'
|
|
2
|
+
import { collectFiles, runValidators } from '../utils/lint.js'
|
|
3
|
+
import { createHumanReporter, createJsonReporter, createGithubReporter } from '../utils/reporter.js'
|
|
4
|
+
import { printHeader, isAgenticCli } from '../utils/eyes.js'
|
|
5
|
+
import * as styles from '../utils/styles.js'
|
|
6
6
|
|
|
7
|
-
const require = createRequire(import.meta.url)
|
|
8
|
-
const pkg = require('../../package.json')
|
|
7
|
+
const require = createRequire(import.meta.url)
|
|
8
|
+
const pkg = require('../../package.json')
|
|
9
9
|
|
|
10
|
-
export const command = 'lint'
|
|
11
|
-
export const order = 1
|
|
12
|
-
export const aliases = ['validate']
|
|
13
|
-
export const category = 'Linting'
|
|
14
|
-
export const description = 'Lint and validate your ReadMe docs'
|
|
10
|
+
export const command = 'lint'
|
|
11
|
+
export const order = 1
|
|
12
|
+
export const aliases = ['validate']
|
|
13
|
+
export const category = 'Linting'
|
|
14
|
+
export const description = 'Lint and validate your ReadMe docs'
|
|
15
15
|
|
|
16
16
|
export function args(cmd) {
|
|
17
|
-
cmd.option('--json', 'Output results as JSON (for CI and automation)')
|
|
18
|
-
cmd.option('--github', 'Output a GitHub PR comment body as markdown')
|
|
19
|
-
cmd.option('--fix', 'Automatically fix issues where possible')
|
|
17
|
+
cmd.option('--json', 'Output results as JSON (for CI and automation)')
|
|
18
|
+
cmd.option('--github', 'Output a GitHub PR comment body as markdown')
|
|
19
|
+
cmd.option('--fix', 'Automatically fix issues where possible')
|
|
20
20
|
}
|
|
21
21
|
|
|
22
22
|
/**
|
|
@@ -25,46 +25,43 @@ export function args(cmd) {
|
|
|
25
25
|
* exit on their own terms.
|
|
26
26
|
*
|
|
27
27
|
* @param {object} [opts]
|
|
28
|
-
* @param {string} [opts.cwd]
|
|
29
|
-
* @param {boolean} [opts.fix]
|
|
28
|
+
* @param {string} [opts.cwd] Repo root to lint (defaults to process.cwd()).
|
|
29
|
+
* @param {boolean} [opts.fix] Apply autofixes where validators support it.
|
|
30
|
+
* @param {boolean} [opts.nonInteractive] Skip any interactive prompts (e.g. rename confirmation). Fixes that would normally prompt are not applied.
|
|
30
31
|
* @param {(file: string) => void} [opts.onFile] Called per file before validation.
|
|
31
32
|
* @param {() => void} [opts.onBeforeCrossFile] Called once before cross-file checks run.
|
|
32
33
|
* @returns {Promise<{ gitRoot: string, files: string[], results: object[], hasErrors: boolean }>}
|
|
33
34
|
*/
|
|
34
|
-
export async function lint({ cwd, fix = false, onFile, onBeforeCrossFile } = {}) {
|
|
35
|
-
const gitRoot = cwd || process.cwd()
|
|
36
|
-
const files = collectFiles(gitRoot)
|
|
37
|
-
const results = await runValidators(files, gitRoot, { onFile, onBeforeCrossFile, fix })
|
|
38
|
-
const hasErrors = results.some((r) => r.severity !== 'warning')
|
|
39
|
-
return { gitRoot, files, results, hasErrors }
|
|
35
|
+
export async function lint({ cwd, fix = false, nonInteractive = false, onFile, onBeforeCrossFile } = {}) {
|
|
36
|
+
const gitRoot = cwd || process.cwd()
|
|
37
|
+
const files = collectFiles(gitRoot)
|
|
38
|
+
const results = await runValidators(files, gitRoot, { onFile, onBeforeCrossFile, fix, nonInteractive })
|
|
39
|
+
const hasErrors = results.some((r) => r.severity !== 'warning')
|
|
40
|
+
return { gitRoot, files, results, hasErrors }
|
|
40
41
|
}
|
|
41
42
|
|
|
42
43
|
export async function run(options, _cmd, ctx) {
|
|
43
|
-
const { gitRoot } = ctx
|
|
44
|
+
const { gitRoot } = ctx
|
|
44
45
|
|
|
45
46
|
if (!options.json && !options.github && !isAgenticCli()) {
|
|
46
|
-
printHeader({ version: pkg.version, binName: styles.binName(), indent: ' ' })
|
|
47
|
-
console.log()
|
|
47
|
+
printHeader({ version: pkg.version, binName: styles.binName(), indent: ' ' })
|
|
48
|
+
console.log()
|
|
48
49
|
}
|
|
49
50
|
|
|
50
|
-
const reporter = options.github
|
|
51
|
-
? createGithubReporter()
|
|
52
|
-
: options.json
|
|
53
|
-
? createJsonReporter()
|
|
54
|
-
: createHumanReporter();
|
|
51
|
+
const reporter = options.github ? createGithubReporter() : options.json ? createJsonReporter() : createHumanReporter()
|
|
55
52
|
|
|
56
53
|
const { files, results, hasErrors } = await lint({
|
|
57
54
|
cwd: gitRoot,
|
|
58
55
|
fix: options.fix,
|
|
59
56
|
onFile: (f) => reporter.onFile(f),
|
|
60
57
|
onBeforeCrossFile: () => reporter.pause(),
|
|
61
|
-
})
|
|
58
|
+
})
|
|
62
59
|
|
|
63
|
-
reporter.finish(files.length, results, files, { fix: options.fix, gitRoot })
|
|
60
|
+
reporter.finish(files.length, results, files, { fix: options.fix, gitRoot })
|
|
64
61
|
|
|
65
62
|
if (hasErrors) {
|
|
66
63
|
// Wait for stdout to flush (important when piped to a file), then exit
|
|
67
|
-
await new Promise((resolve) => process.stdout.write('', resolve))
|
|
68
|
-
process.exit(1)
|
|
64
|
+
await new Promise((resolve) => process.stdout.write('', resolve))
|
|
65
|
+
process.exit(1)
|
|
69
66
|
}
|
|
70
67
|
}
|