@readme/cli 0.0.26 → 0.0.29

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (123) hide show
  1. package/README.md +2 -2
  2. package/package.json +5 -3
  3. package/src/cli.js +6 -1
  4. package/src/commands/components.js +288 -0
  5. package/src/commands/import.js +2456 -376
  6. package/src/commands/import.slug-planner.test.js +86 -0
  7. package/src/commands/import.test.js +136 -0
  8. package/src/commands/lint.js +32 -35
  9. package/src/commands/oas-sync.js +60 -75
  10. package/src/commands/pretty.js +314 -77
  11. package/src/prompts/index.js +187 -24
  12. package/src/utils/lint.js +33 -33
  13. package/src/utils/llms.js +224 -0
  14. package/src/utils/llms.test.js +257 -0
  15. package/src/utils/marketplace.js +208 -0
  16. package/src/utils/multi-spinner.js +71 -0
  17. package/src/utils/promise-pool.js +17 -0
  18. package/src/utils/url-segs.js +124 -0
  19. package/src/utils/url-segs.test.js +239 -0
  20. package/src/validators/components.js +1 -0
  21. package/src/validators/duplicates.js +6 -4
  22. package/src/validators/frontmatter.js +11 -0
  23. package/src/validators/oas-reference.js +1 -31
  24. package/src/validators/ordering.js +169 -83
  25. package/vendor/TOOLS.md +1 -1
  26. package/src/dev/.next/app-build-manifest.json +0 -20
  27. package/src/dev/.next/build-manifest.json +0 -31
  28. package/src/dev/.next/cache/.rscinfo +0 -1
  29. package/src/dev/.next/cache/next-devtools-config.json +0 -1
  30. package/src/dev/.next/cache/webpack/client-development/0.pack.gz +0 -0
  31. package/src/dev/.next/cache/webpack/client-development/1.pack.gz +0 -0
  32. package/src/dev/.next/cache/webpack/client-development/10.pack.gz +0 -0
  33. package/src/dev/.next/cache/webpack/client-development/11.pack.gz +0 -0
  34. package/src/dev/.next/cache/webpack/client-development/2.pack.gz +0 -0
  35. package/src/dev/.next/cache/webpack/client-development/3.pack.gz +0 -0
  36. package/src/dev/.next/cache/webpack/client-development/3.pack.gz_ +0 -0
  37. package/src/dev/.next/cache/webpack/client-development/4.pack.gz +0 -0
  38. package/src/dev/.next/cache/webpack/client-development/5.pack.gz +0 -0
  39. package/src/dev/.next/cache/webpack/client-development/5.pack.gz_ +0 -0
  40. package/src/dev/.next/cache/webpack/client-development/6.pack.gz +0 -0
  41. package/src/dev/.next/cache/webpack/client-development/7.pack.gz +0 -0
  42. package/src/dev/.next/cache/webpack/client-development/7.pack.gz_ +0 -0
  43. package/src/dev/.next/cache/webpack/client-development/8.pack.gz +0 -0
  44. package/src/dev/.next/cache/webpack/client-development/9.pack.gz +0 -0
  45. package/src/dev/.next/cache/webpack/client-development/index.pack.gz.old +0 -0
  46. package/src/dev/.next/cache/webpack/client-development-fallback/0.pack.gz +0 -0
  47. package/src/dev/.next/cache/webpack/client-development-fallback/1.pack.gz +0 -0
  48. package/src/dev/.next/cache/webpack/client-development-fallback/index.pack.gz +0 -0
  49. package/src/dev/.next/cache/webpack/client-development-fallback/index.pack.gz.old +0 -0
  50. package/src/dev/.next/cache/webpack/edge-server-development/0.pack.gz +0 -0
  51. package/src/dev/.next/cache/webpack/edge-server-development/1.pack.gz +0 -0
  52. package/src/dev/.next/cache/webpack/edge-server-development/index.pack.gz +0 -0
  53. package/src/dev/.next/cache/webpack/edge-server-development/index.pack.gz.old +0 -0
  54. package/src/dev/.next/cache/webpack/server-development/0.pack.gz +0 -0
  55. package/src/dev/.next/cache/webpack/server-development/1.pack.gz +0 -0
  56. package/src/dev/.next/cache/webpack/server-development/10.pack.gz +0 -0
  57. package/src/dev/.next/cache/webpack/server-development/11.pack.gz +0 -0
  58. package/src/dev/.next/cache/webpack/server-development/12.pack.gz +0 -0
  59. package/src/dev/.next/cache/webpack/server-development/13.pack.gz +0 -0
  60. package/src/dev/.next/cache/webpack/server-development/14.pack.gz +0 -0
  61. package/src/dev/.next/cache/webpack/server-development/15.pack.gz +0 -0
  62. package/src/dev/.next/cache/webpack/server-development/2.pack.gz +0 -0
  63. package/src/dev/.next/cache/webpack/server-development/2.pack.gz_ +0 -0
  64. package/src/dev/.next/cache/webpack/server-development/3.pack.gz +0 -0
  65. package/src/dev/.next/cache/webpack/server-development/3.pack.gz_ +0 -0
  66. package/src/dev/.next/cache/webpack/server-development/4.pack.gz +0 -0
  67. package/src/dev/.next/cache/webpack/server-development/5.pack.gz +0 -0
  68. package/src/dev/.next/cache/webpack/server-development/6.pack.gz +0 -0
  69. package/src/dev/.next/cache/webpack/server-development/6.pack.gz_ +0 -0
  70. package/src/dev/.next/cache/webpack/server-development/7.pack.gz +0 -0
  71. package/src/dev/.next/cache/webpack/server-development/7.pack.gz_ +0 -0
  72. package/src/dev/.next/cache/webpack/server-development/8.pack.gz +0 -0
  73. package/src/dev/.next/cache/webpack/server-development/9.pack.gz +0 -0
  74. package/src/dev/.next/cache/webpack/server-development/9.pack.gz_ +0 -0
  75. package/src/dev/.next/cache/webpack/server-development/index.pack.gz +0 -0
  76. package/src/dev/.next/cache/webpack/server-development/index.pack.gz.old +0 -0
  77. package/src/dev/.next/package.json +0 -1
  78. package/src/dev/.next/prerender-manifest.json +0 -11
  79. package/src/dev/.next/react-loadable-manifest.json +0 -1
  80. package/src/dev/.next/routes-manifest.json +0 -1
  81. package/src/dev/.next/server/app/[...slug]/page.js +0 -360
  82. package/src/dev/.next/server/app/[...slug]/page_client-reference-manifest.js +0 -1
  83. package/src/dev/.next/server/app/page.js +0 -349
  84. package/src/dev/.next/server/app/page_client-reference-manifest.js +0 -1
  85. package/src/dev/.next/server/app-paths-manifest.json +0 -3
  86. package/src/dev/.next/server/edge-runtime-webpack.js +0 -1151
  87. package/src/dev/.next/server/interception-route-rewrite-manifest.js +0 -1
  88. package/src/dev/.next/server/middleware-build-manifest.js +0 -33
  89. package/src/dev/.next/server/middleware-manifest.json +0 -32
  90. package/src/dev/.next/server/middleware-react-loadable-manifest.js +0 -1
  91. package/src/dev/.next/server/middleware.js +0 -1113
  92. package/src/dev/.next/server/next-font-manifest.js +0 -1
  93. package/src/dev/.next/server/next-font-manifest.json +0 -1
  94. package/src/dev/.next/server/pages-manifest.json +0 -5
  95. package/src/dev/.next/server/server-reference-manifest.js +0 -1
  96. package/src/dev/.next/server/server-reference-manifest.json +0 -5
  97. package/src/dev/.next/server/static/webpack/633457081244afec._.hot-update.json +0 -1
  98. package/src/dev/.next/server/vendor-chunks/@readme.js +0 -25
  99. package/src/dev/.next/server/vendor-chunks/@swc.js +0 -55
  100. package/src/dev/.next/server/vendor-chunks/next.js +0 -3659
  101. package/src/dev/.next/server/webpack-runtime.js +0 -209
  102. package/src/dev/.next/static/chunks/app/[...slug]/loading.js +0 -28
  103. package/src/dev/.next/static/chunks/app/[...slug]/page.js +0 -28
  104. package/src/dev/.next/static/chunks/app/layout.js +0 -171
  105. package/src/dev/.next/static/chunks/app/page.js +0 -28
  106. package/src/dev/.next/static/chunks/app-pages-internals.js +0 -182
  107. package/src/dev/.next/static/chunks/main-app.js +0 -1882
  108. package/src/dev/.next/static/chunks/polyfills.js +0 -1
  109. package/src/dev/.next/static/chunks/webpack.js +0 -1393
  110. package/src/dev/.next/static/css/app/layout.css +0 -559
  111. package/src/dev/.next/static/development/_buildManifest.js +0 -1
  112. package/src/dev/.next/static/development/_ssgManifest.js +0 -1
  113. package/src/dev/.next/static/webpack/633457081244afec._.hot-update.json +0 -1
  114. package/src/dev/.next/static/webpack/ec52a3fce0f78db0.webpack.hot-update.json +0 -1
  115. package/src/dev/.next/static/webpack/webpack.ec52a3fce0f78db0.hot-update.js +0 -12
  116. package/src/dev/.next/trace +0 -21
  117. package/src/dev/.next/types/app/[...slug]/page.ts +0 -84
  118. package/src/dev/.next/types/app/layout.ts +0 -84
  119. package/src/dev/.next/types/app/page.ts +0 -84
  120. package/src/dev/.next/types/cache-life.d.ts +0 -141
  121. package/src/dev/.next/types/package.json +0 -1
  122. package/src/dev/.next/types/routes.d.ts +0 -55
  123. package/src/validators/numbering.js +0 -136
@@ -19,16 +19,33 @@
19
19
 
20
20
  export const slotOrphansSystemPrompt = [
21
21
  'You assign orphan documentation pages to existing categories.',
22
- 'Output ONLY a valid JSON array of integers — one per orphan page, in order.',
22
+ 'Respond with a JSON object of the form {"assignments": [...]} — one integer per orphan page, in order.',
23
23
  'Each integer is the category index (0-based) the orphan belongs in, or -1 if none fit.',
24
24
  '',
25
- 'Example output: [0, 2, 0, 1, -1, 3]',
25
+ 'Example output: {"assignments": [0, 2, 0, 1, -1, 3]}',
26
26
  '',
27
27
  'Guidance:',
28
28
  '- Choose the best semantic fit based on the orphan\'s title and URL path.',
29
29
  '- Only use -1 when no existing category is plausible.',
30
30
  ].join('\n');
31
31
 
32
+ /**
33
+ * Structured-output schema for slotOrphans. Root must be an object (top-level
34
+ * arrays aren't allowed), so the index array is wrapped in `assignments`.
35
+ */
36
+ export const slotOrphansOutputSchema = {
37
+ type: 'object',
38
+ additionalProperties: false,
39
+ required: ['assignments'],
40
+ properties: {
41
+ assignments: {
42
+ type: 'array',
43
+ items: { type: 'integer' },
44
+ description: 'One entry per orphan page, in input order: the 0-based category index, or -1 if none fit.',
45
+ },
46
+ },
47
+ };
48
+
32
49
  /**
33
50
  * @param {object} input
34
51
  * @param {Array<{ title: string, pages?: Array<{ title: string }> }>} input.categories
@@ -54,7 +71,7 @@ export function buildSlotOrphansUserPrompt({ categories, orphans }) {
54
71
  `${orphans.length} orphan pages to slot:`,
55
72
  ...orphanList,
56
73
  '',
57
- `Output a JSON array of ${orphans.length} integers, one per orphan in order (category index 0..${categories.length - 1}, or -1 for none).`,
74
+ `Output {"assignments": [...]} with ${orphans.length} integers, one per orphan in order (category index 0..${categories.length - 1}, or -1 for none).`,
58
75
  ].join('\n');
59
76
  }
60
77
 
@@ -69,12 +86,29 @@ export function slotOrphansPrompt(input) {
69
86
 
70
87
  export const iconizeNavSystemPrompt = [
71
88
  'You assign one FontAwesome Free Solid icon to each documentation category.',
72
- 'Output ONLY a valid JSON array of icon name strings, one per input category, in order.',
73
- 'Use the icon name only (no "fa-" prefix, no object wrapper).',
89
+ 'Respond with a JSON object of the form {"icons": [...]} — one icon name string per input category, in order.',
90
+ 'Use the icon name only (no "fa-" prefix).',
74
91
  '',
75
- 'Example output: ["rocket", "book", "code", "gear"]',
92
+ 'Example output: {"icons": ["rocket", "book", "code", "gear"]}',
76
93
  ].join('\n');
77
94
 
95
+ /**
96
+ * Structured-output schema for iconizeNav. Root must be an object, so the
97
+ * icon-name array is wrapped in `icons`.
98
+ */
99
+ export const iconizeNavOutputSchema = {
100
+ type: 'object',
101
+ additionalProperties: false,
102
+ required: ['icons'],
103
+ properties: {
104
+ icons: {
105
+ type: 'array',
106
+ items: { type: 'string' },
107
+ description: 'One FontAwesome Free Solid icon name per input category, in order (no "fa-" prefix).',
108
+ },
109
+ },
110
+ };
111
+
78
112
  /**
79
113
  * @param {object} input
80
114
  * @param {Array<{ title: string }>} input.categories
@@ -86,7 +120,7 @@ export function buildIconizeNavUserPrompt({ categories }) {
86
120
  '',
87
121
  ...categories.map((c, i) => `${i}. ${c.title}`),
88
122
  '',
89
- 'Return a JSON array of FontAwesome icon names, one per category, in order.',
123
+ 'Return {"icons": [...]} with one FontAwesome icon name per category, in order.',
90
124
  ].join('\n');
91
125
  }
92
126
 
@@ -101,13 +135,15 @@ export function iconizeNavPrompt(input) {
101
135
 
102
136
  export const organizeFromSectionsSystemPrompt = [
103
137
  'You assign a FontAwesome Free Solid icon to each documentation section, and lightly polish the section title.',
104
- 'Output ONLY a valid JSON array — no prose, no markdown, no code fences.',
138
+ 'Respond with a JSON object matching the schema below.',
105
139
  '',
106
140
  'Schema:',
107
- '[',
108
- ' { "title": "<lightly polished Title Case, 1-4 words>", "icon": "<fontawesome icon name, no fa- prefix>" },',
109
- ' ...',
110
- ']',
141
+ '{',
142
+ ' "sections": [',
143
+ ' { "title": "<lightly polished Title Case, 1-4 words>", "icon": "<fontawesome icon name, no fa- prefix>" },',
144
+ ' ...',
145
+ ' ]',
146
+ '}',
111
147
  '',
112
148
  'Rules:',
113
149
  '- Return exactly one entry per input section, in the same order.',
@@ -115,6 +151,31 @@ export const organizeFromSectionsSystemPrompt = [
115
151
  '- Pick an icon that semantically fits the section (e.g. rocket for Getting Started, code for API, plug for Integrations).',
116
152
  ].join('\n');
117
153
 
154
+ /**
155
+ * Structured-output schema for organizeFromSections. Root must be an object,
156
+ * so the per-section entries are wrapped in `sections`.
157
+ */
158
+ export const organizeFromSectionsOutputSchema = {
159
+ type: 'object',
160
+ additionalProperties: false,
161
+ required: ['sections'],
162
+ properties: {
163
+ sections: {
164
+ type: 'array',
165
+ description: 'Exactly one entry per input section, in the same order.',
166
+ items: {
167
+ type: 'object',
168
+ additionalProperties: false,
169
+ required: ['title', 'icon'],
170
+ properties: {
171
+ title: { type: 'string', description: 'Lightly polished Title Case section title, 1-4 words.' },
172
+ icon: { type: 'string', description: 'FontAwesome Free Solid icon name, no "fa-" prefix.' },
173
+ },
174
+ },
175
+ },
176
+ },
177
+ };
178
+
118
179
  /**
119
180
  * @param {object} input
120
181
  * @param {string|null|undefined} input.siteTitle
@@ -128,7 +189,7 @@ export function buildOrganizeFromSectionsUserPrompt({ siteTitle, sections }) {
128
189
  '',
129
190
  ...sections.map((s, i) => `${i}. ${s.title} (${s.items.length} pages)`),
130
191
  '',
131
- 'Output the JSON array now.',
192
+ 'Output the JSON object now.',
132
193
  ].join('\n');
133
194
  }
134
195
 
@@ -166,6 +227,33 @@ export const organizeFromScratchSystemPrompt = [
166
227
  '- Keep category titles human-readable, Title Case, 1-4 words.',
167
228
  ].join('\n');
168
229
 
230
+ /** Structured-output schema for organizeFromScratch. */
231
+ export const organizeFromScratchOutputSchema = {
232
+ type: 'object',
233
+ additionalProperties: false,
234
+ required: ['title', 'categories'],
235
+ properties: {
236
+ title: { type: 'string', description: 'Short site or doc title.' },
237
+ categories: {
238
+ type: 'array',
239
+ items: {
240
+ type: 'object',
241
+ additionalProperties: false,
242
+ required: ['title', 'icon', 'pageIds'],
243
+ properties: {
244
+ title: { type: 'string', description: 'Title Case category name, 1-4 words.' },
245
+ icon: { type: 'string', description: 'FontAwesome Free Solid icon name, no "fa-" prefix.' },
246
+ pageIds: {
247
+ type: 'array',
248
+ items: { type: 'integer' },
249
+ description: 'Integer ids from the input page list. Every input id appears in exactly one category.',
250
+ },
251
+ },
252
+ },
253
+ },
254
+ },
255
+ };
256
+
169
257
  /**
170
258
  * @param {object} input
171
259
  * @param {string|null|undefined} input.siteTitle
@@ -296,16 +384,44 @@ If the page IS an API reference, extract the endpoints into a partial OpenAPI 3.
296
384
  - Use realistic types inferred from the documentation (don't use "object" or "string" for everything)
297
385
  - Include the server/base URL if mentioned
298
386
 
299
- ## Output Format
300
- Respond with ONLY a JSON object (no markdown fences, no explanation):
301
- {
302
- "excerpt": "one-line summary of the page",
303
- "body": "the full MDX-fixed markdown content (no frontmatter)",
304
- "isApiRef": true/false,
305
- "suggestedCategory": "Best Category Name",
306
- "oasPartial": null OR {"paths": {"/endpoint": {"get": {...}}}, "schemas": {"ModelName": {...}}}
307
- }
308
- When isApiRef=false, set oasPartial to null. When isApiRef=true, include the extracted paths and schemas.`;
387
+ ## Output
388
+ The runtime enforces the response shape via a JSON schema, so don't worry about formatting — just populate every field with the right content.
389
+
390
+ - \`body\`: the full MDX-fixed markdown content with no frontmatter. This is the load-bearing field — get the content right, render it human-readable, and don't leave broken MDX.
391
+ - \`isApiRef\`: true if this page primarily documents API endpoints (per the rules above), false otherwise.
392
+ - \`oasPartialJson\`: when \`isApiRef\` is true, set this to a JSON-stringified partial OpenAPI 3.0 fragment of the form \`{"paths": {"/endpoint": {"get": {...}}}, "schemas": {"ModelName": {...}}}\`. The caller will \`JSON.parse\` it. When \`isApiRef\` is false, set it to \`null\`.
393
+ `;
394
+
395
+ /**
396
+ * JSON schema describing the structured output shape returned by the prettify
397
+ * agent. Pass to the Claude Agent SDK's \`outputFormat\` option so the runtime
398
+ * enforces the shape and returns parsed data on the result message's
399
+ * \`structured_output\` field — no manual fence-stripping or JSON.parse needed.
400
+ *
401
+ * \`oasPartialJson\` is a stringified OAS fragment rather than a nested object
402
+ * because Anthropic's structured outputs require \`additionalProperties: false\`
403
+ * on every object, which is incompatible with OpenAPI's free-form path/schema
404
+ * keys. The caller parses the string client-side.
405
+ */
406
+ export const prettifyPageOutputSchema = {
407
+ type: 'object',
408
+ additionalProperties: false,
409
+ required: ['body', 'isApiRef', 'oasPartialJson'],
410
+ properties: {
411
+ body: {
412
+ type: 'string',
413
+ description: 'The MDX-fixed markdown content for the page (no frontmatter).',
414
+ },
415
+ isApiRef: {
416
+ type: 'boolean',
417
+ description: 'True if the page primarily documents REST/GraphQL/gRPC endpoints.',
418
+ },
419
+ oasPartialJson: {
420
+ anyOf: [{ type: 'string' }, { type: 'null' }],
421
+ description: 'JSON-stringified partial OpenAPI 3.0 fragment when isApiRef is true; null otherwise.',
422
+ },
423
+ },
424
+ };
309
425
 
310
426
  /**
311
427
  * @param {object} input
@@ -326,9 +442,56 @@ export function buildPrettifyPageUserPrompt({ source, title, relativePath }) {
326
442
  ].join('\n');
327
443
  }
328
444
 
445
+ /**
446
+ * Build a short addendum describing the user's locally-installed marketplace
447
+ * components so Claude can use them in prettified output. We pull the first
448
+ * usage signature out of each component's mdx (a `<Name ... />` line near the
449
+ * bottom of the file).
450
+ *
451
+ * @param {Array<{ name: string, source: string }>} components
452
+ * @returns {string} — the addendum, or '' if there are no components.
453
+ */
454
+ export function buildLocalComponentsSection(components) {
455
+ if (!components || components.length === 0) return '';
456
+ const lines = ['', '### Project-specific components (installed in ./components/)'];
457
+ lines.push(
458
+ 'The following custom components are installed in this project and may be used in prettified output. Prefer them when they fit the content better than plain markdown — same liberal-upgrade bias as the built-in components above. Do NOT invent components that are not listed here or in the built-in set.',
459
+ '',
460
+ );
461
+ for (const c of components) {
462
+ const sig = (c.source.match(new RegExp(`<${c.name}[\\s\\S]*?(?:/>|</${c.name}>)`, 'm')) || [])[0];
463
+ lines.push(`- \`<${c.name}>\`${sig ? ` — e.g. \`${sig.replace(/\s+/g, ' ').slice(0, 200)}\`` : ''}`);
464
+ }
465
+ return lines.join('\n');
466
+ }
467
+
468
+ /**
469
+ * Build a short section listing additional marketplace components that are
470
+ * NOT yet installed but can be — the CLI will auto-install any of them that
471
+ * the model uses in its output.
472
+ *
473
+ * @param {Array<{ name: string, summary?: string }>} available
474
+ * Marketplace components not currently in ./components/.
475
+ * @returns {string} — the addendum, or '' if there are none.
476
+ */
477
+ export function buildAvailableComponentsSection(available) {
478
+ if (!available || available.length === 0) return '';
479
+ const lines = ['', '### Marketplace components (auto-installed on use)'];
480
+ lines.push(
481
+ 'These components are NOT yet installed locally, but the CLI will copy them into ./components/ automatically if you use any of them. Use them when they clearly fit the content — same upgrade bias as built-in components. Do NOT invent components outside this list and the built-in set.',
482
+ '',
483
+ );
484
+ for (const c of available) {
485
+ lines.push(`- \`<${c.name}>\`${c.summary ? ` — ${c.summary}` : ''}`);
486
+ }
487
+ return lines.join('\n');
488
+ }
489
+
329
490
  export function prettifyPagePrompt(input) {
491
+ const localAddendum = buildLocalComponentsSection(input.localComponents);
492
+ const availableAddendum = buildAvailableComponentsSection(input.availableComponents);
330
493
  return {
331
- systemPrompt: prettifyPageSystemPrompt,
494
+ systemPrompt: prettifyPageSystemPrompt + localAddendum + availableAddendum,
332
495
  userPrompt: buildPrettifyPageUserPrompt(input),
333
496
  };
334
497
  }
package/src/utils/lint.js CHANGED
@@ -1,6 +1,6 @@
1
- import fs from 'node:fs';
2
- import path from 'node:path';
3
- import { pathToFileURL } from 'node:url';
1
+ import fs from 'node:fs'
2
+ import path from 'node:path'
3
+ import { pathToFileURL } from 'node:url'
4
4
 
5
5
  const TARGET_PATTERNS = [
6
6
  { dir: 'custom_blocks', ext: ['.mdx', '.md'] },
@@ -8,28 +8,28 @@ const TARGET_PATTERNS = [
8
8
  { dir: 'reference', ext: '.md' },
9
9
  { dir: 'custom_pages', ext: '.md' },
10
10
  { dir: 'recipes', ext: '.md' },
11
- ];
11
+ ]
12
12
 
13
13
  /**
14
14
  * Collect all target files from the repo root.
15
15
  */
16
16
  export function collectFiles(gitRoot) {
17
- const files = [];
17
+ const files = []
18
18
 
19
19
  for (const { dir, ext } of TARGET_PATTERNS) {
20
- const dirPath = path.join(gitRoot, dir);
21
- if (!fs.existsSync(dirPath)) continue;
20
+ const dirPath = path.join(gitRoot, dir)
21
+ if (!fs.existsSync(dirPath)) continue
22
22
 
23
- const exts = Array.isArray(ext) ? ext : [ext];
24
- const entries = fs.readdirSync(dirPath, { recursive: true });
23
+ const exts = Array.isArray(ext) ? ext : [ext]
24
+ const entries = fs.readdirSync(dirPath, { recursive: true })
25
25
  for (const entry of entries) {
26
26
  if (exts.some((e) => entry.endsWith(e))) {
27
- files.push(path.join(dir, entry));
27
+ files.push(path.join(dir, entry))
28
28
  }
29
29
  }
30
30
  }
31
31
 
32
- return files.sort();
32
+ return files.sort()
33
33
  }
34
34
 
35
35
  /**
@@ -38,18 +38,18 @@ export function collectFiles(gitRoot) {
38
38
  * `validateAll()` for cross-file checks (like ordering).
39
39
  */
40
40
  async function loadValidators() {
41
- const validatorsDir = path.join(path.dirname(new URL(import.meta.url).pathname), '..', 'validators');
42
- const files = fs.readdirSync(validatorsDir).filter((f) => f.endsWith('.js'));
43
- const validators = [];
41
+ const validatorsDir = path.join(path.dirname(new URL(import.meta.url).pathname), '..', 'validators')
42
+ const files = fs.readdirSync(validatorsDir).filter((f) => f.endsWith('.js'))
43
+ const validators = []
44
44
 
45
45
  for (const file of files) {
46
- const mod = await import(pathToFileURL(path.join(validatorsDir, file)).href);
46
+ const mod = await import(pathToFileURL(path.join(validatorsDir, file)).href)
47
47
  if (mod.name && (mod.validate || mod.validateAll)) {
48
- validators.push(mod);
48
+ validators.push(mod)
49
49
  }
50
50
  }
51
51
 
52
- return validators;
52
+ return validators
53
53
  }
54
54
 
55
55
  /**
@@ -57,43 +57,43 @@ async function loadValidators() {
57
57
  * Each result has { file, rule, message, severity? } where severity defaults to 'error'.
58
58
  * Calls `onFile(relativePath)` before processing each file (for progress reporting).
59
59
  */
60
- export async function runValidators(files, gitRoot, { onFile, onBeforeCrossFile, fix } = {}) {
61
- const validators = await loadValidators();
62
- const results = [];
60
+ export async function runValidators(files, gitRoot, { onFile, onBeforeCrossFile, fix, nonInteractive } = {}) {
61
+ const validators = await loadValidators()
62
+ const results = []
63
63
 
64
64
  // Per-file validators.
65
65
  for (const relativePath of files) {
66
- if (onFile) onFile(relativePath);
66
+ if (onFile) onFile(relativePath)
67
67
 
68
- const filePath = path.join(gitRoot, relativePath);
69
- const content = fs.readFileSync(filePath, 'utf-8');
68
+ const filePath = path.join(gitRoot, relativePath)
69
+ const content = fs.readFileSync(filePath, 'utf-8')
70
70
 
71
71
  for (const validator of validators) {
72
- if (!validator.validate) continue;
73
- const result = validator.validate({ filePath, content, relativePath, fix });
72
+ if (!validator.validate) continue
73
+ const result = validator.validate({ filePath, content, relativePath, fix })
74
74
  if (result) {
75
75
  if (Array.isArray(result)) {
76
- results.push(...result);
76
+ results.push(...result)
77
77
  } else {
78
- results.push(result);
78
+ results.push(result)
79
79
  }
80
80
  }
81
81
  }
82
82
  }
83
83
 
84
84
  // Cross-file validators — stop the spinner first so interactive prompts work.
85
- if (onBeforeCrossFile) onBeforeCrossFile();
85
+ if (onBeforeCrossFile) onBeforeCrossFile()
86
86
  for (const validator of validators) {
87
- if (!validator.validateAll) continue;
88
- const result = await validator.validateAll(files, gitRoot, { fix });
87
+ if (!validator.validateAll) continue
88
+ const result = await validator.validateAll(files, gitRoot, { fix, nonInteractive })
89
89
  if (result) {
90
90
  if (Array.isArray(result)) {
91
- results.push(...result);
91
+ results.push(...result)
92
92
  } else {
93
- results.push(result);
93
+ results.push(result)
94
94
  }
95
95
  }
96
96
  }
97
97
 
98
- return results;
98
+ return results
99
99
  }
@@ -0,0 +1,224 @@
1
+ const H1_RE = /^#\s+(.+)$/
2
+ const H2_RE = /^##\s+(.+)$/
3
+ const H3_RE = /^###\s+(.+)$/
4
+
5
+ // llms.txt might have a literal " (section)" marker to every
6
+ // grouping heading (e.g. `## getting-started (section)`). It's internal
7
+ // metadata, not part of the category name — strip it before it reaches the sidebar.
8
+ const SECTION_MARKER_RE = /\s*\(section\)\s*$/i
9
+
10
+ // Max items per section before we consider it "oversized" for usability purposes.
11
+ // Mirrors the cap in usableSections() in import.js.
12
+ const MAX_SECTION_ITEMS = 200
13
+ // Any line whose meaningful content is a markdown link. Accepts:
14
+ // - Standard list rows: `- [text](url)`, `* [text](url) — desc`
15
+ // - Bare link lines: `[text](url)`
16
+ // - Breadcrumb-prefixed rows used by Fern/Mintlify-style indices:
17
+ // `ElevenAgents [Agent WebSockets](url)`
18
+ // `API Reference > Agents > Branches [List branches](url)`
19
+ // - Trailing-parenthetical-breadcrumb rows (AssemblyAI/Fern variant) — the
20
+ // breadcrumb sits between the url and the description and is matched-and-discarded:
21
+ // `[text](url) (Breadcrumb > path): description`
22
+ // Captures: [prefix, text, url, description] (prefix and description may be empty).
23
+ const LINK_LINE_RE = /^\s*(?:[-*+]\s+)?([^\[\n]*?)\s*\[([^\]]+)\] ?\(([^)\s]+)\)(?:\s+\([^)]*\))?(?:\s*[:—–-]\s*(.+?))?\s*$/
24
+ const BLOCKQUOTE_RE = /^\s*>/
25
+ const FENCE_RE = /^\s*(?:```|~~~)/
26
+
27
+ // A llms.txt is considered usable as long as it has at least one link row.
28
+ // Beyond that we enforce a structural ratio (share of "spec-shaped" lines —
29
+ // H1/H2/blockquote/link row/blank — over total lines), but only once the
30
+ // file has enough links for the ratio to be statistically meaningful. A
31
+ // 3-link file with one prose sentence would compute a misleading ratio; we
32
+ // can't tell signal from noise at that size, so we trust it and let the
33
+ // BFS-merge dedupe handle any junk that slips through.
34
+ const RATIO_CHECK_MIN_LINKS = 10
35
+ const MIN_CONFORMING_RATIO = 0.7
36
+
37
+ export function analyzeLlmsTxt(body, llmsUrl, options = {}) {
38
+ const parsed = parseLlmsTxt(body, llmsUrl, options)
39
+ const lineStats = classifyLines(body)
40
+ const linkItems = parsed.sections.reduce((sum, s) => sum + s.items.length, 0)
41
+ const reason = getSkipReason(body, linkItems, lineStats)
42
+
43
+ return {
44
+ parsed,
45
+ usable: !reason,
46
+ reason,
47
+ stats: { ...lineStats, linkItems },
48
+ }
49
+ }
50
+
51
+ export function parseLlmsTxt(body, llmsUrl, options = {}) {
52
+ const h2Result = parseSections(body, llmsUrl, H2_RE, options)
53
+
54
+ // If every H2 section is oversized (or there are none), and the file has H3
55
+ // headings, re-parse treating H3 (###) as section boundaries.
56
+ // For strange LLMS.txt formatting
57
+ const allOversized = h2Result.sections.length === 0 ||
58
+ h2Result.sections.every((s) => s.items.length > MAX_SECTION_ITEMS)
59
+ if (allOversized && /^###\s/m.test(body)) {
60
+ const h3Result = parseSections(body, llmsUrl, H3_RE, options)
61
+ if (h3Result.sections.length > 1) {
62
+ return { ...h3Result, h3Fallback: true }
63
+ }
64
+ }
65
+
66
+ return h2Result
67
+ }
68
+
69
+ function parseSections(body, llmsUrl, sectionRe, options = {}) {
70
+ const lines = body.split(/\r?\n/)
71
+ let title = null
72
+ const sections = []
73
+ let current = null
74
+
75
+ for (const line of lines) {
76
+ const h1 = line.match(H1_RE)
77
+ if (h1 && !title) {
78
+ title = h1[1].trim()
79
+ continue
80
+ }
81
+
82
+ const sectionMatch = line.match(sectionRe)
83
+ if (sectionMatch) {
84
+ current = { title: sectionMatch[1].trim().replace(SECTION_MARKER_RE, '').trim(), items: [] }
85
+ sections.push(current)
86
+ continue
87
+ }
88
+
89
+ const item = parseListLink(line, llmsUrl, options)
90
+ if (!item) continue
91
+
92
+ if (!current) {
93
+ current = { title: 'Resources', items: [] }
94
+ sections.push(current)
95
+ }
96
+ current.items.push(item)
97
+ }
98
+
99
+ return { title, sections }
100
+ }
101
+
102
+ function getSkipReason(body, linkItems, lineStats) {
103
+ if (/^---\r?\n/.test(body)) return 'starts with YAML frontmatter'
104
+ if (linkItems === 0) return 'no link items'
105
+ if (linkItems >= RATIO_CHECK_MIN_LINKS && lineStats.ratio < MIN_CONFORMING_RATIO) {
106
+ return `conforming-line ratio ${lineStats.ratio.toFixed(2)} below ${MIN_CONFORMING_RATIO} (${lineStats.conforming}/${lineStats.total} lines)`
107
+ }
108
+ return null
109
+ }
110
+
111
+ /**
112
+ * Walk the body line-by-line and bucket each line as conforming or not.
113
+ *
114
+ * conforming blank, H1, H2, blockquote, standard link-row
115
+ * non-conforming everything else, plus every line inside (and the delimiters
116
+ * of) a fenced code block
117
+ */
118
+ function classifyLines(body) {
119
+ const lines = body.split(/\r?\n/)
120
+ let inFence = false
121
+ let conforming = 0
122
+ let nonConforming = 0
123
+
124
+ for (const line of lines) {
125
+ if (FENCE_RE.test(line)) {
126
+ nonConforming++
127
+ inFence = !inFence
128
+ continue
129
+ }
130
+ if (inFence) {
131
+ nonConforming++
132
+ continue
133
+ }
134
+ if (line.trim() === '') {
135
+ conforming++
136
+ continue
137
+ }
138
+ if (H1_RE.test(line) || H2_RE.test(line) || LINK_LINE_RE.test(line) || BLOCKQUOTE_RE.test(line)) {
139
+ conforming++
140
+ continue
141
+ }
142
+ nonConforming++
143
+ }
144
+
145
+ const total = conforming + nonConforming
146
+ const ratio = total === 0 ? 0 : conforming / total
147
+ return { conforming, nonConforming, total, ratio }
148
+ }
149
+
150
+ function parseListLink(line, llmsUrl, options = {}) {
151
+ const match = line.match(LINK_LINE_RE)
152
+ if (!match) return null
153
+
154
+ const [, prefix, text, rawUrl, trailingDesc] = match
155
+ const url = normalizeUrl(rawUrl, llmsUrl, options)
156
+ if (!url) return null
157
+
158
+ // Prefer an explicit trailing description (`[text](url) — desc`); fall back
159
+ // to the breadcrumb prefix (`API Reference > Agents [text](url)`) when no
160
+ // explicit description is present. Either gives downstream extra context.
161
+ const description = trailingDesc?.trim() || prefix?.trim() || null
162
+
163
+ return {
164
+ text: text.trim(),
165
+ url,
166
+ description: description || null,
167
+ }
168
+ }
169
+
170
+ function normalizeUrl(rawUrl, llmsUrl, options = {}) {
171
+ const trimmed = String(rawUrl || '').trim().replace(/[.,;]+$/, '')
172
+ if (!trimmed || /^#/.test(trimmed) || /^(mailto|javascript):/i.test(trimmed)) return null
173
+
174
+ try {
175
+ const url = llmsUrl ? new URL(resolveLlmsRelativeUrl(trimmed, llmsUrl, options), llmsUrl) : new URL(trimmed)
176
+ if (url.protocol !== 'http:' && url.protocol !== 'https:') return null
177
+ return url.toString()
178
+ } catch {
179
+ return null
180
+ }
181
+ }
182
+
183
+ function resolveLlmsRelativeUrl(trimmed, llmsUrl, options) {
184
+ if (options.rootRelativeResolution !== 'llms-dir' || !isRootRelativeUrl(trimmed)) return trimmed
185
+
186
+ const nestedPrefix = getLlmsNestedPrefix(llmsUrl)
187
+ if (!nestedPrefix) return trimmed
188
+
189
+ const normallyResolved = new URL(trimmed, llmsUrl)
190
+ if (!shouldResolveRootRelativeFromLlmsDir(trimmed, normallyResolved.pathname, nestedPrefix)) return trimmed
191
+
192
+ return `.${trimmed}`
193
+ }
194
+
195
+ function shouldResolveRootRelativeFromLlmsDir(rawUrl, pathname, nestedPrefix) {
196
+ if (isPathWithinPrefix(pathname, nestedPrefix)) return false
197
+ if (/\/llms\.txt$/i.test(pathname)) return false
198
+
199
+ const lastSegment = pathname.split('/').pop() || ''
200
+ const ext = lastSegment.match(/\.([a-z0-9]+)$/i)?.[1]?.toLowerCase()
201
+ if (ext && !['htm', 'html', 'md', 'mdx'].includes(ext)) return false
202
+
203
+ return isRootRelativeUrl(rawUrl)
204
+ }
205
+
206
+ function getLlmsNestedPrefix(llmsUrl) {
207
+ if (!llmsUrl) return null
208
+
209
+ try {
210
+ const pathname = new URL(llmsUrl).pathname
211
+ const dir = pathname.replace(/llms\.txt$/i, '').replace(/\/+$/, '/')
212
+ return dir === '/' ? null : dir
213
+ } catch {
214
+ return null
215
+ }
216
+ }
217
+
218
+ function isRootRelativeUrl(rawUrl) {
219
+ return /^\/(?!\/)/.test(rawUrl)
220
+ }
221
+
222
+ function isPathWithinPrefix(pathname, prefix) {
223
+ return pathname === prefix.replace(/\/$/, '') || pathname.startsWith(prefix)
224
+ }