@readme/cli 0.0.26 → 0.0.27

package/src/commands/pretty.js

@@ -1,133 +1,370 @@
-import fs from 'node:fs';
-import path from 'node:path';
-import { createRequire } from 'node:module';
-import { query } from '@anthropic-ai/claude-agent-sdk';
-import matter from 'gray-matter';
-import ora from 'ora';
-import { collectFiles } from '../utils/lint.js';
-import { prettifyPagePrompt, stripCodeFences } from '../prompts/index.js';
-import { printHeader, isAgenticCli } from '../utils/eyes.js';
-import * as styles from '../utils/styles.js';
-
-const require = createRequire(import.meta.url);
-const pkg = require('../../package.json');
-
-export const command = 'pretty';
-export const order = 2;
-export const description = 'Polish docs into ReadMe-ready MDX with Claude';
-export const beta = true;
+import fs from 'node:fs'
+import path from 'node:path'
+import { createRequire } from 'node:module'
+import { query } from '@anthropic-ai/claude-agent-sdk'
+import { InvalidArgumentError, Option } from 'commander'
+import matter from 'gray-matter'
+import { collectFiles } from '../utils/lint.js'
+import { prettifyPagePrompt, prettifyPageOutputSchema } from '../prompts/index.js'
+import { printHeader, isAgenticCli } from '../utils/eyes.js'
+import { MultiSpinner } from '../utils/multi-spinner.js'
+import { runPool } from '../utils/promise-pool.js'
+import {
+  listLocalComponents,
+  listMarketplaceComponents,
+  installComponent,
+  readCachedReadme,
+  summarizeReadme,
+} from '../utils/marketplace.js'
+import * as styles from '../utils/styles.js'
+
+const require = createRequire(import.meta.url)
+const pkg = require('../../package.json')
+
+/**
+ * Native ReadMe MDX tags that don't need a marketplace install — the renderer
+ * supports them out of the box. We exclude these from auto-install detection.
+ */
+const NATIVE_TAGS = new Set([
+  'Tabs', 'Tab',
+  'Accordion',
+  'Cards', 'Card',
+  'Columns', 'Column',
+  'Image',
+])
+
+/**
+ * Pull every PascalCase tag name out of a block of mdx. Matches both
+ * `<Foo ... />` and `<Foo>...</Foo>`. Returns a Set of unique names.
+ */
+function extractUsedTags(mdx) {
+  const tags = new Set()
+  const re = /<([A-Z][A-Za-z0-9]*)\b/g
+  let m
+  while ((m = re.exec(mdx)) !== null) tags.add(m[1])
+  return tags
+}
+
+export const command = 'pretty [file]'
+export const order = 2
+export const description = 'Polish docs into ReadMe-ready MDX with Claude'
+export const beta = true
 
 export function args(cmd) {
-  cmd.option('--model <name>', 'Claude model alias: haiku, sonnet, opus', 'sonnet');
-  cmd.option('--dry-run', 'Show what would change without writing files');
+  cmd.addOption(new Option('-m, --model <name>', 'Claude model alias: haiku, sonnet, opus').choices(['haiku', 'sonnet', 'opus']).default('sonnet'))
+
+  cmd.option('--dry-run', 'Show what would change without writing files')
+  cmd.option('--no-auto-install', 'Don\'t auto-install marketplace components used in output')
+  cmd.option(
+    '-j, --jobs <n>',
+    'Number of files to prettify in parallel',
+    (raw) => {
+      const n = Number(raw)
+      if (!Number.isInteger(n) || n < 1) {
+        throw new InvalidArgumentError('must be a positive integer')
+      }
+      return n
+    },
+    1,
+  )
 }
 
 /**
- * Send one page through Claude using the prettify system prompt and parse the
- * JSON response. Returns the parsed object or throws on failure.
+ * Fetch with exponential backoff for transient (5xx / network) failures.
+ * `retries: 3` does up to 4 attempts with delays of 1s, 2s, 4s before each
+ * retry. 4xx responses are definitive and short-circuit so callers can fall
+ * through to the next strategy.
  */
-async function prettifyPage({ source, title, relativePath, model }) {
-  const { systemPrompt, userPrompt } = prettifyPagePrompt({ source, title, relativePath });
+async function fetchWithRetry(url, { headers = {}, retries = 3, baseDelayMs = 1000 } = {}) {
+  const maxAttempts = retries + 1
+  let lastError
+  for (let attempt = 1; attempt <= maxAttempts; attempt++) {
+    try {
+      const res = await fetch(url, {
+        redirect: 'follow',
+        headers: { 'User-Agent': 'readme-cli-pretty', ...headers },
+      })
+      if (!res.ok) {
+        const err = new Error(`HTTP ${res.status} ${res.statusText}`)
+        err.status = res.status
+        throw err
+      }
+      return await res.text()
+    } catch (err) {
+      lastError = err
+      if (err.status && err.status >= 400 && err.status < 500) break
+      if (attempt < maxAttempts) {
+        const delay = baseDelayMs * Math.pow(2, attempt - 1)
+        await new Promise((resolve) => setTimeout(resolve, delay))
+      }
+    }
+  }
+  throw lastError
+}
+
+function withMdExtension(url) {
+  try {
+    const u = new URL(url)
+    if (u.pathname.endsWith('.md')) return null
+    u.pathname = u.pathname.replace(/\/$/, '') + '.md'
+    return u.toString()
+  } catch {
+    if (url.endsWith('.md')) return null
+    return url.replace(/\/$/, '') + '.md'
+  }
+}
+
+/**
+ * Fetch a doc URL, preferring markdown. Falls through in order:
+ *   1. <url>.md (when not already .md)
+ *   2. <url> with Accept: text/markdown
+ *   3. <url> directly
+ * Each strategy gets its own retry budget for transient failures.
+ */
+async function fetchImport(url, opts = {}) {
+  const strategies = []
+  const mdUrl = withMdExtension(url)
+  if (mdUrl) strategies.push({ url: mdUrl })
+  strategies.push({ url, headers: { Accept: 'text/markdown' } })
+  strategies.push({ url })
+
+  let lastError
+  for (const strategy of strategies) {
+    try {
+      return await fetchWithRetry(strategy.url, { ...opts, headers: strategy.headers })
+    } catch (err) {
+      lastError = err
+    }
+  }
+  throw new Error(`failed to fetch ${url}: ${lastError?.message || lastError}`)
+}
 
-  let text = '';
+/**
+ * Send one page through Claude using the prettify system prompt and the agent
+ * SDK's structured-output mode. The runtime enforces `prettifyPageOutputSchema`
+ * and surfaces the parsed object on the result message's `structured_output`
+ * field — no fence-stripping or manual JSON.parse needed.
+ *
+ * `oasPartialJson` arrives as a string (the schema models it as such to dodge
+ * structured outputs' `additionalProperties: false` rule, which is
+ * incompatible with OpenAPI's free-form path/schema keys). We parse it here
+ * and re-expose it as `oasPartial` so callers see the conventional shape.
+ */
+async function prettifyPage({ source, title, relativePath, model, localComponents, availableComponents }) {
+  const { systemPrompt, userPrompt } = prettifyPagePrompt({
+    source, title, relativePath, localComponents, availableComponents,
+  })
+
+  let structured
   for await (const message of query({
     prompt: userPrompt,
     options: {
       systemPrompt,
       allowedTools: [],
+      outputFormat: { type: 'json_schema', schema: prettifyPageOutputSchema },
       ...(model ? { model } : {}),
     },
   })) {
-    if (message.type === 'assistant' && message.message?.content) {
-      for (const block of message.message.content) {
-        if (block.type === 'text' && block.text) text += block.text;
+    if (message.type === 'result') {
+      if (message.subtype !== 'success') {
+        throw new Error(`${message.subtype}${message.error?.message ? ' — ' + message.error.message : ''}`)
       }
-    } else if (message.type === 'result') {
-      if (message.subtype && message.subtype !== 'success') {
-        throw new Error(`${message.subtype}${message.error?.message ? ' — ' + message.error.message : ''}`);
-      }
-      break;
+      structured = message.structured_output
+      break
     }
   }
 
-  const stripped = stripCodeFences(text);
-  try {
-    return JSON.parse(stripped);
-  } catch (e) {
-    throw new Error(`invalid JSON (${e.message}); first 200 chars: ${stripped.slice(0, 200)}`);
+  if (!structured || typeof structured !== 'object') {
+    throw new Error('agent returned no structured output')
+  }
+
+  let oasPartial = null
+  if (typeof structured.oasPartialJson === 'string' && structured.oasPartialJson.length > 0) {
+    try {
+      oasPartial = JSON.parse(structured.oasPartialJson)
+    } catch (e) {
+      throw new Error(`invalid oasPartialJson (${e.message}); first 200 chars: ${structured.oasPartialJson.slice(0, 200)}`)
+    }
   }
+
+  return { body: structured.body, isApiRef: structured.isApiRef, oasPartial }
 }
 
-export async function run(options, _cmd, ctx) {
-  const { gitRoot } = ctx;
+export async function run(file, options, _cmd, ctx) {
+  const { gitRoot } = ctx
 
   if (!isAgenticCli()) {
-    printHeader({ version: pkg.version, binName: styles.binName(), indent: '  ' });
-    console.log();
+    printHeader({ version: pkg.version, binName: styles.binName(), indent: '  ' })
+    console.log()
+  }
+
+  let files
+  if (file) {
+    const absFile = path.resolve(file)
+    if (!fs.existsSync(absFile)) {
+      styles.error(`File not found: ${styles.bold(file)}`)
+      process.exit(1)
+    }
+    const relFile = path.relative(gitRoot, absFile)
+    if (relFile.startsWith('..') || path.isAbsolute(relFile)) {
+      styles.error(`File must be inside the repo: ${styles.bold(file)}`)
+      process.exit(1)
+    }
+    files = [relFile]
+  } else {
+    files = collectFiles(gitRoot)
   }
 
-  const files = collectFiles(gitRoot);
   if (files.length === 0) {
-    styles.warning('No docs files found to prettify.');
-    return;
+    styles.warning('No docs files found to prettify.')
+    return
+  }
+
+  const localComponents = listLocalComponents(gitRoot)
+  const localNames = new Set(localComponents.map((c) => c.name))
+
+  // Pull the full marketplace catalog so Claude knows what else it can use.
+  // Summaries come from disk cache (populated by `readme components list`);
+  // we don't fetch readmes here to keep the prompt path fast.
+  let marketplace = []
+  if (options.autoInstall !== false) {
+    try {
+      marketplace = await listMarketplaceComponents()
+    } catch (err) {
+      styles.warning(`Could not load marketplace catalog: ${err.message} (continuing without it)`)
+    }
+  }
+  const marketplaceNames = new Set(marketplace.map((c) => c.name))
+  const availableComponents = marketplace
+    .filter((c) => !localNames.has(c.name))
+    .map((c) => ({ name: c.name, summary: summarizeReadme(readCachedReadme(c.name)) }))
+
+  // Concurrent workers may both try to install the same component. Coalesce
+  // duplicate installs through a Map<tag, Promise> — the second caller awaits
+  // the first's install instead of clobbering it.
+  const inFlightInstalls = new Map()
+  const autoInstalled = new Set()
+  function ensureInstalled(tag) {
+    if (inFlightInstalls.has(tag)) return inFlightInstalls.get(tag)
+    const p = (async () => {
+      await installComponent(gitRoot, tag)
+      localNames.add(tag)
+      autoInstalled.add(tag)
+    })()
+    inFlightInstalls.set(tag, p)
+    return p
   }
 
   styles.info(
-    `Prettifying ${styles.bold(String(files.length))} file${files.length === 1 ? '' : 's'} with ${styles.bold(options.model)}${options.dryRun ? styles.dim(' (dry run)') : ''}.`,
-  );
-  console.log();
+    `Prettifying ${styles.bold(String(files.length))} file${files.length === 1 ? '' : 's'} with ${styles.bold(options.model)}${options.jobs > 1 ? ` (${styles.bold('-j ' + options.jobs)})` : ''}${options.dryRun ? styles.dim(' (dry run)') : ''}.`,
+  )
+  if (localComponents.length > 0) {
+    styles.info(
+      `Found ${styles.bold(String(localComponents.length))} local component${localComponents.length === 1 ? '' : 's'} in ${styles.bold('./components/')}: ${localComponents.map((c) => c.name).join(', ')}`,
+    )
+  }
+  if (options.autoInstall !== false && availableComponents.length > 0) {
+    styles.info(
+      `${styles.bold(String(availableComponents.length))} additional marketplace component${availableComponents.length === 1 ? '' : 's'} will be auto-installed if used.`,
+    )
+  }
+  console.log()
+
+  let updated = 0
+  let unchanged = 0
+  const failures = []
 
-  const spinner = ora({ color: 'blue' }).start();
-  let updated = 0;
-  let unchanged = 0;
-  const failures = [];
+  const rows = Math.min(options.jobs, files.length)
+  const multi = new MultiSpinner({ rows, color: 'blue' })
+  multi.start()
 
-  for (const relativePath of files) {
-    spinner.text = relativePath;
-    const filePath = path.join(gitRoot, relativePath);
-    const original = fs.readFileSync(filePath, 'utf-8');
-    const parsed = matter(original);
+  await runPool(files, options.jobs, async (relativePath, _i, workerIdx) => {
+    multi.setLine(workerIdx, relativePath)
 
     try {
+      const filePath = path.join(gitRoot, relativePath)
+      const original = fs.readFileSync(filePath, 'utf-8')
+      const parsed = matter(original)
+
+      let source = parsed.content
+      const importUrl = parsed.data?.['x-import']
+      if (importUrl) {
+        multi.setLine(workerIdx, `${relativePath} ${styles.dim(`(fetching ${importUrl})`)}`)
+        source = await fetchImport(importUrl)
+        multi.setLine(workerIdx, relativePath)
+      }
+
       const result = await prettifyPage({
-        source: parsed.content,
+        source,
         title: parsed.data?.title,
         relativePath,
         model: options.model,
-      });
+        localComponents,
+        availableComponents: options.autoInstall === false ? [] : availableComponents,
+      })
 
       if (typeof result?.body !== 'string') {
-        throw new Error('response missing "body" field');
+        throw new Error('response missing "body" field')
+      }
+
+      // Auto-install marketplace components Claude used in the body. We do
+      // this before writing the file so the docs page and its components land
+      // together — and we share the inFlightInstalls map across workers so a
+      // tag is only fetched once.
+      if (options.autoInstall !== false && !options.dryRun) {
+        const used = extractUsedTags(result.body)
+        for (const tag of used) {
+          if (NATIVE_TAGS.has(tag)) continue
+          if (localNames.has(tag)) continue
+          if (!marketplaceNames.has(tag)) continue
+          try {
+            multi.setLine(workerIdx, `${relativePath} ${styles.dim(`(installing <${tag}>)`)}`)
+            await ensureInstalled(tag)
+          } catch (err) {
+            failures.push({ relativePath, message: `auto-install <${tag}> failed: ${err.message}` })
+          }
+        }
+        multi.setLine(workerIdx, relativePath)
       }
 
-      const newFm = { ...parsed.data };
-      if (result.excerpt && !newFm.excerpt) newFm.excerpt = result.excerpt;
-      const next = matter.stringify(result.body, newFm);
+      const newFm = { ...parsed.data }
+      delete newFm.hidden
+      delete newFm['x-import']
+      if (result.excerpt && !newFm.excerpt) newFm.excerpt = result.excerpt
+      const next = matter.stringify(result.body, newFm)
 
       if (next === original) {
-        unchanged++;
+        unchanged++
+        if (!process.stderr.isTTY) multi.log(`  ${styles.dim('·')} ${relativePath}`)
       } else {
-        if (!options.dryRun) fs.writeFileSync(filePath, next, 'utf-8');
-        updated++;
+        if (!options.dryRun) fs.writeFileSync(filePath, next, 'utf-8')
+        updated++
+        if (!process.stderr.isTTY) multi.log(`  ${styles.success('✓')} ${relativePath}`)
       }
     } catch (err) {
-      failures.push({ relativePath, message: err.message || String(err) });
+      failures.push({ relativePath, message: err.message || String(err) })
+      if (!process.stderr.isTTY) {
+        multi.log(`  ${styles.err('●')} ${relativePath} ${styles.dim('— ' + (err.message || String(err)))}`)
+      }
+    } finally {
+      multi.clearLine(workerIdx)
     }
-  }
+  })
 
-  spinner.stop();
+  multi.stop()
 
-  console.log();
-  const verb = options.dryRun ? 'would update' : 'updated';
-  styles.ok(
-    `${styles.bold(String(updated))} ${verb} · ${styles.bold(String(unchanged))} unchanged${failures.length ? ` · ${styles.bold(String(failures.length))} failed` : ''}`,
-  );
+  console.log()
+  const verb = options.dryRun ? 'would update' : 'updated'
+  styles.ok(`${styles.bold(String(updated))} ${verb} · ${styles.bold(String(unchanged))} unchanged${failures.length ? ` · ${styles.bold(String(failures.length))} failed` : ''}`)
+  if (autoInstalled.size > 0) {
+    styles.ok(`Auto-installed ${styles.bold(String(autoInstalled.size))} component${autoInstalled.size === 1 ? '' : 's'}: ${[...autoInstalled].join(', ')}`)
+  }
 
   if (failures.length > 0) {
-    console.log();
+    console.log()
     for (const f of failures) {
-      console.log(`  ${styles.err('●')} ${styles.bold(f.relativePath)} ${styles.dim('— ' + f.message)}`);
+      console.log(`  ${styles.err('●')} ${styles.bold(f.relativePath)} ${styles.dim('— ' + f.message)}`)
     }
-    process.exit(1);
+    process.exit(1)
   }
 }

package/src/prompts/index.js

@@ -296,16 +296,44 @@ If the page IS an API reference, extract the endpoints into a partial OpenAPI 3.
 - Use realistic types inferred from the documentation (don't use "object" or "string" for everything)
 - Include the server/base URL if mentioned
 
-## Output Format
-Respond with ONLY a JSON object (no markdown fences, no explanation):
-{
-  "excerpt": "one-line summary of the page",
-  "body": "the full MDX-fixed markdown content (no frontmatter)",
-  "isApiRef": true/false,
-  "suggestedCategory": "Best Category Name",
-  "oasPartial": null OR {"paths": {"/endpoint": {"get": {...}}}, "schemas": {"ModelName": {...}}}
-}
-When isApiRef=false, set oasPartial to null. When isApiRef=true, include the extracted paths and schemas.`;
+## Output
+The runtime enforces the response shape via a JSON schema, so don't worry about formatting — just populate every field with the right content.
+
+- \`body\`: the full MDX-fixed markdown content with no frontmatter. This is the load-bearing field — get the content right, render it human-readable, and don't leave broken MDX.
+- \`isApiRef\`: true if this page primarily documents API endpoints (per the rules above), false otherwise.
+- \`oasPartialJson\`: when \`isApiRef\` is true, set this to a JSON-stringified partial OpenAPI 3.0 fragment of the form \`{"paths": {"/endpoint": {"get": {...}}}, "schemas": {"ModelName": {...}}}\`. The caller will \`JSON.parse\` it. When \`isApiRef\` is false, set it to \`null\`.
+`;
+
+/**
+ * JSON schema describing the structured output shape returned by the prettify
+ * agent. Pass to the Claude Agent SDK's \`outputFormat\` option so the runtime
+ * enforces the shape and returns parsed data on the result message's
+ * \`structured_output\` field — no manual fence-stripping or JSON.parse needed.
+ *
+ * \`oasPartialJson\` is a stringified OAS fragment rather than a nested object
+ * because Anthropic's structured outputs require \`additionalProperties: false\`
+ * on every object, which is incompatible with OpenAPI's free-form path/schema
+ * keys. The caller parses the string client-side.
+ */
+export const prettifyPageOutputSchema = {
+  type: 'object',
+  additionalProperties: false,
+  required: ['body', 'isApiRef', 'oasPartialJson'],
+  properties: {
+    body: {
+      type: 'string',
+      description: 'The MDX-fixed markdown content for the page (no frontmatter).',
+    },
+    isApiRef: {
+      type: 'boolean',
+      description: 'True if the page primarily documents REST/GraphQL/gRPC endpoints.',
+    },
+    oasPartialJson: {
+      anyOf: [{ type: 'string' }, { type: 'null' }],
+      description: 'JSON-stringified partial OpenAPI 3.0 fragment when isApiRef is true; null otherwise.',
+    },
+  },
+};
 
 /**
  * @param {object} input
@@ -326,9 +354,56 @@ export function buildPrettifyPageUserPrompt({ source, title, relativePath }) {
   ].join('\n');
 }
 
+/**
+ * Build a short addendum describing the user's locally-installed marketplace
+ * components so Claude can use them in prettified output. We pull the first
+ * usage signature out of each component's mdx (a `<Name ... />` line near the
+ * bottom of the file).
+ *
+ * @param {Array<{ name: string, source: string }>} components
+ * @returns {string} — the addendum, or '' if there are no components.
+ */
+export function buildLocalComponentsSection(components) {
+  if (!components || components.length === 0) return '';
+  const lines = ['', '### Project-specific components (installed in ./components/)'];
+  lines.push(
+    'The following custom components are installed in this project and may be used in prettified output. Prefer them when they fit the content better than plain markdown — same liberal-upgrade bias as the built-in components above. Do NOT invent components that are not listed here or in the built-in set.',
+    '',
+  );
+  for (const c of components) {
+    const sig = (c.source.match(new RegExp(`<${c.name}[\\s\\S]*?(?:/>|</${c.name}>)`, 'm')) || [])[0];
+    lines.push(`- \`<${c.name}>\`${sig ? ` — e.g. \`${sig.replace(/\s+/g, ' ').slice(0, 200)}\`` : ''}`);
+  }
+  return lines.join('\n');
+}
+
+/**
+ * Build a short section listing additional marketplace components that are
+ * NOT yet installed but can be — the CLI will auto-install any of them that
+ * the model uses in its output.
+ *
+ * @param {Array<{ name: string, summary?: string }>} available
+ *        Marketplace components not currently in ./components/.
+ * @returns {string} — the addendum, or '' if there are none.
+ */
+export function buildAvailableComponentsSection(available) {
+  if (!available || available.length === 0) return '';
+  const lines = ['', '### Marketplace components (auto-installed on use)'];
+  lines.push(
+    'These components are NOT yet installed locally, but the CLI will copy them into ./components/ automatically if you use any of them. Use them when they clearly fit the content — same upgrade bias as built-in components. Do NOT invent components outside this list and the built-in set.',
+    '',
+  );
+  for (const c of available) {
+    lines.push(`- \`<${c.name}>\`${c.summary ? ` — ${c.summary}` : ''}`);
+  }
+  return lines.join('\n');
+}
+
 export function prettifyPagePrompt(input) {
+  const localAddendum = buildLocalComponentsSection(input.localComponents);
+  const availableAddendum = buildAvailableComponentsSection(input.availableComponents);
   return {
-    systemPrompt: prettifyPageSystemPrompt,
+    systemPrompt: prettifyPageSystemPrompt + localAddendum + availableAddendum,
     userPrompt: buildPrettifyPageUserPrompt(input),
   };
 }