@readme/cli 0.0.27 → 0.0.30

package/src/prompts/index.js

@@ -19,16 +19,33 @@
 
 export const slotOrphansSystemPrompt = [
   'You assign orphan documentation pages to existing categories.',
-  'Output ONLY a valid JSON array of integers — one per orphan page, in order.',
+  'Respond with a JSON object of the form {"assignments": [...]} — one integer per orphan page, in order.',
   'Each integer is the category index (0-based) the orphan belongs in, or -1 if none fit.',
   '',
-  'Example output: [0, 2, 0, 1, -1, 3]',
+  'Example output: {"assignments": [0, 2, 0, 1, -1, 3]}',
   '',
   'Guidance:',
   '- Choose the best semantic fit based on the orphan\'s title and URL path.',
   '- Only use -1 when no existing category is plausible.',
 ].join('\n');
 
+/**
+ * Structured-output schema for slotOrphans. Root must be an object (top-level
+ * arrays aren't allowed), so the index array is wrapped in `assignments`.
+ */
+export const slotOrphansOutputSchema = {
+  type: 'object',
+  additionalProperties: false,
+  required: ['assignments'],
+  properties: {
+    assignments: {
+      type: 'array',
+      items: { type: 'integer' },
+      description: 'One entry per orphan page, in input order: the 0-based category index, or -1 if none fit.',
+    },
+  },
+};
+
 /**
  * @param {object}   input
  * @param {Array<{ title: string, pages?: Array<{ title: string }> }>} input.categories
@@ -54,7 +71,7 @@ export function buildSlotOrphansUserPrompt({ categories, orphans }) {
     `${orphans.length} orphan pages to slot:`,
     ...orphanList,
     '',
-    `Output a JSON array of ${orphans.length} integers, one per orphan in order (category index 0..${categories.length - 1}, or -1 for none).`,
+    `Output {"assignments": [...]} with ${orphans.length} integers, one per orphan in order (category index 0..${categories.length - 1}, or -1 for none).`,
   ].join('\n');
 }
 
@@ -69,12 +86,29 @@ export function slotOrphansPrompt(input) {
 
 export const iconizeNavSystemPrompt = [
   'You assign one FontAwesome Free Solid icon to each documentation category.',
-  'Output ONLY a valid JSON array of icon name strings, one per input category, in order.',
-  'Use the icon name only (no "fa-" prefix, no object wrapper).',
+  'Respond with a JSON object of the form {"icons": [...]} — one icon name string per input category, in order.',
+  'Use the icon name only (no "fa-" prefix).',
   '',
-  'Example output: ["rocket", "book", "code", "gear"]',
+  'Example output: {"icons": ["rocket", "book", "code", "gear"]}',
 ].join('\n');
 
+/**
+ * Structured-output schema for iconizeNav. Root must be an object, so the
+ * icon-name array is wrapped in `icons`.
+ */
+export const iconizeNavOutputSchema = {
+  type: 'object',
+  additionalProperties: false,
+  required: ['icons'],
+  properties: {
+    icons: {
+      type: 'array',
+      items: { type: 'string' },
+      description: 'One FontAwesome Free Solid icon name per input category, in order (no "fa-" prefix).',
+    },
+  },
+};
+
 /**
  * @param {object} input
  * @param {Array<{ title: string }>} input.categories
@@ -86,7 +120,7 @@ export function buildIconizeNavUserPrompt({ categories }) {
     '',
     ...categories.map((c, i) => `${i}. ${c.title}`),
     '',
-    'Return a JSON array of FontAwesome icon names, one per category, in order.',
+    'Return {"icons": [...]} with one FontAwesome icon name per category, in order.',
   ].join('\n');
 }
 
@@ -101,13 +135,15 @@ export function iconizeNavPrompt(input) {
 
 export const organizeFromSectionsSystemPrompt = [
   'You assign a FontAwesome Free Solid icon to each documentation section, and lightly polish the section title.',
-  'Output ONLY a valid JSON array — no prose, no markdown, no code fences.',
+  'Respond with a JSON object matching the schema below.',
   '',
   'Schema:',
-  '[',
-  '  { "title": "<lightly polished Title Case, 1-4 words>", "icon": "<fontawesome icon name, no fa- prefix>" },',
-  '  ...',
-  ']',
+  '{',
+  '  "sections": [',
+  '    { "title": "<lightly polished Title Case, 1-4 words>", "icon": "<fontawesome icon name, no fa- prefix>" },',
+  '    ...',
+  '  ]',
+  '}',
   '',
   'Rules:',
   '- Return exactly one entry per input section, in the same order.',
@@ -115,6 +151,31 @@ export const organizeFromSectionsSystemPrompt = [
   '- Pick an icon that semantically fits the section (e.g. rocket for Getting Started, code for API, plug for Integrations).',
 ].join('\n');
 
+/**
+ * Structured-output schema for organizeFromSections. Root must be an object,
+ * so the per-section entries are wrapped in `sections`.
+ */
+export const organizeFromSectionsOutputSchema = {
+  type: 'object',
+  additionalProperties: false,
+  required: ['sections'],
+  properties: {
+    sections: {
+      type: 'array',
+      description: 'Exactly one entry per input section, in the same order.',
+      items: {
+        type: 'object',
+        additionalProperties: false,
+        required: ['title', 'icon'],
+        properties: {
+          title: { type: 'string', description: 'Lightly polished Title Case section title, 1-4 words.' },
+          icon: { type: 'string', description: 'FontAwesome Free Solid icon name, no "fa-" prefix.' },
+        },
+      },
+    },
+  },
+};
+
 /**
  * @param {object} input
  * @param {string|null|undefined} input.siteTitle
@@ -128,7 +189,7 @@ export function buildOrganizeFromSectionsUserPrompt({ siteTitle, sections }) {
     '',
     ...sections.map((s, i) => `${i}. ${s.title} (${s.items.length} pages)`),
     '',
-    'Output the JSON array now.',
+    'Output the JSON object now.',
   ].join('\n');
 }
 
@@ -166,6 +227,33 @@ export const organizeFromScratchSystemPrompt = [
   '- Keep category titles human-readable, Title Case, 1-4 words.',
 ].join('\n');
 
+/** Structured-output schema for organizeFromScratch. */
+export const organizeFromScratchOutputSchema = {
+  type: 'object',
+  additionalProperties: false,
+  required: ['title', 'categories'],
+  properties: {
+    title: { type: 'string', description: 'Short site or doc title.' },
+    categories: {
+      type: 'array',
+      items: {
+        type: 'object',
+        additionalProperties: false,
+        required: ['title', 'icon', 'pageIds'],
+        properties: {
+          title: { type: 'string', description: 'Title Case category name, 1-4 words.' },
+          icon: { type: 'string', description: 'FontAwesome Free Solid icon name, no "fa-" prefix.' },
+          pageIds: {
+            type: 'array',
+            items: { type: 'integer' },
+            description: 'Integer ids from the input page list. Every input id appears in exactly one category.',
+          },
+        },
+      },
+    },
+  },
+};
+
 /**
  * @param {object} input
  * @param {string|null|undefined} input.siteTitle

package/src/utils/lint.js

@@ -1,6 +1,6 @@
 import fs from 'node:fs'
 import path from 'node:path'
-import { pathToFileURL } from 'node:url'
+import { pathToFileURL, fileURLToPath } from 'node:url'
 
 const TARGET_PATTERNS = [
   { dir: 'custom_blocks', ext: ['.mdx', '.md'] },
@@ -38,7 +38,7 @@ export function collectFiles(gitRoot) {
  * `validateAll()` for cross-file checks (like ordering).
  */
 async function loadValidators() {
-  const validatorsDir = path.join(path.dirname(new URL(import.meta.url).pathname), '..', 'validators')
+  const validatorsDir = path.join(path.dirname(fileURLToPath(import.meta.url)), '..', 'validators')
   const files = fs.readdirSync(validatorsDir).filter((f) => f.endsWith('.js'))
   const validators = []
 

package/src/utils/llms.js

@@ -0,0 +1,224 @@
+const H1_RE = /^#\s+(.+)$/
+const H2_RE = /^##\s+(.+)$/
+const H3_RE = /^###\s+(.+)$/
+
+// llms.txt might have a literal " (section)" marker to every
+// grouping heading (e.g. `## getting-started (section)`). It's internal
+// metadata, not part of the category name — strip it before it reaches the sidebar.
+const SECTION_MARKER_RE = /\s*\(section\)\s*$/i
+
+// Max items per section before we consider it "oversized" for usability purposes.
+// Mirrors the cap in usableSections() in import.js.
+const MAX_SECTION_ITEMS = 200
+// Any line whose meaningful content is a markdown link. Accepts:
+//   - Standard list rows:           `- [text](url)`, `* [text](url) — desc`
+//   - Bare link lines:              `[text](url)`
+//   - Breadcrumb-prefixed rows used by Fern/Mintlify-style indices:
+//       `ElevenAgents [Agent WebSockets](url)`
+//       `API Reference > Agents > Branches [List branches](url)`
+//   - Trailing-parenthetical-breadcrumb rows (AssemblyAI/Fern variant) — the
+//     breadcrumb sits between the url and the description and is matched-and-discarded:
+//       `[text](url) (Breadcrumb > path): description`
+// Captures: [prefix, text, url, description] (prefix and description may be empty).
+const LINK_LINE_RE = /^\s*(?:[-*+]\s+)?([^\[\n]*?)\s*\[([^\]]+)\] ?\(([^)\s]+)\)(?:\s+\([^)]*\))?(?:\s*[:—–-]\s*(.+?))?\s*$/
+const BLOCKQUOTE_RE = /^\s*>/
+const FENCE_RE = /^\s*(?:```|~~~)/
+
+// A llms.txt is considered usable as long as it has at least one link row.
+// Beyond that we enforce a structural ratio (share of "spec-shaped" lines —
+// H1/H2/blockquote/link row/blank — over total lines), but only once the
+// file has enough links for the ratio to be statistically meaningful. A
+// 3-link file with one prose sentence would compute a misleading ratio; we
+// can't tell signal from noise at that size, so we trust it and let the
+// BFS-merge dedupe handle any junk that slips through.
+const RATIO_CHECK_MIN_LINKS = 10
+const MIN_CONFORMING_RATIO = 0.7
+
+export function analyzeLlmsTxt(body, llmsUrl, options = {}) {
+  const parsed = parseLlmsTxt(body, llmsUrl, options)
+  const lineStats = classifyLines(body)
+  const linkItems = parsed.sections.reduce((sum, s) => sum + s.items.length, 0)
+  const reason = getSkipReason(body, linkItems, lineStats)
+
+  return {
+    parsed,
+    usable: !reason,
+    reason,
+    stats: { ...lineStats, linkItems },
+  }
+}
+
+export function parseLlmsTxt(body, llmsUrl, options = {}) {
+  const h2Result = parseSections(body, llmsUrl, H2_RE, options)
+
+  // If every H2 section is oversized (or there are none), and the file has H3
+  // headings, re-parse treating H3 (###) as section boundaries.
+  // For strange LLMS.txt formatting
+  const allOversized = h2Result.sections.length === 0 ||
+    h2Result.sections.every((s) => s.items.length > MAX_SECTION_ITEMS)
+  if (allOversized && /^###\s/m.test(body)) {
+    const h3Result = parseSections(body, llmsUrl, H3_RE, options)
+    if (h3Result.sections.length > 1) {
+      return { ...h3Result, h3Fallback: true }
+    }
+  }
+
+  return h2Result
+}
+
+function parseSections(body, llmsUrl, sectionRe, options = {}) {
+  const lines = body.split(/\r?\n/)
+  let title = null
+  const sections = []
+  let current = null
+
+  for (const line of lines) {
+    const h1 = line.match(H1_RE)
+    if (h1 && !title) {
+      title = h1[1].trim()
+      continue
+    }
+
+    const sectionMatch = line.match(sectionRe)
+    if (sectionMatch) {
+      current = { title: sectionMatch[1].trim().replace(SECTION_MARKER_RE, '').trim(), items: [] }
+      sections.push(current)
+      continue
+    }
+
+    const item = parseListLink(line, llmsUrl, options)
+    if (!item) continue
+
+    if (!current) {
+      current = { title: 'Resources', items: [] }
+      sections.push(current)
+    }
+    current.items.push(item)
+  }
+
+  return { title, sections }
+}
+
+function getSkipReason(body, linkItems, lineStats) {
+  if (/^---\r?\n/.test(body)) return 'starts with YAML frontmatter'
+  if (linkItems === 0) return 'no link items'
+  if (linkItems >= RATIO_CHECK_MIN_LINKS && lineStats.ratio < MIN_CONFORMING_RATIO) {
+    return `conforming-line ratio ${lineStats.ratio.toFixed(2)} below ${MIN_CONFORMING_RATIO} (${lineStats.conforming}/${lineStats.total} lines)`
+  }
+  return null
+}
+
+/**
+ * Walk the body line-by-line and bucket each line as conforming or not.
+ *
+ *   conforming     blank, H1, H2, blockquote, standard link-row
+ *   non-conforming everything else, plus every line inside (and the delimiters
+ *                  of) a fenced code block
+ */
+function classifyLines(body) {
+  const lines = body.split(/\r?\n/)
+  let inFence = false
+  let conforming = 0
+  let nonConforming = 0
+
+  for (const line of lines) {
+    if (FENCE_RE.test(line)) {
+      nonConforming++
+      inFence = !inFence
+      continue
+    }
+    if (inFence) {
+      nonConforming++
+      continue
+    }
+    if (line.trim() === '') {
+      conforming++
+      continue
+    }
+    if (H1_RE.test(line) || H2_RE.test(line) || LINK_LINE_RE.test(line) || BLOCKQUOTE_RE.test(line)) {
+      conforming++
+      continue
+    }
+    nonConforming++
+  }
+
+  const total = conforming + nonConforming
+  const ratio = total === 0 ? 0 : conforming / total
+  return { conforming, nonConforming, total, ratio }
+}
+
+function parseListLink(line, llmsUrl, options = {}) {
+  const match = line.match(LINK_LINE_RE)
+  if (!match) return null
+
+  const [, prefix, text, rawUrl, trailingDesc] = match
+  const url = normalizeUrl(rawUrl, llmsUrl, options)
+  if (!url) return null
+
+  // Prefer an explicit trailing description (`[text](url) — desc`); fall back
+  // to the breadcrumb prefix (`API Reference > Agents [text](url)`) when no
+  // explicit description is present. Either gives downstream extra context.
+  const description = trailingDesc?.trim() || prefix?.trim() || null
+
+  return {
+    text: text.trim(),
+    url,
+    description: description || null,
+  }
+}
+
+function normalizeUrl(rawUrl, llmsUrl, options = {}) {
+  const trimmed = String(rawUrl || '').trim().replace(/[.,;]+$/, '')
+  if (!trimmed || /^#/.test(trimmed) || /^(mailto|javascript):/i.test(trimmed)) return null
+
+  try {
+    const url = llmsUrl ? new URL(resolveLlmsRelativeUrl(trimmed, llmsUrl, options), llmsUrl) : new URL(trimmed)
+    if (url.protocol !== 'http:' && url.protocol !== 'https:') return null
+    return url.toString()
+  } catch {
+    return null
+  }
+}
+
+function resolveLlmsRelativeUrl(trimmed, llmsUrl, options) {
+  if (options.rootRelativeResolution !== 'llms-dir' || !isRootRelativeUrl(trimmed)) return trimmed
+
+  const nestedPrefix = getLlmsNestedPrefix(llmsUrl)
+  if (!nestedPrefix) return trimmed
+
+  const normallyResolved = new URL(trimmed, llmsUrl)
+  if (!shouldResolveRootRelativeFromLlmsDir(trimmed, normallyResolved.pathname, nestedPrefix)) return trimmed
+
+  return `.${trimmed}`
+}
+
+function shouldResolveRootRelativeFromLlmsDir(rawUrl, pathname, nestedPrefix) {
+  if (isPathWithinPrefix(pathname, nestedPrefix)) return false
+  if (/\/llms\.txt$/i.test(pathname)) return false
+
+  const lastSegment = pathname.split('/').pop() || ''
+  const ext = lastSegment.match(/\.([a-z0-9]+)$/i)?.[1]?.toLowerCase()
+  if (ext && !['htm', 'html', 'md', 'mdx'].includes(ext)) return false
+
+  return isRootRelativeUrl(rawUrl)
+}
+
+function getLlmsNestedPrefix(llmsUrl) {
+  if (!llmsUrl) return null
+
+  try {
+    const pathname = new URL(llmsUrl).pathname
+    const dir = pathname.replace(/llms\.txt$/i, '').replace(/\/+$/, '/')
+    return dir === '/' ? null : dir
+  } catch {
+    return null
+  }
+}
+
+function isRootRelativeUrl(rawUrl) {
+  return /^\/(?!\/)/.test(rawUrl)
+}
+
+function isPathWithinPrefix(pathname, prefix) {
+  return pathname === prefix.replace(/\/$/, '') || pathname.startsWith(prefix)
+}