@readme/cli 0.0.27 → 0.0.30

package/src/commands/oas-sync.js

@@ -14,12 +14,6 @@ export const description = 'Sync reference pages with OpenAPI specs';
 
 const HTTP_METHODS = new Set(['get', 'post', 'put', 'patch', 'delete', 'options', 'head', 'trace']);
 
-/**
- * Check if this is a ReadMeConfig spec (internal ReadMe pages — skip title/excerpt updates).
- */
-function isReadMeConfig(spec) {
-  return spec.info?.title === 'ReadMeConfig';
-}
 
 /**
  * Find OAS files at the root of reference/ (JSON or YAML).
@@ -62,29 +56,104 @@ function generateOperationId(method, pathStr) {
 }
 
 /**
- * Extract operations from an OAS spec.
- * Returns a Map of operationId -> { summary, description, tag, operationId }.
- * For operations without an operationId, a synthetic one is generated from the method and path.
+ * Identity key for an operation record (or an existing page's frontmatter),
+ * used everywhere operations/pages are looked up by operationId. `paths` and
+ * `webhooks` are separate namespaces in an OAS document, but both can have
+ * operationId omitted, so their synthetic `<method>_<name>` ids can
+ * legitimately collide (e.g. `POST /orders` and webhook `POST orders` both
+ * synthesize to `post_orders`) — the isWebhook flag disambiguates them so
+ * neither silently overwrites the other in an operationId-only Map.
+ */
+export function operationKey({ operationId, isWebhook }) {
+  return `${isWebhook ? 'webhook' : 'path'}:${operationId}`;
+}
+
+/**
+ * Resolve a `paths`/`webhooks` entry that's a Reference Object (OAS 3.1,
+ * `{ $ref: '#/components/pathItems/Name' }`) against the spec's own
+ * `components.pathItems`, following chained refs (a pathItem that is itself
+ * a $ref to another) until a literal Path Item is reached. Only same-document
+ * refs in that exact form are supported; anything else (external files,
+ * other pointer shapes, an unresolvable name, or a cycle) is left unresolved
+ * and quietly skipped by the caller, same as before this existed.
+ */
+function resolveLocalPathItemRef(entry, spec) {
+  const seen = new Set();
+  let current = entry;
+  // Sibling fields (e.g. an inline operation) alongside a $ref are explicitly
+  // allowed in an OAS 3.1 Path Item Object — accumulate them from every hop
+  // in the chain so they aren't discarded once $ref is followed. A field
+  // declared at an outer/earlier hop wins over the same field found deeper
+  // in the chain (OAS itself leaves this "undefined" when both define it).
+  let overrides = {};
+  const finish = () => ({ ...current, ...overrides });
+
+  while (current && typeof current.$ref === 'string') {
+    const { $ref, ...siblings } = current;
+    overrides = { ...siblings, ...overrides };
+
+    if (seen.has(current.$ref)) return finish();
+    seen.add(current.$ref);
+
+    const match = current.$ref.match(/^#\/components\/pathItems\/(.+)$/);
+    if (!match) return finish();
+
+    let name;
+    try {
+      name = decodeURIComponent(match[1]).replace(/~1/g, '/').replace(/~0/g, '~');
+    } catch {
+      // Malformed percent-escape — leave unresolved rather than throwing and
+      // aborting the whole sync/lint run over one bad $ref.
+      return finish();
+    }
+    const resolved = spec.components?.pathItems?.[name];
+    if (!resolved) return finish();
+
+    current = resolved;
+  }
+
+  return finish();
+}
+
+/**
+ * Extract operations from an OAS spec's `paths`, plus its OAS 3.1 `webhooks`
+ * (callouts the API itself makes to a client-registered URL, not endpoints the
+ * API exposes — a separate top-level sibling of `paths` with the same
+ * Operation Object shape). The platform pages a webhook the same way it pages
+ * a path operation: a synthetic `post_<name>` operationId when none is given,
+ * grouped by its own tag or, absent one, its own category keyed by its raw
+ * name — never merged with `paths` operations of the same name.
+ * Returns a Map keyed by `operationKey()` -> { summary, description, tag,
+ * path, operationId, isWebhook }. For operations without an operationId, a
+ * synthetic one is generated from the method and path (or webhook name).
  */
 export function extractOperations(spec) {
   const ops = new Map();
-  const paths = spec.paths || {};
 
-  for (const [pathStr, methods] of Object.entries(paths)) {
-    for (const [method, operation] of Object.entries(methods)) {
-      if (!HTTP_METHODS.has(method)) continue;
+  function collect(entries, isWebhook) {
+    for (const [pathStr, rawItem] of Object.entries(entries)) {
+      const methods = resolveLocalPathItemRef(rawItem, spec);
+
+      for (const [method, operation] of Object.entries(methods)) {
+        if (!HTTP_METHODS.has(method)) continue;
 
-      const operationId = operation.operationId || generateOperationId(method, pathStr);
+        const operationId = operation.operationId || generateOperationId(method, pathStr);
 
-      ops.set(operationId, {
-        operationId,
-        summary: operation.summary || null,
-        description: operation.description || null,
-        tag: (operation.tags && operation.tags[0]) || null,
-      });
+        ops.set(operationKey({ operationId, isWebhook }), {
+          operationId,
+          summary: operation.summary || null,
+          description: operation.description || null,
+          tag: (operation.tags && operation.tags[0]) || null,
+          path: pathStr,
+          isWebhook,
+        });
+      }
     }
   }
 
+  collect(spec.paths || {}, false);
+  collect(spec.webhooks || {}, true);
+
   return ops;
 }
 
@@ -168,29 +237,178 @@ function removeFromOrder(orderPath, slug) {
   }
 }
 
-function buildPageContent({ oasFilename, operationId, summary, description }) {
+/**
+ * Spec-derived values (info.title, tags, operationIds) become directory and
+ * file names. Collapse path separators and dot-only names into a single safe
+ * segment so a crafted spec can't write outside reference/.
+ */
+function safeSegment(value, fallback) {
+  const segment = String(value).replace(/[/\\]/g, '-').trim();
+  return !segment || segment === '.' || segment === '..' ? fallback : segment;
+}
+
+function isWithin(baseDir, target) {
+  const rel = path.relative(baseDir, target);
+  return (
+    rel !== '' && rel !== '..' && !rel.startsWith(`..${path.sep}`) && !path.isAbsolute(rel)
+  );
+}
+
+/**
+ * Render a frontmatter-only page. `matter.stringify` always appends a blank
+ * body after the closing fence (even for an empty body); the platform's own
+ * generated pages end immediately after the fence with no trailing newline,
+ * so trim it to match.
+ */
+function stringifyFrontmatter(frontmatter) {
+  return matter.stringify('', frontmatter).replace(/\n+$/, '');
+}
+
+function buildPageContent({ oasFilename, operationId, isWebhook }) {
   const frontmatter = {
-    title: summary || operationId,
     api: {
       file: oasFilename,
       operationId,
+      // Marks the page as a webhook (the API calling out to the client)
+      // rather than a path operation (the client calling the API), matching
+      // what the platform stamps on a page generated from `webhooks`.
+      ...(isWebhook ? { webhook: true } : {}),
     },
+    // Mirror the platform's OAS-upload behavior: a newly added endpoint is
+    // always written `hidden: false`, even when its tag and siblings are
+    // `hidden: true`. The backend does not infer this from a missing field, so
+    // it must be written explicitly.
+    //
+    // @todo Honor the `x-internal` OpenAPI extension for page visibility, to
+    // match gitto#2095 (RM-4616 / CX-3303): resolve `hidden` from operation-level
+    // `x-internal`, falling back to root-level, else false; and hide a tag's
+    // index page when all of its operations are `x-internal: true`. Deferred to
+    // keep oas:sync create-only — the resync-side rules (re-applying x-internal
+    // to existing pages, parent hide-ratchet) would require mutating existing
+    // pages, which this command intentionally never does.
+    hidden: false,
   };
 
-  if (description) {
-    frontmatter.excerpt = description;
+  return stringifyFrontmatter(frontmatter);
+}
+
+/**
+ * Build a category landing page (mirrors what the ReadMe platform generates on
+ * OAS upload): `title` is the tag name for a tagged group, or the raw path for
+ * an untagged path-derived group (see `operationGroup`); `excerpt`, when given,
+ * is the tag's description from the spec's top-level `tags` array.
+ */
+function buildTagIndexContent(title, description) {
+  const frontmatter = { title };
+  if (description) frontmatter.excerpt = description;
+  // As with operation pages, upload always stamps hidden: false on new pages.
+  frontmatter.hidden = false;
+
+  return stringifyFrontmatter(frontmatter);
+}
+
+/**
+ * The category-folder grouping for an operation. A tagged operation groups
+ * under its own tag, as before. An untagged operation groups under a folder
+ * derived from its path, with the raw path as the category page's title — one
+ * folder per unique path, not a single shared bucket. This mirrors the
+ * platform's own OAS-upload output: untagged operations are never lumped into
+ * one "Other" folder.
+ */
+function operationGroup(op) {
+  if (op.tag) return { folder: safeSegment(op.tag, 'Other').toLowerCase(), title: op.tag };
+  const folder = safeSegment(op.path.replace(/[/{}]/g, ''), 'operation').toLowerCase();
+  return { folder, title: op.path };
+}
+
+/**
+ * Collect every slug already used across the entire reference/ tree, as a
+ * lowercase-slug -> owner-count map. Reference page slugs share one flat
+ * namespace (docs/ is a separate namespace and is not consulted), so a
+ * generated operation slug must be unique against all of them. A page's slug
+ * is its filename without `.md`; a category page's slug (a folder containing
+ * `index.md`) is the folder name.
+ *
+ * A count, not a Set, because two existing pages or folders can already share
+ * a slug (hand-authored content, or content that predates this uniqueness
+ * logic) — a Set would collapse them to one entry, and releasing one owner
+ * (see `releaseSlug`) would incorrectly free the slug while the other owner
+ * still holds it.
+ */
+function collectReferenceSlugs(refDir) {
+  const counts = new Map();
+
+  function walk(dir) {
+    for (const entry of fs.readdirSync(dir)) {
+      const full = path.join(dir, entry);
+      let stat;
+      try {
+        stat = fs.statSync(full);
+      } catch {
+        continue;
+      }
+      if (stat.isDirectory()) {
+        walk(full);
+      } else if (entry.endsWith('.md')) {
+        // A folder's index.md contributes the folder name as a slug; any other
+        // page contributes its own filename.
+        const slug = entry === 'index.md' ? path.basename(dir) : path.basename(entry, '.md');
+        takeSlug(counts, slug);
+      }
+    }
   }
 
-  return matter.stringify('', frontmatter);
+  walk(refDir);
+  return counts;
+}
+
+function isSlugTaken(takenSlugs, slug) {
+  return (takenSlugs.get(slug.toLowerCase()) || 0) > 0;
+}
+
+/** Record one more owner of `slug`. */
+function takeSlug(takenSlugs, slug) {
+  const key = slug.toLowerCase();
+  takenSlugs.set(key, (takenSlugs.get(key) || 0) + 1);
+}
+
+/** Record one fewer owner of `slug`; only fully frees it once every owner is gone. */
+function releaseSlug(takenSlugs, slug) {
+  const key = slug.toLowerCase();
+  const remaining = (takenSlugs.get(key) || 0) - 1;
+  if (remaining > 0) takenSlugs.set(key, remaining);
+  else takenSlugs.delete(key);
+}
+
+/**
+ * Reserve a unique reference slug. `index` is never usable by an operation (it's
+ * reserved for the tag category page), and any slug already present in the
+ * reference namespace gets a numeric suffix (`-1`, `-2`, ...) until it's free.
+ * The chosen slug gains an owner in `takenSlugs` so later operations see it.
+ */
+function reserveSlug(takenSlugs, base) {
+  let chosen = base;
+  if (base === 'index' || isSlugTaken(takenSlugs, base)) {
+    let n = 1;
+    while (isSlugTaken(takenSlugs, `${base}-${n}`)) n += 1;
+    chosen = `${base}-${n}`;
+  }
+  takeSlug(takenSlugs, chosen);
+  return chosen;
 }
 
 /**
  * Run the sync for a single OAS file. Returns changes for that file.
+ *
+ * `takenSlugs` is the reference-wide set of slugs already in use; it is read and
+ * mutated so slugs stay unique across every spec processed in one sync run.
  */
-function syncOneOas(refDir, oasFilename, spec) {
+function syncOneOas(refDir, oasFilename, spec, takenSlugs) {
   const specOps = extractOperations(spec);
-  const infoTitle = spec.info?.title || path.basename(oasFilename, path.extname(oasFilename));
-  const skipUpdates = isReadMeConfig(spec);
+  const infoTitle = safeSegment(
+    spec.info?.title || path.basename(oasFilename, path.extname(oasFilename)),
+    'api',
+  );
 
   const existingPages = collectExistingPages(refDir).filter(
     (p) => p.data.api.file === oasFilename,
@@ -198,10 +416,21 @@ function syncOneOas(refDir, oasFilename, spec) {
 
   const pagesByOpId = new Map();
   for (const page of existingPages) {
-    pagesByOpId.set(page.data.api.operationId, page);
+    pagesByOpId.set(
+      operationKey({ operationId: page.data.api.operationId, isWebhook: !!page.data.api.webhook }),
+      page,
+    );
   }
 
-  const changes = { added: [], deleted: [], updated: [] };
+  const changes = { added: [], deleted: [], skipped: [] };
+
+  // Tag descriptions from the spec's top-level `tags` array, used for the
+  // per-tag category landing page (index.md).
+  const tagDescriptions = new Map(
+    (Array.isArray(spec.tags) ? spec.tags : [])
+      .filter((t) => t && t.name)
+      .map((t) => [t.name, t.description || null]),
+  );
 
   // Deletes: pages referencing operations that no longer exist.
   for (const [opId, page] of pagesByOpId) {
@@ -209,68 +438,100 @@ function syncOneOas(refDir, oasFilename, spec) {
       fs.unlinkSync(page.filePath);
 
       const pageDir = path.dirname(page.filePath);
-      const slug = path.basename(page.filePath, '.md');
-      removeFromOrder(path.join(pageDir, '_order.yaml'), slug);
+      // A legacy operation page can be literally named index.md (predating
+      // the "index is reserved for the category page" convention). Two
+      // different things need two different values here: pageSlug is what a
+      // pre-refactor tool would have actually written into pageDir's own
+      // _order.yaml ("index", the filename) — that's what removeFromOrder
+      // must remove. referenceSlug is what the reference-wide slug map
+      // reserved for it (its folder name, like any index.md — see
+      // collectReferenceSlugs) — that's what releaseSlug must free.
+      const isIndexPage = path.basename(page.filePath) === 'index.md';
+      const pageSlug = path.basename(page.filePath, '.md');
+      const referenceSlug = isIndexPage ? path.basename(pageDir) : pageSlug;
+      removeFromOrder(path.join(pageDir, '_order.yaml'), pageSlug);
+      releaseSlug(takenSlugs, referenceSlug);
 
       changes.deleted.push(page.relativePath);
     }
   }
 
-  // Adds + Updates.
-  for (const [opId, op] of specOps) {
-    const existing = pagesByOpId.get(opId);
-    const tag = op.tag || 'Other';
+  // Ensure every group (a tag, or a path-derived bucket for untagged
+  // operations) present in the spec has its category landing page (index.md)
+  // and is ordered — independent of whether its operation pages are new. Doing
+  // this as its own pass (rather than only when creating a new op page) backfills
+  // category pages for references first synced by a CLI version that didn't
+  // generate them, and recreates one that was deleted.
+  const groupsByFolder = new Map();
+  for (const op of specOps.values()) {
+    const { folder, title } = operationGroup(op);
+    if (!groupsByFolder.has(folder)) {
+      groupsByFolder.set(folder, { title, description: op.tag ? tagDescriptions.get(op.tag) : null });
+    }
+  }
 
-    if (!existing) {
-      const pageDir = path.join(refDir, infoTitle, tag);
+  // Order groups the way the platform does: a tag keeps the position it's
+  // declared in the spec's own top-level `tags` array, not the order its
+  // operations happen to appear in `paths`. A group with no declared position
+  // (an untagged path-derived group, or a tag used by an operation but never
+  // listed in `tags`) keeps its natural encounter order, appended after every
+  // declared tag.
+  const declaredOrder = (Array.isArray(spec.tags) ? spec.tags : [])
+    .filter((t) => t && t.name)
+    .map((t) => safeSegment(t.name, 'Other').toLowerCase());
+  const orderedFolders = [
+    ...declaredOrder.filter((folder) => groupsByFolder.has(folder)),
+    ...[...groupsByFolder.keys()].filter((folder) => !declaredOrder.includes(folder)),
+  ];
+
+  for (const folder of orderedFolders) {
+    const { title, description } = groupsByFolder.get(folder);
+    const pageDir = path.join(refDir, infoTitle, folder);
+    if (!isWithin(refDir, pageDir)) continue;
+
+    const indexPath = path.join(pageDir, 'index.md');
+    if (!fs.existsSync(indexPath)) {
+      // Never overwrite an existing index.md — it may be a hand-written category.
       fs.mkdirSync(pageDir, { recursive: true });
+      fs.writeFileSync(indexPath, buildTagIndexContent(title, description));
+      changes.added.push(path.relative(refDir, indexPath));
+      // The category page's slug is the folder name; reserve it so no operation
+      // takes it. Only when just-created — an existing index.md was already
+      // counted by collectReferenceSlugs's initial disk walk.
+      takeSlug(takenSlugs, folder);
+    }
+    addToOrder(path.join(refDir, infoTitle, '_order.yaml'), folder);
+    addToOrder(path.join(refDir, '_order.yaml'), infoTitle);
+  }
 
-      const pagePath = path.join(pageDir, `${opId}.md`);
-      const content = buildPageContent({
-        oasFilename,
-        operationId: opId,
-        summary: op.summary,
-        description: op.description,
-      });
-      fs.writeFileSync(pagePath, content);
-
-      addToOrder(path.join(pageDir, '_order.yaml'), opId);
-      addToOrder(path.join(refDir, infoTitle, '_order.yaml'), tag);
-
-      changes.added.push(path.relative(refDir, pagePath));
-    } else if (!skipUpdates) {
-      const expectedTitle = op.summary || opId;
-      const expectedExcerpt = op.description || null;
-      const currentTitle = existing.data.title;
-      const currentExcerpt = existing.data.excerpt || null;
-
-      const titleChanged = currentTitle !== expectedTitle;
-      const excerptChanged = currentExcerpt !== expectedExcerpt;
-
-      if (titleChanged || excerptChanged) {
-        const updated = { ...existing.data };
-        const updateDetails = [];
-
-        if (titleChanged) {
-          updated.title = expectedTitle;
-          updateDetails.push('title');
-        }
-        if (excerptChanged) {
-          if (expectedExcerpt) {
-            updated.excerpt = expectedExcerpt;
-          } else {
-            delete updated.excerpt;
-          }
-          updateDetails.push('excerpt');
-        }
+  // Adds: operation pages with no page yet. Title/excerpt are owned by the OAS
+  // spec at render time, so generated pages carry only the api reference. Slugs
+  // are lowercased to match the platform's OAS-upload output.
+  for (const [key, op] of specOps) {
+    if (pagesByOpId.has(key)) continue;
+
+    const { folder } = operationGroup(op);
+    const pageDir = path.join(refDir, infoTitle, folder);
+    // Reference slugs share one flat namespace, so uniquify against every slug
+    // already in reference/ — a collision (or the reserved `index` slug) gets a
+    // numeric suffix rather than being skipped.
+    const slug = reserveSlug(takenSlugs, safeSegment(op.operationId, 'operation').toLowerCase());
+    const pagePath = path.join(pageDir, `${slug}.md`);
+
+    // Guard against a spec-crafted name escaping reference/, or a stale slug set
+    // vs. disk. reserveSlug already prevents slug collisions.
+    if (!isWithin(refDir, pagePath) || fs.existsSync(pagePath)) {
+      changes.skipped.push({ path: path.relative(refDir, pagePath), operationId: op.operationId });
+      continue;
+    }
+    fs.mkdirSync(pageDir, { recursive: true });
 
-        const body = matter(existing.content).content;
-        const newContent = matter.stringify(body, updated);
-        fs.writeFileSync(existing.filePath, newContent);
+    const content = buildPageContent({ oasFilename, operationId: op.operationId, isWebhook: op.isWebhook });
+    fs.writeFileSync(pagePath, content);
 
-        changes.updated.push(`${existing.relativePath} (${updateDetails.join(', ')})`);
-      }
-    }
+    addToOrder(path.join(pageDir, '_order.yaml'), slug);
+
+    changes.added.push(path.relative(refDir, pagePath));
   }
 
   return changes;
@@ -285,7 +546,7 @@ function syncOneOas(refDir, oasFilename, spec) {
  *
  * @param {string | { cwd?: string }} input  Repo root path, or `{ cwd }` object.
  * @returns {null | Array<{ filename: string, spec: object, opCount: number,
- *   changes: { added: string[], deleted: string[], updated: string[] } }>}
+ *   changes: { added: string[], deleted: string[] } }>}
  *   Returns null if there's no reference/ dir or no specs.
  */
 export function syncOas(input) {
@@ -296,11 +557,14 @@ export function syncOas(input) {
   const oasFiles = findOasFiles(refDir);
   if (oasFiles.length === 0) return null;
 
+  // Reference slugs share one flat namespace across every spec, so build the set
+  // of in-use slugs once and let each spec read/extend it.
+  const takenSlugs = collectReferenceSlugs(refDir);
   const allChanges = [];
 
   for (const { filename, spec } of oasFiles) {
     const ops = extractOperations(spec);
-    const changes = syncOneOas(refDir, filename, spec);
+    const changes = syncOneOas(refDir, filename, spec, takenSlugs);
     allChanges.push({ filename, spec, opCount: ops.size, changes });
   }
 
@@ -325,11 +589,12 @@ export async function run(_options, _cmd, ctx) {
 
   let totalAdded = 0;
   let totalDeleted = 0;
-  let totalUpdated = 0;
+  let totalSkipped = 0;
 
   for (const { filename, spec, opCount, changes } of results) {
     const title = spec.info?.title || filename;
-    const hasChanges = changes.added.length + changes.deleted.length + changes.updated.length > 0;
+    const hasChanges =
+      changes.added.length + changes.deleted.length + changes.skipped.length > 0;
 
     const dot = hasChanges ? styles.warn('●') : styles.success('●');
     console.log();
@@ -345,20 +610,25 @@ export async function run(_options, _cmd, ctx) {
     for (const file of changes.deleted) {
       console.log(`    ${styles.err('−')} Deleted ${file}`);
     }
-    for (const file of changes.updated) {
-      console.log(`    ${styles.warn('~')} Updated ${file}`);
+    for (const { path: file, operationId } of changes.skipped) {
+      console.log(
+        `    ${styles.warn('!')} Skipped ${file} for "${operationId}" (destination already exists)`,
+      );
     }
 
     totalAdded += changes.added.length;
     totalDeleted += changes.deleted.length;
-    totalUpdated += changes.updated.length;
+    totalSkipped += changes.skipped.length;
   }
 
   console.log();
-  const total = totalAdded + totalDeleted + totalUpdated;
-  if (total === 0) {
+  const total = totalAdded + totalDeleted;
+  const skippedNote = totalSkipped > 0 ? `, ${totalSkipped} skipped` : '';
+  if (total === 0 && totalSkipped === 0) {
     styles.ok('Reference pages are already in sync.');
+  } else if (total === 0) {
+    styles.warning(`No pages synced; ${totalSkipped} skipped (destination already exists).`);
   } else {
-    styles.ok(`Synced: ${totalAdded} added, ${totalDeleted} deleted, ${totalUpdated} updated.`);
+    styles.ok(`Synced: ${totalAdded} added, ${totalDeleted} deleted${skippedNote}.`);
   }
 }