@readme/cli 0.0.27 → 0.0.30

package/src/utils/url-segs.test.js

@@ -0,0 +1,239 @@
+import { test } from 'node:test'
+import assert from 'node:assert/strict'
+import { stripSegmentExtensions, urlTrieSegs, extractUrlPathSegments, normalizePath, compareCanonicalUrlPreference, llmsPageDedupeKey } from './url-segs.js'
+
+// ---------------------------------------------------------------------------
+// stripSegmentExtensions
+// ---------------------------------------------------------------------------
+
+test('stripSegmentExtensions: strips single .md', () => {
+  assert.equal(stripSegmentExtensions('index.md'), 'index')
+})
+
+test('stripSegmentExtensions: strips single .html', () => {
+  assert.equal(stripSegmentExtensions('index.html'), 'index')
+})
+
+test('stripSegmentExtensions: strips single .htm', () => {
+  assert.equal(stripSegmentExtensions('index.htm'), 'index')
+})
+
+test('stripSegmentExtensions: strips double .html.md (plaid pattern)', () => {
+  assert.equal(stripSegmentExtensions('index.html.md'), 'index')
+})
+
+test('stripSegmentExtensions: strips .mdx', () => {
+  assert.equal(stripSegmentExtensions('page.mdx'), 'page')
+})
+
+test('stripSegmentExtensions: strips .txt', () => {
+  assert.equal(stripSegmentExtensions('terminal.txt'), 'terminal')
+})
+
+test('stripSegmentExtensions: leaves bare txt segment untouched (no dot prefix)', () => {
+  assert.equal(stripSegmentExtensions('txt'), 'txt')
+})
+
+test('stripSegmentExtensions: leaves plain segments untouched', () => {
+  assert.equal(stripSegmentExtensions('quickstart'), 'quickstart')
+})
+
+test('stripSegmentExtensions: leaves kebab segments untouched', () => {
+  assert.equal(stripSegmentExtensions('getting-started'), 'getting-started')
+})
+
+// ---------------------------------------------------------------------------
+// urlTrieSegs — the double-extension plaid.com pattern
+// ---------------------------------------------------------------------------
+
+test('urlTrieSegs: .html.md index collapses to parent path (plaid pattern)', () => {
+  assert.deepEqual(
+    urlTrieSegs('https://plaid.com/docs/account/index.html.md'),
+    ['docs', 'account'],
+  )
+})
+
+test('urlTrieSegs: .html index collapses to parent path', () => {
+  assert.deepEqual(
+    urlTrieSegs('https://plaid.com/docs/account/index.html'),
+    ['docs', 'account'],
+  )
+})
+
+test('urlTrieSegs: trailing slash collapses correctly', () => {
+  assert.deepEqual(
+    urlTrieSegs('https://plaid.com/docs/account/'),
+    ['docs', 'account'],
+  )
+})
+
+test('urlTrieSegs: non-index .html.md page keeps its segment', () => {
+  assert.deepEqual(
+    urlTrieSegs('https://plaid.com/docs/account/billing/index.html.md'),
+    ['docs', 'account', 'billing'],
+  )
+})
+
+test('urlTrieSegs: orval.dev docs/index.html.md collapses correctly', () => {
+  assert.deepEqual(
+    urlTrieSegs('https://orval.dev/docs/index.html.md'),
+    ['docs'],
+  )
+})
+
+test('urlTrieSegs: plain URL without extension passes through', () => {
+  assert.deepEqual(
+    urlTrieSegs('https://example.com/docs/quickstart'),
+    ['docs', 'quickstart'],
+  )
+})
+
+test('urlTrieSegs: returns null on unparseable input', () => {
+  assert.equal(urlTrieSegs('not-a-url'), null)
+})
+
+test('urlTrieSegs: .txt segment strips extension (warp.dev pattern)', () => {
+  assert.deepEqual(
+    urlTrieSegs('https://docs.warp.dev/_llms-txt/terminal.txt'),
+    ['_llms-txt', 'terminal'],
+  )
+})
+
+// ---------------------------------------------------------------------------
+// extractUrlPathSegments — slug source
+// ---------------------------------------------------------------------------
+
+test('extractUrlPathSegments: .html.md index → parent segments only (plaid pattern)', () => {
+  assert.deepEqual(
+    extractUrlPathSegments('https://plaid.com/docs/account/index.html.md'),
+    ['docs', 'account'],
+  )
+})
+
+test('extractUrlPathSegments: slug from deepened path', () => {
+  assert.deepEqual(
+    extractUrlPathSegments('https://plaid.com/docs/account/billing/index.html.md'),
+    ['docs', 'account', 'billing'],
+  )
+})
+
+test('extractUrlPathSegments: strips leading numeric prefix', () => {
+  assert.deepEqual(
+    extractUrlPathSegments('https://example.com/docs/01-intro.md'),
+    ['docs', 'intro'],
+  )
+})
+
+test('extractUrlPathSegments: plain .md page', () => {
+  assert.deepEqual(
+    extractUrlPathSegments('https://example.com/docs/quickstart.md'),
+    ['docs', 'quickstart'],
+  )
+})
+
+test('extractUrlPathSegments: returns [] for empty/null input', () => {
+  assert.deepEqual(extractUrlPathSegments(''), [])
+  assert.deepEqual(extractUrlPathSegments(null), [])
+})
+
+// ---------------------------------------------------------------------------
+// normalizePath — deduplication keys
+// ---------------------------------------------------------------------------
+
+test('normalizePath: .html.md and .html and bare path all produce same key', () => {
+  const a = normalizePath('https://plaid.com/docs/account/index.html.md')
+  const b = normalizePath('https://plaid.com/docs/account/index.html')
+  const c = normalizePath('https://plaid.com/docs/account/index')
+  assert.equal(a, b)
+  assert.equal(b, c)
+})
+
+test('normalizePath: trailing slash and bare path produce same key', () => {
+  const a = normalizePath('https://example.com/docs/foo/')
+  const b = normalizePath('https://example.com/docs/foo')
+  assert.equal(a, b)
+})
+
+test('normalizePath: index variants and bare path produce same key', () => {
+  const bare = normalizePath('https://example.com/docs/get-started')
+  assert.equal(normalizePath('https://example.com/docs/get-started/'), bare)
+  assert.equal(normalizePath('https://example.com/docs/get-started/index.html'), bare)
+  assert.equal(normalizePath('https://example.com/docs/get-started/index.htm'), bare)
+})
+
+test('normalizePath: lowercases result', () => {
+  assert.equal(normalizePath('https://example.com/Docs/QuickStart'), '/docs/quickstart')
+})
+
+test('normalizePath: falls back for unparseable input', () => {
+  assert.equal(normalizePath('not-a-url'), 'not-a-url')
+})
+
+test('normalizePath: strips .txt extension (warp.dev pattern)', () => {
+  assert.equal(
+    normalizePath('https://docs.warp.dev/_llms-txt/terminal.txt'),
+    '/_llms-txt/terminal',
+  )
+})
+
+// ---------------------------------------------------------------------------
+// llmsPageDedupeKey — llms.txt page-row deduplication
+// ---------------------------------------------------------------------------
+
+test('llmsPageDedupeKey: includes origin so same paths on different hosts stay distinct', () => {
+  assert.notEqual(
+    llmsPageDedupeKey('https://docs.example.com/docs/get-started.md'),
+    llmsPageDedupeKey('https://example.com/docs/get-started.md'),
+  )
+})
+
+test('llmsPageDedupeKey: preserves query params by default', () => {
+  assert.notEqual(
+    llmsPageDedupeKey('https://example.com/docs/search?product=api'),
+    llmsPageDedupeKey('https://example.com/docs/search?product=sdk'),
+  )
+})
+
+test('llmsPageDedupeKey: normalizes query param order', () => {
+  assert.equal(
+    llmsPageDedupeKey('https://example.com/docs/search?b=2&a=1'),
+    llmsPageDedupeKey('https://example.com/docs/search?a=1&b=2'),
+  )
+})
+
+test('llmsPageDedupeKey: ignores fragments but preserves router query state', () => {
+  assert.equal(
+    llmsPageDedupeKey('https://example.com/docs/get-started.md?tab=js#install'),
+    llmsPageDedupeKey('https://example.com/docs/get-started?tab=js#auth'),
+  )
+})
+
+test('llmsPageDedupeKey: collapses doc URL spellings for the same page', () => {
+  const bare = llmsPageDedupeKey('https://example.com/docs/get-started')
+  assert.equal(llmsPageDedupeKey('https://example.com/docs/get-started.md'), bare)
+  assert.equal(llmsPageDedupeKey('https://example.com/docs/get-started.txt'), bare)
+  assert.equal(llmsPageDedupeKey('https://example.com/docs/get-started/index.html'), bare)
+  assert.equal(llmsPageDedupeKey('https://example.com/docs/get-started/'), bare)
+})
+
+// ---------------------------------------------------------------------------
+// compareCanonicalUrlPreference — retained URL selection
+// ---------------------------------------------------------------------------
+
+test('compareCanonicalUrlPreference: prefers md over txt, bare, trailing slash, and index', () => {
+  const urls = [
+    'https://example.com/docs/get-started/index.html',
+    'https://example.com/docs/get-started/',
+    'https://example.com/docs/get-started',
+    'https://example.com/docs/get-started.txt',
+    'https://example.com/docs/get-started.md',
+  ]
+
+  assert.deepEqual([...urls].sort(compareCanonicalUrlPreference), [
+    'https://example.com/docs/get-started.md',
+    'https://example.com/docs/get-started.txt',
+    'https://example.com/docs/get-started',
+    'https://example.com/docs/get-started/',
+    'https://example.com/docs/get-started/index.html',
+  ])
+})

package/src/validators/components.js

@@ -135,19 +135,22 @@ export function validate({ content, relativePath }) {
 
   const results = [];
 
-  // .md snippet files: filename must be PascalCase (it becomes the component tag).
+  // .md snippet files: the filename becomes a component tag <Name />, so it must
+  // be a valid JS identifier starting with an uppercase letter. Underscores and
+  // digits are fine (<Helm_v4_error /> is valid); hyphens and a lowercase initial
+  // are not.
   if (isMd) {
     const slug = filename.replace(/\.md$/, "");
-    const pascalCase = slug
-      .split(/[-_]/)
-      .map((part) => part.charAt(0).toUpperCase() + part.slice(1))
-      .join("");
-    if (slug !== pascalCase) {
+    if (!/^[A-Z][A-Za-z0-9_$]*$/.test(slug)) {
+      const suggestion = slug
+        .split(/[-_]/)
+        .map((part) => part.charAt(0).toUpperCase() + part.slice(1))
+        .join("");
       results.push({
         file: relativePath,
         rule: name,
         severity: "warning",
-        message: `Bad filename: should be PascalCase — rename to "${pascalCase}.md" so it can be used as <${pascalCase} />`,
+        message: `Bad filename: "${slug}" can't be used as a component tag — rename to "${suggestion}.md" so it can be used as <${suggestion} />`,
       });
     }
     return results.length > 0 ? results : null;
@@ -220,6 +223,7 @@ export function validateAll(files, gitRoot) {
         results.push({
           file: relPath,
           rule: name,
+          severity: 'warning',
           message: `Unknown component: <${comp}> is not a built-in or custom block`,
         });
       }

package/src/validators/duplicates.js

@@ -1,4 +1,5 @@
 import path from 'node:path';
+import { slugForEntry } from '../utils/slug-for-entry.js';
 
 export const name = 'duplicates';
 
@@ -8,27 +9,26 @@ const CHECKED_DIRS = ['docs', 'reference'];
 export function validateAll(files) {
   const results = [];
 
-  // Group files by slug.
+  // Group files by "<topDir>:<slug>" so slugs only collide within the same section.
   const slugMap = new Map();
   for (const relPath of files) {
     const topDir = relPath.split('/')[0];
     if (!CHECKED_DIRS.includes(topDir)) continue;
 
-    const filename = path.basename(relPath);
-    if (filename === 'index.md' || filename === 'index.mdx') continue;
-
-    const slug = filename.replace(/\.(md|mdx)$/, '');
-    if (!slugMap.has(slug)) slugMap.set(slug, []);
-    slugMap.get(slug).push(relPath);
+    const slug = slugForEntry(relPath);
+    const key = `${topDir}:${slug}`;
+    if (!slugMap.has(key)) slugMap.set(key, []);
+    slugMap.get(key).push(relPath);
   }
 
-  for (const [slug, paths] of slugMap) {
+  for (const [key, paths] of slugMap) {
     if (paths.length < 2) continue;
 
     // TODO: figure out why ReadMeConfig causes duplicate slugs and handle properly.
     // For now, skip any duplicate set that involves a ReadMeConfig path.
     if (paths.some((p) => p.includes('ReadMeConfig/'))) continue;
 
+    const slug = key.slice(key.indexOf(':') + 1);
     const others = paths.slice(1);
     for (const relPath of others) {
       const otherLocations = paths.filter((p) => p !== relPath).map((p) => path.dirname(p)).join(', ');

package/src/validators/frontmatter.js

@@ -113,6 +113,17 @@ export function validate({ content, filePath, relativePath, fix }) {
     const valid = validateFn(data);
     if (!valid) {
       for (const err of validateFn.errors) {
+        // Reference pages are OAS-backed: title/excerpt come from the spec, so a
+        // missing title is not an error for pages that declare an api.file.
+        if (
+          dir === 'reference' &&
+          data?.api?.file &&
+          err.keyword === 'required' &&
+          err.params?.missingProperty === 'title'
+        ) {
+          continue;
+        }
+
         if (err.keyword === 'not' && err.schema?.properties) {
           const target = err.instancePath ? getNestedValue(data, err.instancePath) : data;
           if (!target || !Object.keys(err.schema.properties).some((k) => k in target)) continue;

package/src/validators/links.js

@@ -1,6 +1,7 @@
 import fs from 'node:fs';
 import path from 'node:path';
 import matter from 'gray-matter';
+import { slugForEntry } from '../utils/slug-for-entry.js';
 
 export const name = 'links';
 
@@ -19,14 +20,7 @@ function collectSlugs(gitRoot) {
     const entries = fs.readdirSync(dirPath, { recursive: true });
     for (const entry of entries) {
       if (!/\.(md|mdx)$/.test(entry)) continue;
-      const filename = path.basename(entry);
-      if (filename === 'index.md' || filename === 'index.mdx') {
-        // index.md represents the parent directory, e.g. plans-and-pricing/index.md -> slug "plans-and-pricing"
-        const parentDir = path.basename(path.dirname(entry));
-        if (parentDir && parentDir !== '.') slugs.add(parentDir);
-      } else {
-        slugs.add(filename.replace(/\.(md|mdx)$/, ''));
-      }
+      slugs.add(slugForEntry(entry));
     }
   }
 

package/src/validators/oas-reference.js

@@ -1,7 +1,7 @@
 import fs from 'node:fs';
 import path from 'node:path';
 import matter from 'gray-matter';
-import { findOasFiles, extractOperations, collectExistingPages, syncOas } from '../commands/oas-sync.js';
+import { findOasFiles, extractOperations, collectExistingPages, syncOas, operationKey } from '../commands/oas-sync.js';
 
 export const name = 'oas-reference';
 
@@ -34,6 +34,7 @@ export function validateAll(files, gitRoot, { fix } = {}) {
 
     const oasFilename = data.api.file;
     const operationId = data.api.operationId;
+    const isWebhook = !!data.api.webhook;
     const oas = oasMap.get(oasFilename);
 
     // Check: OAS file doesn't exist.
@@ -50,7 +51,7 @@ export function validateAll(files, gitRoot, { fix } = {}) {
     if (!operationId) continue;
 
     // Check: operationId doesn't exist in the spec.
-    if (!oas.ops.has(operationId)) {
+    if (!oas.ops.has(operationKey({ operationId, isWebhook }))) {
       results.push({
         file: relPath,
         rule: name,
@@ -60,52 +61,26 @@ export function validateAll(files, gitRoot, { fix } = {}) {
       continue;
     }
 
-    // Skip title/excerpt sync checks for ReadMeConfig (internal ReadMe pages).
-    // Check both the spec title and the page's directory path.
-    const isReadMeConfig = oas.spec.info?.title === 'ReadMeConfig'
-      || relPath.startsWith('reference/ReadMeConfig/');
-    if (isReadMeConfig) continue;
-
-    // Check: title or excerpt out of sync.
-    const op = oas.ops.get(operationId);
-    const expectedTitle = op.summary || operationId;
-    const expectedExcerpt = op.description || null;
-
-    if (data.title !== expectedTitle) {
-      results.push({
-        file: relPath,
-        rule: name,
-        severity: 'warning',
-        message: `Out of sync: title is "${data.title}" but spec summary is "${expectedTitle}"`,
-        fixable: true,
-      });
-    }
-
-    const currentExcerpt = data.excerpt || null;
-    if (currentExcerpt !== expectedExcerpt) {
-      results.push({
-        file: relPath,
-        rule: name,
-        severity: 'warning',
-        message: `Out of sync: excerpt does not match spec description for "${operationId}"`,
-        fixable: true,
-      });
-    }
+    // Title/excerpt are owned by the OAS spec at render time — no sync check here.
   }
 
   // Check for missing pages: operations in the spec with no corresponding page.
   const existingPages = collectExistingPages(refDir);
   for (const [oasFilename, { ops }] of oasMap) {
     const pagesForOas = existingPages.filter((p) => p.data.api.file === oasFilename);
-    const coveredOps = new Set(pagesForOas.map((p) => p.data.api.operationId));
-
-    for (const [opId] of ops) {
-      if (!coveredOps.has(opId)) {
+    const coveredOps = new Set(
+      pagesForOas.map((p) =>
+        operationKey({ operationId: p.data.api.operationId, isWebhook: !!p.data.api.webhook }),
+      ),
+    );
+
+    for (const op of ops.values()) {
+      if (!coveredOps.has(operationKey(op))) {
         results.push({
           file: `reference/${oasFilename}`,
           rule: name,
           severity: 'warning',
-          message: `Missing page: no reference page found for operation "${opId}"`,
+          message: `Missing page: no reference page found for operation "${op.operationId}"`,
           fixable: true,
         });
       }