@readme/cli 0.0.27 → 0.0.29

package/src/utils/llms.js

@@ -0,0 +1,224 @@
+const H1_RE = /^#\s+(.+)$/
+const H2_RE = /^##\s+(.+)$/
+const H3_RE = /^###\s+(.+)$/
+
+// llms.txt might have a literal " (section)" marker to every
+// grouping heading (e.g. `## getting-started (section)`). It's internal
+// metadata, not part of the category name — strip it before it reaches the sidebar.
+const SECTION_MARKER_RE = /\s*\(section\)\s*$/i
+
+// Max items per section before we consider it "oversized" for usability purposes.
+// Mirrors the cap in usableSections() in import.js.
+const MAX_SECTION_ITEMS = 200
+// Any line whose meaningful content is a markdown link. Accepts:
+//   - Standard list rows:           `- [text](url)`, `* [text](url) — desc`
+//   - Bare link lines:              `[text](url)`
+//   - Breadcrumb-prefixed rows used by Fern/Mintlify-style indices:
+//       `ElevenAgents [Agent WebSockets](url)`
+//       `API Reference > Agents > Branches [List branches](url)`
+//   - Trailing-parenthetical-breadcrumb rows (AssemblyAI/Fern variant) — the
+//     breadcrumb sits between the url and the description and is matched-and-discarded:
+//       `[text](url) (Breadcrumb > path): description`
+// Captures: [prefix, text, url, description] (prefix and description may be empty).
+const LINK_LINE_RE = /^\s*(?:[-*+]\s+)?([^\[\n]*?)\s*\[([^\]]+)\] ?\(([^)\s]+)\)(?:\s+\([^)]*\))?(?:\s*[:—–-]\s*(.+?))?\s*$/
+const BLOCKQUOTE_RE = /^\s*>/
+const FENCE_RE = /^\s*(?:```|~~~)/
+
+// A llms.txt is considered usable as long as it has at least one link row.
+// Beyond that we enforce a structural ratio (share of "spec-shaped" lines —
+// H1/H2/blockquote/link row/blank — over total lines), but only once the
+// file has enough links for the ratio to be statistically meaningful. A
+// 3-link file with one prose sentence would compute a misleading ratio; we
+// can't tell signal from noise at that size, so we trust it and let the
+// BFS-merge dedupe handle any junk that slips through.
+const RATIO_CHECK_MIN_LINKS = 10
+const MIN_CONFORMING_RATIO = 0.7
+
+export function analyzeLlmsTxt(body, llmsUrl, options = {}) {
+  const parsed = parseLlmsTxt(body, llmsUrl, options)
+  const lineStats = classifyLines(body)
+  const linkItems = parsed.sections.reduce((sum, s) => sum + s.items.length, 0)
+  const reason = getSkipReason(body, linkItems, lineStats)
+
+  return {
+    parsed,
+    usable: !reason,
+    reason,
+    stats: { ...lineStats, linkItems },
+  }
+}
+
+export function parseLlmsTxt(body, llmsUrl, options = {}) {
+  const h2Result = parseSections(body, llmsUrl, H2_RE, options)
+
+  // If every H2 section is oversized (or there are none), and the file has H3
+  // headings, re-parse treating H3 (###) as section boundaries.
+  // For strange LLMS.txt formatting
+  const allOversized = h2Result.sections.length === 0 ||
+    h2Result.sections.every((s) => s.items.length > MAX_SECTION_ITEMS)
+  if (allOversized && /^###\s/m.test(body)) {
+    const h3Result = parseSections(body, llmsUrl, H3_RE, options)
+    if (h3Result.sections.length > 1) {
+      return { ...h3Result, h3Fallback: true }
+    }
+  }
+
+  return h2Result
+}
+
+function parseSections(body, llmsUrl, sectionRe, options = {}) {
+  const lines = body.split(/\r?\n/)
+  let title = null
+  const sections = []
+  let current = null
+
+  for (const line of lines) {
+    const h1 = line.match(H1_RE)
+    if (h1 && !title) {
+      title = h1[1].trim()
+      continue
+    }
+
+    const sectionMatch = line.match(sectionRe)
+    if (sectionMatch) {
+      current = { title: sectionMatch[1].trim().replace(SECTION_MARKER_RE, '').trim(), items: [] }
+      sections.push(current)
+      continue
+    }
+
+    const item = parseListLink(line, llmsUrl, options)
+    if (!item) continue
+
+    if (!current) {
+      current = { title: 'Resources', items: [] }
+      sections.push(current)
+    }
+    current.items.push(item)
+  }
+
+  return { title, sections }
+}
+
+function getSkipReason(body, linkItems, lineStats) {
+  if (/^---\r?\n/.test(body)) return 'starts with YAML frontmatter'
+  if (linkItems === 0) return 'no link items'
+  if (linkItems >= RATIO_CHECK_MIN_LINKS && lineStats.ratio < MIN_CONFORMING_RATIO) {
+    return `conforming-line ratio ${lineStats.ratio.toFixed(2)} below ${MIN_CONFORMING_RATIO} (${lineStats.conforming}/${lineStats.total} lines)`
+  }
+  return null
+}
+
+/**
+ * Walk the body line-by-line and bucket each line as conforming or not.
+ *
+ *   conforming     blank, H1, H2, blockquote, standard link-row
+ *   non-conforming everything else, plus every line inside (and the delimiters
+ *                  of) a fenced code block
+ */
+function classifyLines(body) {
+  const lines = body.split(/\r?\n/)
+  let inFence = false
+  let conforming = 0
+  let nonConforming = 0
+
+  for (const line of lines) {
+    if (FENCE_RE.test(line)) {
+      nonConforming++
+      inFence = !inFence
+      continue
+    }
+    if (inFence) {
+      nonConforming++
+      continue
+    }
+    if (line.trim() === '') {
+      conforming++
+      continue
+    }
+    if (H1_RE.test(line) || H2_RE.test(line) || LINK_LINE_RE.test(line) || BLOCKQUOTE_RE.test(line)) {
+      conforming++
+      continue
+    }
+    nonConforming++
+  }
+
+  const total = conforming + nonConforming
+  const ratio = total === 0 ? 0 : conforming / total
+  return { conforming, nonConforming, total, ratio }
+}
+
+function parseListLink(line, llmsUrl, options = {}) {
+  const match = line.match(LINK_LINE_RE)
+  if (!match) return null
+
+  const [, prefix, text, rawUrl, trailingDesc] = match
+  const url = normalizeUrl(rawUrl, llmsUrl, options)
+  if (!url) return null
+
+  // Prefer an explicit trailing description (`[text](url) — desc`); fall back
+  // to the breadcrumb prefix (`API Reference > Agents [text](url)`) when no
+  // explicit description is present. Either gives downstream extra context.
+  const description = trailingDesc?.trim() || prefix?.trim() || null
+
+  return {
+    text: text.trim(),
+    url,
+    description: description || null,
+  }
+}
+
+function normalizeUrl(rawUrl, llmsUrl, options = {}) {
+  const trimmed = String(rawUrl || '').trim().replace(/[.,;]+$/, '')
+  if (!trimmed || /^#/.test(trimmed) || /^(mailto|javascript):/i.test(trimmed)) return null
+
+  try {
+    const url = llmsUrl ? new URL(resolveLlmsRelativeUrl(trimmed, llmsUrl, options), llmsUrl) : new URL(trimmed)
+    if (url.protocol !== 'http:' && url.protocol !== 'https:') return null
+    return url.toString()
+  } catch {
+    return null
+  }
+}
+
+function resolveLlmsRelativeUrl(trimmed, llmsUrl, options) {
+  if (options.rootRelativeResolution !== 'llms-dir' || !isRootRelativeUrl(trimmed)) return trimmed
+
+  const nestedPrefix = getLlmsNestedPrefix(llmsUrl)
+  if (!nestedPrefix) return trimmed
+
+  const normallyResolved = new URL(trimmed, llmsUrl)
+  if (!shouldResolveRootRelativeFromLlmsDir(trimmed, normallyResolved.pathname, nestedPrefix)) return trimmed
+
+  return `.${trimmed}`
+}
+
+function shouldResolveRootRelativeFromLlmsDir(rawUrl, pathname, nestedPrefix) {
+  if (isPathWithinPrefix(pathname, nestedPrefix)) return false
+  if (/\/llms\.txt$/i.test(pathname)) return false
+
+  const lastSegment = pathname.split('/').pop() || ''
+  const ext = lastSegment.match(/\.([a-z0-9]+)$/i)?.[1]?.toLowerCase()
+  if (ext && !['htm', 'html', 'md', 'mdx'].includes(ext)) return false
+
+  return isRootRelativeUrl(rawUrl)
+}
+
+function getLlmsNestedPrefix(llmsUrl) {
+  if (!llmsUrl) return null
+
+  try {
+    const pathname = new URL(llmsUrl).pathname
+    const dir = pathname.replace(/llms\.txt$/i, '').replace(/\/+$/, '/')
+    return dir === '/' ? null : dir
+  } catch {
+    return null
+  }
+}
+
+function isRootRelativeUrl(rawUrl) {
+  return /^\/(?!\/)/.test(rawUrl)
+}
+
+function isPathWithinPrefix(pathname, prefix) {
+  return pathname === prefix.replace(/\/$/, '') || pathname.startsWith(prefix)
+}

package/src/utils/llms.test.js

@@ -0,0 +1,257 @@
+import { test } from 'node:test'
+import assert from 'node:assert/strict'
+import { parseLlmsTxt, analyzeLlmsTxt } from './llms.js'
+
+const BASE = 'https://docs.example.com/llms.txt'
+
+// ---------------------------------------------------------------------------
+// parseLlmsTxt — root-relative resolution
+// ---------------------------------------------------------------------------
+
+test('parseLlmsTxt: root-relative links resolve from the origin by default', () => {
+  const parsed = parseLlmsTxt(
+    `# Example\n\n## Guides\n- [Overview](/overview.md)\n- [Install](/docs/install.md)\n`,
+    'https://example.com/docs/llms.txt',
+  )
+
+  const urls = parsed.sections[0].items.map((item) => item.url)
+  assert.deepEqual(urls, [
+    'https://example.com/overview.md',
+    'https://example.com/docs/install.md',
+  ])
+})
+
+test('parseLlmsTxt: explicit llms-dir policy resolves nested root-relative page links from the llms.txt directory', () => {
+  const parsed = parseLlmsTxt(
+    `# Example\n\n## Guides\n- [Overview](/overview.md)\n- [API](/api/reference.md)\n- [Absolute](https://example.com/reference/openapi.md)\n`,
+    'https://example.com/docs/llms.txt',
+    { rootRelativeResolution: 'llms-dir' },
+  )
+
+  const urls = parsed.sections[0].items.map((item) => item.url)
+  assert.deepEqual(urls, [
+    'https://example.com/docs/overview.md',
+    'https://example.com/docs/api/reference.md',
+    'https://example.com/reference/openapi.md',
+  ])
+})
+
+test('parseLlmsTxt: llms-dir policy preserves links already scoped to the llms.txt directory', () => {
+  const parsed = parseLlmsTxt(
+    `# Example\n\n## Guides\n- [Already scoped](/docs/reference.md)\n- [Deep already scoped](/docs/api/reference.md?version=1#auth)\n`,
+    'https://example.com/docs/llms.txt',
+    { rootRelativeResolution: 'llms-dir' },
+  )
+
+  const urls = parsed.sections[0].items.map((item) => item.url)
+  assert.deepEqual(urls, [
+    'https://example.com/docs/reference.md',
+    'https://example.com/docs/api/reference.md?version=1#auth',
+  ])
+})
+
+test('parseLlmsTxt: llms-dir policy does not rewrite explicit nested llms.txt links', () => {
+  const parsed = parseLlmsTxt(
+    `# Example\n\n## Guides\n- [API llms](/api/llms.txt)\n- [Docs API llms](/docs/api/llms.txt)\n`,
+    'https://example.com/docs/llms.txt',
+    { rootRelativeResolution: 'llms-dir' },
+  )
+
+  const urls = parsed.sections[0].items.map((item) => item.url)
+  assert.deepEqual(urls, [
+    'https://example.com/api/llms.txt',
+    'https://example.com/docs/api/llms.txt',
+  ])
+})
+
+test('parseLlmsTxt: llms-dir policy preserves root-scoped non-page files', () => {
+  const parsed = parseLlmsTxt(
+    `# Example\n\n## Guides\n- [Overview](/overview.md)\n- [OpenAPI](/reference/openapi.yaml)\n- [Image](/assets/logo.png)\n`,
+    'https://example.com/docs/llms.txt',
+    { rootRelativeResolution: 'llms-dir' },
+  )
+
+  const urls = parsed.sections[0].items.map((item) => item.url)
+  assert.deepEqual(urls, [
+    'https://example.com/docs/overview.md',
+    'https://example.com/reference/openapi.yaml',
+    'https://example.com/assets/logo.png',
+  ])
+})
+
+test('parseLlmsTxt: llms-dir policy handles query and hash page links', () => {
+  const parsed = parseLlmsTxt(
+    `# Example\n\n## Guides\n- [Overview](/overview.md?tab=js#install)\n- [No extension](/get-started?tab=js#install)\n- [Hash only](#local)\n`,
+    'https://example.com/docs/llms.txt',
+    { rootRelativeResolution: 'llms-dir' },
+  )
+
+  const urls = parsed.sections[0].items.map((item) => item.url)
+  assert.deepEqual(urls, [
+    'https://example.com/docs/overview.md?tab=js#install',
+    'https://example.com/docs/get-started?tab=js#install',
+  ])
+})
+
+test('parseLlmsTxt: malformed and unsupported links are ignored', () => {
+  const parsed = parseLlmsTxt(
+    `# Example\n\n## Guides\n- [Mail](mailto:test@example.com)\n- [JS](javascript:alert(1))\n- [OK](/ok.md)\n`,
+    'https://example.com/docs/llms.txt',
+    { rootRelativeResolution: 'llms-dir' },
+  )
+
+  const urls = parsed.sections[0].items.map((item) => item.url)
+  assert.deepEqual(urls, ['https://example.com/docs/ok.md'])
+})
+
+// ---------------------------------------------------------------------------
+// parseLlmsTxt — standard H2 sections
+// ---------------------------------------------------------------------------
+
+test('parseLlmsTxt: parses H1 title and H2 sections', () => {
+  const body = [
+    '# My Docs',
+    '',
+    '## Getting Started',
+    '- [Overview](https://docs.example.com/overview)',
+    '- [Quickstart](https://docs.example.com/quickstart)',
+    '',
+    '## Reference',
+    '- [API](https://docs.example.com/api)',
+  ].join('\n')
+
+  const result = parseLlmsTxt(body, BASE)
+  assert.equal(result.title, 'My Docs')
+  assert.equal(result.sections.length, 2)
+  assert.equal(result.sections[0].title, 'Getting Started')
+  assert.equal(result.sections[0].items.length, 2)
+  assert.equal(result.sections[1].title, 'Reference')
+  assert.equal(result.h3Fallback, undefined)
+})
+
+test('parseLlmsTxt: items with no H2 fall into implicit Resources section', () => {
+  const body = [
+    '# My Docs',
+    '- [Overview](https://docs.example.com/overview)',
+  ].join('\n')
+
+  const result = parseLlmsTxt(body, BASE)
+  assert.equal(result.sections.length, 1)
+  assert.equal(result.sections[0].title, 'Resources')
+})
+
+// ---------------------------------------------------------------------------
+// parseLlmsTxt — H3 fallback
+// ---------------------------------------------------------------------------
+
+// Generates a Couchbase-style body: one H2 bucket, N H3 sub-sections each
+// with `itemsPerSection` pages. With itemsPerSection >= 68, the single H2
+// section exceeds MAX_SECTION_ITEMS (200) and triggers the H3 fallback.
+function makeCouchbaseBody(sections, itemsPerSection) {
+  const lines = ['# Couchbase', '', '> Official Couchbase docs.', '', '## Docs', '']
+  for (const title of sections) {
+    lines.push(`### ${title}`)
+    for (let i = 0; i < itemsPerSection; i++) {
+      const slug = title.toLowerCase().replace(/[^a-z0-9]+/g, '-')
+      lines.push(`- [Page ${i}](https://docs.couchbase.com/${slug}/page-${i})`)
+    }
+    lines.push('')
+  }
+  return lines.join('\n')
+}
+
+const CB_SECTIONS = ['.NET Analytics SDK (1.0)', '.NET Entity Framework (1.0)', '.NET SDK (3.9)']
+
+test('parseLlmsTxt: falls back to H3 when single H2 section is oversized', () => {
+  const manyItems = Array.from({ length: 201 }, (_, i) =>
+    `- [Page ${i}](https://docs.example.com/page-${i})`
+  ).join('\n')
+
+  const body = [
+    '# Big Site',
+    '',
+    '## Docs',
+    manyItems,
+    '',
+    '### Section A',
+    '- [A1](https://docs.example.com/a1)',
+    '- [A2](https://docs.example.com/a2)',
+    '',
+    '### Section B',
+    '- [B1](https://docs.example.com/b1)',
+  ].join('\n')
+
+  const result = parseLlmsTxt(body, BASE)
+  assert.equal(result.h3Fallback, true)
+  assert.ok(result.sections.length > 1, `expected >1 section, got ${result.sections.length}`)
+
+  const titles = result.sections.map((s) => s.title)
+  assert.ok(titles.includes('Section A'))
+  assert.ok(titles.includes('Section B'))
+})
+
+test('parseLlmsTxt: Couchbase-style H3 structure produces one section per SDK', () => {
+  const body = makeCouchbaseBody(CB_SECTIONS, 70)
+  const result = parseLlmsTxt(body, 'https://docs.couchbase.com/llms.txt')
+
+  assert.equal(result.h3Fallback, true)
+  assert.equal(result.title, 'Couchbase')
+  assert.equal(result.sections.length, 3)
+  assert.equal(result.sections[0].title, '.NET Analytics SDK (1.0)')
+  assert.equal(result.sections[1].title, '.NET Entity Framework (1.0)')
+  assert.equal(result.sections[2].title, '.NET SDK (3.9)')
+  assert.equal(result.sections[0].items.length, 70)
+})
+
+test('parseLlmsTxt: does NOT use H3 fallback when H2 sections are usable', () => {
+  const body = [
+    '# Site',
+    '',
+    '## Section A',
+    '- [A1](https://docs.example.com/a1)',
+    '- [A2](https://docs.example.com/a2)',
+    '',
+    '## Section B',
+    '- [B1](https://docs.example.com/b1)',
+    '',
+    '### Sub B',
+    '- [B2](https://docs.example.com/b2)',
+  ].join('\n')
+
+  const result = parseLlmsTxt(body, BASE)
+  assert.equal(result.h3Fallback, undefined)
+  assert.equal(result.sections.length, 2)
+  assert.equal(result.sections[0].title, 'Section A')
+  assert.equal(result.sections[1].title, 'Section B')
+})
+
+test('parseLlmsTxt: does NOT use H3 fallback when no H3 headings exist', () => {
+  const manyItems = Array.from({ length: 201 }, (_, i) =>
+    `- [Page ${i}](https://docs.example.com/page-${i})`
+  ).join('\n')
+
+  const body = ['# Big Site', '', '## Docs', manyItems].join('\n')
+
+  const result = parseLlmsTxt(body, BASE)
+  assert.equal(result.h3Fallback, undefined)
+  assert.equal(result.sections.length, 1)
+  assert.equal(result.sections[0].title, 'Docs')
+})
+
+// ---------------------------------------------------------------------------
+// analyzeLlmsTxt
+// ---------------------------------------------------------------------------
+
+test('analyzeLlmsTxt: marks usable when it has link items', () => {
+  const body = '## Section\n- [Page](https://docs.example.com/page)'
+  const result = analyzeLlmsTxt(body, BASE)
+  assert.equal(result.usable, true)
+  assert.equal(result.reason, null)
+})
+
+test('analyzeLlmsTxt: marks not usable when no link items', () => {
+  const body = '# Just a title\n\nSome prose with no links.'
+  const result = analyzeLlmsTxt(body, BASE)
+  assert.equal(result.usable, false)
+  assert.ok(result.reason)
+})

package/src/utils/url-segs.js

@@ -0,0 +1,124 @@
+/**
+ * URL segment parsing utilities shared across import.js pipeline stages.
+ * Centralised here so the single-extension-strip bug only lives in one place.
+ */
+
+// Extensions that are implementation artifacts, not meaningful path segments.
+const DOC_FILE_EXTENSIONS = ['md', 'mdx', 'html', 'htm', 'txt']
+const DOC_EXT_RE = new RegExp(`(\\.(${DOC_FILE_EXTENSIONS.join('|')}))+$`, 'i')
+
+/**
+ * Strip all chained doc-file extensions (.html.md, .html, .md, .mdx, .htm, .txt)
+ * from a single URL path segment. Handles the double-extension pattern used
+ * by some SSGs (e.g. `.html.md`).
+ */
+export function stripSegmentExtensions(s) {
+  return s.replace(DOC_EXT_RE, '')
+}
+
+/**
+ * Parse a URL's pathname into segments suitable for trie nesting. Strips file
+ * extensions (including double-extensions like `.html.md`) and drops a trailing
+ * `index` segment so `/foo`, `/foo/`, `/foo/index.html`, `/foo/index.html.md`,
+ * and `/foo/index.md` all collapse to the same logical page.
+ * Returns null if the URL can't be parsed.
+ */
+export function urlTrieSegs(url) {
+  try {
+    const segs = new URL(url).pathname
+      .split('/')
+      .filter(Boolean)
+      .map((s) => stripSegmentExtensions(s))
+      .filter(Boolean)
+    if (segs.length > 0 && segs[segs.length - 1].toLowerCase() === 'index') segs.pop()
+    return segs
+  } catch {
+    return null
+  }
+}
+
+/**
+ * Extract URL path segments for slug planning. Strips file extensions
+ * (including double-extensions like `.html.md`) and leading numeric prefixes
+ * (e.g. `01-intro` → `intro`). Also drops a trailing `index` segment so
+ * `/foo/index.html.md` and `/foo/` both collapse to the same slug base.
+ */
+export function extractUrlPathSegments(url) {
+  if (!url) return []
+  try {
+    const segs = new URL(url).pathname
+      .split('/')
+      .filter(Boolean)
+      .map((s) => stripSegmentExtensions(s).replace(/^\d+[-_.]/, ''))
+      .filter(Boolean)
+    if (segs.length > 0 && segs[segs.length - 1].toLowerCase() === 'index') segs.pop()
+    return segs
+  } catch {
+    return []
+  }
+}
+
+/**
+ * Normalise a URL to a canonical path key for deduplication. Strips trailing
+ * slash, strips all chained doc-file extensions from the path tail, drops a
+ * trailing `index`, and lowercases — so `/foo/bar.html.md`, `/foo/bar.html`,
+ * `/foo/bar/index.html`, and `/foo/bar` all resolve to the same key `/foo/bar`.
+ */
+export function normalizePath(url) {
+  try {
+    const u = new URL(url)
+    let p = u.pathname.replace(/\/$/, '').toLowerCase()
+    p = p.replace(DOC_EXT_RE, '')
+    p = p.replace(/\/index$/i, '')
+    return p || '/'
+  } catch {
+    return String(url).toLowerCase()
+  }
+}
+
+/**
+ * Dedupe key for llms.txt page rows only.
+ *
+ * Invariants:
+ * - includes origin so same-path docs on different hosts stay distinct
+ * - collapses doc URL spellings for the same page tail (.md/.mdx/.html/.htm, /index, trailing slash)
+ * - preserves query params because some docs routers encode page state there
+ * - ignores fragments because they identify anchors within a page, not separate import pages
+ */
+export function llmsPageDedupeKey(url) {
+  try {
+    const u = new URL(url)
+    let p = u.pathname.replace(/\/$/, '').toLowerCase()
+    p = p.replace(DOC_EXT_RE, '')
+    p = p.replace(/\/index$/i, '')
+
+    const params = [...u.searchParams.entries()].sort(([ak, av], [bk, bv]) => (ak === bk ? av.localeCompare(bv) : ak.localeCompare(bk)))
+    const query = params.length > 0 ? `?${params.map(([k, v]) => `${encodeURIComponent(k)}=${encodeURIComponent(v)}`).join('&')}` : ''
+
+    return `${u.origin.toLowerCase()}${p || '/'}${query}`
+  } catch {
+    return String(url).toLowerCase()
+  }
+}
+
+export function compareCanonicalUrlPreference(a, b) {
+  const aRank = canonicalUrlPreferenceRank(a)
+  const bRank = canonicalUrlPreferenceRank(b)
+  if (aRank !== bRank) return aRank - bRank
+  return String(a || '').length - String(b || '').length
+}
+
+function canonicalUrlPreferenceRank(url) {
+  try {
+    const pathname = new URL(url).pathname
+    const lower = pathname.toLowerCase()
+    if (/\.md$/.test(lower)) return 0
+    if (/\.txt$/.test(lower)) return 1
+    if (!/\/$/.test(pathname) && !/\/index\.html?$/.test(lower) && !/\.[^/]+$/.test(lower)) return 2
+    if (/\/$/.test(pathname)) return 3
+    if (/\/index\.html?$/.test(lower)) return 4
+    return 5
+  } catch {
+    return 5
+  }
+}