@tikoci/rosetta 0.10.0 → 0.11.0-alpha.87

package/src/restraml.ts

@@ -2,14 +2,16 @@
  * restraml.ts — Shared helpers for fetching data from tikoci/restraml.
  *
  * restraml publishes inspect.json files to GitHub Pages. Version discovery
- * uses the GitHub API (1 call), but all inspect.json fetches go through
- * GitHub Pages (no rate limit).
+ * uses the authenticated GitHub API when a token is available (1 call), but
+ * all inspect.json fetches go through GitHub Pages (no rate limit).
  */
 
+import { fetchGitHub, githubApiHeaders } from "./github.ts";
+
 /** GitHub Pages base URL — inspect.json files served here (no rate limit) */
 export const RESTRAML_PAGES_URL = "https://tikoci.github.io/restraml";
 
-/** GitHub API endpoint for version directory listing (60 req/hr unauthenticated) */
+/** GitHub API endpoint for version directory listing. */
 const RESTRAML_API_CONTENTS_URL = "https://api.github.com/repos/tikoci/restraml/contents/docs";
 
 export function isHttpUrl(value: string): boolean {
@@ -30,8 +32,8 @@ interface GitHubContentEntry {
  * Uses 1 GitHub API call to list the docs/ directory, returns version strings.
  */
 export async function discoverRemoteVersions(): Promise<string[]> {
-  const response = await fetch(RESTRAML_API_CONTENTS_URL, {
-    headers: { Accept: "application/vnd.github.v3+json" },
+  const response = await fetchGitHub(RESTRAML_API_CONTENTS_URL, {
+    headers: githubApiHeaders(),
   });
 
   if (!response.ok) {

package/src/rosetta-id.test.ts

@@ -0,0 +1,95 @@
+import { describe, expect, test } from "bun:test";
+import { checkCollisions, deriveRosettaId, parseSitemapLocs, rosettaIdToUrl } from "./rosetta-id.ts";
+
+describe("deriveRosettaId", () => {
+  test("strips scheme, host, and leading/trailing slashes", () => {
+    expect(deriveRosettaId("https://manual.mikrotik.com/docs/ip/address/")).toBe("docs/ip/address");
+  });
+
+  test("lowercases mixed-case path segments", () => {
+    expect(deriveRosettaId("https://manual.mikrotik.com/docs/CLI-Reference/Routing/BGP")).toBe(
+      "docs/cli-reference/routing/bgp",
+    );
+  });
+
+  test("passes through a bare path (no scheme/host)", () => {
+    expect(deriveRosettaId("/docs/ip/dhcp-server")).toBe("docs/ip/dhcp-server");
+  });
+
+  test("strips an optional /docs/next/ version prefix", () => {
+    expect(deriveRosettaId("https://manual.mikrotik.com/docs/next/ip/address")).toBe("docs/ip/address");
+  });
+
+  test("strips an optional /docs/<semver>/ version prefix", () => {
+    expect(deriveRosettaId("https://manual.mikrotik.com/docs/7.22/ip/address")).toBe("docs/ip/address");
+  });
+
+  test("does not strip a real path segment that merely looks numeric-ish but isn't a version", () => {
+    // "next" and semver-like segments only get stripped directly after /docs/ — elsewhere they're just path.
+    expect(deriveRosettaId("https://manual.mikrotik.com/docs/ip/next")).toBe("docs/ip/next");
+  });
+
+  test("strips a .md/.mdx extension so a page and a link resolving to it agree", () => {
+    expect(deriveRosettaId("https://manual.mikrotik.com/docs/network-management/dhcp")).toBe(
+      deriveRosettaId("https://manual.mikrotik.com/docs/network-management/dhcp.md"),
+    );
+    expect(deriveRosettaId("https://manual.mikrotik.com/docs/network-management/dhcp.md#dhcp-server")).toBe(
+      "docs/network-management/dhcp",
+    );
+  });
+
+  test("handles non-/docs sections unchanged", () => {
+    expect(deriveRosettaId("https://manual.mikrotik.com/hardware/rb5009")).toBe("hardware/rb5009");
+    expect(deriveRosettaId("https://manual.mikrotik.com/changelog/changelog-2026-05-25")).toBe(
+      "changelog/changelog-2026-05-25",
+    );
+  });
+});
+
+describe("rosettaIdToUrl", () => {
+  test("round-trips with deriveRosettaId for a canonical URL", () => {
+    const url = "https://manual.mikrotik.com/docs/network-management/dhcp";
+    expect(rosettaIdToUrl(deriveRosettaId(url))).toBe(url);
+  });
+});
+
+describe("checkCollisions", () => {
+  test("reports zero collisions for distinct paths", () => {
+    const report = checkCollisions([
+      "https://manual.mikrotik.com/docs/ip/address",
+      "https://manual.mikrotik.com/docs/ip/route",
+    ]);
+    expect(report.collisions.size).toBe(0);
+    expect(report.uniqueIds).toBe(2);
+  });
+
+  test("detects a real collision (case-only difference)", () => {
+    const report = checkCollisions([
+      "https://manual.mikrotik.com/docs/IP/Address",
+      "https://manual.mikrotik.com/docs/ip/address",
+    ]);
+    expect(report.collisions.size).toBe(1);
+    expect(report.collisions.get("docs/ip/address")?.length).toBe(2);
+  });
+
+  test("detects a collision introduced by version-prefix stripping", () => {
+    const report = checkCollisions([
+      "https://manual.mikrotik.com/docs/ip/address",
+      "https://manual.mikrotik.com/docs/next/ip/address",
+    ]);
+    expect(report.collisions.size).toBe(1);
+  });
+});
+
+describe("parseSitemapLocs", () => {
+  test("extracts <loc> entries from sitemap XML", () => {
+    const xml = `<?xml version="1.0"?><urlset>
+      <url><loc>https://manual.mikrotik.com/docs/ip/address</loc></url>
+      <url><loc>https://manual.mikrotik.com/docs/ip/route</loc></url>
+    </urlset>`;
+    expect(parseSitemapLocs(xml)).toEqual([
+      "https://manual.mikrotik.com/docs/ip/address",
+      "https://manual.mikrotik.com/docs/ip/route",
+    ]);
+  });
+});

package/src/rosetta-id.ts

@@ -0,0 +1,151 @@
+/**
+ * rosetta-id.ts — URL-path-derived stable identifiers for Docusaurus-sourced content.
+ *
+ * Promoted from the T-0034 spike (src/spike-docusaurus-rosetta-id.ts, now removed) after
+ * its H7 Option-2 shape (separate `rosetta_id TEXT UNIQUE` column, existing INTEGER PKs
+ * untouched) was validated against a live 20-page prototype and confirmed in
+ * briefings/B-0012-docusaurus-manual-migration.md, "H7 — Identity / rosetta-id design".
+ *
+ * Used by extract-docusaurus.ts to mint `pages.rosetta_id` and to resolve relative
+ * Markdown links inside property descriptions to a canonical id/URL.
+ */
+
+const SITEMAP_URL = "https://manual.mikrotik.com/sitemap.xml";
+
+/**
+ * URL path -> rosetta-id. Pure function so it's independently testable.
+ *
+ * Strips scheme+host, leading/trailing slashes, and lowercases. Also strips an
+ * optional Docusaurus version-prefix segment (/docs/next/... or /docs/<semver>/...)
+ * if present, even though manual.mikrotik.com does not version today — B-0012 H7
+ * flags this as cheap to build in now, expensive to retrofit later.
+ */
+export function deriveRosettaId(urlOrPath: string): string {
+  let path: string;
+  try {
+    path = new URL(urlOrPath).pathname;
+  } catch {
+    path = urlOrPath;
+  }
+
+  path = path.replace(/^\/+|\/+$/g, "").toLowerCase();
+  // Markdown-source links (and the docusaurus-plugin-llms .md endpoint) point at the
+  // same page's .md/.mdx sibling URL — strip it so a page's own canonical id and an
+  // internal link resolving to it collapse to the same rosetta-id (found empirically
+  // during T-0034: without this, ./dhcp.md#anchor resolved to
+  // "docs/network-management/dhcp.md" while the page itself was "docs/.../dhcp").
+  path = path.replace(/\.mdx?$/, "");
+
+  const segments = path.split("/");
+  if (segments[0] === "docs" && segments.length > 1) {
+    const second = segments[1];
+    const isVersionPrefix = second === "next" || /^v?\d+(\.\d+)*(-[a-z0-9.]+)?$/.test(second);
+    if (isVersionPrefix) {
+      segments.splice(1, 1);
+    }
+  }
+
+  return segments.join("/");
+}
+
+/** Build a live manual.mikrotik.com URL from a rosetta-id (inverse of deriveRosettaId, minus version-prefix). */
+export function rosettaIdToUrl(rosettaId: string): string {
+  return `https://manual.mikrotik.com/${rosettaId}`;
+}
+
+export interface CollisionReport {
+  total: number;
+  uniqueIds: number;
+  collisions: Map<string, string[]>;
+  maxLength: number;
+  unexpectedChars: string[];
+}
+
+/** Derive rosetta-ids for a list of URLs and report any collisions. */
+export function checkCollisions(urls: string[]): CollisionReport {
+  const byId = new Map<string, string[]>();
+  let maxLength = 0;
+  const unexpectedChars = new Set<string>();
+
+  for (const url of urls) {
+    const id = deriveRosettaId(url);
+    maxLength = Math.max(maxLength, id.length);
+    for (const ch of id) {
+      if (!/[a-z0-9/_-]/.test(ch)) unexpectedChars.add(ch);
+    }
+    const existing = byId.get(id);
+    if (existing) existing.push(url);
+    else byId.set(id, [url]);
+  }
+
+  const collisions = new Map<string, string[]>();
+  for (const [id, sourceUrls] of byId) {
+    if (sourceUrls.length > 1) collisions.set(id, sourceUrls);
+  }
+
+  return {
+    total: urls.length,
+    uniqueIds: byId.size,
+    collisions,
+    maxLength,
+    unexpectedChars: [...unexpectedChars],
+  };
+}
+
+/** Decode the five predefined XML entities. `&amp;` is decoded last so already-decoded `&` isn't re-processed. */
+function decodeXmlEntities(s: string): string {
+  return s
+    .replace(/&lt;/g, "<")
+    .replace(/&gt;/g, ">")
+    .replace(/&quot;/g, '"')
+    .replace(/&apos;/g, "'")
+    .replace(/&amp;/g, "&");
+}
+
+/** Extract <loc> entries from a sitemap.xml document, decoding XML entities in each URL. */
+export function parseSitemapLocs(xml: string): string[] {
+  return [...xml.matchAll(/<loc>([^<]+)<\/loc>/g)].map((m) => decodeXmlEntities(m[1]));
+}
+
+/** Fetch and parse the live manual.mikrotik.com sitemap, or read a local file if `source` is given. */
+export async function loadSitemapUrls(source?: string): Promise<string[]> {
+  let xml: string;
+  if (source) {
+    xml = await Bun.file(source).text();
+  } else {
+    // 10s timeout matches the per-page/llms.txt fetches in extract-docusaurus.ts so a
+    // hung connection can't stall the whole extractor indefinitely.
+    const res = await fetch(SITEMAP_URL, { signal: AbortSignal.timeout(10_000) });
+    // Fail loud on a non-2xx: an HTML error page has no <loc> matches, which would
+    // otherwise masquerade as "0 pages in scope" and be diagnosed as a filter bug.
+    if (!res.ok) throw new Error(`Failed to fetch sitemap ${SITEMAP_URL}: HTTP ${res.status} ${res.statusText}`);
+    xml = await res.text();
+  }
+  return parseSitemapLocs(xml);
+}
+
+if (import.meta.main) {
+  (async () => {
+    const source = process.argv[2];
+    const urls = await loadSitemapUrls(source);
+    const report = checkCollisions(urls);
+
+    console.log(`Source: ${source ?? SITEMAP_URL}`);
+    console.log(`Total URLs:       ${report.total}`);
+    console.log(`Unique rosetta-ids: ${report.uniqueIds}`);
+    console.log(`Max id length:    ${report.maxLength}`);
+    console.log(`Unexpected chars: ${report.unexpectedChars.length === 0 ? "none" : report.unexpectedChars.join(" ")}`);
+    console.log(`Collisions:       ${report.collisions.size}`);
+
+    if (report.collisions.size > 0) {
+      for (const [id, sourceUrls] of report.collisions) {
+        console.log(`  ${id}:`);
+        for (const u of sourceUrls) console.log(`    ${u}`);
+      }
+      process.exitCode = 1;
+    }
+  })().catch((err) => {
+    console.error(err);
+    process.exitCode = 1;
+  });
+}