@warpgogol/werkstatt-shared 0.2.0

package/src/share/tests/text-normalize.pbt.test.ts

@@ -0,0 +1,143 @@
+/*
+<MODULE_CONTRACT>
+  <purpose>RFC-0347 + ADR-0019: property-based tests for text normalization idempotency and HTML structural invariance.</purpose>
+  <keywords>RFC-0347, ADR-0019, PBT, fast-check, text-normalize, idempotency, tag balance, comment isolation, no content creation</keywords>
+  <responsibilities>
+    <item>Verify normalizeText is idempotent: normalize(normalize(x)) === normalize(x).</item>
+    <item>Verify normalizeHtml is idempotent.</item>
+    <item>Verify normalizeMarkdown is idempotent.</item>
+    <item>ADR-0019: Verify normalizeHtml preserves tag balance — structural tags in input remain balanced in output.</item>
+    <item>ADR-0019: Verify normalizeHtml preserves comment isolation — HTML comments are not created or destroyed.</item>
+    <item>ADR-0019: Verify normalizeHtml does not create HTML tags not present in input.</item>
+  </responsibilities>
+</MODULE_CONTRACT>
+<MODULE_MAP><entry key="pbt-text-normalize">Property-based tests for normalization idempotency and structural invariance.</entry></MODULE_MAP>
+<CHANGE_SUMMARY>
+  <item>RFC-0347: initial PBT illustrative example for text-normalize idempotency.</item>
+  <item>ADR-0019: added tag balance, comment isolation, and no-content-creation properties for normalizeHtml.</item>
+</CHANGE_SUMMARY>
+*/
+
+import { test, expect } from "vitest";
+import fc from "fast-check";
+import {
+  normalizeText,
+  normalizeHtml,
+  normalizeMarkdown,
+  DEFAULT_NORMALIZE_CONFIG,
+} from "../text-normalize.ts";
+
+const textArbitrary = fc
+  .string({ minLength: 0, maxLength: 200 })
+  .filter((s) => !s.includes("\uFFFD"));
+
+test("PBT: normalizeText is idempotent — normalize(normalize(x)) === normalize(x)", () => {
+  fc.assert(
+    fc.property(textArbitrary, (s) => {
+      const once = normalizeText(s, DEFAULT_NORMALIZE_CONFIG);
+      expect(normalizeText(once, DEFAULT_NORMALIZE_CONFIG)).toBe(once);
+    }),
+  );
+});
+
+test("PBT: normalizeHtml is idempotent — normalize(normalize(x)) === normalize(x)", () => {
+  fc.assert(
+    fc.property(textArbitrary, (s) => {
+      const once = normalizeHtml(s, DEFAULT_NORMALIZE_CONFIG);
+      expect(normalizeHtml(once, DEFAULT_NORMALIZE_CONFIG)).toBe(once);
+    }),
+  );
+});
+
+test("PBT: normalizeMarkdown is idempotent — normalize(normalize(x)) === normalize(x)", () => {
+  fc.assert(
+    fc.property(textArbitrary, (s) => {
+      const once = normalizeMarkdown(s, DEFAULT_NORMALIZE_CONFIG);
+      expect(normalizeMarkdown(once, DEFAULT_NORMALIZE_CONFIG)).toBe(once);
+    }),
+  );
+});
+
+// ---------------------------------------------------------------------------
+// ADR-0019: structural invariance properties for normalizeHtml
+// ---------------------------------------------------------------------------
+
+const STRUCTURAL_TAGS = [
+  "div",
+  "span",
+  "p",
+  "main",
+  "section",
+  "article",
+  "header",
+  "footer",
+  "ul",
+  "li",
+] as const;
+
+const htmlTextSegment = fc
+  .string({ minLength: 0, maxLength: 50 })
+  .filter((s) => !s.includes("<") && !s.includes(">") && !s.includes("\uFFFD"));
+
+const htmlComment = fc.tuple(htmlTextSegment).map(([text]) => `<!-- ${text} -->`);
+
+const htmlBalancedTag = fc
+  .tuple(fc.constantFrom(...STRUCTURAL_TAGS), htmlTextSegment)
+  .map(([tag, text]) => `<${tag}>${text}</${tag}>`);
+
+const htmlSegment = fc.oneof(htmlTextSegment, htmlComment, htmlBalancedTag);
+
+const htmlArbitrary = fc
+  .array(htmlSegment, { minLength: 1, maxLength: 8 })
+  .map((segments) => segments.join("\n"));
+
+function countTagOccurrences(html: string, tag: string): number {
+  const openRe = new RegExp(`<${tag}\\b[^>]*>`, "gi");
+  const closeRe = new RegExp(`</${tag}>`, "gi");
+  return (html.match(openRe) ?? []).length + (html.match(closeRe) ?? []).length;
+}
+
+test("PBT: normalizeHtml preserves tag balance — structural tags in input remain balanced in output", () => {
+  fc.assert(
+    fc.property(htmlArbitrary, (input) => {
+      const output = normalizeHtml(input, DEFAULT_NORMALIZE_CONFIG);
+      for (const tag of STRUCTURAL_TAGS) {
+        const inputCount = countTagOccurrences(input, tag);
+        const outputCount = countTagOccurrences(output, tag);
+        expect(outputCount).toBe(inputCount);
+      }
+    }),
+  );
+});
+
+test("PBT: normalizeHtml preserves comment isolation — comments are not created or destroyed", () => {
+  fc.assert(
+    fc.property(htmlArbitrary, (input) => {
+      const output = normalizeHtml(input, DEFAULT_NORMALIZE_CONFIG);
+      const inputCommentCount = (input.match(/<!--[\s\S]*?-->/g) ?? []).length;
+      const outputCommentCount = (output.match(/<!--[\s\S]*?-->/g) ?? []).length;
+      expect(outputCommentCount).toBe(inputCommentCount);
+    }),
+  );
+});
+
+test("PBT: normalizeHtml does not create HTML tags not present in input", () => {
+  fc.assert(
+    fc.property(htmlArbitrary, (input) => {
+      const output = normalizeHtml(input, DEFAULT_NORMALIZE_CONFIG);
+      const tagRe = /<([a-zA-Z][a-zA-Z0-9]*)\b/g;
+      const inputTagNames = new Set<string>();
+      let m: RegExpExecArray | null;
+      while ((m = tagRe.exec(input)) !== null) {
+        inputTagNames.add(m[1].toLowerCase());
+      }
+      const outputTagNames = new Set<string>();
+      while ((m = tagRe.exec(output)) !== null) {
+        outputTagNames.add(m[1].toLowerCase());
+      }
+      for (const tag of outputTagNames) {
+        expect(inputTagNames.has(tag)).toBe(true);
+      }
+    }),
+  );
+});

package/src/share/tests/text-normalize.test.ts

@@ -0,0 +1,285 @@
+import { test, expect } from "vitest";
+import {
+  DEFAULT_NORMALIZE_CONFIG,
+  SIGNAL_IDS,
+  SIGNAL_REGISTRY,
+  resolveNormalizeConfig,
+  normalizeText,
+  normalizeHtml,
+  normalizeJson,
+  normalizeXml,
+  normalizeMarkdown,
+  detectResidual,
+  createDevNormalizeMiddleware,
+  type NormalizeConfig,
+} from "../text-normalize.ts";
+
+// All special/invisible inputs use \u escapes so the test is canonical and the
+// module's matchers must hit the real codepoints. Expected outputs are ASCII.
+const ALL_ON = DEFAULT_NORMALIZE_CONFIG;
+
+function only(id: (typeof SIGNAL_IDS)[number]): NormalizeConfig {
+  const signals = {} as NormalizeConfig["signals"];
+  for (const s of SIGNAL_IDS) signals[s] = s === id;
+  return { enabled: true, signals };
+}
+
+// --- char signals -----------------------------------------------------------
+
+test("dashes: every special dash becomes a hyphen, spaces preserved", () => {
+  const input = "a—b — c–d‒e―f‑g−h";
+  expect(normalizeText(input, ALL_ON)).toBe("a-b - c-d-e-f-g-h");
+});
+
+test("quotes: curly + guillemets become straight", () => {
+  const input = "“hi” «yo» it’s ‘a’ „de“";
+  expect(normalizeText(input, ALL_ON)).toBe(`"hi" "yo" it's 'a' "de"`);
+});
+
+test("ellipsis: single char becomes three dots", () => {
+  expect(normalizeText("wait…", ALL_ON)).toBe("wait...");
+});
+
+test("spaces: nbsp and exotic spaces become a regular space (no collapsing)", () => {
+  const input = "a b c d e　f g";
+  expect(normalizeText(input, ALL_ON)).toBe("a b c d e f g");
+});
+
+test("zero-width cruft is removed", () => {
+  const input = "a​b‌c⁠de­f؜g";
+  expect(normalizeText(input, ALL_ON)).toBe("abcdefg");
+});
+
+test("ZWJ inside an emoji sequence is preserved; standalone ZWJ is stripped", () => {
+  const family = "\u{1F469}‍\u{1F469}‍\u{1F466}"; // woman+woman+boy
+  expect(normalizeText(family, ALL_ON)).toBe(family);
+  expect(normalizeText("a‍b", ALL_ON)).toBe("ab");
+});
+
+// --- html entities -----------------------------------------------------------
+
+test("typographic entities are decoded then normalized; structural entities survive", () => {
+  const input = "A—B C… & <tag> "q"";
+  expect(normalizeText(input, ALL_ON)).toBe("A-B C... & <tag> "q"");
+});
+
+test("numeric typographic entities decode; structural numeric entities do not", () => {
+  // — = em dash (typographic) ; & = & (structural)
+  expect(normalizeText("x—y & z", ALL_ON)).toBe("x-y & z");
+});
+
+test("entity lens respects the dash toggle (decode keeps em-dash when dashes off)", () => {
+  const cfg: NormalizeConfig = { enabled: true, signals: { ...ALL_ON.signals, dashes: false } };
+  expect(normalizeText("a—b", cfg)).toBe("a—b");
+});
+
+// --- per-signal isolation ----------------------------------------------------
+
+test("disabling spaces keeps nbsp while other signals still fire", () => {
+  const cfg: NormalizeConfig = { enabled: true, signals: { ...ALL_ON.signals, spaces: false } };
+  expect(normalizeText("a b—c", cfg)).toBe("a b-c");
+});
+
+test("master switch off is a no-op", () => {
+  const off: NormalizeConfig = { enabled: false, signals: { ...ALL_ON.signals } };
+  const input = "a—b“c”";
+  expect(normalizeText(input, off)).toBe(input);
+});
+
+// --- html --------------------------------------------------------------------
+
+test("html: text nodes normalized, code/script/pre skipped", () => {
+  const input = "<p>he said “hi”—ok</p><code>a—b</code><pre>x—y</pre>";
+  expect(normalizeHtml(input, ALL_ON)).toBe(`<p>he said "hi"-ok</p><code>a—b</code><pre>x—y</pre>`);
+});
+
+test("html: whitelisted attribute normalized and quote re-escaped", () => {
+  const input = `<img alt="say “hi”" src="/x.png">`;
+  expect(normalizeHtml(input, ALL_ON)).toBe(`<img alt="say &quot;hi&quot;" src="/x.png">`);
+});
+
+test("html: non-whitelisted attribute (href) is left untouched", () => {
+  const input = `<a href="/a—b">t—t</a>`;
+  expect(normalizeHtml(input, ALL_ON)).toBe(`<a href="/a—b">t-t</a>`);
+});
+
+test("html: inline JSON-LD string values normalized and stay valid JSON", () => {
+  const input = '<script type="application/ld+json">{"name":"He said “hi”—ok"}</script>';
+  const out = normalizeHtml(input, ALL_ON);
+  const json = out.replace(/^<script[^>]*>/, "").replace(/<\/script>$/, "");
+  const parsed = JSON.parse(json);
+  expect(parsed.name).toBe(`He said "hi"-ok`);
+});
+
+// --- json --------------------------------------------------------------------
+
+test("json: clean input returns byte-identical (no reformat)", () => {
+  const input = '{"a":"plain","b":1}';
+  expect(normalizeJson(input, ALL_ON)).toBe(input);
+});
+
+test("json: values normalized, keys untouched", () => {
+  const out = normalizeJson('{"a\\u2014key":"v\\u2014v"}', ALL_ON);
+  const parsed = JSON.parse(out);
+  expect(Object.keys(parsed)).toEqual(["a—key"]);
+  expect(parsed["a—key"]).toBe("v-v");
+});
+
+// --- xml ---------------------------------------------------------------------
+
+test("xml: element text normalized, tags preserved", () => {
+  expect(normalizeXml("<title>A—B</title>", ALL_ON)).toBe("<title>A-B</title>");
+});
+
+test("xml: CDATA body normalized as html", () => {
+  const input = "<description><![CDATA[<p>a—b</p>]]></description>";
+  expect(normalizeXml(input, ALL_ON)).toBe("<description><![CDATA[<p>a-b</p>]]></description>");
+});
+
+// --- markdown ----------------------------------------------------------------
+
+test("markdown: prose normalized, fenced + inline code protected", () => {
+  const input = "He—said\n```\nx—y\n```\nand `a—b` end";
+  expect(normalizeMarkdown(input, ALL_ON)).toBe("He-said\n```\nx—y\n```\nand `a—b` end");
+});
+
+// --- CSS < in style block (regression) ---------------------------------------
+
+test("html: CSS media query < in <style> does not trap the skip stack", () => {
+  const input = "<style>@media (width<=768px){.x{color:red}}</style><p>a\u2014b</p>";
+  expect(normalizeHtml(input, ALL_ON)).toBe(
+    "<style>@media (width<=768px){.x{color:red}}</style><p>a-b</p>",
+  );
+});
+
+test("html: text after <style> with CSS < is still normalized", () => {
+  const input =
+    "<style>.a{color:red}@media (width<=768px){.a{color:blue}}</style>" +
+    '<span aria-label="Warpgogol \u2014 Home">link</span>' +
+    "<p>\u0426\u0438\u0444\u0440\u043e\u0432\u0438\u0439 \u0444\u0443\u043d\u0434\u0430\u043c\u0435\u043d\u0442 \u2014 \u043f\u0440\u043e\u0434\u0443\u043a\u0442</p>";
+  const out = normalizeHtml(input, ALL_ON);
+  expect(out).toContain('aria-label="Warpgogol - Home"');
+  expect(out).not.toContain("\u2014");
+});
+
+// --- idempotency -------------------------------------------------------------
+
+test("all transforms are idempotent", () => {
+  const html = `<p>“q”—<a title="t…">l</a></p>`;
+  const once = normalizeHtml(html, ALL_ON);
+  expect(normalizeHtml(once, ALL_ON)).toBe(once);
+  const md = "a—b “c”…";
+  const mdOnce = normalizeMarkdown(md, ALL_ON);
+  expect(normalizeMarkdown(mdOnce, ALL_ON)).toBe(mdOnce);
+});
+
+// --- config resolution -------------------------------------------------------
+
+test("resolveNormalizeConfig: absent block ⇒ all on", () => {
+  expect(resolveNormalizeConfig({})).toEqual(ALL_ON);
+  expect(resolveNormalizeConfig(null)).toEqual(ALL_ON);
+});
+
+test("resolveNormalizeConfig: explicit per-signal off respected, others default on", () => {
+  const cfg = resolveNormalizeConfig({ text: { normalize: { signals: { spaces: false } } } });
+  expect(cfg.enabled).toBe(true);
+  expect(cfg.signals.spaces).toBe(false);
+  expect(cfg.signals.dashes).toBe(true);
+});
+
+test("resolveNormalizeConfig: enabled:false honored", () => {
+  const cfg = resolveNormalizeConfig({ text: { normalize: { enabled: false } } });
+  expect(cfg.enabled).toBe(false);
+});
+
+// --- backstop detection ------------------------------------------------------
+
+test("detectResidual: clean content returns null", () => {
+  expect(detectResidual("<p>clean</p>", "html", ALL_ON)).toBe(null);
+});
+
+test("detectResidual: reports the responsible signal and a line", () => {
+  const finding = detectResidual("line1\nse—cond", "txt", ALL_ON);
+  expect(finding).toBeTruthy();
+  expect(finding!.line).toBe(2);
+  expect(finding!.signals).toEqual(["dashes"]);
+});
+
+// --- registry ----------------------------------------------------------------
+
+test("SIGNAL_REGISTRY covers exactly the six signal ids", () => {
+  expect(SIGNAL_REGISTRY.map((s) => s.id).sort()).toEqual([...SIGNAL_IDS].sort());
+});
+
+test("only() isolates a single signal", () => {
+  expect(normalizeText("a—b“c”", only("dashes"))).toBe(`a-b“c”`);
+});
+
+// Codepoint-built inputs — independent of how source glyphs are stored, this proves
+// the matchers hit the exact Unicode points named in the RFC (no false pass).
+test("codepoints: spaces + zero-width matched by exact U+ value", () => {
+  const cp = (...codes: number[]) => String.fromCodePoint(...codes);
+  // U+00A0 U+2009 U+2003 U+202F U+3000 U+1680 U+205F all → single space
+  const spaces = `a${cp(0x00a0)}b${cp(0x2009)}c${cp(0x2003)}d${cp(0x202f)}e${cp(0x3000)}f${cp(0x1680)}g${cp(0x205f)}h`;
+  expect(normalizeText(spaces, ALL_ON)).toBe("a b c d e f g h");
+  // U+200B U+200C U+2060 U+FEFF U+00AD U+061C U+2061 → removed
+  const zw = `a${cp(0x200b)}b${cp(0x200c)}c${cp(0x2060)}d${cp(0xfeff)}e${cp(0x00ad)}f${cp(0x061c)}g${cp(0x2061)}h`;
+  expect(normalizeText(zw, ALL_ON)).toBe("abcdefgh");
+  // U+2014 em dash, U+201C/U+201D quotes, U+2026 ellipsis
+  expect(normalizeText(`x${cp(0x2014)}y ${cp(0x201c)}z${cp(0x201d)}${cp(0x2026)}`, ALL_ON)).toBe(
+    `x-y "z"...`,
+  );
+});
+
+// --- dev middleware (RFC-0569) -----------------------------------------------
+
+function makeResponse(body: string, contentType: string): Response {
+  return new Response(body, { headers: { "content-type": contentType } });
+}
+
+test("dev middleware: enabled config normalizes HTML response", async () => {
+  const mw = createDevNormalizeMiddleware(ALL_ON);
+  const html = "<p>he said \u201chi\u201d\u2014ok</p>";
+  const next = async () => makeResponse(html, "text/html; charset=utf-8");
+  const res = await mw({} as unknown, next);
+  if (!(res instanceof Response)) throw new Error("expected Response");
+  const body = await res.text();
+  expect(body).toBe(`<p>he said "hi"-ok</p>`);
+});
+
+test("dev middleware: disabled config is pass-through", async () => {
+  const off: NormalizeConfig = { enabled: false, signals: { ...ALL_ON.signals } };
+  const mw = createDevNormalizeMiddleware(off);
+  const html = "<p>a\u2014b</p>";
+  const next = async () => makeResponse(html, "text/html");
+  const res = await mw({} as unknown, next);
+  if (!(res instanceof Response)) throw new Error("expected Response");
+  const body = await res.text();
+  expect(body).toBe(html);
+});
+
+test("dev middleware: non-HTML response is pass-through", async () => {
+  const mw = createDevNormalizeMiddleware(ALL_ON);
+  const json = '{"a":"b\u2014c"}';
+  const next = async () => makeResponse(json, "application/json");
+  const res = await mw({} as unknown, next);
+  if (!(res instanceof Response)) throw new Error("expected Response");
+  const body = await res.text();
+  expect(body).toBe(json);
+});
+
+test("dev middleware: try/catch falls back to original on error", async () => {
+  const mw = createDevNormalizeMiddleware(ALL_ON);
+  const html = "<p>valid</p>";
+  let threw = false;
+  const res = await mw({} as unknown, async () => {
+    const r = makeResponse(html, "text/html");
+    r.text = async () => {
+      threw = true;
+      throw new Error("simulated read error");
+    };
+    return r;
+  });
+  expect(threw).toBe(true);
+  expect(res).toBeInstanceOf(Response);
+});

package/src/share/tests/twin-path.test.ts

@@ -0,0 +1,43 @@
+/*
+<MODULE_CONTRACT>
+<purpose>Unit tests for RFC-0306 Markdown twin path helpers.</purpose>
+<keywords>markdown twin, path helper, RFC-0306</keywords>
+<responsibilities>
+  <item>Lock root, language-root, nested, trailing-slash, and non-language slug behavior.</item>
+</responsibilities>
+<non-goals>
+  <item>Do not inspect generated app files.</item>
+</non-goals>
+</MODULE_CONTRACT>
+<MODULE_MAP>
+  <entry key="tests">node:test assertions for markdownTwinRelPath and markdownTwinUrlPath.</entry>
+</MODULE_MAP>
+<CHANGE_SUMMARY>
+  <item>RFC-0306: Added Markdown twin path helper coverage.</item>
+</CHANGE_SUMMARY>
+*/
+
+import { test, expect } from "vitest";
+import { markdownTwinRelPath, markdownTwinUrlPath } from "../semantic/ids.ts";
+
+const supportedLangs = ["de", "en"] as const;
+
+test("markdownTwinRelPath keeps root and language roots at index.md", () => {
+  expect(markdownTwinRelPath("/", { supportedLangs })).toBe("index.md");
+  expect(markdownTwinRelPath("/en", { supportedLangs })).toBe("en/index.md");
+  expect(markdownTwinRelPath("/en/", { supportedLangs })).toBe("en/index.md");
+});
+
+test("markdownTwinRelPath maps non-home pages to sibling md files", () => {
+  expect(markdownTwinRelPath("/team/andrii-syrokomskyi", { supportedLangs })).toBe(
+    "team/andrii-syrokomskyi.md",
+  );
+  expect(markdownTwinRelPath("/en/team/andrii", { supportedLangs })).toBe("en/team/andrii.md");
+  expect(markdownTwinRelPath("/it", { supportedLangs })).toBe("it.md");
+});
+
+test("markdownTwinUrlPath returns the public URL form", () => {
+  expect(markdownTwinUrlPath("/team/andrii-syrokomskyi", { supportedLangs })).toBe(
+    "/team/andrii-syrokomskyi.md",
+  );
+});