@jxsuite/parser 1.4.1 → 1.5.2

package/src/md.ts

@@ -18,6 +18,7 @@ import remarkParseFrontmatter from "remark-parse-frontmatter";
 import remarkGfm from "remark-gfm";
 import remarkDirective from "remark-directive";
 import { readFileSync } from "node:fs";
+import { DEFAULT_FORMAT_LOCALE } from "@jxsuite/schema/intl";
 import { basename, extname, relative, resolve as resolvePath } from "node:path";
 import { globSync } from "glob";
 import { assignHeadingIds, mdastNodeToJx } from "./transpile.ts";
@@ -82,15 +83,62 @@ function mdastToString(node: MdastNode): string {
 
 // ─── Helpers ──────────────────────────────────────────────────────────────────
 
+/** Average reading speed. See {@link readingTime} for why this is one number and not a table. */
+const WORDS_PER_MINUTE = 200;
+
+/**
+ * One segmenter, built once.
+ *
+ * The locale is fixed rather than the host's: word boundaries are a property of the text, and a
+ * word count that changed with the build machine's locale would be the same determinism bug the
+ * `Intl` helpers exist to avoid.
+ */
+const WORD_SEGMENTER = new Intl.Segmenter(DEFAULT_FORMAT_LOCALE, { granularity: "word" });
+
+/** A segment is a word when it carries a letter or a digit. See {@link countWords}. */
+const WORD_LIKE = /[\p{L}\p{N}]/u;
+
+/**
+ * Count the words in a string by **segmenting** it, not by splitting on whitespace (UAX #29).
+ *
+ * Splitting on `\s+` assumes a script that puts spaces between its words. Japanese, Chinese and
+ * Thai do not, so an entire article in any of them counted as **one word** — and therefore one
+ * minute to read, whatever its length. `Intl.Segmenter` applies the Unicode word-boundary rules.
+ *
+ * **A segment counts when it contains a letter or a digit, rather than when `isWordLike` says so.**
+ * That property is what `isWordLike` is defined to mean, but Bun's engine answers `false` for a
+ * mixed alphanumeric segment — `w0`, `v3`, `h1` — so trusting it would drop every token of that
+ * shape from the count, silently and only for some documents. The predicate is spelled out because
+ * it has to be correct on the runtime this actually runs on.
+ *
+ * This runs in the parser under Bun, so there is no browser-support question to weigh.
+ *
+ * @param {string} text
+ * @returns {number}
+ */
+function countWords(text: string): number {
+  let words = 0;
+  for (const { segment } of WORD_SEGMENTER.segment(text)) {
+    if (WORD_LIKE.test(segment)) {
+      words += 1;
+    }
+  }
+  return words;
+}
+
 /**
- * Estimate reading time based on word count (~200 wpm average).
+ * Estimate reading time from the word count.
+ *
+ * **200 wpm, and the limitation is stated rather than modelled.** Real reading speed varies by
+ * script, by density and by reader — a Japanese character carries more than a Latin one, so a
+ * per-script table would be more accurate and would also be a table of guesses nobody could verify.
+ * One honest constant that the docs name beats a plausible one that they cannot.
  *
  * @param {string} text
  * @returns {number} Minutes (rounded up, minimum 1)
  */
 function readingTime(text: string) {
-  const words = text.split(/\s+/).filter(Boolean).length;
-  return Math.max(1, Math.ceil(words / 200));
+  return Math.max(1, Math.ceil(countWords(text) / WORDS_PER_MINUTE));
 }
 
 /**
@@ -199,7 +247,7 @@ export function processMarkdown(
     $excerpt: excerpt,
     $readingTime: readingTime(plainText),
     $toc: toc,
-    $wordCount: plainText.split(/\s+/).filter(Boolean).length,
+    $wordCount: countWords(plainText),
     frontmatter,
     path: filePath,
     slug,

package/src/serialize.ts

@@ -19,7 +19,7 @@
  */
 
 import { unified } from "unified";
-import { isRef } from "@jxsuite/schema/guards";
+import { isRef, tagNameCandidates } from "@jxsuite/schema/guards";
 import remarkStringify from "remark-stringify";
 import remarkDirective from "remark-directive";
 import remarkGfm from "remark-gfm";
@@ -494,7 +494,7 @@ function convertJxNode(
     return null;
   }
 
-  const tag = el.tagName ?? "div";
+  const tag = serializableTag(el.tagName) || "div";
 
   // Not markdown-native or carries Jx-specific props → directive
   if (!allowlist.has(tag) || hasJxProps(el)) {
@@ -690,6 +690,31 @@ const JX_DOLLAR_KEYS = new Set([
   "$elements",
 ]);
 
+/**
+ * The tag, for a serializer that must write ONE name.
+ *
+ * A markdown directive is named by a literal, so a tag chosen at creation cannot round-trip through
+ * this format: picking a candidate would silently rewrite the author's document into one branch of
+ * it, which is the failure the tag-name pattern was added to stop. Content documents do not carry
+ * chosen tags today — the construct is an element-document (component) feature — so this refuses
+ * rather than guesses, and says which candidates it saw.
+ *
+ * @param {unknown} tagName
+ * @returns {string} The literal tag, or "" when none is declared
+ */
+function serializableTag(tagName: unknown): string {
+  if (typeof tagName === "string") {
+    return tagName;
+  }
+  if (tagName == null) {
+    return "";
+  }
+  throw new Error(
+    `Markdown cannot express a tag chosen at creation (candidates: ` +
+      `${tagNameCandidates(tagName).join(", ")}). Keep this element in a JSON component.`,
+  );
+}
+
 /**
  * `$prototype` element types that serialize as a directive named after the prototype (no tagName),
  * e.g. `:::Array`. Mirrors the set in transpile.ts.
@@ -898,7 +923,7 @@ function nodeToMdast(
     return expandArray(node, ctx, scope);
   }
 
-  const tag = node.tagName ?? "div";
+  const tag = serializableTag(node.tagName) || "div";
   const text = resolveText(node.textContent, ctx, scope);
 
   // InnerHTML — convert HTML content to mdast

package/src/transpile.ts

@@ -725,18 +725,35 @@ export function transpileJxMarkdown(source: string) {
 // ─── Heading anchors ─────────────────────────────────────────────────────────
 
 /**
- * Slugify a heading's text into an anchor id: lowercase, punctuation stripped, spaces collapsed to
- * single hyphens. The one algorithm behind rendered heading `id`s, `$toc` entries, and search deep
- * links (specs/parser.md).
+ * Slugify a heading's text into an anchor id: normalized, lowercased, punctuation stripped, spaces
+ * collapsed to single hyphens. The one algorithm behind rendered heading `id`s, `$toc` entries, and
+ * search deep links (specs/parser.md).
+ *
+ * **NFC first, and that is the whole point of the change** (UAX #15). "Café" typed on macOS is `e`
+ * + U+0301; the same word typed on Windows, or pasted from a web page, is U+00E9. They look
+ * identical, they compare unequal, and they produced two different anchors — so a link written
+ * against one heading silently stopped working when someone re-typed the heading. Normalizing puts
+ * both spellings in one form before anything else looks at them.
+ *
+ * **`toLowerCase`, never `toLocaleLowerCase`.** Locale-sensitive casing would make the same heading
+ * produce different anchors in different projects — Turkish maps `I` to `ı`, not `i` — and an
+ * anchor is a URL, which belongs to the document rather than to the reader's locale.
+ *
+ * **`_` stays in the character class deliberately.** The old class was `\w`, which is
+ * `[A-Za-z0-9_]`; `\p{L}\p{N}` is the Unicode-aware replacement and does **not** include `_`.
+ * Keeping it means this function is **byte-identical on every pure-ASCII heading**, so no existing
+ * anchor on an English site moves — `tests/transpile.test.ts` asserts exactly that against a
+ * corpus. What changes is only headings that were being mangled anyway.
  *
  * @param {string} text
  * @returns {string}
  */
 export function slugifyHeading(text: string) {
   return text
+    .normalize("NFC")
     .toLowerCase()
-    .replaceAll(/[^\w\s-]/g, "")
-    .replaceAll(/\s+/g, "-")
+    .replaceAll(/[^\p{L}\p{N}_\s-]/gu, "")
+    .replaceAll(/\s+/gu, "-")
     .replaceAll(/-+/g, "-")
     .replaceAll(/^-|-$/g, "");
 }
@@ -780,7 +797,9 @@ export function assignHeadingIds(children?: (JxElement | string)[]): TocEntry[]
       if (typeof node !== "object" || node === null) {
         continue;
       }
-      const tag = node.tagName;
+      // Heading extraction reads a literal tag; a chosen one is never a heading (every candidate
+      // Would have to be, and the construct exists to vary a WRAPPER), so it is simply not one.
+      const tag = typeof node.tagName === "string" ? node.tagName : "";
       if (tag && HEADING_TAGS.has(tag)) {
         const text = jxNodeText(node).trim();
         let id = typeof node.id === "string" && node.id.length > 0 ? node.id : "";