@jxsuite/parser 1.1.0 → 1.2.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jxsuite/parser",
-  "version": "1.1.0",
+  "version": "1.2.0",
   "description": "Jx markdown parser and external class integration",
   "license": "MIT",
   "repository": {
@@ -48,8 +48,10 @@
     "upgrade": "bunx npm-check-updates -u && bun install"
   },
   "dependencies": {
-    "@jxsuite/markup": "^0.3.0",
-    "@jxsuite/schema": "^1.1.0",
+    "@jxsuite/markup": "^0.3.1",
+    "@jxsuite/schema": "^1.2.0",
+    "@shikijs/langs": "^4.3.1",
+    "@shikijs/themes": "^4.3.1",
     "glob": "^13.0.6",
     "mdast-util-to-string": "^4.0.0",
     "remark-directive": "^4.0.0",
@@ -58,13 +60,14 @@
     "remark-parse": "^11.0.0",
     "remark-parse-frontmatter": "^1.0.3",
     "remark-stringify": "^11.0.0",
+    "shiki": "^4.3.1",
     "unified": "^11.0.5",
     "unist-util-visit": "^5.1.0",
     "yaml": "^2.9.0"
   },
   "devDependencies": {
     "@happy-dom/global-registrator": "^20.10.6",
-    "@jxsuite/runtime": "^1.1.0",
+    "@jxsuite/runtime": "^1.2.0",
     "@types/mdast": "^4.0.4",
     "ajv": "^8.20.0"
   },

package/src/highlight.ts

@@ -0,0 +1,112 @@
+/**
+ * Build-time syntax highlighting for fenced code blocks (node-only).
+ *
+ * A synchronous Shiki core (JavaScript regex engine — no WASM, no async) tokenizes a fixed
+ * grammar set against a light + dark GitHub theme pair. Tokens are emitted as span elements
+ * carrying both colors as CSS custom properties (--shiki-light / --shiki-dark); the page's
+ * stylesheet decides which one paints, so highlighting follows the color-scheme contract
+ * (spec §9.5) for auto and forced schemes alike.
+ *
+ * Lives outside the browser-safe transpile module: only the compile-time markdown path
+ * (processMarkdown in md.ts) pays for the grammars.
+ *
+ * @docs framework/site/jx-markdown
+ * @module @jxsuite/parser/highlight
+ * @license MIT
+ */
+
+import { createHighlighterCoreSync } from "shiki/core";
+import { createJavaScriptRegexEngine } from "shiki/engine/javascript";
+import githubDark from "@shikijs/themes/github-dark-default";
+import githubLight from "@shikijs/themes/github-light-default";
+import css from "@shikijs/langs/css";
+import html from "@shikijs/langs/html";
+import javascript from "@shikijs/langs/javascript";
+import json from "@shikijs/langs/json";
+import markdown from "@shikijs/langs/markdown";
+import shellscript from "@shikijs/langs/shellscript";
+import typescript from "@shikijs/langs/typescript";
+import yaml from "@shikijs/langs/yaml";
+import type { JxElement } from "@jxsuite/schema/types";
+import type { HighlighterCore } from "shiki/core";
+
+const THEMES = { dark: "github-dark-default", light: "github-light-default" } as const;
+
+let _highlighter: HighlighterCore | null = null;
+
+/** Lazy singleton — grammar/theme setup only runs when a fence is actually highlighted. */
+function getHighlighter(): HighlighterCore {
+  _highlighter ??= createHighlighterCoreSync({
+    engine: createJavaScriptRegexEngine(),
+    langs: [json, typescript, javascript, markdown, html, shellscript, css, yaml],
+    themes: [githubLight, githubDark],
+  });
+  return _highlighter;
+}
+
+/**
+ * Tokenize a fenced code block into span elements with dual-theme color variables. Returns null for
+ * unknown languages (callers keep the plain-text fallback).
+ *
+ * @param {string} code
+ * @param {string} lang - Fence info string (grammar name or registered alias)
+ * @returns {(JxElement | string)[] | null}
+ */
+export function highlightFence(code: string, lang: string): (JxElement | string)[] | null {
+  const h = getHighlighter();
+  if (!h.getLoadedLanguages().includes(lang)) {
+    return null;
+  }
+  const { tokens } = h.codeToTokens(code, { defaultColor: false, lang, themes: THEMES });
+  const out: (JxElement | string)[] = [];
+  for (const [i, line] of tokens.entries()) {
+    if (i > 0) {
+      out.push("\n");
+    }
+    for (const token of line) {
+      const span: JxElement = { tagName: "span", textContent: token.content };
+      if (token.htmlStyle && Object.keys(token.htmlStyle).length > 0) {
+        span.style = token.htmlStyle as Record<string, string>;
+      }
+      out.push(span);
+    }
+  }
+  return out;
+}
+
+/**
+ * Walk a transpiled Jx tree and highlight every `pre > code.language-*` block in place. Unknown
+ * languages and bare fences keep their plain textContent.
+ *
+ * @param {(JxElement | string)[]} nodes
+ */
+export function highlightCodeBlocks(nodes: (JxElement | string)[]): void {
+  for (const node of nodes) {
+    if (!node || typeof node !== "object") {
+      continue;
+    }
+    if (node.tagName === "pre" && Array.isArray(node.children)) {
+      for (const child of node.children) {
+        if (!child || typeof child !== "object" || child.tagName !== "code") {
+          continue;
+        }
+        const lang = /^language-(\S+)/.exec(child.className ?? "")?.[1];
+        const code = child.textContent;
+        if (!lang || typeof code !== "string" || code.length === 0) {
+          continue;
+        }
+        const spans = highlightFence(code, lang);
+        if (!spans) {
+          continue;
+        }
+        delete child.textContent;
+        child.children = spans;
+        child.className = `${child.className} shiki`;
+      }
+      continue;
+    }
+    if (Array.isArray(node.children)) {
+      highlightCodeBlocks(node.children as (JxElement | string)[]);
+    }
+  }
+}

package/src/md.ts

@@ -20,8 +20,9 @@ import remarkDirective from "remark-directive";
 import { readFileSync } from "node:fs";
 import { basename, extname, relative, resolve as resolvePath } from "node:path";
 import { globSync } from "glob";
-import { mdastNodeToJx } from "./transpile.ts";
-import type { MarkdownFileResult, MdastNode, TocEntry, UnifiedProcessor } from "./types.ts";
+import { assignHeadingIds, mdastNodeToJx } from "./transpile.ts";
+import { highlightCodeBlocks } from "./highlight.ts";
+import type { MarkdownFileResult, MdastNode, UnifiedProcessor } from "./types.ts";
 import type { JxElement } from "@jxsuite/schema/types";
 
 // ─── Tree utilities (inline to avoid Bun ESM resolution issues with unist-util-*) ──
@@ -92,28 +93,6 @@ function readingTime(text: string) {
   return Math.max(1, Math.ceil(words / 200));
 }
 
-/**
- * Extract table of contents entries from an mdast tree.
- *
- * @param {object} tree - Mdast AST
- * @returns {{ depth: number; text: string; id: string }[]}
- */
-function extractToc(tree: MdastNode) {
-  /** @type {{ depth: number; text: string; id: string }[]} */
-  const entries: TocEntry[] = [];
-  visit(tree, "heading", (node: MdastNode) => {
-    const text = mdastToString(node);
-    const id = text
-      .toLowerCase()
-      .replaceAll(/[^\w\s-]/g, "")
-      .replaceAll(/\s+/g, "-")
-      .replaceAll(/-+/g, "-")
-      .replaceAll(/^-|-$/g, "");
-    entries.push({ depth: node.depth as number, id, text });
-  });
-  return entries;
-}
-
 /**
  * Extract first paragraph as a JX text string from an mdast tree.
  *
@@ -197,7 +176,6 @@ export function processMarkdown(
   const frontmatter = (vfileData.frontmatter ?? {}) as Record<string, unknown>;
   const mdTree = tree as unknown as MdastNode;
   const plainText = mdastToString(mdTree);
-  const toc = extractToc(tree);
   const excerpt = extractExcerpt(tree);
   const slug = deriveSlug(filePath, config.sourceRoot);
 
@@ -209,6 +187,13 @@ export function processMarkdown(
     | string
   )[];
 
+  // Build-time syntax highlighting: fence text becomes dual-theme token spans in place.
+  highlightCodeBlocks($children);
+
+  // One walk assigns deduplicated heading ids AND builds $toc, so rendered anchors and the
+  // Table of contents agree by construction (specs/parser.md).
+  const toc = assignHeadingIds($children);
+
   return {
     $children,
     $excerpt: excerpt,

package/src/transpile.ts

@@ -19,7 +19,13 @@ import remarkGfm from "remark-gfm";
 import remarkDirective from "remark-directive";
 import { htmlToJx } from "@jxsuite/markup/html-to-jx";
 import type { MdastNode, UnifiedProcessor } from "./types.ts";
-import type { JsonValue, JxAttributeValue, JxDocument, JxElement } from "@jxsuite/schema/types";
+import type {
+  JsonValue,
+  JxAttributeValue,
+  JxDocument,
+  JxElement,
+  TocEntry,
+} from "@jxsuite/schema/types";
 
 export { htmlToJx } from "@jxsuite/markup/html-to-jx";
 
@@ -458,6 +464,16 @@ export function mdastNodeToJx(node: MdastNode) {
 
     case "table": {
       const rows = convertChildren(node.children ?? []);
+      // GFM's first table row is the header row, but mdast does not mark its cells — retag them
+      // To <th> so the <thead> carries real column headers (assistive tech relies on this).
+      const [headerRow] = rows;
+      if (headerRow && typeof headerRow === "object" && Array.isArray(headerRow.children)) {
+        for (const cell of headerRow.children) {
+          if (cell && typeof cell === "object" && (cell as JxElement).tagName === "td") {
+            (cell as JxElement).tagName = "th";
+          }
+        }
+      }
       const thead = rows.length > 0 ? { children: [rows[0]], tagName: "thead" } : null;
       const tbody = rows.length > 1 ? { children: rows.slice(1), tagName: "tbody" } : null;
       el.children = [thead, tbody].filter(Boolean) as JxElement[];
@@ -705,3 +721,88 @@ export function transpileJxMarkdown(source: string) {
 
   return doc;
 }
+
+// ─── Heading anchors ─────────────────────────────────────────────────────────
+
+/**
+ * Slugify a heading's text into an anchor id: lowercase, punctuation stripped, spaces collapsed to
+ * single hyphens. The one algorithm behind rendered heading `id`s, `$toc` entries, and search deep
+ * links (specs/parser.md).
+ *
+ * @param {string} text
+ * @returns {string}
+ */
+export function slugifyHeading(text: string) {
+  return text
+    .toLowerCase()
+    .replaceAll(/[^\w\s-]/g, "")
+    .replaceAll(/\s+/g, "-")
+    .replaceAll(/-+/g, "-")
+    .replaceAll(/^-|-$/g, "");
+}
+
+const HEADING_TAGS = new Set(["h1", "h2", "h3", "h4", "h5", "h6"]);
+
+/**
+ * Concatenated text of a Jx node: own textContent plus string/element children, depth-first.
+ *
+ * @param {JxElement | string} node
+ * @returns {string}
+ */
+function jxNodeText(node: JxElement | string): string {
+  if (typeof node === "string") {
+    return node;
+  }
+  let out = typeof node.textContent === "string" ? node.textContent : "";
+  if (Array.isArray(node.children)) {
+    for (const child of node.children) {
+      out += jxNodeText(child as JxElement | string);
+    }
+  }
+  return out;
+}
+
+/**
+ * Assign deduplicated slug ids to `h1`–`h6` elements (depth-first, document order) and return the
+ * matching table-of-contents entries. The first occurrence of a slug is unsuffixed; repeats get
+ * `-2`, `-3`, …. Pre-existing ids are respected and still claim their slug. Because ids and entries
+ * come from one walk, rendered anchors and `$toc` agree by construction.
+ *
+ * @param {(JxElement | string)[] | undefined} children - Jx nodes, mutated in place
+ * @returns {TocEntry[]}
+ */
+export function assignHeadingIds(children?: (JxElement | string)[]): TocEntry[] {
+  const entries: TocEntry[] = [];
+  const seen = new Map<string, number>();
+
+  function walk(nodes: (JxElement | string)[]) {
+    for (const node of nodes) {
+      if (typeof node !== "object" || node === null) {
+        continue;
+      }
+      const tag = node.tagName;
+      if (tag && HEADING_TAGS.has(tag)) {
+        const text = jxNodeText(node).trim();
+        let id = typeof node.id === "string" && node.id.length > 0 ? node.id : "";
+        if (id) {
+          seen.set(id, (seen.get(id) ?? 0) + 1);
+        } else {
+          const base = slugifyHeading(text) || "section";
+          const count = seen.get(base) ?? 0;
+          id = count === 0 ? base : `${base}-${count + 1}`;
+          seen.set(base, count + 1);
+          node.id = id;
+        }
+        entries.push({ depth: Number(tag.slice(1)), id, text });
+      }
+      if (Array.isArray(node.children)) {
+        walk(node.children as (JxElement | string)[]);
+      }
+    }
+  }
+
+  if (children) {
+    walk(children);
+  }
+  return entries;
+}