@jxsuite/parser 1.0.0 → 1.2.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jxsuite/parser",
-  "version": "1.0.0",
+  "version": "1.2.0",
   "description": "Jx markdown parser and external class integration",
   "license": "MIT",
   "repository": {
@@ -48,8 +48,10 @@
     "upgrade": "bunx npm-check-updates -u && bun install"
   },
   "dependencies": {
-    "@jxsuite/markup": "^0.2.0",
-    "@jxsuite/schema": "^1.0.0",
+    "@jxsuite/markup": "^0.3.1",
+    "@jxsuite/schema": "^1.2.0",
+    "@shikijs/langs": "^4.3.1",
+    "@shikijs/themes": "^4.3.1",
     "glob": "^13.0.6",
     "mdast-util-to-string": "^4.0.0",
     "remark-directive": "^4.0.0",
@@ -58,13 +60,14 @@
     "remark-parse": "^11.0.0",
     "remark-parse-frontmatter": "^1.0.3",
     "remark-stringify": "^11.0.0",
+    "shiki": "^4.3.1",
     "unified": "^11.0.5",
     "unist-util-visit": "^5.1.0",
     "yaml": "^2.9.0"
   },
   "devDependencies": {
     "@happy-dom/global-registrator": "^20.10.6",
-    "@jxsuite/runtime": "^1.0.0",
+    "@jxsuite/runtime": "^1.2.0",
     "@types/mdast": "^4.0.4",
     "ajv": "^8.20.0"
   },

package/src/Csv.class.json

@@ -17,7 +17,7 @@
 
   "$studio": {
     "icon": "table",
-    "modes": ["source"]
+    "modes": ["grid", "source"]
   },
 
   "$defs": {

package/src/content-loader.ts

@@ -269,12 +269,15 @@ async function loadContentType(
       })) as string[])
     : [resolvedSource];
 
+  // Directory sources pass their resolved root so formats can derive path-based ids for nested files
+  const sourceRoot = extname(source) ? undefined : resolvedSource;
   const entries: ContentLoaderEntry[] = [];
   for (const filePath of files) {
     entries.push(
       ...((await entry.call("load", filePath, {
         directiveOptions,
         schema,
+        ...(sourceRoot !== undefined && { sourceRoot }),
       })) as ContentLoaderEntry[]),
     );
   }
@@ -374,7 +377,7 @@ function refTargetName(ref: string | undefined): string | undefined {
  * field `$ref` is to-one: a blog post's `author: "jane-doe"` with a schema `$ref` of
  * `#/content/authors` gets replaced by the full author entry. An array field whose `items` carry
  * the `$ref` is to-many: each id in the stored array is replaced by its entry. Unresolvable ids are
- * left untouched.
+ * left untouched, with a warning naming the entry and the missing target.
  *
  * @param {Map<string, ContentLoaderEntry[]>} contentTypes - All loaded content types
  * @param {ContentSection} section - The `content` section value the types were loaded from
@@ -403,6 +406,9 @@ export function resolveContentTypeRefs(
       }
       const refEntries = contentTypes.get(target);
       if (!refEntries) {
+        console.warn(
+          `Content relationships: "${name}" field "${field}" references unknown content type "${target}"`,
+        );
         continue;
       }
 
@@ -412,11 +418,24 @@ export function resolveContentTypeRefs(
           const resolved = refEntries.find((e) => e.id === value);
           if (resolved) {
             entry.data[field] = resolved;
+          } else {
+            console.warn(
+              `Content relationships: "${name}/${entry.id}" field "${field}" references missing "${target}" entry "${value}"`,
+            );
           }
         } else if (toMany && Array.isArray(value)) {
-          entry.data[field] = value.map((id) =>
-            typeof id === "string" ? (refEntries.find((e) => e.id === id) ?? id) : id,
-          );
+          entry.data[field] = value.map((id) => {
+            if (typeof id !== "string") {
+              return id;
+            }
+            const resolved = refEntries.find((e) => e.id === id);
+            if (!resolved) {
+              console.warn(
+                `Content relationships: "${name}/${entry.id}" field "${field}" references missing "${target}" entry "${id}"`,
+              );
+            }
+            return resolved ?? id;
+          });
         }
       }
     }

package/src/highlight.ts

@@ -0,0 +1,112 @@
+/**
+ * Build-time syntax highlighting for fenced code blocks (node-only).
+ *
+ * A synchronous Shiki core (JavaScript regex engine — no WASM, no async) tokenizes a fixed
+ * grammar set against a light + dark GitHub theme pair. Tokens are emitted as span elements
+ * carrying both colors as CSS custom properties (--shiki-light / --shiki-dark); the page's
+ * stylesheet decides which one paints, so highlighting follows the color-scheme contract
+ * (spec §9.5) for auto and forced schemes alike.
+ *
+ * Lives outside the browser-safe transpile module: only the compile-time markdown path
+ * (processMarkdown in md.ts) pays for the grammars.
+ *
+ * @docs framework/site/jx-markdown
+ * @module @jxsuite/parser/highlight
+ * @license MIT
+ */
+
+import { createHighlighterCoreSync } from "shiki/core";
+import { createJavaScriptRegexEngine } from "shiki/engine/javascript";
+import githubDark from "@shikijs/themes/github-dark-default";
+import githubLight from "@shikijs/themes/github-light-default";
+import css from "@shikijs/langs/css";
+import html from "@shikijs/langs/html";
+import javascript from "@shikijs/langs/javascript";
+import json from "@shikijs/langs/json";
+import markdown from "@shikijs/langs/markdown";
+import shellscript from "@shikijs/langs/shellscript";
+import typescript from "@shikijs/langs/typescript";
+import yaml from "@shikijs/langs/yaml";
+import type { JxElement } from "@jxsuite/schema/types";
+import type { HighlighterCore } from "shiki/core";
+
+const THEMES = { dark: "github-dark-default", light: "github-light-default" } as const;
+
+let _highlighter: HighlighterCore | null = null;
+
+/** Lazy singleton — grammar/theme setup only runs when a fence is actually highlighted. */
+function getHighlighter(): HighlighterCore {
+  _highlighter ??= createHighlighterCoreSync({
+    engine: createJavaScriptRegexEngine(),
+    langs: [json, typescript, javascript, markdown, html, shellscript, css, yaml],
+    themes: [githubLight, githubDark],
+  });
+  return _highlighter;
+}
+
+/**
+ * Tokenize a fenced code block into span elements with dual-theme color variables. Returns null for
+ * unknown languages (callers keep the plain-text fallback).
+ *
+ * @param {string} code
+ * @param {string} lang - Fence info string (grammar name or registered alias)
+ * @returns {(JxElement | string)[] | null}
+ */
+export function highlightFence(code: string, lang: string): (JxElement | string)[] | null {
+  const h = getHighlighter();
+  if (!h.getLoadedLanguages().includes(lang)) {
+    return null;
+  }
+  const { tokens } = h.codeToTokens(code, { defaultColor: false, lang, themes: THEMES });
+  const out: (JxElement | string)[] = [];
+  for (const [i, line] of tokens.entries()) {
+    if (i > 0) {
+      out.push("\n");
+    }
+    for (const token of line) {
+      const span: JxElement = { tagName: "span", textContent: token.content };
+      if (token.htmlStyle && Object.keys(token.htmlStyle).length > 0) {
+        span.style = token.htmlStyle as Record<string, string>;
+      }
+      out.push(span);
+    }
+  }
+  return out;
+}
+
+/**
+ * Walk a transpiled Jx tree and highlight every `pre > code.language-*` block in place. Unknown
+ * languages and bare fences keep their plain textContent.
+ *
+ * @param {(JxElement | string)[]} nodes
+ */
+export function highlightCodeBlocks(nodes: (JxElement | string)[]): void {
+  for (const node of nodes) {
+    if (!node || typeof node !== "object") {
+      continue;
+    }
+    if (node.tagName === "pre" && Array.isArray(node.children)) {
+      for (const child of node.children) {
+        if (!child || typeof child !== "object" || child.tagName !== "code") {
+          continue;
+        }
+        const lang = /^language-(\S+)/.exec(child.className ?? "")?.[1];
+        const code = child.textContent;
+        if (!lang || typeof code !== "string" || code.length === 0) {
+          continue;
+        }
+        const spans = highlightFence(code, lang);
+        if (!spans) {
+          continue;
+        }
+        delete child.textContent;
+        child.children = spans;
+        child.className = `${child.className} shiki`;
+      }
+      continue;
+    }
+    if (Array.isArray(node.children)) {
+      highlightCodeBlocks(node.children as (JxElement | string)[]);
+    }
+  }
+}

package/src/markdown.ts

@@ -26,6 +26,12 @@ export interface MarkdownLoadOptions {
   directiveOptions?: unknown;
   /** Enable remark-directive parsing. Implied by directiveOptions. */
   directives?: boolean;
+  /**
+   * Resolved content-source root directory. When set, entries in subdirectories get path-based ids
+   * (`studio/canvas` for `studio/canvas.md`, `/index` stripped); files directly at the root keep
+   * basename ids.
+   */
+  sourceRoot?: string;
 }
 
 /**
@@ -83,6 +89,7 @@ export class Markdown {
       ...(options.directiveOptions !== undefined && {
         directiveOptions: options.directiveOptions,
       }),
+      ...(options.sourceRoot !== undefined && { sourceRoot: options.sourceRoot }),
     });
     const _meta: ContentLoaderEntry["_meta"] = {};
     if (result.$excerpt != null) {

package/src/md.ts

@@ -18,10 +18,11 @@ import remarkParseFrontmatter from "remark-parse-frontmatter";
 import remarkGfm from "remark-gfm";
 import remarkDirective from "remark-directive";
 import { readFileSync } from "node:fs";
-import { basename, extname, resolve as resolvePath } from "node:path";
+import { basename, extname, relative, resolve as resolvePath } from "node:path";
 import { globSync } from "glob";
-import { mdastNodeToJx } from "./transpile.ts";
-import type { MarkdownFileResult, MdastNode, TocEntry, UnifiedProcessor } from "./types.ts";
+import { assignHeadingIds, mdastNodeToJx } from "./transpile.ts";
+import { highlightCodeBlocks } from "./highlight.ts";
+import type { MarkdownFileResult, MdastNode, UnifiedProcessor } from "./types.ts";
 import type { JxElement } from "@jxsuite/schema/types";
 
 // ─── Tree utilities (inline to avoid Bun ESM resolution issues with unist-util-*) ──
@@ -92,28 +93,6 @@ function readingTime(text: string) {
   return Math.max(1, Math.ceil(words / 200));
 }
 
-/**
- * Extract table of contents entries from an mdast tree.
- *
- * @param {object} tree - Mdast AST
- * @returns {{ depth: number; text: string; id: string }[]}
- */
-function extractToc(tree: MdastNode) {
-  /** @type {{ depth: number; text: string; id: string }[]} */
-  const entries: TocEntry[] = [];
-  visit(tree, "heading", (node: MdastNode) => {
-    const text = mdastToString(node);
-    const id = text
-      .toLowerCase()
-      .replaceAll(/[^\w\s-]/g, "")
-      .replaceAll(/\s+/g, "-")
-      .replaceAll(/-+/g, "-")
-      .replaceAll(/^-|-$/g, "");
-    entries.push({ depth: node.depth as number, id, text });
-  });
-  return entries;
-}
-
 /**
  * Extract first paragraph as a JX text string from an mdast tree.
  *
@@ -133,6 +112,30 @@ function extractExcerpt(tree: MdastNode) {
   return mdastToString(firstParagraph);
 }
 
+/**
+ * Derive an entry slug from a file path. Without a source root (or for files directly at the root),
+ * the slug is the basename — the historical behavior every flat collection relies on. Files in
+ * subdirectories of the root get path-based slugs with POSIX separators and a trailing `/index`
+ * stripped, so `studio/canvas.md` and `studio/canvas/index.md` both yield `studio/canvas`.
+ *
+ * @param {string} filePath - Absolute path to the markdown file
+ * @param {string} [sourceRoot] - Resolved content-source root directory
+ * @returns {string} The entry slug
+ */
+function deriveSlug(filePath: string, sourceRoot?: string): string {
+  if (sourceRoot) {
+    const rel = relative(sourceRoot, filePath).split("\\").join("/");
+    if (rel && !rel.startsWith("..") && rel.includes("/")) {
+      let slug = rel.slice(0, rel.length - extname(rel).length);
+      if (slug.endsWith("/index")) {
+        slug = slug.slice(0, -"/index".length);
+      }
+      return slug;
+    }
+  }
+  return basename(filePath, extname(filePath));
+}
+
 /**
  * Process a single markdown source string into a MarkdownFileResult.
  *
@@ -143,6 +146,7 @@ function extractExcerpt(tree: MdastNode) {
  * @param {object} config - Processing options
  * @param {boolean} [config.directives] - Enable directive support
  * @param {unknown} [config.directiveOptions] - Directive plugin options
+ * @param {string} [config.sourceRoot] - Content-source root; files below it get path-based slugs
  * @returns {MarkdownFileResult}
  */
 export function processMarkdown(
@@ -151,6 +155,7 @@ export function processMarkdown(
   config: {
     directives?: boolean;
     directiveOptions?: unknown;
+    sourceRoot?: string;
   } = {},
 ) {
   let processor = (unified as unknown as () => UnifiedProcessor)()
@@ -171,9 +176,8 @@ export function processMarkdown(
   const frontmatter = (vfileData.frontmatter ?? {}) as Record<string, unknown>;
   const mdTree = tree as unknown as MdastNode;
   const plainText = mdastToString(mdTree);
-  const toc = extractToc(tree);
   const excerpt = extractExcerpt(tree);
-  const slug = basename(filePath, extname(filePath));
+  const slug = deriveSlug(filePath, config.sourceRoot);
 
   const bodyNodes = tree.children!.filter(
     (n: MdastNode) => n.type !== "yaml" && n.type !== "toml",
@@ -183,6 +187,13 @@ export function processMarkdown(
     | string
   )[];
 
+  // Build-time syntax highlighting: fence text becomes dual-theme token spans in place.
+  highlightCodeBlocks($children);
+
+  // One walk assigns deduplicated heading ids AND builds $toc, so rendered anchors and the
+  // Table of contents agree by construction (specs/parser.md).
+  const toc = assignHeadingIds($children);
+
   return {
     $children,
     $excerpt: excerpt,

package/src/transpile.ts

@@ -19,7 +19,13 @@ import remarkGfm from "remark-gfm";
 import remarkDirective from "remark-directive";
 import { htmlToJx } from "@jxsuite/markup/html-to-jx";
 import type { MdastNode, UnifiedProcessor } from "./types.ts";
-import type { JsonValue, JxAttributeValue, JxDocument, JxElement } from "@jxsuite/schema/types";
+import type {
+  JsonValue,
+  JxAttributeValue,
+  JxDocument,
+  JxElement,
+  TocEntry,
+} from "@jxsuite/schema/types";
 
 export { htmlToJx } from "@jxsuite/markup/html-to-jx";
 
@@ -458,6 +464,16 @@ export function mdastNodeToJx(node: MdastNode) {
 
     case "table": {
       const rows = convertChildren(node.children ?? []);
+      // GFM's first table row is the header row, but mdast does not mark its cells — retag them
+      // To <th> so the <thead> carries real column headers (assistive tech relies on this).
+      const [headerRow] = rows;
+      if (headerRow && typeof headerRow === "object" && Array.isArray(headerRow.children)) {
+        for (const cell of headerRow.children) {
+          if (cell && typeof cell === "object" && (cell as JxElement).tagName === "td") {
+            (cell as JxElement).tagName = "th";
+          }
+        }
+      }
       const thead = rows.length > 0 ? { children: [rows[0]], tagName: "thead" } : null;
       const tbody = rows.length > 1 ? { children: rows.slice(1), tagName: "tbody" } : null;
       el.children = [thead, tbody].filter(Boolean) as JxElement[];
@@ -705,3 +721,88 @@ export function transpileJxMarkdown(source: string) {
 
   return doc;
 }
+
+// ─── Heading anchors ─────────────────────────────────────────────────────────
+
+/**
+ * Slugify a heading's text into an anchor id: lowercase, punctuation stripped, spaces collapsed to
+ * single hyphens. The one algorithm behind rendered heading `id`s, `$toc` entries, and search deep
+ * links (specs/parser.md).
+ *
+ * @param {string} text
+ * @returns {string}
+ */
+export function slugifyHeading(text: string) {
+  return text
+    .toLowerCase()
+    .replaceAll(/[^\w\s-]/g, "")
+    .replaceAll(/\s+/g, "-")
+    .replaceAll(/-+/g, "-")
+    .replaceAll(/^-|-$/g, "");
+}
+
+const HEADING_TAGS = new Set(["h1", "h2", "h3", "h4", "h5", "h6"]);
+
+/**
+ * Concatenated text of a Jx node: own textContent plus string/element children, depth-first.
+ *
+ * @param {JxElement | string} node
+ * @returns {string}
+ */
+function jxNodeText(node: JxElement | string): string {
+  if (typeof node === "string") {
+    return node;
+  }
+  let out = typeof node.textContent === "string" ? node.textContent : "";
+  if (Array.isArray(node.children)) {
+    for (const child of node.children) {
+      out += jxNodeText(child as JxElement | string);
+    }
+  }
+  return out;
+}
+
+/**
+ * Assign deduplicated slug ids to `h1`–`h6` elements (depth-first, document order) and return the
+ * matching table-of-contents entries. The first occurrence of a slug is unsuffixed; repeats get
+ * `-2`, `-3`, …. Pre-existing ids are respected and still claim their slug. Because ids and entries
+ * come from one walk, rendered anchors and `$toc` agree by construction.
+ *
+ * @param {(JxElement | string)[] | undefined} children - Jx nodes, mutated in place
+ * @returns {TocEntry[]}
+ */
+export function assignHeadingIds(children?: (JxElement | string)[]): TocEntry[] {
+  const entries: TocEntry[] = [];
+  const seen = new Map<string, number>();
+
+  function walk(nodes: (JxElement | string)[]) {
+    for (const node of nodes) {
+      if (typeof node !== "object" || node === null) {
+        continue;
+      }
+      const tag = node.tagName;
+      if (tag && HEADING_TAGS.has(tag)) {
+        const text = jxNodeText(node).trim();
+        let id = typeof node.id === "string" && node.id.length > 0 ? node.id : "";
+        if (id) {
+          seen.set(id, (seen.get(id) ?? 0) + 1);
+        } else {
+          const base = slugifyHeading(text) || "section";
+          const count = seen.get(base) ?? 0;
+          id = count === 0 ? base : `${base}-${count + 1}`;
+          seen.set(base, count + 1);
+          node.id = id;
+        }
+        entries.push({ depth: Number(tag.slice(1)), id, text });
+      }
+      if (Array.isArray(node.children)) {
+        walk(node.children as (JxElement | string)[]);
+      }
+    }
+  }
+
+  if (children) {
+    walk(children);
+  }
+  return entries;
+}