@fuzdev/fuz_code 0.45.1 → 0.46.1

package/dist/syntax_styler.js

@@ -1,5 +1,11 @@
 import { SyntaxToken } from './syntax_token.js';
 import { tokenize_syntax } from './tokenize_syntax.js';
+/**
+ * Maps a matched `&`, `<`, or non-breaking space in text content to its
+ * HTML-safe form. Used as the replacer in `stringify_token` for leaf strings
+ * (non-breaking spaces are normalized to a regular space).
+ */
+const escape_text_char = (ch) => (ch === '&' ? '&amp;' : ch === '<' ? '&lt;' : ' ');
 /**
  * Based on Prism (https://github.com/PrismJS/prism)
  * by Lea Verou (https://lea.verou.me/)
@@ -79,15 +85,14 @@ export class SyntaxStyler {
      * - Custom grammar: `stylize(code, 'ts', customGrammar)` - uses custom grammar but keeps 'ts' label
      * - Extended grammar: `stylize(code, 'custom', this.extend_grammar('ts', extension))` - new language variant
      *
-     * @param text - The source code to syntax highlight.
-     * @param lang - Language identifier (e.g., 'ts', 'css', 'html'). Used for:
-     *   - Grammar lookup when `grammar` is undefined
-     *   - Hook context (`lang` field passed to hooks)
-     *   - Language identification in output
-     * @param grammar - Optional custom grammar object. When undefined, automatically
-     *   looks up the grammar via `this.get_lang(lang)`. Provide this to use a custom
-     *   or modified grammar instead of the registered one.
-     *
+     * @param text - the source code to syntax highlight
+     * @param lang - language identifier (e.g., 'ts', 'css', 'html'), used for:
+     *   - grammar lookup when `grammar` is undefined
+     *   - hook context (`lang` field passed to hooks)
+     *   - language identification in output
+     * @param grammar - optional custom `SyntaxGrammar` object; when undefined, automatically
+     *   looks up the grammar via `this.get_lang(lang)`; provide this to use a custom
+     *   or modified grammar instead of the registered one
      * @returns HTML string with syntax highlighting using CSS classes (`.token_*`)
      *
      * @example
@@ -111,7 +116,32 @@ export class SyntaxStyler {
      * ```
      */
     stylize(text, lang, grammar = this.get_lang(lang)) {
-        var ctx = {
+        // stringify with the post-hook `lang`, which a `before_tokenize` hook may
+        // have rewritten (it flows into each token's `wrap` hook context)
+        const c = this.#tokenize_hooked(text, lang, grammar);
+        return this.stringify_token(c.tokens, c.lang);
+    }
+    /**
+     * Tokenizes `text` into a `SyntaxTokenStream`, running the `before_tokenize`
+     * and `after_tokenize` hooks. This is the tokenization half of `stylize` — use
+     * it when you need the token stream itself (e.g. CSS Custom Highlight API range
+     * highlighting) rather than HTML.
+     *
+     * @param text - source to tokenize
+     * @param lang - language identifier; passed to the tokenize hooks
+     * @param grammar - grammar to tokenize with; defaults to `this.get_lang(lang)`
+     * @returns the resulting token stream
+     */
+    tokenize(text, lang, grammar = this.get_lang(lang)) {
+        return this.#tokenize_hooked(text, lang, grammar).tokens;
+    }
+    /**
+     * Runs `before_tokenize` → `tokenize_syntax` → `after_tokenize`, returning the
+     * resolved context. Shared by `stylize` (which also needs the post-hook `lang`)
+     * and `tokenize` (which only needs `tokens`).
+     */
+    #tokenize_hooked(text, lang, grammar) {
+        const ctx = {
             code: text,
             grammar,
             lang,
@@ -121,7 +151,7 @@ export class SyntaxStyler {
         const c = ctx;
         c.tokens = tokenize_syntax(c.code, c.grammar);
         this.run_hook_after_tokenize(c);
-        return this.stringify_token(c.tokens, c.lang);
+        return c;
     }
     /**
      * Inserts tokens _before_ another token in a language definition or any other grammar.
@@ -140,7 +170,7 @@ export class SyntaxStyler {
      * };
      * ```
      *
-     * then the `style` token will be added (and processed) at the end. `insert_before` allows you to insert tokens
+     * then the `style` token will be added (and processed) at the end. `grammar_insert_before` allows you to insert tokens
      * before existing tokens. For the CSS example above, you would use it like this:
      *
      * ```js
@@ -167,12 +197,12 @@ export class SyntaxStyler {
      *
      * ## Limitations
      *
-     * The main problem `insert_before` has to solve is iteration order. Since ES2015, the iteration order for object
+     * The main problem `grammar_insert_before` has to solve is iteration order. Since ES2015, the iteration order for object
      * properties is guaranteed to be the insertion order (except for integer keys) but some browsers behave
-     * differently when keys are deleted and re-inserted. So `insert_before` can't be implemented by temporarily
+     * differently when keys are deleted and re-inserted. So `grammar_insert_before` can't be implemented by temporarily
      * deleting properties which is necessary to insert at arbitrary positions.
      *
-     * To solve this problem, `insert_before` doesn't actually insert the given tokens into the target object.
+     * To solve this problem, `grammar_insert_before` doesn't actually insert the given tokens into the target object.
      * Instead, it will create a new object and replace all references to the target object with the new one. This
      * can be done without temporarily deleting properties, so the iteration order is well-defined.
      *
@@ -187,16 +217,13 @@ export class SyntaxStyler {
      * assert(newMarkup === syntax_styler.get_lang('markup'));
      * ```
      *
-     * @param inside - The property of `root` (e.g. a language id in `syntax_styler.langs`) that contains the
-     * object to be modified.
-     * @param before - The key to insert before.
-     * @param insert - An object containing the key-value pairs to be inserted.
-     * @param root - The object containing `inside`, i.e. the object that contains the
-     * object to be modified.
-     *
-     * Defaults to `syntax_styler.langs`.
-     *
-     * @returns the new grammar object
+     * @param inside - the property of `root` (e.g. a language id in `syntax_styler.langs`) that contains the
+     * object to be modified
+     * @param before - the key to insert before
+     * @param insert - an object containing the key-value pairs to be inserted
+     * @param root - the object containing `inside`, i.e. the object that contains the
+     * object to be modified; defaults to `syntax_styler.langs`
+     * @returns the new `SyntaxGrammar` object
      */
     grammar_insert_before(inside, before, insert, root = this.langs) {
         var grammar = root[inside];
@@ -231,16 +258,15 @@ export class SyntaxStyler {
      *
      * Runs the `wrap` hook on each `SyntaxToken`.
      *
-     * @param o - The token or token stream to be converted.
-     * @param lang - The name of current language.
-     * @returns The HTML representation of the token or token stream.
+     * @param o - the token or `SyntaxTokenStream` to be converted
+     * @param lang - the name of current language
+     * @returns HTML representation of the token or token stream
      */
     stringify_token(o, lang) {
         if (typeof o === 'string') {
-            return o
-                .replace(/&/g, '&amp;')
-                .replace(/</g, '&lt;')
-                .replace(/\u00a0/g, ' ');
+            // single pass over the leaf text (only `&` and `<` need escaping in text
+            // content; `\u00a0` is normalized to a regular space)
+            return o.replace(/[&<\u00a0]/g, escape_text_char);
         }
         if (Array.isArray(o)) {
             var s = '';
@@ -249,19 +275,25 @@ export class SyntaxStyler {
             }
             return s;
         }
+        var content = this.stringify_token(o.content, lang);
+        // build the class list once; aliases are always an array after normalization
+        var classes = `token_${o.type}`;
+        for (const a of o.alias) {
+            classes += ` token_${a}`;
+        }
+        // fast path: with no `wrap` hooks the tag is always a plain <span> with no
+        // attributes, so skip the per-token context object and hook dispatch
+        if (this.hooks_wrap.length === 0) {
+            return '<span class="' + classes + '">' + content + '</span>';
+        }
         var ctx = {
             type: o.type,
-            content: this.stringify_token(o.content, lang),
+            content,
             tag: 'span',
-            classes: [`token_${o.type}`],
+            classes: classes.split(' '),
             attributes: {},
             lang,
         };
-        var aliases = o.alias;
-        // alias is always an array after normalization
-        for (const a of aliases) {
-            ctx.classes.push(`token_${a}`);
-        }
         this.run_hook_wrap(ctx);
         var attributes = '';
         for (var name in ctx.attributes) {
@@ -294,9 +326,9 @@ export class SyntaxStyler {
      * Therefore, it is encouraged to order overwriting tokens according to the positions of the overwritten tokens.
      * Furthermore, all non-overwriting tokens should be placed after the overwriting ones.
      *
-     * @param base_id - The id of the language to extend. This has to be a key in `syntax_styler.langs`.
-     * @param extension - The new tokens to append.
-     * @returns the new grammar
+     * @param base_id - the id of the language to extend, must be a key in `syntax_styler.langs`
+     * @param extension - the new tokens to append
+     * @returns the new `SyntaxGrammar`
      */
     extend_grammar(base_id, extension) {
         // Merge normalized base with un-normalized extension
@@ -307,7 +339,7 @@ export class SyntaxStyler {
         return extended;
     }
     /**
-     * Normalize a single pattern to have consistent shape.
+     * Normalizes a single pattern to have consistent shape.
      * This ensures all patterns have the same object shape for V8 optimization.
      */
     #normalize_pattern(pattern, visited) {
@@ -339,15 +371,15 @@ export class SyntaxStyler {
         };
     }
     /**
-     * Normalize a grammar to have consistent object shapes.
+     * Normalizes a grammar to have consistent object shapes.
      * This performs several optimizations:
-     * 1. Merges `rest` property into main grammar
-     * 2. Ensures all pattern values are arrays
-     * 3. Normalizes all pattern objects to have consistent shapes
-     * 4. Adds global flag to greedy patterns
+     * 1. Merges `rest` property into main grammar.
+     * 2. Ensures all pattern values are arrays.
+     * 3. Normalizes all pattern objects to have consistent shapes.
+     * 4. Adds global flag to greedy patterns.
      *
      * This is called once at registration time to avoid runtime overhead.
-     * @param visited - Set of grammar object IDs already normalized (for circular references)
+     * @param visited - set of grammar object IDs already normalized (for circular references)
      */
     #normalize_grammar(grammar, visited) {
         // Check if we've already normalized this grammar (circular reference)

package/dist/syntax_token.d.ts

@@ -2,13 +2,13 @@ export declare class SyntaxToken {
     /**
      * The type of the token.
      *
-     * This is usually the key of a pattern in a `Grammar`.
+     * This is usually the key of a pattern in a `SyntaxGrammar`.
      */
     type: string;
     /**
      * The strings or tokens contained by this token.
      *
-     * This will be a token stream if the pattern matched also defined an `inside` grammar.
+     * This will be a `SyntaxTokenStream` if the pattern matched also defined an `inside` grammar.
      */
     content: string | SyntaxTokenStream;
     /**
@@ -22,8 +22,8 @@ export declare class SyntaxToken {
 /**
  * A token stream is an array of strings and `SyntaxToken` objects.
  *
- * Syntax token streams have to fulfill a few properties that are assumed by most functions (mostly internal ones) that process
- * them.
+ * `SyntaxTokenStream` values have to fulfill a few properties that are assumed by most functions
+ * (mostly internal ones) that process them.
  *
  * 1. No adjacent strings.
  * 2. No empty strings.

package/dist/syntax_token.js

@@ -2,13 +2,13 @@ export class SyntaxToken {
     /**
      * The type of the token.
      *
-     * This is usually the key of a pattern in a `Grammar`.
+     * This is usually the key of a pattern in a `SyntaxGrammar`.
      */
     type;
     /**
      * The strings or tokens contained by this token.
      *
-     * This will be a token stream if the pattern matched also defined an `inside` grammar.
+     * This will be a `SyntaxTokenStream` if the pattern matched also defined an `inside` grammar.
      */
     content;
     /**

package/dist/tokenize_syntax.d.ts

@@ -9,11 +9,9 @@ import { type SyntaxTokenStream } from './syntax_token.js';
  * This method could be useful in other contexts as well, as a very crude parser.
  *
  * @param text - a string with the code to be styled
- * @param grammar - an object containing the tokens to use
- *
+ * @param grammar - a `SyntaxGrammar` object containing the tokens to use.
  * Usually a language definition like `syntax_styler.get_lang('markup')`.
- *
- * @returns an array of strings and tokens, a token stream
+ * @returns a `SyntaxTokenStream` array of strings and tokens
  *
  * @example
  * ```ts

package/dist/tokenize_syntax.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"tokenize_syntax.d.ts","sourceRoot":"../src/lib/","sources":["../src/lib/tokenize_syntax.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,aAAa,EAAC,MAAM,oBAAoB,CAAC;AACtD,OAAO,EAAc,KAAK,iBAAiB,EAAC,MAAM,mBAAmB,CAAC;AAEtE;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,eAAO,MAAM,eAAe,GAAI,MAAM,MAAM,EAAE,SAAS,aAAa,KAAG,iBAQtE,CAAC"}
+{"version":3,"file":"tokenize_syntax.d.ts","sourceRoot":"../src/lib/","sources":["../src/lib/tokenize_syntax.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,aAAa,EAAC,MAAM,oBAAoB,CAAC;AACtD,OAAO,EAAc,KAAK,iBAAiB,EAAC,MAAM,mBAAmB,CAAC;AAEtE;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,eAAO,MAAM,eAAe,GAAI,MAAM,MAAM,EAAE,SAAS,aAAa,KAAG,iBAQtE,CAAC"}

package/dist/tokenize_syntax.js

@@ -8,11 +8,9 @@ import { SyntaxToken } from './syntax_token.js';
  * This method could be useful in other contexts as well, as a very crude parser.
  *
  * @param text - a string with the code to be styled
- * @param grammar - an object containing the tokens to use
- *
+ * @param grammar - a `SyntaxGrammar` object containing the tokens to use.
  * Usually a language definition like `syntax_styler.get_lang('markup')`.
- *
- * @returns an array of strings and tokens, a token stream
+ * @returns a `SyntaxTokenStream` array of strings and tokens
  *
  * @example
  * ```ts

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fuzdev/fuz_code",
-  "version": "0.45.1",
+  "version": "0.46.1",
   "description": "syntax styling utilities and components for TypeScript, Svelte, Markdown, and more",
   "glyph": "🎨",
   "logo": "logo.svg",
@@ -24,48 +24,46 @@
     "preview": "vite preview",
     "deploy": "gro deploy",
     "benchmark": "gro run benchmark/run_benchmarks.ts",
-    "benchmark:compare": "gro run benchmark/compare/run_compare.ts",
+    "benchmark:save": "gro run benchmark/run_benchmarks.ts --save",
+    "benchmark:clean": "rm -f benchmark/baseline.json",
+    "benchmark:vs": "gro run benchmark/compare/run_compare.ts",
+    "benchmark:vs:write": "gro run benchmark/compare/run_compare.ts --write",
     "fixtures:update": "gro src/test/fixtures/update"
   },
   "type": "module",
   "engines": {
-    "node": ">=22.15"
+    "node": ">=24.14"
   },
   "peerDependencies": {
-    "@fuzdev/fuz_css": ">=0.47.0",
-    "@fuzdev/fuz_util": ">=0.49.2",
+    "@fuzdev/fuz_css": ">=0.62.0",
+    "@fuzdev/fuz_util": ">=0.65.1",
     "esm-env": "^1",
-    "magic-string": "^0.30",
-    "svelte": "^5",
-    "zimmerframe": "^1"
+    "svelte": "^5"
   },
   "peerDependenciesMeta": {
     "@fuzdev/fuz_css": {
       "optional": true
     },
-    "@fuzdev/fuz_util": {
-      "optional": true
-    },
-    "magic-string": {
-      "optional": true
-    },
     "svelte": {
       "optional": true
-    },
-    "zimmerframe": {
-      "optional": true
     }
   },
+  "dependencies": {
+    "magic-string": "^0.30.21",
+    "zimmerframe": "^1.1.4"
+  },
   "devDependencies": {
     "@changesets/changelog-git": "^0.2.1",
-    "@fuzdev/fuz_css": "^0.50.0",
-    "@fuzdev/fuz_ui": "^0.183.1",
-    "@fuzdev/fuz_util": "^0.51.0",
-    "@fuzdev/gro": "^0.194.0",
-    "@jridgewell/trace-mapping": "^0.3.31",
-    "@ryanatkn/eslint-config": "^0.9.0",
+    "@fuzdev/blake3_wasm": "^0.1.1",
+    "@fuzdev/fuz_css": "^0.63.2",
+    "@fuzdev/fuz_ui": "^0.205.0",
+    "@fuzdev/fuz_util": "^0.65.1",
+    "@fuzdev/gro": "^0.204.0",
+    "@fuzdev/mdz": "^0.1.0",
+    "@ryanatkn/eslint-config": "^0.12.1",
+    "@sveltejs/acorn-typescript": "^1.0.9",
     "@sveltejs/adapter-static": "^3.0.10",
-    "@sveltejs/kit": "^2.50.1",
+    "@sveltejs/kit": "^2.63.0",
     "@sveltejs/package": "^2.5.7",
     "@sveltejs/vite-plugin-svelte": "^6.2.4",
     "@types/estree": "^1.0.8",
@@ -74,17 +72,17 @@
     "eslint": "^9.39.1",
     "eslint-plugin-svelte": "^3.13.1",
     "esm-env": "^1.2.2",
-    "magic-string": "^0.30.21",
     "prettier": "^3.7.4",
     "prettier-plugin-svelte": "^3.4.1",
-    "svelte": "^5.49.1",
-    "svelte-check": "^4.3.6",
-    "svelte2tsx": "^0.7.47",
+    "svelte": "^5.56.2",
+    "svelte-check": "^4.6.0",
+    "svelte-docinfo": "^0.5.1",
+    "svelte2tsx": "^0.7.55",
     "tslib": "^2.8.1",
     "typescript": "^5.9.3",
     "typescript-eslint": "^8.48.1",
+    "vite": "^7.3.1",
     "vitest": "^4.0.15",
-    "zimmerframe": "^1.1.4",
     "zod": "^4.3.6"
   },
   "prettier": {

package/src/lib/grammar_markdown.ts

@@ -16,7 +16,7 @@ interface FenceType {
 }
 
 /**
- * Helper to create fenced code block pattern for a language
+ * Helper to create fenced code block pattern for a language.
  */
 const create_fence_pattern = (
 	backticks: string,
@@ -48,7 +48,7 @@ const create_fence_pattern = (
 };
 
 /**
- * Helper to create catch-all fence pattern (unknown languages)
+ * Helper to create catch-all fence pattern (unknown languages).
  */
 const create_catchall_fence = (backticks: string): SyntaxGrammarTokenRaw => {
 	const pattern = new RegExp(`^${backticks}[^\\n\\r]*(?:\\r?\\n|\\r)[\\s\\S]*?^${backticks}$`, 'm');
@@ -67,7 +67,7 @@ const create_catchall_fence = (backticks: string): SyntaxGrammarTokenRaw => {
 };
 
 /**
- * Helper to create md self-reference placeholder pattern
+ * Helper to create md self-reference placeholder pattern.
  */
 const create_md_placeholder = (backticks: string): SyntaxGrammarTokenRaw => {
 	const pattern = new RegExp(

package/src/lib/grammar_markup.ts

@@ -98,9 +98,10 @@ export const add_grammar_markup: AddSyntaxGrammar = (syntax_styler) => {
  *
  * An example of an inlined language is CSS with `<style>` tags.
  *
- * @param tag_name - The name of the tag that contains the inlined language. This name will be treated as
- * case insensitive.
- * @param lang - The language key.
+ * @param syntax_styler - the `SyntaxStyler` instance to modify
+ * @param tag_name - the name of the tag that contains the inlined language, treated as case insensitive
+ * @param lang - the language key
+ * @param inside_lang - the language to insert into, defaults to `'markup'`
  */
 export const grammar_markup_add_inlined = (
 	syntax_styler: SyntaxStyler,
@@ -145,13 +146,13 @@ export const grammar_markup_add_inlined = (
 };
 
 /**
- * Adds an pattern to style languages embedded in HTML attributes.
+ * Adds a pattern to style languages embedded in HTML attributes.
  *
  * An example of an inlined language is CSS with `style` attributes.
  *
- * @param attr_name - The name of the tag that contains the inlined language. This name will be treated as
- * case insensitive.
- * @param lang - The language key.
+ * @param syntax_styler - the `SyntaxStyler` instance to modify
+ * @param attr_name - the name of the attribute that contains the inlined language, treated as case insensitive
+ * @param lang - the language key
  */
 export const grammar_markup_add_attribute = (
 	syntax_styler: SyntaxStyler,