@fuzdev/fuz_code 0.37.0

package/dist/syntax_styler.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"syntax_styler.d.ts","sourceRoot":"../src/lib/","sources":["../src/lib/syntax_styler.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,WAAW,EAAE,KAAK,iBAAiB,EAAC,MAAM,mBAAmB,CAAC;AAGtE,MAAM,MAAM,gBAAgB,GAAG,CAAC,aAAa,EAAE,YAAY,KAAK,IAAI,CAAC;AAErE;;;;;;;GAOG;AACH,qBAAa,YAAY;IACxB,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,aAAa,GAAG,SAAS,CAAC,CAE9C;IAkBF,QAAQ,CAAC,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,gBAAgB,EAAE,OAAO,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,GAAG,IAAI;IAc9E,iBAAiB,CAChB,OAAO,EAAE,MAAM,EACf,YAAY,EAAE,MAAM,EACpB,SAAS,EAAE,gBAAgB,EAC3B,OAAO,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,GACrB,aAAa;IAahB,QAAQ,CAAC,EAAE,EAAE,MAAM,GAAG,aAAa;IAQnC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA4CG;IACH,OAAO,CACN,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,MAAM,EACZ,OAAO,GAAE,aAAa,GAAG,SAA+B,GACtD,MAAM;IAcT;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA0EG;IACH,qBAAqB,CACpB,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,gBAAgB,EACxB,IAAI,GAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAc,GACpC,aAAa;IAmChB;;;;;;;;OAQG;IACH,eAAe,CAAC,CAAC,EAAE,MAAM,GAAG,WAAW,GAAG,iBAAiB,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM;IAoDlF;;;;;;;;;;;;;;;;;;OAkBG;IACH,cAAc,CAAC,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,gBAAgB,GAAG,aAAa;IAS3E;;;OAGG;IACH,OAAO,CAAC,iBAAiB;IAqCzB;;;;;;;;;;OAUG;IACH,OAAO,CAAC,iBAAiB;IAoCzB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAM;IAGlC,qBAAqB,EAAE,KAAK,CAAC,0BAA0B,CAAC,CAAM;IAC9D,oBAAoB,EAAE,KAAK,CAAC,yBAAyB,CAAC,CAAM;IAC5D,UAAU,EAAE,KAAK,CAAC,gBAAgB,CAAC,CAAM;IAEzC,wBAAwB,CAAC,EAAE,EAAE,0BAA0B,GAAG,IAAI;IAG9D,uBAAuB,CAAC,EAAE,EAAE,yBAAyB,GAAG,IAAI;IAG5D,aAAa,CAAC,EAAE,EAAE,gBAAgB,GAAG,IAAI;IAIzC,wBAAwB,CAAC,GAAG,EAAE,iCAAiC,GAAG,IAAI;IAKtE,uBAAuB,CAAC,GAAG,EAAE,gCAAgC,GAAG,IAAI;IAKpE,aAAa,CAAC,GAAG,EAAE,uBAAuB,GAAG,IAAI;CAKjD;AAED,MAAM,MAAM,qBAAqB,GAC9B,MAAM,GACN,qBAAqB,GACrB,KAAK,CAAC,MAAM,GAAG,qBAAqB,CAAC,CAAC;AAEzC,MAAM,MAAM,gBAAgB,GAAG,MAAM,CAAC,MAAM,EAAE,qBAAqB,GAAG,SAAS,CAAC,GAAG;IAClF,IAAI,CAAC,EAAE,gBAAgB,GAAG,SAAS,CAAC;CACpC,CAAC;AAEF;;;;;;;;;;;;GAYG;AACH,MAAM,WAAW,qBAAqB;IACrC;;OAEG;IACH,OAAO,EAAE,MAAM,CAAC;IAChB;;;;OAIG;IACH,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB;;;OAGG;IACH,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC;IAC/B;;OAEG;IACH,MAAM,CAAC,EAAE,gBAAgB,GAAG,IAAI,CAAC;CACjC;AAED;;;GAGG;AACH,MAAM,WAAW,kBAAkB;IAClC,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,OAAO,CAAC;IACpB,MAAM,EAAE,OAAO,CAAC;IAChB,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACrB,MAAM,EAAE,aAAa,GAAG,IAAI,CAAC;CAC7B;AAED;;;GAGG;AACH,MAAM,MAAM,aAAa,GAAG,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,kBAAkB,CAAC,CAAC,CAAC;AAwBtE,MAAM,MAAM,0BAA0B,GAAG,CAAC,GAAG,EAAE,iCAAiC,KAAK,IAAI,CAAC;AAC1F,MAAM,MAAM,yBAAyB,GAAG,CAAC,GAAG,EAAE,gCAAgC,KAAK,IAAI,CAAC;AACxF,MAAM,MAAM,gBAAgB,GAAG,CAAC,GAAG,EAAE,uBAAuB,KAAK,IAAI,CAAC;AAEtE,MAAM,WAAW,iCAAiC;IACjD,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,aAAa,CAAC;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,SAAS,CAAC;CAClB;AACD,MAAM,WAAW,gCAAgC;IAChD,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,aAAa,CAAC;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,iBAAiB,CAAC;CAC1B;AACD,MAAM,WAAW,uBAAuB;IACvC,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACvB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACnC,IAAI,EAAE,MAAM,CAAC;CACb"}

package/dist/syntax_styler.js

@@ -0,0 +1,426 @@
+import { SyntaxToken } from './syntax_token.js';
+import { tokenize_syntax } from './tokenize_syntax.js';
+/**
+ * Based on Prism (https://github.com/PrismJS/prism)
+ * by Lea Verou (https://lea.verou.me/)
+ *
+ * MIT license
+ *
+ * @see LICENSE
+ */
+export class SyntaxStyler {
+    langs = {
+        plaintext: {},
+    };
+    // constructor() {
+    // TODO this API? problem is the grammars rely on mutating existing grammars in the `syntax_styler`,
+    // so for now adding grammars will remain inherently stateful
+    // export interface SyntaxStylerOptions {
+    // 	grammars?: AddGrammar[];
+    // }
+    // options: SyntaxStylerOptions = {}
+    // const {grammars} = options;
+    // if (grammars) {
+    // 	for (const add_grammar of grammars) {
+    // this.langs[id] =
+    // 		add_grammar(this);
+    // 	}
+    // }
+    // }
+    add_lang(id, grammar, aliases) {
+        // Normalize grammar once at registration for optimal runtime performance
+        // Use a visited set to handle circular references
+        this.normalize_grammar(grammar, new Set());
+        // After normalization, grammar has the shape of SyntaxGrammar
+        const normalized = grammar;
+        this.langs[id] = normalized;
+        if (aliases !== undefined) {
+            for (var alias of aliases) {
+                this.langs[alias] = normalized;
+            }
+        }
+    }
+    add_extended_lang(base_id, extension_id, extension, aliases) {
+        // extend_grammar returns already normalized grammar
+        var grammar = this.extend_grammar(base_id, extension);
+        // Store the normalized grammar directly
+        this.langs[extension_id] = grammar;
+        if (aliases !== undefined) {
+            for (var alias of aliases) {
+                this.langs[alias] = grammar;
+            }
+        }
+        return grammar;
+    }
+    get_lang(id) {
+        var lang = this.langs[id];
+        if (lang === undefined) {
+            throw Error(`The language "${id}" has no grammar.`);
+        }
+        return lang;
+    }
+    /**
+     * Generates HTML with syntax highlighting from source code.
+     *
+     * **Process:**
+     * 1. Runs `before_tokenize` hook
+     * 2. Tokenizes code using the provided or looked-up grammar
+     * 3. Runs `after_tokenize` hook
+     * 4. Runs `wrap` hook on each token
+     * 5. Converts tokens to HTML with CSS classes
+     *
+     * **Parameter Relationship:**
+     * - `lang` is ALWAYS required for hook context and identification
+     * - `grammar` is optional; when undefined, automatically looks up via `this.get_lang(lang)`
+     * - When both are provided, `grammar` is used for tokenization, `lang` for metadata
+     *
+     * **Use cases:**
+     * - Standard usage: `stylize(code, 'ts')` - uses registered TypeScript grammar
+     * - Custom grammar: `stylize(code, 'ts', customGrammar)` - uses custom grammar but keeps 'ts' label
+     * - Extended grammar: `stylize(code, 'custom', this.extend_grammar('ts', extension))` - new language variant
+     *
+     * @param text - The source code to syntax highlight.
+     * @param lang - Language identifier (e.g., 'ts', 'css', 'html'). Used for:
+     *   - Grammar lookup when `grammar` is undefined
+     *   - Hook context (`lang` field passed to hooks)
+     *   - Language identification in output
+     * @param grammar - Optional custom grammar object. When undefined, automatically
+     *   looks up the grammar via `this.get_lang(lang)`. Provide this to use a custom
+     *   or modified grammar instead of the registered one.
+     *
+     * @returns HTML string with syntax highlighting using CSS classes (`.token_*`)
+     *
+     * @example
+     * // Standard usage - uses registered grammar
+     * stylize('var foo = true;', 'ts');
+     *
+     * @example
+     * // Custom grammar - overrides registered grammar
+     * const customGrammar = { keyword: [...], string: [...] };
+     * stylize('var foo = false;', 'ts', customGrammar);
+     *
+     * @example
+     * // Extended grammar - builds on existing grammar
+     * const extended = this.extend_grammar('ts', { customToken: [...] });
+     * stylize('var foo = 42;', 'ts-extended', extended);
+     */
+    stylize(text, lang, grammar = this.get_lang(lang)) {
+        var ctx = {
+            code: text,
+            grammar,
+            lang,
+            tokens: undefined,
+        };
+        this.run_hook_before_tokenize(ctx);
+        const c = ctx;
+        c.tokens = tokenize_syntax(c.code, c.grammar);
+        this.run_hook_after_tokenize(c);
+        return this.stringify_token(c.tokens, c.lang);
+    }
+    /**
+     * Inserts tokens _before_ another token in a language definition or any other grammar.
+     *
+     * ## Usage
+     *
+     * This helper method makes it easy to modify existing languages. For example, the CSS language definition
+     * not only defines CSS styling for CSS documents, but also needs to define styling for CSS embedded
+     * in HTML through `<style>` elements. To do this, it needs to modify `syntax_styler.get_lang('markup')` and add the
+     * appropriate tokens. However, `syntax_styler.get_lang('markup')` is a regular JS object literal, so if you do
+     * this:
+     *
+     * ```js
+     * syntax_styler.get_lang('markup').style = {
+     *     // token
+     * };
+     * ```
+     *
+     * then the `style` token will be added (and processed) at the end. `insert_before` allows you to insert tokens
+     * before existing tokens. For the CSS example above, you would use it like this:
+     *
+     * ```js
+     * grammar_insert_before('markup', 'cdata', {
+     *     'style': {
+     *         // token
+     *     }
+     * });
+     * ```
+     *
+     * ## Special cases
+     *
+     * If the grammars of `inside` and `insert` have tokens with the same name, the tokens in `inside`'s grammar
+     * will be ignored.
+     *
+     * This behavior can be used to insert tokens after `before`:
+     *
+     * ```js
+     * grammar_insert_before('markup', 'comment', {
+     *     'comment': syntax_styler.get_lang('markup').comment,
+     *     // tokens after 'comment'
+     * });
+     * ```
+     *
+     * ## Limitations
+     *
+     * The main problem `insert_before` has to solve is iteration order. Since ES2015, the iteration order for object
+     * properties is guaranteed to be the insertion order (except for integer keys) but some browsers behave
+     * differently when keys are deleted and re-inserted. So `insert_before` can't be implemented by temporarily
+     * deleting properties which is necessary to insert at arbitrary positions.
+     *
+     * To solve this problem, `insert_before` doesn't actually insert the given tokens into the target object.
+     * Instead, it will create a new object and replace all references to the target object with the new one. This
+     * can be done without temporarily deleting properties, so the iteration order is well-defined.
+     *
+     * However, only references that can be reached from `syntax_styler.langs` or `insert` will be replaced. I.e. if
+     * you hold the target object in a variable, then the value of the variable will not change.
+     *
+     * ```js
+     * var oldMarkup = syntax_styler.get_lang('markup');
+     * var newMarkup = grammar_insert_before('markup', 'comment', { ... });
+     *
+     * assert(oldMarkup !== syntax_styler.get_lang('markup'));
+     * assert(newMarkup === syntax_styler.get_lang('markup'));
+     * ```
+     *
+     * @param inside - The property of `root` (e.g. a language id in `syntax_styler.langs`) that contains the
+     * object to be modified.
+     * @param before - The key to insert before.
+     * @param insert - An object containing the key-value pairs to be inserted.
+     * @param root - The object containing `inside`, i.e. the object that contains the
+     * object to be modified.
+     *
+     * Defaults to `syntax_styler.langs`.
+     *
+     * @returns the new grammar object
+     */
+    grammar_insert_before(inside, before, insert, root = this.langs) {
+        var grammar = root[inside];
+        var updated = {};
+        for (var token in grammar) {
+            if (token === before) {
+                for (var new_token in insert) {
+                    updated[new_token] = insert[new_token];
+                }
+            }
+            // Do not insert tokens which also occur in insert.
+            if (!Object.hasOwn(insert, token)) {
+                updated[token] = grammar[token];
+            }
+        }
+        // Normalize the updated grammar to ensure inserted patterns have consistent shape
+        this.normalize_grammar(updated, new Set());
+        // After normalization, cast to SyntaxGrammar
+        const normalized = updated;
+        var old = root[inside];
+        root[inside] = normalized;
+        // Update references in other language definitions
+        depth_first_search(this.langs, (o, key, value) => {
+            if (value === old && key !== inside) {
+                o[key] = normalized;
+            }
+        });
+        return normalized;
+    }
+    /**
+     * Converts the given token or token stream to an HTML representation.
+     *
+     * Runs the `wrap` hook on each `SyntaxToken`.
+     *
+     * @param o - The token or token stream to be converted.
+     * @param lang - The name of current language.
+     * @returns The HTML representation of the token or token stream.
+     */
+    stringify_token(o, lang) {
+        if (typeof o === 'string') {
+            return o
+                .replace(/&/g, '&amp;')
+                .replace(/</g, '&lt;')
+                .replace(/\u00a0/g, ' ');
+        }
+        if (Array.isArray(o)) {
+            var s = '';
+            for (var e of o) {
+                s += this.stringify_token(e, lang);
+            }
+            return s;
+        }
+        var ctx = {
+            type: o.type,
+            content: this.stringify_token(o.content, lang),
+            tag: 'span',
+            classes: [`token_${o.type}`],
+            attributes: {},
+            lang,
+        };
+        var aliases = o.alias;
+        // alias is always an array after normalization
+        for (const a of aliases) {
+            ctx.classes.push(`token_${a}`);
+        }
+        this.run_hook_wrap(ctx);
+        var attributes = '';
+        for (var name in ctx.attributes) {
+            attributes += ' ' + name + '="' + (ctx.attributes[name] || '').replace(/"/g, '&quot;') + '"';
+        }
+        return ('<' +
+            ctx.tag +
+            ' class="' +
+            ctx.classes.join(' ') +
+            '"' +
+            attributes +
+            '>' +
+            ctx.content +
+            '</' +
+            ctx.tag +
+            '>');
+    }
+    /**
+     * Creates a deep copy of the language with the given id and appends the given tokens.
+     *
+     * If a token in `extension` also appears in the copied language, then the existing token in the copied language
+     * will be overwritten at its original position.
+     *
+     * ## Best practices
+     *
+     * Since the position of overwriting tokens (token in `extension` that overwrite tokens in the copied language)
+     * doesn't matter, they can technically be in any order. However, this can be confusing to others that trying to
+     * understand the language definition because, normally, the order of tokens matters in the grammars.
+     *
+     * Therefore, it is encouraged to order overwriting tokens according to the positions of the overwritten tokens.
+     * Furthermore, all non-overwriting tokens should be placed after the overwriting ones.
+     *
+     * @param base_id - The id of the language to extend. This has to be a key in `syntax_styler.langs`.
+     * @param extension - The new tokens to append.
+     * @returns the new grammar
+     */
+    extend_grammar(base_id, extension) {
+        // Merge normalized base with un-normalized extension
+        const extended = { ...structuredClone(this.get_lang(base_id)), ...extension };
+        // Normalize the extension parts
+        this.normalize_grammar(extended, new Set());
+        // Return as SyntaxGrammar
+        return extended;
+    }
+    /**
+     * Normalize a single pattern to have consistent shape.
+     * This ensures all patterns have the same object shape for V8 optimization.
+     */
+    normalize_pattern(pattern, visited) {
+        const p = pattern instanceof RegExp ? { pattern } : pattern;
+        let regex = p.pattern;
+        // Add global flag if greedy and not already present
+        if ((p.greedy ?? false) && !regex.global) {
+            const flags = regex.flags;
+            regex = new RegExp(regex.source, flags.includes('g') ? flags : flags + 'g');
+        }
+        // Normalize alias to always be an array
+        let normalized_alias = [];
+        if (p.alias) {
+            normalized_alias = Array.isArray(p.alias) ? p.alias : [p.alias];
+        }
+        // Recursively normalize the inside grammar if present
+        let normalized_inside = null;
+        if (p.inside) {
+            this.normalize_grammar(p.inside, visited);
+            // After normalization, cast to SyntaxGrammar
+            normalized_inside = p.inside;
+        }
+        return {
+            pattern: regex,
+            lookbehind: p.lookbehind ?? false,
+            greedy: p.greedy ?? false,
+            alias: normalized_alias,
+            inside: normalized_inside,
+        };
+    }
+    /**
+     * Normalize a grammar to have consistent object shapes.
+     * This performs several optimizations:
+     * 1. Merges `rest` property into main grammar
+     * 2. Ensures all pattern values are arrays
+     * 3. Normalizes all pattern objects to have consistent shapes
+     * 4. Adds global flag to greedy patterns
+     *
+     * This is called once at registration time to avoid runtime overhead.
+     * @param visited - Set of grammar object IDs already normalized (for circular references)
+     */
+    normalize_grammar(grammar, visited) {
+        // Check if we've already normalized this grammar (circular reference)
+        const grammar_id = id_of(grammar);
+        if (visited.has(grammar_id)) {
+            return;
+        }
+        visited.add(grammar_id);
+        // Step 1: Merge rest into grammar first
+        if (grammar.rest) {
+            for (const token in grammar.rest) {
+                if (!grammar[token]) {
+                    // Don't overwrite existing tokens
+                    grammar[token] = grammar.rest[token];
+                }
+            }
+            delete grammar.rest;
+        }
+        // Step 2: Normalize all patterns
+        for (const key in grammar) {
+            if (key === 'rest')
+                continue;
+            const value = grammar[key];
+            if (!value) {
+                grammar[key] = [];
+                continue;
+            }
+            // Always store as array of normalized patterns
+            const patterns = Array.isArray(value) ? value : [value];
+            grammar[key] = patterns.map((p) => this.normalize_pattern(p, visited));
+        }
+    }
+    // TODO add some builtins
+    plugins = {};
+    // TODO maybe extend/compose an event listener?
+    hooks_before_tokenize = [];
+    hooks_after_tokenize = [];
+    hooks_wrap = [];
+    add_hook_before_tokenize(cb) {
+        this.hooks_before_tokenize.push(cb);
+    }
+    add_hook_after_tokenize(cb) {
+        this.hooks_after_tokenize.push(cb);
+    }
+    add_hook_wrap(cb) {
+        this.hooks_wrap.push(cb);
+    }
+    run_hook_before_tokenize(ctx) {
+        for (var cb of this.hooks_before_tokenize) {
+            cb(ctx);
+        }
+    }
+    run_hook_after_tokenize(ctx) {
+        for (var cb of this.hooks_after_tokenize) {
+            cb(ctx);
+        }
+    }
+    run_hook_wrap(ctx) {
+        for (var cb of this.hooks_wrap) {
+            cb(ctx);
+        }
+    }
+}
+const depth_first_search = (o, cb, visited = new Set()) => {
+    for (var key in o) {
+        cb(o, key, o[key]);
+        var property = o[key];
+        if (property &&
+            typeof property === 'object' &&
+            !(property instanceof RegExp) &&
+            !visited.has(id_of(property))) {
+            visited.add(id_of(property));
+            depth_first_search(property, cb, visited);
+        }
+    }
+};
+var unique_id = 0;
+/**
+ * Returns a unique number for the given object. Later calls will still return the same number.
+ */
+const ID = Symbol('id');
+const id_of = (obj) => (obj[ID] ??= ++unique_id);

package/dist/syntax_styler_global.d.ts

@@ -0,0 +1,3 @@
+import { SyntaxStyler } from './syntax_styler.js';
+export declare const syntax_styler_global: SyntaxStyler;
+//# sourceMappingURL=syntax_styler_global.d.ts.map

package/dist/syntax_styler_global.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"syntax_styler_global.d.ts","sourceRoot":"../src/lib/","sources":["../src/lib/syntax_styler_global.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAC;AAUhD,eAAO,MAAM,oBAAoB,cAAqB,CAAC"}

package/dist/syntax_styler_global.js

@@ -0,0 +1,18 @@
+import { SyntaxStyler } from './syntax_styler.js';
+import { add_grammar_markup } from './grammar_markup.js';
+import { add_grammar_css } from './grammar_css.js';
+import { add_grammar_clike } from './grammar_clike.js';
+import { add_grammar_js } from './grammar_js.js';
+import { add_grammar_ts } from './grammar_ts.js';
+import { add_grammar_svelte } from './grammar_svelte.js';
+import { add_grammar_json } from './grammar_json.js';
+import { add_grammar_markdown } from './grammar_markdown.js';
+export const syntax_styler_global = new SyntaxStyler();
+add_grammar_markup(syntax_styler_global);
+add_grammar_css(syntax_styler_global);
+add_grammar_clike(syntax_styler_global);
+add_grammar_js(syntax_styler_global);
+add_grammar_ts(syntax_styler_global);
+add_grammar_svelte(syntax_styler_global);
+add_grammar_json(syntax_styler_global);
+add_grammar_markdown(syntax_styler_global);

package/dist/syntax_token.d.ts

@@ -0,0 +1,34 @@
+export declare class SyntaxToken {
+    /**
+     * The type of the token.
+     *
+     * This is usually the key of a pattern in a `Grammar`.
+     */
+    type: string;
+    /**
+     * The strings or tokens contained by this token.
+     *
+     * This will be a token stream if the pattern matched also defined an `inside` grammar.
+     */
+    content: string | SyntaxTokenStream;
+    /**
+     * The alias(es) of the token.
+     * Always an array, even if empty or single value.
+     */
+    alias: Array<string>;
+    length: number;
+    constructor(type: string, content: string | SyntaxTokenStream, alias: string | Array<string> | undefined, matched_str?: string);
+}
+/**
+ * A token stream is an array of strings and `SyntaxToken` objects.
+ *
+ * Syntax token streams have to fulfill a few properties that are assumed by most functions (mostly internal ones) that process
+ * them.
+ *
+ * 1. No adjacent strings.
+ * 2. No empty strings.
+ *
+ *    The only exception here is the token stream that only contains the empty string and nothing else.
+ */
+export type SyntaxTokenStream = Array<string | SyntaxToken>;
+//# sourceMappingURL=syntax_token.d.ts.map

package/dist/syntax_token.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"syntax_token.d.ts","sourceRoot":"../src/lib/","sources":["../src/lib/syntax_token.ts"],"names":[],"mappings":"AAAA,qBAAa,WAAW;IACvB;;;;OAIG;IACH,IAAI,EAAE,MAAM,CAAC;IAEb;;;;OAIG;IACH,OAAO,EAAE,MAAM,GAAG,iBAAiB,CAAC;IAEpC;;;OAGG;IACH,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IAErB,MAAM,EAAE,MAAM,CAAC;gBAGd,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,MAAM,GAAG,iBAAiB,EACnC,KAAK,EAAE,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,GAAG,SAAS,EACzC,WAAW,GAAE,MAAW;CAQzB;AAED;;;;;;;;;;GAUG;AACH,MAAM,MAAM,iBAAiB,GAAG,KAAK,CAAC,MAAM,GAAG,WAAW,CAAC,CAAC"}

package/dist/syntax_token.js

@@ -0,0 +1,27 @@
+export class SyntaxToken {
+    /**
+     * The type of the token.
+     *
+     * This is usually the key of a pattern in a `Grammar`.
+     */
+    type;
+    /**
+     * The strings or tokens contained by this token.
+     *
+     * This will be a token stream if the pattern matched also defined an `inside` grammar.
+     */
+    content;
+    /**
+     * The alias(es) of the token.
+     * Always an array, even if empty or single value.
+     */
+    alias;
+    length;
+    constructor(type, content, alias, matched_str = '') {
+        this.type = type;
+        this.content = content;
+        // Normalize alias to always be an array
+        this.alias = alias ? (Array.isArray(alias) ? alias : [alias]) : [];
+        this.length = matched_str.length;
+    }
+}

package/dist/theme.css

@@ -0,0 +1,98 @@
+.token_processing_instruction,
+.token_doctype,
+.token_cdata,
+.token_punctuation {
+	color: var(--text_color_5);
+}
+
+.token_tag,
+.token_constant,
+.token_symbol,
+.token_deleted,
+.token_keyword,
+.token_null,
+.token_boolean,
+.token_interpolation_punctuation,
+.token_heading,
+.token_heading_punctuation,
+.token_tag_punctuation {
+	color: var(--color_a_5);
+}
+
+.token_comment,
+.token_char,
+.token_inserted,
+.token_blockquote,
+.token_blockquote_punctuation {
+	color: var(--color_b_5);
+}
+
+.token_builtin,
+.token_class_name,
+.token_number {
+	color: var(--color_j_5);
+}
+
+.token_attr_value,
+.token_attr_quote,
+.token_string,
+.token_template_punctuation,
+.token_inline_code,
+.token_code_punctuation {
+	color: var(--color_h_5);
+}
+
+/* attr_equals must be after attr_value */
+.token_attr_equals {
+	color: var(--text_color_5);
+}
+
+.token_selector,
+.token_function,
+.token_regex,
+.token_important,
+.token_variable {
+	color: var(--color_e_5);
+}
+
+.token_atrule {
+	color: var(--color_f_5);
+}
+
+.token_attr_name,
+.token_property,
+.token_decorator,
+.token_decorator_name,
+.token_link_text_wrapper,
+.token_link_text,
+.token_link_punctuation {
+	color: var(--color_i_5);
+}
+
+.token_special_keyword,
+.token_namespace,
+.token_rule {
+	color: var(--color_g_5);
+}
+
+.token_at_keyword,
+.token_url {
+	color: var(--color_d_5);
+}
+
+.token_url {
+	text-decoration: underline;
+}
+
+.token_strikethrough {
+	text-decoration: line-through;
+}
+
+.token_important,
+.token_bold {
+	font-weight: bold;
+}
+
+.token_italic {
+	font-style: italic;
+}