@heartlandone/vega 2.75.0 → 2.76.0

package/dist/collection/polyfill/prism/test/tokenizer.test.js

@@ -0,0 +1,180 @@
+import { tokenize } from '../tokenizer';
+describe('Syntax Highlight Tokenizer', () => {
+  describe('tokenize', () => {
+    it('should return the original string when grammar is not provided', () => {
+      const result = tokenize('hello world', undefined);
+      expect(result).toEqual(['hello world']);
+    });
+    it('should return the original string when code is empty', () => {
+      const result = tokenize('', { keyword: { pattern: /\bvar\b/ } });
+      expect(result).toEqual(['']);
+    });
+    it('should tokenize a simple keyword', () => {
+      const grammar = {
+        keyword: { pattern: /\bvar\b/ },
+      };
+      const result = tokenize('var x = 1', grammar);
+      expect(result.length).toBe(2);
+      expect(result[0].type).toBe('keyword');
+      expect(result[0].content).toBe('var');
+      expect(result[1]).toBe(' x = 1');
+    });
+    it('should tokenize multiple keywords in a string', () => {
+      const grammar = {
+        keyword: { pattern: /\b(?:var|let|const)\b/ },
+      };
+      const result = tokenize('var x; let y; const z;', grammar);
+      const keywords = result.filter((t) => typeof t !== 'string' && t.type === 'keyword');
+      expect(keywords.length).toBe(3);
+      expect(keywords[0].content).toBe('var');
+      expect(keywords[1].content).toBe('let');
+      expect(keywords[2].content).toBe('const');
+    });
+    it('should handle multiple grammar rules', () => {
+      const grammar = {
+        keyword: { pattern: /\b(?:var|let|const)\b/ },
+        number: { pattern: /\b\d+\b/ },
+      };
+      const result = tokenize('var x = 42', grammar);
+      const tokens = result.filter((t) => typeof t !== 'string');
+      expect(tokens.length).toBe(2);
+      expect(tokens[0].type).toBe('keyword');
+      expect(tokens[0].content).toBe('var');
+      expect(tokens[1].type).toBe('number');
+      expect(tokens[1].content).toBe('42');
+    });
+    it('should handle greedy matching', () => {
+      const grammar = {
+        string: {
+          pattern: /(["'])(?:\\[\s\S]|(?!\1)[^\\\n\r])*\1/,
+          greedy: true,
+        },
+      };
+      const result = tokenize('"hello world"', grammar);
+      expect(result.length).toBe(1);
+      expect(result[0].type).toBe('string');
+      expect(result[0].content).toBe('"hello world"');
+    });
+    it('should handle lookbehind patterns', () => {
+      const grammar = {
+        function: {
+          pattern: /(\b)(\w+)(?=\s*\()/,
+          lookbehind: true,
+        },
+      };
+      const result = tokenize('call foo()', grammar);
+      const funcTokens = result.filter((t) => typeof t !== 'string' && t.type === 'function');
+      expect(funcTokens.length).toBeGreaterThanOrEqual(1);
+    });
+    it('should handle array of patterns for a single token type', () => {
+      const grammar = {
+        comment: [
+          { pattern: /\/\/.*/, greedy: true },
+          { pattern: /\/\*[\s\S]*?\*\//, greedy: true },
+        ],
+      };
+      const singleLineResult = tokenize('x = 1; // comment', grammar);
+      const multiLineResult = tokenize('x = 1; /* block */', grammar);
+      const singleComments = singleLineResult.filter((t) => typeof t !== 'string' && t.type === 'comment');
+      const multiComments = multiLineResult.filter((t) => typeof t !== 'string' && t.type === 'comment');
+      expect(singleComments.length).toBe(1);
+      expect(multiComments.length).toBe(1);
+    });
+    it('should handle nested grammar via inside property', () => {
+      const grammar = {
+        'template-string': {
+          pattern: /`[^`]*`/,
+          greedy: true,
+          inside: {
+            'template-punctuation': { pattern: /^`|`$/ },
+            'string': { pattern: /[\s\S]+/ },
+          },
+        },
+      };
+      const result = tokenize('`hello`', grammar);
+      expect(result.length).toBe(1);
+      const templateToken = result[0];
+      expect(templateToken.type).toBe('template-string');
+      expect(Array.isArray(templateToken.content)).toBe(true);
+    });
+    it('should preserve unmatched text as plain strings', () => {
+      const grammar = {
+        keyword: { pattern: /\bvar\b/ },
+      };
+      const result = tokenize('var x = hello', grammar);
+      const strings = result.filter((t) => typeof t === 'string');
+      expect(strings.length).toBe(1);
+      expect(strings[0]).toBe(' x = hello');
+    });
+    it('should handle code with no matches', () => {
+      const grammar = {
+        keyword: { pattern: /\bxyz\b/ },
+      };
+      const result = tokenize('hello world', grammar);
+      expect(result).toEqual(['hello world']);
+    });
+    it('greedy string patterns should prevent keywords inside strings from being tokenized', () => {
+      const grammar = {
+        string: {
+          pattern: /(["'])(?:\\[\s\S]|(?!\1)[^\\\n\r])*\1/,
+          greedy: true,
+        },
+        keyword: { pattern: /\b(?:var|let|const|function|return)\b/ },
+      };
+      const code = 'var x = "function return let"';
+      const result = tokenize(code, grammar);
+      const keywords = result.filter((t) => typeof t !== 'string' && t.type === 'keyword');
+      const strings = result.filter((t) => typeof t !== 'string' && t.type === 'string');
+      // Only 'var' should be a keyword; keywords inside the string should NOT be tokenized
+      expect(keywords.length).toBe(1);
+      expect(keywords[0].content).toBe('var');
+      expect(strings.length).toBe(1);
+      expect(strings[0].content).toBe('"function return let"');
+    });
+    it('greedy comment patterns should prevent tokens inside comments from being tokenized', () => {
+      const grammar = {
+        comment: {
+          pattern: /\/\*[\s\S]*?\*\//,
+          greedy: true,
+        },
+        keyword: { pattern: /\b(?:var|let|const)\b/ },
+        number: { pattern: /\b\d+\b/ },
+      };
+      const code = 'var x = 1; /* let y = 42 */';
+      const result = tokenize(code, grammar);
+      const keywords = result.filter((t) => typeof t !== 'string' && t.type === 'keyword');
+      const numbers = result.filter((t) => typeof t !== 'string' && t.type === 'number');
+      const comments = result.filter((t) => typeof t !== 'string' && t.type === 'comment');
+      expect(keywords.length).toBe(1);
+      expect(keywords[0].content).toBe('var');
+      expect(numbers.length).toBe(1);
+      expect(numbers[0].content).toBe('1');
+      expect(comments.length).toBe(1);
+      expect(comments[0].content).toBe('/* let y = 42 */');
+    });
+    it('multiple greedy patterns should not interfere with each other', () => {
+      const grammar = {
+        string: {
+          pattern: /(["'])(?:\\[\s\S]|(?!\1)[^\\\n\r])*\1/,
+          greedy: true,
+        },
+        comment: {
+          pattern: /\/\/.*|\/\*[\s\S]*?\*\//,
+          greedy: true,
+        },
+        keyword: { pattern: /\b(?:var|let|const)\b/ },
+      };
+      const code = 'var x = "hello"; // let y';
+      const result = tokenize(code, grammar);
+      const keywords = result.filter((t) => typeof t !== 'string' && t.type === 'keyword');
+      const strings = result.filter((t) => typeof t !== 'string' && t.type === 'string');
+      const comments = result.filter((t) => typeof t !== 'string' && t.type === 'comment');
+      expect(keywords.length).toBe(1);
+      expect(keywords[0].content).toBe('var');
+      expect(strings.length).toBe(1);
+      expect(strings[0].content).toBe('"hello"');
+      expect(comments.length).toBe(1);
+      expect(comments[0].content).toBe('// let y');
+    });
+  });
+});

package/dist/collection/polyfill/prism/test/tsx.test.js

@@ -0,0 +1,87 @@
+import { tokenize } from '../tokenizer';
+import { tsxGrammar } from '../languages/tsx';
+import { findTokensByType } from './test-utils';
+describe('TSX Language Grammar', () => {
+  it('should tokenize JSX tags in TSX', () => {
+    const code = '<div>Hello</div>';
+    const tokens = tokenize(code, tsxGrammar);
+    const tags = findTokensByType(tokens, 'tag');
+    expect(tags.length).toBe(2);
+  });
+  it('should tokenize TypeScript keywords in TSX', () => {
+    const code = 'interface Props { name: string; }';
+    const tokens = tokenize(code, tsxGrammar);
+    const keywords = findTokensByType(tokens, 'keyword');
+    const interfaceKw = keywords.find((t) => t.content === 'interface');
+    expect(interfaceKw).toBeDefined();
+  });
+  it('should tokenize TypeScript builtin types in TSX', () => {
+    const code = 'const x: number = 42;';
+    const tokens = tokenize(code, tsxGrammar);
+    const builtins = findTokensByType(tokens, 'builtin');
+    expect(builtins.length).toBeGreaterThanOrEqual(1);
+  });
+  it('should tokenize a typed TSX component', () => {
+    const code = `const App: React.FC<Props> = ({ name }) => {
+  return <div>{name}</div>;
+};`;
+    const tokens = tokenize(code, tsxGrammar);
+    const keywords = findTokensByType(tokens, 'keyword');
+    const tags = findTokensByType(tokens, 'tag');
+    expect(keywords.length).toBeGreaterThanOrEqual(2);
+    // 3 tags: <Props> (generic false positive), <div>, </div>
+    expect(tags.length).toBe(3);
+  });
+  it('should tokenize TSX with generics and JSX', () => {
+    const code = `function useHook<T>(initial: T): T {
+  return initial;
+}`;
+    const tokens = tokenize(code, tsxGrammar);
+    const keywords = findTokensByType(tokens, 'keyword');
+    expect(keywords.length).toBeGreaterThanOrEqual(2);
+  });
+  it('should tokenize enum in TSX', () => {
+    const code = 'enum Direction { Up, Down, Left, Right }';
+    const tokens = tokenize(code, tsxGrammar);
+    const keywords = findTokensByType(tokens, 'keyword');
+    const enumKw = keywords.find((t) => t.content === 'enum');
+    expect(enumKw).toBeDefined();
+  });
+  it('should tokenize function names before parentheses in TSX', () => {
+    const code = `interface MyComponentProps {
+      name: string;
+    }
+
+    function MyComponent({ name }: MyComponentProps) {
+      return (
+          <h1>Hello, {name}!</h1>
+      );
+    }`;
+    const tokens = tokenize(code, tsxGrammar);
+    const functions = findTokensByType(tokens, 'function');
+    const myComponent = functions.find((t) => t.content === 'MyComponent');
+    expect(myComponent).toBeDefined();
+  });
+  it('should tokenize complete TSX component with types', () => {
+    const code = `interface CardProps {
+  title: string;
+  count: number;
+}
+
+const Card: React.FC<CardProps> = ({ title, count }) => {
+  return (
+    <div className="card">
+      <h1>{title}</h1>
+      <span>{count}</span>
+    </div>
+  );
+};`;
+    const tokens = tokenize(code, tsxGrammar);
+    const keywords = findTokensByType(tokens, 'keyword');
+    const builtins = findTokensByType(tokens, 'builtin');
+    const tags = findTokensByType(tokens, 'tag');
+    expect(keywords.length).toBeGreaterThanOrEqual(2);
+    expect(builtins.length).toBeGreaterThanOrEqual(2);
+    expect(tags.length).toBeGreaterThan(0);
+  });
+});

package/dist/collection/polyfill/prism/test/typescript.test.js

@@ -0,0 +1,86 @@
+import { tokenize } from '../tokenizer';
+import { typescriptGrammar } from '../languages/typescript';
+import { findTokensByType } from './test-utils';
+describe('TypeScript Language Grammar', () => {
+  it('should tokenize TypeScript-specific keywords', () => {
+    const code = 'interface Foo { bar: string; }';
+    const tokens = tokenize(code, typescriptGrammar);
+    const keywords = findTokensByType(tokens, 'keyword');
+    const interfaceKw = keywords.find((t) => t.content === 'interface');
+    expect(interfaceKw).toBeDefined();
+  });
+  it('should tokenize type annotations with builtin types', () => {
+    const code = 'const x: number = 42;';
+    const tokens = tokenize(code, typescriptGrammar);
+    const builtins = findTokensByType(tokens, 'builtin');
+    expect(builtins.length).toBeGreaterThanOrEqual(1);
+  });
+  it('should tokenize enum declarations', () => {
+    const code = 'enum Color { Red, Green, Blue }';
+    const tokens = tokenize(code, typescriptGrammar);
+    const keywords = findTokensByType(tokens, 'keyword');
+    const enumKw = keywords.find((t) => t.content === 'enum');
+    expect(enumKw).toBeDefined();
+  });
+  it('should tokenize access modifiers', () => {
+    const code = 'class Foo { private bar: string; protected baz: number; public qux: boolean; }';
+    const tokens = tokenize(code, typescriptGrammar);
+    const keywords = findTokensByType(tokens, 'keyword');
+    const privateKw = keywords.find((t) => t.content === 'private');
+    const protectedKw = keywords.find((t) => t.content === 'protected');
+    const publicKw = keywords.find((t) => t.content === 'public');
+    expect(privateKw).toBeDefined();
+    expect(protectedKw).toBeDefined();
+    expect(publicKw).toBeDefined();
+  });
+  it('should tokenize readonly keyword', () => {
+    const code = 'readonly name: string;';
+    const tokens = tokenize(code, typescriptGrammar);
+    const keywords = findTokensByType(tokens, 'keyword');
+    const readonlyKw = keywords.find((t) => t.content === 'readonly');
+    expect(readonlyKw).toBeDefined();
+  });
+  it('should tokenize type keyword', () => {
+    const code = 'type Result = string | number;';
+    const tokens = tokenize(code, typescriptGrammar);
+    const keywords = findTokensByType(tokens, 'keyword');
+    const typeKw = keywords.find((t) => t.content === 'type');
+    expect(typeKw).toBeDefined();
+  });
+  it('should tokenize builtin types', () => {
+    const code = 'let a: string; let b: number; let c: boolean; let d: void;';
+    const tokens = tokenize(code, typescriptGrammar);
+    const builtins = findTokensByType(tokens, 'builtin');
+    expect(builtins.length).toBe(4);
+  });
+  it('should still tokenize JS-inherited features', () => {
+    const code = 'const fn = async () => { await fetch(url); };';
+    const tokens = tokenize(code, typescriptGrammar);
+    const keywords = findTokensByType(tokens, 'keyword');
+    const constKw = keywords.find((t) => t.content === 'const');
+    const asyncKw = keywords.find((t) => t.content === 'async');
+    const awaitKw = keywords.find((t) => t.content === 'await');
+    expect(constKw).toBeDefined();
+    expect(asyncKw).toBeDefined();
+    expect(awaitKw).toBeDefined();
+  });
+  it('should tokenize a complete TypeScript interface', () => {
+    const code = `interface User {
+  name: string;
+  age: number;
+  isActive: boolean;
+}`;
+    const tokens = tokenize(code, typescriptGrammar);
+    const keywords = findTokensByType(tokens, 'keyword');
+    const builtins = findTokensByType(tokens, 'builtin');
+    expect(keywords.length).toBeGreaterThanOrEqual(1);
+    expect(builtins.length).toBe(3);
+  });
+  it('should tokenize namespace keyword', () => {
+    const code = 'namespace MyNamespace { }';
+    const tokens = tokenize(code, typescriptGrammar);
+    const keywords = findTokensByType(tokens, 'keyword');
+    const namespaceKw = keywords.find((t) => t.content === 'namespace');
+    expect(namespaceKw).toBeDefined();
+  });
+});

package/dist/collection/polyfill/prism/tokenizer.js

@@ -0,0 +1,242 @@
+/**
+ * Unique placeholder used to mark greedy-matched regions so they are not
+ * re-tokenized by later grammar rules.  The surrounding curly-brace pairs
+ * make an accidental collision with real source code extremely unlikely.
+ */
+const GREEDY_PLACEHOLDER = '\x00GREEDY_PLACEHOLDER\x00';
+/**
+ * Core tokenizer that converts source code into syntax tokens based on a grammar definition.
+ * This is a local replacement for Prism tokenize.
+ *
+ * The tokenizer works by:
+ * 1. Pre-processing greedy patterns to claim their matched regions first
+ * 2. Iterating through all grammar rules in order
+ * 3. Finding regex matches in the source text
+ * 4. Splitting the text into matched (typed) and unmatched (plain) segments
+ * 5. Recursively tokenizing nested content when an `inside` grammar is specified
+ *
+ * @param {string} code - The source code string to tokenize.
+ * @param {Grammar} grammar - The grammar rules to apply for tokenization.
+ * @returns {(string | SyntaxToken)[]} An array of plain strings and syntax tokens.
+ */
+export function tokenize(code, grammar) {
+  if (!code || !grammar) {
+    return [code];
+  }
+  const tokens = [code];
+  matchGrammar(tokens, grammar);
+  return tokens;
+}
+/**
+ * Apply grammar rules to tokenize string entries in the token list.
+ * Greedy patterns are applied first: their matched regions are replaced with
+ * opaque placeholder tokens so that subsequent (non-greedy) rules cannot
+ * match inside those regions.  After all rules have run the placeholders are
+ * restored to the original matched text.
+ *
+ * @param {(string | SyntaxToken)[]} tokens - The current token list to process in-place.
+ * @param {Grammar} grammar - The grammar rules to apply.
+ */
+function matchGrammar(tokens, grammar) {
+  const greedyEntries = [];
+  const nonGreedyEntries = [];
+  for (const tokenType of Object.keys(grammar)) {
+    const grammarEntry = grammar[tokenType];
+    const patterns = Array.isArray(grammarEntry) ? grammarEntry : [grammarEntry];
+    for (const patternDef of patterns) {
+      if (patternDef.greedy) {
+        greedyEntries.push({ tokenType, patternDef });
+      }
+      else {
+        nonGreedyEntries.push({ tokenType, patternDef });
+      }
+    }
+  }
+  // --- Phase 1: Apply greedy patterns first --------------------------------
+  // Greedy patterns claim regions of the source text exclusively.
+  // We replace matched regions with a placeholder string of equal length so
+  // that character offsets remain stable.  Each greedy match is recorded so
+  // we can later restore the original text inside the token.
+  if (greedyEntries.length > 0) {
+    applyGreedyPatterns(tokens, greedyEntries);
+  }
+  // --- Phase 2: Apply remaining non-greedy patterns ------------------------
+  for (const { tokenType, patternDef } of nonGreedyEntries) {
+    applyPattern(tokens, tokenType, patternDef);
+  }
+}
+/**
+ * Process all greedy patterns against the current token list.
+ *
+ * For every plain-string entry in the list the function builds a *mask* string
+ * of the same length.  Greedy matches are recorded and their corresponding
+ * character ranges in the mask are overwritten with a placeholder character so
+ * that later greedy patterns (and all non-greedy patterns) will never match
+ * inside an already-claimed region.
+ *
+ * After all greedy patterns have been collected the affected string entries are
+ * rebuilt by splicing in SyntaxToken objects for each greedy match.
+ *
+ * @param {(string | SyntaxToken)[]} tokens - The token list to process in-place.
+ * @param {{ tokenType: string; patternDef: GrammarPattern }[]} greedyEntries - Greedy patterns to apply.
+ */
+function applyGreedyPatterns(tokens, greedyEntries) {
+  for (let i = 0; i < tokens.length; i++) {
+    const token = tokens[i];
+    /* istanbul ignore next -- defensive: greedy phase only sees strings from initial tokenize() call */
+    if (typeof token !== 'string') {
+      continue;
+    }
+    // Build the full text by concatenating all consecutive string segments
+    // so greedy regex can match across them — but for simplicity we operate
+    // per-segment, which matches prismjs behavior for the patterns we use.
+    const originalText = token;
+    // Mask tracks which characters have been claimed.  A GREEDY_PLACEHOLDER
+    // character at position j means that character is already part of a greedy match.
+    let mask = originalText;
+    const matches = [];
+    for (const { tokenType, patternDef } of greedyEntries) {
+      const { pattern, lookbehind, inside } = patternDef;
+      const regex = new RegExp(pattern.source, pattern.flags.replace(/g/g, '') + 'g');
+      let execResult;
+      while ((execResult = regex.exec(mask)) !== null) {
+        let matchIndex = execResult.index;
+        let match = execResult[0];
+        /* istanbul ignore next -- defensive: prevents infinite loop on zero-length regex matches */
+        if (match.length === 0) {
+          regex.lastIndex++;
+          continue;
+        }
+        // If matched text contains a placeholder char it overlaps a
+        // previously claimed region — skip it.
+        /* istanbul ignore next -- defensive: prevents re-tokenizing already-claimed greedy regions */
+        if (match.indexOf(GREEDY_PLACEHOLDER) !== -1) {
+          continue;
+        }
+        if (lookbehind && execResult[1]) {
+          const lookbehindLength = execResult[1].length;
+          matchIndex += lookbehindLength;
+          match = match.slice(lookbehindLength);
+        }
+        // Record the match and blank out the region in the mask.
+        const matchEnd = matchIndex + match.length;
+        matches.push({
+          tokenType,
+          matchStart: matchIndex,
+          matchEnd,
+          matchText: originalText.slice(matchIndex, matchEnd),
+          inside,
+        });
+        // Replace matched region in the mask with placeholder so later
+        // patterns cannot match inside it.
+        mask =
+          mask.slice(0, matchIndex) +
+            GREEDY_PLACEHOLDER.repeat(match.length).slice(0, match.length) +
+            mask.slice(matchEnd);
+      }
+    }
+    if (matches.length === 0) {
+      continue;
+    }
+    // Sort matches by start position for deterministic splicing.
+    matches.sort((a, b) => a.matchStart - b.matchStart);
+    // Build replacement array.
+    const replacement = [];
+    let lastIndex = 0;
+    for (const m of matches) {
+      if (m.matchStart > lastIndex) {
+        replacement.push(originalText.slice(lastIndex, m.matchStart));
+      }
+      let content = m.matchText;
+      if (m.inside) {
+        const nested = tokenize(m.matchText, m.inside);
+        content = nested.length === 1 ? nested[0] : nested;
+      }
+      replacement.push({ type: m.tokenType, content });
+      lastIndex = m.matchEnd;
+    }
+    if (lastIndex < originalText.length) {
+      replacement.push(originalText.slice(lastIndex));
+    }
+    tokens.splice(i, 1, ...replacement);
+    i += replacement.length - 1;
+  }
+}
+/**
+ * Apply a single pattern definition across all string entries in the token list.
+ * For each string entry, finds ALL matches of the pattern and splits the string
+ * into alternating unmatched/matched segments in one pass.
+ *
+ * @param {(string | SyntaxToken)[]} tokens - The token list to process.
+ * @param {string} tokenType - The token type name for matched content.
+ * @param {GrammarPattern} patternDef - The pattern definition to apply.
+ */
+function applyPattern(tokens, tokenType, patternDef) {
+  for (let i = 0; i < tokens.length; i++) {
+    const token = tokens[i];
+    // Only process string tokens, skip already-tokenized entries
+    if (typeof token !== 'string') {
+      continue;
+    }
+    const replacement = tokenizeStringByPattern(token, tokenType, patternDef);
+    // Only splice if matches were found (replacement differs from original)
+    if (replacement.length === 1 && typeof replacement[0] === 'string') {
+      continue;
+    }
+    tokens.splice(i, 1, ...replacement);
+    // Skip past the entire replacement — all matches in this string are handled
+    i += replacement.length - 1;
+  }
+}
+/**
+ * Find all matches of a pattern within a string and return the split result
+ * as an array of alternating plain strings and syntax tokens.
+ *
+ * @param {string} text - The string to tokenize.
+ * @param {string} tokenType - The token type name for matched content.
+ * @param {GrammarPattern} patternDef - The pattern definition to apply.
+ * @returns {(string | SyntaxToken)[]} The tokenized result.
+ */
+function tokenizeStringByPattern(text, tokenType, patternDef) {
+  const { pattern, lookbehind, inside } = patternDef;
+  const regex = new RegExp(pattern.source, pattern.flags.replace(/g/g, '') + 'g');
+  const result = [];
+  let lastIndex = 0;
+  let execResult;
+  while ((execResult = regex.exec(text)) !== null) {
+    let matchIndex = execResult.index;
+    let match = execResult[0];
+    // Prevent zero-length match infinite loops
+    if (match.length === 0) {
+      regex.lastIndex++;
+      continue;
+    }
+    // Handle lookbehind: adjust match position and content
+    if (lookbehind && execResult[1]) {
+      const lookbehindLength = execResult[1].length;
+      matchIndex += lookbehindLength;
+      match = match.slice(lookbehindLength);
+    }
+    // Add unmatched text before this match
+    if (matchIndex > lastIndex) {
+      result.push(text.slice(lastIndex, matchIndex));
+    }
+    // Process nested grammar if specified
+    let content = match;
+    if (inside) {
+      const nestedTokens = tokenize(match, inside);
+      content = nestedTokens.length === 1 ? nestedTokens[0] : nestedTokens;
+    }
+    result.push({ type: tokenType, content });
+    lastIndex = matchIndex + match.length;
+  }
+  // Add remaining unmatched text
+  if (lastIndex < text.length) {
+    result.push(text.slice(lastIndex));
+  }
+  /* istanbul ignore next -- defensive: tokenizeStringByPattern always has remainder text */
+  if (result.length === 0) {
+    return [text];
+  }
+  return result;
+}

package/dist/collection/polyfill/prism/types.js

@@ -0,0 +1 @@
+export {};