@shikijs/engine-javascript 1.25.0 → 1.26.0

package/dist/engine-compile.d.mts

@@ -0,0 +1,4 @@
+import '@shikijs/types';
+import 'oniguruma-to-es';
+export { J as JavaScriptRegexEngineOptions, c as createJavaScriptRegexEngine, d as defaultJavaScriptRegexConstructor } from './shared/engine-javascript.DCvMc5Xx.mjs';
+import '@shikijs/vscode-textmate';

package/dist/engine-compile.d.ts

@@ -0,0 +1,4 @@
+import '@shikijs/types';
+import 'oniguruma-to-es';
+export { J as JavaScriptRegexEngineOptions, c as createJavaScriptRegexEngine, d as defaultJavaScriptRegexConstructor } from './shared/engine-javascript.DCvMc5Xx.js';
+import '@shikijs/vscode-textmate';

package/dist/engine-compile.mjs

@@ -0,0 +1,46 @@
+import { toRegExp } from 'oniguruma-to-es';
+import { J as JavaScriptScanner } from './shared/engine-javascript.hzpS1_41.mjs';
+
+function defaultJavaScriptRegexConstructor(pattern, options) {
+  return toRegExp(
+    pattern,
+    {
+      global: true,
+      hasIndices: true,
+      rules: {
+        // Needed since TextMate grammars merge backrefs across patterns
+        allowOrphanBackrefs: true,
+        // Improves search performance for generated regexes
+        asciiWordBoundaries: true,
+        // Follow `vscode-oniguruma` which enables this Oniguruma option by default
+        captureGroup: true,
+        // Removing `\G` anchors in cases when they're not supported for emulation allows
+        // supporting more grammars, but also allows some mismatches
+        ignoreUnsupportedGAnchors: true
+      },
+      ...options
+    }
+  );
+}
+function createJavaScriptRegexEngine(options = {}) {
+  const _options = Object.assign(
+    {
+      target: "auto",
+      cache: /* @__PURE__ */ new Map()
+    },
+    options
+  );
+  _options.regexConstructor ||= (pattern) => defaultJavaScriptRegexConstructor(pattern, { target: _options.target });
+  return {
+    createScanner(patterns) {
+      return new JavaScriptScanner(patterns, _options);
+    },
+    createString(s) {
+      return {
+        content: s
+      };
+    }
+  };
+}
+
+export { createJavaScriptRegexEngine, defaultJavaScriptRegexConstructor };

package/dist/engine-raw.d.mts

@@ -0,0 +1,12 @@
+import { RegexEngine } from '@shikijs/types';
+
+/**
+ * Raw JavaScript regex engine that only supports precompiled grammars.
+ *
+ * This further simplifies the engine by excluding the regex compilation step.
+ *
+ * Zero dependencies.
+ */
+declare function createJavaScriptRawEngine(): RegexEngine;
+
+export { createJavaScriptRawEngine };

package/dist/engine-raw.d.ts

@@ -0,0 +1,12 @@
+import { RegexEngine } from '@shikijs/types';
+
+/**
+ * Raw JavaScript regex engine that only supports precompiled grammars.
+ *
+ * This further simplifies the engine by excluding the regex compilation step.
+ *
+ * Zero dependencies.
+ */
+declare function createJavaScriptRawEngine(): RegexEngine;
+
+export { createJavaScriptRawEngine };

package/dist/engine-raw.mjs

@@ -0,0 +1,22 @@
+import { J as JavaScriptScanner } from './shared/engine-javascript.hzpS1_41.mjs';
+
+function createJavaScriptRawEngine() {
+  const options = {
+    cache: /* @__PURE__ */ new Map(),
+    regexConstructor: () => {
+      throw new Error("JavaScriptRawEngine: only support precompiled grammar");
+    }
+  };
+  return {
+    createScanner(patterns) {
+      return new JavaScriptScanner(patterns, options);
+    },
+    createString(s) {
+      return {
+        content: s
+      };
+    }
+  };
+}
+
+export { createJavaScriptRawEngine };

package/dist/index.d.mts

@@ -1,60 +1,5 @@
-import { PatternScanner, RegexEngineString, RegexEngine } from '@shikijs/types';
-import { IOnigMatch } from '@shikijs/vscode-textmate';
-import { OnigurumaToEsOptions } from 'oniguruma-to-es';
-
-interface JavaScriptRegexEngineOptions {
-    /**
-     * Whether to allow invalid regex patterns.
-     *
-     * @default false
-     */
-    forgiving?: boolean;
-    /**
-     * The target ECMAScript version.
-     *
-     * Oniguruma-To-ES uses RegExp features from later versions of ECMAScript to provide improved
-     * accuracy and add support for more grammars. If using target `ES2024` or later, the RegExp `v`
-     * flag is used which requires Node.js 20+ or Chrome 112+.
-     * @see https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/RegExp/unicodeSets
-     *
-     * For maximum compatibility, you can set it to `ES2018` which uses the RegExp `u` flag but
-     * supports a few less grammars.
-     *
-     * Set to `auto` to automatically detect the latest version supported by the environment.
-     *
-     * @default 'auto'
-     */
-    target?: 'auto' | 'ES2025' | 'ES2024' | 'ES2018';
-    /**
-     * Cache for regex patterns.
-     */
-    cache?: Map<string, RegExp | Error> | null;
-    /**
-     * Custom pattern to RegExp constructor.
-     *
-     * By default `oniguruma-to-es` is used.
-     */
-    regexConstructor?: (pattern: string) => RegExp;
-}
-/**
- * The default RegExp constructor for JavaScript regex engine.
- */
-declare function defaultJavaScriptRegexConstructor(pattern: string, options?: OnigurumaToEsOptions): RegExp;
-declare class JavaScriptScanner implements PatternScanner {
-    patterns: string[];
-    options: JavaScriptRegexEngineOptions;
-    regexps: (RegExp | null)[];
-    constructor(patterns: string[], options?: JavaScriptRegexEngineOptions);
-    findNextMatchSync(string: string | RegexEngineString, startPosition: number, _options: number): IOnigMatch | null;
-}
-/**
- * Use the modern JavaScript RegExp engine to implement the OnigScanner.
- *
- * As Oniguruma supports some features that can't be emulated using native JavaScript regexes, some
- * patterns are not supported. Errors will be thrown when parsing TextMate grammars with
- * unsupported patterns, and when the grammar includes patterns that use invalid Oniguruma syntax.
- * Set `forgiving` to `true` to ignore these errors and skip any unsupported or invalid patterns.
- */
-declare function createJavaScriptRegexEngine(options?: JavaScriptRegexEngineOptions): RegexEngine;
-
-export { type JavaScriptRegexEngineOptions, JavaScriptScanner, createJavaScriptRegexEngine, defaultJavaScriptRegexConstructor };
+export { J as JavaScriptRegexEngineOptions, a as JavaScriptRegexScannerOptions, b as JavaScriptScanner, c as createJavaScriptRegexEngine, d as defaultJavaScriptRegexConstructor } from './shared/engine-javascript.DCvMc5Xx.mjs';
+export { createJavaScriptRawEngine } from './engine-raw.mjs';
+import '@shikijs/types';
+import 'oniguruma-to-es';
+import '@shikijs/vscode-textmate';

package/dist/index.d.ts

@@ -1,60 +1,5 @@
-import { PatternScanner, RegexEngineString, RegexEngine } from '@shikijs/types';
-import { IOnigMatch } from '@shikijs/vscode-textmate';
-import { OnigurumaToEsOptions } from 'oniguruma-to-es';
-
-interface JavaScriptRegexEngineOptions {
-    /**
-     * Whether to allow invalid regex patterns.
-     *
-     * @default false
-     */
-    forgiving?: boolean;
-    /**
-     * The target ECMAScript version.
-     *
-     * Oniguruma-To-ES uses RegExp features from later versions of ECMAScript to provide improved
-     * accuracy and add support for more grammars. If using target `ES2024` or later, the RegExp `v`
-     * flag is used which requires Node.js 20+ or Chrome 112+.
-     * @see https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/RegExp/unicodeSets
-     *
-     * For maximum compatibility, you can set it to `ES2018` which uses the RegExp `u` flag but
-     * supports a few less grammars.
-     *
-     * Set to `auto` to automatically detect the latest version supported by the environment.
-     *
-     * @default 'auto'
-     */
-    target?: 'auto' | 'ES2025' | 'ES2024' | 'ES2018';
-    /**
-     * Cache for regex patterns.
-     */
-    cache?: Map<string, RegExp | Error> | null;
-    /**
-     * Custom pattern to RegExp constructor.
-     *
-     * By default `oniguruma-to-es` is used.
-     */
-    regexConstructor?: (pattern: string) => RegExp;
-}
-/**
- * The default RegExp constructor for JavaScript regex engine.
- */
-declare function defaultJavaScriptRegexConstructor(pattern: string, options?: OnigurumaToEsOptions): RegExp;
-declare class JavaScriptScanner implements PatternScanner {
-    patterns: string[];
-    options: JavaScriptRegexEngineOptions;
-    regexps: (RegExp | null)[];
-    constructor(patterns: string[], options?: JavaScriptRegexEngineOptions);
-    findNextMatchSync(string: string | RegexEngineString, startPosition: number, _options: number): IOnigMatch | null;
-}
-/**
- * Use the modern JavaScript RegExp engine to implement the OnigScanner.
- *
- * As Oniguruma supports some features that can't be emulated using native JavaScript regexes, some
- * patterns are not supported. Errors will be thrown when parsing TextMate grammars with
- * unsupported patterns, and when the grammar includes patterns that use invalid Oniguruma syntax.
- * Set `forgiving` to `true` to ignore these errors and skip any unsupported or invalid patterns.
- */
-declare function createJavaScriptRegexEngine(options?: JavaScriptRegexEngineOptions): RegexEngine;
-
-export { type JavaScriptRegexEngineOptions, JavaScriptScanner, createJavaScriptRegexEngine, defaultJavaScriptRegexConstructor };
+export { J as JavaScriptRegexEngineOptions, a as JavaScriptRegexScannerOptions, b as JavaScriptScanner, c as createJavaScriptRegexEngine, d as defaultJavaScriptRegexConstructor } from './shared/engine-javascript.DCvMc5Xx.js';
+export { createJavaScriptRawEngine } from './engine-raw.js';
+import '@shikijs/types';
+import 'oniguruma-to-es';
+import '@shikijs/vscode-textmate';

package/dist/index.mjs

@@ -1,127 +1,4 @@
-import { toRegExp } from 'oniguruma-to-es';
-
-const MAX = 4294967295;
-function defaultJavaScriptRegexConstructor(pattern, options) {
-  return toRegExp(
-    pattern,
-    {
-      global: true,
-      hasIndices: true,
-      rules: {
-        // Needed since TextMate grammars merge backrefs across patterns
-        allowOrphanBackrefs: true,
-        // Improves search performance for generated regexes
-        asciiWordBoundaries: true,
-        // Follow `vscode-oniguruma` which enables this Oniguruma option by default
-        captureGroup: true,
-        // Removing `\G` anchors in cases when they're not supported for emulation allows
-        // supporting more grammars, but also allows some mismatches
-        ignoreUnsupportedGAnchors: true
-      },
-      ...options
-    }
-  );
-}
-class JavaScriptScanner {
-  constructor(patterns, options = {}) {
-    this.patterns = patterns;
-    this.options = options;
-    const {
-      forgiving = false,
-      cache,
-      target = "auto",
-      regexConstructor = (pattern) => defaultJavaScriptRegexConstructor(pattern, { target })
-    } = options;
-    this.regexps = patterns.map((p) => {
-      const cached = cache?.get(p);
-      if (cached) {
-        if (cached instanceof RegExp) {
-          return cached;
-        }
-        if (forgiving)
-          return null;
-        throw cached;
-      }
-      try {
-        const regex = regexConstructor(p);
-        cache?.set(p, regex);
-        return regex;
-      } catch (e) {
-        cache?.set(p, e);
-        if (forgiving)
-          return null;
-        throw e;
-      }
-    });
-  }
-  regexps;
-  findNextMatchSync(string, startPosition, _options) {
-    const str = typeof string === "string" ? string : string.content;
-    const pending = [];
-    function toResult(index, match, offset = 0) {
-      return {
-        index,
-        captureIndices: match.indices.map((indice) => {
-          if (indice == null) {
-            return {
-              start: MAX,
-              end: MAX,
-              length: 0
-            };
-          }
-          return {
-            start: indice[0] + offset,
-            end: indice[1] + offset,
-            length: indice[1] - indice[0]
-          };
-        })
-      };
-    }
-    for (let i = 0; i < this.regexps.length; i++) {
-      const regexp = this.regexps[i];
-      if (!regexp)
-        continue;
-      try {
-        regexp.lastIndex = startPosition;
-        const match = regexp.exec(str);
-        if (!match)
-          continue;
-        if (match.index === startPosition) {
-          return toResult(i, match, 0);
-        }
-        pending.push([i, match, 0]);
-      } catch (e) {
-        if (this.options.forgiving)
-          continue;
-        throw e;
-      }
-    }
-    if (pending.length) {
-      const minIndex = Math.min(...pending.map((m) => m[1].index));
-      for (const [i, match, offset] of pending) {
-        if (match.index === minIndex) {
-          return toResult(i, match, offset);
-        }
-      }
-    }
-    return null;
-  }
-}
-function createJavaScriptRegexEngine(options = {}) {
-  const _options = {
-    cache: /* @__PURE__ */ new Map(),
-    ...options
-  };
-  return {
-    createScanner(patterns) {
-      return new JavaScriptScanner(patterns, _options);
-    },
-    createString(s) {
-      return {
-        content: s
-      };
-    }
-  };
-}
-
-export { JavaScriptScanner, createJavaScriptRegexEngine, defaultJavaScriptRegexConstructor };
+export { createJavaScriptRegexEngine, defaultJavaScriptRegexConstructor } from './engine-compile.mjs';
+export { createJavaScriptRawEngine } from './engine-raw.mjs';
+export { J as JavaScriptScanner } from './shared/engine-javascript.hzpS1_41.mjs';
+import 'oniguruma-to-es';

package/dist/shared/engine-javascript.DCvMc5Xx.d.mts

@@ -0,0 +1,63 @@
+import { PatternScanner, RegexEngineString, RegexEngine } from '@shikijs/types';
+import { OnigurumaToEsOptions } from 'oniguruma-to-es';
+import { IOnigMatch } from '@shikijs/vscode-textmate';
+
+interface JavaScriptRegexScannerOptions {
+    /**
+     * Whether to allow invalid regex patterns.
+     *
+     * @default false
+     */
+    forgiving?: boolean;
+    /**
+     * Cache for regex patterns.
+     */
+    cache?: Map<string, RegExp | Error> | null;
+    /**
+     * Custom pattern to RegExp constructor.
+     *
+     * By default `oniguruma-to-es` is used.
+     */
+    regexConstructor?: (pattern: string) => RegExp;
+}
+declare class JavaScriptScanner implements PatternScanner {
+    patterns: (string | RegExp)[];
+    options: JavaScriptRegexScannerOptions;
+    regexps: (RegExp | null)[];
+    constructor(patterns: (string | RegExp)[], options?: JavaScriptRegexScannerOptions);
+    findNextMatchSync(string: string | RegexEngineString, startPosition: number, _options: number): IOnigMatch | null;
+}
+
+interface JavaScriptRegexEngineOptions extends JavaScriptRegexScannerOptions {
+    /**
+     * The target ECMAScript version.
+     *
+     * Oniguruma-To-ES uses RegExp features from later versions of ECMAScript to provide improved
+     * accuracy and add support for more grammars. If using target `ES2024` or later, the RegExp `v`
+     * flag is used which requires Node.js 20+ or Chrome 112+.
+     * @see https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/RegExp/unicodeSets
+     *
+     * For maximum compatibility, you can set it to `ES2018` which uses the RegExp `u` flag but
+     * supports a few less grammars.
+     *
+     * Set to `auto` to automatically detect the latest version supported by the environment.
+     *
+     * @default 'auto'
+     */
+    target?: 'auto' | 'ES2025' | 'ES2024' | 'ES2018';
+}
+/**
+ * The default RegExp constructor for JavaScript regex engine.
+ */
+declare function defaultJavaScriptRegexConstructor(pattern: string, options?: OnigurumaToEsOptions): RegExp;
+/**
+ * Use the modern JavaScript RegExp engine to implement the OnigScanner.
+ *
+ * As Oniguruma supports some features that can't be emulated using native JavaScript regexes, some
+ * patterns are not supported. Errors will be thrown when parsing TextMate grammars with
+ * unsupported patterns, and when the grammar includes patterns that use invalid Oniguruma syntax.
+ * Set `forgiving` to `true` to ignore these errors and skip any unsupported or invalid patterns.
+ */
+declare function createJavaScriptRegexEngine(options?: JavaScriptRegexEngineOptions): RegexEngine;
+
+export { type JavaScriptRegexEngineOptions as J, type JavaScriptRegexScannerOptions as a, JavaScriptScanner as b, createJavaScriptRegexEngine as c, defaultJavaScriptRegexConstructor as d };

package/dist/shared/engine-javascript.DCvMc5Xx.d.ts

@@ -0,0 +1,63 @@
+import { PatternScanner, RegexEngineString, RegexEngine } from '@shikijs/types';
+import { OnigurumaToEsOptions } from 'oniguruma-to-es';
+import { IOnigMatch } from '@shikijs/vscode-textmate';
+
+interface JavaScriptRegexScannerOptions {
+    /**
+     * Whether to allow invalid regex patterns.
+     *
+     * @default false
+     */
+    forgiving?: boolean;
+    /**
+     * Cache for regex patterns.
+     */
+    cache?: Map<string, RegExp | Error> | null;
+    /**
+     * Custom pattern to RegExp constructor.
+     *
+     * By default `oniguruma-to-es` is used.
+     */
+    regexConstructor?: (pattern: string) => RegExp;
+}
+declare class JavaScriptScanner implements PatternScanner {
+    patterns: (string | RegExp)[];
+    options: JavaScriptRegexScannerOptions;
+    regexps: (RegExp | null)[];
+    constructor(patterns: (string | RegExp)[], options?: JavaScriptRegexScannerOptions);
+    findNextMatchSync(string: string | RegexEngineString, startPosition: number, _options: number): IOnigMatch | null;
+}
+
+interface JavaScriptRegexEngineOptions extends JavaScriptRegexScannerOptions {
+    /**
+     * The target ECMAScript version.
+     *
+     * Oniguruma-To-ES uses RegExp features from later versions of ECMAScript to provide improved
+     * accuracy and add support for more grammars. If using target `ES2024` or later, the RegExp `v`
+     * flag is used which requires Node.js 20+ or Chrome 112+.
+     * @see https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/RegExp/unicodeSets
+     *
+     * For maximum compatibility, you can set it to `ES2018` which uses the RegExp `u` flag but
+     * supports a few less grammars.
+     *
+     * Set to `auto` to automatically detect the latest version supported by the environment.
+     *
+     * @default 'auto'
+     */
+    target?: 'auto' | 'ES2025' | 'ES2024' | 'ES2018';
+}
+/**
+ * The default RegExp constructor for JavaScript regex engine.
+ */
+declare function defaultJavaScriptRegexConstructor(pattern: string, options?: OnigurumaToEsOptions): RegExp;
+/**
+ * Use the modern JavaScript RegExp engine to implement the OnigScanner.
+ *
+ * As Oniguruma supports some features that can't be emulated using native JavaScript regexes, some
+ * patterns are not supported. Errors will be thrown when parsing TextMate grammars with
+ * unsupported patterns, and when the grammar includes patterns that use invalid Oniguruma syntax.
+ * Set `forgiving` to `true` to ignore these errors and skip any unsupported or invalid patterns.
+ */
+declare function createJavaScriptRegexEngine(options?: JavaScriptRegexEngineOptions): RegexEngine;
+
+export { type JavaScriptRegexEngineOptions as J, type JavaScriptRegexScannerOptions as a, JavaScriptScanner as b, createJavaScriptRegexEngine as c, defaultJavaScriptRegexConstructor as d };

package/dist/shared/engine-javascript.hzpS1_41.mjs

@@ -0,0 +1,93 @@
+const MAX = 4294967295;
+class JavaScriptScanner {
+  constructor(patterns, options = {}) {
+    this.patterns = patterns;
+    this.options = options;
+    const {
+      forgiving = false,
+      cache,
+      regexConstructor
+    } = options;
+    if (!regexConstructor) {
+      throw new Error("Option `regexConstructor` is not provided");
+    }
+    this.regexps = patterns.map((p) => {
+      if (typeof p !== "string") {
+        return p;
+      }
+      const cached = cache?.get(p);
+      if (cached) {
+        if (cached instanceof RegExp) {
+          return cached;
+        }
+        if (forgiving)
+          return null;
+        throw cached;
+      }
+      try {
+        const regex = regexConstructor(p);
+        cache?.set(p, regex);
+        return regex;
+      } catch (e) {
+        cache?.set(p, e);
+        if (forgiving)
+          return null;
+        throw e;
+      }
+    });
+  }
+  regexps;
+  findNextMatchSync(string, startPosition, _options) {
+    const str = typeof string === "string" ? string : string.content;
+    const pending = [];
+    function toResult(index, match, offset = 0) {
+      return {
+        index,
+        captureIndices: match.indices.map((indice) => {
+          if (indice == null) {
+            return {
+              start: MAX,
+              end: MAX,
+              length: 0
+            };
+          }
+          return {
+            start: indice[0] + offset,
+            end: indice[1] + offset,
+            length: indice[1] - indice[0]
+          };
+        })
+      };
+    }
+    for (let i = 0; i < this.regexps.length; i++) {
+      const regexp = this.regexps[i];
+      if (!regexp)
+        continue;
+      try {
+        regexp.lastIndex = startPosition;
+        const match = regexp.exec(str);
+        if (!match)
+          continue;
+        if (match.index === startPosition) {
+          return toResult(i, match, 0);
+        }
+        pending.push([i, match, 0]);
+      } catch (e) {
+        if (this.options.forgiving)
+          continue;
+        throw e;
+      }
+    }
+    if (pending.length) {
+      const minIndex = Math.min(...pending.map((m) => m[1].index));
+      for (const [i, match, offset] of pending) {
+        if (match.index === minIndex) {
+          return toResult(i, match, offset);
+        }
+      }
+    }
+    return null;
+  }
+}
+
+export { JavaScriptScanner as J };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@shikijs/engine-javascript",
   "type": "module",
-  "version": "1.25.0",
+  "version": "1.26.0",
   "description": "Engine for Shiki using JavaScript's native RegExp",
   "author": "Anthony Fu <anthonyfu117@hotmail.com>",
   "license": "MIT",
@@ -21,6 +21,10 @@
     ".": {
       "types": "./dist/index.d.mts",
       "default": "./dist/index.mjs"
+    },
+    "./raw": {
+      "types": "./dist/engine-raw.d.mts",
+      "default": "./dist/engine-raw.mjs"
     }
   },
   "main": "./dist/index.mjs",
@@ -30,9 +34,9 @@
     "dist"
   ],
   "dependencies": {
-    "@shikijs/vscode-textmate": "^9.3.1",
+    "@shikijs/vscode-textmate": "^10.0.1",
     "oniguruma-to-es": "0.10.0",
-    "@shikijs/types": "1.25.0"
+    "@shikijs/types": "1.26.0"
   },
   "scripts": {
     "build": "unbuild",