@shikijs/engine-javascript 1.22.1 → 1.23.0

package/README.md

@@ -1,6 +1,6 @@
 # @shikijs/engine-javascript
 
-Engine for Shiki using JavaScript's native RegExp (experimental).
+Engine for Shiki using JavaScript's native RegExp (experimental). Uses [Oniguruma-To-ES](https://github.com/slevithan/oniguruma-to-es) to transpile regex syntax and behavior.
 
 [Documentation](https://shiki.style/guide/regex-engines)
 

package/dist/index.d.mts

@@ -1,5 +1,6 @@
 import { PatternScanner, RegexEngineString, RegexEngine } from '@shikijs/types';
 import { IOnigMatch } from '@shikijs/vscode-textmate';
+import { Options } from 'oniguruma-to-es';
 
 interface JavaScriptRegexEngineOptions {
     /**
@@ -9,11 +10,25 @@ interface JavaScriptRegexEngineOptions {
      */
     forgiving?: boolean;
     /**
-     * Use JavaScript to simulate some unsupported regex features.
+     * Cleanup some grammar patterns before use.
      *
      * @default true
      */
     simulation?: boolean;
+    /**
+     * The target ECMAScript version.
+     *
+     * For the best accuracy, Oniguruma-to-ES needs the `v` flag support in RegExp which is landed in ES2024.
+     * Which requires Node.js 20+ or Chrome 112+.
+     * @see https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/RegExp/unicodeSets
+     *
+     * For the maximum compatibility, you can set it to `ES2018`. Which will use the `u` flag to simulate and will be less accurate.
+     *
+     * Set to `auto` to detect the target version automatically.
+     *
+     * @default 'auto'
+     */
+    target?: 'ES2024' | 'ES2025' | 'ES2018' | 'auto';
     /**
      * Cache for regex patterns.
      */
@@ -21,28 +36,28 @@ interface JavaScriptRegexEngineOptions {
     /**
      * Custom pattern to RegExp constructor.
      *
-     * By default `oniguruma-to-js` is used.
+     * By default `oniguruma-to-es` is used.
      */
     regexConstructor?: (pattern: string) => RegExp;
 }
 /**
  * The default RegExp constructor for JavaScript regex engine.
  */
-declare function defaultJavaScriptRegexConstructor(pattern: string): RegExp;
+declare function defaultJavaScriptRegexConstructor(pattern: string, options?: Options): RegExp;
 declare class JavaScriptScanner implements PatternScanner {
     patterns: string[];
     options: JavaScriptRegexEngineOptions;
     regexps: (RegExp | null)[];
-    contiguousAnchorSimulation: boolean[];
     constructor(patterns: string[], options?: JavaScriptRegexEngineOptions);
     findNextMatchSync(string: string | RegexEngineString, startPosition: number): IOnigMatch | null;
 }
 /**
  * Use the modern JavaScript RegExp engine to implement the OnigScanner.
  *
- * As Oniguruma regex is more powerful than JavaScript regex, some patterns may not be supported.
- * Errors will be thrown when parsing TextMate grammars with unsupported patterns.
- * Set `forgiving` to `true` to ignore these errors and skip the unsupported patterns.
+ * As Oniguruma supports some features that can't be emulated using native JavaScript regexes, some
+ * patterns are not supported. Errors will be thrown when parsing TextMate grammars with
+ * unsupported patterns, and when the grammar includes patterns that use invalid Oniguruma syntax.
+ * Set `forgiving` to `true` to ignore these errors and skip any unsupported or invalid patterns.
  *
  * @experimental
  */

package/dist/index.d.ts

@@ -1,5 +1,6 @@
 import { PatternScanner, RegexEngineString, RegexEngine } from '@shikijs/types';
 import { IOnigMatch } from '@shikijs/vscode-textmate';
+import { Options } from 'oniguruma-to-es';
 
 interface JavaScriptRegexEngineOptions {
     /**
@@ -9,11 +10,25 @@ interface JavaScriptRegexEngineOptions {
      */
     forgiving?: boolean;
     /**
-     * Use JavaScript to simulate some unsupported regex features.
+     * Cleanup some grammar patterns before use.
      *
      * @default true
      */
     simulation?: boolean;
+    /**
+     * The target ECMAScript version.
+     *
+     * For the best accuracy, Oniguruma-to-ES needs the `v` flag support in RegExp which is landed in ES2024.
+     * Which requires Node.js 20+ or Chrome 112+.
+     * @see https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/RegExp/unicodeSets
+     *
+     * For the maximum compatibility, you can set it to `ES2018`. Which will use the `u` flag to simulate and will be less accurate.
+     *
+     * Set to `auto` to detect the target version automatically.
+     *
+     * @default 'auto'
+     */
+    target?: 'ES2024' | 'ES2025' | 'ES2018' | 'auto';
     /**
      * Cache for regex patterns.
      */
@@ -21,28 +36,28 @@ interface JavaScriptRegexEngineOptions {
     /**
      * Custom pattern to RegExp constructor.
      *
-     * By default `oniguruma-to-js` is used.
+     * By default `oniguruma-to-es` is used.
      */
     regexConstructor?: (pattern: string) => RegExp;
 }
 /**
  * The default RegExp constructor for JavaScript regex engine.
  */
-declare function defaultJavaScriptRegexConstructor(pattern: string): RegExp;
+declare function defaultJavaScriptRegexConstructor(pattern: string, options?: Options): RegExp;
 declare class JavaScriptScanner implements PatternScanner {
     patterns: string[];
     options: JavaScriptRegexEngineOptions;
     regexps: (RegExp | null)[];
-    contiguousAnchorSimulation: boolean[];
     constructor(patterns: string[], options?: JavaScriptRegexEngineOptions);
     findNextMatchSync(string: string | RegexEngineString, startPosition: number): IOnigMatch | null;
 }
 /**
  * Use the modern JavaScript RegExp engine to implement the OnigScanner.
  *
- * As Oniguruma regex is more powerful than JavaScript regex, some patterns may not be supported.
- * Errors will be thrown when parsing TextMate grammars with unsupported patterns.
- * Set `forgiving` to `true` to ignore these errors and skip the unsupported patterns.
+ * As Oniguruma supports some features that can't be emulated using native JavaScript regexes, some
+ * patterns are not supported. Errors will be thrown when parsing TextMate grammars with
+ * unsupported patterns, and when the grammar includes patterns that use invalid Oniguruma syntax.
+ * Set `forgiving` to `true` to ignore these errors and skip any unsupported or invalid patterns.
  *
  * @experimental
  */

package/dist/index.mjs

@@ -1,15 +1,4 @@
-import { onigurumaToRegexp } from 'oniguruma-to-js';
-
-const replacements = [
-  [
-    "(?<square>[^\\[\\]\\\\]|\\\\.|\\[\\g<square>*+\\])",
-    "(?<square>[^\\[\\]\\\\]|\\\\.|\\[(?:[^\\[\\]\\\\]|\\\\.|\\[(?:[^\\[\\]\\\\]|\\\\.|\\[(?:[^\\[\\]\\\\])*+\\])*+\\])*+\\])"
-  ],
-  [
-    "(?<url>(?>[^\\s()]+)|\\(\\g<url>*\\))",
-    "(?<url>(?>[^\\s()]+)|\\((?:(?>[^\\s()]+)|\\((?:(?>[^\\s()]+)|\\((?>[^\\s()]+)*\\))*\\))*\\))"
-  ]
-];
+import { toRegExp } from 'oniguruma-to-es';
 
 var __defProp = Object.defineProperty;
 var __defNormalProp = (obj, key, value) => key in obj ? __defProp(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value;
@@ -18,12 +7,28 @@ var __publicField = (obj, key, value) => {
   return value;
 };
 const MAX = 4294967295;
-function defaultJavaScriptRegexConstructor(pattern) {
-  return onigurumaToRegexp(
+let supportedRegExpTarget;
+function detectRegExpTarget() {
+  if (supportedRegExpTarget != null)
+    return supportedRegExpTarget;
+  supportedRegExpTarget = "ES2018";
+  try {
+    new RegExp("a", "v");
+    supportedRegExpTarget = "ES2024";
+  } catch {
+    supportedRegExpTarget = "ES2018";
+  }
+  return supportedRegExpTarget;
+}
+function defaultJavaScriptRegexConstructor(pattern, options) {
+  return toRegExp(
     pattern,
     {
-      flags: "dgm",
-      ignoreContiguousAnchors: true
+      accuracy: "loose",
+      global: true,
+      hasIndices: true,
+      tmGrammar: true,
+      ...options
     }
   );
 }
@@ -32,19 +37,18 @@ class JavaScriptScanner {
     this.patterns = patterns;
     this.options = options;
     __publicField(this, "regexps");
-    __publicField(this, "contiguousAnchorSimulation");
     const {
       forgiving = false,
       cache,
+      target = "auto",
       simulation = true,
-      regexConstructor = defaultJavaScriptRegexConstructor
+      regexConstructor = (pattern) => defaultJavaScriptRegexConstructor(pattern, {
+        target: target === "auto" ? detectRegExpTarget() : target
+      })
     } = options;
-    this.contiguousAnchorSimulation = Array.from({ length: patterns.length }, () => false);
-    this.regexps = patterns.map((p, idx) => {
+    this.regexps = patterns.map((p) => {
       if (simulation)
         p = p.replaceAll("(^|\\\uFFFF)", "(^|\\G)");
-      if (simulation && (p.startsWith("(^|\\G)") || p.startsWith("(\\G|^)")))
-        this.contiguousAnchorSimulation[idx] = true;
       const cached = cache?.get(p);
       if (cached) {
         if (cached instanceof RegExp) {
@@ -55,13 +59,7 @@ class JavaScriptScanner {
         throw cached;
       }
       try {
-        let pattern = p;
-        if (simulation) {
-          for (const [from, to] of replacements) {
-            pattern = pattern.replaceAll(from, to);
-          }
-        }
-        const regex = regexConstructor(pattern);
+        const regex = regexConstructor(p);
         cache?.set(p, regex);
         return regex;
       } catch (e) {
@@ -99,20 +97,14 @@ class JavaScriptScanner {
       if (!regexp)
         continue;
       try {
-        let offset = 0;
         regexp.lastIndex = startPosition;
-        let match = regexp.exec(str);
-        if (!match && this.contiguousAnchorSimulation[i]) {
-          offset = startPosition;
-          regexp.lastIndex = 0;
-          match = regexp.exec(str.slice(startPosition));
-        }
+        const match = regexp.exec(str);
         if (!match)
           continue;
         if (match.index === startPosition) {
-          return toResult(i, match, offset);
+          return toResult(i, match, 0);
         }
-        pending.push([i, match, offset]);
+        pending.push([i, match, 0]);
       } catch (e) {
         if (this.options.forgiving)
           continue;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@shikijs/engine-javascript",
   "type": "module",
-  "version": "1.22.1",
+  "version": "1.23.0",
   "description": "Engine for Shiki using JavaScript's native RegExp",
   "author": "Anthony Fu <anthonyfu117@hotmail.com>",
   "license": "MIT",
@@ -31,8 +31,8 @@
   ],
   "dependencies": {
     "@shikijs/vscode-textmate": "^9.3.0",
-    "oniguruma-to-js": "0.4.3",
-    "@shikijs/types": "1.22.1"
+    "oniguruma-to-es": "0.1.2",
+    "@shikijs/types": "1.23.0"
   },
   "scripts": {
     "build": "unbuild",