@shikijs/engine-javascript 1.17.4 → 1.17.6

package/dist/index.d.mts

@@ -3,12 +3,20 @@ import { PatternScanner, RegexEngineString, RegexEngine } from '@shikijs/types';
 interface JavaScriptRegexEngineOptions {
     /**
      * Whether to allow invalid regex patterns.
+     *
+     * @default false
      */
     forgiving?: boolean;
+    /**
+     * Use JavaScript to simulate some unsupported regex features.
+     *
+     * @default true
+     */
+    simulation?: boolean;
     /**
      * Cache for regex patterns.
      */
-    cache?: Map<string, RegExp | Error>;
+    cache?: Map<string, RegExp | Error> | null;
     /**
      * Custom pattern to RegExp constructor.
      *
@@ -22,11 +30,10 @@ interface JavaScriptRegexEngineOptions {
 declare function defaultJavaScriptRegexConstructor(pattern: string): RegExp;
 declare class JavaScriptScanner implements PatternScanner {
     patterns: string[];
-    cache: Map<string, RegExp | Error>;
-    forgiving: boolean;
-    regexConstructor: (pattern: string) => RegExp;
+    options: JavaScriptRegexEngineOptions;
     regexps: (RegExp | null)[];
-    constructor(patterns: string[], cache: Map<string, RegExp | Error>, forgiving: boolean, regexConstructor?: (pattern: string) => RegExp);
+    contiguousAnchorSimulation: boolean[];
+    constructor(patterns: string[], options?: JavaScriptRegexEngineOptions);
     findNextMatchSync(string: string | RegexEngineString, startPosition: number): {
         index: number;
         captureIndices: {

package/dist/index.d.ts

@@ -3,12 +3,20 @@ import { PatternScanner, RegexEngineString, RegexEngine } from '@shikijs/types';
 interface JavaScriptRegexEngineOptions {
     /**
      * Whether to allow invalid regex patterns.
+     *
+     * @default false
      */
     forgiving?: boolean;
+    /**
+     * Use JavaScript to simulate some unsupported regex features.
+     *
+     * @default true
+     */
+    simulation?: boolean;
     /**
      * Cache for regex patterns.
      */
-    cache?: Map<string, RegExp | Error>;
+    cache?: Map<string, RegExp | Error> | null;
     /**
      * Custom pattern to RegExp constructor.
      *
@@ -22,11 +30,10 @@ interface JavaScriptRegexEngineOptions {
 declare function defaultJavaScriptRegexConstructor(pattern: string): RegExp;
 declare class JavaScriptScanner implements PatternScanner {
     patterns: string[];
-    cache: Map<string, RegExp | Error>;
-    forgiving: boolean;
-    regexConstructor: (pattern: string) => RegExp;
+    options: JavaScriptRegexEngineOptions;
     regexps: (RegExp | null)[];
-    constructor(patterns: string[], cache: Map<string, RegExp | Error>, forgiving: boolean, regexConstructor?: (pattern: string) => RegExp);
+    contiguousAnchorSimulation: boolean[];
+    constructor(patterns: string[], options?: JavaScriptRegexEngineOptions);
     findNextMatchSync(string: string | RegexEngineString, startPosition: number): {
         index: number;
         captureIndices: {

package/dist/index.mjs

@@ -1,5 +1,16 @@
 import { onigurumaToRegexp } from 'oniguruma-to-js';
 
+const replacements = [
+  [
+    "(?<square>[^\\[\\]\\\\]|\\\\.|\\[\\g<square>*+\\])",
+    "(?<square>[^\\[\\]\\\\]|\\\\.|\\[(?:[^\\[\\]\\\\]|\\\\.|\\[(?:[^\\[\\]\\\\]|\\\\.|\\[(?:[^\\[\\]\\\\])*+\\])*+\\])*+\\])"
+  ],
+  [
+    "(?<url>(?>[^\\s()]+)|\\(\\g<url>*\\))",
+    "(?<url>(?>[^\\s()]+)|\\((?:(?>[^\\s()]+)|\\((?:(?>[^\\s()]+)|\\((?>[^\\s()]+)*\\))*\\))*\\))"
+  ]
+];
+
 var __defProp = Object.defineProperty;
 var __defNormalProp = (obj, key, value) => key in obj ? __defProp(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value;
 var __publicField = (obj, key, value) => {
@@ -17,13 +28,23 @@ function defaultJavaScriptRegexConstructor(pattern) {
   );
 }
 class JavaScriptScanner {
-  constructor(patterns, cache, forgiving, regexConstructor = defaultJavaScriptRegexConstructor) {
+  constructor(patterns, options = {}) {
     this.patterns = patterns;
-    this.cache = cache;
-    this.forgiving = forgiving;
-    this.regexConstructor = regexConstructor;
+    this.options = options;
     __publicField(this, "regexps");
-    this.regexps = patterns.map((p) => {
+    __publicField(this, "contiguousAnchorSimulation");
+    const {
+      forgiving = false,
+      cache,
+      simulation = true,
+      regexConstructor = defaultJavaScriptRegexConstructor
+    } = options;
+    this.contiguousAnchorSimulation = Array.from({ length: patterns.length }, () => false);
+    this.regexps = patterns.map((p, idx) => {
+      if (simulation)
+        p = p.replaceAll("(^|\\\uFFFF)", "(^|\\G)");
+      if (simulation && (p.startsWith("(^|\\G)") || p.startsWith("(\\G|^)")))
+        this.contiguousAnchorSimulation[idx] = true;
       const cached = cache?.get(p);
       if (cached) {
         if (cached instanceof RegExp) {
@@ -34,7 +55,13 @@ class JavaScriptScanner {
         throw cached;
       }
       try {
-        const regex = regexConstructor(p);
+        let pattern = p;
+        if (simulation) {
+          for (const [from, to] of replacements) {
+            pattern = pattern.replaceAll(from, to);
+          }
+        }
+        const regex = regexConstructor(pattern);
         cache?.set(p, regex);
         return regex;
       } catch (e) {
@@ -48,7 +75,7 @@ class JavaScriptScanner {
   findNextMatchSync(string, startPosition) {
     const str = typeof string === "string" ? string : string.content;
     const pending = [];
-    function toResult(index, match) {
+    function toResult(index, match, offset = 0) {
       return {
         index,
         captureIndices: match.indices.map((indice) => {
@@ -60,9 +87,9 @@ class JavaScriptScanner {
             };
           }
           return {
-            start: indice[0],
+            start: indice[0] + offset,
             length: indice[1] - indice[0],
-            end: indice[1]
+            end: indice[1] + offset
           };
         })
       };
@@ -72,25 +99,31 @@ class JavaScriptScanner {
       if (!regexp)
         continue;
       try {
+        let offset = 0;
         regexp.lastIndex = startPosition;
-        const match = regexp.exec(str);
+        let match = regexp.exec(str);
+        if (!match && this.contiguousAnchorSimulation[i]) {
+          offset = startPosition;
+          regexp.lastIndex = 0;
+          match = regexp.exec(str.slice(startPosition));
+        }
         if (!match)
           continue;
         if (match.index === startPosition) {
-          return toResult(i, match);
+          return toResult(i, match, offset);
         }
-        pending.push([i, match]);
+        pending.push([i, match, offset]);
       } catch (e) {
-        if (this.forgiving)
+        if (this.options.forgiving)
           continue;
         throw e;
       }
     }
     if (pending.length) {
       const minIndex = Math.min(...pending.map((m) => m[1].index));
-      for (const [i, match] of pending) {
+      for (const [i, match, offset] of pending) {
         if (match.index === minIndex) {
-          return toResult(i, match);
+          return toResult(i, match, offset);
         }
       }
     }
@@ -98,13 +131,13 @@ class JavaScriptScanner {
   }
 }
 function createJavaScriptRegexEngine(options = {}) {
-  const {
-    forgiving = false,
-    cache = /* @__PURE__ */ new Map()
-  } = options;
+  const _options = {
+    cache: /* @__PURE__ */ new Map(),
+    ...options
+  };
   return {
     createScanner(patterns) {
-      return new JavaScriptScanner(patterns, cache, forgiving, options.regexConstructor);
+      return new JavaScriptScanner(patterns, _options);
     },
     createString(s) {
       return {

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@shikijs/engine-javascript",
   "type": "module",
-  "version": "1.17.4",
+  "version": "1.17.6",
   "description": "Engine for Shiki using JavaScript's native RegExp",
   "author": "Anthony Fu <anthonyfu117@hotmail.com>",
   "license": "MIT",
@@ -30,8 +30,8 @@
     "dist"
   ],
   "dependencies": {
-    "oniguruma-to-js": "0.4.0",
-    "@shikijs/types": "1.17.4"
+    "oniguruma-to-js": "0.4.3",
+    "@shikijs/types": "1.17.6"
   },
   "scripts": {
     "build": "unbuild",