@shikijs/engine-javascript 1.17.0

package/LICENSE

@@ -0,0 +1,22 @@
+MIT License
+
+Copyright (c) 2021 Pine Wu
+Copyright (c) 2023 Anthony Fu <https://github.com/antfu>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,9 @@
+# @shikijs/engine-javascript
+
+Engine for Shiki using JavaScript's native RegExp (experimental).
+
+[Documentation](https://shiki.style/guide/regex-engines)
+
+## License
+
+MIT

package/dist/index.d.mts

@@ -0,0 +1,50 @@
+import { PatternScanner, RegexEngineString, RegexEngine } from '@shikijs/types';
+
+interface JavaScriptRegexEngineOptions {
+    /**
+     * Whether to allow invalid regex patterns.
+     */
+    forgiving?: boolean;
+    /**
+     * Cache for regex patterns.
+     */
+    cache?: Map<string, RegExp | Error>;
+    /**
+     * Custom pattern to RegExp constructor.
+     *
+     * By default `oniguruma-to-js` is used.
+     */
+    regexConstructor?: (pattern: string) => RegExp;
+}
+/**
+ * The default RegExp constructor for JavaScript regex engine.
+ */
+declare function defaultJavaScriptRegexConstructor(pattern: string): RegExp;
+declare class JavaScriptScanner implements PatternScanner {
+    patterns: string[];
+    cache: Map<string, RegExp | Error>;
+    forgiving: boolean;
+    regexConstructor: (pattern: string) => RegExp;
+    regexps: (RegExp | null)[];
+    constructor(patterns: string[], cache: Map<string, RegExp | Error>, forgiving: boolean, regexConstructor?: (pattern: string) => RegExp);
+    findNextMatchSync(string: string | RegexEngineString, startPosition: number): {
+        index: number;
+        captureIndices: {
+            end: number;
+            start: number;
+            length: number;
+        }[];
+    } | null;
+}
+/**
+ * Use the modern JavaScript RegExp engine to implement the OnigScanner.
+ *
+ * As Oniguruma regex is more powerful than JavaScript regex, some patterns may not be supported.
+ * Errors will be thrown when parsing TextMate grammars with unsupported patterns.
+ * Set `forgiving` to `true` to ignore these errors and skip the unsupported patterns.
+ *
+ * @experimental
+ */
+declare function createJavaScriptRegexEngine(options?: JavaScriptRegexEngineOptions): RegexEngine;
+
+export { type JavaScriptRegexEngineOptions, JavaScriptScanner, createJavaScriptRegexEngine, defaultJavaScriptRegexConstructor };

package/dist/index.d.ts

@@ -0,0 +1,50 @@
+import { PatternScanner, RegexEngineString, RegexEngine } from '@shikijs/types';
+
+interface JavaScriptRegexEngineOptions {
+    /**
+     * Whether to allow invalid regex patterns.
+     */
+    forgiving?: boolean;
+    /**
+     * Cache for regex patterns.
+     */
+    cache?: Map<string, RegExp | Error>;
+    /**
+     * Custom pattern to RegExp constructor.
+     *
+     * By default `oniguruma-to-js` is used.
+     */
+    regexConstructor?: (pattern: string) => RegExp;
+}
+/**
+ * The default RegExp constructor for JavaScript regex engine.
+ */
+declare function defaultJavaScriptRegexConstructor(pattern: string): RegExp;
+declare class JavaScriptScanner implements PatternScanner {
+    patterns: string[];
+    cache: Map<string, RegExp | Error>;
+    forgiving: boolean;
+    regexConstructor: (pattern: string) => RegExp;
+    regexps: (RegExp | null)[];
+    constructor(patterns: string[], cache: Map<string, RegExp | Error>, forgiving: boolean, regexConstructor?: (pattern: string) => RegExp);
+    findNextMatchSync(string: string | RegexEngineString, startPosition: number): {
+        index: number;
+        captureIndices: {
+            end: number;
+            start: number;
+            length: number;
+        }[];
+    } | null;
+}
+/**
+ * Use the modern JavaScript RegExp engine to implement the OnigScanner.
+ *
+ * As Oniguruma regex is more powerful than JavaScript regex, some patterns may not be supported.
+ * Errors will be thrown when parsing TextMate grammars with unsupported patterns.
+ * Set `forgiving` to `true` to ignore these errors and skip the unsupported patterns.
+ *
+ * @experimental
+ */
+declare function createJavaScriptRegexEngine(options?: JavaScriptRegexEngineOptions): RegexEngine;
+
+export { type JavaScriptRegexEngineOptions, JavaScriptScanner, createJavaScriptRegexEngine, defaultJavaScriptRegexConstructor };

package/dist/index.mjs

@@ -0,0 +1,128 @@
+import { onigurumaToRegexp } from 'oniguruma-to-js';
+import { rewrite } from 'regex';
+
+var __defProp = Object.defineProperty;
+var __defNormalProp = (obj, key, value) => key in obj ? __defProp(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value;
+var __publicField = (obj, key, value) => {
+  __defNormalProp(obj, typeof key !== "symbol" ? key + "" : key, value);
+  return value;
+};
+const MAX = 4294967295;
+function defaultJavaScriptRegexConstructor(pattern) {
+  pattern = pattern.replaceAll("[^\\s[-?:,\\[\\]{}#&*!|>'\"%@`]]", "[^\\s\\-?:,\\[\\]{}#&*!|>'\"%@`]");
+  const rewritten = rewrite(pattern, {
+    flags: "dgm",
+    unicodeSetsPlugin: null,
+    disable: {
+      n: true,
+      v: true,
+      x: true
+    }
+  });
+  return onigurumaToRegexp(
+    rewritten.expression,
+    {
+      flags: "dgm",
+      ignoreContiguousAnchors: true
+    }
+  );
+}
+class JavaScriptScanner {
+  constructor(patterns, cache, forgiving, regexConstructor = defaultJavaScriptRegexConstructor) {
+    this.patterns = patterns;
+    this.cache = cache;
+    this.forgiving = forgiving;
+    this.regexConstructor = regexConstructor;
+    __publicField(this, "regexps");
+    this.regexps = patterns.map((p) => {
+      const cached = cache?.get(p);
+      if (cached) {
+        if (cached instanceof RegExp) {
+          return cached;
+        }
+        if (forgiving)
+          return null;
+        throw cached;
+      }
+      try {
+        const regex = regexConstructor(p);
+        cache?.set(p, regex);
+        return regex;
+      } catch (e) {
+        cache?.set(p, e);
+        if (forgiving)
+          return null;
+        throw e;
+      }
+    });
+  }
+  findNextMatchSync(string, startPosition) {
+    const str = typeof string === "string" ? string : string.content;
+    const pending = [];
+    function toResult(index, match) {
+      return {
+        index,
+        captureIndices: match.indices.map((indice) => {
+          if (indice == null) {
+            return {
+              end: MAX,
+              start: MAX,
+              length: 0
+            };
+          }
+          return {
+            start: indice[0],
+            length: indice[1] - indice[0],
+            end: indice[1]
+          };
+        })
+      };
+    }
+    for (let i = 0; i < this.regexps.length; i++) {
+      const regexp = this.regexps[i];
+      if (!regexp)
+        continue;
+      try {
+        regexp.lastIndex = startPosition;
+        const match = regexp.exec(str);
+        if (!match)
+          continue;
+        if (match.index === startPosition) {
+          return toResult(i, match);
+        }
+        pending.push([i, match]);
+      } catch (e) {
+        if (this.forgiving)
+          continue;
+        throw e;
+      }
+    }
+    if (pending.length) {
+      const minIndex = Math.min(...pending.map((m) => m[1].index));
+      for (const [i, match] of pending) {
+        if (match.index === minIndex) {
+          return toResult(i, match);
+        }
+      }
+    }
+    return null;
+  }
+}
+function createJavaScriptRegexEngine(options = {}) {
+  const {
+    forgiving = false,
+    cache = /* @__PURE__ */ new Map()
+  } = options;
+  return {
+    createScanner(patterns) {
+      return new JavaScriptScanner(patterns, cache, forgiving, options.regexConstructor);
+    },
+    createString(s) {
+      return {
+        content: s
+      };
+    }
+  };
+}
+
+export { JavaScriptScanner, createJavaScriptRegexEngine, defaultJavaScriptRegexConstructor };

package/package.json

@@ -0,0 +1,41 @@
+{
+  "name": "@shikijs/engine-javascript",
+  "type": "module",
+  "version": "1.17.0",
+  "description": "Engine for Shiki using JavaScript's native RegExp",
+  "author": "Anthony Fu <anthonyfu117@hotmail.com>",
+  "license": "MIT",
+  "homepage": "https://github.com/shikijs/shiki#readme",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/shikijs/shiki.git",
+    "directory": "packages/transformers"
+  },
+  "bugs": "https://github.com/shikijs/shiki/issues",
+  "keywords": [
+    "shiki",
+    "@shikijs/transformers"
+  ],
+  "sideEffects": false,
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.mts",
+      "default": "./dist/index.mjs"
+    }
+  },
+  "main": "./dist/index.mjs",
+  "module": "./dist/index.mjs",
+  "types": "./dist/index.d.mts",
+  "files": [
+    "dist"
+  ],
+  "dependencies": {
+    "oniguruma-to-js": "0.3.3",
+    "regex": "4.3.2",
+    "@shikijs/types": "1.17.0"
+  },
+  "scripts": {
+    "build": "unbuild",
+    "dev": "unbuild --stub"
+  }
+}