goatlint-parser 0.125.0

package/src-js/raw-transfer/eager.js

@@ -0,0 +1,254 @@
+import { createRequire } from "node:module";
+import { TOKENS_OFFSET_POS_32, TOKENS_LEN_POS_32 } from "../generated/constants.js";
+import { isJsAst, parseAsyncRawImpl, parseSyncRawImpl, returnBufferToCache } from "./common.js";
+
+const require = createRequire(import.meta.url);
+
+/**
+ * Parse JS/TS source synchronously on current thread, using raw transfer to speed up deserialization.
+ *
+ * @param {string} filename - Filename
+ * @param {string} sourceText - Source text of file
+ * @param {Object} options - Parsing options
+ * @returns {Object} - Object with property getters for `program`, `module`, `comments`, and `errors`
+ */
+export function parseSyncRaw(filename, sourceText, options) {
+  return parseSyncRawImpl(filename, sourceText, options, deserialize);
+}
+
+/**
+ * Parse JS/TS source asynchronously, using raw transfer to speed up deserialization.
+ *
+ * Note that not all of the workload can happen on a separate thread.
+ * Parsing on Rust side does happen in a separate thread, but deserialization of the AST to JS objects
+ * has to happen on current thread. This synchronous deserialization work typically outweighs
+ * the asynchronous parsing by a factor of around 3.
+ *
+ * i.e. the majority of the workload cannot be parallelized by using this method.
+ *
+ * Generally `parseSyncRaw` is preferable to use as it does not have the overhead of spawning a thread.
+ * If you need to parallelize parsing multiple files, it is recommended to use worker threads.
+ *
+ * @param {string} filename - Filename
+ * @param {string} sourceText - Source text of file
+ * @param {Object} options - Parsing options
+ * @returns {Object} - Object with property getters for `program`, `module`, `comments`, and `errors`
+ */
+export function parse(filename, sourceText, options) {
+  return parseAsyncRawImpl(filename, sourceText, options, deserialize);
+}
+
+// Deserializers are large files, so lazy-loaded.
+// `deserialize` functions are stored in this array once loaded.
+// Index into these arrays is `isJs * 1 + range * 2 + experimentalParent * 4`.
+const deserializers = [null, null, null, null, null, null, null, null];
+const deserializerNames = [
+  "ts",
+  "js",
+  "ts_range",
+  "js_range",
+  "ts_parent",
+  "js_parent",
+  "ts_range_parent",
+  "js_range_parent",
+];
+
+/**
+ * Deserialize whole AST from buffer.
+ *
+ * @param {Uint8Array} buffer - Buffer containing AST in raw form
+ * @param {string} sourceText - Source for the file
+ * @param {number} sourceByteLen - Length of source text in UTF-8 bytes
+ * @param {Object} options - Parsing options
+ * @returns {Object} - Object with property getters for `program`, `module`, `comments`, and `errors`
+ */
+function deserialize(buffer, sourceText, sourceByteLen, options) {
+  const isJs = isJsAst(buffer),
+    range = !!options.range,
+    parent = !!options.experimentalParent;
+
+  // Lazy load deserializer, and deserialize buffer to JS objects
+  const deserializerIndex = +isJs | (+range << 1) | (+parent << 2);
+  let deserializeThis = deserializers[deserializerIndex];
+  if (deserializeThis === null) {
+    deserializeThis = deserializers[deserializerIndex] = require(
+      `../generated/deserialize/${deserializerNames[deserializerIndex]}.js`,
+    ).deserialize;
+  }
+
+  const data = deserializeThis(buffer, sourceText, sourceByteLen);
+
+  // Add a line comment for hashbang if JS.
+  // Do not add comment if TS, to match `@typescript-eslint/parser`.
+  // See https://github.com/oxc-project/oxc/blob/ea784f5f082e4c53c98afde9bf983afd0b95e44e/napi/parser/src/lib.rs#L106-L130
+  if (isJs) {
+    const { hashbang } = data.program;
+    if (hashbang !== null) {
+      data.comments.unshift(
+        range
+          ? {
+              type: "Line",
+              value: hashbang.value,
+              start: hashbang.start,
+              end: hashbang.end,
+              range: hashbang.range,
+            }
+          : { type: "Line", value: hashbang.value, start: hashbang.start, end: hashbang.end },
+      );
+    }
+  }
+
+  // Deserialize tokens
+  const tokens = options.experimentalTokens ? deserializeTokens(buffer, sourceText, isJs) : null;
+
+  // Return buffer to cache, to be reused
+  returnBufferToCache(buffer);
+
+  // We cannot lazily deserialize in the getters, because the buffer might be re-used to parse
+  // another file before the getter is called
+  if (tokens !== null) {
+    return {
+      get program() {
+        return data.program;
+      },
+      get module() {
+        return data.module;
+      },
+      get comments() {
+        return data.comments;
+      },
+      get tokens() {
+        return tokens;
+      },
+      get errors() {
+        return data.errors;
+      },
+    };
+  }
+
+  return {
+    get program() {
+      return data.program;
+    },
+    get module() {
+      return data.module;
+    },
+    get comments() {
+      return data.comments;
+    },
+    get errors() {
+      return data.errors;
+    },
+  };
+}
+
+// `ESTreeKind` discriminants (set by Rust side)
+const PRIVATE_IDENTIFIER_KIND = 2;
+const REGEXP_KIND = 8;
+
+// Indexed by `ESTreeKind` discriminant (matches `ESTreeKind` enum in `estree_kind.rs`)
+const TOKEN_TYPES = [
+  "Identifier",
+  "Keyword",
+  "PrivateIdentifier",
+  "Punctuator",
+  "Numeric",
+  "String",
+  "Boolean",
+  "Null",
+  "RegularExpression",
+  "Template",
+  "JSXText",
+  "JSXIdentifier",
+];
+
+// Mask for active bits in `ESTreeKind` discriminants
+const TOKEN_KIND_MASK = 15;
+
+// Details of Rust `Token` type
+const TOKEN_SIZE = 16;
+
+/**
+ * Deserialize tokens from buffer.
+ * @param {Uint8Array} buffer - Buffer containing AST in raw form
+ * @param {string} sourceText - Source for the file
+ * @param {boolean} isJs - `true` if parsing in JS mode
+ * @returns {Object[]} - Array of token objects
+ */
+function deserializeTokens(buffer, sourceText, isJs) {
+  const { int32 } = buffer;
+
+  let pos = int32[TOKENS_OFFSET_POS_32];
+  const len = int32[TOKENS_LEN_POS_32];
+  const endPos = pos + len * TOKEN_SIZE;
+
+  const tokens = [];
+  while (pos < endPos) {
+    tokens.push(deserializeToken(pos, int32, sourceText, isJs));
+    pos += TOKEN_SIZE;
+  }
+  return tokens;
+}
+
+/**
+ * Deserialize a token from buffer at position `pos`.
+ * @param {number} pos - Position in buffer containing Rust `Token` type
+ * @param {Int32Array} int32 - Buffer containing AST in raw form as an `Int32Array`
+ * @param {string} sourceText - Source for the file
+ * @param {boolean} isJs - `true` if parsing in JS mode
+ * @returns {Object} - Token object
+ */
+function deserializeToken(pos, int32, sourceText, isJs) {
+  const pos32 = pos >> 2,
+    start = int32[pos32],
+    end = int32[pos32 + 1],
+    kindAndFlags = int32[pos32 + 2];
+
+  let value = sourceText.slice(start, end);
+
+  // `Kind` is byte at index 8 in `Token`.
+  // `Kind` has 12 variants numbered from 0 to 11.
+  // We have to mask the bottom byte (`& 0xFF`), so may as well mask off bits which can't be set in `Kind` at same time.
+  // This may allow V8 to generate more efficient code for `TOKEN_TYPES[kind]`.
+  const kind = kindAndFlags & TOKEN_KIND_MASK;
+
+  if (kind === REGEXP_KIND) {
+    const patternEnd = value.lastIndexOf("/");
+    return {
+      type: "RegularExpression",
+      value,
+      regex: {
+        pattern: value.slice(1, patternEnd),
+        flags: value.slice(patternEnd + 1),
+      },
+      start,
+      end,
+    };
+  }
+
+  // Strip leading `#` from private identifiers
+  if (kind === PRIVATE_IDENTIFIER_KIND) value = value.slice(1);
+
+  // Unescape identifiers, keywords, and private identifiers in JS mode.
+  // `is_escaped` flag is in byte 10 of `Token`, and is a `bool`.
+  if (isJs && kind <= PRIVATE_IDENTIFIER_KIND && (kindAndFlags & 0x10000) !== 0) {
+    value = unescapeIdentifier(value);
+  }
+
+  return { type: TOKEN_TYPES[kind], value, start, end };
+}
+
+/**
+ * Unescape an identifier.
+ *
+ * We do this on JS side, because escaped identifiers are so extremely rare that this function
+ * is never called in practice anyway.
+ *
+ * @param {string} name - Identifier name to unescape
+ * @returns {string} - Unescaped identifier name
+ */
+function unescapeIdentifier(name) {
+  return name.replace(/\\u(?:\{([0-9a-fA-F]+)\}|([0-9a-fA-F]{4}))/g, (_, hex1, hex2) =>
+    String.fromCodePoint(parseInt(hex1 ?? hex2, 16)),
+  );
+}

package/src-js/raw-transfer/lazy-common.js

@@ -0,0 +1,11 @@
+// Unique token which is not exposed publicly.
+// Used to prevent user calling class constructors.
+export const TOKEN = {};
+
+/**
+ * Throw error when restricted class constructor is called by user code.
+ * @throws {Error}
+ */
+export function constructorError() {
+  throw new Error("Constructor is for internal use only");
+}

package/src-js/raw-transfer/lazy.js

@@ -0,0 +1,153 @@
+import { DATA_POINTER_POS_32, PROGRAM_OFFSET } from "../generated/constants.js";
+import { RawTransferData } from "../generated/lazy/constructors.js";
+import { walkProgram } from "../generated/lazy/walk.js";
+import { parseAsyncRawImpl, parseSyncRawImpl, returnBufferToCache } from "./common.js";
+import { TOKEN } from "./lazy-common.js";
+import { getVisitorsArr } from "./visitor.js";
+export { Visitor } from "./visitor.js";
+
+/**
+ * Parse JS/TS source synchronously on current thread.
+ *
+ * The data in buffer is not deserialized. Is deserialized to JS objects lazily, when accessing the
+ * properties of objects.
+ *
+ * e.g. `program` in returned object is an instance of `Program` class, with getters for `start`, `end`,
+ * `body` etc.
+ *
+ * Returned object contains a `visit` function which can be used to visit the AST with a `Visitor`
+ * (`Visitor` class can be obtained by calling `experimentalGetLazyVisitor()`).
+ *
+ * Returned object contains a `dispose` method. When finished with this AST, it's advisable to call
+ * `dispose`, to return the buffer to the cache, so it can be reused.
+ * Garbage collector should do this anyway at some point, but on an unpredictable schedule,
+ * so it's preferable to call `dispose` manually, to ensure the buffer can be reused immediately.
+ *
+ * @param {string} filename - Filename
+ * @param {string} sourceText - Source text of file
+ * @param {Object} options - Parsing options
+ * @returns {Object} - Object with property getters for `program`, `module`, `comments`, and `errors`,
+ *   and `dispose` and `visit` methods
+ */
+export function parseSyncLazy(filename, sourceText, options) {
+  return parseSyncRawImpl(filename, sourceText, options, construct);
+}
+
+/**
+ * Parse JS/TS source asynchronously on a separate thread.
+ *
+ * The data in buffer is not deserialized. Is deserialized to JS objects lazily, when accessing the
+ * properties of objects.
+ *
+ * e.g. `program` in returned object is an instance of `Program` class, with getters for `start`, `end`,
+ * `body` etc.
+ *
+ * Because this function does not deserialize the AST, unlike `parse`, very little work happens
+ * on current thread in this function. Deserialization work only occurs when properties of the objects
+ * are accessed.
+ *
+ * Returned object contains a `visit` function which can be used to visit the AST with a `Visitor`
+ * (`Visitor` class can be obtained by calling `experimentalGetLazyVisitor()`).
+ *
+ * Returned object contains a `dispose` method. When finished with this AST, it's advisable to call
+ * `dispose`, to return the buffer to the cache, so it can be reused.
+ * Garbage collector should do this anyway at some point, but on an unpredictable schedule,
+ * so it's preferable to call `dispose` manually, to ensure the buffer can be reused immediately.
+ *
+ * @param {string} filename - Filename
+ * @param {string} sourceText - Source text of file
+ * @param {Object} options - Parsing options
+ * @returns {Object} - Object with property getters for `program`, `module`, `comments`, and `errors`,
+ *   and `dispose` and `visit` methods
+ */
+export function parse(filename, sourceText, options) {
+  return parseAsyncRawImpl(filename, sourceText, options, construct);
+}
+
+// Registry for buffers which are held by lazily-deserialized ASTs.
+// Returns buffer to cache when the `ast` wrapper is garbage collected.
+//
+// Check for existence of `FinalizationRegistry`, to avoid errors on old versions of NodeJS
+// which don't support it. e.g. Prettier supports NodeJS v14.
+// Raw transfer is disabled on NodeJS before v22, so it doesn't matter if this is `null` on old NodeJS
+// - it'll never be accessed in that case.
+const bufferRecycleRegistry =
+  typeof FinalizationRegistry === "undefined"
+    ? null
+    : new FinalizationRegistry(returnBufferToCache);
+
+/**
+ * Get an object with getters which lazy deserialize AST and other data from buffer.
+ *
+ * Object also includes `dispose` and `visit` functions.
+ *
+ * @param {Uint8Array} buffer - Buffer containing AST in raw form
+ * @param {string} sourceText - Source for the file
+ * @param {number} sourceByteLen - Length of source text in UTF-8 bytes
+ * @param {Object} _options - Parsing options
+ * @returns {Object} - Object with property getters for `program`, `module`, `comments`, and `errors`,
+ *   and `dispose` and `visit` methods
+ */
+function construct(buffer, sourceText, sourceByteLen, _options) {
+  // Create AST object
+  const sourceIsAscii = sourceText.length === sourceByteLen;
+  const ast = { buffer, sourceText, sourceByteLen, sourceIsAscii, nodes: new Map(), token: TOKEN };
+
+  // Register `ast` with the recycle registry so buffer is returned to cache
+  // when `ast` is garbage collected
+  bufferRecycleRegistry.register(ast, buffer, ast);
+
+  // Get root data class instance
+  const rawDataPos = buffer.int32[DATA_POINTER_POS_32];
+  const data = new RawTransferData(rawDataPos, ast);
+
+  return {
+    get program() {
+      return data.program;
+    },
+    get module() {
+      return data.module;
+    },
+    get comments() {
+      return data.comments;
+    },
+    get errors() {
+      return data.errors;
+    },
+    dispose: dispose.bind(null, ast),
+    visit(visitor) {
+      walkProgram(rawDataPos + PROGRAM_OFFSET, ast, getVisitorsArr(visitor));
+    },
+  };
+}
+
+/**
+ * Dispose of this AST.
+ *
+ * After calling this method, trying to read any nodes from this AST may cause an error.
+ *
+ * Buffer is returned to the cache to be reused.
+ *
+ * The buffer would be returned to the cache anyway, once all nodes of the AST are garbage collected,
+ * but calling `dispose` is preferable, as it will happen immediately.
+ * Otherwise, garbage collector may take time to collect the `ast` object, and new buffers may be created
+ * in the meantime, when we could have reused this one.
+ *
+ * @param {Object} ast - AST object containing buffer etc
+ * @returns {undefined}
+ */
+function dispose(ast) {
+  // Return buffer to cache, to be reused
+  returnBufferToCache(ast.buffer);
+
+  // Remove connection between `ast` and the buffer
+  ast.buffer = null;
+
+  // Clear other contents of `ast`, so they can be garbage collected
+  ast.sourceText = null;
+  ast.nodes = null;
+
+  // Remove `ast` from recycling register.
+  // When `ast` is garbage collected, there's no longer any action to be taken.
+  bufferRecycleRegistry.unregister(ast);
+}