@herb-tools/node 0.1.0

package/extension/prism/src/util/pm_strpbrk.c

@@ -0,0 +1,206 @@
+#include "prism/util/pm_strpbrk.h"
+
+/**
+ * Add an invalid multibyte character error to the parser.
+ */
+static inline void
+pm_strpbrk_invalid_multibyte_character(pm_parser_t *parser, const uint8_t *start, const uint8_t *end) {
+    pm_diagnostic_list_append_format(&parser->error_list, start, end, PM_ERR_INVALID_MULTIBYTE_CHARACTER, *start);
+}
+
+/**
+ * Set the explicit encoding for the parser to the current encoding.
+ */
+static inline void
+pm_strpbrk_explicit_encoding_set(pm_parser_t *parser, const uint8_t *source, size_t width) {
+    if (parser->explicit_encoding != NULL) {
+        if (parser->explicit_encoding == parser->encoding) {
+            // Okay, we already locked to this encoding.
+        } else if (parser->explicit_encoding == PM_ENCODING_UTF_8_ENTRY) {
+            // Not okay, we already found a Unicode escape sequence and this
+            // conflicts.
+            pm_diagnostic_list_append_format(&parser->error_list, source, source + width, PM_ERR_MIXED_ENCODING, parser->encoding->name);
+        } else {
+            // Should not be anything else.
+            assert(false && "unreachable");
+        }
+    }
+
+    parser->explicit_encoding = parser->encoding;
+}
+
+/**
+ * This is the default path.
+ */
+static inline const uint8_t *
+pm_strpbrk_utf8(pm_parser_t *parser, const uint8_t *source, const uint8_t *charset, size_t maximum, bool validate) {
+    size_t index = 0;
+
+    while (index < maximum) {
+        if (strchr((const char *) charset, source[index]) != NULL) {
+            return source + index;
+        }
+
+        if (source[index] < 0x80) {
+            index++;
+        } else {
+            size_t width = pm_encoding_utf_8_char_width(source + index, (ptrdiff_t) (maximum - index));
+
+            if (width > 0) {
+                index += width;
+            } else if (!validate) {
+                index++;
+            } else {
+                // At this point we know we have an invalid multibyte character.
+                // We'll walk forward as far as we can until we find the next
+                // valid character so that we don't spam the user with a ton of
+                // the same kind of error.
+                const size_t start = index;
+
+                do {
+                    index++;
+                } while (index < maximum && pm_encoding_utf_8_char_width(source + index, (ptrdiff_t) (maximum - index)) == 0);
+
+                pm_strpbrk_invalid_multibyte_character(parser, source + start, source + index);
+            }
+        }
+    }
+
+    return NULL;
+}
+
+/**
+ * This is the path when the encoding is ASCII-8BIT.
+ */
+static inline const uint8_t *
+pm_strpbrk_ascii_8bit(pm_parser_t *parser, const uint8_t *source, const uint8_t *charset, size_t maximum, bool validate) {
+    size_t index = 0;
+
+    while (index < maximum) {
+        if (strchr((const char *) charset, source[index]) != NULL) {
+            return source + index;
+        }
+
+        if (validate && source[index] >= 0x80) pm_strpbrk_explicit_encoding_set(parser, source, 1);
+        index++;
+    }
+
+    return NULL;
+}
+
+/**
+ * This is the slow path that does care about the encoding.
+ */
+static inline const uint8_t *
+pm_strpbrk_multi_byte(pm_parser_t *parser, const uint8_t *source, const uint8_t *charset, size_t maximum, bool validate) {
+    size_t index = 0;
+    const pm_encoding_t *encoding = parser->encoding;
+
+    while (index < maximum) {
+        if (strchr((const char *) charset, source[index]) != NULL) {
+            return source + index;
+        }
+
+        if (source[index] < 0x80) {
+            index++;
+        } else {
+            size_t width = encoding->char_width(source + index, (ptrdiff_t) (maximum - index));
+            if (validate) pm_strpbrk_explicit_encoding_set(parser, source, width);
+
+            if (width > 0) {
+                index += width;
+            } else if (!validate) {
+                index++;
+            } else {
+                // At this point we know we have an invalid multibyte character.
+                // We'll walk forward as far as we can until we find the next
+                // valid character so that we don't spam the user with a ton of
+                // the same kind of error.
+                const size_t start = index;
+
+                do {
+                    index++;
+                } while (index < maximum && encoding->char_width(source + index, (ptrdiff_t) (maximum - index)) == 0);
+
+                pm_strpbrk_invalid_multibyte_character(parser, source + start, source + index);
+            }
+        }
+    }
+
+    return NULL;
+}
+
+/**
+ * This is the fast path that does not care about the encoding because we know
+ * the encoding only supports single-byte characters.
+ */
+static inline const uint8_t *
+pm_strpbrk_single_byte(pm_parser_t *parser, const uint8_t *source, const uint8_t *charset, size_t maximum, bool validate) {
+    size_t index = 0;
+    const pm_encoding_t *encoding = parser->encoding;
+
+    while (index < maximum) {
+        if (strchr((const char *) charset, source[index]) != NULL) {
+            return source + index;
+        }
+
+        if (source[index] < 0x80 || !validate) {
+            index++;
+        } else {
+            size_t width = encoding->char_width(source + index, (ptrdiff_t) (maximum - index));
+            pm_strpbrk_explicit_encoding_set(parser, source, width);
+
+            if (width > 0) {
+                index += width;
+            } else {
+                // At this point we know we have an invalid multibyte character.
+                // We'll walk forward as far as we can until we find the next
+                // valid character so that we don't spam the user with a ton of
+                // the same kind of error.
+                const size_t start = index;
+
+                do {
+                    index++;
+                } while (index < maximum && encoding->char_width(source + index, (ptrdiff_t) (maximum - index)) == 0);
+
+                pm_strpbrk_invalid_multibyte_character(parser, source + start, source + index);
+            }
+        }
+    }
+
+    return NULL;
+}
+
+/**
+ * Here we have rolled our own version of strpbrk. The standard library strpbrk
+ * has undefined behavior when the source string is not null-terminated. We want
+ * to support strings that are not null-terminated because pm_parse does not
+ * have the contract that the string is null-terminated. (This is desirable
+ * because it means the extension can call pm_parse with the result of a call to
+ * mmap).
+ *
+ * The standard library strpbrk also does not support passing a maximum length
+ * to search. We want to support this for the reason mentioned above, but we
+ * also don't want it to stop on null bytes. Ruby actually allows null bytes
+ * within strings, comments, regular expressions, etc. So we need to be able to
+ * skip past them.
+ *
+ * Finally, we want to support encodings wherein the charset could contain
+ * characters that are trailing bytes of multi-byte characters. For example, in
+ * Shift_JIS, the backslash character can be a trailing byte. In that case we
+ * need to take a slower path and iterate one multi-byte character at a time.
+ */
+const uint8_t *
+pm_strpbrk(pm_parser_t *parser, const uint8_t *source, const uint8_t *charset, ptrdiff_t length, bool validate) {
+    if (length <= 0) {
+        return NULL;
+    } else if (!parser->encoding_changed) {
+        return pm_strpbrk_utf8(parser, source, charset, (size_t) length, validate);
+    } else if (parser->encoding == PM_ENCODING_ASCII_8BIT_ENTRY) {
+        return pm_strpbrk_ascii_8bit(parser, source, charset, (size_t) length, validate);
+    } else if (parser->encoding->multibyte) {
+        return pm_strpbrk_multi_byte(parser, source, charset, (size_t) length, validate);
+    } else {
+        return pm_strpbrk_single_byte(parser, source, charset, (size_t) length, validate);
+    }
+}

package/package.json

@@ -0,0 +1,74 @@
+{
+  "name": "@herb-tools/node",
+  "version": "0.1.0",
+  "type": "module",
+  "license": "MIT",
+  "homepage": "https://herb-tools.dev",
+  "bugs": "https://github.com/marcoroth/herb/issues/new?title=Package%20%60@herb-tools/node%60:%20",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/marcoroth/herb.git",
+    "directory": "javascript/packages/node"
+  },
+  "gypfile": true,
+  "main": "./dist/herb-node.cjs",
+  "module": "./dist/herb-node.esm.js",
+  "types": "./dist/types/index-esm.d.mts",
+  "exports": {
+    "./package.json": "./package.json",
+    ".": {
+      "types": "./dist/types/index-esm.d.mts",
+      "import": "./dist/herb-node.esm.js",
+      "require": "./dist/herb-node.cjs",
+      "default": "./dist/herb-node.esm.js"
+    }
+  },
+  "scripts": {
+    "install": "yarn vendor:libherb && yarn vendor:prism && node-pre-gyp install --fallback-to-build",
+    "dev": "rollup -c -w",
+    "build": "yarn rebuild",
+    "build:javascript": "rollup -c",
+    "vendor:libherb": "yarn clean && rake templates && cp -r ../../../src ./extension/libherb",
+    "vendor:prism": "node ./bin/vendor-prism.cjs",
+    "rebuild": "yarn clean && yarn vendor:libherb && yarn vendor:prism && node-pre-gyp rebuild && yarn build:javascript",
+    "configure": "node-pre-gyp configure",
+    "package": "node-pre-gyp package",
+    "clean": "node-pre-gyp clean && rimraf dist/ && rimraf extension/libherb/ && rimraf build/",
+    "test": "vitest run",
+    "test:watch": "vitest --watch",
+    "prepublishOnly": "yarn clean && yarn build && yarn test"
+  },
+  "binary": {
+    "module_name": "herb",
+    "module_path": "./build/{configuration}/{node_abi}-{platform}-{arch}/",
+    "remote_path": "{version}",
+    "package_name": "{module_name}-v{version}-{node_abi}-{platform}-{arch}.tar.gz",
+    "host": "https://github.com/marcoroth/herb/releases/download/"
+  },
+  "dependencies": {
+    "@herb-tools/core": "0.1.0",
+    "@mapbox/node-pre-gyp": "^2.0.0"
+  },
+  "devDependencies": {
+    "@rollup/plugin-json": "^6.1.0",
+    "@rollup/plugin-node-resolve": "^16.0.1",
+    "@rollup/plugin-typescript": "^12.1.2",
+    "node-addon-api": "^5.1.0",
+    "node-pre-gyp-github": "^2.0.0",
+    "rimraf": "^6.0.1",
+    "rollup": "^4.35.0",
+    "tslib": "^2.8.1",
+    "typescript": "^5.8.2",
+    "vitest": "^3.0.0"
+  },
+  "files": [
+    "package.json",
+    "README.md",
+    "binding.gyp",
+    "dist/",
+    "src/",
+    "extension/**/*.c",
+    "extension/**/*.cpp",
+    "extension/**/*.h"
+  ]
+}

package/src/index-cjs.cts

@@ -0,0 +1,22 @@
+const path = require("path")
+const binary = require("@mapbox/node-pre-gyp")
+
+const { Visitor } = require("@herb-tools/core")
+const { HerbBackendNode } = require("./node-backend.js")
+
+const packagePath = path.resolve(__dirname, "../package.json")
+const libherbPath = binary.find(packagePath)
+const libHerbBinary = require(libherbPath)
+
+/**
+ * An instance of the `Herb` class using a Node.js backend.
+ * This loads `libherb` in a Node.js C++ native extension.
+ */
+const Herb = new HerbBackendNode(
+  new Promise((resolve, _reject) => resolve(libHerbBinary)),
+)
+
+module.exports = {
+  Herb: Herb,
+  Visitor: Visitor,
+}

package/src/index-esm.mts

@@ -0,0 +1,28 @@
+export * from "@herb-tools/core"
+
+import path from "path"
+import binary from "@mapbox/node-pre-gyp"
+
+import { createRequire } from "module"
+import { fileURLToPath } from "url"
+
+const __filename = fileURLToPath(import.meta.url)
+const __dirname = path.dirname(__filename)
+const require = createRequire(import.meta.url)
+
+const packagePath = path.resolve(__dirname, "../package.json")
+const libherbPath = binary.find(packagePath)
+
+const libHerbBinary = require(libherbPath)
+
+import { HerbBackendNode } from "./node-backend.js"
+
+/**
+ * An instance of the `Herb` class using a Node.js backend.
+ * This loads `libherb` in a Node.js C++ native extension.
+ */
+const Herb = new HerbBackendNode(
+  () => new Promise((resolve, _reject) => resolve(libHerbBinary)),
+)
+
+export { Herb, HerbBackendNode }

package/src/node-backend.ts

@@ -0,0 +1,9 @@
+import packageJSON from "../package.json" with { type: "json" }
+
+import { HerbBackend } from "@herb-tools/core"
+
+export class HerbBackendNode extends HerbBackend {
+  backendVersion(): string {
+    return `${packageJSON.name}@${packageJSON.version}`
+  }
+}

package/src/types/global.d.ts

@@ -0,0 +1,18 @@
+declare module "*.node" {
+  const content: any
+  export default content
+}
+
+declare module "@mapbox/node-pre-gyp" {
+  export interface FindOptions {
+    module_root?: string
+    [key: string]: any
+  }
+
+  export interface NodePreGyp {
+    find(packageJsonPath: string, opts?: FindOptions): string
+  }
+
+  declare const nodePreGyp: NodePreGyp
+  export default nodePreGyp
+}

package/src/util.ts

@@ -0,0 +1,35 @@
+import path from "path"
+import { stat as fileExists } from "node:fs/promises"
+import { ensureString } from "@herb-tools/core"
+
+export async function ensureFile(object: any): Promise<string> {
+  const string = ensureString(object)
+
+  if (await fileExists(string)) {
+    return string
+  }
+
+  throw new TypeError("Argument must be a string")
+}
+
+export function resolvePath(relativePath: string) {
+  let basePath
+  // Check if we"re in ESM or CJS context
+  if (typeof __dirname !== "undefined") {
+    // CommonJS environment
+    basePath = __dirname
+  } else {
+    // ESM environment - need to use import.meta.url
+    // This needs to be in a try/catch for bundlers and environments that don"t support it
+    try {
+      const { fileURLToPath } = require("url")
+      const currentFileUrl = import.meta.url
+      basePath = path.dirname(fileURLToPath(currentFileUrl))
+    } catch (error) {
+      // Fallback for environments where neither is available
+      basePath = process.cwd()
+    }
+  }
+
+  return path.join(basePath, relativePath)
+}