clip-tokenizer-rs 0.1.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2020 N-API for Rust
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,69 @@
+# clip-tokenizer-rs
+
+Fast CLIP text tokenizer for Node.js/Bun, powered by Rust and napi-rs.
+
+## Features
+
+- Fast: Native Rust implementation via napi-rs
+- Zero runtime deps: No Python required
+- Accurate: Official CLIP BPE vocabulary (49408 tokens)
+- Cross-platform: macOS and Linux (x64, arm64)
+
+## Installation
+
+```bash
+bun add clip-tokenizer-rs
+# or
+npm install clip-tokenizer-rs
+```
+
+Requires Node.js 10+ or Bun 1.0+ (NAPI v4).
+
+## Usage
+
+```ts
+import { encode, count } from "clip-tokenizer-rs";
+
+// Tokenize text
+const tokens = encode("a photo of a cat");
+// => [320, 1125, 539, 320, 2368]
+
+// Count tokens without returning array
+const tokenCount = count("a photo of a cat");
+// => 5
+```
+
+## API
+
+### `encode(text: string): number[]`
+Tokenizes text and returns an array of token IDs.
+
+### `count(text: string): number`
+Returns the number of tokens.
+
+## Limitations
+
+- Returns raw tokens; does not truncate/pad to 77 tokens
+- Apply truncation at model inference if needed for CLIP
+- Windows not currently supported
+
+## Platform Support
+
+| Platform | Architecture |
+|----------|-------------|
+| macOS | x64, arm64 |
+| Linux | x64, arm64 |
+
+## Development
+
+```bash
+# Build native module
+cargo build --release
+
+# Run tests
+bun test
+```
+
+## License
+
+MIT

package/index.d.ts

@@ -0,0 +1,5 @@
+/* auto-generated by NAPI-RS */
+/* eslint-disable */
+export declare function count(text: string): number
+
+export declare function encode(text: string): Array<number>

package/index.js

@@ -0,0 +1,580 @@
+// prettier-ignore
+/* eslint-disable */
+// @ts-nocheck
+/* auto-generated by NAPI-RS */
+
+const { readFileSync } = require('node:fs')
+let nativeBinding = null
+const loadErrors = []
+
+const isMusl = () => {
+  let musl = false
+  if (process.platform === 'linux') {
+    musl = isMuslFromFilesystem()
+    if (musl === null) {
+      musl = isMuslFromReport()
+    }
+    if (musl === null) {
+      musl = isMuslFromChildProcess()
+    }
+  }
+  return musl
+}
+
+const isFileMusl = (f) => f.includes('libc.musl-') || f.includes('ld-musl-')
+
+const isMuslFromFilesystem = () => {
+  try {
+    return readFileSync('/usr/bin/ldd', 'utf-8').includes('musl')
+  } catch {
+    return null
+  }
+}
+
+const isMuslFromReport = () => {
+  let report = null
+  if (typeof process.report?.getReport === 'function') {
+    process.report.excludeNetwork = true
+    report = process.report.getReport()
+  }
+  if (!report) {
+    return null
+  }
+  if (report.header && report.header.glibcVersionRuntime) {
+    return false
+  }
+  if (Array.isArray(report.sharedObjects)) {
+    if (report.sharedObjects.some(isFileMusl)) {
+      return true
+    }
+  }
+  return false
+}
+
+const isMuslFromChildProcess = () => {
+  try {
+    return require('child_process').execSync('ldd --version', { encoding: 'utf8' }).includes('musl')
+  } catch (e) {
+    // If we reach this case, we don't know if the system is musl or not, so is better to just fallback to false
+    return false
+  }
+}
+
+function requireNative() {
+  if (process.env.NAPI_RS_NATIVE_LIBRARY_PATH) {
+    try {
+      return require(process.env.NAPI_RS_NATIVE_LIBRARY_PATH);
+    } catch (err) {
+      loadErrors.push(err)
+    }
+  } else if (process.platform === 'android') {
+    if (process.arch === 'arm64') {
+      try {
+        return require('./clip_tokenizer.android-arm64.node')
+      } catch (e) {
+        loadErrors.push(e)
+      }
+      try {
+        const binding = require('clip-tokenizer-rs-android-arm64')
+        const bindingPackageVersion = require('clip-tokenizer-rs-android-arm64/package.json').version
+        if (bindingPackageVersion !== '0.1.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 0.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        }
+        return binding
+      } catch (e) {
+        loadErrors.push(e)
+      }
+    } else if (process.arch === 'arm') {
+      try {
+        return require('./clip_tokenizer.android-arm-eabi.node')
+      } catch (e) {
+        loadErrors.push(e)
+      }
+      try {
+        const binding = require('clip-tokenizer-rs-android-arm-eabi')
+        const bindingPackageVersion = require('clip-tokenizer-rs-android-arm-eabi/package.json').version
+        if (bindingPackageVersion !== '0.1.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 0.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        }
+        return binding
+      } catch (e) {
+        loadErrors.push(e)
+      }
+    } else {
+      loadErrors.push(new Error(`Unsupported architecture on Android ${process.arch}`))
+    }
+  } else if (process.platform === 'win32') {
+    if (process.arch === 'x64') {
+      if (process.config?.variables?.shlib_suffix === 'dll.a' || process.config?.variables?.node_target_type === 'shared_library') {
+        try {
+        return require('./clip_tokenizer.win32-x64-gnu.node')
+      } catch (e) {
+        loadErrors.push(e)
+      }
+      try {
+        const binding = require('clip-tokenizer-rs-win32-x64-gnu')
+        const bindingPackageVersion = require('clip-tokenizer-rs-win32-x64-gnu/package.json').version
+        if (bindingPackageVersion !== '0.1.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 0.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        }
+        return binding
+      } catch (e) {
+        loadErrors.push(e)
+      }
+      } else {
+        try {
+        return require('./clip_tokenizer.win32-x64-msvc.node')
+      } catch (e) {
+        loadErrors.push(e)
+      }
+      try {
+        const binding = require('clip-tokenizer-rs-win32-x64-msvc')
+        const bindingPackageVersion = require('clip-tokenizer-rs-win32-x64-msvc/package.json').version
+        if (bindingPackageVersion !== '0.1.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 0.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        }
+        return binding
+      } catch (e) {
+        loadErrors.push(e)
+      }
+      }
+    } else if (process.arch === 'ia32') {
+      try {
+        return require('./clip_tokenizer.win32-ia32-msvc.node')
+      } catch (e) {
+        loadErrors.push(e)
+      }
+      try {
+        const binding = require('clip-tokenizer-rs-win32-ia32-msvc')
+        const bindingPackageVersion = require('clip-tokenizer-rs-win32-ia32-msvc/package.json').version
+        if (bindingPackageVersion !== '0.1.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 0.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        }
+        return binding
+      } catch (e) {
+        loadErrors.push(e)
+      }
+    } else if (process.arch === 'arm64') {
+      try {
+        return require('./clip_tokenizer.win32-arm64-msvc.node')
+      } catch (e) {
+        loadErrors.push(e)
+      }
+      try {
+        const binding = require('clip-tokenizer-rs-win32-arm64-msvc')
+        const bindingPackageVersion = require('clip-tokenizer-rs-win32-arm64-msvc/package.json').version
+        if (bindingPackageVersion !== '0.1.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 0.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        }
+        return binding
+      } catch (e) {
+        loadErrors.push(e)
+      }
+    } else {
+      loadErrors.push(new Error(`Unsupported architecture on Windows: ${process.arch}`))
+    }
+  } else if (process.platform === 'darwin') {
+    try {
+      return require('./clip_tokenizer.darwin-universal.node')
+    } catch (e) {
+      loadErrors.push(e)
+    }
+    try {
+      const binding = require('clip-tokenizer-rs-darwin-universal')
+      const bindingPackageVersion = require('clip-tokenizer-rs-darwin-universal/package.json').version
+      if (bindingPackageVersion !== '0.1.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+        throw new Error(`Native binding package version mismatch, expected 0.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+      }
+      return binding
+    } catch (e) {
+      loadErrors.push(e)
+    }
+    if (process.arch === 'x64') {
+      try {
+        return require('./clip_tokenizer.darwin-x64.node')
+      } catch (e) {
+        loadErrors.push(e)
+      }
+      try {
+        const binding = require('clip-tokenizer-rs-darwin-x64')
+        const bindingPackageVersion = require('clip-tokenizer-rs-darwin-x64/package.json').version
+        if (bindingPackageVersion !== '0.1.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 0.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        }
+        return binding
+      } catch (e) {
+        loadErrors.push(e)
+      }
+    } else if (process.arch === 'arm64') {
+      try {
+        return require('./clip_tokenizer.darwin-arm64.node')
+      } catch (e) {
+        loadErrors.push(e)
+      }
+      try {
+        const binding = require('clip-tokenizer-rs-darwin-arm64')
+        const bindingPackageVersion = require('clip-tokenizer-rs-darwin-arm64/package.json').version
+        if (bindingPackageVersion !== '0.1.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 0.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        }
+        return binding
+      } catch (e) {
+        loadErrors.push(e)
+      }
+    } else {
+      loadErrors.push(new Error(`Unsupported architecture on macOS: ${process.arch}`))
+    }
+  } else if (process.platform === 'freebsd') {
+    if (process.arch === 'x64') {
+      try {
+        return require('./clip_tokenizer.freebsd-x64.node')
+      } catch (e) {
+        loadErrors.push(e)
+      }
+      try {
+        const binding = require('clip-tokenizer-rs-freebsd-x64')
+        const bindingPackageVersion = require('clip-tokenizer-rs-freebsd-x64/package.json').version
+        if (bindingPackageVersion !== '0.1.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 0.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        }
+        return binding
+      } catch (e) {
+        loadErrors.push(e)
+      }
+    } else if (process.arch === 'arm64') {
+      try {
+        return require('./clip_tokenizer.freebsd-arm64.node')
+      } catch (e) {
+        loadErrors.push(e)
+      }
+      try {
+        const binding = require('clip-tokenizer-rs-freebsd-arm64')
+        const bindingPackageVersion = require('clip-tokenizer-rs-freebsd-arm64/package.json').version
+        if (bindingPackageVersion !== '0.1.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 0.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        }
+        return binding
+      } catch (e) {
+        loadErrors.push(e)
+      }
+    } else {
+      loadErrors.push(new Error(`Unsupported architecture on FreeBSD: ${process.arch}`))
+    }
+  } else if (process.platform === 'linux') {
+    if (process.arch === 'x64') {
+      if (isMusl()) {
+        try {
+          return require('./clip_tokenizer.linux-x64-musl.node')
+        } catch (e) {
+          loadErrors.push(e)
+        }
+        try {
+          const binding = require('clip-tokenizer-rs-linux-x64-musl')
+          const bindingPackageVersion = require('clip-tokenizer-rs-linux-x64-musl/package.json').version
+          if (bindingPackageVersion !== '0.1.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+            throw new Error(`Native binding package version mismatch, expected 0.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+          }
+          return binding
+        } catch (e) {
+          loadErrors.push(e)
+        }
+      } else {
+        try {
+          return require('./clip_tokenizer.linux-x64-gnu.node')
+        } catch (e) {
+          loadErrors.push(e)
+        }
+        try {
+          const binding = require('clip-tokenizer-rs-linux-x64-gnu')
+          const bindingPackageVersion = require('clip-tokenizer-rs-linux-x64-gnu/package.json').version
+          if (bindingPackageVersion !== '0.1.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+            throw new Error(`Native binding package version mismatch, expected 0.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+          }
+          return binding
+        } catch (e) {
+          loadErrors.push(e)
+        }
+      }
+    } else if (process.arch === 'arm64') {
+      if (isMusl()) {
+        try {
+          return require('./clip_tokenizer.linux-arm64-musl.node')
+        } catch (e) {
+          loadErrors.push(e)
+        }
+        try {
+          const binding = require('clip-tokenizer-rs-linux-arm64-musl')
+          const bindingPackageVersion = require('clip-tokenizer-rs-linux-arm64-musl/package.json').version
+          if (bindingPackageVersion !== '0.1.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+            throw new Error(`Native binding package version mismatch, expected 0.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+          }
+          return binding
+        } catch (e) {
+          loadErrors.push(e)
+        }
+      } else {
+        try {
+          return require('./clip_tokenizer.linux-arm64-gnu.node')
+        } catch (e) {
+          loadErrors.push(e)
+        }
+        try {
+          const binding = require('clip-tokenizer-rs-linux-arm64-gnu')
+          const bindingPackageVersion = require('clip-tokenizer-rs-linux-arm64-gnu/package.json').version
+          if (bindingPackageVersion !== '0.1.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+            throw new Error(`Native binding package version mismatch, expected 0.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+          }
+          return binding
+        } catch (e) {
+          loadErrors.push(e)
+        }
+      }
+    } else if (process.arch === 'arm') {
+      if (isMusl()) {
+        try {
+          return require('./clip_tokenizer.linux-arm-musleabihf.node')
+        } catch (e) {
+          loadErrors.push(e)
+        }
+        try {
+          const binding = require('clip-tokenizer-rs-linux-arm-musleabihf')
+          const bindingPackageVersion = require('clip-tokenizer-rs-linux-arm-musleabihf/package.json').version
+          if (bindingPackageVersion !== '0.1.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+            throw new Error(`Native binding package version mismatch, expected 0.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+          }
+          return binding
+        } catch (e) {
+          loadErrors.push(e)
+        }
+      } else {
+        try {
+          return require('./clip_tokenizer.linux-arm-gnueabihf.node')
+        } catch (e) {
+          loadErrors.push(e)
+        }
+        try {
+          const binding = require('clip-tokenizer-rs-linux-arm-gnueabihf')
+          const bindingPackageVersion = require('clip-tokenizer-rs-linux-arm-gnueabihf/package.json').version
+          if (bindingPackageVersion !== '0.1.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+            throw new Error(`Native binding package version mismatch, expected 0.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+          }
+          return binding
+        } catch (e) {
+          loadErrors.push(e)
+        }
+      }
+    } else if (process.arch === 'loong64') {
+      if (isMusl()) {
+        try {
+          return require('./clip_tokenizer.linux-loong64-musl.node')
+        } catch (e) {
+          loadErrors.push(e)
+        }
+        try {
+          const binding = require('clip-tokenizer-rs-linux-loong64-musl')
+          const bindingPackageVersion = require('clip-tokenizer-rs-linux-loong64-musl/package.json').version
+          if (bindingPackageVersion !== '0.1.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+            throw new Error(`Native binding package version mismatch, expected 0.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+          }
+          return binding
+        } catch (e) {
+          loadErrors.push(e)
+        }
+      } else {
+        try {
+          return require('./clip_tokenizer.linux-loong64-gnu.node')
+        } catch (e) {
+          loadErrors.push(e)
+        }
+        try {
+          const binding = require('clip-tokenizer-rs-linux-loong64-gnu')
+          const bindingPackageVersion = require('clip-tokenizer-rs-linux-loong64-gnu/package.json').version
+          if (bindingPackageVersion !== '0.1.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+            throw new Error(`Native binding package version mismatch, expected 0.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+          }
+          return binding
+        } catch (e) {
+          loadErrors.push(e)
+        }
+      }
+    } else if (process.arch === 'riscv64') {
+      if (isMusl()) {
+        try {
+          return require('./clip_tokenizer.linux-riscv64-musl.node')
+        } catch (e) {
+          loadErrors.push(e)
+        }
+        try {
+          const binding = require('clip-tokenizer-rs-linux-riscv64-musl')
+          const bindingPackageVersion = require('clip-tokenizer-rs-linux-riscv64-musl/package.json').version
+          if (bindingPackageVersion !== '0.1.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+            throw new Error(`Native binding package version mismatch, expected 0.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+          }
+          return binding
+        } catch (e) {
+          loadErrors.push(e)
+        }
+      } else {
+        try {
+          return require('./clip_tokenizer.linux-riscv64-gnu.node')
+        } catch (e) {
+          loadErrors.push(e)
+        }
+        try {
+          const binding = require('clip-tokenizer-rs-linux-riscv64-gnu')
+          const bindingPackageVersion = require('clip-tokenizer-rs-linux-riscv64-gnu/package.json').version
+          if (bindingPackageVersion !== '0.1.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+            throw new Error(`Native binding package version mismatch, expected 0.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+          }
+          return binding
+        } catch (e) {
+          loadErrors.push(e)
+        }
+      }
+    } else if (process.arch === 'ppc64') {
+      try {
+        return require('./clip_tokenizer.linux-ppc64-gnu.node')
+      } catch (e) {
+        loadErrors.push(e)
+      }
+      try {
+        const binding = require('clip-tokenizer-rs-linux-ppc64-gnu')
+        const bindingPackageVersion = require('clip-tokenizer-rs-linux-ppc64-gnu/package.json').version
+        if (bindingPackageVersion !== '0.1.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 0.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        }
+        return binding
+      } catch (e) {
+        loadErrors.push(e)
+      }
+    } else if (process.arch === 's390x') {
+      try {
+        return require('./clip_tokenizer.linux-s390x-gnu.node')
+      } catch (e) {
+        loadErrors.push(e)
+      }
+      try {
+        const binding = require('clip-tokenizer-rs-linux-s390x-gnu')
+        const bindingPackageVersion = require('clip-tokenizer-rs-linux-s390x-gnu/package.json').version
+        if (bindingPackageVersion !== '0.1.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 0.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        }
+        return binding
+      } catch (e) {
+        loadErrors.push(e)
+      }
+    } else {
+      loadErrors.push(new Error(`Unsupported architecture on Linux: ${process.arch}`))
+    }
+  } else if (process.platform === 'openharmony') {
+    if (process.arch === 'arm64') {
+      try {
+        return require('./clip_tokenizer.openharmony-arm64.node')
+      } catch (e) {
+        loadErrors.push(e)
+      }
+      try {
+        const binding = require('clip-tokenizer-rs-openharmony-arm64')
+        const bindingPackageVersion = require('clip-tokenizer-rs-openharmony-arm64/package.json').version
+        if (bindingPackageVersion !== '0.1.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 0.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        }
+        return binding
+      } catch (e) {
+        loadErrors.push(e)
+      }
+    } else if (process.arch === 'x64') {
+      try {
+        return require('./clip_tokenizer.openharmony-x64.node')
+      } catch (e) {
+        loadErrors.push(e)
+      }
+      try {
+        const binding = require('clip-tokenizer-rs-openharmony-x64')
+        const bindingPackageVersion = require('clip-tokenizer-rs-openharmony-x64/package.json').version
+        if (bindingPackageVersion !== '0.1.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 0.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        }
+        return binding
+      } catch (e) {
+        loadErrors.push(e)
+      }
+    } else if (process.arch === 'arm') {
+      try {
+        return require('./clip_tokenizer.openharmony-arm.node')
+      } catch (e) {
+        loadErrors.push(e)
+      }
+      try {
+        const binding = require('clip-tokenizer-rs-openharmony-arm')
+        const bindingPackageVersion = require('clip-tokenizer-rs-openharmony-arm/package.json').version
+        if (bindingPackageVersion !== '0.1.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 0.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        }
+        return binding
+      } catch (e) {
+        loadErrors.push(e)
+      }
+    } else {
+      loadErrors.push(new Error(`Unsupported architecture on OpenHarmony: ${process.arch}`))
+    }
+  } else {
+    loadErrors.push(new Error(`Unsupported OS: ${process.platform}, architecture: ${process.arch}`))
+  }
+}
+
+nativeBinding = requireNative()
+
+if (!nativeBinding || process.env.NAPI_RS_FORCE_WASI) {
+  let wasiBinding = null
+  let wasiBindingError = null
+  try {
+    wasiBinding = require('./clip_tokenizer.wasi.cjs')
+    nativeBinding = wasiBinding
+  } catch (err) {
+    if (process.env.NAPI_RS_FORCE_WASI) {
+      wasiBindingError = err
+    }
+  }
+  if (!nativeBinding || process.env.NAPI_RS_FORCE_WASI) {
+    try {
+      wasiBinding = require('clip-tokenizer-rs-wasm32-wasi')
+      nativeBinding = wasiBinding
+    } catch (err) {
+      if (process.env.NAPI_RS_FORCE_WASI) {
+        if (!wasiBindingError) {
+          wasiBindingError = err
+        } else {
+          wasiBindingError.cause = err
+        }
+        loadErrors.push(err)
+      }
+    }
+  }
+  if (process.env.NAPI_RS_FORCE_WASI === 'error' && !wasiBinding) {
+    const error = new Error('WASI binding not found and NAPI_RS_FORCE_WASI is set to error')
+    error.cause = wasiBindingError
+    throw error
+  }
+}
+
+if (!nativeBinding) {
+  if (loadErrors.length > 0) {
+    throw new Error(
+      `Cannot find native binding. ` +
+        `npm has a bug related to optional dependencies (https://github.com/npm/cli/issues/4828). ` +
+        'Please try `npm i` again after removing both package-lock.json and node_modules directory.',
+      {
+        cause: loadErrors.reduce((err, cur) => {
+          cur.cause = err
+          return cur
+        }),
+      },
+    )
+  }
+  throw new Error(`Failed to load native binding`)
+}
+
+module.exports = nativeBinding
+module.exports.count = nativeBinding.count
+module.exports.encode = nativeBinding.encode

package/package.json

@@ -0,0 +1,45 @@
+{
+  "name": "clip-tokenizer-rs",
+  "version": "0.1.0",
+  "type": "module",
+  "description": "Fast CLIP tokenizer using Rust via napi-rs",
+  "main": "index.js",
+  "types": "index.d.ts",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/nstandif/clip-tokenizer-rs.git"
+  },
+  "bugs": {
+    "url": "https://github.com/nstandif/clip-tokenizer-rs/issues"
+  },
+  "homepage": "https://github.com/nstandif/clip-tokenizer-rs#readme",
+  "files": [
+    "index.js",
+    "index.d.ts",
+    "*.node"
+  ],
+  "napi": {
+    "name": "clip_tokenizer",
+    "targets": [
+      "x86_64-apple-darwin",
+      "aarch64-apple-darwin",
+      "x86_64-unknown-linux-gnu",
+      "aarch64-unknown-linux-gnu"
+    ]
+  },
+  "scripts": {
+    "build": "napi build --release --platform --strip",
+    "build:debug": "napi build --platform",
+    "test": "bun test",
+    "ci:publish": "napi artifacts && npm publish --access public --provenance"
+  },
+  "devDependencies": {
+    "@napi-rs/cli": "^3.2.0",
+    "@semantic-release/changelog": "^6.0.3",
+    "@semantic-release/exec": "^6.0.3",
+    "@semantic-release/git": "^10.0.1",
+    "@semantic-release/github": "^11.0.1",
+    "semantic-release": "^24.2.0"
+  }
+}