openvino-tokenizers-node 2024.6.0-preview

package/README.md

@@ -0,0 +1,30 @@
+# OpenVINO Tokenizers
+
+This package contains binaries extension for
+[openvino-node](https://www.npmjs.com/package/openvino-node) package
+
+## Installation
+
+```bash
+npm install openvino-tokenizers-node
+```
+After package installation script that downloads and extracts binaries will be executed. Binaries will be placed in the `node_modules/openvino-tokenizers-node/bin` directory.
+
+This package is a part of [openvino-node](https://www.npmjs.com/package/openvino-node) package and should be used with it. Version of this package should be the same as version of openvino-node package.
+
+## Usage
+
+```javascript
+const { addon: ov } = require('openvino-node');
+const openvinoTokenizers = require('openvino-tokenizers-node');
+
+const core = new ov.Core();
+core.addExtension(openvinoTokenizers.path); // Add tokenizers extension
+
+// Load tokenizers model
+// ...
+```
+
+[License](https://github.com/openvinotoolkit/openvino/blob/master/LICENSE)
+
+Copyright © 2018-2024 Intel Corporation

package/openvino-tokenizers.js

@@ -0,0 +1,60 @@
+const os = require('node:os');
+const path = require('node:path');
+
+module.exports = {
+  path: getPathToBinary({
+    platform: os.platform(),
+    arch: os.arch(),
+  }),
+  getPathToBinary,
+};
+
+function getPathToBinary(osProps) {
+  const { arch, platform } = osProps;
+
+  if (platform === 'win32' && arch !== 'x64')
+    throw new Error(`Version for windows and '${arch}' is not supported.`);
+
+  return path.join(
+    __dirname,
+    'bin/runtime',
+    libOrBin(platform),
+    getDirnameByArch(arch),
+    platform === 'linux' ? '' : 'Release',
+    getBinaryFilename(platform),
+  );
+}
+
+function getDirnameByArch(arch) {
+  switch (arch) {
+    case 'x64':
+      return 'intel64';
+    case 'arm64':
+    case 'armhf':
+      return 'arm64';
+    default:
+      throw new Error(`Unsupported architecture: ${arch}`);
+  }
+}
+
+function libOrBin(platform) {
+  switch (platform) {
+    case 'win32':
+      return 'bin';
+    default:
+      return 'lib';
+  }
+}
+
+function getBinaryFilename(platform) {
+  switch (platform) {
+    case 'win32':
+      return 'openvino_tokenizers.dll';
+    case 'linux':
+      return 'libopenvino_tokenizers.so';
+    case 'darwin':
+      return 'libopenvino_tokenizers.dylib';
+    default:
+      throw new Error(`Unsupported platform: ${platform}`);
+  }
+}

package/package.json

@@ -0,0 +1,46 @@
+{
+  "name": "openvino-tokenizers-node",
+  "version": "2024.6.0-preview",
+  "description": "OpenVINO™ Tokenizers adds text processing operations to openvino-node package",
+  "repository": {
+    "url": "https://github.com/openvinotoolkit/openvino.git",
+    "type": "git"
+  },
+  "license": "Apache-2.0",
+  "main": "openvino-tokenizers.js",
+  "os": [
+    "win32",
+    "darwin",
+    "linux"
+  ],
+  "scripts": {
+    "postinstall": "npm run install_runtime",
+    "download_runtime": "node ./scripts/download-runtime.js",
+    "install_runtime": "npm run download_runtime -- --ignore-if-exists",
+    "test": "node --test ./tests/*.test.js"
+  },
+  "binary": {
+    "version": "2024.6.0.0",
+    "module_path": "./bin/",
+    "remote_path": "./repositories/openvino_tokenizers/packages/{version}/",
+    "package_name": "openvino_tokenizers_{platform}_{version}_{arch}.{extension}",
+    "host": "https://storage.openvinotoolkit.org"
+  },
+  "keywords": [
+    "OpenVINO",
+    "openvino",
+    "tokenizers",
+    "OpenVINO tokenizers",
+    "openvino tokenizers",
+    "openvino-tokenizers",
+    "openvino-tokenizers-node",
+    "tokenization",
+    "text processing",
+    "string tensors",
+    "text tensors"
+  ],
+  "dependencies": {
+    "adm-zip": "^0.5.16",
+    "openvino-node": "2024.6.0"
+  }
+}

package/scripts/download-runtime.js

@@ -0,0 +1,91 @@
+
+const os = require('node:os');
+const AdmZip = require('adm-zip');
+const { join } = require('node:path');
+const BinaryManager = require('openvino-node/scripts/lib/binary-manager');
+
+const packageJson = require('../package.json');
+
+class TokenizersBinaryManager extends BinaryManager {
+  getVersion() {
+    return `${ this.binaryConfig.version || this.version }.0`;
+  }
+
+  getPlatformLabel() {
+    return this.convertPlatformLabel(os.platform());
+  }
+
+  getArchLabel() {
+    return this.convertArchLabel(os.arch());
+  }
+
+  getExtension() {
+    return os.platform() === 'win32' ? 'zip' : 'tar.gz';
+  }
+
+  convertPlatformLabel(platform) {
+    switch(platform) {
+      case 'win32':
+        return 'windows';
+      case 'linux':
+        return 'centos7';
+      case 'darwin':
+        return 'macos_12_6';
+    }
+  }
+
+  convertArchLabel(arch) {
+    switch(arch) {
+      case 'x64':
+        return 'x86_64';
+      case 'arm64':
+      case 'armhf':
+        return 'arm64';
+    }
+  }
+
+  /**
+   * Unarchive tar, tar.gz or zip archives.
+   *
+   * @function unarchive
+   * @param {string} archivePath - Path to archive.
+   * @param {string} dest - Path where to unpack.
+   * @returns {Promise<void>}
+   */
+  static unarchive(archivePath, dest) {
+    if (archivePath.endsWith('.zip')) {
+      const zip = new AdmZip(archivePath);
+
+      return new Promise((resolve, reject) => {
+        zip.extractAllToAsync(dest, true, (err) => {
+          if (err) {
+            reject(err);
+          } else {
+            resolve();
+          }
+        });
+      });
+    }
+
+    return BinaryManager.unarchive(archivePath, dest);
+  }
+}
+
+if (require.main === module) main();
+
+async function main() {
+  if (!TokenizersBinaryManager.isCompatible()) process.exit(1);
+
+  const force = process.argv.includes('-f');
+  const ignoreIfExists = process.argv.includes('--ignore-if-exists');
+
+  const { env } = process;
+  const proxy = env.http_proxy || env.HTTP_PROXY || env.npm_config_proxy;
+
+  await TokenizersBinaryManager.prepareBinary(
+    join(__dirname, '..'),
+    packageJson.binary.version || packageJson.version,
+    packageJson.binary,
+    { force, ignoreIfExists, proxy },
+  );
+}