@cyberlangke/tokkit-microsoft 1.1.0

package/dist/index.cjs

@@ -0,0 +1,111 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __reExport = (target, mod, secondTarget) => (__copyProps(target, mod, "default"), secondTarget && __copyProps(secondTarget, mod, "default"));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var index_exports = {};
+__export(index_exports, {
+  registerBuiltins: () => registerBuiltins
+});
+module.exports = __toCommonJS(index_exports);
+var import_tokkit_core = require("@cyberlangke/tokkit-core");
+__reExport(index_exports, require("@cyberlangke/tokkit-core"), module.exports);
+const BUILTIN_FAMILIES = [
+  {
+    family: "phi-1",
+    aliases: ["phi1", "phi_1"],
+    models: ["microsoft/phi-1", "microsoft/phi-1_5", "microsoft/phi-2"],
+    modulePath: "./generated/phi_1.js"
+  },
+  {
+    family: "phi-3-mini",
+    aliases: ["phi3-mini", "phi_3_mini"],
+    models: ["microsoft/Phi-3-mini-4k-instruct", "microsoft/Phi-3-mini-128k-instruct"],
+    modulePath: "./generated/phi_3_mini.js"
+  },
+  {
+    family: "phi-3-medium",
+    aliases: ["phi3-medium", "phi_3_medium"],
+    models: [
+      "microsoft/Phi-3-medium-4k-instruct",
+      "microsoft/Phi-3-medium-128k-instruct"
+    ],
+    modulePath: "./generated/phi_3_medium.js"
+  },
+  {
+    family: "phi-3.5",
+    aliases: ["phi3.5", "phi_3_5"],
+    models: ["microsoft/Phi-3.5-mini-instruct", "microsoft/Phi-3.5-MoE-instruct"],
+    modulePath: "./generated/phi_3_5.js"
+  },
+  {
+    family: "phi-4",
+    aliases: ["phi4", "phi_4"],
+    models: ["microsoft/phi-4"],
+    modulePath: "./generated/phi_4.js"
+  },
+  {
+    family: "phi-4-mini",
+    aliases: ["phi4-mini", "phi_4_mini"],
+    models: ["microsoft/Phi-4-mini-instruct"],
+    modulePath: "./generated/phi_4_mini.js"
+  },
+  {
+    family: "phi-4-mini-flash",
+    aliases: ["phi4-mini-flash", "phi_4_mini_flash"],
+    models: ["microsoft/Phi-4-mini-flash-reasoning"],
+    modulePath: "./generated/phi_4_mini_flash.js"
+  },
+  {
+    family: "phi-4-mini-reasoning",
+    aliases: ["phi4-mini-reasoning", "phi_4_mini_reasoning"],
+    models: ["microsoft/Phi-4-mini-reasoning"],
+    modulePath: "./generated/phi_4_mini_reasoning.js"
+  },
+  {
+    family: "phi-4-reasoning",
+    aliases: ["phi4-reasoning", "phi_4_reasoning"],
+    models: ["microsoft/Phi-4-reasoning", "microsoft/Phi-4-reasoning-plus"],
+    modulePath: "./generated/phi_4_reasoning.js"
+  },
+  {
+    family: "phi-moe",
+    aliases: ["phi_moe"],
+    models: ["microsoft/Phi-mini-MoE-instruct", "microsoft/Phi-tiny-MoE-instruct"],
+    modulePath: "./generated/phi_moe.js"
+  }
+];
+function registerBuiltins() {
+  for (const spec of BUILTIN_FAMILIES) {
+    (0, import_tokkit_core.registerTokenizerFamily)({
+      family: spec.family,
+      aliases: [...spec.aliases],
+      models: [...spec.models],
+      load: () => loadFamilyAsset(spec.modulePath)
+    });
+  }
+}
+async function loadFamilyAsset(modulePath) {
+  const module2 = await import(modulePath);
+  return (0, import_tokkit_core.unpackPackedAsset)(module2.default);
+}
+registerBuiltins();
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  registerBuiltins,
+  ...require("@cyberlangke/tokkit-core")
+});

package/dist/index.d.cts

@@ -0,0 +1,16 @@
+export * from '@cyberlangke/tokkit-core';
+
+/**
+ * Microsoft family 包公共入口。
+ * 输入：Microsoft family 名称或其模型别名。
+ * 输出：自动注册 Microsoft 内置 family，并复用 core 的公共 API。
+ */
+
+/**
+ * 注册 Microsoft 子包内置的 family。
+ * 输入：无。
+ * 输出：Microsoft 相关 family 被写入全局注册表。
+ */
+declare function registerBuiltins(): void;
+
+export { registerBuiltins };

package/dist/index.d.ts

@@ -0,0 +1,16 @@
+export * from '@cyberlangke/tokkit-core';
+
+/**
+ * Microsoft family 包公共入口。
+ * 输入：Microsoft family 名称或其模型别名。
+ * 输出：自动注册 Microsoft 内置 family，并复用 core 的公共 API。
+ */
+
+/**
+ * 注册 Microsoft 子包内置的 family。
+ * 输入：无。
+ * 输出：Microsoft 相关 family 被写入全局注册表。
+ */
+declare function registerBuiltins(): void;
+
+export { registerBuiltins };

package/dist/index.js

@@ -0,0 +1,85 @@
+import { registerTokenizerFamily, unpackPackedAsset } from "@cyberlangke/tokkit-core";
+export * from "@cyberlangke/tokkit-core";
+const BUILTIN_FAMILIES = [
+  {
+    family: "phi-1",
+    aliases: ["phi1", "phi_1"],
+    models: ["microsoft/phi-1", "microsoft/phi-1_5", "microsoft/phi-2"],
+    modulePath: "./generated/phi_1.js"
+  },
+  {
+    family: "phi-3-mini",
+    aliases: ["phi3-mini", "phi_3_mini"],
+    models: ["microsoft/Phi-3-mini-4k-instruct", "microsoft/Phi-3-mini-128k-instruct"],
+    modulePath: "./generated/phi_3_mini.js"
+  },
+  {
+    family: "phi-3-medium",
+    aliases: ["phi3-medium", "phi_3_medium"],
+    models: [
+      "microsoft/Phi-3-medium-4k-instruct",
+      "microsoft/Phi-3-medium-128k-instruct"
+    ],
+    modulePath: "./generated/phi_3_medium.js"
+  },
+  {
+    family: "phi-3.5",
+    aliases: ["phi3.5", "phi_3_5"],
+    models: ["microsoft/Phi-3.5-mini-instruct", "microsoft/Phi-3.5-MoE-instruct"],
+    modulePath: "./generated/phi_3_5.js"
+  },
+  {
+    family: "phi-4",
+    aliases: ["phi4", "phi_4"],
+    models: ["microsoft/phi-4"],
+    modulePath: "./generated/phi_4.js"
+  },
+  {
+    family: "phi-4-mini",
+    aliases: ["phi4-mini", "phi_4_mini"],
+    models: ["microsoft/Phi-4-mini-instruct"],
+    modulePath: "./generated/phi_4_mini.js"
+  },
+  {
+    family: "phi-4-mini-flash",
+    aliases: ["phi4-mini-flash", "phi_4_mini_flash"],
+    models: ["microsoft/Phi-4-mini-flash-reasoning"],
+    modulePath: "./generated/phi_4_mini_flash.js"
+  },
+  {
+    family: "phi-4-mini-reasoning",
+    aliases: ["phi4-mini-reasoning", "phi_4_mini_reasoning"],
+    models: ["microsoft/Phi-4-mini-reasoning"],
+    modulePath: "./generated/phi_4_mini_reasoning.js"
+  },
+  {
+    family: "phi-4-reasoning",
+    aliases: ["phi4-reasoning", "phi_4_reasoning"],
+    models: ["microsoft/Phi-4-reasoning", "microsoft/Phi-4-reasoning-plus"],
+    modulePath: "./generated/phi_4_reasoning.js"
+  },
+  {
+    family: "phi-moe",
+    aliases: ["phi_moe"],
+    models: ["microsoft/Phi-mini-MoE-instruct", "microsoft/Phi-tiny-MoE-instruct"],
+    modulePath: "./generated/phi_moe.js"
+  }
+];
+function registerBuiltins() {
+  for (const spec of BUILTIN_FAMILIES) {
+    registerTokenizerFamily({
+      family: spec.family,
+      aliases: [...spec.aliases],
+      models: [...spec.models],
+      load: () => loadFamilyAsset(spec.modulePath)
+    });
+  }
+}
+async function loadFamilyAsset(modulePath) {
+  const module = await import(modulePath);
+  return unpackPackedAsset(module.default);
+}
+registerBuiltins();
+export {
+  registerBuiltins
+};

package/package.json

@@ -0,0 +1,40 @@
+{
+  "name": "@cyberlangke/tokkit-microsoft",
+  "version": "1.1.0",
+  "description": "Microsoft tokenizer families for tokkit.",
+  "type": "module",
+  "main": "./dist/index.cjs",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js",
+      "require": "./dist/index.cjs"
+    }
+  },
+  "files": [
+    "dist",
+    "COPYRIGHT"
+  ],
+  "scripts": {
+    "build": "tsup",
+    "dev": "tsup --watch"
+  },
+  "dependencies": {
+    "@cyberlangke/tokkit-core": "1.1.0"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/1cyberlangke1/tokkit.git"
+  },
+  "homepage": "https://github.com/1cyberlangke1/tokkit#readme",
+  "bugs": {
+    "url": "https://github.com/1cyberlangke1/tokkit/issues"
+  },
+  "author": "cyberlangke",
+  "license": "MIT"
+}