kiri-mcp-server 0.5.0 → 0.6.0

package/dist/src/shared/tokenizer.js

@@ -2,4 +2,39 @@ export function encode(text) {
     const codePoints = Array.from(text);
     return codePoints.map((_, index) => index);
 }
+/**
+ * 環境変数からトークン化戦略を取得
+ */
+export function getTokenizationStrategy() {
+    const strategy = process.env.KIRI_TOKENIZATION_STRATEGY?.toLowerCase();
+    if (strategy === "legacy" || strategy === "hybrid") {
+        return strategy;
+    }
+    return "phrase-aware"; // デフォルト
+}
+/**
+ * テキストをトークンに分割
+ * Unicode文字に対応し、戦略に応じてハイフンの扱いを変更
+ *
+ * @param text - トークン化するテキスト
+ * @param strategy - トークン化戦略（省略時は環境変数から取得）
+ * @returns トークンの配列
+ */
+export function tokenizeText(text, strategy) {
+    const effectiveStrategy = strategy ?? getTokenizationStrategy();
+    // レガシーモード: ハイフンも分割（従来の動作）
+    if (effectiveStrategy === "legacy") {
+        return text
+            .toLowerCase()
+            .split(/[^\p{L}\p{N}_]+/u)
+            .map((token) => token.trim())
+            .filter((token) => token.length > 0);
+    }
+    // phrase-aware または hybrid モード: ハイフンを保持
+    return text
+        .toLowerCase()
+        .split(/[^\p{L}\p{N}_-]+/u)
+        .map((token) => token.trim())
+        .filter((token) => token.length > 0);
+}
 //# sourceMappingURL=tokenizer.js.map

package/dist/src/shared/tokenizer.js.map

@@ -1 +1 @@
-{"version":3,"file":"tokenizer.js","sourceRoot":"","sources":["../../../src/shared/tokenizer.ts"],"names":[],"mappings":"AAAA,MAAM,UAAU,MAAM,CAAC,IAAY;IACjC,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACpC,OAAO,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC;AAC7C,CAAC"}
+{"version":3,"file":"tokenizer.js","sourceRoot":"","sources":["../../../src/shared/tokenizer.ts"],"names":[],"mappings":"AAAA,MAAM,UAAU,MAAM,CAAC,IAAY;IACjC,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACpC,OAAO,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC;AAC7C,CAAC;AASD;;GAEG;AACH,MAAM,UAAU,uBAAuB;IACrC,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,0BAA0B,EAAE,WAAW,EAAE,CAAC;IACvE,IAAI,QAAQ,KAAK,QAAQ,IAAI,QAAQ,KAAK,QAAQ,EAAE,CAAC;QACnD,OAAO,QAAQ,CAAC;IAClB,CAAC;IACD,OAAO,cAAc,CAAC,CAAC,QAAQ;AACjC,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,YAAY,CAAC,IAAY,EAAE,QAA+B;IACxE,MAAM,iBAAiB,GAAG,QAAQ,IAAI,uBAAuB,EAAE,CAAC;IAEhE,0BAA0B;IAC1B,IAAI,iBAAiB,KAAK,QAAQ,EAAE,CAAC;QACnC,OAAO,IAAI;aACR,WAAW,EAAE;aACb,KAAK,CAAC,kBAAkB,CAAC;aACzB,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;aAC5B,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACzC,CAAC;IAED,uCAAuC;IACvC,OAAO,IAAI;SACR,WAAW,EAAE;SACb,KAAK,CAAC,mBAAmB,CAAC;SAC1B,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;SAC5B,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AACzC,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "kiri-mcp-server",
-  "version": "0.5.0",
+  "version": "0.6.0",
   "description": "KIRI context extraction platform",
   "type": "module",
   "packageManager": "pnpm@9.0.0",