@agentmemory/agentmemory 0.9.12 → 0.9.13

package/dist/index.mjs

@@ -1,4 +1,5 @@
 #!/usr/bin/env node
+import { createRequire } from "node:module";
 import { TriggerAction, registerWorker } from "iii-sdk";
 import { constants, existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
 import { basename, dirname, extname, join, resolve, sep } from "node:path";
@@ -82,7 +83,7 @@ function detectProvider(env) {
 		if (!hasRealValue(env["GEMINI_API_KEY"]) && hasRealValue(env["GOOGLE_API_KEY"])) process.stderr.write("[agentmemory] GOOGLE_API_KEY detected — treating as GEMINI_API_KEY. Set GEMINI_API_KEY in ~/.agentmemory/.env to silence this warning.\n");
 		return {
 			provider: "gemini",
-			model: env["GEMINI_MODEL"] || "gemini-2.0-flash",
+			model: env["GEMINI_MODEL"] || "gemini-2.5-flash",
 			maxTokens
 		};
 	}
@@ -570,7 +571,8 @@ var FallbackChainProvider = class {
 //#endregion
 //#region src/providers/embedding/gemini.ts
 const BATCH_LIMIT = 100;
-const API_BASE = "https://generativelanguage.googleapis.com/v1beta/models/text-embedding-004:batchEmbedContent";
+const MODEL = "models/gemini-embedding-001";
+const API_BASE = `https://generativelanguage.googleapis.com/v1beta/${MODEL}:batchEmbedContents`;
 var GeminiEmbeddingProvider = class {
 	name = "gemini";
 	dimensions = 768;
@@ -591,8 +593,9 @@ var GeminiEmbeddingProvider = class {
 				method: "POST",
 				headers: { "Content-Type": "application/json" },
 				body: JSON.stringify({ requests: chunk.map((t) => ({
-					model: "models/text-embedding-004",
-					content: { parts: [{ text: t }] }
+					model: MODEL,
+					content: { parts: [{ text: t }] },
+					outputDimensionality: this.dimensions
 				})) })
 			});
 			if (!response.ok) {
@@ -600,11 +603,26 @@ var GeminiEmbeddingProvider = class {
 				throw new Error(`Gemini embedding failed (${response.status}): ${err}`);
 			}
 			const data = await response.json();
-			for (const emb of data.embeddings) results.push(new Float32Array(emb.values));
+			for (const emb of data.embeddings) results.push(l2Normalize(new Float32Array(emb.values)));
 		}
 		return results;
 	}
 };
+let zeroNormWarned = false;
+function l2Normalize(vec) {
+	let sum = 0;
+	for (let i = 0; i < vec.length; i++) sum += vec[i] * vec[i];
+	const norm = Math.sqrt(sum);
+	if (norm === 0) {
+		if (!zeroNormWarned) {
+			zeroNormWarned = true;
+			process.stderr.write(`[agentmemory] warn: gemini-embedding-001 returned a zero-norm embedding (length=${vec.length}); leaving it un-normalized. Subsequent zero-norm vectors will not be reported.\n`);
+		}
+		return vec;
+	}
+	for (let i = 0; i < vec.length; i++) vec[i] = vec[i] / norm;
+	return vec;
+}
 
 //#endregion
 //#region src/providers/embedding/openai.ts
@@ -2075,6 +2093,110 @@ function getSynonyms(stemmedTerm) {
 	return syns ? [...syns] : [];
 }
 
+//#endregion
+//#region src/state/cjk-segmenter.ts
+const cjkRequire = createRequire(import.meta.url);
+const CJK_RE = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]/u;
+const KANA_RE = /[\p{Script=Hiragana}\p{Script=Katakana}]/u;
+const HANGUL_RE = /\p{Script=Hangul}/u;
+const CJK_RUN_RE = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]+/gu;
+const HANGUL_BLOCK_RE = /[가-힯]+/g;
+const hintShown = /* @__PURE__ */ new Set();
+function hasCjk(text) {
+	return CJK_RE.test(text);
+}
+function showHintOnce(key, message) {
+	if (hintShown.has(key)) return;
+	hintShown.add(key);
+	if (typeof process !== "undefined" && process.stderr?.write) process.stderr.write(`agentmemory: ${message}\n`);
+}
+let jiebaInstance = null;
+let jiebaLoaded = false;
+function getJieba() {
+	if (jiebaLoaded) return jiebaInstance;
+	jiebaLoaded = true;
+	try {
+		const mod = cjkRequire("@node-rs/jieba");
+		try {
+			const dictMod = cjkRequire("@node-rs/jieba/dict");
+			jiebaInstance = mod.Jieba.withDict(dictMod.dict);
+		} catch {
+			jiebaInstance = new mod.Jieba();
+		}
+		return jiebaInstance;
+	} catch {
+		showHintOnce("jieba", "install @node-rs/jieba to improve Chinese search; falling back to whole-string tokenization");
+		return null;
+	}
+}
+let jaSegmenterInstance = null;
+let jaSegmenterLoaded = false;
+function getJaSegmenter() {
+	if (jaSegmenterLoaded) return jaSegmenterInstance;
+	jaSegmenterLoaded = true;
+	try {
+		jaSegmenterInstance = new (cjkRequire("tiny-segmenter"))();
+		return jaSegmenterInstance;
+	} catch {
+		showHintOnce("tiny-segmenter", "install tiny-segmenter to improve Japanese search; falling back to whole-string tokenization");
+		return null;
+	}
+}
+function cleanTokens(tokens) {
+	const out = [];
+	for (const t of tokens) {
+		const trimmed = t.trim();
+		if (trimmed) out.push(trimmed);
+	}
+	return out;
+}
+function segmentHan(text) {
+	const j = getJieba();
+	if (!j) return [text];
+	try {
+		return cleanTokens(j.cut(text, true));
+	} catch {
+		return [text];
+	}
+}
+function segmentKana(text) {
+	const s = getJaSegmenter();
+	if (!s) return [text];
+	try {
+		return cleanTokens(s.segment(text));
+	} catch {
+		return [text];
+	}
+}
+function segmentHangul(text) {
+	const out = [];
+	for (const m of text.matchAll(HANGUL_BLOCK_RE)) if (m[0]) out.push(m[0]);
+	return out;
+}
+function segmentCjk(text) {
+	if (!hasCjk(text)) return [text];
+	const out = [];
+	let cursor = 0;
+	for (const m of text.matchAll(CJK_RUN_RE)) {
+		const start = m.index ?? 0;
+		const run = m[0];
+		const end = start + run.length;
+		if (start > cursor) {
+			const piece = text.slice(cursor, start).trim();
+			if (piece) out.push(piece);
+		}
+		if (HANGUL_RE.test(run)) out.push(...segmentHangul(run));
+		else if (KANA_RE.test(run)) out.push(...segmentKana(run));
+		else out.push(...segmentHan(run));
+		cursor = end;
+	}
+	if (cursor < text.length) {
+		const trailing = text.slice(cursor).trim();
+		if (trailing) out.push(trailing);
+	}
+	return out;
+}
+
 //#endregion
 //#region src/state/search-index.ts
 var SearchIndex = class SearchIndex {
@@ -2231,7 +2353,15 @@ var SearchIndex = class SearchIndex {
 		return this.tokenize(parts.join(" ").toLowerCase());
 	}
 	tokenize(text) {
-		return text.replace(/[^\p{L}\p{N}\s/.\\-_]/gu, " ").split(/\s+/).filter((t) => t.length > 1).map((t) => stem(t));
+		const cleaned = text.replace(/[^\p{L}\p{N}\s/.\\-_]/gu, " ");
+		const out = [];
+		for (const raw of cleaned.split(/\s+/)) {
+			if (raw.length < 2) continue;
+			if (hasCjk(raw)) {
+				for (const seg of segmentCjk(raw)) if (seg.length >= 1) out.push(seg);
+			} else out.push(stem(raw));
+		}
+		return out;
 	}
 	getSortedTerms() {
 		if (!this.sortedTerms) this.sortedTerms = Array.from(this.invertedIndex.keys()).sort();
@@ -5826,7 +5956,7 @@ function registerAutoForgetFunction(sdk, kv) {
 
 //#endregion
 //#region src/version.ts
-const VERSION = "0.9.12";
+const VERSION = "0.9.13";
 
 //#endregion
 //#region src/functions/export-import.ts
@@ -5956,7 +6086,8 @@ function registerExportImportFunction(sdk, kv) {
 			"0.9.9",
 			"0.9.10",
 			"0.9.11",
-			"0.9.12"
+			"0.9.12",
+			"0.9.13"
 		]).has(importData.version)) return {
 			success: false,
 			error: `Unsupported export version: ${importData.version}`