@agentmemory/agentmemory 0.9.11 → 0.9.13

package/dist/index.mjs

@@ -1,4 +1,5 @@
 #!/usr/bin/env node
+import { createRequire } from "node:module";
 import { TriggerAction, registerWorker } from "iii-sdk";
 import { constants, existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
 import { basename, dirname, extname, join, resolve, sep } from "node:path";
@@ -82,7 +83,7 @@ function detectProvider(env) {
 		if (!hasRealValue(env["GEMINI_API_KEY"]) && hasRealValue(env["GOOGLE_API_KEY"])) process.stderr.write("[agentmemory] GOOGLE_API_KEY detected — treating as GEMINI_API_KEY. Set GEMINI_API_KEY in ~/.agentmemory/.env to silence this warning.\n");
 		return {
 			provider: "gemini",
-			model: env["GEMINI_MODEL"] || "gemini-2.0-flash",
+			model: env["GEMINI_MODEL"] || "gemini-2.5-flash",
 			maxTokens
 		};
 	}
@@ -570,7 +571,8 @@ var FallbackChainProvider = class {
 //#endregion
 //#region src/providers/embedding/gemini.ts
 const BATCH_LIMIT = 100;
-const API_BASE = "https://generativelanguage.googleapis.com/v1beta/models/text-embedding-004:batchEmbedContent";
+const MODEL = "models/gemini-embedding-001";
+const API_BASE = `https://generativelanguage.googleapis.com/v1beta/${MODEL}:batchEmbedContents`;
 var GeminiEmbeddingProvider = class {
 	name = "gemini";
 	dimensions = 768;
@@ -591,8 +593,9 @@ var GeminiEmbeddingProvider = class {
 				method: "POST",
 				headers: { "Content-Type": "application/json" },
 				body: JSON.stringify({ requests: chunk.map((t) => ({
-					model: "models/text-embedding-004",
-					content: { parts: [{ text: t }] }
+					model: MODEL,
+					content: { parts: [{ text: t }] },
+					outputDimensionality: this.dimensions
 				})) })
 			});
 			if (!response.ok) {
@@ -600,11 +603,26 @@ var GeminiEmbeddingProvider = class {
 				throw new Error(`Gemini embedding failed (${response.status}): ${err}`);
 			}
 			const data = await response.json();
-			for (const emb of data.embeddings) results.push(new Float32Array(emb.values));
+			for (const emb of data.embeddings) results.push(l2Normalize(new Float32Array(emb.values)));
 		}
 		return results;
 	}
 };
+let zeroNormWarned = false;
+function l2Normalize(vec) {
+	let sum = 0;
+	for (let i = 0; i < vec.length; i++) sum += vec[i] * vec[i];
+	const norm = Math.sqrt(sum);
+	if (norm === 0) {
+		if (!zeroNormWarned) {
+			zeroNormWarned = true;
+			process.stderr.write(`[agentmemory] warn: gemini-embedding-001 returned a zero-norm embedding (length=${vec.length}); leaving it un-normalized. Subsequent zero-norm vectors will not be reported.\n`);
+		}
+		return vec;
+	}
+	for (let i = 0; i < vec.length; i++) vec[i] = vec[i] / norm;
+	return vec;
+}
 
 //#endregion
 //#region src/providers/embedding/openai.ts
@@ -2075,6 +2093,110 @@ function getSynonyms(stemmedTerm) {
 	return syns ? [...syns] : [];
 }
 
+//#endregion
+//#region src/state/cjk-segmenter.ts
+const cjkRequire = createRequire(import.meta.url);
+const CJK_RE = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]/u;
+const KANA_RE = /[\p{Script=Hiragana}\p{Script=Katakana}]/u;
+const HANGUL_RE = /\p{Script=Hangul}/u;
+const CJK_RUN_RE = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]+/gu;
+const HANGUL_BLOCK_RE = /[가-힯]+/g;
+const hintShown = /* @__PURE__ */ new Set();
+function hasCjk(text) {
+	return CJK_RE.test(text);
+}
+function showHintOnce(key, message) {
+	if (hintShown.has(key)) return;
+	hintShown.add(key);
+	if (typeof process !== "undefined" && process.stderr?.write) process.stderr.write(`agentmemory: ${message}\n`);
+}
+let jiebaInstance = null;
+let jiebaLoaded = false;
+function getJieba() {
+	if (jiebaLoaded) return jiebaInstance;
+	jiebaLoaded = true;
+	try {
+		const mod = cjkRequire("@node-rs/jieba");
+		try {
+			const dictMod = cjkRequire("@node-rs/jieba/dict");
+			jiebaInstance = mod.Jieba.withDict(dictMod.dict);
+		} catch {
+			jiebaInstance = new mod.Jieba();
+		}
+		return jiebaInstance;
+	} catch {
+		showHintOnce("jieba", "install @node-rs/jieba to improve Chinese search; falling back to whole-string tokenization");
+		return null;
+	}
+}
+let jaSegmenterInstance = null;
+let jaSegmenterLoaded = false;
+function getJaSegmenter() {
+	if (jaSegmenterLoaded) return jaSegmenterInstance;
+	jaSegmenterLoaded = true;
+	try {
+		jaSegmenterInstance = new (cjkRequire("tiny-segmenter"))();
+		return jaSegmenterInstance;
+	} catch {
+		showHintOnce("tiny-segmenter", "install tiny-segmenter to improve Japanese search; falling back to whole-string tokenization");
+		return null;
+	}
+}
+function cleanTokens(tokens) {
+	const out = [];
+	for (const t of tokens) {
+		const trimmed = t.trim();
+		if (trimmed) out.push(trimmed);
+	}
+	return out;
+}
+function segmentHan(text) {
+	const j = getJieba();
+	if (!j) return [text];
+	try {
+		return cleanTokens(j.cut(text, true));
+	} catch {
+		return [text];
+	}
+}
+function segmentKana(text) {
+	const s = getJaSegmenter();
+	if (!s) return [text];
+	try {
+		return cleanTokens(s.segment(text));
+	} catch {
+		return [text];
+	}
+}
+function segmentHangul(text) {
+	const out = [];
+	for (const m of text.matchAll(HANGUL_BLOCK_RE)) if (m[0]) out.push(m[0]);
+	return out;
+}
+function segmentCjk(text) {
+	if (!hasCjk(text)) return [text];
+	const out = [];
+	let cursor = 0;
+	for (const m of text.matchAll(CJK_RUN_RE)) {
+		const start = m.index ?? 0;
+		const run = m[0];
+		const end = start + run.length;
+		if (start > cursor) {
+			const piece = text.slice(cursor, start).trim();
+			if (piece) out.push(piece);
+		}
+		if (HANGUL_RE.test(run)) out.push(...segmentHangul(run));
+		else if (KANA_RE.test(run)) out.push(...segmentKana(run));
+		else out.push(...segmentHan(run));
+		cursor = end;
+	}
+	if (cursor < text.length) {
+		const trailing = text.slice(cursor).trim();
+		if (trailing) out.push(trailing);
+	}
+	return out;
+}
+
 //#endregion
 //#region src/state/search-index.ts
 var SearchIndex = class SearchIndex {
@@ -2231,7 +2353,15 @@ var SearchIndex = class SearchIndex {
 		return this.tokenize(parts.join(" ").toLowerCase());
 	}
 	tokenize(text) {
-		return text.replace(/[^\w\s/.\-_]/g, " ").split(/\s+/).filter((t) => t.length > 1).map((t) => stem(t));
+		const cleaned = text.replace(/[^\p{L}\p{N}\s/.\\-_]/gu, " ");
+		const out = [];
+		for (const raw of cleaned.split(/\s+/)) {
+			if (raw.length < 2) continue;
+			if (hasCjk(raw)) {
+				for (const seg of segmentCjk(raw)) if (seg.length >= 1) out.push(seg);
+			} else out.push(stem(raw));
+		}
+		return out;
 	}
 	getSortedTerms() {
 		if (!this.sortedTerms) this.sortedTerms = Array.from(this.invertedIndex.keys()).sort();
@@ -2530,13 +2660,55 @@ async function deleteAccessLog(kv, memoryId) {
 //#endregion
 //#region src/functions/search.ts
 let index = null;
+let vectorIndex = null;
+let currentEmbeddingProvider = null;
 function getSearchIndex() {
 	if (!index) index = new SearchIndex();
 	return index;
 }
+function setVectorIndex(idx) {
+	vectorIndex = idx;
+}
+function setEmbeddingProvider(provider) {
+	currentEmbeddingProvider = provider;
+}
+const EMBED_MAX_CHARS = 16e3;
+function clipEmbedInput(text) {
+	if (text.length <= EMBED_MAX_CHARS) return text;
+	return text.slice(0, EMBED_MAX_CHARS);
+}
+async function vectorIndexAddGuarded(id, sessionId, text, context) {
+	const vi = vectorIndex;
+	const ep = currentEmbeddingProvider;
+	if (!vi || !ep) return false;
+	try {
+		const embedding = await ep.embed(clipEmbedInput(text));
+		if (embedding.length !== ep.dimensions) {
+			logger.warn("vector-index add: dimension mismatch — skipping", {
+				kind: context.kind,
+				id: context.logId,
+				provider: ep.name,
+				expected: ep.dimensions,
+				received: embedding.length
+			});
+			return false;
+		}
+		vi.add(id, sessionId, embedding);
+		return true;
+	} catch (err) {
+		logger.warn("vector-index add: embed failed — skipping", {
+			kind: context.kind,
+			id: context.logId,
+			provider: ep.name,
+			error: err instanceof Error ? err.message : String(err)
+		});
+		return false;
+	}
+}
 async function rebuildIndex(kv) {
 	const idx = getSearchIndex();
 	idx.clear();
+	vectorIndex?.clear();
 	let count = 0;
 	try {
 		const memories = await kv.list(KV.memories);
@@ -2544,6 +2716,10 @@ async function rebuildIndex(kv) {
 			if (memory.isLatest === false) continue;
 			if (!memory.title || !memory.content) continue;
 			idx.add(memoryToObservation(memory));
+			await vectorIndexAddGuarded(memory.id, memory.sessionIds[0] ?? "memory", memory.title + " " + memory.content, {
+				kind: "memory",
+				logId: memory.id
+			});
 			count++;
 		}
 	} catch (err) {
@@ -2568,6 +2744,10 @@ async function rebuildIndex(kv) {
 	if (failedSessions.length > 0) logger.warn("rebuildIndex: failed to load observations for sessions", { failedSessions });
 	for (const observations of obsPerSession) for (const obs of observations) if (obs.title && obs.narrative) {
 		idx.add(obs);
+		await vectorIndexAddGuarded(obs.id, obs.sessionId, obs.title + " " + obs.narrative, {
+			kind: "observation",
+			logId: obs.id
+		});
 		count++;
 	}
 	return count;
@@ -2874,6 +3054,10 @@ function registerObserveFunction(sdk, kv, dedupMap, maxObservationsPerSession) {
 				const synthetic = buildSyntheticCompression(raw);
 				await kv.set(KV.observations(payload.sessionId), obsId, synthetic);
 				getSearchIndex().add(synthetic);
+				await vectorIndexAddGuarded(synthetic.id, synthetic.sessionId, synthetic.title + " " + (synthetic.narrative || ""), {
+					kind: "synthetic",
+					logId: synthetic.id
+				});
 				await sdk.trigger({
 					function_id: "stream::set",
 					payload: {
@@ -4118,7 +4302,20 @@ function registerCompressFunction(sdk, kv, provider, metricsStore) {
 				...data.raw.imageData ? { imageRef: data.raw.imageData } : {}
 			};
 			await kv.set(KV.observations(data.sessionId), data.observationId, compressed);
-			getSearchIndex().add(compressed);
+			try {
+				getSearchIndex().add(compressed);
+			} catch (err) {
+				logger.warn("Failed to index compressed observation into BM25", {
+					obsId: compressed.id,
+					sessionId: compressed.sessionId,
+					title: compressed.title,
+					error: err instanceof Error ? err.message : String(err)
+				});
+			}
+			await vectorIndexAddGuarded(compressed.id, compressed.sessionId, compressed.title + " " + (compressed.narrative || ""), {
+				kind: "observation",
+				logId: compressed.id
+			});
 			const streamResults = await Promise.allSettled([sdk.trigger({
 				function_id: "stream::set",
 				payload: {
@@ -4958,6 +5155,10 @@ function registerRememberFunction(sdk, kv) {
 					error: err instanceof Error ? err.message : String(err)
 				});
 			}
+			await vectorIndexAddGuarded(memory.id, memory.sessionIds[0] ?? "memory", memory.title + " " + memory.content, {
+				kind: "memory",
+				logId: memory.id
+			});
 			if (supersededId) await sdk.trigger({
 				function_id: "mem::cascade-update",
 				payload: { supersededMemoryId: supersededId },
@@ -5755,7 +5956,7 @@ function registerAutoForgetFunction(sdk, kv) {
 
 //#endregion
 //#region src/version.ts
-const VERSION = "0.9.11";
+const VERSION = "0.9.13";
 
 //#endregion
 //#region src/functions/export-import.ts
@@ -5884,7 +6085,9 @@ function registerExportImportFunction(sdk, kv) {
 			"0.9.8",
 			"0.9.9",
 			"0.9.10",
-			"0.9.11"
+			"0.9.11",
+			"0.9.12",
+			"0.9.13"
 		]).has(importData.version)) return {
 			success: false,
 			error: `Unsupported export version: ${importData.version}`
@@ -19451,6 +19654,8 @@ async function main() {
 	const metricsStore = new MetricsStore(kv);
 	const dedupMap = new DedupMap();
 	const vectorIndex = embeddingProvider ? new VectorIndex() : null;
+	setVectorIndex(vectorIndex);
+	setEmbeddingProvider(embeddingProvider);
 	initMetrics(hasGetMeter(sdk) ? sdk.getMeter.bind(sdk) : void 0);
 	registerPrivacyFunction(sdk);
 	registerObserveFunction(sdk, kv, dedupMap, config.maxObservationsPerSession);