@agentmemory/agentmemory 0.9.11 → 0.9.13

package/dist/{src-IbjZ5iJi.mjs → src-Ca9oX6Hq.mjs}

@@ -1,12 +1,14 @@
 import { a as jaccardSimilarity, i as generateId, n as STREAM, r as fingerprintId, t as KV } from "./cli.mjs";
-import { _ as loadSnapshotConfig, a as detectLlmProviderKind, d as isContextInjectionEnabled, f as isGraphExtractionEnabled, g as loadFallbackConfig, h as loadEmbeddingConfig, i as detectEmbeddingProvider, l as isAutoCompressEnabled, m as loadConfig, n as getVisibleTools, o as getConsolidationDecayDays, p as loadClaudeBridgeConfig, r as VERSION, s as getEnvVar, t as getAllTools, u as isConsolidationEnabled, v as loadTeamConfig } from "./tools-registry-CzSNN8Ca.mjs";
+import { a as isManagedImagePath, getImageRefCount, i as getMaxBytes, n as IMAGES_DIR, r as deleteImage, t as withKeyedLock } from "./image-refs-HVu22rfu.mjs";
+import { _ as loadSnapshotConfig, a as detectLlmProviderKind, d as isContextInjectionEnabled, f as isGraphExtractionEnabled, g as loadFallbackConfig, h as loadEmbeddingConfig, i as detectEmbeddingProvider, l as isAutoCompressEnabled, m as loadConfig, n as getVisibleTools, o as getConsolidationDecayDays, p as loadClaudeBridgeConfig, r as VERSION, s as getEnvVar, t as getAllTools, u as isConsolidationEnabled, v as loadTeamConfig } from "./tools-registry-D5l632PP.mjs";
+import { createRequire } from "node:module";
 import { execFile } from "node:child_process";
 import { constants, existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
 import { basename, dirname, extname, join, resolve, sep } from "node:path";
 import { fileURLToPath } from "node:url";
 import { homedir } from "node:os";
 import { createHash, createHmac, randomBytes, timingSafeEqual } from "node:crypto";
-import { lstat, mkdir, open, readFile, readdir, stat, unlink, utimes, writeFile } from "node:fs/promises";
+import { lstat, mkdir, open, readFile, readdir, stat, writeFile } from "node:fs/promises";
 import { TriggerAction, registerWorker } from "iii-sdk";
 import Anthropic from "@anthropic-ai/sdk";
 import { z } from "zod";
@@ -15,113 +17,6 @@ import { lookup } from "node:dns/promises";
 import { isIP } from "node:net";
 import { createServer } from "node:http";
 
-//#region src/utils/image-store.ts
-const IMAGES_DIR = join(homedir(), ".agentmemory", "images");
-const DEFAULT_MAX_BYTES = 500 * 1024 * 1024;
-function getMaxBytes() {
-	return Number(process.env.AGENTMEMORY_IMAGE_STORE_MAX_BYTES) || DEFAULT_MAX_BYTES;
-}
-function isManagedImagePath(filePath) {
-	const resolved = resolve(filePath);
-	const normalizedImagesDir = resolve(IMAGES_DIR);
-	return resolved.startsWith(normalizedImagesDir + sep) || resolved === normalizedImagesDir;
-}
-function contentHash(data) {
-	return createHash("sha256").update(data).digest("hex");
-}
-async function saveImageToDisk(base64Data) {
-	if (!base64Data) return {
-		filePath: "",
-		bytesWritten: 0
-	};
-	if (!existsSync(IMAGES_DIR)) await mkdir(IMAGES_DIR, { recursive: true });
-	let cleanBase64 = base64Data;
-	let ext = "png";
-	if (base64Data.startsWith("data:image/")) {
-		const commaIdx = base64Data.indexOf(",");
-		if (commaIdx !== -1) {
-			const meta = base64Data.substring(0, commaIdx);
-			if (meta.includes("jpeg") || meta.includes("jpg")) ext = "jpg";
-			else if (meta.includes("webp")) ext = "webp";
-			else if (meta.includes("gif")) ext = "gif";
-			cleanBase64 = base64Data.substring(commaIdx + 1);
-		}
-	} else if (base64Data.startsWith("/9j/")) ext = "jpg";
-	const filePath = join(IMAGES_DIR, `${contentHash(cleanBase64)}.${ext}`);
-	if (existsSync(filePath)) return {
-		filePath,
-		bytesWritten: 0
-	};
-	await writeFile(filePath, Buffer.from(cleanBase64, "base64"));
-	return {
-		filePath,
-		bytesWritten: (await stat(filePath)).size
-	};
-}
-async function deleteImage(filePath) {
-	if (!filePath) return { deletedBytes: 0 };
-	if (!isManagedImagePath(filePath)) return { deletedBytes: 0 };
-	try {
-		if (existsSync(filePath)) {
-			const size = (await stat(filePath)).size;
-			await unlink(filePath);
-			return { deletedBytes: size };
-		}
-	} catch (err) {
-		console.error("[agentmemory] Failed to delete image context:", err);
-	}
-	return { deletedBytes: 0 };
-}
-/** Touch an image file to update its mtime (marking it as recently used for LRU eviction) */
-async function touchImage(filePath) {
-	if (!filePath || !isManagedImagePath(filePath)) return;
-	try {
-		if (existsSync(filePath)) {
-			const now = /* @__PURE__ */ new Date();
-			await utimes(filePath, now, now);
-		}
-	} catch (err) {}
-}
-
-//#endregion
-//#region src/state/keyed-mutex.ts
-const locks = /* @__PURE__ */ new Map();
-function withKeyedLock(key, fn) {
-	const next = (locks.get(key) ?? Promise.resolve()).then(fn, fn);
-	const cleanup = next.then(() => {}, () => {});
-	locks.set(key, cleanup);
-	cleanup.then(() => {
-		if (locks.get(key) === cleanup) locks.delete(key);
-	});
-	return next;
-}
-
-//#endregion
-//#region src/functions/image-refs.ts
-async function getImageRefCount(kv, filePath) {
-	const count = await kv.get(KV.imageRefs, filePath);
-	return count ? Number(count) : 0;
-}
-async function incrementImageRef(kv, filePath) {
-	return withKeyedLock(`imgRef:${filePath}`, async () => {
-		const current = await getImageRefCount(kv, filePath);
-		await kv.set(KV.imageRefs, filePath, current + 1);
-		await touchImage(filePath);
-	});
-}
-async function decrementImageRef(kv, sdk, filePath) {
-	return withKeyedLock(`imgRef:${filePath}`, async () => {
-		const current = await getImageRefCount(kv, filePath);
-		if (current <= 1) {
-			await kv.delete(KV.imageEmbeddings, filePath);
-			await kv.delete(KV.imageRefs, filePath);
-			const { deletedBytes } = await deleteImage(filePath);
-			if (deletedBytes > 0) sdk.triggerVoid("mem::disk-size-delta", { deltaBytes: -deletedBytes });
-		} else await kv.set(KV.imageRefs, filePath, current - 1);
-	});
-}
-
-//#endregion
 //#region src/providers/agent-sdk.ts
 var AgentSDKProvider = class {
 	name = "agent-sdk";
@@ -462,7 +357,8 @@ var FallbackChainProvider = class {
 //#endregion
 //#region src/providers/embedding/gemini.ts
 const BATCH_LIMIT = 100;
-const API_BASE = "https://generativelanguage.googleapis.com/v1beta/models/text-embedding-004:batchEmbedContent";
+const MODEL = "models/gemini-embedding-001";
+const API_BASE = `https://generativelanguage.googleapis.com/v1beta/${MODEL}:batchEmbedContents`;
 var GeminiEmbeddingProvider = class {
 	name = "gemini";
 	dimensions = 768;
@@ -483,8 +379,9 @@ var GeminiEmbeddingProvider = class {
 				method: "POST",
 				headers: { "Content-Type": "application/json" },
 				body: JSON.stringify({ requests: chunk.map((t) => ({
-					model: "models/text-embedding-004",
-					content: { parts: [{ text: t }] }
+					model: MODEL,
+					content: { parts: [{ text: t }] },
+					outputDimensionality: this.dimensions
 				})) })
 			});
 			if (!response.ok) {
@@ -492,11 +389,26 @@ var GeminiEmbeddingProvider = class {
 				throw new Error(`Gemini embedding failed (${response.status}): ${err}`);
 			}
 			const data = await response.json();
-			for (const emb of data.embeddings) results.push(new Float32Array(emb.values));
+			for (const emb of data.embeddings) results.push(l2Normalize(new Float32Array(emb.values)));
 		}
 		return results;
 	}
 };
+let zeroNormWarned = false;
+function l2Normalize(vec) {
+	let sum = 0;
+	for (let i = 0; i < vec.length; i++) sum += vec[i] * vec[i];
+	const norm = Math.sqrt(sum);
+	if (norm === 0) {
+		if (!zeroNormWarned) {
+			zeroNormWarned = true;
+			process.stderr.write(`[agentmemory] warn: gemini-embedding-001 returned a zero-norm embedding (length=${vec.length}); leaving it un-normalized. Subsequent zero-norm vectors will not be reported.\n`);
+		}
+		return vec;
+	}
+	for (let i = 0; i < vec.length; i++) vec[i] = vec[i] / norm;
+	return vec;
+}
 
 //#endregion
 //#region src/providers/embedding/openai.ts
@@ -1898,6 +1810,110 @@ function getSynonyms(stemmedTerm) {
 	return syns ? [...syns] : [];
 }
 
+//#endregion
+//#region src/state/cjk-segmenter.ts
+const cjkRequire = createRequire(import.meta.url);
+const CJK_RE = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]/u;
+const KANA_RE = /[\p{Script=Hiragana}\p{Script=Katakana}]/u;
+const HANGUL_RE = /\p{Script=Hangul}/u;
+const CJK_RUN_RE = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]+/gu;
+const HANGUL_BLOCK_RE = /[가-힯]+/g;
+const hintShown = /* @__PURE__ */ new Set();
+function hasCjk(text) {
+	return CJK_RE.test(text);
+}
+function showHintOnce(key, message) {
+	if (hintShown.has(key)) return;
+	hintShown.add(key);
+	if (typeof process !== "undefined" && process.stderr?.write) process.stderr.write(`agentmemory: ${message}\n`);
+}
+let jiebaInstance = null;
+let jiebaLoaded = false;
+function getJieba() {
+	if (jiebaLoaded) return jiebaInstance;
+	jiebaLoaded = true;
+	try {
+		const mod = cjkRequire("@node-rs/jieba");
+		try {
+			const dictMod = cjkRequire("@node-rs/jieba/dict");
+			jiebaInstance = mod.Jieba.withDict(dictMod.dict);
+		} catch {
+			jiebaInstance = new mod.Jieba();
+		}
+		return jiebaInstance;
+	} catch {
+		showHintOnce("jieba", "install @node-rs/jieba to improve Chinese search; falling back to whole-string tokenization");
+		return null;
+	}
+}
+let jaSegmenterInstance = null;
+let jaSegmenterLoaded = false;
+function getJaSegmenter() {
+	if (jaSegmenterLoaded) return jaSegmenterInstance;
+	jaSegmenterLoaded = true;
+	try {
+		jaSegmenterInstance = new (cjkRequire("tiny-segmenter"))();
+		return jaSegmenterInstance;
+	} catch {
+		showHintOnce("tiny-segmenter", "install tiny-segmenter to improve Japanese search; falling back to whole-string tokenization");
+		return null;
+	}
+}
+function cleanTokens(tokens) {
+	const out = [];
+	for (const t of tokens) {
+		const trimmed = t.trim();
+		if (trimmed) out.push(trimmed);
+	}
+	return out;
+}
+function segmentHan(text) {
+	const j = getJieba();
+	if (!j) return [text];
+	try {
+		return cleanTokens(j.cut(text, true));
+	} catch {
+		return [text];
+	}
+}
+function segmentKana(text) {
+	const s = getJaSegmenter();
+	if (!s) return [text];
+	try {
+		return cleanTokens(s.segment(text));
+	} catch {
+		return [text];
+	}
+}
+function segmentHangul(text) {
+	const out = [];
+	for (const m of text.matchAll(HANGUL_BLOCK_RE)) if (m[0]) out.push(m[0]);
+	return out;
+}
+function segmentCjk(text) {
+	if (!hasCjk(text)) return [text];
+	const out = [];
+	let cursor = 0;
+	for (const m of text.matchAll(CJK_RUN_RE)) {
+		const start = m.index ?? 0;
+		const run = m[0];
+		const end = start + run.length;
+		if (start > cursor) {
+			const piece = text.slice(cursor, start).trim();
+			if (piece) out.push(piece);
+		}
+		if (HANGUL_RE.test(run)) out.push(...segmentHangul(run));
+		else if (KANA_RE.test(run)) out.push(...segmentKana(run));
+		else out.push(...segmentHan(run));
+		cursor = end;
+	}
+	if (cursor < text.length) {
+		const trailing = text.slice(cursor).trim();
+		if (trailing) out.push(trailing);
+	}
+	return out;
+}
+
 //#endregion
 //#region src/state/search-index.ts
 var SearchIndex = class SearchIndex {
@@ -2054,7 +2070,15 @@ var SearchIndex = class SearchIndex {
 		return this.tokenize(parts.join(" ").toLowerCase());
 	}
 	tokenize(text) {
-		return text.replace(/[^\w\s/.\-_]/g, " ").split(/\s+/).filter((t) => t.length > 1).map((t) => stem(t));
+		const cleaned = text.replace(/[^\p{L}\p{N}\s/.\\-_]/gu, " ");
+		const out = [];
+		for (const raw of cleaned.split(/\s+/)) {
+			if (raw.length < 2) continue;
+			if (hasCjk(raw)) {
+				for (const seg of segmentCjk(raw)) if (seg.length >= 1) out.push(seg);
+			} else out.push(stem(raw));
+		}
+		return out;
 	}
 	getSortedTerms() {
 		if (!this.sortedTerms) this.sortedTerms = Array.from(this.invertedIndex.keys()).sort();
@@ -2340,13 +2364,55 @@ async function deleteAccessLog(kv, memoryId) {
 //#endregion
 //#region src/functions/search.ts
 let index = null;
+let vectorIndex = null;
+let currentEmbeddingProvider = null;
 function getSearchIndex() {
 	if (!index) index = new SearchIndex();
 	return index;
 }
+function setVectorIndex(idx) {
+	vectorIndex = idx;
+}
+function setEmbeddingProvider(provider) {
+	currentEmbeddingProvider = provider;
+}
+const EMBED_MAX_CHARS = 16e3;
+function clipEmbedInput(text) {
+	if (text.length <= EMBED_MAX_CHARS) return text;
+	return text.slice(0, EMBED_MAX_CHARS);
+}
+async function vectorIndexAddGuarded(id, sessionId, text, context) {
+	const vi = vectorIndex;
+	const ep = currentEmbeddingProvider;
+	if (!vi || !ep) return false;
+	try {
+		const embedding = await ep.embed(clipEmbedInput(text));
+		if (embedding.length !== ep.dimensions) {
+			logger.warn("vector-index add: dimension mismatch — skipping", {
+				kind: context.kind,
+				id: context.logId,
+				provider: ep.name,
+				expected: ep.dimensions,
+				received: embedding.length
+			});
+			return false;
+		}
+		vi.add(id, sessionId, embedding);
+		return true;
+	} catch (err) {
+		logger.warn("vector-index add: embed failed — skipping", {
+			kind: context.kind,
+			id: context.logId,
+			provider: ep.name,
+			error: err instanceof Error ? err.message : String(err)
+		});
+		return false;
+	}
+}
 async function rebuildIndex(kv) {
 	const idx = getSearchIndex();
 	idx.clear();
+	vectorIndex?.clear();
 	let count = 0;
 	try {
 		const memories = await kv.list(KV.memories);
@@ -2354,6 +2420,10 @@ async function rebuildIndex(kv) {
 			if (memory.isLatest === false) continue;
 			if (!memory.title || !memory.content) continue;
 			idx.add(memoryToObservation(memory));
+			await vectorIndexAddGuarded(memory.id, memory.sessionIds[0] ?? "memory", memory.title + " " + memory.content, {
+				kind: "memory",
+				logId: memory.id
+			});
 			count++;
 		}
 	} catch (err) {
@@ -2378,6 +2448,10 @@ async function rebuildIndex(kv) {
 	if (failedSessions.length > 0) logger.warn("rebuildIndex: failed to load observations for sessions", { failedSessions });
 	for (const observations of obsPerSession) for (const obs of observations) if (obs.title && obs.narrative) {
 		idx.add(obs);
+		await vectorIndexAddGuarded(obs.id, obs.sessionId, obs.title + " " + obs.narrative, {
+			kind: "observation",
+			logId: obs.id
+		});
 		count++;
 	}
 	return count;
@@ -2600,10 +2674,10 @@ function registerObserveFunction(sdk, kv, dedupMap, maxObservationsPerSession) {
 				};
 			}
 			if (pendingImageData && (pendingImageData.startsWith("data:image/") || pendingImageData.startsWith("iVBORw0KGgo") || pendingImageData.startsWith("/9j/"))) {
-				const { saveImageToDisk } = await import("./image-store-DR4g1FAW.mjs");
+				const { saveImageToDisk } = await import("./image-store-BfN1vDbj.mjs");
 				const { filePath, bytesWritten } = await saveImageToDisk(pendingImageData);
 				raw.imageData = filePath;
-				const { incrementImageRef } = await import("./image-refs-D4M1sJAz.mjs");
+				const { incrementImageRef } = await import("./image-refs-HVu22rfu.mjs");
 				await incrementImageRef(kv, filePath);
 				sdk.triggerVoid("mem::disk-size-delta", { deltaBytes: bytesWritten });
 				if (process.env["AGENTMEMORY_IMAGE_EMBEDDINGS"] === "true") sdk.triggerVoid("mem::vision-embed", {
@@ -2616,7 +2690,7 @@ function registerObserveFunction(sdk, kv, dedupMap, maxObservationsPerSession) {
 				await kv.set(KV.observations(payload.sessionId), obsId, raw);
 			} catch (error) {
 				if (raw.imageData) {
-					const { deleteImage } = await import("./image-store-DR4g1FAW.mjs");
+					const { deleteImage } = await import("./image-store-BfN1vDbj.mjs");
 					const { deletedBytes } = await deleteImage(raw.imageData);
 					if (deletedBytes > 0) sdk.triggerVoid("mem::disk-size-delta", { deltaBytes: -deletedBytes });
 				}
@@ -2684,6 +2758,10 @@ function registerObserveFunction(sdk, kv, dedupMap, maxObservationsPerSession) {
 				const synthetic = buildSyntheticCompression(raw);
 				await kv.set(KV.observations(payload.sessionId), obsId, synthetic);
 				getSearchIndex().add(synthetic);
+				await vectorIndexAddGuarded(synthetic.id, synthetic.sessionId, synthetic.title + " " + (synthetic.narrative || ""), {
+					kind: "synthetic",
+					logId: synthetic.id
+				});
 				await sdk.trigger({
 					function_id: "stream::set",
 					payload: {
@@ -3821,7 +3899,20 @@ function registerCompressFunction(sdk, kv, provider, metricsStore) {
 				...data.raw.imageData ? { imageRef: data.raw.imageData } : {}
 			};
 			await kv.set(KV.observations(data.sessionId), data.observationId, compressed);
-			getSearchIndex().add(compressed);
+			try {
+				getSearchIndex().add(compressed);
+			} catch (err) {
+				logger.warn("Failed to index compressed observation into BM25", {
+					obsId: compressed.id,
+					sessionId: compressed.sessionId,
+					title: compressed.title,
+					error: err instanceof Error ? err.message : String(err)
+				});
+			}
+			await vectorIndexAddGuarded(compressed.id, compressed.sessionId, compressed.title + " " + (compressed.narrative || ""), {
+				kind: "observation",
+				logId: compressed.id
+			});
 			const streamResults = await Promise.allSettled([sdk.trigger({
 				function_id: "stream::set",
 				payload: {
@@ -4661,6 +4752,10 @@ function registerRememberFunction(sdk, kv) {
 					error: err instanceof Error ? err.message : String(err)
 				});
 			}
+			await vectorIndexAddGuarded(memory.id, memory.sessionIds[0] ?? "memory", memory.title + " " + memory.content, {
+				kind: "memory",
+				logId: memory.id
+			});
 			if (supersededId) await sdk.trigger({
 				function_id: "mem::cascade-update",
 				payload: { supersededMemoryId: supersededId },
@@ -4681,7 +4776,7 @@ function registerRememberFunction(sdk, kv) {
 		const deletedMemoryIds = [];
 		const deletedObservationIds = [];
 		let deletedSession = false;
-		const { decrementImageRef } = await import("./image-refs-D4M1sJAz.mjs");
+		const { decrementImageRef } = await import("./image-refs-HVu22rfu.mjs");
 		if (data.memoryId) {
 			const mem = await kv.get(KV.memories, data.memoryId);
 			await kv.delete(KV.memories, data.memoryId);
@@ -4740,7 +4835,7 @@ const DEFAULTS$1 = {
 function registerEvictFunction(sdk, kv) {
 	sdk.registerFunction("mem::evict", async (data) => {
 		const dryRun = data?.dryRun ?? false;
-		const { decrementImageRef } = await import("./image-refs-D4M1sJAz.mjs");
+		const { decrementImageRef } = await import("./image-refs-HVu22rfu.mjs");
 		const configOverride = await kv.get(KV.config, "eviction").catch(() => null);
 		const cfg = {
 			...DEFAULTS$1,
@@ -5343,7 +5438,7 @@ function registerAutoForgetFunction(sdk, kv) {
 	sdk.registerFunction("mem::auto-forget", async (data) => {
 		const dryRun = data?.dryRun ?? false;
 		const now = Date.now();
-		const { decrementImageRef } = await import("./image-refs-D4M1sJAz.mjs");
+		const { decrementImageRef } = await import("./image-refs-HVu22rfu.mjs");
 		const result = {
 			ttlExpired: [],
 			contradictions: [],
@@ -5583,7 +5678,9 @@ function registerExportImportFunction(sdk, kv) {
 			"0.9.8",
 			"0.9.9",
 			"0.9.10",
-			"0.9.11"
+			"0.9.11",
+			"0.9.12",
+			"0.9.13"
 		]).has(importData.version)) return {
 			success: false,
 			error: `Unsupported export version: ${importData.version}`
@@ -11940,7 +12037,7 @@ function registerRetentionFunctions(sdk, kv) {
 		const threshold = typeof data?.threshold === "number" && Number.isFinite(data.threshold) ? data.threshold : DEFAULT_DECAY.tierThresholds.cold;
 		const maxEvictRaw = typeof data?.maxEvict === "number" && Number.isInteger(data.maxEvict) ? data.maxEvict : 50;
 		const maxEvict = Math.min(1e3, Math.max(0, maxEvictRaw));
-		const { decrementImageRef } = await import("./image-refs-D4M1sJAz.mjs");
+		const { decrementImageRef } = await import("./image-refs-HVu22rfu.mjs");
 		const candidates = (await kv.list(KV.retentionScores)).filter((s) => s.score < threshold).sort((a, b) => a.score - b.score).slice(0, maxEvict);
 		if (data?.dryRun) return {
 			success: true,
@@ -18077,6 +18174,8 @@ async function main() {
 	const metricsStore = new MetricsStore(kv);
 	const dedupMap = new DedupMap();
 	const vectorIndex = embeddingProvider ? new VectorIndex() : null;
+	setVectorIndex(vectorIndex);
+	setEmbeddingProvider(embeddingProvider);
 	initMetrics(hasGetMeter(sdk) ? sdk.getMeter.bind(sdk) : void 0);
 	registerPrivacyFunction(sdk);
 	registerObserveFunction(sdk, kv, dedupMap, config.maxObservationsPerSession);
@@ -18293,5 +18392,5 @@ main().catch((err) => {
 });
 
 //#endregion
-export { deleteImage as a, saveImageToDisk as c, IMAGES_DIR as i, touchImage as l, getImageRefCount as n, getMaxBytes as o, incrementImageRef as r, isManagedImagePath as s, decrementImageRef as t };
-//# sourceMappingURL=src-IbjZ5iJi.mjs.map
+export {  };
+//# sourceMappingURL=src-Ca9oX6Hq.mjs.map