@agentmemory/agentmemory 0.9.20 → 0.9.21

package/dist/index.mjs

@@ -137,6 +137,9 @@ function getMergedEnv(overrides) {
 function getEnvVar(key) {
 	return getMergedEnv()[key];
 }
+function isDropStaleIndexEnabled() {
+	return getMergedEnv()["AGENTMEMORY_DROP_STALE_INDEX"] === "true";
+}
 function detectLlmProviderKind() {
 	const env = getMergedEnv();
 	if (hasRealValue(env["ANTHROPIC_API_KEY"]) || hasRealValue(env["GEMINI_API_KEY"]) || hasRealValue(env["GOOGLE_API_KEY"]) || hasRealValue(env["OPENROUTER_API_KEY"]) || hasRealValue(env["MINIMAX_API_KEY"]) || hasRealValue(env["OPENAI_API_KEY"]) && env["OPENAI_API_KEY_FOR_LLM"] !== "false") return "llm";
@@ -2985,20 +2988,108 @@ async function vectorIndexAddGuarded(id, sessionId, text, context) {
 		return false;
 	}
 }
+async function vectorIndexAddBatchGuarded(items) {
+	const vi = vectorIndex;
+	const ep = currentEmbeddingProvider;
+	if (!vi || !ep || items.length === 0) return {
+		ok: 0,
+		fail: 0
+	};
+	let embeddings;
+	try {
+		embeddings = await ep.embedBatch(items.map((i) => clipEmbedInput(i.text)));
+	} catch (err) {
+		logger.warn("vector-index add batch: embed failed — skipping batch", {
+			batchSize: items.length,
+			provider: ep.name,
+			error: err instanceof Error ? err.message : String(err)
+		});
+		return {
+			ok: 0,
+			fail: items.length
+		};
+	}
+	if (embeddings.length !== items.length) {
+		logger.warn("vector-index add batch: provider returned wrong length — skipping batch", {
+			batchSize: items.length,
+			returned: embeddings.length,
+			provider: ep.name
+		});
+		return {
+			ok: 0,
+			fail: items.length
+		};
+	}
+	let ok = 0;
+	let fail = 0;
+	for (let i = 0; i < items.length; i++) {
+		const item = items[i];
+		const embedding = embeddings[i];
+		if (embedding.length !== ep.dimensions) {
+			logger.warn("vector-index add batch: dimension mismatch — skipping item", {
+				kind: item.context.kind,
+				id: item.context.logId,
+				provider: ep.name,
+				expected: ep.dimensions,
+				received: embedding.length
+			});
+			fail++;
+			continue;
+		}
+		try {
+			vi.add(item.id, item.sessionId, embedding);
+			ok++;
+		} catch (err) {
+			logger.warn("vector-index add batch: index write failed — skipping item", {
+				kind: item.context.kind,
+				id: item.context.logId,
+				error: err instanceof Error ? err.message : String(err)
+			});
+			fail++;
+		}
+	}
+	return {
+		ok,
+		fail
+	};
+}
+const DEFAULT_REBUILD_EMBED_BATCH = 32;
+function getRebuildEmbedBatchSize() {
+	const raw = process.env.REBUILD_EMBED_BATCH_SIZE;
+	if (!raw) return DEFAULT_REBUILD_EMBED_BATCH;
+	const n = parseInt(raw, 10);
+	return Number.isFinite(n) && n > 0 ? n : DEFAULT_REBUILD_EMBED_BATCH;
+}
 async function rebuildIndex(kv) {
 	const idx = getSearchIndex();
 	idx.clear();
 	vectorIndex?.clear();
+	const batchSize = getRebuildEmbedBatchSize();
+	const pending = [];
 	let count = 0;
+	const flush = async () => {
+		if (pending.length === 0) return;
+		await vectorIndexAddBatchGuarded(pending);
+		pending.length = 0;
+	};
+	const enqueue = async (job) => {
+		pending.push(job);
+		if (pending.length >= batchSize) await flush();
+	};
 	try {
 		const memories = await kv.list(KV.memories);
 		for (const memory of memories) {
 			if (memory.isLatest === false) continue;
 			if (!memory.title || !memory.content) continue;
 			idx.add(memoryToObservation(memory));
-			await vectorIndexAddGuarded(memory.id, memory.sessionIds[0] ?? "memory", memory.title + " " + memory.content, {
-				kind: "memory",
-				logId: memory.id
+			await enqueue({
+				id: memory.id,
+				sessionId: memory.sessionIds[0] ?? "memory",
+				text: memory.title + " " + memory.content,
+				context: {
+					kind: "memory",
+					logId: memory.id
+				}
 			});
 			count++;
 		}
@@ -3006,7 +3097,10 @@ async function rebuildIndex(kv) {
 		logger.warn("rebuildIndex: failed to load memories", { error: err instanceof Error ? err.message : String(err) });
 	}
 	const sessions = await kv.list(KV.sessions);
-	if (!sessions.length) return count;
+	if (!sessions.length) {
+		await flush();
+		return count;
+	}
 	const obsPerSession = [];
 	const failedSessions = [];
 	for (let batch = 0; batch < sessions.length; batch += 10) {
@@ -3024,12 +3118,18 @@ async function rebuildIndex(kv) {
 	if (failedSessions.length > 0) logger.warn("rebuildIndex: failed to load observations for sessions", { failedSessions });
 	for (const observations of obsPerSession) for (const obs of observations) if (obs.title && obs.narrative) {
 		idx.add(obs);
-		await vectorIndexAddGuarded(obs.id, obs.sessionId, obs.title + " " + obs.narrative, {
-			kind: "observation",
-			logId: obs.id
+		await enqueue({
+			id: obs.id,
+			sessionId: obs.sessionId,
+			text: obs.title + " " + obs.narrative,
+			context: {
+				kind: "observation",
+				logId: obs.id
+			}
 		});
 		count++;
 	}
+	await flush();
 	return count;
 }
 function registerSearchFunction(sdk, kv) {
@@ -4814,9 +4914,134 @@ function buildSummaryPrompt(observations) {
 	});
 	return `Session observations (${observations.length} total):\n\n${lines.join("\n\n---\n\n")}`;
 }
+const REDUCE_SYSTEM = `You are merging multiple partial summaries of the SAME coding session into one final session summary. The partials are chronological chunks of one continuous session — not separate sessions.
+
+Output EXACTLY this XML format with no additional text:
+
+<summary>
+  <title>Short session title (max 100 chars)</title>
+  <narrative>3-5 sentence narrative covering the whole session</narrative>
+  <decisions>
+    <decision>Key technical decision made</decision>
+  </decisions>
+  <files>
+    <file>path/to/modified/file</file>
+  </files>
+  <concepts>
+    <concept>key concept from session</concept>
+  </concepts>
+</summary>
+
+Rules:
+- Synthesize a single narrative that reflects the whole arc, not a chunk-by-chunk recap
+- Preserve every distinct decision across chunks
+- Union (deduplicate) all files and concepts
+- Title should capture the session's overall outcome`;
+function buildReducePrompt(partials) {
+	const sections = partials.map((p, i) => {
+		const decisions = p.keyDecisions.map((d) => `  - ${d}`).join("\n");
+		const files = p.filesModified.map((f) => `  - ${f}`).join("\n");
+		const concepts = p.concepts.join(", ");
+		return `[Chunk ${i + 1} of ${partials.length} — obs ${p.obsRangeStart}-${p.obsRangeEnd}]
+Title: ${p.title}
+Narrative: ${p.narrative}
+Decisions:
+${decisions}
+Files:
+${files}
+Concepts: ${concepts}`;
+	});
+	return `Partial summaries (${partials.length} chunks of one session, chronological):\n\n${sections.join("\n\n---\n\n")}`;
+}
 
 //#endregion
 //#region src/functions/summarize.ts
+const CHUNK_SIZE_DEFAULT = 400;
+const CHUNK_CONCURRENCY_DEFAULT = 6;
+const MAX_SKIP_RATIO = .5;
+function getChunkSize() {
+	const raw = process.env.SUMMARIZE_CHUNK_SIZE;
+	if (!raw) return CHUNK_SIZE_DEFAULT;
+	const n = parseInt(raw, 10);
+	return Number.isFinite(n) && n > 0 ? n : CHUNK_SIZE_DEFAULT;
+}
+function getChunkConcurrency() {
+	const raw = process.env.SUMMARIZE_CHUNK_CONCURRENCY;
+	if (!raw) return CHUNK_CONCURRENCY_DEFAULT;
+	const n = parseInt(raw, 10);
+	return Number.isFinite(n) && n > 0 ? n : CHUNK_CONCURRENCY_DEFAULT;
+}
+async function summarizeChunkWithRetry(provider, chunk, sessionId, project, idx, total) {
+	for (let attempt = 1; attempt <= 2; attempt++) try {
+		const parsed = parseSummaryXml(await provider.summarize(SUMMARY_SYSTEM, buildSummaryPrompt(chunk)), sessionId, project, chunk.length);
+		if (parsed) return parsed;
+		logger.warn("Summarize chunk parse failed", {
+			sessionId,
+			chunk: `${idx + 1}/${total}`,
+			attempt
+		});
+	} catch (err) {
+		logger.warn("Summarize chunk LLM call failed", {
+			sessionId,
+			chunk: `${idx + 1}/${total}`,
+			attempt,
+			error: err instanceof Error ? err.message : String(err)
+		});
+	}
+	return null;
+}
+async function produceSummaryXml(provider, compressed, sessionId, project) {
+	const chunkSize = getChunkSize();
+	if (compressed.length <= chunkSize) return {
+		response: await provider.summarize(SUMMARY_SYSTEM, buildSummaryPrompt(compressed)),
+		mode: "single",
+		chunks: 1
+	};
+	const chunks = [];
+	for (let i = 0; i < compressed.length; i += chunkSize) chunks.push(compressed.slice(i, i + chunkSize));
+	const concurrency = getChunkConcurrency();
+	logger.info("Summarize chunking session", {
+		sessionId,
+		chunks: chunks.length,
+		chunkSize,
+		concurrency,
+		totalObservations: compressed.length
+	});
+	const partialByIdx = new Array(chunks.length).fill(null);
+	for (let batchStart = 0; batchStart < chunks.length; batchStart += concurrency) {
+		const batch = chunks.slice(batchStart, batchStart + concurrency);
+		await Promise.all(batch.map(async (chunk, j) => {
+			const idx = batchStart + j;
+			partialByIdx[idx] = await summarizeChunkWithRetry(provider, chunk, sessionId, project, idx, chunks.length);
+		}));
+	}
+	const skipped = partialByIdx.filter((p) => p === null).length;
+	const partials = partialByIdx.filter((p) => p !== null);
+	if (skipped > Math.floor(chunks.length * MAX_SKIP_RATIO)) throw new Error(`too_many_chunks_skipped: ${skipped}/${chunks.length} chunks failed to parse after retry`);
+	if (skipped > 0) logger.warn("Summarize chunks partially skipped", {
+		sessionId,
+		skipped,
+		total: chunks.length
+	});
+	const reduceInput = partials.map((p) => {
+		const originalIdx = partialByIdx.indexOf(p);
+		return {
+			title: p.title,
+			narrative: p.narrative,
+			keyDecisions: p.keyDecisions,
+			filesModified: p.filesModified,
+			concepts: p.concepts,
+			obsRangeStart: originalIdx * chunkSize + 1,
+			obsRangeEnd: Math.min((originalIdx + 1) * chunkSize, compressed.length)
+		};
+	});
+	return {
+		response: await provider.summarize(REDUCE_SYSTEM, buildReducePrompt(reduceInput)),
+		mode: "chunked",
+		chunks: chunks.length,
+		skipped
+	};
+}
 function parseSummaryXml(xml, sessionId, project, obsCount) {
 	const title = getXmlTag(xml, "title");
 	if (!title) return null;
@@ -4865,16 +5090,15 @@ function registerSummarizeFunction(sdk, kv, provider, metricsStore) {
 			};
 		}
 		try {
-			const prompt = buildSummaryPrompt(compressed);
-			const response = await provider.summarize(SUMMARY_SYSTEM, prompt);
+			const { response, mode, chunks } = await produceSummaryXml(provider, compressed, sessionId, session.project);
 			if (!response || !response.trim()) {
 				const latencyMs = Date.now() - startMs;
 				if (metricsStore) await metricsStore.record("mem::summarize", latencyMs, false);
 				logger.warn("Empty provider response on summarize", {
 					sessionId,
 					provider: provider.name,
-					promptBytes: prompt.length,
-					systemBytes: SUMMARY_SYSTEM.length,
+					mode,
+					chunks,
 					observationCount: compressed.length
 				});
 				return {
@@ -6031,6 +6255,7 @@ async function findByKeyword(kv, keyword, project) {
 
 //#endregion
 //#region src/functions/smart-search.ts
+const LESSON_CONTENT_PREVIEW_CHARS = 240;
 function registerSmartSearchFunction(sdk, kv, searchFn) {
 	sdk.registerFunction("mem::smart-search", async (data) => {
 		if (data.expandIds && data.expandIds.length > 0) {
@@ -6073,7 +6298,10 @@ function registerSmartSearchFunction(sdk, kv, searchFn) {
 			error: "query is required"
 		};
 		const limit = Math.max(1, Math.min(data.limit ?? 20, 100));
-		const compact = (await searchFn(data.query, limit)).map((r) => ({
+		const lessonLimit = Math.min(limit, 10);
+		const includeLessons = data.includeLessons !== false;
+		const [hybridResults, lessons] = await Promise.all([searchFn(data.query, limit), includeLessons ? recallLessons(sdk, data.query, lessonLimit, data.project) : Promise.resolve([])]);
+		const compact = hybridResults.map((r) => ({
 			obsId: r.observation.id,
 			sessionId: r.sessionId,
 			title: r.observation.title,
@@ -6084,14 +6312,42 @@ function registerSmartSearchFunction(sdk, kv, searchFn) {
 		recordAccessBatch(kv, compact.map((r) => r.obsId));
 		logger.info("Smart search compact", {
 			query: data.query,
-			results: compact.length
+			results: compact.length,
+			lessons: lessons.length
 		});
-		return {
+		const response = {
 			mode: "compact",
 			results: compact
 		};
+		if (includeLessons) response.lessons = lessons;
+		return response;
 	});
 }
+async function recallLessons(sdk, query, limit, project) {
+	try {
+		const result = await sdk.trigger({
+			function_id: "mem::lesson-recall",
+			payload: {
+				query,
+				limit,
+				project
+			}
+		});
+		if (!result?.success || !Array.isArray(result.lessons)) return [];
+		return result.lessons.map((l) => ({
+			lessonId: l.id,
+			content: l.content.length > LESSON_CONTENT_PREVIEW_CHARS ? l.content.slice(0, LESSON_CONTENT_PREVIEW_CHARS) + "…" : l.content,
+			confidence: l.confidence,
+			score: l.score ?? l.confidence,
+			createdAt: l.createdAt,
+			project: l.project,
+			tags: l.tags ?? []
+		}));
+	} catch (err) {
+		logger.warn("Smart search: mem::lesson-recall failed; returning empty lesson list", { error: err instanceof Error ? err.message : String(err) });
+		return [];
+	}
+}
 async function findObservation$1(kv, obsId, sessionIdHint) {
 	if (sessionIdHint) {
 		const obs = await kv.get(KV.observations(sessionIdHint), obsId).catch(() => null);
@@ -6317,7 +6573,7 @@ function registerAutoForgetFunction(sdk, kv) {
 
 //#endregion
 //#region src/version.ts
-const VERSION = "0.9.20";
+const VERSION = "0.9.21";
 
 //#endregion
 //#region src/functions/export-import.ts
@@ -6455,7 +6711,8 @@ function registerExportImportFunction(sdk, kv) {
 			"0.9.17",
 			"0.9.18",
 			"0.9.19",
-			"0.9.20"
+			"0.9.20",
+			"0.9.21"
 		]).has(importData.version)) return {
 			success: false,
 			error: `Unsupported export version: ${importData.version}`
@@ -10157,6 +10414,12 @@ const ALL_CATEGORIES = [
 	"signals",
 	"sessions",
 	"memories",
+	"lessons",
+	"summaries",
+	"semantic",
+	"procedural",
+	"crystals",
+	"insights",
 	"mesh"
 ];
 const TWENTY_FOUR_HOURS_MS = 1440 * 60 * 1e3;
@@ -10389,6 +10652,133 @@ function registerDiagnosticsFunction(sdk, kv) {
 				fixable: false
 			});
 		}
+		if (categories.includes("lessons")) {
+			const lessons = await kv.list(KV.lessons);
+			const live = lessons.filter((l) => !l.deleted);
+			let lessonIssues = 0;
+			for (const l of live) if (!Number.isFinite(l.confidence) || l.confidence < 0 || l.confidence > 1) {
+				checks.push({
+					name: `lesson-bad-confidence:${l.id}`,
+					category: "lessons",
+					status: "warn",
+					message: `Lesson ${l.id} has confidence ${l.confidence} (expected finite number in 0..1)`,
+					fixable: false
+				});
+				lessonIssues++;
+			}
+			if (lessonIssues === 0) checks.push({
+				name: "lessons-ok",
+				category: "lessons",
+				status: "pass",
+				message: `All ${live.length} lessons are healthy (${lessons.length - live.length} tombstoned)`,
+				fixable: false
+			});
+		}
+		if (categories.includes("summaries")) {
+			const summaries = await kv.list(KV.summaries);
+			let summaryIssues = 0;
+			for (const s of summaries) if (typeof s.title !== "string" || s.title.trim().length === 0) {
+				checks.push({
+					name: `summary-missing-title:${s.sessionId}`,
+					category: "summaries",
+					status: "warn",
+					message: `Summary for session ${s.sessionId} has no title`,
+					fixable: false
+				});
+				summaryIssues++;
+			}
+			if (summaryIssues === 0) checks.push({
+				name: "summaries-ok",
+				category: "summaries",
+				status: "pass",
+				message: `All ${summaries.length} session summaries are consistent`,
+				fixable: false
+			});
+		}
+		if (categories.includes("semantic")) {
+			const semantic = await kv.list(KV.semantic);
+			let semanticIssues = 0;
+			for (const s of semantic) if (!Number.isFinite(s.confidence) || s.confidence < 0 || s.confidence > 1) {
+				checks.push({
+					name: `semantic-bad-confidence:${s.id}`,
+					category: "semantic",
+					status: "warn",
+					message: `Semantic fact ${s.id} has confidence ${s.confidence} (expected finite number in 0..1)`,
+					fixable: false
+				});
+				semanticIssues++;
+			}
+			if (semanticIssues === 0) checks.push({
+				name: "semantic-ok",
+				category: "semantic",
+				status: "pass",
+				message: `All ${semantic.length} semantic memories are consistent`,
+				fixable: false
+			});
+		}
+		if (categories.includes("procedural")) {
+			const procedural = await kv.list(KV.procedural);
+			let proceduralIssues = 0;
+			for (const p of procedural) if (!Array.isArray(p.steps) || p.steps.length === 0) {
+				checks.push({
+					name: `procedural-empty-steps:${p.id}`,
+					category: "procedural",
+					status: "warn",
+					message: `Procedural memory "${p.name}" (${p.id}) has no steps`,
+					fixable: false
+				});
+				proceduralIssues++;
+			}
+			if (proceduralIssues === 0) checks.push({
+				name: "procedural-ok",
+				category: "procedural",
+				status: "pass",
+				message: `All ${procedural.length} procedural memories are consistent`,
+				fixable: false
+			});
+		}
+		if (categories.includes("crystals")) {
+			const crystals = await kv.list(KV.crystals);
+			let crystalIssues = 0;
+			for (const c of crystals) if (typeof c.narrative !== "string" || c.narrative.trim().length === 0) {
+				checks.push({
+					name: `crystal-empty-narrative:${c.id}`,
+					category: "crystals",
+					status: "warn",
+					message: `Crystal ${c.id} has empty narrative`,
+					fixable: false
+				});
+				crystalIssues++;
+			}
+			if (crystalIssues === 0) checks.push({
+				name: "crystals-ok",
+				category: "crystals",
+				status: "pass",
+				message: `All ${crystals.length} crystals are consistent`,
+				fixable: false
+			});
+		}
+		if (categories.includes("insights")) {
+			const insights = await kv.list(KV.insights);
+			let insightIssues = 0;
+			for (const i of insights) if (!Number.isFinite(i.confidence) || i.confidence < 0 || i.confidence > 1) {
+				checks.push({
+					name: `insight-bad-confidence:${i.id}`,
+					category: "insights",
+					status: "warn",
+					message: `Insight ${i.id} has confidence ${i.confidence} (expected finite number in 0..1)`,
+					fixable: false
+				});
+				insightIssues++;
+			}
+			if (insightIssues === 0) checks.push({
+				name: "insights-ok",
+				category: "insights",
+				status: "pass",
+				message: `All ${insights.length} insights are consistent`,
+				fixable: false
+			});
+		}
 		if (categories.includes("mesh")) {
 			const peers = await kv.list(KV.mesh);
 			let meshIssues = 0;
@@ -14199,13 +14589,16 @@ function registerApiTriggers(sdk, kv, secret, metricsStore, provider) {
 			status_code: 400,
 			body: { error: "sessionId, project, and cwd are required non-empty strings" }
 		};
+		const title = typeof body.title === "string" ? body.title.trim() : void 0;
 		const session = {
 			id: sessionId,
 			project,
 			cwd,
 			startedAt: (/* @__PURE__ */ new Date()).toISOString(),
 			status: "active",
-			observationCount: 0
+			observationCount: 0,
+			...title ? { summary: title.slice(0, 200) } : {},
+			...title ? { firstPrompt: title.slice(0, 200) } : {}
 		};
 		await kv.set(KV.sessions, sessionId, session);
 		return {
@@ -20386,23 +20779,22 @@ async function main() {
 		if (mismatches.length > 0) {
 			const sample = mismatches.slice(0, 5).map((m) => `${m.obsId} (dim=${m.dim})`).join(", ");
 			const distinct = Array.from(seenDimensions).sort((a, b) => a - b).join(", ");
-			if (process.env["AGENTMEMORY_DROP_STALE_INDEX"] === "true") console.warn(`[agentmemory] Persisted vector index has ${mismatches.length} of ${loaded.vector.size} vectors with the wrong dimension. Active provider (${embeddingProvider?.name}) declares ${activeDim}; dimensions seen on disk: ${distinct}. AGENTMEMORY_DROP_STALE_INDEX=true is set — discarding the persisted vectors. Live observations will rebuild the index over time.`);
+			if (isDropStaleIndexEnabled()) console.warn(`[agentmemory] Persisted vector index has ${mismatches.length} of ${loaded.vector.size} vectors with the wrong dimension. Active provider (${embeddingProvider?.name}) declares ${activeDim}; dimensions seen on disk: ${distinct}. AGENTMEMORY_DROP_STALE_INDEX=true is set — discarding the persisted vectors. Live observations will rebuild the index over time.`);
 			else throw new Error(`[agentmemory] Refusing to start: persisted vector index has ${mismatches.length} of ${loaded.vector.size} vectors with the wrong dimension. Active provider (${embeddingProvider?.name}) declares ${activeDim}; dimensions seen on disk: ${distinct}. First mismatched obsIds: ${sample}. Loading would silently corrupt search (cross-dimension cosine returns 0). Choose one:\n  - Re-embed the existing index against the new provider, then start.\n  - Set AGENTMEMORY_DROP_STALE_INDEX=true to discard the persisted vectors and rebuild from live observations.\n  - Switch the embedding provider back to the one that wrote the index.`);
 		} else {
 			vectorIndex.restoreFrom(loaded.vector);
 			bootLog(`Loaded persisted vector index (${vectorIndex.size} vectors)`);
 		}
 	}
-	if (bm25Index.size === 0) {
-		const indexCount = await rebuildIndex(kv).catch((err) => {
-			console.warn(`[agentmemory] Failed to rebuild search index:`, err);
-			return 0;
-		});
+	if (bm25Index.size === 0) rebuildIndex(kv).then((indexCount) => {
 		if (indexCount > 0) {
 			bootLog(`Search index rebuilt: ${indexCount} entries`);
 			indexPersistence.scheduleSave();
 		}
-	} else try {
+	}).catch((err) => {
+		console.warn(`[agentmemory] Failed to rebuild search index:`, err);
+	});
+	else try {
 		const memories = await kv.list(KV.memories);
 		let backfilled = 0;
 		for (const memory of memories) {