@agentmemory/agentmemory 0.7.4 → 0.7.6

package/dist/index.mjs

@@ -1183,12 +1183,66 @@ function extractEntitiesFromQuery(query) {
 	return [...new Set(entities)];
 }
 
+//#endregion
+//#region src/state/reranker.ts
+let pipeline = null;
+let pipelineLoading = null;
+let pipelineUnavailable = false;
+async function loadPipeline() {
+	if (pipelineUnavailable) return null;
+	if (pipeline) return pipeline;
+	if (pipelineLoading) return pipelineLoading;
+	pipelineLoading = (async () => {
+		try {
+			const { pipeline: createPipeline } = await import("./transformers-BX_tgxdO.mjs");
+			pipeline = await createPipeline("text-classification", "Xenova/ms-marco-MiniLM-L-6-v2", { quantized: true });
+			return pipeline;
+		} catch {
+			pipeline = null;
+			pipelineUnavailable = true;
+			return null;
+		} finally {
+			pipelineLoading = null;
+		}
+	})();
+	return pipelineLoading;
+}
+async function rerank(query, results, topK = 20) {
+	if (results.length <= 1) return results;
+	const reranker = await loadPipeline();
+	if (!reranker) return results;
+	const pairs = results.slice(0, Math.min(results.length, topK)).map((r) => ({
+		text: `${query} [SEP] ${r.observation.title || ""} ${r.observation.narrative || ""}`.slice(0, 512),
+		result: r
+	}));
+	const scores = [];
+	for (const pair of pairs) try {
+		const output = await reranker(pair.text);
+		const score = Array.isArray(output) ? output[0]?.score ?? 0 : 0;
+		scores.push({
+			result: pair.result,
+			rerankScore: score
+		});
+	} catch {
+		scores.push({
+			result: pair.result,
+			rerankScore: pair.result.combinedScore
+		});
+	}
+	scores.sort((a, b) => b.rerankScore - a.rerankScore);
+	return scores.map((s, i) => ({
+		...s.result,
+		combinedScore: s.rerankScore,
+		rerankPosition: i + 1
+	}));
+}
+
 //#endregion
 //#region src/state/hybrid-search.ts
 const RRF_K = 60;
 var HybridSearch = class {
 	graphRetrieval;
-	constructor(bm25, vector, embeddingProvider, kv, bm25Weight = .4, vectorWeight = .6, graphWeight = .3) {
+	constructor(bm25, vector, embeddingProvider, kv, bm25Weight = .4, vectorWeight = .6, graphWeight = .3, rerankEnabled = process.env.RERANK_ENABLED === "true") {
 		this.bm25 = bm25;
 		this.vector = vector;
 		this.embeddingProvider = embeddingProvider;
@@ -1196,6 +1250,7 @@ var HybridSearch = class {
 		this.bm25Weight = bm25Weight;
 		this.vectorWeight = vectorWeight;
 		this.graphWeight = graphWeight;
+		this.rerankEnabled = rerankEnabled;
 		this.graphRetrieval = new GraphRetrieval(kv);
 	}
 	async search(query, limit = 20) {
@@ -1299,8 +1354,18 @@ var HybridSearch = class {
 			combinedScore: effectiveBm25W * (1 / (RRF_K + s.bm25Rank)) + effectiveVectorW * (1 / (RRF_K + s.vectorRank)) + effectiveGraphW * (1 / (RRF_K + s.graphRank))
 		}));
 		combined.sort((a, b) => b.combinedScore - a.combinedScore);
-		const diversified = this.diversifyBySession(combined, limit);
-		return this.enrichResults(diversified, limit);
+		const retrievalDepth = Math.max(limit, 20);
+		const rerankWindow = 20;
+		const diversified = this.diversifyBySession(combined, retrievalDepth);
+		const enriched = await this.enrichResults(diversified, retrievalDepth);
+		if (this.rerankEnabled && enriched.length > 1) try {
+			const head = enriched.slice(0, rerankWindow);
+			const tail = enriched.slice(rerankWindow);
+			return (await rerank(query, head, rerankWindow)).concat(tail).slice(0, limit);
+		} catch {
+			return enriched.slice(0, limit);
+		}
+		return enriched.slice(0, limit);
 	}
 	diversifyBySession(results, limit, maxPerSession = 3) {
 		const selected = [];
@@ -2414,7 +2479,7 @@ function registerCompressFunction(sdk, kv, provider, metricsStore) {
 
 //#endregion
 //#region src/functions/context.ts
-function estimateTokens(text) {
+function estimateTokens$1(text) {
 	return Math.ceil(text.length / 3);
 }
 function escapeXmlAttr(s) {
@@ -2440,7 +2505,7 @@ function registerContextFunction(sdk, kv, tokenBudget) {
 				blocks.push({
 					type: "memory",
 					content: profileContent,
-					tokens: estimateTokens(profileContent),
+					tokens: estimateTokens$1(profileContent),
 					recency: new Date(profile.updatedAt).getTime()
 				});
 			}
@@ -2455,7 +2520,7 @@ function registerContextFunction(sdk, kv, tokenBudget) {
 				blocks.push({
 					type: "summary",
 					content,
-					tokens: estimateTokens(content),
+					tokens: estimateTokens$1(content),
 					recency: new Date(summary.createdAt).getTime()
 				});
 			} else sessionsNeedingObs.push(i);
@@ -2470,7 +2535,7 @@ function registerContextFunction(sdk, kv, tokenBudget) {
 				blocks.push({
 					type: "observation",
 					content,
-					tokens: estimateTokens(content),
+					tokens: estimateTokens$1(content),
 					recency: new Date(sessions[i].startedAt).getTime()
 				});
 			}
@@ -2480,7 +2545,7 @@ function registerContextFunction(sdk, kv, tokenBudget) {
 		const selected = [];
 		const header = `<agentmemory-context project="${escapeXmlAttr(data.project)}">`;
 		const footer = `</agentmemory-context>`;
-		usedTokens += estimateTokens(header) + estimateTokens(footer);
+		usedTokens += estimateTokens$1(header) + estimateTokens$1(footer);
 		for (const block of blocks) {
 			if (usedTokens + block.tokens > budget) break;
 			selected.push(block.content);
@@ -3740,7 +3805,7 @@ function registerAutoForgetFunction(sdk, kv) {
 
 //#endregion
 //#region src/version.ts
-const VERSION = "0.7.4";
+const VERSION = "0.7.6";
 
 //#endregion
 //#region src/functions/export-import.ts
@@ -3841,7 +3906,9 @@ function registerExportImportFunction(sdk, kv) {
 			"0.7.0",
 			"0.7.2",
 			"0.7.3",
-			"0.7.4"
+			"0.7.4",
+			"0.7.5",
+			"0.7.6"
 		]).has(importData.version)) return {
 			success: false,
 			error: `Unsupported export version: ${importData.version}`
@@ -8743,6 +8810,407 @@ function registerReflectFunctions(sdk, kv, provider) {
 	});
 }
 
+//#endregion
+//#region src/functions/working-memory.ts
+const CORE_SCOPE = "mem:core-memory";
+function estimateTokens(text) {
+	return Math.ceil(text.length / 3);
+}
+function scoreEntry(entry, now) {
+	const recencyScore = 1 / (1 + (now - new Date(entry.lastAccessedAt).getTime()) / (1e3 * 60 * 60 * 24) * .1);
+	const accessScore = Math.log2(entry.accessCount + 1) / 10;
+	return entry.importance / 10 * .5 + recencyScore * .3 + accessScore * .2;
+}
+function registerWorkingMemoryFunctions(sdk, kv, tokenBudget) {
+	sdk.registerFunction({
+		id: "mem::core-add",
+		description: "Add a fact to core memory (always included in context)"
+	}, async (data) => {
+		if (!data?.content?.trim()) return {
+			success: false,
+			error: "content is required"
+		};
+		const now = (/* @__PURE__ */ new Date()).toISOString();
+		const entry = {
+			id: generateId("core"),
+			content: data.content.trim(),
+			importance: Math.min(10, Math.max(1, data.importance ?? 7)),
+			pinned: data.pinned ?? false,
+			accessCount: 0,
+			lastAccessedAt: now,
+			createdAt: now
+		};
+		await kv.set(CORE_SCOPE, entry.id, entry);
+		try {
+			await recordAudit(kv, "core_add", "mem::core-add", [entry.id], {
+				content: entry.content.slice(0, 100),
+				importance: entry.importance,
+				pinned: entry.pinned
+			});
+		} catch {}
+		return {
+			success: true,
+			id: entry.id
+		};
+	});
+	sdk.registerFunction({
+		id: "mem::core-remove",
+		description: "Remove a fact from core memory"
+	}, async (data) => {
+		if (!data?.id) return {
+			success: false,
+			error: "id is required"
+		};
+		await kv.delete(CORE_SCOPE, data.id);
+		try {
+			await recordAudit(kv, "core_remove", "mem::core-remove", [data.id], {});
+		} catch {}
+		return { success: true };
+	});
+	sdk.registerFunction({
+		id: "mem::core-list",
+		description: "List all core memory entries"
+	}, async () => {
+		const entries = await kv.list(CORE_SCOPE);
+		entries.sort((a, b) => b.importance - a.importance);
+		return {
+			success: true,
+			entries,
+			totalTokens: entries.reduce((sum, e) => sum + estimateTokens(e.content), 0)
+		};
+	});
+	sdk.registerFunction({
+		id: "mem::working-context",
+		description: "Build working context: core memory (pinned) + paged archival (by relevance/recency)"
+	}, async (data) => {
+		const ctx = getContext();
+		const budget = data.budget || tokenBudget;
+		const now = Date.now();
+		let usedTokens = 0;
+		const coreEntries = await kv.list(CORE_SCOPE);
+		const pinned = coreEntries.filter((e) => e.pinned);
+		const unpinned = coreEntries.filter((e) => !e.pinned).sort((a, b) => scoreEntry(b, now) - scoreEntry(a, now));
+		const coreLines = [];
+		const coreBudget = Math.floor(budget * .3);
+		const accessUpdates = [];
+		const accessTimestamp = (/* @__PURE__ */ new Date()).toISOString();
+		for (const entry of [...pinned, ...unpinned]) {
+			const tokens = estimateTokens(entry.content);
+			if (usedTokens + tokens > coreBudget && !entry.pinned) continue;
+			coreLines.push(`- ${entry.content}`);
+			usedTokens += tokens;
+			entry.accessCount++;
+			entry.lastAccessedAt = accessTimestamp;
+			accessUpdates.push({
+				id: entry.id,
+				entry
+			});
+		}
+		Promise.allSettled(accessUpdates.map(({ id, entry }) => kv.set(CORE_SCOPE, id, entry))).catch(() => {});
+		const archivalLines = [];
+		const active = (await kv.list(KV.memories)).filter((m) => m.isLatest !== false).sort((a, b) => {
+			const strengthDiff = b.strength - a.strength;
+			if (Math.abs(strengthDiff) > .2) return strengthDiff;
+			return new Date(b.updatedAt).getTime() - new Date(a.updatedAt).getTime();
+		});
+		for (const mem of active) {
+			const tokens = estimateTokens(mem.content);
+			if (usedTokens + tokens > budget) continue;
+			archivalLines.push(`- [${mem.type}] ${mem.title}: ${mem.content}`);
+			usedTokens += tokens;
+		}
+		const pagedOut = active.length - archivalLines.length;
+		const sections = [];
+		if (coreLines.length > 0) sections.push(`## Core Memory\n${coreLines.join("\n")}`);
+		if (archivalLines.length > 0) sections.push(`## Archival Memory\n${archivalLines.join("\n")}`);
+		if (pagedOut > 0) sections.push(`_${pagedOut} memories paged to archival (use mem::search to retrieve)_`);
+		const context = sections.join("\n\n");
+		ctx.logger.info("Working context built", {
+			coreEntries: coreLines.length,
+			archivalEntries: archivalLines.length,
+			pagedOut,
+			tokens: usedTokens,
+			budget
+		});
+		return {
+			success: true,
+			context,
+			coreEntries: coreLines.length,
+			archivalEntries: archivalLines.length,
+			pagedOut,
+			tokens: usedTokens,
+			budget
+		};
+	});
+	sdk.registerFunction({
+		id: "mem::auto-page",
+		description: "Automatically page low-value core memory entries to archival when over budget"
+	}, async (data) => {
+		const budget = data?.budget || tokenBudget;
+		const coreBudget = Math.floor(budget * .3);
+		const entries = await kv.list(CORE_SCOPE);
+		let totalTokens = entries.reduce((sum, e) => sum + estimateTokens(e.content), 0);
+		if (totalTokens <= coreBudget) return {
+			success: true,
+			paged: 0,
+			totalTokens,
+			budget: coreBudget
+		};
+		const now = Date.now();
+		const unpinned = entries.filter((e) => !e.pinned).sort((a, b) => scoreEntry(a, now) - scoreEntry(b, now));
+		let paged = 0;
+		const pagedIds = [];
+		for (const entry of unpinned) {
+			if (totalTokens <= coreBudget) break;
+			const tokens = estimateTokens(entry.content);
+			const archivalMemory = {
+				id: generateId("mem"),
+				createdAt: entry.createdAt,
+				updatedAt: (/* @__PURE__ */ new Date()).toISOString(),
+				type: "fact",
+				title: entry.content.slice(0, 80),
+				content: entry.content,
+				concepts: [],
+				files: [],
+				sessionIds: [],
+				strength: entry.importance / 10,
+				version: 1,
+				isLatest: true
+			};
+			await kv.set(KV.memories, archivalMemory.id, archivalMemory);
+			await kv.delete(CORE_SCOPE, entry.id);
+			totalTokens -= tokens;
+			paged++;
+			pagedIds.push(entry.id);
+		}
+		if (paged > 0) try {
+			await recordAudit(kv, "auto_page", "mem::auto-page", pagedIds, {
+				paged,
+				budget: coreBudget
+			});
+		} catch {}
+		return {
+			success: true,
+			paged,
+			totalTokens,
+			budget: coreBudget
+		};
+	});
+}
+
+//#endregion
+//#region src/functions/skill-extract.ts
+const SKILL_EXTRACT_SYSTEM = `You are a skill extraction engine. Given a completed multi-step task session, extract a reusable procedural skill document.
+
+Output format:
+<skill>
+<trigger>When the agent encounters [specific situation/pattern]</trigger>
+<title>Short skill title</title>
+<steps>
+<step>First concrete action</step>
+<step>Second concrete action</step>
+</steps>
+<expected_outcome>What success looks like</expected_outcome>
+<tags>comma,separated,tags</tags>
+</skill>
+
+Rules:
+- Extract ONLY if the session shows a clear multi-step procedure that succeeded
+- Steps must be concrete and actionable, not vague
+- The trigger should describe WHEN to apply this skill
+- If the session is exploratory with no clear procedure, output <no-skill/>
+- Maximum 10 steps per skill`;
+function buildSkillPrompt(summary, observations) {
+	const obsText = observations.filter((o) => o.importance >= 4).sort((a, b) => new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime()).slice(0, 30).map((o) => `[${o.type}] ${o.title}${o.narrative ? ": " + o.narrative : ""}`).join("\n");
+	return `## Session Summary
+Title: ${summary.title}
+Narrative: ${summary.narrative}
+Key Decisions: ${summary.keyDecisions.join("; ")}
+Files Modified: ${summary.filesModified.join(", ")}
+Concepts: ${summary.concepts.join(", ")}
+
+## Observations (${observations.length} total, showing top by importance)
+${obsText}`;
+}
+function parseSkillXml(xml) {
+	if (xml.includes("<no-skill/>")) return null;
+	const triggerMatch = xml.match(/<trigger>([\s\S]*?)<\/trigger>/);
+	const titleMatch = xml.match(/<title>([\s\S]*?)<\/title>/);
+	const stepsMatch = xml.match(/<steps>([\s\S]*?)<\/steps>/);
+	const outcomeMatch = xml.match(/<expected_outcome>([\s\S]*?)<\/expected_outcome>/);
+	const tagsMatch = xml.match(/<tags>([\s\S]*?)<\/tags>/);
+	if (!triggerMatch || !titleMatch || !stepsMatch) return null;
+	const stepRegex = /<step>([\s\S]*?)<\/step>/g;
+	const steps = [];
+	let match;
+	while ((match = stepRegex.exec(stepsMatch[1])) !== null) {
+		const step = match[1].trim();
+		if (step) steps.push(step);
+	}
+	if (steps.length < 2) return null;
+	return {
+		trigger: triggerMatch[1].trim(),
+		title: titleMatch[1].trim(),
+		steps,
+		expectedOutcome: outcomeMatch?.[1]?.trim() || "",
+		tags: tagsMatch?.[1]?.split(",").map((t) => t.trim()).filter(Boolean) || []
+	};
+}
+function registerSkillExtractFunctions(sdk, kv, provider) {
+	sdk.registerFunction({
+		id: "mem::skill-extract",
+		description: "Extract a reusable skill document from a completed session"
+	}, async (data) => {
+		const ctx = getContext();
+		if (!data?.sessionId) return {
+			success: false,
+			error: "sessionId is required"
+		};
+		const session = await kv.get(KV.sessions, data.sessionId).catch(() => null);
+		if (!session) return {
+			success: false,
+			error: "session not found"
+		};
+		if (session.status !== "completed") return {
+			success: false,
+			error: "session must be completed before skill extraction"
+		};
+		const [summary, observations] = await Promise.all([kv.get(KV.summaries, data.sessionId).catch(() => null), kv.list(KV.observations(data.sessionId)).catch(() => [])]);
+		if (!summary) return {
+			success: false,
+			error: "no summary — run mem::summarize first"
+		};
+		if (observations.length < 3) return {
+			success: false,
+			error: "too few observations for skill extraction"
+		};
+		try {
+			const prompt = buildSkillPrompt(summary, observations);
+			const parsed = parseSkillXml(await provider.summarize(SKILL_EXTRACT_SYSTEM, prompt));
+			if (!parsed) {
+				ctx.logger.info("No skill extracted — session was exploratory", { sessionId: data.sessionId });
+				return {
+					success: true,
+					extracted: false,
+					reason: "no clear procedure found"
+				};
+			}
+			const fp = fingerprintId("skill", JSON.stringify({
+				title: parsed.title.toLowerCase(),
+				trigger: parsed.trigger.toLowerCase(),
+				steps: parsed.steps.map((s) => s.toLowerCase().trim())
+			}));
+			const existing = await kv.get(KV.procedural, fp).catch(() => null);
+			if (existing) {
+				if (!existing.sourceSessionIds.includes(data.sessionId)) {
+					existing.strength = Math.min(1, existing.strength + .15);
+					existing.frequency++;
+					existing.sourceSessionIds = [...existing.sourceSessionIds, data.sessionId];
+				}
+				existing.updatedAt = (/* @__PURE__ */ new Date()).toISOString();
+				await kv.set(KV.procedural, existing.id, existing);
+				try {
+					await recordAudit(kv, "skill_extract", "mem::skill-extract", [], {
+						skillId: existing.id,
+						reinforced: true,
+						sessionId: data.sessionId
+					});
+				} catch {}
+				ctx.logger.info("Skill reinforced", {
+					id: existing.id,
+					name: parsed.title
+				});
+				return {
+					success: true,
+					extracted: true,
+					reinforced: true,
+					skill: existing
+				};
+			}
+			const now = (/* @__PURE__ */ new Date()).toISOString();
+			const skill = {
+				id: fp,
+				name: parsed.title,
+				triggerCondition: parsed.trigger,
+				steps: parsed.steps,
+				expectedOutcome: parsed.expectedOutcome,
+				strength: .6,
+				frequency: 1,
+				tags: parsed.tags,
+				concepts: summary.concepts,
+				sourceSessionIds: [data.sessionId],
+				sourceObservationIds: observations.slice(0, 10).map((o) => o.id),
+				createdAt: now,
+				updatedAt: now
+			};
+			await kv.set(KV.procedural, skill.id, skill);
+			try {
+				await recordAudit(kv, "skill_extract", "mem::skill-extract", [], {
+					skillId: skill.id,
+					title: parsed.title,
+					steps: parsed.steps.length,
+					sessionId: data.sessionId
+				});
+			} catch {}
+			ctx.logger.info("Skill extracted", {
+				id: skill.id,
+				title: parsed.title,
+				steps: parsed.steps.length
+			});
+			return {
+				success: true,
+				extracted: true,
+				reinforced: false,
+				skill
+			};
+		} catch (err) {
+			const msg = err instanceof Error ? err.message : String(err);
+			ctx.logger.error("Skill extraction failed", { error: msg });
+			return {
+				success: false,
+				error: msg
+			};
+		}
+	});
+	sdk.registerFunction({
+		id: "mem::skill-list",
+		description: "List extracted procedural skills"
+	}, async (data) => {
+		const limit = data?.limit ?? 50;
+		const sorted = (await kv.list(KV.procedural)).sort((a, b) => b.strength - a.strength);
+		return {
+			success: true,
+			skills: sorted.slice(0, limit),
+			total: sorted.length
+		};
+	});
+	sdk.registerFunction({
+		id: "mem::skill-match",
+		description: "Find skills relevant to a given task description"
+	}, async (data) => {
+		if (!data?.query?.trim()) return {
+			success: false,
+			error: "query is required"
+		};
+		const limit = data.limit ?? 5;
+		const terms = data.query.toLowerCase().split(/\s+/).filter((t) => t.length > 2);
+		const scored = (await kv.list(KV.procedural)).map((skill) => {
+			const text = `${skill.name} ${skill.triggerCondition} ${(skill.tags || []).join(" ")} ${skill.steps.join(" ")}`.toLowerCase();
+			const matchCount = terms.filter((t) => text.includes(t)).length;
+			if (matchCount === 0) return null;
+			return {
+				skill,
+				score: matchCount / terms.length * skill.strength
+			};
+		}).filter(Boolean);
+		scored.sort((a, b) => b.score - a.score);
+		return {
+			success: true,
+			matches: scored.slice(0, limit)
+		};
+	});
+}
+
 //#endregion
 //#region src/functions/sliding-window.ts
 const SLIDING_WINDOW_SYSTEM = `You are a contextual enrichment engine. Given a primary observation and its surrounding context window (previous and next observations from the same session), produce an enriched version.
@@ -13895,6 +14363,10 @@ function startViewerServer(port, _kv, _sdk, secret, restPort) {
 			json(res, 502, { error: "upstream error" }, req);
 		}
 	});
+	server.on("error", (err) => {
+		if (err.code === "EADDRINUSE") console.warn(`[agentmemory] Viewer port ${port} already in use, skipping viewer.`);
+		else console.error(`[agentmemory] Viewer error:`, err.message);
+	});
 	server.listen(port, "127.0.0.1", () => {
 		console.log(`[agentmemory] Viewer: http://localhost:${port}`);
 	});
@@ -14153,6 +14625,8 @@ async function main() {
 	registerLessonsFunctions(sdk, kv);
 	registerObsidianExportFunction(sdk, kv);
 	registerReflectFunctions(sdk, kv, provider);
+	registerWorkingMemoryFunctions(sdk, kv, config.tokenBudget);
+	registerSkillExtractFunctions(sdk, kv, provider);
 	registerCascadeFunction(sdk, kv);
 	registerSlidingWindowFunction(sdk, kv, provider);
 	registerQueryExpansionFunction(sdk, provider);