mnemocyte 0.1.0 → 0.1.2

package/dist/index.mjs

@@ -24,12 +24,16 @@ const vector = customType({
 		return `vector(${typeof config === "number" ? config : 1536})`;
 	},
 	toDriver(value) {
-		return `[${value.join(",")}]`;
+		return `[${value.map(formatVectorComponent$1).join(",")}]`;
 	},
 	fromDriver(value) {
 		return value.slice(1, -1).split(",").filter(Boolean).map(Number);
 	}
 });
+function formatVectorComponent$1(value) {
+	if (!Number.isFinite(value)) throw new Error("Vector values must be finite numbers.");
+	return Object.is(value, -0) ? "0" : value.toFixed(17);
+}
 const memoriesTable = pgTable("mnemocyte_memories", {
 	id: text("id").primaryKey(),
 	entityId: text("entity_id").notNull(),
@@ -43,7 +47,7 @@ const memoriesTable = pgTable("mnemocyte_memories", {
 	embedding: vector("embedding", { dimensions: 1536 }),
 	embeddingModel: text("embedding_model").notNull(),
 	embeddingDimensions: integer("embedding_dimensions").notNull(),
-	supersededBy: text("superseded_by"),
+	supersededBy: text("superseded_by").references(() => memoriesTable.id),
 	supersededAt: timestamp("superseded_at", { withTimezone: true }),
 	expiresAt: timestamp("expires_at", { withTimezone: true }),
 	lastAccessedAt: timestamp("last_accessed_at", { withTimezone: true }),
@@ -138,7 +142,11 @@ function formatMetadata(memory) {
 	if (memory.tags.length > 0) parts.push(`tags=${memory.tags.join(",")}`);
 	return parts.join("; ");
 }
-function formatMarkdown(memories, query) {
+function markdownFence(value) {
+	const longestBacktickRun = Math.max(0, ...Array.from(value.matchAll(/`+/g), (match) => match[0].length));
+	return "`".repeat(Math.max(3, longestBacktickRun + 1));
+}
+function formatMarkdown(memories, query, omittedCount) {
 	const lines = [
 		"# Memory Context",
 		"",
@@ -146,10 +154,14 @@ function formatMarkdown(memories, query) {
 		"",
 		"## Relevant Memories"
 	];
-	for (const memory of memories) lines.push("", `- ${memory.content}`, `  - ${formatMetadata(memory)}`);
+	for (const memory of memories) {
+		const fence = markdownFence(memory.content);
+		lines.push("", `- ${formatMetadata(memory)}`, `  ${fence}text`, ...memory.content.split("\n").map((line) => `  ${line}`), `  ${fence}`);
+	}
+	if (omittedCount > 0) lines.push("", `[${omittedCount} memories omitted to fit token budget]`);
 	return lines.join("\n");
 }
-function formatPlain(memories, query) {
+function formatPlain(memories, query, omittedCount) {
 	const lines = [
 		"MEMORY CONTEXT",
 		"",
@@ -157,20 +169,22 @@ function formatPlain(memories, query) {
 		"",
 		"RELEVANT MEMORIES"
 	];
-	for (const memory of memories) lines.push("", memory.content, formatMetadata(memory));
+	for (const [index, memory] of memories.entries()) lines.push("", `--- MEMORY ${index + 1} START ---`, formatMetadata(memory), memory.content, `--- MEMORY ${index + 1} END ---`);
+	if (omittedCount > 0) lines.push("", `[${omittedCount} memories omitted to fit token budget]`);
 	return lines.join("\n");
 }
-function formatXml(memories, query) {
+function formatXml(memories, query, omittedCount) {
 	const lines = [`<memory_context query="${escapeXml(query)}">`];
 	for (const memory of memories) lines.push(`  <memory id="${escapeXml(memory.id)}" type="${escapeXml(memory.type)}" importance="${escapeXml(memory.importance)}" score="${memory.score.toFixed(3)}">`, `    <content>${escapeXml(memory.content)}</content>`, `    <metadata>${escapeXml(formatMetadata(memory))}</metadata>`, "  </memory>");
+	if (omittedCount > 0) lines.push(`  <omitted count="${omittedCount}" reason="token_budget" />`);
 	lines.push("</memory_context>");
 	return lines.join("\n");
 }
-function formatContext(memories, query, format) {
+function formatContext(memories, query, format, omittedCount = 0) {
 	switch (format) {
-		case "markdown": return formatMarkdown(memories, query);
-		case "plain": return formatPlain(memories, query);
-		case "xml": return formatXml(memories, query);
+		case "markdown": return formatMarkdown(memories, query, omittedCount);
+		case "plain": return formatPlain(memories, query, omittedCount);
+		case "xml": return formatXml(memories, query, omittedCount);
 	}
 }
 //#endregion
@@ -199,7 +213,21 @@ async function buildContext(options) {
 	const format = options.input.format ?? "markdown";
 	const maxTokens = options.input.maxTokens ?? 1200;
 	const tokenCounter = options.input.tokenCounter ?? heuristicTokenCounter;
-	return trimToTokenBudget(formatContext(await options.recall(options.input), options.input.query, format), maxTokens, tokenCounter);
+	const memories = await options.recall(options.input);
+	let low = 0;
+	let high = memories.length;
+	let bestContext = "";
+	while (low <= high) {
+		const mid = Math.floor((low + high) / 2);
+		const omittedCount = memories.length - mid;
+		const context = formatContext(memories.slice(0, mid), options.input.query, format, omittedCount);
+		if (tokenCounter.count(context) <= maxTokens) {
+			bestContext = context;
+			low = mid + 1;
+		} else high = mid - 1;
+	}
+	if (bestContext || memories.length === 0) return bestContext;
+	return trimToTokenBudget(formatContext([], options.input.query, format, memories.length), maxTokens, tokenCounter);
 }
 //#endregion
 //#region src/observability.ts
@@ -499,6 +527,22 @@ async function embedOne(embedder, text, options = {}) {
 	if (embedding.length !== embedder.dimensions) throw new MnemocyteError("Embedder returned an embedding with unexpected dimensions.", "EMBEDDING");
 	return embedding;
 }
+async function embedMany(embedder, texts, options = {}) {
+	if (texts.length === 0) return [];
+	let embeddings;
+	try {
+		embeddings = await withResilience((signal) => signal === void 0 ? embedder.embed(texts) : embedder.embed(texts, { signal }), {
+			...options.signal === void 0 ? {} : { signal: options.signal },
+			...options.resilience === void 0 ? {} : { resilience: options.resilience }
+		});
+	} catch (error) {
+		if (error instanceof MnemocyteError && (error.code === "TIMEOUT" || error.code === "ABORTED")) throw error;
+		throw new MnemocyteError("Failed to embed texts.", "EMBEDDING", error);
+	}
+	if (embeddings.length !== texts.length) throw new MnemocyteError(`Embedder returned ${embeddings.length} embeddings for ${texts.length} texts.`, "EMBEDDING");
+	for (const embedding of embeddings) if (embedding.length !== embedder.dimensions) throw new MnemocyteError("Embedder returned an embedding with unexpected dimensions.", "EMBEDDING");
+	return embeddings;
+}
 function validateRememberInput(input) {
 	assertNonEmptyString(input.entityId, "entityId");
 	assertNonEmptyString(input.content, "content");
@@ -778,8 +822,45 @@ function createInMemoryClient(config) {
 		async rememberMany(inputs) {
 			return observe(config, "in-memory", "rememberMany", { count: inputs.length }, async () => {
 				assertOpen();
+				if (inputs.length === 0) return [];
+				for (const input of inputs) validateRememberInput(input);
+				const embeddings = await embedMany(config.embedder, inputs.map((i) => i.content), {
+					...inputs[0]?.signal === void 0 ? {} : { signal: inputs[0].signal },
+					...config.provider === void 0 ? {} : { resilience: config.provider }
+				});
+				const now = /* @__PURE__ */ new Date();
 				const result = [];
-				for (const input of inputs) result.push(await remember(input));
+				for (let i = 0; i < inputs.length; i++) {
+					const input = inputs[i];
+					const memory = {
+						id: createId(),
+						entityId: input.entityId,
+						content: input.content,
+						type: input.type ?? "fact",
+						importance: input.importance ?? "normal",
+						tags: normalizeTags(input.tags),
+						source: input.source ?? null,
+						metadata: input.metadata ?? {},
+						confidence: input.confidence ?? 1,
+						embeddingModel: config.embedder.model,
+						embeddingDimensions: config.embedder.dimensions,
+						supersededBy: null,
+						supersededAt: null,
+						expiresAt: input.expiresAt ?? null,
+						lastAccessedAt: null,
+						accessCount: 0,
+						createdAt: now,
+						updatedAt: now,
+						embedding: embeddings[i]
+					};
+					memories.set(memory.id, memory);
+					recordAudit(memory.entityId, "memory.created", {
+						memoryId: memory.id,
+						type: memory.type,
+						importance: memory.importance
+					});
+					result.push(cloneMemory(memory));
+				}
 				return result;
 			}, (result) => ({ count: result.length }));
 		},
@@ -1009,16 +1090,24 @@ async function listEvents(db, filter) {
 //#endregion
 //#region src/db/queries/memories.ts
 function vectorLiteral(embedding) {
-	return `[${embedding.join(",")}]`;
+	return `[${embedding.map(formatVectorComponent).join(",")}]`;
+}
+function formatVectorComponent(value) {
+	if (!Number.isFinite(value)) throw new Error("Vector values must be finite numbers.");
+	return Object.is(value, -0) ? "0" : value.toFixed(17);
 }
 function filterConditions(filter) {
-	return and(eq(memoriesTable.entityId, filter.entityId), filter.includeSuperseded ? void 0 : isNull(memoriesTable.supersededBy), filter.includeExpired ? void 0 : sql`(${memoriesTable.expiresAt} IS NULL OR ${memoriesTable.expiresAt} > now())`, filter.types && filter.types.length > 0 ? inArray(memoriesTable.type, [...filter.types]) : void 0, filter.tags && filter.tags.length > 0 ? sql`${memoriesTable.tags} && ${filter.tags}` : void 0, filter.before ? lt(memoriesTable.createdAt, filter.before) : void 0, filter.after ? gt(memoriesTable.createdAt, filter.after) : void 0);
+	return and(eq(memoriesTable.entityId, filter.entityId), filter.includeSuperseded ? void 0 : isNull(memoriesTable.supersededBy), filter.includeExpired ? void 0 : sql`(${memoriesTable.expiresAt} IS NULL OR ${memoriesTable.expiresAt} > now())`, filter.types && filter.types.length > 0 ? inArray(memoriesTable.type, [...filter.types]) : void 0, filter.tags && filter.tags.length > 0 ? sql`${memoriesTable.tags} @> ${filter.tags}` : void 0, filter.before ? lt(memoriesTable.createdAt, filter.before) : void 0, filter.after ? gt(memoriesTable.createdAt, filter.after) : void 0);
 }
 async function insertMemory(db, row) {
 	const inserted = (await db.insert(memoriesTable).values(row).returning())[0];
 	if (!inserted) throw new Error("Memory insert returned no rows.");
 	return inserted;
 }
+async function insertMemories(db, rows) {
+	if (rows.length === 0) return [];
+	return db.insert(memoriesTable).values(rows).returning();
+}
 async function getMemoryById(db, entityId, memoryId) {
 	return (await db.select().from(memoriesTable).where(and(eq(memoriesTable.entityId, entityId), eq(memoriesTable.id, memoryId))).limit(1))[0] ?? null;
 }
@@ -1210,7 +1299,7 @@ async function vectorSearch(db, input) {
 			${input.includeSuperseded ? sql`` : sql`AND superseded_by IS NULL`}
 			${input.includeExpired ? sql`` : sql`AND (expires_at IS NULL OR expires_at > now())`}
 			${input.types && input.types.length > 0 ? sql`AND type IN (${sql.join(input.types.map((type) => sql`${type}`), sql`, `)})` : sql``}
-			${input.tags && input.tags.length > 0 ? sql`AND tags && ${input.tags}` : sql``}
+			${input.tags && input.tags.length > 0 ? sql`AND tags @> ${input.tags}` : sql``}
 			${input.before ? sql`AND created_at < ${input.before}` : sql``}
 			${input.after ? sql`AND created_at > ${input.after}` : sql``}
 			AND 1 - (embedding <=> ${embedding}::vector) >= ${minScore}
@@ -1247,7 +1336,7 @@ async function lexicalSearch(db, input) {
 			${input.includeSuperseded ? sql`` : sql`AND superseded_by IS NULL`}
 			${input.includeExpired ? sql`` : sql`AND (expires_at IS NULL OR expires_at > now())`}
 			${input.types && input.types.length > 0 ? sql`AND type IN (${sql.join(input.types.map((type) => sql`${type}`), sql`, `)})` : sql``}
-			${input.tags && input.tags.length > 0 ? sql`AND tags && ${input.tags}` : sql``}
+			${input.tags && input.tags.length > 0 ? sql`AND tags @> ${input.tags}` : sql``}
 			${input.before ? sql`AND created_at < ${input.before}` : sql``}
 			${input.after ? sql`AND created_at > ${input.after}` : sql``}
 			AND to_tsvector('english', content) @@ websearch_to_tsquery('english', ${input.query})
@@ -1277,16 +1366,20 @@ async function hybridRecall(input) {
 	for (const row of vectorRows) merged.set(row.id, {
 		row,
 		vectorScore: row.vectorScore,
-		lexicalScore: 0
+		lexicalScore: lexicalScore(row.content, input.input.query)
 	});
 	for (const row of lexicalRows) {
 		const existing = merged.get(row.id);
 		if (existing) existing.lexicalScore = row.lexicalScore;
-		else merged.set(row.id, {
-			row,
-			vectorScore: 0,
-			lexicalScore: row.lexicalScore
-		});
+		else {
+			const embedding = row.embedding;
+			const sim = embedding ? cosineSimilarity(embedding, queryEmbedding) : 0;
+			merged.set(row.id, {
+				row,
+				vectorScore: Math.max(0, sim),
+				lexicalScore: row.lexicalScore
+			});
+		}
 	}
 	const scored = Array.from(merged.values()).map((entry) => toScoredMemory(rowToMemory(entry.row), Math.max(0, entry.vectorScore), Math.max(0, entry.lexicalScore), input.input, input.retrieval)).filter((memory) => memory.score >= input.minScore).sort((a, b) => b.score - a.score).slice(0, input.limit);
 	await markMemoryAccessed(input.db, scored.map((memory) => memory.id));
@@ -1387,9 +1480,41 @@ function createPostgresClient(config, handle) {
 		async rememberMany(inputs) {
 			return observe(config, "postgres", "rememberMany", { count: inputs.length }, async () => {
 				assertOpen();
-				const result = [];
-				for (const input of inputs) result.push(await remember(input));
-				return result;
+				if (inputs.length === 0) return [];
+				for (const input of inputs) validateRememberInput(input);
+				const embeddings = await embedMany(config.embedder, inputs.map((i) => i.content), {
+					...inputs[0]?.signal === void 0 ? {} : { signal: inputs[0].signal },
+					...config.provider === void 0 ? {} : { resilience: config.provider }
+				});
+				const now = /* @__PURE__ */ new Date();
+				const rows = inputs.map((input, idx) => ({
+					id: createId(),
+					entityId: input.entityId,
+					content: input.content,
+					type: input.type ?? "fact",
+					importance: input.importance ?? "normal",
+					tags: normalizeTags(input.tags),
+					source: input.source ?? null,
+					metadata: input.metadata ?? {},
+					confidence: input.confidence ?? 1,
+					embedding: embeddings[idx],
+					embeddingModel: config.embedder.model,
+					embeddingDimensions: config.embedder.dimensions,
+					supersededBy: null,
+					supersededAt: null,
+					expiresAt: input.expiresAt ?? null,
+					lastAccessedAt: null,
+					accessCount: 0,
+					createdAt: now,
+					updatedAt: now
+				}));
+				const memories = (await insertMemories(handle.db, rows)).map(rowToMemory);
+				for (const memory of memories) recordAudit(memory.entityId, "memory.created", {
+					memoryId: memory.id,
+					type: memory.type,
+					importance: memory.importance
+				});
+				return memories;
 			}, (result) => ({ count: result.length }));
 		},
 		async recall(input) {
@@ -1557,27 +1682,39 @@ function createPostgresClient(config, handle) {
 					supersededIds: []
 				};
 				const now = /* @__PURE__ */ new Date();
-				const updated = await markMemoriesSuperseded(handle.db, {
-					survivorId: survivor.id,
-					entityId: input.entityId,
-					ids: losers.map((loser) => loser.id),
-					now
-				});
-				const newSupersededIds = updated.map((row) => row.id);
-				for (const id of newSupersededIds) await recordAudit(input.entityId, "memory.superseded", {
-					memoryId: id,
-					supersededBy: survivor.id
-				});
-				if (input.mergeTags !== false && updated.length > 0) {
-					const mergedTags = new Set(survivor.tags);
-					for (const row of updated) for (const tag of row.tags) mergedTags.add(tag);
-					if (mergedTags.size !== survivor.tags.length) await setMemoryTags(handle.db, {
+				const { newSupersededIds } = await handle.db.transaction(async (tx) => {
+					const updated = await markMemoriesSuperseded(tx, {
+						survivorId: survivor.id,
 						entityId: input.entityId,
-						memoryId: survivor.id,
-						tags: [...mergedTags],
+						ids: losers.map((loser) => loser.id),
 						now
 					});
-				}
+					const newSupersededIds = updated.map((row) => row.id);
+					for (const id of newSupersededIds) await insertEvent(tx, {
+						id: createEventId(),
+						entityId: input.entityId,
+						description: "memory.superseded",
+						metadata: {
+							memoryId: id,
+							supersededBy: survivor.id
+						},
+						timestamp: now
+					});
+					if (input.mergeTags !== false && updated.length > 0) {
+						const mergedTags = new Set(survivor.tags);
+						for (const row of updated) for (const tag of row.tags) mergedTags.add(tag);
+						if (mergedTags.size !== survivor.tags.length) await setMemoryTags(tx, {
+							entityId: input.entityId,
+							memoryId: survivor.id,
+							tags: [...mergedTags],
+							now
+						});
+					}
+					return {
+						updated,
+						newSupersededIds
+					};
+				});
 				return {
 					survivorId: survivor.id,
 					supersededCount: newSupersededIds.length,