@oh-my-pi/pi-mnemopi 16.3.3 → 16.3.5

package/CHANGELOG.md

@@ -2,6 +2,12 @@
 
 ## [Unreleased]
 
+## [16.3.5] - 2026-07-04
+
+### Fixed
+
+- Fixed `remember(..., { embedText })` so hosts can store full transcripts while embedding, FTS-indexing, and rebuild-reembedding a marker-free projection. ([#4395](https://github.com/can1357/oh-my-pi/issues/4395))
+
 ## [16.2.2] - 2026-06-27
 
 ### Fixed

package/dist/types/core/beam/store.d.ts

@@ -12,6 +12,7 @@ type StoreRememberOptions = RememberOptions & {
     extractEntities?: boolean;
     extract_entities?: boolean;
     extract_text?: string;
+    embed_text?: string;
     channelId?: string | null;
     channel_id?: string | null;
 };

package/dist/types/core/beam/types.d.ts

@@ -108,6 +108,11 @@ export interface RememberOptions {
      * as user `Instruction:` memories.
      */
     extractText?: string;
+    /**
+     * Override the text passed to embeddings and FTS indexing. Stored `content`
+     * remains unchanged; when unset, embeddings and FTS use `content`.
+     */
+    embedText?: string;
     veracity?: Veracity;
     memoryType?: string;
     scope?: MemoryScope;

package/dist/types/core/memory.d.ts

@@ -44,6 +44,8 @@ export interface RememberInput extends MemoryInput {
     readonly extract_entities?: boolean;
     readonly extractText?: string | null;
     readonly extract_text?: string | null;
+    readonly embedText?: string | null;
+    readonly embed_text?: string | null;
     readonly trustTier?: string | null;
     readonly trust_tier?: string | null;
     readonly memoryType?: string | null;
@@ -65,6 +67,12 @@ export interface RememberFacadeOptions {
      */
     readonly extractText?: string | null;
     readonly extract_text?: string | null;
+    /**
+     * Override the text passed to embeddings and FTS indexing. Stored content
+     * remains unchanged; when unset, embeddings and FTS use stored content.
+     */
+    readonly embedText?: string | null;
+    readonly embed_text?: string | null;
     readonly trustTier?: string | null;
     readonly trust_tier?: string | null;
     readonly timestamp?: string | Date | null;

package/package.json

@@ -1,7 +1,7 @@
 {
 	"type": "module",
 	"name": "@oh-my-pi/pi-mnemopi",
-	"version": "16.3.3",
+	"version": "16.3.5",
 	"description": "Local SQLite memory engine for Oh My Pi agents",
 	"homepage": "https://omp.sh",
 	"author": "Can Boluk",
@@ -39,9 +39,9 @@
 		"fmt": "biome format --write ."
 	},
 	"dependencies": {
-		"@oh-my-pi/pi-ai": "16.3.3",
-		"@oh-my-pi/pi-catalog": "16.3.3",
-		"@oh-my-pi/pi-utils": "16.3.3",
+		"@oh-my-pi/pi-ai": "16.3.5",
+		"@oh-my-pi/pi-catalog": "16.3.5",
+		"@oh-my-pi/pi-utils": "16.3.5",
 		"lru-cache": "11.5.1"
 	},
 	"peerDependencies": {

package/src/core/beam/consolidate.ts

@@ -896,7 +896,7 @@ function eligibleWorkingRows(beam: BeamMemoryState, sessionId: string): Row[] {
 	return asRows(
 		beam.db
 			.query(
-				`SELECT id, content, source, timestamp, importance, metadata_json, scope, valid_until, veracity
+				`SELECT id, COALESCE(embed_text, content) AS content, source, timestamp, importance, metadata_json, scope, valid_until, veracity
 		 FROM working_memory
 		 WHERE COALESCE(session_id, 'default') = ? AND timestamp < ? AND consolidated_at IS NULL
 		 ORDER BY timestamp ASC LIMIT ?`,

package/src/core/beam/recall.ts

@@ -507,6 +507,8 @@ function buildWhere(
 
 const MEMORY_COLUMNS =
 	"id, content, source, timestamp, session_id, importance, metadata_json, veracity, memory_type, recall_count, last_recalled, valid_until, superseded_by, scope, author_id, author_type, channel_id, event_date, event_date_precision, temporal_tags";
+const WORKING_MEMORY_COLUMNS =
+	"id, content, embed_text, source, timestamp, session_id, importance, metadata_json, veracity, memory_type, recall_count, last_recalled, valid_until, superseded_by, scope, author_id, author_type, channel_id, event_date, event_date_precision, temporal_tags";
 const EPISODIC_COLUMNS = `${MEMORY_COLUMNS}, rowid, summary_of, tier`;
 
 function ftsRows(
@@ -624,7 +626,7 @@ function fetchCandidates(
 	if (idsOrRowids.length === 0) return [];
 	const table = tierLabel === "working" ? "working_memory" : "episodic_memory";
 	const keyColumn = tierLabel === "working" ? "id" : "rowid";
-	const columns = tierLabel === "working" ? MEMORY_COLUMNS : EPISODIC_COLUMNS;
+	const columns = tierLabel === "working" ? WORKING_MEMORY_COLUMNS : EPISODIC_COLUMNS;
 	const { where, params } = buildWhere(beam, "m", options);
 	const rows = queryAll(
 		beam,
@@ -662,7 +664,7 @@ function fallbackCandidates(
 	options: RecallOptionsInternal,
 ): MemoryCandidate[] {
 	const table = tierLabel === "working" ? "working_memory" : "episodic_memory";
-	const columns = tierLabel === "working" ? MEMORY_COLUMNS : EPISODIC_COLUMNS;
+	const columns = tierLabel === "working" ? WORKING_MEMORY_COLUMNS : EPISODIC_COLUMNS;
 	const { where, params } = buildWhere(beam, "", options);
 	const rows = queryAll(beam, `SELECT ${columns} FROM ${table} WHERE ${where} ORDER BY timestamp DESC LIMIT ?`, [
 		...params,
@@ -684,10 +686,11 @@ function scoreCandidate(
 	options: RecallOptionsInternal,
 ): RecallResult | null {
 	const content = asString(candidate.row.content);
+	const searchableContent = asString(candidate.row.embed_text) || content;
 	const lexical =
 		queryGroups.length > 0
-			? lexicalGroupRelevance(queryGroups, content, normalizedQueryLower)
-			: lexicalRelevance(queryTokens, content, normalizedQueryLower);
+			? lexicalGroupRelevance(queryGroups, searchableContent, normalizedQueryLower)
+			: lexicalRelevance(queryTokens, searchableContent, normalizedQueryLower);
 	const minRel = minimumRelevance(queryTokens);
 	if (lexical < minRel && candidate.signals.dense < 0.65) return null;
 	const [vecWeight, ftsWeight, importanceWeight] = weights;
@@ -732,7 +735,7 @@ function scoreCandidate(
 		const tierWeight = degradationTier === 1 ? 1 : degradationTier === 2 ? 0.85 : 0.7;
 		score *= tierWeight;
 	}
-	score *= veracityWeight * currentContentAdjustment(content, options.currentSensitive === true);
+	score *= veracityWeight * currentContentAdjustment(searchableContent, options.currentSensitive === true);
 	const result: RecallResult = {
 		...candidate.row,
 		id: asString(candidate.row.id),

package/src/core/beam/schema.ts

@@ -26,6 +26,7 @@ export function initBeam(db: Database): void {
 		CREATE TABLE IF NOT EXISTS working_memory (
 			id TEXT PRIMARY KEY,
 			content TEXT NOT NULL,
+			embed_text TEXT DEFAULT NULL,
 			source TEXT,
 			timestamp TEXT,
 			session_id TEXT DEFAULT 'default',
@@ -105,6 +106,7 @@ export function initBeam(db: Database): void {
 	addColumnIfMissing(db, "working_memory", "veracity", "TEXT DEFAULT 'unknown'");
 	addColumnIfMissing(db, "episodic_memory", "veracity", "TEXT DEFAULT 'unknown'");
 	addColumnIfMissing(db, "working_memory", "memory_type", "TEXT DEFAULT 'unknown'");
+	addColumnIfMissing(db, "working_memory", "embed_text", "TEXT DEFAULT NULL");
 	addColumnIfMissing(db, "episodic_memory", "memory_type", "TEXT DEFAULT 'unknown'");
 	addColumnIfMissing(db, "episodic_memory", "binary_vector", "BLOB");
 	const consolidatedAtAdded = addColumnIfMissing(db, "working_memory", "consolidated_at", "TEXT");
@@ -150,16 +152,17 @@ export function initBeam(db: Database): void {
 			INSERT INTO fts_episodes(fts_episodes, rowid, content) VALUES ('delete', old.rowid, old.content);
 			INSERT INTO fts_episodes(rowid, content) VALUES (new.rowid, new.content);
 		END`,
+		"DROP TRIGGER IF EXISTS wm_ai",
 		`CREATE TRIGGER IF NOT EXISTS wm_ai AFTER INSERT ON working_memory BEGIN
-			INSERT INTO fts_working(id, content) VALUES (new.id, new.content);
+			INSERT INTO fts_working(id, content) VALUES (new.id, COALESCE(new.embed_text, new.content));
 		END`,
 		`CREATE TRIGGER IF NOT EXISTS wm_ad AFTER DELETE ON working_memory BEGIN
 			DELETE FROM fts_working WHERE id = old.id;
 		END`,
 		"DROP TRIGGER IF EXISTS wm_au",
-		`CREATE TRIGGER IF NOT EXISTS wm_au AFTER UPDATE OF content ON working_memory BEGIN
+		`CREATE TRIGGER IF NOT EXISTS wm_au AFTER UPDATE OF content, embed_text ON working_memory BEGIN
 			DELETE FROM fts_working WHERE id = old.id;
-			INSERT INTO fts_working(id, content) VALUES (new.id, new.content);
+			INSERT INTO fts_working(id, content) VALUES (new.id, COALESCE(new.embed_text, new.content));
 		END`,
 	]);
 

package/src/core/beam/store.ts

@@ -37,6 +37,7 @@ type StoreRememberOptions = RememberOptions & {
 	extractEntities?: boolean;
 	extract_entities?: boolean;
 	extract_text?: string;
+	embed_text?: string;
 	channelId?: string | null;
 	channel_id?: string | null;
 };
@@ -89,6 +90,14 @@ function sqlBinding(value: unknown, fallback: SQLQueryBindings): SQLQueryBinding
 	return isSqlBinding(value) ? value : fallback;
 }
 
+function embeddingText(content: string, options: { embedText?: string; embed_text?: string }): string {
+	return options.embedText ?? options.embed_text ?? content;
+}
+
+function storedEmbeddingText(content: string, embedText: string): string | null {
+	return embedText === content ? null : embedText;
+}
+
 function clampVeracity(value: unknown): Veracity {
 	if (typeof value !== "string") return "unknown";
 	const normalized = value.trim().toLowerCase();
@@ -301,7 +310,7 @@ export function reconcileEmbeddingModel(beam: BeamMemoryState): void {
 			.all(active) as { model: string | null }[];
 		const live = beam.db
 			.query(`
-				SELECT id AS memoryId, content FROM working_memory WHERE superseded_by IS NULL
+				SELECT id AS memoryId, COALESCE(embed_text, content) AS content FROM working_memory WHERE superseded_by IS NULL
 				UNION ALL
 				SELECT id AS memoryId, content FROM episodic_memory WHERE superseded_by IS NULL
 			`)
@@ -334,7 +343,7 @@ export function reconcileEmbeddingModel(beam: BeamMemoryState): void {
 	// row still missing an active-model embedding.
 	const missing = beam.db
 		.query(`
-			SELECT id AS memoryId, content FROM working_memory
+			SELECT id AS memoryId, COALESCE(embed_text, content) AS content FROM working_memory
 			WHERE superseded_by IS NULL AND id NOT IN (SELECT memory_id FROM memory_embeddings WHERE model = ?)
 			UNION ALL
 			SELECT id AS memoryId, content FROM episodic_memory
@@ -359,6 +368,7 @@ export function remember(beam: BeamMemoryState, content: string, options: StoreR
 	const authorType = options.authorType ?? options.author_type ?? beam.authorType;
 	const channelId = options.channelId ?? options.channel_id ?? beam.channelId;
 	const metadata = options.metadata ?? null;
+	const embedText = embeddingText(content, options);
 
 	const existingId = findDuplicate(beam, content);
 	if (existingId !== null) {
@@ -374,6 +384,7 @@ export function remember(beam: BeamMemoryState, content: string, options: StoreR
 					memory_type = COALESCE(?, memory_type),
 					veracity = CASE WHEN ? != 'unknown' THEN ? ELSE veracity END,
 					trust_tier = COALESCE(?, trust_tier),
+					embed_text = COALESCE(?, embed_text),
 					consolidated_at = NULL
 				WHERE id = ? AND session_id = ?
 			`)
@@ -390,6 +401,7 @@ export function remember(beam: BeamMemoryState, content: string, options: StoreR
 				veracity,
 				veracity,
 				trustTier,
+				storedEmbeddingText(content, embedText),
 				existingId,
 				beam.sessionId,
 			);
@@ -400,6 +412,7 @@ export function remember(beam: BeamMemoryState, content: string, options: StoreR
 			importance,
 			metadata: metadata ?? undefined,
 		});
+		if (embedText !== content) scheduleEmbedding(beam, [{ memoryId: existingId, content: embedText }]);
 		invalidateCaches(beam);
 		return existingId;
 	}
@@ -408,13 +421,14 @@ export function remember(beam: BeamMemoryState, content: string, options: StoreR
 	beam.db
 		.prepare(`
 			INSERT INTO working_memory
-			(id, content, source, timestamp, session_id, importance, metadata_json, valid_until, scope,
+			(id, content, embed_text, source, timestamp, session_id, importance, metadata_json, valid_until, scope,
 			 author_id, author_type, channel_id, veracity, memory_type, trust_tier)
-			VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+			VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
 		`)
 		.run(
 			memoryId,
 			content,
+			storedEmbeddingText(content, embedText),
 			source,
 			timestamp,
 			beam.sessionId,
@@ -448,7 +462,7 @@ export function remember(beam: BeamMemoryState, content: string, options: StoreR
 		importance,
 		metadata: metadata ?? undefined,
 	});
-	scheduleEmbedding(beam, [{ memoryId, content }]);
+	scheduleEmbedding(beam, [{ memoryId, content: embedText }]);
 	if (options.extract === true) scheduleFactExtraction(beam, memoryId, extractionSource);
 	invalidateCaches(beam);
 	return memoryId;
@@ -469,9 +483,9 @@ export function rememberBatch(
 	transaction(beam.db, () => {
 		const statement = beam.db.prepare(`
 			INSERT INTO working_memory
-			(id, content, source, timestamp, session_id, importance, metadata_json,
+			(id, content, embed_text, source, timestamp, session_id, importance, metadata_json,
 			 author_id, author_type, channel_id, memory_type, veracity, trust_tier, scope)
-			VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+			VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
 		`);
 		for (const item of items) {
 			const itemTimestamp = item.timestamp ?? timestamp;
@@ -479,6 +493,7 @@ export function rememberBatch(
 			ids.push(memoryId);
 			const source = item.source ?? "conversation";
 			const storeItem = item as StoreRememberOptions;
+			const embedText = embeddingText(item.content, storeItem);
 			const itemVeracity = forceVeracity
 				? defaultVeracity
 				: item.veracity !== undefined
@@ -487,6 +502,7 @@ export function rememberBatch(
 			statement.run(
 				memoryId,
 				item.content,
+				storedEmbeddingText(item.content, embedText),
 				source,
 				itemTimestamp,
 				beam.sessionId,
@@ -516,7 +532,7 @@ export function rememberBatch(
 	items.forEach((item, index) => {
 		const id = ids[index];
 		if (id === undefined) return;
-		embeddingItems.push({ memoryId: id, content: item.content });
+		embeddingItems.push({ memoryId: id, content: embeddingText(item.content, item as StoreRememberOptions) });
 	});
 	scheduleEmbedding(beam, embeddingItems);
 	items.forEach((item, index) => {
@@ -611,7 +627,7 @@ export function updateWorking(
 	const assignments: string[] = [];
 	const params: SQLQueryBindings[] = [];
 	if (content !== null) {
-		assignments.push("content = ?");
+		assignments.push("content = ?", "embed_text = NULL");
 		params.push(content);
 	}
 	if (importance !== null) {
@@ -710,6 +726,7 @@ export function exportToDict(beam: BeamMemoryState): Record<string, unknown> {
 		working_memory: db
 			.prepare(`
 				SELECT id, content, source, timestamp, session_id, importance,
+					   embed_text,
 					   metadata_json, valid_until, superseded_by, scope,
 					   recall_count, last_recalled, created_at, veracity, consolidated_at,
 					   memory_type, author_id, author_type, channel_id, trust_tier,
@@ -777,9 +794,9 @@ export function importFromDict(beam: BeamMemoryState, data: Record<string, unkno
 				INSERT INTO working_memory
 				(id, content, source, timestamp, session_id, importance, metadata_json,
 				 valid_until, superseded_by, scope, recall_count, last_recalled, created_at,
-				 veracity, consolidated_at, memory_type, author_id, author_type, channel_id,
+				 veracity, consolidated_at, memory_type, embed_text, author_id, author_type, channel_id,
 				 trust_tier, event_date, event_date_precision, temporal_tags)
-				VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+				VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
 			`).run(
 				id,
 				sqlBinding(item.content, ""),
@@ -797,6 +814,7 @@ export function importFromDict(beam: BeamMemoryState, data: Record<string, unkno
 				clampVeracity(item.veracity),
 				sqlBinding(item.consolidated_at, null),
 				sqlBinding(item.memory_type, "unknown"),
+				sqlBinding(item.embed_text, null),
 				sqlBinding(item.author_id, null),
 				sqlBinding(item.author_type, null),
 				sqlBinding(item.channel_id, null),

package/src/core/beam/types.ts

@@ -129,6 +129,11 @@ export interface RememberOptions {
 	 * as user `Instruction:` memories.
 	 */
 	extractText?: string;
+	/**
+	 * Override the text passed to embeddings and FTS indexing. Stored `content`
+	 * remains unchanged; when unset, embeddings and FTS use `content`.
+	 */
+	embedText?: string;
 	veracity?: Veracity;
 	memoryType?: string;
 	scope?: MemoryScope;

package/src/core/extraction.ts

@@ -115,9 +115,7 @@ export function parseFacts(rawOutput: string | null | undefined): string[] {
 						}
 					}
 				}
-				if (out.length > 0) {
-					return out.slice(0, 5);
-				}
+				return out.slice(0, 5);
 			}
 		} catch {
 			const matches = [...raw.matchAll(/"([^"]{10,})"/g)].map(m => m[1]).filter((v): v is string => v !== undefined);

package/src/core/memory.ts

@@ -61,6 +61,8 @@ export interface RememberInput extends MemoryInput {
 	readonly extract_entities?: boolean;
 	readonly extractText?: string | null;
 	readonly extract_text?: string | null;
+	readonly embedText?: string | null;
+	readonly embed_text?: string | null;
 	readonly trustTier?: string | null;
 	readonly trust_tier?: string | null;
 	readonly memoryType?: string | null;
@@ -83,6 +85,12 @@ export interface RememberFacadeOptions {
 	 */
 	readonly extractText?: string | null;
 	readonly extract_text?: string | null;
+	/**
+	 * Override the text passed to embeddings and FTS indexing. Stored content
+	 * remains unchanged; when unset, embeddings and FTS use stored content.
+	 */
+	readonly embedText?: string | null;
+	readonly embed_text?: string | null;
 	readonly trustTier?: string | null;
 	readonly trust_tier?: string | null;
 	readonly timestamp?: string | Date | null;
@@ -148,6 +156,7 @@ type FacadeRememberOptions = {
 	extractEntities: boolean;
 	extract: boolean;
 	extractText: string | undefined;
+	embedText: string | undefined;
 	trustTier: string | undefined;
 	veracity: string | undefined;
 	memoryType: string | undefined;
@@ -270,6 +279,7 @@ function toRememberOptions(input: string | RememberInput, options: RememberFacad
 	const timestamp = normalizeDate(options.timestamp ?? memory?.timestamp);
 	const extractText =
 		options.extractText ?? options.extract_text ?? memory?.extractText ?? memory?.extract_text ?? null;
+	const embedText = options.embedText ?? options.embed_text ?? memory?.embedText ?? memory?.embed_text ?? null;
 	const rememberOptions: FacadeRememberOptions = {
 		source: options.source ?? memory?.source ?? "conversation",
 		importance: options.importance ?? memory?.importance ?? 0.5,
@@ -284,6 +294,7 @@ function toRememberOptions(input: string | RememberInput, options: RememberFacad
 			false,
 		extract: options.extract ?? memory?.extract ?? false,
 		extractText: extractText ?? undefined,
+		embedText: embedText ?? undefined,
 		trustTier: options.trustTier ?? options.trust_tier ?? memory?.trustTier ?? memory?.trust_tier ?? undefined,
 		veracity: options.veracity ?? memory?.veracity ?? undefined,
 		memoryType: options.memoryType ?? options.memory_type ?? memory?.memoryType ?? memory?.memory_type ?? undefined,