@oh-my-pi/pi-mnemopi 18.3.1 → 18.3.3

package/CHANGELOG.md

@@ -2,6 +2,13 @@
 
 ## [Unreleased]
 
+## [18.3.3] - 2026-09-27
+
+### Fixed
+
+- Fixed recall search matching query terms inside unrelated words, improving result relevance and ensuring exact matches are prioritized.
+- Fixed sleep summaries replacing shortened terms within larger words, preventing unintended word corruption.
+
 ## [18.2.5] - 2026-09-17
 
 ### Fixed

package/README.md

@@ -29,6 +29,8 @@ console.log(id, results[0]?.content);
 memory.close();
 ```
 
+Lexical recall matches whole tokens, declared synonyms, underscore-separated identifier parts, and other forms of a word of at least four characters: the word plus `s`, `es`, `d`, `ed`, `ing`, `er`, `ers`, `ment` or `ments`, a final `e` replaced by `ing`/`ion`/`ions`, or a final `y` replaced by `ies`/`ied`. `backup` matches `backups`, `facts` matches `fact` and `caching` matches `cache`, but `1Password` does not match `pass` or `word`, and `redis` does not match `redistribution`.
+
 ## Configuration
 
 `Mnemopi` accepts LLM and embedding options directly. `MNEMOPI_*` environment variables remain fallbacks/defaults when the matching constructor option is omitted.

package/dist/types/core/beam/helpers.d.ts

@@ -30,8 +30,6 @@ export declare function parseQueryTime(queryTime?: string | Date | null): Date;
 export declare function parseTimestampFast(ts: string | null | undefined, beam?: Pick<BeamMemoryState, "caches"> | null): Date | null;
 export declare function recencyDecay(timestamp: string | null | undefined, halflifeHours?: number, now?: Date): number;
 export declare function temporalBoost(memoryTimestamp: string | null | undefined, queryTime: Date | string, halflifeHours?: number, beam?: Pick<BeamMemoryState, "caches"> | null): number;
-export declare function lexicalRelevance(queryTokens: readonly string[], content: string, queryLower?: string): number;
-export declare function strictFactMatches(query: string, factText: string): boolean;
 export declare function buildFtsQuery(query: string): string;
 export declare function cjkLikeSearch(db: Database, query: string, k?: number, working?: boolean): Array<FtsRankResult | WorkingFtsRankResult>;
 export declare function ftsSearch(db: Database, query: string, k?: number): FtsRankResult[];

package/dist/types/util/regex.d.ts

@@ -3,8 +3,15 @@ export declare const RECALL_SYNONYMS: Readonly<Record<string, readonly string[]>
 export declare function isCjkChar(ch: string): boolean;
 export declare function hasCjk(text: string): boolean;
 export declare const containsSpacelessCjk: typeof hasCjk;
+/**
+ * Weak match between two forms of one word, in either direction: the longer token is the shorter
+ * one plus an inflectional or common derivational ending (`facts`/`fact`, `deploy`/`deployment`,
+ * `caching`/`cache`, `stories`/`story`). The shorter token needs at least four characters. Any
+ * other extension or fragment never matches, so `1password`, `password` and `passport` do not
+ * find `pass`, and `redis` does not find `redistribution`. This is deliberately not a stemmer.
+ */
+export declare function matchesWordForm(left: string, right: string): boolean;
 export declare function recallTokens(text: string): string[];
-export declare function factMatchTokens(text: string): Set<string>;
 export declare function expandedQueryTokens(tokens: readonly string[]): string[];
 export declare function minimumRecallRelevance(queryTokens: readonly string[]): number;
 export declare function cjkFtsTerms(text: string): string[];

package/package.json

@@ -1,7 +1,7 @@
 {
 	"type": "module",
 	"name": "@oh-my-pi/pi-mnemopi",
-	"version": "18.3.1",
+	"version": "18.3.3",
 	"description": "Local SQLite memory engine for omp agents",
 	"homepage": "https://omp.sh",
 	"author": {
@@ -42,10 +42,10 @@
 		"fmt": "oxfmt --no-error-on-unmatched-pattern 'src/**/*.{ts,tsx}' '{test,bench,examples,scripts}/**/*.ts' '*.ts'"
 	},
 	"dependencies": {
-		"@oh-my-pi/pi-ai": "18.3.1",
-		"@oh-my-pi/pi-catalog": "18.3.1",
-		"@oh-my-pi/pi-natives": "18.3.1",
-		"@oh-my-pi/pi-utils": "18.3.1"
+		"@oh-my-pi/pi-ai": "18.3.3",
+		"@oh-my-pi/pi-catalog": "18.3.3",
+		"@oh-my-pi/pi-natives": "18.3.3",
+		"@oh-my-pi/pi-utils": "18.3.3"
 	},
 	"peerDependencies": {
 		"fastembed": "2.1.0",

package/src/core/aaak.ts

@@ -80,9 +80,29 @@ function reverseMap<const T extends Readonly<Record<string, string>>>(source: T)
 
 export const REV_CATEGORY = reverseMap(CATEGORY_MAP);
 
-const SORTED_PHRASES = Object.entries(PHRASE_MAP).sort(([left], [right]) => right.length - left.length);
+/**
+ * Literal pattern for `phrase` that never matches inside a longer word: an edge that is a
+ * word character must sit on a word boundary, so `complete` leaves `incomplete` and
+ * `completed` intact. Space and punctuation edges match as before.
+ */
+function wholeWordPattern(phrase: string): RegExp {
+	const start = /^[\p{L}\p{N}_]/u.test(phrase) ? "(?<![\\p{L}\\p{N}_])" : "";
+	const end = /[\p{L}\p{N}_]$/u.test(phrase) ? "(?![\\p{L}\\p{N}_])" : "";
+	return new RegExp(`${start}${RegExp.escape(phrase)}${end}`, "gu");
+}
+
+const PHRASE_PATTERNS = Object.entries(PHRASE_MAP)
+	.sort(([left], [right]) => right.length - left.length)
+	.map(([phrase, shorthand]) => [wholeWordPattern(phrase), shorthand] as const);
 export const REV_PHRASE = reverseMap(PHRASE_MAP);
 
+const STATUS_SHORTHANDS = [
+	[wholeWordPattern("working correctly"), "OK"],
+	[wholeWordPattern("working"), "OK"],
+	[wholeWordPattern("complete"), "DONE"],
+	[wholeWordPattern("completed"), "DONE"],
+] as const;
+
 function replaceAllLiteral(text: string, pattern: string, replacement: string): string {
 	return text.replaceAll(pattern, replacement);
 }
@@ -100,8 +120,8 @@ export function applyCategoryPrefixes(text: string): string {
 
 export function applyPhrases(text: string): string {
 	let result = text;
-	for (const [phrase, shorthand] of SORTED_PHRASES) {
-		result = replaceAllLiteral(result, phrase, shorthand);
+	for (const [pattern, shorthand] of PHRASE_PATTERNS) {
+		result = result.replace(pattern, shorthand);
 	}
 	return result;
 }
@@ -132,10 +152,9 @@ export function encode(text: string): string {
 	result = applyPhrases(result);
 	result = applyStructural(result);
 	result = compactParens(result);
-	result = result.replaceAll("working correctly", "OK");
-	result = result.replaceAll("working", "OK");
-	result = result.replaceAll("complete", "DONE");
-	result = result.replaceAll("completed", "DONE");
+	for (const [pattern, shorthand] of STATUS_SHORTHANDS) {
+		result = result.replace(pattern, shorthand);
+	}
 	return result.trim();
 }
 

package/src/core/beam/helpers.ts

@@ -1,17 +1,7 @@
 import type { Database } from "bun:sqlite";
 import { logger } from "@oh-my-pi/pi-utils";
 import { generateId as generateTimedId, sha256Hex16, stableMemoryId } from "../../util/ids";
-import {
-	cjkFtsTerms,
-	containsSpacelessCjk,
-	FACT_MATCH_STOPWORDS,
-	factMatchTokens,
-	ftsQueryTerms,
-	hasCjk,
-	isCjkChar,
-	RECALL_SYNONYMS,
-	recallTokens,
-} from "../../util/regex";
+import { cjkFtsTerms, containsSpacelessCjk, ftsQueryTerms, hasCjk, isCjkChar, recallTokens } from "../../util/regex";
 import { currentEmbeddingModel, embed } from "../embeddings";
 import { getMnemopiRuntimeOptions, mnemopiDebugEnabled, withMnemopiRuntimeOptions } from "../runtime-options";
 import { buildExactVectorIndex, searchExactVectorIndex } from "../vector-index";
@@ -46,7 +36,6 @@ const DEFAULT_WEIGHTS: HybridWeights = [0.5, 0.3, 0.2];
 const TS_CACHE_MAX = 2000;
 const moduleTimestampCache = new Map<string, Date>();
 
-const SPLIT_TOKEN_RE = /[_:/.-]+/g;
 const WORD_RE = /[\p{L}\p{N}_]+/gu;
 function envNumber(name: string, fallback: number): number {
 	const raw = process.env[name];
@@ -187,68 +176,6 @@ export function temporalBoost(
 	return Math.exp(-hoursDelta / halflife);
 }
 
-export function lexicalRelevance(queryTokens: readonly string[], content: string, queryLower = ""): number {
-	const contentLower = content.toLowerCase();
-	const queryCjk = new Set(Array.from(queryLower).filter(isCjkChar));
-	if (queryTokens.length === 0 && queryCjk.size === 0) return 0;
-
-	const contentTokens = new Set(recallTokens(contentLower));
-	for (const token of Array.from(contentTokens)) {
-		for (const part of token.split(SPLIT_TOKEN_RE)) {
-			if (part.length >= 3 && !FACT_MATCH_STOPWORDS.has(part) && !/^\d+$/.test(part)) contentTokens.add(part);
-		}
-	}
-	if (contentTokens.size === 0 && queryCjk.size === 0) return 0;
-
-	let exact = 0;
-	let partial = 0;
-	for (const token of queryTokens) {
-		if (contentTokens.has(token)) {
-			exact += 1;
-			continue;
-		}
-		const synonyms = RECALL_SYNONYMS[token] ?? [];
-		if (synonyms.some(syn => contentTokens.has(syn))) {
-			partial += 0.75;
-			continue;
-		}
-		if (
-			token.length >= 4 &&
-			Array.from(contentTokens).some(
-				contentToken => contentToken.length >= 4 && (token.includes(contentToken) || contentToken.includes(token)),
-			)
-		) {
-			partial += 0.4;
-		}
-	}
-
-	const fullMatch = queryLower !== "" && contentLower.includes(queryLower) ? 1 : 0;
-	let score = (exact + partial + fullMatch) / Math.max(queryTokens.length, 1);
-	if (score === 0 && queryCjk.size > 0) {
-		const contentCjk = new Set(Array.from(contentLower).filter(isCjkChar));
-		let overlap = 0;
-		for (const ch of queryCjk) if (contentCjk.has(ch)) overlap += 1;
-		score = overlap / queryCjk.size;
-	}
-	return Math.min(score, 1);
-}
-
-export function strictFactMatches(query: string, factText: string): boolean {
-	const queryLower = query.toLowerCase().trim();
-	const factLower = factText.toLowerCase().trim();
-	if (!queryLower || !factLower) return false;
-	if (factLower.includes(queryLower)) return true;
-	const queryTokens = factMatchTokens(queryLower);
-	const factTokens = factMatchTokens(factLower);
-	if (queryTokens.size === 0 || factTokens.size === 0) return false;
-	const overlap = Array.from(queryTokens).filter(token => factTokens.has(token));
-	if (overlap.length >= 2) return true;
-	const token = overlap[0];
-	if (token === undefined) return false;
-	if (token.length >= 8 && /[./:_-]/.test(token)) return true;
-	return token.length >= 5;
-}
-
 export function buildFtsQuery(query: string): string {
 	return ftsQueryTerms(query).join(" OR ");
 }

package/src/core/beam/recall.ts

@@ -1,8 +1,9 @@
 import { normalizedRecallWeights, temporalHalflifeHours } from "../../config";
+import { hasCjk, matchesWordForm } from "../../util/regex";
 import { embedQuery } from "../embeddings";
 import { mmrRerank } from "../mmr";
 import { adjustWeights, classifyIntent } from "../query-intent";
-import { getSynonyms, normalizeQuery, STOP_WORDS as QUERY_STOP_WORDS } from "../synonyms";
+import { getSynonyms, STOP_WORDS as QUERY_STOP_WORDS } from "../synonyms";
 import { extractTemporal } from "../temporal-parser";
 import { cosineSimilarity } from "../vector-math";
 import type { BeamMemoryState, RecallEnhancedOptions, RecallOptions, RecallResult } from "./types";
@@ -251,79 +252,53 @@ function factExpandedTokenGroups(query: string, content: string): string[][] {
 	return groups;
 }
 
-function tokensFromGroups(groups: readonly (readonly string[])[]): string[] {
-	const seen = new Set<string>();
-	for (const group of groups) {
-		for (const token of group) seen.add(token);
-	}
-	return [...seen];
-}
-
-function contentMatchesToken(contentLower: string, contentTokens: ReadonlySet<string>, token: string): boolean {
-	if (contentTokens.has(token) || contentLower.includes(token)) return true;
-	for (const contentToken of contentTokens) {
-		if (
-			contentToken.length >= 4 &&
-			token.length >= 4 &&
-			(contentToken.includes(token) || token.includes(contentToken))
-		) {
-			return true;
+// Match whole tokens or another form of the same word; CJK text need not separate words with spaces.
+function lexicalGroupRelevance(queryGroups: readonly (readonly string[])[], content: string): number {
+	if (queryGroups.length === 0) return 0;
+	const tokens = tokenize(content);
+	const contentTokens = new Set(tokens);
+	// Split document identifiers only; a query synonym such as data_store must stay atomic.
+	for (const token of tokens) {
+		if (!token.includes("_")) continue;
+		for (const part of token.split("_")) {
+			if (part.length > 0) contentTokens.add(part);
 		}
 	}
-	return false;
-}
-
-function lexicalGroupRelevance(
-	queryGroups: readonly (readonly string[])[],
-	content: string,
-	normalizedQuery: string,
-): number {
-	if (queryGroups.length === 0) return 0;
-	const contentLower = content.toLowerCase();
-	if (queryGroups.length > 1 && normalizedQuery.length > 0 && contentLower.includes(normalizedQuery)) return 1;
-	const contentTokens = new Set(tokenize(contentLower));
-	let exact = 0;
+	const cjkContent = queryGroups.some(group => group.some(hasCjk)) ? content.toLowerCase() : null;
+	let matched = 0;
 	let partial = 0;
 	for (const group of queryGroups) {
-		let matched = false;
-		for (const token of group) {
-			if (contentMatchesToken(contentLower, contentTokens, token)) {
-				matched = true;
-				break;
-			}
-		}
-		if (matched) exact += 1;
-		else {
-			for (const token of group) {
+		if (group.some(token => contentTokens.has(token) || (hasCjk(token) && cjkContent?.includes(token)))) {
+			matched += 1;
+		} else if (
+			group.some(token => {
 				for (const contentToken of contentTokens) {
-					if (
-						contentToken.length >= 4 &&
-						token.length >= 4 &&
-						(contentToken.includes(token) || token.includes(contentToken))
-					) {
-						partial += 1;
-						matched = true;
-						break;
-					}
+					if (matchesWordForm(token, contentToken)) return true;
 				}
-				if (matched) break;
-			}
+				return false;
+			})
+		) {
+			partial += 1;
 		}
 	}
-	if (queryGroups.length === 1) {
-		if (exact === 0 && partial === 0) return 0;
-		const token = queryGroups[0]?.[0] ?? "";
-		let count = 0;
-		let offset = 0;
-		while (token.length > 0) {
-			const idx = contentLower.indexOf(token, offset);
-			if (idx < 0) break;
+	if (queryGroups.length !== 1) return (matched + partial * 0.5) / queryGroups.length;
+	if (matched === 0) return partial > 0 ? 0.35 : 0;
+	const token = queryGroups[0]?.[0];
+	let count = 0;
+	if (token && hasCjk(token) && cjkContent !== null) {
+		for (
+			let offset = cjkContent.indexOf(token);
+			offset >= 0;
+			offset = cjkContent.indexOf(token, offset + token.length)
+		) {
 			count += 1;
-			offset = idx + token.length;
 		}
-		return clamp01(0.7 + Math.min(Math.max(count - 1, 0), 3) * 0.1);
+	} else {
+		for (const contentToken of tokens) {
+			if (contentToken === token) count += 1;
+		}
 	}
-	return clamp01((exact + partial * 0.5) / queryGroups.length);
+	return clamp01(0.7 + Math.min(Math.max(count - 1, 0), 3) * 0.1);
 }
 
 function queryAsksCurrent(query: string): boolean {
@@ -346,45 +321,6 @@ function minimumRelevance(tokens: readonly string[]): number {
 	return 0.22;
 }
 
-function lexicalRelevance(queryTokens: readonly string[], content: string, normalizedQuery: string): number {
-	if (queryTokens.length === 0) return 0;
-	const contentLower = content.toLowerCase();
-	if (queryTokens.length > 1 && normalizedQuery.length > 0 && contentLower.includes(normalizedQuery)) return 1;
-	if (queryTokens.length === 1) {
-		const token = queryTokens[0] ?? "";
-		if (token.length === 0 || !contentLower.includes(token)) return 0;
-		let count = 0;
-		let offset = 0;
-		while (true) {
-			const idx = contentLower.indexOf(token, offset);
-			if (idx < 0) break;
-			count += 1;
-			offset = idx + token.length;
-		}
-		return clamp01(0.7 + Math.min(Math.max(count - 1, 0), 3) * 0.1);
-	}
-	const contentTokens = new Set(tokenize(contentLower));
-	let exact = 0;
-	let partial = 0;
-	for (const token of queryTokens) {
-		if (contentTokens.has(token) || contentLower.includes(token)) {
-			exact += 1;
-			continue;
-		}
-		for (const contentToken of contentTokens) {
-			if (
-				contentToken.length >= 4 &&
-				token.length >= 4 &&
-				(contentToken.includes(token) || token.includes(contentToken))
-			) {
-				partial += 1;
-				break;
-			}
-		}
-	}
-	return clamp01((exact + partial * 0.5) / queryTokens.length);
-}
-
 function recencyDecay(timestamp: unknown, halfLifeHours = 72): number {
 	const raw = asString(timestamp);
 	if (raw.length === 0) return 0;
@@ -723,16 +659,12 @@ function scoreCandidate(
 	candidate: MemoryCandidate,
 	queryTokens: readonly string[],
 	queryGroups: readonly (readonly string[])[],
-	normalizedQueryLower: string,
 	weights: readonly [number, number, number],
 	options: RecallOptionsInternal,
 ): RecallResult | null {
 	const content = asString(candidate.row.content);
 	const searchableContent = asString(candidate.row.embed_text) || content;
-	const lexical =
-		queryGroups.length > 0
-			? lexicalGroupRelevance(queryGroups, searchableContent, normalizedQueryLower)
-			: lexicalRelevance(queryTokens, searchableContent, normalizedQueryLower);
+	const lexical = lexicalGroupRelevance(queryGroups, searchableContent);
 	const minRel = minimumRelevance(queryTokens);
 	if (lexical < minRel && candidate.signals.dense < 0.65) return null;
 	const [vecWeight, ftsWeight, importanceWeight] = weights;
@@ -993,11 +925,10 @@ export async function recall(
 	const useSynonyms = options.useSynonyms !== false;
 	const tokens = expandedTokens(query, useSynonyms);
 	const tokenGroups = expandedTokenGroups(query, useSynonyms);
-	const normalized = normalizeQuery(query).toLowerCase();
 	const candidates = collectMemoryCandidates(beam, query, topK, temporalOptions);
 	const scored: RecallResult[] = [];
 	for (const candidate of candidates) {
-		const result = scoreCandidate(candidate, tokens, tokenGroups, normalized, weights, temporalOptions);
+		const result = scoreCandidate(candidate, tokens, tokenGroups, weights, temporalOptions);
 		if (result !== null) scored.push(result);
 	}
 	scored.sort((left, right) => (right.score ?? 0) - (left.score ?? 0));
@@ -1184,7 +1115,6 @@ export function factRecall(beam: BeamMemoryState, query: string, topK = 30): Fac
 	if (rowids.length === 0) return [];
 	const visibility = factVisibilityWhere(beam, "");
 	const ranks = normalizeRanks(matched, "rowid");
-	const normalized = normalizeQuery(query).toLowerCase();
 	const rows = queryAll(
 		beam,
 		`SELECT rowid, fact_id, subject, predicate, object, timestamp, confidence
@@ -1203,11 +1133,7 @@ export function factRecall(beam: BeamMemoryState, query: string, topK = 30): Fac
 			const content = object.length > 0 ? object : `${subject} ${predicate}`.trim();
 			const searchable = factSearchableText(subject, predicate, object);
 			const queryGroups = factExpandedTokenGroups(query, searchable);
-			const queryTokens = tokensFromGroups(queryGroups);
-			const lexical =
-				queryGroups.length > 0
-					? lexicalGroupRelevance(queryGroups, searchable, normalized)
-					: lexicalRelevance(queryTokens, searchable, normalized);
+			const lexical = lexicalGroupRelevance(queryGroups, searchable);
 			const rank = ranks.get(asNumber(row.rowid)) ?? 0;
 			const result: FactRecallResult = {
 				id: asString(row.fact_id),

package/src/util/regex.ts

@@ -79,6 +79,32 @@ export function hasCjk(text: string): boolean {
 
 export const containsSpacelessCjk = hasCjk;
 
+/** Endings that turn one form of a word into another: `backup`/`backups`, `deploy`/`deployment`. */
+const WORD_FORM_ENDINGS: readonly string[] = ["s", "es", "d", "ed", "ing", "er", "ers", "ment", "ments"];
+/** Endings that replace a final `e`: `cache`/`caching`, `create`/`creation`. */
+const SILENT_E_ENDINGS: readonly string[] = ["ing", "ion", "ions"];
+/** Endings that replace a final `y`: `story`/`stories`, `copy`/`copied`. */
+const FINAL_Y_ENDINGS: readonly string[] = ["ies", "ied"];
+
+/**
+ * Weak match between two forms of one word, in either direction: the longer token is the shorter
+ * one plus an inflectional or common derivational ending (`facts`/`fact`, `deploy`/`deployment`,
+ * `caching`/`cache`, `stories`/`story`). The shorter token needs at least four characters. Any
+ * other extension or fragment never matches, so `1password`, `password` and `passport` do not
+ * find `pass`, and `redis` does not find `redistribution`. This is deliberately not a stemmer.
+ */
+export function matchesWordForm(left: string, right: string): boolean {
+	const [shorter, longer] = left.length <= right.length ? [left, right] : [right, left];
+	if (shorter.length < 4 || longer.length === shorter.length) return false;
+	if (longer.startsWith(shorter) && WORD_FORM_ENDINGS.includes(longer.slice(shorter.length))) return true;
+	const stem = shorter.slice(0, -1);
+	if (!longer.startsWith(stem)) return false;
+	const ending = longer.slice(stem.length);
+	if (shorter.endsWith("e")) return SILENT_E_ENDINGS.includes(ending);
+	if (shorter.endsWith("y")) return FINAL_Y_ENDINGS.includes(ending);
+	return false;
+}
+
 export function recallTokens(text: string): string[] {
 	RECALL_TOKEN_RE.lastIndex = 0;
 	const tokens: string[] = [];
@@ -92,10 +118,6 @@ export function recallTokens(text: string): string[] {
 	return tokens;
 }
 
-export function factMatchTokens(text: string): Set<string> {
-	return new Set(recallTokens(text));
-}
-
 export function expandedQueryTokens(tokens: readonly string[]): string[] {
 	const expanded: string[] = [];
 	const seen = new Set<string>();