@oh-my-pi/pi-mnemopi 16.1.13 → 16.1.14

package/dist/types/core/beam/helpers.d.ts

@@ -1,5 +1,6 @@
 import type { Database } from "bun:sqlite";
 import { sha256Hex16 } from "../../util/ids";
+import { cjkFtsTerms, containsSpacelessCjk, ftsQueryTerms, recallTokens } from "../../util/regex";
 import type { BeamMemoryState, Metadata } from "./types";
 export type Vector = number[];
 export type HybridWeights = readonly [vecWeight: number, ftsWeight: number, importanceWeight: number];
@@ -29,16 +30,8 @@ export declare function parseQueryTime(queryTime?: string | Date | null): Date;
 export declare function parseTimestampFast(ts: string | null | undefined, beam?: Pick<BeamMemoryState, "caches"> | null): Date | null;
 export declare function recencyDecay(timestamp: string | null | undefined, halflifeHours?: number, now?: Date): number;
 export declare function temporalBoost(memoryTimestamp: string | null | undefined, queryTime: Date | string, halflifeHours?: number, beam?: Pick<BeamMemoryState, "caches"> | null): number;
-export declare function recallTokens(text: string): string[];
-export declare function expandedQueryTokens(tokens: readonly string[]): string[];
-export declare function minimumRecallRelevance(queryTokens: readonly string[]): number;
-export declare function factMatchTokens(text: string): Set<string>;
-export declare function containsSpacelessCjk(text: string): boolean;
-export declare function hasCjk(text: string): boolean;
-export declare function cjkFtsTerms(text: string): string[];
 export declare function lexicalRelevance(queryTokens: readonly string[], content: string, queryLower?: string): number;
 export declare function strictFactMatches(query: string, factText: string): boolean;
-export declare function ftsQueryTerms(query: string): string[];
 export declare function buildFtsQuery(query: string): string;
 export declare function cjkLikeSearch(db: Database, query: string, k?: number, working?: boolean): Array<FtsRankResult | WorkingFtsRankResult>;
 export declare function ftsSearch(db: Database, query: string, k?: number): FtsRankResult[];
@@ -56,7 +49,7 @@ export declare function metadataJson(input: unknown): string;
 export declare function detectLanguage(text: string): string;
 export declare function memoryRowMetadata(row: unknown): Metadata;
 export { cosineSimilarity, hammingDistance, informationTheoreticScore, maximallyInformativeBinarization, quantizeInt8, } from "../binary-vectors";
-export { sha256Hex16 };
+export { cjkFtsTerms, containsSpacelessCjk, ftsQueryTerms, recallTokens, sha256Hex16 };
 /** Identifies one freshly stored memory whose embedding still needs to be derived. */
 export interface EmbedItem {
     readonly memoryId: string;

package/dist/types/util/regex.d.ts

@@ -1,5 +1,6 @@
 export declare const FACT_MATCH_STOPWORDS: Set<string>;
 export declare const RECALL_SYNONYMS: Readonly<Record<string, readonly string[]>>;
+export declare function isCjkChar(ch: string): boolean;
 export declare function hasCjk(text: string): boolean;
 export declare const containsSpacelessCjk: typeof hasCjk;
 export declare function recallTokens(text: string): string[];

package/package.json

@@ -1,7 +1,7 @@
 {
 	"type": "module",
 	"name": "@oh-my-pi/pi-mnemopi",
-	"version": "16.1.13",
+	"version": "16.1.14",
 	"description": "Local SQLite memory engine for Oh My Pi agents",
 	"homepage": "https://omp.sh",
 	"author": "Can Boluk",
@@ -39,9 +39,9 @@
 		"fmt": "biome format --write ."
 	},
 	"dependencies": {
-		"@oh-my-pi/pi-ai": "16.1.13",
-		"@oh-my-pi/pi-catalog": "16.1.13",
-		"@oh-my-pi/pi-utils": "16.1.13",
+		"@oh-my-pi/pi-ai": "16.1.14",
+		"@oh-my-pi/pi-catalog": "16.1.14",
+		"@oh-my-pi/pi-utils": "16.1.14",
 		"lru-cache": "11.5.1"
 	},
 	"peerDependencies": {

package/src/core/beam/helpers.ts

@@ -1,6 +1,17 @@
 import type { Database } from "bun:sqlite";
 import { logger } from "@oh-my-pi/pi-utils";
 import { generateId as generateTimedId, sha256Hex16, stableMemoryId } from "../../util/ids";
+import {
+	cjkFtsTerms,
+	containsSpacelessCjk,
+	FACT_MATCH_STOPWORDS,
+	factMatchTokens,
+	ftsQueryTerms,
+	hasCjk,
+	isCjkChar,
+	RECALL_SYNONYMS,
+	recallTokens,
+} from "../../util/regex";
 import { currentEmbeddingModel, embed } from "../embeddings";
 import { getMnemopiRuntimeOptions, mnemopiDebugEnabled, withMnemopiRuntimeOptions } from "../runtime-options";
 import { buildExactVectorIndex, searchExactVectorIndex } from "../vector-index";
@@ -35,76 +46,8 @@ const DEFAULT_WEIGHTS: HybridWeights = [0.5, 0.3, 0.2];
 const TS_CACHE_MAX = 2000;
 const moduleTimestampCache = new Map<string, Date>();
 
-const FACT_MATCH_STOPWORDS = new Set([
-	"a",
-	"an",
-	"and",
-	"are",
-	"as",
-	"at",
-	"be",
-	"by",
-	"can",
-	"could",
-	"did",
-	"do",
-	"does",
-	"for",
-	"from",
-	"had",
-	"has",
-	"have",
-	"how",
-	"i",
-	"in",
-	"is",
-	"it",
-	"its",
-	"me",
-	"my",
-	"of",
-	"on",
-	"or",
-	"our",
-	"related",
-	"should",
-	"that",
-	"the",
-	"their",
-	"there",
-	"this",
-	"to",
-	"totally",
-	"unrelated",
-	"use",
-	"uses",
-	"was",
-	"we",
-	"what",
-	"when",
-	"where",
-	"which",
-	"who",
-	"why",
-	"with",
-	"you",
-	"your",
-]);
-
-const RECALL_SYNONYMS: Readonly<Record<string, readonly string[]>> = {
-	branding: ["brand", "positioning", "identity", "wording"],
-	preference: ["prefer", "prefers", "want", "wants", "reject", "rejects", "avoid", "grounded"],
-	professional: ["software", "builder"],
-	url: ["link", "profile"],
-	current: ["now", "live", "latest"],
-	feeling: ["feel", "feels"],
-	imposter: ["self-doubt", "doubt", "insecure"],
-};
-
-const RECALL_TOKEN_RE = /[a-z0-9][a-z0-9_.:/+-]*/g;
 const SPLIT_TOKEN_RE = /[_:/.-]+/g;
 const WORD_RE = /[\p{L}\p{N}_]+/gu;
-
 function envNumber(name: string, fallback: number): number {
 	const raw = process.env[name];
 	if (raw === undefined || raw.trim() === "") return fallback;
@@ -123,12 +66,6 @@ function asFiniteNonNegative(value: number): number {
 	return Number.isFinite(value) && value > 0 ? value : 0;
 }
 
-function isCjkChar(ch: string): boolean {
-	return (
-		(ch >= "\u4e00" && ch <= "\u9fff") || (ch >= "\u3040" && ch <= "\u30ff") || (ch >= "\uac00" && ch <= "\ud7af")
-	);
-}
-
 function tableExists(db: Database, table: string): boolean {
 	try {
 		return (
@@ -250,73 +187,6 @@ export function temporalBoost(
 	return Math.exp(-hoursDelta / halflife);
 }
 
-export function recallTokens(text: string): string[] {
-	const out: string[] = [];
-	for (const match of text.toLowerCase().matchAll(RECALL_TOKEN_RE)) {
-		const token = match[0] ?? "";
-		if (token.length >= 3 && !FACT_MATCH_STOPWORDS.has(token) && !/^\d+$/.test(token)) out.push(token);
-	}
-	return out;
-}
-
-export function expandedQueryTokens(tokens: readonly string[]): string[] {
-	const expanded: string[] = [];
-	const seen = new Set<string>();
-	for (const token of tokens) {
-		const synonyms = RECALL_SYNONYMS[token] ?? [];
-		for (const candidate of [token, ...synonyms]) {
-			if (!seen.has(candidate)) {
-				seen.add(candidate);
-				expanded.push(candidate);
-			}
-		}
-	}
-	return expanded;
-}
-
-export function minimumRecallRelevance(queryTokens: readonly string[]): number {
-	if (queryTokens.length >= 4) return 0.3;
-	if (queryTokens.length === 3) return 0.5;
-	return 0.15;
-}
-
-export function factMatchTokens(text: string): Set<string> {
-	return new Set(recallTokens(text));
-}
-
-export function containsSpacelessCjk(text: string): boolean {
-	return hasCjk(text);
-}
-
-export function hasCjk(text: string): boolean {
-	for (const ch of text) if (isCjkChar(ch)) return true;
-	return false;
-}
-
-export function cjkFtsTerms(text: string): string[] {
-	const chars = Array.from(text).filter(isCjkChar);
-	if (chars.length === 0) return [];
-	const terms: string[] = [];
-	const seen = new Set<string>();
-	for (const ch of chars) {
-		if (!seen.has(ch)) {
-			seen.add(ch);
-			terms.push(ch);
-		}
-	}
-	for (let i = 0; i < chars.length - 1; i += 1) {
-		const left = chars[i];
-		const right = chars[i + 1];
-		if (left === undefined || right === undefined) continue;
-		const bigram = left + right;
-		if (!seen.has(bigram)) {
-			seen.add(bigram);
-			terms.push(`"${bigram}"`);
-		}
-	}
-	return terms;
-}
-
 export function lexicalRelevance(queryTokens: readonly string[], content: string, queryLower = ""): number {
 	const contentLower = content.toLowerCase();
 	const queryCjk = new Set(Array.from(queryLower).filter(isCjkChar));
@@ -379,15 +249,6 @@ export function strictFactMatches(query: string, factText: string): boolean {
 	return token.length >= 5;
 }
 
-export function ftsQueryTerms(query: string): string[] {
-	const terms: string[] = [];
-	for (const term of expandedQueryTokens(recallTokens(query))) {
-		const escaped = term.replaceAll('"', '""').trim();
-		if (escaped) terms.push(`"${escaped}"`);
-	}
-	return terms;
-}
-
 export function buildFtsQuery(query: string): string {
 	return ftsQueryTerms(query).join(" OR ");
 }
@@ -908,7 +769,7 @@ export {
 	maximallyInformativeBinarization,
 	quantizeInt8,
 } from "../binary-vectors";
-export { sha256Hex16 };
+export { cjkFtsTerms, containsSpacelessCjk, ftsQueryTerms, recallTokens, sha256Hex16 };
 
 /** Identifies one freshly stored memory whose embedding still needs to be derived. */
 export interface EmbedItem {

package/src/util/regex.ts

@@ -67,6 +67,12 @@ export const RECALL_SYNONYMS: Readonly<Record<string, readonly string[]>> = {
 	imposter: ["self-doubt", "doubt", "insecure"],
 };
 
+export function isCjkChar(ch: string): boolean {
+	return (
+		(ch >= "\u4e00" && ch <= "\u9fff") || (ch >= "\u3040" && ch <= "\u30ff") || (ch >= "\uac00" && ch <= "\ud7af")
+	);
+}
+
 export function hasCjk(text: string): boolean {
 	return CJK_RE.test(text);
 }
@@ -119,7 +125,7 @@ export function cjkFtsTerms(text: string): string[] {
 	const chars: string[] = [];
 	for (let i = 0; i < text.length; i++) {
 		const ch = text.charAt(i);
-		if (isCjkCodeUnit(ch.charCodeAt(0))) chars.push(ch);
+		if (isCjkChar(ch)) chars.push(ch);
 	}
 	if (chars.length === 0) return [];
 	const terms: string[] = [];
@@ -157,9 +163,3 @@ function isAsciiDigits(value: string): boolean {
 	}
 	return value.length > 0;
 }
-
-function isCjkCodeUnit(code: number): boolean {
-	return (
-		(code >= 0x4e00 && code <= 0x9fff) || (code >= 0x3040 && code <= 0x30ff) || (code >= 0xac00 && code <= 0xd7af)
-	);
-}