@oh-my-pi/pi-mnemopi 18.5.0 → 18.5.1

package/CHANGELOG.md

@@ -2,6 +2,17 @@
 
 ## [Unreleased]
 
+## [18.5.1] - 2026-10-03
+
+### Changed
+
+- Upgraded local embedding support to fastembed 3.0.0. Models now download from Hugging Face into the updated cache layout, with interrupted downloads resuming automatically; existing models are migrated on first use while producing the same vectors.
+
+### Fixed
+
+- Fixed enhanced recall returning cached results from an unrelated longer query when answering a shorter query.
+- Fixed local embedding setup on fresh caches and Linux ARM64, including compatibility with current model downloads and ARM64 tokenization support.
+
 ## [18.4.4] - 2026-09-29
 
 ### Added

package/dist/types/core/embeddings.d.ts

@@ -32,30 +32,11 @@ export type LocalModelInitializer = (options: LocalModelInitOptions) => Promise<
  */
 export declare function quarantineCorruptModelFile(message: string, cacheDir?: string): Promise<boolean>;
 /**
- * Recover from a partially-extracted fastembed model cache. An interrupted
- * archive download/extraction leaves `<cacheDir>/<model>/` populated with
- * sidecars and a truncated `model.onnx_data` but WITHOUT the graph file
- * (`model.onnx` / `model_optimized.onnx`), so `FlagEmbedding.init` throws
- * `Model file not found at .../model.onnx`. Upstream `retrieveModel`
- * short-circuits on the existing model dir and never re-downloads, and
- * `downloadFileFromGCS` reuses a leftover partial `<model>.tar.gz` as-is, so
- * the cache is stuck broken forever: recall silently dies and the rebuild
- * queue grows every session. Delete the incomplete model dir AND the leftover
- * partial archive so the retry re-downloads from scratch. The model path is
- * error-message CONTENT, so the dir is only removed when it is a direct child
- * of the fastembed cache root — never touch a directory a dependency merely
- * names. Returns true when a retry is worth attempting (the incomplete cache
- * was cleared, or already gone from a concurrent heal).
- * @internal exported for tests
- */
-export declare function clearIncompleteModelCache(message: string, cacheDir?: string): Promise<boolean>;
-/**
- * Shared local-model initializer: FlagEmbedding.init with BOTH cache heals.
- * Missing sidecars (config/tokenizer/tokens map) re-fetch and retry; a
- * corrupt model blob (Protobuf parse failure) quarantines the file and
- * retries THROUGH the sidecar heal, so a cache that is broken in both ways
- * still recovers in one pass. Also the initializer the embed worker uses in
- * its subprocess; the in-process seam stays {@link setLocalModelInitializer}.
+ * Shared local-model initializer: FlagEmbedding.init with a corrupt-cache heal.
+ * A corrupt model blob (Protobuf parse failure) quarantines the file and
+ * retries once; fastembed re-downloads any file missing from the model's
+ * cache directory. Also the initializer the embed worker uses in its
+ * subprocess; the in-process seam stays {@link setLocalModelInitializer}.
  */
 export declare function defaultLocalModelInitializer(options: LocalModelInitOptions): Promise<LocalEmbeddingModel>;
 export declare function embeddingsDisabled(): boolean;

package/dist/types/core/fastembed-model-cache.d.ts

@@ -1,2 +1,8 @@
-/** Download missing config/tokenizer sidecars into a fastembed model cache directory. */
-export declare function ensureFastembedModelSidecars(model: string, cacheDir?: string): Promise<boolean>;
+/**
+ * Whether `FlagEmbedding.init` can load `model` from `cacheDir` without a
+ * download. fastembed caches each file under `<cacheDir>/<repo with / → _>/`
+ * and fetches only the files that are missing, so a model is cached exactly
+ * when every file is present. Unknown models report false. The agent's embed
+ * client uses this to decide whether an init shows download activity.
+ */
+export declare function isFastembedModelCached(model: string, cacheDir: string): Promise<boolean>;

package/dist/types/core/query-cache.d.ts

@@ -31,7 +31,9 @@ export declare function isQueryCacheEnabled(useCache?: boolean, env?: Env, confi
  *
  * - Tier 1: exact normalized query match.
  * - Tiers 2/3: query embedding cosine >= 0.88, or >= 0.78 plus word overlap.
- * - Tier 4: normalized word overlap.
+ * - Tier 4: shared distinct words cover at least 70% of the new query and half of the
+ *   cached one, so a reworded or trimmed query hits but a short query never lands
+ *   inside a long cached prompt that merely contains its words.
  */
 export declare class QueryCache<T = QueryCacheResult> {
     #private;

package/package.json

@@ -1,7 +1,7 @@
 {
 	"type": "module",
 	"name": "@oh-my-pi/pi-mnemopi",
-	"version": "18.5.0",
+	"version": "18.5.1",
 	"description": "Local SQLite memory engine for omp agents",
 	"homepage": "https://omp.sh",
 	"author": {
@@ -42,13 +42,13 @@
 		"fmt": "oxfmt --no-error-on-unmatched-pattern 'src/**/*.{ts,tsx}' '{test,bench,examples,scripts}/**/*.ts' '*.ts'"
 	},
 	"dependencies": {
-		"@oh-my-pi/pi-ai": "18.5.0",
-		"@oh-my-pi/pi-catalog": "18.5.0",
-		"@oh-my-pi/pi-natives": "18.5.0",
-		"@oh-my-pi/pi-utils": "18.5.0"
+		"@oh-my-pi/pi-ai": "18.5.1",
+		"@oh-my-pi/pi-catalog": "18.5.1",
+		"@oh-my-pi/pi-natives": "18.5.1",
+		"@oh-my-pi/pi-utils": "18.5.1"
 	},
 	"peerDependencies": {
-		"fastembed": "2.1.0",
+		"fastembed": "3.0.0",
 		"onnxruntime-node": "1.21.0"
 	},
 	"peerDependenciesMeta": {
@@ -61,7 +61,7 @@
 	},
 	"devDependencies": {
 		"@types/bun": "^1.3.14",
-		"fastembed": "2.1.0",
+		"fastembed": "3.0.0",
 		"onnxruntime-node": "1.26.0"
 	},
 	"engines": {

package/src/core/embeddings.ts

@@ -14,7 +14,6 @@ import {
 } from "@oh-my-pi/pi-utils";
 import { LRUCache } from "@oh-my-pi/pi-utils/lru";
 import type { EmbeddingModel } from "fastembed";
-import { ensureFastembedModelSidecars } from "./fastembed-model-cache";
 import { loadFastembed } from "./fastembed-runtime";
 import {
 	type EmbeddingOutput,
@@ -93,74 +92,22 @@ export async function quarantineCorruptModelFile(message: string, cacheDir?: str
 }
 
 /**
- * Recover from a partially-extracted fastembed model cache. An interrupted
- * archive download/extraction leaves `<cacheDir>/<model>/` populated with
- * sidecars and a truncated `model.onnx_data` but WITHOUT the graph file
- * (`model.onnx` / `model_optimized.onnx`), so `FlagEmbedding.init` throws
- * `Model file not found at .../model.onnx`. Upstream `retrieveModel`
- * short-circuits on the existing model dir and never re-downloads, and
- * `downloadFileFromGCS` reuses a leftover partial `<model>.tar.gz` as-is, so
- * the cache is stuck broken forever: recall silently dies and the rebuild
- * queue grows every session. Delete the incomplete model dir AND the leftover
- * partial archive so the retry re-downloads from scratch. The model path is
- * error-message CONTENT, so the dir is only removed when it is a direct child
- * of the fastembed cache root — never touch a directory a dependency merely
- * names. Returns true when a retry is worth attempting (the incomplete cache
- * was cleared, or already gone from a concurrent heal).
- * @internal exported for tests
- */
-export async function clearIncompleteModelCache(message: string, cacheDir?: string): Promise<boolean> {
-	const match = /Model file not found at (.+?\.onnx)\b/i.exec(message);
-	if (!match) return false;
-	const modelFile = nodePath.resolve(match[1]);
-	const modelDir = nodePath.dirname(modelFile);
-	const cacheRoot = nodePath.resolve(cacheDir ?? getFastembedCacheDir());
-	// Only a direct child of the cache root: `<cacheRoot>/<model>`.
-	if (nodePath.dirname(modelDir) !== cacheRoot) return false;
-	try {
-		await fsp.rm(modelDir, { recursive: true, force: true });
-		await fsp.rm(`${modelDir}.tar.gz`, { force: true });
-		logger.warn("mnemopi: cleared incomplete local embedding model cache; re-downloading", { modelDir });
-	} catch {
-		// Concurrent heal or vanished dir: the single retry stays safe.
-	}
-	return true;
-}
-
-const SIDECAR_ERROR_RE =
-	/(?:Config file not found at .*config|Tokenizer file not found at .*tokenizer|Tokens map file not found at .*special_tokens_map)/u;
-
-/**
- * Shared local-model initializer: FlagEmbedding.init with BOTH cache heals.
- * Missing sidecars (config/tokenizer/tokens map) re-fetch and retry; a
- * corrupt model blob (Protobuf parse failure) quarantines the file and
- * retries THROUGH the sidecar heal, so a cache that is broken in both ways
- * still recovers in one pass. Also the initializer the embed worker uses in
- * its subprocess; the in-process seam stays {@link setLocalModelInitializer}.
+ * Shared local-model initializer: FlagEmbedding.init with a corrupt-cache heal.
+ * A corrupt model blob (Protobuf parse failure) quarantines the file and
+ * retries once; fastembed re-downloads any file missing from the model's
+ * cache directory. Also the initializer the embed worker uses in its
+ * subprocess; the in-process seam stays {@link setLocalModelInitializer}.
  */
 export async function defaultLocalModelInitializer(options: LocalModelInitOptions): Promise<LocalEmbeddingModel> {
 	const cacheDir = options.cacheDir ?? getFastembedCacheDir();
 	const initOptions = options.cacheDir === undefined ? { ...options, cacheDir } : options;
 	const { FlagEmbedding } = await loadFastembed();
-	const initWithSidecarHeal = async (): Promise<LocalEmbeddingModel> => {
-		try {
-			return await FlagEmbedding.init(initOptions);
-		} catch (error) {
-			const message = error instanceof Error ? error.message : "";
-			if (!SIDECAR_ERROR_RE.test(message)) throw error;
-			if (!(await ensureFastembedModelSidecars(options.model, cacheDir))) throw error;
-			return FlagEmbedding.init(initOptions);
-		}
-	};
 	try {
-		return await initWithSidecarHeal();
+		return await FlagEmbedding.init(initOptions);
 	} catch (error) {
 		const message = error instanceof Error ? error.message : "";
 		if (/Protobuf parsing failed/i.test(message) && (await quarantineCorruptModelFile(message, cacheDir))) {
-			return initWithSidecarHeal();
-		}
-		if (await clearIncompleteModelCache(message, cacheDir)) {
-			return initWithSidecarHeal();
+			return FlagEmbedding.init(initOptions);
 		}
 		throw error;
 	}

package/src/core/fastembed-model-cache.ts

@@ -1,39 +1,49 @@
+import * as fs from "node:fs/promises";
 import * as path from "node:path";
 
-const FASTEMBED_MODEL_SIDECARS = [
-	"config.json",
+/**
+ * Hugging Face repo and weight files fastembed downloads for each built-in
+ * model. Mirrors fastembed's private `MODEL_SOURCES` table; fastembed
+ * exposes no API for its cache layout.
+ */
+const FASTEMBED_MODEL_SOURCES: Record<string, { repo: string; files: readonly string[] }> = {
+	"fast-all-MiniLM-L6-v2": { repo: "Qdrant/all-MiniLM-L6-v2-onnx", files: ["model.onnx"] },
+	"fast-bge-base-en": { repo: "Qdrant/fast-bge-base-en", files: ["model_optimized.onnx"] },
+	"fast-bge-base-en-v1.5": { repo: "Qdrant/bge-base-en-v1.5-onnx-Q", files: ["model_optimized.onnx"] },
+	"fast-bge-small-en": { repo: "Qdrant/bge-small-en", files: ["model_optimized.onnx"] },
+	"fast-bge-small-en-v1.5": { repo: "Qdrant/bge-small-en-v1.5-onnx-Q", files: ["model_optimized.onnx"] },
+	"fast-bge-small-zh-v1.5": { repo: "Qdrant/bge-small-zh-v1.5", files: ["model_optimized.onnx"] },
+	"fast-multilingual-e5-large": {
+		repo: "Qdrant/multilingual-e5-large-onnx",
+		files: ["model.onnx", "model.onnx_data"],
+	},
+};
+
+const FASTEMBED_TOKENIZER_FILES = [
 	"tokenizer.json",
 	"tokenizer_config.json",
+	"config.json",
 	"special_tokens_map.json",
 ] as const;
 
-const FASTEMBED_HF_REPOS: Record<string, string> = {
-	"fast-all-MiniLM-L6-v2": "sentence-transformers/all-MiniLM-L6-v2",
-	"fast-bge-base-en": "BAAI/bge-base-en",
-	"fast-bge-base-en-v1.5": "BAAI/bge-base-en-v1.5",
-	"fast-bge-small-en": "BAAI/bge-small-en",
-	"fast-bge-small-en-v1.5": "BAAI/bge-small-en-v1.5",
-	"fast-bge-small-zh-v1.5": "BAAI/bge-small-zh-v1.5",
-	"fast-multilingual-e5-large": "intfloat/multilingual-e5-large",
-};
-
-/** Download missing config/tokenizer sidecars into a fastembed model cache directory. */
-export async function ensureFastembedModelSidecars(model: string, cacheDir = "local_cache"): Promise<boolean> {
-	const repo = FASTEMBED_HF_REPOS[model];
-	if (repo === undefined) return false;
-
-	const modelDir = path.join(cacheDir, model);
-	for (const fileName of FASTEMBED_MODEL_SIDECARS) {
-		const target = path.join(modelDir, fileName);
-		if (await Bun.file(target).exists()) continue;
-
-		const response = await fetch(`https://huggingface.co/${repo}/resolve/main/${fileName}`);
-		if (!response.ok) {
-			throw new Error(
-				`Failed to download ${model} ${fileName} from ${repo}: ${response.status} ${response.statusText}`,
-			);
-		}
-		await Bun.write(target, await response.arrayBuffer());
-	}
-	return true;
+/**
+ * Whether `FlagEmbedding.init` can load `model` from `cacheDir` without a
+ * download. fastembed caches each file under `<cacheDir>/<repo with / → _>/`
+ * and fetches only the files that are missing, so a model is cached exactly
+ * when every file is present. Unknown models report false. The agent's embed
+ * client uses this to decide whether an init shows download activity.
+ */
+export async function isFastembedModelCached(model: string, cacheDir: string): Promise<boolean> {
+	const source = FASTEMBED_MODEL_SOURCES[model];
+	if (source === undefined) return false;
+	const modelDir = path.join(cacheDir, source.repo.replace("/", "_"));
+	const present = await Promise.all(
+		[...source.files, ...FASTEMBED_TOKENIZER_FILES].map(file =>
+			fs.access(path.join(modelDir, file)).then(
+				() => true,
+				() => false,
+			),
+		),
+	);
+	return present.every(Boolean);
 }

package/src/core/fastembed-runtime.ts

@@ -33,15 +33,26 @@ export interface FastembedRuntimeInstallPlan {
  * manifest is inlined into a bundle. The runtime install deliberately does not
  * override fastembed's `onnxruntime-node` dependency: the prebuilt native addon
  * links against that package's bundled ORT dylib/so/dll name.
+ *
+ * It does override `@anush008/tokenizers`: every fastembed release pins
+ * `^0.0.0`, which resolves to a build without linux-arm64 bindings (#14083).
+ * {@link TOKENIZERS_SPEC} keeps the identical JS API and adds them.
  */
 const FASTEMBED_SPEC = packageManifest.peerDependencies.fastembed;
 
+/** `@anush008/tokenizers` release with linux-arm64 gnu/musl bindings (added in 0.5.0). */
+const TOKENIZERS_SPEC = "0.6.0";
+
 /** Build the deterministic fastembed runtime install plan used by local embeddings. */
 export function fastembedRuntimeInstallPlan(): FastembedRuntimeInstallPlan {
 	return {
-		versionKey: `fastembed-${FASTEMBED_SPEC}_transitive-ort`.replace(/[^A-Za-z0-9._-]/g, "_"),
+		versionKey: `fastembed-${FASTEMBED_SPEC}_transitive-ort_tokenizers-${TOKENIZERS_SPEC}`.replace(
+			/[^A-Za-z0-9._-]/g,
+			"_",
+		),
 		install: {
 			dependencies: { fastembed: FASTEMBED_SPEC },
+			overrides: { "@anush008/tokenizers": TOKENIZERS_SPEC },
 			trustedDependencies: ["onnxruntime-node"],
 		},
 	};

package/src/core/query-cache.ts

@@ -31,6 +31,8 @@ export interface QueryCacheStats {
 interface CacheEntry<T> {
 	readonly scope: string;
 	readonly normalized: string;
+	/** Distinct words of {@link normalized}, for the tier-4 overlap test. */
+	readonly words: ReadonlySet<string>;
 	readonly results: readonly T[];
 	readonly embedding: QueryEmbedding | null;
 }
@@ -66,7 +68,9 @@ export function isQueryCacheEnabled(useCache = true, env: Env = process.env, con
  *
  * - Tier 1: exact normalized query match.
  * - Tiers 2/3: query embedding cosine >= 0.88, or >= 0.78 plus word overlap.
- * - Tier 4: normalized word overlap.
+ * - Tier 4: shared distinct words cover at least 70% of the new query and half of the
+ *   cached one, so a reworded or trimmed query hits but a short query never lands
+ *   inside a long cached prompt that merely contains its words.
  */
 export class QueryCache<T = QueryCacheResult> {
 	readonly maxSize: number;
@@ -122,7 +126,13 @@ export class QueryCache<T = QueryCacheResult> {
 					const results = JSON.parse(row.results_json) as T[];
 					const embedding = row.embedding_json === null ? null : (JSON.parse(row.embedding_json) as number[]);
 					const { scope, normalized } = splitEntryKey(row.normalized);
-					this.#entries.set(row.normalized, { scope, normalized, results, embedding });
+					this.#entries.set(row.normalized, {
+						scope,
+						normalized,
+						words: this.#wordSet(normalized),
+						results,
+						embedding,
+					});
 					this.#rememberKey(row.normalized, now);
 				} catch {
 					// Match Python's best-effort persistence loading: corrupt rows are ignored.
@@ -197,11 +207,11 @@ export class QueryCache<T = QueryCacheResult> {
 		for (const [cachedKey, cached] of this.#entries) {
 			if (cached.scope !== scope) continue;
 			if (this.#isExpired(cachedKey, now)) continue;
-			queryWords ??= new Set(normalized.split(/\s+/));
+			queryWords ??= this.#wordSet(normalized);
 			if (queryWords.size === 0) continue;
-			let overlap = 0;
-			for (const cachedWord of cached.normalized.split(/\s+/)) if (queryWords.has(cachedWord)) overlap += 1;
-			if (overlap >= queryWords.size * 0.7 && overlap >= 2) {
+			let shared = 0;
+			for (const word of queryWords) if (cached.words.has(word)) shared += 1;
+			if (shared >= 2 && shared >= queryWords.size * 0.7 && shared >= cached.words.size * 0.5) {
 				this.#touchKey(cachedKey);
 				this.hits += 1;
 				this.tier4Hits += 1;
@@ -222,7 +232,13 @@ export class QueryCache<T = QueryCacheResult> {
 		const storedEmbedding =
 			embedding !== undefined && embedding !== null && embedding.length !== 0 ? embedding : null;
 		this.#entries.delete(key);
-		this.#entries.set(key, { scope, normalized, results, embedding: storedEmbedding });
+		this.#entries.set(key, {
+			scope,
+			normalized,
+			words: this.#wordSet(normalized),
+			results,
+			embedding: storedEmbedding,
+		});
 		this.#rememberKey(key, now);
 		this.#putPersistent(key, results, storedEmbedding);
 		this.#evictIfNeeded();