@linxiraos/pi-mnemopi 1.1.26 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +13 -2
- package/README.md +1 -1
- package/dist/types/core/beam/index.d.ts +1 -0
- package/dist/types/core/embeddings.d.ts +5 -24
- package/dist/types/core/episodic-graph.d.ts +0 -1
- package/dist/types/core/fastembed-model-cache.d.ts +8 -2
- package/dist/types/core/query-cache.d.ts +3 -1
- package/dist/types/core/text-window.d.ts +4 -0
- package/package.json +8 -8
- package/src/core/beam/helpers.ts +27 -23
- package/src/core/beam/index.ts +1 -0
- package/src/core/beam/recall.ts +74 -62
- package/src/core/beam/store.ts +8 -6
- package/src/core/embeddings.ts +25 -93
- package/src/core/episodic-graph.ts +111 -97
- package/src/core/extraction.ts +96 -84
- package/src/core/fastembed-model-cache.ts +41 -31
- package/src/core/fastembed-runtime.ts +12 -1
- package/src/core/memory.ts +101 -83
- package/src/core/polyphonic-recall.ts +12 -8
- package/src/core/query-cache.ts +23 -7
- package/src/core/text-window.ts +14 -0
- package/src/db.ts +6 -4
package/CHANGELOG.md
CHANGED
|
@@ -1,7 +1,18 @@
|
|
|
1
|
-
# Changelog
|
|
2
|
-
|
|
3
1
|
## [Unreleased]
|
|
4
2
|
|
|
3
|
+
## [1.2.0] - 2026-10-12
|
|
4
|
+
|
|
5
|
+
- Fixed proactive linking freezing the host for seconds per stored memory on large banks; a memory's graph links are now written in one commit instead of one per link ([#14998](https://github.com/can1357/oh-my-pi/issues/14998)).
|
|
6
|
+
- Fixed long retained transcripts exhausting local memory extraction models by limiting each extraction input to 8192 characters while keeping the opening context and newest turns ([#14956](https://github.com/can1357/oh-my-pi/issues/14956)).
|
|
7
|
+
|
|
8
|
+
### Fixed
|
|
9
|
+
|
|
10
|
+
- Fixed new memories gaining graph links (`related_to`, `references`, `ctx`) to memories that were already invalidated, superseded, or expired; proactive linking and consolidation now link only to memories recall can still return, so the graph stops growing toward retired memories ([#14427](https://github.com/can1357/oh-my-pi/pull/14427) by [@tickernelz](https://github.com/tickernelz)).
|
|
11
|
+
|
|
12
|
+
## [1.1.27] - 2026-10-07
|
|
13
|
+
|
|
14
|
+
- 版本线推进至 1.1.27;本版无独立用户可见变化。
|
|
15
|
+
|
|
5
16
|
## [1.1.26] - 2026-10-03
|
|
6
17
|
|
|
7
18
|
- 版本线推进至 1.1.26;本版无独立用户可见变化。
|
package/README.md
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { Database } from "bun:sqlite";
|
|
2
2
|
import type { VeracityConsolidator } from "../veracity-consolidation.js";
|
|
3
3
|
import type { BeamCaches, BeamConfig, BeamEvent, BeamMemoryOptions, BeamMemoryState, BeamStats, ImportStats, MemoriaRetrieveResult, Metadata, RecallEnhancedOptions, RecallOptions, RecallResult, RememberBatchItem, RememberBatchOptions, RememberOptions, SleepResult } from "./types.js";
|
|
4
|
+
export { clipRecallContent } from "./recall.js";
|
|
4
5
|
export { initBeam } from "./schema.js";
|
|
5
6
|
export type * from "./types.js";
|
|
6
7
|
export declare class BeamMemory implements BeamMemoryState {
|
|
@@ -32,30 +32,11 @@ export type LocalModelInitializer = (options: LocalModelInitOptions) => Promise<
|
|
|
32
32
|
*/
|
|
33
33
|
export declare function quarantineCorruptModelFile(message: string, cacheDir?: string): Promise<boolean>;
|
|
34
34
|
/**
|
|
35
|
-
*
|
|
36
|
-
*
|
|
37
|
-
*
|
|
38
|
-
*
|
|
39
|
-
*
|
|
40
|
-
* short-circuits on the existing model dir and never re-downloads, and
|
|
41
|
-
* `downloadFileFromGCS` reuses a leftover partial `<model>.tar.gz` as-is, so
|
|
42
|
-
* the cache is stuck broken forever: recall silently dies and the rebuild
|
|
43
|
-
* queue grows every session. Delete the incomplete model dir AND the leftover
|
|
44
|
-
* partial archive so the retry re-downloads from scratch. The model path is
|
|
45
|
-
* error-message CONTENT, so the dir is only removed when it is a direct child
|
|
46
|
-
* of the fastembed cache root — never touch a directory a dependency merely
|
|
47
|
-
* names. Returns true when a retry is worth attempting (the incomplete cache
|
|
48
|
-
* was cleared, or already gone from a concurrent heal).
|
|
49
|
-
* @internal exported for tests
|
|
50
|
-
*/
|
|
51
|
-
export declare function clearIncompleteModelCache(message: string, cacheDir?: string): Promise<boolean>;
|
|
52
|
-
/**
|
|
53
|
-
* Shared local-model initializer: FlagEmbedding.init with BOTH cache heals.
|
|
54
|
-
* Missing sidecars (config/tokenizer/tokens map) re-fetch and retry; a
|
|
55
|
-
* corrupt model blob (Protobuf parse failure) quarantines the file and
|
|
56
|
-
* retries THROUGH the sidecar heal, so a cache that is broken in both ways
|
|
57
|
-
* still recovers in one pass. Also the initializer the embed worker uses in
|
|
58
|
-
* its subprocess; the in-process seam stays {@link setLocalModelInitializer}.
|
|
35
|
+
* Shared local-model initializer: FlagEmbedding.init with a corrupt-cache heal.
|
|
36
|
+
* A corrupt model blob (Protobuf parse failure) quarantines the file and
|
|
37
|
+
* retries once; fastembed re-downloads any file missing from the model's
|
|
38
|
+
* cache directory. Also the initializer the embed worker uses in its
|
|
39
|
+
* subprocess; the in-process seam stays {@link setLocalModelInitializer}.
|
|
59
40
|
*/
|
|
60
41
|
export declare function defaultLocalModelInitializer(options: LocalModelInitOptions): Promise<LocalEmbeddingModel>;
|
|
61
42
|
export declare function embeddingsDisabled(): boolean;
|
|
@@ -1,2 +1,8 @@
|
|
|
1
|
-
/**
|
|
2
|
-
|
|
1
|
+
/**
|
|
2
|
+
* Whether `FlagEmbedding.init` can load `model` from `cacheDir` without a
|
|
3
|
+
* download. fastembed caches each file under `<cacheDir>/<repo with / → _>/`
|
|
4
|
+
* and fetches only the files that are missing, so a model is cached exactly
|
|
5
|
+
* when every file is present. Unknown models report false. The agent's embed
|
|
6
|
+
* client uses this to decide whether an init shows download activity.
|
|
7
|
+
*/
|
|
8
|
+
export declare function isFastembedModelCached(model: string, cacheDir: string): Promise<boolean>;
|
|
@@ -31,7 +31,9 @@ export declare function isQueryCacheEnabled(useCache?: boolean, env?: Env, confi
|
|
|
31
31
|
*
|
|
32
32
|
* - Tier 1: exact normalized query match.
|
|
33
33
|
* - Tiers 2/3: query embedding cosine >= 0.88, or >= 0.78 plus word overlap.
|
|
34
|
-
* - Tier 4:
|
|
34
|
+
* - Tier 4: shared distinct words cover at least 70% of the new query and half of the
|
|
35
|
+
* cached one, so a reworded or trimmed query hits but a short query never lands
|
|
36
|
+
* inside a long cached prompt that merely contains its words.
|
|
35
37
|
*/
|
|
36
38
|
export declare class QueryCache<T = QueryCacheResult> {
|
|
37
39
|
#private;
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
/** Default character budget for a single embedding or fact-extraction input. */
|
|
2
|
+
export declare const DEFAULT_INPUT_CHARS = 8192;
|
|
3
|
+
/** Preserve the opening context and newest turns of an oversized chronological input. */
|
|
4
|
+
export declare function clipToWindow(text: string, max: number): string;
|
package/package.json
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
{
|
|
2
2
|
"type": "module",
|
|
3
3
|
"name": "@linxiraos/pi-mnemopi",
|
|
4
|
-
"version": "1.
|
|
5
|
-
"description": "Local SQLite memory engine for
|
|
4
|
+
"version": "1.2.0",
|
|
5
|
+
"description": "Local SQLite memory engine for Zeta agents",
|
|
6
6
|
"homepage": "https://linxira-os.github.io/zeta/",
|
|
7
7
|
"author": "Linxira-OS",
|
|
8
8
|
"contributors": [
|
|
@@ -39,13 +39,13 @@
|
|
|
39
39
|
"fmt": "oxfmt --no-error-on-unmatched-pattern 'src/**/*.{ts,tsx}' '{test,bench,examples,scripts}/**/*.ts' '*.ts'"
|
|
40
40
|
},
|
|
41
41
|
"dependencies": {
|
|
42
|
-
"@linxiraos/pi-ai": "1.
|
|
43
|
-
"@linxiraos/pi-catalog": "1.
|
|
44
|
-
"@linxiraos/pi-natives": "1.
|
|
45
|
-
"@linxiraos/pi-utils": "1.
|
|
42
|
+
"@linxiraos/pi-ai": "1.2.0",
|
|
43
|
+
"@linxiraos/pi-catalog": "1.2.0",
|
|
44
|
+
"@linxiraos/pi-natives": "1.2.0",
|
|
45
|
+
"@linxiraos/pi-utils": "1.2.0"
|
|
46
46
|
},
|
|
47
47
|
"peerDependencies": {
|
|
48
|
-
"fastembed": "
|
|
48
|
+
"fastembed": "3.0.0",
|
|
49
49
|
"onnxruntime-node": "1.21.0"
|
|
50
50
|
},
|
|
51
51
|
"peerDependenciesMeta": {
|
|
@@ -58,7 +58,7 @@
|
|
|
58
58
|
},
|
|
59
59
|
"devDependencies": {
|
|
60
60
|
"@types/bun": "^1.3.14",
|
|
61
|
-
"fastembed": "
|
|
61
|
+
"fastembed": "3.0.0",
|
|
62
62
|
"onnxruntime-node": "1.26.0"
|
|
63
63
|
},
|
|
64
64
|
"engines": {
|
package/src/core/beam/helpers.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { Database } from "bun:sqlite";
|
|
2
|
-
import { logger } from "@linxiraos/pi-utils";
|
|
2
|
+
import { logger, withLoopPhase } from "@linxiraos/pi-utils";
|
|
3
3
|
import { generateId as generateTimedId, sha256Hex16, stableMemoryId } from "../../util/ids";
|
|
4
4
|
import { cjkFtsTerms, containsSpacelessCjk, ftsQueryTerms, hasCjk, isCjkChar, recallTokens } from "../../util/regex";
|
|
5
5
|
import { currentEmbeddingModel, embed } from "../embeddings";
|
|
@@ -727,29 +727,33 @@ async function runEmbedding(beam: BeamMemoryState, items: readonly EmbedItem[]):
|
|
|
727
727
|
try {
|
|
728
728
|
const matrix = await embed(items.map(item => item.content));
|
|
729
729
|
if (matrix === null) return;
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
730
|
+
// Persisting the vectors runs synchronously on the caller's event loop; label it so a
|
|
731
|
+
// stall is attributable. The embedding itself is awaited above, outside the label.
|
|
732
|
+
withLoopPhase("mnemopi.embed-persist", () => {
|
|
733
|
+
const model = currentEmbeddingModel();
|
|
734
|
+
using insertEmbedding = beam.db.prepare(
|
|
735
|
+
"INSERT OR REPLACE INTO memory_embeddings(memory_id, embedding_json, model) VALUES (?, ?, ?)",
|
|
736
|
+
);
|
|
737
|
+
let committed = 0;
|
|
738
|
+
const insertMany = beam.db.transaction((rows: readonly EmbedItem[]) => {
|
|
739
|
+
for (let i = 0; i < rows.length; i += 1) {
|
|
740
|
+
const vector = matrix[i];
|
|
741
|
+
const item = rows[i];
|
|
742
|
+
if (vector === undefined || item === undefined) continue;
|
|
743
|
+
insertEmbedding.run(item.memoryId, JSON.stringify(Array.from(vector)), model);
|
|
744
|
+
committed += 1;
|
|
745
|
+
}
|
|
746
|
+
});
|
|
747
|
+
insertMany(items);
|
|
748
|
+
// A recall taken while these vectors were still generating cached an FTS-only ranking, and
|
|
749
|
+
// committing vectors changes what recall returns -- so that cache must be dropped, or the
|
|
750
|
+
// pre-embedding order keeps being served for the whole cache TTL. Gated on `committed`: a
|
|
751
|
+
// batch whose provider returned a short or empty matrix inserts nothing, changes no ranking,
|
|
752
|
+
// and invalidating there would only discard a still-valid cache. Only the query cache is
|
|
753
|
+
// affected; the polyphonic subject dictionary is built from facts/gists, which an embedding
|
|
754
|
+
// batch never touches.
|
|
755
|
+
if (committed > 0) beam.caches.queryCache?.invalidate();
|
|
743
756
|
});
|
|
744
|
-
insertMany(items);
|
|
745
|
-
// A recall taken while these vectors were still generating cached an FTS-only ranking, and
|
|
746
|
-
// committing vectors changes what recall returns -- so that cache must be dropped, or the
|
|
747
|
-
// pre-embedding order keeps being served for the whole cache TTL. Gated on `committed`: a
|
|
748
|
-
// batch whose provider returned a short or empty matrix inserts nothing, changes no ranking,
|
|
749
|
-
// and invalidating there would only discard a still-valid cache. Only the query cache is
|
|
750
|
-
// affected; the polyphonic subject dictionary is built from facts/gists, which an embedding
|
|
751
|
-
// batch never touches.
|
|
752
|
-
if (committed > 0) beam.caches.queryCache?.invalidate();
|
|
753
757
|
} catch (error) {
|
|
754
758
|
// Background embedding generation is best-effort: a failing provider, a closed DB
|
|
755
759
|
// during shutdown, or a transient API error must never disrupt the synchronous
|
package/src/core/beam/index.ts
CHANGED
package/src/core/beam/recall.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { popLoopPhase, pushLoopPhase, withLoopPhase } from "@linxiraos/pi-utils";
|
|
1
2
|
import { normalizedRecallWeights, polyphonicRecallEnabled, temporalHalflifeHours } from "../../config";
|
|
2
3
|
import { hasCjk, matchesWordForm } from "../../util/regex";
|
|
3
4
|
import { embedQuery } from "../embeddings";
|
|
@@ -923,35 +924,37 @@ export async function recall(
|
|
|
923
924
|
const derived = query.length > 0 ? await embedQuery(query) : null;
|
|
924
925
|
temporalOptions.queryEmbedding = derived === null ? null : Array.from(derived);
|
|
925
926
|
}
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
const
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
927
|
+
return withLoopPhase("mnemopi.recall", () => {
|
|
928
|
+
let weights = normalizedRecallWeights(
|
|
929
|
+
options.vecWeight ?? beam.config.vecWeight,
|
|
930
|
+
options.ftsWeight ?? beam.config.ftsWeight,
|
|
931
|
+
options.importanceWeight ?? beam.config.importanceWeight,
|
|
932
|
+
);
|
|
933
|
+
if (options.useIntent === true) {
|
|
934
|
+
const intent = classifyIntent(query);
|
|
935
|
+
weights = adjustWeights(weights[0], weights[1], weights[2], intent);
|
|
936
|
+
}
|
|
937
|
+
const useSynonyms = options.useSynonyms !== false;
|
|
938
|
+
const tokens = expandedTokens(query, useSynonyms);
|
|
939
|
+
const tokenGroups = expandedTokenGroups(query, useSynonyms);
|
|
940
|
+
const candidates = collectMemoryCandidates(beam, query, topK, temporalOptions);
|
|
941
|
+
const scored: RecallResult[] = [];
|
|
942
|
+
for (const candidate of candidates) {
|
|
943
|
+
const result = scoreCandidate(candidate, tokens, tokenGroups, weights, temporalOptions);
|
|
944
|
+
if (result !== null) scored.push(result);
|
|
945
|
+
}
|
|
946
|
+
scored.sort((left, right) => (right.score ?? 0) - (left.score ?? 0));
|
|
947
|
+
let finalResults = dedupCrossTierSummaryLinks(beam, dedupeResults(scored));
|
|
948
|
+
if (query.length > 0 && tokens.length >= 4 && finalResults.length > topK)
|
|
949
|
+
finalResults = diversifyByCoverage(finalResults, tokens, topK);
|
|
950
|
+
if (options.useMmr === true && finalResults.length > 1) {
|
|
951
|
+
finalResults = rerankRecallResults(finalResults, options.mmrLambda ?? 0.7, topK);
|
|
952
|
+
} else {
|
|
953
|
+
finalResults = finalResults.slice(0, topK);
|
|
954
|
+
}
|
|
955
|
+
if (temporalOptions.updateRecallCounts !== false) updateRecallCounts(beam, finalResults, temporalOptions);
|
|
956
|
+
return finalResults;
|
|
957
|
+
});
|
|
955
958
|
}
|
|
956
959
|
|
|
957
960
|
function diversifyByCoverage(
|
|
@@ -1034,37 +1037,44 @@ export async function recallEnhanced(
|
|
|
1034
1037
|
let cache: QueryCache<RecallResult> | null = null;
|
|
1035
1038
|
let token = "";
|
|
1036
1039
|
let scope = "";
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
beam.caches.queryCacheToken
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
const
|
|
1054
|
-
if (
|
|
1055
|
-
|
|
1040
|
+
pushLoopPhase("mnemopi.recall");
|
|
1041
|
+
try {
|
|
1042
|
+
if (cacheEnabled) {
|
|
1043
|
+
beam.caches.queryCache ??= new QueryCache<RecallResult>({
|
|
1044
|
+
maxSize: ENHANCED_RECALL_CACHE_MAX_ENTRIES,
|
|
1045
|
+
ttlSeconds: ENHANCED_RECALL_CACHE_TTL_SECONDS,
|
|
1046
|
+
});
|
|
1047
|
+
cache = beam.caches.queryCache;
|
|
1048
|
+
// Explicit hooks invalidate on this beam's own writes; the token also catches writes
|
|
1049
|
+
// no hook covers (sleep, graph ingest) and commits from other connections.
|
|
1050
|
+
token = databaseWriteToken(beam);
|
|
1051
|
+
if (beam.caches.queryCacheToken !== token) {
|
|
1052
|
+
cache.invalidate();
|
|
1053
|
+
beam.caches.queryCacheToken = token;
|
|
1054
|
+
}
|
|
1055
|
+
scope = enhancedRecallCacheScope(beam, topK, options, polyphonic);
|
|
1056
|
+
const cached = cache.get(query, queryEmbedding, scope);
|
|
1057
|
+
if (cached !== null) {
|
|
1058
|
+
const results = structuredClone(cached) as RecallResult[];
|
|
1059
|
+
if (countRecalls) countRecallsKeepingCache(beam, results, runOptions, token);
|
|
1060
|
+
return results;
|
|
1061
|
+
}
|
|
1056
1062
|
}
|
|
1063
|
+
} finally {
|
|
1064
|
+
popLoopPhase();
|
|
1057
1065
|
}
|
|
1058
1066
|
|
|
1059
1067
|
const results = polyphonic
|
|
1060
1068
|
? await polyphonicRecallEnhanced(beam, query, topK, runOptions)
|
|
1061
1069
|
: await linearRecallEnhanced(beam, query, topK, runOptions);
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1070
|
+
return withLoopPhase("mnemopi.recall", () => {
|
|
1071
|
+
// A write committed while recall awaited may be missing from this ranking: skip the
|
|
1072
|
+
// put and keep the stale token so the next lookup starts from an empty cache.
|
|
1073
|
+
const cacheable = cache !== null && databaseWriteToken(beam) === token;
|
|
1074
|
+
if (cacheable) cache?.put(query, structuredClone(results), queryEmbedding, scope);
|
|
1075
|
+
if (countRecalls) countRecallsKeepingCache(beam, results, runOptions, cacheable ? token : null);
|
|
1076
|
+
return results;
|
|
1077
|
+
});
|
|
1068
1078
|
}
|
|
1069
1079
|
|
|
1070
1080
|
async function linearRecallEnhanced(
|
|
@@ -1084,14 +1094,16 @@ async function linearRecallEnhanced(
|
|
|
1084
1094
|
...enhancedOptions,
|
|
1085
1095
|
updateRecallCounts: false,
|
|
1086
1096
|
});
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
|
|
1097
|
+
return withLoopPhase("mnemopi.recall", () => {
|
|
1098
|
+
if (options.includeFacts === true) {
|
|
1099
|
+
const facts = factRecall(beam, query, factRecallLimit(topK));
|
|
1100
|
+
results.push(...facts);
|
|
1101
|
+
}
|
|
1102
|
+
results.sort((left, right) => (right.score ?? 0) - (left.score ?? 0));
|
|
1103
|
+
const finalResults = rerankRecallResults(results, options.mmrLambda ?? 0.7, topK);
|
|
1104
|
+
if (enhancedOptions.updateRecallCounts !== false) updateRecallCounts(beam, finalResults, enhancedOptions);
|
|
1105
|
+
return finalResults;
|
|
1106
|
+
});
|
|
1095
1107
|
}
|
|
1096
1108
|
|
|
1097
1109
|
/**
|
package/src/core/beam/store.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { Database, SQLQueryBindings } from "bun:sqlite";
|
|
2
|
-
import { logger } from "@linxiraos/pi-utils";
|
|
2
|
+
import { logger, withLoopPhase } from "@linxiraos/pi-utils";
|
|
3
3
|
import { transaction } from "../../db";
|
|
4
4
|
import { toUtcIso } from "../../util/datetime";
|
|
5
5
|
import { generateId } from "../../util/ids";
|
|
@@ -307,9 +307,11 @@ function proactiveLinkIfEnabled(
|
|
|
307
307
|
async function runFactExtraction(beam: BeamMemoryState, memoryId: string, content: string): Promise<void> {
|
|
308
308
|
try {
|
|
309
309
|
const extracted = await extractFactCategoriesSafe(content);
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
310
|
+
withLoopPhase("mnemopi.extract", () => {
|
|
311
|
+
if (countExtractedFactCategories(extracted) === 0) return;
|
|
312
|
+
storeExtractedFactCategories(beam, extracted, 0, memoryId);
|
|
313
|
+
invalidateCaches(beam);
|
|
314
|
+
});
|
|
313
315
|
} catch {
|
|
314
316
|
// Background fact extraction is best-effort and never surfaces to the caller.
|
|
315
317
|
}
|
|
@@ -740,7 +742,7 @@ export function updateWorking(
|
|
|
740
742
|
export function get(beam: BeamMemoryState, memoryId: string): Row | null {
|
|
741
743
|
using workingStatement = beam.db.prepare(`
|
|
742
744
|
SELECT id, content, source, timestamp, session_id,
|
|
743
|
-
importance, metadata_json, veracity, created_at
|
|
745
|
+
importance, metadata_json, veracity, created_at, valid_until, superseded_by
|
|
744
746
|
FROM working_memory
|
|
745
747
|
WHERE id = ?
|
|
746
748
|
`);
|
|
@@ -749,7 +751,7 @@ export function get(beam: BeamMemoryState, memoryId: string): Row | null {
|
|
|
749
751
|
|
|
750
752
|
using episodicStatement = beam.db.prepare(`
|
|
751
753
|
SELECT id, content, source, timestamp, session_id,
|
|
752
|
-
importance, metadata_json, veracity, created_at
|
|
754
|
+
importance, metadata_json, veracity, created_at, valid_until, superseded_by
|
|
753
755
|
FROM episodic_memory
|
|
754
756
|
WHERE id = ? AND (session_id = ? OR scope = 'global')
|
|
755
757
|
`);
|
package/src/core/embeddings.ts
CHANGED
|
@@ -11,10 +11,10 @@ import {
|
|
|
11
11
|
fetchWithRetry,
|
|
12
12
|
getFastembedCacheDir,
|
|
13
13
|
logger,
|
|
14
|
+
withLoopPhase,
|
|
14
15
|
} from "@linxiraos/pi-utils";
|
|
15
16
|
import { LRUCache } from "@linxiraos/pi-utils/lru";
|
|
16
17
|
import type { EmbeddingModel } from "fastembed";
|
|
17
|
-
import { ensureFastembedModelSidecars } from "./fastembed-model-cache";
|
|
18
18
|
import { loadFastembed } from "./fastembed-runtime";
|
|
19
19
|
import {
|
|
20
20
|
type EmbeddingOutput,
|
|
@@ -22,6 +22,7 @@ import {
|
|
|
22
22
|
mnemopiDebugEnabled,
|
|
23
23
|
resolveEmbeddingProvider,
|
|
24
24
|
} from "./runtime-options";
|
|
25
|
+
import { clipToWindow, DEFAULT_INPUT_CHARS } from "./text-window";
|
|
25
26
|
|
|
26
27
|
export type { EmbeddingOutput } from "./runtime-options";
|
|
27
28
|
export { cosineSimilarity } from "./vector-math";
|
|
@@ -93,74 +94,22 @@ export async function quarantineCorruptModelFile(message: string, cacheDir?: str
|
|
|
93
94
|
}
|
|
94
95
|
|
|
95
96
|
/**
|
|
96
|
-
*
|
|
97
|
-
*
|
|
98
|
-
*
|
|
99
|
-
*
|
|
100
|
-
*
|
|
101
|
-
* short-circuits on the existing model dir and never re-downloads, and
|
|
102
|
-
* `downloadFileFromGCS` reuses a leftover partial `<model>.tar.gz` as-is, so
|
|
103
|
-
* the cache is stuck broken forever: recall silently dies and the rebuild
|
|
104
|
-
* queue grows every session. Delete the incomplete model dir AND the leftover
|
|
105
|
-
* partial archive so the retry re-downloads from scratch. The model path is
|
|
106
|
-
* error-message CONTENT, so the dir is only removed when it is a direct child
|
|
107
|
-
* of the fastembed cache root — never touch a directory a dependency merely
|
|
108
|
-
* names. Returns true when a retry is worth attempting (the incomplete cache
|
|
109
|
-
* was cleared, or already gone from a concurrent heal).
|
|
110
|
-
* @internal exported for tests
|
|
111
|
-
*/
|
|
112
|
-
export async function clearIncompleteModelCache(message: string, cacheDir?: string): Promise<boolean> {
|
|
113
|
-
const match = /Model file not found at (.+?\.onnx)\b/i.exec(message);
|
|
114
|
-
if (!match) return false;
|
|
115
|
-
const modelFile = nodePath.resolve(match[1]);
|
|
116
|
-
const modelDir = nodePath.dirname(modelFile);
|
|
117
|
-
const cacheRoot = nodePath.resolve(cacheDir ?? getFastembedCacheDir());
|
|
118
|
-
// Only a direct child of the cache root: `<cacheRoot>/<model>`.
|
|
119
|
-
if (nodePath.dirname(modelDir) !== cacheRoot) return false;
|
|
120
|
-
try {
|
|
121
|
-
await fsp.rm(modelDir, { recursive: true, force: true });
|
|
122
|
-
await fsp.rm(`${modelDir}.tar.gz`, { force: true });
|
|
123
|
-
logger.warn("mnemopi: cleared incomplete local embedding model cache; re-downloading", { modelDir });
|
|
124
|
-
} catch {
|
|
125
|
-
// Concurrent heal or vanished dir: the single retry stays safe.
|
|
126
|
-
}
|
|
127
|
-
return true;
|
|
128
|
-
}
|
|
129
|
-
|
|
130
|
-
const SIDECAR_ERROR_RE =
|
|
131
|
-
/(?:Config file not found at .*config|Tokenizer file not found at .*tokenizer|Tokens map file not found at .*special_tokens_map)/u;
|
|
132
|
-
|
|
133
|
-
/**
|
|
134
|
-
* Shared local-model initializer: FlagEmbedding.init with BOTH cache heals.
|
|
135
|
-
* Missing sidecars (config/tokenizer/tokens map) re-fetch and retry; a
|
|
136
|
-
* corrupt model blob (Protobuf parse failure) quarantines the file and
|
|
137
|
-
* retries THROUGH the sidecar heal, so a cache that is broken in both ways
|
|
138
|
-
* still recovers in one pass. Also the initializer the embed worker uses in
|
|
139
|
-
* its subprocess; the in-process seam stays {@link setLocalModelInitializer}.
|
|
97
|
+
* Shared local-model initializer: FlagEmbedding.init with a corrupt-cache heal.
|
|
98
|
+
* A corrupt model blob (Protobuf parse failure) quarantines the file and
|
|
99
|
+
* retries once; fastembed re-downloads any file missing from the model's
|
|
100
|
+
* cache directory. Also the initializer the embed worker uses in its
|
|
101
|
+
* subprocess; the in-process seam stays {@link setLocalModelInitializer}.
|
|
140
102
|
*/
|
|
141
103
|
export async function defaultLocalModelInitializer(options: LocalModelInitOptions): Promise<LocalEmbeddingModel> {
|
|
142
104
|
const cacheDir = options.cacheDir ?? getFastembedCacheDir();
|
|
143
105
|
const initOptions = options.cacheDir === undefined ? { ...options, cacheDir } : options;
|
|
144
106
|
const { FlagEmbedding } = await loadFastembed();
|
|
145
|
-
const initWithSidecarHeal = async (): Promise<LocalEmbeddingModel> => {
|
|
146
|
-
try {
|
|
147
|
-
return await FlagEmbedding.init(initOptions);
|
|
148
|
-
} catch (error) {
|
|
149
|
-
const message = error instanceof Error ? error.message : "";
|
|
150
|
-
if (!SIDECAR_ERROR_RE.test(message)) throw error;
|
|
151
|
-
if (!(await ensureFastembedModelSidecars(options.model, cacheDir))) throw error;
|
|
152
|
-
return FlagEmbedding.init(initOptions);
|
|
153
|
-
}
|
|
154
|
-
};
|
|
155
107
|
try {
|
|
156
|
-
return await
|
|
108
|
+
return await FlagEmbedding.init(initOptions);
|
|
157
109
|
} catch (error) {
|
|
158
110
|
const message = error instanceof Error ? error.message : "";
|
|
159
111
|
if (/Protobuf parsing failed/i.test(message) && (await quarantineCorruptModelFile(message, cacheDir))) {
|
|
160
|
-
return
|
|
161
|
-
}
|
|
162
|
-
if (await clearIncompleteModelCache(message, cacheDir)) {
|
|
163
|
-
return initWithSidecarHeal();
|
|
112
|
+
return FlagEmbedding.init(initOptions);
|
|
164
113
|
}
|
|
165
114
|
throw error;
|
|
166
115
|
}
|
|
@@ -219,29 +168,7 @@ function effectiveMaxInputChars(): number {
|
|
|
219
168
|
if (override !== undefined) return Math.max(0, Math.trunc(override));
|
|
220
169
|
const envValue = Number.parseInt($env.MNEMOPI_EMBEDDING_MAX_INPUT_CHARS ?? "", 10);
|
|
221
170
|
if (Number.isFinite(envValue) && envValue >= 0) return envValue;
|
|
222
|
-
return
|
|
223
|
-
}
|
|
224
|
-
|
|
225
|
-
/** Elision marker injected between the retained head and tail of an oversized input. */
|
|
226
|
-
const EMBEDDING_ELISION_MARKER = "\n\n[...]\n\n";
|
|
227
|
-
|
|
228
|
-
/**
|
|
229
|
-
* Right-clip a single oversized input to {@link max} chars while preserving
|
|
230
|
-
* both ends. Retention transcripts are chronological (oldest → newest), so a
|
|
231
|
-
* naive `slice(0, max)` would drop the most recent — and most semantically
|
|
232
|
-
* loaded — turns once a session passed the cap, leaving every later retained
|
|
233
|
-
* episode with essentially the same prefix vector. Keeping a head/tail split
|
|
234
|
-
* lets the embedding capture the topic setup at the start AND the latest
|
|
235
|
-
* exchanges at the end. Falls back to a tail-only clip when `max` is too
|
|
236
|
-
* small to fit the elision marker plus a useful slice on either side.
|
|
237
|
-
*/
|
|
238
|
-
function clipToWindow(text: string, max: number): string {
|
|
239
|
-
if (text.length <= max) return text;
|
|
240
|
-
if (max <= EMBEDDING_ELISION_MARKER.length + 16) return text.slice(text.length - max);
|
|
241
|
-
const budget = max - EMBEDDING_ELISION_MARKER.length;
|
|
242
|
-
const headLen = budget >>> 1;
|
|
243
|
-
const tailLen = budget - headLen;
|
|
244
|
-
return text.slice(0, headLen) + EMBEDDING_ELISION_MARKER + text.slice(text.length - tailLen);
|
|
171
|
+
return DEFAULT_INPUT_CHARS;
|
|
245
172
|
}
|
|
246
173
|
|
|
247
174
|
/**
|
|
@@ -368,9 +295,11 @@ export function embeddingDimFor(modelName: string): number {
|
|
|
368
295
|
async function collectMatrix(batches: EmbeddingOutput): Promise<EmbeddingMatrix> {
|
|
369
296
|
const rows: Vector[] = [];
|
|
370
297
|
for await (const batch of batches) {
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
298
|
+
withLoopPhase("mnemopi.embed", () => {
|
|
299
|
+
for (const row of batch) {
|
|
300
|
+
rows.push(new Float32Array(row));
|
|
301
|
+
}
|
|
302
|
+
});
|
|
374
303
|
}
|
|
375
304
|
return rows;
|
|
376
305
|
}
|
|
@@ -462,12 +391,15 @@ async function embedApi(texts: readonly string[]): Promise<EmbeddingMatrix | nul
|
|
|
462
391
|
if (!response.ok) {
|
|
463
392
|
return null;
|
|
464
393
|
}
|
|
465
|
-
const
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
394
|
+
const data = (await response.json()) as { data?: Array<{ embedding: number[] }> };
|
|
395
|
+
return withLoopPhase("mnemopi.embed", () => {
|
|
396
|
+
const { data: rows } = data;
|
|
397
|
+
if (rows === undefined) {
|
|
398
|
+
return null;
|
|
399
|
+
}
|
|
400
|
+
apiCallCount += 1;
|
|
401
|
+
return rows.map(row => new Float32Array(row.embedding));
|
|
402
|
+
});
|
|
471
403
|
} catch (error) {
|
|
472
404
|
logger.debug("mnemopi embedding request failed", { status: extractHttpStatusFromError(error) });
|
|
473
405
|
return null;
|
|
@@ -599,7 +531,7 @@ export async function embed(texts: readonly string[]): Promise<EmbeddingMatrix |
|
|
|
599
531
|
return null;
|
|
600
532
|
}
|
|
601
533
|
try {
|
|
602
|
-
const vectors = await collectMatrix(model.embed([...texts]));
|
|
534
|
+
const vectors = await withLoopPhase("mnemopi.embed", () => collectMatrix(model.embed([...texts])));
|
|
603
535
|
if (vectors.length === 1) {
|
|
604
536
|
const vector = vectors[0];
|
|
605
537
|
if (vector !== undefined) {
|