@mnemoai/core 1.1.0 → 1.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.d.ts +2 -0
- package/dist/cli.d.ts.map +1 -0
- package/dist/cli.js +7 -0
- package/dist/cli.js.map +7 -0
- package/dist/index.d.ts +136 -0
- package/dist/index.d.ts.map +1 -0
- package/{index.ts → dist/index.js} +537 -1333
- package/dist/index.js.map +7 -0
- package/dist/src/access-tracker.d.ts +97 -0
- package/dist/src/access-tracker.d.ts.map +1 -0
- package/dist/src/access-tracker.js +184 -0
- package/dist/src/access-tracker.js.map +7 -0
- package/dist/src/adapters/chroma.d.ts +31 -0
- package/dist/src/adapters/chroma.d.ts.map +1 -0
- package/{src/adapters/chroma.ts → dist/src/adapters/chroma.js} +45 -107
- package/dist/src/adapters/chroma.js.map +7 -0
- package/dist/src/adapters/lancedb.d.ts +29 -0
- package/dist/src/adapters/lancedb.d.ts.map +1 -0
- package/{src/adapters/lancedb.ts → dist/src/adapters/lancedb.js} +41 -109
- package/dist/src/adapters/lancedb.js.map +7 -0
- package/dist/src/adapters/pgvector.d.ts +33 -0
- package/dist/src/adapters/pgvector.d.ts.map +1 -0
- package/{src/adapters/pgvector.ts → dist/src/adapters/pgvector.js} +42 -104
- package/dist/src/adapters/pgvector.js.map +7 -0
- package/dist/src/adapters/qdrant.d.ts +34 -0
- package/dist/src/adapters/qdrant.d.ts.map +1 -0
- package/dist/src/adapters/qdrant.js +132 -0
- package/dist/src/adapters/qdrant.js.map +7 -0
- package/dist/src/adaptive-retrieval.d.ts +14 -0
- package/dist/src/adaptive-retrieval.d.ts.map +1 -0
- package/dist/src/adaptive-retrieval.js +52 -0
- package/dist/src/adaptive-retrieval.js.map +7 -0
- package/dist/src/audit-log.d.ts +56 -0
- package/dist/src/audit-log.d.ts.map +1 -0
- package/dist/src/audit-log.js +139 -0
- package/dist/src/audit-log.js.map +7 -0
- package/dist/src/chunker.d.ts +45 -0
- package/dist/src/chunker.d.ts.map +1 -0
- package/dist/src/chunker.js +157 -0
- package/dist/src/chunker.js.map +7 -0
- package/dist/src/config.d.ts +70 -0
- package/dist/src/config.d.ts.map +1 -0
- package/dist/src/config.js +142 -0
- package/dist/src/config.js.map +7 -0
- package/dist/src/decay-engine.d.ts +73 -0
- package/dist/src/decay-engine.d.ts.map +1 -0
- package/dist/src/decay-engine.js +119 -0
- package/dist/src/decay-engine.js.map +7 -0
- package/dist/src/embedder.d.ts +94 -0
- package/dist/src/embedder.d.ts.map +1 -0
- package/{src/embedder.ts → dist/src/embedder.js} +119 -317
- package/dist/src/embedder.js.map +7 -0
- package/dist/src/extraction-prompts.d.ts +12 -0
- package/dist/src/extraction-prompts.d.ts.map +1 -0
- package/dist/src/extraction-prompts.js +311 -0
- package/dist/src/extraction-prompts.js.map +7 -0
- package/dist/src/license.d.ts +29 -0
- package/dist/src/license.d.ts.map +1 -0
- package/{src/license.ts → dist/src/license.js} +42 -113
- package/dist/src/license.js.map +7 -0
- package/dist/src/llm-client.d.ts +23 -0
- package/dist/src/llm-client.d.ts.map +1 -0
- package/{src/llm-client.ts → dist/src/llm-client.js} +22 -55
- package/dist/src/llm-client.js.map +7 -0
- package/dist/src/logger.d.ts +33 -0
- package/dist/src/logger.d.ts.map +1 -0
- package/dist/src/logger.js +35 -0
- package/dist/src/logger.js.map +7 -0
- package/dist/src/mcp-server.d.ts +16 -0
- package/dist/src/mcp-server.d.ts.map +1 -0
- package/{src/mcp-server.ts → dist/src/mcp-server.js} +81 -181
- package/dist/src/mcp-server.js.map +7 -0
- package/dist/src/memory-categories.d.ts +40 -0
- package/dist/src/memory-categories.d.ts.map +1 -0
- package/dist/src/memory-categories.js +33 -0
- package/dist/src/memory-categories.js.map +7 -0
- package/dist/src/memory-upgrader.d.ts +71 -0
- package/dist/src/memory-upgrader.d.ts.map +1 -0
- package/dist/src/memory-upgrader.js +238 -0
- package/dist/src/memory-upgrader.js.map +7 -0
- package/dist/src/migrate.d.ts +47 -0
- package/dist/src/migrate.d.ts.map +1 -0
- package/{src/migrate.ts → dist/src/migrate.js} +57 -165
- package/dist/src/migrate.js.map +7 -0
- package/dist/src/mnemo.d.ts +67 -0
- package/dist/src/mnemo.d.ts.map +1 -0
- package/dist/src/mnemo.js +66 -0
- package/dist/src/mnemo.js.map +7 -0
- package/dist/src/noise-filter.d.ts +23 -0
- package/dist/src/noise-filter.d.ts.map +1 -0
- package/dist/src/noise-filter.js +62 -0
- package/dist/src/noise-filter.js.map +7 -0
- package/dist/src/noise-prototypes.d.ts +40 -0
- package/dist/src/noise-prototypes.d.ts.map +1 -0
- package/dist/src/noise-prototypes.js +116 -0
- package/dist/src/noise-prototypes.js.map +7 -0
- package/dist/src/observability.d.ts +16 -0
- package/dist/src/observability.d.ts.map +1 -0
- package/dist/src/observability.js +53 -0
- package/dist/src/observability.js.map +7 -0
- package/dist/src/query-tracker.d.ts +27 -0
- package/dist/src/query-tracker.d.ts.map +1 -0
- package/dist/src/query-tracker.js +32 -0
- package/dist/src/query-tracker.js.map +7 -0
- package/dist/src/reflection-event-store.d.ts +44 -0
- package/dist/src/reflection-event-store.d.ts.map +1 -0
- package/dist/src/reflection-event-store.js +50 -0
- package/dist/src/reflection-event-store.js.map +7 -0
- package/dist/src/reflection-item-store.d.ts +58 -0
- package/dist/src/reflection-item-store.d.ts.map +1 -0
- package/dist/src/reflection-item-store.js +69 -0
- package/dist/src/reflection-item-store.js.map +7 -0
- package/dist/src/reflection-mapped-metadata.d.ts +47 -0
- package/dist/src/reflection-mapped-metadata.d.ts.map +1 -0
- package/dist/src/reflection-mapped-metadata.js +40 -0
- package/dist/src/reflection-mapped-metadata.js.map +7 -0
- package/dist/src/reflection-metadata.d.ts +11 -0
- package/dist/src/reflection-metadata.d.ts.map +1 -0
- package/dist/src/reflection-metadata.js +24 -0
- package/dist/src/reflection-metadata.js.map +7 -0
- package/dist/src/reflection-ranking.d.ts +13 -0
- package/dist/src/reflection-ranking.d.ts.map +1 -0
- package/{src/reflection-ranking.ts → dist/src/reflection-ranking.js} +12 -21
- package/dist/src/reflection-ranking.js.map +7 -0
- package/dist/src/reflection-retry.d.ts +30 -0
- package/dist/src/reflection-retry.d.ts.map +1 -0
- package/{src/reflection-retry.ts → dist/src/reflection-retry.js} +24 -64
- package/dist/src/reflection-retry.js.map +7 -0
- package/dist/src/reflection-slices.d.ts +42 -0
- package/dist/src/reflection-slices.d.ts.map +1 -0
- package/{src/reflection-slices.ts → dist/src/reflection-slices.js} +60 -136
- package/dist/src/reflection-slices.js.map +7 -0
- package/dist/src/reflection-store.d.ts +85 -0
- package/dist/src/reflection-store.d.ts.map +1 -0
- package/dist/src/reflection-store.js +407 -0
- package/dist/src/reflection-store.js.map +7 -0
- package/dist/src/resonance-state.d.ts +19 -0
- package/dist/src/resonance-state.d.ts.map +1 -0
- package/{src/resonance-state.ts → dist/src/resonance-state.js} +13 -42
- package/dist/src/resonance-state.js.map +7 -0
- package/dist/src/retriever.d.ts +228 -0
- package/dist/src/retriever.d.ts.map +1 -0
- package/dist/src/retriever.js +1006 -0
- package/dist/src/retriever.js.map +7 -0
- package/dist/src/scopes.d.ts +58 -0
- package/dist/src/scopes.d.ts.map +1 -0
- package/dist/src/scopes.js +252 -0
- package/dist/src/scopes.js.map +7 -0
- package/dist/src/self-improvement-files.d.ts +20 -0
- package/dist/src/self-improvement-files.d.ts.map +1 -0
- package/{src/self-improvement-files.ts → dist/src/self-improvement-files.js} +24 -49
- package/dist/src/self-improvement-files.js.map +7 -0
- package/dist/src/semantic-gate.d.ts +24 -0
- package/dist/src/semantic-gate.d.ts.map +1 -0
- package/dist/src/semantic-gate.js +86 -0
- package/dist/src/semantic-gate.js.map +7 -0
- package/dist/src/session-recovery.d.ts +9 -0
- package/dist/src/session-recovery.d.ts.map +1 -0
- package/{src/session-recovery.ts → dist/src/session-recovery.js} +40 -57
- package/dist/src/session-recovery.js.map +7 -0
- package/dist/src/smart-extractor.d.ts +107 -0
- package/dist/src/smart-extractor.d.ts.map +1 -0
- package/{src/smart-extractor.ts → dist/src/smart-extractor.js} +130 -383
- package/dist/src/smart-extractor.js.map +7 -0
- package/dist/src/smart-metadata.d.ts +103 -0
- package/dist/src/smart-metadata.d.ts.map +1 -0
- package/dist/src/smart-metadata.js +361 -0
- package/dist/src/smart-metadata.js.map +7 -0
- package/dist/src/storage-adapter.d.ts +102 -0
- package/dist/src/storage-adapter.d.ts.map +1 -0
- package/dist/src/storage-adapter.js +22 -0
- package/dist/src/storage-adapter.js.map +7 -0
- package/dist/src/store.d.ts +108 -0
- package/dist/src/store.d.ts.map +1 -0
- package/dist/src/store.js +939 -0
- package/dist/src/store.js.map +7 -0
- package/dist/src/tier-manager.d.ts +57 -0
- package/dist/src/tier-manager.d.ts.map +1 -0
- package/dist/src/tier-manager.js +80 -0
- package/dist/src/tier-manager.js.map +7 -0
- package/dist/src/tools.d.ts +43 -0
- package/dist/src/tools.d.ts.map +1 -0
- package/dist/src/tools.js +1075 -0
- package/dist/src/tools.js.map +7 -0
- package/dist/src/wal-recovery.d.ts +30 -0
- package/dist/src/wal-recovery.d.ts.map +1 -0
- package/{src/wal-recovery.ts → dist/src/wal-recovery.js} +26 -79
- package/dist/src/wal-recovery.js.map +7 -0
- package/package.json +21 -2
- package/openclaw.plugin.json +0 -815
- package/src/access-tracker.ts +0 -341
- package/src/adapters/README.md +0 -78
- package/src/adapters/qdrant.ts +0 -191
- package/src/adaptive-retrieval.ts +0 -90
- package/src/audit-log.ts +0 -238
- package/src/chunker.ts +0 -254
- package/src/config.ts +0 -271
- package/src/decay-engine.ts +0 -238
- package/src/extraction-prompts.ts +0 -339
- package/src/memory-categories.ts +0 -71
- package/src/memory-upgrader.ts +0 -388
- package/src/mnemo.ts +0 -142
- package/src/noise-filter.ts +0 -97
- package/src/noise-prototypes.ts +0 -164
- package/src/observability.ts +0 -81
- package/src/query-tracker.ts +0 -57
- package/src/reflection-event-store.ts +0 -98
- package/src/reflection-item-store.ts +0 -112
- package/src/reflection-mapped-metadata.ts +0 -84
- package/src/reflection-metadata.ts +0 -23
- package/src/reflection-store.ts +0 -602
- package/src/retriever.ts +0 -1510
- package/src/scopes.ts +0 -375
- package/src/semantic-gate.ts +0 -121
- package/src/smart-metadata.ts +0 -561
- package/src/storage-adapter.ts +0 -153
- package/src/store.ts +0 -1330
- package/src/tier-manager.ts +0 -189
- package/src/tools.ts +0 -1292
- package/test/core.test.mjs +0 -301
|
@@ -1,55 +1,22 @@
|
|
|
1
|
-
// SPDX-License-Identifier: MIT
|
|
2
|
-
/**
|
|
3
|
-
* Smart Memory Extractor — LLM-powered extraction pipeline
|
|
4
|
-
* Replaces regex-triggered capture with intelligent 6-category extraction.
|
|
5
|
-
*
|
|
6
|
-
* Pipeline: conversation → LLM extract → candidates → dedup → persist
|
|
7
|
-
*
|
|
8
|
-
*/
|
|
9
|
-
|
|
10
|
-
import type { MemoryStore, MemorySearchResult } from "./store.js";
|
|
11
|
-
import type { Embedder } from "./embedder.js";
|
|
12
|
-
import type { LlmClient } from "./llm-client.js";
|
|
13
1
|
import {
|
|
14
2
|
buildExtractionPrompt,
|
|
15
3
|
buildChineseExtractionPrompt,
|
|
16
4
|
buildDedupPrompt,
|
|
17
|
-
buildMergePrompt
|
|
5
|
+
buildMergePrompt
|
|
18
6
|
} from "./extraction-prompts.js";
|
|
19
7
|
import {
|
|
20
|
-
type CandidateMemory,
|
|
21
|
-
type DedupDecision,
|
|
22
|
-
type DedupResult,
|
|
23
|
-
type ExtractionStats,
|
|
24
|
-
type MemoryCategory,
|
|
25
8
|
ALWAYS_MERGE_CATEGORIES,
|
|
26
9
|
MERGE_SUPPORTED_CATEGORIES,
|
|
27
|
-
|
|
28
|
-
normalizeCategory,
|
|
10
|
+
normalizeCategory
|
|
29
11
|
} from "./memory-categories.js";
|
|
30
12
|
import { isNoise } from "./noise-filter.js";
|
|
31
|
-
import type { NoisePrototypeBank } from "./noise-prototypes.js";
|
|
32
13
|
import { buildSmartMetadata, parseSmartMetadata, stringifySmartMetadata, parseSupportInfo, updateSupportStats } from "./smart-metadata.js";
|
|
33
|
-
|
|
34
|
-
// ============================================================================
|
|
35
|
-
// Constants
|
|
36
|
-
// ============================================================================
|
|
37
|
-
|
|
14
|
+
import { log as _log } from "./logger.js";
|
|
38
15
|
const SIMILARITY_THRESHOLD = 0.7;
|
|
39
16
|
const MAX_SIMILAR_FOR_PROMPT = 3;
|
|
40
17
|
const MAX_MEMORIES_PER_EXTRACTION = 5;
|
|
41
|
-
const VALID_DECISIONS = new Set
|
|
42
|
-
|
|
43
|
-
// ============================================================================
|
|
44
|
-
// CJK Detection
|
|
45
|
-
// ============================================================================
|
|
46
|
-
|
|
47
|
-
/**
|
|
48
|
-
* Detect whether a text is predominantly CJK (Chinese/Japanese/Korean).
|
|
49
|
-
* Returns true if CJK characters make up > 30% of non-whitespace characters.
|
|
50
|
-
*/
|
|
51
|
-
function isCjkDominant(text: string): boolean {
|
|
52
|
-
// CJK Unified Ideographs + CJK Extension A/B + CJK Compatibility + Kana + Hangul
|
|
18
|
+
const VALID_DECISIONS = /* @__PURE__ */ new Set(["create", "merge", "skip", "support", "contextualize", "contradict"]);
|
|
19
|
+
function isCjkDominant(text) {
|
|
53
20
|
const cjkRegex = /[\u4e00-\u9fff\u3400-\u4dbf\u3040-\u309f\u30a0-\u30ff\uac00-\ud7af\uf900-\ufaff]/g;
|
|
54
21
|
const nonWhitespace = text.replace(/\s/g, "");
|
|
55
22
|
if (nonWhitespace.length === 0) return false;
|
|
@@ -57,84 +24,38 @@ function isCjkDominant(text: string): boolean {
|
|
|
57
24
|
const cjkCount = cjkMatches ? cjkMatches.length : 0;
|
|
58
25
|
return cjkCount / nonWhitespace.length > 0.3;
|
|
59
26
|
}
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
extractMinMessages?: number;
|
|
70
|
-
/** Maximum characters of conversation text to process. */
|
|
71
|
-
extractMaxChars?: number;
|
|
72
|
-
/** Default scope for new memories. */
|
|
73
|
-
defaultScope?: string;
|
|
74
|
-
/** Logger function. */
|
|
75
|
-
log?: (msg: string) => void;
|
|
76
|
-
/** Debug logger function. */
|
|
77
|
-
debugLog?: (msg: string) => void;
|
|
78
|
-
/** Optional embedding-based noise prototype bank for language-agnostic noise filtering. */
|
|
79
|
-
noiseBank?: NoisePrototypeBank;
|
|
80
|
-
}
|
|
81
|
-
|
|
82
|
-
export interface ExtractPersistOptions {
|
|
83
|
-
/** Target scope for newly created memories. */
|
|
84
|
-
scope?: string;
|
|
85
|
-
/** Scopes visible to the current agent for dedup/merge. */
|
|
86
|
-
scopeFilter?: string[];
|
|
87
|
-
}
|
|
88
|
-
|
|
89
|
-
export class SmartExtractor {
|
|
90
|
-
private log: (msg: string) => void;
|
|
91
|
-
private debugLog: (msg: string) => void;
|
|
92
|
-
|
|
93
|
-
constructor(
|
|
94
|
-
private store: MemoryStore,
|
|
95
|
-
private embedder: Embedder,
|
|
96
|
-
private llm: LlmClient,
|
|
97
|
-
private config: SmartExtractorConfig = {},
|
|
98
|
-
) {
|
|
99
|
-
this.log = config.log ?? ((msg: string) => console.log(msg));
|
|
100
|
-
this.debugLog = config.debugLog ?? (() => { });
|
|
27
|
+
class SmartExtractor {
|
|
28
|
+
constructor(store, embedder, llm, config = {}) {
|
|
29
|
+
this.store = store;
|
|
30
|
+
this.embedder = embedder;
|
|
31
|
+
this.llm = llm;
|
|
32
|
+
this.config = config;
|
|
33
|
+
this.log = config.log ?? ((msg) => _log.info(msg));
|
|
34
|
+
this.debugLog = config.debugLog ?? (() => {
|
|
35
|
+
});
|
|
101
36
|
}
|
|
102
|
-
|
|
37
|
+
log;
|
|
38
|
+
debugLog;
|
|
103
39
|
// --------------------------------------------------------------------------
|
|
104
40
|
// Main entry point
|
|
105
41
|
// --------------------------------------------------------------------------
|
|
106
|
-
|
|
107
42
|
/**
|
|
108
43
|
* Extract memories from a conversation text and persist them.
|
|
109
44
|
* Returns extraction statistics.
|
|
110
45
|
*/
|
|
111
|
-
async extractAndPersist(
|
|
112
|
-
|
|
113
|
-
sessionKey: string = "unknown",
|
|
114
|
-
options: ExtractPersistOptions = {},
|
|
115
|
-
): Promise<ExtractionStats> {
|
|
116
|
-
const stats: ExtractionStats = { created: 0, merged: 0, skipped: 0 };
|
|
46
|
+
async extractAndPersist(conversationText, sessionKey = "unknown", options = {}) {
|
|
47
|
+
const stats = { created: 0, merged: 0, skipped: 0 };
|
|
117
48
|
const targetScope = options.scope ?? this.config.defaultScope ?? "global";
|
|
118
|
-
const scopeFilter =
|
|
119
|
-
options.scopeFilter && options.scopeFilter.length > 0
|
|
120
|
-
? options.scopeFilter
|
|
121
|
-
: [targetScope];
|
|
122
|
-
|
|
123
|
-
// Step 1: LLM extraction
|
|
49
|
+
const scopeFilter = options.scopeFilter && options.scopeFilter.length > 0 ? options.scopeFilter : [targetScope];
|
|
124
50
|
const candidates = await this.extractCandidates(conversationText);
|
|
125
|
-
|
|
126
51
|
if (candidates.length === 0) {
|
|
127
52
|
this.log("memory-pro: smart-extractor: no memories extracted");
|
|
128
|
-
// LLM returned zero candidates → strongest noise signal → feedback to noise bank
|
|
129
53
|
this.learnAsNoise(conversationText);
|
|
130
54
|
return stats;
|
|
131
55
|
}
|
|
132
|
-
|
|
133
56
|
this.log(
|
|
134
|
-
`memory-pro: smart-extractor: extracted ${candidates.length} candidate(s)
|
|
57
|
+
`memory-pro: smart-extractor: extracted ${candidates.length} candidate(s)`
|
|
135
58
|
);
|
|
136
|
-
|
|
137
|
-
// Step 2: Process each candidate through dedup pipeline
|
|
138
59
|
for (const candidate of candidates.slice(0, MAX_MEMORIES_PER_EXTRACTION)) {
|
|
139
60
|
try {
|
|
140
61
|
await this.processCandidate(
|
|
@@ -142,39 +63,33 @@ export class SmartExtractor {
|
|
|
142
63
|
sessionKey,
|
|
143
64
|
stats,
|
|
144
65
|
targetScope,
|
|
145
|
-
scopeFilter
|
|
66
|
+
scopeFilter
|
|
146
67
|
);
|
|
147
68
|
} catch (err) {
|
|
148
69
|
this.log(
|
|
149
|
-
`memory-pro: smart-extractor: failed to process candidate [${candidate.category}]: ${String(err)}
|
|
70
|
+
`memory-pro: smart-extractor: failed to process candidate [${candidate.category}]: ${String(err)}`
|
|
150
71
|
);
|
|
151
72
|
}
|
|
152
73
|
}
|
|
153
|
-
|
|
154
74
|
return stats;
|
|
155
75
|
}
|
|
156
|
-
|
|
157
76
|
// --------------------------------------------------------------------------
|
|
158
77
|
// Embedding Noise Pre-Filter
|
|
159
78
|
// --------------------------------------------------------------------------
|
|
160
|
-
|
|
161
79
|
/**
|
|
162
80
|
* Filter out texts that match noise prototypes by embedding similarity.
|
|
163
81
|
* Long texts (>300 chars) are passed through without checking.
|
|
164
82
|
* Only active when noiseBank is configured and initialized.
|
|
165
83
|
*/
|
|
166
|
-
async filterNoiseByEmbedding(texts
|
|
84
|
+
async filterNoiseByEmbedding(texts) {
|
|
167
85
|
const noiseBank = this.config.noiseBank;
|
|
168
86
|
if (!noiseBank || !noiseBank.initialized) return texts;
|
|
169
|
-
|
|
170
|
-
const result: string[] = [];
|
|
87
|
+
const result = [];
|
|
171
88
|
for (const text of texts) {
|
|
172
|
-
// Very short texts lack semantic signal — skip noise check to avoid false positives
|
|
173
89
|
if (text.length <= 8) {
|
|
174
90
|
result.push(text);
|
|
175
91
|
continue;
|
|
176
92
|
}
|
|
177
|
-
// Long texts are unlikely to be pure noise queries
|
|
178
93
|
if (text.length > 300) {
|
|
179
94
|
result.push(text);
|
|
180
95
|
continue;
|
|
@@ -185,25 +100,22 @@ export class SmartExtractor {
|
|
|
185
100
|
result.push(text);
|
|
186
101
|
} else {
|
|
187
102
|
this.debugLog(
|
|
188
|
-
`mnemo: smart-extractor: embedding noise filtered: ${text.slice(0, 80)}
|
|
103
|
+
`mnemo: smart-extractor: embedding noise filtered: ${text.slice(0, 80)}`
|
|
189
104
|
);
|
|
190
105
|
}
|
|
191
106
|
} catch {
|
|
192
|
-
// Embedding failed — pass text through
|
|
193
107
|
result.push(text);
|
|
194
108
|
}
|
|
195
109
|
}
|
|
196
110
|
return result;
|
|
197
111
|
}
|
|
198
|
-
|
|
199
112
|
/**
|
|
200
113
|
* Feed back conversation text to the noise prototype bank.
|
|
201
114
|
* Called when LLM extraction returns zero candidates (strongest noise signal).
|
|
202
115
|
*/
|
|
203
|
-
|
|
116
|
+
async learnAsNoise(conversationText) {
|
|
204
117
|
const noiseBank = this.config.noiseBank;
|
|
205
118
|
if (!noiseBank || !noiseBank.initialized) return;
|
|
206
|
-
|
|
207
119
|
try {
|
|
208
120
|
const tail = conversationText.slice(-300);
|
|
209
121
|
const vec = await this.embedder.embed(tail);
|
|
@@ -212,59 +124,36 @@ export class SmartExtractor {
|
|
|
212
124
|
this.debugLog("mnemo: smart-extractor: learned noise from zero-extraction");
|
|
213
125
|
}
|
|
214
126
|
} catch {
|
|
215
|
-
// Non-critical — silently skip
|
|
216
127
|
}
|
|
217
128
|
}
|
|
218
|
-
|
|
219
129
|
// --------------------------------------------------------------------------
|
|
220
130
|
// Step 1: LLM Extraction
|
|
221
131
|
// --------------------------------------------------------------------------
|
|
222
|
-
|
|
223
132
|
/**
|
|
224
133
|
* Call LLM to extract candidate memories from conversation text.
|
|
225
134
|
*/
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
const maxChars = this.config.extractMaxChars ?? 8000;
|
|
230
|
-
const truncated =
|
|
231
|
-
conversationText.length > maxChars
|
|
232
|
-
? conversationText.slice(-maxChars)
|
|
233
|
-
: conversationText;
|
|
234
|
-
|
|
135
|
+
async extractCandidates(conversationText) {
|
|
136
|
+
const maxChars = this.config.extractMaxChars ?? 8e3;
|
|
137
|
+
const truncated = conversationText.length > maxChars ? conversationText.slice(-maxChars) : conversationText;
|
|
235
138
|
const user = this.config.user ?? "User";
|
|
236
|
-
const prompt = isCjkDominant(truncated)
|
|
237
|
-
|
|
238
|
-
: buildExtractionPrompt(truncated, user);
|
|
239
|
-
|
|
240
|
-
const result = await this.llm.completeJson<{
|
|
241
|
-
memories: Array<{
|
|
242
|
-
category: string;
|
|
243
|
-
abstract: string;
|
|
244
|
-
overview: string;
|
|
245
|
-
content: string;
|
|
246
|
-
}>;
|
|
247
|
-
}>(prompt, "extract-candidates");
|
|
248
|
-
|
|
139
|
+
const prompt = isCjkDominant(truncated) ? buildChineseExtractionPrompt(truncated, user) : buildExtractionPrompt(truncated, user);
|
|
140
|
+
const result = await this.llm.completeJson(prompt, "extract-candidates");
|
|
249
141
|
if (!result) {
|
|
250
142
|
this.debugLog(
|
|
251
|
-
"mnemo: smart-extractor: extract-candidates returned null"
|
|
143
|
+
"mnemo: smart-extractor: extract-candidates returned null"
|
|
252
144
|
);
|
|
253
145
|
return [];
|
|
254
146
|
}
|
|
255
147
|
if (!result.memories || !Array.isArray(result.memories)) {
|
|
256
148
|
this.debugLog(
|
|
257
|
-
`mnemo: smart-extractor: extract-candidates returned unexpected shape keys=${Object.keys(result).join(",") || "(none)"}
|
|
149
|
+
`mnemo: smart-extractor: extract-candidates returned unexpected shape keys=${Object.keys(result).join(",") || "(none)"}`
|
|
258
150
|
);
|
|
259
151
|
return [];
|
|
260
152
|
}
|
|
261
|
-
|
|
262
153
|
this.debugLog(
|
|
263
|
-
`mnemo: smart-extractor: extract-candidates raw memories=${result.memories.length}
|
|
154
|
+
`mnemo: smart-extractor: extract-candidates raw memories=${result.memories.length}`
|
|
264
155
|
);
|
|
265
|
-
|
|
266
|
-
// Validate and normalize candidates
|
|
267
|
-
const candidates: CandidateMemory[] = [];
|
|
156
|
+
const candidates = [];
|
|
268
157
|
let invalidCategoryCount = 0;
|
|
269
158
|
let shortAbstractCount = 0;
|
|
270
159
|
let noiseAbstractCount = 0;
|
|
@@ -273,68 +162,51 @@ export class SmartExtractor {
|
|
|
273
162
|
if (!category) {
|
|
274
163
|
invalidCategoryCount++;
|
|
275
164
|
this.debugLog(
|
|
276
|
-
`mnemo: smart-extractor: dropping candidate due to invalid category rawCategory=${JSON.stringify(raw.category ?? "")} abstract=${JSON.stringify((raw.abstract ?? "").trim().slice(0, 120))}
|
|
165
|
+
`mnemo: smart-extractor: dropping candidate due to invalid category rawCategory=${JSON.stringify(raw.category ?? "")} abstract=${JSON.stringify((raw.abstract ?? "").trim().slice(0, 120))}`
|
|
277
166
|
);
|
|
278
167
|
continue;
|
|
279
168
|
}
|
|
280
|
-
|
|
281
169
|
const abstract = (raw.abstract ?? "").trim();
|
|
282
170
|
const overview = (raw.overview ?? "").trim();
|
|
283
171
|
const content = (raw.content ?? "").trim();
|
|
284
|
-
|
|
285
|
-
// Skip empty or noise
|
|
286
172
|
if (!abstract || abstract.length < 5) {
|
|
287
173
|
shortAbstractCount++;
|
|
288
174
|
this.debugLog(
|
|
289
|
-
`mnemo: smart-extractor: dropping candidate due to short abstract category=${category} abstract=${JSON.stringify(abstract)}
|
|
175
|
+
`mnemo: smart-extractor: dropping candidate due to short abstract category=${category} abstract=${JSON.stringify(abstract)}`
|
|
290
176
|
);
|
|
291
177
|
continue;
|
|
292
178
|
}
|
|
293
179
|
if (isNoise(abstract)) {
|
|
294
180
|
noiseAbstractCount++;
|
|
295
181
|
this.debugLog(
|
|
296
|
-
`mnemo: smart-extractor: dropping candidate due to noise abstract category=${category} abstract=${JSON.stringify(abstract.slice(0, 120))}
|
|
182
|
+
`mnemo: smart-extractor: dropping candidate due to noise abstract category=${category} abstract=${JSON.stringify(abstract.slice(0, 120))}`
|
|
297
183
|
);
|
|
298
184
|
continue;
|
|
299
185
|
}
|
|
300
|
-
|
|
301
186
|
candidates.push({ category, abstract, overview, content });
|
|
302
187
|
}
|
|
303
|
-
|
|
304
188
|
this.debugLog(
|
|
305
|
-
`mnemo: smart-extractor: validation summary accepted=${candidates.length}, invalidCategory=${invalidCategoryCount}, shortAbstract=${shortAbstractCount}, noiseAbstract=${noiseAbstractCount}
|
|
189
|
+
`mnemo: smart-extractor: validation summary accepted=${candidates.length}, invalidCategory=${invalidCategoryCount}, shortAbstract=${shortAbstractCount}, noiseAbstract=${noiseAbstractCount}`
|
|
306
190
|
);
|
|
307
|
-
|
|
308
191
|
return candidates;
|
|
309
192
|
}
|
|
310
|
-
|
|
311
193
|
// --------------------------------------------------------------------------
|
|
312
194
|
// Step 2: Dedup + Persist
|
|
313
195
|
// --------------------------------------------------------------------------
|
|
314
|
-
|
|
315
196
|
/**
|
|
316
197
|
* Process a single candidate memory: dedup → merge/create → store
|
|
317
198
|
*/
|
|
318
|
-
|
|
319
|
-
candidate: CandidateMemory,
|
|
320
|
-
sessionKey: string,
|
|
321
|
-
stats: ExtractionStats,
|
|
322
|
-
targetScope: string,
|
|
323
|
-
scopeFilter: string[],
|
|
324
|
-
): Promise<void> {
|
|
325
|
-
// Profile always merges (skip dedup)
|
|
199
|
+
async processCandidate(candidate, sessionKey, stats, targetScope, scopeFilter) {
|
|
326
200
|
if (ALWAYS_MERGE_CATEGORIES.has(candidate.category)) {
|
|
327
201
|
await this.handleProfileMerge(
|
|
328
202
|
candidate,
|
|
329
203
|
sessionKey,
|
|
330
204
|
targetScope,
|
|
331
|
-
scopeFilter
|
|
205
|
+
scopeFilter
|
|
332
206
|
);
|
|
333
207
|
stats.merged++;
|
|
334
208
|
return;
|
|
335
209
|
}
|
|
336
|
-
|
|
337
|
-
// Embed the candidate for vector dedup
|
|
338
210
|
const embeddingText = `${candidate.abstract} ${candidate.content}`;
|
|
339
211
|
const vector = await this.embedder.embed(embeddingText);
|
|
340
212
|
if (!vector || vector.length === 0) {
|
|
@@ -343,43 +215,33 @@ export class SmartExtractor {
|
|
|
343
215
|
stats.created++;
|
|
344
216
|
return;
|
|
345
217
|
}
|
|
346
|
-
|
|
347
|
-
// Dedup pipeline
|
|
348
218
|
const dedupResult = await this.deduplicate(candidate, vector, scopeFilter);
|
|
349
|
-
|
|
350
219
|
switch (dedupResult.decision) {
|
|
351
220
|
case "create":
|
|
352
221
|
await this.storeCandidate(candidate, vector, sessionKey, targetScope);
|
|
353
222
|
stats.created++;
|
|
354
223
|
break;
|
|
355
|
-
|
|
356
224
|
case "merge":
|
|
357
|
-
if (
|
|
358
|
-
dedupResult.matchId &&
|
|
359
|
-
MERGE_SUPPORTED_CATEGORIES.has(candidate.category)
|
|
360
|
-
) {
|
|
225
|
+
if (dedupResult.matchId && MERGE_SUPPORTED_CATEGORIES.has(candidate.category)) {
|
|
361
226
|
await this.handleMerge(
|
|
362
227
|
candidate,
|
|
363
228
|
dedupResult.matchId,
|
|
364
229
|
scopeFilter,
|
|
365
230
|
targetScope,
|
|
366
|
-
dedupResult.contextLabel
|
|
231
|
+
dedupResult.contextLabel
|
|
367
232
|
);
|
|
368
233
|
stats.merged++;
|
|
369
234
|
} else {
|
|
370
|
-
// Category doesn't support merge → create instead
|
|
371
235
|
await this.storeCandidate(candidate, vector, sessionKey, targetScope);
|
|
372
236
|
stats.created++;
|
|
373
237
|
}
|
|
374
238
|
break;
|
|
375
|
-
|
|
376
239
|
case "skip":
|
|
377
240
|
this.log(
|
|
378
|
-
`memory-pro: smart-extractor: skipped [${candidate.category}] ${candidate.abstract.slice(0, 60)}
|
|
241
|
+
`memory-pro: smart-extractor: skipped [${candidate.category}] ${candidate.abstract.slice(0, 60)}`
|
|
379
242
|
);
|
|
380
243
|
stats.skipped++;
|
|
381
244
|
break;
|
|
382
|
-
|
|
383
245
|
case "support":
|
|
384
246
|
if (dedupResult.matchId) {
|
|
385
247
|
await this.handleSupport(dedupResult.matchId, scopeFilter, { session: sessionKey, timestamp: Date.now() }, dedupResult.reason, dedupResult.contextLabel);
|
|
@@ -389,7 +251,6 @@ export class SmartExtractor {
|
|
|
389
251
|
stats.created++;
|
|
390
252
|
}
|
|
391
253
|
break;
|
|
392
|
-
|
|
393
254
|
case "contextualize":
|
|
394
255
|
if (dedupResult.matchId) {
|
|
395
256
|
await this.handleContextualize(candidate, vector, dedupResult.matchId, sessionKey, targetScope, scopeFilter, dedupResult.contextLabel);
|
|
@@ -399,7 +260,6 @@ export class SmartExtractor {
|
|
|
399
260
|
stats.created++;
|
|
400
261
|
}
|
|
401
262
|
break;
|
|
402
|
-
|
|
403
263
|
case "contradict":
|
|
404
264
|
if (dedupResult.matchId) {
|
|
405
265
|
await this.handleContradict(candidate, vector, dedupResult.matchId, sessionKey, targetScope, scopeFilter, dedupResult.contextLabel);
|
|
@@ -411,127 +271,88 @@ export class SmartExtractor {
|
|
|
411
271
|
break;
|
|
412
272
|
}
|
|
413
273
|
}
|
|
414
|
-
|
|
415
274
|
// --------------------------------------------------------------------------
|
|
416
275
|
// Dedup Pipeline (vector pre-filter + LLM decision)
|
|
417
276
|
// --------------------------------------------------------------------------
|
|
418
|
-
|
|
419
277
|
/**
|
|
420
278
|
* Two-stage dedup: vector similarity search → LLM decision.
|
|
421
279
|
*/
|
|
422
|
-
|
|
423
|
-
candidate: CandidateMemory,
|
|
424
|
-
candidateVector: number[],
|
|
425
|
-
scopeFilter: string[],
|
|
426
|
-
): Promise<DedupResult> {
|
|
427
|
-
// Stage 1: Vector pre-filter — find similar memories
|
|
280
|
+
async deduplicate(candidate, candidateVector, scopeFilter) {
|
|
428
281
|
const similar = await this.store.vectorSearch(
|
|
429
282
|
candidateVector,
|
|
430
283
|
5,
|
|
431
284
|
SIMILARITY_THRESHOLD,
|
|
432
|
-
scopeFilter
|
|
285
|
+
scopeFilter
|
|
433
286
|
);
|
|
434
|
-
|
|
435
287
|
if (similar.length === 0) {
|
|
436
288
|
return { decision: "create", reason: "No similar memories found" };
|
|
437
289
|
}
|
|
438
|
-
|
|
439
|
-
// Stage 2: LLM decision
|
|
440
290
|
return this.llmDedupDecision(candidate, similar);
|
|
441
291
|
}
|
|
442
|
-
|
|
443
|
-
private async llmDedupDecision(
|
|
444
|
-
candidate: CandidateMemory,
|
|
445
|
-
similar: MemorySearchResult[],
|
|
446
|
-
): Promise<DedupResult> {
|
|
292
|
+
async llmDedupDecision(candidate, similar) {
|
|
447
293
|
const topSimilar = similar.slice(0, MAX_SIMILAR_FOR_PROMPT);
|
|
448
|
-
const existingFormatted = topSimilar
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
294
|
+
const existingFormatted = topSimilar.map((r, i) => {
|
|
295
|
+
let metaObj = {};
|
|
296
|
+
try {
|
|
297
|
+
metaObj = JSON.parse(r.entry.metadata || "{}");
|
|
298
|
+
} catch {
|
|
299
|
+
}
|
|
300
|
+
const abstract = metaObj.l0_abstract || r.entry.text;
|
|
301
|
+
const overview = metaObj.l1_overview || "";
|
|
302
|
+
return `${i + 1}. [${metaObj.memory_category || r.entry.category}] ${abstract}
|
|
303
|
+
Overview: ${overview}
|
|
304
|
+
Score: ${r.score.toFixed(3)}`;
|
|
305
|
+
}).join("\n");
|
|
461
306
|
const prompt = buildDedupPrompt(
|
|
462
307
|
candidate.abstract,
|
|
463
308
|
candidate.overview,
|
|
464
309
|
candidate.content,
|
|
465
|
-
existingFormatted
|
|
310
|
+
existingFormatted
|
|
466
311
|
);
|
|
467
|
-
|
|
468
312
|
try {
|
|
469
|
-
const data = await this.llm.completeJson
|
|
470
|
-
decision: string;
|
|
471
|
-
reason: string;
|
|
472
|
-
match_index?: number;
|
|
473
|
-
}>(prompt, "dedup-decision");
|
|
474
|
-
|
|
313
|
+
const data = await this.llm.completeJson(prompt, "dedup-decision");
|
|
475
314
|
if (!data) {
|
|
476
315
|
this.log(
|
|
477
|
-
"memory-pro: smart-extractor: dedup LLM returned unparseable response, defaulting to CREATE"
|
|
316
|
+
"memory-pro: smart-extractor: dedup LLM returned unparseable response, defaulting to CREATE"
|
|
478
317
|
);
|
|
479
318
|
return { decision: "create", reason: "LLM response unparseable" };
|
|
480
319
|
}
|
|
481
|
-
|
|
482
|
-
const decision = (data.decision?.toLowerCase() ??
|
|
483
|
-
"create") as DedupDecision;
|
|
320
|
+
const decision = data.decision?.toLowerCase() ?? "create";
|
|
484
321
|
if (!VALID_DECISIONS.has(decision)) {
|
|
485
322
|
return {
|
|
486
323
|
decision: "create",
|
|
487
|
-
reason: `Unknown decision: ${data.decision}
|
|
324
|
+
reason: `Unknown decision: ${data.decision}`
|
|
488
325
|
};
|
|
489
326
|
}
|
|
490
|
-
|
|
491
|
-
// Resolve merge target from LLM's match_index (1-based)
|
|
492
327
|
const idx = data.match_index;
|
|
493
|
-
const matchEntry =
|
|
494
|
-
typeof idx === "number" && idx >= 1 && idx <= topSimilar.length
|
|
495
|
-
? topSimilar[idx - 1]
|
|
496
|
-
: topSimilar[0];
|
|
497
|
-
|
|
328
|
+
const matchEntry = typeof idx === "number" && idx >= 1 && idx <= topSimilar.length ? topSimilar[idx - 1] : topSimilar[0];
|
|
498
329
|
return {
|
|
499
330
|
decision,
|
|
500
331
|
reason: data.reason ?? "",
|
|
501
|
-
matchId: ["merge", "support", "contextualize", "contradict"].includes(decision) ? matchEntry?.entry.id :
|
|
502
|
-
contextLabel: typeof
|
|
332
|
+
matchId: ["merge", "support", "contextualize", "contradict"].includes(decision) ? matchEntry?.entry.id : void 0,
|
|
333
|
+
contextLabel: typeof data.context_label === "string" ? data.context_label : void 0
|
|
503
334
|
};
|
|
504
335
|
} catch (err) {
|
|
505
336
|
this.log(
|
|
506
|
-
`memory-pro: smart-extractor: dedup LLM failed: ${String(err)}
|
|
337
|
+
`memory-pro: smart-extractor: dedup LLM failed: ${String(err)}`
|
|
507
338
|
);
|
|
508
339
|
return { decision: "create", reason: `LLM failed: ${String(err)}` };
|
|
509
340
|
}
|
|
510
341
|
}
|
|
511
|
-
|
|
512
342
|
// --------------------------------------------------------------------------
|
|
513
343
|
// Merge Logic
|
|
514
344
|
// --------------------------------------------------------------------------
|
|
515
|
-
|
|
516
345
|
/**
|
|
517
346
|
* Profile always-merge: read existing profile, merge with LLM, upsert.
|
|
518
347
|
*/
|
|
519
|
-
|
|
520
|
-
candidate: CandidateMemory,
|
|
521
|
-
sessionKey: string,
|
|
522
|
-
targetScope: string,
|
|
523
|
-
scopeFilter: string[],
|
|
524
|
-
): Promise<void> {
|
|
525
|
-
// Find existing profile memory by category
|
|
348
|
+
async handleProfileMerge(candidate, sessionKey, targetScope, scopeFilter) {
|
|
526
349
|
const embeddingText = `${candidate.abstract} ${candidate.content}`;
|
|
527
350
|
const vector = await this.embedder.embed(embeddingText);
|
|
528
|
-
|
|
529
|
-
// Search for existing profile memories
|
|
530
351
|
const existing = await this.store.vectorSearch(
|
|
531
352
|
vector || [],
|
|
532
353
|
1,
|
|
533
354
|
0.3,
|
|
534
|
-
scopeFilter
|
|
355
|
+
scopeFilter
|
|
535
356
|
);
|
|
536
357
|
const profileMatch = existing.find((r) => {
|
|
537
358
|
try {
|
|
@@ -541,60 +362,47 @@ export class SmartExtractor {
|
|
|
541
362
|
return false;
|
|
542
363
|
}
|
|
543
364
|
});
|
|
544
|
-
|
|
545
365
|
if (profileMatch) {
|
|
546
366
|
await this.handleMerge(
|
|
547
367
|
candidate,
|
|
548
368
|
profileMatch.entry.id,
|
|
549
369
|
scopeFilter,
|
|
550
|
-
targetScope
|
|
370
|
+
targetScope
|
|
551
371
|
);
|
|
552
372
|
} else {
|
|
553
|
-
// No existing profile — create new
|
|
554
373
|
await this.storeCandidate(candidate, vector || [], sessionKey, targetScope);
|
|
555
374
|
}
|
|
556
375
|
}
|
|
557
|
-
|
|
558
376
|
/**
|
|
559
377
|
* Merge a candidate into an existing memory using LLM.
|
|
560
378
|
*/
|
|
561
|
-
|
|
562
|
-
candidate: CandidateMemory,
|
|
563
|
-
matchId: string,
|
|
564
|
-
scopeFilter: string[],
|
|
565
|
-
targetScope: string,
|
|
566
|
-
contextLabel?: string,
|
|
567
|
-
): Promise<void> {
|
|
379
|
+
async handleMerge(candidate, matchId, scopeFilter, targetScope, contextLabel) {
|
|
568
380
|
let existingAbstract = "";
|
|
569
381
|
let existingOverview = "";
|
|
570
382
|
let existingContent = "";
|
|
571
|
-
|
|
572
383
|
try {
|
|
573
|
-
const
|
|
574
|
-
if (
|
|
575
|
-
const meta = parseSmartMetadata(
|
|
576
|
-
existingAbstract = meta.l0_abstract ||
|
|
384
|
+
const existing2 = await this.store.getById(matchId, scopeFilter);
|
|
385
|
+
if (existing2) {
|
|
386
|
+
const meta = parseSmartMetadata(existing2.metadata, existing2);
|
|
387
|
+
existingAbstract = meta.l0_abstract || existing2.text;
|
|
577
388
|
existingOverview = meta.l1_overview || "";
|
|
578
|
-
existingContent = meta.l2_content ||
|
|
389
|
+
existingContent = meta.l2_content || existing2.text;
|
|
579
390
|
}
|
|
580
391
|
} catch {
|
|
581
|
-
// Fallback: store as new
|
|
582
392
|
this.log(
|
|
583
|
-
`memory-pro: smart-extractor: could not read existing memory ${matchId}, storing as new
|
|
393
|
+
`memory-pro: smart-extractor: could not read existing memory ${matchId}, storing as new`
|
|
584
394
|
);
|
|
585
395
|
const vector = await this.embedder.embed(
|
|
586
|
-
`${candidate.abstract} ${candidate.content}
|
|
396
|
+
`${candidate.abstract} ${candidate.content}`
|
|
587
397
|
);
|
|
588
398
|
await this.storeCandidate(
|
|
589
399
|
candidate,
|
|
590
400
|
vector || [],
|
|
591
401
|
"merge-fallback",
|
|
592
|
-
targetScope
|
|
402
|
+
targetScope
|
|
593
403
|
);
|
|
594
404
|
return;
|
|
595
405
|
}
|
|
596
|
-
|
|
597
|
-
// Call LLM to merge
|
|
598
406
|
const prompt = buildMergePrompt(
|
|
599
407
|
existingAbstract,
|
|
600
408
|
existingOverview,
|
|
@@ -602,25 +410,15 @@ export class SmartExtractor {
|
|
|
602
410
|
candidate.abstract,
|
|
603
411
|
candidate.overview,
|
|
604
412
|
candidate.content,
|
|
605
|
-
candidate.category
|
|
413
|
+
candidate.category
|
|
606
414
|
);
|
|
607
|
-
|
|
608
|
-
const merged = await this.llm.completeJson<{
|
|
609
|
-
abstract: string;
|
|
610
|
-
overview: string;
|
|
611
|
-
content: string;
|
|
612
|
-
}>(prompt, "merge-memory");
|
|
613
|
-
|
|
415
|
+
const merged = await this.llm.completeJson(prompt, "merge-memory");
|
|
614
416
|
if (!merged) {
|
|
615
417
|
this.log("memory-pro: smart-extractor: merge LLM failed, skipping merge");
|
|
616
418
|
return;
|
|
617
419
|
}
|
|
618
|
-
|
|
619
|
-
// Re-embed the merged content
|
|
620
420
|
const mergedText = `${merged.abstract} ${merged.content}`;
|
|
621
421
|
const newVector = await this.embedder.embed(mergedText);
|
|
622
|
-
|
|
623
|
-
// Update existing memory via store.update()
|
|
624
422
|
const existing = await this.store.getById(matchId, scopeFilter);
|
|
625
423
|
const metadata = stringifySmartMetadata(
|
|
626
424
|
buildSmartMetadata(existing ?? { text: merged.abstract }, {
|
|
@@ -629,21 +427,18 @@ export class SmartExtractor {
|
|
|
629
427
|
l2_content: merged.content,
|
|
630
428
|
memory_category: candidate.category,
|
|
631
429
|
tier: "working",
|
|
632
|
-
confidence: 0.8
|
|
633
|
-
})
|
|
430
|
+
confidence: 0.8
|
|
431
|
+
})
|
|
634
432
|
);
|
|
635
|
-
|
|
636
433
|
await this.store.update(
|
|
637
434
|
matchId,
|
|
638
435
|
{
|
|
639
436
|
text: merged.abstract,
|
|
640
437
|
vector: newVector,
|
|
641
|
-
metadata
|
|
438
|
+
metadata
|
|
642
439
|
},
|
|
643
|
-
scopeFilter
|
|
440
|
+
scopeFilter
|
|
644
441
|
);
|
|
645
|
-
|
|
646
|
-
// Update support stats on the merged memory
|
|
647
442
|
try {
|
|
648
443
|
const updatedEntry = await this.store.getById(matchId, scopeFilter);
|
|
649
444
|
if (updatedEntry) {
|
|
@@ -654,106 +449,71 @@ export class SmartExtractor {
|
|
|
654
449
|
await this.store.update(matchId, { metadata: finalMetadata }, scopeFilter);
|
|
655
450
|
}
|
|
656
451
|
} catch {
|
|
657
|
-
// Non-critical: merge succeeded, support stats update is best-effort
|
|
658
452
|
}
|
|
659
|
-
|
|
660
453
|
this.log(
|
|
661
|
-
`memory-pro: smart-extractor: merged [${candidate.category}]${contextLabel ? ` [${contextLabel}]` : ""} into ${matchId.slice(0, 8)}
|
|
454
|
+
`memory-pro: smart-extractor: merged [${candidate.category}]${contextLabel ? ` [${contextLabel}]` : ""} into ${matchId.slice(0, 8)}`
|
|
662
455
|
);
|
|
663
456
|
}
|
|
664
|
-
|
|
665
457
|
// --------------------------------------------------------------------------
|
|
666
458
|
// Context-Aware Handlers (support / contextualize / contradict)
|
|
667
459
|
// --------------------------------------------------------------------------
|
|
668
|
-
|
|
669
460
|
/**
|
|
670
461
|
* Handle SUPPORT: update support stats on existing memory for a specific context.
|
|
671
462
|
*/
|
|
672
|
-
|
|
673
|
-
matchId: string,
|
|
674
|
-
scopeFilter: string[],
|
|
675
|
-
source: { session: string; timestamp: number },
|
|
676
|
-
reason: string,
|
|
677
|
-
contextLabel?: string,
|
|
678
|
-
): Promise<void> {
|
|
463
|
+
async handleSupport(matchId, scopeFilter, source, reason, contextLabel) {
|
|
679
464
|
const existing = await this.store.getById(matchId, scopeFilter);
|
|
680
465
|
if (!existing) return;
|
|
681
|
-
|
|
682
466
|
const meta = parseSmartMetadata(existing.metadata, existing);
|
|
683
467
|
const supportInfo = parseSupportInfo(meta.support_info);
|
|
684
468
|
const updated = updateSupportStats(supportInfo, contextLabel, "support");
|
|
685
469
|
meta.support_info = updated;
|
|
686
|
-
|
|
687
470
|
await this.store.update(
|
|
688
471
|
matchId,
|
|
689
472
|
{ metadata: stringifySmartMetadata(meta) },
|
|
690
|
-
scopeFilter
|
|
473
|
+
scopeFilter
|
|
691
474
|
);
|
|
692
|
-
|
|
693
475
|
this.log(
|
|
694
|
-
`memory-pro: smart-extractor: support [${contextLabel || "general"}] on ${matchId.slice(0, 8)}
|
|
476
|
+
`memory-pro: smart-extractor: support [${contextLabel || "general"}] on ${matchId.slice(0, 8)} \u2014 ${reason}`
|
|
695
477
|
);
|
|
696
478
|
}
|
|
697
|
-
|
|
698
479
|
/**
|
|
699
480
|
* Handle CONTEXTUALIZE: create a new entry that adds situational nuance,
|
|
700
481
|
* linked to the original via a relation in metadata.
|
|
701
482
|
*/
|
|
702
|
-
|
|
703
|
-
candidate: CandidateMemory,
|
|
704
|
-
vector: number[],
|
|
705
|
-
matchId: string,
|
|
706
|
-
sessionKey: string,
|
|
707
|
-
targetScope: string,
|
|
708
|
-
scopeFilter: string[],
|
|
709
|
-
contextLabel?: string,
|
|
710
|
-
): Promise<void> {
|
|
483
|
+
async handleContextualize(candidate, vector, matchId, sessionKey, targetScope, scopeFilter, contextLabel) {
|
|
711
484
|
const storeCategory = this.mapToStoreCategory(candidate.category);
|
|
712
485
|
const metadata = stringifySmartMetadata({
|
|
713
486
|
l0_abstract: candidate.abstract,
|
|
714
487
|
l1_overview: candidate.overview,
|
|
715
488
|
l2_content: candidate.content,
|
|
716
489
|
memory_category: candidate.category,
|
|
717
|
-
tier: "working"
|
|
490
|
+
tier: "working",
|
|
718
491
|
access_count: 0,
|
|
719
492
|
confidence: 0.7,
|
|
720
493
|
last_accessed_at: Date.now(),
|
|
721
494
|
source_session: sessionKey,
|
|
722
495
|
contexts: contextLabel ? [contextLabel] : [],
|
|
723
|
-
relations: [{ type: "contextualizes", targetId: matchId }]
|
|
496
|
+
relations: [{ type: "contextualizes", targetId: matchId }]
|
|
724
497
|
});
|
|
725
|
-
|
|
726
498
|
await this.store.store({
|
|
727
499
|
text: candidate.abstract,
|
|
728
500
|
vector,
|
|
729
501
|
category: storeCategory,
|
|
730
502
|
scope: targetScope,
|
|
731
503
|
importance: this.getDefaultImportance(candidate.category),
|
|
732
|
-
metadata
|
|
504
|
+
metadata
|
|
733
505
|
});
|
|
734
|
-
|
|
735
506
|
this.log(
|
|
736
|
-
`memory-pro: smart-extractor: contextualize [${contextLabel || "general"}] new entry linked to ${matchId.slice(0, 8)}
|
|
507
|
+
`memory-pro: smart-extractor: contextualize [${contextLabel || "general"}] new entry linked to ${matchId.slice(0, 8)}`
|
|
737
508
|
);
|
|
738
509
|
}
|
|
739
|
-
|
|
740
510
|
/**
|
|
741
511
|
* Handle CONTRADICT: create contradicting entry + record contradiction evidence
|
|
742
512
|
* on the original memory's support stats.
|
|
743
513
|
*/
|
|
744
|
-
|
|
745
|
-
candidate: CandidateMemory,
|
|
746
|
-
vector: number[],
|
|
747
|
-
matchId: string,
|
|
748
|
-
sessionKey: string,
|
|
749
|
-
targetScope: string,
|
|
750
|
-
scopeFilter: string[],
|
|
751
|
-
contextLabel?: string,
|
|
752
|
-
): Promise<void> {
|
|
514
|
+
async handleContradict(candidate, vector, matchId, sessionKey, targetScope, scopeFilter, contextLabel) {
|
|
753
515
|
const now = Date.now();
|
|
754
516
|
const nowIso = new Date(now).toISOString();
|
|
755
|
-
|
|
756
|
-
// 1. Demote + expire the contradicted memory
|
|
757
517
|
const existing = await this.store.getById(matchId, scopeFilter);
|
|
758
518
|
if (existing) {
|
|
759
519
|
const meta = parseSmartMetadata(existing.metadata, existing);
|
|
@@ -767,12 +527,10 @@ export class SmartExtractor {
|
|
|
767
527
|
matchId,
|
|
768
528
|
{
|
|
769
529
|
importance: Math.max(0.05, (existing.importance ?? 0.7) * 0.2),
|
|
770
|
-
metadata: stringifySmartMetadata(meta)
|
|
530
|
+
metadata: stringifySmartMetadata(meta)
|
|
771
531
|
},
|
|
772
|
-
scopeFilter
|
|
532
|
+
scopeFilter
|
|
773
533
|
);
|
|
774
|
-
|
|
775
|
-
// 2. Expire in Graphiti (fire-and-forget)
|
|
776
534
|
if (process.env.GRAPHITI_ENABLED === "true") {
|
|
777
535
|
const graphitiBase = process.env.GRAPHITI_BASE_URL || "http://127.0.0.1:18799";
|
|
778
536
|
fetch(`${graphitiBase}/facts/expire`, {
|
|
@@ -781,25 +539,23 @@ export class SmartExtractor {
|
|
|
781
539
|
body: JSON.stringify({
|
|
782
540
|
text: existing.text,
|
|
783
541
|
expired_at: nowIso,
|
|
784
|
-
reason: `contradicted: ${candidate.abstract.slice(0, 80)}
|
|
542
|
+
reason: `contradicted: ${candidate.abstract.slice(0, 80)}`
|
|
785
543
|
}),
|
|
786
|
-
signal: AbortSignal.timeout(
|
|
787
|
-
}).catch(() => {
|
|
544
|
+
signal: AbortSignal.timeout(5e3)
|
|
545
|
+
}).catch(() => {
|
|
546
|
+
});
|
|
788
547
|
}
|
|
789
|
-
|
|
790
548
|
this.log(
|
|
791
|
-
`memory-pro: smart-extractor: expired old memory ${matchId.slice(0, 8)} (imp ${(existing.importance ?? 0.7).toFixed(2)}
|
|
549
|
+
`memory-pro: smart-extractor: expired old memory ${matchId.slice(0, 8)} (imp ${(existing.importance ?? 0.7).toFixed(2)}\u2192${Math.max(0.05, (existing.importance ?? 0.7) * 0.2).toFixed(2)})`
|
|
792
550
|
);
|
|
793
551
|
}
|
|
794
|
-
|
|
795
|
-
// 3. Store the new (contradicting) entry with supersedes relation
|
|
796
552
|
const storeCategory = this.mapToStoreCategory(candidate.category);
|
|
797
553
|
const metadata = stringifySmartMetadata({
|
|
798
554
|
l0_abstract: candidate.abstract,
|
|
799
555
|
l1_overview: candidate.overview,
|
|
800
556
|
l2_content: candidate.content,
|
|
801
557
|
memory_category: candidate.category,
|
|
802
|
-
tier: "working"
|
|
558
|
+
tier: "working",
|
|
803
559
|
access_count: 0,
|
|
804
560
|
confidence: 0.85,
|
|
805
561
|
last_accessed_at: now,
|
|
@@ -807,46 +563,35 @@ export class SmartExtractor {
|
|
|
807
563
|
contexts: contextLabel ? [contextLabel] : [],
|
|
808
564
|
relations: [
|
|
809
565
|
{ type: "contradicts", targetId: matchId },
|
|
810
|
-
{ type: "supersedes", targetId: matchId }
|
|
566
|
+
{ type: "supersedes", targetId: matchId }
|
|
811
567
|
],
|
|
812
|
-
valid_from: nowIso
|
|
568
|
+
valid_from: nowIso
|
|
813
569
|
});
|
|
814
|
-
|
|
815
570
|
await this.store.store({
|
|
816
571
|
text: candidate.abstract,
|
|
817
572
|
vector,
|
|
818
573
|
category: storeCategory,
|
|
819
574
|
scope: targetScope,
|
|
820
|
-
importance: Math.min(1
|
|
821
|
-
metadata
|
|
575
|
+
importance: Math.min(1, this.getDefaultImportance(candidate.category) + 0.1),
|
|
576
|
+
metadata
|
|
822
577
|
});
|
|
823
|
-
|
|
824
578
|
this.log(
|
|
825
|
-
`memory-pro: smart-extractor: contradict [${contextLabel || "general"}] superseded ${matchId.slice(0, 8)}
|
|
579
|
+
`memory-pro: smart-extractor: contradict [${contextLabel || "general"}] superseded ${matchId.slice(0, 8)} \u2192 new entry (imp ${(this.getDefaultImportance(candidate.category) + 0.1).toFixed(2)})`
|
|
826
580
|
);
|
|
827
581
|
}
|
|
828
|
-
|
|
829
582
|
// --------------------------------------------------------------------------
|
|
830
583
|
// Store Helper
|
|
831
584
|
// --------------------------------------------------------------------------
|
|
832
|
-
|
|
833
585
|
/**
|
|
834
586
|
* Store a candidate memory as a new entry with L0/L1/L2 metadata.
|
|
835
587
|
*/
|
|
836
|
-
|
|
837
|
-
candidate: CandidateMemory,
|
|
838
|
-
vector: number[],
|
|
839
|
-
sessionKey: string,
|
|
840
|
-
targetScope: string,
|
|
841
|
-
): Promise<void> {
|
|
842
|
-
// Map 6-category to existing store categories for backward compatibility
|
|
588
|
+
async storeCandidate(candidate, vector, sessionKey, targetScope) {
|
|
843
589
|
const storeCategory = this.mapToStoreCategory(candidate.category);
|
|
844
|
-
|
|
845
590
|
const metadata = stringifySmartMetadata(
|
|
846
591
|
buildSmartMetadata(
|
|
847
592
|
{
|
|
848
593
|
text: candidate.abstract,
|
|
849
|
-
category: this.mapToStoreCategory(candidate.category)
|
|
594
|
+
category: this.mapToStoreCategory(candidate.category)
|
|
850
595
|
},
|
|
851
596
|
{
|
|
852
597
|
l0_abstract: candidate.abstract,
|
|
@@ -856,31 +601,27 @@ export class SmartExtractor {
|
|
|
856
601
|
tier: "working",
|
|
857
602
|
access_count: 0,
|
|
858
603
|
confidence: 0.7,
|
|
859
|
-
source_session: sessionKey
|
|
860
|
-
}
|
|
861
|
-
)
|
|
604
|
+
source_session: sessionKey
|
|
605
|
+
}
|
|
606
|
+
)
|
|
862
607
|
);
|
|
863
|
-
|
|
864
608
|
await this.store.store({
|
|
865
|
-
text: candidate.abstract,
|
|
609
|
+
text: candidate.abstract,
|
|
610
|
+
// L0 used as the searchable text
|
|
866
611
|
vector,
|
|
867
612
|
category: storeCategory,
|
|
868
613
|
scope: targetScope,
|
|
869
614
|
importance: this.getDefaultImportance(candidate.category),
|
|
870
|
-
metadata
|
|
615
|
+
metadata
|
|
871
616
|
});
|
|
872
|
-
|
|
873
617
|
this.log(
|
|
874
|
-
`memory-pro: smart-extractor: created [${candidate.category}] ${candidate.abstract.slice(0, 60)}
|
|
618
|
+
`memory-pro: smart-extractor: created [${candidate.category}] ${candidate.abstract.slice(0, 60)}`
|
|
875
619
|
);
|
|
876
620
|
}
|
|
877
|
-
|
|
878
621
|
/**
|
|
879
622
|
* Map 6-category to existing 5-category store type for backward compatibility.
|
|
880
623
|
*/
|
|
881
|
-
|
|
882
|
-
category: MemoryCategory,
|
|
883
|
-
): "preference" | "fact" | "decision" | "entity" | "other" {
|
|
624
|
+
mapToStoreCategory(category) {
|
|
884
625
|
switch (category) {
|
|
885
626
|
case "profile":
|
|
886
627
|
return "fact";
|
|
@@ -898,14 +639,14 @@ export class SmartExtractor {
|
|
|
898
639
|
return "other";
|
|
899
640
|
}
|
|
900
641
|
}
|
|
901
|
-
|
|
902
642
|
/**
|
|
903
643
|
* Get default importance score by category.
|
|
904
644
|
*/
|
|
905
|
-
|
|
645
|
+
getDefaultImportance(category) {
|
|
906
646
|
switch (category) {
|
|
907
647
|
case "profile":
|
|
908
|
-
return 0.9;
|
|
648
|
+
return 0.9;
|
|
649
|
+
// Identity is very important
|
|
909
650
|
case "preferences":
|
|
910
651
|
return 0.8;
|
|
911
652
|
case "entities":
|
|
@@ -913,11 +654,17 @@ export class SmartExtractor {
|
|
|
913
654
|
case "events":
|
|
914
655
|
return 0.6;
|
|
915
656
|
case "cases":
|
|
916
|
-
return 0.8;
|
|
657
|
+
return 0.8;
|
|
658
|
+
// Problem-solution pairs are high value
|
|
917
659
|
case "patterns":
|
|
918
|
-
return 0.85;
|
|
660
|
+
return 0.85;
|
|
661
|
+
// Reusable processes are high value
|
|
919
662
|
default:
|
|
920
663
|
return 0.5;
|
|
921
664
|
}
|
|
922
665
|
}
|
|
923
666
|
}
|
|
667
|
+
export {
|
|
668
|
+
SmartExtractor
|
|
669
|
+
};
|
|
670
|
+
//# sourceMappingURL=smart-extractor.js.map
|