dsh-tdai-memory 0.2.13 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -21
- package/README.md +15 -13
- package/README.zh.md +9 -7
- package/THIRD-PARTY-NOTICES.md +20 -20
- package/client.js +21 -4
- package/cordis.patch.yml +8 -8
- package/index.js +36 -14
- package/package.json +77 -78
- package/recall-inject.js +86 -86
- package/vendor/tdai/adapters/index.js +15 -15
- package/vendor/tdai/adapters/openclaw/host-adapter.js +83 -83
- package/vendor/tdai/adapters/openclaw/index.js +5 -5
- package/vendor/tdai/adapters/openclaw/llm-runner.js +69 -69
- package/vendor/tdai/adapters/standalone/host-adapter.js +58 -58
- package/vendor/tdai/adapters/standalone/index.js +5 -5
- package/vendor/tdai/adapters/standalone/llm-runner.js +34 -6
- package/vendor/tdai/config.js +3 -0
- package/vendor/tdai/core/conversation/l0-recorder.js +420 -420
- package/vendor/tdai/core/hooks/auto-capture.js +236 -236
- package/vendor/tdai/core/hooks/auto-recall.js +640 -640
- package/vendor/tdai/core/index.js +8 -8
- package/vendor/tdai/core/persona/persona-generator.js +184 -184
- package/vendor/tdai/core/persona/persona-trigger.js +94 -94
- package/vendor/tdai/core/profile/profile-sync.js +200 -200
- package/vendor/tdai/core/prompts/l1-dedup.js +72 -72
- package/vendor/tdai/core/prompts/l1-extraction.js +34 -34
- package/vendor/tdai/core/prompts/persona-generation.js +35 -35
- package/vendor/tdai/core/prompts/scene-extraction.js +45 -45
- package/vendor/tdai/core/record/l1-dedup.js +318 -318
- package/vendor/tdai/core/record/l1-extractor.js +379 -379
- package/vendor/tdai/core/record/l1-reader.js +178 -178
- package/vendor/tdai/core/record/l1-writer.js +141 -141
- package/vendor/tdai/core/report/reporter.js +86 -86
- package/vendor/tdai/core/scene/filename-normalizer.js +161 -161
- package/vendor/tdai/core/scene/scene-extractor.js +410 -410
- package/vendor/tdai/core/scene/scene-format.js +53 -53
- package/vendor/tdai/core/scene/scene-index.js +82 -82
- package/vendor/tdai/core/scene/scene-navigation.js +57 -57
- package/vendor/tdai/core/seed/input.js +404 -404
- package/vendor/tdai/core/seed/seed-runtime.js +4 -0
- package/vendor/tdai/core/seed/types.js +9 -9
- package/vendor/tdai/core/store/bm25-client.js +117 -117
- package/vendor/tdai/core/store/bm25-local.js +68 -68
- package/vendor/tdai/core/store/embedding.js +632 -632
- package/vendor/tdai/core/store/factory.js +95 -95
- package/vendor/tdai/core/store/search-utils.js +54 -54
- package/vendor/tdai/core/store/sqlite.js +1632 -1632
- package/vendor/tdai/core/store/tcvdb-client.js +221 -221
- package/vendor/tdai/core/store/tcvdb.js +1047 -1047
- package/vendor/tdai/core/store/types.js +17 -17
- package/vendor/tdai/core/tdai-core.js +4 -0
- package/vendor/tdai/core/tools/conversation-search.js +199 -199
- package/vendor/tdai/core/tools/memory-search.js +207 -207
- package/vendor/tdai/core/types.js +12 -12
- package/vendor/tdai/utils/backup.js +189 -189
- package/vendor/tdai/utils/checkpoint.js +360 -360
- package/vendor/tdai/utils/clean-context-runner.js +472 -472
- package/vendor/tdai/utils/ensure-hook-policy.js +213 -213
- package/vendor/tdai/utils/env.js +13 -13
- package/vendor/tdai/utils/managed-timer.js +127 -127
- package/vendor/tdai/utils/manifest.js +91 -91
- package/vendor/tdai/utils/memory-cleaner.js +300 -300
- package/vendor/tdai/utils/no-think-fetch.js +109 -109
- package/vendor/tdai/utils/openclaw-state-dir.js +26 -26
- package/vendor/tdai/utils/pipeline-factory.js +517 -517
- package/vendor/tdai/utils/pipeline-manager.js +892 -892
- package/vendor/tdai/utils/sanitize.js +360 -360
- package/vendor/tdai/utils/serial-queue.js +103 -103
- package/vendor/tdai/utils/session-filter.js +88 -88
- package/vendor/tdai/utils/session-header-fetch.js +88 -0
- package/vendor/tdai/utils/text-utils.js +29 -29
- package/vendor/tdai/utils/time.js +252 -252
|
@@ -1,318 +1,318 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* L1 Memory Conflict Detection (Batch Mode): decides how to handle multiple new
|
|
3
|
-
* memories against existing records in a single LLM call.
|
|
4
|
-
*
|
|
5
|
-
* v4: Removed JSONL-based Jaccard fallback. Candidate recall now relies exclusively
|
|
6
|
-
* on vector search (primary) and FTS5 BM25 (degraded). If neither is available,
|
|
7
|
-
* conflict detection is skipped entirely — all memories go straight to store.
|
|
8
|
-
*
|
|
9
|
-
* Two-phase approach:
|
|
10
|
-
* 1. Candidate search per new memory — vector recall or FTS5 keyword recall (fast, no LLM)
|
|
11
|
-
* 2. Batch LLM judgment on all new memories + their candidate pools (single call)
|
|
12
|
-
*/
|
|
13
|
-
import { CONFLICT_DETECTION_SYSTEM_PROMPT, formatBatchConflictPrompt } from "../prompts/l1-dedup.js";
|
|
14
|
-
import { CleanContextRunner } from "../../utils/clean-context-runner.js";
|
|
15
|
-
import { sanitizeJsonForParse } from "../../utils/sanitize.js";
|
|
16
|
-
import { buildFtsQuery } from "../store/sqlite.js";
|
|
17
|
-
const TAG = "[memory-tdai][l1-dedup]";
|
|
18
|
-
// ============================
|
|
19
|
-
// Core function (batch mode)
|
|
20
|
-
// ============================
|
|
21
|
-
/**
|
|
22
|
-
* Batch conflict detection: compare all new memories against existing records
|
|
23
|
-
* in a single LLM call.
|
|
24
|
-
*
|
|
25
|
-
* Candidate recall strategy (3-tier degradation):
|
|
26
|
-
* 1. Vector recall (vectorStore + embeddingService) — cosine similarity (best)
|
|
27
|
-
* 2. FTS5 keyword recall (vectorStore with FTS available) — BM25 ranking (degraded)
|
|
28
|
-
* 3. Skip conflict detection entirely — all memories go straight to "store"
|
|
29
|
-
*
|
|
30
|
-
* The old JSONL-based Jaccard fallback has been removed. If neither vector search
|
|
31
|
-
* nor FTS is available, we skip dedup rather than paying the O(N) full-file-scan cost.
|
|
32
|
-
*
|
|
33
|
-
* @param memories - Newly extracted memories (with record_id)
|
|
34
|
-
* @param config - OpenClaw config (for LLM access)
|
|
35
|
-
* @param logger - Optional logger
|
|
36
|
-
* @param model - Optional model override
|
|
37
|
-
* @param vectorStore - Optional vector store for cosine similarity search
|
|
38
|
-
* @param embeddingService - Optional embedding service for computing query vectors
|
|
39
|
-
* @param conflictRecallTopK - Top-K candidates to recall per new memory (default: 5)
|
|
40
|
-
* @returns Array of dedup decisions, one per new memory
|
|
41
|
-
*/
|
|
42
|
-
export async function batchDedup(params) {
|
|
43
|
-
const { memories, config, logger, model, vectorStore, embeddingService, llmRunner } = params;
|
|
44
|
-
const topK = params.conflictRecallTopK ?? 5;
|
|
45
|
-
if (memories.length === 0) {
|
|
46
|
-
return [];
|
|
47
|
-
}
|
|
48
|
-
const storeAll = () => memories.map((m) => ({
|
|
49
|
-
record_id: m.record_id,
|
|
50
|
-
action: "store",
|
|
51
|
-
target_ids: [],
|
|
52
|
-
}));
|
|
53
|
-
// Determine what recall capabilities are available
|
|
54
|
-
const hasVectorData = vectorStore && (await vectorStore.countL1()) > 0;
|
|
55
|
-
const hasFts = vectorStore?.isFtsAvailable() ?? false;
|
|
56
|
-
// Fast path: no recall capability at all → skip dedup
|
|
57
|
-
if (!hasVectorData && !hasFts) {
|
|
58
|
-
logger?.debug?.(`${TAG} No vector data and no FTS available, skipping conflict detection for ${memories.length} memories`);
|
|
59
|
-
return storeAll();
|
|
60
|
-
}
|
|
61
|
-
// Phase 1: Find candidates
|
|
62
|
-
//
|
|
63
|
-
// Decision tree (after the fast-path guard above, vectorStore is guaranteed non-null):
|
|
64
|
-
// hasVectorData + embeddingService → Tier 1 vector recall (FTS fallback on error)
|
|
65
|
-
// otherwise hasFts → Tier 2 FTS keyword recall
|
|
66
|
-
// otherwise → skip dedup (defensive; shouldn't reach here)
|
|
67
|
-
let matches;
|
|
68
|
-
if (hasVectorData && embeddingService) {
|
|
69
|
-
// === Tier 1: Vector recall mode ===
|
|
70
|
-
logger?.debug?.(`${TAG} Using vector recall mode (topK=${topK})`);
|
|
71
|
-
try {
|
|
72
|
-
matches = await findCandidatesByVector(memories, vectorStore, embeddingService, topK, logger, params.embeddingTimeoutMs);
|
|
73
|
-
}
|
|
74
|
-
catch (err) {
|
|
75
|
-
logger?.warn?.(`${TAG} Vector recall failed, falling back to FTS keyword: ${err instanceof Error ? err.message : String(err)}`);
|
|
76
|
-
// Degrade to FTS keyword recall
|
|
77
|
-
if (hasFts) {
|
|
78
|
-
matches = await findCandidatesByFts(memories, vectorStore, logger);
|
|
79
|
-
}
|
|
80
|
-
else {
|
|
81
|
-
logger?.debug?.(`${TAG} FTS not available either, skipping conflict detection`);
|
|
82
|
-
return storeAll();
|
|
83
|
-
}
|
|
84
|
-
}
|
|
85
|
-
}
|
|
86
|
-
else if (hasFts) {
|
|
87
|
-
// === Tier 2: FTS keyword recall ===
|
|
88
|
-
logger?.debug?.(`${TAG} Using FTS keyword recall mode (no embedding service or no vector data)`);
|
|
89
|
-
matches = await findCandidatesByFts(memories, vectorStore, logger);
|
|
90
|
-
}
|
|
91
|
-
else {
|
|
92
|
-
// Shouldn't reach here given the fast-path check above, but be defensive
|
|
93
|
-
logger?.debug?.(`${TAG} No usable recall path, skipping conflict detection`);
|
|
94
|
-
return storeAll();
|
|
95
|
-
}
|
|
96
|
-
// Check if any memory has candidates
|
|
97
|
-
const hasAnyCandidates = matches.some((m) => m.candidates.length > 0);
|
|
98
|
-
if (!hasAnyCandidates) {
|
|
99
|
-
logger?.debug?.(`${TAG} No similar records found for any memory, all will be stored`);
|
|
100
|
-
return storeAll();
|
|
101
|
-
}
|
|
102
|
-
// Phase 2: Batch LLM judgment
|
|
103
|
-
return runLlmJudgment(matches, memories, config, logger, model, llmRunner);
|
|
104
|
-
}
|
|
105
|
-
/**
|
|
106
|
-
* Phase 2: Run batch LLM judgment on candidate matches.
|
|
107
|
-
*/
|
|
108
|
-
async function runLlmJudgment(matches, memories, config, logger, model, llmRunner) {
|
|
109
|
-
logger?.debug?.(`${TAG} Running batch conflict detection for ${memories.length} memories`);
|
|
110
|
-
try {
|
|
111
|
-
const userPrompt = formatBatchConflictPrompt(matches);
|
|
112
|
-
let result;
|
|
113
|
-
if (llmRunner) {
|
|
114
|
-
// Use the host-neutral LLMRunner interface
|
|
115
|
-
result = await llmRunner.run({
|
|
116
|
-
prompt: userPrompt,
|
|
117
|
-
systemPrompt: CONFLICT_DETECTION_SYSTEM_PROMPT,
|
|
118
|
-
taskId: "l1-conflict-detection",
|
|
119
|
-
timeoutMs: 180_000,
|
|
120
|
-
});
|
|
121
|
-
}
|
|
122
|
-
else {
|
|
123
|
-
// Fallback: create CleanContextRunner (OpenClaw path)
|
|
124
|
-
const runner = new CleanContextRunner({
|
|
125
|
-
config,
|
|
126
|
-
modelRef: model,
|
|
127
|
-
enableTools: false,
|
|
128
|
-
logger,
|
|
129
|
-
});
|
|
130
|
-
result = await runner.run({
|
|
131
|
-
prompt: userPrompt,
|
|
132
|
-
systemPrompt: CONFLICT_DETECTION_SYSTEM_PROMPT,
|
|
133
|
-
taskId: "l1-conflict-detection",
|
|
134
|
-
timeoutMs: 180_000,
|
|
135
|
-
});
|
|
136
|
-
}
|
|
137
|
-
const decisions = parseBatchResult(result, memories, logger);
|
|
138
|
-
return decisions;
|
|
139
|
-
}
|
|
140
|
-
catch (err) {
|
|
141
|
-
logger?.warn?.(`${TAG} Batch conflict detection failed, defaulting all to store: ${err instanceof Error ? err.message : String(err)}`);
|
|
142
|
-
return memories.map((m) => ({
|
|
143
|
-
record_id: m.record_id,
|
|
144
|
-
action: "store",
|
|
145
|
-
target_ids: [],
|
|
146
|
-
}));
|
|
147
|
-
}
|
|
148
|
-
}
|
|
149
|
-
// ============================
|
|
150
|
-
// Candidate recall strategies
|
|
151
|
-
// ============================
|
|
152
|
-
/**
|
|
153
|
-
* Vector-based candidate recall (aligned with prototype):
|
|
154
|
-
* batch-embed new memories → cosine search in VectorStore → exclude self-batch → return candidates.
|
|
155
|
-
*/
|
|
156
|
-
async function findCandidatesByVector(memories, vectorStore, embeddingService, topK, logger, embeddingTimeoutMs) {
|
|
157
|
-
const newRecordIds = new Set(memories.map((m) => m.record_id));
|
|
158
|
-
// Batch-compute embeddings for all new memories
|
|
159
|
-
const texts = memories.map((m) => m.content);
|
|
160
|
-
const embeddings = await embeddingService.embedBatch(texts, embeddingTimeoutMs ? { timeoutMs: embeddingTimeoutMs } : undefined);
|
|
161
|
-
const matches = [];
|
|
162
|
-
for (let i = 0; i < memories.length; i++) {
|
|
163
|
-
const mem = memories[i];
|
|
164
|
-
const queryVec = embeddings[i];
|
|
165
|
-
// Vector search top-K (request extra to account for self-batch filtering)
|
|
166
|
-
const searchResults = await vectorStore.searchL1Vector(queryVec, topK + memories.length, mem.content);
|
|
167
|
-
// Exclude records from current batch, convert to MemoryRecord format
|
|
168
|
-
const candidates = searchResults
|
|
169
|
-
.filter((r) => !newRecordIds.has(r.record_id))
|
|
170
|
-
.slice(0, topK)
|
|
171
|
-
.map((r) => ({
|
|
172
|
-
id: r.record_id,
|
|
173
|
-
content: r.content,
|
|
174
|
-
type: r.type,
|
|
175
|
-
priority: r.priority,
|
|
176
|
-
scene_name: r.scene_name,
|
|
177
|
-
source_message_ids: [],
|
|
178
|
-
metadata: {},
|
|
179
|
-
timestamps: [r.timestamp_str].filter(Boolean),
|
|
180
|
-
createdAt: "",
|
|
181
|
-
updatedAt: "",
|
|
182
|
-
sessionKey: r.session_key,
|
|
183
|
-
sessionId: r.session_id,
|
|
184
|
-
}));
|
|
185
|
-
matches.push({ newMemory: mem, candidates });
|
|
186
|
-
}
|
|
187
|
-
logger?.debug?.(`${TAG} Vector recall: ${matches.map((m) => `${m.newMemory.record_id}→${m.candidates.length}`).join(", ")}`);
|
|
188
|
-
return matches;
|
|
189
|
-
}
|
|
190
|
-
/**
|
|
191
|
-
* FTS5-based candidate recall:
|
|
192
|
-
* Uses the FTS index for efficient BM25-ranked keyword matching.
|
|
193
|
-
* This replaces the old Jaccard word-overlap fallback entirely.
|
|
194
|
-
*/
|
|
195
|
-
async function findCandidatesByFts(memories, vectorStore, _logger) {
|
|
196
|
-
const newRecordIds = new Set(memories.map((m) => m.record_id));
|
|
197
|
-
const matches = [];
|
|
198
|
-
for (const mem of memories) {
|
|
199
|
-
const ftsQuery = buildFtsQuery(mem.content);
|
|
200
|
-
if (ftsQuery) {
|
|
201
|
-
const ftsResults = await vectorStore.searchL1Fts(ftsQuery, 10);
|
|
202
|
-
// Filter out records from the current batch
|
|
203
|
-
const candidates = ftsResults
|
|
204
|
-
.filter((r) => !newRecordIds.has(r.record_id))
|
|
205
|
-
.slice(0, 5)
|
|
206
|
-
.map((r) => ({
|
|
207
|
-
id: r.record_id,
|
|
208
|
-
content: r.content,
|
|
209
|
-
type: r.type,
|
|
210
|
-
priority: r.priority,
|
|
211
|
-
scene_name: r.scene_name,
|
|
212
|
-
source_message_ids: [],
|
|
213
|
-
metadata: r.metadata_json ? (() => { try {
|
|
214
|
-
return JSON.parse(r.metadata_json);
|
|
215
|
-
}
|
|
216
|
-
catch {
|
|
217
|
-
return {};
|
|
218
|
-
} })() : {},
|
|
219
|
-
timestamps: [r.timestamp_str].filter(Boolean),
|
|
220
|
-
createdAt: "",
|
|
221
|
-
updatedAt: "",
|
|
222
|
-
sessionKey: r.session_key,
|
|
223
|
-
sessionId: r.session_id,
|
|
224
|
-
}));
|
|
225
|
-
matches.push({ newMemory: mem, candidates });
|
|
226
|
-
}
|
|
227
|
-
else {
|
|
228
|
-
matches.push({ newMemory: mem, candidates: [] });
|
|
229
|
-
}
|
|
230
|
-
}
|
|
231
|
-
_logger?.debug?.(`${TAG} FTS keyword recall: ${matches.map((m) => `${m.newMemory.record_id}→${m.candidates.length}`).join(", ")}`);
|
|
232
|
-
return matches;
|
|
233
|
-
}
|
|
234
|
-
// ============================
|
|
235
|
-
// Result parsing
|
|
236
|
-
// ============================
|
|
237
|
-
const VALID_TYPES = ["persona", "episodic", "instruction"];
|
|
238
|
-
/**
|
|
239
|
-
* Parse the LLM's batch conflict detection JSON response.
|
|
240
|
-
*
|
|
241
|
-
* Expected format: [{record_id, action, target_ids, merged_content, merged_type, merged_priority, merged_timestamps}]
|
|
242
|
-
*/
|
|
243
|
-
function parseBatchResult(raw, memories, logger) {
|
|
244
|
-
try {
|
|
245
|
-
// Strip markdown code block wrappers
|
|
246
|
-
let cleaned = raw.trim();
|
|
247
|
-
if (cleaned.startsWith("```")) {
|
|
248
|
-
cleaned = cleaned.replace(/^```(?:json)?\s*\n?/, "").replace(/\n?```\s*$/, "");
|
|
249
|
-
}
|
|
250
|
-
// Extract JSON array
|
|
251
|
-
const arrayMatch = cleaned.match(/\[[\s\S]*\]/);
|
|
252
|
-
if (!arrayMatch) {
|
|
253
|
-
logger?.warn?.(`${TAG} No JSON array found in conflict detection response`);
|
|
254
|
-
return fallbackStoreAll(memories);
|
|
255
|
-
}
|
|
256
|
-
// Sanitize control characters inside JSON string literals that LLM may produce
|
|
257
|
-
const sanitized = sanitizeJsonForParse(arrayMatch[0]);
|
|
258
|
-
const parsed = JSON.parse(sanitized);
|
|
259
|
-
if (!Array.isArray(parsed)) {
|
|
260
|
-
logger?.warn?.(`${TAG} Conflict detection response is not an array`);
|
|
261
|
-
return fallbackStoreAll(memories);
|
|
262
|
-
}
|
|
263
|
-
// Build decisions from LLM output
|
|
264
|
-
const decisions = [];
|
|
265
|
-
const validActions = ["store", "update", "merge", "skip"];
|
|
266
|
-
for (const item of parsed) {
|
|
267
|
-
if (!item || typeof item !== "object")
|
|
268
|
-
continue;
|
|
269
|
-
const d = item;
|
|
270
|
-
const recordId = String(d.record_id ?? "");
|
|
271
|
-
// Skip entries with empty/missing record_id — they are LLM hallucinations
|
|
272
|
-
if (!recordId) {
|
|
273
|
-
logger?.debug?.(`${TAG} Skipping decision with empty record_id`);
|
|
274
|
-
continue;
|
|
275
|
-
}
|
|
276
|
-
const action = String(d.action ?? "store");
|
|
277
|
-
if (!validActions.includes(action)) {
|
|
278
|
-
logger?.warn?.(`${TAG} Invalid action "${action}" for record ${recordId}, defaulting to store`);
|
|
279
|
-
}
|
|
280
|
-
decisions.push({
|
|
281
|
-
record_id: recordId,
|
|
282
|
-
action: validActions.includes(action) ? action : "store",
|
|
283
|
-
target_ids: Array.isArray(d.target_ids) ? d.target_ids.map(String) : [],
|
|
284
|
-
merged_content: typeof d.merged_content === "string" ? d.merged_content : undefined,
|
|
285
|
-
merged_type: VALID_TYPES.includes(d.merged_type) ? d.merged_type : undefined,
|
|
286
|
-
merged_priority: typeof d.merged_priority === "number" ? d.merged_priority : undefined,
|
|
287
|
-
merged_timestamps: Array.isArray(d.merged_timestamps) ? d.merged_timestamps.map(String) : undefined,
|
|
288
|
-
});
|
|
289
|
-
}
|
|
290
|
-
// Ensure all memories have a decision (fill missing with "store")
|
|
291
|
-
const decidedIds = new Set(decisions.map((d) => d.record_id));
|
|
292
|
-
for (const mem of memories) {
|
|
293
|
-
if (!decidedIds.has(mem.record_id)) {
|
|
294
|
-
logger?.debug?.(`${TAG} No decision for record ${mem.record_id}, defaulting to store`);
|
|
295
|
-
decisions.push({
|
|
296
|
-
record_id: mem.record_id,
|
|
297
|
-
action: "store",
|
|
298
|
-
target_ids: [],
|
|
299
|
-
});
|
|
300
|
-
}
|
|
301
|
-
}
|
|
302
|
-
return decisions;
|
|
303
|
-
}
|
|
304
|
-
catch (err) {
|
|
305
|
-
logger?.warn?.(`${TAG} Failed to parse conflict detection result: ${err instanceof Error ? err.message : String(err)}`);
|
|
306
|
-
return fallbackStoreAll(memories);
|
|
307
|
-
}
|
|
308
|
-
}
|
|
309
|
-
/**
|
|
310
|
-
* Fallback: store all memories when parsing fails.
|
|
311
|
-
*/
|
|
312
|
-
function fallbackStoreAll(memories) {
|
|
313
|
-
return memories.map((m) => ({
|
|
314
|
-
record_id: m.record_id,
|
|
315
|
-
action: "store",
|
|
316
|
-
target_ids: [],
|
|
317
|
-
}));
|
|
318
|
-
}
|
|
1
|
+
/**
|
|
2
|
+
* L1 Memory Conflict Detection (Batch Mode): decides how to handle multiple new
|
|
3
|
+
* memories against existing records in a single LLM call.
|
|
4
|
+
*
|
|
5
|
+
* v4: Removed JSONL-based Jaccard fallback. Candidate recall now relies exclusively
|
|
6
|
+
* on vector search (primary) and FTS5 BM25 (degraded). If neither is available,
|
|
7
|
+
* conflict detection is skipped entirely — all memories go straight to store.
|
|
8
|
+
*
|
|
9
|
+
* Two-phase approach:
|
|
10
|
+
* 1. Candidate search per new memory — vector recall or FTS5 keyword recall (fast, no LLM)
|
|
11
|
+
* 2. Batch LLM judgment on all new memories + their candidate pools (single call)
|
|
12
|
+
*/
|
|
13
|
+
import { CONFLICT_DETECTION_SYSTEM_PROMPT, formatBatchConflictPrompt } from "../prompts/l1-dedup.js";
|
|
14
|
+
import { CleanContextRunner } from "../../utils/clean-context-runner.js";
|
|
15
|
+
import { sanitizeJsonForParse } from "../../utils/sanitize.js";
|
|
16
|
+
import { buildFtsQuery } from "../store/sqlite.js";
|
|
17
|
+
const TAG = "[memory-tdai][l1-dedup]";
|
|
18
|
+
// ============================
|
|
19
|
+
// Core function (batch mode)
|
|
20
|
+
// ============================
|
|
21
|
+
/**
|
|
22
|
+
* Batch conflict detection: compare all new memories against existing records
|
|
23
|
+
* in a single LLM call.
|
|
24
|
+
*
|
|
25
|
+
* Candidate recall strategy (3-tier degradation):
|
|
26
|
+
* 1. Vector recall (vectorStore + embeddingService) — cosine similarity (best)
|
|
27
|
+
* 2. FTS5 keyword recall (vectorStore with FTS available) — BM25 ranking (degraded)
|
|
28
|
+
* 3. Skip conflict detection entirely — all memories go straight to "store"
|
|
29
|
+
*
|
|
30
|
+
* The old JSONL-based Jaccard fallback has been removed. If neither vector search
|
|
31
|
+
* nor FTS is available, we skip dedup rather than paying the O(N) full-file-scan cost.
|
|
32
|
+
*
|
|
33
|
+
* @param memories - Newly extracted memories (with record_id)
|
|
34
|
+
* @param config - OpenClaw config (for LLM access)
|
|
35
|
+
* @param logger - Optional logger
|
|
36
|
+
* @param model - Optional model override
|
|
37
|
+
* @param vectorStore - Optional vector store for cosine similarity search
|
|
38
|
+
* @param embeddingService - Optional embedding service for computing query vectors
|
|
39
|
+
* @param conflictRecallTopK - Top-K candidates to recall per new memory (default: 5)
|
|
40
|
+
* @returns Array of dedup decisions, one per new memory
|
|
41
|
+
*/
|
|
42
|
+
export async function batchDedup(params) {
|
|
43
|
+
const { memories, config, logger, model, vectorStore, embeddingService, llmRunner } = params;
|
|
44
|
+
const topK = params.conflictRecallTopK ?? 5;
|
|
45
|
+
if (memories.length === 0) {
|
|
46
|
+
return [];
|
|
47
|
+
}
|
|
48
|
+
const storeAll = () => memories.map((m) => ({
|
|
49
|
+
record_id: m.record_id,
|
|
50
|
+
action: "store",
|
|
51
|
+
target_ids: [],
|
|
52
|
+
}));
|
|
53
|
+
// Determine what recall capabilities are available
|
|
54
|
+
const hasVectorData = vectorStore && (await vectorStore.countL1()) > 0;
|
|
55
|
+
const hasFts = vectorStore?.isFtsAvailable() ?? false;
|
|
56
|
+
// Fast path: no recall capability at all → skip dedup
|
|
57
|
+
if (!hasVectorData && !hasFts) {
|
|
58
|
+
logger?.debug?.(`${TAG} No vector data and no FTS available, skipping conflict detection for ${memories.length} memories`);
|
|
59
|
+
return storeAll();
|
|
60
|
+
}
|
|
61
|
+
// Phase 1: Find candidates
|
|
62
|
+
//
|
|
63
|
+
// Decision tree (after the fast-path guard above, vectorStore is guaranteed non-null):
|
|
64
|
+
// hasVectorData + embeddingService → Tier 1 vector recall (FTS fallback on error)
|
|
65
|
+
// otherwise hasFts → Tier 2 FTS keyword recall
|
|
66
|
+
// otherwise → skip dedup (defensive; shouldn't reach here)
|
|
67
|
+
let matches;
|
|
68
|
+
if (hasVectorData && embeddingService) {
|
|
69
|
+
// === Tier 1: Vector recall mode ===
|
|
70
|
+
logger?.debug?.(`${TAG} Using vector recall mode (topK=${topK})`);
|
|
71
|
+
try {
|
|
72
|
+
matches = await findCandidatesByVector(memories, vectorStore, embeddingService, topK, logger, params.embeddingTimeoutMs);
|
|
73
|
+
}
|
|
74
|
+
catch (err) {
|
|
75
|
+
logger?.warn?.(`${TAG} Vector recall failed, falling back to FTS keyword: ${err instanceof Error ? err.message : String(err)}`);
|
|
76
|
+
// Degrade to FTS keyword recall
|
|
77
|
+
if (hasFts) {
|
|
78
|
+
matches = await findCandidatesByFts(memories, vectorStore, logger);
|
|
79
|
+
}
|
|
80
|
+
else {
|
|
81
|
+
logger?.debug?.(`${TAG} FTS not available either, skipping conflict detection`);
|
|
82
|
+
return storeAll();
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
else if (hasFts) {
|
|
87
|
+
// === Tier 2: FTS keyword recall ===
|
|
88
|
+
logger?.debug?.(`${TAG} Using FTS keyword recall mode (no embedding service or no vector data)`);
|
|
89
|
+
matches = await findCandidatesByFts(memories, vectorStore, logger);
|
|
90
|
+
}
|
|
91
|
+
else {
|
|
92
|
+
// Shouldn't reach here given the fast-path check above, but be defensive
|
|
93
|
+
logger?.debug?.(`${TAG} No usable recall path, skipping conflict detection`);
|
|
94
|
+
return storeAll();
|
|
95
|
+
}
|
|
96
|
+
// Check if any memory has candidates
|
|
97
|
+
const hasAnyCandidates = matches.some((m) => m.candidates.length > 0);
|
|
98
|
+
if (!hasAnyCandidates) {
|
|
99
|
+
logger?.debug?.(`${TAG} No similar records found for any memory, all will be stored`);
|
|
100
|
+
return storeAll();
|
|
101
|
+
}
|
|
102
|
+
// Phase 2: Batch LLM judgment
|
|
103
|
+
return runLlmJudgment(matches, memories, config, logger, model, llmRunner);
|
|
104
|
+
}
|
|
105
|
+
/**
|
|
106
|
+
* Phase 2: Run batch LLM judgment on candidate matches.
|
|
107
|
+
*/
|
|
108
|
+
async function runLlmJudgment(matches, memories, config, logger, model, llmRunner) {
|
|
109
|
+
logger?.debug?.(`${TAG} Running batch conflict detection for ${memories.length} memories`);
|
|
110
|
+
try {
|
|
111
|
+
const userPrompt = formatBatchConflictPrompt(matches);
|
|
112
|
+
let result;
|
|
113
|
+
if (llmRunner) {
|
|
114
|
+
// Use the host-neutral LLMRunner interface
|
|
115
|
+
result = await llmRunner.run({
|
|
116
|
+
prompt: userPrompt,
|
|
117
|
+
systemPrompt: CONFLICT_DETECTION_SYSTEM_PROMPT,
|
|
118
|
+
taskId: "l1-conflict-detection",
|
|
119
|
+
timeoutMs: 180_000,
|
|
120
|
+
});
|
|
121
|
+
}
|
|
122
|
+
else {
|
|
123
|
+
// Fallback: create CleanContextRunner (OpenClaw path)
|
|
124
|
+
const runner = new CleanContextRunner({
|
|
125
|
+
config,
|
|
126
|
+
modelRef: model,
|
|
127
|
+
enableTools: false,
|
|
128
|
+
logger,
|
|
129
|
+
});
|
|
130
|
+
result = await runner.run({
|
|
131
|
+
prompt: userPrompt,
|
|
132
|
+
systemPrompt: CONFLICT_DETECTION_SYSTEM_PROMPT,
|
|
133
|
+
taskId: "l1-conflict-detection",
|
|
134
|
+
timeoutMs: 180_000,
|
|
135
|
+
});
|
|
136
|
+
}
|
|
137
|
+
const decisions = parseBatchResult(result, memories, logger);
|
|
138
|
+
return decisions;
|
|
139
|
+
}
|
|
140
|
+
catch (err) {
|
|
141
|
+
logger?.warn?.(`${TAG} Batch conflict detection failed, defaulting all to store: ${err instanceof Error ? err.message : String(err)}`);
|
|
142
|
+
return memories.map((m) => ({
|
|
143
|
+
record_id: m.record_id,
|
|
144
|
+
action: "store",
|
|
145
|
+
target_ids: [],
|
|
146
|
+
}));
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
// ============================
|
|
150
|
+
// Candidate recall strategies
|
|
151
|
+
// ============================
|
|
152
|
+
/**
|
|
153
|
+
* Vector-based candidate recall (aligned with prototype):
|
|
154
|
+
* batch-embed new memories → cosine search in VectorStore → exclude self-batch → return candidates.
|
|
155
|
+
*/
|
|
156
|
+
async function findCandidatesByVector(memories, vectorStore, embeddingService, topK, logger, embeddingTimeoutMs) {
|
|
157
|
+
const newRecordIds = new Set(memories.map((m) => m.record_id));
|
|
158
|
+
// Batch-compute embeddings for all new memories
|
|
159
|
+
const texts = memories.map((m) => m.content);
|
|
160
|
+
const embeddings = await embeddingService.embedBatch(texts, embeddingTimeoutMs ? { timeoutMs: embeddingTimeoutMs } : undefined);
|
|
161
|
+
const matches = [];
|
|
162
|
+
for (let i = 0; i < memories.length; i++) {
|
|
163
|
+
const mem = memories[i];
|
|
164
|
+
const queryVec = embeddings[i];
|
|
165
|
+
// Vector search top-K (request extra to account for self-batch filtering)
|
|
166
|
+
const searchResults = await vectorStore.searchL1Vector(queryVec, topK + memories.length, mem.content);
|
|
167
|
+
// Exclude records from current batch, convert to MemoryRecord format
|
|
168
|
+
const candidates = searchResults
|
|
169
|
+
.filter((r) => !newRecordIds.has(r.record_id))
|
|
170
|
+
.slice(0, topK)
|
|
171
|
+
.map((r) => ({
|
|
172
|
+
id: r.record_id,
|
|
173
|
+
content: r.content,
|
|
174
|
+
type: r.type,
|
|
175
|
+
priority: r.priority,
|
|
176
|
+
scene_name: r.scene_name,
|
|
177
|
+
source_message_ids: [],
|
|
178
|
+
metadata: {},
|
|
179
|
+
timestamps: [r.timestamp_str].filter(Boolean),
|
|
180
|
+
createdAt: "",
|
|
181
|
+
updatedAt: "",
|
|
182
|
+
sessionKey: r.session_key,
|
|
183
|
+
sessionId: r.session_id,
|
|
184
|
+
}));
|
|
185
|
+
matches.push({ newMemory: mem, candidates });
|
|
186
|
+
}
|
|
187
|
+
logger?.debug?.(`${TAG} Vector recall: ${matches.map((m) => `${m.newMemory.record_id}→${m.candidates.length}`).join(", ")}`);
|
|
188
|
+
return matches;
|
|
189
|
+
}
|
|
190
|
+
/**
|
|
191
|
+
* FTS5-based candidate recall:
|
|
192
|
+
* Uses the FTS index for efficient BM25-ranked keyword matching.
|
|
193
|
+
* This replaces the old Jaccard word-overlap fallback entirely.
|
|
194
|
+
*/
|
|
195
|
+
async function findCandidatesByFts(memories, vectorStore, _logger) {
|
|
196
|
+
const newRecordIds = new Set(memories.map((m) => m.record_id));
|
|
197
|
+
const matches = [];
|
|
198
|
+
for (const mem of memories) {
|
|
199
|
+
const ftsQuery = buildFtsQuery(mem.content);
|
|
200
|
+
if (ftsQuery) {
|
|
201
|
+
const ftsResults = await vectorStore.searchL1Fts(ftsQuery, 10);
|
|
202
|
+
// Filter out records from the current batch
|
|
203
|
+
const candidates = ftsResults
|
|
204
|
+
.filter((r) => !newRecordIds.has(r.record_id))
|
|
205
|
+
.slice(0, 5)
|
|
206
|
+
.map((r) => ({
|
|
207
|
+
id: r.record_id,
|
|
208
|
+
content: r.content,
|
|
209
|
+
type: r.type,
|
|
210
|
+
priority: r.priority,
|
|
211
|
+
scene_name: r.scene_name,
|
|
212
|
+
source_message_ids: [],
|
|
213
|
+
metadata: r.metadata_json ? (() => { try {
|
|
214
|
+
return JSON.parse(r.metadata_json);
|
|
215
|
+
}
|
|
216
|
+
catch {
|
|
217
|
+
return {};
|
|
218
|
+
} })() : {},
|
|
219
|
+
timestamps: [r.timestamp_str].filter(Boolean),
|
|
220
|
+
createdAt: "",
|
|
221
|
+
updatedAt: "",
|
|
222
|
+
sessionKey: r.session_key,
|
|
223
|
+
sessionId: r.session_id,
|
|
224
|
+
}));
|
|
225
|
+
matches.push({ newMemory: mem, candidates });
|
|
226
|
+
}
|
|
227
|
+
else {
|
|
228
|
+
matches.push({ newMemory: mem, candidates: [] });
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
_logger?.debug?.(`${TAG} FTS keyword recall: ${matches.map((m) => `${m.newMemory.record_id}→${m.candidates.length}`).join(", ")}`);
|
|
232
|
+
return matches;
|
|
233
|
+
}
|
|
234
|
+
// ============================
|
|
235
|
+
// Result parsing
|
|
236
|
+
// ============================
|
|
237
|
+
const VALID_TYPES = ["persona", "episodic", "instruction"];
|
|
238
|
+
/**
|
|
239
|
+
* Parse the LLM's batch conflict detection JSON response.
|
|
240
|
+
*
|
|
241
|
+
* Expected format: [{record_id, action, target_ids, merged_content, merged_type, merged_priority, merged_timestamps}]
|
|
242
|
+
*/
|
|
243
|
+
function parseBatchResult(raw, memories, logger) {
|
|
244
|
+
try {
|
|
245
|
+
// Strip markdown code block wrappers
|
|
246
|
+
let cleaned = raw.trim();
|
|
247
|
+
if (cleaned.startsWith("```")) {
|
|
248
|
+
cleaned = cleaned.replace(/^```(?:json)?\s*\n?/, "").replace(/\n?```\s*$/, "");
|
|
249
|
+
}
|
|
250
|
+
// Extract JSON array
|
|
251
|
+
const arrayMatch = cleaned.match(/\[[\s\S]*\]/);
|
|
252
|
+
if (!arrayMatch) {
|
|
253
|
+
logger?.warn?.(`${TAG} No JSON array found in conflict detection response`);
|
|
254
|
+
return fallbackStoreAll(memories);
|
|
255
|
+
}
|
|
256
|
+
// Sanitize control characters inside JSON string literals that LLM may produce
|
|
257
|
+
const sanitized = sanitizeJsonForParse(arrayMatch[0]);
|
|
258
|
+
const parsed = JSON.parse(sanitized);
|
|
259
|
+
if (!Array.isArray(parsed)) {
|
|
260
|
+
logger?.warn?.(`${TAG} Conflict detection response is not an array`);
|
|
261
|
+
return fallbackStoreAll(memories);
|
|
262
|
+
}
|
|
263
|
+
// Build decisions from LLM output
|
|
264
|
+
const decisions = [];
|
|
265
|
+
const validActions = ["store", "update", "merge", "skip"];
|
|
266
|
+
for (const item of parsed) {
|
|
267
|
+
if (!item || typeof item !== "object")
|
|
268
|
+
continue;
|
|
269
|
+
const d = item;
|
|
270
|
+
const recordId = String(d.record_id ?? "");
|
|
271
|
+
// Skip entries with empty/missing record_id — they are LLM hallucinations
|
|
272
|
+
if (!recordId) {
|
|
273
|
+
logger?.debug?.(`${TAG} Skipping decision with empty record_id`);
|
|
274
|
+
continue;
|
|
275
|
+
}
|
|
276
|
+
const action = String(d.action ?? "store");
|
|
277
|
+
if (!validActions.includes(action)) {
|
|
278
|
+
logger?.warn?.(`${TAG} Invalid action "${action}" for record ${recordId}, defaulting to store`);
|
|
279
|
+
}
|
|
280
|
+
decisions.push({
|
|
281
|
+
record_id: recordId,
|
|
282
|
+
action: validActions.includes(action) ? action : "store",
|
|
283
|
+
target_ids: Array.isArray(d.target_ids) ? d.target_ids.map(String) : [],
|
|
284
|
+
merged_content: typeof d.merged_content === "string" ? d.merged_content : undefined,
|
|
285
|
+
merged_type: VALID_TYPES.includes(d.merged_type) ? d.merged_type : undefined,
|
|
286
|
+
merged_priority: typeof d.merged_priority === "number" ? d.merged_priority : undefined,
|
|
287
|
+
merged_timestamps: Array.isArray(d.merged_timestamps) ? d.merged_timestamps.map(String) : undefined,
|
|
288
|
+
});
|
|
289
|
+
}
|
|
290
|
+
// Ensure all memories have a decision (fill missing with "store")
|
|
291
|
+
const decidedIds = new Set(decisions.map((d) => d.record_id));
|
|
292
|
+
for (const mem of memories) {
|
|
293
|
+
if (!decidedIds.has(mem.record_id)) {
|
|
294
|
+
logger?.debug?.(`${TAG} No decision for record ${mem.record_id}, defaulting to store`);
|
|
295
|
+
decisions.push({
|
|
296
|
+
record_id: mem.record_id,
|
|
297
|
+
action: "store",
|
|
298
|
+
target_ids: [],
|
|
299
|
+
});
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
return decisions;
|
|
303
|
+
}
|
|
304
|
+
catch (err) {
|
|
305
|
+
logger?.warn?.(`${TAG} Failed to parse conflict detection result: ${err instanceof Error ? err.message : String(err)}`);
|
|
306
|
+
return fallbackStoreAll(memories);
|
|
307
|
+
}
|
|
308
|
+
}
|
|
309
|
+
/**
|
|
310
|
+
* Fallback: store all memories when parsing fails.
|
|
311
|
+
*/
|
|
312
|
+
function fallbackStoreAll(memories) {
|
|
313
|
+
return memories.map((m) => ({
|
|
314
|
+
record_id: m.record_id,
|
|
315
|
+
action: "store",
|
|
316
|
+
target_ids: [],
|
|
317
|
+
}));
|
|
318
|
+
}
|