dsh-tdai-memory 0.2.13 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. package/LICENSE +21 -21
  2. package/README.md +15 -13
  3. package/README.zh.md +9 -7
  4. package/THIRD-PARTY-NOTICES.md +20 -20
  5. package/client.js +21 -4
  6. package/cordis.patch.yml +8 -8
  7. package/index.js +36 -14
  8. package/package.json +77 -78
  9. package/recall-inject.js +86 -86
  10. package/vendor/tdai/adapters/index.js +15 -15
  11. package/vendor/tdai/adapters/openclaw/host-adapter.js +83 -83
  12. package/vendor/tdai/adapters/openclaw/index.js +5 -5
  13. package/vendor/tdai/adapters/openclaw/llm-runner.js +69 -69
  14. package/vendor/tdai/adapters/standalone/host-adapter.js +58 -58
  15. package/vendor/tdai/adapters/standalone/index.js +5 -5
  16. package/vendor/tdai/adapters/standalone/llm-runner.js +34 -6
  17. package/vendor/tdai/config.js +3 -0
  18. package/vendor/tdai/core/conversation/l0-recorder.js +420 -420
  19. package/vendor/tdai/core/hooks/auto-capture.js +236 -236
  20. package/vendor/tdai/core/hooks/auto-recall.js +640 -640
  21. package/vendor/tdai/core/index.js +8 -8
  22. package/vendor/tdai/core/persona/persona-generator.js +184 -184
  23. package/vendor/tdai/core/persona/persona-trigger.js +94 -94
  24. package/vendor/tdai/core/profile/profile-sync.js +200 -200
  25. package/vendor/tdai/core/prompts/l1-dedup.js +72 -72
  26. package/vendor/tdai/core/prompts/l1-extraction.js +34 -34
  27. package/vendor/tdai/core/prompts/persona-generation.js +35 -35
  28. package/vendor/tdai/core/prompts/scene-extraction.js +45 -45
  29. package/vendor/tdai/core/record/l1-dedup.js +318 -318
  30. package/vendor/tdai/core/record/l1-extractor.js +379 -379
  31. package/vendor/tdai/core/record/l1-reader.js +178 -178
  32. package/vendor/tdai/core/record/l1-writer.js +141 -141
  33. package/vendor/tdai/core/report/reporter.js +86 -86
  34. package/vendor/tdai/core/scene/filename-normalizer.js +161 -161
  35. package/vendor/tdai/core/scene/scene-extractor.js +410 -410
  36. package/vendor/tdai/core/scene/scene-format.js +53 -53
  37. package/vendor/tdai/core/scene/scene-index.js +82 -82
  38. package/vendor/tdai/core/scene/scene-navigation.js +57 -57
  39. package/vendor/tdai/core/seed/input.js +404 -404
  40. package/vendor/tdai/core/seed/seed-runtime.js +4 -0
  41. package/vendor/tdai/core/seed/types.js +9 -9
  42. package/vendor/tdai/core/store/bm25-client.js +117 -117
  43. package/vendor/tdai/core/store/bm25-local.js +68 -68
  44. package/vendor/tdai/core/store/embedding.js +632 -632
  45. package/vendor/tdai/core/store/factory.js +95 -95
  46. package/vendor/tdai/core/store/search-utils.js +54 -54
  47. package/vendor/tdai/core/store/sqlite.js +1632 -1632
  48. package/vendor/tdai/core/store/tcvdb-client.js +221 -221
  49. package/vendor/tdai/core/store/tcvdb.js +1047 -1047
  50. package/vendor/tdai/core/store/types.js +17 -17
  51. package/vendor/tdai/core/tdai-core.js +4 -0
  52. package/vendor/tdai/core/tools/conversation-search.js +199 -199
  53. package/vendor/tdai/core/tools/memory-search.js +207 -207
  54. package/vendor/tdai/core/types.js +12 -12
  55. package/vendor/tdai/utils/backup.js +189 -189
  56. package/vendor/tdai/utils/checkpoint.js +360 -360
  57. package/vendor/tdai/utils/clean-context-runner.js +472 -472
  58. package/vendor/tdai/utils/ensure-hook-policy.js +213 -213
  59. package/vendor/tdai/utils/env.js +13 -13
  60. package/vendor/tdai/utils/managed-timer.js +127 -127
  61. package/vendor/tdai/utils/manifest.js +91 -91
  62. package/vendor/tdai/utils/memory-cleaner.js +300 -300
  63. package/vendor/tdai/utils/no-think-fetch.js +109 -109
  64. package/vendor/tdai/utils/openclaw-state-dir.js +26 -26
  65. package/vendor/tdai/utils/pipeline-factory.js +517 -517
  66. package/vendor/tdai/utils/pipeline-manager.js +892 -892
  67. package/vendor/tdai/utils/sanitize.js +360 -360
  68. package/vendor/tdai/utils/serial-queue.js +103 -103
  69. package/vendor/tdai/utils/session-filter.js +88 -88
  70. package/vendor/tdai/utils/session-header-fetch.js +88 -0
  71. package/vendor/tdai/utils/text-utils.js +29 -29
  72. package/vendor/tdai/utils/time.js +252 -252
@@ -1,318 +1,318 @@
1
- /**
2
- * L1 Memory Conflict Detection (Batch Mode): decides how to handle multiple new
3
- * memories against existing records in a single LLM call.
4
- *
5
- * v4: Removed JSONL-based Jaccard fallback. Candidate recall now relies exclusively
6
- * on vector search (primary) and FTS5 BM25 (degraded). If neither is available,
7
- * conflict detection is skipped entirely — all memories go straight to store.
8
- *
9
- * Two-phase approach:
10
- * 1. Candidate search per new memory — vector recall or FTS5 keyword recall (fast, no LLM)
11
- * 2. Batch LLM judgment on all new memories + their candidate pools (single call)
12
- */
13
- import { CONFLICT_DETECTION_SYSTEM_PROMPT, formatBatchConflictPrompt } from "../prompts/l1-dedup.js";
14
- import { CleanContextRunner } from "../../utils/clean-context-runner.js";
15
- import { sanitizeJsonForParse } from "../../utils/sanitize.js";
16
- import { buildFtsQuery } from "../store/sqlite.js";
17
- const TAG = "[memory-tdai][l1-dedup]";
18
- // ============================
19
- // Core function (batch mode)
20
- // ============================
21
- /**
22
- * Batch conflict detection: compare all new memories against existing records
23
- * in a single LLM call.
24
- *
25
- * Candidate recall strategy (3-tier degradation):
26
- * 1. Vector recall (vectorStore + embeddingService) — cosine similarity (best)
27
- * 2. FTS5 keyword recall (vectorStore with FTS available) — BM25 ranking (degraded)
28
- * 3. Skip conflict detection entirely — all memories go straight to "store"
29
- *
30
- * The old JSONL-based Jaccard fallback has been removed. If neither vector search
31
- * nor FTS is available, we skip dedup rather than paying the O(N) full-file-scan cost.
32
- *
33
- * @param memories - Newly extracted memories (with record_id)
34
- * @param config - OpenClaw config (for LLM access)
35
- * @param logger - Optional logger
36
- * @param model - Optional model override
37
- * @param vectorStore - Optional vector store for cosine similarity search
38
- * @param embeddingService - Optional embedding service for computing query vectors
39
- * @param conflictRecallTopK - Top-K candidates to recall per new memory (default: 5)
40
- * @returns Array of dedup decisions, one per new memory
41
- */
42
- export async function batchDedup(params) {
43
- const { memories, config, logger, model, vectorStore, embeddingService, llmRunner } = params;
44
- const topK = params.conflictRecallTopK ?? 5;
45
- if (memories.length === 0) {
46
- return [];
47
- }
48
- const storeAll = () => memories.map((m) => ({
49
- record_id: m.record_id,
50
- action: "store",
51
- target_ids: [],
52
- }));
53
- // Determine what recall capabilities are available
54
- const hasVectorData = vectorStore && (await vectorStore.countL1()) > 0;
55
- const hasFts = vectorStore?.isFtsAvailable() ?? false;
56
- // Fast path: no recall capability at all → skip dedup
57
- if (!hasVectorData && !hasFts) {
58
- logger?.debug?.(`${TAG} No vector data and no FTS available, skipping conflict detection for ${memories.length} memories`);
59
- return storeAll();
60
- }
61
- // Phase 1: Find candidates
62
- //
63
- // Decision tree (after the fast-path guard above, vectorStore is guaranteed non-null):
64
- // hasVectorData + embeddingService → Tier 1 vector recall (FTS fallback on error)
65
- // otherwise hasFts → Tier 2 FTS keyword recall
66
- // otherwise → skip dedup (defensive; shouldn't reach here)
67
- let matches;
68
- if (hasVectorData && embeddingService) {
69
- // === Tier 1: Vector recall mode ===
70
- logger?.debug?.(`${TAG} Using vector recall mode (topK=${topK})`);
71
- try {
72
- matches = await findCandidatesByVector(memories, vectorStore, embeddingService, topK, logger, params.embeddingTimeoutMs);
73
- }
74
- catch (err) {
75
- logger?.warn?.(`${TAG} Vector recall failed, falling back to FTS keyword: ${err instanceof Error ? err.message : String(err)}`);
76
- // Degrade to FTS keyword recall
77
- if (hasFts) {
78
- matches = await findCandidatesByFts(memories, vectorStore, logger);
79
- }
80
- else {
81
- logger?.debug?.(`${TAG} FTS not available either, skipping conflict detection`);
82
- return storeAll();
83
- }
84
- }
85
- }
86
- else if (hasFts) {
87
- // === Tier 2: FTS keyword recall ===
88
- logger?.debug?.(`${TAG} Using FTS keyword recall mode (no embedding service or no vector data)`);
89
- matches = await findCandidatesByFts(memories, vectorStore, logger);
90
- }
91
- else {
92
- // Shouldn't reach here given the fast-path check above, but be defensive
93
- logger?.debug?.(`${TAG} No usable recall path, skipping conflict detection`);
94
- return storeAll();
95
- }
96
- // Check if any memory has candidates
97
- const hasAnyCandidates = matches.some((m) => m.candidates.length > 0);
98
- if (!hasAnyCandidates) {
99
- logger?.debug?.(`${TAG} No similar records found for any memory, all will be stored`);
100
- return storeAll();
101
- }
102
- // Phase 2: Batch LLM judgment
103
- return runLlmJudgment(matches, memories, config, logger, model, llmRunner);
104
- }
105
- /**
106
- * Phase 2: Run batch LLM judgment on candidate matches.
107
- */
108
- async function runLlmJudgment(matches, memories, config, logger, model, llmRunner) {
109
- logger?.debug?.(`${TAG} Running batch conflict detection for ${memories.length} memories`);
110
- try {
111
- const userPrompt = formatBatchConflictPrompt(matches);
112
- let result;
113
- if (llmRunner) {
114
- // Use the host-neutral LLMRunner interface
115
- result = await llmRunner.run({
116
- prompt: userPrompt,
117
- systemPrompt: CONFLICT_DETECTION_SYSTEM_PROMPT,
118
- taskId: "l1-conflict-detection",
119
- timeoutMs: 180_000,
120
- });
121
- }
122
- else {
123
- // Fallback: create CleanContextRunner (OpenClaw path)
124
- const runner = new CleanContextRunner({
125
- config,
126
- modelRef: model,
127
- enableTools: false,
128
- logger,
129
- });
130
- result = await runner.run({
131
- prompt: userPrompt,
132
- systemPrompt: CONFLICT_DETECTION_SYSTEM_PROMPT,
133
- taskId: "l1-conflict-detection",
134
- timeoutMs: 180_000,
135
- });
136
- }
137
- const decisions = parseBatchResult(result, memories, logger);
138
- return decisions;
139
- }
140
- catch (err) {
141
- logger?.warn?.(`${TAG} Batch conflict detection failed, defaulting all to store: ${err instanceof Error ? err.message : String(err)}`);
142
- return memories.map((m) => ({
143
- record_id: m.record_id,
144
- action: "store",
145
- target_ids: [],
146
- }));
147
- }
148
- }
149
- // ============================
150
- // Candidate recall strategies
151
- // ============================
152
- /**
153
- * Vector-based candidate recall (aligned with prototype):
154
- * batch-embed new memories → cosine search in VectorStore → exclude self-batch → return candidates.
155
- */
156
- async function findCandidatesByVector(memories, vectorStore, embeddingService, topK, logger, embeddingTimeoutMs) {
157
- const newRecordIds = new Set(memories.map((m) => m.record_id));
158
- // Batch-compute embeddings for all new memories
159
- const texts = memories.map((m) => m.content);
160
- const embeddings = await embeddingService.embedBatch(texts, embeddingTimeoutMs ? { timeoutMs: embeddingTimeoutMs } : undefined);
161
- const matches = [];
162
- for (let i = 0; i < memories.length; i++) {
163
- const mem = memories[i];
164
- const queryVec = embeddings[i];
165
- // Vector search top-K (request extra to account for self-batch filtering)
166
- const searchResults = await vectorStore.searchL1Vector(queryVec, topK + memories.length, mem.content);
167
- // Exclude records from current batch, convert to MemoryRecord format
168
- const candidates = searchResults
169
- .filter((r) => !newRecordIds.has(r.record_id))
170
- .slice(0, topK)
171
- .map((r) => ({
172
- id: r.record_id,
173
- content: r.content,
174
- type: r.type,
175
- priority: r.priority,
176
- scene_name: r.scene_name,
177
- source_message_ids: [],
178
- metadata: {},
179
- timestamps: [r.timestamp_str].filter(Boolean),
180
- createdAt: "",
181
- updatedAt: "",
182
- sessionKey: r.session_key,
183
- sessionId: r.session_id,
184
- }));
185
- matches.push({ newMemory: mem, candidates });
186
- }
187
- logger?.debug?.(`${TAG} Vector recall: ${matches.map((m) => `${m.newMemory.record_id}→${m.candidates.length}`).join(", ")}`);
188
- return matches;
189
- }
190
- /**
191
- * FTS5-based candidate recall:
192
- * Uses the FTS index for efficient BM25-ranked keyword matching.
193
- * This replaces the old Jaccard word-overlap fallback entirely.
194
- */
195
- async function findCandidatesByFts(memories, vectorStore, _logger) {
196
- const newRecordIds = new Set(memories.map((m) => m.record_id));
197
- const matches = [];
198
- for (const mem of memories) {
199
- const ftsQuery = buildFtsQuery(mem.content);
200
- if (ftsQuery) {
201
- const ftsResults = await vectorStore.searchL1Fts(ftsQuery, 10);
202
- // Filter out records from the current batch
203
- const candidates = ftsResults
204
- .filter((r) => !newRecordIds.has(r.record_id))
205
- .slice(0, 5)
206
- .map((r) => ({
207
- id: r.record_id,
208
- content: r.content,
209
- type: r.type,
210
- priority: r.priority,
211
- scene_name: r.scene_name,
212
- source_message_ids: [],
213
- metadata: r.metadata_json ? (() => { try {
214
- return JSON.parse(r.metadata_json);
215
- }
216
- catch {
217
- return {};
218
- } })() : {},
219
- timestamps: [r.timestamp_str].filter(Boolean),
220
- createdAt: "",
221
- updatedAt: "",
222
- sessionKey: r.session_key,
223
- sessionId: r.session_id,
224
- }));
225
- matches.push({ newMemory: mem, candidates });
226
- }
227
- else {
228
- matches.push({ newMemory: mem, candidates: [] });
229
- }
230
- }
231
- _logger?.debug?.(`${TAG} FTS keyword recall: ${matches.map((m) => `${m.newMemory.record_id}→${m.candidates.length}`).join(", ")}`);
232
- return matches;
233
- }
234
- // ============================
235
- // Result parsing
236
- // ============================
237
- const VALID_TYPES = ["persona", "episodic", "instruction"];
238
- /**
239
- * Parse the LLM's batch conflict detection JSON response.
240
- *
241
- * Expected format: [{record_id, action, target_ids, merged_content, merged_type, merged_priority, merged_timestamps}]
242
- */
243
- function parseBatchResult(raw, memories, logger) {
244
- try {
245
- // Strip markdown code block wrappers
246
- let cleaned = raw.trim();
247
- if (cleaned.startsWith("```")) {
248
- cleaned = cleaned.replace(/^```(?:json)?\s*\n?/, "").replace(/\n?```\s*$/, "");
249
- }
250
- // Extract JSON array
251
- const arrayMatch = cleaned.match(/\[[\s\S]*\]/);
252
- if (!arrayMatch) {
253
- logger?.warn?.(`${TAG} No JSON array found in conflict detection response`);
254
- return fallbackStoreAll(memories);
255
- }
256
- // Sanitize control characters inside JSON string literals that LLM may produce
257
- const sanitized = sanitizeJsonForParse(arrayMatch[0]);
258
- const parsed = JSON.parse(sanitized);
259
- if (!Array.isArray(parsed)) {
260
- logger?.warn?.(`${TAG} Conflict detection response is not an array`);
261
- return fallbackStoreAll(memories);
262
- }
263
- // Build decisions from LLM output
264
- const decisions = [];
265
- const validActions = ["store", "update", "merge", "skip"];
266
- for (const item of parsed) {
267
- if (!item || typeof item !== "object")
268
- continue;
269
- const d = item;
270
- const recordId = String(d.record_id ?? "");
271
- // Skip entries with empty/missing record_id — they are LLM hallucinations
272
- if (!recordId) {
273
- logger?.debug?.(`${TAG} Skipping decision with empty record_id`);
274
- continue;
275
- }
276
- const action = String(d.action ?? "store");
277
- if (!validActions.includes(action)) {
278
- logger?.warn?.(`${TAG} Invalid action "${action}" for record ${recordId}, defaulting to store`);
279
- }
280
- decisions.push({
281
- record_id: recordId,
282
- action: validActions.includes(action) ? action : "store",
283
- target_ids: Array.isArray(d.target_ids) ? d.target_ids.map(String) : [],
284
- merged_content: typeof d.merged_content === "string" ? d.merged_content : undefined,
285
- merged_type: VALID_TYPES.includes(d.merged_type) ? d.merged_type : undefined,
286
- merged_priority: typeof d.merged_priority === "number" ? d.merged_priority : undefined,
287
- merged_timestamps: Array.isArray(d.merged_timestamps) ? d.merged_timestamps.map(String) : undefined,
288
- });
289
- }
290
- // Ensure all memories have a decision (fill missing with "store")
291
- const decidedIds = new Set(decisions.map((d) => d.record_id));
292
- for (const mem of memories) {
293
- if (!decidedIds.has(mem.record_id)) {
294
- logger?.debug?.(`${TAG} No decision for record ${mem.record_id}, defaulting to store`);
295
- decisions.push({
296
- record_id: mem.record_id,
297
- action: "store",
298
- target_ids: [],
299
- });
300
- }
301
- }
302
- return decisions;
303
- }
304
- catch (err) {
305
- logger?.warn?.(`${TAG} Failed to parse conflict detection result: ${err instanceof Error ? err.message : String(err)}`);
306
- return fallbackStoreAll(memories);
307
- }
308
- }
309
- /**
310
- * Fallback: store all memories when parsing fails.
311
- */
312
- function fallbackStoreAll(memories) {
313
- return memories.map((m) => ({
314
- record_id: m.record_id,
315
- action: "store",
316
- target_ids: [],
317
- }));
318
- }
1
+ /**
2
+ * L1 Memory Conflict Detection (Batch Mode): decides how to handle multiple new
3
+ * memories against existing records in a single LLM call.
4
+ *
5
+ * v4: Removed JSONL-based Jaccard fallback. Candidate recall now relies exclusively
6
+ * on vector search (primary) and FTS5 BM25 (degraded). If neither is available,
7
+ * conflict detection is skipped entirely — all memories go straight to store.
8
+ *
9
+ * Two-phase approach:
10
+ * 1. Candidate search per new memory — vector recall or FTS5 keyword recall (fast, no LLM)
11
+ * 2. Batch LLM judgment on all new memories + their candidate pools (single call)
12
+ */
13
+ import { CONFLICT_DETECTION_SYSTEM_PROMPT, formatBatchConflictPrompt } from "../prompts/l1-dedup.js";
14
+ import { CleanContextRunner } from "../../utils/clean-context-runner.js";
15
+ import { sanitizeJsonForParse } from "../../utils/sanitize.js";
16
+ import { buildFtsQuery } from "../store/sqlite.js";
17
+ const TAG = "[memory-tdai][l1-dedup]";
18
+ // ============================
19
+ // Core function (batch mode)
20
+ // ============================
21
+ /**
22
+ * Batch conflict detection: compare all new memories against existing records
23
+ * in a single LLM call.
24
+ *
25
+ * Candidate recall strategy (3-tier degradation):
26
+ * 1. Vector recall (vectorStore + embeddingService) — cosine similarity (best)
27
+ * 2. FTS5 keyword recall (vectorStore with FTS available) — BM25 ranking (degraded)
28
+ * 3. Skip conflict detection entirely — all memories go straight to "store"
29
+ *
30
+ * The old JSONL-based Jaccard fallback has been removed. If neither vector search
31
+ * nor FTS is available, we skip dedup rather than paying the O(N) full-file-scan cost.
32
+ *
33
+ * @param memories - Newly extracted memories (with record_id)
34
+ * @param config - OpenClaw config (for LLM access)
35
+ * @param logger - Optional logger
36
+ * @param model - Optional model override
37
+ * @param vectorStore - Optional vector store for cosine similarity search
38
+ * @param embeddingService - Optional embedding service for computing query vectors
39
+ * @param conflictRecallTopK - Top-K candidates to recall per new memory (default: 5)
40
+ * @returns Array of dedup decisions, one per new memory
41
+ */
42
+ export async function batchDedup(params) {
43
+ const { memories, config, logger, model, vectorStore, embeddingService, llmRunner } = params;
44
+ const topK = params.conflictRecallTopK ?? 5;
45
+ if (memories.length === 0) {
46
+ return [];
47
+ }
48
+ const storeAll = () => memories.map((m) => ({
49
+ record_id: m.record_id,
50
+ action: "store",
51
+ target_ids: [],
52
+ }));
53
+ // Determine what recall capabilities are available
54
+ const hasVectorData = vectorStore && (await vectorStore.countL1()) > 0;
55
+ const hasFts = vectorStore?.isFtsAvailable() ?? false;
56
+ // Fast path: no recall capability at all → skip dedup
57
+ if (!hasVectorData && !hasFts) {
58
+ logger?.debug?.(`${TAG} No vector data and no FTS available, skipping conflict detection for ${memories.length} memories`);
59
+ return storeAll();
60
+ }
61
+ // Phase 1: Find candidates
62
+ //
63
+ // Decision tree (after the fast-path guard above, vectorStore is guaranteed non-null):
64
+ // hasVectorData + embeddingService → Tier 1 vector recall (FTS fallback on error)
65
+ // otherwise hasFts → Tier 2 FTS keyword recall
66
+ // otherwise → skip dedup (defensive; shouldn't reach here)
67
+ let matches;
68
+ if (hasVectorData && embeddingService) {
69
+ // === Tier 1: Vector recall mode ===
70
+ logger?.debug?.(`${TAG} Using vector recall mode (topK=${topK})`);
71
+ try {
72
+ matches = await findCandidatesByVector(memories, vectorStore, embeddingService, topK, logger, params.embeddingTimeoutMs);
73
+ }
74
+ catch (err) {
75
+ logger?.warn?.(`${TAG} Vector recall failed, falling back to FTS keyword: ${err instanceof Error ? err.message : String(err)}`);
76
+ // Degrade to FTS keyword recall
77
+ if (hasFts) {
78
+ matches = await findCandidatesByFts(memories, vectorStore, logger);
79
+ }
80
+ else {
81
+ logger?.debug?.(`${TAG} FTS not available either, skipping conflict detection`);
82
+ return storeAll();
83
+ }
84
+ }
85
+ }
86
+ else if (hasFts) {
87
+ // === Tier 2: FTS keyword recall ===
88
+ logger?.debug?.(`${TAG} Using FTS keyword recall mode (no embedding service or no vector data)`);
89
+ matches = await findCandidatesByFts(memories, vectorStore, logger);
90
+ }
91
+ else {
92
+ // Shouldn't reach here given the fast-path check above, but be defensive
93
+ logger?.debug?.(`${TAG} No usable recall path, skipping conflict detection`);
94
+ return storeAll();
95
+ }
96
+ // Check if any memory has candidates
97
+ const hasAnyCandidates = matches.some((m) => m.candidates.length > 0);
98
+ if (!hasAnyCandidates) {
99
+ logger?.debug?.(`${TAG} No similar records found for any memory, all will be stored`);
100
+ return storeAll();
101
+ }
102
+ // Phase 2: Batch LLM judgment
103
+ return runLlmJudgment(matches, memories, config, logger, model, llmRunner);
104
+ }
105
+ /**
106
+ * Phase 2: Run batch LLM judgment on candidate matches.
107
+ */
108
+ async function runLlmJudgment(matches, memories, config, logger, model, llmRunner) {
109
+ logger?.debug?.(`${TAG} Running batch conflict detection for ${memories.length} memories`);
110
+ try {
111
+ const userPrompt = formatBatchConflictPrompt(matches);
112
+ let result;
113
+ if (llmRunner) {
114
+ // Use the host-neutral LLMRunner interface
115
+ result = await llmRunner.run({
116
+ prompt: userPrompt,
117
+ systemPrompt: CONFLICT_DETECTION_SYSTEM_PROMPT,
118
+ taskId: "l1-conflict-detection",
119
+ timeoutMs: 180_000,
120
+ });
121
+ }
122
+ else {
123
+ // Fallback: create CleanContextRunner (OpenClaw path)
124
+ const runner = new CleanContextRunner({
125
+ config,
126
+ modelRef: model,
127
+ enableTools: false,
128
+ logger,
129
+ });
130
+ result = await runner.run({
131
+ prompt: userPrompt,
132
+ systemPrompt: CONFLICT_DETECTION_SYSTEM_PROMPT,
133
+ taskId: "l1-conflict-detection",
134
+ timeoutMs: 180_000,
135
+ });
136
+ }
137
+ const decisions = parseBatchResult(result, memories, logger);
138
+ return decisions;
139
+ }
140
+ catch (err) {
141
+ logger?.warn?.(`${TAG} Batch conflict detection failed, defaulting all to store: ${err instanceof Error ? err.message : String(err)}`);
142
+ return memories.map((m) => ({
143
+ record_id: m.record_id,
144
+ action: "store",
145
+ target_ids: [],
146
+ }));
147
+ }
148
+ }
149
+ // ============================
150
+ // Candidate recall strategies
151
+ // ============================
152
+ /**
153
+ * Vector-based candidate recall (aligned with prototype):
154
+ * batch-embed new memories → cosine search in VectorStore → exclude self-batch → return candidates.
155
+ */
156
+ async function findCandidatesByVector(memories, vectorStore, embeddingService, topK, logger, embeddingTimeoutMs) {
157
+ const newRecordIds = new Set(memories.map((m) => m.record_id));
158
+ // Batch-compute embeddings for all new memories
159
+ const texts = memories.map((m) => m.content);
160
+ const embeddings = await embeddingService.embedBatch(texts, embeddingTimeoutMs ? { timeoutMs: embeddingTimeoutMs } : undefined);
161
+ const matches = [];
162
+ for (let i = 0; i < memories.length; i++) {
163
+ const mem = memories[i];
164
+ const queryVec = embeddings[i];
165
+ // Vector search top-K (request extra to account for self-batch filtering)
166
+ const searchResults = await vectorStore.searchL1Vector(queryVec, topK + memories.length, mem.content);
167
+ // Exclude records from current batch, convert to MemoryRecord format
168
+ const candidates = searchResults
169
+ .filter((r) => !newRecordIds.has(r.record_id))
170
+ .slice(0, topK)
171
+ .map((r) => ({
172
+ id: r.record_id,
173
+ content: r.content,
174
+ type: r.type,
175
+ priority: r.priority,
176
+ scene_name: r.scene_name,
177
+ source_message_ids: [],
178
+ metadata: {},
179
+ timestamps: [r.timestamp_str].filter(Boolean),
180
+ createdAt: "",
181
+ updatedAt: "",
182
+ sessionKey: r.session_key,
183
+ sessionId: r.session_id,
184
+ }));
185
+ matches.push({ newMemory: mem, candidates });
186
+ }
187
+ logger?.debug?.(`${TAG} Vector recall: ${matches.map((m) => `${m.newMemory.record_id}→${m.candidates.length}`).join(", ")}`);
188
+ return matches;
189
+ }
190
+ /**
191
+ * FTS5-based candidate recall:
192
+ * Uses the FTS index for efficient BM25-ranked keyword matching.
193
+ * This replaces the old Jaccard word-overlap fallback entirely.
194
+ */
195
+ async function findCandidatesByFts(memories, vectorStore, _logger) {
196
+ const newRecordIds = new Set(memories.map((m) => m.record_id));
197
+ const matches = [];
198
+ for (const mem of memories) {
199
+ const ftsQuery = buildFtsQuery(mem.content);
200
+ if (ftsQuery) {
201
+ const ftsResults = await vectorStore.searchL1Fts(ftsQuery, 10);
202
+ // Filter out records from the current batch
203
+ const candidates = ftsResults
204
+ .filter((r) => !newRecordIds.has(r.record_id))
205
+ .slice(0, 5)
206
+ .map((r) => ({
207
+ id: r.record_id,
208
+ content: r.content,
209
+ type: r.type,
210
+ priority: r.priority,
211
+ scene_name: r.scene_name,
212
+ source_message_ids: [],
213
+ metadata: r.metadata_json ? (() => { try {
214
+ return JSON.parse(r.metadata_json);
215
+ }
216
+ catch {
217
+ return {};
218
+ } })() : {},
219
+ timestamps: [r.timestamp_str].filter(Boolean),
220
+ createdAt: "",
221
+ updatedAt: "",
222
+ sessionKey: r.session_key,
223
+ sessionId: r.session_id,
224
+ }));
225
+ matches.push({ newMemory: mem, candidates });
226
+ }
227
+ else {
228
+ matches.push({ newMemory: mem, candidates: [] });
229
+ }
230
+ }
231
+ _logger?.debug?.(`${TAG} FTS keyword recall: ${matches.map((m) => `${m.newMemory.record_id}→${m.candidates.length}`).join(", ")}`);
232
+ return matches;
233
+ }
234
+ // ============================
235
+ // Result parsing
236
+ // ============================
237
+ const VALID_TYPES = ["persona", "episodic", "instruction"];
238
+ /**
239
+ * Parse the LLM's batch conflict detection JSON response.
240
+ *
241
+ * Expected format: [{record_id, action, target_ids, merged_content, merged_type, merged_priority, merged_timestamps}]
242
+ */
243
+ function parseBatchResult(raw, memories, logger) {
244
+ try {
245
+ // Strip markdown code block wrappers
246
+ let cleaned = raw.trim();
247
+ if (cleaned.startsWith("```")) {
248
+ cleaned = cleaned.replace(/^```(?:json)?\s*\n?/, "").replace(/\n?```\s*$/, "");
249
+ }
250
+ // Extract JSON array
251
+ const arrayMatch = cleaned.match(/\[[\s\S]*\]/);
252
+ if (!arrayMatch) {
253
+ logger?.warn?.(`${TAG} No JSON array found in conflict detection response`);
254
+ return fallbackStoreAll(memories);
255
+ }
256
+ // Sanitize control characters inside JSON string literals that LLM may produce
257
+ const sanitized = sanitizeJsonForParse(arrayMatch[0]);
258
+ const parsed = JSON.parse(sanitized);
259
+ if (!Array.isArray(parsed)) {
260
+ logger?.warn?.(`${TAG} Conflict detection response is not an array`);
261
+ return fallbackStoreAll(memories);
262
+ }
263
+ // Build decisions from LLM output
264
+ const decisions = [];
265
+ const validActions = ["store", "update", "merge", "skip"];
266
+ for (const item of parsed) {
267
+ if (!item || typeof item !== "object")
268
+ continue;
269
+ const d = item;
270
+ const recordId = String(d.record_id ?? "");
271
+ // Skip entries with empty/missing record_id — they are LLM hallucinations
272
+ if (!recordId) {
273
+ logger?.debug?.(`${TAG} Skipping decision with empty record_id`);
274
+ continue;
275
+ }
276
+ const action = String(d.action ?? "store");
277
+ if (!validActions.includes(action)) {
278
+ logger?.warn?.(`${TAG} Invalid action "${action}" for record ${recordId}, defaulting to store`);
279
+ }
280
+ decisions.push({
281
+ record_id: recordId,
282
+ action: validActions.includes(action) ? action : "store",
283
+ target_ids: Array.isArray(d.target_ids) ? d.target_ids.map(String) : [],
284
+ merged_content: typeof d.merged_content === "string" ? d.merged_content : undefined,
285
+ merged_type: VALID_TYPES.includes(d.merged_type) ? d.merged_type : undefined,
286
+ merged_priority: typeof d.merged_priority === "number" ? d.merged_priority : undefined,
287
+ merged_timestamps: Array.isArray(d.merged_timestamps) ? d.merged_timestamps.map(String) : undefined,
288
+ });
289
+ }
290
+ // Ensure all memories have a decision (fill missing with "store")
291
+ const decidedIds = new Set(decisions.map((d) => d.record_id));
292
+ for (const mem of memories) {
293
+ if (!decidedIds.has(mem.record_id)) {
294
+ logger?.debug?.(`${TAG} No decision for record ${mem.record_id}, defaulting to store`);
295
+ decisions.push({
296
+ record_id: mem.record_id,
297
+ action: "store",
298
+ target_ids: [],
299
+ });
300
+ }
301
+ }
302
+ return decisions;
303
+ }
304
+ catch (err) {
305
+ logger?.warn?.(`${TAG} Failed to parse conflict detection result: ${err instanceof Error ? err.message : String(err)}`);
306
+ return fallbackStoreAll(memories);
307
+ }
308
+ }
309
+ /**
310
+ * Fallback: store all memories when parsing fails.
311
+ */
312
+ function fallbackStoreAll(memories) {
313
+ return memories.map((m) => ({
314
+ record_id: m.record_id,
315
+ action: "store",
316
+ target_ids: [],
317
+ }));
318
+ }