dsh-tdai-memory 0.4.2 → 0.4.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. package/LICENSE +21 -21
  2. package/README.md +4 -0
  3. package/README.zh.md +4 -0
  4. package/THIRD-PARTY-NOTICES.md +20 -20
  5. package/cordis.patch.yml +8 -8
  6. package/package.json +102 -100
  7. package/vendor/tdai/adapters/index.js +15 -15
  8. package/vendor/tdai/adapters/openclaw/host-adapter.js +83 -83
  9. package/vendor/tdai/adapters/openclaw/index.js +5 -5
  10. package/vendor/tdai/adapters/openclaw/llm-runner.js +69 -69
  11. package/vendor/tdai/adapters/standalone/host-adapter.js +58 -58
  12. package/vendor/tdai/adapters/standalone/index.js +5 -5
  13. package/vendor/tdai/core/conversation/l0-recorder.js +420 -420
  14. package/vendor/tdai/core/hooks/auto-capture.js +236 -236
  15. package/vendor/tdai/core/hooks/auto-recall.js +640 -640
  16. package/vendor/tdai/core/index.js +8 -8
  17. package/vendor/tdai/core/persona/persona-generator.js +184 -184
  18. package/vendor/tdai/core/persona/persona-trigger.js +94 -94
  19. package/vendor/tdai/core/profile/profile-sync.js +200 -200
  20. package/vendor/tdai/core/prompts/l1-dedup.js +72 -72
  21. package/vendor/tdai/core/prompts/l1-extraction.js +34 -34
  22. package/vendor/tdai/core/prompts/persona-generation.js +35 -35
  23. package/vendor/tdai/core/prompts/scene-extraction.js +45 -45
  24. package/vendor/tdai/core/record/l1-dedup.js +318 -318
  25. package/vendor/tdai/core/record/l1-extractor.js +379 -379
  26. package/vendor/tdai/core/record/l1-reader.js +178 -178
  27. package/vendor/tdai/core/record/l1-writer.js +141 -141
  28. package/vendor/tdai/core/report/reporter.js +86 -86
  29. package/vendor/tdai/core/scene/filename-normalizer.js +161 -161
  30. package/vendor/tdai/core/scene/scene-extractor.js +410 -410
  31. package/vendor/tdai/core/scene/scene-format.js +53 -53
  32. package/vendor/tdai/core/scene/scene-index.js +82 -82
  33. package/vendor/tdai/core/scene/scene-navigation.js +57 -57
  34. package/vendor/tdai/core/seed/input.js +404 -404
  35. package/vendor/tdai/core/seed/types.js +9 -9
  36. package/vendor/tdai/core/store/bm25-client.js +117 -117
  37. package/vendor/tdai/core/store/bm25-local.js +68 -68
  38. package/vendor/tdai/core/store/embedding.js +632 -632
  39. package/vendor/tdai/core/store/factory.js +95 -95
  40. package/vendor/tdai/core/store/search-utils.js +54 -54
  41. package/vendor/tdai/core/store/sqlite.js +1632 -1632
  42. package/vendor/tdai/core/store/tcvdb-client.js +221 -221
  43. package/vendor/tdai/core/store/tcvdb.js +1047 -1047
  44. package/vendor/tdai/core/store/types.js +17 -17
  45. package/vendor/tdai/core/tools/conversation-search.js +199 -199
  46. package/vendor/tdai/core/tools/memory-search.js +207 -207
  47. package/vendor/tdai/core/types.js +12 -12
  48. package/vendor/tdai/utils/backup.js +189 -189
  49. package/vendor/tdai/utils/checkpoint.js +360 -360
  50. package/vendor/tdai/utils/clean-context-runner.js +472 -472
  51. package/vendor/tdai/utils/ensure-hook-policy.js +213 -213
  52. package/vendor/tdai/utils/env.js +13 -13
  53. package/vendor/tdai/utils/managed-timer.js +127 -127
  54. package/vendor/tdai/utils/manifest.js +91 -91
  55. package/vendor/tdai/utils/memory-cleaner.js +300 -300
  56. package/vendor/tdai/utils/no-think-fetch.js +109 -109
  57. package/vendor/tdai/utils/openclaw-state-dir.js +26 -26
  58. package/vendor/tdai/utils/pipeline-factory.js +517 -517
  59. package/vendor/tdai/utils/pipeline-manager.js +892 -892
  60. package/vendor/tdai/utils/sanitize.js +360 -360
  61. package/vendor/tdai/utils/serial-queue.js +103 -103
  62. package/vendor/tdai/utils/session-filter.js +88 -88
  63. package/vendor/tdai/utils/text-utils.js +29 -29
  64. package/vendor/tdai/utils/time.js +252 -252
@@ -1,29 +1,29 @@
1
- /**
2
- * auto-recall hook (v3): injects relevant memories + persona into agent context
3
- * before the agent starts processing.
4
- *
5
- * - Searches L1 memories using configurable strategy (keyword / embedding / hybrid)
6
- * - keyword: FTS5 BM25 (requires FTS5; returns empty if unavailable)
7
- * - embedding: VectorStore cosine similarity
8
- * - hybrid: keyword + embedding merged with RRF
9
- * - L3 persona injection
10
- * - L2 scene navigation (full injection, LLM decides relevance)
11
- */
12
- import fs from "node:fs/promises";
13
- import path from "node:path";
14
- import { formatForLLM } from "../../utils/time.js";
15
- import { readSceneIndex } from "../scene/scene-index.js";
16
- import { generateSceneNavigation, stripSceneNavigation } from "../scene/scene-navigation.js";
17
- import { buildFtsQuery } from "../store/sqlite.js";
18
- import { sanitizeText } from "../../utils/sanitize.js";
19
- const TAG = "[memory-tdai] [recall]";
20
- const RECALL_TRUNCATION_SUFFIX = "…(已截断;可用 tdai_memory_search 或 tdai_conversation_search 查看详情)";
21
- const MIN_TRUNCATED_RECALL_LINE_CHARS = 40;
22
- const RECALL_LINE_SEPARATOR = "\n";
23
- /**
24
- * Memory tools usage guide — injected at the end of memory context so the
25
- * main agent knows how to actively retrieve deeper information.
26
- */
1
+ /**
2
+ * auto-recall hook (v3): injects relevant memories + persona into agent context
3
+ * before the agent starts processing.
4
+ *
5
+ * - Searches L1 memories using configurable strategy (keyword / embedding / hybrid)
6
+ * - keyword: FTS5 BM25 (requires FTS5; returns empty if unavailable)
7
+ * - embedding: VectorStore cosine similarity
8
+ * - hybrid: keyword + embedding merged with RRF
9
+ * - L3 persona injection
10
+ * - L2 scene navigation (full injection, LLM decides relevance)
11
+ */
12
+ import fs from "node:fs/promises";
13
+ import path from "node:path";
14
+ import { formatForLLM } from "../../utils/time.js";
15
+ import { readSceneIndex } from "../scene/scene-index.js";
16
+ import { generateSceneNavigation, stripSceneNavigation } from "../scene/scene-navigation.js";
17
+ import { buildFtsQuery } from "../store/sqlite.js";
18
+ import { sanitizeText } from "../../utils/sanitize.js";
19
+ const TAG = "[memory-tdai] [recall]";
20
+ const RECALL_TRUNCATION_SUFFIX = "…(已截断;可用 tdai_memory_search 或 tdai_conversation_search 查看详情)";
21
+ const MIN_TRUNCATED_RECALL_LINE_CHARS = 40;
22
+ const RECALL_LINE_SEPARATOR = "\n";
23
+ /**
24
+ * Memory tools usage guide — injected at the end of memory context so the
25
+ * main agent knows how to actively retrieve deeper information.
26
+ */
27
27
  const MEMORY_TOOLS_GUIDE = `<memory-tools-guide>
28
28
  ## 记忆工具调用指南
29
29
 
@@ -37,617 +37,617 @@ const MEMORY_TOOLS_GUIDE = `<memory-tools-guide>
37
37
  每轮对话中,tdai_memory_search 和 tdai_conversation_search **合计最多调用 3 次**。
38
38
  - 首次搜索无结果时,可换关键词或换工具重试,但总调用次数不要超过 3 次。
39
39
  - 若 3 次搜索后仍无结果,说明该信息不在记忆中,请直接根据已有信息回复用户,不要继续搜索。
40
- </memory-tools-guide>`;
41
- export async function performAutoRecall(params) {
42
- const { cfg, logger } = params;
43
- const timeoutMs = cfg.recall.timeoutMs ?? 5000;
44
- let timer;
45
- return Promise.race([
46
- performAutoRecallInner(params).finally(() => {
47
- if (timer)
48
- clearTimeout(timer);
49
- }),
50
- new Promise((resolve) => {
51
- timer = setTimeout(() => {
52
- logger?.warn?.(`${TAG} ⚠️ Recall timed out after ${timeoutMs}ms — skipping memory injection to avoid blocking the user`);
53
- resolve(undefined);
54
- }, timeoutMs);
55
- }),
56
- ]);
57
- }
58
- async function performAutoRecallInner(params) {
59
- const { userText, cfg, pluginDataDir, logger, vectorStore, embeddingService } = params;
60
- const tRecallStart = performance.now();
61
- // Search relevant memories (L1 layer) — skip only when userText is empty/undefined
62
- const tSearchStart = performance.now();
63
- let memoryLines = [];
64
- let effectiveStrategy = "skipped";
65
- let recalledL1Memories = [];
66
- let searchTiming = { ftsMs: 0, embeddingMs: 0, ftsHits: 0, embeddingHits: 0 };
67
- if (!userText || userText.length === 0) {
68
- logger?.debug?.(`${TAG} User text empty/undefined, skipping memory search (persona/scene still injected)`);
69
- }
70
- else {
71
- effectiveStrategy = cfg.recall.strategy ?? "hybrid";
72
- const searchResult = await searchMemories(userText, pluginDataDir, cfg, logger, effectiveStrategy, vectorStore, embeddingService);
73
- memoryLines = searchResult.lines;
74
- searchTiming = searchResult.timing;
75
- memoryLines = applyRecallBudget(memoryLines, cfg.recall, logger);
76
- // Extract structured RecalledMemory from formatted lines for metric reporting
77
- recalledL1Memories = memoryLines.map((line) => {
78
- const match = line.match(/^-\s+\[([^\]]+)\]\s+(.+?)(?:\s*\(活动时间:.*\))?$/);
79
- if (match) {
80
- const tag = match[1];
81
- const content = match[2].trim();
82
- const typePart = tag.includes("|") ? tag.split("|")[0] : tag;
83
- return { content, score: 0, type: typePart };
84
- }
85
- return { content: line, score: 0, type: "unknown" };
86
- });
87
- }
88
- const tSearchEnd = performance.now();
89
- // Read persona (L3 layer)
90
- const tPersonaStart = performance.now();
91
- let personaContent;
92
- try {
93
- const personaPath = path.join(pluginDataDir, "persona.md");
94
- const raw = await fs.readFile(personaPath, "utf-8");
95
- personaContent = stripSceneNavigation(raw).trim();
96
- if (!personaContent)
97
- personaContent = undefined;
98
- logger?.debug?.(`${TAG} Persona loaded: ${personaContent ? `${personaContent.length} chars` : "empty"}`);
99
- }
100
- catch {
101
- logger?.debug?.(`${TAG} No persona file found (expected for new users)`);
102
- }
103
- const tPersonaEnd = performance.now();
104
- // Load full scene navigation (L2 layer)
105
- const tSceneStart = performance.now();
106
- let sceneNavigation;
107
- try {
108
- const sceneIndex = await readSceneIndex(pluginDataDir);
109
- if (sceneIndex.length > 0) {
110
- sceneNavigation = generateSceneNavigation(sceneIndex, pluginDataDir);
111
- logger?.debug?.(`${TAG} Scene navigation generated: ${sceneIndex.length} scenes`);
112
- }
113
- }
114
- catch {
115
- logger?.debug?.(`${TAG} No scene index found`);
116
- }
117
- const tSceneEnd = performance.now();
118
- if (memoryLines.length === 0 && !personaContent && !sceneNavigation) {
119
- const totalMs = performance.now() - tRecallStart;
120
- logger?.info(`${TAG} ⏱ Recall timing: total=${totalMs.toFixed(0)}ms, ` +
121
- `search=${(tSearchEnd - tSearchStart).toFixed(0)}ms(strategy=${effectiveStrategy},hits=${memoryLines.length},` +
122
- `fts=${searchTiming.ftsMs.toFixed(0)}ms/${searchTiming.ftsHits}hits,` +
123
- `vec=${searchTiming.embeddingMs.toFixed(0)}ms/${searchTiming.embeddingHits}hits), ` +
124
- `persona=${(tPersonaEnd - tPersonaStart).toFixed(0)}ms, ` +
125
- `scene=${(tSceneEnd - tSceneStart).toFixed(0)}ms — no context to inject`);
126
- logger?.debug?.(`${TAG} No memories/persona/scenes to inject`);
127
- return undefined;
128
- }
129
- // Split recall context into stable and dynamic parts to optimize prompt caching.
130
- //
131
- // appendSystemContext (system prompt end — stable, cacheable):
132
- // persona, scene navigation, memory tools guide
133
- // These change infrequently; when content is identical across turns,
134
- // providers with prompt caching (Anthropic/OpenAI) can cache this region.
135
- //
136
- // prependContext (user prompt prefix — dynamic, per-turn):
137
- // L1 relevant memories — different every turn, moved out of system prompt
138
- // so it doesn't bust the system prompt cache.
139
- const stableParts = [];
140
- if (personaContent) {
141
- stableParts.push(`<user-persona>\n${personaContent}\n</user-persona>`);
142
- }
143
- if (sceneNavigation) {
144
- stableParts.push(`<scene-navigation>\n${sceneNavigation}\n</scene-navigation>`);
145
- }
146
- // Dynamic part: L1 relevant memories (changes every turn) → prependContext (user prompt)
147
- let prependContext;
148
- if (memoryLines.length > 0) {
149
- prependContext =
150
- `<relevant-memories>\n以下是当前对话召回的相关记忆,不代表当前任务进程,仅作为参考:\n\n${memoryLines.join(RECALL_LINE_SEPARATOR)}\n</relevant-memories>`;
151
- }
152
- // Append memory tools usage guide to the stable part so the agent knows
153
- // how to actively retrieve deeper context when the injected snippets
154
- // are not enough. This is static content and benefits from caching.
155
- if (stableParts.length > 0 || prependContext) {
156
- stableParts.push(MEMORY_TOOLS_GUIDE);
157
- }
158
- const appendSystemContext = stableParts.length > 0 ? stableParts.join("\n\n") : undefined;
159
- const totalMs = performance.now() - tRecallStart;
160
- logger?.info(`${TAG} ⏱ Recall timing: total=${totalMs.toFixed(0)}ms, ` +
161
- `search=${(tSearchEnd - tSearchStart).toFixed(0)}ms(strategy=${effectiveStrategy},hits=${memoryLines.length},` +
162
- `fts=${searchTiming.ftsMs.toFixed(0)}ms/${searchTiming.ftsHits}hits,` +
163
- `vec=${searchTiming.embeddingMs.toFixed(0)}ms/${searchTiming.embeddingHits}hits), ` +
164
- `persona=${(tPersonaEnd - tPersonaStart).toFixed(0)}ms(${personaContent ? `${personaContent.length}chars` : "none"}), ` +
165
- `scene=${(tSceneEnd - tSceneStart).toFixed(0)}ms(${sceneNavigation ? "loaded" : "none"})`);
166
- if (!appendSystemContext && !prependContext) {
167
- return undefined;
168
- }
169
- return {
170
- prependContext,
171
- appendSystemContext,
172
- recalledL1Memories,
173
- recalledL3Persona: personaContent ?? null,
174
- recallStrategy: effectiveStrategy,
175
- };
176
- }
177
- /**
178
- * Search memories and return both formatted lines and structured details.
179
- *
180
- * This is a thin wrapper around `searchMemories` that also captures
181
- * the recalled memory metadata for metric reporting (agent_turn event).
182
- * It parses the returned formatted lines to extract type/content info.
183
- */
184
- async function searchMemoriesWithDetails(userText, pluginDataDir, cfg, logger, strategy, vectorStore, embeddingService) {
185
- const result = await searchMemories(userText, pluginDataDir, cfg, logger, strategy, vectorStore, embeddingService);
186
- // Extract structured data from formatted memory lines.
187
- // Format: "- [type|scene] content (活动时间: ...)" or "- [type] content"
188
- const memories = result.lines.map((line) => {
189
- const match = line.match(/^-\s+\[([^\]]+)\]\s+(.+?)(?:\s*\(活动时间:.*\))?$/);
190
- if (match) {
191
- const tag = match[1];
192
- const content = match[2].trim();
193
- const typePart = tag.includes("|") ? tag.split("|")[0] : tag;
194
- return { content, score: 0, type: typePart };
195
- }
196
- return { content: line, score: 0, type: "unknown" };
197
- });
198
- return { lines: result.lines, memories, timing: result.timing };
199
- }
200
- /**
201
- * Search memories using the configured strategy.
202
- *
203
- * - "keyword": JSONL keyword-based (Jaccard similarity) — no embedding needed
204
- * - "embedding": VectorStore cosine similarity — requires vectorStore + embeddingService
205
- * - "hybrid": merge both keyword and embedding results with RRF (Reciprocal Rank Fusion)
206
- *
207
- * Falls back to keyword if embedding resources are unavailable.
208
- */
209
- async function searchMemories(userText, pluginDataDir, cfg, logger, strategy, vectorStore, embeddingService) {
210
- const emptyResult = { lines: [], timing: { ftsMs: 0, embeddingMs: 0, ftsHits: 0, embeddingHits: 0 } };
211
- // Strip gateway-injected inbound metadata (Sender, timestamps, media markers,
212
- // base64 image data, etc.) so FTS / embedding queries are based on pure user intent.
213
- const cleanText = sanitizeText(userText);
214
- if (cleanText.length < 2) {
215
- logger?.debug?.(`${TAG} Query too short for memory search (raw=${userText.length}, clean=${cleanText.length})`);
216
- return emptyResult;
217
- }
218
- if (cleanText.length !== userText.length) {
219
- logger?.debug?.(`${TAG} userText sanitized: ${userText.length} → ${cleanText.length} chars`);
220
- }
221
- const maxResults = cfg.recall.maxResults ?? 5;
222
- const threshold = cfg.recall.scoreThreshold ?? 0.3;
223
- const embeddingAvailable = !!vectorStore && !!embeddingService;
224
- logger?.debug?.(`${TAG} [searchMemories] strategy=${strategy}, embeddingAvailable=${embeddingAvailable}, ` +
225
- `vectorStore=${vectorStore ? "available" : "UNAVAILABLE"}, ` +
226
- `embeddingService=${embeddingService ? "available" : "UNAVAILABLE"}, ` +
227
- `maxResults=${maxResults}, threshold=${threshold}`);
228
- // Determine effective strategy (fall back to keyword if embedding not available)
229
- let effectiveStrategy = strategy;
230
- if ((strategy === "embedding" || strategy === "hybrid") && !embeddingAvailable) {
231
- logger?.warn?.(`${TAG} Strategy "${strategy}" requested but EmbeddingService not available, falling back to keyword`);
232
- effectiveStrategy = "keyword";
233
- }
234
- logger?.debug?.(`${TAG} Search strategy: ${effectiveStrategy} (configured: ${strategy})`);
235
- // Resolve per-call embedding timeout for recall path.
236
- // Falls back to global embedding.timeoutMs when recallTimeoutMs is not configured.
237
- const recallEmbeddingTimeoutMs = cfg.embedding?.recallTimeoutMs ?? cfg.embedding?.timeoutMs;
238
- const embeddingCallOpts = { timeoutMs: recallEmbeddingTimeoutMs };
239
- try {
240
- if (effectiveStrategy === "keyword") {
241
- const tFts = performance.now();
242
- const lines = await searchByKeyword(cleanText, pluginDataDir, maxResults, threshold, logger, vectorStore);
243
- return { lines, timing: { ftsMs: performance.now() - tFts, embeddingMs: 0, ftsHits: lines.length, embeddingHits: 0 } };
244
- }
245
- if (effectiveStrategy === "embedding") {
246
- const tEmb = performance.now();
247
- const lines = await searchByEmbedding(cleanText, maxResults, threshold, vectorStore, embeddingService, logger, embeddingCallOpts);
248
- return { lines, timing: { ftsMs: 0, embeddingMs: performance.now() - tEmb, ftsHits: 0, embeddingHits: lines.length } };
249
- }
250
- // Hybrid: if the store natively supports hybrid search (e.g. TCVDB does
251
- // server-side dense + sparse + RRF in a single API call), short-circuit
252
- // to avoid a redundant second HTTP request and a wasted local embed().
253
- if (vectorStore?.getCapabilities().nativeHybridSearch) {
254
- const tNative = performance.now();
255
- const results = await vectorStore.searchL1Hybrid({ query: cleanText, topK: maxResults });
256
- const nativeMs = performance.now() - tNative;
257
- logger?.debug?.(`${TAG} [hybrid-native] Single-call hybrid: ${results.length} results in ${nativeMs.toFixed(0)}ms`);
258
- const lines = results.map((r) => formatMemoryLine(vectorResultToFormatable(r)));
259
- return { lines, timing: { ftsMs: 0, embeddingMs: nativeMs, ftsHits: 0, embeddingHits: results.length } };
260
- }
261
- // Fallback: run keyword + embedding in parallel, merge with client-side RRF (SQLite path)
262
- return await searchHybrid(cleanText, pluginDataDir, maxResults, threshold, vectorStore, embeddingService, logger, embeddingCallOpts);
263
- }
264
- catch (err) {
265
- logger?.warn?.(`${TAG} Memory search failed (strategy=${effectiveStrategy}): ${err instanceof Error ? err.message : String(err)}`);
266
- return emptyResult;
267
- }
268
- }
269
- // ============================
270
- // Strategy: Keyword (FTS5 BM25, no in-memory fallback)
271
- // ============================
272
- async function searchByKeyword(userText, _pluginDataDir, maxResults, threshold, logger, vectorStore) {
273
- // Prefer FTS5 if available
274
- if (vectorStore?.isFtsAvailable()) {
275
- const ftsQuery = buildFtsQuery(userText);
276
- if (ftsQuery) {
277
- logger?.debug?.(`${TAG} [keyword-fts] Using FTS5 BM25 search: query="${ftsQuery}"`);
278
- const ftsResults = await vectorStore.searchL1Fts(ftsQuery, maxResults * 2);
279
- if (ftsResults.length > 0) {
280
- logger?.debug?.(`${TAG} [keyword-fts] FTS5 raw results (${ftsResults.length}): ` +
281
- ftsResults.map((r) => `id=${r.record_id} score=${r.score.toFixed(6)}`).join(", "));
282
- const filtered = ftsResults
283
- .filter((r) => r.score >= threshold)
284
- .slice(0, maxResults);
285
- if (filtered.length > 0) {
286
- logger?.debug?.(`${TAG} [keyword-fts] FTS5 found ${filtered.length} results (from ${ftsResults.length} raw, threshold=${threshold})`);
287
- return filtered.map((r) => formatMemoryLine(ftsResultToFormatable(r)));
288
- }
289
- // BM25 absolute scores are unreliable when the document set is very
290
- // small (e.g. 1–3 records) because IDF approaches 0. In that case,
291
- // trust FTS5's MATCH + rank ordering and return the top results anyway.
292
- if (ftsResults.length <= maxResults) {
293
- logger?.debug?.(`${TAG} [keyword-fts] All ${ftsResults.length} results below threshold=${threshold} ` +
294
- `but document set is small — returning all matched results`);
295
- return ftsResults.slice(0, maxResults).map((r) => formatMemoryLine(ftsResultToFormatable(r)));
296
- }
297
- logger?.debug?.(`${TAG} [keyword-fts] FTS5 returned 0 results above threshold (from ${ftsResults.length} raw)`);
298
- }
299
- }
300
- }
301
- // FTS5 not available or returned no results — skip in-memory fallback to avoid O(N) full scan
302
- logger?.debug?.(`${TAG} [keyword] FTS5 unavailable or no results, skipping keyword search`);
303
- return [];
304
- }
305
- // ============================
306
- // Strategy: Embedding (VectorStore cosine)
307
- // ============================
308
- async function searchByEmbedding(userText, maxResults, threshold, vectorStore, embeddingService, logger, embeddingCallOpts) {
309
- logger?.debug?.(`${TAG} [embedding-search] START query="${userText.slice(0, 80)}...", maxResults=${maxResults}, threshold=${threshold}`);
310
- const queryEmbedding = await embeddingService.embed(userText, embeddingCallOpts);
311
- logger?.debug?.(`${TAG} [embedding-search] Query embedding OK: dims=${queryEmbedding.length}, ` +
312
- `norm=${Math.sqrt(Array.from(queryEmbedding).reduce((s, v) => s + v * v, 0)).toFixed(4)}, ` +
313
- `searching top-${maxResults * 2}...`);
314
- // Retrieve more candidates for subsequent filtering
315
- const vecResults = await vectorStore.searchL1Vector(queryEmbedding, maxResults * 2);
316
- if (vecResults.length === 0) {
317
- logger?.debug?.(`${TAG} [embedding-search] Returned 0 results`);
318
- return [];
319
- }
320
- logger?.debug?.(`${TAG} [embedding-search] Got ${vecResults.length} candidates, filtering by threshold=${threshold}`);
321
- for (const r of vecResults) {
322
- logger?.debug?.(`${TAG} [embedding-search] candidate id=${r.record_id}, score=${r.score.toFixed(4)}, ` +
323
- `type=${r.type}, content="${r.content.slice(0, 60)}..."`);
324
- }
325
- const filtered = vecResults
326
- .filter((r) => r.score >= threshold)
327
- .slice(0, maxResults);
328
- if (filtered.length > 0) {
329
- logger?.debug?.(`${TAG} [embedding-search] Found ${filtered.length} relevant memories above threshold (from ${vecResults.length} candidates)`);
330
- return filtered.map((r) => formatMemoryLine(vectorResultToFormatable(r)));
331
- }
332
- logger?.debug?.(`${TAG} [embedding-search] No results above threshold ${threshold}`);
333
- return [];
334
- }
335
- // ============================
336
- // Strategy: Hybrid (Keyword + Embedding + RRF)
337
- // ============================
338
- /**
339
- * Hybrid search: run keyword (FTS5) and embedding in parallel, merge with
340
- * Reciprocal Rank Fusion (RRF) to combine rank lists.
341
- *
342
- * RRF score for a record at rank r = 1 / (k + r), where k=60 is a constant.
343
- * If a record appears in both lists, its RRF scores are summed.
344
- *
345
- * If FTS5 is unavailable, the keyword side returns empty and RRF uses
346
- * embedding results only.
347
- */
348
- async function searchHybrid(userText, _pluginDataDir, maxResults, _threshold, vectorStore, embeddingService, logger, embeddingCallOpts) {
349
- // Run keyword and embedding searches in parallel
350
- const candidateK = maxResults * 3; // retrieve more for merging
351
- const [keywordResult, embeddingResult] = await Promise.all([
352
- // Keyword search: FTS5 only (no in-memory fallback)
353
- (async () => {
354
- const tStart = performance.now();
355
- try {
356
- // Try FTS5 first
357
- if (vectorStore.isFtsAvailable()) {
358
- const ftsQuery = buildFtsQuery(userText);
359
- if (ftsQuery) {
360
- const ftsResults = await vectorStore.searchL1Fts(ftsQuery, candidateK);
361
- if (ftsResults.length > 0) {
362
- logger?.debug?.(`${TAG} [hybrid-keyword-fts] FTS5 found ${ftsResults.length} candidates`);
363
- // Convert FtsSearchResult to ScoredRecord for RRF merge
364
- const records = ftsResults.map((r) => ({
365
- record: {
366
- id: r.record_id,
367
- content: r.content,
368
- type: r.type,
369
- priority: r.priority,
370
- scene_name: r.scene_name,
371
- source_message_ids: [],
372
- metadata: r.metadata_json ? (() => { try {
373
- return JSON.parse(r.metadata_json);
374
- }
375
- catch {
376
- return {};
377
- } })() : {},
378
- timestamps: [r.timestamp_str].filter(Boolean),
379
- createdAt: "",
380
- updatedAt: "",
381
- sessionKey: r.session_key,
382
- sessionId: r.session_id,
383
- },
384
- score: r.score,
385
- }));
386
- return { records, ms: performance.now() - tStart };
387
- }
388
- }
389
- }
390
- // FTS5 not available or returned no results — skip in-memory fallback
391
- logger?.debug?.(`${TAG} [hybrid-keyword] FTS5 unavailable or no results, skipping keyword part`);
392
- return { records: [], ms: performance.now() - tStart };
393
- }
394
- catch (err) {
395
- logger?.warn?.(`${TAG} Hybrid: keyword part failed: ${err instanceof Error ? err.message : String(err)}`);
396
- return { records: [], ms: performance.now() - tStart };
397
- }
398
- })(),
399
- // Embedding search
400
- (async () => {
401
- const tStart = performance.now();
402
- try {
403
- logger?.debug?.(`${TAG} [hybrid-embedding] Generating query embedding...`);
404
- const queryEmbedding = await embeddingService.embed(userText, embeddingCallOpts);
405
- logger?.debug?.(`${TAG} [hybrid-embedding] Embedding OK, dims=${queryEmbedding.length}, searching top-${candidateK}...`);
406
- const results = await vectorStore.searchL1Vector(queryEmbedding, candidateK, userText);
407
- logger?.debug?.(`${TAG} [hybrid-embedding] Got ${results.length} candidates`);
408
- return { results, ms: performance.now() - tStart };
409
- }
410
- catch (err) {
411
- logger?.warn?.(`${TAG} Hybrid: embedding part failed: ${err instanceof Error ? err.message : String(err)}`);
412
- return { results: [], ms: performance.now() - tStart };
413
- }
414
- })(),
415
- ]);
416
- const keywordResults = keywordResult.records;
417
- const embeddingResults = embeddingResult.results;
418
- const timing = {
419
- ftsMs: keywordResult.ms,
420
- embeddingMs: embeddingResult.ms,
421
- ftsHits: keywordResults.length,
422
- embeddingHits: embeddingResults.length,
423
- };
424
- if (keywordResults.length === 0 && embeddingResults.length === 0) {
425
- logger?.debug?.(`${TAG} Hybrid search: both strategies returned 0 results`);
426
- return { lines: [], timing };
427
- }
428
- // RRF merge: k=60 is a standard constant from the RRF paper
429
- const RRF_K = 60;
430
- // Map: record_id → { rrfScore, formatable }
431
- const mergedMap = new Map();
432
- // Process keyword results
433
- for (let rank = 0; rank < keywordResults.length; rank++) {
434
- const r = keywordResults[rank];
435
- const id = r.record.id;
436
- const rrfScore = 1 / (RRF_K + rank + 1);
437
- const existing = mergedMap.get(id);
438
- if (existing) {
439
- existing.rrfScore += rrfScore;
440
- }
441
- else {
442
- mergedMap.set(id, { rrfScore, formatable: recordToFormatable(r.record) });
443
- }
444
- }
445
- // Process embedding results
446
- for (let rank = 0; rank < embeddingResults.length; rank++) {
447
- const r = embeddingResults[rank];
448
- const id = r.record_id;
449
- const rrfScore = 1 / (RRF_K + rank + 1);
450
- const existing = mergedMap.get(id);
451
- if (existing) {
452
- existing.rrfScore += rrfScore;
453
- }
454
- else {
455
- mergedMap.set(id, { rrfScore, formatable: vectorResultToFormatable(r) });
456
- }
457
- }
458
- // Sort by combined RRF score and take top results
459
- const sorted = [...mergedMap.entries()]
460
- .sort((a, b) => b[1].rrfScore - a[1].rrfScore)
461
- .slice(0, maxResults);
462
- if (sorted.length > 0) {
463
- logger?.debug?.(`${TAG} Hybrid search found ${sorted.length} results ` +
464
- `(keyword=${keywordResults.length}, embedding=${embeddingResults.length})`);
465
- return { lines: sorted.map(([, { formatable }]) => formatMemoryLine(formatable)), timing };
466
- }
467
- logger?.debug?.(`${TAG} Hybrid search: no results after merge`);
468
- return { lines: [], timing };
469
- }
470
- function formatMemoryLine(m) {
471
- // 1. Type tag + optional scene name
472
- const tag = m.scene_name ? `${m.type}|${m.scene_name}` : m.type;
473
- // 2. Content (core)
474
- let line = `- [${tag}] ${m.content}`;
475
- // 3. Time info — prefer activity_start/end range; fall back to timestamp as point-in-time
476
- const start = formatTimestamp(m.activity_start_time);
477
- const end = formatTimestamp(m.activity_end_time);
478
- const point = formatTimestamp(m.timestamp);
479
- if (start && end) {
480
- // 段时间: both start and end
481
- line += ` (活动时间: ${start} ~ ${end})`;
482
- }
483
- else if (start) {
484
- // 段时间: only start
485
- line += ` (活动时间: ${start}起)`;
486
- }
487
- else if (end) {
488
- // 段时间: only end
489
- line += ` (活动时间: 至${end})`;
490
- }
491
- else if (point) {
492
- // 点时间: single timestamp
493
- line += ` (活动时间: ${point})`;
494
- }
495
- // If all three are empty → no time info appended (graceful)
496
- return line;
497
- }
498
- function applyRecallBudget(lines, recall, logger) {
499
- const maxCharsPerMemory = normalizeBudgetLimit(recall.maxCharsPerMemory);
500
- const maxTotalRecallChars = normalizeBudgetLimit(recall.maxTotalRecallChars);
501
- if (!maxCharsPerMemory && !maxTotalRecallChars) {
502
- return lines;
503
- }
504
- const budgeted = [];
505
- let usedChars = 0;
506
- let truncatedCount = 0;
507
- let droppedCount = 0;
508
- for (let i = 0; i < lines.length; i++) {
509
- const line = lines[i];
510
- const perMemoryBounded = maxCharsPerMemory
511
- ? truncateRecallLine(line, maxCharsPerMemory)
512
- : line;
513
- let wasTruncated = perMemoryBounded !== line;
514
- if (!maxTotalRecallChars) {
515
- budgeted.push(perMemoryBounded);
516
- if (wasTruncated)
517
- truncatedCount++;
518
- continue;
519
- }
520
- const separatorChars = budgeted.length > 0 ? RECALL_LINE_SEPARATOR.length : 0;
521
- const remainingChars = maxTotalRecallChars - usedChars - separatorChars;
522
- if (remainingChars <= 0) {
523
- droppedCount += lines.length - i;
524
- break;
525
- }
526
- if (perMemoryBounded.length > remainingChars) {
527
- const canFit = remainingChars >= MIN_TRUNCATED_RECALL_LINE_CHARS;
528
- if (canFit) {
529
- const totalBounded = truncateRecallLine(perMemoryBounded, remainingChars);
530
- budgeted.push(totalBounded);
531
- usedChars += separatorChars + totalBounded.length;
532
- wasTruncated ||= totalBounded !== perMemoryBounded;
533
- if (wasTruncated)
534
- truncatedCount++;
535
- }
536
- droppedCount += lines.length - i - (canFit ? 1 : 0);
537
- break;
538
- }
539
- budgeted.push(perMemoryBounded);
540
- usedChars += separatorChars + perMemoryBounded.length;
541
- if (wasTruncated)
542
- truncatedCount++;
543
- }
544
- if (truncatedCount > 0 || droppedCount > 0) {
545
- logger?.debug?.(`${TAG} Recall budget applied: input=${lines.length}, output=${budgeted.length}, ` +
546
- `truncated=${truncatedCount}, dropped=${droppedCount}, ` +
547
- `maxCharsPerMemory=${recall.maxCharsPerMemory}, maxTotalRecallChars=${recall.maxTotalRecallChars}`);
548
- }
549
- return budgeted;
550
- }
551
- function normalizeBudgetLimit(value) {
552
- if (value == null || !Number.isFinite(value) || value <= 0)
553
- return undefined;
554
- return Math.floor(value);
555
- }
556
- function truncateRecallLine(line, maxChars) {
557
- // Count and slice by code point, not UTF-16 code unit, so a cut never lands
558
- // between the halves of a surrogate pair (which would corrupt a non-BMP
559
- // character to U+FFFD when the line is UTF-8 encoded for the request).
560
- const cps = Array.from(line);
561
- if (cps.length <= maxChars)
562
- return line;
563
- if (maxChars <= RECALL_TRUNCATION_SUFFIX.length) {
564
- return cps.slice(0, maxChars).join("");
565
- }
566
- return `${cps.slice(0, maxChars - RECALL_TRUNCATION_SUFFIX.length).join("").trimEnd()}${RECALL_TRUNCATION_SUFFIX}`;
567
- }
568
- /**
569
- * Format an ISO 8601 timestamp to a concise, timezone-aware string for display.
570
- * Uses the configured timezone (via time module).
571
- * - If the time part is 00:00:00 → show date only (e.g. "2025-03-01")
572
- * - Otherwise → show full ISO 8601 with offset (e.g. "2025-03-01T14:30:00+08:00")
573
- * - Returns undefined for empty/invalid inputs.
574
- */
575
- function formatTimestamp(ts) {
576
- if (!ts)
577
- return undefined;
578
- const d = new Date(ts);
579
- if (isNaN(d.getTime()))
580
- return undefined;
581
- // Check if time part is midnight UTC (date-only semantics)
582
- const match = ts.match(/^(\d{4}-\d{2}-\d{2})(?:T(\d{2}:\d{2})(?::\d{2})?)?/);
583
- if (match) {
584
- const timePart = match[2];
585
- if (!timePart || timePart === "00:00") {
586
- return match[1]; // date-only, no timezone conversion needed
587
- }
588
- }
589
- return formatForLLM(ts);
590
- }
591
- /**
592
- * Build a FormatableMemory from a full MemoryRecord (keyword search path).
593
- * Handles empty metadata, empty timestamps array gracefully.
594
- */
595
- function recordToFormatable(record) {
596
- const meta = record.metadata;
597
- return {
598
- type: record.type,
599
- content: record.content,
600
- scene_name: record.scene_name || undefined,
601
- activity_start_time: meta?.activity_start_time || undefined,
602
- activity_end_time: meta?.activity_end_time || undefined,
603
- timestamp: (record.timestamps && record.timestamps.length > 0) ? record.timestamps[0] : undefined,
604
- };
605
- }
606
- /**
607
- * Build a FormatableMemory from a VectorSearchResult (embedding search path).
608
- * Handles empty/invalid metadata_json, empty timestamp_str gracefully.
609
- */
610
- function vectorResultToFormatable(r) {
611
- let activityStart;
612
- let activityEnd;
613
- if (r.metadata_json && r.metadata_json !== "{}") {
614
- try {
615
- const meta = typeof r.metadata_json === "string" ? JSON.parse(r.metadata_json) : r.metadata_json;
616
- activityStart = meta?.activity_start_time || undefined;
617
- activityEnd = meta?.activity_end_time || undefined;
618
- }
619
- catch { /* ignore parse errors — treat as no metadata */ }
620
- }
621
- return {
622
- type: r.type,
623
- content: r.content,
624
- scene_name: r.scene_name || undefined,
625
- activity_start_time: activityStart,
626
- activity_end_time: activityEnd,
627
- timestamp: r.timestamp_str || undefined,
628
- };
629
- }
630
- /**
631
- * Build a FormatableMemory from an FtsSearchResult (FTS5 keyword search path).
632
- * Handles empty/invalid metadata_json, empty timestamp_str gracefully.
633
- */
634
- function ftsResultToFormatable(r) {
635
- let activityStart;
636
- let activityEnd;
637
- if (r.metadata_json && r.metadata_json !== "{}") {
638
- try {
639
- const meta = typeof r.metadata_json === "string" ? JSON.parse(r.metadata_json) : r.metadata_json;
640
- activityStart = meta?.activity_start_time || undefined;
641
- activityEnd = meta?.activity_end_time || undefined;
642
- }
643
- catch { /* ignore parse errors — treat as no metadata */ }
644
- }
645
- return {
646
- type: r.type,
647
- content: r.content,
648
- scene_name: r.scene_name || undefined,
649
- activity_start_time: activityStart,
650
- activity_end_time: activityEnd,
651
- timestamp: r.timestamp_str || undefined,
652
- };
653
- }
40
+ </memory-tools-guide>`;
41
+ export async function performAutoRecall(params) {
42
+ const { cfg, logger } = params;
43
+ const timeoutMs = cfg.recall.timeoutMs ?? 5000;
44
+ let timer;
45
+ return Promise.race([
46
+ performAutoRecallInner(params).finally(() => {
47
+ if (timer)
48
+ clearTimeout(timer);
49
+ }),
50
+ new Promise((resolve) => {
51
+ timer = setTimeout(() => {
52
+ logger?.warn?.(`${TAG} ⚠️ Recall timed out after ${timeoutMs}ms — skipping memory injection to avoid blocking the user`);
53
+ resolve(undefined);
54
+ }, timeoutMs);
55
+ }),
56
+ ]);
57
+ }
58
+ async function performAutoRecallInner(params) {
59
+ const { userText, cfg, pluginDataDir, logger, vectorStore, embeddingService } = params;
60
+ const tRecallStart = performance.now();
61
+ // Search relevant memories (L1 layer) — skip only when userText is empty/undefined
62
+ const tSearchStart = performance.now();
63
+ let memoryLines = [];
64
+ let effectiveStrategy = "skipped";
65
+ let recalledL1Memories = [];
66
+ let searchTiming = { ftsMs: 0, embeddingMs: 0, ftsHits: 0, embeddingHits: 0 };
67
+ if (!userText || userText.length === 0) {
68
+ logger?.debug?.(`${TAG} User text empty/undefined, skipping memory search (persona/scene still injected)`);
69
+ }
70
+ else {
71
+ effectiveStrategy = cfg.recall.strategy ?? "hybrid";
72
+ const searchResult = await searchMemories(userText, pluginDataDir, cfg, logger, effectiveStrategy, vectorStore, embeddingService);
73
+ memoryLines = searchResult.lines;
74
+ searchTiming = searchResult.timing;
75
+ memoryLines = applyRecallBudget(memoryLines, cfg.recall, logger);
76
+ // Extract structured RecalledMemory from formatted lines for metric reporting
77
+ recalledL1Memories = memoryLines.map((line) => {
78
+ const match = line.match(/^-\s+\[([^\]]+)\]\s+(.+?)(?:\s*\(活动时间:.*\))?$/);
79
+ if (match) {
80
+ const tag = match[1];
81
+ const content = match[2].trim();
82
+ const typePart = tag.includes("|") ? tag.split("|")[0] : tag;
83
+ return { content, score: 0, type: typePart };
84
+ }
85
+ return { content: line, score: 0, type: "unknown" };
86
+ });
87
+ }
88
+ const tSearchEnd = performance.now();
89
+ // Read persona (L3 layer)
90
+ const tPersonaStart = performance.now();
91
+ let personaContent;
92
+ try {
93
+ const personaPath = path.join(pluginDataDir, "persona.md");
94
+ const raw = await fs.readFile(personaPath, "utf-8");
95
+ personaContent = stripSceneNavigation(raw).trim();
96
+ if (!personaContent)
97
+ personaContent = undefined;
98
+ logger?.debug?.(`${TAG} Persona loaded: ${personaContent ? `${personaContent.length} chars` : "empty"}`);
99
+ }
100
+ catch {
101
+ logger?.debug?.(`${TAG} No persona file found (expected for new users)`);
102
+ }
103
+ const tPersonaEnd = performance.now();
104
+ // Load full scene navigation (L2 layer)
105
+ const tSceneStart = performance.now();
106
+ let sceneNavigation;
107
+ try {
108
+ const sceneIndex = await readSceneIndex(pluginDataDir);
109
+ if (sceneIndex.length > 0) {
110
+ sceneNavigation = generateSceneNavigation(sceneIndex, pluginDataDir);
111
+ logger?.debug?.(`${TAG} Scene navigation generated: ${sceneIndex.length} scenes`);
112
+ }
113
+ }
114
+ catch {
115
+ logger?.debug?.(`${TAG} No scene index found`);
116
+ }
117
+ const tSceneEnd = performance.now();
118
+ if (memoryLines.length === 0 && !personaContent && !sceneNavigation) {
119
+ const totalMs = performance.now() - tRecallStart;
120
+ logger?.info(`${TAG} ⏱ Recall timing: total=${totalMs.toFixed(0)}ms, ` +
121
+ `search=${(tSearchEnd - tSearchStart).toFixed(0)}ms(strategy=${effectiveStrategy},hits=${memoryLines.length},` +
122
+ `fts=${searchTiming.ftsMs.toFixed(0)}ms/${searchTiming.ftsHits}hits,` +
123
+ `vec=${searchTiming.embeddingMs.toFixed(0)}ms/${searchTiming.embeddingHits}hits), ` +
124
+ `persona=${(tPersonaEnd - tPersonaStart).toFixed(0)}ms, ` +
125
+ `scene=${(tSceneEnd - tSceneStart).toFixed(0)}ms — no context to inject`);
126
+ logger?.debug?.(`${TAG} No memories/persona/scenes to inject`);
127
+ return undefined;
128
+ }
129
+ // Split recall context into stable and dynamic parts to optimize prompt caching.
130
+ //
131
+ // appendSystemContext (system prompt end — stable, cacheable):
132
+ // persona, scene navigation, memory tools guide
133
+ // These change infrequently; when content is identical across turns,
134
+ // providers with prompt caching (Anthropic/OpenAI) can cache this region.
135
+ //
136
+ // prependContext (user prompt prefix — dynamic, per-turn):
137
+ // L1 relevant memories — different every turn, moved out of system prompt
138
+ // so it doesn't bust the system prompt cache.
139
+ const stableParts = [];
140
+ if (personaContent) {
141
+ stableParts.push(`<user-persona>\n${personaContent}\n</user-persona>`);
142
+ }
143
+ if (sceneNavigation) {
144
+ stableParts.push(`<scene-navigation>\n${sceneNavigation}\n</scene-navigation>`);
145
+ }
146
+ // Dynamic part: L1 relevant memories (changes every turn) → prependContext (user prompt)
147
+ let prependContext;
148
+ if (memoryLines.length > 0) {
149
+ prependContext =
150
+ `<relevant-memories>\n以下是当前对话召回的相关记忆,不代表当前任务进程,仅作为参考:\n\n${memoryLines.join(RECALL_LINE_SEPARATOR)}\n</relevant-memories>`;
151
+ }
152
+ // Append memory tools usage guide to the stable part so the agent knows
153
+ // how to actively retrieve deeper context when the injected snippets
154
+ // are not enough. This is static content and benefits from caching.
155
+ if (stableParts.length > 0 || prependContext) {
156
+ stableParts.push(MEMORY_TOOLS_GUIDE);
157
+ }
158
+ const appendSystemContext = stableParts.length > 0 ? stableParts.join("\n\n") : undefined;
159
+ const totalMs = performance.now() - tRecallStart;
160
+ logger?.info(`${TAG} ⏱ Recall timing: total=${totalMs.toFixed(0)}ms, ` +
161
+ `search=${(tSearchEnd - tSearchStart).toFixed(0)}ms(strategy=${effectiveStrategy},hits=${memoryLines.length},` +
162
+ `fts=${searchTiming.ftsMs.toFixed(0)}ms/${searchTiming.ftsHits}hits,` +
163
+ `vec=${searchTiming.embeddingMs.toFixed(0)}ms/${searchTiming.embeddingHits}hits), ` +
164
+ `persona=${(tPersonaEnd - tPersonaStart).toFixed(0)}ms(${personaContent ? `${personaContent.length}chars` : "none"}), ` +
165
+ `scene=${(tSceneEnd - tSceneStart).toFixed(0)}ms(${sceneNavigation ? "loaded" : "none"})`);
166
+ if (!appendSystemContext && !prependContext) {
167
+ return undefined;
168
+ }
169
+ return {
170
+ prependContext,
171
+ appendSystemContext,
172
+ recalledL1Memories,
173
+ recalledL3Persona: personaContent ?? null,
174
+ recallStrategy: effectiveStrategy,
175
+ };
176
+ }
177
+ /**
178
+ * Search memories and return both formatted lines and structured details.
179
+ *
180
+ * This is a thin wrapper around `searchMemories` that also captures
181
+ * the recalled memory metadata for metric reporting (agent_turn event).
182
+ * It parses the returned formatted lines to extract type/content info.
183
+ */
184
+ async function searchMemoriesWithDetails(userText, pluginDataDir, cfg, logger, strategy, vectorStore, embeddingService) {
185
+ const result = await searchMemories(userText, pluginDataDir, cfg, logger, strategy, vectorStore, embeddingService);
186
+ // Extract structured data from formatted memory lines.
187
+ // Format: "- [type|scene] content (活动时间: ...)" or "- [type] content"
188
+ const memories = result.lines.map((line) => {
189
+ const match = line.match(/^-\s+\[([^\]]+)\]\s+(.+?)(?:\s*\(活动时间:.*\))?$/);
190
+ if (match) {
191
+ const tag = match[1];
192
+ const content = match[2].trim();
193
+ const typePart = tag.includes("|") ? tag.split("|")[0] : tag;
194
+ return { content, score: 0, type: typePart };
195
+ }
196
+ return { content: line, score: 0, type: "unknown" };
197
+ });
198
+ return { lines: result.lines, memories, timing: result.timing };
199
+ }
200
+ /**
201
+ * Search memories using the configured strategy.
202
+ *
203
+ * - "keyword": JSONL keyword-based (Jaccard similarity) — no embedding needed
204
+ * - "embedding": VectorStore cosine similarity — requires vectorStore + embeddingService
205
+ * - "hybrid": merge both keyword and embedding results with RRF (Reciprocal Rank Fusion)
206
+ *
207
+ * Falls back to keyword if embedding resources are unavailable.
208
+ */
209
+ async function searchMemories(userText, pluginDataDir, cfg, logger, strategy, vectorStore, embeddingService) {
210
+ const emptyResult = { lines: [], timing: { ftsMs: 0, embeddingMs: 0, ftsHits: 0, embeddingHits: 0 } };
211
+ // Strip gateway-injected inbound metadata (Sender, timestamps, media markers,
212
+ // base64 image data, etc.) so FTS / embedding queries are based on pure user intent.
213
+ const cleanText = sanitizeText(userText);
214
+ if (cleanText.length < 2) {
215
+ logger?.debug?.(`${TAG} Query too short for memory search (raw=${userText.length}, clean=${cleanText.length})`);
216
+ return emptyResult;
217
+ }
218
+ if (cleanText.length !== userText.length) {
219
+ logger?.debug?.(`${TAG} userText sanitized: ${userText.length} → ${cleanText.length} chars`);
220
+ }
221
+ const maxResults = cfg.recall.maxResults ?? 5;
222
+ const threshold = cfg.recall.scoreThreshold ?? 0.3;
223
+ const embeddingAvailable = !!vectorStore && !!embeddingService;
224
+ logger?.debug?.(`${TAG} [searchMemories] strategy=${strategy}, embeddingAvailable=${embeddingAvailable}, ` +
225
+ `vectorStore=${vectorStore ? "available" : "UNAVAILABLE"}, ` +
226
+ `embeddingService=${embeddingService ? "available" : "UNAVAILABLE"}, ` +
227
+ `maxResults=${maxResults}, threshold=${threshold}`);
228
+ // Determine effective strategy (fall back to keyword if embedding not available)
229
+ let effectiveStrategy = strategy;
230
+ if ((strategy === "embedding" || strategy === "hybrid") && !embeddingAvailable) {
231
+ logger?.warn?.(`${TAG} Strategy "${strategy}" requested but EmbeddingService not available, falling back to keyword`);
232
+ effectiveStrategy = "keyword";
233
+ }
234
+ logger?.debug?.(`${TAG} Search strategy: ${effectiveStrategy} (configured: ${strategy})`);
235
+ // Resolve per-call embedding timeout for recall path.
236
+ // Falls back to global embedding.timeoutMs when recallTimeoutMs is not configured.
237
+ const recallEmbeddingTimeoutMs = cfg.embedding?.recallTimeoutMs ?? cfg.embedding?.timeoutMs;
238
+ const embeddingCallOpts = { timeoutMs: recallEmbeddingTimeoutMs };
239
+ try {
240
+ if (effectiveStrategy === "keyword") {
241
+ const tFts = performance.now();
242
+ const lines = await searchByKeyword(cleanText, pluginDataDir, maxResults, threshold, logger, vectorStore);
243
+ return { lines, timing: { ftsMs: performance.now() - tFts, embeddingMs: 0, ftsHits: lines.length, embeddingHits: 0 } };
244
+ }
245
+ if (effectiveStrategy === "embedding") {
246
+ const tEmb = performance.now();
247
+ const lines = await searchByEmbedding(cleanText, maxResults, threshold, vectorStore, embeddingService, logger, embeddingCallOpts);
248
+ return { lines, timing: { ftsMs: 0, embeddingMs: performance.now() - tEmb, ftsHits: 0, embeddingHits: lines.length } };
249
+ }
250
+ // Hybrid: if the store natively supports hybrid search (e.g. TCVDB does
251
+ // server-side dense + sparse + RRF in a single API call), short-circuit
252
+ // to avoid a redundant second HTTP request and a wasted local embed().
253
+ if (vectorStore?.getCapabilities().nativeHybridSearch) {
254
+ const tNative = performance.now();
255
+ const results = await vectorStore.searchL1Hybrid({ query: cleanText, topK: maxResults });
256
+ const nativeMs = performance.now() - tNative;
257
+ logger?.debug?.(`${TAG} [hybrid-native] Single-call hybrid: ${results.length} results in ${nativeMs.toFixed(0)}ms`);
258
+ const lines = results.map((r) => formatMemoryLine(vectorResultToFormatable(r)));
259
+ return { lines, timing: { ftsMs: 0, embeddingMs: nativeMs, ftsHits: 0, embeddingHits: results.length } };
260
+ }
261
+ // Fallback: run keyword + embedding in parallel, merge with client-side RRF (SQLite path)
262
+ return await searchHybrid(cleanText, pluginDataDir, maxResults, threshold, vectorStore, embeddingService, logger, embeddingCallOpts);
263
+ }
264
+ catch (err) {
265
+ logger?.warn?.(`${TAG} Memory search failed (strategy=${effectiveStrategy}): ${err instanceof Error ? err.message : String(err)}`);
266
+ return emptyResult;
267
+ }
268
+ }
269
+ // ============================
270
+ // Strategy: Keyword (FTS5 BM25, no in-memory fallback)
271
+ // ============================
272
+ async function searchByKeyword(userText, _pluginDataDir, maxResults, threshold, logger, vectorStore) {
273
+ // Prefer FTS5 if available
274
+ if (vectorStore?.isFtsAvailable()) {
275
+ const ftsQuery = buildFtsQuery(userText);
276
+ if (ftsQuery) {
277
+ logger?.debug?.(`${TAG} [keyword-fts] Using FTS5 BM25 search: query="${ftsQuery}"`);
278
+ const ftsResults = await vectorStore.searchL1Fts(ftsQuery, maxResults * 2);
279
+ if (ftsResults.length > 0) {
280
+ logger?.debug?.(`${TAG} [keyword-fts] FTS5 raw results (${ftsResults.length}): ` +
281
+ ftsResults.map((r) => `id=${r.record_id} score=${r.score.toFixed(6)}`).join(", "));
282
+ const filtered = ftsResults
283
+ .filter((r) => r.score >= threshold)
284
+ .slice(0, maxResults);
285
+ if (filtered.length > 0) {
286
+ logger?.debug?.(`${TAG} [keyword-fts] FTS5 found ${filtered.length} results (from ${ftsResults.length} raw, threshold=${threshold})`);
287
+ return filtered.map((r) => formatMemoryLine(ftsResultToFormatable(r)));
288
+ }
289
+ // BM25 absolute scores are unreliable when the document set is very
290
+ // small (e.g. 1–3 records) because IDF approaches 0. In that case,
291
+ // trust FTS5's MATCH + rank ordering and return the top results anyway.
292
+ if (ftsResults.length <= maxResults) {
293
+ logger?.debug?.(`${TAG} [keyword-fts] All ${ftsResults.length} results below threshold=${threshold} ` +
294
+ `but document set is small — returning all matched results`);
295
+ return ftsResults.slice(0, maxResults).map((r) => formatMemoryLine(ftsResultToFormatable(r)));
296
+ }
297
+ logger?.debug?.(`${TAG} [keyword-fts] FTS5 returned 0 results above threshold (from ${ftsResults.length} raw)`);
298
+ }
299
+ }
300
+ }
301
+ // FTS5 not available or returned no results — skip in-memory fallback to avoid O(N) full scan
302
+ logger?.debug?.(`${TAG} [keyword] FTS5 unavailable or no results, skipping keyword search`);
303
+ return [];
304
+ }
305
+ // ============================
306
+ // Strategy: Embedding (VectorStore cosine)
307
+ // ============================
308
+ async function searchByEmbedding(userText, maxResults, threshold, vectorStore, embeddingService, logger, embeddingCallOpts) {
309
+ logger?.debug?.(`${TAG} [embedding-search] START query="${userText.slice(0, 80)}...", maxResults=${maxResults}, threshold=${threshold}`);
310
+ const queryEmbedding = await embeddingService.embed(userText, embeddingCallOpts);
311
+ logger?.debug?.(`${TAG} [embedding-search] Query embedding OK: dims=${queryEmbedding.length}, ` +
312
+ `norm=${Math.sqrt(Array.from(queryEmbedding).reduce((s, v) => s + v * v, 0)).toFixed(4)}, ` +
313
+ `searching top-${maxResults * 2}...`);
314
+ // Retrieve more candidates for subsequent filtering
315
+ const vecResults = await vectorStore.searchL1Vector(queryEmbedding, maxResults * 2);
316
+ if (vecResults.length === 0) {
317
+ logger?.debug?.(`${TAG} [embedding-search] Returned 0 results`);
318
+ return [];
319
+ }
320
+ logger?.debug?.(`${TAG} [embedding-search] Got ${vecResults.length} candidates, filtering by threshold=${threshold}`);
321
+ for (const r of vecResults) {
322
+ logger?.debug?.(`${TAG} [embedding-search] candidate id=${r.record_id}, score=${r.score.toFixed(4)}, ` +
323
+ `type=${r.type}, content="${r.content.slice(0, 60)}..."`);
324
+ }
325
+ const filtered = vecResults
326
+ .filter((r) => r.score >= threshold)
327
+ .slice(0, maxResults);
328
+ if (filtered.length > 0) {
329
+ logger?.debug?.(`${TAG} [embedding-search] Found ${filtered.length} relevant memories above threshold (from ${vecResults.length} candidates)`);
330
+ return filtered.map((r) => formatMemoryLine(vectorResultToFormatable(r)));
331
+ }
332
+ logger?.debug?.(`${TAG} [embedding-search] No results above threshold ${threshold}`);
333
+ return [];
334
+ }
335
+ // ============================
336
+ // Strategy: Hybrid (Keyword + Embedding + RRF)
337
+ // ============================
338
+ /**
339
+ * Hybrid search: run keyword (FTS5) and embedding in parallel, merge with
340
+ * Reciprocal Rank Fusion (RRF) to combine rank lists.
341
+ *
342
+ * RRF score for a record at rank r = 1 / (k + r), where k=60 is a constant.
343
+ * If a record appears in both lists, its RRF scores are summed.
344
+ *
345
+ * If FTS5 is unavailable, the keyword side returns empty and RRF uses
346
+ * embedding results only.
347
+ */
348
+ async function searchHybrid(userText, _pluginDataDir, maxResults, _threshold, vectorStore, embeddingService, logger, embeddingCallOpts) {
349
+ // Run keyword and embedding searches in parallel
350
+ const candidateK = maxResults * 3; // retrieve more for merging
351
+ const [keywordResult, embeddingResult] = await Promise.all([
352
+ // Keyword search: FTS5 only (no in-memory fallback)
353
+ (async () => {
354
+ const tStart = performance.now();
355
+ try {
356
+ // Try FTS5 first
357
+ if (vectorStore.isFtsAvailable()) {
358
+ const ftsQuery = buildFtsQuery(userText);
359
+ if (ftsQuery) {
360
+ const ftsResults = await vectorStore.searchL1Fts(ftsQuery, candidateK);
361
+ if (ftsResults.length > 0) {
362
+ logger?.debug?.(`${TAG} [hybrid-keyword-fts] FTS5 found ${ftsResults.length} candidates`);
363
+ // Convert FtsSearchResult to ScoredRecord for RRF merge
364
+ const records = ftsResults.map((r) => ({
365
+ record: {
366
+ id: r.record_id,
367
+ content: r.content,
368
+ type: r.type,
369
+ priority: r.priority,
370
+ scene_name: r.scene_name,
371
+ source_message_ids: [],
372
+ metadata: r.metadata_json ? (() => { try {
373
+ return JSON.parse(r.metadata_json);
374
+ }
375
+ catch {
376
+ return {};
377
+ } })() : {},
378
+ timestamps: [r.timestamp_str].filter(Boolean),
379
+ createdAt: "",
380
+ updatedAt: "",
381
+ sessionKey: r.session_key,
382
+ sessionId: r.session_id,
383
+ },
384
+ score: r.score,
385
+ }));
386
+ return { records, ms: performance.now() - tStart };
387
+ }
388
+ }
389
+ }
390
+ // FTS5 not available or returned no results — skip in-memory fallback
391
+ logger?.debug?.(`${TAG} [hybrid-keyword] FTS5 unavailable or no results, skipping keyword part`);
392
+ return { records: [], ms: performance.now() - tStart };
393
+ }
394
+ catch (err) {
395
+ logger?.warn?.(`${TAG} Hybrid: keyword part failed: ${err instanceof Error ? err.message : String(err)}`);
396
+ return { records: [], ms: performance.now() - tStart };
397
+ }
398
+ })(),
399
+ // Embedding search
400
+ (async () => {
401
+ const tStart = performance.now();
402
+ try {
403
+ logger?.debug?.(`${TAG} [hybrid-embedding] Generating query embedding...`);
404
+ const queryEmbedding = await embeddingService.embed(userText, embeddingCallOpts);
405
+ logger?.debug?.(`${TAG} [hybrid-embedding] Embedding OK, dims=${queryEmbedding.length}, searching top-${candidateK}...`);
406
+ const results = await vectorStore.searchL1Vector(queryEmbedding, candidateK, userText);
407
+ logger?.debug?.(`${TAG} [hybrid-embedding] Got ${results.length} candidates`);
408
+ return { results, ms: performance.now() - tStart };
409
+ }
410
+ catch (err) {
411
+ logger?.warn?.(`${TAG} Hybrid: embedding part failed: ${err instanceof Error ? err.message : String(err)}`);
412
+ return { results: [], ms: performance.now() - tStart };
413
+ }
414
+ })(),
415
+ ]);
416
+ const keywordResults = keywordResult.records;
417
+ const embeddingResults = embeddingResult.results;
418
+ const timing = {
419
+ ftsMs: keywordResult.ms,
420
+ embeddingMs: embeddingResult.ms,
421
+ ftsHits: keywordResults.length,
422
+ embeddingHits: embeddingResults.length,
423
+ };
424
+ if (keywordResults.length === 0 && embeddingResults.length === 0) {
425
+ logger?.debug?.(`${TAG} Hybrid search: both strategies returned 0 results`);
426
+ return { lines: [], timing };
427
+ }
428
+ // RRF merge: k=60 is a standard constant from the RRF paper
429
+ const RRF_K = 60;
430
+ // Map: record_id → { rrfScore, formatable }
431
+ const mergedMap = new Map();
432
+ // Process keyword results
433
+ for (let rank = 0; rank < keywordResults.length; rank++) {
434
+ const r = keywordResults[rank];
435
+ const id = r.record.id;
436
+ const rrfScore = 1 / (RRF_K + rank + 1);
437
+ const existing = mergedMap.get(id);
438
+ if (existing) {
439
+ existing.rrfScore += rrfScore;
440
+ }
441
+ else {
442
+ mergedMap.set(id, { rrfScore, formatable: recordToFormatable(r.record) });
443
+ }
444
+ }
445
+ // Process embedding results
446
+ for (let rank = 0; rank < embeddingResults.length; rank++) {
447
+ const r = embeddingResults[rank];
448
+ const id = r.record_id;
449
+ const rrfScore = 1 / (RRF_K + rank + 1);
450
+ const existing = mergedMap.get(id);
451
+ if (existing) {
452
+ existing.rrfScore += rrfScore;
453
+ }
454
+ else {
455
+ mergedMap.set(id, { rrfScore, formatable: vectorResultToFormatable(r) });
456
+ }
457
+ }
458
+ // Sort by combined RRF score and take top results
459
+ const sorted = [...mergedMap.entries()]
460
+ .sort((a, b) => b[1].rrfScore - a[1].rrfScore)
461
+ .slice(0, maxResults);
462
+ if (sorted.length > 0) {
463
+ logger?.debug?.(`${TAG} Hybrid search found ${sorted.length} results ` +
464
+ `(keyword=${keywordResults.length}, embedding=${embeddingResults.length})`);
465
+ return { lines: sorted.map(([, { formatable }]) => formatMemoryLine(formatable)), timing };
466
+ }
467
+ logger?.debug?.(`${TAG} Hybrid search: no results after merge`);
468
+ return { lines: [], timing };
469
+ }
470
+ function formatMemoryLine(m) {
471
+ // 1. Type tag + optional scene name
472
+ const tag = m.scene_name ? `${m.type}|${m.scene_name}` : m.type;
473
+ // 2. Content (core)
474
+ let line = `- [${tag}] ${m.content}`;
475
+ // 3. Time info — prefer activity_start/end range; fall back to timestamp as point-in-time
476
+ const start = formatTimestamp(m.activity_start_time);
477
+ const end = formatTimestamp(m.activity_end_time);
478
+ const point = formatTimestamp(m.timestamp);
479
+ if (start && end) {
480
+ // 段时间: both start and end
481
+ line += ` (活动时间: ${start} ~ ${end})`;
482
+ }
483
+ else if (start) {
484
+ // 段时间: only start
485
+ line += ` (活动时间: ${start}起)`;
486
+ }
487
+ else if (end) {
488
+ // 段时间: only end
489
+ line += ` (活动时间: 至${end})`;
490
+ }
491
+ else if (point) {
492
+ // 点时间: single timestamp
493
+ line += ` (活动时间: ${point})`;
494
+ }
495
+ // If all three are empty → no time info appended (graceful)
496
+ return line;
497
+ }
498
+ function applyRecallBudget(lines, recall, logger) {
499
+ const maxCharsPerMemory = normalizeBudgetLimit(recall.maxCharsPerMemory);
500
+ const maxTotalRecallChars = normalizeBudgetLimit(recall.maxTotalRecallChars);
501
+ if (!maxCharsPerMemory && !maxTotalRecallChars) {
502
+ return lines;
503
+ }
504
+ const budgeted = [];
505
+ let usedChars = 0;
506
+ let truncatedCount = 0;
507
+ let droppedCount = 0;
508
+ for (let i = 0; i < lines.length; i++) {
509
+ const line = lines[i];
510
+ const perMemoryBounded = maxCharsPerMemory
511
+ ? truncateRecallLine(line, maxCharsPerMemory)
512
+ : line;
513
+ let wasTruncated = perMemoryBounded !== line;
514
+ if (!maxTotalRecallChars) {
515
+ budgeted.push(perMemoryBounded);
516
+ if (wasTruncated)
517
+ truncatedCount++;
518
+ continue;
519
+ }
520
+ const separatorChars = budgeted.length > 0 ? RECALL_LINE_SEPARATOR.length : 0;
521
+ const remainingChars = maxTotalRecallChars - usedChars - separatorChars;
522
+ if (remainingChars <= 0) {
523
+ droppedCount += lines.length - i;
524
+ break;
525
+ }
526
+ if (perMemoryBounded.length > remainingChars) {
527
+ const canFit = remainingChars >= MIN_TRUNCATED_RECALL_LINE_CHARS;
528
+ if (canFit) {
529
+ const totalBounded = truncateRecallLine(perMemoryBounded, remainingChars);
530
+ budgeted.push(totalBounded);
531
+ usedChars += separatorChars + totalBounded.length;
532
+ wasTruncated ||= totalBounded !== perMemoryBounded;
533
+ if (wasTruncated)
534
+ truncatedCount++;
535
+ }
536
+ droppedCount += lines.length - i - (canFit ? 1 : 0);
537
+ break;
538
+ }
539
+ budgeted.push(perMemoryBounded);
540
+ usedChars += separatorChars + perMemoryBounded.length;
541
+ if (wasTruncated)
542
+ truncatedCount++;
543
+ }
544
+ if (truncatedCount > 0 || droppedCount > 0) {
545
+ logger?.debug?.(`${TAG} Recall budget applied: input=${lines.length}, output=${budgeted.length}, ` +
546
+ `truncated=${truncatedCount}, dropped=${droppedCount}, ` +
547
+ `maxCharsPerMemory=${recall.maxCharsPerMemory}, maxTotalRecallChars=${recall.maxTotalRecallChars}`);
548
+ }
549
+ return budgeted;
550
+ }
551
+ function normalizeBudgetLimit(value) {
552
+ if (value == null || !Number.isFinite(value) || value <= 0)
553
+ return undefined;
554
+ return Math.floor(value);
555
+ }
556
+ function truncateRecallLine(line, maxChars) {
557
+ // Count and slice by code point, not UTF-16 code unit, so a cut never lands
558
+ // between the halves of a surrogate pair (which would corrupt a non-BMP
559
+ // character to U+FFFD when the line is UTF-8 encoded for the request).
560
+ const cps = Array.from(line);
561
+ if (cps.length <= maxChars)
562
+ return line;
563
+ if (maxChars <= RECALL_TRUNCATION_SUFFIX.length) {
564
+ return cps.slice(0, maxChars).join("");
565
+ }
566
+ return `${cps.slice(0, maxChars - RECALL_TRUNCATION_SUFFIX.length).join("").trimEnd()}${RECALL_TRUNCATION_SUFFIX}`;
567
+ }
568
+ /**
569
+ * Format an ISO 8601 timestamp to a concise, timezone-aware string for display.
570
+ * Uses the configured timezone (via time module).
571
+ * - If the time part is 00:00:00 → show date only (e.g. "2025-03-01")
572
+ * - Otherwise → show full ISO 8601 with offset (e.g. "2025-03-01T14:30:00+08:00")
573
+ * - Returns undefined for empty/invalid inputs.
574
+ */
575
+ function formatTimestamp(ts) {
576
+ if (!ts)
577
+ return undefined;
578
+ const d = new Date(ts);
579
+ if (isNaN(d.getTime()))
580
+ return undefined;
581
+ // Check if time part is midnight UTC (date-only semantics)
582
+ const match = ts.match(/^(\d{4}-\d{2}-\d{2})(?:T(\d{2}:\d{2})(?::\d{2})?)?/);
583
+ if (match) {
584
+ const timePart = match[2];
585
+ if (!timePart || timePart === "00:00") {
586
+ return match[1]; // date-only, no timezone conversion needed
587
+ }
588
+ }
589
+ return formatForLLM(ts);
590
+ }
591
+ /**
592
+ * Build a FormatableMemory from a full MemoryRecord (keyword search path).
593
+ * Handles empty metadata, empty timestamps array gracefully.
594
+ */
595
+ function recordToFormatable(record) {
596
+ const meta = record.metadata;
597
+ return {
598
+ type: record.type,
599
+ content: record.content,
600
+ scene_name: record.scene_name || undefined,
601
+ activity_start_time: meta?.activity_start_time || undefined,
602
+ activity_end_time: meta?.activity_end_time || undefined,
603
+ timestamp: (record.timestamps && record.timestamps.length > 0) ? record.timestamps[0] : undefined,
604
+ };
605
+ }
606
+ /**
607
+ * Build a FormatableMemory from a VectorSearchResult (embedding search path).
608
+ * Handles empty/invalid metadata_json, empty timestamp_str gracefully.
609
+ */
610
+ function vectorResultToFormatable(r) {
611
+ let activityStart;
612
+ let activityEnd;
613
+ if (r.metadata_json && r.metadata_json !== "{}") {
614
+ try {
615
+ const meta = typeof r.metadata_json === "string" ? JSON.parse(r.metadata_json) : r.metadata_json;
616
+ activityStart = meta?.activity_start_time || undefined;
617
+ activityEnd = meta?.activity_end_time || undefined;
618
+ }
619
+ catch { /* ignore parse errors — treat as no metadata */ }
620
+ }
621
+ return {
622
+ type: r.type,
623
+ content: r.content,
624
+ scene_name: r.scene_name || undefined,
625
+ activity_start_time: activityStart,
626
+ activity_end_time: activityEnd,
627
+ timestamp: r.timestamp_str || undefined,
628
+ };
629
+ }
630
+ /**
631
+ * Build a FormatableMemory from an FtsSearchResult (FTS5 keyword search path).
632
+ * Handles empty/invalid metadata_json, empty timestamp_str gracefully.
633
+ */
634
+ function ftsResultToFormatable(r) {
635
+ let activityStart;
636
+ let activityEnd;
637
+ if (r.metadata_json && r.metadata_json !== "{}") {
638
+ try {
639
+ const meta = typeof r.metadata_json === "string" ? JSON.parse(r.metadata_json) : r.metadata_json;
640
+ activityStart = meta?.activity_start_time || undefined;
641
+ activityEnd = meta?.activity_end_time || undefined;
642
+ }
643
+ catch { /* ignore parse errors — treat as no metadata */ }
644
+ }
645
+ return {
646
+ type: r.type,
647
+ content: r.content,
648
+ scene_name: r.scene_name || undefined,
649
+ activity_start_time: activityStart,
650
+ activity_end_time: activityEnd,
651
+ timestamp: r.timestamp_str || undefined,
652
+ };
653
+ }