dsh-tdai-memory 0.2.13 → 0.2.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. package/LICENSE +21 -21
  2. package/THIRD-PARTY-NOTICES.md +20 -20
  3. package/client.js +8 -4
  4. package/cordis.patch.yml +8 -8
  5. package/index.js +414 -414
  6. package/package.json +1 -1
  7. package/recall-inject.js +86 -86
  8. package/vendor/tdai/adapters/index.js +15 -15
  9. package/vendor/tdai/adapters/openclaw/host-adapter.js +83 -83
  10. package/vendor/tdai/adapters/openclaw/index.js +5 -5
  11. package/vendor/tdai/adapters/openclaw/llm-runner.js +69 -69
  12. package/vendor/tdai/adapters/standalone/host-adapter.js +58 -58
  13. package/vendor/tdai/adapters/standalone/index.js +5 -5
  14. package/vendor/tdai/adapters/standalone/llm-runner.js +246 -246
  15. package/vendor/tdai/config.js +363 -363
  16. package/vendor/tdai/core/conversation/l0-recorder.js +420 -420
  17. package/vendor/tdai/core/hooks/auto-capture.js +236 -236
  18. package/vendor/tdai/core/hooks/auto-recall.js +653 -653
  19. package/vendor/tdai/core/index.js +8 -8
  20. package/vendor/tdai/core/persona/persona-generator.js +184 -184
  21. package/vendor/tdai/core/persona/persona-trigger.js +94 -94
  22. package/vendor/tdai/core/profile/profile-sync.js +200 -200
  23. package/vendor/tdai/core/prompts/l1-dedup.js +136 -136
  24. package/vendor/tdai/core/prompts/l1-extraction.js +132 -132
  25. package/vendor/tdai/core/prompts/persona-generation.js +160 -160
  26. package/vendor/tdai/core/prompts/scene-extraction.js +269 -269
  27. package/vendor/tdai/core/record/l1-dedup.js +318 -318
  28. package/vendor/tdai/core/record/l1-extractor.js +379 -379
  29. package/vendor/tdai/core/record/l1-reader.js +178 -178
  30. package/vendor/tdai/core/record/l1-writer.js +141 -141
  31. package/vendor/tdai/core/report/reporter.js +86 -86
  32. package/vendor/tdai/core/scene/filename-normalizer.js +161 -161
  33. package/vendor/tdai/core/scene/scene-extractor.js +410 -410
  34. package/vendor/tdai/core/scene/scene-format.js +53 -53
  35. package/vendor/tdai/core/scene/scene-index.js +82 -82
  36. package/vendor/tdai/core/scene/scene-navigation.js +60 -60
  37. package/vendor/tdai/core/seed/input.js +404 -404
  38. package/vendor/tdai/core/seed/seed-runtime.js +314 -314
  39. package/vendor/tdai/core/seed/types.js +9 -9
  40. package/vendor/tdai/core/store/bm25-client.js +117 -117
  41. package/vendor/tdai/core/store/bm25-local.js +68 -68
  42. package/vendor/tdai/core/store/embedding.js +632 -632
  43. package/vendor/tdai/core/store/factory.js +95 -95
  44. package/vendor/tdai/core/store/search-utils.js +54 -54
  45. package/vendor/tdai/core/store/sqlite.js +1822 -1822
  46. package/vendor/tdai/core/store/tcvdb-client.js +221 -221
  47. package/vendor/tdai/core/store/tcvdb.js +1047 -1047
  48. package/vendor/tdai/core/store/types.js +17 -17
  49. package/vendor/tdai/core/tdai-core.js +446 -446
  50. package/vendor/tdai/core/tools/conversation-search.js +199 -199
  51. package/vendor/tdai/core/tools/memory-search.js +207 -207
  52. package/vendor/tdai/core/types.js +12 -12
  53. package/vendor/tdai/utils/backup.js +189 -189
  54. package/vendor/tdai/utils/checkpoint.js +360 -360
  55. package/vendor/tdai/utils/clean-context-runner.js +472 -472
  56. package/vendor/tdai/utils/ensure-hook-policy.js +213 -213
  57. package/vendor/tdai/utils/env.js +13 -13
  58. package/vendor/tdai/utils/managed-timer.js +127 -127
  59. package/vendor/tdai/utils/manifest.js +91 -91
  60. package/vendor/tdai/utils/memory-cleaner.js +300 -300
  61. package/vendor/tdai/utils/no-think-fetch.js +109 -109
  62. package/vendor/tdai/utils/openclaw-state-dir.js +26 -26
  63. package/vendor/tdai/utils/pipeline-factory.js +517 -517
  64. package/vendor/tdai/utils/pipeline-manager.js +892 -892
  65. package/vendor/tdai/utils/sanitize.js +360 -360
  66. package/vendor/tdai/utils/serial-queue.js +103 -103
  67. package/vendor/tdai/utils/session-filter.js +88 -88
  68. package/vendor/tdai/utils/text-utils.js +29 -29
  69. package/vendor/tdai/utils/time.js +252 -252
@@ -1,236 +1,236 @@
1
- /**
2
- * auto-capture hook (v3): records conversation messages locally (L0),
3
- * then notifies the MemoryPipelineManager for L1/L2/L3 scheduling.
4
- *
5
- * Key design decisions:
6
- * - Always write L0 locally via l0-recorder.
7
- * - When VectorStore + EmbeddingService are available, also write L0 vector index.
8
- * - Notify MemoryPipelineManager for L1/L2/L3 trigger evaluation.
9
- * - L1 Runner reads from VectorStore DB (primary) or L0 JSONL files (fallback).
10
- * - Extraction is NOT triggered here. The pipeline manager decides when.
11
- */
12
- import crypto from "node:crypto";
13
- import { CheckpointManager } from "../../utils/checkpoint.js";
14
- import { recordConversation } from "../conversation/l0-recorder.js";
15
- const TAG = "[memory-tdai] [capture]";
16
- /**
17
- * Generate a unique L0 record ID for vector indexing.
18
- * Includes an index to distinguish multiple messages within the same round.
19
- */
20
- function generateL0RecordId(sessionKey, index) {
21
- return `l0_${sessionKey}_${Date.now()}_${index}_${crypto.randomBytes(3).toString("hex")}`;
22
- }
23
- export async function performAutoCapture(params) {
24
- const { messages, sessionKey, sessionId, cfg, pluginDataDir, logger, scheduler, originalUserText, originalUserMessageCount, pluginStartTimestamp, vectorStore, embeddingService, bgTaskRegistry, } = params;
25
- const tCaptureStart = performance.now();
26
- const checkpoint = new CheckpointManager(pluginDataDir, logger);
27
- // ============================
28
- // Step 1 + 2: L0 recording + checkpoint update (ATOMIC)
29
- // ============================
30
- // These steps are combined inside captureAtomically() to prevent the race
31
- // condition where two concurrent agent_end events both read the same stale
32
- // cursor and produce duplicate L0 records. The file lock is held for the
33
- // entire read-cursor → recordConversation → advance-cursor sequence.
34
- const tL0RecordStart = performance.now();
35
- let filteredMessages = [];
36
- try {
37
- await checkpoint.captureAtomically(sessionKey, pluginStartTimestamp, async (afterTimestamp) => {
38
- logger?.debug?.(`${TAG} L0 capture cursor (per-session, atomic): afterTimestamp=${afterTimestamp} session=${sessionKey}`);
39
- if (afterTimestamp === pluginStartTimestamp && pluginStartTimestamp && pluginStartTimestamp > 0) {
40
- logger?.debug?.(`${TAG} No per-session checkpoint cursor found for session=${sessionKey} — ` +
41
- `using pluginStartTimestamp as floor: ` +
42
- `${afterTimestamp} (${new Date(afterTimestamp).toISOString()})`);
43
- }
44
- filteredMessages = await recordConversation({
45
- sessionKey,
46
- sessionId,
47
- rawMessages: messages,
48
- baseDir: pluginDataDir,
49
- logger,
50
- originalUserText,
51
- afterTimestamp,
52
- originalUserMessageCount,
53
- });
54
- if (filteredMessages.length === 0) {
55
- return null; // Nothing captured — cursor stays unchanged
56
- }
57
- logger?.debug?.(`${TAG} L0 recorded: ${filteredMessages.length} messages for session ${sessionKey}`);
58
- const maxTs = Math.max(...filteredMessages.map((m) => m.timestamp));
59
- return { maxTimestamp: maxTs, messageCount: filteredMessages.length };
60
- });
61
- }
62
- catch (err) {
63
- logger?.error(`${TAG} L0 recording failed: ${err instanceof Error ? err.message : String(err)}`);
64
- }
65
- const tL0RecordEnd = performance.now();
66
- // ============================
67
- // Step 1.5: L0 vector indexing
68
- // ============================
69
- // Two paths depending on store capabilities:
70
- //
71
- // A) Store supports updateL0Embedding (sqlite):
72
- // - Write metadata + FTS immediately WITHOUT embedding (~ms)
73
- // - Fire-and-forget background task: embedBatch + updateL0Embedding
74
- // - PERF: avoids blocking agent_end with 2-3s embedding calls
75
- //
76
- // B) Store does NOT support updateL0Embedding (VDB / remote):
77
- // - Embed synchronously, then upsertL0 with embedding in one call
78
- // - VDB backends handle embedding server-side or need it upfront
79
- const tL0VecStart = performance.now();
80
- let l0VectorsWritten = 0;
81
- let l0EmbedTotalMs = 0;
82
- let l0UpsertTotalMs = 0;
83
- logger?.debug?.(`${TAG} [L0-vec-index] Check: filteredMessages=${filteredMessages.length}, ` +
84
- `vectorStore=${vectorStore ? "available" : "UNAVAILABLE"}, ` +
85
- `embeddingService=${embeddingService ? "available" : "UNAVAILABLE"}`);
86
- const supportsBgEmbed = vectorStore?.supportsDeferredEmbedding === true;
87
- if (filteredMessages.length > 0 && vectorStore) {
88
- const now = new Date().toISOString();
89
- const bgRecords = [];
90
- logger?.debug?.(`${TAG} [L0-vec-index] START indexing ${filteredMessages.length} message(s) for session ${sessionKey} ` +
91
- `(mode=${supportsBgEmbed ? "async-bg" : "sync"})`);
92
- for (let i = 0; i < filteredMessages.length; i++) {
93
- const msg = filteredMessages[i];
94
- try {
95
- const l0Record = {
96
- id: generateL0RecordId(sessionKey, i),
97
- sessionKey,
98
- sessionId: sessionId || "",
99
- role: msg.role,
100
- messageText: msg.content,
101
- recordedAt: now,
102
- timestamp: msg.timestamp,
103
- };
104
- let embedding;
105
- if (!supportsBgEmbed && embeddingService) {
106
- // Path B (VDB): embed synchronously — needed for upsertL0
107
- // Skip local embed when using server-side embedding (NoopEmbeddingService, dims=0)
108
- if (embeddingService.getDimensions() === 0) {
109
- logger?.debug?.(`${TAG} [L0-vec-index] Server-side embedding (dims=0), skipping local embed for message ${i}`);
110
- }
111
- else {
112
- const tEmbedStart = performance.now();
113
- try {
114
- embedding = await embeddingService.embed(msg.content);
115
- l0EmbedTotalMs += performance.now() - tEmbedStart;
116
- logger?.debug?.(`${TAG} [L0-vec-index] Embedding OK: dims=${embedding.length}, ` +
117
- `norm=${Math.sqrt(Array.from(embedding).reduce((s, v) => s + v * v, 0)).toFixed(4)}`);
118
- }
119
- catch (embedErr) {
120
- l0EmbedTotalMs += performance.now() - tEmbedStart;
121
- logger?.warn(`${TAG} [L0-vec-index] Embedding FAILED for message ${i}, ` +
122
- `will write metadata only: ${embedErr instanceof Error ? embedErr.message : String(embedErr)}`);
123
- }
124
- }
125
- }
126
- // Path A (sqlite): pass undefined embedding — metadata + FTS only
127
- // Path B (VDB): pass embedding (may be undefined on failure)
128
- const tUpsertStart = performance.now();
129
- const upsertOk = await vectorStore.upsertL0(l0Record, supportsBgEmbed ? undefined : embedding);
130
- l0UpsertTotalMs += performance.now() - tUpsertStart;
131
- if (upsertOk) {
132
- l0VectorsWritten++;
133
- if (supportsBgEmbed) {
134
- bgRecords.push({ recordId: l0Record.id, content: msg.content });
135
- }
136
- }
137
- else {
138
- logger?.warn(`${TAG} [L0-vec-index] upsertL0 returned false for message ${i}`);
139
- }
140
- }
141
- catch (err) {
142
- logger?.warn?.(`${TAG} [L0-vec-index] FAILED for message ${i} (non-blocking): ${err instanceof Error ? err.message : String(err)}`);
143
- }
144
- }
145
- const modeLabel = supportsBgEmbed ? "metadata-only, embed=background" : `embed=${l0EmbedTotalMs.toFixed(0)}ms, upsert=${l0UpsertTotalMs.toFixed(0)}ms`;
146
- logger?.debug?.(`${TAG} [L0-vec-index] DONE: ${l0VectorsWritten}/${filteredMessages.length} records written (${modeLabel})`);
147
- // Path A only: fire-and-forget background embedding for sqlite stores
148
- if (supportsBgEmbed && bgRecords.length > 0 && embeddingService) {
149
- const bgVectorStore = vectorStore;
150
- const bgEmbeddingService = embeddingService;
151
- const bgSnapshot = [...bgRecords];
152
- const bgLogger = logger;
153
- // Do NOT await — runs in background after response is sent.
154
- //
155
- // Register the task in bgTaskRegistry (if provided) so TdaiCore.destroy()
156
- // can await it before closing vectorStore / embeddingService. The
157
- // ``.finally`` clean-up ensures the entry is removed on both success
158
- // and failure; without that the set would leak and eventually block
159
- // shutdown indefinitely.
160
- const bgPromise = (async () => {
161
- const tBgStart = performance.now();
162
- try {
163
- const texts = bgSnapshot.map((r) => r.content);
164
- const embeddings = await bgEmbeddingService.embedBatch(texts);
165
- let bgUpdated = 0;
166
- for (let i = 0; i < bgSnapshot.length; i++) {
167
- try {
168
- const ok = await bgVectorStore.updateL0Embedding(bgSnapshot[i].recordId, embeddings[i]);
169
- if (ok)
170
- bgUpdated++;
171
- }
172
- catch (err) {
173
- bgLogger?.warn?.(`${TAG} [L0-vec-index-bg] Failed to update embedding for ${bgSnapshot[i].recordId}: ` +
174
- `${err instanceof Error ? err.message : String(err)}`);
175
- }
176
- }
177
- const bgMs = performance.now() - tBgStart;
178
- bgLogger?.debug?.(`${TAG} [L0-vec-index-bg] Background embedding complete: ${bgUpdated}/${bgSnapshot.length} vectors updated (${bgMs.toFixed(0)}ms)`);
179
- }
180
- catch (err) {
181
- const bgMs = performance.now() - tBgStart;
182
- bgLogger?.warn?.(`${TAG} [L0-vec-index-bg] Background embedding failed (${bgMs.toFixed(0)}ms, non-fatal): ` +
183
- `${err instanceof Error ? err.message : String(err)}`);
184
- }
185
- })();
186
- if (bgTaskRegistry) {
187
- bgTaskRegistry.add(bgPromise);
188
- void bgPromise.finally(() => {
189
- bgTaskRegistry.delete(bgPromise);
190
- });
191
- }
192
- }
193
- }
194
- else if (filteredMessages.length > 0) {
195
- logger?.warn(`${TAG} [L0-vec-index] SKIPPED: vectorStore not available`);
196
- }
197
- const tL0VecEnd = performance.now();
198
- // ============================
199
- // Step 3: Notify scheduler of this conversation round
200
- // ============================
201
- const tNotifyStart = performance.now();
202
- // Pass empty array: L1 Runner reads from VectorStore DB (or L0 JSONL fallback), not from in-memory buffers.
203
- if (scheduler) {
204
- await scheduler.notifyConversation(sessionKey, []);
205
- logger?.debug?.(`${TAG} Scheduler notified of conversation round (sessionKey=${sessionKey})`);
206
- const totalMs = performance.now() - tCaptureStart;
207
- const vecDetail = supportsBgEmbed
208
- ? `metadata-only, embed=background, msgs=${filteredMessages.length}`
209
- : `embed=${l0EmbedTotalMs.toFixed(0)}ms, upsert=${l0UpsertTotalMs.toFixed(0)}ms, msgs=${filteredMessages.length}`;
210
- logger?.info(`${TAG} ⏱ Capture timing: total=${totalMs.toFixed(0)}ms, ` +
211
- `l0Record+checkpoint=${(tL0RecordEnd - tL0RecordStart).toFixed(0)}ms, ` +
212
- `l0VecIndex=${(tL0VecEnd - tL0VecStart).toFixed(0)}ms (${vecDetail}), ` +
213
- `notify=${(performance.now() - tNotifyStart).toFixed(0)}ms`);
214
- return {
215
- schedulerNotified: true,
216
- l0RecordedCount: filteredMessages.length,
217
- l0VectorsWritten,
218
- filteredMessages,
219
- };
220
- }
221
- const totalMs = performance.now() - tCaptureStart;
222
- const vecDetail = supportsBgEmbed
223
- ? `metadata-only, embed=background, msgs=${filteredMessages.length}`
224
- : `embed=${l0EmbedTotalMs.toFixed(0)}ms, upsert=${l0UpsertTotalMs.toFixed(0)}ms, msgs=${filteredMessages.length}`;
225
- logger?.info(`${TAG} ⏱ Capture timing: total=${totalMs.toFixed(0)}ms, ` +
226
- `l0Record+checkpoint=${(tL0RecordEnd - tL0RecordStart).toFixed(0)}ms, ` +
227
- `l0VecIndex=${(tL0VecEnd - tL0VecStart).toFixed(0)}ms (${vecDetail}), ` +
228
- `notify=${(performance.now() - tNotifyStart).toFixed(0)}ms`);
229
- logger?.debug?.(`${TAG} No scheduler provided, skipping notification`);
230
- return {
231
- schedulerNotified: false,
232
- l0RecordedCount: filteredMessages.length,
233
- l0VectorsWritten,
234
- filteredMessages,
235
- };
236
- }
1
+ /**
2
+ * auto-capture hook (v3): records conversation messages locally (L0),
3
+ * then notifies the MemoryPipelineManager for L1/L2/L3 scheduling.
4
+ *
5
+ * Key design decisions:
6
+ * - Always write L0 locally via l0-recorder.
7
+ * - When VectorStore + EmbeddingService are available, also write L0 vector index.
8
+ * - Notify MemoryPipelineManager for L1/L2/L3 trigger evaluation.
9
+ * - L1 Runner reads from VectorStore DB (primary) or L0 JSONL files (fallback).
10
+ * - Extraction is NOT triggered here. The pipeline manager decides when.
11
+ */
12
+ import crypto from "node:crypto";
13
+ import { CheckpointManager } from "../../utils/checkpoint.js";
14
+ import { recordConversation } from "../conversation/l0-recorder.js";
15
+ const TAG = "[memory-tdai] [capture]";
16
+ /**
17
+ * Generate a unique L0 record ID for vector indexing.
18
+ * Includes an index to distinguish multiple messages within the same round.
19
+ */
20
+ function generateL0RecordId(sessionKey, index) {
21
+ return `l0_${sessionKey}_${Date.now()}_${index}_${crypto.randomBytes(3).toString("hex")}`;
22
+ }
23
+ export async function performAutoCapture(params) {
24
+ const { messages, sessionKey, sessionId, cfg, pluginDataDir, logger, scheduler, originalUserText, originalUserMessageCount, pluginStartTimestamp, vectorStore, embeddingService, bgTaskRegistry, } = params;
25
+ const tCaptureStart = performance.now();
26
+ const checkpoint = new CheckpointManager(pluginDataDir, logger);
27
+ // ============================
28
+ // Step 1 + 2: L0 recording + checkpoint update (ATOMIC)
29
+ // ============================
30
+ // These steps are combined inside captureAtomically() to prevent the race
31
+ // condition where two concurrent agent_end events both read the same stale
32
+ // cursor and produce duplicate L0 records. The file lock is held for the
33
+ // entire read-cursor → recordConversation → advance-cursor sequence.
34
+ const tL0RecordStart = performance.now();
35
+ let filteredMessages = [];
36
+ try {
37
+ await checkpoint.captureAtomically(sessionKey, pluginStartTimestamp, async (afterTimestamp) => {
38
+ logger?.debug?.(`${TAG} L0 capture cursor (per-session, atomic): afterTimestamp=${afterTimestamp} session=${sessionKey}`);
39
+ if (afterTimestamp === pluginStartTimestamp && pluginStartTimestamp && pluginStartTimestamp > 0) {
40
+ logger?.debug?.(`${TAG} No per-session checkpoint cursor found for session=${sessionKey} — ` +
41
+ `using pluginStartTimestamp as floor: ` +
42
+ `${afterTimestamp} (${new Date(afterTimestamp).toISOString()})`);
43
+ }
44
+ filteredMessages = await recordConversation({
45
+ sessionKey,
46
+ sessionId,
47
+ rawMessages: messages,
48
+ baseDir: pluginDataDir,
49
+ logger,
50
+ originalUserText,
51
+ afterTimestamp,
52
+ originalUserMessageCount,
53
+ });
54
+ if (filteredMessages.length === 0) {
55
+ return null; // Nothing captured — cursor stays unchanged
56
+ }
57
+ logger?.debug?.(`${TAG} L0 recorded: ${filteredMessages.length} messages for session ${sessionKey}`);
58
+ const maxTs = Math.max(...filteredMessages.map((m) => m.timestamp));
59
+ return { maxTimestamp: maxTs, messageCount: filteredMessages.length };
60
+ });
61
+ }
62
+ catch (err) {
63
+ logger?.error(`${TAG} L0 recording failed: ${err instanceof Error ? err.message : String(err)}`);
64
+ }
65
+ const tL0RecordEnd = performance.now();
66
+ // ============================
67
+ // Step 1.5: L0 vector indexing
68
+ // ============================
69
+ // Two paths depending on store capabilities:
70
+ //
71
+ // A) Store supports updateL0Embedding (sqlite):
72
+ // - Write metadata + FTS immediately WITHOUT embedding (~ms)
73
+ // - Fire-and-forget background task: embedBatch + updateL0Embedding
74
+ // - PERF: avoids blocking agent_end with 2-3s embedding calls
75
+ //
76
+ // B) Store does NOT support updateL0Embedding (VDB / remote):
77
+ // - Embed synchronously, then upsertL0 with embedding in one call
78
+ // - VDB backends handle embedding server-side or need it upfront
79
+ const tL0VecStart = performance.now();
80
+ let l0VectorsWritten = 0;
81
+ let l0EmbedTotalMs = 0;
82
+ let l0UpsertTotalMs = 0;
83
+ logger?.debug?.(`${TAG} [L0-vec-index] Check: filteredMessages=${filteredMessages.length}, ` +
84
+ `vectorStore=${vectorStore ? "available" : "UNAVAILABLE"}, ` +
85
+ `embeddingService=${embeddingService ? "available" : "UNAVAILABLE"}`);
86
+ const supportsBgEmbed = vectorStore?.supportsDeferredEmbedding === true;
87
+ if (filteredMessages.length > 0 && vectorStore) {
88
+ const now = new Date().toISOString();
89
+ const bgRecords = [];
90
+ logger?.debug?.(`${TAG} [L0-vec-index] START indexing ${filteredMessages.length} message(s) for session ${sessionKey} ` +
91
+ `(mode=${supportsBgEmbed ? "async-bg" : "sync"})`);
92
+ for (let i = 0; i < filteredMessages.length; i++) {
93
+ const msg = filteredMessages[i];
94
+ try {
95
+ const l0Record = {
96
+ id: generateL0RecordId(sessionKey, i),
97
+ sessionKey,
98
+ sessionId: sessionId || "",
99
+ role: msg.role,
100
+ messageText: msg.content,
101
+ recordedAt: now,
102
+ timestamp: msg.timestamp,
103
+ };
104
+ let embedding;
105
+ if (!supportsBgEmbed && embeddingService) {
106
+ // Path B (VDB): embed synchronously — needed for upsertL0
107
+ // Skip local embed when using server-side embedding (NoopEmbeddingService, dims=0)
108
+ if (embeddingService.getDimensions() === 0) {
109
+ logger?.debug?.(`${TAG} [L0-vec-index] Server-side embedding (dims=0), skipping local embed for message ${i}`);
110
+ }
111
+ else {
112
+ const tEmbedStart = performance.now();
113
+ try {
114
+ embedding = await embeddingService.embed(msg.content);
115
+ l0EmbedTotalMs += performance.now() - tEmbedStart;
116
+ logger?.debug?.(`${TAG} [L0-vec-index] Embedding OK: dims=${embedding.length}, ` +
117
+ `norm=${Math.sqrt(Array.from(embedding).reduce((s, v) => s + v * v, 0)).toFixed(4)}`);
118
+ }
119
+ catch (embedErr) {
120
+ l0EmbedTotalMs += performance.now() - tEmbedStart;
121
+ logger?.warn(`${TAG} [L0-vec-index] Embedding FAILED for message ${i}, ` +
122
+ `will write metadata only: ${embedErr instanceof Error ? embedErr.message : String(embedErr)}`);
123
+ }
124
+ }
125
+ }
126
+ // Path A (sqlite): pass undefined embedding — metadata + FTS only
127
+ // Path B (VDB): pass embedding (may be undefined on failure)
128
+ const tUpsertStart = performance.now();
129
+ const upsertOk = await vectorStore.upsertL0(l0Record, supportsBgEmbed ? undefined : embedding);
130
+ l0UpsertTotalMs += performance.now() - tUpsertStart;
131
+ if (upsertOk) {
132
+ l0VectorsWritten++;
133
+ if (supportsBgEmbed) {
134
+ bgRecords.push({ recordId: l0Record.id, content: msg.content });
135
+ }
136
+ }
137
+ else {
138
+ logger?.warn(`${TAG} [L0-vec-index] upsertL0 returned false for message ${i}`);
139
+ }
140
+ }
141
+ catch (err) {
142
+ logger?.warn?.(`${TAG} [L0-vec-index] FAILED for message ${i} (non-blocking): ${err instanceof Error ? err.message : String(err)}`);
143
+ }
144
+ }
145
+ const modeLabel = supportsBgEmbed ? "metadata-only, embed=background" : `embed=${l0EmbedTotalMs.toFixed(0)}ms, upsert=${l0UpsertTotalMs.toFixed(0)}ms`;
146
+ logger?.debug?.(`${TAG} [L0-vec-index] DONE: ${l0VectorsWritten}/${filteredMessages.length} records written (${modeLabel})`);
147
+ // Path A only: fire-and-forget background embedding for sqlite stores
148
+ if (supportsBgEmbed && bgRecords.length > 0 && embeddingService) {
149
+ const bgVectorStore = vectorStore;
150
+ const bgEmbeddingService = embeddingService;
151
+ const bgSnapshot = [...bgRecords];
152
+ const bgLogger = logger;
153
+ // Do NOT await — runs in background after response is sent.
154
+ //
155
+ // Register the task in bgTaskRegistry (if provided) so TdaiCore.destroy()
156
+ // can await it before closing vectorStore / embeddingService. The
157
+ // ``.finally`` clean-up ensures the entry is removed on both success
158
+ // and failure; without that the set would leak and eventually block
159
+ // shutdown indefinitely.
160
+ const bgPromise = (async () => {
161
+ const tBgStart = performance.now();
162
+ try {
163
+ const texts = bgSnapshot.map((r) => r.content);
164
+ const embeddings = await bgEmbeddingService.embedBatch(texts);
165
+ let bgUpdated = 0;
166
+ for (let i = 0; i < bgSnapshot.length; i++) {
167
+ try {
168
+ const ok = await bgVectorStore.updateL0Embedding(bgSnapshot[i].recordId, embeddings[i]);
169
+ if (ok)
170
+ bgUpdated++;
171
+ }
172
+ catch (err) {
173
+ bgLogger?.warn?.(`${TAG} [L0-vec-index-bg] Failed to update embedding for ${bgSnapshot[i].recordId}: ` +
174
+ `${err instanceof Error ? err.message : String(err)}`);
175
+ }
176
+ }
177
+ const bgMs = performance.now() - tBgStart;
178
+ bgLogger?.debug?.(`${TAG} [L0-vec-index-bg] Background embedding complete: ${bgUpdated}/${bgSnapshot.length} vectors updated (${bgMs.toFixed(0)}ms)`);
179
+ }
180
+ catch (err) {
181
+ const bgMs = performance.now() - tBgStart;
182
+ bgLogger?.warn?.(`${TAG} [L0-vec-index-bg] Background embedding failed (${bgMs.toFixed(0)}ms, non-fatal): ` +
183
+ `${err instanceof Error ? err.message : String(err)}`);
184
+ }
185
+ })();
186
+ if (bgTaskRegistry) {
187
+ bgTaskRegistry.add(bgPromise);
188
+ void bgPromise.finally(() => {
189
+ bgTaskRegistry.delete(bgPromise);
190
+ });
191
+ }
192
+ }
193
+ }
194
+ else if (filteredMessages.length > 0) {
195
+ logger?.warn(`${TAG} [L0-vec-index] SKIPPED: vectorStore not available`);
196
+ }
197
+ const tL0VecEnd = performance.now();
198
+ // ============================
199
+ // Step 3: Notify scheduler of this conversation round
200
+ // ============================
201
+ const tNotifyStart = performance.now();
202
+ // Pass empty array: L1 Runner reads from VectorStore DB (or L0 JSONL fallback), not from in-memory buffers.
203
+ if (scheduler) {
204
+ await scheduler.notifyConversation(sessionKey, []);
205
+ logger?.debug?.(`${TAG} Scheduler notified of conversation round (sessionKey=${sessionKey})`);
206
+ const totalMs = performance.now() - tCaptureStart;
207
+ const vecDetail = supportsBgEmbed
208
+ ? `metadata-only, embed=background, msgs=${filteredMessages.length}`
209
+ : `embed=${l0EmbedTotalMs.toFixed(0)}ms, upsert=${l0UpsertTotalMs.toFixed(0)}ms, msgs=${filteredMessages.length}`;
210
+ logger?.info(`${TAG} ⏱ Capture timing: total=${totalMs.toFixed(0)}ms, ` +
211
+ `l0Record+checkpoint=${(tL0RecordEnd - tL0RecordStart).toFixed(0)}ms, ` +
212
+ `l0VecIndex=${(tL0VecEnd - tL0VecStart).toFixed(0)}ms (${vecDetail}), ` +
213
+ `notify=${(performance.now() - tNotifyStart).toFixed(0)}ms`);
214
+ return {
215
+ schedulerNotified: true,
216
+ l0RecordedCount: filteredMessages.length,
217
+ l0VectorsWritten,
218
+ filteredMessages,
219
+ };
220
+ }
221
+ const totalMs = performance.now() - tCaptureStart;
222
+ const vecDetail = supportsBgEmbed
223
+ ? `metadata-only, embed=background, msgs=${filteredMessages.length}`
224
+ : `embed=${l0EmbedTotalMs.toFixed(0)}ms, upsert=${l0UpsertTotalMs.toFixed(0)}ms, msgs=${filteredMessages.length}`;
225
+ logger?.info(`${TAG} ⏱ Capture timing: total=${totalMs.toFixed(0)}ms, ` +
226
+ `l0Record+checkpoint=${(tL0RecordEnd - tL0RecordStart).toFixed(0)}ms, ` +
227
+ `l0VecIndex=${(tL0VecEnd - tL0VecStart).toFixed(0)}ms (${vecDetail}), ` +
228
+ `notify=${(performance.now() - tNotifyStart).toFixed(0)}ms`);
229
+ logger?.debug?.(`${TAG} No scheduler provided, skipping notification`);
230
+ return {
231
+ schedulerNotified: false,
232
+ l0RecordedCount: filteredMessages.length,
233
+ l0VectorsWritten,
234
+ filteredMessages,
235
+ };
236
+ }