dsh-tdai-memory 0.4.2 → 0.4.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. package/LICENSE +21 -21
  2. package/README.md +8 -0
  3. package/README.zh.md +8 -0
  4. package/THIRD-PARTY-NOTICES.md +20 -20
  5. package/cordis.patch.yml +8 -8
  6. package/package.json +103 -100
  7. package/vendor/tdai/adapters/index.js +15 -15
  8. package/vendor/tdai/adapters/openclaw/host-adapter.js +83 -83
  9. package/vendor/tdai/adapters/openclaw/index.js +5 -5
  10. package/vendor/tdai/adapters/openclaw/llm-runner.js +69 -69
  11. package/vendor/tdai/adapters/standalone/host-adapter.js +58 -58
  12. package/vendor/tdai/adapters/standalone/index.js +5 -5
  13. package/vendor/tdai/core/conversation/l0-recorder.js +420 -420
  14. package/vendor/tdai/core/hooks/auto-capture.js +236 -236
  15. package/vendor/tdai/core/hooks/auto-recall.js +640 -640
  16. package/vendor/tdai/core/index.js +8 -8
  17. package/vendor/tdai/core/persona/persona-generator.js +184 -184
  18. package/vendor/tdai/core/persona/persona-trigger.js +94 -94
  19. package/vendor/tdai/core/profile/profile-sync.js +200 -200
  20. package/vendor/tdai/core/prompts/l1-dedup.js +72 -72
  21. package/vendor/tdai/core/prompts/l1-extraction.js +34 -34
  22. package/vendor/tdai/core/prompts/persona-generation.js +35 -35
  23. package/vendor/tdai/core/prompts/scene-extraction.js +45 -45
  24. package/vendor/tdai/core/record/l1-dedup.js +318 -318
  25. package/vendor/tdai/core/record/l1-extractor.js +379 -379
  26. package/vendor/tdai/core/record/l1-reader.js +178 -178
  27. package/vendor/tdai/core/record/l1-writer.js +141 -141
  28. package/vendor/tdai/core/report/reporter.js +86 -86
  29. package/vendor/tdai/core/scene/filename-normalizer.js +161 -161
  30. package/vendor/tdai/core/scene/scene-extractor.js +410 -410
  31. package/vendor/tdai/core/scene/scene-format.js +53 -53
  32. package/vendor/tdai/core/scene/scene-index.js +82 -82
  33. package/vendor/tdai/core/scene/scene-navigation.js +57 -57
  34. package/vendor/tdai/core/seed/input.js +404 -404
  35. package/vendor/tdai/core/seed/types.js +9 -9
  36. package/vendor/tdai/core/store/bm25-client.js +117 -117
  37. package/vendor/tdai/core/store/bm25-local.js +68 -68
  38. package/vendor/tdai/core/store/embedding.js +632 -632
  39. package/vendor/tdai/core/store/factory.js +95 -95
  40. package/vendor/tdai/core/store/search-utils.js +54 -54
  41. package/vendor/tdai/core/store/sqlite.js +1632 -1632
  42. package/vendor/tdai/core/store/tcvdb-client.js +221 -221
  43. package/vendor/tdai/core/store/tcvdb.js +1047 -1047
  44. package/vendor/tdai/core/store/types.js +17 -17
  45. package/vendor/tdai/core/tools/conversation-search.js +199 -199
  46. package/vendor/tdai/core/tools/memory-search.js +207 -207
  47. package/vendor/tdai/core/types.js +12 -12
  48. package/vendor/tdai/utils/backup.js +189 -189
  49. package/vendor/tdai/utils/checkpoint.js +360 -360
  50. package/vendor/tdai/utils/clean-context-runner.js +472 -472
  51. package/vendor/tdai/utils/ensure-hook-policy.js +213 -213
  52. package/vendor/tdai/utils/env.js +13 -13
  53. package/vendor/tdai/utils/managed-timer.js +127 -127
  54. package/vendor/tdai/utils/manifest.js +91 -91
  55. package/vendor/tdai/utils/memory-cleaner.js +300 -300
  56. package/vendor/tdai/utils/no-think-fetch.js +109 -109
  57. package/vendor/tdai/utils/openclaw-state-dir.js +26 -26
  58. package/vendor/tdai/utils/pipeline-factory.js +517 -517
  59. package/vendor/tdai/utils/pipeline-manager.js +892 -892
  60. package/vendor/tdai/utils/sanitize.js +360 -360
  61. package/vendor/tdai/utils/serial-queue.js +103 -103
  62. package/vendor/tdai/utils/session-filter.js +88 -88
  63. package/vendor/tdai/utils/text-utils.js +29 -29
  64. package/vendor/tdai/utils/time.js +252 -252
@@ -1,374 +1,374 @@
1
- /**
2
- * VectorStore: SQLite-based vector storage using sqlite-vec extension.
3
- *
4
- * Manages two layers of vector-indexed data in a single SQLite database:
5
- *
6
- * **L1 (structured memories):**
7
- * 1. `l1_records` — relational metadata table (content, type, priority, scene, timestamps)
8
- * 2. `l1_vec` — vec0 virtual table for cosine similarity search
9
- *
10
- * **L0 (raw conversations):**
11
- * 3. `l0_conversations` — relational metadata table (session_key, role, message text, timestamps)
12
- * 4. `l0_vec` — vec0 virtual table for cosine similarity search on individual messages
13
- *
14
- * Dependencies: Node.js built-in `node:sqlite` (Node 22+) + `sqlite-vec` (from root workspace).
15
- *
16
- * Design:
17
- * - All operations are synchronous (DatabaseSync API).
18
- * - Writes use manual BEGIN/COMMIT transactions for atomicity (metadata + vector).
19
- * - vec0 virtual table does NOT support ON CONFLICT, so upsert = delete + insert.
20
- * - Thread-safe via WAL mode.
21
- */
22
- import { createRequire } from "node:module";
23
- const TAG = "[memory-tdai][sqlite]";
24
- // Use createRequire to load the experimental node:sqlite module
25
- const require = createRequire(import.meta.url);
26
- function requireNodeSqlite() {
27
- return require("node:sqlite");
28
- }
29
- let _jieba; // undefined = not yet tried
30
- function getJieba() {
31
- if (_jieba !== undefined)
32
- return _jieba;
33
- try {
34
- // eslint-disable-next-line @typescript-eslint/no-require-imports
35
- const { Jieba } = require("@node-rs/jieba");
36
- // eslint-disable-next-line @typescript-eslint/no-require-imports
37
- const { dict } = require("@node-rs/jieba/dict");
38
- _jieba = Jieba.withDict(dict);
39
- }
40
- catch {
41
- _jieba = null; // mark as unavailable — won't retry
42
- }
43
- return _jieba;
44
- }
45
- /**
46
- * Common Chinese stop-words that add noise to FTS5 queries.
47
- * Kept small on purpose — only high-frequency function words.
48
- */
49
- const ZH_STOP_WORDS = new Set([
50
- "的", "了", "在", "是", "我", "有", "和", "就", "不", "人", "都", "一",
51
- "一个", "上", "也", "很", "到", "说", "要", "去", "你", "会", "着",
52
- "没有", "看", "好", "自己", "这", "他", "她", "它", "们", "那",
53
- "吗", "吧", "呢", "啊", "呀", "哦", "嗯",
54
- ]);
55
- /**
56
- * Build an FTS5 MATCH query from raw text.
57
- *
58
- * When `@node-rs/jieba` is available, uses jieba's search-engine mode
59
- * (`cutForSearch`) for accurate Chinese word segmentation, producing
60
- * much better recall than the previous regex-only approach.
61
- *
62
- * Falls back to Unicode-regex splitting (`/[\p{L}\p{N}_]+/gu`) if
63
- * jieba is not installed.
64
- *
65
- * Tokens are OR-joined as quoted FTS5 phrase terms so that a document
66
- * matching *any* token is returned. BM25 naturally ranks documents that
67
- * match more tokens higher, so precision is preserved while recall is
68
- * significantly improved — especially for longer queries and when running
69
- * in FTS-only fallback mode (no embedding available).
70
- *
71
- * Example (with jieba):
72
- * "用户喜欢编程和TypeScript" → '"用户" OR "喜欢" OR "编程" OR "TypeScript"'
73
- * Example (fallback):
74
- * "旅行计划 API" → '"旅行计划" OR "API"'
75
- */
76
- export function buildFtsQuery(raw) {
77
- const jieba = getJieba();
78
- let tokens;
79
- if (jieba) {
80
- // jieba cutForSearch: splits long words further for better recall
81
- // e.g. "北京烤鸭" → ["北京", "烤鸭", "北京烤鸭"]
82
- tokens = jieba
83
- .cutForSearch(raw, true)
84
- .map((t) => t.trim())
85
- .filter((t) => {
86
- if (!t)
87
- return false;
88
- // Remove pure whitespace / punctuation tokens
89
- if (!/[\p{L}\p{N}]/u.test(t))
90
- return false;
91
- // Remove common Chinese stop-words to reduce noise
92
- if (ZH_STOP_WORDS.has(t))
93
- return false;
94
- return true;
95
- });
96
- // Deduplicate (cutForSearch may produce duplicates for sub-words)
97
- tokens = [...new Set(tokens)];
98
- }
99
- else {
100
- // Fallback: simple Unicode regex split
101
- tokens =
102
- raw
103
- .match(/[\p{L}\p{N}_]+/gu)
104
- ?.map((t) => t.trim())
105
- .filter(Boolean) ?? [];
106
- }
107
- if (tokens.length === 0)
108
- return null;
109
- const quoted = tokens.map((t) => `"${t.replaceAll('"', "")}"`);
110
- return quoted.join(" OR ");
111
- }
112
- /**
113
- * Tokenize text for FTS5 indexing (write-side).
114
- *
115
- * Uses jieba `cutForSearch()` (search-engine mode) to segment Chinese text,
116
- * then joins tokens with spaces. The resulting string is stored in the FTS5
117
- * `content` column so that `unicode61` tokenizer can split it into meaningful
118
- * words — including both full words and their sub-words.
119
- *
120
- * Using `cutForSearch` (instead of `cut`) ensures that the index contains
121
- * the same sub-word tokens that `buildFtsQuery()` produces on the query side.
122
- * For example, "人工智能" is indexed as "人工 智能 人工智能", so queries for
123
- * either the full term or sub-words will match.
124
- *
125
- * Falls back to the original text if jieba is unavailable.
126
- *
127
- * Example (with jieba):
128
- * "用户五月去日本旅行" → "用户 五月 去 日本 旅行"
129
- * "人工智能的分支" → "人工 智能 人工智能 的 分支"
130
- * Example (fallback):
131
- * "用户五月去日本旅行" → "用户五月去日本旅行" (unchanged)
132
- */
133
- export function tokenizeForFts(raw) {
134
- const jieba = getJieba();
135
- if (!jieba)
136
- return raw;
137
- // Use `cutForSearch` (search-engine mode) for indexing — it produces both
138
- // full words AND their sub-word components. This ensures that query-side
139
- // tokens (also produced by `cutForSearch` in `buildFtsQuery`) will always
140
- // find a match in the index.
141
- const tokens = jieba.cutForSearch(raw, true);
142
- // Join with spaces so `unicode61` tokenizer can split them.
143
- // Punctuation tokens are kept — unicode61 treats them as separators anyway.
144
- return tokens.join(" ");
145
- }
146
- /**
147
- * Reset jieba state so next call to `buildFtsQuery` re-initialises.
148
- * Exported for testing only.
149
- * @internal
150
- */
151
- export function _resetJiebaForTest() {
152
- _jieba = undefined;
153
- }
154
- /**
155
- * Override jieba instance (or set to `null` to force fallback).
156
- * Exported for testing only.
157
- * @internal
158
- */
159
- export function _setJiebaForTest(instance) {
160
- _jieba = instance;
161
- }
162
- /**
163
- * Convert a BM25 rank (negative = more relevant) to a 0–1 score.
164
- * Mirrors the formula in openclaw core `hybrid.ts`.
165
- */
166
- export function bm25RankToScore(rank) {
167
- if (!Number.isFinite(rank))
168
- return 1 / (1 + 999);
169
- if (rank < 0) {
170
- const relevance = -rank;
171
- return relevance / (1 + relevance);
172
- }
173
- return 1 / (1 + rank);
174
- }
175
- // ============================
176
- // VectorStore class
177
- // ============================
178
- export class VectorStore {
179
- db;
180
- dimensions;
181
- logger;
182
- /** @see IMemoryStore.supportsDeferredEmbedding */
183
- supportsDeferredEmbedding = true;
184
- /**
185
- * When `true`, the store is in a degraded state (e.g. sqlite-vec failed to
186
- * load, or init() encountered an unrecoverable error). All public methods
187
- * become safe no-ops so the plugin never blocks the main OpenClaw flow.
188
- */
189
- degraded = false;
190
- /** Tracks whether close() has been called to prevent double-close errors. */
191
- closed = false;
192
- /**
193
- * `true` when vec0 virtual tables (l1_vec / l0_vec) have been created and
194
- * their prepared statements are ready. When `dimensions === 0` (i.e.
195
- * provider="none"), vec0 tables are deferred and this stays `false`.
196
- */
197
- vecTablesReady = false;
198
- // Prepared statements — L1 (initialized in init())
199
- stmtUpsertMeta;
200
- stmtDeleteVec; // optional — only set when vecTablesReady
201
- stmtInsertVec; // optional — only set when vecTablesReady
202
- stmtDeleteMeta;
203
- stmtGetMeta;
204
- stmtSearchVec; // optional — only set when vecTablesReady
205
- stmtQueryBySessionId;
206
- stmtQueryBySessionIdSince;
207
- stmtQueryBySessionKey;
208
- stmtQueryBySessionKeySince;
209
- stmtQueryAll;
210
- stmtQueryAllSince;
211
- // Prepared statements — L0 (initialized in init())
212
- stmtL0UpsertMeta;
213
- stmtL0DeleteVec; // optional — only set when vecTablesReady
214
- stmtL0InsertVec; // optional — only set when vecTablesReady
215
- stmtL0DeleteMeta;
216
- stmtL0GetMeta;
217
- stmtL0SearchVec; // optional — only set when vecTablesReady
218
- /** L0 query for L1 runner: all messages for a session key */
219
- stmtL0QueryAll;
220
- /** L0 query for L1 runner: messages after a timestamp cursor */
221
- stmtL0QueryAfter;
222
- /** L1 cursor-based pagination for migration (by PK) */
223
- stmtL1QueryMigrationCursor;
224
- /** L0 cursor-based pagination for migration (by PK) */
225
- stmtL0QueryMigrationCursor;
226
- // FTS5 tables availability flag (created best-effort — may be false if fts5 is not compiled in)
227
- ftsAvailable = false;
228
- // Prepared statements — FTS5 L1 (initialized in init())
229
- stmtL1FtsInsert;
230
- stmtL1FtsDelete;
231
- stmtL1FtsSearch;
232
- // Prepared statements — FTS5 L0 (initialized in init())
233
- stmtL0FtsInsert;
234
- stmtL0FtsDelete;
235
- stmtL0FtsSearch;
236
- /**
237
- * Create a VectorStore instance.
238
- *
239
- * Note: After construction, you MUST call `init()` to load the sqlite-vec
240
- * extension and create the schema.
241
- */
242
- constructor(dbPath, dimensions, logger) {
243
- this.dimensions = dimensions;
244
- this.logger = logger;
245
- // Open database with extension support enabled
246
- const { DatabaseSync: DbSync } = requireNodeSqlite();
247
- this.db = new DbSync(dbPath, { allowExtension: true });
248
- // Set busy timeout so concurrent processes retry instead of failing with SQLITE_BUSY
249
- this.db.exec("PRAGMA busy_timeout = 5000");
250
- // Enable WAL mode for better concurrent read performance
251
- this.db.exec("PRAGMA journal_mode = WAL");
252
- // Cap page cache at 64 MB
253
- this.db.exec("PRAGMA cache_size = -65536");
254
- // Cap memory-mapped I/O at 128 MB to bound RSS growth
255
- this.db.exec("PRAGMA mmap_size = 134217728");
256
- // Auto-checkpoint WAL every 1000 pages (~4 MB) to keep WAL file compact
257
- this.db.exec("PRAGMA wal_autocheckpoint = 1000");
258
- }
259
- /**
260
- * Whether the store is in degraded mode (e.g. sqlite-vec failed to load).
261
- * When degraded, all write/search operations become safe no-ops.
262
- */
263
- isDegraded() {
264
- return this.degraded;
265
- }
266
- /**
267
- * Load sqlite-vec extension and initialize database schema.
268
- * Must be called once after construction.
269
- *
270
- * @param providerInfo Current embedding provider info. When provided,
271
- * the store compares it against the persisted metadata. If the provider,
272
- * model, or dimensions changed, the vector tables are dropped and
273
- * re-created with the new dimensions, and `needsReindex: true` is returned
274
- * so the caller can schedule a full re-embed.
275
- */
276
- init(providerInfo) {
277
- // Load sqlite-vec extension (same approach as root project's sqlite-vec.ts)
278
- try {
279
- // eslint-disable-next-line @typescript-eslint/no-require-imports
280
- const sqliteVec = require("sqlite-vec");
281
- this.db.enableLoadExtension(true);
282
- sqliteVec.load(this.db);
283
- }
284
- catch (err) {
285
- const message = err instanceof Error ? err.message : String(err);
286
- this.logger?.error(`${TAG} Failed to load sqlite-vec extension: ${message}. ` +
287
- `VectorStore entering degraded mode — all operations will be no-ops.`);
288
- this.degraded = true;
289
- return { needsReindex: false, reason: `sqlite-vec load failed: ${message}` };
290
- }
291
- // ── Schema creation & prepared statements ──────────────────────────────
292
- // Wrapped in try-catch: if anything fails during schema init (e.g. the DB
293
- // is corrupted, disk full, etc.), we degrade gracefully instead of crashing.
294
- try {
295
- return this.initSchema(providerInfo);
296
- }
297
- catch (err) {
298
- const message = err instanceof Error ? err.message : String(err);
299
- this.logger?.error(`${TAG} Schema initialization failed: ${message}. ` +
300
- `VectorStore entering degraded mode.`);
301
- this.degraded = true;
302
- return { needsReindex: false, reason: `schema init failed: ${message}` };
303
- }
304
- }
305
- /**
306
- * Internal schema initialization — separated from init() so we can
307
- * catch errors at the top level and degrade gracefully.
308
- */
309
- initSchema(providerInfo) {
310
- // Tracks which provider/model/dimensions were used to generate vectors.
1
+ /**
2
+ * VectorStore: SQLite-based vector storage using sqlite-vec extension.
3
+ *
4
+ * Manages two layers of vector-indexed data in a single SQLite database:
5
+ *
6
+ * **L1 (structured memories):**
7
+ * 1. `l1_records` — relational metadata table (content, type, priority, scene, timestamps)
8
+ * 2. `l1_vec` — vec0 virtual table for cosine similarity search
9
+ *
10
+ * **L0 (raw conversations):**
11
+ * 3. `l0_conversations` — relational metadata table (session_key, role, message text, timestamps)
12
+ * 4. `l0_vec` — vec0 virtual table for cosine similarity search on individual messages
13
+ *
14
+ * Dependencies: Node.js built-in `node:sqlite` (Node 22+) + `sqlite-vec` (from root workspace).
15
+ *
16
+ * Design:
17
+ * - All operations are synchronous (DatabaseSync API).
18
+ * - Writes use manual BEGIN/COMMIT transactions for atomicity (metadata + vector).
19
+ * - vec0 virtual table does NOT support ON CONFLICT, so upsert = delete + insert.
20
+ * - Thread-safe via WAL mode.
21
+ */
22
+ import { createRequire } from "node:module";
23
+ const TAG = "[memory-tdai][sqlite]";
24
+ // Use createRequire to load the experimental node:sqlite module
25
+ const require = createRequire(import.meta.url);
26
+ function requireNodeSqlite() {
27
+ return require("node:sqlite");
28
+ }
29
+ let _jieba; // undefined = not yet tried
30
+ function getJieba() {
31
+ if (_jieba !== undefined)
32
+ return _jieba;
33
+ try {
34
+ // eslint-disable-next-line @typescript-eslint/no-require-imports
35
+ const { Jieba } = require("@node-rs/jieba");
36
+ // eslint-disable-next-line @typescript-eslint/no-require-imports
37
+ const { dict } = require("@node-rs/jieba/dict");
38
+ _jieba = Jieba.withDict(dict);
39
+ }
40
+ catch {
41
+ _jieba = null; // mark as unavailable — won't retry
42
+ }
43
+ return _jieba;
44
+ }
45
+ /**
46
+ * Common Chinese stop-words that add noise to FTS5 queries.
47
+ * Kept small on purpose — only high-frequency function words.
48
+ */
49
+ const ZH_STOP_WORDS = new Set([
50
+ "的", "了", "在", "是", "我", "有", "和", "就", "不", "人", "都", "一",
51
+ "一个", "上", "也", "很", "到", "说", "要", "去", "你", "会", "着",
52
+ "没有", "看", "好", "自己", "这", "他", "她", "它", "们", "那",
53
+ "吗", "吧", "呢", "啊", "呀", "哦", "嗯",
54
+ ]);
55
+ /**
56
+ * Build an FTS5 MATCH query from raw text.
57
+ *
58
+ * When `@node-rs/jieba` is available, uses jieba's search-engine mode
59
+ * (`cutForSearch`) for accurate Chinese word segmentation, producing
60
+ * much better recall than the previous regex-only approach.
61
+ *
62
+ * Falls back to Unicode-regex splitting (`/[\p{L}\p{N}_]+/gu`) if
63
+ * jieba is not installed.
64
+ *
65
+ * Tokens are OR-joined as quoted FTS5 phrase terms so that a document
66
+ * matching *any* token is returned. BM25 naturally ranks documents that
67
+ * match more tokens higher, so precision is preserved while recall is
68
+ * significantly improved — especially for longer queries and when running
69
+ * in FTS-only fallback mode (no embedding available).
70
+ *
71
+ * Example (with jieba):
72
+ * "用户喜欢编程和TypeScript" → '"用户" OR "喜欢" OR "编程" OR "TypeScript"'
73
+ * Example (fallback):
74
+ * "旅行计划 API" → '"旅行计划" OR "API"'
75
+ */
76
+ export function buildFtsQuery(raw) {
77
+ const jieba = getJieba();
78
+ let tokens;
79
+ if (jieba) {
80
+ // jieba cutForSearch: splits long words further for better recall
81
+ // e.g. "北京烤鸭" → ["北京", "烤鸭", "北京烤鸭"]
82
+ tokens = jieba
83
+ .cutForSearch(raw, true)
84
+ .map((t) => t.trim())
85
+ .filter((t) => {
86
+ if (!t)
87
+ return false;
88
+ // Remove pure whitespace / punctuation tokens
89
+ if (!/[\p{L}\p{N}]/u.test(t))
90
+ return false;
91
+ // Remove common Chinese stop-words to reduce noise
92
+ if (ZH_STOP_WORDS.has(t))
93
+ return false;
94
+ return true;
95
+ });
96
+ // Deduplicate (cutForSearch may produce duplicates for sub-words)
97
+ tokens = [...new Set(tokens)];
98
+ }
99
+ else {
100
+ // Fallback: simple Unicode regex split
101
+ tokens =
102
+ raw
103
+ .match(/[\p{L}\p{N}_]+/gu)
104
+ ?.map((t) => t.trim())
105
+ .filter(Boolean) ?? [];
106
+ }
107
+ if (tokens.length === 0)
108
+ return null;
109
+ const quoted = tokens.map((t) => `"${t.replaceAll('"', "")}"`);
110
+ return quoted.join(" OR ");
111
+ }
112
+ /**
113
+ * Tokenize text for FTS5 indexing (write-side).
114
+ *
115
+ * Uses jieba `cutForSearch()` (search-engine mode) to segment Chinese text,
116
+ * then joins tokens with spaces. The resulting string is stored in the FTS5
117
+ * `content` column so that `unicode61` tokenizer can split it into meaningful
118
+ * words — including both full words and their sub-words.
119
+ *
120
+ * Using `cutForSearch` (instead of `cut`) ensures that the index contains
121
+ * the same sub-word tokens that `buildFtsQuery()` produces on the query side.
122
+ * For example, "人工智能" is indexed as "人工 智能 人工智能", so queries for
123
+ * either the full term or sub-words will match.
124
+ *
125
+ * Falls back to the original text if jieba is unavailable.
126
+ *
127
+ * Example (with jieba):
128
+ * "用户五月去日本旅行" → "用户 五月 去 日本 旅行"
129
+ * "人工智能的分支" → "人工 智能 人工智能 的 分支"
130
+ * Example (fallback):
131
+ * "用户五月去日本旅行" → "用户五月去日本旅行" (unchanged)
132
+ */
133
+ export function tokenizeForFts(raw) {
134
+ const jieba = getJieba();
135
+ if (!jieba)
136
+ return raw;
137
+ // Use `cutForSearch` (search-engine mode) for indexing — it produces both
138
+ // full words AND their sub-word components. This ensures that query-side
139
+ // tokens (also produced by `cutForSearch` in `buildFtsQuery`) will always
140
+ // find a match in the index.
141
+ const tokens = jieba.cutForSearch(raw, true);
142
+ // Join with spaces so `unicode61` tokenizer can split them.
143
+ // Punctuation tokens are kept — unicode61 treats them as separators anyway.
144
+ return tokens.join(" ");
145
+ }
146
+ /**
147
+ * Reset jieba state so next call to `buildFtsQuery` re-initialises.
148
+ * Exported for testing only.
149
+ * @internal
150
+ */
151
+ export function _resetJiebaForTest() {
152
+ _jieba = undefined;
153
+ }
154
+ /**
155
+ * Override jieba instance (or set to `null` to force fallback).
156
+ * Exported for testing only.
157
+ * @internal
158
+ */
159
+ export function _setJiebaForTest(instance) {
160
+ _jieba = instance;
161
+ }
162
+ /**
163
+ * Convert a BM25 rank (negative = more relevant) to a 0–1 score.
164
+ * Mirrors the formula in openclaw core `hybrid.ts`.
165
+ */
166
+ export function bm25RankToScore(rank) {
167
+ if (!Number.isFinite(rank))
168
+ return 1 / (1 + 999);
169
+ if (rank < 0) {
170
+ const relevance = -rank;
171
+ return relevance / (1 + relevance);
172
+ }
173
+ return 1 / (1 + rank);
174
+ }
175
+ // ============================
176
+ // VectorStore class
177
+ // ============================
178
+ export class VectorStore {
179
+ db;
180
+ dimensions;
181
+ logger;
182
+ /** @see IMemoryStore.supportsDeferredEmbedding */
183
+ supportsDeferredEmbedding = true;
184
+ /**
185
+ * When `true`, the store is in a degraded state (e.g. sqlite-vec failed to
186
+ * load, or init() encountered an unrecoverable error). All public methods
187
+ * become safe no-ops so the plugin never blocks the main OpenClaw flow.
188
+ */
189
+ degraded = false;
190
+ /** Tracks whether close() has been called to prevent double-close errors. */
191
+ closed = false;
192
+ /**
193
+ * `true` when vec0 virtual tables (l1_vec / l0_vec) have been created and
194
+ * their prepared statements are ready. When `dimensions === 0` (i.e.
195
+ * provider="none"), vec0 tables are deferred and this stays `false`.
196
+ */
197
+ vecTablesReady = false;
198
+ // Prepared statements — L1 (initialized in init())
199
+ stmtUpsertMeta;
200
+ stmtDeleteVec; // optional — only set when vecTablesReady
201
+ stmtInsertVec; // optional — only set when vecTablesReady
202
+ stmtDeleteMeta;
203
+ stmtGetMeta;
204
+ stmtSearchVec; // optional — only set when vecTablesReady
205
+ stmtQueryBySessionId;
206
+ stmtQueryBySessionIdSince;
207
+ stmtQueryBySessionKey;
208
+ stmtQueryBySessionKeySince;
209
+ stmtQueryAll;
210
+ stmtQueryAllSince;
211
+ // Prepared statements — L0 (initialized in init())
212
+ stmtL0UpsertMeta;
213
+ stmtL0DeleteVec; // optional — only set when vecTablesReady
214
+ stmtL0InsertVec; // optional — only set when vecTablesReady
215
+ stmtL0DeleteMeta;
216
+ stmtL0GetMeta;
217
+ stmtL0SearchVec; // optional — only set when vecTablesReady
218
+ /** L0 query for L1 runner: all messages for a session key */
219
+ stmtL0QueryAll;
220
+ /** L0 query for L1 runner: messages after a timestamp cursor */
221
+ stmtL0QueryAfter;
222
+ /** L1 cursor-based pagination for migration (by PK) */
223
+ stmtL1QueryMigrationCursor;
224
+ /** L0 cursor-based pagination for migration (by PK) */
225
+ stmtL0QueryMigrationCursor;
226
+ // FTS5 tables availability flag (created best-effort — may be false if fts5 is not compiled in)
227
+ ftsAvailable = false;
228
+ // Prepared statements — FTS5 L1 (initialized in init())
229
+ stmtL1FtsInsert;
230
+ stmtL1FtsDelete;
231
+ stmtL1FtsSearch;
232
+ // Prepared statements — FTS5 L0 (initialized in init())
233
+ stmtL0FtsInsert;
234
+ stmtL0FtsDelete;
235
+ stmtL0FtsSearch;
236
+ /**
237
+ * Create a VectorStore instance.
238
+ *
239
+ * Note: After construction, you MUST call `init()` to load the sqlite-vec
240
+ * extension and create the schema.
241
+ */
242
+ constructor(dbPath, dimensions, logger) {
243
+ this.dimensions = dimensions;
244
+ this.logger = logger;
245
+ // Open database with extension support enabled
246
+ const { DatabaseSync: DbSync } = requireNodeSqlite();
247
+ this.db = new DbSync(dbPath, { allowExtension: true });
248
+ // Set busy timeout so concurrent processes retry instead of failing with SQLITE_BUSY
249
+ this.db.exec("PRAGMA busy_timeout = 5000");
250
+ // Enable WAL mode for better concurrent read performance
251
+ this.db.exec("PRAGMA journal_mode = WAL");
252
+ // Cap page cache at 64 MB
253
+ this.db.exec("PRAGMA cache_size = -65536");
254
+ // Cap memory-mapped I/O at 128 MB to bound RSS growth
255
+ this.db.exec("PRAGMA mmap_size = 134217728");
256
+ // Auto-checkpoint WAL every 1000 pages (~4 MB) to keep WAL file compact
257
+ this.db.exec("PRAGMA wal_autocheckpoint = 1000");
258
+ }
259
+ /**
260
+ * Whether the store is in degraded mode (e.g. sqlite-vec failed to load).
261
+ * When degraded, all write/search operations become safe no-ops.
262
+ */
263
+ isDegraded() {
264
+ return this.degraded;
265
+ }
266
+ /**
267
+ * Load sqlite-vec extension and initialize database schema.
268
+ * Must be called once after construction.
269
+ *
270
+ * @param providerInfo Current embedding provider info. When provided,
271
+ * the store compares it against the persisted metadata. If the provider,
272
+ * model, or dimensions changed, the vector tables are dropped and
273
+ * re-created with the new dimensions, and `needsReindex: true` is returned
274
+ * so the caller can schedule a full re-embed.
275
+ */
276
+ init(providerInfo) {
277
+ // Load sqlite-vec extension (same approach as root project's sqlite-vec.ts)
278
+ try {
279
+ // eslint-disable-next-line @typescript-eslint/no-require-imports
280
+ const sqliteVec = require("sqlite-vec");
281
+ this.db.enableLoadExtension(true);
282
+ sqliteVec.load(this.db);
283
+ }
284
+ catch (err) {
285
+ const message = err instanceof Error ? err.message : String(err);
286
+ this.logger?.error(`${TAG} Failed to load sqlite-vec extension: ${message}. ` +
287
+ `VectorStore entering degraded mode — all operations will be no-ops.`);
288
+ this.degraded = true;
289
+ return { needsReindex: false, reason: `sqlite-vec load failed: ${message}` };
290
+ }
291
+ // ── Schema creation & prepared statements ──────────────────────────────
292
+ // Wrapped in try-catch: if anything fails during schema init (e.g. the DB
293
+ // is corrupted, disk full, etc.), we degrade gracefully instead of crashing.
294
+ try {
295
+ return this.initSchema(providerInfo);
296
+ }
297
+ catch (err) {
298
+ const message = err instanceof Error ? err.message : String(err);
299
+ this.logger?.error(`${TAG} Schema initialization failed: ${message}. ` +
300
+ `VectorStore entering degraded mode.`);
301
+ this.degraded = true;
302
+ return { needsReindex: false, reason: `schema init failed: ${message}` };
303
+ }
304
+ }
305
+ /**
306
+ * Internal schema initialization — separated from init() so we can
307
+ * catch errors at the top level and degrade gracefully.
308
+ */
309
+ initSchema(providerInfo) {
310
+ // Tracks which provider/model/dimensions were used to generate vectors.
311
311
  this.db.exec(`
312
312
  CREATE TABLE IF NOT EXISTS embedding_meta (
313
313
  key TEXT PRIMARY KEY,
314
314
  value TEXT NOT NULL
315
315
  )
316
- `);
317
- // Detect whether re-index is needed
318
- let needsReindex = false;
319
- let reindexReason;
320
- const savedMeta = this.readEmbeddingMeta();
321
- if (providerInfo) {
322
- if (savedMeta) {
323
- const providerChanged = savedMeta.provider !== providerInfo.provider;
324
- const modelChanged = savedMeta.model !== providerInfo.model;
325
- const dimsChanged = savedMeta.dimensions !== this.dimensions;
326
- if (providerChanged || modelChanged || dimsChanged) {
327
- const reasons = [];
328
- if (providerChanged)
329
- reasons.push(`provider: ${savedMeta.provider} → ${providerInfo.provider}`);
330
- if (modelChanged)
331
- reasons.push(`model: ${savedMeta.model} → ${providerInfo.model}`);
332
- if (dimsChanged)
333
- reasons.push(`dimensions: ${savedMeta.dimensions} → ${this.dimensions}`);
334
- reindexReason = reasons.join(", ");
335
- this.logger?.info(`${TAG} Embedding config changed (${reindexReason}). ` +
336
- `Dropping vector tables for rebuild...`);
337
- // Drop and re-create vector tables with new dimensions
338
- this.dropVectorTables();
339
- needsReindex = true;
340
- }
341
- }
342
- else {
343
- // No saved meta — first run or legacy DB without meta table.
344
- // Two cases require dropping vector tables:
345
- // 1. Existing data created without meta tracking (legacy DB) — need re-embed
346
- // 2. vec0 tables exist with wrong dimensions (e.g. previously created with
347
- // provider="none" placeholder 768D, now switching to a real provider
348
- // with different dimensions) — must rebuild even if data tables are empty
349
- const l1Count = this.tableRowCount("l1_records");
350
- const l0Count = this.tableRowCount("l0_conversations");
351
- const existingVecDims = this.getVecTableDimensions();
352
- if (l1Count > 0 || l0Count > 0) {
353
- this.logger?.info(`${TAG} No embedding_meta found but existing data exists ` +
354
- `(L1=${l1Count}, L0=${l0Count}). Dropping vector tables for safety...`);
355
- this.dropVectorTables();
356
- needsReindex = true;
357
- reindexReason = "legacy DB without embedding_meta — cannot verify vector compatibility";
358
- }
359
- else if (existingVecDims !== null && existingVecDims !== this.dimensions) {
360
- // vec0 tables exist (from a previous provider="none" placeholder or
361
- // different config) but with mismatched dimensions. Drop them so they
362
- // get re-created with the correct dimensions below.
363
- this.logger?.info(`${TAG} vec0 table dimension mismatch (existing=${existingVecDims}, ` +
364
- `required=${this.dimensions}). Dropping vector tables for rebuild...`);
365
- this.dropVectorTables();
366
- // No needsReindex — there's no data to re-embed
367
- }
368
- }
369
- }
370
- // ── L1 schema ──────────────────────────────────
371
- // Metadata table
316
+ `);
317
+ // Detect whether re-index is needed
318
+ let needsReindex = false;
319
+ let reindexReason;
320
+ const savedMeta = this.readEmbeddingMeta();
321
+ if (providerInfo) {
322
+ if (savedMeta) {
323
+ const providerChanged = savedMeta.provider !== providerInfo.provider;
324
+ const modelChanged = savedMeta.model !== providerInfo.model;
325
+ const dimsChanged = savedMeta.dimensions !== this.dimensions;
326
+ if (providerChanged || modelChanged || dimsChanged) {
327
+ const reasons = [];
328
+ if (providerChanged)
329
+ reasons.push(`provider: ${savedMeta.provider} → ${providerInfo.provider}`);
330
+ if (modelChanged)
331
+ reasons.push(`model: ${savedMeta.model} → ${providerInfo.model}`);
332
+ if (dimsChanged)
333
+ reasons.push(`dimensions: ${savedMeta.dimensions} → ${this.dimensions}`);
334
+ reindexReason = reasons.join(", ");
335
+ this.logger?.info(`${TAG} Embedding config changed (${reindexReason}). ` +
336
+ `Dropping vector tables for rebuild...`);
337
+ // Drop and re-create vector tables with new dimensions
338
+ this.dropVectorTables();
339
+ needsReindex = true;
340
+ }
341
+ }
342
+ else {
343
+ // No saved meta — first run or legacy DB without meta table.
344
+ // Two cases require dropping vector tables:
345
+ // 1. Existing data created without meta tracking (legacy DB) — need re-embed
346
+ // 2. vec0 tables exist with wrong dimensions (e.g. previously created with
347
+ // provider="none" placeholder 768D, now switching to a real provider
348
+ // with different dimensions) — must rebuild even if data tables are empty
349
+ const l1Count = this.tableRowCount("l1_records");
350
+ const l0Count = this.tableRowCount("l0_conversations");
351
+ const existingVecDims = this.getVecTableDimensions();
352
+ if (l1Count > 0 || l0Count > 0) {
353
+ this.logger?.info(`${TAG} No embedding_meta found but existing data exists ` +
354
+ `(L1=${l1Count}, L0=${l0Count}). Dropping vector tables for safety...`);
355
+ this.dropVectorTables();
356
+ needsReindex = true;
357
+ reindexReason = "legacy DB without embedding_meta — cannot verify vector compatibility";
358
+ }
359
+ else if (existingVecDims !== null && existingVecDims !== this.dimensions) {
360
+ // vec0 tables exist (from a previous provider="none" placeholder or
361
+ // different config) but with mismatched dimensions. Drop them so they
362
+ // get re-created with the correct dimensions below.
363
+ this.logger?.info(`${TAG} vec0 table dimension mismatch (existing=${existingVecDims}, ` +
364
+ `required=${this.dimensions}). Dropping vector tables for rebuild...`);
365
+ this.dropVectorTables();
366
+ // No needsReindex — there's no data to re-embed
367
+ }
368
+ }
369
+ }
370
+ // ── L1 schema ──────────────────────────────────
371
+ // Metadata table
372
372
  this.db.exec(`
373
373
  CREATE TABLE IF NOT EXISTS l1_records (
374
374
  record_id TEXT PRIMARY KEY,
@@ -385,31 +385,31 @@ export class VectorStore {
385
385
  updated_time TEXT DEFAULT '',
386
386
  metadata_json TEXT DEFAULT '{}'
387
387
  )
388
- `);
389
- // Indexes for common queries
390
- this.db.exec("CREATE INDEX IF NOT EXISTS idx_l1_type ON l1_records(type)");
391
- this.db.exec("CREATE INDEX IF NOT EXISTS idx_l1_session_key ON l1_records(session_key)");
392
- this.db.exec("CREATE INDEX IF NOT EXISTS idx_l1_session_id ON l1_records(session_id)");
393
- this.db.exec("CREATE INDEX IF NOT EXISTS idx_l1_scene ON l1_records(scene_name)");
394
- this.db.exec("CREATE INDEX IF NOT EXISTS idx_l1_ts_start ON l1_records(timestamp_start)");
395
- this.db.exec("CREATE INDEX IF NOT EXISTS idx_l1_ts_end ON l1_records(timestamp_end)");
396
- // Composite index: session_id exact match + updated_time range scan (for incremental L2 queries)
397
- this.db.exec("CREATE INDEX IF NOT EXISTS idx_l1_session_updated ON l1_records(session_id, updated_time)");
398
- // Composite index: session_key exact match + updated_time range scan (for pipeline cursor queries)
399
- this.db.exec("CREATE INDEX IF NOT EXISTS idx_l1_sessionkey_updated ON l1_records(session_key, updated_time)");
400
- // Vector virtual table (cosine distance) — only created when dimensions > 0.
401
- // When provider="none", dimensions=0 and vec0 tables are deferred until a
402
- // real embedding provider is configured.
403
- if (this.dimensions > 0) {
388
+ `);
389
+ // Indexes for common queries
390
+ this.db.exec("CREATE INDEX IF NOT EXISTS idx_l1_type ON l1_records(type)");
391
+ this.db.exec("CREATE INDEX IF NOT EXISTS idx_l1_session_key ON l1_records(session_key)");
392
+ this.db.exec("CREATE INDEX IF NOT EXISTS idx_l1_session_id ON l1_records(session_id)");
393
+ this.db.exec("CREATE INDEX IF NOT EXISTS idx_l1_scene ON l1_records(scene_name)");
394
+ this.db.exec("CREATE INDEX IF NOT EXISTS idx_l1_ts_start ON l1_records(timestamp_start)");
395
+ this.db.exec("CREATE INDEX IF NOT EXISTS idx_l1_ts_end ON l1_records(timestamp_end)");
396
+ // Composite index: session_id exact match + updated_time range scan (for incremental L2 queries)
397
+ this.db.exec("CREATE INDEX IF NOT EXISTS idx_l1_session_updated ON l1_records(session_id, updated_time)");
398
+ // Composite index: session_key exact match + updated_time range scan (for pipeline cursor queries)
399
+ this.db.exec("CREATE INDEX IF NOT EXISTS idx_l1_sessionkey_updated ON l1_records(session_key, updated_time)");
400
+ // Vector virtual table (cosine distance) — only created when dimensions > 0.
401
+ // When provider="none", dimensions=0 and vec0 tables are deferred until a
402
+ // real embedding provider is configured.
403
+ if (this.dimensions > 0) {
404
404
  this.db.exec(`
405
405
  CREATE VIRTUAL TABLE IF NOT EXISTS l1_vec USING vec0(
406
406
  record_id TEXT PRIMARY KEY,
407
407
  embedding float[${this.dimensions}] distance_metric=cosine,
408
408
  updated_time TEXT DEFAULT ''
409
409
  )
410
- `);
411
- }
412
- // Prepare statements for reuse
410
+ `);
411
+ }
412
+ // Prepare statements for reuse
413
413
  this.stmtUpsertMeta = this.db.prepare(`
414
414
  INSERT INTO l1_records (
415
415
  record_id, content, type, priority, scene_name, session_key, session_id,
@@ -426,28 +426,28 @@ export class VectorStore {
426
426
  timestamp_end=excluded.timestamp_end,
427
427
  updated_time=excluded.updated_time,
428
428
  metadata_json=excluded.metadata_json
429
- `);
430
- if (this.dimensions > 0) {
431
- this.stmtDeleteVec = this.db.prepare("DELETE FROM l1_vec WHERE record_id = ?");
432
- this.stmtInsertVec = this.db.prepare("INSERT INTO l1_vec (record_id, embedding, updated_time) VALUES (?, ?, ?)");
433
- }
434
- this.stmtDeleteMeta = this.db.prepare("DELETE FROM l1_records WHERE record_id = ?");
429
+ `);
430
+ if (this.dimensions > 0) {
431
+ this.stmtDeleteVec = this.db.prepare("DELETE FROM l1_vec WHERE record_id = ?");
432
+ this.stmtInsertVec = this.db.prepare("INSERT INTO l1_vec (record_id, embedding, updated_time) VALUES (?, ?, ?)");
433
+ }
434
+ this.stmtDeleteMeta = this.db.prepare("DELETE FROM l1_records WHERE record_id = ?");
435
435
  this.stmtGetMeta = this.db.prepare(`
436
436
  SELECT content, type, priority, scene_name, session_key, session_id,
437
437
  timestamp_str, timestamp_start, timestamp_end, metadata_json
438
438
  FROM l1_records WHERE record_id = ?
439
- `);
440
- if (this.dimensions > 0) {
439
+ `);
440
+ if (this.dimensions > 0) {
441
441
  this.stmtSearchVec = this.db.prepare(`
442
442
  SELECT record_id, distance
443
443
  FROM l1_vec
444
444
  WHERE embedding MATCH ?
445
445
  AND k = ?
446
446
  ORDER BY distance
447
- `);
448
- }
449
- // ── L0 schema ──────────────────────────────────
450
- // L0 metadata table: stores individual messages for vector search
447
+ `);
448
+ }
449
+ // ── L0 schema ──────────────────────────────────
450
+ // L0 metadata table: stores individual messages for vector search
451
451
  this.db.exec(`
452
452
  CREATE TABLE IF NOT EXISTS l0_conversations (
453
453
  record_id TEXT PRIMARY KEY,
@@ -458,31 +458,31 @@ export class VectorStore {
458
458
  recorded_at TEXT DEFAULT '',
459
459
  timestamp INTEGER DEFAULT 0
460
460
  )
461
- `);
462
- // Migration: add timestamp column if missing (existing DBs pre-v3.x)
463
- try {
464
- this.db.exec("ALTER TABLE l0_conversations ADD COLUMN timestamp INTEGER DEFAULT 0");
465
- this.logger?.debug?.(`${TAG} Migrated l0_conversations: added timestamp column`);
466
- }
467
- catch {
468
- // Column already exists — expected on non-first run
469
- }
470
- // Indexes for L0 queries
471
- this.db.exec("CREATE INDEX IF NOT EXISTS idx_l0_session ON l0_conversations(session_key)");
472
- this.db.exec("CREATE INDEX IF NOT EXISTS idx_l0_session_id ON l0_conversations(session_id)");
473
- this.db.exec("CREATE INDEX IF NOT EXISTS idx_l0_recorded ON l0_conversations(recorded_at)");
474
- this.db.exec("CREATE INDEX IF NOT EXISTS idx_l0_timestamp ON l0_conversations(timestamp)");
475
- // L0 vector virtual table (cosine distance, same dimensions as L1) — deferred when dimensions=0
476
- if (this.dimensions > 0) {
461
+ `);
462
+ // Migration: add timestamp column if missing (existing DBs pre-v3.x)
463
+ try {
464
+ this.db.exec("ALTER TABLE l0_conversations ADD COLUMN timestamp INTEGER DEFAULT 0");
465
+ this.logger?.debug?.(`${TAG} Migrated l0_conversations: added timestamp column`);
466
+ }
467
+ catch {
468
+ // Column already exists — expected on non-first run
469
+ }
470
+ // Indexes for L0 queries
471
+ this.db.exec("CREATE INDEX IF NOT EXISTS idx_l0_session ON l0_conversations(session_key)");
472
+ this.db.exec("CREATE INDEX IF NOT EXISTS idx_l0_session_id ON l0_conversations(session_id)");
473
+ this.db.exec("CREATE INDEX IF NOT EXISTS idx_l0_recorded ON l0_conversations(recorded_at)");
474
+ this.db.exec("CREATE INDEX IF NOT EXISTS idx_l0_timestamp ON l0_conversations(timestamp)");
475
+ // L0 vector virtual table (cosine distance, same dimensions as L1) — deferred when dimensions=0
476
+ if (this.dimensions > 0) {
477
477
  this.db.exec(`
478
478
  CREATE VIRTUAL TABLE IF NOT EXISTS l0_vec USING vec0(
479
479
  record_id TEXT PRIMARY KEY,
480
480
  embedding float[${this.dimensions}] distance_metric=cosine,
481
481
  recorded_at TEXT DEFAULT ''
482
482
  )
483
- `);
484
- }
485
- // L0 prepared statements
483
+ `);
484
+ }
485
+ // L0 prepared statements
486
486
  this.stmtL0UpsertMeta = this.db.prepare(`
487
487
  INSERT INTO l0_conversations (
488
488
  record_id, session_key, session_id, role, message_text, recorded_at, timestamp
@@ -491,61 +491,61 @@ export class VectorStore {
491
491
  message_text=excluded.message_text,
492
492
  recorded_at=excluded.recorded_at,
493
493
  timestamp=excluded.timestamp
494
- `);
495
- if (this.dimensions > 0) {
496
- this.stmtL0DeleteVec = this.db.prepare("DELETE FROM l0_vec WHERE record_id = ?");
497
- this.stmtL0InsertVec = this.db.prepare("INSERT INTO l0_vec (record_id, embedding, recorded_at) VALUES (?, ?, ?)");
498
- }
499
- this.stmtL0DeleteMeta = this.db.prepare("DELETE FROM l0_conversations WHERE record_id = ?");
494
+ `);
495
+ if (this.dimensions > 0) {
496
+ this.stmtL0DeleteVec = this.db.prepare("DELETE FROM l0_vec WHERE record_id = ?");
497
+ this.stmtL0InsertVec = this.db.prepare("INSERT INTO l0_vec (record_id, embedding, recorded_at) VALUES (?, ?, ?)");
498
+ }
499
+ this.stmtL0DeleteMeta = this.db.prepare("DELETE FROM l0_conversations WHERE record_id = ?");
500
500
  this.stmtL0GetMeta = this.db.prepare(`
501
501
  SELECT session_key, session_id, role, message_text, recorded_at, timestamp
502
502
  FROM l0_conversations WHERE record_id = ?
503
- `);
504
- if (this.dimensions > 0) {
503
+ `);
504
+ if (this.dimensions > 0) {
505
505
  this.stmtL0SearchVec = this.db.prepare(`
506
506
  SELECT record_id, distance
507
507
  FROM l0_vec
508
508
  WHERE embedding MATCH ?
509
509
  AND k = ?
510
510
  ORDER BY distance
511
- `);
512
- }
513
- // L0 query statements for L1 runner (newest-first + LIMIT to bound memory)
514
- // Sort/filter by recorded_at (write time) instead of timestamp (conversation time)
515
- // because L1 cursor uses recorded_at semantics. ISO 8601 string comparison preserves time order.
511
+ `);
512
+ }
513
+ // L0 query statements for L1 runner (newest-first + LIMIT to bound memory)
514
+ // Sort/filter by recorded_at (write time) instead of timestamp (conversation time)
515
+ // because L1 cursor uses recorded_at semantics. ISO 8601 string comparison preserves time order.
516
516
  this.stmtL0QueryAll = this.db.prepare(`
517
517
  SELECT record_id, session_key, session_id, role, message_text, recorded_at, timestamp
518
518
  FROM l0_conversations
519
519
  WHERE session_key = ?
520
520
  ORDER BY recorded_at DESC
521
521
  LIMIT ?
522
- `);
522
+ `);
523
523
  this.stmtL0QueryAfter = this.db.prepare(`
524
524
  SELECT record_id, session_key, session_id, role, message_text, recorded_at, timestamp
525
525
  FROM l0_conversations
526
526
  WHERE session_key = ? AND recorded_at > ?
527
527
  ORDER BY recorded_at DESC
528
528
  LIMIT ?
529
- `);
529
+ `);
530
530
  this.stmtL0QueryMigrationCursor = this.db.prepare(`
531
531
  SELECT record_id, session_key, session_id, role, message_text, recorded_at, timestamp
532
532
  FROM l0_conversations
533
533
  WHERE record_id > ?
534
534
  ORDER BY record_id ASC
535
535
  LIMIT ?
536
- `);
537
- // ── FTS5 tables (best-effort — gracefully degrade if fts5 is not compiled in) ──
538
- // Schema v2: `content` column stores jieba-segmented text (for indexing),
539
- // `content_original` (UNINDEXED) stores the raw text (for display).
540
- // If old v1 tables exist (no content_original column), drop + recreate.
541
- try {
542
- // ── Migrate old FTS5 tables (v1 → v2) ──
543
- // v1 tables stored raw text in the `content` column. v2 stores segmented
544
- // text in `content` and raw text in `content_original` / `message_text_original`.
545
- // FTS5 virtual tables don't support ALTER TABLE ADD COLUMN, so we must
546
- // drop and recreate. The data will be repopulated by `rebuildFtsIndex()`.
547
- const needsFtsRebuild = this.migrateFtsTablesIfNeeded();
548
- // L1 FTS5 virtual table (v2 schema)
536
+ `);
537
+ // ── FTS5 tables (best-effort — gracefully degrade if fts5 is not compiled in) ──
538
+ // Schema v2: `content` column stores jieba-segmented text (for indexing),
539
+ // `content_original` (UNINDEXED) stores the raw text (for display).
540
+ // If old v1 tables exist (no content_original column), drop + recreate.
541
+ try {
542
+ // ── Migrate old FTS5 tables (v1 → v2) ──
543
+ // v1 tables stored raw text in the `content` column. v2 stores segmented
544
+ // text in `content` and raw text in `content_original` / `message_text_original`.
545
+ // FTS5 virtual tables don't support ALTER TABLE ADD COLUMN, so we must
546
+ // drop and recreate. The data will be repopulated by `rebuildFtsIndex()`.
547
+ const needsFtsRebuild = this.migrateFtsTablesIfNeeded();
548
+ // L1 FTS5 virtual table (v2 schema)
549
549
  this.db.exec(`
550
550
  CREATE VIRTUAL TABLE IF NOT EXISTS l1_fts USING fts5(
551
551
  content,
@@ -561,8 +561,8 @@ export class VectorStore {
561
561
  timestamp_end UNINDEXED,
562
562
  metadata_json UNINDEXED
563
563
  )
564
- `);
565
- // L0 FTS5 virtual table (v2 schema)
564
+ `);
565
+ // L0 FTS5 virtual table (v2 schema)
566
566
  this.db.exec(`
567
567
  CREATE VIRTUAL TABLE IF NOT EXISTS l0_fts USING fts5(
568
568
  message_text,
@@ -574,14 +574,14 @@ export class VectorStore {
574
574
  recorded_at UNINDEXED,
575
575
  timestamp UNINDEXED
576
576
  )
577
- `);
578
- // L1 FTS prepared statements
577
+ `);
578
+ // L1 FTS prepared statements
579
579
  this.stmtL1FtsInsert = this.db.prepare(`
580
580
  INSERT INTO l1_fts (content, content_original, record_id, type, priority, scene_name,
581
581
  session_key, session_id, timestamp_str, timestamp_start, timestamp_end, metadata_json)
582
582
  VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
583
- `);
584
- this.stmtL1FtsDelete = this.db.prepare("DELETE FROM l1_fts WHERE record_id = ?");
583
+ `);
584
+ this.stmtL1FtsDelete = this.db.prepare("DELETE FROM l1_fts WHERE record_id = ?");
585
585
  this.stmtL1FtsSearch = this.db.prepare(`
586
586
  SELECT record_id, content_original AS content, type, priority, scene_name,
587
587
  session_key, session_id, timestamp_str, timestamp_start, timestamp_end,
@@ -591,13 +591,13 @@ export class VectorStore {
591
591
  WHERE l1_fts MATCH ?
592
592
  ORDER BY rank ASC
593
593
  LIMIT ?
594
- `);
595
- // L0 FTS prepared statements
594
+ `);
595
+ // L0 FTS prepared statements
596
596
  this.stmtL0FtsInsert = this.db.prepare(`
597
597
  INSERT INTO l0_fts (message_text, message_text_original, record_id, session_key, session_id, role, recorded_at, timestamp)
598
598
  VALUES (?, ?, ?, ?, ?, ?, ?, ?)
599
- `);
600
- this.stmtL0FtsDelete = this.db.prepare("DELETE FROM l0_fts WHERE record_id = ?");
599
+ `);
600
+ this.stmtL0FtsDelete = this.db.prepare("DELETE FROM l0_fts WHERE record_id = ?");
601
601
  this.stmtL0FtsSearch = this.db.prepare(`
602
602
  SELECT record_id, message_text_original AS message_text, session_key, session_id, role, recorded_at, timestamp,
603
603
  bm25(l0_fts) AS rank
@@ -605,1218 +605,1218 @@ export class VectorStore {
605
605
  WHERE l0_fts MATCH ?
606
606
  ORDER BY rank ASC
607
607
  LIMIT ?
608
- `);
609
- this.ftsAvailable = true;
610
- this.logger?.debug?.(`${TAG} FTS5 tables initialized (l1_fts, l0_fts) [schema v2 — jieba segmented]`);
611
- // Rebuild FTS index if migrated from v1 or tables were freshly created
612
- if (needsFtsRebuild) {
613
- this.rebuildFtsIndex();
614
- }
615
- }
616
- catch (err) {
617
- const message = err instanceof Error ? err.message : String(err);
618
- this.ftsAvailable = false;
619
- this.logger?.warn(`${TAG} FTS5 tables NOT available (fts5 may not be compiled in): ${message}. ` +
620
- `FTS-based keyword search will be unavailable; recall will use in-memory scoring if needed.`);
621
- }
622
- // Save current embedding meta (write after schema is ready)
623
- if (providerInfo) {
624
- this.writeEmbeddingMeta({
625
- provider: providerInfo.provider,
626
- model: providerInfo.model,
627
- dimensions: this.dimensions,
628
- });
629
- }
630
- // Mark vec0 tables as ready only when they were actually created
631
- this.vecTablesReady = this.dimensions > 0;
632
- // L1 query statements (for l1-reader)
608
+ `);
609
+ this.ftsAvailable = true;
610
+ this.logger?.debug?.(`${TAG} FTS5 tables initialized (l1_fts, l0_fts) [schema v2 — jieba segmented]`);
611
+ // Rebuild FTS index if migrated from v1 or tables were freshly created
612
+ if (needsFtsRebuild) {
613
+ this.rebuildFtsIndex();
614
+ }
615
+ }
616
+ catch (err) {
617
+ const message = err instanceof Error ? err.message : String(err);
618
+ this.ftsAvailable = false;
619
+ this.logger?.warn(`${TAG} FTS5 tables NOT available (fts5 may not be compiled in): ${message}. ` +
620
+ `FTS-based keyword search will be unavailable; recall will use in-memory scoring if needed.`);
621
+ }
622
+ // Save current embedding meta (write after schema is ready)
623
+ if (providerInfo) {
624
+ this.writeEmbeddingMeta({
625
+ provider: providerInfo.provider,
626
+ model: providerInfo.model,
627
+ dimensions: this.dimensions,
628
+ });
629
+ }
630
+ // Mark vec0 tables as ready only when they were actually created
631
+ this.vecTablesReady = this.dimensions > 0;
632
+ // L1 query statements (for l1-reader)
633
633
  const l1QueryCols = `record_id, content, type, priority, scene_name, session_key, session_id,
634
634
  timestamp_str, timestamp_start, timestamp_end,
635
- created_time, updated_time, metadata_json`;
635
+ created_time, updated_time, metadata_json`;
636
636
  this.stmtQueryBySessionId = this.db.prepare(`
637
637
  SELECT ${l1QueryCols} FROM l1_records
638
638
  WHERE session_id = ?
639
639
  ORDER BY updated_time ASC
640
- `);
640
+ `);
641
641
  this.stmtQueryBySessionIdSince = this.db.prepare(`
642
642
  SELECT ${l1QueryCols} FROM l1_records
643
643
  WHERE session_id = ? AND updated_time > ?
644
644
  ORDER BY updated_time ASC
645
- `);
645
+ `);
646
646
  this.stmtQueryBySessionKey = this.db.prepare(`
647
647
  SELECT ${l1QueryCols} FROM l1_records
648
648
  WHERE session_key = ?
649
649
  ORDER BY updated_time ASC
650
- `);
650
+ `);
651
651
  this.stmtQueryBySessionKeySince = this.db.prepare(`
652
652
  SELECT ${l1QueryCols} FROM l1_records
653
653
  WHERE session_key = ? AND updated_time > ?
654
654
  ORDER BY updated_time ASC
655
- `);
655
+ `);
656
656
  this.stmtQueryAll = this.db.prepare(`
657
657
  SELECT ${l1QueryCols} FROM l1_records
658
658
  ORDER BY updated_time ASC
659
- `);
659
+ `);
660
660
  this.stmtQueryAllSince = this.db.prepare(`
661
661
  SELECT ${l1QueryCols} FROM l1_records
662
662
  WHERE updated_time > ?
663
663
  ORDER BY updated_time ASC
664
- `);
664
+ `);
665
665
  this.stmtL1QueryMigrationCursor = this.db.prepare(`
666
666
  SELECT ${l1QueryCols} FROM l1_records
667
667
  WHERE record_id > ?
668
668
  ORDER BY record_id ASC
669
669
  LIMIT ?
670
- `);
671
- this.logger?.debug?.(`${TAG} Initialized (dimensions=${this.dimensions})`);
672
- return { needsReindex, reason: reindexReason };
673
- }
674
- // ── Embedding meta helpers ──────────────────────────────
675
- readEmbeddingMeta() {
676
- try {
677
- const row = this.db
678
- .prepare("SELECT value FROM embedding_meta WHERE key = ?")
679
- .get("embedding_provider_info");
680
- if (!row)
681
- return null;
682
- return JSON.parse(row.value);
683
- }
684
- catch {
685
- return null;
686
- }
687
- }
688
- writeEmbeddingMeta(meta) {
689
- this.db.prepare("INSERT INTO embedding_meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value=excluded.value").run("embedding_provider_info", JSON.stringify(meta));
690
- }
691
- /** Allowed table names for row counting (whitelist to prevent SQL injection). */
692
- static COUNTABLE_TABLES = new Set(["l1_records", "l0_conversations"]);
693
- /**
694
- * Extra rows to retrieve from vec0 KNN search to compensate for legacy
695
- * zero-vector placeholders that may still linger from older data.
696
- */
697
- static ZERO_VEC_BUFFER = 10;
698
- /** Default result limit for FTS5 keyword searches. */
699
- static FTS_DEFAULT_LIMIT = 20;
700
- tableRowCount(table) {
701
- if (!VectorStore.COUNTABLE_TABLES.has(table)) {
702
- this.logger?.warn(`${TAG} tableRowCount: rejected unknown table name "${table}"`);
703
- return 0;
704
- }
705
- try {
706
- const row = this.db
707
- .prepare(`SELECT COUNT(*) AS cnt FROM ${table}`)
708
- .get();
709
- return row?.cnt ?? 0;
710
- }
711
- catch {
712
- return 0;
713
- }
714
- }
715
- /**
716
- * Detect the embedding dimension of an existing vec0 table by inspecting
717
- * the DDL stored in sqlite_master. Returns `null` if the table doesn't
718
- * exist or the dimension cannot be determined.
719
- *
720
- * The vec0 DDL looks like:
721
- * CREATE VIRTUAL TABLE l1_vec USING vec0(... embedding float[768] ...)
722
- * We parse the number inside `float[N]`.
723
- */
724
- getVecTableDimensions() {
725
- try {
726
- const row = this.db
727
- .prepare("SELECT sql FROM sqlite_master WHERE type='table' AND name=?")
728
- .get("l1_vec");
729
- if (!row?.sql)
730
- return null;
731
- const match = row.sql.match(/float\[(\d+)\]/);
732
- return match ? Number(match[1]) : null;
733
- }
734
- catch {
735
- return null;
736
- }
737
- }
738
- /**
739
- * Drop both L1 and L0 vector virtual tables.
740
- * Metadata tables (l1_records, l0_conversations) are preserved — only
741
- * the vec0 tables need to be rebuilt with the new dimensions.
742
- */
743
- dropVectorTables() {
744
- this.db.exec("DROP TABLE IF EXISTS l1_vec");
745
- this.db.exec("DROP TABLE IF EXISTS l0_vec");
746
- this.logger?.info(`${TAG} Dropped vector tables (l1_vec, l0_vec)`);
747
- }
748
- /**
749
- * Write or update a memory record (metadata + vector).
750
- * Uses a manual transaction for atomicity.
751
- *
752
- * If `embedding` is `undefined` or a zero vector (all elements are 0), only
753
- * the metadata row is written — the vec0 table is left untouched. This
754
- * allows callers without an EmbeddingService to still persist metadata + FTS
755
- * without constructing a throwaway zero-vector, and prevents placeholder
756
- * zero vectors (from embedding-service failures) from polluting KNN search
757
- * results with null / NaN distances.
758
- *
759
- * **Fault-tolerant**: catches all errors internally so that a vector store
760
- * failure never propagates to the caller / main OpenClaw flow.
761
- * Returns `true` on success, `false` on failure (logged as warning).
762
- */
763
- upsertL1(record, embedding) {
764
- if (this.degraded) {
765
- this.logger?.warn(`${TAG} [L1-upsert] SKIPPED (degraded mode) id=${record.id}`);
766
- return false;
767
- }
768
- try {
769
- const { id: recordId, timestamps } = record;
770
- const tsStr = timestamps[0] ?? "";
771
- const tsStart = timestamps.length > 0
772
- ? timestamps.reduce((a, b) => (a < b ? a : b))
773
- : tsStr;
774
- const tsEnd = timestamps.length > 0
775
- ? timestamps.reduce((a, b) => (a > b ? a : b))
776
- : tsStr;
777
- const skipVec = !embedding || embedding.every(v => v === 0) || !this.vecTablesReady;
778
- this.logger?.debug?.(`${TAG} [L1-upsert] START id=${recordId}, type=${record.type}, ` +
779
- `content="${record.content.slice(0, 60)}..."` +
780
- (embedding
781
- ? `, embeddingDims=${embedding.length}, ` +
782
- `embeddingNorm=${Math.sqrt(Array.from(embedding).reduce((s, v) => s + v * v, 0)).toFixed(4)}` +
783
- `${skipVec ? " (ZERO VECTOR or vec tables not ready — vec write will be skipped)" : ""}`
784
- : " (no embedding — metadata-only write)"));
785
- this.db.exec("BEGIN");
786
- try {
787
- // Upsert metadata (INSERT OR UPDATE)
788
- this.stmtUpsertMeta.run(recordId, record.content, record.type, record.priority, record.scene_name, record.sessionKey, record.sessionId, tsStr, tsStart, tsEnd, record.createdAt, record.updatedAt, JSON.stringify(record.metadata));
789
- if (!skipVec) {
790
- // vec0 does not support ON CONFLICT → delete then insert
791
- this.stmtDeleteVec.run(recordId);
792
- this.stmtInsertVec.run(recordId, Buffer.from(embedding.buffer), record.updatedAt);
793
- }
794
- else {
795
- this.logger?.debug?.(`${TAG} [L1-upsert] Skipping vec write (${embedding ? "zero vector" : "no embedding"}) id=${recordId}`);
796
- }
797
- // Sync FTS5 (delete + re-insert to handle updates)
798
- if (this.ftsAvailable) {
799
- try {
800
- this.stmtL1FtsDelete.run(recordId);
801
- this.stmtL1FtsInsert.run(tokenizeForFts(record.content), // content — segmented for indexing
802
- record.content, // content_original — raw for display
803
- recordId, record.type, record.priority, record.scene_name, record.sessionKey, record.sessionId, tsStr, tsStart, tsEnd, JSON.stringify(record.metadata));
804
- }
805
- catch (ftsErr) {
806
- // FTS write failure is non-fatal — log and continue
807
- this.logger?.warn(`${TAG} [L1-upsert] FTS write failed (non-fatal) id=${recordId}: ${ftsErr instanceof Error ? ftsErr.message : String(ftsErr)}`);
808
- }
809
- }
810
- this.db.exec("COMMIT");
811
- }
812
- catch (err) {
813
- try {
814
- this.db.exec("ROLLBACK");
815
- }
816
- catch { /* ignore rollback errors */ }
817
- throw err;
818
- }
819
- this.logger?.debug?.(`${TAG} [L1-upsert] OK id=${recordId}${skipVec ? " (meta-only)" : ""}`);
820
- return true;
821
- }
822
- catch (err) {
823
- this.logger?.warn(`${TAG} [L1-upsert] FAILED (non-fatal) id=${record.id}: ${err instanceof Error ? err.message : String(err)}`);
824
- return false;
825
- }
826
- }
827
- /**
828
- * Vector similarity search (cosine distance).
829
- * Returns top-k results sorted by similarity (highest first).
830
- *
831
- * **Fault-tolerant**: returns an empty array on any error (e.g. dimension
832
- * mismatch, corrupted DB) so callers can fall back to keyword search.
833
- */
834
- searchL1Vector(queryEmbedding, topK = 5) {
835
- if (this.degraded || !this.vecTablesReady) {
836
- if (this.degraded)
837
- this.logger?.warn(`${TAG} [L1-search] SKIPPED (degraded mode)`);
838
- return [];
839
- }
840
- try {
841
- // Over-retrieve to compensate for legacy zero-vector placeholders that
842
- // may still exist in the vec0 table. New zero vectors are no longer
843
- // inserted (upsert() skips vec write for zero vectors since v3.x), but
844
- // older data may still contain them — they surface as NULL/NaN distance
845
- // in KNN results. A small buffer of 10 is sufficient for remnants.
846
- // NOTE: "AND distance IS NOT NULL" is NOT usable because vec0 does not
847
- // support that constraint — it causes an empty result set.
848
- const ZERO_VEC_BUFFER = 10;
849
- const retrieveCount = topK + ZERO_VEC_BUFFER;
850
- this.logger?.debug?.(`${TAG} [L1-search] START topK=${topK}, retrieveCount=${retrieveCount}, ` +
851
- `queryEmbeddingDims=${queryEmbedding.length}, ` +
852
- `queryNorm=${Math.sqrt(Array.from(queryEmbedding).reduce((s, v) => s + v * v, 0)).toFixed(4)}`);
853
- const rows = this.stmtSearchVec.all(Buffer.from(queryEmbedding.buffer), retrieveCount);
854
- this.logger?.debug?.(`${TAG} [L1-search] vec0 returned ${rows.length} candidate(s)`);
855
- if (rows.length === 0)
856
- return [];
857
- const results = [];
858
- for (const { record_id, distance } of rows) {
859
- // sqlite-vec returns null distance for zero vectors (cosine undefined when ‖v‖=0).
860
- // Skip these — they are placeholder vectors from embedding-service-unavailable fallback.
861
- if (distance == null || Number.isNaN(distance)) {
862
- this.logger?.warn(`${TAG} [L1-search] record_id=${record_id} has null/NaN distance (likely zero vector) — skipping`);
863
- continue;
864
- }
865
- const meta = this.stmtGetMeta.get(record_id);
866
- if (!meta) {
867
- this.logger?.warn(`${TAG} [L1-search] record_id=${record_id} has vector but NO metadata (orphan)`);
868
- continue;
869
- }
870
- const score = 1.0 - distance;
871
- this.logger?.debug?.(`${TAG} [L1-search] HIT id=${record_id}, distance=${distance.toFixed(4)}, score=${score.toFixed(4)}, ` +
872
- `type=${meta.type}, content="${meta.content.slice(0, 60)}..."`);
873
- results.push({
874
- record_id,
875
- content: meta.content,
876
- type: meta.type,
877
- priority: meta.priority,
878
- scene_name: meta.scene_name,
879
- score,
880
- timestamp_str: meta.timestamp_str,
881
- timestamp_start: meta.timestamp_start,
882
- timestamp_end: meta.timestamp_end,
883
- session_key: meta.session_key,
884
- session_id: meta.session_id,
885
- metadata_json: meta.metadata_json,
886
- });
887
- }
888
- // Trim back to the caller's requested topK (we over-fetched above).
889
- const trimmed = results.slice(0, topK);
890
- this.logger?.info(`${TAG} [L1-search] DONE returning ${trimmed.length} result(s) (from ${results.length} valid, ${rows.length} raw)`);
891
- return trimmed;
892
- }
893
- catch (err) {
894
- this.logger?.warn(`${TAG} [L1-search] FAILED (non-fatal, returning empty): ${err instanceof Error ? err.message : String(err)}`);
895
- return [];
896
- }
897
- }
898
- /**
899
- * Delete a single record (metadata + vector).
900
- *
901
- * **Fault-tolerant**: logs a warning on failure, never throws.
902
- */
903
- deleteL1(recordId) {
904
- if (this.degraded)
905
- return false;
906
- try {
907
- this.db.exec("BEGIN");
908
- try {
909
- this.stmtDeleteMeta.run(recordId);
910
- if (this.vecTablesReady)
911
- this.stmtDeleteVec.run(recordId);
912
- if (this.ftsAvailable) {
913
- try {
914
- this.stmtL1FtsDelete.run(recordId);
915
- }
916
- catch { /* non-fatal */ }
917
- }
918
- this.db.exec("COMMIT");
919
- }
920
- catch (err) {
921
- try {
922
- this.db.exec("ROLLBACK");
923
- }
924
- catch { /* ignore rollback errors */ }
925
- throw err;
926
- }
927
- return true;
928
- }
929
- catch (err) {
930
- this.logger?.warn(`${TAG} delete failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
931
- return false;
932
- }
933
- }
934
- /**
935
- * Delete multiple records (metadata + vector).
936
- *
937
- * **Fault-tolerant**: logs a warning on failure, never throws.
938
- */
939
- deleteL1Batch(recordIds) {
940
- if (this.degraded)
941
- return false;
942
- if (recordIds.length === 0)
943
- return true;
944
- try {
945
- this.db.exec("BEGIN");
946
- try {
947
- for (const id of recordIds) {
948
- this.stmtDeleteMeta.run(id);
949
- if (this.vecTablesReady)
950
- this.stmtDeleteVec.run(id);
951
- if (this.ftsAvailable) {
952
- try {
953
- this.stmtL1FtsDelete.run(id);
954
- }
955
- catch { /* non-fatal */ }
956
- }
957
- }
958
- this.db.exec("COMMIT");
959
- }
960
- catch (err) {
961
- try {
962
- this.db.exec("ROLLBACK");
963
- }
964
- catch { /* ignore rollback errors */ }
965
- throw err;
966
- }
967
- return true;
968
- }
969
- catch (err) {
970
- this.logger?.warn(`${TAG} deleteBatch failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
971
- return false;
972
- }
973
- }
974
- /**
975
- * Get the total number of L1 records in the store.
976
- *
977
- * **Fault-tolerant**: returns 0 on failure.
978
- * TTL cleanup by updated_time.
979
- *
980
- * Deletes expired rows from l1_records and matching vectors from l1_vec
981
- * in a single transaction to guarantee consistency.
982
- */
983
- deleteL1Expired(cutoffIso) {
984
- if (this.degraded) {
985
- this.logger?.warn(`${TAG} [deleteExpired] SKIPPED (degraded mode)`);
986
- return 0;
987
- }
988
- try {
989
- const row = this.db.prepare("SELECT COUNT(*) AS cnt FROM l1_records WHERE updated_time != '' AND updated_time < ?").get(cutoffIso);
990
- const expiredCount = row?.cnt ?? 0;
991
- if (expiredCount <= 0)
992
- return 0;
993
- // Ratio protection: refuse to delete > 80% in one pass
994
- const totalRow = this.db.prepare("SELECT COUNT(*) AS cnt FROM l1_records").get();
995
- const total = totalRow.cnt;
996
- const ratio = total > 0 ? expiredCount / total : 0;
997
- if (ratio > 0.8) {
998
- this.logger?.warn(`${TAG} [L1-deleteExpired] BLOCKED: would delete ${expiredCount}/${total} ` +
999
- `(${(ratio * 100).toFixed(1)}%) — exceeds 80% safety threshold, cutoff=${cutoffIso}`);
1000
- return 0;
1001
- }
1002
- this.db.exec("BEGIN");
1003
- try {
1004
- if (this.vecTablesReady) {
1005
- this.db.prepare("DELETE FROM l1_vec WHERE updated_time != '' AND updated_time < ?").run(cutoffIso);
1006
- }
1007
- this.db.prepare("DELETE FROM l1_records WHERE updated_time != '' AND updated_time < ?").run(cutoffIso);
1008
- this.db.exec("COMMIT");
1009
- this.logger?.info?.(`${TAG} [L1-deleteExpired] Deleted ${expiredCount}/${total} records (cutoff=${cutoffIso})`);
1010
- return expiredCount;
1011
- }
1012
- catch (err) {
1013
- try {
1014
- this.db.exec("ROLLBACK");
1015
- }
1016
- catch { /* ignore rollback errors */ }
1017
- throw err;
1018
- }
1019
- }
1020
- catch (err) {
1021
- this.logger?.warn(`${TAG} deleteL1ExpiredByUpdatedTime failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
1022
- return 0;
1023
- }
1024
- }
1025
- /**
1026
- * Get the total number of records in the store.
1027
- */
1028
- countL1() {
1029
- if (this.degraded)
1030
- return 0;
1031
- try {
1032
- const row = this.db
1033
- .prepare("SELECT COUNT(*) AS cnt FROM l1_records")
1034
- .get();
1035
- this.logger?.debug?.(`${TAG} [L1-count] total=${row.cnt}`);
1036
- return row.cnt;
1037
- }
1038
- catch (err) {
1039
- this.logger?.warn(`${TAG} count failed (non-fatal, returning 0): ${err instanceof Error ? err.message : String(err)}`);
1040
- return 0;
1041
- }
1042
- }
1043
- /**
1044
- * Query L1 records with optional session and time filters.
1045
- *
1046
- * Uses the composite index `idx_l1_session_updated(session_id, updated_time)`
1047
- * for efficient filtering. All timestamps are compared as UTC ISO 8601 strings.
1048
- *
1049
- * **Fault-tolerant**: returns an empty array on any error (degraded mode, DB issues).
1050
- */
1051
- queryL1Records(filter) {
1052
- if (this.degraded) {
1053
- this.logger?.warn(`${TAG} [L1-query] SKIPPED (degraded mode)`);
1054
- return [];
1055
- }
1056
- try {
1057
- const { sessionKey, sessionId, updatedAfter } = filter ?? {};
1058
- let raw;
1059
- // Priority: sessionId > sessionKey (sessionId is more specific)
1060
- if (sessionId && updatedAfter) {
1061
- raw = this.stmtQueryBySessionIdSince.all(sessionId, updatedAfter);
1062
- }
1063
- else if (sessionId) {
1064
- raw = this.stmtQueryBySessionId.all(sessionId);
1065
- }
1066
- else if (sessionKey && updatedAfter) {
1067
- raw = this.stmtQueryBySessionKeySince.all(sessionKey, updatedAfter);
1068
- }
1069
- else if (sessionKey) {
1070
- raw = this.stmtQueryBySessionKey.all(sessionKey);
1071
- }
1072
- else if (updatedAfter) {
1073
- raw = this.stmtQueryAllSince.all(updatedAfter);
1074
- }
1075
- else {
1076
- raw = this.stmtQueryAll.all();
1077
- }
1078
- // Runtime sanity check: verify first row has expected columns (guards against schema drift)
1079
- if (raw.length > 0 && !("record_id" in raw[0] && "content" in raw[0])) {
1080
- this.logger?.warn(`${TAG} [L1-query] Schema mismatch: first row missing expected columns. ` +
1081
- `Got keys: [${Object.keys(raw[0]).join(", ")}]`);
1082
- return [];
1083
- }
1084
- const rows = raw;
1085
- this.logger?.info(`${TAG} [L1-query] filter={sessionKey=${sessionKey ?? "(all)"}, sessionId=${sessionId ?? "(all)"}, updatedAfter=${updatedAfter ?? "(none)"}}, ` +
1086
- `returned ${rows.length} record(s)`);
1087
- return rows;
1088
- }
1089
- catch (err) {
1090
- this.logger?.warn(`${TAG} [L1-query] FAILED (non-fatal, returning empty): ${err instanceof Error ? err.message : String(err)}`);
1091
- return [];
1092
- }
1093
- }
1094
- // ── L0 operations ──────────────────────────────────
1095
- /**
1096
- * Write or update an L0 single-message record (metadata + vector).
1097
- * Uses a manual transaction for atomicity.
1098
- *
1099
- * If `embedding` is `undefined` or a zero vector (all elements are 0), only
1100
- * the metadata row (`l0_conversations`) is written — the vec0 table
1101
- * (`l0_vec`) is left untouched. This allows callers without an
1102
- * EmbeddingService to still persist metadata + FTS without constructing a
1103
- * throwaway zero-vector, and prevents placeholder zero vectors (from
1104
- * embedding-service failures) from polluting KNN search results.
1105
- *
1106
- * **Fault-tolerant**: catches all errors internally, never throws.
1107
- * Returns `true` on success, `false` on failure (logged as warning).
1108
- */
1109
- upsertL0(record, embedding) {
1110
- if (this.degraded) {
1111
- this.logger?.warn(`${TAG} [L0-upsert] SKIPPED (degraded mode) id=${record.id}`);
1112
- return false;
1113
- }
1114
- try {
1115
- const skipVec = !embedding || embedding.every(v => v === 0) || !this.vecTablesReady;
1116
- this.logger?.debug?.(`${TAG} [L0-upsert] START id=${record.id}, session=${record.sessionKey}, role=${record.role}, ` +
1117
- `text="${record.messageText.slice(0, 60)}..."` +
1118
- (embedding
1119
- ? `, embeddingDims=${embedding.length}, ` +
1120
- `embeddingNorm=${Math.sqrt(Array.from(embedding).reduce((s, v) => s + v * v, 0)).toFixed(4)}` +
1121
- `${skipVec ? " (ZERO VECTOR or vec tables not ready — vec write will be skipped)" : ""}`
1122
- : " (no embedding — metadata-only write)"));
1123
- this.db.exec("BEGIN");
1124
- try {
1125
- this.stmtL0UpsertMeta.run(record.id, record.sessionKey, record.sessionId, record.role, record.messageText, record.recordedAt, record.timestamp);
1126
- if (!skipVec) {
1127
- // vec0 does not support ON CONFLICT → delete then insert
1128
- this.stmtL0DeleteVec.run(record.id);
1129
- this.stmtL0InsertVec.run(record.id, Buffer.from(embedding.buffer), record.recordedAt);
1130
- }
1131
- else {
1132
- this.logger?.debug?.(`${TAG} [L0-upsert] Skipping vec write (${embedding ? "zero vector" : "no embedding"}) id=${record.id}`);
1133
- }
1134
- // Sync FTS5 (delete + re-insert to handle updates)
1135
- if (this.ftsAvailable) {
1136
- try {
1137
- this.stmtL0FtsDelete.run(record.id);
1138
- this.stmtL0FtsInsert.run(tokenizeForFts(record.messageText), // message_text — segmented for indexing
1139
- record.messageText, // message_text_original — raw for display
1140
- record.id, record.sessionKey, record.sessionId, record.role, record.recordedAt, record.timestamp);
1141
- }
1142
- catch (ftsErr) {
1143
- // FTS write failure is non-fatal — log and continue
1144
- this.logger?.warn(`${TAG} [L0-upsert] FTS write failed (non-fatal) id=${record.id}: ${ftsErr instanceof Error ? ftsErr.message : String(ftsErr)}`);
1145
- }
1146
- }
1147
- this.db.exec("COMMIT");
1148
- }
1149
- catch (err) {
1150
- try {
1151
- this.db.exec("ROLLBACK");
1152
- }
1153
- catch { /* ignore rollback errors */ }
1154
- throw err;
1155
- }
1156
- this.logger?.debug?.(`${TAG} [L0-upsert] OK id=${record.id}${skipVec ? " (meta-only)" : ""}`);
1157
- return true;
1158
- }
1159
- catch (err) {
1160
- this.logger?.warn(`${TAG} [L0-upsert] FAILED (non-fatal) id=${record.id}: ${err instanceof Error ? err.message : String(err)}`);
1161
- return false;
1162
- }
1163
- }
1164
- /**
1165
- * Update ONLY the vector embedding for an existing L0 record.
1166
- * The metadata row must already exist in l0_conversations (written by upsertL0).
1167
- *
1168
- * This is used by the background embedding task in auto-capture:
1169
- * 1. upsertL0() writes metadata + FTS synchronously (no embedding)
1170
- * 2. Background task calls embedBatch() then updateL0Embedding() for each record
1171
- *
1172
- * **Fault-tolerant**: catches all errors internally, never throws.
1173
- * Returns `true` on success, `false` on failure.
1174
- */
1175
- updateL0Embedding(recordId, embedding) {
1176
- if (this.degraded || !this.vecTablesReady) {
1177
- return false;
1178
- }
1179
- if (!embedding || embedding.every(v => v === 0)) {
1180
- this.logger?.debug?.(`${TAG} [L0-update-embedding] Skipping zero vector for ${recordId}`);
1181
- return false;
1182
- }
1183
- try {
1184
- // Look up recorded_at from metadata for the vec0 row
1185
- const meta = this.stmtL0GetMeta.get(recordId);
1186
- if (!meta) {
1187
- this.logger?.warn(`${TAG} [L0-update-embedding] No metadata found for ${recordId}, skipping`);
1188
- return false;
1189
- }
1190
- this.db.exec("BEGIN");
1191
- try {
1192
- this.stmtL0DeleteVec.run(recordId);
1193
- this.stmtL0InsertVec.run(recordId, Buffer.from(embedding.buffer), meta.recorded_at);
1194
- this.db.exec("COMMIT");
1195
- }
1196
- catch (err) {
1197
- try {
1198
- this.db.exec("ROLLBACK");
1199
- }
1200
- catch { /* ignore */ }
1201
- throw err;
1202
- }
1203
- return true;
1204
- }
1205
- catch (err) {
1206
- this.logger?.warn(`${TAG} [L0-update-embedding] FAILED (non-fatal) id=${recordId}: ${err instanceof Error ? err.message : String(err)}`);
1207
- return false;
1208
- }
1209
- }
1210
- /**
1211
- * Vector similarity search on L0 individual messages (cosine distance).
1212
- * Returns top-k results sorted by similarity (highest first).
1213
- *
1214
- * **Fault-tolerant**: returns an empty array on any error.
1215
- */
1216
- searchL0Vector(queryEmbedding, topK = 5) {
1217
- if (this.degraded || !this.vecTablesReady) {
1218
- if (this.degraded)
1219
- this.logger?.warn(`${TAG} [L0-search] SKIPPED (degraded mode)`);
1220
- return [];
1221
- }
1222
- try {
1223
- // Over-retrieve to compensate for legacy zero-vector placeholders that
1224
- // may still exist in the vec0 table. New zero vectors are no longer
1225
- // inserted (upsertL0() skips vec write for zero vectors since v3.x), but
1226
- // older data may still contain them — they surface as NULL/NaN distance
1227
- // in KNN results.
1228
- // NOTE: "AND distance IS NOT NULL" is NOT usable because vec0 does not
1229
- // support that constraint — it causes an empty result set.
1230
- const retrieveCount = topK + VectorStore.ZERO_VEC_BUFFER;
1231
- this.logger?.debug?.(`${TAG} [L0-search] START topK=${topK}, retrieveCount=${retrieveCount}, ` +
1232
- `queryEmbeddingDims=${queryEmbedding.length}, ` +
1233
- `queryNorm=${Math.sqrt(Array.from(queryEmbedding).reduce((s, v) => s + v * v, 0)).toFixed(4)}`);
1234
- const rows = this.stmtL0SearchVec.all(Buffer.from(queryEmbedding.buffer), retrieveCount);
1235
- this.logger?.debug?.(`${TAG} [L0-search] vec0 returned ${rows.length} candidate(s)`);
1236
- if (rows.length === 0)
1237
- return [];
1238
- const results = [];
1239
- for (const { record_id, distance } of rows) {
1240
- // sqlite-vec returns null distance for zero vectors (cosine undefined when ‖v‖=0).
1241
- // Skip these — they are placeholder vectors from embedding-service-unavailable fallback.
1242
- if (distance == null || Number.isNaN(distance)) {
1243
- this.logger?.warn(`${TAG} [L0-search] record_id=${record_id} has null/NaN distance (likely zero vector) — skipping`);
1244
- continue;
1245
- }
1246
- const meta = this.stmtL0GetMeta.get(record_id);
1247
- if (!meta) {
1248
- this.logger?.warn(`${TAG} [L0-search] record_id=${record_id} has vector but NO metadata (orphan)`);
1249
- continue;
1250
- }
1251
- const score = 1.0 - distance;
1252
- this.logger?.debug?.(`${TAG} [L0-search] HIT id=${record_id}, distance=${distance.toFixed(4)}, score=${score.toFixed(4)}, ` +
1253
- `role=${meta.role}, session=${meta.session_key}, text="${meta.message_text.slice(0, 60)}..."`);
1254
- results.push({
1255
- record_id,
1256
- session_key: meta.session_key,
1257
- session_id: meta.session_id,
1258
- role: meta.role,
1259
- message_text: meta.message_text,
1260
- score,
1261
- recorded_at: meta.recorded_at,
1262
- timestamp: meta.timestamp ?? 0,
1263
- });
1264
- }
1265
- // Trim back to the caller's requested topK (we over-fetched above).
1266
- const trimmed = results.slice(0, topK);
1267
- this.logger?.info(`${TAG} [L0-search] DONE returning ${trimmed.length} result(s) (from ${results.length} valid, ${rows.length} raw)`);
1268
- return trimmed;
1269
- }
1270
- catch (err) {
1271
- this.logger?.warn(`${TAG} [L0-search] FAILED (non-fatal, returning empty): ${err instanceof Error ? err.message : String(err)}`);
1272
- return [];
1273
- }
1274
- }
1275
- /**
1276
- * Delete a single L0 record (metadata + vector).
1277
- *
1278
- * **Fault-tolerant**: logs a warning on failure, never throws.
1279
- */
1280
- deleteL0(recordId) {
1281
- if (this.degraded)
1282
- return false;
1283
- try {
1284
- this.db.exec("BEGIN");
1285
- try {
1286
- this.stmtL0DeleteMeta.run(recordId);
1287
- if (this.vecTablesReady)
1288
- this.stmtL0DeleteVec.run(recordId);
1289
- if (this.ftsAvailable) {
1290
- try {
1291
- this.stmtL0FtsDelete.run(recordId);
1292
- }
1293
- catch { /* non-fatal */ }
1294
- }
1295
- this.db.exec("COMMIT");
1296
- }
1297
- catch (err) {
1298
- try {
1299
- this.db.exec("ROLLBACK");
1300
- }
1301
- catch { /* ignore rollback errors */ }
1302
- throw err;
1303
- }
1304
- return true;
1305
- }
1306
- catch (err) {
1307
- this.logger?.warn(`${TAG} deleteL0 failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
1308
- return false;
1309
- }
1310
- }
1311
- /**
1312
- * TTL cleanup by recorded_at (ISO string) for L0 records.
1313
- *
1314
- * Deletes expired rows from l0_conversations and matching vectors from l0_vec
1315
- * in a single transaction to guarantee consistency.
1316
- */
1317
- deleteL0Expired(cutoffIso) {
1318
- if (this.degraded) {
1319
- this.logger?.warn(`${TAG} [deleteExpiredL0] SKIPPED (degraded mode)`);
1320
- return 0;
1321
- }
1322
- try {
1323
- const row = this.db.prepare("SELECT COUNT(*) AS cnt FROM l0_conversations WHERE recorded_at != '' AND recorded_at < ?").get(cutoffIso);
1324
- const expiredCount = row?.cnt ?? 0;
1325
- if (expiredCount <= 0)
1326
- return 0;
1327
- // Ratio protection: refuse to delete > 80% in one pass
1328
- const totalRow = this.db.prepare("SELECT COUNT(*) AS cnt FROM l0_conversations").get();
1329
- const total = totalRow.cnt;
1330
- const ratio = total > 0 ? expiredCount / total : 0;
1331
- if (ratio > 0.8) {
1332
- this.logger?.warn(`${TAG} [L0-deleteExpired] BLOCKED: would delete ${expiredCount}/${total} ` +
1333
- `(${(ratio * 100).toFixed(1)}%) — exceeds 80% safety threshold, cutoff=${cutoffIso}`);
1334
- return 0;
1335
- }
1336
- this.db.exec("BEGIN");
1337
- try {
1338
- if (this.vecTablesReady) {
1339
- this.db.prepare("DELETE FROM l0_vec WHERE recorded_at != '' AND recorded_at < ?").run(cutoffIso);
1340
- }
1341
- this.db.prepare("DELETE FROM l0_conversations WHERE recorded_at != '' AND recorded_at < ?").run(cutoffIso);
1342
- this.db.exec("COMMIT");
1343
- this.logger?.info?.(`${TAG} [L0-deleteExpired] Deleted ${expiredCount}/${total} records (cutoff=${cutoffIso})`);
1344
- return expiredCount;
1345
- }
1346
- catch (err) {
1347
- try {
1348
- this.db.exec("ROLLBACK");
1349
- }
1350
- catch { /* ignore rollback errors */ }
1351
- throw err;
1352
- }
1353
- }
1354
- catch (err) {
1355
- this.logger?.warn(`${TAG} deleteL0ExpiredByRecordedAt failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
1356
- return 0;
1357
- }
1358
- }
1359
- /**
1360
- * Get the total number of L0 message records in the store.
1361
- *
1362
- * **Fault-tolerant**: returns 0 on failure.
1363
- */
1364
- countL0() {
1365
- if (this.degraded)
1366
- return 0;
1367
- try {
1368
- const row = this.db
1369
- .prepare("SELECT COUNT(*) AS cnt FROM l0_conversations")
1370
- .get();
1371
- this.logger?.debug?.(`${TAG} [L0-count] total=${row.cnt}`);
1372
- return row.cnt;
1373
- }
1374
- catch (err) {
1375
- this.logger?.warn(`${TAG} countL0 failed (non-fatal, returning 0): ${err instanceof Error ? err.message : String(err)}`);
1376
- return 0;
1377
- }
1378
- }
1379
- // ── Re-index operations ──────────────────────────────────
1380
- /**
1381
- * Get all L1 record texts for re-embedding.
1382
- * Returns record_id → content pairs.
1383
- */
1384
- getAllL1Texts() {
1385
- if (this.degraded)
1386
- return [];
1387
- try {
1388
- return this.db
1389
- .prepare("SELECT record_id, content, updated_time FROM l1_records")
1390
- .all();
1391
- }
1392
- catch (err) {
1393
- this.logger?.warn(`${TAG} getAllL1Texts failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
1394
- return [];
1395
- }
1396
- }
1397
- /**
1398
- * Get all L0 message texts for re-embedding.
1399
- * Returns record_id → message_text/recorded_at tuples.
1400
- */
1401
- getAllL0Texts() {
1402
- if (this.degraded)
1403
- return [];
1404
- try {
1405
- return this.db
1406
- .prepare("SELECT record_id, message_text, recorded_at FROM l0_conversations")
1407
- .all();
1408
- }
1409
- catch (err) {
1410
- this.logger?.warn(`${TAG} getAllL0Texts failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
1411
- return [];
1412
- }
1413
- }
1414
- /**
1415
- * Re-embed all existing L1 and L0 texts with a new embedding function.
1416
- *
1417
- * This is called after `init()` returns `needsReindex: true` — the vector
1418
- * tables have already been dropped and re-created with the correct dimensions.
1419
- * This method reads every text from the metadata tables and writes fresh
1420
- * embeddings into the new vector tables.
1421
- *
1422
- * @param embedFn A function that converts text → Float32Array embedding.
1423
- * @param onProgress Optional callback for progress reporting.
1424
- */
1425
- async reindexAll(embedFn, onProgress) {
1426
- if (this.degraded || !this.vecTablesReady) {
1427
- if (this.degraded)
1428
- this.logger?.warn(`${TAG} reindexAll skipped: VectorStore is in degraded mode`);
1429
- return { l1Count: 0, l0Count: 0 };
1430
- }
1431
- try {
1432
- // ── Re-embed L1 ──
1433
- const l1Rows = this.getAllL1Texts();
1434
- let l1Done = 0;
1435
- for (const { record_id, content, updated_time } of l1Rows) {
1436
- try {
1437
- const embedding = await embedFn(content);
1438
- // Wrap delete+insert in a transaction to prevent orphan vectors
1439
- this.db.exec("BEGIN");
1440
- try {
1441
- this.stmtDeleteVec.run(record_id);
1442
- this.stmtInsertVec.run(record_id, Buffer.from(embedding.buffer), updated_time);
1443
- this.db.exec("COMMIT");
1444
- }
1445
- catch (txErr) {
1446
- try {
1447
- this.db.exec("ROLLBACK");
1448
- }
1449
- catch { /* ignore */ }
1450
- throw txErr;
1451
- }
1452
- }
1453
- catch (err) {
1454
- this.logger?.warn?.(`${TAG} reindex L1 skip ${record_id}: ${err instanceof Error ? err.message : String(err)}`);
1455
- }
1456
- l1Done++;
1457
- onProgress?.(l1Done, l1Rows.length, "L1");
1458
- }
1459
- // ── Re-embed L0 ──
1460
- const l0Rows = this.getAllL0Texts();
1461
- let l0Done = 0;
1462
- for (const { record_id, message_text, recorded_at } of l0Rows) {
1463
- try {
1464
- const embedding = await embedFn(message_text);
1465
- // Wrap delete+insert in a transaction to prevent orphan vectors
1466
- this.db.exec("BEGIN");
1467
- try {
1468
- this.stmtL0DeleteVec.run(record_id);
1469
- this.stmtL0InsertVec.run(record_id, Buffer.from(embedding.buffer), recorded_at);
1470
- this.db.exec("COMMIT");
1471
- }
1472
- catch (txErr) {
1473
- try {
1474
- this.db.exec("ROLLBACK");
1475
- }
1476
- catch { /* ignore */ }
1477
- throw txErr;
1478
- }
1479
- }
1480
- catch (err) {
1481
- this.logger?.warn?.(`${TAG} reindex L0 skip ${record_id}: ${err instanceof Error ? err.message : String(err)}`);
1482
- }
1483
- l0Done++;
1484
- onProgress?.(l0Done, l0Rows.length, "L0");
1485
- }
1486
- this.logger?.info(`${TAG} Reindex complete: L1=${l1Done}/${l1Rows.length}, L0=${l0Done}/${l0Rows.length}`);
1487
- return { l1Count: l1Done, l0Count: l0Done };
1488
- }
1489
- catch (err) {
1490
- this.logger?.error(`${TAG} reindexAll failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
1491
- return { l1Count: 0, l0Count: 0 };
1492
- }
1493
- }
1494
- // ── L0 query operations (for L1 runner) ──────────────────────────────────
1495
- /**
1496
- * Query L0 messages for a given session key, optionally filtered by recorded_at cursor.
1497
- * Returns messages ordered by recorded_at ASC (chronological write order).
1498
- *
1499
- * Used by L1 runner to read L0 data from DB instead of JSONL files.
1500
- */
1501
- queryL0ForL1(sessionKey, afterRecordedAtMs, limit = 50) {
1502
- if (this.degraded) {
1503
- this.logger?.warn(`${TAG} [L0-query] SKIPPED (degraded mode)`);
1504
- return [];
1505
- }
1506
- try {
1507
- // Query newest-first (DESC) with LIMIT, then reverse to chronological order
1508
- let rows;
1509
- if (afterRecordedAtMs && afterRecordedAtMs > 0) {
1510
- // Convert epoch ms to ISO string for recorded_at comparison
1511
- const afterRecordedAtIso = new Date(afterRecordedAtMs).toISOString();
1512
- rows = this.stmtL0QueryAfter.all(sessionKey, afterRecordedAtIso, limit);
1513
- }
1514
- else {
1515
- rows = this.stmtL0QueryAll.all(sessionKey, limit);
1516
- }
1517
- this.logger?.info(`${TAG} [L0-query] session=${sessionKey}, afterRecordedAtMs=${afterRecordedAtMs ?? "(all)"}, ` +
1518
- `limit=${limit}, returned ${rows.length} row(s)`);
1519
- // Reverse: SQL returns newest-first (DESC), callers expect chronological order
1520
- return rows.map((r) => ({
1521
- record_id: r.record_id,
1522
- session_key: r.session_key,
1523
- session_id: r.session_id || "",
1524
- role: r.role,
1525
- message_text: r.message_text,
1526
- recorded_at: r.recorded_at || "",
1527
- timestamp: r.timestamp || 0,
1528
- })).reverse();
1529
- }
1530
- catch (err) {
1531
- this.logger?.warn(`${TAG} [L0-query] FAILED (non-fatal, returning empty): ${err instanceof Error ? err.message : String(err)}`);
1532
- return [];
1533
- }
1534
- }
1535
- /**
1536
- * Query L0 messages for a given session key, grouped by session_id.
1537
- * Each group's messages are in chronological order (recorded_at ASC).
1538
- * Groups are sorted by earliest message timestamp.
1539
- *
1540
- * Used by L1 runner to replace readConversationMessagesGroupedBySessionId().
1541
- */
1542
- queryL0GroupedBySessionId(sessionKey, afterRecordedAtMs, limit = 50) {
1543
- if (this.degraded) {
1544
- this.logger?.warn(`${TAG} [L0-query-grouped] SKIPPED (degraded mode)`);
1545
- return [];
1546
- }
1547
- try {
1548
- const rows = this.queryL0ForL1(sessionKey, afterRecordedAtMs, limit);
1549
- // Group by session_id
1550
- const groupMap = new Map();
1551
- for (const row of rows) {
1552
- const sid = row.session_id || "";
1553
- let group = groupMap.get(sid);
1554
- if (!group) {
1555
- group = [];
1556
- groupMap.set(sid, group);
1557
- }
1558
- group.push({
1559
- id: row.record_id,
1560
- role: row.role,
1561
- content: row.message_text,
1562
- timestamp: row.timestamp,
1563
- recordedAtMs: row.recorded_at ? Date.parse(row.recorded_at) || 0 : 0,
1564
- });
1565
- }
1566
- // Convert to array, sorted by earliest message timestamp
1567
- const groups = [];
1568
- for (const [sessionId, messages] of groupMap) {
1569
- if (messages.length > 0) {
1570
- groups.push({ sessionId, messages });
1571
- }
1572
- }
1573
- groups.sort((a, b) => a.messages[0].timestamp - b.messages[0].timestamp);
1574
- this.logger?.info(`${TAG} [L0-query-grouped] session=${sessionKey}, afterRecordedAtMs=${afterRecordedAtMs ?? "(all)"}, ` +
1575
- `${rows.length} messages across ${groups.length} group(s)`);
1576
- return groups;
1577
- }
1578
- catch (err) {
1579
- this.logger?.warn(`${TAG} [L0-query-grouped] FAILED (non-fatal, returning empty): ${err instanceof Error ? err.message : String(err)}`);
1580
- return [];
1581
- }
1582
- }
1583
- // ── Cursor-based pagination for migration ──────────────────
1584
- /**
1585
- * Read a page of L1 records using primary key cursor.
1586
- * Returns rows with `record_id > afterId`, ordered by PK, limited to `pageSize`.
1587
- * Pass `""` as `afterId` for the first page.
1588
- */
1589
- queryL1RecordsCursor(afterId, pageSize) {
1590
- if (this.degraded)
1591
- return [];
1592
- try {
1593
- return this.stmtL1QueryMigrationCursor.all(afterId, pageSize);
1594
- }
1595
- catch (err) {
1596
- this.logger?.warn(`${TAG} [L1-query-cursor] FAILED (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
1597
- return [];
1598
- }
1599
- }
1600
- /**
1601
- * Read a page of L0 records using primary key cursor.
1602
- * Returns rows with `record_id > afterId`, ordered by PK, limited to `pageSize`.
1603
- * Pass `""` as `afterId` for the first page.
1604
- */
1605
- queryL0RecordsCursor(afterId, pageSize) {
1606
- if (this.degraded)
1607
- return [];
1608
- try {
1609
- return this.stmtL0QueryMigrationCursor.all(afterId, pageSize);
1610
- }
1611
- catch (err) {
1612
- this.logger?.warn(`${TAG} [L0-query-cursor] FAILED (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
1613
- return [];
1614
- }
1615
- }
1616
- // ── FTS5 search operations ──────────────────────────────────
1617
- /**
1618
- * Whether FTS5 full-text search is available.
1619
- * When `false`, callers should skip keyword-based recall entirely.
1620
- */
1621
- isFtsAvailable() {
1622
- return this.ftsAvailable;
1623
- }
1624
- /**
1625
- * FTS5 keyword search on L1 records.
1626
- * Returns top-`limit` results sorted by BM25 relevance (highest first).
1627
- *
1628
- * @param ftsQuery A pre-built FTS5 MATCH expression (from `buildFtsQuery()`).
1629
- * @param limit Maximum number of results to return.
1630
- *
1631
- * **Fault-tolerant**: returns an empty array on any error.
1632
- */
1633
- searchL1Fts(ftsQuery, limit = 20) {
1634
- if (this.degraded || !this.ftsAvailable)
1635
- return [];
1636
- try {
1637
- const rows = this.stmtL1FtsSearch.all(ftsQuery, limit);
1638
- return rows.map((r) => ({
1639
- record_id: r.record_id,
1640
- content: r.content,
1641
- type: r.type,
1642
- priority: r.priority,
1643
- scene_name: r.scene_name,
1644
- score: bm25RankToScore(r.rank),
1645
- timestamp_str: r.timestamp_str,
1646
- timestamp_start: r.timestamp_start,
1647
- timestamp_end: r.timestamp_end,
1648
- session_key: r.session_key,
1649
- session_id: r.session_id,
1650
- metadata_json: r.metadata_json,
1651
- }));
1652
- }
1653
- catch (err) {
1654
- this.logger?.warn(`${TAG} [L1-fts-search] FAILED (non-fatal, returning empty): ${err instanceof Error ? err.message : String(err)}`);
1655
- return [];
1656
- }
1657
- }
1658
- /**
1659
- * FTS5 keyword search on L0 conversation messages.
1660
- * Returns top-`limit` results sorted by BM25 relevance (highest first).
1661
- *
1662
- * @param ftsQuery A pre-built FTS5 MATCH expression (from `buildFtsQuery()`).
1663
- * @param limit Maximum number of results to return.
1664
- *
1665
- * **Fault-tolerant**: returns an empty array on any error.
1666
- */
1667
- searchL0Fts(ftsQuery, limit = VectorStore.FTS_DEFAULT_LIMIT) {
1668
- if (this.degraded || !this.ftsAvailable)
1669
- return [];
1670
- try {
1671
- const rows = this.stmtL0FtsSearch.all(ftsQuery, limit);
1672
- return rows.map((r) => ({
1673
- record_id: r.record_id,
1674
- session_key: r.session_key,
1675
- session_id: r.session_id,
1676
- role: r.role,
1677
- message_text: r.message_text,
1678
- score: bm25RankToScore(r.rank),
1679
- recorded_at: r.recorded_at,
1680
- timestamp: r.timestamp ?? 0,
1681
- }));
1682
- }
1683
- catch (err) {
1684
- this.logger?.warn(`${TAG} [L0-fts-search] FAILED (non-fatal, returning empty): ${err instanceof Error ? err.message : String(err)}`);
1685
- return [];
1686
- }
1687
- }
1688
- // ── FTS5 migration & rebuild ──────────────────────────────────────────────
1689
- /**
1690
- * Detect old FTS5 v1 schema (no `content_original` column) and drop the
1691
- * tables so they can be recreated with the v2 schema.
1692
- *
1693
- * FTS5 virtual tables do NOT support `ALTER TABLE ADD COLUMN`, so the only
1694
- * migration path is DROP + recreate + repopulate.
1695
- *
1696
- * @returns `true` if migration was performed (= FTS index needs rebuilding).
1697
- * @internal
1698
- */
1699
- migrateFtsTablesIfNeeded() {
1700
- try {
1701
- // Check if l1_fts exists at all
1702
- const l1Exists = this.db
1703
- .prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='l1_fts'")
1704
- .get();
1705
- if (!l1Exists) {
1706
- // Fresh install — tables will be created with v2 schema.
1707
- // Still need rebuild if there's existing data in l1_records.
1708
- const hasData = this.db.prepare("SELECT 1 FROM l1_records LIMIT 1").get();
1709
- return !!hasData;
1710
- }
1711
- // Check if the v2 column `content_original` exists.
1712
- // FTS5 tables appear in pragma_table_info with their column names.
1713
- const cols = this.db
1714
- .prepare("SELECT name FROM pragma_table_info('l1_fts')")
1715
- .all();
1716
- const hasV2Col = cols.some((c) => c.name === "content_original");
1717
- if (hasV2Col) {
1718
- return false; // Already v2 — no migration needed
1719
- }
1720
- // v1 → v2: drop both FTS tables (data will be repopulated by rebuildFtsIndex)
1721
- this.logger?.info(`${TAG} Migrating FTS5 tables from v1 to v2 (jieba segmented)`);
1722
- this.db.exec("DROP TABLE IF EXISTS l1_fts");
1723
- this.db.exec("DROP TABLE IF EXISTS l0_fts");
1724
- return true;
1725
- }
1726
- catch (err) {
1727
- this.logger?.warn(`${TAG} FTS migration check failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
1728
- return false;
1729
- }
1730
- }
1731
- /**
1732
- * Rebuild the FTS5 index from scratch by reading all records from the
1733
- * metadata tables and re-inserting them with jieba-segmented text.
1734
- *
1735
- * Called automatically after:
1736
- * - Schema migration from v1 to v2
1737
- * - Fresh table creation when existing data exists
1738
- *
1739
- * Safe to call multiple times (idempotent — clears FTS tables first).
1740
- */
1741
- rebuildFtsIndex() {
1742
- if (!this.ftsAvailable)
1743
- return;
1744
- try {
1745
- this.logger?.info(`${TAG} Rebuilding FTS5 index with jieba segmentation…`);
1746
- // ── Rebuild L1 FTS ──
1747
- // Clear existing FTS data
1748
- this.db.exec("DELETE FROM l1_fts");
1749
- // Read all L1 records from metadata table
1750
- const l1Rows = this.db
670
+ `);
671
+ this.logger?.debug?.(`${TAG} Initialized (dimensions=${this.dimensions})`);
672
+ return { needsReindex, reason: reindexReason };
673
+ }
674
+ // ── Embedding meta helpers ──────────────────────────────
675
+ readEmbeddingMeta() {
676
+ try {
677
+ const row = this.db
678
+ .prepare("SELECT value FROM embedding_meta WHERE key = ?")
679
+ .get("embedding_provider_info");
680
+ if (!row)
681
+ return null;
682
+ return JSON.parse(row.value);
683
+ }
684
+ catch {
685
+ return null;
686
+ }
687
+ }
688
+ writeEmbeddingMeta(meta) {
689
+ this.db.prepare("INSERT INTO embedding_meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value=excluded.value").run("embedding_provider_info", JSON.stringify(meta));
690
+ }
691
+ /** Allowed table names for row counting (whitelist to prevent SQL injection). */
692
+ static COUNTABLE_TABLES = new Set(["l1_records", "l0_conversations"]);
693
+ /**
694
+ * Extra rows to retrieve from vec0 KNN search to compensate for legacy
695
+ * zero-vector placeholders that may still linger from older data.
696
+ */
697
+ static ZERO_VEC_BUFFER = 10;
698
+ /** Default result limit for FTS5 keyword searches. */
699
+ static FTS_DEFAULT_LIMIT = 20;
700
+ tableRowCount(table) {
701
+ if (!VectorStore.COUNTABLE_TABLES.has(table)) {
702
+ this.logger?.warn(`${TAG} tableRowCount: rejected unknown table name "${table}"`);
703
+ return 0;
704
+ }
705
+ try {
706
+ const row = this.db
707
+ .prepare(`SELECT COUNT(*) AS cnt FROM ${table}`)
708
+ .get();
709
+ return row?.cnt ?? 0;
710
+ }
711
+ catch {
712
+ return 0;
713
+ }
714
+ }
715
+ /**
716
+ * Detect the embedding dimension of an existing vec0 table by inspecting
717
+ * the DDL stored in sqlite_master. Returns `null` if the table doesn't
718
+ * exist or the dimension cannot be determined.
719
+ *
720
+ * The vec0 DDL looks like:
721
+ * CREATE VIRTUAL TABLE l1_vec USING vec0(... embedding float[768] ...)
722
+ * We parse the number inside `float[N]`.
723
+ */
724
+ getVecTableDimensions() {
725
+ try {
726
+ const row = this.db
727
+ .prepare("SELECT sql FROM sqlite_master WHERE type='table' AND name=?")
728
+ .get("l1_vec");
729
+ if (!row?.sql)
730
+ return null;
731
+ const match = row.sql.match(/float\[(\d+)\]/);
732
+ return match ? Number(match[1]) : null;
733
+ }
734
+ catch {
735
+ return null;
736
+ }
737
+ }
738
+ /**
739
+ * Drop both L1 and L0 vector virtual tables.
740
+ * Metadata tables (l1_records, l0_conversations) are preserved — only
741
+ * the vec0 tables need to be rebuilt with the new dimensions.
742
+ */
743
+ dropVectorTables() {
744
+ this.db.exec("DROP TABLE IF EXISTS l1_vec");
745
+ this.db.exec("DROP TABLE IF EXISTS l0_vec");
746
+ this.logger?.info(`${TAG} Dropped vector tables (l1_vec, l0_vec)`);
747
+ }
748
+ /**
749
+ * Write or update a memory record (metadata + vector).
750
+ * Uses a manual transaction for atomicity.
751
+ *
752
+ * If `embedding` is `undefined` or a zero vector (all elements are 0), only
753
+ * the metadata row is written — the vec0 table is left untouched. This
754
+ * allows callers without an EmbeddingService to still persist metadata + FTS
755
+ * without constructing a throwaway zero-vector, and prevents placeholder
756
+ * zero vectors (from embedding-service failures) from polluting KNN search
757
+ * results with null / NaN distances.
758
+ *
759
+ * **Fault-tolerant**: catches all errors internally so that a vector store
760
+ * failure never propagates to the caller / main OpenClaw flow.
761
+ * Returns `true` on success, `false` on failure (logged as warning).
762
+ */
763
+ upsertL1(record, embedding) {
764
+ if (this.degraded) {
765
+ this.logger?.warn(`${TAG} [L1-upsert] SKIPPED (degraded mode) id=${record.id}`);
766
+ return false;
767
+ }
768
+ try {
769
+ const { id: recordId, timestamps } = record;
770
+ const tsStr = timestamps[0] ?? "";
771
+ const tsStart = timestamps.length > 0
772
+ ? timestamps.reduce((a, b) => (a < b ? a : b))
773
+ : tsStr;
774
+ const tsEnd = timestamps.length > 0
775
+ ? timestamps.reduce((a, b) => (a > b ? a : b))
776
+ : tsStr;
777
+ const skipVec = !embedding || embedding.every(v => v === 0) || !this.vecTablesReady;
778
+ this.logger?.debug?.(`${TAG} [L1-upsert] START id=${recordId}, type=${record.type}, ` +
779
+ `content="${record.content.slice(0, 60)}..."` +
780
+ (embedding
781
+ ? `, embeddingDims=${embedding.length}, ` +
782
+ `embeddingNorm=${Math.sqrt(Array.from(embedding).reduce((s, v) => s + v * v, 0)).toFixed(4)}` +
783
+ `${skipVec ? " (ZERO VECTOR or vec tables not ready — vec write will be skipped)" : ""}`
784
+ : " (no embedding — metadata-only write)"));
785
+ this.db.exec("BEGIN");
786
+ try {
787
+ // Upsert metadata (INSERT OR UPDATE)
788
+ this.stmtUpsertMeta.run(recordId, record.content, record.type, record.priority, record.scene_name, record.sessionKey, record.sessionId, tsStr, tsStart, tsEnd, record.createdAt, record.updatedAt, JSON.stringify(record.metadata));
789
+ if (!skipVec) {
790
+ // vec0 does not support ON CONFLICT → delete then insert
791
+ this.stmtDeleteVec.run(recordId);
792
+ this.stmtInsertVec.run(recordId, Buffer.from(embedding.buffer), record.updatedAt);
793
+ }
794
+ else {
795
+ this.logger?.debug?.(`${TAG} [L1-upsert] Skipping vec write (${embedding ? "zero vector" : "no embedding"}) id=${recordId}`);
796
+ }
797
+ // Sync FTS5 (delete + re-insert to handle updates)
798
+ if (this.ftsAvailable) {
799
+ try {
800
+ this.stmtL1FtsDelete.run(recordId);
801
+ this.stmtL1FtsInsert.run(tokenizeForFts(record.content), // content — segmented for indexing
802
+ record.content, // content_original — raw for display
803
+ recordId, record.type, record.priority, record.scene_name, record.sessionKey, record.sessionId, tsStr, tsStart, tsEnd, JSON.stringify(record.metadata));
804
+ }
805
+ catch (ftsErr) {
806
+ // FTS write failure is non-fatal — log and continue
807
+ this.logger?.warn(`${TAG} [L1-upsert] FTS write failed (non-fatal) id=${recordId}: ${ftsErr instanceof Error ? ftsErr.message : String(ftsErr)}`);
808
+ }
809
+ }
810
+ this.db.exec("COMMIT");
811
+ }
812
+ catch (err) {
813
+ try {
814
+ this.db.exec("ROLLBACK");
815
+ }
816
+ catch { /* ignore rollback errors */ }
817
+ throw err;
818
+ }
819
+ this.logger?.debug?.(`${TAG} [L1-upsert] OK id=${recordId}${skipVec ? " (meta-only)" : ""}`);
820
+ return true;
821
+ }
822
+ catch (err) {
823
+ this.logger?.warn(`${TAG} [L1-upsert] FAILED (non-fatal) id=${record.id}: ${err instanceof Error ? err.message : String(err)}`);
824
+ return false;
825
+ }
826
+ }
827
+ /**
828
+ * Vector similarity search (cosine distance).
829
+ * Returns top-k results sorted by similarity (highest first).
830
+ *
831
+ * **Fault-tolerant**: returns an empty array on any error (e.g. dimension
832
+ * mismatch, corrupted DB) so callers can fall back to keyword search.
833
+ */
834
+ searchL1Vector(queryEmbedding, topK = 5) {
835
+ if (this.degraded || !this.vecTablesReady) {
836
+ if (this.degraded)
837
+ this.logger?.warn(`${TAG} [L1-search] SKIPPED (degraded mode)`);
838
+ return [];
839
+ }
840
+ try {
841
+ // Over-retrieve to compensate for legacy zero-vector placeholders that
842
+ // may still exist in the vec0 table. New zero vectors are no longer
843
+ // inserted (upsert() skips vec write for zero vectors since v3.x), but
844
+ // older data may still contain them — they surface as NULL/NaN distance
845
+ // in KNN results. A small buffer of 10 is sufficient for remnants.
846
+ // NOTE: "AND distance IS NOT NULL" is NOT usable because vec0 does not
847
+ // support that constraint — it causes an empty result set.
848
+ const ZERO_VEC_BUFFER = 10;
849
+ const retrieveCount = topK + ZERO_VEC_BUFFER;
850
+ this.logger?.debug?.(`${TAG} [L1-search] START topK=${topK}, retrieveCount=${retrieveCount}, ` +
851
+ `queryEmbeddingDims=${queryEmbedding.length}, ` +
852
+ `queryNorm=${Math.sqrt(Array.from(queryEmbedding).reduce((s, v) => s + v * v, 0)).toFixed(4)}`);
853
+ const rows = this.stmtSearchVec.all(Buffer.from(queryEmbedding.buffer), retrieveCount);
854
+ this.logger?.debug?.(`${TAG} [L1-search] vec0 returned ${rows.length} candidate(s)`);
855
+ if (rows.length === 0)
856
+ return [];
857
+ const results = [];
858
+ for (const { record_id, distance } of rows) {
859
+ // sqlite-vec returns null distance for zero vectors (cosine undefined when ‖v‖=0).
860
+ // Skip these — they are placeholder vectors from embedding-service-unavailable fallback.
861
+ if (distance == null || Number.isNaN(distance)) {
862
+ this.logger?.warn(`${TAG} [L1-search] record_id=${record_id} has null/NaN distance (likely zero vector) — skipping`);
863
+ continue;
864
+ }
865
+ const meta = this.stmtGetMeta.get(record_id);
866
+ if (!meta) {
867
+ this.logger?.warn(`${TAG} [L1-search] record_id=${record_id} has vector but NO metadata (orphan)`);
868
+ continue;
869
+ }
870
+ const score = 1.0 - distance;
871
+ this.logger?.debug?.(`${TAG} [L1-search] HIT id=${record_id}, distance=${distance.toFixed(4)}, score=${score.toFixed(4)}, ` +
872
+ `type=${meta.type}, content="${meta.content.slice(0, 60)}..."`);
873
+ results.push({
874
+ record_id,
875
+ content: meta.content,
876
+ type: meta.type,
877
+ priority: meta.priority,
878
+ scene_name: meta.scene_name,
879
+ score,
880
+ timestamp_str: meta.timestamp_str,
881
+ timestamp_start: meta.timestamp_start,
882
+ timestamp_end: meta.timestamp_end,
883
+ session_key: meta.session_key,
884
+ session_id: meta.session_id,
885
+ metadata_json: meta.metadata_json,
886
+ });
887
+ }
888
+ // Trim back to the caller's requested topK (we over-fetched above).
889
+ const trimmed = results.slice(0, topK);
890
+ this.logger?.info(`${TAG} [L1-search] DONE returning ${trimmed.length} result(s) (from ${results.length} valid, ${rows.length} raw)`);
891
+ return trimmed;
892
+ }
893
+ catch (err) {
894
+ this.logger?.warn(`${TAG} [L1-search] FAILED (non-fatal, returning empty): ${err instanceof Error ? err.message : String(err)}`);
895
+ return [];
896
+ }
897
+ }
898
+ /**
899
+ * Delete a single record (metadata + vector).
900
+ *
901
+ * **Fault-tolerant**: logs a warning on failure, never throws.
902
+ */
903
+ deleteL1(recordId) {
904
+ if (this.degraded)
905
+ return false;
906
+ try {
907
+ this.db.exec("BEGIN");
908
+ try {
909
+ this.stmtDeleteMeta.run(recordId);
910
+ if (this.vecTablesReady)
911
+ this.stmtDeleteVec.run(recordId);
912
+ if (this.ftsAvailable) {
913
+ try {
914
+ this.stmtL1FtsDelete.run(recordId);
915
+ }
916
+ catch { /* non-fatal */ }
917
+ }
918
+ this.db.exec("COMMIT");
919
+ }
920
+ catch (err) {
921
+ try {
922
+ this.db.exec("ROLLBACK");
923
+ }
924
+ catch { /* ignore rollback errors */ }
925
+ throw err;
926
+ }
927
+ return true;
928
+ }
929
+ catch (err) {
930
+ this.logger?.warn(`${TAG} delete failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
931
+ return false;
932
+ }
933
+ }
934
+ /**
935
+ * Delete multiple records (metadata + vector).
936
+ *
937
+ * **Fault-tolerant**: logs a warning on failure, never throws.
938
+ */
939
+ deleteL1Batch(recordIds) {
940
+ if (this.degraded)
941
+ return false;
942
+ if (recordIds.length === 0)
943
+ return true;
944
+ try {
945
+ this.db.exec("BEGIN");
946
+ try {
947
+ for (const id of recordIds) {
948
+ this.stmtDeleteMeta.run(id);
949
+ if (this.vecTablesReady)
950
+ this.stmtDeleteVec.run(id);
951
+ if (this.ftsAvailable) {
952
+ try {
953
+ this.stmtL1FtsDelete.run(id);
954
+ }
955
+ catch { /* non-fatal */ }
956
+ }
957
+ }
958
+ this.db.exec("COMMIT");
959
+ }
960
+ catch (err) {
961
+ try {
962
+ this.db.exec("ROLLBACK");
963
+ }
964
+ catch { /* ignore rollback errors */ }
965
+ throw err;
966
+ }
967
+ return true;
968
+ }
969
+ catch (err) {
970
+ this.logger?.warn(`${TAG} deleteBatch failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
971
+ return false;
972
+ }
973
+ }
974
+ /**
975
+ * Get the total number of L1 records in the store.
976
+ *
977
+ * **Fault-tolerant**: returns 0 on failure.
978
+ * TTL cleanup by updated_time.
979
+ *
980
+ * Deletes expired rows from l1_records and matching vectors from l1_vec
981
+ * in a single transaction to guarantee consistency.
982
+ */
983
+ deleteL1Expired(cutoffIso) {
984
+ if (this.degraded) {
985
+ this.logger?.warn(`${TAG} [deleteExpired] SKIPPED (degraded mode)`);
986
+ return 0;
987
+ }
988
+ try {
989
+ const row = this.db.prepare("SELECT COUNT(*) AS cnt FROM l1_records WHERE updated_time != '' AND updated_time < ?").get(cutoffIso);
990
+ const expiredCount = row?.cnt ?? 0;
991
+ if (expiredCount <= 0)
992
+ return 0;
993
+ // Ratio protection: refuse to delete > 80% in one pass
994
+ const totalRow = this.db.prepare("SELECT COUNT(*) AS cnt FROM l1_records").get();
995
+ const total = totalRow.cnt;
996
+ const ratio = total > 0 ? expiredCount / total : 0;
997
+ if (ratio > 0.8) {
998
+ this.logger?.warn(`${TAG} [L1-deleteExpired] BLOCKED: would delete ${expiredCount}/${total} ` +
999
+ `(${(ratio * 100).toFixed(1)}%) — exceeds 80% safety threshold, cutoff=${cutoffIso}`);
1000
+ return 0;
1001
+ }
1002
+ this.db.exec("BEGIN");
1003
+ try {
1004
+ if (this.vecTablesReady) {
1005
+ this.db.prepare("DELETE FROM l1_vec WHERE updated_time != '' AND updated_time < ?").run(cutoffIso);
1006
+ }
1007
+ this.db.prepare("DELETE FROM l1_records WHERE updated_time != '' AND updated_time < ?").run(cutoffIso);
1008
+ this.db.exec("COMMIT");
1009
+ this.logger?.info?.(`${TAG} [L1-deleteExpired] Deleted ${expiredCount}/${total} records (cutoff=${cutoffIso})`);
1010
+ return expiredCount;
1011
+ }
1012
+ catch (err) {
1013
+ try {
1014
+ this.db.exec("ROLLBACK");
1015
+ }
1016
+ catch { /* ignore rollback errors */ }
1017
+ throw err;
1018
+ }
1019
+ }
1020
+ catch (err) {
1021
+ this.logger?.warn(`${TAG} deleteL1ExpiredByUpdatedTime failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
1022
+ return 0;
1023
+ }
1024
+ }
1025
+ /**
1026
+ * Get the total number of records in the store.
1027
+ */
1028
+ countL1() {
1029
+ if (this.degraded)
1030
+ return 0;
1031
+ try {
1032
+ const row = this.db
1033
+ .prepare("SELECT COUNT(*) AS cnt FROM l1_records")
1034
+ .get();
1035
+ this.logger?.debug?.(`${TAG} [L1-count] total=${row.cnt}`);
1036
+ return row.cnt;
1037
+ }
1038
+ catch (err) {
1039
+ this.logger?.warn(`${TAG} count failed (non-fatal, returning 0): ${err instanceof Error ? err.message : String(err)}`);
1040
+ return 0;
1041
+ }
1042
+ }
1043
+ /**
1044
+ * Query L1 records with optional session and time filters.
1045
+ *
1046
+ * Uses the composite index `idx_l1_session_updated(session_id, updated_time)`
1047
+ * for efficient filtering. All timestamps are compared as UTC ISO 8601 strings.
1048
+ *
1049
+ * **Fault-tolerant**: returns an empty array on any error (degraded mode, DB issues).
1050
+ */
1051
+ queryL1Records(filter) {
1052
+ if (this.degraded) {
1053
+ this.logger?.warn(`${TAG} [L1-query] SKIPPED (degraded mode)`);
1054
+ return [];
1055
+ }
1056
+ try {
1057
+ const { sessionKey, sessionId, updatedAfter } = filter ?? {};
1058
+ let raw;
1059
+ // Priority: sessionId > sessionKey (sessionId is more specific)
1060
+ if (sessionId && updatedAfter) {
1061
+ raw = this.stmtQueryBySessionIdSince.all(sessionId, updatedAfter);
1062
+ }
1063
+ else if (sessionId) {
1064
+ raw = this.stmtQueryBySessionId.all(sessionId);
1065
+ }
1066
+ else if (sessionKey && updatedAfter) {
1067
+ raw = this.stmtQueryBySessionKeySince.all(sessionKey, updatedAfter);
1068
+ }
1069
+ else if (sessionKey) {
1070
+ raw = this.stmtQueryBySessionKey.all(sessionKey);
1071
+ }
1072
+ else if (updatedAfter) {
1073
+ raw = this.stmtQueryAllSince.all(updatedAfter);
1074
+ }
1075
+ else {
1076
+ raw = this.stmtQueryAll.all();
1077
+ }
1078
+ // Runtime sanity check: verify first row has expected columns (guards against schema drift)
1079
+ if (raw.length > 0 && !("record_id" in raw[0] && "content" in raw[0])) {
1080
+ this.logger?.warn(`${TAG} [L1-query] Schema mismatch: first row missing expected columns. ` +
1081
+ `Got keys: [${Object.keys(raw[0]).join(", ")}]`);
1082
+ return [];
1083
+ }
1084
+ const rows = raw;
1085
+ this.logger?.info(`${TAG} [L1-query] filter={sessionKey=${sessionKey ?? "(all)"}, sessionId=${sessionId ?? "(all)"}, updatedAfter=${updatedAfter ?? "(none)"}}, ` +
1086
+ `returned ${rows.length} record(s)`);
1087
+ return rows;
1088
+ }
1089
+ catch (err) {
1090
+ this.logger?.warn(`${TAG} [L1-query] FAILED (non-fatal, returning empty): ${err instanceof Error ? err.message : String(err)}`);
1091
+ return [];
1092
+ }
1093
+ }
1094
+ // ── L0 operations ──────────────────────────────────
1095
+ /**
1096
+ * Write or update an L0 single-message record (metadata + vector).
1097
+ * Uses a manual transaction for atomicity.
1098
+ *
1099
+ * If `embedding` is `undefined` or a zero vector (all elements are 0), only
1100
+ * the metadata row (`l0_conversations`) is written — the vec0 table
1101
+ * (`l0_vec`) is left untouched. This allows callers without an
1102
+ * EmbeddingService to still persist metadata + FTS without constructing a
1103
+ * throwaway zero-vector, and prevents placeholder zero vectors (from
1104
+ * embedding-service failures) from polluting KNN search results.
1105
+ *
1106
+ * **Fault-tolerant**: catches all errors internally, never throws.
1107
+ * Returns `true` on success, `false` on failure (logged as warning).
1108
+ */
1109
+ upsertL0(record, embedding) {
1110
+ if (this.degraded) {
1111
+ this.logger?.warn(`${TAG} [L0-upsert] SKIPPED (degraded mode) id=${record.id}`);
1112
+ return false;
1113
+ }
1114
+ try {
1115
+ const skipVec = !embedding || embedding.every(v => v === 0) || !this.vecTablesReady;
1116
+ this.logger?.debug?.(`${TAG} [L0-upsert] START id=${record.id}, session=${record.sessionKey}, role=${record.role}, ` +
1117
+ `text="${record.messageText.slice(0, 60)}..."` +
1118
+ (embedding
1119
+ ? `, embeddingDims=${embedding.length}, ` +
1120
+ `embeddingNorm=${Math.sqrt(Array.from(embedding).reduce((s, v) => s + v * v, 0)).toFixed(4)}` +
1121
+ `${skipVec ? " (ZERO VECTOR or vec tables not ready — vec write will be skipped)" : ""}`
1122
+ : " (no embedding — metadata-only write)"));
1123
+ this.db.exec("BEGIN");
1124
+ try {
1125
+ this.stmtL0UpsertMeta.run(record.id, record.sessionKey, record.sessionId, record.role, record.messageText, record.recordedAt, record.timestamp);
1126
+ if (!skipVec) {
1127
+ // vec0 does not support ON CONFLICT → delete then insert
1128
+ this.stmtL0DeleteVec.run(record.id);
1129
+ this.stmtL0InsertVec.run(record.id, Buffer.from(embedding.buffer), record.recordedAt);
1130
+ }
1131
+ else {
1132
+ this.logger?.debug?.(`${TAG} [L0-upsert] Skipping vec write (${embedding ? "zero vector" : "no embedding"}) id=${record.id}`);
1133
+ }
1134
+ // Sync FTS5 (delete + re-insert to handle updates)
1135
+ if (this.ftsAvailable) {
1136
+ try {
1137
+ this.stmtL0FtsDelete.run(record.id);
1138
+ this.stmtL0FtsInsert.run(tokenizeForFts(record.messageText), // message_text — segmented for indexing
1139
+ record.messageText, // message_text_original — raw for display
1140
+ record.id, record.sessionKey, record.sessionId, record.role, record.recordedAt, record.timestamp);
1141
+ }
1142
+ catch (ftsErr) {
1143
+ // FTS write failure is non-fatal — log and continue
1144
+ this.logger?.warn(`${TAG} [L0-upsert] FTS write failed (non-fatal) id=${record.id}: ${ftsErr instanceof Error ? ftsErr.message : String(ftsErr)}`);
1145
+ }
1146
+ }
1147
+ this.db.exec("COMMIT");
1148
+ }
1149
+ catch (err) {
1150
+ try {
1151
+ this.db.exec("ROLLBACK");
1152
+ }
1153
+ catch { /* ignore rollback errors */ }
1154
+ throw err;
1155
+ }
1156
+ this.logger?.debug?.(`${TAG} [L0-upsert] OK id=${record.id}${skipVec ? " (meta-only)" : ""}`);
1157
+ return true;
1158
+ }
1159
+ catch (err) {
1160
+ this.logger?.warn(`${TAG} [L0-upsert] FAILED (non-fatal) id=${record.id}: ${err instanceof Error ? err.message : String(err)}`);
1161
+ return false;
1162
+ }
1163
+ }
1164
+ /**
1165
+ * Update ONLY the vector embedding for an existing L0 record.
1166
+ * The metadata row must already exist in l0_conversations (written by upsertL0).
1167
+ *
1168
+ * This is used by the background embedding task in auto-capture:
1169
+ * 1. upsertL0() writes metadata + FTS synchronously (no embedding)
1170
+ * 2. Background task calls embedBatch() then updateL0Embedding() for each record
1171
+ *
1172
+ * **Fault-tolerant**: catches all errors internally, never throws.
1173
+ * Returns `true` on success, `false` on failure.
1174
+ */
1175
+ updateL0Embedding(recordId, embedding) {
1176
+ if (this.degraded || !this.vecTablesReady) {
1177
+ return false;
1178
+ }
1179
+ if (!embedding || embedding.every(v => v === 0)) {
1180
+ this.logger?.debug?.(`${TAG} [L0-update-embedding] Skipping zero vector for ${recordId}`);
1181
+ return false;
1182
+ }
1183
+ try {
1184
+ // Look up recorded_at from metadata for the vec0 row
1185
+ const meta = this.stmtL0GetMeta.get(recordId);
1186
+ if (!meta) {
1187
+ this.logger?.warn(`${TAG} [L0-update-embedding] No metadata found for ${recordId}, skipping`);
1188
+ return false;
1189
+ }
1190
+ this.db.exec("BEGIN");
1191
+ try {
1192
+ this.stmtL0DeleteVec.run(recordId);
1193
+ this.stmtL0InsertVec.run(recordId, Buffer.from(embedding.buffer), meta.recorded_at);
1194
+ this.db.exec("COMMIT");
1195
+ }
1196
+ catch (err) {
1197
+ try {
1198
+ this.db.exec("ROLLBACK");
1199
+ }
1200
+ catch { /* ignore */ }
1201
+ throw err;
1202
+ }
1203
+ return true;
1204
+ }
1205
+ catch (err) {
1206
+ this.logger?.warn(`${TAG} [L0-update-embedding] FAILED (non-fatal) id=${recordId}: ${err instanceof Error ? err.message : String(err)}`);
1207
+ return false;
1208
+ }
1209
+ }
1210
+ /**
1211
+ * Vector similarity search on L0 individual messages (cosine distance).
1212
+ * Returns top-k results sorted by similarity (highest first).
1213
+ *
1214
+ * **Fault-tolerant**: returns an empty array on any error.
1215
+ */
1216
+ searchL0Vector(queryEmbedding, topK = 5) {
1217
+ if (this.degraded || !this.vecTablesReady) {
1218
+ if (this.degraded)
1219
+ this.logger?.warn(`${TAG} [L0-search] SKIPPED (degraded mode)`);
1220
+ return [];
1221
+ }
1222
+ try {
1223
+ // Over-retrieve to compensate for legacy zero-vector placeholders that
1224
+ // may still exist in the vec0 table. New zero vectors are no longer
1225
+ // inserted (upsertL0() skips vec write for zero vectors since v3.x), but
1226
+ // older data may still contain them — they surface as NULL/NaN distance
1227
+ // in KNN results.
1228
+ // NOTE: "AND distance IS NOT NULL" is NOT usable because vec0 does not
1229
+ // support that constraint — it causes an empty result set.
1230
+ const retrieveCount = topK + VectorStore.ZERO_VEC_BUFFER;
1231
+ this.logger?.debug?.(`${TAG} [L0-search] START topK=${topK}, retrieveCount=${retrieveCount}, ` +
1232
+ `queryEmbeddingDims=${queryEmbedding.length}, ` +
1233
+ `queryNorm=${Math.sqrt(Array.from(queryEmbedding).reduce((s, v) => s + v * v, 0)).toFixed(4)}`);
1234
+ const rows = this.stmtL0SearchVec.all(Buffer.from(queryEmbedding.buffer), retrieveCount);
1235
+ this.logger?.debug?.(`${TAG} [L0-search] vec0 returned ${rows.length} candidate(s)`);
1236
+ if (rows.length === 0)
1237
+ return [];
1238
+ const results = [];
1239
+ for (const { record_id, distance } of rows) {
1240
+ // sqlite-vec returns null distance for zero vectors (cosine undefined when ‖v‖=0).
1241
+ // Skip these — they are placeholder vectors from embedding-service-unavailable fallback.
1242
+ if (distance == null || Number.isNaN(distance)) {
1243
+ this.logger?.warn(`${TAG} [L0-search] record_id=${record_id} has null/NaN distance (likely zero vector) — skipping`);
1244
+ continue;
1245
+ }
1246
+ const meta = this.stmtL0GetMeta.get(record_id);
1247
+ if (!meta) {
1248
+ this.logger?.warn(`${TAG} [L0-search] record_id=${record_id} has vector but NO metadata (orphan)`);
1249
+ continue;
1250
+ }
1251
+ const score = 1.0 - distance;
1252
+ this.logger?.debug?.(`${TAG} [L0-search] HIT id=${record_id}, distance=${distance.toFixed(4)}, score=${score.toFixed(4)}, ` +
1253
+ `role=${meta.role}, session=${meta.session_key}, text="${meta.message_text.slice(0, 60)}..."`);
1254
+ results.push({
1255
+ record_id,
1256
+ session_key: meta.session_key,
1257
+ session_id: meta.session_id,
1258
+ role: meta.role,
1259
+ message_text: meta.message_text,
1260
+ score,
1261
+ recorded_at: meta.recorded_at,
1262
+ timestamp: meta.timestamp ?? 0,
1263
+ });
1264
+ }
1265
+ // Trim back to the caller's requested topK (we over-fetched above).
1266
+ const trimmed = results.slice(0, topK);
1267
+ this.logger?.info(`${TAG} [L0-search] DONE returning ${trimmed.length} result(s) (from ${results.length} valid, ${rows.length} raw)`);
1268
+ return trimmed;
1269
+ }
1270
+ catch (err) {
1271
+ this.logger?.warn(`${TAG} [L0-search] FAILED (non-fatal, returning empty): ${err instanceof Error ? err.message : String(err)}`);
1272
+ return [];
1273
+ }
1274
+ }
1275
+ /**
1276
+ * Delete a single L0 record (metadata + vector).
1277
+ *
1278
+ * **Fault-tolerant**: logs a warning on failure, never throws.
1279
+ */
1280
+ deleteL0(recordId) {
1281
+ if (this.degraded)
1282
+ return false;
1283
+ try {
1284
+ this.db.exec("BEGIN");
1285
+ try {
1286
+ this.stmtL0DeleteMeta.run(recordId);
1287
+ if (this.vecTablesReady)
1288
+ this.stmtL0DeleteVec.run(recordId);
1289
+ if (this.ftsAvailable) {
1290
+ try {
1291
+ this.stmtL0FtsDelete.run(recordId);
1292
+ }
1293
+ catch { /* non-fatal */ }
1294
+ }
1295
+ this.db.exec("COMMIT");
1296
+ }
1297
+ catch (err) {
1298
+ try {
1299
+ this.db.exec("ROLLBACK");
1300
+ }
1301
+ catch { /* ignore rollback errors */ }
1302
+ throw err;
1303
+ }
1304
+ return true;
1305
+ }
1306
+ catch (err) {
1307
+ this.logger?.warn(`${TAG} deleteL0 failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
1308
+ return false;
1309
+ }
1310
+ }
1311
+ /**
1312
+ * TTL cleanup by recorded_at (ISO string) for L0 records.
1313
+ *
1314
+ * Deletes expired rows from l0_conversations and matching vectors from l0_vec
1315
+ * in a single transaction to guarantee consistency.
1316
+ */
1317
+ deleteL0Expired(cutoffIso) {
1318
+ if (this.degraded) {
1319
+ this.logger?.warn(`${TAG} [deleteExpiredL0] SKIPPED (degraded mode)`);
1320
+ return 0;
1321
+ }
1322
+ try {
1323
+ const row = this.db.prepare("SELECT COUNT(*) AS cnt FROM l0_conversations WHERE recorded_at != '' AND recorded_at < ?").get(cutoffIso);
1324
+ const expiredCount = row?.cnt ?? 0;
1325
+ if (expiredCount <= 0)
1326
+ return 0;
1327
+ // Ratio protection: refuse to delete > 80% in one pass
1328
+ const totalRow = this.db.prepare("SELECT COUNT(*) AS cnt FROM l0_conversations").get();
1329
+ const total = totalRow.cnt;
1330
+ const ratio = total > 0 ? expiredCount / total : 0;
1331
+ if (ratio > 0.8) {
1332
+ this.logger?.warn(`${TAG} [L0-deleteExpired] BLOCKED: would delete ${expiredCount}/${total} ` +
1333
+ `(${(ratio * 100).toFixed(1)}%) — exceeds 80% safety threshold, cutoff=${cutoffIso}`);
1334
+ return 0;
1335
+ }
1336
+ this.db.exec("BEGIN");
1337
+ try {
1338
+ if (this.vecTablesReady) {
1339
+ this.db.prepare("DELETE FROM l0_vec WHERE recorded_at != '' AND recorded_at < ?").run(cutoffIso);
1340
+ }
1341
+ this.db.prepare("DELETE FROM l0_conversations WHERE recorded_at != '' AND recorded_at < ?").run(cutoffIso);
1342
+ this.db.exec("COMMIT");
1343
+ this.logger?.info?.(`${TAG} [L0-deleteExpired] Deleted ${expiredCount}/${total} records (cutoff=${cutoffIso})`);
1344
+ return expiredCount;
1345
+ }
1346
+ catch (err) {
1347
+ try {
1348
+ this.db.exec("ROLLBACK");
1349
+ }
1350
+ catch { /* ignore rollback errors */ }
1351
+ throw err;
1352
+ }
1353
+ }
1354
+ catch (err) {
1355
+ this.logger?.warn(`${TAG} deleteL0ExpiredByRecordedAt failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
1356
+ return 0;
1357
+ }
1358
+ }
1359
+ /**
1360
+ * Get the total number of L0 message records in the store.
1361
+ *
1362
+ * **Fault-tolerant**: returns 0 on failure.
1363
+ */
1364
+ countL0() {
1365
+ if (this.degraded)
1366
+ return 0;
1367
+ try {
1368
+ const row = this.db
1369
+ .prepare("SELECT COUNT(*) AS cnt FROM l0_conversations")
1370
+ .get();
1371
+ this.logger?.debug?.(`${TAG} [L0-count] total=${row.cnt}`);
1372
+ return row.cnt;
1373
+ }
1374
+ catch (err) {
1375
+ this.logger?.warn(`${TAG} countL0 failed (non-fatal, returning 0): ${err instanceof Error ? err.message : String(err)}`);
1376
+ return 0;
1377
+ }
1378
+ }
1379
+ // ── Re-index operations ──────────────────────────────────
1380
+ /**
1381
+ * Get all L1 record texts for re-embedding.
1382
+ * Returns record_id → content pairs.
1383
+ */
1384
+ getAllL1Texts() {
1385
+ if (this.degraded)
1386
+ return [];
1387
+ try {
1388
+ return this.db
1389
+ .prepare("SELECT record_id, content, updated_time FROM l1_records")
1390
+ .all();
1391
+ }
1392
+ catch (err) {
1393
+ this.logger?.warn(`${TAG} getAllL1Texts failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
1394
+ return [];
1395
+ }
1396
+ }
1397
+ /**
1398
+ * Get all L0 message texts for re-embedding.
1399
+ * Returns record_id → message_text/recorded_at tuples.
1400
+ */
1401
+ getAllL0Texts() {
1402
+ if (this.degraded)
1403
+ return [];
1404
+ try {
1405
+ return this.db
1406
+ .prepare("SELECT record_id, message_text, recorded_at FROM l0_conversations")
1407
+ .all();
1408
+ }
1409
+ catch (err) {
1410
+ this.logger?.warn(`${TAG} getAllL0Texts failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
1411
+ return [];
1412
+ }
1413
+ }
1414
+ /**
1415
+ * Re-embed all existing L1 and L0 texts with a new embedding function.
1416
+ *
1417
+ * This is called after `init()` returns `needsReindex: true` — the vector
1418
+ * tables have already been dropped and re-created with the correct dimensions.
1419
+ * This method reads every text from the metadata tables and writes fresh
1420
+ * embeddings into the new vector tables.
1421
+ *
1422
+ * @param embedFn A function that converts text → Float32Array embedding.
1423
+ * @param onProgress Optional callback for progress reporting.
1424
+ */
1425
+ async reindexAll(embedFn, onProgress) {
1426
+ if (this.degraded || !this.vecTablesReady) {
1427
+ if (this.degraded)
1428
+ this.logger?.warn(`${TAG} reindexAll skipped: VectorStore is in degraded mode`);
1429
+ return { l1Count: 0, l0Count: 0 };
1430
+ }
1431
+ try {
1432
+ // ── Re-embed L1 ──
1433
+ const l1Rows = this.getAllL1Texts();
1434
+ let l1Done = 0;
1435
+ for (const { record_id, content, updated_time } of l1Rows) {
1436
+ try {
1437
+ const embedding = await embedFn(content);
1438
+ // Wrap delete+insert in a transaction to prevent orphan vectors
1439
+ this.db.exec("BEGIN");
1440
+ try {
1441
+ this.stmtDeleteVec.run(record_id);
1442
+ this.stmtInsertVec.run(record_id, Buffer.from(embedding.buffer), updated_time);
1443
+ this.db.exec("COMMIT");
1444
+ }
1445
+ catch (txErr) {
1446
+ try {
1447
+ this.db.exec("ROLLBACK");
1448
+ }
1449
+ catch { /* ignore */ }
1450
+ throw txErr;
1451
+ }
1452
+ }
1453
+ catch (err) {
1454
+ this.logger?.warn?.(`${TAG} reindex L1 skip ${record_id}: ${err instanceof Error ? err.message : String(err)}`);
1455
+ }
1456
+ l1Done++;
1457
+ onProgress?.(l1Done, l1Rows.length, "L1");
1458
+ }
1459
+ // ── Re-embed L0 ──
1460
+ const l0Rows = this.getAllL0Texts();
1461
+ let l0Done = 0;
1462
+ for (const { record_id, message_text, recorded_at } of l0Rows) {
1463
+ try {
1464
+ const embedding = await embedFn(message_text);
1465
+ // Wrap delete+insert in a transaction to prevent orphan vectors
1466
+ this.db.exec("BEGIN");
1467
+ try {
1468
+ this.stmtL0DeleteVec.run(record_id);
1469
+ this.stmtL0InsertVec.run(record_id, Buffer.from(embedding.buffer), recorded_at);
1470
+ this.db.exec("COMMIT");
1471
+ }
1472
+ catch (txErr) {
1473
+ try {
1474
+ this.db.exec("ROLLBACK");
1475
+ }
1476
+ catch { /* ignore */ }
1477
+ throw txErr;
1478
+ }
1479
+ }
1480
+ catch (err) {
1481
+ this.logger?.warn?.(`${TAG} reindex L0 skip ${record_id}: ${err instanceof Error ? err.message : String(err)}`);
1482
+ }
1483
+ l0Done++;
1484
+ onProgress?.(l0Done, l0Rows.length, "L0");
1485
+ }
1486
+ this.logger?.info(`${TAG} Reindex complete: L1=${l1Done}/${l1Rows.length}, L0=${l0Done}/${l0Rows.length}`);
1487
+ return { l1Count: l1Done, l0Count: l0Done };
1488
+ }
1489
+ catch (err) {
1490
+ this.logger?.error(`${TAG} reindexAll failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
1491
+ return { l1Count: 0, l0Count: 0 };
1492
+ }
1493
+ }
1494
+ // ── L0 query operations (for L1 runner) ──────────────────────────────────
1495
+ /**
1496
+ * Query L0 messages for a given session key, optionally filtered by recorded_at cursor.
1497
+ * Returns messages ordered by recorded_at ASC (chronological write order).
1498
+ *
1499
+ * Used by L1 runner to read L0 data from DB instead of JSONL files.
1500
+ */
1501
+ queryL0ForL1(sessionKey, afterRecordedAtMs, limit = 50) {
1502
+ if (this.degraded) {
1503
+ this.logger?.warn(`${TAG} [L0-query] SKIPPED (degraded mode)`);
1504
+ return [];
1505
+ }
1506
+ try {
1507
+ // Query newest-first (DESC) with LIMIT, then reverse to chronological order
1508
+ let rows;
1509
+ if (afterRecordedAtMs && afterRecordedAtMs > 0) {
1510
+ // Convert epoch ms to ISO string for recorded_at comparison
1511
+ const afterRecordedAtIso = new Date(afterRecordedAtMs).toISOString();
1512
+ rows = this.stmtL0QueryAfter.all(sessionKey, afterRecordedAtIso, limit);
1513
+ }
1514
+ else {
1515
+ rows = this.stmtL0QueryAll.all(sessionKey, limit);
1516
+ }
1517
+ this.logger?.info(`${TAG} [L0-query] session=${sessionKey}, afterRecordedAtMs=${afterRecordedAtMs ?? "(all)"}, ` +
1518
+ `limit=${limit}, returned ${rows.length} row(s)`);
1519
+ // Reverse: SQL returns newest-first (DESC), callers expect chronological order
1520
+ return rows.map((r) => ({
1521
+ record_id: r.record_id,
1522
+ session_key: r.session_key,
1523
+ session_id: r.session_id || "",
1524
+ role: r.role,
1525
+ message_text: r.message_text,
1526
+ recorded_at: r.recorded_at || "",
1527
+ timestamp: r.timestamp || 0,
1528
+ })).reverse();
1529
+ }
1530
+ catch (err) {
1531
+ this.logger?.warn(`${TAG} [L0-query] FAILED (non-fatal, returning empty): ${err instanceof Error ? err.message : String(err)}`);
1532
+ return [];
1533
+ }
1534
+ }
1535
+ /**
1536
+ * Query L0 messages for a given session key, grouped by session_id.
1537
+ * Each group's messages are in chronological order (recorded_at ASC).
1538
+ * Groups are sorted by earliest message timestamp.
1539
+ *
1540
+ * Used by L1 runner to replace readConversationMessagesGroupedBySessionId().
1541
+ */
1542
+ queryL0GroupedBySessionId(sessionKey, afterRecordedAtMs, limit = 50) {
1543
+ if (this.degraded) {
1544
+ this.logger?.warn(`${TAG} [L0-query-grouped] SKIPPED (degraded mode)`);
1545
+ return [];
1546
+ }
1547
+ try {
1548
+ const rows = this.queryL0ForL1(sessionKey, afterRecordedAtMs, limit);
1549
+ // Group by session_id
1550
+ const groupMap = new Map();
1551
+ for (const row of rows) {
1552
+ const sid = row.session_id || "";
1553
+ let group = groupMap.get(sid);
1554
+ if (!group) {
1555
+ group = [];
1556
+ groupMap.set(sid, group);
1557
+ }
1558
+ group.push({
1559
+ id: row.record_id,
1560
+ role: row.role,
1561
+ content: row.message_text,
1562
+ timestamp: row.timestamp,
1563
+ recordedAtMs: row.recorded_at ? Date.parse(row.recorded_at) || 0 : 0,
1564
+ });
1565
+ }
1566
+ // Convert to array, sorted by earliest message timestamp
1567
+ const groups = [];
1568
+ for (const [sessionId, messages] of groupMap) {
1569
+ if (messages.length > 0) {
1570
+ groups.push({ sessionId, messages });
1571
+ }
1572
+ }
1573
+ groups.sort((a, b) => a.messages[0].timestamp - b.messages[0].timestamp);
1574
+ this.logger?.info(`${TAG} [L0-query-grouped] session=${sessionKey}, afterRecordedAtMs=${afterRecordedAtMs ?? "(all)"}, ` +
1575
+ `${rows.length} messages across ${groups.length} group(s)`);
1576
+ return groups;
1577
+ }
1578
+ catch (err) {
1579
+ this.logger?.warn(`${TAG} [L0-query-grouped] FAILED (non-fatal, returning empty): ${err instanceof Error ? err.message : String(err)}`);
1580
+ return [];
1581
+ }
1582
+ }
1583
+ // ── Cursor-based pagination for migration ──────────────────
1584
+ /**
1585
+ * Read a page of L1 records using primary key cursor.
1586
+ * Returns rows with `record_id > afterId`, ordered by PK, limited to `pageSize`.
1587
+ * Pass `""` as `afterId` for the first page.
1588
+ */
1589
+ queryL1RecordsCursor(afterId, pageSize) {
1590
+ if (this.degraded)
1591
+ return [];
1592
+ try {
1593
+ return this.stmtL1QueryMigrationCursor.all(afterId, pageSize);
1594
+ }
1595
+ catch (err) {
1596
+ this.logger?.warn(`${TAG} [L1-query-cursor] FAILED (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
1597
+ return [];
1598
+ }
1599
+ }
1600
+ /**
1601
+ * Read a page of L0 records using primary key cursor.
1602
+ * Returns rows with `record_id > afterId`, ordered by PK, limited to `pageSize`.
1603
+ * Pass `""` as `afterId` for the first page.
1604
+ */
1605
+ queryL0RecordsCursor(afterId, pageSize) {
1606
+ if (this.degraded)
1607
+ return [];
1608
+ try {
1609
+ return this.stmtL0QueryMigrationCursor.all(afterId, pageSize);
1610
+ }
1611
+ catch (err) {
1612
+ this.logger?.warn(`${TAG} [L0-query-cursor] FAILED (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
1613
+ return [];
1614
+ }
1615
+ }
1616
+ // ── FTS5 search operations ──────────────────────────────────
1617
+ /**
1618
+ * Whether FTS5 full-text search is available.
1619
+ * When `false`, callers should skip keyword-based recall entirely.
1620
+ */
1621
+ isFtsAvailable() {
1622
+ return this.ftsAvailable;
1623
+ }
1624
+ /**
1625
+ * FTS5 keyword search on L1 records.
1626
+ * Returns top-`limit` results sorted by BM25 relevance (highest first).
1627
+ *
1628
+ * @param ftsQuery A pre-built FTS5 MATCH expression (from `buildFtsQuery()`).
1629
+ * @param limit Maximum number of results to return.
1630
+ *
1631
+ * **Fault-tolerant**: returns an empty array on any error.
1632
+ */
1633
+ searchL1Fts(ftsQuery, limit = 20) {
1634
+ if (this.degraded || !this.ftsAvailable)
1635
+ return [];
1636
+ try {
1637
+ const rows = this.stmtL1FtsSearch.all(ftsQuery, limit);
1638
+ return rows.map((r) => ({
1639
+ record_id: r.record_id,
1640
+ content: r.content,
1641
+ type: r.type,
1642
+ priority: r.priority,
1643
+ scene_name: r.scene_name,
1644
+ score: bm25RankToScore(r.rank),
1645
+ timestamp_str: r.timestamp_str,
1646
+ timestamp_start: r.timestamp_start,
1647
+ timestamp_end: r.timestamp_end,
1648
+ session_key: r.session_key,
1649
+ session_id: r.session_id,
1650
+ metadata_json: r.metadata_json,
1651
+ }));
1652
+ }
1653
+ catch (err) {
1654
+ this.logger?.warn(`${TAG} [L1-fts-search] FAILED (non-fatal, returning empty): ${err instanceof Error ? err.message : String(err)}`);
1655
+ return [];
1656
+ }
1657
+ }
1658
+ /**
1659
+ * FTS5 keyword search on L0 conversation messages.
1660
+ * Returns top-`limit` results sorted by BM25 relevance (highest first).
1661
+ *
1662
+ * @param ftsQuery A pre-built FTS5 MATCH expression (from `buildFtsQuery()`).
1663
+ * @param limit Maximum number of results to return.
1664
+ *
1665
+ * **Fault-tolerant**: returns an empty array on any error.
1666
+ */
1667
+ searchL0Fts(ftsQuery, limit = VectorStore.FTS_DEFAULT_LIMIT) {
1668
+ if (this.degraded || !this.ftsAvailable)
1669
+ return [];
1670
+ try {
1671
+ const rows = this.stmtL0FtsSearch.all(ftsQuery, limit);
1672
+ return rows.map((r) => ({
1673
+ record_id: r.record_id,
1674
+ session_key: r.session_key,
1675
+ session_id: r.session_id,
1676
+ role: r.role,
1677
+ message_text: r.message_text,
1678
+ score: bm25RankToScore(r.rank),
1679
+ recorded_at: r.recorded_at,
1680
+ timestamp: r.timestamp ?? 0,
1681
+ }));
1682
+ }
1683
+ catch (err) {
1684
+ this.logger?.warn(`${TAG} [L0-fts-search] FAILED (non-fatal, returning empty): ${err instanceof Error ? err.message : String(err)}`);
1685
+ return [];
1686
+ }
1687
+ }
1688
+ // ── FTS5 migration & rebuild ──────────────────────────────────────────────
1689
+ /**
1690
+ * Detect old FTS5 v1 schema (no `content_original` column) and drop the
1691
+ * tables so they can be recreated with the v2 schema.
1692
+ *
1693
+ * FTS5 virtual tables do NOT support `ALTER TABLE ADD COLUMN`, so the only
1694
+ * migration path is DROP + recreate + repopulate.
1695
+ *
1696
+ * @returns `true` if migration was performed (= FTS index needs rebuilding).
1697
+ * @internal
1698
+ */
1699
+ migrateFtsTablesIfNeeded() {
1700
+ try {
1701
+ // Check if l1_fts exists at all
1702
+ const l1Exists = this.db
1703
+ .prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='l1_fts'")
1704
+ .get();
1705
+ if (!l1Exists) {
1706
+ // Fresh install — tables will be created with v2 schema.
1707
+ // Still need rebuild if there's existing data in l1_records.
1708
+ const hasData = this.db.prepare("SELECT 1 FROM l1_records LIMIT 1").get();
1709
+ return !!hasData;
1710
+ }
1711
+ // Check if the v2 column `content_original` exists.
1712
+ // FTS5 tables appear in pragma_table_info with their column names.
1713
+ const cols = this.db
1714
+ .prepare("SELECT name FROM pragma_table_info('l1_fts')")
1715
+ .all();
1716
+ const hasV2Col = cols.some((c) => c.name === "content_original");
1717
+ if (hasV2Col) {
1718
+ return false; // Already v2 — no migration needed
1719
+ }
1720
+ // v1 → v2: drop both FTS tables (data will be repopulated by rebuildFtsIndex)
1721
+ this.logger?.info(`${TAG} Migrating FTS5 tables from v1 to v2 (jieba segmented)`);
1722
+ this.db.exec("DROP TABLE IF EXISTS l1_fts");
1723
+ this.db.exec("DROP TABLE IF EXISTS l0_fts");
1724
+ return true;
1725
+ }
1726
+ catch (err) {
1727
+ this.logger?.warn(`${TAG} FTS migration check failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
1728
+ return false;
1729
+ }
1730
+ }
1731
+ /**
1732
+ * Rebuild the FTS5 index from scratch by reading all records from the
1733
+ * metadata tables and re-inserting them with jieba-segmented text.
1734
+ *
1735
+ * Called automatically after:
1736
+ * - Schema migration from v1 to v2
1737
+ * - Fresh table creation when existing data exists
1738
+ *
1739
+ * Safe to call multiple times (idempotent — clears FTS tables first).
1740
+ */
1741
+ rebuildFtsIndex() {
1742
+ if (!this.ftsAvailable)
1743
+ return;
1744
+ try {
1745
+ this.logger?.info(`${TAG} Rebuilding FTS5 index with jieba segmentation…`);
1746
+ // ── Rebuild L1 FTS ──
1747
+ // Clear existing FTS data
1748
+ this.db.exec("DELETE FROM l1_fts");
1749
+ // Read all L1 records from metadata table
1750
+ const l1Rows = this.db
1751
1751
  .prepare(`
1752
1752
  SELECT record_id, content, type, priority, scene_name,
1753
1753
  session_key, session_id, timestamp_str, timestamp_start, timestamp_end, metadata_json
1754
1754
  FROM l1_records
1755
- `)
1756
- .all();
1757
- let l1Count = 0;
1758
- for (const r of l1Rows) {
1759
- try {
1760
- this.stmtL1FtsInsert.run(tokenizeForFts(r.content), // content — segmented
1761
- r.content, // content_original — raw
1762
- r.record_id, r.type, r.priority, r.scene_name, r.session_key, r.session_id, r.timestamp_str, r.timestamp_start, r.timestamp_end, r.metadata_json);
1763
- l1Count++;
1764
- }
1765
- catch (err) {
1766
- this.logger?.warn?.(`${TAG} FTS rebuild skip L1 ${r.record_id}: ${err instanceof Error ? err.message : String(err)}`);
1767
- }
1768
- }
1769
- // ── Rebuild L0 FTS ──
1770
- this.db.exec("DELETE FROM l0_fts");
1771
- const l0Rows = this.db
1755
+ `)
1756
+ .all();
1757
+ let l1Count = 0;
1758
+ for (const r of l1Rows) {
1759
+ try {
1760
+ this.stmtL1FtsInsert.run(tokenizeForFts(r.content), // content — segmented
1761
+ r.content, // content_original — raw
1762
+ r.record_id, r.type, r.priority, r.scene_name, r.session_key, r.session_id, r.timestamp_str, r.timestamp_start, r.timestamp_end, r.metadata_json);
1763
+ l1Count++;
1764
+ }
1765
+ catch (err) {
1766
+ this.logger?.warn?.(`${TAG} FTS rebuild skip L1 ${r.record_id}: ${err instanceof Error ? err.message : String(err)}`);
1767
+ }
1768
+ }
1769
+ // ── Rebuild L0 FTS ──
1770
+ this.db.exec("DELETE FROM l0_fts");
1771
+ const l0Rows = this.db
1772
1772
  .prepare(`
1773
1773
  SELECT record_id, message_text, session_key, session_id, role, recorded_at, timestamp
1774
1774
  FROM l0_conversations
1775
- `)
1776
- .all();
1777
- let l0Count = 0;
1778
- for (const r of l0Rows) {
1779
- try {
1780
- this.stmtL0FtsInsert.run(tokenizeForFts(r.message_text), // message_text — segmented
1781
- r.message_text, // message_text_original — raw
1782
- r.record_id, r.session_key, r.session_id, r.role, r.recorded_at, r.timestamp);
1783
- l0Count++;
1784
- }
1785
- catch (err) {
1786
- this.logger?.warn?.(`${TAG} FTS rebuild skip L0 ${r.record_id}: ${err instanceof Error ? err.message : String(err)}`);
1787
- }
1788
- }
1789
- this.logger?.info(`${TAG} FTS5 rebuild complete: L1=${l1Count}/${l1Rows.length}, L0=${l0Count}/${l0Rows.length}`);
1790
- }
1791
- catch (err) {
1792
- this.logger?.warn(`${TAG} FTS5 rebuild failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
1793
- }
1794
- }
1795
- // ============================
1796
- // IMemoryStore interface implementation
1797
- // ============================
1798
- /** Query the store's search capabilities. */
1799
- getCapabilities() {
1800
- return {
1801
- vectorSearch: this.vecTablesReady,
1802
- ftsSearch: this.ftsAvailable,
1803
- nativeHybridSearch: false,
1804
- sparseVectors: false,
1805
- };
1806
- }
1807
- /**
1808
- * Close the database connection.
1809
- * Should be called on shutdown. Idempotent — safe to call multiple times.
1810
- */
1811
- close() {
1812
- if (this.closed)
1813
- return;
1814
- this.closed = true;
1815
- try {
1816
- this.db.close();
1817
- }
1818
- catch (err) {
1819
- this.logger?.warn?.(`${TAG} Error closing database: ${err instanceof Error ? err.message : String(err)}`);
1820
- }
1821
- }
1822
- }
1775
+ `)
1776
+ .all();
1777
+ let l0Count = 0;
1778
+ for (const r of l0Rows) {
1779
+ try {
1780
+ this.stmtL0FtsInsert.run(tokenizeForFts(r.message_text), // message_text — segmented
1781
+ r.message_text, // message_text_original — raw
1782
+ r.record_id, r.session_key, r.session_id, r.role, r.recorded_at, r.timestamp);
1783
+ l0Count++;
1784
+ }
1785
+ catch (err) {
1786
+ this.logger?.warn?.(`${TAG} FTS rebuild skip L0 ${r.record_id}: ${err instanceof Error ? err.message : String(err)}`);
1787
+ }
1788
+ }
1789
+ this.logger?.info(`${TAG} FTS5 rebuild complete: L1=${l1Count}/${l1Rows.length}, L0=${l0Count}/${l0Rows.length}`);
1790
+ }
1791
+ catch (err) {
1792
+ this.logger?.warn(`${TAG} FTS5 rebuild failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
1793
+ }
1794
+ }
1795
+ // ============================
1796
+ // IMemoryStore interface implementation
1797
+ // ============================
1798
+ /** Query the store's search capabilities. */
1799
+ getCapabilities() {
1800
+ return {
1801
+ vectorSearch: this.vecTablesReady,
1802
+ ftsSearch: this.ftsAvailable,
1803
+ nativeHybridSearch: false,
1804
+ sparseVectors: false,
1805
+ };
1806
+ }
1807
+ /**
1808
+ * Close the database connection.
1809
+ * Should be called on shutdown. Idempotent — safe to call multiple times.
1810
+ */
1811
+ close() {
1812
+ if (this.closed)
1813
+ return;
1814
+ this.closed = true;
1815
+ try {
1816
+ this.db.close();
1817
+ }
1818
+ catch (err) {
1819
+ this.logger?.warn?.(`${TAG} Error closing database: ${err instanceof Error ? err.message : String(err)}`);
1820
+ }
1821
+ }
1822
+ }