dsh-tdai-memory 0.3.2 → 0.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -21
- package/README.md +9 -1
- package/README.zh.md +8 -1
- package/THIRD-PARTY-NOTICES.md +20 -20
- package/cordis.patch.yml +8 -8
- package/package.json +16 -4
- package/recall-inject.js +86 -86
- package/vendor/tdai/adapters/index.js +15 -15
- package/vendor/tdai/adapters/openclaw/host-adapter.js +83 -83
- package/vendor/tdai/adapters/openclaw/index.js +5 -5
- package/vendor/tdai/adapters/openclaw/llm-runner.js +69 -69
- package/vendor/tdai/adapters/standalone/host-adapter.js +58 -58
- package/vendor/tdai/adapters/standalone/index.js +5 -5
- package/vendor/tdai/core/conversation/l0-recorder.js +420 -420
- package/vendor/tdai/core/hooks/auto-capture.js +236 -236
- package/vendor/tdai/core/hooks/auto-recall.js +640 -640
- package/vendor/tdai/core/index.js +8 -8
- package/vendor/tdai/core/persona/persona-generator.js +184 -184
- package/vendor/tdai/core/persona/persona-trigger.js +94 -94
- package/vendor/tdai/core/profile/profile-sync.js +200 -200
- package/vendor/tdai/core/prompts/l1-dedup.js +72 -72
- package/vendor/tdai/core/prompts/l1-extraction.js +34 -34
- package/vendor/tdai/core/prompts/persona-generation.js +35 -35
- package/vendor/tdai/core/prompts/scene-extraction.js +45 -45
- package/vendor/tdai/core/record/l1-dedup.js +318 -318
- package/vendor/tdai/core/record/l1-extractor.js +379 -379
- package/vendor/tdai/core/record/l1-reader.js +178 -178
- package/vendor/tdai/core/record/l1-writer.js +141 -141
- package/vendor/tdai/core/report/reporter.js +86 -86
- package/vendor/tdai/core/scene/filename-normalizer.js +161 -161
- package/vendor/tdai/core/scene/scene-extractor.js +410 -410
- package/vendor/tdai/core/scene/scene-format.js +53 -53
- package/vendor/tdai/core/scene/scene-index.js +82 -82
- package/vendor/tdai/core/scene/scene-navigation.js +57 -57
- package/vendor/tdai/core/seed/input.js +404 -404
- package/vendor/tdai/core/seed/types.js +9 -9
- package/vendor/tdai/core/store/bm25-client.js +117 -117
- package/vendor/tdai/core/store/bm25-local.js +68 -68
- package/vendor/tdai/core/store/embedding.js +632 -632
- package/vendor/tdai/core/store/factory.js +95 -95
- package/vendor/tdai/core/store/search-utils.js +54 -54
- package/vendor/tdai/core/store/sqlite.js +1632 -1632
- package/vendor/tdai/core/store/tcvdb-client.js +221 -221
- package/vendor/tdai/core/store/tcvdb.js +1047 -1047
- package/vendor/tdai/core/store/types.js +17 -17
- package/vendor/tdai/core/tools/conversation-search.js +199 -199
- package/vendor/tdai/core/tools/memory-search.js +207 -207
- package/vendor/tdai/core/types.js +12 -12
- package/vendor/tdai/utils/backup.js +189 -189
- package/vendor/tdai/utils/checkpoint.js +360 -360
- package/vendor/tdai/utils/clean-context-runner.js +472 -472
- package/vendor/tdai/utils/ensure-hook-policy.js +213 -213
- package/vendor/tdai/utils/env.js +13 -13
- package/vendor/tdai/utils/managed-timer.js +127 -127
- package/vendor/tdai/utils/manifest.js +91 -91
- package/vendor/tdai/utils/memory-cleaner.js +300 -300
- package/vendor/tdai/utils/no-think-fetch.js +109 -109
- package/vendor/tdai/utils/openclaw-state-dir.js +26 -26
- package/vendor/tdai/utils/pipeline-factory.js +517 -517
- package/vendor/tdai/utils/pipeline-manager.js +892 -892
- package/vendor/tdai/utils/sanitize.js +360 -360
- package/vendor/tdai/utils/serial-queue.js +103 -103
- package/vendor/tdai/utils/session-filter.js +88 -88
- package/vendor/tdai/utils/text-utils.js +29 -29
- package/vendor/tdai/utils/time.js +252 -252
|
@@ -1,374 +1,374 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* VectorStore: SQLite-based vector storage using sqlite-vec extension.
|
|
3
|
-
*
|
|
4
|
-
* Manages two layers of vector-indexed data in a single SQLite database:
|
|
5
|
-
*
|
|
6
|
-
* **L1 (structured memories):**
|
|
7
|
-
* 1. `l1_records` — relational metadata table (content, type, priority, scene, timestamps)
|
|
8
|
-
* 2. `l1_vec` — vec0 virtual table for cosine similarity search
|
|
9
|
-
*
|
|
10
|
-
* **L0 (raw conversations):**
|
|
11
|
-
* 3. `l0_conversations` — relational metadata table (session_key, role, message text, timestamps)
|
|
12
|
-
* 4. `l0_vec` — vec0 virtual table for cosine similarity search on individual messages
|
|
13
|
-
*
|
|
14
|
-
* Dependencies: Node.js built-in `node:sqlite` (Node 22+) + `sqlite-vec` (from root workspace).
|
|
15
|
-
*
|
|
16
|
-
* Design:
|
|
17
|
-
* - All operations are synchronous (DatabaseSync API).
|
|
18
|
-
* - Writes use manual BEGIN/COMMIT transactions for atomicity (metadata + vector).
|
|
19
|
-
* - vec0 virtual table does NOT support ON CONFLICT, so upsert = delete + insert.
|
|
20
|
-
* - Thread-safe via WAL mode.
|
|
21
|
-
*/
|
|
22
|
-
import { createRequire } from "node:module";
|
|
23
|
-
const TAG = "[memory-tdai][sqlite]";
|
|
24
|
-
// Use createRequire to load the experimental node:sqlite module
|
|
25
|
-
const require = createRequire(import.meta.url);
|
|
26
|
-
function requireNodeSqlite() {
|
|
27
|
-
return require("node:sqlite");
|
|
28
|
-
}
|
|
29
|
-
let _jieba; // undefined = not yet tried
|
|
30
|
-
function getJieba() {
|
|
31
|
-
if (_jieba !== undefined)
|
|
32
|
-
return _jieba;
|
|
33
|
-
try {
|
|
34
|
-
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
|
35
|
-
const { Jieba } = require("@node-rs/jieba");
|
|
36
|
-
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
|
37
|
-
const { dict } = require("@node-rs/jieba/dict");
|
|
38
|
-
_jieba = Jieba.withDict(dict);
|
|
39
|
-
}
|
|
40
|
-
catch {
|
|
41
|
-
_jieba = null; // mark as unavailable — won't retry
|
|
42
|
-
}
|
|
43
|
-
return _jieba;
|
|
44
|
-
}
|
|
45
|
-
/**
|
|
46
|
-
* Common Chinese stop-words that add noise to FTS5 queries.
|
|
47
|
-
* Kept small on purpose — only high-frequency function words.
|
|
48
|
-
*/
|
|
49
|
-
const ZH_STOP_WORDS = new Set([
|
|
50
|
-
"的", "了", "在", "是", "我", "有", "和", "就", "不", "人", "都", "一",
|
|
51
|
-
"一个", "上", "也", "很", "到", "说", "要", "去", "你", "会", "着",
|
|
52
|
-
"没有", "看", "好", "自己", "这", "他", "她", "它", "们", "那",
|
|
53
|
-
"吗", "吧", "呢", "啊", "呀", "哦", "嗯",
|
|
54
|
-
]);
|
|
55
|
-
/**
|
|
56
|
-
* Build an FTS5 MATCH query from raw text.
|
|
57
|
-
*
|
|
58
|
-
* When `@node-rs/jieba` is available, uses jieba's search-engine mode
|
|
59
|
-
* (`cutForSearch`) for accurate Chinese word segmentation, producing
|
|
60
|
-
* much better recall than the previous regex-only approach.
|
|
61
|
-
*
|
|
62
|
-
* Falls back to Unicode-regex splitting (`/[\p{L}\p{N}_]+/gu`) if
|
|
63
|
-
* jieba is not installed.
|
|
64
|
-
*
|
|
65
|
-
* Tokens are OR-joined as quoted FTS5 phrase terms so that a document
|
|
66
|
-
* matching *any* token is returned. BM25 naturally ranks documents that
|
|
67
|
-
* match more tokens higher, so precision is preserved while recall is
|
|
68
|
-
* significantly improved — especially for longer queries and when running
|
|
69
|
-
* in FTS-only fallback mode (no embedding available).
|
|
70
|
-
*
|
|
71
|
-
* Example (with jieba):
|
|
72
|
-
* "用户喜欢编程和TypeScript" → '"用户" OR "喜欢" OR "编程" OR "TypeScript"'
|
|
73
|
-
* Example (fallback):
|
|
74
|
-
* "旅行计划 API" → '"旅行计划" OR "API"'
|
|
75
|
-
*/
|
|
76
|
-
export function buildFtsQuery(raw) {
|
|
77
|
-
const jieba = getJieba();
|
|
78
|
-
let tokens;
|
|
79
|
-
if (jieba) {
|
|
80
|
-
// jieba cutForSearch: splits long words further for better recall
|
|
81
|
-
// e.g. "北京烤鸭" → ["北京", "烤鸭", "北京烤鸭"]
|
|
82
|
-
tokens = jieba
|
|
83
|
-
.cutForSearch(raw, true)
|
|
84
|
-
.map((t) => t.trim())
|
|
85
|
-
.filter((t) => {
|
|
86
|
-
if (!t)
|
|
87
|
-
return false;
|
|
88
|
-
// Remove pure whitespace / punctuation tokens
|
|
89
|
-
if (!/[\p{L}\p{N}]/u.test(t))
|
|
90
|
-
return false;
|
|
91
|
-
// Remove common Chinese stop-words to reduce noise
|
|
92
|
-
if (ZH_STOP_WORDS.has(t))
|
|
93
|
-
return false;
|
|
94
|
-
return true;
|
|
95
|
-
});
|
|
96
|
-
// Deduplicate (cutForSearch may produce duplicates for sub-words)
|
|
97
|
-
tokens = [...new Set(tokens)];
|
|
98
|
-
}
|
|
99
|
-
else {
|
|
100
|
-
// Fallback: simple Unicode regex split
|
|
101
|
-
tokens =
|
|
102
|
-
raw
|
|
103
|
-
.match(/[\p{L}\p{N}_]+/gu)
|
|
104
|
-
?.map((t) => t.trim())
|
|
105
|
-
.filter(Boolean) ?? [];
|
|
106
|
-
}
|
|
107
|
-
if (tokens.length === 0)
|
|
108
|
-
return null;
|
|
109
|
-
const quoted = tokens.map((t) => `"${t.replaceAll('"', "")}"`);
|
|
110
|
-
return quoted.join(" OR ");
|
|
111
|
-
}
|
|
112
|
-
/**
|
|
113
|
-
* Tokenize text for FTS5 indexing (write-side).
|
|
114
|
-
*
|
|
115
|
-
* Uses jieba `cutForSearch()` (search-engine mode) to segment Chinese text,
|
|
116
|
-
* then joins tokens with spaces. The resulting string is stored in the FTS5
|
|
117
|
-
* `content` column so that `unicode61` tokenizer can split it into meaningful
|
|
118
|
-
* words — including both full words and their sub-words.
|
|
119
|
-
*
|
|
120
|
-
* Using `cutForSearch` (instead of `cut`) ensures that the index contains
|
|
121
|
-
* the same sub-word tokens that `buildFtsQuery()` produces on the query side.
|
|
122
|
-
* For example, "人工智能" is indexed as "人工 智能 人工智能", so queries for
|
|
123
|
-
* either the full term or sub-words will match.
|
|
124
|
-
*
|
|
125
|
-
* Falls back to the original text if jieba is unavailable.
|
|
126
|
-
*
|
|
127
|
-
* Example (with jieba):
|
|
128
|
-
* "用户五月去日本旅行" → "用户 五月 去 日本 旅行"
|
|
129
|
-
* "人工智能的分支" → "人工 智能 人工智能 的 分支"
|
|
130
|
-
* Example (fallback):
|
|
131
|
-
* "用户五月去日本旅行" → "用户五月去日本旅行" (unchanged)
|
|
132
|
-
*/
|
|
133
|
-
export function tokenizeForFts(raw) {
|
|
134
|
-
const jieba = getJieba();
|
|
135
|
-
if (!jieba)
|
|
136
|
-
return raw;
|
|
137
|
-
// Use `cutForSearch` (search-engine mode) for indexing — it produces both
|
|
138
|
-
// full words AND their sub-word components. This ensures that query-side
|
|
139
|
-
// tokens (also produced by `cutForSearch` in `buildFtsQuery`) will always
|
|
140
|
-
// find a match in the index.
|
|
141
|
-
const tokens = jieba.cutForSearch(raw, true);
|
|
142
|
-
// Join with spaces so `unicode61` tokenizer can split them.
|
|
143
|
-
// Punctuation tokens are kept — unicode61 treats them as separators anyway.
|
|
144
|
-
return tokens.join(" ");
|
|
145
|
-
}
|
|
146
|
-
/**
|
|
147
|
-
* Reset jieba state so next call to `buildFtsQuery` re-initialises.
|
|
148
|
-
* Exported for testing only.
|
|
149
|
-
* @internal
|
|
150
|
-
*/
|
|
151
|
-
export function _resetJiebaForTest() {
|
|
152
|
-
_jieba = undefined;
|
|
153
|
-
}
|
|
154
|
-
/**
|
|
155
|
-
* Override jieba instance (or set to `null` to force fallback).
|
|
156
|
-
* Exported for testing only.
|
|
157
|
-
* @internal
|
|
158
|
-
*/
|
|
159
|
-
export function _setJiebaForTest(instance) {
|
|
160
|
-
_jieba = instance;
|
|
161
|
-
}
|
|
162
|
-
/**
|
|
163
|
-
* Convert a BM25 rank (negative = more relevant) to a 0–1 score.
|
|
164
|
-
* Mirrors the formula in openclaw core `hybrid.ts`.
|
|
165
|
-
*/
|
|
166
|
-
export function bm25RankToScore(rank) {
|
|
167
|
-
if (!Number.isFinite(rank))
|
|
168
|
-
return 1 / (1 + 999);
|
|
169
|
-
if (rank < 0) {
|
|
170
|
-
const relevance = -rank;
|
|
171
|
-
return relevance / (1 + relevance);
|
|
172
|
-
}
|
|
173
|
-
return 1 / (1 + rank);
|
|
174
|
-
}
|
|
175
|
-
// ============================
|
|
176
|
-
// VectorStore class
|
|
177
|
-
// ============================
|
|
178
|
-
export class VectorStore {
|
|
179
|
-
db;
|
|
180
|
-
dimensions;
|
|
181
|
-
logger;
|
|
182
|
-
/** @see IMemoryStore.supportsDeferredEmbedding */
|
|
183
|
-
supportsDeferredEmbedding = true;
|
|
184
|
-
/**
|
|
185
|
-
* When `true`, the store is in a degraded state (e.g. sqlite-vec failed to
|
|
186
|
-
* load, or init() encountered an unrecoverable error). All public methods
|
|
187
|
-
* become safe no-ops so the plugin never blocks the main OpenClaw flow.
|
|
188
|
-
*/
|
|
189
|
-
degraded = false;
|
|
190
|
-
/** Tracks whether close() has been called to prevent double-close errors. */
|
|
191
|
-
closed = false;
|
|
192
|
-
/**
|
|
193
|
-
* `true` when vec0 virtual tables (l1_vec / l0_vec) have been created and
|
|
194
|
-
* their prepared statements are ready. When `dimensions === 0` (i.e.
|
|
195
|
-
* provider="none"), vec0 tables are deferred and this stays `false`.
|
|
196
|
-
*/
|
|
197
|
-
vecTablesReady = false;
|
|
198
|
-
// Prepared statements — L1 (initialized in init())
|
|
199
|
-
stmtUpsertMeta;
|
|
200
|
-
stmtDeleteVec; // optional — only set when vecTablesReady
|
|
201
|
-
stmtInsertVec; // optional — only set when vecTablesReady
|
|
202
|
-
stmtDeleteMeta;
|
|
203
|
-
stmtGetMeta;
|
|
204
|
-
stmtSearchVec; // optional — only set when vecTablesReady
|
|
205
|
-
stmtQueryBySessionId;
|
|
206
|
-
stmtQueryBySessionIdSince;
|
|
207
|
-
stmtQueryBySessionKey;
|
|
208
|
-
stmtQueryBySessionKeySince;
|
|
209
|
-
stmtQueryAll;
|
|
210
|
-
stmtQueryAllSince;
|
|
211
|
-
// Prepared statements — L0 (initialized in init())
|
|
212
|
-
stmtL0UpsertMeta;
|
|
213
|
-
stmtL0DeleteVec; // optional — only set when vecTablesReady
|
|
214
|
-
stmtL0InsertVec; // optional — only set when vecTablesReady
|
|
215
|
-
stmtL0DeleteMeta;
|
|
216
|
-
stmtL0GetMeta;
|
|
217
|
-
stmtL0SearchVec; // optional — only set when vecTablesReady
|
|
218
|
-
/** L0 query for L1 runner: all messages for a session key */
|
|
219
|
-
stmtL0QueryAll;
|
|
220
|
-
/** L0 query for L1 runner: messages after a timestamp cursor */
|
|
221
|
-
stmtL0QueryAfter;
|
|
222
|
-
/** L1 cursor-based pagination for migration (by PK) */
|
|
223
|
-
stmtL1QueryMigrationCursor;
|
|
224
|
-
/** L0 cursor-based pagination for migration (by PK) */
|
|
225
|
-
stmtL0QueryMigrationCursor;
|
|
226
|
-
// FTS5 tables availability flag (created best-effort — may be false if fts5 is not compiled in)
|
|
227
|
-
ftsAvailable = false;
|
|
228
|
-
// Prepared statements — FTS5 L1 (initialized in init())
|
|
229
|
-
stmtL1FtsInsert;
|
|
230
|
-
stmtL1FtsDelete;
|
|
231
|
-
stmtL1FtsSearch;
|
|
232
|
-
// Prepared statements — FTS5 L0 (initialized in init())
|
|
233
|
-
stmtL0FtsInsert;
|
|
234
|
-
stmtL0FtsDelete;
|
|
235
|
-
stmtL0FtsSearch;
|
|
236
|
-
/**
|
|
237
|
-
* Create a VectorStore instance.
|
|
238
|
-
*
|
|
239
|
-
* Note: After construction, you MUST call `init()` to load the sqlite-vec
|
|
240
|
-
* extension and create the schema.
|
|
241
|
-
*/
|
|
242
|
-
constructor(dbPath, dimensions, logger) {
|
|
243
|
-
this.dimensions = dimensions;
|
|
244
|
-
this.logger = logger;
|
|
245
|
-
// Open database with extension support enabled
|
|
246
|
-
const { DatabaseSync: DbSync } = requireNodeSqlite();
|
|
247
|
-
this.db = new DbSync(dbPath, { allowExtension: true });
|
|
248
|
-
// Set busy timeout so concurrent processes retry instead of failing with SQLITE_BUSY
|
|
249
|
-
this.db.exec("PRAGMA busy_timeout = 5000");
|
|
250
|
-
// Enable WAL mode for better concurrent read performance
|
|
251
|
-
this.db.exec("PRAGMA journal_mode = WAL");
|
|
252
|
-
// Cap page cache at 64 MB
|
|
253
|
-
this.db.exec("PRAGMA cache_size = -65536");
|
|
254
|
-
// Cap memory-mapped I/O at 128 MB to bound RSS growth
|
|
255
|
-
this.db.exec("PRAGMA mmap_size = 134217728");
|
|
256
|
-
// Auto-checkpoint WAL every 1000 pages (~4 MB) to keep WAL file compact
|
|
257
|
-
this.db.exec("PRAGMA wal_autocheckpoint = 1000");
|
|
258
|
-
}
|
|
259
|
-
/**
|
|
260
|
-
* Whether the store is in degraded mode (e.g. sqlite-vec failed to load).
|
|
261
|
-
* When degraded, all write/search operations become safe no-ops.
|
|
262
|
-
*/
|
|
263
|
-
isDegraded() {
|
|
264
|
-
return this.degraded;
|
|
265
|
-
}
|
|
266
|
-
/**
|
|
267
|
-
* Load sqlite-vec extension and initialize database schema.
|
|
268
|
-
* Must be called once after construction.
|
|
269
|
-
*
|
|
270
|
-
* @param providerInfo Current embedding provider info. When provided,
|
|
271
|
-
* the store compares it against the persisted metadata. If the provider,
|
|
272
|
-
* model, or dimensions changed, the vector tables are dropped and
|
|
273
|
-
* re-created with the new dimensions, and `needsReindex: true` is returned
|
|
274
|
-
* so the caller can schedule a full re-embed.
|
|
275
|
-
*/
|
|
276
|
-
init(providerInfo) {
|
|
277
|
-
// Load sqlite-vec extension (same approach as root project's sqlite-vec.ts)
|
|
278
|
-
try {
|
|
279
|
-
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
|
280
|
-
const sqliteVec = require("sqlite-vec");
|
|
281
|
-
this.db.enableLoadExtension(true);
|
|
282
|
-
sqliteVec.load(this.db);
|
|
283
|
-
}
|
|
284
|
-
catch (err) {
|
|
285
|
-
const message = err instanceof Error ? err.message : String(err);
|
|
286
|
-
this.logger?.error(`${TAG} Failed to load sqlite-vec extension: ${message}. ` +
|
|
287
|
-
`VectorStore entering degraded mode — all operations will be no-ops.`);
|
|
288
|
-
this.degraded = true;
|
|
289
|
-
return { needsReindex: false, reason: `sqlite-vec load failed: ${message}` };
|
|
290
|
-
}
|
|
291
|
-
// ── Schema creation & prepared statements ──────────────────────────────
|
|
292
|
-
// Wrapped in try-catch: if anything fails during schema init (e.g. the DB
|
|
293
|
-
// is corrupted, disk full, etc.), we degrade gracefully instead of crashing.
|
|
294
|
-
try {
|
|
295
|
-
return this.initSchema(providerInfo);
|
|
296
|
-
}
|
|
297
|
-
catch (err) {
|
|
298
|
-
const message = err instanceof Error ? err.message : String(err);
|
|
299
|
-
this.logger?.error(`${TAG} Schema initialization failed: ${message}. ` +
|
|
300
|
-
`VectorStore entering degraded mode.`);
|
|
301
|
-
this.degraded = true;
|
|
302
|
-
return { needsReindex: false, reason: `schema init failed: ${message}` };
|
|
303
|
-
}
|
|
304
|
-
}
|
|
305
|
-
/**
|
|
306
|
-
* Internal schema initialization — separated from init() so we can
|
|
307
|
-
* catch errors at the top level and degrade gracefully.
|
|
308
|
-
*/
|
|
309
|
-
initSchema(providerInfo) {
|
|
310
|
-
// Tracks which provider/model/dimensions were used to generate vectors.
|
|
1
|
+
/**
|
|
2
|
+
* VectorStore: SQLite-based vector storage using sqlite-vec extension.
|
|
3
|
+
*
|
|
4
|
+
* Manages two layers of vector-indexed data in a single SQLite database:
|
|
5
|
+
*
|
|
6
|
+
* **L1 (structured memories):**
|
|
7
|
+
* 1. `l1_records` — relational metadata table (content, type, priority, scene, timestamps)
|
|
8
|
+
* 2. `l1_vec` — vec0 virtual table for cosine similarity search
|
|
9
|
+
*
|
|
10
|
+
* **L0 (raw conversations):**
|
|
11
|
+
* 3. `l0_conversations` — relational metadata table (session_key, role, message text, timestamps)
|
|
12
|
+
* 4. `l0_vec` — vec0 virtual table for cosine similarity search on individual messages
|
|
13
|
+
*
|
|
14
|
+
* Dependencies: Node.js built-in `node:sqlite` (Node 22+) + `sqlite-vec` (from root workspace).
|
|
15
|
+
*
|
|
16
|
+
* Design:
|
|
17
|
+
* - All operations are synchronous (DatabaseSync API).
|
|
18
|
+
* - Writes use manual BEGIN/COMMIT transactions for atomicity (metadata + vector).
|
|
19
|
+
* - vec0 virtual table does NOT support ON CONFLICT, so upsert = delete + insert.
|
|
20
|
+
* - Thread-safe via WAL mode.
|
|
21
|
+
*/
|
|
22
|
+
import { createRequire } from "node:module";
|
|
23
|
+
const TAG = "[memory-tdai][sqlite]";
|
|
24
|
+
// Use createRequire to load the experimental node:sqlite module
|
|
25
|
+
const require = createRequire(import.meta.url);
|
|
26
|
+
function requireNodeSqlite() {
|
|
27
|
+
return require("node:sqlite");
|
|
28
|
+
}
|
|
29
|
+
let _jieba; // undefined = not yet tried
|
|
30
|
+
function getJieba() {
|
|
31
|
+
if (_jieba !== undefined)
|
|
32
|
+
return _jieba;
|
|
33
|
+
try {
|
|
34
|
+
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
|
35
|
+
const { Jieba } = require("@node-rs/jieba");
|
|
36
|
+
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
|
37
|
+
const { dict } = require("@node-rs/jieba/dict");
|
|
38
|
+
_jieba = Jieba.withDict(dict);
|
|
39
|
+
}
|
|
40
|
+
catch {
|
|
41
|
+
_jieba = null; // mark as unavailable — won't retry
|
|
42
|
+
}
|
|
43
|
+
return _jieba;
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* Common Chinese stop-words that add noise to FTS5 queries.
|
|
47
|
+
* Kept small on purpose — only high-frequency function words.
|
|
48
|
+
*/
|
|
49
|
+
const ZH_STOP_WORDS = new Set([
|
|
50
|
+
"的", "了", "在", "是", "我", "有", "和", "就", "不", "人", "都", "一",
|
|
51
|
+
"一个", "上", "也", "很", "到", "说", "要", "去", "你", "会", "着",
|
|
52
|
+
"没有", "看", "好", "自己", "这", "他", "她", "它", "们", "那",
|
|
53
|
+
"吗", "吧", "呢", "啊", "呀", "哦", "嗯",
|
|
54
|
+
]);
|
|
55
|
+
/**
|
|
56
|
+
* Build an FTS5 MATCH query from raw text.
|
|
57
|
+
*
|
|
58
|
+
* When `@node-rs/jieba` is available, uses jieba's search-engine mode
|
|
59
|
+
* (`cutForSearch`) for accurate Chinese word segmentation, producing
|
|
60
|
+
* much better recall than the previous regex-only approach.
|
|
61
|
+
*
|
|
62
|
+
* Falls back to Unicode-regex splitting (`/[\p{L}\p{N}_]+/gu`) if
|
|
63
|
+
* jieba is not installed.
|
|
64
|
+
*
|
|
65
|
+
* Tokens are OR-joined as quoted FTS5 phrase terms so that a document
|
|
66
|
+
* matching *any* token is returned. BM25 naturally ranks documents that
|
|
67
|
+
* match more tokens higher, so precision is preserved while recall is
|
|
68
|
+
* significantly improved — especially for longer queries and when running
|
|
69
|
+
* in FTS-only fallback mode (no embedding available).
|
|
70
|
+
*
|
|
71
|
+
* Example (with jieba):
|
|
72
|
+
* "用户喜欢编程和TypeScript" → '"用户" OR "喜欢" OR "编程" OR "TypeScript"'
|
|
73
|
+
* Example (fallback):
|
|
74
|
+
* "旅行计划 API" → '"旅行计划" OR "API"'
|
|
75
|
+
*/
|
|
76
|
+
export function buildFtsQuery(raw) {
|
|
77
|
+
const jieba = getJieba();
|
|
78
|
+
let tokens;
|
|
79
|
+
if (jieba) {
|
|
80
|
+
// jieba cutForSearch: splits long words further for better recall
|
|
81
|
+
// e.g. "北京烤鸭" → ["北京", "烤鸭", "北京烤鸭"]
|
|
82
|
+
tokens = jieba
|
|
83
|
+
.cutForSearch(raw, true)
|
|
84
|
+
.map((t) => t.trim())
|
|
85
|
+
.filter((t) => {
|
|
86
|
+
if (!t)
|
|
87
|
+
return false;
|
|
88
|
+
// Remove pure whitespace / punctuation tokens
|
|
89
|
+
if (!/[\p{L}\p{N}]/u.test(t))
|
|
90
|
+
return false;
|
|
91
|
+
// Remove common Chinese stop-words to reduce noise
|
|
92
|
+
if (ZH_STOP_WORDS.has(t))
|
|
93
|
+
return false;
|
|
94
|
+
return true;
|
|
95
|
+
});
|
|
96
|
+
// Deduplicate (cutForSearch may produce duplicates for sub-words)
|
|
97
|
+
tokens = [...new Set(tokens)];
|
|
98
|
+
}
|
|
99
|
+
else {
|
|
100
|
+
// Fallback: simple Unicode regex split
|
|
101
|
+
tokens =
|
|
102
|
+
raw
|
|
103
|
+
.match(/[\p{L}\p{N}_]+/gu)
|
|
104
|
+
?.map((t) => t.trim())
|
|
105
|
+
.filter(Boolean) ?? [];
|
|
106
|
+
}
|
|
107
|
+
if (tokens.length === 0)
|
|
108
|
+
return null;
|
|
109
|
+
const quoted = tokens.map((t) => `"${t.replaceAll('"', "")}"`);
|
|
110
|
+
return quoted.join(" OR ");
|
|
111
|
+
}
|
|
112
|
+
/**
|
|
113
|
+
* Tokenize text for FTS5 indexing (write-side).
|
|
114
|
+
*
|
|
115
|
+
* Uses jieba `cutForSearch()` (search-engine mode) to segment Chinese text,
|
|
116
|
+
* then joins tokens with spaces. The resulting string is stored in the FTS5
|
|
117
|
+
* `content` column so that `unicode61` tokenizer can split it into meaningful
|
|
118
|
+
* words — including both full words and their sub-words.
|
|
119
|
+
*
|
|
120
|
+
* Using `cutForSearch` (instead of `cut`) ensures that the index contains
|
|
121
|
+
* the same sub-word tokens that `buildFtsQuery()` produces on the query side.
|
|
122
|
+
* For example, "人工智能" is indexed as "人工 智能 人工智能", so queries for
|
|
123
|
+
* either the full term or sub-words will match.
|
|
124
|
+
*
|
|
125
|
+
* Falls back to the original text if jieba is unavailable.
|
|
126
|
+
*
|
|
127
|
+
* Example (with jieba):
|
|
128
|
+
* "用户五月去日本旅行" → "用户 五月 去 日本 旅行"
|
|
129
|
+
* "人工智能的分支" → "人工 智能 人工智能 的 分支"
|
|
130
|
+
* Example (fallback):
|
|
131
|
+
* "用户五月去日本旅行" → "用户五月去日本旅行" (unchanged)
|
|
132
|
+
*/
|
|
133
|
+
export function tokenizeForFts(raw) {
|
|
134
|
+
const jieba = getJieba();
|
|
135
|
+
if (!jieba)
|
|
136
|
+
return raw;
|
|
137
|
+
// Use `cutForSearch` (search-engine mode) for indexing — it produces both
|
|
138
|
+
// full words AND their sub-word components. This ensures that query-side
|
|
139
|
+
// tokens (also produced by `cutForSearch` in `buildFtsQuery`) will always
|
|
140
|
+
// find a match in the index.
|
|
141
|
+
const tokens = jieba.cutForSearch(raw, true);
|
|
142
|
+
// Join with spaces so `unicode61` tokenizer can split them.
|
|
143
|
+
// Punctuation tokens are kept — unicode61 treats them as separators anyway.
|
|
144
|
+
return tokens.join(" ");
|
|
145
|
+
}
|
|
146
|
+
/**
|
|
147
|
+
* Reset jieba state so next call to `buildFtsQuery` re-initialises.
|
|
148
|
+
* Exported for testing only.
|
|
149
|
+
* @internal
|
|
150
|
+
*/
|
|
151
|
+
export function _resetJiebaForTest() {
|
|
152
|
+
_jieba = undefined;
|
|
153
|
+
}
|
|
154
|
+
/**
|
|
155
|
+
* Override jieba instance (or set to `null` to force fallback).
|
|
156
|
+
* Exported for testing only.
|
|
157
|
+
* @internal
|
|
158
|
+
*/
|
|
159
|
+
export function _setJiebaForTest(instance) {
|
|
160
|
+
_jieba = instance;
|
|
161
|
+
}
|
|
162
|
+
/**
|
|
163
|
+
* Convert a BM25 rank (negative = more relevant) to a 0–1 score.
|
|
164
|
+
* Mirrors the formula in openclaw core `hybrid.ts`.
|
|
165
|
+
*/
|
|
166
|
+
export function bm25RankToScore(rank) {
|
|
167
|
+
if (!Number.isFinite(rank))
|
|
168
|
+
return 1 / (1 + 999);
|
|
169
|
+
if (rank < 0) {
|
|
170
|
+
const relevance = -rank;
|
|
171
|
+
return relevance / (1 + relevance);
|
|
172
|
+
}
|
|
173
|
+
return 1 / (1 + rank);
|
|
174
|
+
}
|
|
175
|
+
// ============================
|
|
176
|
+
// VectorStore class
|
|
177
|
+
// ============================
|
|
178
|
+
export class VectorStore {
|
|
179
|
+
db;
|
|
180
|
+
dimensions;
|
|
181
|
+
logger;
|
|
182
|
+
/** @see IMemoryStore.supportsDeferredEmbedding */
|
|
183
|
+
supportsDeferredEmbedding = true;
|
|
184
|
+
/**
|
|
185
|
+
* When `true`, the store is in a degraded state (e.g. sqlite-vec failed to
|
|
186
|
+
* load, or init() encountered an unrecoverable error). All public methods
|
|
187
|
+
* become safe no-ops so the plugin never blocks the main OpenClaw flow.
|
|
188
|
+
*/
|
|
189
|
+
degraded = false;
|
|
190
|
+
/** Tracks whether close() has been called to prevent double-close errors. */
|
|
191
|
+
closed = false;
|
|
192
|
+
/**
|
|
193
|
+
* `true` when vec0 virtual tables (l1_vec / l0_vec) have been created and
|
|
194
|
+
* their prepared statements are ready. When `dimensions === 0` (i.e.
|
|
195
|
+
* provider="none"), vec0 tables are deferred and this stays `false`.
|
|
196
|
+
*/
|
|
197
|
+
vecTablesReady = false;
|
|
198
|
+
// Prepared statements — L1 (initialized in init())
|
|
199
|
+
stmtUpsertMeta;
|
|
200
|
+
stmtDeleteVec; // optional — only set when vecTablesReady
|
|
201
|
+
stmtInsertVec; // optional — only set when vecTablesReady
|
|
202
|
+
stmtDeleteMeta;
|
|
203
|
+
stmtGetMeta;
|
|
204
|
+
stmtSearchVec; // optional — only set when vecTablesReady
|
|
205
|
+
stmtQueryBySessionId;
|
|
206
|
+
stmtQueryBySessionIdSince;
|
|
207
|
+
stmtQueryBySessionKey;
|
|
208
|
+
stmtQueryBySessionKeySince;
|
|
209
|
+
stmtQueryAll;
|
|
210
|
+
stmtQueryAllSince;
|
|
211
|
+
// Prepared statements — L0 (initialized in init())
|
|
212
|
+
stmtL0UpsertMeta;
|
|
213
|
+
stmtL0DeleteVec; // optional — only set when vecTablesReady
|
|
214
|
+
stmtL0InsertVec; // optional — only set when vecTablesReady
|
|
215
|
+
stmtL0DeleteMeta;
|
|
216
|
+
stmtL0GetMeta;
|
|
217
|
+
stmtL0SearchVec; // optional — only set when vecTablesReady
|
|
218
|
+
/** L0 query for L1 runner: all messages for a session key */
|
|
219
|
+
stmtL0QueryAll;
|
|
220
|
+
/** L0 query for L1 runner: messages after a timestamp cursor */
|
|
221
|
+
stmtL0QueryAfter;
|
|
222
|
+
/** L1 cursor-based pagination for migration (by PK) */
|
|
223
|
+
stmtL1QueryMigrationCursor;
|
|
224
|
+
/** L0 cursor-based pagination for migration (by PK) */
|
|
225
|
+
stmtL0QueryMigrationCursor;
|
|
226
|
+
// FTS5 tables availability flag (created best-effort — may be false if fts5 is not compiled in)
|
|
227
|
+
ftsAvailable = false;
|
|
228
|
+
// Prepared statements — FTS5 L1 (initialized in init())
|
|
229
|
+
stmtL1FtsInsert;
|
|
230
|
+
stmtL1FtsDelete;
|
|
231
|
+
stmtL1FtsSearch;
|
|
232
|
+
// Prepared statements — FTS5 L0 (initialized in init())
|
|
233
|
+
stmtL0FtsInsert;
|
|
234
|
+
stmtL0FtsDelete;
|
|
235
|
+
stmtL0FtsSearch;
|
|
236
|
+
/**
|
|
237
|
+
* Create a VectorStore instance.
|
|
238
|
+
*
|
|
239
|
+
* Note: After construction, you MUST call `init()` to load the sqlite-vec
|
|
240
|
+
* extension and create the schema.
|
|
241
|
+
*/
|
|
242
|
+
constructor(dbPath, dimensions, logger) {
|
|
243
|
+
this.dimensions = dimensions;
|
|
244
|
+
this.logger = logger;
|
|
245
|
+
// Open database with extension support enabled
|
|
246
|
+
const { DatabaseSync: DbSync } = requireNodeSqlite();
|
|
247
|
+
this.db = new DbSync(dbPath, { allowExtension: true });
|
|
248
|
+
// Set busy timeout so concurrent processes retry instead of failing with SQLITE_BUSY
|
|
249
|
+
this.db.exec("PRAGMA busy_timeout = 5000");
|
|
250
|
+
// Enable WAL mode for better concurrent read performance
|
|
251
|
+
this.db.exec("PRAGMA journal_mode = WAL");
|
|
252
|
+
// Cap page cache at 64 MB
|
|
253
|
+
this.db.exec("PRAGMA cache_size = -65536");
|
|
254
|
+
// Cap memory-mapped I/O at 128 MB to bound RSS growth
|
|
255
|
+
this.db.exec("PRAGMA mmap_size = 134217728");
|
|
256
|
+
// Auto-checkpoint WAL every 1000 pages (~4 MB) to keep WAL file compact
|
|
257
|
+
this.db.exec("PRAGMA wal_autocheckpoint = 1000");
|
|
258
|
+
}
|
|
259
|
+
/**
|
|
260
|
+
* Whether the store is in degraded mode (e.g. sqlite-vec failed to load).
|
|
261
|
+
* When degraded, all write/search operations become safe no-ops.
|
|
262
|
+
*/
|
|
263
|
+
isDegraded() {
|
|
264
|
+
return this.degraded;
|
|
265
|
+
}
|
|
266
|
+
/**
|
|
267
|
+
* Load sqlite-vec extension and initialize database schema.
|
|
268
|
+
* Must be called once after construction.
|
|
269
|
+
*
|
|
270
|
+
* @param providerInfo Current embedding provider info. When provided,
|
|
271
|
+
* the store compares it against the persisted metadata. If the provider,
|
|
272
|
+
* model, or dimensions changed, the vector tables are dropped and
|
|
273
|
+
* re-created with the new dimensions, and `needsReindex: true` is returned
|
|
274
|
+
* so the caller can schedule a full re-embed.
|
|
275
|
+
*/
|
|
276
|
+
init(providerInfo) {
|
|
277
|
+
// Load sqlite-vec extension (same approach as root project's sqlite-vec.ts)
|
|
278
|
+
try {
|
|
279
|
+
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
|
280
|
+
const sqliteVec = require("sqlite-vec");
|
|
281
|
+
this.db.enableLoadExtension(true);
|
|
282
|
+
sqliteVec.load(this.db);
|
|
283
|
+
}
|
|
284
|
+
catch (err) {
|
|
285
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
286
|
+
this.logger?.error(`${TAG} Failed to load sqlite-vec extension: ${message}. ` +
|
|
287
|
+
`VectorStore entering degraded mode — all operations will be no-ops.`);
|
|
288
|
+
this.degraded = true;
|
|
289
|
+
return { needsReindex: false, reason: `sqlite-vec load failed: ${message}` };
|
|
290
|
+
}
|
|
291
|
+
// ── Schema creation & prepared statements ──────────────────────────────
|
|
292
|
+
// Wrapped in try-catch: if anything fails during schema init (e.g. the DB
|
|
293
|
+
// is corrupted, disk full, etc.), we degrade gracefully instead of crashing.
|
|
294
|
+
try {
|
|
295
|
+
return this.initSchema(providerInfo);
|
|
296
|
+
}
|
|
297
|
+
catch (err) {
|
|
298
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
299
|
+
this.logger?.error(`${TAG} Schema initialization failed: ${message}. ` +
|
|
300
|
+
`VectorStore entering degraded mode.`);
|
|
301
|
+
this.degraded = true;
|
|
302
|
+
return { needsReindex: false, reason: `schema init failed: ${message}` };
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
/**
|
|
306
|
+
* Internal schema initialization — separated from init() so we can
|
|
307
|
+
* catch errors at the top level and degrade gracefully.
|
|
308
|
+
*/
|
|
309
|
+
initSchema(providerInfo) {
|
|
310
|
+
// Tracks which provider/model/dimensions were used to generate vectors.
|
|
311
311
|
this.db.exec(`
|
|
312
312
|
CREATE TABLE IF NOT EXISTS embedding_meta (
|
|
313
313
|
key TEXT PRIMARY KEY,
|
|
314
314
|
value TEXT NOT NULL
|
|
315
315
|
)
|
|
316
|
-
`);
|
|
317
|
-
// Detect whether re-index is needed
|
|
318
|
-
let needsReindex = false;
|
|
319
|
-
let reindexReason;
|
|
320
|
-
const savedMeta = this.readEmbeddingMeta();
|
|
321
|
-
if (providerInfo) {
|
|
322
|
-
if (savedMeta) {
|
|
323
|
-
const providerChanged = savedMeta.provider !== providerInfo.provider;
|
|
324
|
-
const modelChanged = savedMeta.model !== providerInfo.model;
|
|
325
|
-
const dimsChanged = savedMeta.dimensions !== this.dimensions;
|
|
326
|
-
if (providerChanged || modelChanged || dimsChanged) {
|
|
327
|
-
const reasons = [];
|
|
328
|
-
if (providerChanged)
|
|
329
|
-
reasons.push(`provider: ${savedMeta.provider} → ${providerInfo.provider}`);
|
|
330
|
-
if (modelChanged)
|
|
331
|
-
reasons.push(`model: ${savedMeta.model} → ${providerInfo.model}`);
|
|
332
|
-
if (dimsChanged)
|
|
333
|
-
reasons.push(`dimensions: ${savedMeta.dimensions} → ${this.dimensions}`);
|
|
334
|
-
reindexReason = reasons.join(", ");
|
|
335
|
-
this.logger?.info(`${TAG} Embedding config changed (${reindexReason}). ` +
|
|
336
|
-
`Dropping vector tables for rebuild...`);
|
|
337
|
-
// Drop and re-create vector tables with new dimensions
|
|
338
|
-
this.dropVectorTables();
|
|
339
|
-
needsReindex = true;
|
|
340
|
-
}
|
|
341
|
-
}
|
|
342
|
-
else {
|
|
343
|
-
// No saved meta — first run or legacy DB without meta table.
|
|
344
|
-
// Two cases require dropping vector tables:
|
|
345
|
-
// 1. Existing data created without meta tracking (legacy DB) — need re-embed
|
|
346
|
-
// 2. vec0 tables exist with wrong dimensions (e.g. previously created with
|
|
347
|
-
// provider="none" placeholder 768D, now switching to a real provider
|
|
348
|
-
// with different dimensions) — must rebuild even if data tables are empty
|
|
349
|
-
const l1Count = this.tableRowCount("l1_records");
|
|
350
|
-
const l0Count = this.tableRowCount("l0_conversations");
|
|
351
|
-
const existingVecDims = this.getVecTableDimensions();
|
|
352
|
-
if (l1Count > 0 || l0Count > 0) {
|
|
353
|
-
this.logger?.info(`${TAG} No embedding_meta found but existing data exists ` +
|
|
354
|
-
`(L1=${l1Count}, L0=${l0Count}). Dropping vector tables for safety...`);
|
|
355
|
-
this.dropVectorTables();
|
|
356
|
-
needsReindex = true;
|
|
357
|
-
reindexReason = "legacy DB without embedding_meta — cannot verify vector compatibility";
|
|
358
|
-
}
|
|
359
|
-
else if (existingVecDims !== null && existingVecDims !== this.dimensions) {
|
|
360
|
-
// vec0 tables exist (from a previous provider="none" placeholder or
|
|
361
|
-
// different config) but with mismatched dimensions. Drop them so they
|
|
362
|
-
// get re-created with the correct dimensions below.
|
|
363
|
-
this.logger?.info(`${TAG} vec0 table dimension mismatch (existing=${existingVecDims}, ` +
|
|
364
|
-
`required=${this.dimensions}). Dropping vector tables for rebuild...`);
|
|
365
|
-
this.dropVectorTables();
|
|
366
|
-
// No needsReindex — there's no data to re-embed
|
|
367
|
-
}
|
|
368
|
-
}
|
|
369
|
-
}
|
|
370
|
-
// ── L1 schema ──────────────────────────────────
|
|
371
|
-
// Metadata table
|
|
316
|
+
`);
|
|
317
|
+
// Detect whether re-index is needed
|
|
318
|
+
let needsReindex = false;
|
|
319
|
+
let reindexReason;
|
|
320
|
+
const savedMeta = this.readEmbeddingMeta();
|
|
321
|
+
if (providerInfo) {
|
|
322
|
+
if (savedMeta) {
|
|
323
|
+
const providerChanged = savedMeta.provider !== providerInfo.provider;
|
|
324
|
+
const modelChanged = savedMeta.model !== providerInfo.model;
|
|
325
|
+
const dimsChanged = savedMeta.dimensions !== this.dimensions;
|
|
326
|
+
if (providerChanged || modelChanged || dimsChanged) {
|
|
327
|
+
const reasons = [];
|
|
328
|
+
if (providerChanged)
|
|
329
|
+
reasons.push(`provider: ${savedMeta.provider} → ${providerInfo.provider}`);
|
|
330
|
+
if (modelChanged)
|
|
331
|
+
reasons.push(`model: ${savedMeta.model} → ${providerInfo.model}`);
|
|
332
|
+
if (dimsChanged)
|
|
333
|
+
reasons.push(`dimensions: ${savedMeta.dimensions} → ${this.dimensions}`);
|
|
334
|
+
reindexReason = reasons.join(", ");
|
|
335
|
+
this.logger?.info(`${TAG} Embedding config changed (${reindexReason}). ` +
|
|
336
|
+
`Dropping vector tables for rebuild...`);
|
|
337
|
+
// Drop and re-create vector tables with new dimensions
|
|
338
|
+
this.dropVectorTables();
|
|
339
|
+
needsReindex = true;
|
|
340
|
+
}
|
|
341
|
+
}
|
|
342
|
+
else {
|
|
343
|
+
// No saved meta — first run or legacy DB without meta table.
|
|
344
|
+
// Two cases require dropping vector tables:
|
|
345
|
+
// 1. Existing data created without meta tracking (legacy DB) — need re-embed
|
|
346
|
+
// 2. vec0 tables exist with wrong dimensions (e.g. previously created with
|
|
347
|
+
// provider="none" placeholder 768D, now switching to a real provider
|
|
348
|
+
// with different dimensions) — must rebuild even if data tables are empty
|
|
349
|
+
const l1Count = this.tableRowCount("l1_records");
|
|
350
|
+
const l0Count = this.tableRowCount("l0_conversations");
|
|
351
|
+
const existingVecDims = this.getVecTableDimensions();
|
|
352
|
+
if (l1Count > 0 || l0Count > 0) {
|
|
353
|
+
this.logger?.info(`${TAG} No embedding_meta found but existing data exists ` +
|
|
354
|
+
`(L1=${l1Count}, L0=${l0Count}). Dropping vector tables for safety...`);
|
|
355
|
+
this.dropVectorTables();
|
|
356
|
+
needsReindex = true;
|
|
357
|
+
reindexReason = "legacy DB without embedding_meta — cannot verify vector compatibility";
|
|
358
|
+
}
|
|
359
|
+
else if (existingVecDims !== null && existingVecDims !== this.dimensions) {
|
|
360
|
+
// vec0 tables exist (from a previous provider="none" placeholder or
|
|
361
|
+
// different config) but with mismatched dimensions. Drop them so they
|
|
362
|
+
// get re-created with the correct dimensions below.
|
|
363
|
+
this.logger?.info(`${TAG} vec0 table dimension mismatch (existing=${existingVecDims}, ` +
|
|
364
|
+
`required=${this.dimensions}). Dropping vector tables for rebuild...`);
|
|
365
|
+
this.dropVectorTables();
|
|
366
|
+
// No needsReindex — there's no data to re-embed
|
|
367
|
+
}
|
|
368
|
+
}
|
|
369
|
+
}
|
|
370
|
+
// ── L1 schema ──────────────────────────────────
|
|
371
|
+
// Metadata table
|
|
372
372
|
this.db.exec(`
|
|
373
373
|
CREATE TABLE IF NOT EXISTS l1_records (
|
|
374
374
|
record_id TEXT PRIMARY KEY,
|
|
@@ -385,31 +385,31 @@ export class VectorStore {
|
|
|
385
385
|
updated_time TEXT DEFAULT '',
|
|
386
386
|
metadata_json TEXT DEFAULT '{}'
|
|
387
387
|
)
|
|
388
|
-
`);
|
|
389
|
-
// Indexes for common queries
|
|
390
|
-
this.db.exec("CREATE INDEX IF NOT EXISTS idx_l1_type ON l1_records(type)");
|
|
391
|
-
this.db.exec("CREATE INDEX IF NOT EXISTS idx_l1_session_key ON l1_records(session_key)");
|
|
392
|
-
this.db.exec("CREATE INDEX IF NOT EXISTS idx_l1_session_id ON l1_records(session_id)");
|
|
393
|
-
this.db.exec("CREATE INDEX IF NOT EXISTS idx_l1_scene ON l1_records(scene_name)");
|
|
394
|
-
this.db.exec("CREATE INDEX IF NOT EXISTS idx_l1_ts_start ON l1_records(timestamp_start)");
|
|
395
|
-
this.db.exec("CREATE INDEX IF NOT EXISTS idx_l1_ts_end ON l1_records(timestamp_end)");
|
|
396
|
-
// Composite index: session_id exact match + updated_time range scan (for incremental L2 queries)
|
|
397
|
-
this.db.exec("CREATE INDEX IF NOT EXISTS idx_l1_session_updated ON l1_records(session_id, updated_time)");
|
|
398
|
-
// Composite index: session_key exact match + updated_time range scan (for pipeline cursor queries)
|
|
399
|
-
this.db.exec("CREATE INDEX IF NOT EXISTS idx_l1_sessionkey_updated ON l1_records(session_key, updated_time)");
|
|
400
|
-
// Vector virtual table (cosine distance) — only created when dimensions > 0.
|
|
401
|
-
// When provider="none", dimensions=0 and vec0 tables are deferred until a
|
|
402
|
-
// real embedding provider is configured.
|
|
403
|
-
if (this.dimensions > 0) {
|
|
388
|
+
`);
|
|
389
|
+
// Indexes for common queries
|
|
390
|
+
this.db.exec("CREATE INDEX IF NOT EXISTS idx_l1_type ON l1_records(type)");
|
|
391
|
+
this.db.exec("CREATE INDEX IF NOT EXISTS idx_l1_session_key ON l1_records(session_key)");
|
|
392
|
+
this.db.exec("CREATE INDEX IF NOT EXISTS idx_l1_session_id ON l1_records(session_id)");
|
|
393
|
+
this.db.exec("CREATE INDEX IF NOT EXISTS idx_l1_scene ON l1_records(scene_name)");
|
|
394
|
+
this.db.exec("CREATE INDEX IF NOT EXISTS idx_l1_ts_start ON l1_records(timestamp_start)");
|
|
395
|
+
this.db.exec("CREATE INDEX IF NOT EXISTS idx_l1_ts_end ON l1_records(timestamp_end)");
|
|
396
|
+
// Composite index: session_id exact match + updated_time range scan (for incremental L2 queries)
|
|
397
|
+
this.db.exec("CREATE INDEX IF NOT EXISTS idx_l1_session_updated ON l1_records(session_id, updated_time)");
|
|
398
|
+
// Composite index: session_key exact match + updated_time range scan (for pipeline cursor queries)
|
|
399
|
+
this.db.exec("CREATE INDEX IF NOT EXISTS idx_l1_sessionkey_updated ON l1_records(session_key, updated_time)");
|
|
400
|
+
// Vector virtual table (cosine distance) — only created when dimensions > 0.
|
|
401
|
+
// When provider="none", dimensions=0 and vec0 tables are deferred until a
|
|
402
|
+
// real embedding provider is configured.
|
|
403
|
+
if (this.dimensions > 0) {
|
|
404
404
|
this.db.exec(`
|
|
405
405
|
CREATE VIRTUAL TABLE IF NOT EXISTS l1_vec USING vec0(
|
|
406
406
|
record_id TEXT PRIMARY KEY,
|
|
407
407
|
embedding float[${this.dimensions}] distance_metric=cosine,
|
|
408
408
|
updated_time TEXT DEFAULT ''
|
|
409
409
|
)
|
|
410
|
-
`);
|
|
411
|
-
}
|
|
412
|
-
// Prepare statements for reuse
|
|
410
|
+
`);
|
|
411
|
+
}
|
|
412
|
+
// Prepare statements for reuse
|
|
413
413
|
this.stmtUpsertMeta = this.db.prepare(`
|
|
414
414
|
INSERT INTO l1_records (
|
|
415
415
|
record_id, content, type, priority, scene_name, session_key, session_id,
|
|
@@ -426,28 +426,28 @@ export class VectorStore {
|
|
|
426
426
|
timestamp_end=excluded.timestamp_end,
|
|
427
427
|
updated_time=excluded.updated_time,
|
|
428
428
|
metadata_json=excluded.metadata_json
|
|
429
|
-
`);
|
|
430
|
-
if (this.dimensions > 0) {
|
|
431
|
-
this.stmtDeleteVec = this.db.prepare("DELETE FROM l1_vec WHERE record_id = ?");
|
|
432
|
-
this.stmtInsertVec = this.db.prepare("INSERT INTO l1_vec (record_id, embedding, updated_time) VALUES (?, ?, ?)");
|
|
433
|
-
}
|
|
434
|
-
this.stmtDeleteMeta = this.db.prepare("DELETE FROM l1_records WHERE record_id = ?");
|
|
429
|
+
`);
|
|
430
|
+
if (this.dimensions > 0) {
|
|
431
|
+
this.stmtDeleteVec = this.db.prepare("DELETE FROM l1_vec WHERE record_id = ?");
|
|
432
|
+
this.stmtInsertVec = this.db.prepare("INSERT INTO l1_vec (record_id, embedding, updated_time) VALUES (?, ?, ?)");
|
|
433
|
+
}
|
|
434
|
+
this.stmtDeleteMeta = this.db.prepare("DELETE FROM l1_records WHERE record_id = ?");
|
|
435
435
|
this.stmtGetMeta = this.db.prepare(`
|
|
436
436
|
SELECT content, type, priority, scene_name, session_key, session_id,
|
|
437
437
|
timestamp_str, timestamp_start, timestamp_end, metadata_json
|
|
438
438
|
FROM l1_records WHERE record_id = ?
|
|
439
|
-
`);
|
|
440
|
-
if (this.dimensions > 0) {
|
|
439
|
+
`);
|
|
440
|
+
if (this.dimensions > 0) {
|
|
441
441
|
this.stmtSearchVec = this.db.prepare(`
|
|
442
442
|
SELECT record_id, distance
|
|
443
443
|
FROM l1_vec
|
|
444
444
|
WHERE embedding MATCH ?
|
|
445
445
|
AND k = ?
|
|
446
446
|
ORDER BY distance
|
|
447
|
-
`);
|
|
448
|
-
}
|
|
449
|
-
// ── L0 schema ──────────────────────────────────
|
|
450
|
-
// L0 metadata table: stores individual messages for vector search
|
|
447
|
+
`);
|
|
448
|
+
}
|
|
449
|
+
// ── L0 schema ──────────────────────────────────
|
|
450
|
+
// L0 metadata table: stores individual messages for vector search
|
|
451
451
|
this.db.exec(`
|
|
452
452
|
CREATE TABLE IF NOT EXISTS l0_conversations (
|
|
453
453
|
record_id TEXT PRIMARY KEY,
|
|
@@ -458,31 +458,31 @@ export class VectorStore {
|
|
|
458
458
|
recorded_at TEXT DEFAULT '',
|
|
459
459
|
timestamp INTEGER DEFAULT 0
|
|
460
460
|
)
|
|
461
|
-
`);
|
|
462
|
-
// Migration: add timestamp column if missing (existing DBs pre-v3.x)
|
|
463
|
-
try {
|
|
464
|
-
this.db.exec("ALTER TABLE l0_conversations ADD COLUMN timestamp INTEGER DEFAULT 0");
|
|
465
|
-
this.logger?.debug?.(`${TAG} Migrated l0_conversations: added timestamp column`);
|
|
466
|
-
}
|
|
467
|
-
catch {
|
|
468
|
-
// Column already exists — expected on non-first run
|
|
469
|
-
}
|
|
470
|
-
// Indexes for L0 queries
|
|
471
|
-
this.db.exec("CREATE INDEX IF NOT EXISTS idx_l0_session ON l0_conversations(session_key)");
|
|
472
|
-
this.db.exec("CREATE INDEX IF NOT EXISTS idx_l0_session_id ON l0_conversations(session_id)");
|
|
473
|
-
this.db.exec("CREATE INDEX IF NOT EXISTS idx_l0_recorded ON l0_conversations(recorded_at)");
|
|
474
|
-
this.db.exec("CREATE INDEX IF NOT EXISTS idx_l0_timestamp ON l0_conversations(timestamp)");
|
|
475
|
-
// L0 vector virtual table (cosine distance, same dimensions as L1) — deferred when dimensions=0
|
|
476
|
-
if (this.dimensions > 0) {
|
|
461
|
+
`);
|
|
462
|
+
// Migration: add timestamp column if missing (existing DBs pre-v3.x)
|
|
463
|
+
try {
|
|
464
|
+
this.db.exec("ALTER TABLE l0_conversations ADD COLUMN timestamp INTEGER DEFAULT 0");
|
|
465
|
+
this.logger?.debug?.(`${TAG} Migrated l0_conversations: added timestamp column`);
|
|
466
|
+
}
|
|
467
|
+
catch {
|
|
468
|
+
// Column already exists — expected on non-first run
|
|
469
|
+
}
|
|
470
|
+
// Indexes for L0 queries
|
|
471
|
+
this.db.exec("CREATE INDEX IF NOT EXISTS idx_l0_session ON l0_conversations(session_key)");
|
|
472
|
+
this.db.exec("CREATE INDEX IF NOT EXISTS idx_l0_session_id ON l0_conversations(session_id)");
|
|
473
|
+
this.db.exec("CREATE INDEX IF NOT EXISTS idx_l0_recorded ON l0_conversations(recorded_at)");
|
|
474
|
+
this.db.exec("CREATE INDEX IF NOT EXISTS idx_l0_timestamp ON l0_conversations(timestamp)");
|
|
475
|
+
// L0 vector virtual table (cosine distance, same dimensions as L1) — deferred when dimensions=0
|
|
476
|
+
if (this.dimensions > 0) {
|
|
477
477
|
this.db.exec(`
|
|
478
478
|
CREATE VIRTUAL TABLE IF NOT EXISTS l0_vec USING vec0(
|
|
479
479
|
record_id TEXT PRIMARY KEY,
|
|
480
480
|
embedding float[${this.dimensions}] distance_metric=cosine,
|
|
481
481
|
recorded_at TEXT DEFAULT ''
|
|
482
482
|
)
|
|
483
|
-
`);
|
|
484
|
-
}
|
|
485
|
-
// L0 prepared statements
|
|
483
|
+
`);
|
|
484
|
+
}
|
|
485
|
+
// L0 prepared statements
|
|
486
486
|
this.stmtL0UpsertMeta = this.db.prepare(`
|
|
487
487
|
INSERT INTO l0_conversations (
|
|
488
488
|
record_id, session_key, session_id, role, message_text, recorded_at, timestamp
|
|
@@ -491,61 +491,61 @@ export class VectorStore {
|
|
|
491
491
|
message_text=excluded.message_text,
|
|
492
492
|
recorded_at=excluded.recorded_at,
|
|
493
493
|
timestamp=excluded.timestamp
|
|
494
|
-
`);
|
|
495
|
-
if (this.dimensions > 0) {
|
|
496
|
-
this.stmtL0DeleteVec = this.db.prepare("DELETE FROM l0_vec WHERE record_id = ?");
|
|
497
|
-
this.stmtL0InsertVec = this.db.prepare("INSERT INTO l0_vec (record_id, embedding, recorded_at) VALUES (?, ?, ?)");
|
|
498
|
-
}
|
|
499
|
-
this.stmtL0DeleteMeta = this.db.prepare("DELETE FROM l0_conversations WHERE record_id = ?");
|
|
494
|
+
`);
|
|
495
|
+
if (this.dimensions > 0) {
|
|
496
|
+
this.stmtL0DeleteVec = this.db.prepare("DELETE FROM l0_vec WHERE record_id = ?");
|
|
497
|
+
this.stmtL0InsertVec = this.db.prepare("INSERT INTO l0_vec (record_id, embedding, recorded_at) VALUES (?, ?, ?)");
|
|
498
|
+
}
|
|
499
|
+
this.stmtL0DeleteMeta = this.db.prepare("DELETE FROM l0_conversations WHERE record_id = ?");
|
|
500
500
|
this.stmtL0GetMeta = this.db.prepare(`
|
|
501
501
|
SELECT session_key, session_id, role, message_text, recorded_at, timestamp
|
|
502
502
|
FROM l0_conversations WHERE record_id = ?
|
|
503
|
-
`);
|
|
504
|
-
if (this.dimensions > 0) {
|
|
503
|
+
`);
|
|
504
|
+
if (this.dimensions > 0) {
|
|
505
505
|
this.stmtL0SearchVec = this.db.prepare(`
|
|
506
506
|
SELECT record_id, distance
|
|
507
507
|
FROM l0_vec
|
|
508
508
|
WHERE embedding MATCH ?
|
|
509
509
|
AND k = ?
|
|
510
510
|
ORDER BY distance
|
|
511
|
-
`);
|
|
512
|
-
}
|
|
513
|
-
// L0 query statements for L1 runner (newest-first + LIMIT to bound memory)
|
|
514
|
-
// Sort/filter by recorded_at (write time) instead of timestamp (conversation time)
|
|
515
|
-
// because L1 cursor uses recorded_at semantics. ISO 8601 string comparison preserves time order.
|
|
511
|
+
`);
|
|
512
|
+
}
|
|
513
|
+
// L0 query statements for L1 runner (newest-first + LIMIT to bound memory)
|
|
514
|
+
// Sort/filter by recorded_at (write time) instead of timestamp (conversation time)
|
|
515
|
+
// because L1 cursor uses recorded_at semantics. ISO 8601 string comparison preserves time order.
|
|
516
516
|
this.stmtL0QueryAll = this.db.prepare(`
|
|
517
517
|
SELECT record_id, session_key, session_id, role, message_text, recorded_at, timestamp
|
|
518
518
|
FROM l0_conversations
|
|
519
519
|
WHERE session_key = ?
|
|
520
520
|
ORDER BY recorded_at DESC
|
|
521
521
|
LIMIT ?
|
|
522
|
-
`);
|
|
522
|
+
`);
|
|
523
523
|
this.stmtL0QueryAfter = this.db.prepare(`
|
|
524
524
|
SELECT record_id, session_key, session_id, role, message_text, recorded_at, timestamp
|
|
525
525
|
FROM l0_conversations
|
|
526
526
|
WHERE session_key = ? AND recorded_at > ?
|
|
527
527
|
ORDER BY recorded_at DESC
|
|
528
528
|
LIMIT ?
|
|
529
|
-
`);
|
|
529
|
+
`);
|
|
530
530
|
this.stmtL0QueryMigrationCursor = this.db.prepare(`
|
|
531
531
|
SELECT record_id, session_key, session_id, role, message_text, recorded_at, timestamp
|
|
532
532
|
FROM l0_conversations
|
|
533
533
|
WHERE record_id > ?
|
|
534
534
|
ORDER BY record_id ASC
|
|
535
535
|
LIMIT ?
|
|
536
|
-
`);
|
|
537
|
-
// ── FTS5 tables (best-effort — gracefully degrade if fts5 is not compiled in) ──
|
|
538
|
-
// Schema v2: `content` column stores jieba-segmented text (for indexing),
|
|
539
|
-
// `content_original` (UNINDEXED) stores the raw text (for display).
|
|
540
|
-
// If old v1 tables exist (no content_original column), drop + recreate.
|
|
541
|
-
try {
|
|
542
|
-
// ── Migrate old FTS5 tables (v1 → v2) ──
|
|
543
|
-
// v1 tables stored raw text in the `content` column. v2 stores segmented
|
|
544
|
-
// text in `content` and raw text in `content_original` / `message_text_original`.
|
|
545
|
-
// FTS5 virtual tables don't support ALTER TABLE ADD COLUMN, so we must
|
|
546
|
-
// drop and recreate. The data will be repopulated by `rebuildFtsIndex()`.
|
|
547
|
-
const needsFtsRebuild = this.migrateFtsTablesIfNeeded();
|
|
548
|
-
// L1 FTS5 virtual table (v2 schema)
|
|
536
|
+
`);
|
|
537
|
+
// ── FTS5 tables (best-effort — gracefully degrade if fts5 is not compiled in) ──
|
|
538
|
+
// Schema v2: `content` column stores jieba-segmented text (for indexing),
|
|
539
|
+
// `content_original` (UNINDEXED) stores the raw text (for display).
|
|
540
|
+
// If old v1 tables exist (no content_original column), drop + recreate.
|
|
541
|
+
try {
|
|
542
|
+
// ── Migrate old FTS5 tables (v1 → v2) ──
|
|
543
|
+
// v1 tables stored raw text in the `content` column. v2 stores segmented
|
|
544
|
+
// text in `content` and raw text in `content_original` / `message_text_original`.
|
|
545
|
+
// FTS5 virtual tables don't support ALTER TABLE ADD COLUMN, so we must
|
|
546
|
+
// drop and recreate. The data will be repopulated by `rebuildFtsIndex()`.
|
|
547
|
+
const needsFtsRebuild = this.migrateFtsTablesIfNeeded();
|
|
548
|
+
// L1 FTS5 virtual table (v2 schema)
|
|
549
549
|
this.db.exec(`
|
|
550
550
|
CREATE VIRTUAL TABLE IF NOT EXISTS l1_fts USING fts5(
|
|
551
551
|
content,
|
|
@@ -561,8 +561,8 @@ export class VectorStore {
|
|
|
561
561
|
timestamp_end UNINDEXED,
|
|
562
562
|
metadata_json UNINDEXED
|
|
563
563
|
)
|
|
564
|
-
`);
|
|
565
|
-
// L0 FTS5 virtual table (v2 schema)
|
|
564
|
+
`);
|
|
565
|
+
// L0 FTS5 virtual table (v2 schema)
|
|
566
566
|
this.db.exec(`
|
|
567
567
|
CREATE VIRTUAL TABLE IF NOT EXISTS l0_fts USING fts5(
|
|
568
568
|
message_text,
|
|
@@ -574,14 +574,14 @@ export class VectorStore {
|
|
|
574
574
|
recorded_at UNINDEXED,
|
|
575
575
|
timestamp UNINDEXED
|
|
576
576
|
)
|
|
577
|
-
`);
|
|
578
|
-
// L1 FTS prepared statements
|
|
577
|
+
`);
|
|
578
|
+
// L1 FTS prepared statements
|
|
579
579
|
this.stmtL1FtsInsert = this.db.prepare(`
|
|
580
580
|
INSERT INTO l1_fts (content, content_original, record_id, type, priority, scene_name,
|
|
581
581
|
session_key, session_id, timestamp_str, timestamp_start, timestamp_end, metadata_json)
|
|
582
582
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
583
|
-
`);
|
|
584
|
-
this.stmtL1FtsDelete = this.db.prepare("DELETE FROM l1_fts WHERE record_id = ?");
|
|
583
|
+
`);
|
|
584
|
+
this.stmtL1FtsDelete = this.db.prepare("DELETE FROM l1_fts WHERE record_id = ?");
|
|
585
585
|
this.stmtL1FtsSearch = this.db.prepare(`
|
|
586
586
|
SELECT record_id, content_original AS content, type, priority, scene_name,
|
|
587
587
|
session_key, session_id, timestamp_str, timestamp_start, timestamp_end,
|
|
@@ -591,13 +591,13 @@ export class VectorStore {
|
|
|
591
591
|
WHERE l1_fts MATCH ?
|
|
592
592
|
ORDER BY rank ASC
|
|
593
593
|
LIMIT ?
|
|
594
|
-
`);
|
|
595
|
-
// L0 FTS prepared statements
|
|
594
|
+
`);
|
|
595
|
+
// L0 FTS prepared statements
|
|
596
596
|
this.stmtL0FtsInsert = this.db.prepare(`
|
|
597
597
|
INSERT INTO l0_fts (message_text, message_text_original, record_id, session_key, session_id, role, recorded_at, timestamp)
|
|
598
598
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
599
|
-
`);
|
|
600
|
-
this.stmtL0FtsDelete = this.db.prepare("DELETE FROM l0_fts WHERE record_id = ?");
|
|
599
|
+
`);
|
|
600
|
+
this.stmtL0FtsDelete = this.db.prepare("DELETE FROM l0_fts WHERE record_id = ?");
|
|
601
601
|
this.stmtL0FtsSearch = this.db.prepare(`
|
|
602
602
|
SELECT record_id, message_text_original AS message_text, session_key, session_id, role, recorded_at, timestamp,
|
|
603
603
|
bm25(l0_fts) AS rank
|
|
@@ -605,1218 +605,1218 @@ export class VectorStore {
|
|
|
605
605
|
WHERE l0_fts MATCH ?
|
|
606
606
|
ORDER BY rank ASC
|
|
607
607
|
LIMIT ?
|
|
608
|
-
`);
|
|
609
|
-
this.ftsAvailable = true;
|
|
610
|
-
this.logger?.debug?.(`${TAG} FTS5 tables initialized (l1_fts, l0_fts) [schema v2 — jieba segmented]`);
|
|
611
|
-
// Rebuild FTS index if migrated from v1 or tables were freshly created
|
|
612
|
-
if (needsFtsRebuild) {
|
|
613
|
-
this.rebuildFtsIndex();
|
|
614
|
-
}
|
|
615
|
-
}
|
|
616
|
-
catch (err) {
|
|
617
|
-
const message = err instanceof Error ? err.message : String(err);
|
|
618
|
-
this.ftsAvailable = false;
|
|
619
|
-
this.logger?.warn(`${TAG} FTS5 tables NOT available (fts5 may not be compiled in): ${message}. ` +
|
|
620
|
-
`FTS-based keyword search will be unavailable; recall will use in-memory scoring if needed.`);
|
|
621
|
-
}
|
|
622
|
-
// Save current embedding meta (write after schema is ready)
|
|
623
|
-
if (providerInfo) {
|
|
624
|
-
this.writeEmbeddingMeta({
|
|
625
|
-
provider: providerInfo.provider,
|
|
626
|
-
model: providerInfo.model,
|
|
627
|
-
dimensions: this.dimensions,
|
|
628
|
-
});
|
|
629
|
-
}
|
|
630
|
-
// Mark vec0 tables as ready only when they were actually created
|
|
631
|
-
this.vecTablesReady = this.dimensions > 0;
|
|
632
|
-
// L1 query statements (for l1-reader)
|
|
608
|
+
`);
|
|
609
|
+
this.ftsAvailable = true;
|
|
610
|
+
this.logger?.debug?.(`${TAG} FTS5 tables initialized (l1_fts, l0_fts) [schema v2 — jieba segmented]`);
|
|
611
|
+
// Rebuild FTS index if migrated from v1 or tables were freshly created
|
|
612
|
+
if (needsFtsRebuild) {
|
|
613
|
+
this.rebuildFtsIndex();
|
|
614
|
+
}
|
|
615
|
+
}
|
|
616
|
+
catch (err) {
|
|
617
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
618
|
+
this.ftsAvailable = false;
|
|
619
|
+
this.logger?.warn(`${TAG} FTS5 tables NOT available (fts5 may not be compiled in): ${message}. ` +
|
|
620
|
+
`FTS-based keyword search will be unavailable; recall will use in-memory scoring if needed.`);
|
|
621
|
+
}
|
|
622
|
+
// Save current embedding meta (write after schema is ready)
|
|
623
|
+
if (providerInfo) {
|
|
624
|
+
this.writeEmbeddingMeta({
|
|
625
|
+
provider: providerInfo.provider,
|
|
626
|
+
model: providerInfo.model,
|
|
627
|
+
dimensions: this.dimensions,
|
|
628
|
+
});
|
|
629
|
+
}
|
|
630
|
+
// Mark vec0 tables as ready only when they were actually created
|
|
631
|
+
this.vecTablesReady = this.dimensions > 0;
|
|
632
|
+
// L1 query statements (for l1-reader)
|
|
633
633
|
const l1QueryCols = `record_id, content, type, priority, scene_name, session_key, session_id,
|
|
634
634
|
timestamp_str, timestamp_start, timestamp_end,
|
|
635
|
-
created_time, updated_time, metadata_json`;
|
|
635
|
+
created_time, updated_time, metadata_json`;
|
|
636
636
|
this.stmtQueryBySessionId = this.db.prepare(`
|
|
637
637
|
SELECT ${l1QueryCols} FROM l1_records
|
|
638
638
|
WHERE session_id = ?
|
|
639
639
|
ORDER BY updated_time ASC
|
|
640
|
-
`);
|
|
640
|
+
`);
|
|
641
641
|
this.stmtQueryBySessionIdSince = this.db.prepare(`
|
|
642
642
|
SELECT ${l1QueryCols} FROM l1_records
|
|
643
643
|
WHERE session_id = ? AND updated_time > ?
|
|
644
644
|
ORDER BY updated_time ASC
|
|
645
|
-
`);
|
|
645
|
+
`);
|
|
646
646
|
this.stmtQueryBySessionKey = this.db.prepare(`
|
|
647
647
|
SELECT ${l1QueryCols} FROM l1_records
|
|
648
648
|
WHERE session_key = ?
|
|
649
649
|
ORDER BY updated_time ASC
|
|
650
|
-
`);
|
|
650
|
+
`);
|
|
651
651
|
this.stmtQueryBySessionKeySince = this.db.prepare(`
|
|
652
652
|
SELECT ${l1QueryCols} FROM l1_records
|
|
653
653
|
WHERE session_key = ? AND updated_time > ?
|
|
654
654
|
ORDER BY updated_time ASC
|
|
655
|
-
`);
|
|
655
|
+
`);
|
|
656
656
|
this.stmtQueryAll = this.db.prepare(`
|
|
657
657
|
SELECT ${l1QueryCols} FROM l1_records
|
|
658
658
|
ORDER BY updated_time ASC
|
|
659
|
-
`);
|
|
659
|
+
`);
|
|
660
660
|
this.stmtQueryAllSince = this.db.prepare(`
|
|
661
661
|
SELECT ${l1QueryCols} FROM l1_records
|
|
662
662
|
WHERE updated_time > ?
|
|
663
663
|
ORDER BY updated_time ASC
|
|
664
|
-
`);
|
|
664
|
+
`);
|
|
665
665
|
this.stmtL1QueryMigrationCursor = this.db.prepare(`
|
|
666
666
|
SELECT ${l1QueryCols} FROM l1_records
|
|
667
667
|
WHERE record_id > ?
|
|
668
668
|
ORDER BY record_id ASC
|
|
669
669
|
LIMIT ?
|
|
670
|
-
`);
|
|
671
|
-
this.logger?.debug?.(`${TAG} Initialized (dimensions=${this.dimensions})`);
|
|
672
|
-
return { needsReindex, reason: reindexReason };
|
|
673
|
-
}
|
|
674
|
-
// ── Embedding meta helpers ──────────────────────────────
|
|
675
|
-
readEmbeddingMeta() {
|
|
676
|
-
try {
|
|
677
|
-
const row = this.db
|
|
678
|
-
.prepare("SELECT value FROM embedding_meta WHERE key = ?")
|
|
679
|
-
.get("embedding_provider_info");
|
|
680
|
-
if (!row)
|
|
681
|
-
return null;
|
|
682
|
-
return JSON.parse(row.value);
|
|
683
|
-
}
|
|
684
|
-
catch {
|
|
685
|
-
return null;
|
|
686
|
-
}
|
|
687
|
-
}
|
|
688
|
-
writeEmbeddingMeta(meta) {
|
|
689
|
-
this.db.prepare("INSERT INTO embedding_meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value=excluded.value").run("embedding_provider_info", JSON.stringify(meta));
|
|
690
|
-
}
|
|
691
|
-
/** Allowed table names for row counting (whitelist to prevent SQL injection). */
|
|
692
|
-
static COUNTABLE_TABLES = new Set(["l1_records", "l0_conversations"]);
|
|
693
|
-
/**
|
|
694
|
-
* Extra rows to retrieve from vec0 KNN search to compensate for legacy
|
|
695
|
-
* zero-vector placeholders that may still linger from older data.
|
|
696
|
-
*/
|
|
697
|
-
static ZERO_VEC_BUFFER = 10;
|
|
698
|
-
/** Default result limit for FTS5 keyword searches. */
|
|
699
|
-
static FTS_DEFAULT_LIMIT = 20;
|
|
700
|
-
tableRowCount(table) {
|
|
701
|
-
if (!VectorStore.COUNTABLE_TABLES.has(table)) {
|
|
702
|
-
this.logger?.warn(`${TAG} tableRowCount: rejected unknown table name "${table}"`);
|
|
703
|
-
return 0;
|
|
704
|
-
}
|
|
705
|
-
try {
|
|
706
|
-
const row = this.db
|
|
707
|
-
.prepare(`SELECT COUNT(*) AS cnt FROM ${table}`)
|
|
708
|
-
.get();
|
|
709
|
-
return row?.cnt ?? 0;
|
|
710
|
-
}
|
|
711
|
-
catch {
|
|
712
|
-
return 0;
|
|
713
|
-
}
|
|
714
|
-
}
|
|
715
|
-
/**
|
|
716
|
-
* Detect the embedding dimension of an existing vec0 table by inspecting
|
|
717
|
-
* the DDL stored in sqlite_master. Returns `null` if the table doesn't
|
|
718
|
-
* exist or the dimension cannot be determined.
|
|
719
|
-
*
|
|
720
|
-
* The vec0 DDL looks like:
|
|
721
|
-
* CREATE VIRTUAL TABLE l1_vec USING vec0(... embedding float[768] ...)
|
|
722
|
-
* We parse the number inside `float[N]`.
|
|
723
|
-
*/
|
|
724
|
-
getVecTableDimensions() {
|
|
725
|
-
try {
|
|
726
|
-
const row = this.db
|
|
727
|
-
.prepare("SELECT sql FROM sqlite_master WHERE type='table' AND name=?")
|
|
728
|
-
.get("l1_vec");
|
|
729
|
-
if (!row?.sql)
|
|
730
|
-
return null;
|
|
731
|
-
const match = row.sql.match(/float\[(\d+)\]/);
|
|
732
|
-
return match ? Number(match[1]) : null;
|
|
733
|
-
}
|
|
734
|
-
catch {
|
|
735
|
-
return null;
|
|
736
|
-
}
|
|
737
|
-
}
|
|
738
|
-
/**
|
|
739
|
-
* Drop both L1 and L0 vector virtual tables.
|
|
740
|
-
* Metadata tables (l1_records, l0_conversations) are preserved — only
|
|
741
|
-
* the vec0 tables need to be rebuilt with the new dimensions.
|
|
742
|
-
*/
|
|
743
|
-
dropVectorTables() {
|
|
744
|
-
this.db.exec("DROP TABLE IF EXISTS l1_vec");
|
|
745
|
-
this.db.exec("DROP TABLE IF EXISTS l0_vec");
|
|
746
|
-
this.logger?.info(`${TAG} Dropped vector tables (l1_vec, l0_vec)`);
|
|
747
|
-
}
|
|
748
|
-
/**
|
|
749
|
-
* Write or update a memory record (metadata + vector).
|
|
750
|
-
* Uses a manual transaction for atomicity.
|
|
751
|
-
*
|
|
752
|
-
* If `embedding` is `undefined` or a zero vector (all elements are 0), only
|
|
753
|
-
* the metadata row is written — the vec0 table is left untouched. This
|
|
754
|
-
* allows callers without an EmbeddingService to still persist metadata + FTS
|
|
755
|
-
* without constructing a throwaway zero-vector, and prevents placeholder
|
|
756
|
-
* zero vectors (from embedding-service failures) from polluting KNN search
|
|
757
|
-
* results with null / NaN distances.
|
|
758
|
-
*
|
|
759
|
-
* **Fault-tolerant**: catches all errors internally so that a vector store
|
|
760
|
-
* failure never propagates to the caller / main OpenClaw flow.
|
|
761
|
-
* Returns `true` on success, `false` on failure (logged as warning).
|
|
762
|
-
*/
|
|
763
|
-
upsertL1(record, embedding) {
|
|
764
|
-
if (this.degraded) {
|
|
765
|
-
this.logger?.warn(`${TAG} [L1-upsert] SKIPPED (degraded mode) id=${record.id}`);
|
|
766
|
-
return false;
|
|
767
|
-
}
|
|
768
|
-
try {
|
|
769
|
-
const { id: recordId, timestamps } = record;
|
|
770
|
-
const tsStr = timestamps[0] ?? "";
|
|
771
|
-
const tsStart = timestamps.length > 0
|
|
772
|
-
? timestamps.reduce((a, b) => (a < b ? a : b))
|
|
773
|
-
: tsStr;
|
|
774
|
-
const tsEnd = timestamps.length > 0
|
|
775
|
-
? timestamps.reduce((a, b) => (a > b ? a : b))
|
|
776
|
-
: tsStr;
|
|
777
|
-
const skipVec = !embedding || embedding.every(v => v === 0) || !this.vecTablesReady;
|
|
778
|
-
this.logger?.debug?.(`${TAG} [L1-upsert] START id=${recordId}, type=${record.type}, ` +
|
|
779
|
-
`content="${record.content.slice(0, 60)}..."` +
|
|
780
|
-
(embedding
|
|
781
|
-
? `, embeddingDims=${embedding.length}, ` +
|
|
782
|
-
`embeddingNorm=${Math.sqrt(Array.from(embedding).reduce((s, v) => s + v * v, 0)).toFixed(4)}` +
|
|
783
|
-
`${skipVec ? " (ZERO VECTOR or vec tables not ready — vec write will be skipped)" : ""}`
|
|
784
|
-
: " (no embedding — metadata-only write)"));
|
|
785
|
-
this.db.exec("BEGIN");
|
|
786
|
-
try {
|
|
787
|
-
// Upsert metadata (INSERT OR UPDATE)
|
|
788
|
-
this.stmtUpsertMeta.run(recordId, record.content, record.type, record.priority, record.scene_name, record.sessionKey, record.sessionId, tsStr, tsStart, tsEnd, record.createdAt, record.updatedAt, JSON.stringify(record.metadata));
|
|
789
|
-
if (!skipVec) {
|
|
790
|
-
// vec0 does not support ON CONFLICT → delete then insert
|
|
791
|
-
this.stmtDeleteVec.run(recordId);
|
|
792
|
-
this.stmtInsertVec.run(recordId, Buffer.from(embedding.buffer), record.updatedAt);
|
|
793
|
-
}
|
|
794
|
-
else {
|
|
795
|
-
this.logger?.debug?.(`${TAG} [L1-upsert] Skipping vec write (${embedding ? "zero vector" : "no embedding"}) id=${recordId}`);
|
|
796
|
-
}
|
|
797
|
-
// Sync FTS5 (delete + re-insert to handle updates)
|
|
798
|
-
if (this.ftsAvailable) {
|
|
799
|
-
try {
|
|
800
|
-
this.stmtL1FtsDelete.run(recordId);
|
|
801
|
-
this.stmtL1FtsInsert.run(tokenizeForFts(record.content), // content — segmented for indexing
|
|
802
|
-
record.content, // content_original — raw for display
|
|
803
|
-
recordId, record.type, record.priority, record.scene_name, record.sessionKey, record.sessionId, tsStr, tsStart, tsEnd, JSON.stringify(record.metadata));
|
|
804
|
-
}
|
|
805
|
-
catch (ftsErr) {
|
|
806
|
-
// FTS write failure is non-fatal — log and continue
|
|
807
|
-
this.logger?.warn(`${TAG} [L1-upsert] FTS write failed (non-fatal) id=${recordId}: ${ftsErr instanceof Error ? ftsErr.message : String(ftsErr)}`);
|
|
808
|
-
}
|
|
809
|
-
}
|
|
810
|
-
this.db.exec("COMMIT");
|
|
811
|
-
}
|
|
812
|
-
catch (err) {
|
|
813
|
-
try {
|
|
814
|
-
this.db.exec("ROLLBACK");
|
|
815
|
-
}
|
|
816
|
-
catch { /* ignore rollback errors */ }
|
|
817
|
-
throw err;
|
|
818
|
-
}
|
|
819
|
-
this.logger?.debug?.(`${TAG} [L1-upsert] OK id=${recordId}${skipVec ? " (meta-only)" : ""}`);
|
|
820
|
-
return true;
|
|
821
|
-
}
|
|
822
|
-
catch (err) {
|
|
823
|
-
this.logger?.warn(`${TAG} [L1-upsert] FAILED (non-fatal) id=${record.id}: ${err instanceof Error ? err.message : String(err)}`);
|
|
824
|
-
return false;
|
|
825
|
-
}
|
|
826
|
-
}
|
|
827
|
-
/**
|
|
828
|
-
* Vector similarity search (cosine distance).
|
|
829
|
-
* Returns top-k results sorted by similarity (highest first).
|
|
830
|
-
*
|
|
831
|
-
* **Fault-tolerant**: returns an empty array on any error (e.g. dimension
|
|
832
|
-
* mismatch, corrupted DB) so callers can fall back to keyword search.
|
|
833
|
-
*/
|
|
834
|
-
searchL1Vector(queryEmbedding, topK = 5) {
|
|
835
|
-
if (this.degraded || !this.vecTablesReady) {
|
|
836
|
-
if (this.degraded)
|
|
837
|
-
this.logger?.warn(`${TAG} [L1-search] SKIPPED (degraded mode)`);
|
|
838
|
-
return [];
|
|
839
|
-
}
|
|
840
|
-
try {
|
|
841
|
-
// Over-retrieve to compensate for legacy zero-vector placeholders that
|
|
842
|
-
// may still exist in the vec0 table. New zero vectors are no longer
|
|
843
|
-
// inserted (upsert() skips vec write for zero vectors since v3.x), but
|
|
844
|
-
// older data may still contain them — they surface as NULL/NaN distance
|
|
845
|
-
// in KNN results. A small buffer of 10 is sufficient for remnants.
|
|
846
|
-
// NOTE: "AND distance IS NOT NULL" is NOT usable because vec0 does not
|
|
847
|
-
// support that constraint — it causes an empty result set.
|
|
848
|
-
const ZERO_VEC_BUFFER = 10;
|
|
849
|
-
const retrieveCount = topK + ZERO_VEC_BUFFER;
|
|
850
|
-
this.logger?.debug?.(`${TAG} [L1-search] START topK=${topK}, retrieveCount=${retrieveCount}, ` +
|
|
851
|
-
`queryEmbeddingDims=${queryEmbedding.length}, ` +
|
|
852
|
-
`queryNorm=${Math.sqrt(Array.from(queryEmbedding).reduce((s, v) => s + v * v, 0)).toFixed(4)}`);
|
|
853
|
-
const rows = this.stmtSearchVec.all(Buffer.from(queryEmbedding.buffer), retrieveCount);
|
|
854
|
-
this.logger?.debug?.(`${TAG} [L1-search] vec0 returned ${rows.length} candidate(s)`);
|
|
855
|
-
if (rows.length === 0)
|
|
856
|
-
return [];
|
|
857
|
-
const results = [];
|
|
858
|
-
for (const { record_id, distance } of rows) {
|
|
859
|
-
// sqlite-vec returns null distance for zero vectors (cosine undefined when ‖v‖=0).
|
|
860
|
-
// Skip these — they are placeholder vectors from embedding-service-unavailable fallback.
|
|
861
|
-
if (distance == null || Number.isNaN(distance)) {
|
|
862
|
-
this.logger?.warn(`${TAG} [L1-search] record_id=${record_id} has null/NaN distance (likely zero vector) — skipping`);
|
|
863
|
-
continue;
|
|
864
|
-
}
|
|
865
|
-
const meta = this.stmtGetMeta.get(record_id);
|
|
866
|
-
if (!meta) {
|
|
867
|
-
this.logger?.warn(`${TAG} [L1-search] record_id=${record_id} has vector but NO metadata (orphan)`);
|
|
868
|
-
continue;
|
|
869
|
-
}
|
|
870
|
-
const score = 1.0 - distance;
|
|
871
|
-
this.logger?.debug?.(`${TAG} [L1-search] HIT id=${record_id}, distance=${distance.toFixed(4)}, score=${score.toFixed(4)}, ` +
|
|
872
|
-
`type=${meta.type}, content="${meta.content.slice(0, 60)}..."`);
|
|
873
|
-
results.push({
|
|
874
|
-
record_id,
|
|
875
|
-
content: meta.content,
|
|
876
|
-
type: meta.type,
|
|
877
|
-
priority: meta.priority,
|
|
878
|
-
scene_name: meta.scene_name,
|
|
879
|
-
score,
|
|
880
|
-
timestamp_str: meta.timestamp_str,
|
|
881
|
-
timestamp_start: meta.timestamp_start,
|
|
882
|
-
timestamp_end: meta.timestamp_end,
|
|
883
|
-
session_key: meta.session_key,
|
|
884
|
-
session_id: meta.session_id,
|
|
885
|
-
metadata_json: meta.metadata_json,
|
|
886
|
-
});
|
|
887
|
-
}
|
|
888
|
-
// Trim back to the caller's requested topK (we over-fetched above).
|
|
889
|
-
const trimmed = results.slice(0, topK);
|
|
890
|
-
this.logger?.info(`${TAG} [L1-search] DONE returning ${trimmed.length} result(s) (from ${results.length} valid, ${rows.length} raw)`);
|
|
891
|
-
return trimmed;
|
|
892
|
-
}
|
|
893
|
-
catch (err) {
|
|
894
|
-
this.logger?.warn(`${TAG} [L1-search] FAILED (non-fatal, returning empty): ${err instanceof Error ? err.message : String(err)}`);
|
|
895
|
-
return [];
|
|
896
|
-
}
|
|
897
|
-
}
|
|
898
|
-
/**
|
|
899
|
-
* Delete a single record (metadata + vector).
|
|
900
|
-
*
|
|
901
|
-
* **Fault-tolerant**: logs a warning on failure, never throws.
|
|
902
|
-
*/
|
|
903
|
-
deleteL1(recordId) {
|
|
904
|
-
if (this.degraded)
|
|
905
|
-
return false;
|
|
906
|
-
try {
|
|
907
|
-
this.db.exec("BEGIN");
|
|
908
|
-
try {
|
|
909
|
-
this.stmtDeleteMeta.run(recordId);
|
|
910
|
-
if (this.vecTablesReady)
|
|
911
|
-
this.stmtDeleteVec.run(recordId);
|
|
912
|
-
if (this.ftsAvailable) {
|
|
913
|
-
try {
|
|
914
|
-
this.stmtL1FtsDelete.run(recordId);
|
|
915
|
-
}
|
|
916
|
-
catch { /* non-fatal */ }
|
|
917
|
-
}
|
|
918
|
-
this.db.exec("COMMIT");
|
|
919
|
-
}
|
|
920
|
-
catch (err) {
|
|
921
|
-
try {
|
|
922
|
-
this.db.exec("ROLLBACK");
|
|
923
|
-
}
|
|
924
|
-
catch { /* ignore rollback errors */ }
|
|
925
|
-
throw err;
|
|
926
|
-
}
|
|
927
|
-
return true;
|
|
928
|
-
}
|
|
929
|
-
catch (err) {
|
|
930
|
-
this.logger?.warn(`${TAG} delete failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
|
|
931
|
-
return false;
|
|
932
|
-
}
|
|
933
|
-
}
|
|
934
|
-
/**
|
|
935
|
-
* Delete multiple records (metadata + vector).
|
|
936
|
-
*
|
|
937
|
-
* **Fault-tolerant**: logs a warning on failure, never throws.
|
|
938
|
-
*/
|
|
939
|
-
deleteL1Batch(recordIds) {
|
|
940
|
-
if (this.degraded)
|
|
941
|
-
return false;
|
|
942
|
-
if (recordIds.length === 0)
|
|
943
|
-
return true;
|
|
944
|
-
try {
|
|
945
|
-
this.db.exec("BEGIN");
|
|
946
|
-
try {
|
|
947
|
-
for (const id of recordIds) {
|
|
948
|
-
this.stmtDeleteMeta.run(id);
|
|
949
|
-
if (this.vecTablesReady)
|
|
950
|
-
this.stmtDeleteVec.run(id);
|
|
951
|
-
if (this.ftsAvailable) {
|
|
952
|
-
try {
|
|
953
|
-
this.stmtL1FtsDelete.run(id);
|
|
954
|
-
}
|
|
955
|
-
catch { /* non-fatal */ }
|
|
956
|
-
}
|
|
957
|
-
}
|
|
958
|
-
this.db.exec("COMMIT");
|
|
959
|
-
}
|
|
960
|
-
catch (err) {
|
|
961
|
-
try {
|
|
962
|
-
this.db.exec("ROLLBACK");
|
|
963
|
-
}
|
|
964
|
-
catch { /* ignore rollback errors */ }
|
|
965
|
-
throw err;
|
|
966
|
-
}
|
|
967
|
-
return true;
|
|
968
|
-
}
|
|
969
|
-
catch (err) {
|
|
970
|
-
this.logger?.warn(`${TAG} deleteBatch failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
|
|
971
|
-
return false;
|
|
972
|
-
}
|
|
973
|
-
}
|
|
974
|
-
/**
|
|
975
|
-
* Get the total number of L1 records in the store.
|
|
976
|
-
*
|
|
977
|
-
* **Fault-tolerant**: returns 0 on failure.
|
|
978
|
-
* TTL cleanup by updated_time.
|
|
979
|
-
*
|
|
980
|
-
* Deletes expired rows from l1_records and matching vectors from l1_vec
|
|
981
|
-
* in a single transaction to guarantee consistency.
|
|
982
|
-
*/
|
|
983
|
-
deleteL1Expired(cutoffIso) {
|
|
984
|
-
if (this.degraded) {
|
|
985
|
-
this.logger?.warn(`${TAG} [deleteExpired] SKIPPED (degraded mode)`);
|
|
986
|
-
return 0;
|
|
987
|
-
}
|
|
988
|
-
try {
|
|
989
|
-
const row = this.db.prepare("SELECT COUNT(*) AS cnt FROM l1_records WHERE updated_time != '' AND updated_time < ?").get(cutoffIso);
|
|
990
|
-
const expiredCount = row?.cnt ?? 0;
|
|
991
|
-
if (expiredCount <= 0)
|
|
992
|
-
return 0;
|
|
993
|
-
// Ratio protection: refuse to delete > 80% in one pass
|
|
994
|
-
const totalRow = this.db.prepare("SELECT COUNT(*) AS cnt FROM l1_records").get();
|
|
995
|
-
const total = totalRow.cnt;
|
|
996
|
-
const ratio = total > 0 ? expiredCount / total : 0;
|
|
997
|
-
if (ratio > 0.8) {
|
|
998
|
-
this.logger?.warn(`${TAG} [L1-deleteExpired] BLOCKED: would delete ${expiredCount}/${total} ` +
|
|
999
|
-
`(${(ratio * 100).toFixed(1)}%) — exceeds 80% safety threshold, cutoff=${cutoffIso}`);
|
|
1000
|
-
return 0;
|
|
1001
|
-
}
|
|
1002
|
-
this.db.exec("BEGIN");
|
|
1003
|
-
try {
|
|
1004
|
-
if (this.vecTablesReady) {
|
|
1005
|
-
this.db.prepare("DELETE FROM l1_vec WHERE updated_time != '' AND updated_time < ?").run(cutoffIso);
|
|
1006
|
-
}
|
|
1007
|
-
this.db.prepare("DELETE FROM l1_records WHERE updated_time != '' AND updated_time < ?").run(cutoffIso);
|
|
1008
|
-
this.db.exec("COMMIT");
|
|
1009
|
-
this.logger?.info?.(`${TAG} [L1-deleteExpired] Deleted ${expiredCount}/${total} records (cutoff=${cutoffIso})`);
|
|
1010
|
-
return expiredCount;
|
|
1011
|
-
}
|
|
1012
|
-
catch (err) {
|
|
1013
|
-
try {
|
|
1014
|
-
this.db.exec("ROLLBACK");
|
|
1015
|
-
}
|
|
1016
|
-
catch { /* ignore rollback errors */ }
|
|
1017
|
-
throw err;
|
|
1018
|
-
}
|
|
1019
|
-
}
|
|
1020
|
-
catch (err) {
|
|
1021
|
-
this.logger?.warn(`${TAG} deleteL1ExpiredByUpdatedTime failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
|
|
1022
|
-
return 0;
|
|
1023
|
-
}
|
|
1024
|
-
}
|
|
1025
|
-
/**
|
|
1026
|
-
* Get the total number of records in the store.
|
|
1027
|
-
*/
|
|
1028
|
-
countL1() {
|
|
1029
|
-
if (this.degraded)
|
|
1030
|
-
return 0;
|
|
1031
|
-
try {
|
|
1032
|
-
const row = this.db
|
|
1033
|
-
.prepare("SELECT COUNT(*) AS cnt FROM l1_records")
|
|
1034
|
-
.get();
|
|
1035
|
-
this.logger?.debug?.(`${TAG} [L1-count] total=${row.cnt}`);
|
|
1036
|
-
return row.cnt;
|
|
1037
|
-
}
|
|
1038
|
-
catch (err) {
|
|
1039
|
-
this.logger?.warn(`${TAG} count failed (non-fatal, returning 0): ${err instanceof Error ? err.message : String(err)}`);
|
|
1040
|
-
return 0;
|
|
1041
|
-
}
|
|
1042
|
-
}
|
|
1043
|
-
/**
|
|
1044
|
-
* Query L1 records with optional session and time filters.
|
|
1045
|
-
*
|
|
1046
|
-
* Uses the composite index `idx_l1_session_updated(session_id, updated_time)`
|
|
1047
|
-
* for efficient filtering. All timestamps are compared as UTC ISO 8601 strings.
|
|
1048
|
-
*
|
|
1049
|
-
* **Fault-tolerant**: returns an empty array on any error (degraded mode, DB issues).
|
|
1050
|
-
*/
|
|
1051
|
-
queryL1Records(filter) {
|
|
1052
|
-
if (this.degraded) {
|
|
1053
|
-
this.logger?.warn(`${TAG} [L1-query] SKIPPED (degraded mode)`);
|
|
1054
|
-
return [];
|
|
1055
|
-
}
|
|
1056
|
-
try {
|
|
1057
|
-
const { sessionKey, sessionId, updatedAfter } = filter ?? {};
|
|
1058
|
-
let raw;
|
|
1059
|
-
// Priority: sessionId > sessionKey (sessionId is more specific)
|
|
1060
|
-
if (sessionId && updatedAfter) {
|
|
1061
|
-
raw = this.stmtQueryBySessionIdSince.all(sessionId, updatedAfter);
|
|
1062
|
-
}
|
|
1063
|
-
else if (sessionId) {
|
|
1064
|
-
raw = this.stmtQueryBySessionId.all(sessionId);
|
|
1065
|
-
}
|
|
1066
|
-
else if (sessionKey && updatedAfter) {
|
|
1067
|
-
raw = this.stmtQueryBySessionKeySince.all(sessionKey, updatedAfter);
|
|
1068
|
-
}
|
|
1069
|
-
else if (sessionKey) {
|
|
1070
|
-
raw = this.stmtQueryBySessionKey.all(sessionKey);
|
|
1071
|
-
}
|
|
1072
|
-
else if (updatedAfter) {
|
|
1073
|
-
raw = this.stmtQueryAllSince.all(updatedAfter);
|
|
1074
|
-
}
|
|
1075
|
-
else {
|
|
1076
|
-
raw = this.stmtQueryAll.all();
|
|
1077
|
-
}
|
|
1078
|
-
// Runtime sanity check: verify first row has expected columns (guards against schema drift)
|
|
1079
|
-
if (raw.length > 0 && !("record_id" in raw[0] && "content" in raw[0])) {
|
|
1080
|
-
this.logger?.warn(`${TAG} [L1-query] Schema mismatch: first row missing expected columns. ` +
|
|
1081
|
-
`Got keys: [${Object.keys(raw[0]).join(", ")}]`);
|
|
1082
|
-
return [];
|
|
1083
|
-
}
|
|
1084
|
-
const rows = raw;
|
|
1085
|
-
this.logger?.info(`${TAG} [L1-query] filter={sessionKey=${sessionKey ?? "(all)"}, sessionId=${sessionId ?? "(all)"}, updatedAfter=${updatedAfter ?? "(none)"}}, ` +
|
|
1086
|
-
`returned ${rows.length} record(s)`);
|
|
1087
|
-
return rows;
|
|
1088
|
-
}
|
|
1089
|
-
catch (err) {
|
|
1090
|
-
this.logger?.warn(`${TAG} [L1-query] FAILED (non-fatal, returning empty): ${err instanceof Error ? err.message : String(err)}`);
|
|
1091
|
-
return [];
|
|
1092
|
-
}
|
|
1093
|
-
}
|
|
1094
|
-
// ── L0 operations ──────────────────────────────────
|
|
1095
|
-
/**
|
|
1096
|
-
* Write or update an L0 single-message record (metadata + vector).
|
|
1097
|
-
* Uses a manual transaction for atomicity.
|
|
1098
|
-
*
|
|
1099
|
-
* If `embedding` is `undefined` or a zero vector (all elements are 0), only
|
|
1100
|
-
* the metadata row (`l0_conversations`) is written — the vec0 table
|
|
1101
|
-
* (`l0_vec`) is left untouched. This allows callers without an
|
|
1102
|
-
* EmbeddingService to still persist metadata + FTS without constructing a
|
|
1103
|
-
* throwaway zero-vector, and prevents placeholder zero vectors (from
|
|
1104
|
-
* embedding-service failures) from polluting KNN search results.
|
|
1105
|
-
*
|
|
1106
|
-
* **Fault-tolerant**: catches all errors internally, never throws.
|
|
1107
|
-
* Returns `true` on success, `false` on failure (logged as warning).
|
|
1108
|
-
*/
|
|
1109
|
-
upsertL0(record, embedding) {
|
|
1110
|
-
if (this.degraded) {
|
|
1111
|
-
this.logger?.warn(`${TAG} [L0-upsert] SKIPPED (degraded mode) id=${record.id}`);
|
|
1112
|
-
return false;
|
|
1113
|
-
}
|
|
1114
|
-
try {
|
|
1115
|
-
const skipVec = !embedding || embedding.every(v => v === 0) || !this.vecTablesReady;
|
|
1116
|
-
this.logger?.debug?.(`${TAG} [L0-upsert] START id=${record.id}, session=${record.sessionKey}, role=${record.role}, ` +
|
|
1117
|
-
`text="${record.messageText.slice(0, 60)}..."` +
|
|
1118
|
-
(embedding
|
|
1119
|
-
? `, embeddingDims=${embedding.length}, ` +
|
|
1120
|
-
`embeddingNorm=${Math.sqrt(Array.from(embedding).reduce((s, v) => s + v * v, 0)).toFixed(4)}` +
|
|
1121
|
-
`${skipVec ? " (ZERO VECTOR or vec tables not ready — vec write will be skipped)" : ""}`
|
|
1122
|
-
: " (no embedding — metadata-only write)"));
|
|
1123
|
-
this.db.exec("BEGIN");
|
|
1124
|
-
try {
|
|
1125
|
-
this.stmtL0UpsertMeta.run(record.id, record.sessionKey, record.sessionId, record.role, record.messageText, record.recordedAt, record.timestamp);
|
|
1126
|
-
if (!skipVec) {
|
|
1127
|
-
// vec0 does not support ON CONFLICT → delete then insert
|
|
1128
|
-
this.stmtL0DeleteVec.run(record.id);
|
|
1129
|
-
this.stmtL0InsertVec.run(record.id, Buffer.from(embedding.buffer), record.recordedAt);
|
|
1130
|
-
}
|
|
1131
|
-
else {
|
|
1132
|
-
this.logger?.debug?.(`${TAG} [L0-upsert] Skipping vec write (${embedding ? "zero vector" : "no embedding"}) id=${record.id}`);
|
|
1133
|
-
}
|
|
1134
|
-
// Sync FTS5 (delete + re-insert to handle updates)
|
|
1135
|
-
if (this.ftsAvailable) {
|
|
1136
|
-
try {
|
|
1137
|
-
this.stmtL0FtsDelete.run(record.id);
|
|
1138
|
-
this.stmtL0FtsInsert.run(tokenizeForFts(record.messageText), // message_text — segmented for indexing
|
|
1139
|
-
record.messageText, // message_text_original — raw for display
|
|
1140
|
-
record.id, record.sessionKey, record.sessionId, record.role, record.recordedAt, record.timestamp);
|
|
1141
|
-
}
|
|
1142
|
-
catch (ftsErr) {
|
|
1143
|
-
// FTS write failure is non-fatal — log and continue
|
|
1144
|
-
this.logger?.warn(`${TAG} [L0-upsert] FTS write failed (non-fatal) id=${record.id}: ${ftsErr instanceof Error ? ftsErr.message : String(ftsErr)}`);
|
|
1145
|
-
}
|
|
1146
|
-
}
|
|
1147
|
-
this.db.exec("COMMIT");
|
|
1148
|
-
}
|
|
1149
|
-
catch (err) {
|
|
1150
|
-
try {
|
|
1151
|
-
this.db.exec("ROLLBACK");
|
|
1152
|
-
}
|
|
1153
|
-
catch { /* ignore rollback errors */ }
|
|
1154
|
-
throw err;
|
|
1155
|
-
}
|
|
1156
|
-
this.logger?.debug?.(`${TAG} [L0-upsert] OK id=${record.id}${skipVec ? " (meta-only)" : ""}`);
|
|
1157
|
-
return true;
|
|
1158
|
-
}
|
|
1159
|
-
catch (err) {
|
|
1160
|
-
this.logger?.warn(`${TAG} [L0-upsert] FAILED (non-fatal) id=${record.id}: ${err instanceof Error ? err.message : String(err)}`);
|
|
1161
|
-
return false;
|
|
1162
|
-
}
|
|
1163
|
-
}
|
|
1164
|
-
/**
|
|
1165
|
-
* Update ONLY the vector embedding for an existing L0 record.
|
|
1166
|
-
* The metadata row must already exist in l0_conversations (written by upsertL0).
|
|
1167
|
-
*
|
|
1168
|
-
* This is used by the background embedding task in auto-capture:
|
|
1169
|
-
* 1. upsertL0() writes metadata + FTS synchronously (no embedding)
|
|
1170
|
-
* 2. Background task calls embedBatch() then updateL0Embedding() for each record
|
|
1171
|
-
*
|
|
1172
|
-
* **Fault-tolerant**: catches all errors internally, never throws.
|
|
1173
|
-
* Returns `true` on success, `false` on failure.
|
|
1174
|
-
*/
|
|
1175
|
-
updateL0Embedding(recordId, embedding) {
|
|
1176
|
-
if (this.degraded || !this.vecTablesReady) {
|
|
1177
|
-
return false;
|
|
1178
|
-
}
|
|
1179
|
-
if (!embedding || embedding.every(v => v === 0)) {
|
|
1180
|
-
this.logger?.debug?.(`${TAG} [L0-update-embedding] Skipping zero vector for ${recordId}`);
|
|
1181
|
-
return false;
|
|
1182
|
-
}
|
|
1183
|
-
try {
|
|
1184
|
-
// Look up recorded_at from metadata for the vec0 row
|
|
1185
|
-
const meta = this.stmtL0GetMeta.get(recordId);
|
|
1186
|
-
if (!meta) {
|
|
1187
|
-
this.logger?.warn(`${TAG} [L0-update-embedding] No metadata found for ${recordId}, skipping`);
|
|
1188
|
-
return false;
|
|
1189
|
-
}
|
|
1190
|
-
this.db.exec("BEGIN");
|
|
1191
|
-
try {
|
|
1192
|
-
this.stmtL0DeleteVec.run(recordId);
|
|
1193
|
-
this.stmtL0InsertVec.run(recordId, Buffer.from(embedding.buffer), meta.recorded_at);
|
|
1194
|
-
this.db.exec("COMMIT");
|
|
1195
|
-
}
|
|
1196
|
-
catch (err) {
|
|
1197
|
-
try {
|
|
1198
|
-
this.db.exec("ROLLBACK");
|
|
1199
|
-
}
|
|
1200
|
-
catch { /* ignore */ }
|
|
1201
|
-
throw err;
|
|
1202
|
-
}
|
|
1203
|
-
return true;
|
|
1204
|
-
}
|
|
1205
|
-
catch (err) {
|
|
1206
|
-
this.logger?.warn(`${TAG} [L0-update-embedding] FAILED (non-fatal) id=${recordId}: ${err instanceof Error ? err.message : String(err)}`);
|
|
1207
|
-
return false;
|
|
1208
|
-
}
|
|
1209
|
-
}
|
|
1210
|
-
/**
|
|
1211
|
-
* Vector similarity search on L0 individual messages (cosine distance).
|
|
1212
|
-
* Returns top-k results sorted by similarity (highest first).
|
|
1213
|
-
*
|
|
1214
|
-
* **Fault-tolerant**: returns an empty array on any error.
|
|
1215
|
-
*/
|
|
1216
|
-
searchL0Vector(queryEmbedding, topK = 5) {
|
|
1217
|
-
if (this.degraded || !this.vecTablesReady) {
|
|
1218
|
-
if (this.degraded)
|
|
1219
|
-
this.logger?.warn(`${TAG} [L0-search] SKIPPED (degraded mode)`);
|
|
1220
|
-
return [];
|
|
1221
|
-
}
|
|
1222
|
-
try {
|
|
1223
|
-
// Over-retrieve to compensate for legacy zero-vector placeholders that
|
|
1224
|
-
// may still exist in the vec0 table. New zero vectors are no longer
|
|
1225
|
-
// inserted (upsertL0() skips vec write for zero vectors since v3.x), but
|
|
1226
|
-
// older data may still contain them — they surface as NULL/NaN distance
|
|
1227
|
-
// in KNN results.
|
|
1228
|
-
// NOTE: "AND distance IS NOT NULL" is NOT usable because vec0 does not
|
|
1229
|
-
// support that constraint — it causes an empty result set.
|
|
1230
|
-
const retrieveCount = topK + VectorStore.ZERO_VEC_BUFFER;
|
|
1231
|
-
this.logger?.debug?.(`${TAG} [L0-search] START topK=${topK}, retrieveCount=${retrieveCount}, ` +
|
|
1232
|
-
`queryEmbeddingDims=${queryEmbedding.length}, ` +
|
|
1233
|
-
`queryNorm=${Math.sqrt(Array.from(queryEmbedding).reduce((s, v) => s + v * v, 0)).toFixed(4)}`);
|
|
1234
|
-
const rows = this.stmtL0SearchVec.all(Buffer.from(queryEmbedding.buffer), retrieveCount);
|
|
1235
|
-
this.logger?.debug?.(`${TAG} [L0-search] vec0 returned ${rows.length} candidate(s)`);
|
|
1236
|
-
if (rows.length === 0)
|
|
1237
|
-
return [];
|
|
1238
|
-
const results = [];
|
|
1239
|
-
for (const { record_id, distance } of rows) {
|
|
1240
|
-
// sqlite-vec returns null distance for zero vectors (cosine undefined when ‖v‖=0).
|
|
1241
|
-
// Skip these — they are placeholder vectors from embedding-service-unavailable fallback.
|
|
1242
|
-
if (distance == null || Number.isNaN(distance)) {
|
|
1243
|
-
this.logger?.warn(`${TAG} [L0-search] record_id=${record_id} has null/NaN distance (likely zero vector) — skipping`);
|
|
1244
|
-
continue;
|
|
1245
|
-
}
|
|
1246
|
-
const meta = this.stmtL0GetMeta.get(record_id);
|
|
1247
|
-
if (!meta) {
|
|
1248
|
-
this.logger?.warn(`${TAG} [L0-search] record_id=${record_id} has vector but NO metadata (orphan)`);
|
|
1249
|
-
continue;
|
|
1250
|
-
}
|
|
1251
|
-
const score = 1.0 - distance;
|
|
1252
|
-
this.logger?.debug?.(`${TAG} [L0-search] HIT id=${record_id}, distance=${distance.toFixed(4)}, score=${score.toFixed(4)}, ` +
|
|
1253
|
-
`role=${meta.role}, session=${meta.session_key}, text="${meta.message_text.slice(0, 60)}..."`);
|
|
1254
|
-
results.push({
|
|
1255
|
-
record_id,
|
|
1256
|
-
session_key: meta.session_key,
|
|
1257
|
-
session_id: meta.session_id,
|
|
1258
|
-
role: meta.role,
|
|
1259
|
-
message_text: meta.message_text,
|
|
1260
|
-
score,
|
|
1261
|
-
recorded_at: meta.recorded_at,
|
|
1262
|
-
timestamp: meta.timestamp ?? 0,
|
|
1263
|
-
});
|
|
1264
|
-
}
|
|
1265
|
-
// Trim back to the caller's requested topK (we over-fetched above).
|
|
1266
|
-
const trimmed = results.slice(0, topK);
|
|
1267
|
-
this.logger?.info(`${TAG} [L0-search] DONE returning ${trimmed.length} result(s) (from ${results.length} valid, ${rows.length} raw)`);
|
|
1268
|
-
return trimmed;
|
|
1269
|
-
}
|
|
1270
|
-
catch (err) {
|
|
1271
|
-
this.logger?.warn(`${TAG} [L0-search] FAILED (non-fatal, returning empty): ${err instanceof Error ? err.message : String(err)}`);
|
|
1272
|
-
return [];
|
|
1273
|
-
}
|
|
1274
|
-
}
|
|
1275
|
-
/**
|
|
1276
|
-
* Delete a single L0 record (metadata + vector).
|
|
1277
|
-
*
|
|
1278
|
-
* **Fault-tolerant**: logs a warning on failure, never throws.
|
|
1279
|
-
*/
|
|
1280
|
-
deleteL0(recordId) {
|
|
1281
|
-
if (this.degraded)
|
|
1282
|
-
return false;
|
|
1283
|
-
try {
|
|
1284
|
-
this.db.exec("BEGIN");
|
|
1285
|
-
try {
|
|
1286
|
-
this.stmtL0DeleteMeta.run(recordId);
|
|
1287
|
-
if (this.vecTablesReady)
|
|
1288
|
-
this.stmtL0DeleteVec.run(recordId);
|
|
1289
|
-
if (this.ftsAvailable) {
|
|
1290
|
-
try {
|
|
1291
|
-
this.stmtL0FtsDelete.run(recordId);
|
|
1292
|
-
}
|
|
1293
|
-
catch { /* non-fatal */ }
|
|
1294
|
-
}
|
|
1295
|
-
this.db.exec("COMMIT");
|
|
1296
|
-
}
|
|
1297
|
-
catch (err) {
|
|
1298
|
-
try {
|
|
1299
|
-
this.db.exec("ROLLBACK");
|
|
1300
|
-
}
|
|
1301
|
-
catch { /* ignore rollback errors */ }
|
|
1302
|
-
throw err;
|
|
1303
|
-
}
|
|
1304
|
-
return true;
|
|
1305
|
-
}
|
|
1306
|
-
catch (err) {
|
|
1307
|
-
this.logger?.warn(`${TAG} deleteL0 failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
|
|
1308
|
-
return false;
|
|
1309
|
-
}
|
|
1310
|
-
}
|
|
1311
|
-
/**
|
|
1312
|
-
* TTL cleanup by recorded_at (ISO string) for L0 records.
|
|
1313
|
-
*
|
|
1314
|
-
* Deletes expired rows from l0_conversations and matching vectors from l0_vec
|
|
1315
|
-
* in a single transaction to guarantee consistency.
|
|
1316
|
-
*/
|
|
1317
|
-
deleteL0Expired(cutoffIso) {
|
|
1318
|
-
if (this.degraded) {
|
|
1319
|
-
this.logger?.warn(`${TAG} [deleteExpiredL0] SKIPPED (degraded mode)`);
|
|
1320
|
-
return 0;
|
|
1321
|
-
}
|
|
1322
|
-
try {
|
|
1323
|
-
const row = this.db.prepare("SELECT COUNT(*) AS cnt FROM l0_conversations WHERE recorded_at != '' AND recorded_at < ?").get(cutoffIso);
|
|
1324
|
-
const expiredCount = row?.cnt ?? 0;
|
|
1325
|
-
if (expiredCount <= 0)
|
|
1326
|
-
return 0;
|
|
1327
|
-
// Ratio protection: refuse to delete > 80% in one pass
|
|
1328
|
-
const totalRow = this.db.prepare("SELECT COUNT(*) AS cnt FROM l0_conversations").get();
|
|
1329
|
-
const total = totalRow.cnt;
|
|
1330
|
-
const ratio = total > 0 ? expiredCount / total : 0;
|
|
1331
|
-
if (ratio > 0.8) {
|
|
1332
|
-
this.logger?.warn(`${TAG} [L0-deleteExpired] BLOCKED: would delete ${expiredCount}/${total} ` +
|
|
1333
|
-
`(${(ratio * 100).toFixed(1)}%) — exceeds 80% safety threshold, cutoff=${cutoffIso}`);
|
|
1334
|
-
return 0;
|
|
1335
|
-
}
|
|
1336
|
-
this.db.exec("BEGIN");
|
|
1337
|
-
try {
|
|
1338
|
-
if (this.vecTablesReady) {
|
|
1339
|
-
this.db.prepare("DELETE FROM l0_vec WHERE recorded_at != '' AND recorded_at < ?").run(cutoffIso);
|
|
1340
|
-
}
|
|
1341
|
-
this.db.prepare("DELETE FROM l0_conversations WHERE recorded_at != '' AND recorded_at < ?").run(cutoffIso);
|
|
1342
|
-
this.db.exec("COMMIT");
|
|
1343
|
-
this.logger?.info?.(`${TAG} [L0-deleteExpired] Deleted ${expiredCount}/${total} records (cutoff=${cutoffIso})`);
|
|
1344
|
-
return expiredCount;
|
|
1345
|
-
}
|
|
1346
|
-
catch (err) {
|
|
1347
|
-
try {
|
|
1348
|
-
this.db.exec("ROLLBACK");
|
|
1349
|
-
}
|
|
1350
|
-
catch { /* ignore rollback errors */ }
|
|
1351
|
-
throw err;
|
|
1352
|
-
}
|
|
1353
|
-
}
|
|
1354
|
-
catch (err) {
|
|
1355
|
-
this.logger?.warn(`${TAG} deleteL0ExpiredByRecordedAt failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
|
|
1356
|
-
return 0;
|
|
1357
|
-
}
|
|
1358
|
-
}
|
|
1359
|
-
/**
|
|
1360
|
-
* Get the total number of L0 message records in the store.
|
|
1361
|
-
*
|
|
1362
|
-
* **Fault-tolerant**: returns 0 on failure.
|
|
1363
|
-
*/
|
|
1364
|
-
countL0() {
|
|
1365
|
-
if (this.degraded)
|
|
1366
|
-
return 0;
|
|
1367
|
-
try {
|
|
1368
|
-
const row = this.db
|
|
1369
|
-
.prepare("SELECT COUNT(*) AS cnt FROM l0_conversations")
|
|
1370
|
-
.get();
|
|
1371
|
-
this.logger?.debug?.(`${TAG} [L0-count] total=${row.cnt}`);
|
|
1372
|
-
return row.cnt;
|
|
1373
|
-
}
|
|
1374
|
-
catch (err) {
|
|
1375
|
-
this.logger?.warn(`${TAG} countL0 failed (non-fatal, returning 0): ${err instanceof Error ? err.message : String(err)}`);
|
|
1376
|
-
return 0;
|
|
1377
|
-
}
|
|
1378
|
-
}
|
|
1379
|
-
// ── Re-index operations ──────────────────────────────────
|
|
1380
|
-
/**
|
|
1381
|
-
* Get all L1 record texts for re-embedding.
|
|
1382
|
-
* Returns record_id → content pairs.
|
|
1383
|
-
*/
|
|
1384
|
-
getAllL1Texts() {
|
|
1385
|
-
if (this.degraded)
|
|
1386
|
-
return [];
|
|
1387
|
-
try {
|
|
1388
|
-
return this.db
|
|
1389
|
-
.prepare("SELECT record_id, content, updated_time FROM l1_records")
|
|
1390
|
-
.all();
|
|
1391
|
-
}
|
|
1392
|
-
catch (err) {
|
|
1393
|
-
this.logger?.warn(`${TAG} getAllL1Texts failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
|
|
1394
|
-
return [];
|
|
1395
|
-
}
|
|
1396
|
-
}
|
|
1397
|
-
/**
|
|
1398
|
-
* Get all L0 message texts for re-embedding.
|
|
1399
|
-
* Returns record_id → message_text/recorded_at tuples.
|
|
1400
|
-
*/
|
|
1401
|
-
getAllL0Texts() {
|
|
1402
|
-
if (this.degraded)
|
|
1403
|
-
return [];
|
|
1404
|
-
try {
|
|
1405
|
-
return this.db
|
|
1406
|
-
.prepare("SELECT record_id, message_text, recorded_at FROM l0_conversations")
|
|
1407
|
-
.all();
|
|
1408
|
-
}
|
|
1409
|
-
catch (err) {
|
|
1410
|
-
this.logger?.warn(`${TAG} getAllL0Texts failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
|
|
1411
|
-
return [];
|
|
1412
|
-
}
|
|
1413
|
-
}
|
|
1414
|
-
/**
|
|
1415
|
-
* Re-embed all existing L1 and L0 texts with a new embedding function.
|
|
1416
|
-
*
|
|
1417
|
-
* This is called after `init()` returns `needsReindex: true` — the vector
|
|
1418
|
-
* tables have already been dropped and re-created with the correct dimensions.
|
|
1419
|
-
* This method reads every text from the metadata tables and writes fresh
|
|
1420
|
-
* embeddings into the new vector tables.
|
|
1421
|
-
*
|
|
1422
|
-
* @param embedFn A function that converts text → Float32Array embedding.
|
|
1423
|
-
* @param onProgress Optional callback for progress reporting.
|
|
1424
|
-
*/
|
|
1425
|
-
async reindexAll(embedFn, onProgress) {
|
|
1426
|
-
if (this.degraded || !this.vecTablesReady) {
|
|
1427
|
-
if (this.degraded)
|
|
1428
|
-
this.logger?.warn(`${TAG} reindexAll skipped: VectorStore is in degraded mode`);
|
|
1429
|
-
return { l1Count: 0, l0Count: 0 };
|
|
1430
|
-
}
|
|
1431
|
-
try {
|
|
1432
|
-
// ── Re-embed L1 ──
|
|
1433
|
-
const l1Rows = this.getAllL1Texts();
|
|
1434
|
-
let l1Done = 0;
|
|
1435
|
-
for (const { record_id, content, updated_time } of l1Rows) {
|
|
1436
|
-
try {
|
|
1437
|
-
const embedding = await embedFn(content);
|
|
1438
|
-
// Wrap delete+insert in a transaction to prevent orphan vectors
|
|
1439
|
-
this.db.exec("BEGIN");
|
|
1440
|
-
try {
|
|
1441
|
-
this.stmtDeleteVec.run(record_id);
|
|
1442
|
-
this.stmtInsertVec.run(record_id, Buffer.from(embedding.buffer), updated_time);
|
|
1443
|
-
this.db.exec("COMMIT");
|
|
1444
|
-
}
|
|
1445
|
-
catch (txErr) {
|
|
1446
|
-
try {
|
|
1447
|
-
this.db.exec("ROLLBACK");
|
|
1448
|
-
}
|
|
1449
|
-
catch { /* ignore */ }
|
|
1450
|
-
throw txErr;
|
|
1451
|
-
}
|
|
1452
|
-
}
|
|
1453
|
-
catch (err) {
|
|
1454
|
-
this.logger?.warn?.(`${TAG} reindex L1 skip ${record_id}: ${err instanceof Error ? err.message : String(err)}`);
|
|
1455
|
-
}
|
|
1456
|
-
l1Done++;
|
|
1457
|
-
onProgress?.(l1Done, l1Rows.length, "L1");
|
|
1458
|
-
}
|
|
1459
|
-
// ── Re-embed L0 ──
|
|
1460
|
-
const l0Rows = this.getAllL0Texts();
|
|
1461
|
-
let l0Done = 0;
|
|
1462
|
-
for (const { record_id, message_text, recorded_at } of l0Rows) {
|
|
1463
|
-
try {
|
|
1464
|
-
const embedding = await embedFn(message_text);
|
|
1465
|
-
// Wrap delete+insert in a transaction to prevent orphan vectors
|
|
1466
|
-
this.db.exec("BEGIN");
|
|
1467
|
-
try {
|
|
1468
|
-
this.stmtL0DeleteVec.run(record_id);
|
|
1469
|
-
this.stmtL0InsertVec.run(record_id, Buffer.from(embedding.buffer), recorded_at);
|
|
1470
|
-
this.db.exec("COMMIT");
|
|
1471
|
-
}
|
|
1472
|
-
catch (txErr) {
|
|
1473
|
-
try {
|
|
1474
|
-
this.db.exec("ROLLBACK");
|
|
1475
|
-
}
|
|
1476
|
-
catch { /* ignore */ }
|
|
1477
|
-
throw txErr;
|
|
1478
|
-
}
|
|
1479
|
-
}
|
|
1480
|
-
catch (err) {
|
|
1481
|
-
this.logger?.warn?.(`${TAG} reindex L0 skip ${record_id}: ${err instanceof Error ? err.message : String(err)}`);
|
|
1482
|
-
}
|
|
1483
|
-
l0Done++;
|
|
1484
|
-
onProgress?.(l0Done, l0Rows.length, "L0");
|
|
1485
|
-
}
|
|
1486
|
-
this.logger?.info(`${TAG} Reindex complete: L1=${l1Done}/${l1Rows.length}, L0=${l0Done}/${l0Rows.length}`);
|
|
1487
|
-
return { l1Count: l1Done, l0Count: l0Done };
|
|
1488
|
-
}
|
|
1489
|
-
catch (err) {
|
|
1490
|
-
this.logger?.error(`${TAG} reindexAll failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
|
|
1491
|
-
return { l1Count: 0, l0Count: 0 };
|
|
1492
|
-
}
|
|
1493
|
-
}
|
|
1494
|
-
// ── L0 query operations (for L1 runner) ──────────────────────────────────
|
|
1495
|
-
/**
|
|
1496
|
-
* Query L0 messages for a given session key, optionally filtered by recorded_at cursor.
|
|
1497
|
-
* Returns messages ordered by recorded_at ASC (chronological write order).
|
|
1498
|
-
*
|
|
1499
|
-
* Used by L1 runner to read L0 data from DB instead of JSONL files.
|
|
1500
|
-
*/
|
|
1501
|
-
queryL0ForL1(sessionKey, afterRecordedAtMs, limit = 50) {
|
|
1502
|
-
if (this.degraded) {
|
|
1503
|
-
this.logger?.warn(`${TAG} [L0-query] SKIPPED (degraded mode)`);
|
|
1504
|
-
return [];
|
|
1505
|
-
}
|
|
1506
|
-
try {
|
|
1507
|
-
// Query newest-first (DESC) with LIMIT, then reverse to chronological order
|
|
1508
|
-
let rows;
|
|
1509
|
-
if (afterRecordedAtMs && afterRecordedAtMs > 0) {
|
|
1510
|
-
// Convert epoch ms to ISO string for recorded_at comparison
|
|
1511
|
-
const afterRecordedAtIso = new Date(afterRecordedAtMs).toISOString();
|
|
1512
|
-
rows = this.stmtL0QueryAfter.all(sessionKey, afterRecordedAtIso, limit);
|
|
1513
|
-
}
|
|
1514
|
-
else {
|
|
1515
|
-
rows = this.stmtL0QueryAll.all(sessionKey, limit);
|
|
1516
|
-
}
|
|
1517
|
-
this.logger?.info(`${TAG} [L0-query] session=${sessionKey}, afterRecordedAtMs=${afterRecordedAtMs ?? "(all)"}, ` +
|
|
1518
|
-
`limit=${limit}, returned ${rows.length} row(s)`);
|
|
1519
|
-
// Reverse: SQL returns newest-first (DESC), callers expect chronological order
|
|
1520
|
-
return rows.map((r) => ({
|
|
1521
|
-
record_id: r.record_id,
|
|
1522
|
-
session_key: r.session_key,
|
|
1523
|
-
session_id: r.session_id || "",
|
|
1524
|
-
role: r.role,
|
|
1525
|
-
message_text: r.message_text,
|
|
1526
|
-
recorded_at: r.recorded_at || "",
|
|
1527
|
-
timestamp: r.timestamp || 0,
|
|
1528
|
-
})).reverse();
|
|
1529
|
-
}
|
|
1530
|
-
catch (err) {
|
|
1531
|
-
this.logger?.warn(`${TAG} [L0-query] FAILED (non-fatal, returning empty): ${err instanceof Error ? err.message : String(err)}`);
|
|
1532
|
-
return [];
|
|
1533
|
-
}
|
|
1534
|
-
}
|
|
1535
|
-
/**
|
|
1536
|
-
* Query L0 messages for a given session key, grouped by session_id.
|
|
1537
|
-
* Each group's messages are in chronological order (recorded_at ASC).
|
|
1538
|
-
* Groups are sorted by earliest message timestamp.
|
|
1539
|
-
*
|
|
1540
|
-
* Used by L1 runner to replace readConversationMessagesGroupedBySessionId().
|
|
1541
|
-
*/
|
|
1542
|
-
queryL0GroupedBySessionId(sessionKey, afterRecordedAtMs, limit = 50) {
|
|
1543
|
-
if (this.degraded) {
|
|
1544
|
-
this.logger?.warn(`${TAG} [L0-query-grouped] SKIPPED (degraded mode)`);
|
|
1545
|
-
return [];
|
|
1546
|
-
}
|
|
1547
|
-
try {
|
|
1548
|
-
const rows = this.queryL0ForL1(sessionKey, afterRecordedAtMs, limit);
|
|
1549
|
-
// Group by session_id
|
|
1550
|
-
const groupMap = new Map();
|
|
1551
|
-
for (const row of rows) {
|
|
1552
|
-
const sid = row.session_id || "";
|
|
1553
|
-
let group = groupMap.get(sid);
|
|
1554
|
-
if (!group) {
|
|
1555
|
-
group = [];
|
|
1556
|
-
groupMap.set(sid, group);
|
|
1557
|
-
}
|
|
1558
|
-
group.push({
|
|
1559
|
-
id: row.record_id,
|
|
1560
|
-
role: row.role,
|
|
1561
|
-
content: row.message_text,
|
|
1562
|
-
timestamp: row.timestamp,
|
|
1563
|
-
recordedAtMs: row.recorded_at ? Date.parse(row.recorded_at) || 0 : 0,
|
|
1564
|
-
});
|
|
1565
|
-
}
|
|
1566
|
-
// Convert to array, sorted by earliest message timestamp
|
|
1567
|
-
const groups = [];
|
|
1568
|
-
for (const [sessionId, messages] of groupMap) {
|
|
1569
|
-
if (messages.length > 0) {
|
|
1570
|
-
groups.push({ sessionId, messages });
|
|
1571
|
-
}
|
|
1572
|
-
}
|
|
1573
|
-
groups.sort((a, b) => a.messages[0].timestamp - b.messages[0].timestamp);
|
|
1574
|
-
this.logger?.info(`${TAG} [L0-query-grouped] session=${sessionKey}, afterRecordedAtMs=${afterRecordedAtMs ?? "(all)"}, ` +
|
|
1575
|
-
`${rows.length} messages across ${groups.length} group(s)`);
|
|
1576
|
-
return groups;
|
|
1577
|
-
}
|
|
1578
|
-
catch (err) {
|
|
1579
|
-
this.logger?.warn(`${TAG} [L0-query-grouped] FAILED (non-fatal, returning empty): ${err instanceof Error ? err.message : String(err)}`);
|
|
1580
|
-
return [];
|
|
1581
|
-
}
|
|
1582
|
-
}
|
|
1583
|
-
// ── Cursor-based pagination for migration ──────────────────
|
|
1584
|
-
/**
|
|
1585
|
-
* Read a page of L1 records using primary key cursor.
|
|
1586
|
-
* Returns rows with `record_id > afterId`, ordered by PK, limited to `pageSize`.
|
|
1587
|
-
* Pass `""` as `afterId` for the first page.
|
|
1588
|
-
*/
|
|
1589
|
-
queryL1RecordsCursor(afterId, pageSize) {
|
|
1590
|
-
if (this.degraded)
|
|
1591
|
-
return [];
|
|
1592
|
-
try {
|
|
1593
|
-
return this.stmtL1QueryMigrationCursor.all(afterId, pageSize);
|
|
1594
|
-
}
|
|
1595
|
-
catch (err) {
|
|
1596
|
-
this.logger?.warn(`${TAG} [L1-query-cursor] FAILED (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
|
|
1597
|
-
return [];
|
|
1598
|
-
}
|
|
1599
|
-
}
|
|
1600
|
-
/**
|
|
1601
|
-
* Read a page of L0 records using primary key cursor.
|
|
1602
|
-
* Returns rows with `record_id > afterId`, ordered by PK, limited to `pageSize`.
|
|
1603
|
-
* Pass `""` as `afterId` for the first page.
|
|
1604
|
-
*/
|
|
1605
|
-
queryL0RecordsCursor(afterId, pageSize) {
|
|
1606
|
-
if (this.degraded)
|
|
1607
|
-
return [];
|
|
1608
|
-
try {
|
|
1609
|
-
return this.stmtL0QueryMigrationCursor.all(afterId, pageSize);
|
|
1610
|
-
}
|
|
1611
|
-
catch (err) {
|
|
1612
|
-
this.logger?.warn(`${TAG} [L0-query-cursor] FAILED (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
|
|
1613
|
-
return [];
|
|
1614
|
-
}
|
|
1615
|
-
}
|
|
1616
|
-
// ── FTS5 search operations ──────────────────────────────────
|
|
1617
|
-
/**
|
|
1618
|
-
* Whether FTS5 full-text search is available.
|
|
1619
|
-
* When `false`, callers should skip keyword-based recall entirely.
|
|
1620
|
-
*/
|
|
1621
|
-
isFtsAvailable() {
|
|
1622
|
-
return this.ftsAvailable;
|
|
1623
|
-
}
|
|
1624
|
-
/**
|
|
1625
|
-
* FTS5 keyword search on L1 records.
|
|
1626
|
-
* Returns top-`limit` results sorted by BM25 relevance (highest first).
|
|
1627
|
-
*
|
|
1628
|
-
* @param ftsQuery A pre-built FTS5 MATCH expression (from `buildFtsQuery()`).
|
|
1629
|
-
* @param limit Maximum number of results to return.
|
|
1630
|
-
*
|
|
1631
|
-
* **Fault-tolerant**: returns an empty array on any error.
|
|
1632
|
-
*/
|
|
1633
|
-
searchL1Fts(ftsQuery, limit = 20) {
|
|
1634
|
-
if (this.degraded || !this.ftsAvailable)
|
|
1635
|
-
return [];
|
|
1636
|
-
try {
|
|
1637
|
-
const rows = this.stmtL1FtsSearch.all(ftsQuery, limit);
|
|
1638
|
-
return rows.map((r) => ({
|
|
1639
|
-
record_id: r.record_id,
|
|
1640
|
-
content: r.content,
|
|
1641
|
-
type: r.type,
|
|
1642
|
-
priority: r.priority,
|
|
1643
|
-
scene_name: r.scene_name,
|
|
1644
|
-
score: bm25RankToScore(r.rank),
|
|
1645
|
-
timestamp_str: r.timestamp_str,
|
|
1646
|
-
timestamp_start: r.timestamp_start,
|
|
1647
|
-
timestamp_end: r.timestamp_end,
|
|
1648
|
-
session_key: r.session_key,
|
|
1649
|
-
session_id: r.session_id,
|
|
1650
|
-
metadata_json: r.metadata_json,
|
|
1651
|
-
}));
|
|
1652
|
-
}
|
|
1653
|
-
catch (err) {
|
|
1654
|
-
this.logger?.warn(`${TAG} [L1-fts-search] FAILED (non-fatal, returning empty): ${err instanceof Error ? err.message : String(err)}`);
|
|
1655
|
-
return [];
|
|
1656
|
-
}
|
|
1657
|
-
}
|
|
1658
|
-
/**
|
|
1659
|
-
* FTS5 keyword search on L0 conversation messages.
|
|
1660
|
-
* Returns top-`limit` results sorted by BM25 relevance (highest first).
|
|
1661
|
-
*
|
|
1662
|
-
* @param ftsQuery A pre-built FTS5 MATCH expression (from `buildFtsQuery()`).
|
|
1663
|
-
* @param limit Maximum number of results to return.
|
|
1664
|
-
*
|
|
1665
|
-
* **Fault-tolerant**: returns an empty array on any error.
|
|
1666
|
-
*/
|
|
1667
|
-
searchL0Fts(ftsQuery, limit = VectorStore.FTS_DEFAULT_LIMIT) {
|
|
1668
|
-
if (this.degraded || !this.ftsAvailable)
|
|
1669
|
-
return [];
|
|
1670
|
-
try {
|
|
1671
|
-
const rows = this.stmtL0FtsSearch.all(ftsQuery, limit);
|
|
1672
|
-
return rows.map((r) => ({
|
|
1673
|
-
record_id: r.record_id,
|
|
1674
|
-
session_key: r.session_key,
|
|
1675
|
-
session_id: r.session_id,
|
|
1676
|
-
role: r.role,
|
|
1677
|
-
message_text: r.message_text,
|
|
1678
|
-
score: bm25RankToScore(r.rank),
|
|
1679
|
-
recorded_at: r.recorded_at,
|
|
1680
|
-
timestamp: r.timestamp ?? 0,
|
|
1681
|
-
}));
|
|
1682
|
-
}
|
|
1683
|
-
catch (err) {
|
|
1684
|
-
this.logger?.warn(`${TAG} [L0-fts-search] FAILED (non-fatal, returning empty): ${err instanceof Error ? err.message : String(err)}`);
|
|
1685
|
-
return [];
|
|
1686
|
-
}
|
|
1687
|
-
}
|
|
1688
|
-
// ── FTS5 migration & rebuild ──────────────────────────────────────────────
|
|
1689
|
-
/**
|
|
1690
|
-
* Detect old FTS5 v1 schema (no `content_original` column) and drop the
|
|
1691
|
-
* tables so they can be recreated with the v2 schema.
|
|
1692
|
-
*
|
|
1693
|
-
* FTS5 virtual tables do NOT support `ALTER TABLE ADD COLUMN`, so the only
|
|
1694
|
-
* migration path is DROP + recreate + repopulate.
|
|
1695
|
-
*
|
|
1696
|
-
* @returns `true` if migration was performed (= FTS index needs rebuilding).
|
|
1697
|
-
* @internal
|
|
1698
|
-
*/
|
|
1699
|
-
migrateFtsTablesIfNeeded() {
|
|
1700
|
-
try {
|
|
1701
|
-
// Check if l1_fts exists at all
|
|
1702
|
-
const l1Exists = this.db
|
|
1703
|
-
.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='l1_fts'")
|
|
1704
|
-
.get();
|
|
1705
|
-
if (!l1Exists) {
|
|
1706
|
-
// Fresh install — tables will be created with v2 schema.
|
|
1707
|
-
// Still need rebuild if there's existing data in l1_records.
|
|
1708
|
-
const hasData = this.db.prepare("SELECT 1 FROM l1_records LIMIT 1").get();
|
|
1709
|
-
return !!hasData;
|
|
1710
|
-
}
|
|
1711
|
-
// Check if the v2 column `content_original` exists.
|
|
1712
|
-
// FTS5 tables appear in pragma_table_info with their column names.
|
|
1713
|
-
const cols = this.db
|
|
1714
|
-
.prepare("SELECT name FROM pragma_table_info('l1_fts')")
|
|
1715
|
-
.all();
|
|
1716
|
-
const hasV2Col = cols.some((c) => c.name === "content_original");
|
|
1717
|
-
if (hasV2Col) {
|
|
1718
|
-
return false; // Already v2 — no migration needed
|
|
1719
|
-
}
|
|
1720
|
-
// v1 → v2: drop both FTS tables (data will be repopulated by rebuildFtsIndex)
|
|
1721
|
-
this.logger?.info(`${TAG} Migrating FTS5 tables from v1 to v2 (jieba segmented)`);
|
|
1722
|
-
this.db.exec("DROP TABLE IF EXISTS l1_fts");
|
|
1723
|
-
this.db.exec("DROP TABLE IF EXISTS l0_fts");
|
|
1724
|
-
return true;
|
|
1725
|
-
}
|
|
1726
|
-
catch (err) {
|
|
1727
|
-
this.logger?.warn(`${TAG} FTS migration check failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
|
|
1728
|
-
return false;
|
|
1729
|
-
}
|
|
1730
|
-
}
|
|
1731
|
-
/**
|
|
1732
|
-
* Rebuild the FTS5 index from scratch by reading all records from the
|
|
1733
|
-
* metadata tables and re-inserting them with jieba-segmented text.
|
|
1734
|
-
*
|
|
1735
|
-
* Called automatically after:
|
|
1736
|
-
* - Schema migration from v1 to v2
|
|
1737
|
-
* - Fresh table creation when existing data exists
|
|
1738
|
-
*
|
|
1739
|
-
* Safe to call multiple times (idempotent — clears FTS tables first).
|
|
1740
|
-
*/
|
|
1741
|
-
rebuildFtsIndex() {
|
|
1742
|
-
if (!this.ftsAvailable)
|
|
1743
|
-
return;
|
|
1744
|
-
try {
|
|
1745
|
-
this.logger?.info(`${TAG} Rebuilding FTS5 index with jieba segmentation…`);
|
|
1746
|
-
// ── Rebuild L1 FTS ──
|
|
1747
|
-
// Clear existing FTS data
|
|
1748
|
-
this.db.exec("DELETE FROM l1_fts");
|
|
1749
|
-
// Read all L1 records from metadata table
|
|
1750
|
-
const l1Rows = this.db
|
|
670
|
+
`);
|
|
671
|
+
this.logger?.debug?.(`${TAG} Initialized (dimensions=${this.dimensions})`);
|
|
672
|
+
return { needsReindex, reason: reindexReason };
|
|
673
|
+
}
|
|
674
|
+
// ── Embedding meta helpers ──────────────────────────────
|
|
675
|
+
readEmbeddingMeta() {
|
|
676
|
+
try {
|
|
677
|
+
const row = this.db
|
|
678
|
+
.prepare("SELECT value FROM embedding_meta WHERE key = ?")
|
|
679
|
+
.get("embedding_provider_info");
|
|
680
|
+
if (!row)
|
|
681
|
+
return null;
|
|
682
|
+
return JSON.parse(row.value);
|
|
683
|
+
}
|
|
684
|
+
catch {
|
|
685
|
+
return null;
|
|
686
|
+
}
|
|
687
|
+
}
|
|
688
|
+
writeEmbeddingMeta(meta) {
|
|
689
|
+
this.db.prepare("INSERT INTO embedding_meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value=excluded.value").run("embedding_provider_info", JSON.stringify(meta));
|
|
690
|
+
}
|
|
691
|
+
/** Allowed table names for row counting (whitelist to prevent SQL injection). */
|
|
692
|
+
static COUNTABLE_TABLES = new Set(["l1_records", "l0_conversations"]);
|
|
693
|
+
/**
|
|
694
|
+
* Extra rows to retrieve from vec0 KNN search to compensate for legacy
|
|
695
|
+
* zero-vector placeholders that may still linger from older data.
|
|
696
|
+
*/
|
|
697
|
+
static ZERO_VEC_BUFFER = 10;
|
|
698
|
+
/** Default result limit for FTS5 keyword searches. */
|
|
699
|
+
static FTS_DEFAULT_LIMIT = 20;
|
|
700
|
+
tableRowCount(table) {
|
|
701
|
+
if (!VectorStore.COUNTABLE_TABLES.has(table)) {
|
|
702
|
+
this.logger?.warn(`${TAG} tableRowCount: rejected unknown table name "${table}"`);
|
|
703
|
+
return 0;
|
|
704
|
+
}
|
|
705
|
+
try {
|
|
706
|
+
const row = this.db
|
|
707
|
+
.prepare(`SELECT COUNT(*) AS cnt FROM ${table}`)
|
|
708
|
+
.get();
|
|
709
|
+
return row?.cnt ?? 0;
|
|
710
|
+
}
|
|
711
|
+
catch {
|
|
712
|
+
return 0;
|
|
713
|
+
}
|
|
714
|
+
}
|
|
715
|
+
/**
|
|
716
|
+
* Detect the embedding dimension of an existing vec0 table by inspecting
|
|
717
|
+
* the DDL stored in sqlite_master. Returns `null` if the table doesn't
|
|
718
|
+
* exist or the dimension cannot be determined.
|
|
719
|
+
*
|
|
720
|
+
* The vec0 DDL looks like:
|
|
721
|
+
* CREATE VIRTUAL TABLE l1_vec USING vec0(... embedding float[768] ...)
|
|
722
|
+
* We parse the number inside `float[N]`.
|
|
723
|
+
*/
|
|
724
|
+
getVecTableDimensions() {
|
|
725
|
+
try {
|
|
726
|
+
const row = this.db
|
|
727
|
+
.prepare("SELECT sql FROM sqlite_master WHERE type='table' AND name=?")
|
|
728
|
+
.get("l1_vec");
|
|
729
|
+
if (!row?.sql)
|
|
730
|
+
return null;
|
|
731
|
+
const match = row.sql.match(/float\[(\d+)\]/);
|
|
732
|
+
return match ? Number(match[1]) : null;
|
|
733
|
+
}
|
|
734
|
+
catch {
|
|
735
|
+
return null;
|
|
736
|
+
}
|
|
737
|
+
}
|
|
738
|
+
/**
|
|
739
|
+
* Drop both L1 and L0 vector virtual tables.
|
|
740
|
+
* Metadata tables (l1_records, l0_conversations) are preserved — only
|
|
741
|
+
* the vec0 tables need to be rebuilt with the new dimensions.
|
|
742
|
+
*/
|
|
743
|
+
dropVectorTables() {
|
|
744
|
+
this.db.exec("DROP TABLE IF EXISTS l1_vec");
|
|
745
|
+
this.db.exec("DROP TABLE IF EXISTS l0_vec");
|
|
746
|
+
this.logger?.info(`${TAG} Dropped vector tables (l1_vec, l0_vec)`);
|
|
747
|
+
}
|
|
748
|
+
/**
|
|
749
|
+
* Write or update a memory record (metadata + vector).
|
|
750
|
+
* Uses a manual transaction for atomicity.
|
|
751
|
+
*
|
|
752
|
+
* If `embedding` is `undefined` or a zero vector (all elements are 0), only
|
|
753
|
+
* the metadata row is written — the vec0 table is left untouched. This
|
|
754
|
+
* allows callers without an EmbeddingService to still persist metadata + FTS
|
|
755
|
+
* without constructing a throwaway zero-vector, and prevents placeholder
|
|
756
|
+
* zero vectors (from embedding-service failures) from polluting KNN search
|
|
757
|
+
* results with null / NaN distances.
|
|
758
|
+
*
|
|
759
|
+
* **Fault-tolerant**: catches all errors internally so that a vector store
|
|
760
|
+
* failure never propagates to the caller / main OpenClaw flow.
|
|
761
|
+
* Returns `true` on success, `false` on failure (logged as warning).
|
|
762
|
+
*/
|
|
763
|
+
upsertL1(record, embedding) {
|
|
764
|
+
if (this.degraded) {
|
|
765
|
+
this.logger?.warn(`${TAG} [L1-upsert] SKIPPED (degraded mode) id=${record.id}`);
|
|
766
|
+
return false;
|
|
767
|
+
}
|
|
768
|
+
try {
|
|
769
|
+
const { id: recordId, timestamps } = record;
|
|
770
|
+
const tsStr = timestamps[0] ?? "";
|
|
771
|
+
const tsStart = timestamps.length > 0
|
|
772
|
+
? timestamps.reduce((a, b) => (a < b ? a : b))
|
|
773
|
+
: tsStr;
|
|
774
|
+
const tsEnd = timestamps.length > 0
|
|
775
|
+
? timestamps.reduce((a, b) => (a > b ? a : b))
|
|
776
|
+
: tsStr;
|
|
777
|
+
const skipVec = !embedding || embedding.every(v => v === 0) || !this.vecTablesReady;
|
|
778
|
+
this.logger?.debug?.(`${TAG} [L1-upsert] START id=${recordId}, type=${record.type}, ` +
|
|
779
|
+
`content="${record.content.slice(0, 60)}..."` +
|
|
780
|
+
(embedding
|
|
781
|
+
? `, embeddingDims=${embedding.length}, ` +
|
|
782
|
+
`embeddingNorm=${Math.sqrt(Array.from(embedding).reduce((s, v) => s + v * v, 0)).toFixed(4)}` +
|
|
783
|
+
`${skipVec ? " (ZERO VECTOR or vec tables not ready — vec write will be skipped)" : ""}`
|
|
784
|
+
: " (no embedding — metadata-only write)"));
|
|
785
|
+
this.db.exec("BEGIN");
|
|
786
|
+
try {
|
|
787
|
+
// Upsert metadata (INSERT OR UPDATE)
|
|
788
|
+
this.stmtUpsertMeta.run(recordId, record.content, record.type, record.priority, record.scene_name, record.sessionKey, record.sessionId, tsStr, tsStart, tsEnd, record.createdAt, record.updatedAt, JSON.stringify(record.metadata));
|
|
789
|
+
if (!skipVec) {
|
|
790
|
+
// vec0 does not support ON CONFLICT → delete then insert
|
|
791
|
+
this.stmtDeleteVec.run(recordId);
|
|
792
|
+
this.stmtInsertVec.run(recordId, Buffer.from(embedding.buffer), record.updatedAt);
|
|
793
|
+
}
|
|
794
|
+
else {
|
|
795
|
+
this.logger?.debug?.(`${TAG} [L1-upsert] Skipping vec write (${embedding ? "zero vector" : "no embedding"}) id=${recordId}`);
|
|
796
|
+
}
|
|
797
|
+
// Sync FTS5 (delete + re-insert to handle updates)
|
|
798
|
+
if (this.ftsAvailable) {
|
|
799
|
+
try {
|
|
800
|
+
this.stmtL1FtsDelete.run(recordId);
|
|
801
|
+
this.stmtL1FtsInsert.run(tokenizeForFts(record.content), // content — segmented for indexing
|
|
802
|
+
record.content, // content_original — raw for display
|
|
803
|
+
recordId, record.type, record.priority, record.scene_name, record.sessionKey, record.sessionId, tsStr, tsStart, tsEnd, JSON.stringify(record.metadata));
|
|
804
|
+
}
|
|
805
|
+
catch (ftsErr) {
|
|
806
|
+
// FTS write failure is non-fatal — log and continue
|
|
807
|
+
this.logger?.warn(`${TAG} [L1-upsert] FTS write failed (non-fatal) id=${recordId}: ${ftsErr instanceof Error ? ftsErr.message : String(ftsErr)}`);
|
|
808
|
+
}
|
|
809
|
+
}
|
|
810
|
+
this.db.exec("COMMIT");
|
|
811
|
+
}
|
|
812
|
+
catch (err) {
|
|
813
|
+
try {
|
|
814
|
+
this.db.exec("ROLLBACK");
|
|
815
|
+
}
|
|
816
|
+
catch { /* ignore rollback errors */ }
|
|
817
|
+
throw err;
|
|
818
|
+
}
|
|
819
|
+
this.logger?.debug?.(`${TAG} [L1-upsert] OK id=${recordId}${skipVec ? " (meta-only)" : ""}`);
|
|
820
|
+
return true;
|
|
821
|
+
}
|
|
822
|
+
catch (err) {
|
|
823
|
+
this.logger?.warn(`${TAG} [L1-upsert] FAILED (non-fatal) id=${record.id}: ${err instanceof Error ? err.message : String(err)}`);
|
|
824
|
+
return false;
|
|
825
|
+
}
|
|
826
|
+
}
|
|
827
|
+
/**
|
|
828
|
+
* Vector similarity search (cosine distance).
|
|
829
|
+
* Returns top-k results sorted by similarity (highest first).
|
|
830
|
+
*
|
|
831
|
+
* **Fault-tolerant**: returns an empty array on any error (e.g. dimension
|
|
832
|
+
* mismatch, corrupted DB) so callers can fall back to keyword search.
|
|
833
|
+
*/
|
|
834
|
+
searchL1Vector(queryEmbedding, topK = 5) {
|
|
835
|
+
if (this.degraded || !this.vecTablesReady) {
|
|
836
|
+
if (this.degraded)
|
|
837
|
+
this.logger?.warn(`${TAG} [L1-search] SKIPPED (degraded mode)`);
|
|
838
|
+
return [];
|
|
839
|
+
}
|
|
840
|
+
try {
|
|
841
|
+
// Over-retrieve to compensate for legacy zero-vector placeholders that
|
|
842
|
+
// may still exist in the vec0 table. New zero vectors are no longer
|
|
843
|
+
// inserted (upsert() skips vec write for zero vectors since v3.x), but
|
|
844
|
+
// older data may still contain them — they surface as NULL/NaN distance
|
|
845
|
+
// in KNN results. A small buffer of 10 is sufficient for remnants.
|
|
846
|
+
// NOTE: "AND distance IS NOT NULL" is NOT usable because vec0 does not
|
|
847
|
+
// support that constraint — it causes an empty result set.
|
|
848
|
+
const ZERO_VEC_BUFFER = 10;
|
|
849
|
+
const retrieveCount = topK + ZERO_VEC_BUFFER;
|
|
850
|
+
this.logger?.debug?.(`${TAG} [L1-search] START topK=${topK}, retrieveCount=${retrieveCount}, ` +
|
|
851
|
+
`queryEmbeddingDims=${queryEmbedding.length}, ` +
|
|
852
|
+
`queryNorm=${Math.sqrt(Array.from(queryEmbedding).reduce((s, v) => s + v * v, 0)).toFixed(4)}`);
|
|
853
|
+
const rows = this.stmtSearchVec.all(Buffer.from(queryEmbedding.buffer), retrieveCount);
|
|
854
|
+
this.logger?.debug?.(`${TAG} [L1-search] vec0 returned ${rows.length} candidate(s)`);
|
|
855
|
+
if (rows.length === 0)
|
|
856
|
+
return [];
|
|
857
|
+
const results = [];
|
|
858
|
+
for (const { record_id, distance } of rows) {
|
|
859
|
+
// sqlite-vec returns null distance for zero vectors (cosine undefined when ‖v‖=0).
|
|
860
|
+
// Skip these — they are placeholder vectors from embedding-service-unavailable fallback.
|
|
861
|
+
if (distance == null || Number.isNaN(distance)) {
|
|
862
|
+
this.logger?.warn(`${TAG} [L1-search] record_id=${record_id} has null/NaN distance (likely zero vector) — skipping`);
|
|
863
|
+
continue;
|
|
864
|
+
}
|
|
865
|
+
const meta = this.stmtGetMeta.get(record_id);
|
|
866
|
+
if (!meta) {
|
|
867
|
+
this.logger?.warn(`${TAG} [L1-search] record_id=${record_id} has vector but NO metadata (orphan)`);
|
|
868
|
+
continue;
|
|
869
|
+
}
|
|
870
|
+
const score = 1.0 - distance;
|
|
871
|
+
this.logger?.debug?.(`${TAG} [L1-search] HIT id=${record_id}, distance=${distance.toFixed(4)}, score=${score.toFixed(4)}, ` +
|
|
872
|
+
`type=${meta.type}, content="${meta.content.slice(0, 60)}..."`);
|
|
873
|
+
results.push({
|
|
874
|
+
record_id,
|
|
875
|
+
content: meta.content,
|
|
876
|
+
type: meta.type,
|
|
877
|
+
priority: meta.priority,
|
|
878
|
+
scene_name: meta.scene_name,
|
|
879
|
+
score,
|
|
880
|
+
timestamp_str: meta.timestamp_str,
|
|
881
|
+
timestamp_start: meta.timestamp_start,
|
|
882
|
+
timestamp_end: meta.timestamp_end,
|
|
883
|
+
session_key: meta.session_key,
|
|
884
|
+
session_id: meta.session_id,
|
|
885
|
+
metadata_json: meta.metadata_json,
|
|
886
|
+
});
|
|
887
|
+
}
|
|
888
|
+
// Trim back to the caller's requested topK (we over-fetched above).
|
|
889
|
+
const trimmed = results.slice(0, topK);
|
|
890
|
+
this.logger?.info(`${TAG} [L1-search] DONE returning ${trimmed.length} result(s) (from ${results.length} valid, ${rows.length} raw)`);
|
|
891
|
+
return trimmed;
|
|
892
|
+
}
|
|
893
|
+
catch (err) {
|
|
894
|
+
this.logger?.warn(`${TAG} [L1-search] FAILED (non-fatal, returning empty): ${err instanceof Error ? err.message : String(err)}`);
|
|
895
|
+
return [];
|
|
896
|
+
}
|
|
897
|
+
}
|
|
898
|
+
/**
|
|
899
|
+
* Delete a single record (metadata + vector).
|
|
900
|
+
*
|
|
901
|
+
* **Fault-tolerant**: logs a warning on failure, never throws.
|
|
902
|
+
*/
|
|
903
|
+
deleteL1(recordId) {
|
|
904
|
+
if (this.degraded)
|
|
905
|
+
return false;
|
|
906
|
+
try {
|
|
907
|
+
this.db.exec("BEGIN");
|
|
908
|
+
try {
|
|
909
|
+
this.stmtDeleteMeta.run(recordId);
|
|
910
|
+
if (this.vecTablesReady)
|
|
911
|
+
this.stmtDeleteVec.run(recordId);
|
|
912
|
+
if (this.ftsAvailable) {
|
|
913
|
+
try {
|
|
914
|
+
this.stmtL1FtsDelete.run(recordId);
|
|
915
|
+
}
|
|
916
|
+
catch { /* non-fatal */ }
|
|
917
|
+
}
|
|
918
|
+
this.db.exec("COMMIT");
|
|
919
|
+
}
|
|
920
|
+
catch (err) {
|
|
921
|
+
try {
|
|
922
|
+
this.db.exec("ROLLBACK");
|
|
923
|
+
}
|
|
924
|
+
catch { /* ignore rollback errors */ }
|
|
925
|
+
throw err;
|
|
926
|
+
}
|
|
927
|
+
return true;
|
|
928
|
+
}
|
|
929
|
+
catch (err) {
|
|
930
|
+
this.logger?.warn(`${TAG} delete failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
|
|
931
|
+
return false;
|
|
932
|
+
}
|
|
933
|
+
}
|
|
934
|
+
/**
|
|
935
|
+
* Delete multiple records (metadata + vector).
|
|
936
|
+
*
|
|
937
|
+
* **Fault-tolerant**: logs a warning on failure, never throws.
|
|
938
|
+
*/
|
|
939
|
+
deleteL1Batch(recordIds) {
|
|
940
|
+
if (this.degraded)
|
|
941
|
+
return false;
|
|
942
|
+
if (recordIds.length === 0)
|
|
943
|
+
return true;
|
|
944
|
+
try {
|
|
945
|
+
this.db.exec("BEGIN");
|
|
946
|
+
try {
|
|
947
|
+
for (const id of recordIds) {
|
|
948
|
+
this.stmtDeleteMeta.run(id);
|
|
949
|
+
if (this.vecTablesReady)
|
|
950
|
+
this.stmtDeleteVec.run(id);
|
|
951
|
+
if (this.ftsAvailable) {
|
|
952
|
+
try {
|
|
953
|
+
this.stmtL1FtsDelete.run(id);
|
|
954
|
+
}
|
|
955
|
+
catch { /* non-fatal */ }
|
|
956
|
+
}
|
|
957
|
+
}
|
|
958
|
+
this.db.exec("COMMIT");
|
|
959
|
+
}
|
|
960
|
+
catch (err) {
|
|
961
|
+
try {
|
|
962
|
+
this.db.exec("ROLLBACK");
|
|
963
|
+
}
|
|
964
|
+
catch { /* ignore rollback errors */ }
|
|
965
|
+
throw err;
|
|
966
|
+
}
|
|
967
|
+
return true;
|
|
968
|
+
}
|
|
969
|
+
catch (err) {
|
|
970
|
+
this.logger?.warn(`${TAG} deleteBatch failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
|
|
971
|
+
return false;
|
|
972
|
+
}
|
|
973
|
+
}
|
|
974
|
+
/**
|
|
975
|
+
* Get the total number of L1 records in the store.
|
|
976
|
+
*
|
|
977
|
+
* **Fault-tolerant**: returns 0 on failure.
|
|
978
|
+
* TTL cleanup by updated_time.
|
|
979
|
+
*
|
|
980
|
+
* Deletes expired rows from l1_records and matching vectors from l1_vec
|
|
981
|
+
* in a single transaction to guarantee consistency.
|
|
982
|
+
*/
|
|
983
|
+
deleteL1Expired(cutoffIso) {
|
|
984
|
+
if (this.degraded) {
|
|
985
|
+
this.logger?.warn(`${TAG} [deleteExpired] SKIPPED (degraded mode)`);
|
|
986
|
+
return 0;
|
|
987
|
+
}
|
|
988
|
+
try {
|
|
989
|
+
const row = this.db.prepare("SELECT COUNT(*) AS cnt FROM l1_records WHERE updated_time != '' AND updated_time < ?").get(cutoffIso);
|
|
990
|
+
const expiredCount = row?.cnt ?? 0;
|
|
991
|
+
if (expiredCount <= 0)
|
|
992
|
+
return 0;
|
|
993
|
+
// Ratio protection: refuse to delete > 80% in one pass
|
|
994
|
+
const totalRow = this.db.prepare("SELECT COUNT(*) AS cnt FROM l1_records").get();
|
|
995
|
+
const total = totalRow.cnt;
|
|
996
|
+
const ratio = total > 0 ? expiredCount / total : 0;
|
|
997
|
+
if (ratio > 0.8) {
|
|
998
|
+
this.logger?.warn(`${TAG} [L1-deleteExpired] BLOCKED: would delete ${expiredCount}/${total} ` +
|
|
999
|
+
`(${(ratio * 100).toFixed(1)}%) — exceeds 80% safety threshold, cutoff=${cutoffIso}`);
|
|
1000
|
+
return 0;
|
|
1001
|
+
}
|
|
1002
|
+
this.db.exec("BEGIN");
|
|
1003
|
+
try {
|
|
1004
|
+
if (this.vecTablesReady) {
|
|
1005
|
+
this.db.prepare("DELETE FROM l1_vec WHERE updated_time != '' AND updated_time < ?").run(cutoffIso);
|
|
1006
|
+
}
|
|
1007
|
+
this.db.prepare("DELETE FROM l1_records WHERE updated_time != '' AND updated_time < ?").run(cutoffIso);
|
|
1008
|
+
this.db.exec("COMMIT");
|
|
1009
|
+
this.logger?.info?.(`${TAG} [L1-deleteExpired] Deleted ${expiredCount}/${total} records (cutoff=${cutoffIso})`);
|
|
1010
|
+
return expiredCount;
|
|
1011
|
+
}
|
|
1012
|
+
catch (err) {
|
|
1013
|
+
try {
|
|
1014
|
+
this.db.exec("ROLLBACK");
|
|
1015
|
+
}
|
|
1016
|
+
catch { /* ignore rollback errors */ }
|
|
1017
|
+
throw err;
|
|
1018
|
+
}
|
|
1019
|
+
}
|
|
1020
|
+
catch (err) {
|
|
1021
|
+
this.logger?.warn(`${TAG} deleteL1ExpiredByUpdatedTime failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
|
|
1022
|
+
return 0;
|
|
1023
|
+
}
|
|
1024
|
+
}
|
|
1025
|
+
/**
|
|
1026
|
+
* Get the total number of records in the store.
|
|
1027
|
+
*/
|
|
1028
|
+
countL1() {
|
|
1029
|
+
if (this.degraded)
|
|
1030
|
+
return 0;
|
|
1031
|
+
try {
|
|
1032
|
+
const row = this.db
|
|
1033
|
+
.prepare("SELECT COUNT(*) AS cnt FROM l1_records")
|
|
1034
|
+
.get();
|
|
1035
|
+
this.logger?.debug?.(`${TAG} [L1-count] total=${row.cnt}`);
|
|
1036
|
+
return row.cnt;
|
|
1037
|
+
}
|
|
1038
|
+
catch (err) {
|
|
1039
|
+
this.logger?.warn(`${TAG} count failed (non-fatal, returning 0): ${err instanceof Error ? err.message : String(err)}`);
|
|
1040
|
+
return 0;
|
|
1041
|
+
}
|
|
1042
|
+
}
|
|
1043
|
+
/**
|
|
1044
|
+
* Query L1 records with optional session and time filters.
|
|
1045
|
+
*
|
|
1046
|
+
* Uses the composite index `idx_l1_session_updated(session_id, updated_time)`
|
|
1047
|
+
* for efficient filtering. All timestamps are compared as UTC ISO 8601 strings.
|
|
1048
|
+
*
|
|
1049
|
+
* **Fault-tolerant**: returns an empty array on any error (degraded mode, DB issues).
|
|
1050
|
+
*/
|
|
1051
|
+
queryL1Records(filter) {
|
|
1052
|
+
if (this.degraded) {
|
|
1053
|
+
this.logger?.warn(`${TAG} [L1-query] SKIPPED (degraded mode)`);
|
|
1054
|
+
return [];
|
|
1055
|
+
}
|
|
1056
|
+
try {
|
|
1057
|
+
const { sessionKey, sessionId, updatedAfter } = filter ?? {};
|
|
1058
|
+
let raw;
|
|
1059
|
+
// Priority: sessionId > sessionKey (sessionId is more specific)
|
|
1060
|
+
if (sessionId && updatedAfter) {
|
|
1061
|
+
raw = this.stmtQueryBySessionIdSince.all(sessionId, updatedAfter);
|
|
1062
|
+
}
|
|
1063
|
+
else if (sessionId) {
|
|
1064
|
+
raw = this.stmtQueryBySessionId.all(sessionId);
|
|
1065
|
+
}
|
|
1066
|
+
else if (sessionKey && updatedAfter) {
|
|
1067
|
+
raw = this.stmtQueryBySessionKeySince.all(sessionKey, updatedAfter);
|
|
1068
|
+
}
|
|
1069
|
+
else if (sessionKey) {
|
|
1070
|
+
raw = this.stmtQueryBySessionKey.all(sessionKey);
|
|
1071
|
+
}
|
|
1072
|
+
else if (updatedAfter) {
|
|
1073
|
+
raw = this.stmtQueryAllSince.all(updatedAfter);
|
|
1074
|
+
}
|
|
1075
|
+
else {
|
|
1076
|
+
raw = this.stmtQueryAll.all();
|
|
1077
|
+
}
|
|
1078
|
+
// Runtime sanity check: verify first row has expected columns (guards against schema drift)
|
|
1079
|
+
if (raw.length > 0 && !("record_id" in raw[0] && "content" in raw[0])) {
|
|
1080
|
+
this.logger?.warn(`${TAG} [L1-query] Schema mismatch: first row missing expected columns. ` +
|
|
1081
|
+
`Got keys: [${Object.keys(raw[0]).join(", ")}]`);
|
|
1082
|
+
return [];
|
|
1083
|
+
}
|
|
1084
|
+
const rows = raw;
|
|
1085
|
+
this.logger?.info(`${TAG} [L1-query] filter={sessionKey=${sessionKey ?? "(all)"}, sessionId=${sessionId ?? "(all)"}, updatedAfter=${updatedAfter ?? "(none)"}}, ` +
|
|
1086
|
+
`returned ${rows.length} record(s)`);
|
|
1087
|
+
return rows;
|
|
1088
|
+
}
|
|
1089
|
+
catch (err) {
|
|
1090
|
+
this.logger?.warn(`${TAG} [L1-query] FAILED (non-fatal, returning empty): ${err instanceof Error ? err.message : String(err)}`);
|
|
1091
|
+
return [];
|
|
1092
|
+
}
|
|
1093
|
+
}
|
|
1094
|
+
// ── L0 operations ──────────────────────────────────
|
|
1095
|
+
/**
|
|
1096
|
+
* Write or update an L0 single-message record (metadata + vector).
|
|
1097
|
+
* Uses a manual transaction for atomicity.
|
|
1098
|
+
*
|
|
1099
|
+
* If `embedding` is `undefined` or a zero vector (all elements are 0), only
|
|
1100
|
+
* the metadata row (`l0_conversations`) is written — the vec0 table
|
|
1101
|
+
* (`l0_vec`) is left untouched. This allows callers without an
|
|
1102
|
+
* EmbeddingService to still persist metadata + FTS without constructing a
|
|
1103
|
+
* throwaway zero-vector, and prevents placeholder zero vectors (from
|
|
1104
|
+
* embedding-service failures) from polluting KNN search results.
|
|
1105
|
+
*
|
|
1106
|
+
* **Fault-tolerant**: catches all errors internally, never throws.
|
|
1107
|
+
* Returns `true` on success, `false` on failure (logged as warning).
|
|
1108
|
+
*/
|
|
1109
|
+
upsertL0(record, embedding) {
|
|
1110
|
+
if (this.degraded) {
|
|
1111
|
+
this.logger?.warn(`${TAG} [L0-upsert] SKIPPED (degraded mode) id=${record.id}`);
|
|
1112
|
+
return false;
|
|
1113
|
+
}
|
|
1114
|
+
try {
|
|
1115
|
+
const skipVec = !embedding || embedding.every(v => v === 0) || !this.vecTablesReady;
|
|
1116
|
+
this.logger?.debug?.(`${TAG} [L0-upsert] START id=${record.id}, session=${record.sessionKey}, role=${record.role}, ` +
|
|
1117
|
+
`text="${record.messageText.slice(0, 60)}..."` +
|
|
1118
|
+
(embedding
|
|
1119
|
+
? `, embeddingDims=${embedding.length}, ` +
|
|
1120
|
+
`embeddingNorm=${Math.sqrt(Array.from(embedding).reduce((s, v) => s + v * v, 0)).toFixed(4)}` +
|
|
1121
|
+
`${skipVec ? " (ZERO VECTOR or vec tables not ready — vec write will be skipped)" : ""}`
|
|
1122
|
+
: " (no embedding — metadata-only write)"));
|
|
1123
|
+
this.db.exec("BEGIN");
|
|
1124
|
+
try {
|
|
1125
|
+
this.stmtL0UpsertMeta.run(record.id, record.sessionKey, record.sessionId, record.role, record.messageText, record.recordedAt, record.timestamp);
|
|
1126
|
+
if (!skipVec) {
|
|
1127
|
+
// vec0 does not support ON CONFLICT → delete then insert
|
|
1128
|
+
this.stmtL0DeleteVec.run(record.id);
|
|
1129
|
+
this.stmtL0InsertVec.run(record.id, Buffer.from(embedding.buffer), record.recordedAt);
|
|
1130
|
+
}
|
|
1131
|
+
else {
|
|
1132
|
+
this.logger?.debug?.(`${TAG} [L0-upsert] Skipping vec write (${embedding ? "zero vector" : "no embedding"}) id=${record.id}`);
|
|
1133
|
+
}
|
|
1134
|
+
// Sync FTS5 (delete + re-insert to handle updates)
|
|
1135
|
+
if (this.ftsAvailable) {
|
|
1136
|
+
try {
|
|
1137
|
+
this.stmtL0FtsDelete.run(record.id);
|
|
1138
|
+
this.stmtL0FtsInsert.run(tokenizeForFts(record.messageText), // message_text — segmented for indexing
|
|
1139
|
+
record.messageText, // message_text_original — raw for display
|
|
1140
|
+
record.id, record.sessionKey, record.sessionId, record.role, record.recordedAt, record.timestamp);
|
|
1141
|
+
}
|
|
1142
|
+
catch (ftsErr) {
|
|
1143
|
+
// FTS write failure is non-fatal — log and continue
|
|
1144
|
+
this.logger?.warn(`${TAG} [L0-upsert] FTS write failed (non-fatal) id=${record.id}: ${ftsErr instanceof Error ? ftsErr.message : String(ftsErr)}`);
|
|
1145
|
+
}
|
|
1146
|
+
}
|
|
1147
|
+
this.db.exec("COMMIT");
|
|
1148
|
+
}
|
|
1149
|
+
catch (err) {
|
|
1150
|
+
try {
|
|
1151
|
+
this.db.exec("ROLLBACK");
|
|
1152
|
+
}
|
|
1153
|
+
catch { /* ignore rollback errors */ }
|
|
1154
|
+
throw err;
|
|
1155
|
+
}
|
|
1156
|
+
this.logger?.debug?.(`${TAG} [L0-upsert] OK id=${record.id}${skipVec ? " (meta-only)" : ""}`);
|
|
1157
|
+
return true;
|
|
1158
|
+
}
|
|
1159
|
+
catch (err) {
|
|
1160
|
+
this.logger?.warn(`${TAG} [L0-upsert] FAILED (non-fatal) id=${record.id}: ${err instanceof Error ? err.message : String(err)}`);
|
|
1161
|
+
return false;
|
|
1162
|
+
}
|
|
1163
|
+
}
|
|
1164
|
+
/**
|
|
1165
|
+
* Update ONLY the vector embedding for an existing L0 record.
|
|
1166
|
+
* The metadata row must already exist in l0_conversations (written by upsertL0).
|
|
1167
|
+
*
|
|
1168
|
+
* This is used by the background embedding task in auto-capture:
|
|
1169
|
+
* 1. upsertL0() writes metadata + FTS synchronously (no embedding)
|
|
1170
|
+
* 2. Background task calls embedBatch() then updateL0Embedding() for each record
|
|
1171
|
+
*
|
|
1172
|
+
* **Fault-tolerant**: catches all errors internally, never throws.
|
|
1173
|
+
* Returns `true` on success, `false` on failure.
|
|
1174
|
+
*/
|
|
1175
|
+
updateL0Embedding(recordId, embedding) {
|
|
1176
|
+
if (this.degraded || !this.vecTablesReady) {
|
|
1177
|
+
return false;
|
|
1178
|
+
}
|
|
1179
|
+
if (!embedding || embedding.every(v => v === 0)) {
|
|
1180
|
+
this.logger?.debug?.(`${TAG} [L0-update-embedding] Skipping zero vector for ${recordId}`);
|
|
1181
|
+
return false;
|
|
1182
|
+
}
|
|
1183
|
+
try {
|
|
1184
|
+
// Look up recorded_at from metadata for the vec0 row
|
|
1185
|
+
const meta = this.stmtL0GetMeta.get(recordId);
|
|
1186
|
+
if (!meta) {
|
|
1187
|
+
this.logger?.warn(`${TAG} [L0-update-embedding] No metadata found for ${recordId}, skipping`);
|
|
1188
|
+
return false;
|
|
1189
|
+
}
|
|
1190
|
+
this.db.exec("BEGIN");
|
|
1191
|
+
try {
|
|
1192
|
+
this.stmtL0DeleteVec.run(recordId);
|
|
1193
|
+
this.stmtL0InsertVec.run(recordId, Buffer.from(embedding.buffer), meta.recorded_at);
|
|
1194
|
+
this.db.exec("COMMIT");
|
|
1195
|
+
}
|
|
1196
|
+
catch (err) {
|
|
1197
|
+
try {
|
|
1198
|
+
this.db.exec("ROLLBACK");
|
|
1199
|
+
}
|
|
1200
|
+
catch { /* ignore */ }
|
|
1201
|
+
throw err;
|
|
1202
|
+
}
|
|
1203
|
+
return true;
|
|
1204
|
+
}
|
|
1205
|
+
catch (err) {
|
|
1206
|
+
this.logger?.warn(`${TAG} [L0-update-embedding] FAILED (non-fatal) id=${recordId}: ${err instanceof Error ? err.message : String(err)}`);
|
|
1207
|
+
return false;
|
|
1208
|
+
}
|
|
1209
|
+
}
|
|
1210
|
+
/**
|
|
1211
|
+
* Vector similarity search on L0 individual messages (cosine distance).
|
|
1212
|
+
* Returns top-k results sorted by similarity (highest first).
|
|
1213
|
+
*
|
|
1214
|
+
* **Fault-tolerant**: returns an empty array on any error.
|
|
1215
|
+
*/
|
|
1216
|
+
searchL0Vector(queryEmbedding, topK = 5) {
|
|
1217
|
+
if (this.degraded || !this.vecTablesReady) {
|
|
1218
|
+
if (this.degraded)
|
|
1219
|
+
this.logger?.warn(`${TAG} [L0-search] SKIPPED (degraded mode)`);
|
|
1220
|
+
return [];
|
|
1221
|
+
}
|
|
1222
|
+
try {
|
|
1223
|
+
// Over-retrieve to compensate for legacy zero-vector placeholders that
|
|
1224
|
+
// may still exist in the vec0 table. New zero vectors are no longer
|
|
1225
|
+
// inserted (upsertL0() skips vec write for zero vectors since v3.x), but
|
|
1226
|
+
// older data may still contain them — they surface as NULL/NaN distance
|
|
1227
|
+
// in KNN results.
|
|
1228
|
+
// NOTE: "AND distance IS NOT NULL" is NOT usable because vec0 does not
|
|
1229
|
+
// support that constraint — it causes an empty result set.
|
|
1230
|
+
const retrieveCount = topK + VectorStore.ZERO_VEC_BUFFER;
|
|
1231
|
+
this.logger?.debug?.(`${TAG} [L0-search] START topK=${topK}, retrieveCount=${retrieveCount}, ` +
|
|
1232
|
+
`queryEmbeddingDims=${queryEmbedding.length}, ` +
|
|
1233
|
+
`queryNorm=${Math.sqrt(Array.from(queryEmbedding).reduce((s, v) => s + v * v, 0)).toFixed(4)}`);
|
|
1234
|
+
const rows = this.stmtL0SearchVec.all(Buffer.from(queryEmbedding.buffer), retrieveCount);
|
|
1235
|
+
this.logger?.debug?.(`${TAG} [L0-search] vec0 returned ${rows.length} candidate(s)`);
|
|
1236
|
+
if (rows.length === 0)
|
|
1237
|
+
return [];
|
|
1238
|
+
const results = [];
|
|
1239
|
+
for (const { record_id, distance } of rows) {
|
|
1240
|
+
// sqlite-vec returns null distance for zero vectors (cosine undefined when ‖v‖=0).
|
|
1241
|
+
// Skip these — they are placeholder vectors from embedding-service-unavailable fallback.
|
|
1242
|
+
if (distance == null || Number.isNaN(distance)) {
|
|
1243
|
+
this.logger?.warn(`${TAG} [L0-search] record_id=${record_id} has null/NaN distance (likely zero vector) — skipping`);
|
|
1244
|
+
continue;
|
|
1245
|
+
}
|
|
1246
|
+
const meta = this.stmtL0GetMeta.get(record_id);
|
|
1247
|
+
if (!meta) {
|
|
1248
|
+
this.logger?.warn(`${TAG} [L0-search] record_id=${record_id} has vector but NO metadata (orphan)`);
|
|
1249
|
+
continue;
|
|
1250
|
+
}
|
|
1251
|
+
const score = 1.0 - distance;
|
|
1252
|
+
this.logger?.debug?.(`${TAG} [L0-search] HIT id=${record_id}, distance=${distance.toFixed(4)}, score=${score.toFixed(4)}, ` +
|
|
1253
|
+
`role=${meta.role}, session=${meta.session_key}, text="${meta.message_text.slice(0, 60)}..."`);
|
|
1254
|
+
results.push({
|
|
1255
|
+
record_id,
|
|
1256
|
+
session_key: meta.session_key,
|
|
1257
|
+
session_id: meta.session_id,
|
|
1258
|
+
role: meta.role,
|
|
1259
|
+
message_text: meta.message_text,
|
|
1260
|
+
score,
|
|
1261
|
+
recorded_at: meta.recorded_at,
|
|
1262
|
+
timestamp: meta.timestamp ?? 0,
|
|
1263
|
+
});
|
|
1264
|
+
}
|
|
1265
|
+
// Trim back to the caller's requested topK (we over-fetched above).
|
|
1266
|
+
const trimmed = results.slice(0, topK);
|
|
1267
|
+
this.logger?.info(`${TAG} [L0-search] DONE returning ${trimmed.length} result(s) (from ${results.length} valid, ${rows.length} raw)`);
|
|
1268
|
+
return trimmed;
|
|
1269
|
+
}
|
|
1270
|
+
catch (err) {
|
|
1271
|
+
this.logger?.warn(`${TAG} [L0-search] FAILED (non-fatal, returning empty): ${err instanceof Error ? err.message : String(err)}`);
|
|
1272
|
+
return [];
|
|
1273
|
+
}
|
|
1274
|
+
}
|
|
1275
|
+
/**
|
|
1276
|
+
* Delete a single L0 record (metadata + vector).
|
|
1277
|
+
*
|
|
1278
|
+
* **Fault-tolerant**: logs a warning on failure, never throws.
|
|
1279
|
+
*/
|
|
1280
|
+
deleteL0(recordId) {
|
|
1281
|
+
if (this.degraded)
|
|
1282
|
+
return false;
|
|
1283
|
+
try {
|
|
1284
|
+
this.db.exec("BEGIN");
|
|
1285
|
+
try {
|
|
1286
|
+
this.stmtL0DeleteMeta.run(recordId);
|
|
1287
|
+
if (this.vecTablesReady)
|
|
1288
|
+
this.stmtL0DeleteVec.run(recordId);
|
|
1289
|
+
if (this.ftsAvailable) {
|
|
1290
|
+
try {
|
|
1291
|
+
this.stmtL0FtsDelete.run(recordId);
|
|
1292
|
+
}
|
|
1293
|
+
catch { /* non-fatal */ }
|
|
1294
|
+
}
|
|
1295
|
+
this.db.exec("COMMIT");
|
|
1296
|
+
}
|
|
1297
|
+
catch (err) {
|
|
1298
|
+
try {
|
|
1299
|
+
this.db.exec("ROLLBACK");
|
|
1300
|
+
}
|
|
1301
|
+
catch { /* ignore rollback errors */ }
|
|
1302
|
+
throw err;
|
|
1303
|
+
}
|
|
1304
|
+
return true;
|
|
1305
|
+
}
|
|
1306
|
+
catch (err) {
|
|
1307
|
+
this.logger?.warn(`${TAG} deleteL0 failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
|
|
1308
|
+
return false;
|
|
1309
|
+
}
|
|
1310
|
+
}
|
|
1311
|
+
/**
|
|
1312
|
+
* TTL cleanup by recorded_at (ISO string) for L0 records.
|
|
1313
|
+
*
|
|
1314
|
+
* Deletes expired rows from l0_conversations and matching vectors from l0_vec
|
|
1315
|
+
* in a single transaction to guarantee consistency.
|
|
1316
|
+
*/
|
|
1317
|
+
deleteL0Expired(cutoffIso) {
|
|
1318
|
+
if (this.degraded) {
|
|
1319
|
+
this.logger?.warn(`${TAG} [deleteExpiredL0] SKIPPED (degraded mode)`);
|
|
1320
|
+
return 0;
|
|
1321
|
+
}
|
|
1322
|
+
try {
|
|
1323
|
+
const row = this.db.prepare("SELECT COUNT(*) AS cnt FROM l0_conversations WHERE recorded_at != '' AND recorded_at < ?").get(cutoffIso);
|
|
1324
|
+
const expiredCount = row?.cnt ?? 0;
|
|
1325
|
+
if (expiredCount <= 0)
|
|
1326
|
+
return 0;
|
|
1327
|
+
// Ratio protection: refuse to delete > 80% in one pass
|
|
1328
|
+
const totalRow = this.db.prepare("SELECT COUNT(*) AS cnt FROM l0_conversations").get();
|
|
1329
|
+
const total = totalRow.cnt;
|
|
1330
|
+
const ratio = total > 0 ? expiredCount / total : 0;
|
|
1331
|
+
if (ratio > 0.8) {
|
|
1332
|
+
this.logger?.warn(`${TAG} [L0-deleteExpired] BLOCKED: would delete ${expiredCount}/${total} ` +
|
|
1333
|
+
`(${(ratio * 100).toFixed(1)}%) — exceeds 80% safety threshold, cutoff=${cutoffIso}`);
|
|
1334
|
+
return 0;
|
|
1335
|
+
}
|
|
1336
|
+
this.db.exec("BEGIN");
|
|
1337
|
+
try {
|
|
1338
|
+
if (this.vecTablesReady) {
|
|
1339
|
+
this.db.prepare("DELETE FROM l0_vec WHERE recorded_at != '' AND recorded_at < ?").run(cutoffIso);
|
|
1340
|
+
}
|
|
1341
|
+
this.db.prepare("DELETE FROM l0_conversations WHERE recorded_at != '' AND recorded_at < ?").run(cutoffIso);
|
|
1342
|
+
this.db.exec("COMMIT");
|
|
1343
|
+
this.logger?.info?.(`${TAG} [L0-deleteExpired] Deleted ${expiredCount}/${total} records (cutoff=${cutoffIso})`);
|
|
1344
|
+
return expiredCount;
|
|
1345
|
+
}
|
|
1346
|
+
catch (err) {
|
|
1347
|
+
try {
|
|
1348
|
+
this.db.exec("ROLLBACK");
|
|
1349
|
+
}
|
|
1350
|
+
catch { /* ignore rollback errors */ }
|
|
1351
|
+
throw err;
|
|
1352
|
+
}
|
|
1353
|
+
}
|
|
1354
|
+
catch (err) {
|
|
1355
|
+
this.logger?.warn(`${TAG} deleteL0ExpiredByRecordedAt failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
|
|
1356
|
+
return 0;
|
|
1357
|
+
}
|
|
1358
|
+
}
|
|
1359
|
+
/**
|
|
1360
|
+
* Get the total number of L0 message records in the store.
|
|
1361
|
+
*
|
|
1362
|
+
* **Fault-tolerant**: returns 0 on failure.
|
|
1363
|
+
*/
|
|
1364
|
+
countL0() {
|
|
1365
|
+
if (this.degraded)
|
|
1366
|
+
return 0;
|
|
1367
|
+
try {
|
|
1368
|
+
const row = this.db
|
|
1369
|
+
.prepare("SELECT COUNT(*) AS cnt FROM l0_conversations")
|
|
1370
|
+
.get();
|
|
1371
|
+
this.logger?.debug?.(`${TAG} [L0-count] total=${row.cnt}`);
|
|
1372
|
+
return row.cnt;
|
|
1373
|
+
}
|
|
1374
|
+
catch (err) {
|
|
1375
|
+
this.logger?.warn(`${TAG} countL0 failed (non-fatal, returning 0): ${err instanceof Error ? err.message : String(err)}`);
|
|
1376
|
+
return 0;
|
|
1377
|
+
}
|
|
1378
|
+
}
|
|
1379
|
+
// ── Re-index operations ──────────────────────────────────
|
|
1380
|
+
/**
|
|
1381
|
+
* Get all L1 record texts for re-embedding.
|
|
1382
|
+
* Returns record_id → content pairs.
|
|
1383
|
+
*/
|
|
1384
|
+
getAllL1Texts() {
|
|
1385
|
+
if (this.degraded)
|
|
1386
|
+
return [];
|
|
1387
|
+
try {
|
|
1388
|
+
return this.db
|
|
1389
|
+
.prepare("SELECT record_id, content, updated_time FROM l1_records")
|
|
1390
|
+
.all();
|
|
1391
|
+
}
|
|
1392
|
+
catch (err) {
|
|
1393
|
+
this.logger?.warn(`${TAG} getAllL1Texts failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
|
|
1394
|
+
return [];
|
|
1395
|
+
}
|
|
1396
|
+
}
|
|
1397
|
+
/**
|
|
1398
|
+
* Get all L0 message texts for re-embedding.
|
|
1399
|
+
* Returns record_id → message_text/recorded_at tuples.
|
|
1400
|
+
*/
|
|
1401
|
+
getAllL0Texts() {
|
|
1402
|
+
if (this.degraded)
|
|
1403
|
+
return [];
|
|
1404
|
+
try {
|
|
1405
|
+
return this.db
|
|
1406
|
+
.prepare("SELECT record_id, message_text, recorded_at FROM l0_conversations")
|
|
1407
|
+
.all();
|
|
1408
|
+
}
|
|
1409
|
+
catch (err) {
|
|
1410
|
+
this.logger?.warn(`${TAG} getAllL0Texts failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
|
|
1411
|
+
return [];
|
|
1412
|
+
}
|
|
1413
|
+
}
|
|
1414
|
+
/**
|
|
1415
|
+
* Re-embed all existing L1 and L0 texts with a new embedding function.
|
|
1416
|
+
*
|
|
1417
|
+
* This is called after `init()` returns `needsReindex: true` — the vector
|
|
1418
|
+
* tables have already been dropped and re-created with the correct dimensions.
|
|
1419
|
+
* This method reads every text from the metadata tables and writes fresh
|
|
1420
|
+
* embeddings into the new vector tables.
|
|
1421
|
+
*
|
|
1422
|
+
* @param embedFn A function that converts text → Float32Array embedding.
|
|
1423
|
+
* @param onProgress Optional callback for progress reporting.
|
|
1424
|
+
*/
|
|
1425
|
+
async reindexAll(embedFn, onProgress) {
|
|
1426
|
+
if (this.degraded || !this.vecTablesReady) {
|
|
1427
|
+
if (this.degraded)
|
|
1428
|
+
this.logger?.warn(`${TAG} reindexAll skipped: VectorStore is in degraded mode`);
|
|
1429
|
+
return { l1Count: 0, l0Count: 0 };
|
|
1430
|
+
}
|
|
1431
|
+
try {
|
|
1432
|
+
// ── Re-embed L1 ──
|
|
1433
|
+
const l1Rows = this.getAllL1Texts();
|
|
1434
|
+
let l1Done = 0;
|
|
1435
|
+
for (const { record_id, content, updated_time } of l1Rows) {
|
|
1436
|
+
try {
|
|
1437
|
+
const embedding = await embedFn(content);
|
|
1438
|
+
// Wrap delete+insert in a transaction to prevent orphan vectors
|
|
1439
|
+
this.db.exec("BEGIN");
|
|
1440
|
+
try {
|
|
1441
|
+
this.stmtDeleteVec.run(record_id);
|
|
1442
|
+
this.stmtInsertVec.run(record_id, Buffer.from(embedding.buffer), updated_time);
|
|
1443
|
+
this.db.exec("COMMIT");
|
|
1444
|
+
}
|
|
1445
|
+
catch (txErr) {
|
|
1446
|
+
try {
|
|
1447
|
+
this.db.exec("ROLLBACK");
|
|
1448
|
+
}
|
|
1449
|
+
catch { /* ignore */ }
|
|
1450
|
+
throw txErr;
|
|
1451
|
+
}
|
|
1452
|
+
}
|
|
1453
|
+
catch (err) {
|
|
1454
|
+
this.logger?.warn?.(`${TAG} reindex L1 skip ${record_id}: ${err instanceof Error ? err.message : String(err)}`);
|
|
1455
|
+
}
|
|
1456
|
+
l1Done++;
|
|
1457
|
+
onProgress?.(l1Done, l1Rows.length, "L1");
|
|
1458
|
+
}
|
|
1459
|
+
// ── Re-embed L0 ──
|
|
1460
|
+
const l0Rows = this.getAllL0Texts();
|
|
1461
|
+
let l0Done = 0;
|
|
1462
|
+
for (const { record_id, message_text, recorded_at } of l0Rows) {
|
|
1463
|
+
try {
|
|
1464
|
+
const embedding = await embedFn(message_text);
|
|
1465
|
+
// Wrap delete+insert in a transaction to prevent orphan vectors
|
|
1466
|
+
this.db.exec("BEGIN");
|
|
1467
|
+
try {
|
|
1468
|
+
this.stmtL0DeleteVec.run(record_id);
|
|
1469
|
+
this.stmtL0InsertVec.run(record_id, Buffer.from(embedding.buffer), recorded_at);
|
|
1470
|
+
this.db.exec("COMMIT");
|
|
1471
|
+
}
|
|
1472
|
+
catch (txErr) {
|
|
1473
|
+
try {
|
|
1474
|
+
this.db.exec("ROLLBACK");
|
|
1475
|
+
}
|
|
1476
|
+
catch { /* ignore */ }
|
|
1477
|
+
throw txErr;
|
|
1478
|
+
}
|
|
1479
|
+
}
|
|
1480
|
+
catch (err) {
|
|
1481
|
+
this.logger?.warn?.(`${TAG} reindex L0 skip ${record_id}: ${err instanceof Error ? err.message : String(err)}`);
|
|
1482
|
+
}
|
|
1483
|
+
l0Done++;
|
|
1484
|
+
onProgress?.(l0Done, l0Rows.length, "L0");
|
|
1485
|
+
}
|
|
1486
|
+
this.logger?.info(`${TAG} Reindex complete: L1=${l1Done}/${l1Rows.length}, L0=${l0Done}/${l0Rows.length}`);
|
|
1487
|
+
return { l1Count: l1Done, l0Count: l0Done };
|
|
1488
|
+
}
|
|
1489
|
+
catch (err) {
|
|
1490
|
+
this.logger?.error(`${TAG} reindexAll failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
|
|
1491
|
+
return { l1Count: 0, l0Count: 0 };
|
|
1492
|
+
}
|
|
1493
|
+
}
|
|
1494
|
+
// ── L0 query operations (for L1 runner) ──────────────────────────────────
|
|
1495
|
+
/**
|
|
1496
|
+
* Query L0 messages for a given session key, optionally filtered by recorded_at cursor.
|
|
1497
|
+
* Returns messages ordered by recorded_at ASC (chronological write order).
|
|
1498
|
+
*
|
|
1499
|
+
* Used by L1 runner to read L0 data from DB instead of JSONL files.
|
|
1500
|
+
*/
|
|
1501
|
+
queryL0ForL1(sessionKey, afterRecordedAtMs, limit = 50) {
|
|
1502
|
+
if (this.degraded) {
|
|
1503
|
+
this.logger?.warn(`${TAG} [L0-query] SKIPPED (degraded mode)`);
|
|
1504
|
+
return [];
|
|
1505
|
+
}
|
|
1506
|
+
try {
|
|
1507
|
+
// Query newest-first (DESC) with LIMIT, then reverse to chronological order
|
|
1508
|
+
let rows;
|
|
1509
|
+
if (afterRecordedAtMs && afterRecordedAtMs > 0) {
|
|
1510
|
+
// Convert epoch ms to ISO string for recorded_at comparison
|
|
1511
|
+
const afterRecordedAtIso = new Date(afterRecordedAtMs).toISOString();
|
|
1512
|
+
rows = this.stmtL0QueryAfter.all(sessionKey, afterRecordedAtIso, limit);
|
|
1513
|
+
}
|
|
1514
|
+
else {
|
|
1515
|
+
rows = this.stmtL0QueryAll.all(sessionKey, limit);
|
|
1516
|
+
}
|
|
1517
|
+
this.logger?.info(`${TAG} [L0-query] session=${sessionKey}, afterRecordedAtMs=${afterRecordedAtMs ?? "(all)"}, ` +
|
|
1518
|
+
`limit=${limit}, returned ${rows.length} row(s)`);
|
|
1519
|
+
// Reverse: SQL returns newest-first (DESC), callers expect chronological order
|
|
1520
|
+
return rows.map((r) => ({
|
|
1521
|
+
record_id: r.record_id,
|
|
1522
|
+
session_key: r.session_key,
|
|
1523
|
+
session_id: r.session_id || "",
|
|
1524
|
+
role: r.role,
|
|
1525
|
+
message_text: r.message_text,
|
|
1526
|
+
recorded_at: r.recorded_at || "",
|
|
1527
|
+
timestamp: r.timestamp || 0,
|
|
1528
|
+
})).reverse();
|
|
1529
|
+
}
|
|
1530
|
+
catch (err) {
|
|
1531
|
+
this.logger?.warn(`${TAG} [L0-query] FAILED (non-fatal, returning empty): ${err instanceof Error ? err.message : String(err)}`);
|
|
1532
|
+
return [];
|
|
1533
|
+
}
|
|
1534
|
+
}
|
|
1535
|
+
/**
|
|
1536
|
+
* Query L0 messages for a given session key, grouped by session_id.
|
|
1537
|
+
* Each group's messages are in chronological order (recorded_at ASC).
|
|
1538
|
+
* Groups are sorted by earliest message timestamp.
|
|
1539
|
+
*
|
|
1540
|
+
* Used by L1 runner to replace readConversationMessagesGroupedBySessionId().
|
|
1541
|
+
*/
|
|
1542
|
+
queryL0GroupedBySessionId(sessionKey, afterRecordedAtMs, limit = 50) {
|
|
1543
|
+
if (this.degraded) {
|
|
1544
|
+
this.logger?.warn(`${TAG} [L0-query-grouped] SKIPPED (degraded mode)`);
|
|
1545
|
+
return [];
|
|
1546
|
+
}
|
|
1547
|
+
try {
|
|
1548
|
+
const rows = this.queryL0ForL1(sessionKey, afterRecordedAtMs, limit);
|
|
1549
|
+
// Group by session_id
|
|
1550
|
+
const groupMap = new Map();
|
|
1551
|
+
for (const row of rows) {
|
|
1552
|
+
const sid = row.session_id || "";
|
|
1553
|
+
let group = groupMap.get(sid);
|
|
1554
|
+
if (!group) {
|
|
1555
|
+
group = [];
|
|
1556
|
+
groupMap.set(sid, group);
|
|
1557
|
+
}
|
|
1558
|
+
group.push({
|
|
1559
|
+
id: row.record_id,
|
|
1560
|
+
role: row.role,
|
|
1561
|
+
content: row.message_text,
|
|
1562
|
+
timestamp: row.timestamp,
|
|
1563
|
+
recordedAtMs: row.recorded_at ? Date.parse(row.recorded_at) || 0 : 0,
|
|
1564
|
+
});
|
|
1565
|
+
}
|
|
1566
|
+
// Convert to array, sorted by earliest message timestamp
|
|
1567
|
+
const groups = [];
|
|
1568
|
+
for (const [sessionId, messages] of groupMap) {
|
|
1569
|
+
if (messages.length > 0) {
|
|
1570
|
+
groups.push({ sessionId, messages });
|
|
1571
|
+
}
|
|
1572
|
+
}
|
|
1573
|
+
groups.sort((a, b) => a.messages[0].timestamp - b.messages[0].timestamp);
|
|
1574
|
+
this.logger?.info(`${TAG} [L0-query-grouped] session=${sessionKey}, afterRecordedAtMs=${afterRecordedAtMs ?? "(all)"}, ` +
|
|
1575
|
+
`${rows.length} messages across ${groups.length} group(s)`);
|
|
1576
|
+
return groups;
|
|
1577
|
+
}
|
|
1578
|
+
catch (err) {
|
|
1579
|
+
this.logger?.warn(`${TAG} [L0-query-grouped] FAILED (non-fatal, returning empty): ${err instanceof Error ? err.message : String(err)}`);
|
|
1580
|
+
return [];
|
|
1581
|
+
}
|
|
1582
|
+
}
|
|
1583
|
+
// ── Cursor-based pagination for migration ──────────────────
|
|
1584
|
+
/**
|
|
1585
|
+
* Read a page of L1 records using primary key cursor.
|
|
1586
|
+
* Returns rows with `record_id > afterId`, ordered by PK, limited to `pageSize`.
|
|
1587
|
+
* Pass `""` as `afterId` for the first page.
|
|
1588
|
+
*/
|
|
1589
|
+
queryL1RecordsCursor(afterId, pageSize) {
|
|
1590
|
+
if (this.degraded)
|
|
1591
|
+
return [];
|
|
1592
|
+
try {
|
|
1593
|
+
return this.stmtL1QueryMigrationCursor.all(afterId, pageSize);
|
|
1594
|
+
}
|
|
1595
|
+
catch (err) {
|
|
1596
|
+
this.logger?.warn(`${TAG} [L1-query-cursor] FAILED (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
|
|
1597
|
+
return [];
|
|
1598
|
+
}
|
|
1599
|
+
}
|
|
1600
|
+
/**
|
|
1601
|
+
* Read a page of L0 records using primary key cursor.
|
|
1602
|
+
* Returns rows with `record_id > afterId`, ordered by PK, limited to `pageSize`.
|
|
1603
|
+
* Pass `""` as `afterId` for the first page.
|
|
1604
|
+
*/
|
|
1605
|
+
queryL0RecordsCursor(afterId, pageSize) {
|
|
1606
|
+
if (this.degraded)
|
|
1607
|
+
return [];
|
|
1608
|
+
try {
|
|
1609
|
+
return this.stmtL0QueryMigrationCursor.all(afterId, pageSize);
|
|
1610
|
+
}
|
|
1611
|
+
catch (err) {
|
|
1612
|
+
this.logger?.warn(`${TAG} [L0-query-cursor] FAILED (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
|
|
1613
|
+
return [];
|
|
1614
|
+
}
|
|
1615
|
+
}
|
|
1616
|
+
// ── FTS5 search operations ──────────────────────────────────
|
|
1617
|
+
/**
|
|
1618
|
+
* Whether FTS5 full-text search is available.
|
|
1619
|
+
* When `false`, callers should skip keyword-based recall entirely.
|
|
1620
|
+
*/
|
|
1621
|
+
isFtsAvailable() {
|
|
1622
|
+
return this.ftsAvailable;
|
|
1623
|
+
}
|
|
1624
|
+
/**
|
|
1625
|
+
* FTS5 keyword search on L1 records.
|
|
1626
|
+
* Returns top-`limit` results sorted by BM25 relevance (highest first).
|
|
1627
|
+
*
|
|
1628
|
+
* @param ftsQuery A pre-built FTS5 MATCH expression (from `buildFtsQuery()`).
|
|
1629
|
+
* @param limit Maximum number of results to return.
|
|
1630
|
+
*
|
|
1631
|
+
* **Fault-tolerant**: returns an empty array on any error.
|
|
1632
|
+
*/
|
|
1633
|
+
searchL1Fts(ftsQuery, limit = 20) {
|
|
1634
|
+
if (this.degraded || !this.ftsAvailable)
|
|
1635
|
+
return [];
|
|
1636
|
+
try {
|
|
1637
|
+
const rows = this.stmtL1FtsSearch.all(ftsQuery, limit);
|
|
1638
|
+
return rows.map((r) => ({
|
|
1639
|
+
record_id: r.record_id,
|
|
1640
|
+
content: r.content,
|
|
1641
|
+
type: r.type,
|
|
1642
|
+
priority: r.priority,
|
|
1643
|
+
scene_name: r.scene_name,
|
|
1644
|
+
score: bm25RankToScore(r.rank),
|
|
1645
|
+
timestamp_str: r.timestamp_str,
|
|
1646
|
+
timestamp_start: r.timestamp_start,
|
|
1647
|
+
timestamp_end: r.timestamp_end,
|
|
1648
|
+
session_key: r.session_key,
|
|
1649
|
+
session_id: r.session_id,
|
|
1650
|
+
metadata_json: r.metadata_json,
|
|
1651
|
+
}));
|
|
1652
|
+
}
|
|
1653
|
+
catch (err) {
|
|
1654
|
+
this.logger?.warn(`${TAG} [L1-fts-search] FAILED (non-fatal, returning empty): ${err instanceof Error ? err.message : String(err)}`);
|
|
1655
|
+
return [];
|
|
1656
|
+
}
|
|
1657
|
+
}
|
|
1658
|
+
/**
|
|
1659
|
+
* FTS5 keyword search on L0 conversation messages.
|
|
1660
|
+
* Returns top-`limit` results sorted by BM25 relevance (highest first).
|
|
1661
|
+
*
|
|
1662
|
+
* @param ftsQuery A pre-built FTS5 MATCH expression (from `buildFtsQuery()`).
|
|
1663
|
+
* @param limit Maximum number of results to return.
|
|
1664
|
+
*
|
|
1665
|
+
* **Fault-tolerant**: returns an empty array on any error.
|
|
1666
|
+
*/
|
|
1667
|
+
searchL0Fts(ftsQuery, limit = VectorStore.FTS_DEFAULT_LIMIT) {
|
|
1668
|
+
if (this.degraded || !this.ftsAvailable)
|
|
1669
|
+
return [];
|
|
1670
|
+
try {
|
|
1671
|
+
const rows = this.stmtL0FtsSearch.all(ftsQuery, limit);
|
|
1672
|
+
return rows.map((r) => ({
|
|
1673
|
+
record_id: r.record_id,
|
|
1674
|
+
session_key: r.session_key,
|
|
1675
|
+
session_id: r.session_id,
|
|
1676
|
+
role: r.role,
|
|
1677
|
+
message_text: r.message_text,
|
|
1678
|
+
score: bm25RankToScore(r.rank),
|
|
1679
|
+
recorded_at: r.recorded_at,
|
|
1680
|
+
timestamp: r.timestamp ?? 0,
|
|
1681
|
+
}));
|
|
1682
|
+
}
|
|
1683
|
+
catch (err) {
|
|
1684
|
+
this.logger?.warn(`${TAG} [L0-fts-search] FAILED (non-fatal, returning empty): ${err instanceof Error ? err.message : String(err)}`);
|
|
1685
|
+
return [];
|
|
1686
|
+
}
|
|
1687
|
+
}
|
|
1688
|
+
// ── FTS5 migration & rebuild ──────────────────────────────────────────────
|
|
1689
|
+
/**
|
|
1690
|
+
* Detect old FTS5 v1 schema (no `content_original` column) and drop the
|
|
1691
|
+
* tables so they can be recreated with the v2 schema.
|
|
1692
|
+
*
|
|
1693
|
+
* FTS5 virtual tables do NOT support `ALTER TABLE ADD COLUMN`, so the only
|
|
1694
|
+
* migration path is DROP + recreate + repopulate.
|
|
1695
|
+
*
|
|
1696
|
+
* @returns `true` if migration was performed (= FTS index needs rebuilding).
|
|
1697
|
+
* @internal
|
|
1698
|
+
*/
|
|
1699
|
+
migrateFtsTablesIfNeeded() {
|
|
1700
|
+
try {
|
|
1701
|
+
// Check if l1_fts exists at all
|
|
1702
|
+
const l1Exists = this.db
|
|
1703
|
+
.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='l1_fts'")
|
|
1704
|
+
.get();
|
|
1705
|
+
if (!l1Exists) {
|
|
1706
|
+
// Fresh install — tables will be created with v2 schema.
|
|
1707
|
+
// Still need rebuild if there's existing data in l1_records.
|
|
1708
|
+
const hasData = this.db.prepare("SELECT 1 FROM l1_records LIMIT 1").get();
|
|
1709
|
+
return !!hasData;
|
|
1710
|
+
}
|
|
1711
|
+
// Check if the v2 column `content_original` exists.
|
|
1712
|
+
// FTS5 tables appear in pragma_table_info with their column names.
|
|
1713
|
+
const cols = this.db
|
|
1714
|
+
.prepare("SELECT name FROM pragma_table_info('l1_fts')")
|
|
1715
|
+
.all();
|
|
1716
|
+
const hasV2Col = cols.some((c) => c.name === "content_original");
|
|
1717
|
+
if (hasV2Col) {
|
|
1718
|
+
return false; // Already v2 — no migration needed
|
|
1719
|
+
}
|
|
1720
|
+
// v1 → v2: drop both FTS tables (data will be repopulated by rebuildFtsIndex)
|
|
1721
|
+
this.logger?.info(`${TAG} Migrating FTS5 tables from v1 to v2 (jieba segmented)`);
|
|
1722
|
+
this.db.exec("DROP TABLE IF EXISTS l1_fts");
|
|
1723
|
+
this.db.exec("DROP TABLE IF EXISTS l0_fts");
|
|
1724
|
+
return true;
|
|
1725
|
+
}
|
|
1726
|
+
catch (err) {
|
|
1727
|
+
this.logger?.warn(`${TAG} FTS migration check failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
|
|
1728
|
+
return false;
|
|
1729
|
+
}
|
|
1730
|
+
}
|
|
1731
|
+
/**
|
|
1732
|
+
* Rebuild the FTS5 index from scratch by reading all records from the
|
|
1733
|
+
* metadata tables and re-inserting them with jieba-segmented text.
|
|
1734
|
+
*
|
|
1735
|
+
* Called automatically after:
|
|
1736
|
+
* - Schema migration from v1 to v2
|
|
1737
|
+
* - Fresh table creation when existing data exists
|
|
1738
|
+
*
|
|
1739
|
+
* Safe to call multiple times (idempotent — clears FTS tables first).
|
|
1740
|
+
*/
|
|
1741
|
+
rebuildFtsIndex() {
|
|
1742
|
+
if (!this.ftsAvailable)
|
|
1743
|
+
return;
|
|
1744
|
+
try {
|
|
1745
|
+
this.logger?.info(`${TAG} Rebuilding FTS5 index with jieba segmentation…`);
|
|
1746
|
+
// ── Rebuild L1 FTS ──
|
|
1747
|
+
// Clear existing FTS data
|
|
1748
|
+
this.db.exec("DELETE FROM l1_fts");
|
|
1749
|
+
// Read all L1 records from metadata table
|
|
1750
|
+
const l1Rows = this.db
|
|
1751
1751
|
.prepare(`
|
|
1752
1752
|
SELECT record_id, content, type, priority, scene_name,
|
|
1753
1753
|
session_key, session_id, timestamp_str, timestamp_start, timestamp_end, metadata_json
|
|
1754
1754
|
FROM l1_records
|
|
1755
|
-
`)
|
|
1756
|
-
.all();
|
|
1757
|
-
let l1Count = 0;
|
|
1758
|
-
for (const r of l1Rows) {
|
|
1759
|
-
try {
|
|
1760
|
-
this.stmtL1FtsInsert.run(tokenizeForFts(r.content), // content — segmented
|
|
1761
|
-
r.content, // content_original — raw
|
|
1762
|
-
r.record_id, r.type, r.priority, r.scene_name, r.session_key, r.session_id, r.timestamp_str, r.timestamp_start, r.timestamp_end, r.metadata_json);
|
|
1763
|
-
l1Count++;
|
|
1764
|
-
}
|
|
1765
|
-
catch (err) {
|
|
1766
|
-
this.logger?.warn?.(`${TAG} FTS rebuild skip L1 ${r.record_id}: ${err instanceof Error ? err.message : String(err)}`);
|
|
1767
|
-
}
|
|
1768
|
-
}
|
|
1769
|
-
// ── Rebuild L0 FTS ──
|
|
1770
|
-
this.db.exec("DELETE FROM l0_fts");
|
|
1771
|
-
const l0Rows = this.db
|
|
1755
|
+
`)
|
|
1756
|
+
.all();
|
|
1757
|
+
let l1Count = 0;
|
|
1758
|
+
for (const r of l1Rows) {
|
|
1759
|
+
try {
|
|
1760
|
+
this.stmtL1FtsInsert.run(tokenizeForFts(r.content), // content — segmented
|
|
1761
|
+
r.content, // content_original — raw
|
|
1762
|
+
r.record_id, r.type, r.priority, r.scene_name, r.session_key, r.session_id, r.timestamp_str, r.timestamp_start, r.timestamp_end, r.metadata_json);
|
|
1763
|
+
l1Count++;
|
|
1764
|
+
}
|
|
1765
|
+
catch (err) {
|
|
1766
|
+
this.logger?.warn?.(`${TAG} FTS rebuild skip L1 ${r.record_id}: ${err instanceof Error ? err.message : String(err)}`);
|
|
1767
|
+
}
|
|
1768
|
+
}
|
|
1769
|
+
// ── Rebuild L0 FTS ──
|
|
1770
|
+
this.db.exec("DELETE FROM l0_fts");
|
|
1771
|
+
const l0Rows = this.db
|
|
1772
1772
|
.prepare(`
|
|
1773
1773
|
SELECT record_id, message_text, session_key, session_id, role, recorded_at, timestamp
|
|
1774
1774
|
FROM l0_conversations
|
|
1775
|
-
`)
|
|
1776
|
-
.all();
|
|
1777
|
-
let l0Count = 0;
|
|
1778
|
-
for (const r of l0Rows) {
|
|
1779
|
-
try {
|
|
1780
|
-
this.stmtL0FtsInsert.run(tokenizeForFts(r.message_text), // message_text — segmented
|
|
1781
|
-
r.message_text, // message_text_original — raw
|
|
1782
|
-
r.record_id, r.session_key, r.session_id, r.role, r.recorded_at, r.timestamp);
|
|
1783
|
-
l0Count++;
|
|
1784
|
-
}
|
|
1785
|
-
catch (err) {
|
|
1786
|
-
this.logger?.warn?.(`${TAG} FTS rebuild skip L0 ${r.record_id}: ${err instanceof Error ? err.message : String(err)}`);
|
|
1787
|
-
}
|
|
1788
|
-
}
|
|
1789
|
-
this.logger?.info(`${TAG} FTS5 rebuild complete: L1=${l1Count}/${l1Rows.length}, L0=${l0Count}/${l0Rows.length}`);
|
|
1790
|
-
}
|
|
1791
|
-
catch (err) {
|
|
1792
|
-
this.logger?.warn(`${TAG} FTS5 rebuild failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
|
|
1793
|
-
}
|
|
1794
|
-
}
|
|
1795
|
-
// ============================
|
|
1796
|
-
// IMemoryStore interface implementation
|
|
1797
|
-
// ============================
|
|
1798
|
-
/** Query the store's search capabilities. */
|
|
1799
|
-
getCapabilities() {
|
|
1800
|
-
return {
|
|
1801
|
-
vectorSearch: this.vecTablesReady,
|
|
1802
|
-
ftsSearch: this.ftsAvailable,
|
|
1803
|
-
nativeHybridSearch: false,
|
|
1804
|
-
sparseVectors: false,
|
|
1805
|
-
};
|
|
1806
|
-
}
|
|
1807
|
-
/**
|
|
1808
|
-
* Close the database connection.
|
|
1809
|
-
* Should be called on shutdown. Idempotent — safe to call multiple times.
|
|
1810
|
-
*/
|
|
1811
|
-
close() {
|
|
1812
|
-
if (this.closed)
|
|
1813
|
-
return;
|
|
1814
|
-
this.closed = true;
|
|
1815
|
-
try {
|
|
1816
|
-
this.db.close();
|
|
1817
|
-
}
|
|
1818
|
-
catch (err) {
|
|
1819
|
-
this.logger?.warn?.(`${TAG} Error closing database: ${err instanceof Error ? err.message : String(err)}`);
|
|
1820
|
-
}
|
|
1821
|
-
}
|
|
1822
|
-
}
|
|
1775
|
+
`)
|
|
1776
|
+
.all();
|
|
1777
|
+
let l0Count = 0;
|
|
1778
|
+
for (const r of l0Rows) {
|
|
1779
|
+
try {
|
|
1780
|
+
this.stmtL0FtsInsert.run(tokenizeForFts(r.message_text), // message_text — segmented
|
|
1781
|
+
r.message_text, // message_text_original — raw
|
|
1782
|
+
r.record_id, r.session_key, r.session_id, r.role, r.recorded_at, r.timestamp);
|
|
1783
|
+
l0Count++;
|
|
1784
|
+
}
|
|
1785
|
+
catch (err) {
|
|
1786
|
+
this.logger?.warn?.(`${TAG} FTS rebuild skip L0 ${r.record_id}: ${err instanceof Error ? err.message : String(err)}`);
|
|
1787
|
+
}
|
|
1788
|
+
}
|
|
1789
|
+
this.logger?.info(`${TAG} FTS5 rebuild complete: L1=${l1Count}/${l1Rows.length}, L0=${l0Count}/${l0Rows.length}`);
|
|
1790
|
+
}
|
|
1791
|
+
catch (err) {
|
|
1792
|
+
this.logger?.warn(`${TAG} FTS5 rebuild failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
|
|
1793
|
+
}
|
|
1794
|
+
}
|
|
1795
|
+
// ============================
|
|
1796
|
+
// IMemoryStore interface implementation
|
|
1797
|
+
// ============================
|
|
1798
|
+
/** Query the store's search capabilities. */
|
|
1799
|
+
getCapabilities() {
|
|
1800
|
+
return {
|
|
1801
|
+
vectorSearch: this.vecTablesReady,
|
|
1802
|
+
ftsSearch: this.ftsAvailable,
|
|
1803
|
+
nativeHybridSearch: false,
|
|
1804
|
+
sparseVectors: false,
|
|
1805
|
+
};
|
|
1806
|
+
}
|
|
1807
|
+
/**
|
|
1808
|
+
* Close the database connection.
|
|
1809
|
+
* Should be called on shutdown. Idempotent — safe to call multiple times.
|
|
1810
|
+
*/
|
|
1811
|
+
close() {
|
|
1812
|
+
if (this.closed)
|
|
1813
|
+
return;
|
|
1814
|
+
this.closed = true;
|
|
1815
|
+
try {
|
|
1816
|
+
this.db.close();
|
|
1817
|
+
}
|
|
1818
|
+
catch (err) {
|
|
1819
|
+
this.logger?.warn?.(`${TAG} Error closing database: ${err instanceof Error ? err.message : String(err)}`);
|
|
1820
|
+
}
|
|
1821
|
+
}
|
|
1822
|
+
}
|