dsh-prime-memory 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.en.md +28 -0
- package/CHANGELOG.ja.md +30 -0
- package/CHANGELOG.ko.md +30 -0
- package/CHANGELOG.md +1220 -0
- package/ENGINEERING-NOTES.md +452 -0
- package/INSTALL.en.md +92 -0
- package/INSTALL.ja.md +92 -0
- package/INSTALL.ko.md +92 -0
- package/INSTALL.md +92 -0
- package/LICENSE +21 -0
- package/README.en.md +458 -0
- package/README.ja.md +306 -0
- package/README.ko.md +306 -0
- package/README.md +425 -0
- package/assets/changelog/0.8.10/01-write-only-pill.png +0 -0
- package/assets/changelog/0.8.9/01-panel.png +0 -0
- package/assets/changelog/0.8.9/02-halo.png +0 -0
- package/assets/changelog/0.8.9/03-layer-segmented-panel.png +0 -0
- package/assets/changelog/0.8.9/04-layer-l1-panel.png +0 -0
- package/assets/img/EmbeddingSource.png +0 -0
- package/assets/img/Hero.png +0 -0
- package/assets/img/Layers.png +0 -0
- package/assets/img/MemoryTools.png +0 -0
- package/assets/img/Modes.png +0 -0
- package/assets/img/ToolTrajectory.png +0 -0
- package/assets/img/ui-dark.jpg +0 -0
- package/assets/img/ui-light.jpg +0 -0
- package/assets/readme/bench-dialog.svg +70 -0
- package/assets/readme/bench-workflow.svg +79 -0
- package/assets/readme/flow.svg +189 -0
- package/assets/readme/storage.svg +115 -0
- package/cordis.patch.yml +16 -0
- package/dist/bench-control.d.ts +34 -0
- package/dist/bench-control.js +16 -0
- package/dist/client.js +4293 -0
- package/dist/config.d.ts +683 -0
- package/dist/config.js +129 -0
- package/dist/contract.d.ts +820 -0
- package/dist/contract.js +1 -0
- package/dist/embedding-worker.cjs +176 -0
- package/dist/graph/apply.d.ts +37 -0
- package/dist/graph/apply.js +270 -0
- package/dist/graph/constraints.d.ts +47 -0
- package/dist/graph/constraints.js +38 -0
- package/dist/graph/search.d.ts +16 -0
- package/dist/graph/search.js +115 -0
- package/dist/graph/types.d.ts +142 -0
- package/dist/graph/types.js +14 -0
- package/dist/hooks/capture.d.ts +32 -0
- package/dist/hooks/capture.js +194 -0
- package/dist/hooks/recall.d.ts +63 -0
- package/dist/hooks/recall.js +429 -0
- package/dist/index.d.ts +534 -0
- package/dist/index.js +344 -0
- package/dist/llm-usage.d.ts +26 -0
- package/dist/llm-usage.js +37 -0
- package/dist/llm.d.ts +153 -0
- package/dist/llm.js +530 -0
- package/dist/pipeline/graph.d.ts +35 -0
- package/dist/pipeline/graph.js +104 -0
- package/dist/pipeline/l1.d.ts +19 -0
- package/dist/pipeline/l1.js +271 -0
- package/dist/pipeline/l2.d.ts +13 -0
- package/dist/pipeline/l2.js +83 -0
- package/dist/pipeline/l3.d.ts +15 -0
- package/dist/pipeline/l3.js +78 -0
- package/dist/pipeline/rebuild.d.ts +61 -0
- package/dist/pipeline/rebuild.js +307 -0
- package/dist/pipeline/ruminate.d.ts +89 -0
- package/dist/pipeline/ruminate.js +298 -0
- package/dist/pipeline/runner.d.ts +167 -0
- package/dist/pipeline/runner.js +638 -0
- package/dist/pipeline/trigger.d.ts +40 -0
- package/dist/pipeline/trigger.js +75 -0
- package/dist/prompts/graph-projection.d.ts +70 -0
- package/dist/prompts/graph-projection.js +167 -0
- package/dist/prompts/l1-dedup.d.ts +22 -0
- package/dist/prompts/l1-dedup.js +251 -0
- package/dist/prompts/l1-extraction.d.ts +22 -0
- package/dist/prompts/l1-extraction.js +457 -0
- package/dist/prompts/persona.d.ts +23 -0
- package/dist/prompts/persona.js +240 -0
- package/dist/prompts/scene.d.ts +32 -0
- package/dist/prompts/scene.js +414 -0
- package/dist/runtime-package-lock.json +982 -0
- package/dist/settings.d.ts +50 -0
- package/dist/settings.js +355 -0
- package/dist/stats.d.ts +109 -0
- package/dist/stats.js +929 -0
- package/dist/store/bm25.d.ts +19 -0
- package/dist/store/bm25.js +63 -0
- package/dist/store/cost-ledger.d.ts +75 -0
- package/dist/store/cost-ledger.js +171 -0
- package/dist/store/download-queue.d.ts +79 -0
- package/dist/store/download-queue.js +424 -0
- package/dist/store/embedding-source.d.ts +118 -0
- package/dist/store/embedding-source.js +443 -0
- package/dist/store/embedding.d.ts +90 -0
- package/dist/store/embedding.js +206 -0
- package/dist/store/graph-store.d.ts +94 -0
- package/dist/store/graph-store.js +641 -0
- package/dist/store/l0.d.ts +40 -0
- package/dist/store/l0.js +197 -0
- package/dist/store/l1.d.ts +93 -0
- package/dist/store/l1.js +297 -0
- package/dist/store/local-embedding.d.ts +89 -0
- package/dist/store/local-embedding.js +227 -0
- package/dist/store/model-catalog.d.ts +48 -0
- package/dist/store/model-catalog.js +81 -0
- package/dist/store/occupancy.d.ts +30 -0
- package/dist/store/occupancy.js +134 -0
- package/dist/store/pending.d.ts +36 -0
- package/dist/store/pending.js +103 -0
- package/dist/store/persona.d.ts +15 -0
- package/dist/store/persona.js +60 -0
- package/dist/store/recall-dedupe.d.ts +26 -0
- package/dist/store/recall-dedupe.js +138 -0
- package/dist/store/runtime-installer.d.ts +59 -0
- package/dist/store/runtime-installer.js +243 -0
- package/dist/store/scenes.d.ts +24 -0
- package/dist/store/scenes.js +160 -0
- package/dist/store/search-utils.d.ts +38 -0
- package/dist/store/search-utils.js +100 -0
- package/dist/store/session-modes.d.ts +35 -0
- package/dist/store/session-modes.js +144 -0
- package/dist/store/sqlite.d.ts +246 -0
- package/dist/store/sqlite.js +1491 -0
- package/dist/store/state.d.ts +41 -0
- package/dist/store/state.js +72 -0
- package/dist/token-cost.d.ts +23 -0
- package/dist/token-cost.js +185 -0
- package/dist/tools/index.d.ts +34 -0
- package/dist/tools/index.js +758 -0
- package/dist/types.d.ts +139 -0
- package/dist/types.js +38 -0
- package/dist/util/context-occupancy.d.ts +68 -0
- package/dist/util/context-occupancy.js +92 -0
- package/dist/util/filelog.d.ts +6 -0
- package/dist/util/filelog.js +108 -0
- package/dist/util/io.d.ts +18 -0
- package/dist/util/io.js +97 -0
- package/dist/util/recall-budget.d.ts +31 -0
- package/dist/util/recall-budget.js +84 -0
- package/dist/util/sanitize.d.ts +11 -0
- package/dist/util/sanitize.js +67 -0
- package/dist/util/text.d.ts +16 -0
- package/dist/util/text.js +61 -0
- package/dist/util/tokenizer.d.ts +9 -0
- package/dist/util/tokenizer.js +50 -0
- package/dsh.plugin.json +22 -0
- package/package.json +118 -0
|
@@ -0,0 +1,1491 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* SQLite 主检索引擎(单机裁剪版)。
|
|
3
|
+
*
|
|
4
|
+
* 双写架构:JSONL 追加文件是备份/恢复的事实源,本库承担全部检索——L0/L1 的
|
|
5
|
+
* 检索不再扫文件、不再全量载入内存。
|
|
6
|
+
*
|
|
7
|
+
* - l1_records / l0_conversations:结构化元数据表;
|
|
8
|
+
* - l1_fts / l0_fts:FTS5 BM25 全文索引(content 列存分词结果,其余列 UNINDEXED 随行携带);
|
|
9
|
+
* - l1_vec / l0_vec:sqlite-vec vec0 余弦向量表(仅 embedding 启用且维度 > 0 时创建)。
|
|
10
|
+
*
|
|
11
|
+
* 降级规则(degrade-don't-crash):
|
|
12
|
+
* - sqlite-vec 加载失败 → 纯 FTS 模式,能力位 vectorSearch=false;
|
|
13
|
+
* - FTS5 建表失败 → ftsSearch=false;
|
|
14
|
+
* - 任何一步 schema 初始化失败 → degraded=true,全部读写变为安全 no-op,
|
|
15
|
+
* 由上层走 storageOk=false 降级链路,绝不拖垮宿主启动。
|
|
16
|
+
*
|
|
17
|
+
* DDL/行格式是磁盘契约:建表语句、列名、timestamp_str 逗号连接 ISO 格式、
|
|
18
|
+
* embedding_meta 的 kv 约定一律不可更动,否则老用户的库不可读。
|
|
19
|
+
*/
|
|
20
|
+
import { createRequire } from 'node:module';
|
|
21
|
+
import { existsSync, mkdirSync } from 'node:fs';
|
|
22
|
+
import * as path from 'node:path';
|
|
23
|
+
import { familyForType, normPersistence } from '../types.js';
|
|
24
|
+
import { bm25RankToScore, buildFtsQuery, tokenizeForFts } from './search-utils.js';
|
|
25
|
+
import { describeTokenizer, ensureTokenizer, tokenizerStamp } from '../util/tokenizer.js';
|
|
26
|
+
const require = createRequire(import.meta.url);
|
|
27
|
+
const TAG = '[memory][sqlite]';
|
|
28
|
+
// 成本账本(token_cost 表族)独立职责,此处 re-export 不断裂既有引用;
|
|
29
|
+
// MemoryDb 的四个公开方法为一行委托。
|
|
30
|
+
import { CostLedger } from './cost-ledger.js';
|
|
31
|
+
// 图谱存储(graph_* 表族)同为独立职责类;init 失败仅图谱 no-op,不传染主库降级
|
|
32
|
+
import { GraphStore } from './graph-store.js';
|
|
33
|
+
/** vec0 KNN 对遗留零向量的补偿缓冲。 */
|
|
34
|
+
const ZERO_VEC_BUFFER = 10;
|
|
35
|
+
/** IN 查询/删除的分块大小(保守避开 SQLite 变量数上限:现代构建 32766,老版 999)。 */
|
|
36
|
+
const IN_CHUNK = 900;
|
|
37
|
+
/** 零向量 skip 集上限(≤ IN_CHUNK:notInClause 不分块,占位符数即集合大小)。 */
|
|
38
|
+
const VEC_SKIP_CAP = 900;
|
|
39
|
+
/** 把 id 列表切成 ≤IN_CHUNK 的块(分块执行后合并语义等价于单次 IN)。 */
|
|
40
|
+
function chunkIds(ids) {
|
|
41
|
+
const out = [];
|
|
42
|
+
for (let i = 0; i < ids.length; i += IN_CHUNK)
|
|
43
|
+
out.push(ids.slice(i, i + IN_CHUNK));
|
|
44
|
+
return out;
|
|
45
|
+
}
|
|
46
|
+
export class MemoryDb {
|
|
47
|
+
db;
|
|
48
|
+
degraded = false;
|
|
49
|
+
ftsAvailable = false;
|
|
50
|
+
vecLoaded = false;
|
|
51
|
+
vecLoadWarned = false;
|
|
52
|
+
/** 向量维度:活切换嵌入源时会变——vec0 表随维度重建。 */
|
|
53
|
+
dimensions;
|
|
54
|
+
logger;
|
|
55
|
+
stmtUpsertL1;
|
|
56
|
+
stmtGetL1;
|
|
57
|
+
/** 主表存在性点查(防御性 FTS 删除的前置判断,走主键索引)。 */
|
|
58
|
+
stmtL1Exists;
|
|
59
|
+
stmtDeleteL1Meta;
|
|
60
|
+
stmtDeleteL1Vec;
|
|
61
|
+
stmtInsertL1Vec;
|
|
62
|
+
stmtSearchL1Vec;
|
|
63
|
+
stmtL1FtsInsert;
|
|
64
|
+
stmtL1FtsDelete;
|
|
65
|
+
stmtL1FtsSearch;
|
|
66
|
+
stmtL1FtsSearchFamily;
|
|
67
|
+
/** 成本账本(token_cost 表族;init 内初始化,未就绪时方法返回零值)。 */
|
|
68
|
+
costLedger = new CostLedger();
|
|
69
|
+
/** 图谱存储(graph_* 表族;init 独立 try/catch,失败仅图谱 no-op)。 */
|
|
70
|
+
graphStore = new GraphStore();
|
|
71
|
+
stmtUpsertL0;
|
|
72
|
+
stmtGetL0;
|
|
73
|
+
stmtL0Exists;
|
|
74
|
+
stmtDeleteL0Vec;
|
|
75
|
+
stmtInsertL0Vec;
|
|
76
|
+
stmtSearchL0Vec;
|
|
77
|
+
stmtL0FtsInsert;
|
|
78
|
+
stmtL0FtsDelete;
|
|
79
|
+
stmtL0FtsSearch;
|
|
80
|
+
/** 按块缓存的 IN 语句(表名/动作/尺寸 → 预编译语句):热路径不再每次动态 prepare。 */
|
|
81
|
+
inStmts = new Map();
|
|
82
|
+
constructor(dbPath, dimensions, logger) {
|
|
83
|
+
this.dimensions = dimensions;
|
|
84
|
+
this.logger = logger;
|
|
85
|
+
// 构造永不抛出(storage-degrade 不变量):开库/PRAGMA 任一失败 → 降级模式,
|
|
86
|
+
// 全部读写变为安全 no-op,宿主照常启动。
|
|
87
|
+
try {
|
|
88
|
+
const dbDir = path.dirname(dbPath);
|
|
89
|
+
if (!existsSync(dbDir))
|
|
90
|
+
mkdirSync(dbDir, { recursive: true });
|
|
91
|
+
const { DatabaseSync: DbSync } = require('node:sqlite');
|
|
92
|
+
this.db = new DbSync(dbPath, { allowExtension: true });
|
|
93
|
+
// 并发读优化 + 有界内存;synchronous NORMAL 是 WAL 下的官方推荐——批量写从
|
|
94
|
+
// "每事务一次 fsync"降为"每 checkpoint 一次",重嵌入/导入提速明显;
|
|
95
|
+
// 代价仅是断电时丢最后若干已提交事务(只丢不损,无损坏风险)
|
|
96
|
+
this.db.exec('PRAGMA busy_timeout = 5000');
|
|
97
|
+
this.db.exec('PRAGMA journal_mode = WAL');
|
|
98
|
+
this.db.exec('PRAGMA synchronous = NORMAL');
|
|
99
|
+
this.db.exec('PRAGMA cache_size = -65536');
|
|
100
|
+
this.db.exec('PRAGMA mmap_size = 134217728');
|
|
101
|
+
this.db.exec('PRAGMA wal_autocheckpoint = 1000');
|
|
102
|
+
}
|
|
103
|
+
catch (err) {
|
|
104
|
+
this.degraded = true;
|
|
105
|
+
this.logger?.error(`${TAG} 数据库打开失败,存储进入降级模式(记忆读写停用): ${err instanceof Error ? err.message : String(err)}`);
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
isDegraded() {
|
|
109
|
+
return this.degraded;
|
|
110
|
+
}
|
|
111
|
+
getCapabilities() {
|
|
112
|
+
return {
|
|
113
|
+
ftsSearch: this.ftsAvailable && !this.degraded,
|
|
114
|
+
vectorSearch: this.vecLoaded && this.dimensions > 0 && !this.degraded,
|
|
115
|
+
};
|
|
116
|
+
}
|
|
117
|
+
/** 统一事务边界:fn 抛出即 ROLLBACK 并把错误上抛(替代散落的手写 BEGIN/COMMIT/ROLLBACK)。 */
|
|
118
|
+
withTransaction(fn) {
|
|
119
|
+
this.db.exec('BEGIN');
|
|
120
|
+
try {
|
|
121
|
+
const result = fn();
|
|
122
|
+
this.db.exec('COMMIT');
|
|
123
|
+
return result;
|
|
124
|
+
}
|
|
125
|
+
catch (err) {
|
|
126
|
+
try {
|
|
127
|
+
this.db.exec('ROLLBACK');
|
|
128
|
+
}
|
|
129
|
+
catch {
|
|
130
|
+
/* ignore */
|
|
131
|
+
}
|
|
132
|
+
throw err;
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
/**
|
|
136
|
+
* 加载 sqlite-vec 扩展并建 schema。构造后必须调用一次。
|
|
137
|
+
* providerInfo 变化(provider/model/维度)时 drop 向量表并返回 needsReindex。
|
|
138
|
+
*/
|
|
139
|
+
init(providerInfo) {
|
|
140
|
+
// 构造期已降级(开库失败)→ 直接短路
|
|
141
|
+
if (this.degraded)
|
|
142
|
+
return { needsReindex: false, reason: 'database open failed' };
|
|
143
|
+
// dimensions=0 是合法的"纯 FTS 模式",不能因 sqlite-vec 缺失而降级;
|
|
144
|
+
// 后续活切换本地嵌入(维度 > 0)时由 swapProvider 补加载
|
|
145
|
+
this.ensureVecLoaded();
|
|
146
|
+
// 分词器在首次 FTS 写入(迁移回灌)前定死模式:jieba 就绪 info / 回退 warn 一次
|
|
147
|
+
ensureTokenizer();
|
|
148
|
+
this.logger?.info(`${TAG} 分词器:${describeTokenizer()}`);
|
|
149
|
+
try {
|
|
150
|
+
return this.initSchema(providerInfo);
|
|
151
|
+
}
|
|
152
|
+
catch (err) {
|
|
153
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
154
|
+
this.logger?.error(`${TAG} schema 初始化失败,存储进入降级模式: ${message}`);
|
|
155
|
+
this.degraded = true;
|
|
156
|
+
return { needsReindex: false, reason: `schema init failed: ${message}` };
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
/** 惰性加载 sqlite-vec(纯 FTS 起步后切本地嵌入时补加载);失败只停用向量能力并告警一次。 */
|
|
160
|
+
ensureVecLoaded() {
|
|
161
|
+
if (this.vecLoaded || this.dimensions <= 0)
|
|
162
|
+
return;
|
|
163
|
+
try {
|
|
164
|
+
const sqliteVec = require('sqlite-vec');
|
|
165
|
+
this.db.enableLoadExtension(true);
|
|
166
|
+
try {
|
|
167
|
+
sqliteVec.load(this.db);
|
|
168
|
+
this.vecLoaded = true;
|
|
169
|
+
}
|
|
170
|
+
finally {
|
|
171
|
+
// 加载失败也必须复位扩展开关,不留常开的扩展加载面
|
|
172
|
+
this.db.enableLoadExtension(false);
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
catch (err) {
|
|
176
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
177
|
+
if (!this.vecLoadWarned) {
|
|
178
|
+
this.vecLoadWarned = true;
|
|
179
|
+
this.logger?.warn(`${TAG} sqlite-vec 加载失败,向量检索停用(降级为纯 FTS): ${message}`);
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
/**
|
|
184
|
+
* 活切换嵌入源:provider/model/维度任一变化 → drop 向量表按新维度重建,
|
|
185
|
+
* 返回 needsReindex=true(调用方后台重嵌,全部成功后 markEmbeddingSynced);
|
|
186
|
+
* 配置未变化 → false(切回同一模型不重嵌)。
|
|
187
|
+
* 新维度 > 0 但 sqlite-vec 不可用 → ok=false(调用方向用户说明,维持 FTS)。
|
|
188
|
+
*/
|
|
189
|
+
swapProvider(info) {
|
|
190
|
+
if (this.degraded)
|
|
191
|
+
return { ok: false, needsReindex: false, error: '数据库降级模式' };
|
|
192
|
+
if (info.dimensions <= 0) {
|
|
193
|
+
// 关闭档不走此路径(只换服务实例);防御性兜底:不动表,无需重嵌
|
|
194
|
+
this.dimensions = 0;
|
|
195
|
+
return { ok: true, needsReindex: false };
|
|
196
|
+
}
|
|
197
|
+
// 先置新维度再懒加载(ensureVecLoaded 以 dimensions>0 为门)——0 维起步的库
|
|
198
|
+
// 在首次活切本地/远程嵌入时此处补加载 sqlite-vec
|
|
199
|
+
this.dimensions = info.dimensions;
|
|
200
|
+
this.ensureVecLoaded();
|
|
201
|
+
if (!this.vecLoaded) {
|
|
202
|
+
return { ok: false, needsReindex: false, error: 'sqlite-vec 扩展不可用,无法启用向量检索' };
|
|
203
|
+
}
|
|
204
|
+
// unchanged 判据不能只信 embedding_meta:取消/崩溃路径可能留下"meta=旧 provider、
|
|
205
|
+
// 物理表=新维度"的错位(切换已 drop 并按新维度重建,但重嵌取消/失败没写 meta)。
|
|
206
|
+
// 只比对 meta 会在切回旧源时跳过 drop → 旧维度服务往新维度表写向量 → vec0 抛错
|
|
207
|
+
// → upsert 整体回滚 → 记录从检索库静默消失。必须同时校验物理表的真实维度。
|
|
208
|
+
const saved = this.readEmbeddingMeta();
|
|
209
|
+
const physical = this.physicalVecDims();
|
|
210
|
+
const unchanged = saved &&
|
|
211
|
+
saved.provider === info.provider &&
|
|
212
|
+
saved.model === info.model &&
|
|
213
|
+
saved.dimensions === info.dimensions &&
|
|
214
|
+
physical === info.dimensions;
|
|
215
|
+
if (unchanged)
|
|
216
|
+
return { ok: true, needsReindex: false };
|
|
217
|
+
this.dropVectorTables();
|
|
218
|
+
return { ok: true, needsReindex: true };
|
|
219
|
+
}
|
|
220
|
+
/** l1_vec 物理表的向量维度(建表 DDL 里的 float[N]);无表返回 null。 */
|
|
221
|
+
physicalVecDims() {
|
|
222
|
+
try {
|
|
223
|
+
const row = this.db
|
|
224
|
+
.prepare("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'l1_vec'")
|
|
225
|
+
.get();
|
|
226
|
+
if (!row?.sql)
|
|
227
|
+
return null;
|
|
228
|
+
const m = /float\[(\d+)\]/.exec(row.sql);
|
|
229
|
+
return m ? Number(m[1]) : null;
|
|
230
|
+
}
|
|
231
|
+
catch {
|
|
232
|
+
return null;
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
initSchema(providerInfo) {
|
|
236
|
+
this.db.exec(`
|
|
237
|
+
CREATE TABLE IF NOT EXISTS embedding_meta (
|
|
238
|
+
key TEXT PRIMARY KEY,
|
|
239
|
+
value TEXT NOT NULL
|
|
240
|
+
)
|
|
241
|
+
`);
|
|
242
|
+
// ── embedding 配置变化检测 → 重建向量表 ──
|
|
243
|
+
let needsReindex = false;
|
|
244
|
+
let reason;
|
|
245
|
+
const savedMeta = this.readEmbeddingMeta();
|
|
246
|
+
if (providerInfo) {
|
|
247
|
+
if (savedMeta) {
|
|
248
|
+
const providerChanged = savedMeta.provider !== providerInfo.provider;
|
|
249
|
+
const modelChanged = savedMeta.model !== providerInfo.model;
|
|
250
|
+
const dimsChanged = savedMeta.dimensions !== this.dimensions;
|
|
251
|
+
if (providerChanged || modelChanged || dimsChanged) {
|
|
252
|
+
const reasons = [];
|
|
253
|
+
if (providerChanged)
|
|
254
|
+
reasons.push(`provider: ${savedMeta.provider} → ${providerInfo.provider}`);
|
|
255
|
+
if (modelChanged)
|
|
256
|
+
reasons.push(`model: ${savedMeta.model} → ${providerInfo.model}`);
|
|
257
|
+
if (dimsChanged)
|
|
258
|
+
reasons.push(`dimensions: ${savedMeta.dimensions} → ${this.dimensions}`);
|
|
259
|
+
reason = reasons.join(', ');
|
|
260
|
+
this.logger?.info(`${TAG} embedding 配置变化(${reason}),重建向量表`);
|
|
261
|
+
this.dropVectorTables();
|
|
262
|
+
needsReindex = true;
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
else if (this.countL1() > 0 || this.countL0() > 0) {
|
|
266
|
+
// 已有数据但无 meta 记录(首次启用 embedding)→ 需要全量重嵌入
|
|
267
|
+
this.dropVectorTables();
|
|
268
|
+
needsReindex = true;
|
|
269
|
+
reason = 'embedding 首次启用,已有数据需要重嵌入';
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
// ── L1 schema(DDL 为磁盘契约,逐字节不可变) ──
|
|
273
|
+
this.db.exec(`
|
|
274
|
+
CREATE TABLE IF NOT EXISTS l1_records (
|
|
275
|
+
record_id TEXT PRIMARY KEY,
|
|
276
|
+
content TEXT NOT NULL,
|
|
277
|
+
type TEXT DEFAULT '',
|
|
278
|
+
priority INTEGER DEFAULT 50,
|
|
279
|
+
scene_name TEXT DEFAULT '',
|
|
280
|
+
session_id TEXT DEFAULT 'default',
|
|
281
|
+
version INTEGER NOT NULL DEFAULT 0,
|
|
282
|
+
timestamp_str TEXT DEFAULT '',
|
|
283
|
+
timestamp_start TEXT DEFAULT '',
|
|
284
|
+
timestamp_end TEXT DEFAULT '',
|
|
285
|
+
created_time TEXT DEFAULT '',
|
|
286
|
+
updated_time TEXT DEFAULT '',
|
|
287
|
+
metadata_json TEXT DEFAULT '{}',
|
|
288
|
+
family TEXT NOT NULL DEFAULT 'chat'
|
|
289
|
+
)
|
|
290
|
+
`);
|
|
291
|
+
// 旧库缺 family 列 → ALTER 补列,并按 type 前缀回填(幂等:已正确的行不再命中)
|
|
292
|
+
if (!this.hasColumn('l1_records', 'family')) {
|
|
293
|
+
this.db.exec("ALTER TABLE l1_records ADD COLUMN family TEXT NOT NULL DEFAULT 'chat'");
|
|
294
|
+
this.logger?.info(`${TAG} l1_records 补 family 列(旧数据按 type 前缀回填)`);
|
|
295
|
+
}
|
|
296
|
+
const backfilled = this.db
|
|
297
|
+
.prepare("UPDATE l1_records SET family = 'work' WHERE type LIKE 'work\\_%' ESCAPE '\\' AND family != 'work'")
|
|
298
|
+
.run().changes;
|
|
299
|
+
if (backfilled > 0)
|
|
300
|
+
this.logger?.info(`${TAG} family 回填 ${backfilled} 条 work 记录`);
|
|
301
|
+
this.ensureTemporalColumns();
|
|
302
|
+
this.db.exec('CREATE INDEX IF NOT EXISTS idx_l1_type ON l1_records(type)');
|
|
303
|
+
this.db.exec('CREATE INDEX IF NOT EXISTS idx_l1_scene ON l1_records(scene_name)');
|
|
304
|
+
this.db.exec('CREATE INDEX IF NOT EXISTS idx_l1_ts_start ON l1_records(timestamp_start)');
|
|
305
|
+
this.db.exec('CREATE INDEX IF NOT EXISTS idx_l1_updated ON l1_records(updated_time)');
|
|
306
|
+
this.db.exec('CREATE INDEX IF NOT EXISTS idx_l1_family ON l1_records(family)');
|
|
307
|
+
this.db.exec('CREATE INDEX IF NOT EXISTS idx_l1_valid_from ON l1_records(valid_from)');
|
|
308
|
+
this.stmtUpsertL1 = this.db.prepare(`
|
|
309
|
+
INSERT INTO l1_records (
|
|
310
|
+
record_id, content, type, priority, scene_name, session_id, version,
|
|
311
|
+
timestamp_str, timestamp_start, timestamp_end, created_time, updated_time, metadata_json, family,
|
|
312
|
+
valid_from, valid_to, persistence
|
|
313
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
314
|
+
ON CONFLICT(record_id) DO UPDATE SET
|
|
315
|
+
content=excluded.content,
|
|
316
|
+
type=excluded.type,
|
|
317
|
+
priority=excluded.priority,
|
|
318
|
+
scene_name=excluded.scene_name,
|
|
319
|
+
version=excluded.version,
|
|
320
|
+
timestamp_str=excluded.timestamp_str,
|
|
321
|
+
timestamp_start=excluded.timestamp_start,
|
|
322
|
+
timestamp_end=excluded.timestamp_end,
|
|
323
|
+
updated_time=excluded.updated_time,
|
|
324
|
+
metadata_json=excluded.metadata_json,
|
|
325
|
+
family=excluded.family,
|
|
326
|
+
valid_from=excluded.valid_from,
|
|
327
|
+
valid_to=excluded.valid_to,
|
|
328
|
+
persistence=excluded.persistence
|
|
329
|
+
`);
|
|
330
|
+
this.stmtGetL1 = this.db.prepare(`
|
|
331
|
+
SELECT record_id, content, type, priority, scene_name, version, timestamp_str,
|
|
332
|
+
timestamp_start, timestamp_end, created_time, updated_time, metadata_json, family,
|
|
333
|
+
valid_from, valid_to, persistence
|
|
334
|
+
FROM l1_records WHERE record_id = ?
|
|
335
|
+
`);
|
|
336
|
+
this.stmtL1Exists = this.db.prepare('SELECT 1 FROM l1_records WHERE record_id = ?');
|
|
337
|
+
this.stmtDeleteL1Meta = this.db.prepare('DELETE FROM l1_records WHERE record_id = ?');
|
|
338
|
+
this.prepareL1VecStatements();
|
|
339
|
+
// ── L0 schema ──
|
|
340
|
+
this.db.exec(`
|
|
341
|
+
CREATE TABLE IF NOT EXISTS l0_conversations (
|
|
342
|
+
record_id TEXT PRIMARY KEY,
|
|
343
|
+
session_id TEXT DEFAULT 'default',
|
|
344
|
+
role TEXT NOT NULL DEFAULT '',
|
|
345
|
+
message_text TEXT NOT NULL,
|
|
346
|
+
recorded_at TEXT DEFAULT '',
|
|
347
|
+
timestamp INTEGER DEFAULT 0
|
|
348
|
+
)
|
|
349
|
+
`);
|
|
350
|
+
this.db.exec('CREATE INDEX IF NOT EXISTS idx_l0_session_id ON l0_conversations(session_id)');
|
|
351
|
+
this.db.exec('CREATE INDEX IF NOT EXISTS idx_l0_recorded ON l0_conversations(recorded_at)');
|
|
352
|
+
this.db.exec('CREATE INDEX IF NOT EXISTS idx_l0_timestamp ON l0_conversations(timestamp)');
|
|
353
|
+
this.stmtUpsertL0 = this.db.prepare(`
|
|
354
|
+
INSERT INTO l0_conversations (record_id, session_id, role, message_text, recorded_at, timestamp)
|
|
355
|
+
VALUES (?, ?, ?, ?, ?, ?)
|
|
356
|
+
ON CONFLICT(record_id) DO UPDATE SET
|
|
357
|
+
session_id=excluded.session_id,
|
|
358
|
+
role=excluded.role,
|
|
359
|
+
message_text=excluded.message_text,
|
|
360
|
+
recorded_at=excluded.recorded_at,
|
|
361
|
+
timestamp=excluded.timestamp
|
|
362
|
+
`);
|
|
363
|
+
this.stmtGetL0 = this.db.prepare('SELECT session_id, role, message_text, recorded_at, timestamp FROM l0_conversations WHERE record_id = ?');
|
|
364
|
+
this.stmtL0Exists = this.db.prepare('SELECT 1 FROM l0_conversations WHERE record_id = ?');
|
|
365
|
+
this.prepareL0VecStatements();
|
|
366
|
+
// ── token_cost:蒸馏成本明细表(成本账本自治) ──
|
|
367
|
+
this.costLedger.init(this.db, this.logger);
|
|
368
|
+
// ── graph_*:知识图谱投影表族(GraphStore.init 自带 try/catch,失败仅图谱 no-op) ──
|
|
369
|
+
this.graphStore.init(this.db, this.logger);
|
|
370
|
+
// ── FTS5 全文索引(建表失败仅停用 FTS,不降级整个库) ──
|
|
371
|
+
try {
|
|
372
|
+
// 索引重建判据(FTS5 无法 ALTER,只能 drop 后从源表全量回灌):
|
|
373
|
+
// a) 旧 l1_fts 无 family 列;b) FTS 分词器版本戳 ≠ 当前生效分词器
|
|
374
|
+
// (无戳 = jieba 引入前的二元组索引;切换后旧 token 形态不再匹配,须重建)。
|
|
375
|
+
const wantStamp = tokenizerStamp();
|
|
376
|
+
const savedStamp = this.readMetaString('fts_tokenizer') ?? 'bigram-v1';
|
|
377
|
+
const tokenizerChanged = savedStamp !== wantStamp;
|
|
378
|
+
let ftsRebuilt = false;
|
|
379
|
+
if (this.tableExists('l1_fts') && (!this.hasColumn('l1_fts', 'family') || tokenizerChanged)) {
|
|
380
|
+
this.db.exec('DROP TABLE l1_fts');
|
|
381
|
+
ftsRebuilt = true;
|
|
382
|
+
this.logger?.info(`${TAG} l1_fts 缺 family 列或分词器已变更(${savedStamp} → ${wantStamp}),重建全文索引`);
|
|
383
|
+
}
|
|
384
|
+
let l0FtsRebuilt = false;
|
|
385
|
+
if (this.tableExists('l0_fts') && tokenizerChanged) {
|
|
386
|
+
this.db.exec('DROP TABLE l0_fts');
|
|
387
|
+
l0FtsRebuilt = true;
|
|
388
|
+
this.logger?.info(`${TAG} l0_fts 分词器已变更(${savedStamp} → ${wantStamp}),重建全文索引`);
|
|
389
|
+
}
|
|
390
|
+
this.db.exec(`
|
|
391
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS l1_fts USING fts5(
|
|
392
|
+
content,
|
|
393
|
+
content_original UNINDEXED,
|
|
394
|
+
record_id UNINDEXED,
|
|
395
|
+
type UNINDEXED,
|
|
396
|
+
priority UNINDEXED,
|
|
397
|
+
scene_name UNINDEXED,
|
|
398
|
+
session_id UNINDEXED,
|
|
399
|
+
version UNINDEXED,
|
|
400
|
+
timestamp_str UNINDEXED,
|
|
401
|
+
timestamp_start UNINDEXED,
|
|
402
|
+
timestamp_end UNINDEXED,
|
|
403
|
+
metadata_json UNINDEXED,
|
|
404
|
+
family UNINDEXED
|
|
405
|
+
)
|
|
406
|
+
`);
|
|
407
|
+
this.db.exec(`
|
|
408
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS l0_fts USING fts5(
|
|
409
|
+
message_text,
|
|
410
|
+
message_text_original UNINDEXED,
|
|
411
|
+
record_id UNINDEXED,
|
|
412
|
+
session_id UNINDEXED,
|
|
413
|
+
role UNINDEXED,
|
|
414
|
+
recorded_at UNINDEXED,
|
|
415
|
+
timestamp UNINDEXED
|
|
416
|
+
)
|
|
417
|
+
`);
|
|
418
|
+
this.stmtL1FtsInsert = this.db.prepare(`
|
|
419
|
+
INSERT INTO l1_fts (content, content_original, record_id, type, priority, scene_name,
|
|
420
|
+
session_id, version, timestamp_str, timestamp_start, timestamp_end, metadata_json, family)
|
|
421
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
422
|
+
`);
|
|
423
|
+
this.stmtL1FtsDelete = this.db.prepare('DELETE FROM l1_fts WHERE record_id = ?');
|
|
424
|
+
this.stmtL1FtsSearch = this.db.prepare(`
|
|
425
|
+
SELECT record_id, content_original AS content, type, priority, scene_name, version,
|
|
426
|
+
timestamp_str, timestamp_start, timestamp_end, metadata_json, family,
|
|
427
|
+
bm25(l1_fts) AS rank
|
|
428
|
+
FROM l1_fts
|
|
429
|
+
WHERE l1_fts MATCH ?
|
|
430
|
+
ORDER BY rank ASC
|
|
431
|
+
LIMIT ?
|
|
432
|
+
`);
|
|
433
|
+
// 族过滤版(FTS5 UNINDEXED 列可作行级过滤条件)
|
|
434
|
+
this.stmtL1FtsSearchFamily = this.db.prepare(`
|
|
435
|
+
SELECT record_id, content_original AS content, type, priority, scene_name, version,
|
|
436
|
+
timestamp_str, timestamp_start, timestamp_end, metadata_json, family,
|
|
437
|
+
bm25(l1_fts) AS rank
|
|
438
|
+
FROM l1_fts
|
|
439
|
+
WHERE l1_fts MATCH ? AND family = ?
|
|
440
|
+
ORDER BY rank ASC
|
|
441
|
+
LIMIT ?
|
|
442
|
+
`);
|
|
443
|
+
if (ftsRebuilt)
|
|
444
|
+
this.backfillL1Fts();
|
|
445
|
+
this.stmtL0FtsInsert = this.db.prepare(`
|
|
446
|
+
INSERT INTO l0_fts (message_text, message_text_original, record_id, session_id, role, recorded_at, timestamp)
|
|
447
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
448
|
+
`);
|
|
449
|
+
this.stmtL0FtsDelete = this.db.prepare('DELETE FROM l0_fts WHERE record_id = ?');
|
|
450
|
+
this.stmtL0FtsSearch = this.db.prepare(`
|
|
451
|
+
SELECT record_id, message_text_original AS message_text, session_id, role, recorded_at, timestamp,
|
|
452
|
+
bm25(l0_fts) AS rank
|
|
453
|
+
FROM l0_fts
|
|
454
|
+
WHERE l0_fts MATCH ?
|
|
455
|
+
ORDER BY rank ASC
|
|
456
|
+
LIMIT ?
|
|
457
|
+
`);
|
|
458
|
+
if (l0FtsRebuilt)
|
|
459
|
+
this.backfillL0Fts();
|
|
460
|
+
// 戳如实记录"构建当前 FTS 内容的分词器"(含全新空表:后续写入即该分词器)
|
|
461
|
+
try {
|
|
462
|
+
this.writeMetaString('fts_tokenizer', wantStamp);
|
|
463
|
+
}
|
|
464
|
+
catch {
|
|
465
|
+
/* 戳写失败只影响下次启动多一次重建,不阻断 */
|
|
466
|
+
}
|
|
467
|
+
this.ftsAvailable = true;
|
|
468
|
+
}
|
|
469
|
+
catch (err) {
|
|
470
|
+
this.ftsAvailable = false;
|
|
471
|
+
this.logger?.warn(`${TAG} FTS5 不可用(可能未编译进 SQLite): ${err instanceof Error ? err.message : String(err)}`);
|
|
472
|
+
}
|
|
473
|
+
if (providerInfo && this.countL1() === 0 && this.countL0() === 0) {
|
|
474
|
+
// 空库 + 全新 meta:无历史向量,直接标记同步完成
|
|
475
|
+
this.writeEmbeddingMeta(providerInfo);
|
|
476
|
+
}
|
|
477
|
+
return { needsReindex, reason };
|
|
478
|
+
}
|
|
479
|
+
prepareL1VecStatements() {
|
|
480
|
+
if (this.vecLoaded && this.dimensions > 0) {
|
|
481
|
+
this.db.exec(`
|
|
482
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS l1_vec USING vec0(
|
|
483
|
+
record_id TEXT PRIMARY KEY,
|
|
484
|
+
embedding float[${this.dimensions}] distance_metric=cosine,
|
|
485
|
+
updated_time TEXT DEFAULT ''
|
|
486
|
+
)
|
|
487
|
+
`);
|
|
488
|
+
this.stmtDeleteL1Vec = this.db.prepare('DELETE FROM l1_vec WHERE record_id = ?');
|
|
489
|
+
this.stmtInsertL1Vec = this.db.prepare('INSERT INTO l1_vec (record_id, embedding, updated_time) VALUES (?, ?, ?)');
|
|
490
|
+
this.stmtSearchL1Vec = this.db.prepare(`
|
|
491
|
+
SELECT record_id, distance
|
|
492
|
+
FROM l1_vec
|
|
493
|
+
WHERE embedding MATCH ?
|
|
494
|
+
AND k = ?
|
|
495
|
+
ORDER BY distance
|
|
496
|
+
`);
|
|
497
|
+
}
|
|
498
|
+
else {
|
|
499
|
+
this.stmtDeleteL1Vec = undefined;
|
|
500
|
+
this.stmtInsertL1Vec = undefined;
|
|
501
|
+
this.stmtSearchL1Vec = undefined;
|
|
502
|
+
}
|
|
503
|
+
}
|
|
504
|
+
prepareL0VecStatements() {
|
|
505
|
+
if (this.vecLoaded && this.dimensions > 0) {
|
|
506
|
+
this.db.exec(`
|
|
507
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS l0_vec USING vec0(
|
|
508
|
+
record_id TEXT PRIMARY KEY,
|
|
509
|
+
embedding float[${this.dimensions}] distance_metric=cosine,
|
|
510
|
+
recorded_at TEXT DEFAULT ''
|
|
511
|
+
)
|
|
512
|
+
`);
|
|
513
|
+
this.stmtDeleteL0Vec = this.db.prepare('DELETE FROM l0_vec WHERE record_id = ?');
|
|
514
|
+
this.stmtInsertL0Vec = this.db.prepare('INSERT INTO l0_vec (record_id, embedding, recorded_at) VALUES (?, ?, ?)');
|
|
515
|
+
this.stmtSearchL0Vec = this.db.prepare(`
|
|
516
|
+
SELECT record_id, distance
|
|
517
|
+
FROM l0_vec
|
|
518
|
+
WHERE embedding MATCH ?
|
|
519
|
+
AND k = ?
|
|
520
|
+
ORDER BY distance
|
|
521
|
+
`);
|
|
522
|
+
}
|
|
523
|
+
else {
|
|
524
|
+
this.stmtDeleteL0Vec = undefined;
|
|
525
|
+
this.stmtInsertL0Vec = undefined;
|
|
526
|
+
this.stmtSearchL0Vec = undefined;
|
|
527
|
+
}
|
|
528
|
+
}
|
|
529
|
+
dropVectorTables() {
|
|
530
|
+
this.db.exec('DROP TABLE IF EXISTS l1_vec');
|
|
531
|
+
this.db.exec('DROP TABLE IF EXISTS l0_vec');
|
|
532
|
+
// 表已重建:作废按块缓存的 IN 语句(旧语句指向已删表,不能依赖引擎自动重编译兜底)
|
|
533
|
+
this.inStmts.clear();
|
|
534
|
+
// provider/model/维度已变:旧的"不可嵌入"判定作废(新 provider 可能嵌入得了),skip 集清空
|
|
535
|
+
this.clearVecSkipIds('l1');
|
|
536
|
+
this.clearVecSkipIds('l0');
|
|
537
|
+
this.prepareL1VecStatements();
|
|
538
|
+
this.prepareL0VecStatements();
|
|
539
|
+
}
|
|
540
|
+
tableExists(table) {
|
|
541
|
+
const row = this.db
|
|
542
|
+
.prepare("SELECT name FROM sqlite_master WHERE type IN ('table','view') AND name = ?")
|
|
543
|
+
.get(table);
|
|
544
|
+
return row !== undefined;
|
|
545
|
+
}
|
|
546
|
+
hasColumn(table, column) {
|
|
547
|
+
const rows = this.db.prepare(`PRAGMA table_info(${table})`).all();
|
|
548
|
+
return rows.some((r) => r.name === column);
|
|
549
|
+
}
|
|
550
|
+
/**
|
|
551
|
+
* 时间增强列:valid_from / valid_to / persistence。
|
|
552
|
+
*
|
|
553
|
+
* 与 family 列同款增量迁移——DDL 契约不改,只在缺列时补,幂等。
|
|
554
|
+
* 存储形态与 created_time/updated_time 一致(ISO-8601 UTC 的 TEXT),
|
|
555
|
+
* 于是区间比较既可按字典序,也能沿用 idx_l1_updated 的既有用法。
|
|
556
|
+
*
|
|
557
|
+
* 存量数据的 metadata.activity_start_time/activity_end_time 是这两列的前身,不回填:
|
|
558
|
+
* 图谱时间锚仍读 metadata,列由新写入路径填充。
|
|
559
|
+
*/
|
|
560
|
+
ensureTemporalColumns() {
|
|
561
|
+
for (const column of ['valid_from', 'valid_to', 'persistence']) {
|
|
562
|
+
if (this.hasColumn('l1_records', column))
|
|
563
|
+
continue;
|
|
564
|
+
this.db.exec(`ALTER TABLE l1_records ADD COLUMN ${column} TEXT DEFAULT ''`);
|
|
565
|
+
this.logger?.info(`${TAG} l1_records 补 ${column} 列(时间增强)`);
|
|
566
|
+
}
|
|
567
|
+
}
|
|
568
|
+
/** 重建后的 l1_fts 从 l1_records 全量回灌(仅在 drop 重建时调用;iterate 流式防大库内存峰值)。 */
|
|
569
|
+
backfillL1Fts() {
|
|
570
|
+
let count = 0;
|
|
571
|
+
const stmt = this.db
|
|
572
|
+
.prepare(`SELECT record_id, content, type, priority, scene_name, session_id, version,
|
|
573
|
+
timestamp_str, timestamp_start, timestamp_end, metadata_json, family FROM l1_records`);
|
|
574
|
+
for (const r of stmt.iterate()) {
|
|
575
|
+
try {
|
|
576
|
+
this.stmtL1FtsInsert.run(tokenizeForFts(String(r.content ?? '')), String(r.content ?? ''), String(r.record_id ?? ''), String(r.type ?? ''), Number(r.priority ?? 50), String(r.scene_name ?? ''), String(r.session_id ?? 'default'), Number(r.version ?? 0), String(r.timestamp_str ?? ''), String(r.timestamp_start ?? ''), String(r.timestamp_end ?? ''), String(r.metadata_json ?? '{}'), String(r.family ?? 'chat'));
|
|
577
|
+
count++;
|
|
578
|
+
}
|
|
579
|
+
catch {
|
|
580
|
+
/* 单行失败跳过(吞+计数:count 与总量比对可发现大面积失败) */
|
|
581
|
+
}
|
|
582
|
+
}
|
|
583
|
+
if (count > 0)
|
|
584
|
+
this.logger?.info(`${TAG} l1_fts 回灌 ${count} 行`);
|
|
585
|
+
}
|
|
586
|
+
/** 重建后的 l0_fts 从 l0_conversations 全量回灌(仅 drop 重建时调用;iterate 流式)。 */
|
|
587
|
+
backfillL0Fts() {
|
|
588
|
+
let count = 0;
|
|
589
|
+
const stmt = this.db.prepare('SELECT record_id, session_id, role, message_text, recorded_at, timestamp FROM l0_conversations');
|
|
590
|
+
for (const r of stmt.iterate()) {
|
|
591
|
+
try {
|
|
592
|
+
this.stmtL0FtsInsert.run(tokenizeForFts(String(r.message_text ?? '')), String(r.message_text ?? ''), String(r.record_id ?? ''), String(r.session_id ?? 'default'), String(r.role ?? ''), String(r.recorded_at ?? ''), Number(r.timestamp ?? 0));
|
|
593
|
+
count++;
|
|
594
|
+
}
|
|
595
|
+
catch {
|
|
596
|
+
/* 单行失败跳过 */
|
|
597
|
+
}
|
|
598
|
+
}
|
|
599
|
+
if (count > 0)
|
|
600
|
+
this.logger?.info(`${TAG} l0_fts 回灌 ${count} 行`);
|
|
601
|
+
}
|
|
602
|
+
readEmbeddingMeta() {
|
|
603
|
+
try {
|
|
604
|
+
const row = this.db
|
|
605
|
+
.prepare('SELECT value FROM embedding_meta WHERE key = ?')
|
|
606
|
+
.get('embedding_provider_info');
|
|
607
|
+
if (!row)
|
|
608
|
+
return undefined;
|
|
609
|
+
const parsed = JSON.parse(row.value);
|
|
610
|
+
if (typeof parsed.provider !== 'string')
|
|
611
|
+
return undefined;
|
|
612
|
+
return {
|
|
613
|
+
provider: parsed.provider,
|
|
614
|
+
model: parsed.model ?? '',
|
|
615
|
+
dimensions: parsed.dimensions ?? 0,
|
|
616
|
+
};
|
|
617
|
+
}
|
|
618
|
+
catch {
|
|
619
|
+
return undefined;
|
|
620
|
+
}
|
|
621
|
+
}
|
|
622
|
+
writeEmbeddingMeta(info) {
|
|
623
|
+
this.db
|
|
624
|
+
.prepare('INSERT INTO embedding_meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value=excluded.value')
|
|
625
|
+
.run('embedding_provider_info', JSON.stringify(info));
|
|
626
|
+
}
|
|
627
|
+
/** 通用字符串 kv(embedding_meta 表兼作元数据 kv 存储,如 FTS 分词器版本戳)。 */
|
|
628
|
+
readMetaString(key) {
|
|
629
|
+
try {
|
|
630
|
+
const row = this.db
|
|
631
|
+
.prepare('SELECT value FROM embedding_meta WHERE key = ?')
|
|
632
|
+
.get(key);
|
|
633
|
+
return row?.value ?? null;
|
|
634
|
+
}
|
|
635
|
+
catch {
|
|
636
|
+
return null;
|
|
637
|
+
}
|
|
638
|
+
}
|
|
639
|
+
writeMetaString(key, value) {
|
|
640
|
+
this.db
|
|
641
|
+
.prepare('INSERT INTO embedding_meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value=excluded.value')
|
|
642
|
+
.run(key, value);
|
|
643
|
+
}
|
|
644
|
+
/**
|
|
645
|
+
* 持久化 embedding meta(语义:物理向量表当前对应的 provider/维度)。
|
|
646
|
+
* 活切换在 swapProvider 成功后即写(表已是新维度);启动/补齐链在
|
|
647
|
+
* 缺失向量补齐收敛(missing=0)后写——缺失行补齐判据是行数差,不依赖 meta。
|
|
648
|
+
*/
|
|
649
|
+
markEmbeddingSynced(info) {
|
|
650
|
+
if (this.degraded)
|
|
651
|
+
return;
|
|
652
|
+
try {
|
|
653
|
+
this.writeEmbeddingMeta(info);
|
|
654
|
+
}
|
|
655
|
+
catch (err) {
|
|
656
|
+
this.logger?.warn(`${TAG} embedding meta 写入失败: ${err instanceof Error ? err.message : String(err)}`);
|
|
657
|
+
}
|
|
658
|
+
}
|
|
659
|
+
// ============================
|
|
660
|
+
// L1 写入 / 删除 / 读取
|
|
661
|
+
// ============================
|
|
662
|
+
/** upsert 一条 L1(元数据 + FTS 同步;embedding 非零时写向量)。失败返回 false 不抛。 */
|
|
663
|
+
upsertL1(record, embedding) {
|
|
664
|
+
if (this.degraded)
|
|
665
|
+
return false;
|
|
666
|
+
try {
|
|
667
|
+
this.withTransaction(() => this.upsertL1InTx(record, embedding));
|
|
668
|
+
return true;
|
|
669
|
+
}
|
|
670
|
+
catch (err) {
|
|
671
|
+
this.logger?.warn(`${TAG} L1 upsert 失败(非致命)id=${record.id}: ${err instanceof Error ? err.message : String(err)}`);
|
|
672
|
+
return false;
|
|
673
|
+
}
|
|
674
|
+
}
|
|
675
|
+
/**
|
|
676
|
+
* 批量 upsert L1(单事务;与单条同语义:FTS 失败整批回滚)。
|
|
677
|
+
* 追加/导入热路径用它——逐条开事务在 WAL FULL 下每条一次 fsync。
|
|
678
|
+
* 整批失败时回退逐条写入:好记录照常入库、坏记录只丢自身——否则
|
|
679
|
+
* JSONL 事实源已先行追加,检索库却整批缺失且无自动重导路径(批次空洞)。
|
|
680
|
+
*/
|
|
681
|
+
upsertL1Batch(records, embeddings) {
|
|
682
|
+
if (this.degraded || records.length === 0)
|
|
683
|
+
return false;
|
|
684
|
+
try {
|
|
685
|
+
this.withTransaction(() => {
|
|
686
|
+
for (let i = 0; i < records.length; i++) {
|
|
687
|
+
this.upsertL1InTx(records[i], embeddings?.[i]);
|
|
688
|
+
}
|
|
689
|
+
});
|
|
690
|
+
return true;
|
|
691
|
+
}
|
|
692
|
+
catch (err) {
|
|
693
|
+
this.logger?.warn(`${TAG} L1 批量写入失败,回退逐条写入: ${err instanceof Error ? err.message : String(err)}`);
|
|
694
|
+
const failed = [];
|
|
695
|
+
for (let i = 0; i < records.length; i++) {
|
|
696
|
+
if (!this.upsertL1(records[i], embeddings?.[i]))
|
|
697
|
+
failed.push(records[i]?.id ?? `#${i}`);
|
|
698
|
+
}
|
|
699
|
+
if (failed.length > 0) {
|
|
700
|
+
this.logger?.warn(`${TAG} 逐条回退后仍失败 ${failed.length}/${records.length} 条: ${failed.slice(0, 5).join(', ')}${failed.length > 5 ? '…' : ''}`);
|
|
701
|
+
}
|
|
702
|
+
return failed.length === 0;
|
|
703
|
+
}
|
|
704
|
+
}
|
|
705
|
+
/** 事务内的单条写入体(upsertL1 / upsertL1Batch 共用;调用方负责事务)。 */
|
|
706
|
+
upsertL1InTx(record, embedding) {
|
|
707
|
+
const ts = timestampsToDb(record.timestamps);
|
|
708
|
+
// 绑定层字段兜底(取 schema 列默认):旧版 JSONL 等外部数据缺字段时 undefined
|
|
709
|
+
// 无法绑定(node:sqlite 拒绝绑定),曾致旧版导入逐条全挂、每次启动无限重试。
|
|
710
|
+
// 主表与 FTS 两条语句共用同源归一化值;type 归一化后 familyForType 也不再收到 undefined。
|
|
711
|
+
const type = record.type ?? '';
|
|
712
|
+
const priority = record.priority ?? 50;
|
|
713
|
+
const sceneName = record.scene_name ?? '';
|
|
714
|
+
const family = record.family ?? familyForType(type);
|
|
715
|
+
// 防御性 FTS 删除的前置点查(主键索引,微秒级):record_id 在 FTS 表是 UNINDEXED,
|
|
716
|
+
// 按 id DELETE 是 O(N) 全表扫描——导入/重建/重嵌等"全新增"路径曾为每条记录白付一次
|
|
717
|
+
// 全扫(批量写整体 O(N²))。只有主表已有该行(覆盖/合并)才可能有旧 FTS 行需要删。
|
|
718
|
+
// 同批重复 id 也能正确处理:首条插入后,第二条的点查在同一事务内已见新行。
|
|
719
|
+
const ftsExisted = this.ftsAvailable ? this.stmtL1Exists.get(record.id) !== undefined : false;
|
|
720
|
+
this.stmtUpsertL1.run(record.id, record.content, type, priority, sceneName, record.sessionId ?? 'default', record.version ?? 0, ts.str, ts.start, ts.end, toIso(record.createdAt), toIso(record.updatedAt), JSON.stringify(record.metadata ?? {}), family, toIso(record.validFrom), toIso(record.validTo), normPersistence(record.persistence) ?? '');
|
|
721
|
+
// vec0 不支持 ON CONFLICT → 先删后插;零向量跳过(cosine 未定义)
|
|
722
|
+
if (this.stmtDeleteL1Vec && this.stmtInsertL1Vec) {
|
|
723
|
+
this.stmtDeleteL1Vec.run(record.id);
|
|
724
|
+
if (embedding && !isZeroVector(embedding)) {
|
|
725
|
+
this.stmtInsertL1Vec.run(record.id, vecToBuffer(embedding), toIso(record.updatedAt));
|
|
726
|
+
}
|
|
727
|
+
}
|
|
728
|
+
// FTS 删除/插入与元数据同事务:失败必须整体回滚——若只吞 FTS 错误照常 COMMIT,
|
|
729
|
+
// 已执行的 DELETE 会让该 id 的索引行被删未补,记录从此全文检索不可见(静默丢数据)。
|
|
730
|
+
if (this.ftsAvailable) {
|
|
731
|
+
if (ftsExisted)
|
|
732
|
+
this.stmtL1FtsDelete.run(record.id);
|
|
733
|
+
this.stmtL1FtsInsert.run(tokenizeForFts(record.content), record.content, record.id, type, priority, sceneName, record.sessionId ?? 'default', record.version ?? 0, ts.str, ts.start, ts.end, JSON.stringify(record.metadata ?? {}), family);
|
|
734
|
+
}
|
|
735
|
+
}
|
|
736
|
+
/** 批量删除 L1(元数据 + 向量 + FTS),返回删除条数。IN 按 ≤900 分块(避变量数上限)。
|
|
737
|
+
* 删除成功后触发图谱删除传播(来源全失效的节点/边惰性标 archived;失败不影响删除结果)。 */
|
|
738
|
+
deleteL1Batch(ids) {
|
|
739
|
+
if (this.degraded || ids.length === 0)
|
|
740
|
+
return 0;
|
|
741
|
+
try {
|
|
742
|
+
this.withTransaction(() => {
|
|
743
|
+
for (const chunk of chunkIds(ids))
|
|
744
|
+
this.inStatement('l1_records', 'delete', chunk.length).run(...chunk);
|
|
745
|
+
if (this.stmtDeleteL1Vec) {
|
|
746
|
+
for (const chunk of chunkIds(ids))
|
|
747
|
+
this.inStatement('l1_vec', 'delete', chunk.length).run(...chunk);
|
|
748
|
+
}
|
|
749
|
+
if (this.ftsAvailable) {
|
|
750
|
+
for (const chunk of chunkIds(ids))
|
|
751
|
+
this.inStatement('l1_fts', 'delete', chunk.length).run(...chunk);
|
|
752
|
+
}
|
|
753
|
+
});
|
|
754
|
+
this.graphStore.markSourcesDeleted(ids);
|
|
755
|
+
return ids.length;
|
|
756
|
+
}
|
|
757
|
+
catch (err) {
|
|
758
|
+
this.logger?.warn(`${TAG} L1 批量删除失败: ${err instanceof Error ? err.message : String(err)}`);
|
|
759
|
+
return 0;
|
|
760
|
+
}
|
|
761
|
+
}
|
|
762
|
+
inStatement(table, action, size) {
|
|
763
|
+
const key = `${table}:${action}:${size}`;
|
|
764
|
+
let stmt = this.inStmts.get(key);
|
|
765
|
+
if (!stmt) {
|
|
766
|
+
const ph = Array.from({ length: size }, () => '?').join(',');
|
|
767
|
+
// 时间增强列只存在于主表;l1_fts / l1_vec 没有这三列,共用列清单会直接报 no such column。
|
|
768
|
+
// 主表自身也按实际形状探测:未迁移的旧库不应因缺列而让按 id 取记录整条路径失败。
|
|
769
|
+
const temporal = table === 'l1_records' && this.hasColumn('l1_records', 'valid_from')
|
|
770
|
+
? ', valid_from, valid_to, persistence'
|
|
771
|
+
: '';
|
|
772
|
+
const metaCols = 'record_id, content, type, priority, scene_name, version, timestamp_str, created_time, updated_time, metadata_json, family' +
|
|
773
|
+
temporal;
|
|
774
|
+
stmt =
|
|
775
|
+
action === 'delete'
|
|
776
|
+
? this.db.prepare(`DELETE FROM ${table} WHERE record_id IN (${ph})`)
|
|
777
|
+
: this.db.prepare(`SELECT ${metaCols} FROM ${table} WHERE record_id IN (${ph})`);
|
|
778
|
+
this.inStmts.set(key, stmt);
|
|
779
|
+
}
|
|
780
|
+
return stmt;
|
|
781
|
+
}
|
|
782
|
+
/**
|
|
783
|
+
* 清空 L1 检索库全部数据(重建用)。records/FTS 直接 DELETE;
|
|
784
|
+
* 向量表走 DROP + 重建(vec0 的全表 DELETE 语义不可靠,dropVectorTables
|
|
785
|
+
* 会连 l0_vec 一起删——L0 向量必须保留——故此处单独处理 l1_vec)。
|
|
786
|
+
* L0 表与 embedding_meta 不动:backfill 的行数比对天然重新一致。
|
|
787
|
+
* 图谱表族一并清空——图谱是 L1 的可重建投影,记录清空即投影作废(B2)。
|
|
788
|
+
*/
|
|
789
|
+
clearL1() {
|
|
790
|
+
if (this.degraded)
|
|
791
|
+
return false;
|
|
792
|
+
try {
|
|
793
|
+
this.withTransaction(() => {
|
|
794
|
+
this.db.exec('DELETE FROM l1_records');
|
|
795
|
+
if (this.ftsAvailable)
|
|
796
|
+
this.db.exec('DELETE FROM l1_fts');
|
|
797
|
+
});
|
|
798
|
+
this.graphStore.resetAll();
|
|
799
|
+
// vec0 全表 DELETE 语义不可靠 → DROP + 重建空表;放事务外(vtab DDL 事务性弱)。
|
|
800
|
+
// 中间态:向量行短暂孤儿——检索侧对缺 meta 的向量本就跳过(searchL1Vector 回查过滤)。
|
|
801
|
+
if (this.stmtDeleteL1Vec) {
|
|
802
|
+
this.db.exec('DROP TABLE IF EXISTS l1_vec');
|
|
803
|
+
this.prepareL1VecStatements();
|
|
804
|
+
this.inStmts.clear();
|
|
805
|
+
}
|
|
806
|
+
// 重建后 L1 id 全新,旧 skip 集是死数据,清空让新记录获得一次嵌入机会
|
|
807
|
+
this.clearVecSkipIds('l1');
|
|
808
|
+
this.logger?.info(`${TAG} L1 检索库已清空(重建)`);
|
|
809
|
+
return true;
|
|
810
|
+
}
|
|
811
|
+
catch (err) {
|
|
812
|
+
this.logger?.warn(`${TAG} L1 清空失败: ${err instanceof Error ? err.message : String(err)}`);
|
|
813
|
+
return false;
|
|
814
|
+
}
|
|
815
|
+
}
|
|
816
|
+
countL1() {
|
|
817
|
+
if (this.degraded)
|
|
818
|
+
return 0;
|
|
819
|
+
try {
|
|
820
|
+
const row = this.db.prepare('SELECT COUNT(*) AS n FROM l1_records').get();
|
|
821
|
+
return row?.n ?? 0;
|
|
822
|
+
}
|
|
823
|
+
catch {
|
|
824
|
+
// 建表前(init 的 embedding 变更检测)会先调用计数
|
|
825
|
+
return 0;
|
|
826
|
+
}
|
|
827
|
+
}
|
|
828
|
+
/** 全量读取(调试/迁移/重嵌入用;检索请走 FTS/向量)。 */
|
|
829
|
+
getAllL1() {
|
|
830
|
+
if (this.degraded)
|
|
831
|
+
return [];
|
|
832
|
+
const rows = this.db
|
|
833
|
+
.prepare('SELECT record_id, content, type, priority, scene_name, version, timestamp_str, created_time, updated_time, metadata_json, family, valid_from, valid_to, persistence FROM l1_records')
|
|
834
|
+
.all();
|
|
835
|
+
return rows.map(rowToRecord);
|
|
836
|
+
}
|
|
837
|
+
getL1ByIds(ids) {
|
|
838
|
+
if (this.degraded || ids.length === 0)
|
|
839
|
+
return [];
|
|
840
|
+
const rows = [];
|
|
841
|
+
for (const chunk of chunkIds(ids)) {
|
|
842
|
+
rows.push(...this.inStatement('l1_records', 'select', chunk.length).all(...chunk));
|
|
843
|
+
}
|
|
844
|
+
return rows.map(rowToRecord);
|
|
845
|
+
}
|
|
846
|
+
/** 浏览列表(UI 用):按更新时间倒序,支持类型/场景/族/Hall 过滤与分页。失败返回空。 */
|
|
847
|
+
listL1(opts) {
|
|
848
|
+
if (this.degraded)
|
|
849
|
+
return { items: [], total: 0 };
|
|
850
|
+
try {
|
|
851
|
+
const where = [];
|
|
852
|
+
const params = [];
|
|
853
|
+
if (opts.type) {
|
|
854
|
+
where.push('type = ?');
|
|
855
|
+
params.push(opts.type);
|
|
856
|
+
}
|
|
857
|
+
if (opts.scene) {
|
|
858
|
+
where.push('scene_name = ?');
|
|
859
|
+
params.push(opts.scene);
|
|
860
|
+
}
|
|
861
|
+
if (opts.family) {
|
|
862
|
+
where.push('family = ?');
|
|
863
|
+
params.push(opts.family);
|
|
864
|
+
}
|
|
865
|
+
if (opts.hall) {
|
|
866
|
+
// Hall 存于 metadata_json,用 json_extract 过滤(表小,逐行代价可接受)
|
|
867
|
+
where.push(`json_extract(metadata_json, '$.hall') = ?`);
|
|
868
|
+
params.push(opts.hall);
|
|
869
|
+
}
|
|
870
|
+
const whereSql = where.length > 0 ? ` WHERE ${where.join(' AND ')}` : '';
|
|
871
|
+
const totalRow = this.db.prepare(`SELECT COUNT(*) AS n FROM l1_records${whereSql}`).get(...params);
|
|
872
|
+
const rows = this.db
|
|
873
|
+
.prepare(`SELECT record_id, content, type, priority, scene_name, version, timestamp_str, created_time, updated_time, metadata_json, family, valid_from, valid_to, persistence FROM l1_records${whereSql} ORDER BY updated_time DESC LIMIT ? OFFSET ?`)
|
|
874
|
+
.all(...params, opts.limit, opts.offset);
|
|
875
|
+
return { items: rows.map(rowToRecord), total: totalRow?.n ?? 0 };
|
|
876
|
+
}
|
|
877
|
+
catch (err) {
|
|
878
|
+
this.logger?.warn(`${TAG} L1 浏览列表查询失败(返回空): ${err instanceof Error ? err.message : String(err)}`);
|
|
879
|
+
return { items: [], total: 0 };
|
|
880
|
+
}
|
|
881
|
+
}
|
|
882
|
+
/** 场景名去重列表(UI 筛选器数据源)。失败返回空。 */
|
|
883
|
+
distinctL1Scenes() {
|
|
884
|
+
if (this.degraded)
|
|
885
|
+
return [];
|
|
886
|
+
try {
|
|
887
|
+
const rows = this.db
|
|
888
|
+
.prepare('SELECT DISTINCT scene_name FROM l1_records ORDER BY scene_name')
|
|
889
|
+
.all();
|
|
890
|
+
return rows.map((r) => r.scene_name).filter(Boolean);
|
|
891
|
+
}
|
|
892
|
+
catch {
|
|
893
|
+
return [];
|
|
894
|
+
}
|
|
895
|
+
}
|
|
896
|
+
// ============================
|
|
897
|
+
// L1 检索
|
|
898
|
+
// ============================
|
|
899
|
+
/** FTS5 BM25 检索(family 缺省不过滤)。失败返回空数组(调用方降级)。 */
|
|
900
|
+
searchL1Fts(query, limit, family) {
|
|
901
|
+
if (this.degraded || !this.ftsAvailable || limit <= 0)
|
|
902
|
+
return [];
|
|
903
|
+
const ftsQuery = buildFtsQuery(query);
|
|
904
|
+
if (!ftsQuery)
|
|
905
|
+
return [];
|
|
906
|
+
try {
|
|
907
|
+
const rows = (family
|
|
908
|
+
? this.stmtL1FtsSearchFamily.all(ftsQuery, family, limit)
|
|
909
|
+
: this.stmtL1FtsSearch.all(ftsQuery, limit));
|
|
910
|
+
return rows.map((r) => ({
|
|
911
|
+
id: r.record_id,
|
|
912
|
+
content: r.content,
|
|
913
|
+
type: r.type,
|
|
914
|
+
priority: r.priority,
|
|
915
|
+
scene_name: r.scene_name,
|
|
916
|
+
family: normFamily(r.family, r.type),
|
|
917
|
+
score: bm25RankToScore(r.rank),
|
|
918
|
+
}));
|
|
919
|
+
}
|
|
920
|
+
catch (err) {
|
|
921
|
+
this.logger?.warn(`${TAG} L1 FTS 检索失败(返回空): ${err instanceof Error ? err.message : String(err)}`);
|
|
922
|
+
return [];
|
|
923
|
+
}
|
|
924
|
+
}
|
|
925
|
+
/** vec0 余弦 KNN 检索(score = 1 - cosine distance;family 过滤走过度召回 + 回查过滤,vec0 无法 WHERE)。失败返回空数组。 */
|
|
926
|
+
searchL1Vector(embedding, topK, family) {
|
|
927
|
+
if (this.degraded || !this.stmtSearchL1Vec || topK <= 0)
|
|
928
|
+
return [];
|
|
929
|
+
try {
|
|
930
|
+
// 过度召回补偿遗留零向量;带族过滤时再放大(不命中本族的行会被丢弃)
|
|
931
|
+
const retrieveCount = (topK + ZERO_VEC_BUFFER) * (family ? 3 : 1);
|
|
932
|
+
const rows = this.stmtSearchL1Vec.all(vecToBuffer(embedding), retrieveCount);
|
|
933
|
+
const hits = [];
|
|
934
|
+
for (const { record_id, distance } of rows) {
|
|
935
|
+
if (distance == null || Number.isNaN(distance))
|
|
936
|
+
continue;
|
|
937
|
+
const meta = this.stmtGetL1.get(record_id);
|
|
938
|
+
if (!meta)
|
|
939
|
+
continue;
|
|
940
|
+
if (family && normFamily(meta.family, meta.type) !== family)
|
|
941
|
+
continue;
|
|
942
|
+
hits.push({
|
|
943
|
+
id: record_id,
|
|
944
|
+
content: meta.content,
|
|
945
|
+
type: meta.type,
|
|
946
|
+
priority: meta.priority,
|
|
947
|
+
scene_name: meta.scene_name,
|
|
948
|
+
family: normFamily(meta.family, meta.type),
|
|
949
|
+
score: 1.0 - distance,
|
|
950
|
+
});
|
|
951
|
+
}
|
|
952
|
+
return hits.slice(0, topK);
|
|
953
|
+
}
|
|
954
|
+
catch (err) {
|
|
955
|
+
this.logger?.warn(`${TAG} L1 向量检索失败(返回空): ${err instanceof Error ? err.message : String(err)}`);
|
|
956
|
+
return [];
|
|
957
|
+
}
|
|
958
|
+
}
|
|
959
|
+
// ============================
|
|
960
|
+
// L0 写入 / 删除 / 读取
|
|
961
|
+
// ============================
|
|
962
|
+
/** 批量 upsert L0 消息(元数据 + FTS;embeddings 与 records 等长,可省略)。 */
|
|
963
|
+
upsertL0Batch(records, embeddings) {
|
|
964
|
+
if (this.degraded || records.length === 0)
|
|
965
|
+
return false;
|
|
966
|
+
try {
|
|
967
|
+
this.withTransaction(() => {
|
|
968
|
+
for (let i = 0; i < records.length; i++) {
|
|
969
|
+
const rec = records[i];
|
|
970
|
+
// 绑定层字段兜底(取 schema 列默认):同 upsertL1InTx——外部数据缺字段时
|
|
971
|
+
// undefined 无法绑定;主表/向量/FTS 共用同源归一化值
|
|
972
|
+
const sessionId = rec.sessionId ?? 'default';
|
|
973
|
+
const role = rec.role ?? '';
|
|
974
|
+
const content = rec.content ?? '';
|
|
975
|
+
const recordedAt = rec.recordedAt ?? '';
|
|
976
|
+
const timestamp = rec.timestamp ?? 0;
|
|
977
|
+
// 同 upsertL1 的点查预判:全新增路径跳过 UNINDEXED 列的 FTS 全扫删除
|
|
978
|
+
const ftsExisted = this.ftsAvailable ? this.stmtL0Exists.get(rec.id) !== undefined : false;
|
|
979
|
+
this.stmtUpsertL0.run(rec.id, sessionId, role, content, recordedAt, timestamp);
|
|
980
|
+
if (this.stmtDeleteL0Vec && this.stmtInsertL0Vec) {
|
|
981
|
+
this.stmtDeleteL0Vec.run(rec.id);
|
|
982
|
+
const vec = embeddings?.[i];
|
|
983
|
+
if (vec && !isZeroVector(vec)) {
|
|
984
|
+
this.stmtInsertL0Vec.run(rec.id, vecToBuffer(vec), recordedAt);
|
|
985
|
+
}
|
|
986
|
+
}
|
|
987
|
+
if (this.ftsAvailable) {
|
|
988
|
+
// 同 upsertL1:FTS 失败冒泡触发整批回滚,禁止"删了没补"的索引空洞。
|
|
989
|
+
if (ftsExisted)
|
|
990
|
+
this.stmtL0FtsDelete.run(rec.id);
|
|
991
|
+
this.stmtL0FtsInsert.run(tokenizeForFts(content), content, rec.id, sessionId, role, recordedAt, timestamp);
|
|
992
|
+
}
|
|
993
|
+
}
|
|
994
|
+
});
|
|
995
|
+
return true;
|
|
996
|
+
}
|
|
997
|
+
catch (err) {
|
|
998
|
+
this.logger?.warn(`${TAG} L0 批量写入失败: ${err instanceof Error ? err.message : String(err)}`);
|
|
999
|
+
return false;
|
|
1000
|
+
}
|
|
1001
|
+
}
|
|
1002
|
+
/** 记录一次蒸馏调用成本(委托 cost-ledger;语义见 CostLedger.insertCostCall)。 */
|
|
1003
|
+
insertCostCall(provider, model, layer, inputChars, outputTokens, reasoningTokens, retentionDays) {
|
|
1004
|
+
this.costLedger.insertCostCall(provider, model, layer, inputChars, outputTokens, reasoningTokens, retentionDays);
|
|
1005
|
+
}
|
|
1006
|
+
/** 查询 token_cost 单窗口聚合(委托 cost-ledger;降级/异常返回零值)。 */
|
|
1007
|
+
aggregateCost(since) {
|
|
1008
|
+
return this.costLedger.aggregateCost(since);
|
|
1009
|
+
}
|
|
1010
|
+
/** 按层级归并聚合(委托 cost-ledger;降级/异常返回空数组)。 */
|
|
1011
|
+
aggregateCostByLayer(since) {
|
|
1012
|
+
return this.costLedger.aggregateCostByLayer(since);
|
|
1013
|
+
}
|
|
1014
|
+
/** 按时间桶 + model 聚合(委托 cost-ledger;趋势图与日均/周均/月均共用)。 */
|
|
1015
|
+
aggregateByBucket(bucketMs, offsetMs, since, layer) {
|
|
1016
|
+
return this.costLedger.aggregateByBucket(bucketMs, offsetMs, since, layer);
|
|
1017
|
+
}
|
|
1018
|
+
countL0() {
|
|
1019
|
+
if (this.degraded)
|
|
1020
|
+
return 0;
|
|
1021
|
+
try {
|
|
1022
|
+
const row = this.db.prepare('SELECT COUNT(*) AS n FROM l0_conversations').get();
|
|
1023
|
+
return row?.n ?? 0;
|
|
1024
|
+
}
|
|
1025
|
+
catch {
|
|
1026
|
+
return 0;
|
|
1027
|
+
}
|
|
1028
|
+
}
|
|
1029
|
+
/** 统计 recorded_at >= iso 的消息数(状态面板"今日捕获"用)。 */
|
|
1030
|
+
countL0Since(iso) {
|
|
1031
|
+
if (this.degraded)
|
|
1032
|
+
return 0;
|
|
1033
|
+
try {
|
|
1034
|
+
const row = this.db
|
|
1035
|
+
.prepare('SELECT COUNT(*) AS n FROM l0_conversations WHERE recorded_at >= ?')
|
|
1036
|
+
.get(iso);
|
|
1037
|
+
return row?.n ?? 0;
|
|
1038
|
+
}
|
|
1039
|
+
catch {
|
|
1040
|
+
return 0;
|
|
1041
|
+
}
|
|
1042
|
+
}
|
|
1043
|
+
/** 统计某会话已捕获消息数(session-stats 数据源;idx_l0_session_id 索引点查)。 */
|
|
1044
|
+
countL0BySession(sessionId) {
|
|
1045
|
+
if (this.degraded)
|
|
1046
|
+
return 0;
|
|
1047
|
+
try {
|
|
1048
|
+
const row = this.db
|
|
1049
|
+
.prepare('SELECT COUNT(*) AS n FROM l0_conversations WHERE session_id = ?')
|
|
1050
|
+
.get(sessionId);
|
|
1051
|
+
return row?.n ?? 0;
|
|
1052
|
+
}
|
|
1053
|
+
catch {
|
|
1054
|
+
return 0;
|
|
1055
|
+
}
|
|
1056
|
+
}
|
|
1057
|
+
/** 按会话取最近消息(时间升序返回;走 idx_l0_session_id 索引)。
|
|
1058
|
+
* 蒸馏背景参考专用——按会话现查替代全局内存数组(ADR-0003)。 */
|
|
1059
|
+
recentL0BySession(sessionId, limit) {
|
|
1060
|
+
if (this.degraded || limit <= 0)
|
|
1061
|
+
return [];
|
|
1062
|
+
try {
|
|
1063
|
+
const rows = this.db
|
|
1064
|
+
.prepare('SELECT record_id, session_id, role, message_text, recorded_at, timestamp FROM l0_conversations WHERE session_id = ? ORDER BY timestamp DESC, rowid DESC LIMIT ?')
|
|
1065
|
+
.all(sessionId, limit);
|
|
1066
|
+
return rows
|
|
1067
|
+
.map((r) => ({
|
|
1068
|
+
sessionId: r.session_id,
|
|
1069
|
+
recordedAt: r.recorded_at,
|
|
1070
|
+
id: r.record_id,
|
|
1071
|
+
role: r.role,
|
|
1072
|
+
content: r.message_text,
|
|
1073
|
+
timestamp: r.timestamp ?? 0,
|
|
1074
|
+
}))
|
|
1075
|
+
.reverse();
|
|
1076
|
+
}
|
|
1077
|
+
catch (err) {
|
|
1078
|
+
this.logger?.warn(`[memory] L0 按会话取最近消息失败(返回空): ${err instanceof Error ? err.message : String(err)}`);
|
|
1079
|
+
return [];
|
|
1080
|
+
}
|
|
1081
|
+
}
|
|
1082
|
+
/** L0 全量列举(重建快照用;按时间升序,事务一致性避开 JSONL 追加竞态)。 */
|
|
1083
|
+
listL0All() {
|
|
1084
|
+
if (this.degraded)
|
|
1085
|
+
return [];
|
|
1086
|
+
try {
|
|
1087
|
+
const rows = this.db
|
|
1088
|
+
.prepare('SELECT record_id, session_id, role, message_text, recorded_at, timestamp FROM l0_conversations ORDER BY timestamp ASC')
|
|
1089
|
+
.all();
|
|
1090
|
+
return rows.map((r) => ({
|
|
1091
|
+
sessionId: r.session_id,
|
|
1092
|
+
recordedAt: r.recorded_at,
|
|
1093
|
+
id: r.record_id,
|
|
1094
|
+
role: r.role,
|
|
1095
|
+
content: r.message_text,
|
|
1096
|
+
timestamp: r.timestamp ?? 0,
|
|
1097
|
+
}));
|
|
1098
|
+
}
|
|
1099
|
+
catch (err) {
|
|
1100
|
+
this.logger?.warn(`${TAG} L0 全量列举失败(返回空): ${err instanceof Error ? err.message : String(err)}`);
|
|
1101
|
+
return [];
|
|
1102
|
+
}
|
|
1103
|
+
}
|
|
1104
|
+
/** 重建成本预估(一次全表聚合:会话数 / 消息数 / 字符量)。 */
|
|
1105
|
+
l0RebuildEstimate() {
|
|
1106
|
+
if (this.degraded)
|
|
1107
|
+
return { sessions: 0, messages: 0, chars: 0 };
|
|
1108
|
+
try {
|
|
1109
|
+
const row = this.db
|
|
1110
|
+
.prepare('SELECT COUNT(DISTINCT session_id) AS s, COUNT(*) AS n, COALESCE(SUM(LENGTH(message_text)), 0) AS c FROM l0_conversations')
|
|
1111
|
+
.get();
|
|
1112
|
+
return { sessions: row?.s ?? 0, messages: row?.n ?? 0, chars: row?.c ?? 0 };
|
|
1113
|
+
}
|
|
1114
|
+
catch {
|
|
1115
|
+
return { sessions: 0, messages: 0, chars: 0 };
|
|
1116
|
+
}
|
|
1117
|
+
}
|
|
1118
|
+
/** 向量表行数(backfill 判据:与元数据行数的差值即缺失向量数;不可用时返回 -1)。 */
|
|
1119
|
+
countL1Vec() {
|
|
1120
|
+
if (this.degraded || !this.stmtSearchL1Vec)
|
|
1121
|
+
return -1;
|
|
1122
|
+
try {
|
|
1123
|
+
const row = this.db.prepare('SELECT COUNT(*) AS n FROM l1_vec').get();
|
|
1124
|
+
return row?.n ?? -1;
|
|
1125
|
+
}
|
|
1126
|
+
catch {
|
|
1127
|
+
return -1;
|
|
1128
|
+
}
|
|
1129
|
+
}
|
|
1130
|
+
countL0Vec() {
|
|
1131
|
+
if (this.degraded || !this.stmtSearchL0Vec)
|
|
1132
|
+
return -1;
|
|
1133
|
+
try {
|
|
1134
|
+
const row = this.db.prepare('SELECT COUNT(*) AS n FROM l0_vec').get();
|
|
1135
|
+
return row?.n ?? -1;
|
|
1136
|
+
}
|
|
1137
|
+
catch {
|
|
1138
|
+
return -1;
|
|
1139
|
+
}
|
|
1140
|
+
}
|
|
1141
|
+
// ============================
|
|
1142
|
+
// L0 检索
|
|
1143
|
+
// ============================
|
|
1144
|
+
searchL0Fts(query, limit) {
|
|
1145
|
+
if (this.degraded || !this.ftsAvailable || limit <= 0)
|
|
1146
|
+
return [];
|
|
1147
|
+
const ftsQuery = buildFtsQuery(query);
|
|
1148
|
+
if (!ftsQuery)
|
|
1149
|
+
return [];
|
|
1150
|
+
try {
|
|
1151
|
+
const rows = this.stmtL0FtsSearch.all(ftsQuery, limit);
|
|
1152
|
+
return rows.map((r) => ({
|
|
1153
|
+
sessionId: r.session_id,
|
|
1154
|
+
recordedAt: r.recorded_at,
|
|
1155
|
+
id: r.record_id,
|
|
1156
|
+
role: r.role,
|
|
1157
|
+
content: r.message_text,
|
|
1158
|
+
timestamp: r.timestamp ?? 0,
|
|
1159
|
+
score: bm25RankToScore(r.rank),
|
|
1160
|
+
}));
|
|
1161
|
+
}
|
|
1162
|
+
catch (err) {
|
|
1163
|
+
this.logger?.warn(`${TAG} L0 FTS 检索失败(返回空): ${err instanceof Error ? err.message : String(err)}`);
|
|
1164
|
+
return [];
|
|
1165
|
+
}
|
|
1166
|
+
}
|
|
1167
|
+
searchL0Vector(embedding, topK) {
|
|
1168
|
+
if (this.degraded || !this.stmtSearchL0Vec)
|
|
1169
|
+
return [];
|
|
1170
|
+
try {
|
|
1171
|
+
const retrieveCount = topK + ZERO_VEC_BUFFER;
|
|
1172
|
+
const rows = this.stmtSearchL0Vec.all(vecToBuffer(embedding), retrieveCount);
|
|
1173
|
+
const hits = [];
|
|
1174
|
+
for (const { record_id, distance } of rows) {
|
|
1175
|
+
if (distance == null || Number.isNaN(distance))
|
|
1176
|
+
continue;
|
|
1177
|
+
const row = this.stmtGetL0.get(record_id);
|
|
1178
|
+
if (!row)
|
|
1179
|
+
continue;
|
|
1180
|
+
hits.push({
|
|
1181
|
+
sessionId: row.session_id,
|
|
1182
|
+
recordedAt: row.recorded_at,
|
|
1183
|
+
id: record_id,
|
|
1184
|
+
role: row.role,
|
|
1185
|
+
content: row.message_text,
|
|
1186
|
+
timestamp: row.timestamp ?? 0,
|
|
1187
|
+
score: 1.0 - distance,
|
|
1188
|
+
});
|
|
1189
|
+
}
|
|
1190
|
+
return hits.slice(0, topK);
|
|
1191
|
+
}
|
|
1192
|
+
catch (err) {
|
|
1193
|
+
this.logger?.warn(`${TAG} L0 向量检索失败(返回空): ${err instanceof Error ? err.message : String(err)}`);
|
|
1194
|
+
return [];
|
|
1195
|
+
}
|
|
1196
|
+
}
|
|
1197
|
+
// ============================
|
|
1198
|
+
// 重嵌入(reindexAll 用)
|
|
1199
|
+
// ============================
|
|
1200
|
+
/** L1 缺失向量的记录数(排除 skip 集后的补齐判据;向量能力不可用返回 -1)。 */
|
|
1201
|
+
countL1VecMissing(exclude) {
|
|
1202
|
+
return this.countVecMissing('l1', exclude);
|
|
1203
|
+
}
|
|
1204
|
+
/** L0 缺失向量的记录数(同上)。 */
|
|
1205
|
+
countL0VecMissing(exclude) {
|
|
1206
|
+
return this.countVecMissing('l0', exclude);
|
|
1207
|
+
}
|
|
1208
|
+
countVecMissing(kind, exclude) {
|
|
1209
|
+
if (this.degraded)
|
|
1210
|
+
return -1;
|
|
1211
|
+
const vecTable = kind === 'l1' ? 'l1_vec' : 'l0_vec';
|
|
1212
|
+
if (kind === 'l1' ? !this.stmtSearchL1Vec : !this.stmtSearchL0Vec)
|
|
1213
|
+
return -1;
|
|
1214
|
+
const srcTable = kind === 'l1' ? 'l1_records' : 'l0_conversations';
|
|
1215
|
+
try {
|
|
1216
|
+
const row = this.db
|
|
1217
|
+
.prepare(`SELECT COUNT(*) AS n FROM ${srcTable} r
|
|
1218
|
+
LEFT JOIN ${vecTable} v ON v.record_id = r.record_id
|
|
1219
|
+
WHERE v.record_id IS NULL${notInClause('r.record_id', exclude)}`)
|
|
1220
|
+
.all(...notInParams(exclude))[0];
|
|
1221
|
+
return row?.n ?? 0;
|
|
1222
|
+
}
|
|
1223
|
+
catch (err) {
|
|
1224
|
+
// -1 会让补齐判据(> 0)按"无缺失"处理——必须留痕,不能静默停摆
|
|
1225
|
+
this.logger?.warn(`${TAG} ${kind} 缺失向量计数失败(补齐判据按无缺失处理): ${err instanceof Error ? err.message : String(err)}`);
|
|
1226
|
+
return -1;
|
|
1227
|
+
}
|
|
1228
|
+
}
|
|
1229
|
+
/**
|
|
1230
|
+
* 待重嵌入的 L1:只取缺失向量的记录(增量),排除 skip 集里已判定
|
|
1231
|
+
* "当前 provider 下不可嵌入(零向量)"的 id——缺 1 条不再全量重嵌,
|
|
1232
|
+
* 零向量记录也不再反复喂给 embeddings API(死循环双根因)。
|
|
1233
|
+
*/
|
|
1234
|
+
getL1ForReindex(exclude) {
|
|
1235
|
+
if (this.degraded)
|
|
1236
|
+
return [];
|
|
1237
|
+
try {
|
|
1238
|
+
return this.db
|
|
1239
|
+
.prepare(`SELECT r.record_id AS id, r.content FROM l1_records r
|
|
1240
|
+
LEFT JOIN l1_vec v ON v.record_id = r.record_id
|
|
1241
|
+
WHERE v.record_id IS NULL${notInClause('r.record_id', exclude)}`)
|
|
1242
|
+
.all(...notInParams(exclude));
|
|
1243
|
+
}
|
|
1244
|
+
catch (err) {
|
|
1245
|
+
this.logger?.warn(`${TAG} L1 重嵌入取数失败(返回空,本轮跳过): ${err instanceof Error ? err.message : String(err)}`);
|
|
1246
|
+
return [];
|
|
1247
|
+
}
|
|
1248
|
+
}
|
|
1249
|
+
/** 待重嵌入的 L0(增量 + 排除 skip 集,同 getL1ForReindex)。 */
|
|
1250
|
+
getL0ForReindex(exclude) {
|
|
1251
|
+
if (this.degraded)
|
|
1252
|
+
return [];
|
|
1253
|
+
try {
|
|
1254
|
+
return this.db
|
|
1255
|
+
.prepare(`SELECT r.record_id AS id, r.message_text AS text FROM l0_conversations r
|
|
1256
|
+
LEFT JOIN l0_vec v ON v.record_id = r.record_id
|
|
1257
|
+
WHERE v.record_id IS NULL${notInClause('r.record_id', exclude)}`)
|
|
1258
|
+
.all(...notInParams(exclude));
|
|
1259
|
+
}
|
|
1260
|
+
catch (err) {
|
|
1261
|
+
this.logger?.warn(`${TAG} L0 重嵌入取数失败(返回空,本轮跳过): ${err instanceof Error ? err.message : String(err)}`);
|
|
1262
|
+
return [];
|
|
1263
|
+
}
|
|
1264
|
+
}
|
|
1265
|
+
// ── 零向量 skip 集(embedding_meta 持久化;provider 变化时随向量表一起清空) ──
|
|
1266
|
+
getVecSkipSet(kind) {
|
|
1267
|
+
if (this.degraded)
|
|
1268
|
+
return new Set();
|
|
1269
|
+
try {
|
|
1270
|
+
const row = this.db
|
|
1271
|
+
.prepare('SELECT value FROM embedding_meta WHERE key = ?')
|
|
1272
|
+
.get(vecSkipKey(kind));
|
|
1273
|
+
if (!row)
|
|
1274
|
+
return new Set();
|
|
1275
|
+
const parsed = JSON.parse(row.value);
|
|
1276
|
+
if (!Array.isArray(parsed))
|
|
1277
|
+
return new Set();
|
|
1278
|
+
return new Set(parsed.filter((x) => typeof x === 'string'));
|
|
1279
|
+
}
|
|
1280
|
+
catch {
|
|
1281
|
+
return new Set();
|
|
1282
|
+
}
|
|
1283
|
+
}
|
|
1284
|
+
addVecSkippedIds(kind, ids) {
|
|
1285
|
+
if (this.degraded || ids.length === 0)
|
|
1286
|
+
return;
|
|
1287
|
+
try {
|
|
1288
|
+
let merged = [...new Set([...this.getVecSkipSet(kind), ...ids])];
|
|
1289
|
+
// 上限防 NOT IN 占位符无界膨胀(老构建变量上限 999);达到上限本身
|
|
1290
|
+
// 说明 embedding 服务大面积返回零向量,被挤出的旧 id 只是多一次重试
|
|
1291
|
+
if (merged.length > VEC_SKIP_CAP) {
|
|
1292
|
+
merged = merged.slice(-VEC_SKIP_CAP);
|
|
1293
|
+
this.logger?.warn(`${TAG} skip 集达上限 ${VEC_SKIP_CAP}(零向量记录过多,embedding 服务疑似异常)`);
|
|
1294
|
+
}
|
|
1295
|
+
this.db
|
|
1296
|
+
.prepare('INSERT INTO embedding_meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value=excluded.value')
|
|
1297
|
+
.run(vecSkipKey(kind), JSON.stringify(merged));
|
|
1298
|
+
}
|
|
1299
|
+
catch (err) {
|
|
1300
|
+
this.logger?.warn(`${TAG} skip 集写入失败: ${err instanceof Error ? err.message : String(err)}`);
|
|
1301
|
+
}
|
|
1302
|
+
}
|
|
1303
|
+
clearVecSkipIds(kind) {
|
|
1304
|
+
if (this.degraded)
|
|
1305
|
+
return;
|
|
1306
|
+
try {
|
|
1307
|
+
this.db.prepare('DELETE FROM embedding_meta WHERE key = ?').run(vecSkipKey(kind));
|
|
1308
|
+
}
|
|
1309
|
+
catch {
|
|
1310
|
+
/* 空集语义,失败无影响 */
|
|
1311
|
+
}
|
|
1312
|
+
}
|
|
1313
|
+
/** 只更新向量行(重嵌入用)。 */
|
|
1314
|
+
updateL1Vec(id, embedding) {
|
|
1315
|
+
if (this.degraded || !this.stmtDeleteL1Vec || !this.stmtInsertL1Vec)
|
|
1316
|
+
return false;
|
|
1317
|
+
if (isZeroVector(embedding))
|
|
1318
|
+
return false;
|
|
1319
|
+
try {
|
|
1320
|
+
this.stmtDeleteL1Vec.run(id);
|
|
1321
|
+
this.stmtInsertL1Vec.run(id, vecToBuffer(embedding), new Date().toISOString());
|
|
1322
|
+
return true;
|
|
1323
|
+
}
|
|
1324
|
+
catch {
|
|
1325
|
+
return false;
|
|
1326
|
+
}
|
|
1327
|
+
}
|
|
1328
|
+
updateL0Vec(id, embedding, recordedAt) {
|
|
1329
|
+
if (this.degraded || !this.stmtDeleteL0Vec || !this.stmtInsertL0Vec)
|
|
1330
|
+
return false;
|
|
1331
|
+
if (isZeroVector(embedding))
|
|
1332
|
+
return false;
|
|
1333
|
+
try {
|
|
1334
|
+
this.stmtDeleteL0Vec.run(id);
|
|
1335
|
+
this.stmtInsertL0Vec.run(id, vecToBuffer(embedding), recordedAt);
|
|
1336
|
+
return true;
|
|
1337
|
+
}
|
|
1338
|
+
catch {
|
|
1339
|
+
return false;
|
|
1340
|
+
}
|
|
1341
|
+
}
|
|
1342
|
+
/**
|
|
1343
|
+
* 批量更新 L1 向量行(重嵌入热路径):单事务写入整批——逐条每行一次隐式事务,
|
|
1344
|
+
* 批量场景(万级记录重嵌)开销集中在 fsync 上。
|
|
1345
|
+
* 整批失败回退逐条:好行照常入库,坏行只丢自身(向量行 id 寻址,无顺序依赖)。
|
|
1346
|
+
* 返回成功写入的行数(零向量行防御性跳过、不计入)。
|
|
1347
|
+
*/
|
|
1348
|
+
updateL1VecBatch(items) {
|
|
1349
|
+
if (this.degraded || !this.stmtDeleteL1Vec || !this.stmtInsertL1Vec || items.length === 0)
|
|
1350
|
+
return 0;
|
|
1351
|
+
const del = this.stmtDeleteL1Vec;
|
|
1352
|
+
const ins = this.stmtInsertL1Vec;
|
|
1353
|
+
try {
|
|
1354
|
+
return this.withTransaction(() => {
|
|
1355
|
+
let written = 0;
|
|
1356
|
+
for (const it of items) {
|
|
1357
|
+
if (isZeroVector(it.embedding))
|
|
1358
|
+
continue;
|
|
1359
|
+
del.run(it.id);
|
|
1360
|
+
ins.run(it.id, vecToBuffer(it.embedding), new Date().toISOString());
|
|
1361
|
+
written++;
|
|
1362
|
+
}
|
|
1363
|
+
return written;
|
|
1364
|
+
});
|
|
1365
|
+
}
|
|
1366
|
+
catch (err) {
|
|
1367
|
+
this.logger?.warn(`${TAG} L1 向量批量写入失败,回退逐条: ${err instanceof Error ? err.message : String(err)}`);
|
|
1368
|
+
let ok = 0;
|
|
1369
|
+
for (const it of items)
|
|
1370
|
+
if (this.updateL1Vec(it.id, it.embedding))
|
|
1371
|
+
ok++;
|
|
1372
|
+
return ok;
|
|
1373
|
+
}
|
|
1374
|
+
}
|
|
1375
|
+
/** L0 版 updateL1VecBatch(语义同:单事务 + 失败回退逐条)。recordedAt 整批统一。 */
|
|
1376
|
+
updateL0VecBatch(items, recordedAt) {
|
|
1377
|
+
if (this.degraded || !this.stmtDeleteL0Vec || !this.stmtInsertL0Vec || items.length === 0)
|
|
1378
|
+
return 0;
|
|
1379
|
+
const del = this.stmtDeleteL0Vec;
|
|
1380
|
+
const ins = this.stmtInsertL0Vec;
|
|
1381
|
+
try {
|
|
1382
|
+
return this.withTransaction(() => {
|
|
1383
|
+
let written = 0;
|
|
1384
|
+
for (const it of items) {
|
|
1385
|
+
if (isZeroVector(it.embedding))
|
|
1386
|
+
continue;
|
|
1387
|
+
del.run(it.id);
|
|
1388
|
+
ins.run(it.id, vecToBuffer(it.embedding), recordedAt);
|
|
1389
|
+
written++;
|
|
1390
|
+
}
|
|
1391
|
+
return written;
|
|
1392
|
+
});
|
|
1393
|
+
}
|
|
1394
|
+
catch (err) {
|
|
1395
|
+
this.logger?.warn(`${TAG} L0 向量批量写入失败,回退逐条: ${err instanceof Error ? err.message : String(err)}`);
|
|
1396
|
+
let ok = 0;
|
|
1397
|
+
for (const it of items)
|
|
1398
|
+
if (this.updateL0Vec(it.id, it.embedding, recordedAt))
|
|
1399
|
+
ok++;
|
|
1400
|
+
return ok;
|
|
1401
|
+
}
|
|
1402
|
+
}
|
|
1403
|
+
close() {
|
|
1404
|
+
try {
|
|
1405
|
+
this.db.close();
|
|
1406
|
+
}
|
|
1407
|
+
catch {
|
|
1408
|
+
/* ignore */
|
|
1409
|
+
}
|
|
1410
|
+
}
|
|
1411
|
+
}
|
|
1412
|
+
function rowToRecord(row) {
|
|
1413
|
+
let metadata = {};
|
|
1414
|
+
try {
|
|
1415
|
+
metadata = JSON.parse(row.metadata_json || '{}');
|
|
1416
|
+
}
|
|
1417
|
+
catch {
|
|
1418
|
+
/* 坏 JSON 容忍 */
|
|
1419
|
+
}
|
|
1420
|
+
return {
|
|
1421
|
+
id: row.record_id,
|
|
1422
|
+
content: row.content,
|
|
1423
|
+
type: row.type,
|
|
1424
|
+
priority: row.priority,
|
|
1425
|
+
scene_name: row.scene_name,
|
|
1426
|
+
timestamps: dbToTimestamps(row.timestamp_str),
|
|
1427
|
+
createdAt: Date.parse(row.created_time) || 0,
|
|
1428
|
+
updatedAt: Date.parse(row.updated_time) || 0,
|
|
1429
|
+
version: row.version ?? 0,
|
|
1430
|
+
metadata,
|
|
1431
|
+
family: normFamily(row.family, row.type),
|
|
1432
|
+
// 时间轴回读:空串 → undefined(空串表示"未填",不是"时间 0")
|
|
1433
|
+
validFrom: row.valid_from ? Date.parse(row.valid_from) || undefined : undefined,
|
|
1434
|
+
validTo: row.valid_to ? Date.parse(row.valid_to) || undefined : undefined,
|
|
1435
|
+
persistence: normPersistence(row.persistence),
|
|
1436
|
+
};
|
|
1437
|
+
}
|
|
1438
|
+
/** epoch 数组 → {逗号连接 ISO, 首尾 ISO}(磁盘格式契约:逗号连接、升序、ISO)。 */
|
|
1439
|
+
function timestampsToDb(ts) {
|
|
1440
|
+
if (!ts || ts.length === 0)
|
|
1441
|
+
return { str: '', start: '', end: '' };
|
|
1442
|
+
const sorted = [...ts].filter((t) => Number.isFinite(t)).sort((a, b) => a - b);
|
|
1443
|
+
if (sorted.length === 0)
|
|
1444
|
+
return { str: '', start: '', end: '' };
|
|
1445
|
+
const isos = sorted.map((t) => new Date(t).toISOString());
|
|
1446
|
+
return { str: isos.join(','), start: isos[0], end: isos[isos.length - 1] };
|
|
1447
|
+
}
|
|
1448
|
+
function dbToTimestamps(str) {
|
|
1449
|
+
if (!str)
|
|
1450
|
+
return [];
|
|
1451
|
+
return str
|
|
1452
|
+
.split(',')
|
|
1453
|
+
.map((t) => Date.parse(t))
|
|
1454
|
+
.filter((t) => !Number.isNaN(t));
|
|
1455
|
+
}
|
|
1456
|
+
function toIso(epochMs) {
|
|
1457
|
+
if (!epochMs || !Number.isFinite(epochMs))
|
|
1458
|
+
return '';
|
|
1459
|
+
return new Date(epochMs).toISOString();
|
|
1460
|
+
}
|
|
1461
|
+
/** 全零向量(cosine 未定义,不可入向量表)。reindex 侧用它区分"不可嵌入"与"写入失败"。 */
|
|
1462
|
+
export function isZeroVector(vec) {
|
|
1463
|
+
for (const v of vec) {
|
|
1464
|
+
if (v !== 0)
|
|
1465
|
+
return false;
|
|
1466
|
+
}
|
|
1467
|
+
return true;
|
|
1468
|
+
}
|
|
1469
|
+
/** NOT IN 片段(空集 → 空串;配合 notInParams 使用)。 */
|
|
1470
|
+
function notInClause(column, exclude) {
|
|
1471
|
+
if (!exclude || exclude.size === 0)
|
|
1472
|
+
return '';
|
|
1473
|
+
return ` AND ${column} NOT IN (${[...exclude].map(() => '?').join(',')})`;
|
|
1474
|
+
}
|
|
1475
|
+
function notInParams(exclude) {
|
|
1476
|
+
if (!exclude || exclude.size === 0)
|
|
1477
|
+
return [];
|
|
1478
|
+
return [...exclude];
|
|
1479
|
+
}
|
|
1480
|
+
function vecSkipKey(kind) {
|
|
1481
|
+
return kind === 'l1' ? 'embedding_zero_vec_l1' : 'embedding_zero_vec_l0';
|
|
1482
|
+
}
|
|
1483
|
+
/** DB 字符串 → 族(异常值按 type 前缀兜底归一)。 */
|
|
1484
|
+
function normFamily(raw, type) {
|
|
1485
|
+
if (raw === 'work' || raw === 'chat')
|
|
1486
|
+
return raw;
|
|
1487
|
+
return familyForType(type);
|
|
1488
|
+
}
|
|
1489
|
+
function vecToBuffer(vec) {
|
|
1490
|
+
return Buffer.from(vec.buffer, vec.byteOffset, vec.byteLength);
|
|
1491
|
+
}
|