@avantf/dsh-mem 0.4.1 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +19 -1
- package/lib/client.js +40 -9
- package/lib/engine/mem/config/config_files.d.ts +1 -1
- package/lib/engine/mem/db/dao/chunks.d.ts +20 -1
- package/lib/engine/mem/db/dao/entities.d.ts +35 -11
- package/lib/engine/mem/db/dao/fact_sources.d.ts +53 -0
- package/lib/engine/mem/db/dao/facts.d.ts +127 -12
- package/lib/engine/mem/db/invariants.d.ts +26 -0
- package/lib/engine/mem/db/tokenizer.d.ts +23 -0
- package/lib/engine/mem/db/vectors.d.ts +40 -8
- package/lib/engine/mem/entities/extract.d.ts +1 -1
- package/lib/engine/mem/eval/derive.d.ts +120 -0
- package/lib/engine/mem/eval/derived_runner.d.ts +88 -0
- package/lib/engine/mem/eval/metrics.d.ts +24 -0
- package/lib/engine/mem/eval/runner.d.ts +8 -1
- package/lib/engine/mem/index.d.ts +1 -1
- package/lib/engine/mem/modelBootstrap.d.ts +2 -2
- package/lib/engine/mem/runtime.d.ts +1 -1
- package/lib/engine/mem/store/common.d.ts +33 -11
- package/lib/engine/mem/store/entity_leg.d.ts +95 -0
- package/lib/engine/mem/store/floors.d.ts +63 -31
- package/lib/engine/mem/store/hybrid.d.ts +21 -26
- package/lib/engine/mem/store/knowledge.d.ts +18 -9
- package/lib/engine/mem/store/lexical.d.ts +44 -0
- package/lib/engine/mem/store/memory.d.ts +151 -24
- package/lib/engine/mem/store/secret_guard.d.ts +34 -0
- package/lib/engine/mem/store/time_window.d.ts +74 -0
- package/lib/engine/mem-contract/config.d.ts +9 -16
- package/lib/engine/mem-contract/family.d.ts +1 -1
- package/lib/engine/mem-contract/tools.d.ts +10 -0
- package/lib/engine/mem-contract/types.d.ts +164 -9
- package/lib/engine/mem-retrieval/adapters/local_bge.d.ts +37 -7
- package/lib/engine/mem-retrieval/fusion.d.ts +14 -1
- package/lib/engine/mem-retrieval/index.d.ts +1 -3
- package/lib/engine/mem-retrieval/interfaces.d.ts +38 -14
- package/lib/engine/mem-retrieval/registry.d.ts +2 -4
- package/lib/engine/mem-retrieval/representation.d.ts +50 -0
- package/lib/engine/mem-retrieval/stats.d.ts +1 -3
- package/lib/index.js +2003 -565
- package/lib/types/client/index.d.ts.map +1 -1
- package/lib/types/index.d.ts +1 -1
- package/lib/types/index.d.ts.map +1 -1
- package/lib/types/prompt.d.ts +7 -0
- package/lib/types/prompt.d.ts.map +1 -1
- package/lib/types/remote.d.ts +12 -6
- package/lib/types/remote.d.ts.map +1 -1
- package/lib/types/vectorMigration.d.ts +44 -0
- package/lib/types/vectorMigration.d.ts.map +1 -0
- package/package.json +5 -5
- package/lib/engine/mem-retrieval/adapters/local_reranker.d.ts +0 -52
- package/lib/engine/mem-retrieval/adapters/none_reranker.d.ts +0 -10
- package/lib/engine/mem-retrieval/rerank.d.ts +0 -13
package/README.md
CHANGED
|
@@ -30,8 +30,10 @@ DSH(DeepSeek Harness)原生记忆插件:给 agent 一份可长期检索的
|
|
|
30
30
|
(frontmatter 记着它属于哪篇文档)——用你自己的编辑器改,插件会在**其后任一工具调用结束时**自动把改动同步
|
|
31
31
|
回索引(先一次 `stat` 判漂移、再用正文哈希裁决,只重摄入真正改过的),「知识」页的「重新摄入」是手动兜底;
|
|
32
32
|
删除文档会连同这份副本一起删除,`source_uri` 指向的原文件始终不被改动。
|
|
33
|
-
- **可插拔、可降级**:语义后端 /
|
|
33
|
+
- **可插拔、可降级**:语义后端 / 向量库都走内部注册表;模型、hnswlib、分词器任一缺失都自动降级
|
|
34
34
|
(FTS + 实体 / numpy 向量库 / 正则抽取),不会因此退出。
|
|
35
|
+
- **0.5.0 起不再有结果重排**:`Reranker` 接口、`rerank.*` 配置键、公开面的 `registerReranker` 与结果/统计里的
|
|
36
|
+
`rerank_*` 字段已整体移除(旧配置里的 `rerank:` 段会被当作未知键告警并忽略);可注册的检索面只剩语义后端与向量库。
|
|
35
37
|
- **两个主窗口标签页**(排在 对话 / 轨迹 之后):**记忆**(浏览、编辑、归档恢复、反馈、矛盾裁决)与
|
|
36
38
|
**知识**(文档列表,每行可查看切片 / 编辑 / 打开目录 / 重新摄入 / 删除;另有跨库查询与入库——
|
|
37
39
|
URL / 本地文件 / 本地目录 / 粘贴文本共用一个输入)。
|
|
@@ -166,6 +168,22 @@ SQLite 绑定的原因:存储层用运行时自带的 `node:sqlite`,它就
|
|
|
166
168
|
- 规则型:`只有「注销式回购」减少总股本,库存股回购不减少——比较 EPS 影响时要区分。`(当……时,应……,附条件与例外)
|
|
167
169
|
- 反例(不该写成事实):一次调查的过程叙述、数据表、命令与链接——依据属于知识库文档,记忆里只留**结论 + 位置**
|
|
168
170
|
(路径 / 来源 / 名称);敏感信息同理,只记存放位置,不记明文。
|
|
171
|
+
- **自定义过提示词文件的话,升级时需手动补**:提示词**文件优先于内置正文**(文件存在且非空时内置正文完全不参与渲染)。
|
|
172
|
+
0.6.0 起内置正文多了三条行为口径,若你改过 `mem-memory-usage.md`,请把下面三行追加到你的文件末尾;没改过则无需操作:
|
|
173
|
+
```text
|
|
174
|
+
记忆是过去的记录、不是当下指令,无关时可完全忽略。
|
|
175
|
+
同一主题用 `update` 纠正并附核实依据,不同事实新增。
|
|
176
|
+
不要把 `recall` 结果原样回记。
|
|
177
|
+
```
|
|
178
|
+
- **可选:中文时间窗检索腿**(0.6.0 起,**默认关闭**)。它在查询含中文时间表达时,把**事件时间**(写入时用
|
|
179
|
+
`event_date` 给出)落在窗口内的活跃事实作为额外一条腿参与融合——只增加候选、不删已有命中,也不会回退到写入时间。
|
|
180
|
+
在 `~/.avantf/configs/common.yaml` 打开:
|
|
181
|
+
```yaml
|
|
182
|
+
retriever:
|
|
183
|
+
time_window: true
|
|
184
|
+
```
|
|
185
|
+
打开前先看 `mem_admin stats` 的 `validity.coverage`(带事件时间的事实占比):覆盖率接近 0 时这条腿等于没开。
|
|
186
|
+
|
|
169
187
|
- **语义模型**:默认从 `hf-mirror.com` 下载并缓存,也可指向本地模型目录离线运行。
|
|
170
188
|
- **升级前先备份 `~/.avantf`**:数据库迁移是单向的,用旧版本打开新库会被明确拒绝。
|
|
171
189
|
|
package/lib/client.js
CHANGED
|
@@ -5315,7 +5315,10 @@ Set the \`cycles\` parameter to \`"ref"\` to resolve cyclical schemas with defs.
|
|
|
5315
5315
|
fact_id: count.optional(),
|
|
5316
5316
|
category: optionalText.optional(),
|
|
5317
5317
|
ttl_days: count.optional(),
|
|
5318
|
-
reason: optionalText.optional()
|
|
5318
|
+
reason: optionalText.optional(),
|
|
5319
|
+
source_ref: optionalText.optional(),
|
|
5320
|
+
event_date: optionalText.optional(),
|
|
5321
|
+
valid_until: optionalText.optional()
|
|
5319
5322
|
})),
|
|
5320
5323
|
direct("recall", object({
|
|
5321
5324
|
action: optionalText.optional(),
|
|
@@ -5327,6 +5330,8 @@ Set the \`cycles\` parameter to \`"ref"\` to resolve cyclical schemas with defs.
|
|
|
5327
5330
|
obj: optionalText.optional(),
|
|
5328
5331
|
second_pred: optionalText.optional(),
|
|
5329
5332
|
category: optionalText.optional(),
|
|
5333
|
+
source: optionalText.optional(),
|
|
5334
|
+
include_scores: bool.optional(),
|
|
5330
5335
|
limit: count.optional(),
|
|
5331
5336
|
max_tokens: count.optional(),
|
|
5332
5337
|
floors: optionalText.optional()
|
|
@@ -5336,6 +5341,7 @@ Set the \`cycles\` parameter to \`"ref"\` to resolve cyclical schemas with defs.
|
|
|
5336
5341
|
fact_id: count.optional(),
|
|
5337
5342
|
category: optionalText.optional(),
|
|
5338
5343
|
status: optionalText.optional(),
|
|
5344
|
+
source: optionalText.optional(),
|
|
5339
5345
|
limit: count.optional(),
|
|
5340
5346
|
offset: count.optional(),
|
|
5341
5347
|
reason: optionalText.optional(),
|
|
@@ -5366,6 +5372,7 @@ Set the \`cycles\` parameter to \`"ref"\` to resolve cyclical schemas with defs.
|
|
|
5366
5372
|
source: optionalText.optional(),
|
|
5367
5373
|
limit: count.optional(),
|
|
5368
5374
|
max_tokens: count.optional(),
|
|
5375
|
+
include_scores: bool.optional(),
|
|
5369
5376
|
floors: optionalText.optional()
|
|
5370
5377
|
})),
|
|
5371
5378
|
direct("classifySource", object({ text: optionalText.optional() })),
|
|
@@ -5670,9 +5677,9 @@ Set the \`cycles\` parameter to \`"ref"\` to resolve cyclical schemas with defs.
|
|
|
5670
5677
|
* the marker", never a decode failure.
|
|
5671
5678
|
*/
|
|
5672
5679
|
function hostSkew(hostWire) {
|
|
5673
|
-
if (hostWire ===
|
|
5674
|
-
if (hostWire === void 0) return `宿主没有回报 wire 版本(它比本客户端旧):面板可能缺少新字段,重启 dsh web 让两半对齐。(本客户端说 wire=${String(
|
|
5675
|
-
return `宿主回报的 wire 版本是 ${String(hostWire)},本客户端只认识 ${String(
|
|
5680
|
+
if (hostWire === 2) return void 0;
|
|
5681
|
+
if (hostWire === void 0) return `宿主没有回报 wire 版本(它比本客户端旧):面板可能缺少新字段,重启 dsh web 让两半对齐。(本客户端说 wire=${String(2)})`;
|
|
5682
|
+
return `宿主回报的 wire 版本是 ${String(hostWire)},本客户端只认识 ${String(2)}:面板继续显示,但两半可能已经错位——重启 dsh web 后再试。`;
|
|
5676
5683
|
}
|
|
5677
5684
|
/**
|
|
5678
5685
|
* Whether a host reporting `hostWire` can have registered a method introduced at `required`.
|
|
@@ -6440,7 +6447,7 @@ Set the \`cycles\` parameter to \`"ref"\` to resolve cyclical schemas with defs.
|
|
|
6440
6447
|
}));
|
|
6441
6448
|
}
|
|
6442
6449
|
}) : null
|
|
6443
|
-
]), detail?.id === f.fact_id ? h("div", { className: pages_module_css_default.meta }, `创建于 ${detail.data.created_at} / 更新于 ${detail.data.updated_at ?? "—"}`, h("br"), `实体:${(detail.data.entities ?? []).join("、") || "(无)"}`, h("br"), `三元组:${(detail.data.triples ?? []).map((t) => `${t.subj} —${t.pred}→ ${t.obj}(${t.confidence.toFixed(2)})`).join(";") || "(无)"}`, h("br"), `检索次数:${String(detail.data.retrieval_count ?? 0)}${detail.data.supersedes_id ? ` · 修订自 #${String(detail.data.supersedes_id)}` : ""}`) : null))), truncated ? h("button", {
|
|
6450
|
+
]), detail?.id === f.fact_id ? h("div", { className: pages_module_css_default.meta }, `创建于 ${detail.data.created_at} / 更新于 ${detail.data.updated_at ?? "—"}`, h("br"), `实体:${(detail.data.entities ?? []).join("、") || "(无)"}`, h("br"), `三元组:${(detail.data.triples ?? []).map((t) => `${t.subj} —${t.pred}→ ${t.obj}(${t.confidence.toFixed(2)})`).join(";") || "(无)"}`, h("br"), `检索次数:${String(detail.data.retrieval_count ?? 0)}${detail.data.supersedes_id ? ` · 修订自 #${String(detail.data.supersedes_id)}` : ""} · 被断言 ${String(detail.data.assert_count ?? 1)} 次`, h("br"), `有效期:${detail.data.valid_from ?? "未知"} → ${detail.data.valid_to ?? (detail.data.status === "active" ? "仍在有效" : "未知")}` + (detail.data.superseded_by === null || detail.data.superseded_by === void 0 ? "" : ` · 被 #${String(detail.data.superseded_by)} 取代`), h("br"), `来源:${(detail.data.sources ?? []).length === 0 ? "(未记录)" : (detail.data.sources ?? []).map((s) => `${s.kind}:${s.ref}`).join(";")}`) : null))), truncated ? h("button", {
|
|
6444
6451
|
type: "button",
|
|
6445
6452
|
className: pages_module_css_default.more,
|
|
6446
6453
|
disabled: loading,
|
|
@@ -6457,10 +6464,13 @@ Set the \`cycles\` parameter to \`"ref"\` to resolve cyclical schemas with defs.
|
|
|
6457
6464
|
function HealthBlock(props) {
|
|
6458
6465
|
const r = props.stats.retrieval;
|
|
6459
6466
|
if (r === void 0) return h("div", { className: pages_module_css_default.status }, "检索健康度不可用。");
|
|
6467
|
+
const vectors = props.stats.vectors;
|
|
6468
|
+
const pendingVectors = vectors === void 0 ? 0 : vectors.stale + vectors.space_stale;
|
|
6469
|
+
const wal = props.stats.wal;
|
|
6460
6470
|
const pct = (v) => `${String(Math.round(v * 100))}%`;
|
|
6461
6471
|
const kinds = Object.entries(r.by_kind).map(([kind, v]) => `${kind} ${String(v.queries)} 次(空 ${String(v.zero_results)})`).join(" · ");
|
|
6462
|
-
const truncated = r.embedding_truncated + r.
|
|
6463
|
-
return h("div", { className: pages_module_css_default.detail }, h("div", { className: pages_module_css_default.status }, `检索 ${String(r.queries)} 次 · 空结果 ${pct(r.zero_result_rate)} · 平均 ${r.avg_latency_ms.toFixed(1)}ms · 峰值 ${r.max_latency_ms.toFixed(1)}ms`), h("div", { className: pages_module_css_default.meta }, `语义腿在线 ${pct(r.semantic_live_rate)} · 平均命中 ${String(r.avg_results_per_query)}
|
|
6472
|
+
const truncated = r.embedding_truncated + r.output_truncated;
|
|
6473
|
+
return h("div", { className: pages_module_css_default.detail }, pendingVectors === 0 ? null : h("div", { className: pages_module_css_default.status }, `⚠ 向量空间待迁移:${String(pendingVectors)} 条 ACTIVE 向量来自旧嵌入空间(宽度不符 ${String(vectors?.stale ?? 0)} · 其他表示 ${String(vectors?.space_stale ?? 0)})——语义腿对它们失效,正在后台分批重算;手动入口 \`avantf-mem vectors --fix\``), wal?.warning === null || wal?.warning === void 0 ? null : h("div", { className: pages_module_css_default.status }, `⚠ ${wal.warning}(当前 ${String(wal.bytes)} 字节)`), h("div", { className: pages_module_css_default.status }, `检索 ${String(r.queries)} 次 · 空结果 ${pct(r.zero_result_rate)} · 平均 ${r.avg_latency_ms.toFixed(1)}ms · 峰值 ${r.max_latency_ms.toFixed(1)}ms`), h("div", { className: pages_module_css_default.meta }, `语义腿在线 ${pct(r.semantic_live_rate)} · 平均命中 ${String(r.avg_results_per_query)} 条`), kinds === "" ? null : h("div", { className: pages_module_css_default.meta }, kinds), props.stats.sources === void 0 && props.stats.validity === void 0 ? null : h("div", { className: pages_module_css_default.meta }, `来源覆盖 ${props.stats.sources === void 0 ? "—" : pct(props.stats.sources.coverage)}(${String(props.stats.sources?.facts_with_source ?? 0)}/${String(props.stats.sources?.active ?? 0)}) · 事件时间覆盖 ${props.stats.validity === void 0 ? "—" : pct(props.stats.validity.coverage)}(${String(props.stats.validity?.facts_with_valid_from ?? 0)}/${String(props.stats.validity?.active ?? 0)})`), h("div", { className: pages_module_css_default.meta }, truncated === 0 ? "未被任何预算截断。" : `截断:嵌入 ${String(r.embedding_truncated)} · 输出 ${String(r.output_truncated)}${r.output_truncated > 0 ? "(调大 retrieval.max_output_tokens 或用 max_tokens 参数)" : ""}`), r.updated_at === null ? null : h("div", { className: pages_module_css_default.meta }, `更新于 ${r.updated_at}`));
|
|
6464
6474
|
}
|
|
6465
6475
|
/**
|
|
6466
6476
|
* Group cross-store hits for display: memory facts vs `domain → source`.
|
|
@@ -6477,6 +6487,26 @@ Set the \`cycles\` parameter to \`"ref"\` to resolve cyclical schemas with defs.
|
|
|
6477
6487
|
return domain || source || "文档切片";
|
|
6478
6488
|
}
|
|
6479
6489
|
/**
|
|
6490
|
+
* P-01: render the per-leg evidence the query page explicitly asked for (`include_scores: true`).
|
|
6491
|
+
*
|
|
6492
|
+
* `scores` is absent for every other caller (the field is default-off), so this returns null and
|
|
6493
|
+
* the row renders exactly as before. A leg that did not recall the hit is `null` and prints as
|
|
6494
|
+
* `—`, which is the honest reading: the candidate entered through some OTHER leg.
|
|
6495
|
+
*/
|
|
6496
|
+
function legEvidenceLine(hit) {
|
|
6497
|
+
const scores = hit.scores;
|
|
6498
|
+
if (scores === void 0) return null;
|
|
6499
|
+
const label = {
|
|
6500
|
+
semantic: "语义",
|
|
6501
|
+
fts: "FTS",
|
|
6502
|
+
jaccard: "实体",
|
|
6503
|
+
hrr: "HRR"
|
|
6504
|
+
};
|
|
6505
|
+
const parts = Object.entries(scores).map(([leg, score]) => score === null ? `${label[leg] ?? leg} —` : `${label[leg] ?? leg} ${score.normalized.toFixed(2)}(原始 ${score.raw.toFixed(2)})`);
|
|
6506
|
+
const final = hit.final === void 0 ? "" : ` · 融合 ${hit.final.toFixed(3)}`;
|
|
6507
|
+
return `证据:${parts.join(" · ")}${final}`;
|
|
6508
|
+
}
|
|
6509
|
+
/**
|
|
6480
6510
|
* How many candidates one reply's relevance floors removed, across every leg.
|
|
6481
6511
|
*
|
|
6482
6512
|
* This is the difference between the two readings of an empty result ("the floors removed N" vs
|
|
@@ -6573,7 +6603,8 @@ Set the \`cycles\` parameter to \`"ref"\` to resolve cyclical schemas with defs.
|
|
|
6573
6603
|
domain: domain.trim() || void 0,
|
|
6574
6604
|
source: source.trim() || void 0,
|
|
6575
6605
|
limit: 10,
|
|
6576
|
-
floors
|
|
6606
|
+
floors,
|
|
6607
|
+
include_scores: true
|
|
6577
6608
|
})).then((outcome) => {
|
|
6578
6609
|
if (!queryGuard.isCurrent(seq)) return;
|
|
6579
6610
|
if (outcome.ok) {
|
|
@@ -7042,7 +7073,7 @@ Set the \`cycles\` parameter to \`"ref"\` to resolve cyclical schemas with defs.
|
|
|
7042
7073
|
})), props.remote === void 0 ? null : h("div", { className: pages_module_css_default.status }, queryLoading ? "查询中…" : queryStatus), props.remote !== void 0 && !queryLoading && error === void 0 && hits.length === 0 && queryStatus.startsWith("命中") && floors === "strict" && floorDropped > 0 ? h("div", { className: pages_module_css_default.status }, `没有达到相关性门槛的结果(严格门槛丢弃了 ${String(floorDropped)} 条)——可切「宽松」重试。`) : null, props.remote !== void 0 && !queryLoading && error === void 0 && hits.length === 0 && queryStatus.startsWith("命中") && floors === "loose" && floorDropped > 0 ? h("div", { className: pages_module_css_default.status }, `确实没有相关结果(宽松门槛下仍丢弃了 ${String(floorDropped)} 条)。`) : null, props.remote !== void 0 && !queryLoading && error === void 0 && hits.length === 0 && queryStatus.startsWith("命中") && floorDropped === 0 ? h("div", { className: pages_module_css_default.status }, "没有匹配项。知识库为空时先用「入库/导入」添加文档;记忆里也只包含已写入的事实。") : null, groups.map(([groupName, groupHits]) => h("div", { key: groupName }, h("div", { className: pages_module_css_default.groupHead }, groupName), h("ul", { className: pages_module_css_default.list }, groupHits.map((hit) => h("li", {
|
|
7043
7074
|
key: `${hit.source_ref}:${String(hit.ref_id)}`,
|
|
7044
7075
|
className: pages_module_css_default.item
|
|
7045
|
-
}, hit.text, h("span", { className: pages_module_css_default.meta }, `${hit.kind} · score ${hit.score.toFixed(3)} · ${hit.source_ref}` + (hit.entities && hit.entities.length > 0 ? ` · 实体:${hit.entities.join("、")}` : "") + ` · 创建于 ${hit.created_at}` + (hit.updated_at !== null && hit.updated_at !== hit.created_at ? ` · 更新于 ${hit.updated_at}` : "")))))))) : null, showIngest ? h("div", { className: `${pages_module_css_default.item} ${pages_module_css_default.form}` }, h("div", { className: pages_module_css_default.toolbar }, h(DomainField, {
|
|
7076
|
+
}, hit.text, h("span", { className: pages_module_css_default.meta }, `${hit.kind} · score ${hit.score.toFixed(3)} · ${hit.source_ref}` + (hit.entities && hit.entities.length > 0 ? ` · 实体:${hit.entities.join("、")}` : "") + ` · 创建于 ${hit.created_at}` + (hit.updated_at !== null && hit.updated_at !== hit.created_at ? ` · 更新于 ${hit.updated_at}` : "") + (legEvidenceLine(hit) === null ? "" : ` · ${legEvidenceLine(hit) ?? ""}`)))))))) : null, showIngest ? h("div", { className: `${pages_module_css_default.item} ${pages_module_css_default.form}` }, h("div", { className: pages_module_css_default.toolbar }, h(DomainField, {
|
|
7046
7077
|
label: "知识域(domain)",
|
|
7047
7078
|
value: iDomain,
|
|
7048
7079
|
onChange: setIDomain,
|
|
@@ -15,7 +15,7 @@ import type { AvantfLogger } from '../../mem-contract/index.js';
|
|
|
15
15
|
* UNCOMMENTED line is warned about by the loader, so a wrong name here would be a trap rather than a
|
|
16
16
|
* hint. `config_files.spec.ts` pins the ones this template names.
|
|
17
17
|
*/
|
|
18
|
-
export declare const DEFAULT_COMMON_CONFIG = "# avantf-mem \u516C\u5171\u914D\u7F6E\uFF08\u5206\u5C42\u7B2C\u2461\u5C42\uFF09\u3002\u8FD9\u91CC\u5168\u662F\u6CE8\u91CA\uFF1A\u5220\u6389/\u7559\u7A7A = \u7528\u5185\u5EFA\u9ED8\u8BA4\uFF0C\u53D6\u6D88\u6CE8\u91CA\u5373\u751F\u6548\u3002\n# \u5206\u5C42\uFF08\u4F4E\u2192\u9AD8\uFF09\uFF1A\u2460 \u5185\u5EFA\u9ED8\u8BA4 \u2192 \u2461 \u672C\u6587\u4EF6 \u2192 \u2462 \u540C\u76EE\u5F55\u7684 memory.yaml / knowledge.yaml\uFF08\u7F3A\u5931=\u4E0D\u8986\u76D6\uFF09\n# \u2192 \u2463 \u73AF\u5883\u53D8\u91CF \u2192 \u2464 \u63D2\u4EF6/CLI \u663E\u5F0F\u4F20\u53C2\n# \u6240\u6709\u53EF\u7F16\u8F91\u6587\u672C\u90FD\u5728 ~/.avantf \u4E0B\uFF1Aconfigs/*.yaml\uFF08\u672C\u76EE\u5F55\uFF09\u4E0E prompts/*.md\uFF08\u6A21\u578B\u63D0\u793A\u8BCD\uFF09\u3002\n\n# \u6570\u636E\u6839\u4E0D\u5728\u8FD9\u91CC\u914D\uFF1A\u5B83\u7531 \u2463 AVANTF_HOME\uFF08\u6216 CLI --data-home\u3001\u63D2\u4EF6 profile \u7684 dataHome\uFF09\u51B3\u5B9A \u2014\u2014\n# \u672C\u6587\u4EF6\u5C31\u5728\u6570\u636E\u6839\u91CC\u9762\uFF0C\u89E3\u6790\u6839\u7684\u65F6\u5019\u8FD8\u6CA1\u8BFB\u5230\u5B83\uFF0C\u6240\u4EE5\u5199\u5728\u8FD9\u91CC\u7684 dataHome \u5BF9\u6570\u636E\u6839\u65E0\u6548\uFF08DESIGN \u00A73\uFF09\u3002\n\n# semantic: # \u5D4C\u5165\u540E\u7AEF\uFF08\u8BB0\u5FC6\u4E0E\u77E5\u8BC6\u5171\u7528\uFF0C\u8DE8\u5E93\u5206\u6570\u624D\u53EF\u6BD4\uFF09\n# backend: local_bge\n# local_model: Xenova/bge-base-zh-v1.5 # \u5FC5\u987B\u662F ONNX \u4ED3\u5E93\n# dim: 768\n# auto_download: true\n# \u26A0\uFE0F cache_dir / mirror \u7531\u5BB6\u65CF\u5E95\u5EA7 @avantf/dsh-plugin-base \u7BA1\u7406\uFF1A\u5199\u5728\u8FD9\u91CC\u4F1A\u88AB\u5FFD\u7565\u5E76\u544A\u8B66\u3002\n# \u7F13\u5B58\u76EE\u5F55\u7528\u73AF\u5883\u53D8\u91CF AVANTF_MEM_MODEL_CACHE\uFF0C\u955C\u50CF\u7528 AVANTF_MEM_MODEL_MIRROR\uFF08\u6216 HF_ENDPOINT\uFF09\u3002\n\n#
|
|
18
|
+
export declare const DEFAULT_COMMON_CONFIG = "# avantf-mem \u516C\u5171\u914D\u7F6E\uFF08\u5206\u5C42\u7B2C\u2461\u5C42\uFF09\u3002\u8FD9\u91CC\u5168\u662F\u6CE8\u91CA\uFF1A\u5220\u6389/\u7559\u7A7A = \u7528\u5185\u5EFA\u9ED8\u8BA4\uFF0C\u53D6\u6D88\u6CE8\u91CA\u5373\u751F\u6548\u3002\n# \u5206\u5C42\uFF08\u4F4E\u2192\u9AD8\uFF09\uFF1A\u2460 \u5185\u5EFA\u9ED8\u8BA4 \u2192 \u2461 \u672C\u6587\u4EF6 \u2192 \u2462 \u540C\u76EE\u5F55\u7684 memory.yaml / knowledge.yaml\uFF08\u7F3A\u5931=\u4E0D\u8986\u76D6\uFF09\n# \u2192 \u2463 \u73AF\u5883\u53D8\u91CF \u2192 \u2464 \u63D2\u4EF6/CLI \u663E\u5F0F\u4F20\u53C2\n# \u6240\u6709\u53EF\u7F16\u8F91\u6587\u672C\u90FD\u5728 ~/.avantf \u4E0B\uFF1Aconfigs/*.yaml\uFF08\u672C\u76EE\u5F55\uFF09\u4E0E prompts/*.md\uFF08\u6A21\u578B\u63D0\u793A\u8BCD\uFF09\u3002\n\n# \u6570\u636E\u6839\u4E0D\u5728\u8FD9\u91CC\u914D\uFF1A\u5B83\u7531 \u2463 AVANTF_HOME\uFF08\u6216 CLI --data-home\u3001\u63D2\u4EF6 profile \u7684 dataHome\uFF09\u51B3\u5B9A \u2014\u2014\n# \u672C\u6587\u4EF6\u5C31\u5728\u6570\u636E\u6839\u91CC\u9762\uFF0C\u89E3\u6790\u6839\u7684\u65F6\u5019\u8FD8\u6CA1\u8BFB\u5230\u5B83\uFF0C\u6240\u4EE5\u5199\u5728\u8FD9\u91CC\u7684 dataHome \u5BF9\u6570\u636E\u6839\u65E0\u6548\uFF08DESIGN \u00A73\uFF09\u3002\n\n# semantic: # \u5D4C\u5165\u540E\u7AEF\uFF08\u8BB0\u5FC6\u4E0E\u77E5\u8BC6\u5171\u7528\uFF0C\u8DE8\u5E93\u5206\u6570\u624D\u53EF\u6BD4\uFF09\n# backend: local_bge\n# local_model: Xenova/bge-base-zh-v1.5 # \u5FC5\u987B\u662F ONNX \u4ED3\u5E93\n# dim: 768\n# auto_download: true\n# auto_migrate: true # \u6362\u6A21\u578B/\u5BBD\u5EA6\u540E\u81EA\u52A8\u5206\u6279\u91CD\u7B97\u65E7\u7A7A\u95F4\u7684\u5411\u91CF\uFF08\u9ED8\u8BA4\u5F00\uFF09\uFF1B\u5173\u6389\u53EA\u5269\u542F\u52A8\u544A\u8B66 + vectors --fix\n# \u26A0\uFE0F cache_dir / mirror \u7531\u5BB6\u65CF\u5E95\u5EA7 @avantf/dsh-plugin-base \u7BA1\u7406\uFF1A\u5199\u5728\u8FD9\u91CC\u4F1A\u88AB\u5FFD\u7565\u5E76\u544A\u8B66\u3002\n# \u7F13\u5B58\u76EE\u5F55\u7528\u73AF\u5883\u53D8\u91CF AVANTF_MEM_MODEL_CACHE\uFF0C\u955C\u50CF\u7528 AVANTF_MEM_MODEL_MIRROR\uFF08\u6216 HF_ENDPOINT\uFF09\u3002\n\n# vectorStore:\n# backend: auto # auto = \u8D8A\u8FC7\u9608\u503C\u81EA\u52A8\u4ECE local_numpy \u8FC1\u5230 hnswlib\n# auto_thresholds:\n# hnswlib: 2000\n# hnswlib_ef_search: 256 # ANN \u7684\u53EC\u56DE/\u901F\u5EA6\u65CB\u94AE\uFF0C\u8BED\u6599\u53D8\u5927\u5148\u8C03\u5B83\n\n# retriever: # \u4E09\u6761\u68C0\u7D22\u817F\u5728\u878D\u5408\u91CC\u7684\u6743\u91CD\u4E0E\"\u76F8\u5173\u6027\u95E8\u69DB\"\n# weight_semantic: 0.55\n# weight_fts: 0.30\n# weight_jaccard: 0.15\n# # \u7EDD\u5BF9\u95E8\u69DB\u6253\u5728\u6BCF\u6761\u817F\u7684\u539F\u59CB\u5206\u4E0A\u3001fuse() \u4E4B\u524D\uFF08\u878D\u5408\u5206\u53EA\u5728\u4E00\u6B21\u67E5\u8BE2\u5185\u53EF\u6BD4\uFF0C\u4E0D\u80FD\u5F53\u95E8\u69DB\uFF09\u3002\n# # 0 = \u5173\u95ED\u8BE5\u95E8\u69DB\uFF1B\u5206\u6570\u7B49\u4E8E\u95E8\u69DB\u4FDD\u7559\u3002\u8BED\u4E49\u540E\u7AEF\u4E0D\u53EF\u7528\u65F6 min_fts_terms \u7684\u751F\u6548\u503C\u653E\u5BBD\u5230 1\u3002\n# min_semantic_similarity: 0.5 # \u8BED\u4E49\u817F\u4F59\u5F26\uFF1B0.5 \u53EA\u5BF9 bge-base-zh-v1.5/768\uFF08mean pooling\uFF09\u6807\u5B9A\u8FC7\uFF0C\u6362\u6A21\u578B\u8981\u91CD\u6807\n# min_fts_terms: 2 # FTS \u817F\uFF1A\u8FD9\u4E00\u884C\u547D\u4E2D\u51E0\u4E2A\u4E0D\u540C\u7684\u67E5\u8BE2\u8BCD\u5143\uFF08\u62C9\u4E01\u8BCD\u22655\u5B57\u7B26 + CJK 3-gram\uFF09\n# min_jaccard: 0.2 # \u5B9E\u4F53\u817F\uFF1A\u951A\u70B9\u5B9E\u4F53\u7684 Jaccard\uFF08\u4E8B\u5B9E\u5BBD\u5EA6\u9971\u548C\uFF09\uFF1B2 \u5B9E\u4F53\u547D\u4E2D 1 \u4E2A\u6B63\u597D 0.2\n# # P-11 \u4E2D\u6587\u65F6\u95F4\u7A97\u817F\uFF08\u9ED8\u8BA4\u5173\uFF09\uFF1A\u5F00\u542F\u540E\uFF0C\u5E26\u4E2D\u6587\u65F6\u95F4\u8868\u8FBE\u7684\u67E5\u8BE2\uFF08\u4E0A\u4E2A\u6708 / \u6700\u8FD1\u4E09\u5929 / 2026\u5E749\u6708 \u2026\uFF09\n# # \u4F1A\u989D\u5916\u628A\"\u4E8B\u4EF6\u65F6\u95F4 valid_from \u843D\u5728\u8BE5\u7A97\u53E3\"\u7684\u6D3B\u8DC3\u4E8B\u5B9E\u4F5C\u4E3A\u4E00\u6761\u817F\u53C2\u4E0E\u878D\u5408\u3002\u5B83\u662F**\u52A0\u4E00\u6761\u817F\u3001\u4E0D\u662F\u8FC7\u6EE4\u5668**\uFF1A\n# # \u53EA\u589E\u52A0\u5019\u9009\uFF0C\u7EDD\u4E0D\u5220\u6389\u8BED\u4E49 / FTS / \u5B9E\u4F53\u817F\u5DF2\u7ECF\u627E\u5230\u7684\u4E8B\u5B9E\uFF1B\u6CA1\u6709\u4E8B\u4EF6\u65F6\u95F4\u7684\u4E8B\u5B9E\u4E0D\u4F5C\u4E3A\u8BE5\u817F\u5019\u9009\uFF0C\n# # \u4E5F**\u4E0D\u56DE\u9000\u5230\u5199\u5165\u65F6\u95F4 created_at**\u3002\u5B83\u5171\u7528 jaccard \u6743\u91CD\u4F4D\uFF08\u5BF9\u5916\u7684\u4E09\u952E\u6743\u91CD\u4E0D\u53D8\uFF09\u3002\n# # \u4EC0\u4E48\u65F6\u5019\u8BE5\u5F00\uFF1A\u5148\u7528 mem_admin stats \u770B validity.coverage\uFF08\u6709\u591A\u5C11\u6D3B\u8DC3\u4E8B\u5B9E\u5E26\u4E8B\u4EF6\u65F6\u95F4\uFF09\uFF0C\n# # \u518D\u7528\u56DE\u5F52\u7F51\u786E\u8BA4\u5B83\u786E\u5B9E\u6539\u5584\u65F6\u95F4\u7C7B\u67E5\u8BE2\u2014\u2014\u8986\u76D6\u7387\u22480 \u65F6\u5B83\u7B49\u4E8E\u6CA1\u5F00\u3002\u5199\u5165\u4E8B\u4EF6\u65F6\u95F4\u7528 mem_remember \u7684 event_date\u3002\n# time_window: false\n\n# lifecycle:\n# purge_after_archived_days: 365 # \u5F52\u6863\u4E8B\u5B9E\u7684\u7269\u7406\u6E05\u7406\u7A97\u53E3\uFF08\u6D3B\u8DC3\u65E5\uFF09\n# contradiction_threshold: 0.6\n\n# trust:\n# decay_per_day: 0.0055\u2026\n# presence:\n# heartbeat_minutes: 60 # \u5E38\u9A7B\u8FDB\u7A0B\u7684\u5FC3\u8DF3\uFF08\u6D3B\u8DC3\u65E5\u65F6\u949F\u4E0E\u751F\u547D\u5468\u671F\u626B\u63CF\uFF09\n\n# tools: # \u5916\u90E8\u4E8C\u8FDB\u5236\uFF08pandoc\uFF09\u7684\u53D7\u7BA1\u76EE\u5F55\n# dir: '' # \u7559\u7A7A = \u5BB6\u65CF\u6839 tools\uFF08~/.avantf/env/tools\uFF09\uFF1BAVANTF_TOOLS_DIR \u53EF\u8986\u76D6\n# auto_install: true\n";
|
|
19
19
|
/**
|
|
20
20
|
* Ensure the common config exists with content, and hand back the path either way.
|
|
21
21
|
*
|
|
@@ -140,11 +140,30 @@ export declare class ChunksDao {
|
|
|
140
140
|
* the cap makes `scores.size === legCap` — the "was this leg trimmed" signal — report the opposite
|
|
141
141
|
* of the truth. (The memory store's sibling does not batch, so its single `LIMIT` is already exact.)
|
|
142
142
|
*/
|
|
143
|
-
candidatesByEntityNames(names: readonly string[], domain: string | undefined, source: string | undefined, limit: number): number[];
|
|
143
|
+
candidatesByEntityNames(names: readonly string[], domain: string | undefined, source: string | undefined, limit: number, queryWidth: number, widthCap: number): number[];
|
|
144
|
+
/**
|
|
145
|
+
* How many chunks carry each name — the knowledge side of the entity leg's document frequency.
|
|
146
|
+
*
|
|
147
|
+
* Mirrors `EntitiesDao.activeDocFrequency`: only the caller's query names are looked up. There is
|
|
148
|
+
* no status column to filter on (see `candidatesByEntityNames`), so "document frequency" is over
|
|
149
|
+
* all chunks, which is the population the leg ranks.
|
|
150
|
+
*/
|
|
151
|
+
docFrequency(names: readonly string[]): Map<string, number>;
|
|
144
152
|
/** FTS leg (`bm25` rank; negated by the caller so higher = better). */
|
|
145
153
|
ftsSearch(ftsQuery: string, domain?: string, source?: string, limit?: number): {
|
|
146
154
|
id: number;
|
|
147
155
|
rank: number;
|
|
148
156
|
}[];
|
|
157
|
+
/**
|
|
158
|
+
* The knowledge store's SHORT-QUERY fallback leg — the sibling of `FactsDao.ftsSubstringSearch`
|
|
159
|
+
* (see it for the score/tie/verify notes; the predicate itself is shared through
|
|
160
|
+
* `db/tokenizer.ts#likeSubstring`). It reads `doc_chunks` directly rather than the external-content
|
|
161
|
+
* FTS table: no index can serve a two-character pattern anyway, and the `documents` join is where
|
|
162
|
+
* the domain/source filter lives.
|
|
163
|
+
*/
|
|
164
|
+
ftsSubstringSearch(terms: readonly string[], domain?: string, source?: string, limit?: number): {
|
|
165
|
+
id: number;
|
|
166
|
+
rank: number;
|
|
167
|
+
}[];
|
|
149
168
|
}
|
|
150
169
|
export {};
|
|
@@ -7,16 +7,23 @@
|
|
|
7
7
|
* contradiction detection (see `db/chunk.ts`).
|
|
8
8
|
*/
|
|
9
9
|
import type { Db } from '../port.js';
|
|
10
|
+
import type { ExtractedEntity } from '../../entities/extract.js';
|
|
10
11
|
export declare class EntitiesDao {
|
|
11
12
|
private readonly db;
|
|
12
13
|
constructor(db: Db);
|
|
13
14
|
/**
|
|
14
|
-
* Ensure every
|
|
15
|
+
* Ensure every entity exists (with the extractor's `type` / `method`) and link the fact to it.
|
|
15
16
|
*
|
|
16
17
|
* `INSERT OR IGNORE` + the `name` uniqueness make this idempotent, which matters because
|
|
17
18
|
* both `add` and the revive branch of `persistFact` call it for the same fact.
|
|
19
|
+
*
|
|
20
|
+
* FIRST WRITE WINS for `entity_type` / `extraction_method`: a name already in the vocabulary
|
|
21
|
+
* keeps the type/method of whichever write introduced it (`INSERT OR IGNORE` skips the row),
|
|
22
|
+
* so an old row is never rewritten by a later fact that happens to share the name. That is the
|
|
23
|
+
* documented trade-off of not bumping `ENTITY_EXTRACTOR_VERSION` — new entities get real
|
|
24
|
+
* values, pre-existing ones keep theirs.
|
|
18
25
|
*/
|
|
19
|
-
linkFact(factId: number,
|
|
26
|
+
linkFact(factId: number, entities: readonly ExtractedEntity[]): void;
|
|
20
27
|
/**
|
|
21
28
|
* Drop one fact's links, leaving the shared `entities` vocabulary alone.
|
|
22
29
|
*
|
|
@@ -32,6 +39,16 @@ export declare class EntitiesDao {
|
|
|
32
39
|
bagsForFacts(ids: readonly number[]): Map<number, string[]>;
|
|
33
40
|
/** How many entities a fact is linked to — the scorer's `|A|`. */
|
|
34
41
|
countForFact(factId: number): number;
|
|
42
|
+
/**
|
|
43
|
+
* How many ACTIVE facts carry each name — the entity leg's document frequency.
|
|
44
|
+
*
|
|
45
|
+
* The anchor filter (`store/entity_leg.ts`) needs it to drop corpus-wide words from a query's
|
|
46
|
+
* evidence: a name carried by ~half the store is not discriminative, and treating it as an anchor
|
|
47
|
+
* would make the entity leg drag the whole store in. Only the caller's QUERY names are looked up
|
|
48
|
+
* (a handful), so this is one indexed COUNT per name, not a corpus scan. A name absent from the
|
|
49
|
+
* map has frequency 0, i.e. no active fact carries it and it can never be shared.
|
|
50
|
+
*/
|
|
51
|
+
activeDocFrequency(names: readonly string[]): Map<string, number>;
|
|
35
52
|
/**
|
|
36
53
|
* Candidate facts for the embedding leg, narrowed by the scorer's NECESSARY conditions
|
|
37
54
|
* (see {@link ContradictDetector}): at least `minShared` shared entities and at most
|
|
@@ -48,20 +65,27 @@ export declare class EntitiesDao {
|
|
|
48
65
|
activeFactsForEntity(name: string, category?: string): number[];
|
|
49
66
|
/** Active facts linked to ALL of `names` (AND-join, `reason` leg). */
|
|
50
67
|
activeFactsForAllEntities(names: readonly string[]): number[];
|
|
51
|
-
/** Active facts sharing ANY of `names`, in one category (
|
|
68
|
+
/** Active facts sharing ANY of `names`, in one category (the entity leg). */
|
|
52
69
|
/**
|
|
53
|
-
* Candidate facts for an entity
|
|
70
|
+
* Candidate facts for an entity query, ordered by the leg's score and CAPPED.
|
|
54
71
|
*
|
|
55
72
|
* The cap is the whole point: this leg used to return every fact sharing ANY query entity —
|
|
56
73
|
* unbounded in the corpus (measured 6600 rows for a common phrase at 33k facts, and the fusion
|
|
57
74
|
* step then normalized and sorted all of them).
|
|
58
75
|
*
|
|
59
|
-
*
|
|
60
|
-
*
|
|
61
|
-
*
|
|
62
|
-
*
|
|
63
|
-
*
|
|
64
|
-
*
|
|
76
|
+
* The ordering must be the leg's own score order, or the cap drops the best rows: `fuse` scales
|
|
77
|
+
* by each leg's maximum, and a survivor's scaled value is unchanged only when the ordering (and
|
|
78
|
+
* therefore the maximum) survives the trim. The leg scores anchored Jaccard with a saturating
|
|
79
|
+
* union (`store/entity_leg.ts`): `shared / (queryWidth + min(total - shared, widthCap))`, where `queryWidth` is the
|
|
80
|
+
* query's FULL entity count and `names` is its anchors. That ratio — not the raw shared count — is what this orders by, because a
|
|
81
|
+
* narrower fact sharing one entity can outrank a wide one sharing two. `total ASC` is only a
|
|
82
|
+
* deterministic tie-break among equal ratios. The caller still recomputes the score in JS for the
|
|
83
|
+
* survivors, so one formula stays in charge.
|
|
84
|
+
*/
|
|
85
|
+
candidateFactsForAnyEntity(names: readonly string[], category: string | undefined, limit: number, queryWidth: number, widthCap: number,
|
|
86
|
+
/**
|
|
87
|
+
* P-08 provenance filter (`fact_sources.ref`). `EXISTS`, never a JOIN: a fact with several
|
|
88
|
+
* sources would fan out into several rows and eat the leg's `LIMIT`, silently shrinking recall.
|
|
65
89
|
*/
|
|
66
|
-
|
|
90
|
+
source?: string): number[];
|
|
67
91
|
}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `fact_sources` — where a memory fact came from (P-08).
|
|
3
|
+
*
|
|
4
|
+
* One row per `(fact_id, kind, ref)`. `kind` is a closed vocabulary (`session` / `kb_doc` /
|
|
5
|
+
* `tool` / `manual`) so a reader can trust what a reference IS; `ref` is opaque to this layer
|
|
6
|
+
* (a session id, a document id, a tool name, or a free-form note).
|
|
7
|
+
*
|
|
8
|
+
* The SQL lives here like every other aggregate. The store decides WHICH source a write carries
|
|
9
|
+
* (and refuses to invent one when the caller gave none); the DAO only records and reads rows.
|
|
10
|
+
*/
|
|
11
|
+
import type { Db } from '../port.js';
|
|
12
|
+
/** The closed `kind` vocabulary, spelled once so the store and the tests share it. */
|
|
13
|
+
export declare const FACT_SOURCE_KINDS: readonly ["session", "kb_doc", "tool", "manual"];
|
|
14
|
+
export type FactSourceKind = (typeof FACT_SOURCE_KINDS)[number];
|
|
15
|
+
/** One provenance row as the read surfaces render it. */
|
|
16
|
+
export interface FactSource {
|
|
17
|
+
kind: FactSourceKind;
|
|
18
|
+
ref: string;
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* Derive `(kind, ref)` from the single `source_ref` the write face accepts.
|
|
22
|
+
*
|
|
23
|
+
* The tool takes ONE string (the plan adds no second parameter), so the kind axis is spelled by a
|
|
24
|
+
* `kind:` prefix — `session:…` / `kb_doc:…` / `tool:…` — and anything else is `manual` with the
|
|
25
|
+
* reference kept VERBATIM. No shape guessing: a bare `domain:source:title` becomes a `manual`
|
|
26
|
+
* source whose ref is exactly what the caller wrote, which is strictly more honest than inventing
|
|
27
|
+
* a category from how the string looks.
|
|
28
|
+
*/
|
|
29
|
+
export declare function classifySourceRef(sourceRef: string): {
|
|
30
|
+
kind: FactSourceKind;
|
|
31
|
+
ref: string;
|
|
32
|
+
};
|
|
33
|
+
export declare class FactSourcesDao {
|
|
34
|
+
private readonly db;
|
|
35
|
+
constructor(db: Db);
|
|
36
|
+
/**
|
|
37
|
+
* Record one source on one fact. Idempotent by the composite PRIMARY KEY: a repeated write of
|
|
38
|
+
* the same `(kind, ref)` is a no-op, so a caller can safely re-assert provenance.
|
|
39
|
+
*/
|
|
40
|
+
insert(factId: number, kind: FactSourceKind, ref: string): void;
|
|
41
|
+
/** The sources of one fact; empty when no `source_ref` was ever given. */
|
|
42
|
+
listForFact(factId: number): FactSource[];
|
|
43
|
+
/**
|
|
44
|
+
* Coverage over the ACTIVE corpus: how many active facts carry at least one source.
|
|
45
|
+
*
|
|
46
|
+
* The `EXISTS` (rather than `COUNT(DISTINCT)` over a join) keeps the numerator cheap and — more
|
|
47
|
+
* importantly — makes "a fact with three sources" count once, which is what coverage means.
|
|
48
|
+
*/
|
|
49
|
+
activeCoverage(): {
|
|
50
|
+
active: number;
|
|
51
|
+
facts_with_source: number;
|
|
52
|
+
};
|
|
53
|
+
}
|
|
@@ -7,7 +7,8 @@
|
|
|
7
7
|
*/
|
|
8
8
|
import type { Db } from '../port.js';
|
|
9
9
|
/**
|
|
10
|
-
* Every `facts` column EXCEPT the two BLOBs (`hrr_vector` ~8 KB, `semantic_vector` ~2 KB per row)
|
|
10
|
+
* Every `facts` column EXCEPT the two BLOBs (`hrr_vector` ~8 KB, `semantic_vector` ~2 KB per row)
|
|
11
|
+
* and the two DEPRECATED provenance columns (`mirror_source` / `mirror_target`, see below).
|
|
11
12
|
*
|
|
12
13
|
* Multi-row reads that only need scalars used to be `SELECT *`, so a page of the admin list and
|
|
13
14
|
* every recall's trust settlement dragged the whole vector corpus through the driver — tens of
|
|
@@ -17,10 +18,15 @@ import type { Db } from '../port.js';
|
|
|
17
18
|
* `getById` deliberately still selects everything: `restore` re-indexes the persisted vector it
|
|
18
19
|
* reads off that one row.
|
|
19
20
|
*
|
|
20
|
-
* `
|
|
21
|
-
*
|
|
21
|
+
* `mirror_source` was written as the constant `'user'` at the single INSERT below and
|
|
22
|
+
* `mirror_target` never had a writer; neither had a reader left once the model-facing views
|
|
23
|
+
* dropped them (零迁移:the COLUMNS stay in `db/schema.ts` for a later migration to clean up,
|
|
24
|
+
* only this projection and the INSERT stop touching them).
|
|
25
|
+
*
|
|
26
|
+
* `test/db_lifecycle.spec.ts` asserts this list against `PRAGMA table_info(facts)` MINUS those
|
|
27
|
+
* two deprecations, so a new column fails a test instead of silently vanishing from these reads.
|
|
22
28
|
*/
|
|
23
|
-
export declare const FACT_COLUMNS_NO_BLOB = "fact_id, content, category, tags, trust_score, settle_clock,\n pinned, pinned_at, bonus_count, bonus_window_at, last_reinforced_at, archived_clock,\n retrieval_count, helpful_count, last_retrieved_at, embedding_model, vector_store, status,\n supersedes_id, entities_version, conflict_checked, archived_at, archive_reason, ttl_days,\n
|
|
29
|
+
export declare const FACT_COLUMNS_NO_BLOB = "fact_id, content, category, tags, trust_score, settle_clock,\n pinned, pinned_at, bonus_count, bonus_window_at, last_reinforced_at, archived_clock,\n retrieval_count, helpful_count, last_retrieved_at, embedding_model, vector_store, status,\n supersedes_id, entities_version, conflict_checked, archived_at, archive_reason, ttl_days,\n valid_from, valid_to, assert_count, created_at, updated_at";
|
|
24
30
|
/**
|
|
25
31
|
* The FTS leg's exact SQL, exported so the plan guard (`test/fts_plan.spec.ts`) and the bench
|
|
26
32
|
* control group EXPLAIN the production text itself instead of a copy that can silently drift.
|
|
@@ -53,6 +59,21 @@ export declare const FACT_COLUMNS_NO_BLOB = "fact_id, content, category, tags, t
|
|
|
53
59
|
* `INDEXED BY`, which pins the plan against `PRAGMA optimize` flipping it back.
|
|
54
60
|
*/
|
|
55
61
|
export declare const FTS_SEARCH_SQL = "SELECT f.rowid AS id, bm25(facts_fts) AS rank FROM facts_fts f\n JOIN facts fa NOT INDEXED ON fa.fact_id = f.rowid\n WHERE facts_fts MATCH ? AND fa.status = 'active' AND (? IS NULL OR fa.category = ?)\n ORDER BY rank ASC\n LIMIT ?";
|
|
62
|
+
/**
|
|
63
|
+
* {@link FTS_SEARCH_SQL} with the P-08 source predicate, as a SEPARATE statement rather than one
|
|
64
|
+
* more `(? IS NULL OR …)` branch in the shared text.
|
|
65
|
+
*
|
|
66
|
+
* The unfiltered statement's query plan is guarded by `test/fts_plan.spec.ts` (the external-content
|
|
67
|
+
* MATCH must not become the inner loop on a fresh, un-analyzed database), and the guard EXPLAINs
|
|
68
|
+
* `FTS_SEARCH_SQL` itself. Keeping the two texts apart means the default path's plan is provably
|
|
69
|
+
* unchanged, and the filtered path is a NEW statement with its own comment instead of a silent
|
|
70
|
+
* modification of the guarded one.
|
|
71
|
+
*
|
|
72
|
+
* `EXISTS`, never a JOIN on `fact_sources`: a fact with several sources would fan out into several
|
|
73
|
+
* rows, and this leg is `ORDER BY bm25 … LIMIT cap` — the duplicates would eat cap slots and shrink
|
|
74
|
+
* recall behind the caller's back.
|
|
75
|
+
*/
|
|
76
|
+
export declare const FTS_SEARCH_SQL_FILTERED = "SELECT f.rowid AS id, bm25(facts_fts) AS rank FROM facts_fts f\n JOIN facts fa NOT INDEXED ON fa.fact_id = f.rowid\n WHERE facts_fts MATCH ? AND fa.status = 'active' AND (? IS NULL OR fa.category = ?)\n AND (? IS NULL OR EXISTS (\n SELECT 1 FROM fact_sources fs WHERE fs.fact_id = fa.fact_id AND fs.ref = ?\n ))\n ORDER BY rank ASC\n LIMIT ?";
|
|
56
77
|
export declare class FactsDao {
|
|
57
78
|
private readonly db;
|
|
58
79
|
constructor(db: Db);
|
|
@@ -89,10 +110,12 @@ export declare class FactsDao {
|
|
|
89
110
|
rowsByIds(ids: readonly number[]): Record<string, unknown>[];
|
|
90
111
|
/** `fact_id → content` for a hit set (never N+1). */
|
|
91
112
|
textsByIds(ids: readonly number[]): Map<number, string>;
|
|
92
|
-
/** `fact_id → created_at/updated_at` for a hit set (the recall hit's
|
|
113
|
+
/** `fact_id → created_at/updated_at/validity` for a hit set (the recall hit's timestamps). */
|
|
93
114
|
timesByIds(ids: readonly number[]): Map<number, {
|
|
94
115
|
created_at: string;
|
|
95
116
|
updated_at: string | null;
|
|
117
|
+
valid_from: string | null;
|
|
118
|
+
valid_to: string | null;
|
|
96
119
|
}>;
|
|
97
120
|
/** Active rows that carry a persisted vector (index classification). */
|
|
98
121
|
activeVectorRows(): {
|
|
@@ -100,6 +123,18 @@ export declare class FactsDao {
|
|
|
100
123
|
semantic_vector: Uint8Array;
|
|
101
124
|
embedding_model: string | null;
|
|
102
125
|
}[];
|
|
126
|
+
/**
|
|
127
|
+
* Width + recorded space of every ACTIVE persisted vector, WITHOUT reading the bytes.
|
|
128
|
+
*
|
|
129
|
+
* `length(blob)` is the byte count, so `bytes / 4` is the float width — enough to detect a
|
|
130
|
+
* changed `semantic.dim` without decoding a corpus. This is the cheap detection the startup
|
|
131
|
+
* warning and the `/mem` status surface read (see `MemoryStore.vectorSpaceHealth`).
|
|
132
|
+
*/
|
|
133
|
+
activeVectorSpaces(): {
|
|
134
|
+
fact_id: number;
|
|
135
|
+
bytes: number;
|
|
136
|
+
embedding_model: string | null;
|
|
137
|
+
}[];
|
|
103
138
|
/**
|
|
104
139
|
* Facts per archive reason — the trust diagnostics breakdown. One grouped scan rather than
|
|
105
140
|
* a count per reason, because the reasons are few but the table is not.
|
|
@@ -137,10 +172,75 @@ export declare class FactsDao {
|
|
|
137
172
|
pinnedAt: string | null;
|
|
138
173
|
windowAt: string;
|
|
139
174
|
createdAt: string;
|
|
175
|
+
/**
|
|
176
|
+
* P-13: the caller's event time / known end, written straight onto the new row. `null` means
|
|
177
|
+
* "unknown" and is what an omitted argument leaves (the column has no default on purpose).
|
|
178
|
+
* `assert_count` is deliberately NOT named: the column default (1) is the semantics — "this
|
|
179
|
+
* content has been asserted once" — and naming it here would be a second place to keep in step.
|
|
180
|
+
*/
|
|
181
|
+
validFrom?: string | null;
|
|
182
|
+
validTo?: string | null;
|
|
140
183
|
}): {
|
|
141
184
|
changes: number;
|
|
142
185
|
lastInsertRowid: number | bigint;
|
|
143
186
|
};
|
|
187
|
+
/**
|
|
188
|
+
* P-10: land an explicitly given validity on a row that already exists (a duplicate `add` whose
|
|
189
|
+
* caller named `event_date` / `valid_until`). Omitted arguments keep what the row has, exactly
|
|
190
|
+
* like `category`/`ttl_days` (see `add`'s doc).
|
|
191
|
+
*/
|
|
192
|
+
applyValidity(factId: number, validFrom: string | undefined, validTo: string | undefined): void;
|
|
193
|
+
/** P-10: the row's assertion counter (`1` when the column is somehow NULL on a legacy row). */
|
|
194
|
+
assertCount(factId: number): number;
|
|
195
|
+
/**
|
|
196
|
+
* P-10: a verbatim re-assertion of an EXISTING active fact. Returns the new count.
|
|
197
|
+
*
|
|
198
|
+
* Only the pure-duplicate branch calls this: reviving an archived row is a resurrection, not a
|
|
199
|
+
* new assertion (`assert_count` answers "how many times was this said", and the archived row's
|
|
200
|
+
* own count already recorded that). `RETURNING` keeps the read and the write one statement, so
|
|
201
|
+
* two processes cannot interleave between them.
|
|
202
|
+
*/
|
|
203
|
+
incrementAssertCount(factId: number): number;
|
|
204
|
+
/**
|
|
205
|
+
* P-07 audit: stamp `valid_to` on a row that is leaving the active corpus (supersede / a
|
|
206
|
+
* contradiction verdict). `COALESCE` keeps an end the caller already declared (`valid_until`):
|
|
207
|
+
* overwriting it with the archival instant would replace a real answer with an administrative one.
|
|
208
|
+
*/
|
|
209
|
+
markValidTo(factId: number, at: string): void;
|
|
210
|
+
/**
|
|
211
|
+
* P-07: "who superseded this fact", derived from the existing `supersedes_id` link — no
|
|
212
|
+
* redundant column. `MIN` is deterministic when several revisions named the same predecessor
|
|
213
|
+
* (which a merge can produce); the caller renders it as a single direct successor.
|
|
214
|
+
*/
|
|
215
|
+
supersededBy(factId: number): number | null;
|
|
216
|
+
/** P-13 coverage: ACTIVE facts that carry an event time (the `admin stats` numerator). */
|
|
217
|
+
countActiveWithValidFrom(): number;
|
|
218
|
+
/**
|
|
219
|
+
* P-11: the ids of ACTIVE facts whose EVENT time (`valid_from`) falls inside an INCLUSIVE local
|
|
220
|
+
* day range, newest first, capped.
|
|
221
|
+
*
|
|
222
|
+
* The comparison is on the DATE PREFIX (`substr(valid_from, 1, 10)`) rather than on the raw text:
|
|
223
|
+
* `valid_from` is whatever ISO string the caller wrote (`2026-09-15` or
|
|
224
|
+
* `2026-09-15T09:30:00+08:00`), and a lexicographic range over the full strings would put a
|
|
225
|
+
* midnight timestamp on the wrong side of a day boundary. Day granularity is the unit the window
|
|
226
|
+
* parser speaks (`store/time_window.ts`), so `substr` is the matching one. No index can serve it;
|
|
227
|
+
* the leg is default-OFF and capped, and a full scan of `facts` for one enabled query is the
|
|
228
|
+
* accepted cost (see the leg's own comment).
|
|
229
|
+
*
|
|
230
|
+
* The `GLOB` is a SHAPE guard, not decoration: the prefix is compared LEXICOGRAPHICALLY, so a
|
|
231
|
+
* non-padded `2026-9-5` (which `Date.parse` accepts, so `normalizeEventTime` stores it) would sort
|
|
232
|
+
* AFTER `2026-09-30` and leak into an October window. Requiring the canonical zero-padded prefix
|
|
233
|
+
* excludes such a value from the leg entirely — the same "no usable event time" treatment, never a
|
|
234
|
+
* mis-binned one.
|
|
235
|
+
*
|
|
236
|
+
* `valid_from IS NOT NULL` is the P-11 discipline written into SQL: a fact with no event time is
|
|
237
|
+
* NOT a candidate here, and there is deliberately no `COALESCE(valid_from, created_at)` fallback —
|
|
238
|
+
* mixing a write clock into an event-time axis is exactly what this leg must not do.
|
|
239
|
+
*
|
|
240
|
+
* `EXISTS`, never a JOIN on `fact_sources` (the {@link FTS_SEARCH_SQL_FILTERED} rule): the cap is
|
|
241
|
+
* `LIMIT`, and a multi-source fact would eat cap slots with duplicate rows.
|
|
242
|
+
*/
|
|
243
|
+
windowRows(fromDate: string, toDate: string, category: string | undefined, cap: number, source?: string): number[];
|
|
144
244
|
findByContent(content: string): Record<string, unknown> | undefined;
|
|
145
245
|
/**
|
|
146
246
|
* ACTIVE facts whose entity/triple rows were produced by OLDER extraction rules.
|
|
@@ -271,8 +371,8 @@ export declare class FactsDao {
|
|
|
271
371
|
* index in order and filter per row (measured 9.8 ms for a category with 17 rows at 33k),
|
|
272
372
|
* i.e. it would trade a fast path for a slow one to save an index we do not need.
|
|
273
373
|
*/
|
|
274
|
-
page(status: string, category: string | undefined, limit: number, offset: number): Record<string, unknown>[];
|
|
275
|
-
countInStatus(status: string, category?: string): number;
|
|
374
|
+
page(status: string, category: string | undefined, limit: number, offset: number, source?: string): Record<string, unknown>[];
|
|
375
|
+
countInStatus(status: string, category?: string, source?: string): number;
|
|
276
376
|
/** Every ACTIVE fact (the report's `active`). */
|
|
277
377
|
countActive(): number;
|
|
278
378
|
/** ACTIVE and permanent — pinned facts are exempt from decay (the report's `pinned`). */
|
|
@@ -332,7 +432,7 @@ export declare class FactsDao {
|
|
|
332
432
|
* facts with a cap of 200 it silently made 32.8k of them unreachable. Recency is the honest
|
|
333
433
|
* prior for a memory store and `idx_facts_created(status, created_at DESC)` already serves it.
|
|
334
434
|
*/
|
|
335
|
-
activeHrrRows(category?: string, limit?: number): {
|
|
435
|
+
activeHrrRows(category?: string, limit?: number, source?: string): {
|
|
336
436
|
fact_id: number;
|
|
337
437
|
hrr_vector: Uint8Array;
|
|
338
438
|
}[];
|
|
@@ -344,12 +444,12 @@ export declare class FactsDao {
|
|
|
344
444
|
* which is the same information the HRR bundle encodes (its atoms ARE the entity names), so the
|
|
345
445
|
* narrowing removes the O(N) part rather than a meaningful part of the ranking.
|
|
346
446
|
*/
|
|
347
|
-
hrrRowsForFacts(ids: readonly number[], category?: string): {
|
|
447
|
+
hrrRowsForFacts(ids: readonly number[], category?: string, source?: string): {
|
|
348
448
|
fact_id: number;
|
|
349
449
|
hrr_vector: Uint8Array;
|
|
350
450
|
}[];
|
|
351
|
-
/** ACTIVE subset of `ids` in one category (`null` = any) — the semantic leg's filter. */
|
|
352
|
-
activeIdsIn(ids: readonly number[], category?: string): number[];
|
|
451
|
+
/** ACTIVE subset of `ids` in one category/source (`null` = any) — the semantic leg's filter. */
|
|
452
|
+
activeIdsIn(ids: readonly number[], category?: string, source?: string): number[];
|
|
353
453
|
/** `retrieval_count` + the "last used" clock (spec §2.5), batched. */
|
|
354
454
|
touchUsage(ids: readonly number[]): void;
|
|
355
455
|
/** Recall bonus not granted: only the settled trust moved. */
|
|
@@ -364,7 +464,22 @@ export declare class FactsDao {
|
|
|
364
464
|
* that JOIN from becoming the drive (see {@link FTS_SEARCH_SQL}, which carries the plans and
|
|
365
465
|
* measurements).
|
|
366
466
|
*/
|
|
367
|
-
ftsSearch(ftsQuery: string, category?: string, limit?: number): {
|
|
467
|
+
ftsSearch(ftsQuery: string, category?: string, limit?: number, source?: string): {
|
|
468
|
+
id: number;
|
|
469
|
+
rank: number;
|
|
470
|
+
}[];
|
|
471
|
+
/**
|
|
472
|
+
* The SHORT-QUERY fallback leg: rows whose `content` contains at least one of the terms as a
|
|
473
|
+
* contiguous substring, scored by how many it contains.
|
|
474
|
+
*
|
|
475
|
+
* Reached only when `buildFtsQuery` produced nothing (a 2-char CJK query — see
|
|
476
|
+
* `db/tokenizer.ts#likeSubstring` and `store/lexical.ts`), so it cannot move an indexed query.
|
|
477
|
+
* `rank` is the number of distinct terms the row contains, which is the same quantity
|
|
478
|
+
* `applyTermFloor` grades with; ties break on ascending id, matching `fuse`'s own tie rule, so a
|
|
479
|
+
* single-term query (all ranks 1) has a deterministic order. `ORDER BY rank` must read the SELECT
|
|
480
|
+
* alias, so the LIKE expressions are built once and bound twice — same fragment, same order.
|
|
481
|
+
*/
|
|
482
|
+
ftsSubstringSearch(terms: readonly string[], category?: string, limit?: number, source?: string): {
|
|
368
483
|
id: number;
|
|
369
484
|
rank: number;
|
|
370
485
|
}[];
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* P-07 validity invariants, as SQL an operator (or a test) can run against a database.
|
|
3
|
+
*
|
|
4
|
+
* The batch-1 fields are DISPLAY/AUDIT data — no retrieval leg reads them — so nothing in the read
|
|
5
|
+
* path would notice if a write stopped stamping them. These two statements are the check that
|
|
6
|
+
* cannot be satisfied vacuously: each has a planted-violation test in
|
|
7
|
+
* `test/fact_provenance.spec.ts` proving it reports a real inconsistency.
|
|
8
|
+
*
|
|
9
|
+
* ① The two SEMANTIC retirement paths (`update`'s supersede and a `true_positive` contradiction
|
|
10
|
+
* verdict) archive a fact because it stopped being true. Both must leave an end time, or the
|
|
11
|
+
* audit trail says "retired, cause known, end unknown". Lifecycle archival (`ttl` / `forgot` /
|
|
12
|
+
* `idle` / `manual`) is deliberately NOT in the set: those are retention decisions, not statements
|
|
13
|
+
* about when the fact stopped being true.
|
|
14
|
+
*
|
|
15
|
+
* ② A known end cannot precede a known start.
|
|
16
|
+
*/
|
|
17
|
+
import type { Db } from './port.js';
|
|
18
|
+
/** Facts retired by a semantic path (`replaced` / `contradiction`) without a recorded end time. */
|
|
19
|
+
export declare const RETIRED_WITHOUT_VALID_TO_SQL = "SELECT COUNT(*) AS n FROM facts\n WHERE status = 'archived' AND archive_reason IN ('replaced', 'contradiction') AND valid_to IS NULL";
|
|
20
|
+
/** Rows whose known end precedes their known start. */
|
|
21
|
+
export declare const VALID_TO_BEFORE_VALID_FROM_SQL = "SELECT COUNT(*) AS n FROM facts\n WHERE valid_from IS NOT NULL AND valid_to IS NOT NULL AND valid_to < valid_from";
|
|
22
|
+
/** Every P-07 invariant, evaluated. All-zero is the healthy state. */
|
|
23
|
+
export declare function validityInvariantViolations(db: Db): {
|
|
24
|
+
retired_without_valid_to: number;
|
|
25
|
+
valid_to_before_valid_from: number;
|
|
26
|
+
};
|