dsh-prime-memory 0.26.4 → 0.26.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/client.js +36785 -134
- package/dist/clients/drsc.d.ts +93 -0
- package/dist/clients/drsc.js +139 -0
- package/dist/contract.d.ts +74 -0
- package/dist/drs-anchor-backfill.d.ts +88 -0
- package/dist/drs-anchor-backfill.js +150 -0
- package/dist/graph/constraints.d.ts +3 -1
- package/dist/graph/constraints.js +4 -2
- package/dist/graph/preview.d.ts +92 -0
- package/dist/graph/preview.js +186 -0
- package/dist/graph/search.d.ts +22 -5
- package/dist/graph/search.js +69 -7
- package/dist/graph/types.d.ts +26 -1
- package/dist/graph/types.js +26 -1
- package/dist/hooks/capture.d.ts +4 -4
- package/dist/hooks/capture.js +34 -27
- package/dist/hooks/recall.js +8 -1
- package/dist/index.js +58 -20
- package/dist/pipeline/graph.d.ts +9 -0
- package/dist/pipeline/graph.js +29 -2
- package/dist/pipeline/runner.d.ts +17 -4
- package/dist/pipeline/runner.js +33 -6
- package/dist/stats.js +30 -1
- package/dist/store/embedding-source.d.ts +1 -3
- package/dist/store/embedding-source.js +6 -13
- package/dist/store/graph-store.d.ts +40 -3
- package/dist/store/graph-store.js +104 -8
- package/dist/store/l0-watermark.d.ts +46 -0
- package/dist/store/l0-watermark.js +93 -0
- package/dist/store/l0.d.ts +12 -42
- package/dist/store/l0.js +2 -208
- package/dist/store/l1.d.ts +12 -9
- package/dist/store/l1.js +37 -16
- package/dist/store/session-recalled.d.ts +9 -3
- package/dist/store/session-recalled.js +24 -5
- package/dist/store/sqlite.d.ts +3 -0
- package/dist/store/sqlite.js +7 -0
- package/dist/tools/index.d.ts +3 -2
- package/dist/tools/index.js +27 -10
- package/dist/types.d.ts +9 -0
- package/dsh.plugin.json +1 -1
- package/package.json +8 -3
|
@@ -19,8 +19,9 @@
|
|
|
19
19
|
*/
|
|
20
20
|
import { randomBytes } from 'node:crypto';
|
|
21
21
|
import { applyGraphProjection } from '../graph/apply.js';
|
|
22
|
-
import { searchGraphNodes } from '../graph/search.js';
|
|
23
|
-
import {
|
|
22
|
+
import { expandHitRecordIds, searchGraphNodes } from '../graph/search.js';
|
|
23
|
+
import { buildFullView, buildGraphPreview } from '../graph/preview.js';
|
|
24
|
+
import { GRAPH_JOB_BACKOFF_BASE_MS, GRAPH_JOB_BATCH, GRAPH_JOB_MAX_ATTEMPTS, GRAPH_JOB_TRANSIENT_MAX_ATTEMPTS, GRAPH_PRIORITY_BACKFILL, GRAPH_PROJECTOR_VERSION, GRAPH_VEC_SEED_CAP, } from '../graph/types.js';
|
|
24
25
|
import { normPersistence } from '../types.js';
|
|
25
26
|
import { isZeroVector, vecToBuffer } from './vec-utils.js';
|
|
26
27
|
const TAG = '[memory][graph]';
|
|
@@ -460,6 +461,98 @@ export class GraphStore {
|
|
|
460
461
|
: nodes;
|
|
461
462
|
return searchGraphNodes(filtered, edges, query, limit);
|
|
462
463
|
}
|
|
464
|
+
/**
|
|
465
|
+
* §D 第 3 路存储缝(回链记录 id 序;graphLane 直接消费):词法命中 ∪ 路内向量种子
|
|
466
|
+
* → 1-hop 边扩展 → 保序去重的 L1 记录 id 序列。
|
|
467
|
+
*
|
|
468
|
+
* - **向量种子是路内并集、不是新路**:embedding 由调用方传入(装配层惰性查
|
|
469
|
+
* l1.embedText;未就绪/失败传 undefined = 纯词法路,与旧 searchNodes+直连回链
|
|
470
|
+
* 逐位一致);并集不占 RRF 路数名额,"未接线 = 恰为 2 路"不变量不动;
|
|
471
|
+
* - **种子序**:词法命中在前、向量种子追加在后(RRF 用 rank,顺序即权重),
|
|
472
|
+
* 同节点去重;向量种子同样过 visible 族集与 active|disputed 状态闸(宁可漏不可串);
|
|
473
|
+
* - **1-hop 邻居不按族预筛**:来源记录的跨族由 graphLane 尾部记录级 family/scope
|
|
474
|
+
* 双闸兜底(与本路既有纪律同源);
|
|
475
|
+
* - 降级(db 未就绪/空图)返回空数组,不抛。
|
|
476
|
+
*
|
|
477
|
+
* 返回记录 id 而非 hits 是刻意的:回链/扩展属于读路径细节,路侧只需排名序列。
|
|
478
|
+
*/
|
|
479
|
+
searchLaneRecords(query, embedding, limit, families) {
|
|
480
|
+
if (!this.db)
|
|
481
|
+
return [];
|
|
482
|
+
const { nodes, edges } = this.loadGraph();
|
|
483
|
+
const visible = families && families.length > 0
|
|
484
|
+
? nodes.filter((n) => n.families.some((f) => families.includes(f)))
|
|
485
|
+
: nodes;
|
|
486
|
+
const lexical = searchGraphNodes(visible, edges, query, limit);
|
|
487
|
+
let hits = lexical;
|
|
488
|
+
if (embedding && embedding.length > 0) {
|
|
489
|
+
const visibleIds = new Set(visible.map((n) => n.id));
|
|
490
|
+
const seenNodes = new Set(lexical.map((h) => h.node.id));
|
|
491
|
+
const extra = [];
|
|
492
|
+
for (const v of this.searchNodesByVector(embedding, GRAPH_VEC_SEED_CAP)) {
|
|
493
|
+
if (seenNodes.has(v.node.id) || !visibleIds.has(v.node.id))
|
|
494
|
+
continue;
|
|
495
|
+
if (v.node.status !== 'active' && v.node.status !== 'disputed')
|
|
496
|
+
continue;
|
|
497
|
+
seenNodes.add(v.node.id);
|
|
498
|
+
extra.push({ node: v.node, score: v.score, matchedFields: ['name'], matchReason: '向量种子命中' });
|
|
499
|
+
}
|
|
500
|
+
if (extra.length > 0)
|
|
501
|
+
hits = [...lexical, ...extra];
|
|
502
|
+
}
|
|
503
|
+
return expandHitRecordIds(hits, nodes, edges);
|
|
504
|
+
}
|
|
505
|
+
/**
|
|
506
|
+
* 面板图谱预览存储缝(契约 `dsh-memory/graph-preview` 的数据源):
|
|
507
|
+
* loadGraph 一次 + job 计数(pending/running/failed 未定项与全量分母)传入纯函数。
|
|
508
|
+
* 降级(db 未就绪)返**全零结构**不抛——预览是展示面,永不拖垮端点。
|
|
509
|
+
* mode==='full' 时响应附 `full` 轻量块;**缺省时该键缺席**(可选键零漂移纪律,
|
|
510
|
+
* 不是 undefined 占位——消费方 `'full' in res` 与 JSON 形状双判据都干净)。
|
|
511
|
+
*/
|
|
512
|
+
buildPreview(seedId, mode) {
|
|
513
|
+
const zero = { seeds: [], stats: { nodes: 0, edges: 0, causalEdges: 0, pendingJobs: 0, jobsTotal: 0 } };
|
|
514
|
+
if (!this.db)
|
|
515
|
+
return mode === 'full' ? { ...zero, full: { nodes: [], edges: [], truncated: false } } : zero;
|
|
516
|
+
const { nodes, edges } = this.loadGraph();
|
|
517
|
+
let pendingJobs = 0;
|
|
518
|
+
let jobsTotal = 0;
|
|
519
|
+
let deadJobs = 0;
|
|
520
|
+
try {
|
|
521
|
+
pendingJobs = this.db
|
|
522
|
+
.prepare("SELECT COUNT(1) AS c FROM graph_projection_jobs WHERE status IN ('pending','running','failed')")
|
|
523
|
+
.get().c;
|
|
524
|
+
jobsTotal = this.db.prepare('SELECT COUNT(1) AS c FROM graph_projection_jobs').get().c;
|
|
525
|
+
deadJobs = this.db.prepare("SELECT COUNT(1) AS c FROM graph_projection_jobs WHERE status = 'dead'").get().c;
|
|
526
|
+
}
|
|
527
|
+
catch (err) {
|
|
528
|
+
this.logger?.warn(`${TAG} 预览 job 计数失败(计数回零): ${err instanceof Error ? err.message : String(err)}`);
|
|
529
|
+
}
|
|
530
|
+
const r = buildGraphPreview(nodes, edges, { seedId, pendingJobs, jobsTotal, deadJobs });
|
|
531
|
+
if (mode === 'full')
|
|
532
|
+
return { ...r, full: buildFullView(nodes, edges) };
|
|
533
|
+
return r;
|
|
534
|
+
}
|
|
535
|
+
/**
|
|
536
|
+
* 清理历史 dead 作业行(纯审计痕迹删除,不影响已重入队的记录)。
|
|
537
|
+
* olderThanMs: 只删早于 now-olderThanMs 的;缺省删全部 dead。
|
|
538
|
+
*/
|
|
539
|
+
cleanupDead(olderThanMs) {
|
|
540
|
+
if (!this.db)
|
|
541
|
+
return 0;
|
|
542
|
+
try {
|
|
543
|
+
const cutoff = olderThanMs != null && olderThanMs > 0
|
|
544
|
+
? new Date(Date.now() - olderThanMs).toISOString()
|
|
545
|
+
: undefined;
|
|
546
|
+
const result = cutoff
|
|
547
|
+
? this.db.prepare("DELETE FROM graph_projection_jobs WHERE status = 'dead' AND updated_time < ?").run(cutoff)
|
|
548
|
+
: this.db.prepare("DELETE FROM graph_projection_jobs WHERE status = 'dead'").run();
|
|
549
|
+
return Number(result.changes ?? 0);
|
|
550
|
+
}
|
|
551
|
+
catch (err) {
|
|
552
|
+
this.logger?.warn(`${TAG} cleanupDead 失败(忽略): ${err instanceof Error ? err.message : String(err)}`);
|
|
553
|
+
return 0;
|
|
554
|
+
}
|
|
555
|
+
}
|
|
463
556
|
// ============================
|
|
464
557
|
// 投影 job 队列
|
|
465
558
|
// ============================
|
|
@@ -568,7 +661,7 @@ export class GraphStore {
|
|
|
568
661
|
AND status IN ('pending', 'failed')
|
|
569
662
|
AND (next_attempt_at IS NULL OR next_attempt_at <= ?)
|
|
570
663
|
ORDER BY priority DESC, created_time ASC LIMIT 1`)
|
|
571
|
-
.get(GRAPH_PROJECTOR_VERSION,
|
|
664
|
+
.get(GRAPH_PROJECTOR_VERSION, GRAPH_JOB_TRANSIENT_MAX_ATTEMPTS, nowMs);
|
|
572
665
|
if (!row)
|
|
573
666
|
return null;
|
|
574
667
|
db.prepare(`UPDATE graph_projection_jobs SET status = 'running', attempts = attempts + 1, error = NULL, updated_time = ? WHERE job_id = ?`).run(nowIso, row.job_id);
|
|
@@ -645,24 +738,27 @@ export class GraphStore {
|
|
|
645
738
|
}
|
|
646
739
|
}
|
|
647
740
|
/**
|
|
648
|
-
* 投影失败收尾:attempts 已在 claim 时 +1
|
|
649
|
-
*
|
|
741
|
+
* 投影失败收尾:attempts 已在 claim 时 +1——根据错误分类选不同封顶:
|
|
742
|
+
* - 永久错误(校验/解析):attempts >= GRAPH_JOB_MAX_ATTEMPTS(3) → dead
|
|
743
|
+
* - 瞬态错误(aborted/timeout/网络):attempts >= GRAPH_JOB_TRANSIENT_MAX_ATTEMPTS(6) → dead
|
|
744
|
+
* dead 放掉 mapping(允许下次 backfill 重新入队);未封顶转 failed + 指数退避。
|
|
650
745
|
*/
|
|
651
|
-
fail(jobId, error) {
|
|
746
|
+
fail(jobId, error, opts) {
|
|
652
747
|
if (!this.db)
|
|
653
748
|
return;
|
|
654
749
|
try {
|
|
655
750
|
const db = this.db;
|
|
656
751
|
const nowMs = Date.now();
|
|
657
752
|
const nowIso = new Date(nowMs).toISOString();
|
|
753
|
+
const cap = opts?.transient ? GRAPH_JOB_TRANSIENT_MAX_ATTEMPTS : GRAPH_JOB_MAX_ATTEMPTS;
|
|
658
754
|
this.tx(() => {
|
|
659
755
|
const row = db.prepare('SELECT attempts FROM graph_projection_jobs WHERE job_id = ?').get(jobId);
|
|
660
756
|
if (!row)
|
|
661
757
|
return;
|
|
662
|
-
if (row.attempts >=
|
|
758
|
+
if (row.attempts >= cap) {
|
|
663
759
|
db.prepare(`UPDATE graph_projection_jobs SET status = 'dead', error = ?, updated_time = ? WHERE job_id = ?`).run(error.slice(0, 500), nowIso, jobId);
|
|
664
760
|
db.prepare('DELETE FROM graph_job_records WHERE job_id = ?').run(jobId);
|
|
665
|
-
this.logger?.warn(`${TAG}
|
|
761
|
+
this.logger?.warn(`${TAG} 投影任务达重试上限(${opts?.transient ? '瞬态' : '永久'}cap=${cap}),标记 dead(job=${jobId}): ${error.slice(0, 200)}`);
|
|
666
762
|
}
|
|
667
763
|
else {
|
|
668
764
|
const backoffMs = GRAPH_JOB_BACKOFF_BASE_MS * 2 ** (row.attempts - 1);
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* L0 水位表(T8.4 l0 写停,drsc-redact-memory-wiring task_16)。
|
|
3
|
+
*
|
|
4
|
+
* capture 停写 l0_conversations/l0_fts/l0_vec 三表后,§A 崩溃恢复的幂等判据
|
|
5
|
+
* (maxCapturedTurn / hasAnyL0Message)失去数据源——本小表接住:每会话一行,
|
|
6
|
+
* 只存 (max_turn, last_step),几十 KB 量级。正文改由 DRSC 投影供读(历史面),
|
|
7
|
+
* 水位只服务「重启不重放旧 turn」的自检,不再承载任何文本。
|
|
8
|
+
*
|
|
9
|
+
* 语义对齐被退役的 SQL 口径:
|
|
10
|
+
* - `maxTurn(sid)` ≡ 退役前 `MAX(turn)`;只前进不回退(重放/乱序登记无害);
|
|
11
|
+
* - `hasAny(sid, t)` ≡ 退役前 `t <= MAX(turn)` 的存在性判定——恢复窗口
|
|
12
|
+
* (foldRecoverableTurns 只产出 turn > 水位 的括号)内恒 false,行为与
|
|
13
|
+
* MAX 口径等价;写停后正文无「部分写残行」,逐 turn 点查的原始动机消失。
|
|
14
|
+
*
|
|
15
|
+
* 基线物化:写停生效前已存在的 l0_conversations 行由 {@link ensureBaseline}
|
|
16
|
+
* 一次性灌入(INSERT OR IGNORE,存量水位行不覆盖)——没有它,升级后的首次
|
|
17
|
+
* resume 会把水位当空、重放旧 turn 重复蒸馏。每次 init 幂等重跑(空表扫描
|
|
18
|
+
* 零成本;task_22 存量清退删表后同样安静)。
|
|
19
|
+
*
|
|
20
|
+
* 与 CostLedger/GraphStore 同款纪律:独立职责类,唯一耦合是共享 node:sqlite
|
|
21
|
+
* 连接;未 init / 降级时全部方法安全 no-op。
|
|
22
|
+
*/
|
|
23
|
+
import type { DatabaseSync } from 'node:sqlite';
|
|
24
|
+
import type { MemoryLogger } from '../types.js';
|
|
25
|
+
export declare const L0_WATERMARK_DDL = "\nCREATE TABLE IF NOT EXISTS l0_watermark (\n session_id TEXT PRIMARY KEY,\n max_turn INTEGER NOT NULL DEFAULT 0,\n last_step INTEGER,\n updated_at TEXT DEFAULT (datetime('now'))\n)\n";
|
|
26
|
+
export declare class L0WatermarkStore {
|
|
27
|
+
private db;
|
|
28
|
+
private logger;
|
|
29
|
+
init(db: DatabaseSync, logger?: MemoryLogger): void;
|
|
30
|
+
close(): void;
|
|
31
|
+
/**
|
|
32
|
+
* 基线物化:从现存 l0_conversations 把每会话最大 turn 灌进水位表。
|
|
33
|
+
* INSERT OR IGNORE = 已有水位行(活捕获已推进)一律不覆盖;旧行文与
|
|
34
|
+
* 水位并存期两侧取大者由 record 的单调语义兜底。
|
|
35
|
+
*/
|
|
36
|
+
ensureBaseline(): void;
|
|
37
|
+
/**
|
|
38
|
+
* 登记一个已捕获轮次(写停后 capture 的唯一落盘点)。单调:turn 更小/相等的
|
|
39
|
+
* 登记不回退水位(恢复对账与乱序事件都安全);step 取该轮已知最大 step。
|
|
40
|
+
*/
|
|
41
|
+
record(sessionId: string, turn: number, step?: number): void;
|
|
42
|
+
/** §A 水位线:该会话已捕获的最大 turn(空会话/未 init 返回 undefined)。 */
|
|
43
|
+
maxTurn(sessionId: string): number | undefined;
|
|
44
|
+
/** §A 幂等检查:该 (session, turn) 是否已捕获(≡ 退役前 hasAnyL0Message)。 */
|
|
45
|
+
hasAny(sessionId: string, turn: number): boolean;
|
|
46
|
+
}
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
export const L0_WATERMARK_DDL = `
|
|
2
|
+
CREATE TABLE IF NOT EXISTS l0_watermark (
|
|
3
|
+
session_id TEXT PRIMARY KEY,
|
|
4
|
+
max_turn INTEGER NOT NULL DEFAULT 0,
|
|
5
|
+
last_step INTEGER,
|
|
6
|
+
updated_at TEXT DEFAULT (datetime('now'))
|
|
7
|
+
)
|
|
8
|
+
`;
|
|
9
|
+
export class L0WatermarkStore {
|
|
10
|
+
db = null;
|
|
11
|
+
logger;
|
|
12
|
+
init(db, logger) {
|
|
13
|
+
this.db = db;
|
|
14
|
+
this.logger = logger;
|
|
15
|
+
db.exec(L0_WATERMARK_DDL);
|
|
16
|
+
this.ensureBaseline();
|
|
17
|
+
}
|
|
18
|
+
close() {
|
|
19
|
+
this.db = null;
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* 基线物化:从现存 l0_conversations 把每会话最大 turn 灌进水位表。
|
|
23
|
+
* INSERT OR IGNORE = 已有水位行(活捕获已推进)一律不覆盖;旧行文与
|
|
24
|
+
* 水位并存期两侧取大者由 record 的单调语义兜底。
|
|
25
|
+
*/
|
|
26
|
+
ensureBaseline() {
|
|
27
|
+
if (!this.db)
|
|
28
|
+
return;
|
|
29
|
+
try {
|
|
30
|
+
this.db.exec(`
|
|
31
|
+
INSERT OR IGNORE INTO l0_watermark (session_id, max_turn, last_step)
|
|
32
|
+
SELECT session_id, MAX(turn), NULL
|
|
33
|
+
FROM l0_conversations
|
|
34
|
+
WHERE turn IS NOT NULL
|
|
35
|
+
GROUP BY session_id
|
|
36
|
+
`);
|
|
37
|
+
}
|
|
38
|
+
catch (err) {
|
|
39
|
+
// l0_conversations 可能已被存量清退(task_22)删除:表缺 = 无基线可灌,安静跳过
|
|
40
|
+
this.logger?.debug?.(`[memory] L0 水位基线物化跳过: ${err instanceof Error ? err.message : String(err)}`);
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
44
|
+
* 登记一个已捕获轮次(写停后 capture 的唯一落盘点)。单调:turn 更小/相等的
|
|
45
|
+
* 登记不回退水位(恢复对账与乱序事件都安全);step 取该轮已知最大 step。
|
|
46
|
+
*/
|
|
47
|
+
record(sessionId, turn, step) {
|
|
48
|
+
if (!this.db || !sessionId || !Number.isFinite(turn))
|
|
49
|
+
return;
|
|
50
|
+
try {
|
|
51
|
+
this.db
|
|
52
|
+
.prepare(`INSERT INTO l0_watermark (session_id, max_turn, last_step, updated_at)
|
|
53
|
+
VALUES (?, ?, ?, datetime('now'))
|
|
54
|
+
ON CONFLICT(session_id) DO UPDATE SET
|
|
55
|
+
max_turn = MAX(l0_watermark.max_turn, excluded.max_turn),
|
|
56
|
+
last_step = CASE WHEN excluded.max_turn > l0_watermark.max_turn
|
|
57
|
+
THEN excluded.last_step ELSE l0_watermark.last_step END,
|
|
58
|
+
updated_at = excluded.updated_at`)
|
|
59
|
+
.run(sessionId, turn, typeof step === 'number' && Number.isFinite(step) ? step : null);
|
|
60
|
+
}
|
|
61
|
+
catch (err) {
|
|
62
|
+
this.logger?.warn(`[memory] L0 水位登记失败(幂等自检可能重放): ${err instanceof Error ? err.message : String(err)}`);
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
/** §A 水位线:该会话已捕获的最大 turn(空会话/未 init 返回 undefined)。 */
|
|
66
|
+
maxTurn(sessionId) {
|
|
67
|
+
if (!this.db || !sessionId)
|
|
68
|
+
return undefined;
|
|
69
|
+
try {
|
|
70
|
+
const row = this.db
|
|
71
|
+
.prepare('SELECT max_turn FROM l0_watermark WHERE session_id = ?')
|
|
72
|
+
.get(sessionId);
|
|
73
|
+
return typeof row?.max_turn === 'number' ? row.max_turn : undefined;
|
|
74
|
+
}
|
|
75
|
+
catch {
|
|
76
|
+
return undefined;
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
/** §A 幂等检查:该 (session, turn) 是否已捕获(≡ 退役前 hasAnyL0Message)。 */
|
|
80
|
+
hasAny(sessionId, turn) {
|
|
81
|
+
if (!this.db || !sessionId || !Number.isFinite(turn))
|
|
82
|
+
return false;
|
|
83
|
+
try {
|
|
84
|
+
const row = this.db
|
|
85
|
+
.prepare('SELECT 1 FROM l0_watermark WHERE session_id = ? AND max_turn >= ? LIMIT 1')
|
|
86
|
+
.get(sessionId, turn);
|
|
87
|
+
return row !== undefined;
|
|
88
|
+
}
|
|
89
|
+
catch {
|
|
90
|
+
return false;
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
}
|
package/dist/store/l0.d.ts
CHANGED
|
@@ -1,48 +1,18 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
1
|
+
/**
|
|
2
|
+
* L0 存量计数门面(T8.4 l0 写路径退役后)。
|
|
3
|
+
*
|
|
4
|
+
* capture 已停写 l0 正文三表与 JSONL 事实源(见 hooks/capture.ts 与
|
|
5
|
+
* store/l0-watermark.ts),历史读面(recentBySession / search)改经
|
|
6
|
+
* `clients/drsc.ts` 的 DrscClient(硬依赖,无本地回退)——本类只剩状态
|
|
7
|
+
* 面板的两个只读计数(今日捕获 / 按会话累计)。task_22 存量清退后表为
|
|
8
|
+
* 空壳,计数自然归零,读路径无需判空。
|
|
9
|
+
*/
|
|
10
|
+
import type { MemoryDb } from './sqlite.js';
|
|
4
11
|
export declare class L0Store {
|
|
5
12
|
private readonly db;
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
private readonly helper;
|
|
9
|
-
private embedSvc;
|
|
10
|
-
private readonly logger?;
|
|
11
|
-
constructor(dataDir: string, db: MemoryDb, embed?: EmbeddingService, logger?: MemoryLogger);
|
|
12
|
-
init(): Promise<void>;
|
|
13
|
-
/** 旧版 l0/*.jsonl 一次性导入检索库,成功后目录改名 l0.imported/。 */
|
|
14
|
-
private importLegacy;
|
|
15
|
-
append(sessionId: string, messages: ConversationMessage[]): Promise<void>;
|
|
16
|
-
/** 今日已捕获消息数(SQL 计数,不再读整文件)。 */
|
|
13
|
+
constructor(db: MemoryDb);
|
|
14
|
+
/** 今日已捕获消息数(SQL 计数,不再读整文件;写停后归零,面板口径保留)。 */
|
|
17
15
|
countToday(): Promise<number>;
|
|
18
16
|
/** 该会话累计已捕获消息数(session-stats 数据源;索引 COUNT)。 */
|
|
19
17
|
countBySession(sessionId: string): Promise<number>;
|
|
20
|
-
/** 该会话最近 n 条消息(时间升序;蒸馏背景参考用,按会话现查——ADR-0003)。 */
|
|
21
|
-
recentBySession(sessionId: string, limit: number): Promise<ConversationMessage[]>;
|
|
22
|
-
/** §A 水位线:该会话 L0 已落盘的最大 turn(空会话 undefined;委托 MemoryDb)。 */
|
|
23
|
-
maxCapturedTurn(sessionId: string): number | undefined;
|
|
24
|
-
/** §A 幂等检查:该 (session, turn) 是否已有任意 L0 行(委托 MemoryDb)。 */
|
|
25
|
-
hasAnyL0Message(sessionId: string, turn: number): boolean;
|
|
26
|
-
/** 检索:FTS + 向量 hybrid(RRF 融合),返回按相关性排序的消息。 */
|
|
27
|
-
search(query: string, limit: number): Promise<L0MessageRecord[]>;
|
|
28
|
-
/** 活切换嵌入源:同步换底层服务(嵌入源三态切换用)。 */
|
|
29
|
-
setEmbeddingService(svc: EmbeddingService): void;
|
|
30
|
-
/** 向量写入能力是否就绪。`reindex` 在未就绪时**静默短路**成 0/0/0
|
|
31
|
-
* (见本文件 `reindex` 首行),调用方必须自己问这里——否则"根本没跑"
|
|
32
|
-
* 会长得和"跑完了、零条待补"一模一样。 */
|
|
33
|
-
vectorsReady(): boolean;
|
|
34
|
-
/**
|
|
35
|
-
* 增量重嵌入(同 L1Store.reindex:只补缺失向量,零向量记 skipped 并入 skip 集,
|
|
36
|
-
* 不算失败、不阻塞同步标记——保证补齐判据收敛)。onProgress/shouldCancel
|
|
37
|
-
* 供活切换的进度展示与取消。
|
|
38
|
-
*/
|
|
39
|
-
reindex(opts?: {
|
|
40
|
-
onProgress?: (done: number, total: number) => void;
|
|
41
|
-
shouldCancel?: () => boolean;
|
|
42
|
-
}): Promise<{
|
|
43
|
-
written: number;
|
|
44
|
-
failed: number;
|
|
45
|
-
skipped: number;
|
|
46
|
-
cancelled?: boolean;
|
|
47
|
-
}>;
|
|
48
18
|
}
|
package/dist/store/l0.js
CHANGED
|
@@ -1,121 +1,9 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* L0 原始对话存储(双写架构):
|
|
3
|
-
* - conversations/YYYY-MM-DD.jsonl:追加式事实源;
|
|
4
|
-
* - MemoryDb(SQLite):主检索引擎——检索不再按天扫文件、现建内存索引。
|
|
5
|
-
*/
|
|
6
|
-
import { existsSync, promises as fs } from 'node:fs';
|
|
7
|
-
import * as path from 'node:path';
|
|
8
|
-
import { EmbedHelper, NoopEmbeddingService } from './embedding.js';
|
|
9
|
-
import { appendJsonl, dayKey, ensureDir, nowIso, readJsonl } from '../util/io.js';
|
|
10
|
-
import { rrfMerge } from './search-utils.js';
|
|
11
|
-
import { isZeroVector } from './sqlite.js';
|
|
12
|
-
/** 官方过度召回倍数(conversation-search:limit × 3)。 */
|
|
13
|
-
const CANDIDATE_MULTIPLIER = 3;
|
|
14
1
|
export class L0Store {
|
|
15
2
|
db;
|
|
16
|
-
|
|
17
|
-
legacyDir;
|
|
18
|
-
helper;
|
|
19
|
-
embedSvc;
|
|
20
|
-
logger;
|
|
21
|
-
constructor(dataDir, db, embed = new NoopEmbeddingService(), logger) {
|
|
3
|
+
constructor(db) {
|
|
22
4
|
this.db = db;
|
|
23
|
-
this.dir = path.join(dataDir, 'conversations');
|
|
24
|
-
this.legacyDir = path.join(dataDir, 'l0');
|
|
25
|
-
this.embedSvc = embed;
|
|
26
|
-
this.helper = new EmbedHelper(embed, logger);
|
|
27
|
-
this.logger = logger;
|
|
28
5
|
}
|
|
29
|
-
|
|
30
|
-
await ensureDir(this.dir);
|
|
31
|
-
await this.importLegacy();
|
|
32
|
-
}
|
|
33
|
-
/** 旧版 l0/*.jsonl 一次性导入检索库,成功后目录改名 l0.imported/。 */
|
|
34
|
-
async importLegacy() {
|
|
35
|
-
if (!existsSync(this.legacyDir))
|
|
36
|
-
return;
|
|
37
|
-
try {
|
|
38
|
-
const files = await fs.readdir(this.legacyDir).catch(() => []);
|
|
39
|
-
let imported = 0;
|
|
40
|
-
let total = 0;
|
|
41
|
-
for (const f of files.sort()) {
|
|
42
|
-
if (!f.endsWith('.jsonl'))
|
|
43
|
-
continue;
|
|
44
|
-
const records = await readJsonl(path.join(this.legacyDir, f));
|
|
45
|
-
// 最小有效性门(同 L1 的 importLegacy):坏行读取时丢弃,按 valid 数判迁移完成
|
|
46
|
-
const valid = records.filter((r) => r && typeof r.id === 'string' && r.content);
|
|
47
|
-
const badCount = records.length - valid.length;
|
|
48
|
-
if (badCount > 0) {
|
|
49
|
-
this.logger?.warn(`[memory] 旧版 L0 文件 ${f} 丢弃 ${badCount} 条坏行(缺 id/content)`);
|
|
50
|
-
}
|
|
51
|
-
if (valid.length > 0) {
|
|
52
|
-
total += valid.length;
|
|
53
|
-
if (this.db.upsertL0Batch(valid))
|
|
54
|
-
imported += valid.length;
|
|
55
|
-
}
|
|
56
|
-
}
|
|
57
|
-
// 只有全部批次入库成功(或目录为空)才改名,避免数据被改名带走
|
|
58
|
-
if (imported === total) {
|
|
59
|
-
const renamed = await fs
|
|
60
|
-
.rename(this.legacyDir, `${this.legacyDir}.imported`)
|
|
61
|
-
.then(() => true, () => false);
|
|
62
|
-
if (renamed) {
|
|
63
|
-
this.logger?.info(`[memory] 旧版 L0 数据已导入检索库 ${imported} 条(l0/ → l0.imported/)`);
|
|
64
|
-
}
|
|
65
|
-
else {
|
|
66
|
-
this.logger?.warn('[memory] 旧版 L0 导入完成但改名失败(l0.imported/ 已存在?),下次启动会重复导入(幂等,无害)');
|
|
67
|
-
}
|
|
68
|
-
}
|
|
69
|
-
else {
|
|
70
|
-
this.logger?.warn(`[memory] 旧版 L0 导入不完整(${imported}/${total}),保留原目录下次重试`);
|
|
71
|
-
}
|
|
72
|
-
}
|
|
73
|
-
catch (err) {
|
|
74
|
-
this.logger?.warn(`[memory] 旧版 L0 数据导入失败: ${err instanceof Error ? err.message : String(err)}`);
|
|
75
|
-
}
|
|
76
|
-
}
|
|
77
|
-
async append(sessionId, messages) {
|
|
78
|
-
if (messages.length === 0)
|
|
79
|
-
return;
|
|
80
|
-
const records = messages.map((m) => {
|
|
81
|
-
const rec = {
|
|
82
|
-
sessionId,
|
|
83
|
-
recordedAt: nowIso(),
|
|
84
|
-
id: m.id,
|
|
85
|
-
role: m.role,
|
|
86
|
-
content: m.content,
|
|
87
|
-
timestamp: m.timestamp,
|
|
88
|
-
};
|
|
89
|
-
// R7:锚点摊平到事实源与 DB 两处(事实源可跨进程重建,DB 供按坐标查询)。
|
|
90
|
-
// 无锚点时**不写键**——旧行与新增的无坐标消息保持同形。
|
|
91
|
-
if (m.anchor !== undefined) {
|
|
92
|
-
rec.turn = m.anchor.turn;
|
|
93
|
-
if (m.anchor.step !== undefined)
|
|
94
|
-
rec.step = m.anchor.step;
|
|
95
|
-
}
|
|
96
|
-
return rec;
|
|
97
|
-
});
|
|
98
|
-
// 事实源:按天追加
|
|
99
|
-
const byDay = new Map();
|
|
100
|
-
for (const r of records) {
|
|
101
|
-
const k = dayKey(r.timestamp);
|
|
102
|
-
const arr = byDay.get(k) ?? [];
|
|
103
|
-
arr.push(r);
|
|
104
|
-
byDay.set(k, arr);
|
|
105
|
-
}
|
|
106
|
-
for (const [day, list] of byDay) {
|
|
107
|
-
await appendJsonl(path.join(this.dir, `${day}.jsonl`), list);
|
|
108
|
-
}
|
|
109
|
-
// 检索引擎:DB + 向量(嵌入失败只跳过向量,不影响元数据/FTS,backfill 补齐)。
|
|
110
|
-
// 双写失败闭环:JSONL 事实源已先行追加,DB 缺行 = 这些消息检索不可见
|
|
111
|
-
// (conversation_search / 蒸馏背景参考都查不到)——升 error 并给自愈指引。
|
|
112
|
-
const vecs = await this.helper.batch(records.map((r) => r.content));
|
|
113
|
-
if (!this.db.upsertL0Batch(records, vecs)) {
|
|
114
|
-
this.logger?.error(`[memory] L0 检索库批量写入失败(${records.length} 条,JSONL 事实源完好),` +
|
|
115
|
-
'这些消息暂不可检索;可在设置页运行「重建记忆」修复');
|
|
116
|
-
}
|
|
117
|
-
}
|
|
118
|
-
/** 今日已捕获消息数(SQL 计数,不再读整文件)。 */
|
|
6
|
+
/** 今日已捕获消息数(SQL 计数,不再读整文件;写停后归零,面板口径保留)。 */
|
|
119
7
|
async countToday() {
|
|
120
8
|
const d = new Date();
|
|
121
9
|
return this.db.countL0Since(new Date(d.getFullYear(), d.getMonth(), d.getDate()).toISOString());
|
|
@@ -124,98 +12,4 @@ export class L0Store {
|
|
|
124
12
|
async countBySession(sessionId) {
|
|
125
13
|
return this.db.countL0BySession(sessionId);
|
|
126
14
|
}
|
|
127
|
-
/** 该会话最近 n 条消息(时间升序;蒸馏背景参考用,按会话现查——ADR-0003)。 */
|
|
128
|
-
async recentBySession(sessionId, limit) {
|
|
129
|
-
return this.db.recentL0BySession(sessionId, limit);
|
|
130
|
-
}
|
|
131
|
-
/** §A 水位线:该会话 L0 已落盘的最大 turn(空会话 undefined;委托 MemoryDb)。 */
|
|
132
|
-
maxCapturedTurn(sessionId) {
|
|
133
|
-
return this.db.maxCapturedTurn(sessionId);
|
|
134
|
-
}
|
|
135
|
-
/** §A 幂等检查:该 (session, turn) 是否已有任意 L0 行(委托 MemoryDb)。 */
|
|
136
|
-
hasAnyL0Message(sessionId, turn) {
|
|
137
|
-
return this.db.hasAnyL0Message(sessionId, turn);
|
|
138
|
-
}
|
|
139
|
-
/** 检索:FTS + 向量 hybrid(RRF 融合),返回按相关性排序的消息。 */
|
|
140
|
-
async search(query, limit) {
|
|
141
|
-
const caps = this.db.getCapabilities();
|
|
142
|
-
if (!caps.ftsSearch && !caps.vectorSearch)
|
|
143
|
-
return [];
|
|
144
|
-
const candidateK = limit * CANDIDATE_MULTIPLIER;
|
|
145
|
-
if (caps.vectorSearch && this.helper.vectorReady()) {
|
|
146
|
-
const [ftsRaw, vec] = await Promise.all([
|
|
147
|
-
Promise.resolve(this.db.searchL0Fts(query, candidateK)),
|
|
148
|
-
this.helper.query(query),
|
|
149
|
-
]);
|
|
150
|
-
const vecList = vec ? this.db.searchL0Vector(vec, candidateK) : [];
|
|
151
|
-
const merged = rrfMerge([ftsRaw, vecList], (h) => h.id);
|
|
152
|
-
return merged.map(({ rrfScore: _rrf, ...r }) => r).slice(0, limit);
|
|
153
|
-
}
|
|
154
|
-
return this.db.searchL0Fts(query, limit).map(({ score: _score, ...r }) => r);
|
|
155
|
-
}
|
|
156
|
-
/** 活切换嵌入源:同步换底层服务(嵌入源三态切换用)。 */
|
|
157
|
-
setEmbeddingService(svc) {
|
|
158
|
-
this.embedSvc = svc;
|
|
159
|
-
this.helper.setService(svc);
|
|
160
|
-
}
|
|
161
|
-
/** 向量写入能力是否就绪。`reindex` 在未就绪时**静默短路**成 0/0/0
|
|
162
|
-
* (见本文件 `reindex` 首行),调用方必须自己问这里——否则"根本没跑"
|
|
163
|
-
* 会长得和"跑完了、零条待补"一模一样。 */
|
|
164
|
-
vectorsReady() {
|
|
165
|
-
return this.helper.vectorReady();
|
|
166
|
-
}
|
|
167
|
-
/**
|
|
168
|
-
* 增量重嵌入(同 L1Store.reindex:只补缺失向量,零向量记 skipped 并入 skip 集,
|
|
169
|
-
* 不算失败、不阻塞同步标记——保证补齐判据收敛)。onProgress/shouldCancel
|
|
170
|
-
* 供活切换的进度展示与取消。
|
|
171
|
-
*/
|
|
172
|
-
async reindex(opts) {
|
|
173
|
-
if (!this.helper.vectorReady())
|
|
174
|
-
return { written: 0, failed: 0, skipped: 0 };
|
|
175
|
-
const items = this.db.getL0ForReindex(this.db.getVecSkipSet('l0'));
|
|
176
|
-
const total = items.length;
|
|
177
|
-
let done = 0;
|
|
178
|
-
let written = 0;
|
|
179
|
-
let failed = 0;
|
|
180
|
-
let skipped = 0;
|
|
181
|
-
let cancelled = false;
|
|
182
|
-
const skippedNow = [];
|
|
183
|
-
const CHUNK = 32;
|
|
184
|
-
for (let i = 0; i < items.length; i += CHUNK) {
|
|
185
|
-
if (opts?.shouldCancel?.()) {
|
|
186
|
-
cancelled = true;
|
|
187
|
-
break;
|
|
188
|
-
}
|
|
189
|
-
const chunk = items.slice(i, i + CHUNK);
|
|
190
|
-
let vecs;
|
|
191
|
-
try {
|
|
192
|
-
vecs = await this.embedSvc.embedBatch(chunk.map((c) => c.text));
|
|
193
|
-
}
|
|
194
|
-
catch {
|
|
195
|
-
failed += chunk.length;
|
|
196
|
-
done += chunk.length;
|
|
197
|
-
opts?.onProgress?.(done, total);
|
|
198
|
-
continue;
|
|
199
|
-
}
|
|
200
|
-
const pending = [];
|
|
201
|
-
chunk.forEach((c, j) => {
|
|
202
|
-
if (isZeroVector(vecs[j])) {
|
|
203
|
-
skipped++;
|
|
204
|
-
skippedNow.push(c.id);
|
|
205
|
-
return;
|
|
206
|
-
}
|
|
207
|
-
pending.push({ id: c.id, embedding: vecs[j] });
|
|
208
|
-
});
|
|
209
|
-
if (pending.length > 0) {
|
|
210
|
-
const ok = this.db.updateL0VecBatch(pending, '');
|
|
211
|
-
written += ok;
|
|
212
|
-
failed += pending.length - ok;
|
|
213
|
-
}
|
|
214
|
-
done += chunk.length;
|
|
215
|
-
opts?.onProgress?.(done, total);
|
|
216
|
-
}
|
|
217
|
-
if (skippedNow.length > 0)
|
|
218
|
-
this.db.addVecSkippedIds('l0', skippedNow);
|
|
219
|
-
return { written, failed, skipped, cancelled };
|
|
220
|
-
}
|
|
221
15
|
}
|
package/dist/store/l1.d.ts
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import type { L1Hit, MemoryFamily, MemoryLogger, MemoryRecord, RoomCount } from '../types.js';
|
|
2
|
-
import type { GraphNodeSearchResult } from '../graph/types.js';
|
|
3
2
|
import type { L1Receipt, ReceiptQuery } from './receipts.js';
|
|
4
3
|
import type { ConflictClaimGroup, ConflictPair, ConflictRejected, ConflictResolution, ConflictType } from './conflicts.js';
|
|
5
4
|
import { type SupersedeInfo } from './supersede.js';
|
|
@@ -9,11 +8,13 @@ import { type L1MetaLite, type MemoryDb } from './sqlite.js';
|
|
|
9
8
|
import type { L1FileSyncHook } from './memory-md-sync.js';
|
|
10
9
|
export type RecallStrategy = 'keyword' | 'embedding' | 'hybrid';
|
|
11
10
|
/**
|
|
12
|
-
* 图谱路提供者(§D 第 3 路)
|
|
11
|
+
* 图谱路提供者(§D 第 3 路):按查询返回图谱回链的 L1 记录 id 有序列
|
|
12
|
+
* (向量种子并集与 1-hop 边扩展在 provider 内完成,见 graph-store.searchLaneRecords)。
|
|
13
13
|
* 抽成注入式而非直接读 `db.graphStore`,是为了给 hybrid 融合留一个可替换的测试缝,
|
|
14
|
-
* 并让「未接线 = 恰为 2 路」成为默认行为(既有调用方零行为变化)
|
|
14
|
+
* 并让「未接线 = 恰为 2 路」成为默认行为(既有调用方零行为变化)。允许返回 Promise
|
|
15
|
+
* 是给向量种子的 embed 查询(装配层惰性取嵌入源);异常由 graphLane 吞并降级。
|
|
15
16
|
*/
|
|
16
|
-
export type GraphLaneProvider = (query: string, limit: number, family?: MemoryFamily) => readonly
|
|
17
|
+
export type GraphLaneProvider = (query: string, limit: number, family?: MemoryFamily) => readonly string[] | Promise<readonly string[]>;
|
|
17
18
|
export interface L1SearchOptions {
|
|
18
19
|
/** 按记忆类型精确过滤(后置过滤,官方做法)。 */
|
|
19
20
|
type?: string;
|
|
@@ -424,17 +425,19 @@ export declare class L1Store {
|
|
|
424
425
|
*/
|
|
425
426
|
private recencyLane;
|
|
426
427
|
/**
|
|
427
|
-
* §D 第 3 路(图谱路径,hybrid 专用)
|
|
428
|
-
*
|
|
428
|
+
* §D 第 3 路(图谱路径,hybrid 专用):provider 给出回链记录 id 序(路内向量种子
|
|
429
|
+
* 并集 + 1-hop 边扩展在 provider 内完成),本缝只做点查回形 + family/scope 双闸,
|
|
430
|
+
* 作为第 3 条**已排序**列表参与 RRF。
|
|
429
431
|
*
|
|
430
432
|
* 为什么值得:图谱是按实体/关系组织的**可重建派生投影**,能召回词法与向量
|
|
431
433
|
* 都命不中的记录(同义表述、关系可达)——这正是本路相对双路的增量。
|
|
432
434
|
*
|
|
433
435
|
* 三条边界:
|
|
434
436
|
* - **异常降级**:图谱是派生投影,不得因它失败而拖垮主检索 → 记 warn、返回空路,
|
|
435
|
-
* 融合退回双路(路数随之降为 2,分数回到既有量纲);
|
|
436
|
-
*
|
|
437
|
-
*
|
|
437
|
+
* 融合退回双路(路数随之降为 2,分数回到既有量纲);await 在 try 内——
|
|
438
|
+
* provider 的 Promise reject 与同步 throw 同路径降级;
|
|
439
|
+
* - **族隔离**:图谱节点已按族过滤(含向量种子),但 1-hop 邻居与来源记录可能跨族
|
|
440
|
+
* → 这里再按 `family` 过滤一次。宁可漏不可串(与档位隔离同源,§A 的 P0 关注点);
|
|
438
441
|
* - **墓碑边界**:图谱行可能回链到已被删除的 L1 记录 → 取不到就跳过,
|
|
439
442
|
* 不补空占位(占位会在 RRF 里凭空加分)。
|
|
440
443
|
*/
|