@modusensus/dsh-mneme 0.8.6 → 0.8.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +19 -148
- package/bin/cli.mjs +689 -603
- package/lib/api-standalone.js +57 -1
- package/lib/api.js +34 -11
- package/lib/client.js +349 -61
- package/lib/config.js +118 -7
- package/lib/content-hash.js +54 -0
- package/lib/continuity.js +220 -0
- package/lib/document-index.js +94 -0
- package/lib/document.js +93 -11
- package/lib/dream.js +138 -39
- package/lib/guide.js +18 -0
- package/lib/index.js +87 -12
- package/lib/inject.js +6 -2
- package/lib/lang.js +15 -4
- package/lib/maintenance.js +142 -0
- package/lib/mirror.js +147 -15
- package/lib/peak-hours.js +117 -0
- package/lib/recall-stats.js +52 -2
- package/lib/scope.js +34 -6
- package/lib/service.js +86 -7
- package/lib/settings.js +16 -2
- package/lib/store.js +318 -21
- package/lib/summarize.js +52 -113
- package/lib/tools.js +37 -11
- package/lib/write-admission.js +259 -0
- package/package.json +4 -4
- package/src/api-standalone.js +57 -1
- package/src/api.js +34 -11
- package/src/config.js +118 -7
- package/src/content-hash.js +54 -0
- package/src/continuity.js +220 -0
- package/src/document-index.js +94 -0
- package/src/document.js +93 -11
- package/src/dream.js +138 -39
- package/src/guide.js +18 -0
- package/src/index.js +87 -12
- package/src/inject.js +6 -2
- package/src/lang.js +15 -4
- package/src/maintenance.js +142 -0
- package/src/mirror.js +147 -15
- package/src/peak-hours.js +117 -0
- package/src/recall-stats.js +52 -2
- package/src/scope.js +34 -6
- package/src/service.js +86 -7
- package/src/settings.js +16 -2
- package/src/store.js +318 -21
- package/src/summarize.js +52 -113
- package/src/tools.js +37 -11
- package/src/write-admission.js +259 -0
- package/test/api.test.js +21 -3
- package/test/client.test.js +103 -6
- package/test/content-hash.test.js +204 -0
- package/test/continuity.test.js +333 -0
- package/test/document-dir.test.js +233 -0
- package/test/document.test.js +151 -3
- package/test/dream-failure-backoff.test.js +173 -0
- package/test/dream-peak-hours.test.js +194 -0
- package/test/inject-parent-gate.test.js +106 -0
- package/test/inject-pools.test.js +8 -3
- package/test/llm-audit.test.js +42 -0
- package/test/maintenance.test.js +391 -0
- package/test/mirror-types.test.js +208 -0
- package/test/reasoning-effort.test.js +6 -0
- package/test/recall-evals.test.js +18 -0
- package/test/recall-stats.test.js +66 -0
- package/test/summarize-reasoning-effort.test.js +189 -0
- package/test/write-admission.test.js +345 -0
package/lib/store.js
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import { DatabaseSync } from "node:sqlite";
|
|
2
2
|
import { randomUUID } from "node:crypto";
|
|
3
|
+
import { statSync } from "node:fs";
|
|
4
|
+
import { contentHashOf } from "./content-hash.js";
|
|
3
5
|
|
|
4
6
|
const SCHEMA = `
|
|
5
7
|
CREATE TABLE IF NOT EXISTS memories (
|
|
@@ -13,6 +15,7 @@ CREATE TABLE IF NOT EXISTS memories (
|
|
|
13
15
|
archived INTEGER NOT NULL DEFAULT 0,
|
|
14
16
|
source TEXT,
|
|
15
17
|
content_history TEXT,
|
|
18
|
+
content_hash TEXT,
|
|
16
19
|
embedding TEXT,
|
|
17
20
|
epistemic_status TEXT NOT NULL DEFAULT 'subjective',
|
|
18
21
|
last_accessed_at TEXT,
|
|
@@ -192,7 +195,8 @@ CREATE TABLE IF NOT EXISTS llm_audit_logs (
|
|
|
192
195
|
status TEXT NOT NULL, -- success | error | skipped
|
|
193
196
|
error_message TEXT,
|
|
194
197
|
related_memory_ids TEXT, -- JSON: ids the call operated on
|
|
195
|
-
metadata TEXT
|
|
198
|
+
metadata TEXT, -- JSON: free-form extras
|
|
199
|
+
session_key TEXT -- #254 写入准入的会话键(LLM 调用行恒 NULL)
|
|
196
200
|
);
|
|
197
201
|
CREATE INDEX IF NOT EXISTS idx_llm_audit_timestamp ON llm_audit_logs(timestamp);
|
|
198
202
|
CREATE INDEX IF NOT EXISTS idx_llm_audit_source ON llm_audit_logs(trigger_source);
|
|
@@ -272,6 +276,28 @@ CREATE TABLE IF NOT EXISTS mirror_state (
|
|
|
272
276
|
applied_generation INTEGER NOT NULL DEFAULT 0 CHECK (applied_generation >= 0 AND applied_generation <= 9007199254740991 AND applied_generation = CAST(applied_generation AS INTEGER)), -- 已成功应用的轮次
|
|
273
277
|
type_status TEXT -- JSON: 逐 type 状态 {type: {dirty, applied_gen, last_error}}
|
|
274
278
|
);
|
|
279
|
+
|
|
280
|
+
-- #249 N3(压缩边缘双落点):连续性提案。压缩边缘只落**提案**行,不进 memories
|
|
281
|
+
-- ——边缘产出若直接进记忆库,一个长会话就会攒出第 N 条同主题条目,正是 #275 记的
|
|
282
|
+
-- 失败形态;转正通道与 #254 的二次确认共用一套,阶段二才打开,本批只写 pending。
|
|
283
|
+
-- 唯一键 (session_id, kind) 就是形态约定里的「同一会话同一类只留一条」:再次触发
|
|
284
|
+
-- 是刷新同一行,不是新增一行。status 为转正通道预留(pending → promoted/discarded):
|
|
285
|
+
-- 「实际触发率」= 提案行数、「采纳率」= promoted/总数,都能就地统计(#249 §8)。
|
|
286
|
+
-- 时间戳是 ISO(与 memories 同款),created_at 在刷新时不动,updated_at 记最近边缘。
|
|
287
|
+
CREATE TABLE IF NOT EXISTS continuity_proposals (
|
|
288
|
+
id TEXT PRIMARY KEY,
|
|
289
|
+
session_id TEXT NOT NULL,
|
|
290
|
+
kind TEXT NOT NULL,
|
|
291
|
+
current_work TEXT,
|
|
292
|
+
next_step TEXT,
|
|
293
|
+
open_questions TEXT,
|
|
294
|
+
status TEXT NOT NULL DEFAULT 'pending',
|
|
295
|
+
edge_seq INTEGER, -- 触发本次刷新的压缩事件 seq(证据/复现用)
|
|
296
|
+
created_at TEXT NOT NULL,
|
|
297
|
+
updated_at TEXT NOT NULL
|
|
298
|
+
);
|
|
299
|
+
CREATE UNIQUE INDEX IF NOT EXISTS idx_continuity_session_kind ON continuity_proposals(session_id, kind);
|
|
300
|
+
CREATE INDEX IF NOT EXISTS idx_continuity_status ON continuity_proposals(status, updated_at);
|
|
275
301
|
`;
|
|
276
302
|
|
|
277
303
|
// Exported for API-layer type validation (standalone API POST /memories and
|
|
@@ -282,6 +308,12 @@ export const TYPES = new Set(["preference", "project", "decision", "history", "s
|
|
|
282
308
|
// (registerDocument)——saveWithDedupe/updateMemory 另有守卫拒绝旁路铸造。
|
|
283
309
|
"document"]);
|
|
284
310
|
|
|
311
|
+
// #249 N3:连续性提案的 pending 队列上限。「队列满则弃新」是形态约定的一半——满时丢
|
|
312
|
+
// 掉本次触发,**不是**淘汰旧行:旧行是别的会话还没转正的活状态,用"更近的边缘"把它挤
|
|
313
|
+
// 掉,等于让长会话的噪音吃掉短会话的真实工作状态。这个数是存储侧策略,与抽取/注入的
|
|
314
|
+
// 截断上限(continuity.js)是两件事。
|
|
315
|
+
const MAX_CONTINUITY_PENDING = 200;
|
|
316
|
+
|
|
285
317
|
// Epistemic status: what kind of evidence a memory rests on. Defaults to
|
|
286
318
|
// 'subjective' so legacy rows (and rows without any signal) stay compatible.
|
|
287
319
|
const EPISTEMIC_STATUSES = new Set(["observation", "subjective", "inferred"]);
|
|
@@ -405,6 +437,7 @@ function toRow(row) {
|
|
|
405
437
|
type: row.type,
|
|
406
438
|
title: row.title,
|
|
407
439
|
content: row.content,
|
|
440
|
+
content_hash: row.content_hash ?? undefined,
|
|
408
441
|
tags: parseTags(row.tags),
|
|
409
442
|
importance: row.importance,
|
|
410
443
|
forgotten: row.forgotten === 1,
|
|
@@ -607,7 +640,9 @@ function toLlmAudit(row) {
|
|
|
607
640
|
status: row.status,
|
|
608
641
|
error_message: row.error_message ?? undefined,
|
|
609
642
|
related_memory_ids: parseJsonArray(row.related_memory_ids),
|
|
610
|
-
metadata
|
|
643
|
+
metadata,
|
|
644
|
+
// #254 写入准入的会话键;LLM 调用行恒 NULL。
|
|
645
|
+
session_key: row.session_key ?? undefined
|
|
611
646
|
};
|
|
612
647
|
}
|
|
613
648
|
|
|
@@ -711,6 +746,34 @@ export function createStore(path) {
|
|
|
711
746
|
addColumn("memories", "workspace_scope_source", "ALTER TABLE memories ADD COLUMN workspace_scope_source TEXT");
|
|
712
747
|
addColumn("memories", "scope_decided_at", "ALTER TABLE memories ADD COLUMN scope_decided_at TEXT");
|
|
713
748
|
|
|
749
|
+
// #254 计量信号:内容归一化哈希(口径与动机写在 content-hash.js 的文件头)。派生
|
|
750
|
+
// 列,不参与任何判定——只给「同一内容又被写了一次」当等值锚。索引建在加列之后:
|
|
751
|
+
// 老库打开时 SCHEMA 的 CREATE TABLE 对既有表不生效,列还不存在,把索引写进 SCHEMA
|
|
752
|
+
// 会直接报 no such column(与下方 llm_audit_logs.session_key 同理)。
|
|
753
|
+
addColumn("memories", "content_hash", "ALTER TABLE memories ADD COLUMN content_hash TEXT");
|
|
754
|
+
// 列序是 content_hash 打头:哈希几乎唯一,等值 seek 就已收窄到候选行,type 只当同
|
|
755
|
+
// 一次 seek 里的第二列过滤;反过来(type 打头)下面的存量回填就得全表扫——每次打开
|
|
756
|
+
// 都要把全部正文读一遍。
|
|
757
|
+
db.exec("CREATE INDEX IF NOT EXISTS idx_memories_content_hash ON memories(content_hash, type)");
|
|
758
|
+
// 存量回填:老库(含归档区)的行都没有值,而「归档行纳入去重候选集」这条口径正是
|
|
759
|
+
// 要靠既有归档行出数据。只算 NULL 行、幂等;改口径不是补 NULL 能修好的,要整列重算
|
|
760
|
+
// (见 content-hash.js 的文件头)。空标题空正文的行算不出锚,跳过不写——否则每次
|
|
761
|
+
// 打开都要为这些行再跑一遍 UPDATE。
|
|
762
|
+
{
|
|
763
|
+
const updates = db
|
|
764
|
+
.prepare("SELECT id, title, content FROM memories WHERE content_hash IS NULL")
|
|
765
|
+
.all()
|
|
766
|
+
.map((row) => [contentHashOf(row), row.id])
|
|
767
|
+
.filter(([hash]) => hash);
|
|
768
|
+
if (updates.length) {
|
|
769
|
+
const stmt = db.prepare("UPDATE memories SET content_hash = ? WHERE id = ? AND content_hash IS NULL");
|
|
770
|
+
// 一个事务包住整批:逐条自动提交是每行一次 WAL 提交,几千行的存量库上纯属浪费。
|
|
771
|
+
runAtomically(() => {
|
|
772
|
+
for (const [hash, id] of updates) stmt.run(hash, id);
|
|
773
|
+
});
|
|
774
|
+
}
|
|
775
|
+
}
|
|
776
|
+
|
|
714
777
|
// Legacy dream_runs without policy_epoch → backfill with the default epoch.
|
|
715
778
|
addColumn("dream_runs", "policy_epoch", "ALTER TABLE dream_runs ADD COLUMN policy_epoch INTEGER NOT NULL DEFAULT 0");
|
|
716
779
|
addColumn("dream_runs", "run_type", "ALTER TABLE dream_runs ADD COLUMN run_type TEXT NOT NULL DEFAULT 'auto'");
|
|
@@ -724,6 +787,13 @@ export function createStore(path) {
|
|
|
724
787
|
addColumn("mirror_state", "applied_generation", "ALTER TABLE mirror_state ADD COLUMN applied_generation INTEGER NOT NULL DEFAULT 0");
|
|
725
788
|
addColumn("mirror_state", "type_status", "ALTER TABLE mirror_state ADD COLUMN type_status TEXT");
|
|
726
789
|
|
|
790
|
+
// #254 写入准入(第一阶段只计量):准入决策行按会话聚合——「会话内新建了几行」
|
|
791
|
+
// 与「同话题重复间隔」都靠它算出来,所以会话键必须是可等值查询的列,而不是塞在
|
|
792
|
+
// metadata 里做 JSON 匹配。列可空且不带 DEFAULT(存量行零重写)。索引建在加列
|
|
793
|
+
// 之后:老库打开时列还不存在(SCHEMA 的 CREATE TABLE 只对新建库生效)。
|
|
794
|
+
addColumn("llm_audit_logs", "session_key", "ALTER TABLE llm_audit_logs ADD COLUMN session_key TEXT");
|
|
795
|
+
db.exec("CREATE INDEX IF NOT EXISTS idx_llm_audit_session ON llm_audit_logs(session_key);");
|
|
796
|
+
|
|
727
797
|
// Audit peer F: a legacy DB may hold a non-integer generation/applied_generation
|
|
728
798
|
// (pre-v0.3.9 the JS gate truncated with Math.trunc and SQLite's CHECK only
|
|
729
799
|
// enforced >= 0). Such a value is ambiguous — it cannot map to a real applied
|
|
@@ -942,6 +1012,8 @@ export function createStore(path) {
|
|
|
942
1012
|
const importance = Number.isInteger(memory.importance) ? memory.importance : 3;
|
|
943
1013
|
const evidence = Array.isArray(memory.evidence) ? JSON.stringify(memory.evidence) : null;
|
|
944
1014
|
const docPath = typeof memory.doc_path === "string" && memory.doc_path.trim() ? memory.doc_path : null;
|
|
1015
|
+
// #254 计量锚:由本行的 title/content 派生,调用方传什么都以这里算出的为准。
|
|
1016
|
+
const contentHash = contentHashOf(memory);
|
|
945
1017
|
const embedding = Array.isArray(memory.embedding) && memory.embedding.length
|
|
946
1018
|
? JSON.stringify(memory.embedding)
|
|
947
1019
|
: null;
|
|
@@ -952,8 +1024,8 @@ export function createStore(path) {
|
|
|
952
1024
|
: inferEpistemicStatus(memory);
|
|
953
1025
|
runAtomically(() => {
|
|
954
1026
|
db.prepare(
|
|
955
|
-
`INSERT INTO memories (id, type, title, content, tags, importance, forgotten, archived, source, content_history, quality_score, embedding, epistemic_status, agent_scope, workspace_scope, agent_scope_source, workspace_scope_source, scope_decided_at, sensitivity, occurred_at, evidence, doc_path, created_at, updated_at)
|
|
956
|
-
VALUES (?, ?, ?, ?, ?, ?, 0, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
|
1027
|
+
`INSERT INTO memories (id, type, title, content, tags, importance, forgotten, archived, source, content_history, quality_score, embedding, epistemic_status, agent_scope, workspace_scope, agent_scope_source, workspace_scope_source, scope_decided_at, sensitivity, occurred_at, evidence, doc_path, content_hash, created_at, updated_at)
|
|
1028
|
+
VALUES (?, ?, ?, ?, ?, ?, 0, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
|
957
1029
|
).run(
|
|
958
1030
|
id,
|
|
959
1031
|
type,
|
|
@@ -976,6 +1048,7 @@ export function createStore(path) {
|
|
|
976
1048
|
normalizeOccurredAt(memory.occurred_at),
|
|
977
1049
|
evidence,
|
|
978
1050
|
docPath,
|
|
1051
|
+
contentHash,
|
|
979
1052
|
now,
|
|
980
1053
|
now
|
|
981
1054
|
);
|
|
@@ -988,6 +1061,41 @@ export function createStore(path) {
|
|
|
988
1061
|
return getById(id);
|
|
989
1062
|
}
|
|
990
1063
|
|
|
1064
|
+
/**
|
|
1065
|
+
* #254 计量:与本次写入「归一化内容哈希」相同的既有行(去重候选集)。
|
|
1066
|
+
* 结构化锚先收窄成本:按哈希等值 seek(走 idx_memories_content_hash,哈希几乎唯
|
|
1067
|
+
* 一,落在候选集上的行只有几条),再按 type 与 scope 三维过滤——不是全表扫、也不
|
|
1068
|
+
* 逐行比哈希。scope 用 IS 比(NULL 安全):未标注行互相匹配、与已标注行不匹配,
|
|
1069
|
+
* 与 service.js 去重键(scopeKeyOf)同口径;入参先按 store 自己的 normalizeScopeText
|
|
1070
|
+
* 归一,免得调用方传 " foo " 就漏配。
|
|
1071
|
+
* 归档行与已遗忘行都在集内:#275 的分界是「出口止体积、不止重复」——被质量闸归档
|
|
1072
|
+
* 的同一个事实必须仍能判成重复,否则同样的内容再写一次又是一条新行(#254 拍板:
|
|
1073
|
+
* 归档行纳入去重候选集)。
|
|
1074
|
+
* 排序即取舍:活跃行排在归档/遗忘行之前,再按 updated_at 倒序。LIMIT 先于调用方的
|
|
1075
|
+
* 「活区优先」判断执行,而归档动作本身会顶 updated_at——纯按时间倒序时,同键命中一
|
|
1076
|
+
* 旦超过窗口宽度,活跃行就被归档行挤出候选集,调用方只能看到归档命中,把「活跃重复」
|
|
1077
|
+
* 误报成「归档重复」,恰好把这条信号要分流的两类弄反。
|
|
1078
|
+
* @returns {Array<{id: string, archived: boolean, forgotten: boolean}>} 活跃行优先,其后按最近写入
|
|
1079
|
+
*/
|
|
1080
|
+
function findContentHashMatches({ type, hash, agent_scope: agentScope, workspace_scope: workspaceScope, sensitivity, limit = 10 } = {}) {
|
|
1081
|
+
if (!hash || !type) return [];
|
|
1082
|
+
const lim = Number.isInteger(limit) && limit > 0 ? Math.min(limit, 50) : 10;
|
|
1083
|
+
const rows = db.prepare(
|
|
1084
|
+
`SELECT id, archived, forgotten FROM memories
|
|
1085
|
+
WHERE type = ? AND content_hash = ?
|
|
1086
|
+
AND agent_scope IS ? AND workspace_scope IS ? AND sensitivity IS ?
|
|
1087
|
+
ORDER BY archived ASC, forgotten ASC, updated_at DESC, id LIMIT ?`
|
|
1088
|
+
).all(
|
|
1089
|
+
type,
|
|
1090
|
+
hash,
|
|
1091
|
+
normalizeScopeText(agentScope),
|
|
1092
|
+
normalizeScopeText(workspaceScope),
|
|
1093
|
+
normalizeScopeText(sensitivity),
|
|
1094
|
+
lim
|
|
1095
|
+
);
|
|
1096
|
+
return rows.map((row) => ({ id: row.id, archived: row.archived === 1, forgotten: row.forgotten === 1 }));
|
|
1097
|
+
}
|
|
1098
|
+
|
|
991
1099
|
function update(id, patch) {
|
|
992
1100
|
const existing = getById(id);
|
|
993
1101
|
if (!existing) throw new Error(`memory not found: ${id}`);
|
|
@@ -1034,13 +1142,18 @@ export function createStore(path) {
|
|
|
1034
1142
|
const nextDecidedAt = patch.scope_decided_at !== undefined
|
|
1035
1143
|
? normalizeOccurredAt(patch.scope_decided_at)
|
|
1036
1144
|
: (existing.scope_decided_at ?? null);
|
|
1145
|
+
// #254:标题或正文变了,锚跟着重算——它是当前内容的派生物,留旧值等于让「这行
|
|
1146
|
+
// 现在装的什么」和标记对不上。
|
|
1147
|
+
const nextTitle = patch.title ?? existing.title;
|
|
1148
|
+
const nextContent = patch.content ?? existing.content;
|
|
1149
|
+
const contentHash = contentHashOf({ title: nextTitle, content: nextContent });
|
|
1037
1150
|
runAtomically(() => {
|
|
1038
1151
|
db.prepare(
|
|
1039
|
-
`UPDATE memories SET type=?, title=?, content=?, tags=?, importance=?, source=?, content_history=?, quality_score=?, embedding=?, epistemic_status=?, agent_scope=?, workspace_scope=?, agent_scope_source=?, workspace_scope_source=?, scope_decided_at=?, evidence=?, updated_at=? WHERE id=?`
|
|
1152
|
+
`UPDATE memories SET type=?, title=?, content=?, tags=?, importance=?, source=?, content_history=?, quality_score=?, embedding=?, epistemic_status=?, agent_scope=?, workspace_scope=?, agent_scope_source=?, workspace_scope_source=?, scope_decided_at=?, evidence=?, content_hash=?, updated_at=? WHERE id=?`
|
|
1040
1153
|
).run(
|
|
1041
1154
|
type,
|
|
1042
|
-
|
|
1043
|
-
|
|
1155
|
+
nextTitle,
|
|
1156
|
+
nextContent,
|
|
1044
1157
|
JSON.stringify(patch.tags ?? existing.tags),
|
|
1045
1158
|
Number.isInteger(patch.importance) ? patch.importance : existing.importance,
|
|
1046
1159
|
patch.source !== undefined ? patch.source : (existing.source ?? null),
|
|
@@ -1054,6 +1167,7 @@ export function createStore(path) {
|
|
|
1054
1167
|
nextWorkspaceSource,
|
|
1055
1168
|
nextDecidedAt,
|
|
1056
1169
|
evidence,
|
|
1170
|
+
contentHash,
|
|
1057
1171
|
now,
|
|
1058
1172
|
id
|
|
1059
1173
|
);
|
|
@@ -1115,15 +1229,19 @@ export function createStore(path) {
|
|
|
1115
1229
|
// dirty == false — recoverMirror sees no debt and the mirror stays stale.
|
|
1116
1230
|
// Wrapping both in one transaction means a CAS miss rolls back cleanly too
|
|
1117
1231
|
// (no write, no generation bump).
|
|
1232
|
+
// 同 update:#254 的锚随标题/正文重算(CAS 路径也改这两列)。
|
|
1233
|
+
const nextTitle = patch.title ?? existing.title;
|
|
1234
|
+
const nextContent = patch.content ?? existing.content;
|
|
1235
|
+
const contentHash = contentHashOf({ title: nextTitle, content: nextContent });
|
|
1118
1236
|
let applied = false;
|
|
1119
1237
|
runAtomically(() => {
|
|
1120
1238
|
const result = db.prepare(
|
|
1121
|
-
`UPDATE memories SET type=?, title=?, content=?, tags=?, importance=?, source=?, content_history=?, quality_score=?, embedding=?, epistemic_status=?, updated_at=?
|
|
1239
|
+
`UPDATE memories SET type=?, title=?, content=?, tags=?, importance=?, source=?, content_history=?, quality_score=?, embedding=?, epistemic_status=?, content_hash=?, updated_at=?
|
|
1122
1240
|
WHERE id=? AND updated_at=?`
|
|
1123
1241
|
).run(
|
|
1124
1242
|
type,
|
|
1125
|
-
|
|
1126
|
-
|
|
1243
|
+
nextTitle,
|
|
1244
|
+
nextContent,
|
|
1127
1245
|
JSON.stringify(patch.tags ?? existing.tags),
|
|
1128
1246
|
Number.isInteger(patch.importance) ? patch.importance : existing.importance,
|
|
1129
1247
|
patch.source !== undefined ? patch.source : (existing.source ?? null),
|
|
@@ -1131,6 +1249,7 @@ export function createStore(path) {
|
|
|
1131
1249
|
qualityScore,
|
|
1132
1250
|
embedding,
|
|
1133
1251
|
epistemicStatus,
|
|
1252
|
+
contentHash,
|
|
1134
1253
|
now,
|
|
1135
1254
|
id,
|
|
1136
1255
|
expectedUpdatedAt
|
|
@@ -1182,15 +1301,15 @@ export function createStore(path) {
|
|
|
1182
1301
|
function demoteToSummary(id, summary, { minRefTimeMs } = {}) {
|
|
1183
1302
|
let changed = false;
|
|
1184
1303
|
runAtomically(() => {
|
|
1185
|
-
const row = db.prepare("SELECT last_accessed_at, content, _full_content FROM memories WHERE id = ?").get(id);
|
|
1304
|
+
const row = db.prepare("SELECT title, last_accessed_at, content, _full_content FROM memories WHERE id = ?").get(id);
|
|
1186
1305
|
if (!row || row._full_content) return;
|
|
1187
1306
|
if (minRefTimeMs !== undefined && row.last_accessed_at) {
|
|
1188
1307
|
const lastMs = Date.parse(row.last_accessed_at);
|
|
1189
1308
|
if (lastMs >= minRefTimeMs) return; // touched after snapshot — still hot
|
|
1190
1309
|
}
|
|
1191
1310
|
db.prepare(
|
|
1192
|
-
"UPDATE memories SET content = ?, _full_content = ?, updated_at = ? WHERE id = ?"
|
|
1193
|
-
).run(summary, row.content, nowIso(), id);
|
|
1311
|
+
"UPDATE memories SET content = ?, _full_content = ?, content_hash = ?, updated_at = ? WHERE id = ?"
|
|
1312
|
+
).run(summary, row.content, contentHashOf({ title: row.title, content: summary }), nowIso(), id);
|
|
1194
1313
|
incrementGeneration();
|
|
1195
1314
|
changed = true;
|
|
1196
1315
|
});
|
|
@@ -1201,11 +1320,11 @@ export function createStore(path) {
|
|
|
1201
1320
|
function restoreContent(id) {
|
|
1202
1321
|
let changed = false;
|
|
1203
1322
|
runAtomically(() => {
|
|
1204
|
-
const row = db.prepare("SELECT content, _full_content FROM memories WHERE id = ?").get(id);
|
|
1323
|
+
const row = db.prepare("SELECT title, content, _full_content FROM memories WHERE id = ?").get(id);
|
|
1205
1324
|
if (!row || !row._full_content) return;
|
|
1206
1325
|
db.prepare(
|
|
1207
|
-
"UPDATE memories SET content = ?, _full_content = NULL, updated_at = ? WHERE id = ?"
|
|
1208
|
-
).run(row._full_content, nowIso(), id);
|
|
1326
|
+
"UPDATE memories SET content = ?, _full_content = NULL, content_hash = ?, updated_at = ? WHERE id = ?"
|
|
1327
|
+
).run(row._full_content, contentHashOf({ title: row.title, content: row._full_content }), nowIso(), id);
|
|
1209
1328
|
incrementGeneration();
|
|
1210
1329
|
changed = true;
|
|
1211
1330
|
});
|
|
@@ -1766,8 +1885,8 @@ export function createStore(path) {
|
|
|
1766
1885
|
db.prepare(
|
|
1767
1886
|
`INSERT INTO llm_audit_logs (timestamp, trigger_source, operation_type, model_id,
|
|
1768
1887
|
input_tokens, output_tokens, total_tokens, cost_usd, duration_ms, status,
|
|
1769
|
-
error_message, related_memory_ids, metadata)
|
|
1770
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
|
1888
|
+
error_message, related_memory_ids, metadata, session_key)
|
|
1889
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
|
1771
1890
|
).run(
|
|
1772
1891
|
entry.timestamp ?? now,
|
|
1773
1892
|
entry.trigger_source,
|
|
@@ -1783,12 +1902,13 @@ export function createStore(path) {
|
|
|
1783
1902
|
JSON.stringify(entry.related_memory_ids ?? []),
|
|
1784
1903
|
entry.metadata !== undefined
|
|
1785
1904
|
? (typeof entry.metadata === "string" ? entry.metadata : JSON.stringify(entry.metadata))
|
|
1786
|
-
: null
|
|
1905
|
+
: null,
|
|
1906
|
+
entry.session_key ?? null
|
|
1787
1907
|
);
|
|
1788
1908
|
return toLlmAudit(db.prepare("SELECT * FROM llm_audit_logs ORDER BY id DESC LIMIT 1").get());
|
|
1789
1909
|
}
|
|
1790
1910
|
|
|
1791
|
-
function listLlmAudits({ limit = 50, offset = 0, source } = {}) {
|
|
1911
|
+
function listLlmAudits({ limit = 50, offset = 0, source, sessionKey } = {}) {
|
|
1792
1912
|
const { limit: lim, offset: off } = sanitizePage(limit, offset, 50);
|
|
1793
1913
|
const clauses = [];
|
|
1794
1914
|
const params = [];
|
|
@@ -1796,6 +1916,11 @@ export function createStore(path) {
|
|
|
1796
1916
|
clauses.push("trigger_source = ?");
|
|
1797
1917
|
params.push(source);
|
|
1798
1918
|
}
|
|
1919
|
+
// #254 写入准入:会话内的话题回填只需要该会话的行(走 idx_llm_audit_session)。
|
|
1920
|
+
if (sessionKey) {
|
|
1921
|
+
clauses.push("session_key = ?");
|
|
1922
|
+
params.push(sessionKey);
|
|
1923
|
+
}
|
|
1799
1924
|
const where = clauses.length ? `WHERE ${clauses.join(" AND ")}` : "";
|
|
1800
1925
|
const rows = db.prepare(
|
|
1801
1926
|
`SELECT * FROM llm_audit_logs ${where} ORDER BY timestamp DESC, id DESC LIMIT ? OFFSET ?`
|
|
@@ -1803,13 +1928,17 @@ export function createStore(path) {
|
|
|
1803
1928
|
return rows.map(toLlmAudit);
|
|
1804
1929
|
}
|
|
1805
1930
|
|
|
1806
|
-
function countLlmAudits({ source } = {}) {
|
|
1931
|
+
function countLlmAudits({ source, sessionKey } = {}) {
|
|
1807
1932
|
const clauses = [];
|
|
1808
1933
|
const params = [];
|
|
1809
1934
|
if (source) {
|
|
1810
1935
|
clauses.push("trigger_source = ?");
|
|
1811
1936
|
params.push(source);
|
|
1812
1937
|
}
|
|
1938
|
+
if (sessionKey) {
|
|
1939
|
+
clauses.push("session_key = ?");
|
|
1940
|
+
params.push(sessionKey);
|
|
1941
|
+
}
|
|
1813
1942
|
const where = clauses.length ? `WHERE ${clauses.join(" AND ")}` : "";
|
|
1814
1943
|
return db.prepare(`SELECT count(*) AS c FROM llm_audit_logs ${where}`).get(...params).c;
|
|
1815
1944
|
}
|
|
@@ -1858,6 +1987,106 @@ export function createStore(path) {
|
|
|
1858
1987
|
return db.prepare("DELETE FROM llm_audit_logs WHERE timestamp < ?").run(before).changes;
|
|
1859
1988
|
}
|
|
1860
1989
|
|
|
1990
|
+
// --- 存储生命周期:无损回收(#275 第一批)--------------------------------
|
|
1991
|
+
//
|
|
1992
|
+
// 两项都零价值判断、零条数变化:行数不变,只把不可重建/不可达的内容丢掉。
|
|
1993
|
+
// 删行不在本模块里(维护者拍板 3:裁列同意、删行不同意)——dream_runs 的骨架、
|
|
1994
|
+
// LLM 决策原文与 receipt 永不删。
|
|
1995
|
+
|
|
1996
|
+
/**
|
|
1997
|
+
* 历史 run 的输入快照统计(#275 A 项,dry-run 用)。
|
|
1998
|
+
*
|
|
1999
|
+
* `input` 这一列的含义**按 run_type 分叉**:auto / sleep 的 run 存的是当时的记忆库
|
|
2000
|
+
* 快照(可由记忆库重建),organize 的 run 存的是 apply 的重放载荷——`organize.js`
|
|
2001
|
+
* 的 apply 直接读它(`snapshot = Array.isArray(report.input) ? report.input : []`),
|
|
2002
|
+
* 置空会让 apply 找不到候选、静默什么都不做却照写回执并盖上 applied_at,那份报告
|
|
2003
|
+
* 从此永远重放不了。所以 organize 行不在可清范围里(同一列,两种语义,只能按 type 分)。
|
|
2004
|
+
*
|
|
2005
|
+
* `bytes` 是列文本大小,只作上界(真实释放看 VACUUM 前后)。
|
|
2006
|
+
* @returns {{runs: number, bytes: number}}
|
|
2007
|
+
*/
|
|
2008
|
+
function dreamRunInputStats(before) {
|
|
2009
|
+
const row = db.prepare(
|
|
2010
|
+
`SELECT count(*) AS c, COALESCE(SUM(LENGTH(input)), 0) AS b
|
|
2011
|
+
FROM dream_runs
|
|
2012
|
+
WHERE input IS NOT NULL AND created_at < ? AND run_type != 'organize'`
|
|
2013
|
+
).get(before);
|
|
2014
|
+
return { runs: row.c, bytes: row.b };
|
|
2015
|
+
}
|
|
2016
|
+
|
|
2017
|
+
/** 置空历史 run 的输入快照(organize 的重放载荷除外)。返回实际改动的行数。 */
|
|
2018
|
+
function clearDreamRunInputs(before) {
|
|
2019
|
+
return db.prepare(
|
|
2020
|
+
`UPDATE dream_runs SET input = NULL
|
|
2021
|
+
WHERE input IS NOT NULL AND created_at < ? AND run_type != 'organize'`
|
|
2022
|
+
).run(before).changes;
|
|
2023
|
+
}
|
|
2024
|
+
|
|
2025
|
+
/**
|
|
2026
|
+
* 归档行仍带的向量统计(#275 B 项)。检索 SQL 恒带 `archived = 0`,所以这部分
|
|
2027
|
+
* 向量按定义不可达,清掉不改变任何检索结果。
|
|
2028
|
+
* @returns {{rows: number, bytes: number}}
|
|
2029
|
+
*/
|
|
2030
|
+
function archivedEmbeddingStats() {
|
|
2031
|
+
const row = db.prepare(
|
|
2032
|
+
`SELECT count(*) AS c, COALESCE(SUM(LENGTH(embedding)), 0) AS b
|
|
2033
|
+
FROM memories WHERE archived = 1 AND embedding IS NOT NULL`
|
|
2034
|
+
).get();
|
|
2035
|
+
return { rows: row.c, bytes: row.b };
|
|
2036
|
+
}
|
|
2037
|
+
|
|
2038
|
+
/** 清掉归档行的向量。返回实际改动行数。 */
|
|
2039
|
+
function clearArchivedEmbeddings() {
|
|
2040
|
+
const changed = db.prepare(
|
|
2041
|
+
"UPDATE memories SET embedding = NULL WHERE archived = 1 AND embedding IS NOT NULL"
|
|
2042
|
+
).run().changes;
|
|
2043
|
+
// Issue #202 的单条失效点在 setEmbedding 里,批量清走不到它——整表清缓存最省事,
|
|
2044
|
+
// 代价只是活跃行下次检索多解析一次(FIFO 上限 4000,自愈)。
|
|
2045
|
+
if (changed > 0) embeddingCache.clear();
|
|
2046
|
+
return changed;
|
|
2047
|
+
}
|
|
2048
|
+
|
|
2049
|
+
/**
|
|
2050
|
+
* 库文件与页统计(#275 报告口径):回收收益按 VACUUM 前后体积量,不按列字节估——
|
|
2051
|
+
* 实际释放来自溢出页与索引页的回收,列文本大小只是上界。
|
|
2052
|
+
*
|
|
2053
|
+
* 体积是**磁盘足迹**:WAL 模式下主文件之外还有 `-wal` / `-shm`,只量主文件会把
|
|
2054
|
+
* 「刚清完还没落盘」的那部分算漏(实测过:VACUUM 后主文件可能一动不动,要等一次
|
|
2055
|
+
* checkpoint)。所以三份一起量;`:memory:` 库没有文件。
|
|
2056
|
+
*/
|
|
2057
|
+
function storageStats() {
|
|
2058
|
+
const pageSize = db.prepare("PRAGMA page_size").get().page_size;
|
|
2059
|
+
const pageCount = db.prepare("PRAGMA page_count").get().page_count;
|
|
2060
|
+
const freelistCount = db.prepare("PRAGMA freelist_count").get().freelist_count;
|
|
2061
|
+
const sizeOf = (p) => {
|
|
2062
|
+
try {
|
|
2063
|
+
return statSync(p).size;
|
|
2064
|
+
} catch {
|
|
2065
|
+
return 0;
|
|
2066
|
+
}
|
|
2067
|
+
};
|
|
2068
|
+
let fileBytes = null;
|
|
2069
|
+
if (typeof path === "string" && path !== ":memory:") {
|
|
2070
|
+
fileBytes = sizeOf(path) + sizeOf(`${path}-wal`) + sizeOf(`${path}-shm`);
|
|
2071
|
+
}
|
|
2072
|
+
return { path: typeof path === "string" ? path : null, pageSize, pageCount, freelistCount, fileBytes };
|
|
2073
|
+
}
|
|
2074
|
+
|
|
2075
|
+
/**
|
|
2076
|
+
* 整库 VACUUM(#275 的手动步骤):代价 O(库大小) 且需要排他写锁,所以只在显式
|
|
2077
|
+
* 入口里跑,绝不挂启动路径、也不开 auto_vacuum。
|
|
2078
|
+
*
|
|
2079
|
+
* 收尾补一次 `wal_checkpoint(TRUNCATE)`:WAL 模式下 VACUUM 重排的是主文件里的页,
|
|
2080
|
+
* 已回收的空间可能还挂在 WAL 里,不 checkpoint 就量不到体积下降——报告口径要的是
|
|
2081
|
+
* 真实释放量,量不到等于没做(评审实测:VACUUM 后主文件长度不变,checkpoint 后才降)。
|
|
2082
|
+
*/
|
|
2083
|
+
function vacuum() {
|
|
2084
|
+
const started = Date.now();
|
|
2085
|
+
db.exec("VACUUM");
|
|
2086
|
+
db.exec("PRAGMA wal_checkpoint(TRUNCATE)");
|
|
2087
|
+
return { duration_ms: Date.now() - started };
|
|
2088
|
+
}
|
|
2089
|
+
|
|
1861
2090
|
// --- failure memories ----------------------------------------------------
|
|
1862
2091
|
|
|
1863
2092
|
/**
|
|
@@ -2444,6 +2673,60 @@ export function createStore(path) {
|
|
|
2444
2673
|
* Otherwise wrap in BEGIN/COMMIT so a memory write and its desired-generation
|
|
2445
2674
|
* bump commit together: a crash between them can never leave a mutated store
|
|
2446
2675
|
* with generation == applied (audit peer blocker 1, "crash window"). */
|
|
2676
|
+
// --- #249 N3:压缩边缘的连续性提案(唯一写入口 = 压缩边缘监听器) -------------
|
|
2677
|
+
/** Upsert one continuity proposal. (session_id, kind) is the形态约定「同一会话同一类
|
|
2678
|
+
* 只留一条」:命中即刷新(created_at 不动、updated_at 记最近边缘),不新增行。
|
|
2679
|
+
* Queue full → drop the new one and say so (dropped: true) instead of throwing: an edge
|
|
2680
|
+
* that cannot be recorded must not break the host's step. Returns {id, created, dropped}. */
|
|
2681
|
+
function saveContinuityProposal({
|
|
2682
|
+
sessionId, kind, currentWork = null, nextStep = null, openQuestions = null, edgeSeq = null,
|
|
2683
|
+
maxPending = MAX_CONTINUITY_PENDING
|
|
2684
|
+
} = {}) {
|
|
2685
|
+
if (!sessionId || !kind) throw new TypeError("continuity proposal needs sessionId and kind");
|
|
2686
|
+
const now = new Date().toISOString();
|
|
2687
|
+
const existing = db.prepare("SELECT id FROM continuity_proposals WHERE session_id = ? AND kind = ?").get(sessionId, kind);
|
|
2688
|
+
if (!existing) {
|
|
2689
|
+
const pending = db.prepare("SELECT COUNT(*) AS c FROM continuity_proposals WHERE status = 'pending'").get().c;
|
|
2690
|
+
if (pending >= maxPending) return { id: null, created: false, dropped: true };
|
|
2691
|
+
}
|
|
2692
|
+
// 写入本身是一条 UPSERT:唯一索引 (session_id, kind) 是并发的裁决者。两个宿主共用
|
|
2693
|
+
// memoryDir 时(AGENTS.md 的多进程 WAL 场景)先查后插会有一个撞 SQLITE_CONSTRAINT,
|
|
2694
|
+
// 而这里只该有一条语句决定成不成功。上面的 SELECT 只用于「队列满」与 created 标记。
|
|
2695
|
+
// created_at 与 status 刻意不在 SET 里:命中只刷新内容与最近边缘时刻,不把已转正的行打回 pending。
|
|
2696
|
+
const id = randomUUID();
|
|
2697
|
+
db.prepare(`
|
|
2698
|
+
INSERT INTO continuity_proposals (id, session_id, kind, current_work, next_step, open_questions, status, edge_seq, created_at, updated_at)
|
|
2699
|
+
VALUES (?, ?, ?, ?, ?, ?, 'pending', ?, ?, ?)
|
|
2700
|
+
ON CONFLICT(session_id, kind) DO UPDATE SET
|
|
2701
|
+
current_work = excluded.current_work,
|
|
2702
|
+
next_step = excluded.next_step,
|
|
2703
|
+
open_questions = excluded.open_questions,
|
|
2704
|
+
edge_seq = excluded.edge_seq,
|
|
2705
|
+
updated_at = excluded.updated_at
|
|
2706
|
+
`).run(id, sessionId, kind, currentWork, nextStep, openQuestions, edgeSeq, now, now);
|
|
2707
|
+
const row = db.prepare("SELECT id FROM continuity_proposals WHERE session_id = ? AND kind = ?").get(sessionId, kind);
|
|
2708
|
+
return { id: row?.id ?? id, created: !existing, dropped: false };
|
|
2709
|
+
}
|
|
2710
|
+
|
|
2711
|
+
/** The single proposal for (session, kind), or null. */
|
|
2712
|
+
function getContinuityProposal(sessionId, kind) {
|
|
2713
|
+
return db.prepare("SELECT * FROM continuity_proposals WHERE session_id = ? AND kind = ?").get(sessionId, kind) ?? null;
|
|
2714
|
+
}
|
|
2715
|
+
|
|
2716
|
+
/** Newest-first proposals, optionally filtered by status (转正通道的读侧预置)。 */
|
|
2717
|
+
function listContinuityProposals({ status, limit = 50 } = {}) {
|
|
2718
|
+
const bounded = Number.isSafeInteger(limit) && limit > 0 ? limit : 50;
|
|
2719
|
+
return status
|
|
2720
|
+
? db.prepare("SELECT * FROM continuity_proposals WHERE status = ? ORDER BY updated_at DESC, id LIMIT ?").all(status, bounded)
|
|
2721
|
+
: db.prepare("SELECT * FROM continuity_proposals ORDER BY updated_at DESC, id LIMIT ?").all(bounded);
|
|
2722
|
+
}
|
|
2723
|
+
|
|
2724
|
+
function countContinuityProposals({ status } = {}) {
|
|
2725
|
+
return status
|
|
2726
|
+
? db.prepare("SELECT COUNT(*) AS c FROM continuity_proposals WHERE status = ?").get(status).c
|
|
2727
|
+
: db.prepare("SELECT COUNT(*) AS c FROM continuity_proposals").get().c;
|
|
2728
|
+
}
|
|
2729
|
+
|
|
2447
2730
|
function runAtomically(fn) {
|
|
2448
2731
|
if (db.isTransaction) return fn();
|
|
2449
2732
|
db.exec("BEGIN");
|
|
@@ -2485,6 +2768,8 @@ export function createStore(path) {
|
|
|
2485
2768
|
saveDocument,
|
|
2486
2769
|
update,
|
|
2487
2770
|
compareAndUpdate,
|
|
2771
|
+
// #254:写入准入的内容哈希候选集(只读)。
|
|
2772
|
+
findContentHashMatches,
|
|
2488
2773
|
remove,
|
|
2489
2774
|
setForget,
|
|
2490
2775
|
setArchived,
|
|
@@ -2523,6 +2808,13 @@ export function createStore(path) {
|
|
|
2523
2808
|
countLlmAudits,
|
|
2524
2809
|
getLlmAuditStats,
|
|
2525
2810
|
deleteOldLlmAudits,
|
|
2811
|
+
// 存储生命周期(#275 第一批):无损回收的统计、清理与体积口径。
|
|
2812
|
+
dreamRunInputStats,
|
|
2813
|
+
clearDreamRunInputs,
|
|
2814
|
+
archivedEmbeddingStats,
|
|
2815
|
+
clearArchivedEmbeddings,
|
|
2816
|
+
storageStats,
|
|
2817
|
+
vacuum,
|
|
2526
2818
|
saveFailure,
|
|
2527
2819
|
listFailures,
|
|
2528
2820
|
getFailureStats,
|
|
@@ -2559,6 +2851,11 @@ export function createStore(path) {
|
|
|
2559
2851
|
setTypeStatus,
|
|
2560
2852
|
getTypeStatus,
|
|
2561
2853
|
incrementGeneration,
|
|
2854
|
+
// #249 N3:压缩边缘的连续性提案(落提案、转正才进 memories)。
|
|
2855
|
+
saveContinuityProposal,
|
|
2856
|
+
getContinuityProposal,
|
|
2857
|
+
listContinuityProposals,
|
|
2858
|
+
countContinuityProposals,
|
|
2562
2859
|
close() {
|
|
2563
2860
|
db.close();
|
|
2564
2861
|
}
|