@modusensus/dsh-mneme 0.9.0 → 0.9.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +78 -16
- package/bin/cli.mjs +8 -2
- package/docs/CONFIGURATION.md +25 -0
- package/docs/SEMANTIC.md +3 -3
- package/lib/api-standalone.js +78 -12
- package/lib/api.js +282 -57
- package/lib/candidates.js +454 -454
- package/lib/client.js +598 -248
- package/lib/config.js +33 -2
- package/lib/embedding.js +116 -15
- package/lib/index.js +32 -0
- package/lib/inject.js +34 -3
- package/lib/lang.js +124 -0
- package/lib/pipeline-activities.js +178 -0
- package/lib/pipeline-attribution.js +165 -0
- package/lib/pipeline-command.js +279 -0
- package/lib/pipeline-confirm.js +142 -0
- package/lib/pipeline-exec.js +174 -0
- package/lib/pipeline-nudge.js +1281 -0
- package/lib/pipeline-observations.js +181 -0
- package/lib/pipeline-observe.js +133 -0
- package/lib/pipeline-runner.js +346 -0
- package/lib/pipeline-runs.js +442 -0
- package/lib/pipeline.js +128 -7
- package/lib/semantic.js +66 -10
- package/lib/settings.js +132 -1
- package/lib/store.js +77 -3
- package/lib/summarize.js +2 -3
- package/package.json +3 -3
- package/scripts/changelog-prep.mjs +62 -7
- package/scripts/check-sync.js +8 -6
- package/scripts/release-docs.mjs +204 -0
- package/scripts/release-notes.mjs +162 -0
- package/src/api-standalone.js +78 -12
- package/src/api.js +282 -57
- package/src/candidates.js +454 -454
- package/src/config.js +33 -2
- package/src/embedding.js +116 -15
- package/src/index.js +32 -0
- package/src/inject.js +34 -3
- package/src/lang.js +124 -0
- package/src/pipeline-activities.js +178 -0
- package/src/pipeline-attribution.js +165 -0
- package/src/pipeline-command.js +279 -0
- package/src/pipeline-confirm.js +142 -0
- package/src/pipeline-exec.js +174 -0
- package/src/pipeline-nudge.js +1281 -0
- package/src/pipeline-observations.js +181 -0
- package/src/pipeline-observe.js +133 -0
- package/src/pipeline-runner.js +346 -0
- package/src/pipeline-runs.js +442 -0
- package/src/pipeline.js +128 -7
- package/src/semantic.js +66 -10
- package/src/settings.js +132 -1
- package/src/store.js +77 -3
- package/src/summarize.js +2 -3
- package/test/api-hardening.test.js +173 -0
- package/test/api.test.js +98 -1
- package/test/changelog-prep.test.js +74 -8
- package/test/cli-enums.test.js +37 -0
- package/test/client.test.js +152 -3
- package/test/docs-config-keys.test.js +65 -0
- package/test/helpers/write-admission-samples.js +6 -2
- package/test/lib-smoke.test.js +20 -0
- package/test/persona-api.test.js +172 -0
- package/test/persona-flag.test.js +36 -0
- package/test/persona-inject.test.js +145 -0
- package/test/persona-settings.test.js +145 -0
- package/test/pipeline-activities.test.js +155 -0
- package/test/pipeline-attribution.test.js +175 -0
- package/test/pipeline-command.test.js +243 -0
- package/test/pipeline-confirm.test.js +148 -0
- package/test/pipeline-exec.test.js +172 -0
- package/test/pipeline-nudge.test.js +1036 -0
- package/test/pipeline-observations.test.js +157 -0
- package/test/pipeline-observe.test.js +127 -0
- package/test/pipeline-r1r2.test.js +149 -0
- package/test/pipeline-runner.test.js +442 -0
- package/test/pipeline-runs.test.js +767 -0
- package/test/pipeline.test.js +137 -6
- package/test/reindex-backfill.test.js +83 -4
- package/test/release-docs.test.js +130 -0
- package/test/release-notes.test.js +240 -0
- package/test/release-workflow.test.js +16 -0
- package/test/route-policy.test.js +113 -0
- package/test/security-doc.test.js +139 -0
- package/test/sensitive-scan.test.js +44 -0
- package/test/workflow-guards.test.js +156 -0
package/lib/semantic.js
CHANGED
|
@@ -28,29 +28,59 @@ import { LocalReranker } from "./reranker.js";
|
|
|
28
28
|
*/
|
|
29
29
|
export async function backfillMissingEmbeddings({
|
|
30
30
|
store, embedder, vectorIndex, logger,
|
|
31
|
+
// maxTotal: 0 = no cap (drain the whole backlog this run). The default stays
|
|
32
|
+
// modest so a boot never floods a provider, but the cap is a caller choice
|
|
33
|
+
// now instead of a hardcoded one — the fixed 500 meant a 4k-row backlog
|
|
34
|
+
// needed nine restarts to clear, and nothing reported that it had been cut.
|
|
31
35
|
maxTotal = 500, batchSize = 10, rateLimitMs = 200
|
|
32
36
|
}) {
|
|
37
|
+
const startedAt = new Date().toISOString();
|
|
38
|
+
// Model-aware selector: rows whose vector came from another model (or
|
|
39
|
+
// predate per-row provenance) must be re-embedded too, otherwise a model
|
|
40
|
+
// switch leaves uncomparable vectors in the index forever.
|
|
41
|
+
const model = embedder.modelHash;
|
|
42
|
+
const pick = (n) => (typeof store.needsEmbeddingFor === "function"
|
|
43
|
+
? store.needsEmbeddingFor(model, n)
|
|
44
|
+
: store.needsEmbedding(n));
|
|
45
|
+
const unlimited = !(maxTotal > 0);
|
|
33
46
|
let indexed = 0;
|
|
34
|
-
|
|
35
|
-
|
|
47
|
+
let processed = 0;
|
|
48
|
+
let failed = 0;
|
|
49
|
+
for (let done = 0; unlimited || done < maxTotal;) {
|
|
50
|
+
const rows = pick(batchSize);
|
|
36
51
|
if (!rows.length) break;
|
|
37
52
|
for (const row of rows) {
|
|
38
53
|
try {
|
|
39
54
|
const text = [row.title, row.content].filter(Boolean).join("\n");
|
|
40
55
|
const vector = await embedder.embedSingle(text);
|
|
41
56
|
if (vector?.length) {
|
|
42
|
-
store.setEmbedding(row.id, vector);
|
|
57
|
+
store.setEmbedding(row.id, vector, model);
|
|
43
58
|
indexed++;
|
|
59
|
+
} else {
|
|
60
|
+
failed++;
|
|
44
61
|
}
|
|
45
|
-
} catch { /* skip the bad row */ }
|
|
62
|
+
} catch { failed++; /* skip the bad row, but count it */ }
|
|
46
63
|
}
|
|
47
64
|
done += rows.length;
|
|
65
|
+
processed += rows.length;
|
|
48
66
|
// Rate limit: space out batches so the provider is not hammered.
|
|
49
|
-
if (
|
|
67
|
+
if (pick(1).length) await new Promise((r) => setTimeout(r, rateLimitMs));
|
|
50
68
|
}
|
|
51
|
-
if (indexed > 0 &&
|
|
52
|
-
|
|
53
|
-
|
|
69
|
+
if (indexed > 0 && model) vectorIndex.markModel?.(model, embedder.dimension);
|
|
70
|
+
// stillStale answers "did it actually finish?". The old return value was a
|
|
71
|
+
// bare indexed count, so a run that hit the cap looked exactly like a run
|
|
72
|
+
// that drained the backlog.
|
|
73
|
+
const stillStale = pick(1).length > 0;
|
|
74
|
+
const finishReason = stillStale ? "capped" : (failed > 0 && indexed === 0 ? "all-failed" : "drained");
|
|
75
|
+
logger?.info?.(
|
|
76
|
+
`[dsh-mneme] auto-reindex backfilled ${indexed} embeddings (processed=${processed}, failed=${failed}, finish=${finishReason})`
|
|
77
|
+
);
|
|
78
|
+
return {
|
|
79
|
+
indexed, processed, failed, finishReason, stillStale,
|
|
80
|
+
startedAt,
|
|
81
|
+
finishedAt: new Date().toISOString(),
|
|
82
|
+
model: model ?? null
|
|
83
|
+
};
|
|
54
84
|
}
|
|
55
85
|
|
|
56
86
|
/**
|
|
@@ -74,6 +104,9 @@ export function createSemantic({ store, service, settings, cfg, logger, vectorIn
|
|
|
74
104
|
// boot 回填首查计时器。搬移前在 index.js 是裸 setTimeout(不参与卸载清理);
|
|
75
105
|
// 纳入 dispose 是修悬挂,不改变启动行为。
|
|
76
106
|
let reindexTimer = null;
|
|
107
|
+
// Last boot-backfill summary, exposed via backfillStatus() so "did it
|
|
108
|
+
// finish?" has an answer other than a log line.
|
|
109
|
+
let lastBackfill = null;
|
|
77
110
|
|
|
78
111
|
if (lightMode) {
|
|
79
112
|
// Light mode: the whole vector pipeline stays off — no embedder (nothing
|
|
@@ -202,12 +235,31 @@ export function createSemantic({ store, service, settings, cfg, logger, vectorIn
|
|
|
202
235
|
if (tries > 0) reindexTimer = setTimeout(() => attempt(tries - 1), 2000);
|
|
203
236
|
return;
|
|
204
237
|
}
|
|
205
|
-
|
|
238
|
+
const model = embedder.modelHash;
|
|
239
|
+
const stale = typeof store.hasStaleEmbedding === "function"
|
|
240
|
+
? store.hasStaleEmbedding(model)
|
|
241
|
+
: store.needsEmbedding(1).length > 0;
|
|
242
|
+
if (!stale) return; // nothing missing and nothing from another model
|
|
206
243
|
// Issue #128: no fingerprint gate here anymore — a matching fingerprint
|
|
207
244
|
// used to return early and permanently orphan rows whose embed failed
|
|
208
245
|
// at write time. See backfillMissingEmbeddings().
|
|
209
|
-
|
|
246
|
+
// Cap and pacing are configuration now (reindexMaxPerBoot /
|
|
247
|
+
// reindexBatchSize / reindexRateLimitMs) instead of hardcoded 500/10/200.
|
|
248
|
+
lastBackfill = {
|
|
249
|
+
state: "running", startedAt: new Date().toISOString(),
|
|
250
|
+
maxTotal: cfg.reindexMaxPerBoot, batchSize: cfg.reindexBatchSize
|
|
251
|
+
};
|
|
252
|
+
backfillMissingEmbeddings({
|
|
253
|
+
store, embedder, vectorIndex, logger,
|
|
254
|
+
maxTotal: cfg.reindexMaxPerBoot,
|
|
255
|
+
batchSize: cfg.reindexBatchSize,
|
|
256
|
+
rateLimitMs: cfg.reindexRateLimitMs
|
|
257
|
+
})
|
|
258
|
+
.then((summary) => {
|
|
259
|
+
lastBackfill = { ...summary, state: "finished" };
|
|
260
|
+
})
|
|
210
261
|
.catch((error) => {
|
|
262
|
+
lastBackfill = { state: "error", error: String(error), startedAt: lastBackfill?.startedAt };
|
|
211
263
|
logger?.warn?.(`[dsh-mneme] auto-reindex failed: ${String(error)}`);
|
|
212
264
|
});
|
|
213
265
|
} catch (error) {
|
|
@@ -221,6 +273,10 @@ export function createSemantic({ store, service, settings, cfg, logger, vectorIn
|
|
|
221
273
|
return {
|
|
222
274
|
embedder,
|
|
223
275
|
reranker,
|
|
276
|
+
/** Last backfill run: { state, indexed, failed, finishReason, stillStale, ... }. */
|
|
277
|
+
backfillStatus() {
|
|
278
|
+
return lastBackfill;
|
|
279
|
+
},
|
|
224
280
|
dispose() {
|
|
225
281
|
if (embedRetryTimer !== null) { clearTimeout(embedRetryTimer); embedRetryTimer = null; }
|
|
226
282
|
if (reindexTimer !== null) { clearTimeout(reindexTimer); reindexTimer = null; }
|
package/lib/settings.js
CHANGED
|
@@ -31,6 +31,67 @@ function parseList(raw) {
|
|
|
31
31
|
}
|
|
32
32
|
}
|
|
33
33
|
|
|
34
|
+
// persona 对象的字段白名单:写入时只挑这些键,未知字段丢弃(设计稿①:字段从简,
|
|
35
|
+
// beta 收到真实反馈再加;provenance 是 v1.5 协作式人格的预留位,v1 不写管道)。
|
|
36
|
+
// 读侧损坏即回空对象——同一容错口径照 parseList(坏值不抛错、面板显示为空可重写)。
|
|
37
|
+
const PERSONA_FIELDS = ["name", "personality", "replyStyle", "enabled", "provenance"];
|
|
38
|
+
|
|
39
|
+
function parsePersona(raw) {
|
|
40
|
+
try {
|
|
41
|
+
const value = JSON.parse(raw);
|
|
42
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) return {};
|
|
43
|
+
const out = {};
|
|
44
|
+
for (const key of PERSONA_FIELDS) {
|
|
45
|
+
if (value[key] !== undefined) out[key] = value[key];
|
|
46
|
+
}
|
|
47
|
+
return out;
|
|
48
|
+
} catch {
|
|
49
|
+
return {};
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
// 多卡模型(v1.2):存储形为 { default: <卡>, byAgent: { <agentId>: <卡>, … } }。
|
|
54
|
+
// 兼容层:v1 存的是单对象(有 PERSONA_FIELDS 键、无 default/byAgent 键),读侧
|
|
55
|
+
// 判形状——单对象原样落 default 位,旧数据零迁移。byAgent 的键是 agent 身份
|
|
56
|
+
// id(注入时按会话身份寻址,取不到落 default)。default 卡不可删(兜底位)。
|
|
57
|
+
function parsePersonaStore(raw) {
|
|
58
|
+
try {
|
|
59
|
+
const value = JSON.parse(raw);
|
|
60
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
|
61
|
+
return { default: {}, byAgent: {} };
|
|
62
|
+
}
|
|
63
|
+
// 新形:含 default 或 byAgent 键 → 按多卡解析
|
|
64
|
+
if ("default" in value || "byAgent" in value) {
|
|
65
|
+
const byAgent = {};
|
|
66
|
+
if (value.byAgent && typeof value.byAgent === "object" && !Array.isArray(value.byAgent)) {
|
|
67
|
+
for (const [agentId, card] of Object.entries(value.byAgent)) {
|
|
68
|
+
const parsed = parsePersona(JSON.stringify(card ?? {}));
|
|
69
|
+
if (Object.keys(parsed).length) byAgent[String(agentId)] = parsed;
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
return { default: parsePersona(JSON.stringify(value.default ?? {})), byAgent };
|
|
73
|
+
}
|
|
74
|
+
// 旧形:v1 单对象 → 整体作 default(字段白名单在 parsePersona 内裁)
|
|
75
|
+
return { default: parsePersona(raw), byAgent: {} };
|
|
76
|
+
} catch {
|
|
77
|
+
return { default: {}, byAgent: {} };
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
function serializePersonaStore(store) {
|
|
82
|
+
const out = {
|
|
83
|
+
default: parsePersona(JSON.stringify(store?.default ?? {})),
|
|
84
|
+
byAgent: {}
|
|
85
|
+
};
|
|
86
|
+
if (store?.byAgent && typeof store.byAgent === "object") {
|
|
87
|
+
for (const [agentId, card] of Object.entries(store.byAgent)) {
|
|
88
|
+
const parsed = parsePersona(JSON.stringify(card ?? {}));
|
|
89
|
+
if (Object.keys(parsed).length) out.byAgent[String(agentId)] = parsed;
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
return JSON.stringify(out);
|
|
93
|
+
}
|
|
94
|
+
|
|
34
95
|
// --- feature flags(功能开关)白名单 -----------------------------------------
|
|
35
96
|
// 面板可逐项开关的后端能力。设计约束:
|
|
36
97
|
// 1. 键名与类型必须和 config.js schema 同名同型,这份白名单是唯一校验源
|
|
@@ -146,7 +207,18 @@ const FEATURE_FLAG_BOOLEANS = [
|
|
|
146
207
|
// 决定是否注册座位(会话头图标 / 浮层 / 右栏 / 对话行卡片),服务端只负责把
|
|
147
208
|
// 这个键稳定暴露出来——键若不在 effective 里,客户端静默少键、一个座位都不注册。
|
|
148
209
|
// 默认关(beta、opt-in);顶层扁平键,因此不需要 NESTED_FLAG_PATHS 条目。
|
|
149
|
-
"actionPipelineEnabled"
|
|
210
|
+
"actionPipelineEnabled",
|
|
211
|
+
// v0.9.0 第二批:M.O. 执行记账(`/mo-run` 开窗 → 窗口内命令被记成一条 run)。
|
|
212
|
+
// 与上面那个键分开:那个管「看图的座位」,这个管「往账本里写」。默认关(beta、
|
|
213
|
+
// opt-in),顶层扁平键,不需要 NESTED_FLAG_PATHS 条目。
|
|
214
|
+
"actionRunEnabled",
|
|
215
|
+
// 人格定义页(beta):注入器渲染人格段的顶层总闸(双闸门的 config 半边;对象内
|
|
216
|
+
// 的 persona.enabled 是 per-agent 半边,存 user_settings 的 "persona" 键,见
|
|
217
|
+
// getPersona/setPersona)。默认关(beta、opt-in,老用户行为逐字节不变);不进
|
|
218
|
+
// LIGHT_MODE_OFF——人格段是用户自写的一小段轻量常驻文本,不引入任何重型后台
|
|
219
|
+
// 路径,轻量档用户显式勾选仍然赢(同 actionPipelineEnabled 的定性)。顶层扁平
|
|
220
|
+
// 键,不需要 NESTED_FLAG_PATHS 条目。
|
|
221
|
+
"personaEnabled"
|
|
150
222
|
];
|
|
151
223
|
// 整数开关的闭区间,与 config.js 里 z.natural().min().max() 对齐。
|
|
152
224
|
const FEATURE_FLAG_INT_RANGES = {
|
|
@@ -403,6 +475,65 @@ export function createSettings(db) {
|
|
|
403
475
|
setSetting("rules", JSON.stringify(list));
|
|
404
476
|
},
|
|
405
477
|
|
|
478
|
+
/**
|
|
479
|
+
* Agent persona (name / personality / replyStyle + enabled gate), stored as
|
|
480
|
+
* one JSON key. v1.2 多卡:存储形 { default, byAgent },旧 v1 单对象读作
|
|
481
|
+
* default 兼容。getPersona 返回全量多卡形(面板用);getPersonaFor 按 agent
|
|
482
|
+
* 身份取卡(注入用),身份缺失/无卡/卡 enabled=false 均落 default。
|
|
483
|
+
*/
|
|
484
|
+
getPersona() {
|
|
485
|
+
return parsePersonaStore(getSetting("persona") ?? "{}");
|
|
486
|
+
},
|
|
487
|
+
setPersona(input) {
|
|
488
|
+
// 双形兼容:单对象形 = 只更新 default、byAgent 保留(旧面板/旧调用方合并语义);
|
|
489
|
+
// 多卡形 = 全量替换。坏体(非对象)在此不落库不抛错,保值由调用方守卫。
|
|
490
|
+
if (typeof input !== "object" || input === null || Array.isArray(input)) return;
|
|
491
|
+
if ("default" in input || "byAgent" in input) {
|
|
492
|
+
setSetting("persona", serializePersonaStore(input));
|
|
493
|
+
return;
|
|
494
|
+
}
|
|
495
|
+
const current = parsePersonaStore(getSetting("persona") ?? "{}");
|
|
496
|
+
current.default = parsePersona(JSON.stringify(input));
|
|
497
|
+
setSetting("persona", serializePersonaStore(current));
|
|
498
|
+
},
|
|
499
|
+
/** Per-agent lookup for the injector: id card → default fallback. */
|
|
500
|
+
getPersonaFor(agentId) {
|
|
501
|
+
const store = parsePersonaStore(getSetting("persona") ?? "{}");
|
|
502
|
+
const id = typeof agentId === "string" && agentId.trim() ? agentId.trim() : "";
|
|
503
|
+
if (id) {
|
|
504
|
+
const card = store.byAgent[id];
|
|
505
|
+
if (card && card.enabled === true) return card;
|
|
506
|
+
}
|
|
507
|
+
return store.default;
|
|
508
|
+
},
|
|
509
|
+
/** Replace one byAgent card wholesale (panel save); default 不可经此改。 */
|
|
510
|
+
setPersonaFor(agentId, card) {
|
|
511
|
+
const id = typeof agentId === "string" && agentId.trim() ? agentId.trim() : "";
|
|
512
|
+
if (!id || id === "default") return false;
|
|
513
|
+
const store = parsePersonaStore(getSetting("persona") ?? "{}");
|
|
514
|
+
const clean = parsePersona(JSON.stringify(
|
|
515
|
+
typeof card === "object" && card !== null && !Array.isArray(card) ? card : {}
|
|
516
|
+
));
|
|
517
|
+
if (Object.keys(clean).length) store.byAgent[id] = clean;
|
|
518
|
+
else delete store.byAgent[id];
|
|
519
|
+
setSetting("persona", serializePersonaStore(store));
|
|
520
|
+
return true;
|
|
521
|
+
},
|
|
522
|
+
/** byAgent 键列表(面板卡列表用,default 不在其中)。 */
|
|
523
|
+
listPersonas() {
|
|
524
|
+
return Object.keys(parsePersonaStore(getSetting("persona") ?? "{}").byAgent);
|
|
525
|
+
},
|
|
526
|
+
/** 删一张 byAgent 卡;default 不可删(返回 false)。 */
|
|
527
|
+
removePersona(agentId) {
|
|
528
|
+
const id = typeof agentId === "string" && agentId.trim() ? agentId.trim() : "";
|
|
529
|
+
if (!id || id === "default") return false;
|
|
530
|
+
const store = parsePersonaStore(getSetting("persona") ?? "{}");
|
|
531
|
+
if (!(id in store.byAgent)) return false;
|
|
532
|
+
delete store.byAgent[id];
|
|
533
|
+
setSetting("persona", serializePersonaStore(store));
|
|
534
|
+
return true;
|
|
535
|
+
},
|
|
536
|
+
|
|
406
537
|
/** All custom commands, sorted by name. */
|
|
407
538
|
listCommands() {
|
|
408
539
|
const rows = db.prepare("SELECT * FROM custom_commands ORDER BY name ASC").all();
|
package/lib/store.js
CHANGED
|
@@ -835,6 +835,15 @@ export function createStore(path) {
|
|
|
835
835
|
// 老库打开时 SCHEMA 的 CREATE TABLE 对既有表不生效,列还不存在,把索引写进 SCHEMA
|
|
836
836
|
// 会直接报 no such column(与下方 llm_audit_logs.session_key 同理)。
|
|
837
837
|
addColumn("memories", "content_hash", "ALTER TABLE memories ADD COLUMN content_hash TEXT");
|
|
838
|
+
// 每条向量的出身(行级 model hash,如 `qwen3.7-text-embedding-flash#17a6a0b1`)。
|
|
839
|
+
// 为什么必须行级:vector_meta 只有一个 model_hash,它记的是「最后一次成功写入
|
|
840
|
+
// 的模型」,换模型时被整表覆盖——于是上一代模型产出的向量在元数据上看起来完全
|
|
841
|
+
// 合规,实际与新模型的查询向量不可比。没有这一列,「这条向量是不是当前模型产
|
|
842
|
+
// 的」就无从判断(content_hash 是内容归一化哈希,与向量模型无关;dimension 在
|
|
843
|
+
// 同为 1024 维的模型间也区分不出来)。
|
|
844
|
+
// 存量行的值是 NULL,且**不推断**:按 updated_at 猜出的模型名会变成日后审计信
|
|
845
|
+
// 以为真的证据。NULL 一律读作「出身未知」,等价的补法是重算,不是猜。
|
|
846
|
+
addColumn("memories", "embedding_model", "ALTER TABLE memories ADD COLUMN embedding_model TEXT");
|
|
838
847
|
// issue #24 块2(权重演化):entity_relations 加 weight + source。存量关系
|
|
839
848
|
// 行缺列 → 默认 manual 1.0(最保守:既有语义不因迁移而变弱)。
|
|
840
849
|
addColumn("entity_relations", "weight", "ALTER TABLE entity_relations ADD COLUMN weight REAL NOT NULL DEFAULT 1.0");
|
|
@@ -1606,10 +1615,24 @@ export function createStore(path) {
|
|
|
1606
1615
|
return rows.map((row) => row.content ?? "");
|
|
1607
1616
|
}
|
|
1608
1617
|
|
|
1609
|
-
/** Set (or clear with null) the embedding vector of a memory.
|
|
1610
|
-
|
|
1618
|
+
/** Set (or clear with null) the embedding vector of a memory.
|
|
1619
|
+
* `modelHash` records which model produced THIS vector (see the
|
|
1620
|
+
* embedding_model migration note). Clearing the vector clears the
|
|
1621
|
+
* provenance with it, so a NULL vector is never paired with a stale model
|
|
1622
|
+
* name; omitting modelHash leaves the column untouched (callers that
|
|
1623
|
+
* predate the column still work, their rows simply read as unknown). */
|
|
1624
|
+
function setEmbedding(id, vector, modelHash) {
|
|
1611
1625
|
const json = Array.isArray(vector) && vector.length ? JSON.stringify(vector) : null;
|
|
1612
|
-
|
|
1626
|
+
if (json) {
|
|
1627
|
+
if (modelHash !== undefined) {
|
|
1628
|
+
db.prepare("UPDATE memories SET embedding = ?, embedding_model = ? WHERE id = ?")
|
|
1629
|
+
.run(json, modelHash === null ? null : String(modelHash), id);
|
|
1630
|
+
} else {
|
|
1631
|
+
db.prepare("UPDATE memories SET embedding = ? WHERE id = ?").run(json, id);
|
|
1632
|
+
}
|
|
1633
|
+
} else {
|
|
1634
|
+
db.prepare("UPDATE memories SET embedding = NULL, embedding_model = NULL WHERE id = ?").run(id);
|
|
1635
|
+
}
|
|
1613
1636
|
// Issue #202:单一写入口即单一失效点——重写缓存条目(向量在手上不必重解析),
|
|
1614
1637
|
// 清空则移除,删除过的 id 不会被下一次 searchVector 的 id 集合选中。
|
|
1615
1638
|
if (json) {
|
|
@@ -1662,6 +1685,54 @@ export function createStore(path) {
|
|
|
1662
1685
|
).all(limit);
|
|
1663
1686
|
}
|
|
1664
1687
|
|
|
1688
|
+
/**
|
|
1689
|
+
* Rows whose vector is missing OR was produced by a different model than
|
|
1690
|
+
* `modelHash` — the selector the backfill should use whenever the caller
|
|
1691
|
+
* knows which model the vectors are supposed to come from.
|
|
1692
|
+
*
|
|
1693
|
+
* Why `needsEmbedding` alone is not enough: it only matches empty vectors,
|
|
1694
|
+
* so after a model switch every vector written by the previous model stays
|
|
1695
|
+
* eligible for "has a vector" forever, while its similarity to the current
|
|
1696
|
+
* query embedding is meaningless. Those rows keep inflating the status
|
|
1697
|
+
* card's coverage while semantic recall silently degrades.
|
|
1698
|
+
*
|
|
1699
|
+
* `embedding_model IS NULL` counts as stale on purpose: NULL means "written
|
|
1700
|
+
* before per-row provenance existed, producing model unknown", and an
|
|
1701
|
+
* unknown origin cannot be proven to match. Those rows get re-embedded
|
|
1702
|
+
* rather than assumed correct.
|
|
1703
|
+
*
|
|
1704
|
+
* When `modelHash` is falsy (embedder unconfigured / no fingerprint yet)
|
|
1705
|
+
* this degrades to the plain missing-only selector instead of re-embedding
|
|
1706
|
+
* the whole table against nothing.
|
|
1707
|
+
*/
|
|
1708
|
+
function needsEmbeddingFor(modelHash, limit = 50) {
|
|
1709
|
+
if (!modelHash) return needsEmbedding(limit);
|
|
1710
|
+
return db.prepare(
|
|
1711
|
+
`SELECT id, title, content, embedding_model FROM memories
|
|
1712
|
+
WHERE ((embedding IS NULL OR embedding = '')
|
|
1713
|
+
OR embedding_model IS NULL
|
|
1714
|
+
OR embedding_model != ?)
|
|
1715
|
+
AND forgotten = 0 AND archived = 0
|
|
1716
|
+
ORDER BY updated_at DESC LIMIT ?`
|
|
1717
|
+
).all(String(modelHash), limit);
|
|
1718
|
+
}
|
|
1719
|
+
|
|
1720
|
+
/** True when any active row would be picked by needsEmbeddingFor(). */
|
|
1721
|
+
function hasStaleEmbedding(modelHash) {
|
|
1722
|
+
return needsEmbeddingFor(modelHash, 1).length > 0;
|
|
1723
|
+
}
|
|
1724
|
+
|
|
1725
|
+
/** Per-model provenance counts over active rows ('' = unknown origin). */
|
|
1726
|
+
function embeddingModelBreakdown() {
|
|
1727
|
+
return db.prepare(
|
|
1728
|
+
`SELECT COALESCE(embedding_model, '') AS model, count(*) AS c
|
|
1729
|
+
FROM memories
|
|
1730
|
+
WHERE embedding IS NOT NULL AND embedding != ''
|
|
1731
|
+
AND forgotten = 0 AND archived = 0
|
|
1732
|
+
GROUP BY COALESCE(embedding_model, '')`
|
|
1733
|
+
).all();
|
|
1734
|
+
}
|
|
1735
|
+
|
|
1665
1736
|
function search(query, { limit = 20, includeArchived = false } = {}) {
|
|
1666
1737
|
const q = String(query).trim();
|
|
1667
1738
|
if (!q) return [];
|
|
@@ -3159,6 +3230,9 @@ export function createStore(path) {
|
|
|
3159
3230
|
getParsedEmbedding,
|
|
3160
3231
|
embeddedCount,
|
|
3161
3232
|
needsEmbedding,
|
|
3233
|
+
needsEmbeddingFor,
|
|
3234
|
+
hasStaleEmbedding,
|
|
3235
|
+
embeddingModelBreakdown,
|
|
3162
3236
|
searchVector,
|
|
3163
3237
|
saveDreamRun,
|
|
3164
3238
|
lastDreamRunAt,
|
package/lib/summarize.js
CHANGED
|
@@ -514,8 +514,7 @@ export function createSummarizer(ctx, service, config, deps = {}) {
|
|
|
514
514
|
const clearTimer = typeof deps.clearTimeoutFn === "function" ? deps.clearTimeoutFn : clearTimeout;
|
|
515
515
|
|
|
516
516
|
// v0.9.0(M.O. 项目记忆):蒸馏写入也要盖作用域章。此前只有 memory_save 那条工具
|
|
517
|
-
// 路径盖章(tools.js 的
|
|
518
|
-
// 2026-10-13 在 :173-193),而机器提炼是写入量的绝大多数——于是「项目记忆」
|
|
517
|
+
// 路径盖章(tools.js 的 resolveSessionScope 盖章块,2026-10-14 在 :174-181),而机器提炼是写入量的绝大多数——于是「项目记忆」
|
|
519
518
|
// 在库里根本不存在(全库带 scope 的行不足 3%,且带 scope 的全部来自工具路径)。
|
|
520
519
|
// 这里不另造口径:同一把解析器(scope.js 的 createScopeResolver,2026-10-13 在 :24)、同一把闸门
|
|
521
520
|
// (config.scopeEnabled === true)、同一对列(*_scope + *_scope_source='auto'),
|
|
@@ -888,7 +887,7 @@ export function createSummarizer(ctx, service, config, deps = {}) {
|
|
|
888
887
|
// v0.9.0(M.O. 项目记忆):一次窗口解析一次——同一蒸馏窗口里所有条目同属一个
|
|
889
888
|
// 工作区、一个 agent 身份,逐条解析是白花钱。解析不到就落 NULL(NULL = 未标注 =
|
|
890
889
|
// 全局可见),且解析器绝不抛错(scope.js 的硬约束:记忆写入不该被作用域拖挂)。
|
|
891
|
-
// 闸门与 tools.js 的 autoStamping(2026-10-
|
|
890
|
+
// 闸门与 tools.js 的 autoStamping(2026-10-14 在 :175)同一把:关着就一个字段都不加,老行为不变。
|
|
892
891
|
const writeScope = config?.scopeEnabled === true
|
|
893
892
|
? resolveSessionScope({ agent: { session } })
|
|
894
893
|
: null;
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@modusensus/dsh-mneme",
|
|
3
|
-
"description": "Cross-session memory plugin for DeepSeek Harness with autoDream consolidation: SQLite store, Markdown mirrors,
|
|
4
|
-
"version": "0.9.
|
|
3
|
+
"description": "Cross-session memory plugin for DeepSeek Harness with autoDream consolidation: SQLite store, Markdown mirrors, 10 model tools, automatic injection, session summarization, user profile/rules, custom slash commands, vector (semantic) search, and a Web GUI panel",
|
|
4
|
+
"version": "0.9.1",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
7
7
|
"type": "git",
|
|
@@ -64,7 +64,7 @@
|
|
|
64
64
|
"@deepseek-ai/dsh-tools": ">=0.1.0-rc.6 <0.2.0 || >=0.1.5-rc.0 <0.3.0 || >=0.2.0-rc.0 <0.3.0",
|
|
65
65
|
"@deepseek-ai/schemastery": "^3.18.1",
|
|
66
66
|
"@huggingface/transformers": ">=4.2.0 <5",
|
|
67
|
-
"dsh-better-sidebar": "
|
|
67
|
+
"dsh-better-sidebar": ">=0.18.0 <1"
|
|
68
68
|
},
|
|
69
69
|
"peerDependenciesMeta": {
|
|
70
70
|
"dsh-better-sidebar": {
|
|
@@ -14,23 +14,78 @@ export function detectEol(text) {
|
|
|
14
14
|
}
|
|
15
15
|
|
|
16
16
|
/**
|
|
17
|
-
*
|
|
17
|
+
* 只剥掉整段两端的空行,保留每行自己的缩进(正文原样搬走的前提)。
|
|
18
|
+
* 重新拼回时用原文的行尾:CRLF 的 CHANGELOG 若在这里退化成 LF,整份文件就成了混合行尾。
|
|
19
|
+
*/
|
|
20
|
+
function stripBlankEdges(text, eol) {
|
|
21
|
+
const lines = text.split(/\r?\n/);
|
|
22
|
+
let from = 0;
|
|
23
|
+
let to = lines.length;
|
|
24
|
+
while (from < to && lines[from].trim() === "") from += 1;
|
|
25
|
+
while (to > from && lines[to - 1].trim() === "") to -= 1;
|
|
26
|
+
return lines.slice(from, to).join(eol);
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* 插入 `## [version] - date` 节(幂等:已含该版本则不改)。
|
|
31
|
+
*
|
|
32
|
+
* **两个位置约定,都是被真实发版逼出来的**:
|
|
33
|
+
*
|
|
34
|
+
* 1. 新版本节排在 `## [Unreleased]` **下面**,不是 `# Changelog` 下面。Keep a Changelog
|
|
35
|
+
* 的约定是未发布区永远在最上;插反了会写出「[0.9.1] 在 [Unreleased] 之上」的顺序颠倒。
|
|
36
|
+
* v0.9.0 那次就是靠事后一笔人工提交(77df0fb)掰回来的——机器算得对的事不该留给人补。
|
|
37
|
+
*
|
|
38
|
+
* 2. `[Unreleased]` 里攒着的条目**整段搬进新版本节**,Unreleased 留空。原先只插一个
|
|
39
|
+
* 「🐛 修复 / (待填)」空占位,攒着的条目原地不动,于是发版后文件里同时存在
|
|
40
|
+
* 「新版本节是空的」和「旧条目还挂在 Unreleased 名下」——搬运这一步照样是人工补的
|
|
41
|
+
* (同一个 77df0fb)。搬的是位置不是内容:原样移动、一个字不改,所以「新版本节里
|
|
42
|
+
* 该写什么」这个判断仍然只由往 Unreleased 里写条目的人做。
|
|
18
43
|
*
|
|
19
44
|
* 返回 `{ok:false, reason}` 而不是静默返回原文——调用方必须能区分「没什么可做」与
|
|
20
45
|
* 「该做但没做成」,后者要报错退出。`reason` 取值:`already-present` / `header-not-found`。
|
|
46
|
+
* 成功时带两个字段,**必须分开用**:
|
|
47
|
+
* - `movedContent`:是否真的从 [Unreleased] 搬了非空正文(决定「搬了」还是「插了空占位」);
|
|
48
|
+
* - `moved`:其中以 `- ` 开头的条目数,只用于把日志说得更具体。
|
|
49
|
+
* 用 `moved` 当「有没有搬」的判据是错的:正文非空但一条 `-` 都没有(例如全是正文段落或
|
|
50
|
+
* 缩进代码块)时 `moved` 是 0,日志会说「[Unreleased] 本来是空的」——而它是非空的。
|
|
21
51
|
*
|
|
22
52
|
* @param {string} text CHANGELOG 全文
|
|
23
53
|
* @param {string} version 如 "0.8.13"
|
|
24
54
|
* @param {string} date 如 "2026-10-03"
|
|
25
|
-
* @returns {{ok: true, text: string} | {ok: false, reason: "already-present" | "header-not-found"}}
|
|
55
|
+
* @returns {{ok: true, text: string, moved: number, movedContent: boolean} | {ok: false, reason: "already-present" | "header-not-found"}}
|
|
26
56
|
*/
|
|
27
57
|
export function insertReleaseSection(text, version, date) {
|
|
28
58
|
if (text.includes(`## [${version}]`)) return { ok: false, reason: "already-present" };
|
|
29
59
|
const eol = detectEol(text);
|
|
30
|
-
const
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
60
|
+
const placeholder = `## 🐛 修复${eol}${eol}- (待填)${eol}`;
|
|
61
|
+
|
|
62
|
+
const m = /^## \[Unreleased\][^\r\n]*\r?\n/m.exec(text);
|
|
63
|
+
if (m) {
|
|
64
|
+
const start = m.index + m[0].length;
|
|
65
|
+
const rest = text.slice(start);
|
|
66
|
+
const nextRel = rest.search(/^## \[/m);
|
|
67
|
+
const body = nextRel === -1 ? rest : rest.slice(0, nextRel);
|
|
68
|
+
const tail = nextRel === -1 ? "" : rest.slice(nextRel);
|
|
69
|
+
// 只剥节边界上的空行,**不 trim 正文**:正文首行若是缩进代码块(Markdown 的四空格),
|
|
70
|
+
// trim 会把前导空格吃掉,代码块降级成普通段落——那就不是「原样搬」了。
|
|
71
|
+
const pending = stripBlankEdges(body, eol);
|
|
72
|
+
const head = `## [${version}] - ${date}${eol}${eol}`;
|
|
73
|
+
|
|
74
|
+
// Unreleased 里攒着东西:整段搬进新版本节,Unreleased 留空。
|
|
75
|
+
// 搬的是位置不是内容——原样移动,一个字不改。这样「新版本节里有什么」这个判断
|
|
76
|
+
// 仍然只由往 Unreleased 里写条目的人做,脚本只负责把攒好的东西挪对地方。
|
|
77
|
+
if (pending) {
|
|
78
|
+
const moved = pending.split(/\r?\n/).filter((l) => l.startsWith("- ")).length;
|
|
79
|
+
return { ok: true, moved, movedContent: true, text: text.slice(0, start) + eol + head + pending + eol + eol + tail };
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
// Unreleased 是空的:插占位节,等人写内容。
|
|
83
|
+
return { ok: true, moved: 0, movedContent: false, text: text.slice(0, start) + eol + head + placeholder + eol + tail };
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
// 没有 [Unreleased] 的旧文件:退回标题之后。BOM 可选——带 BOM 的检出上 `^#` 同样匹配不到。
|
|
87
|
+
const section = `## [${version}] - ${date}${eol}${eol}${placeholder}${eol}`;
|
|
88
|
+
const out = text.replace(/^(\uFEFF?# Changelog\r?\n\r?\n)/, "$1" + section);
|
|
34
89
|
if (out === text) return { ok: false, reason: "header-not-found" };
|
|
35
|
-
return { ok: true, text: out };
|
|
90
|
+
return { ok: true, moved: 0, movedContent: false, text: out };
|
|
36
91
|
}
|
package/scripts/check-sync.js
CHANGED
|
@@ -1,11 +1,13 @@
|
|
|
1
|
-
//
|
|
1
|
+
// 断言 src/ 与 lib/ 一致——发布前的平价闸门。
|
|
2
2
|
//
|
|
3
|
-
// npm 包实际加载 lib/(package main → lib/index.js
|
|
4
|
-
//
|
|
5
|
-
// lib,v0.7.8 发出去的 npm 包跑的还是旧代码(issue #65)。此脚本让这类漂移在发布时
|
|
6
|
-
// 直接 fail,而不是带着旧产物上线。
|
|
3
|
+
// npm 包实际加载 lib/(package main → lib/index.js)。历史教训:PR #60 只改了 src
|
|
4
|
+
// 忘了同步 lib,v0.7.8 发出去的 npm 包跑的还是旧代码,用户侧静默失效(issue #65)。
|
|
7
5
|
//
|
|
8
|
-
//
|
|
6
|
+
// 注意这里**只校验、不同步**:prepack 钩子跑的是 `npm run sync`(把 src 覆盖到
|
|
7
|
+
// lib),同步后必然一致,所以它永远不会因漂移失败。真正的闸门是这道脚本,跑在
|
|
8
|
+
// CI 每个 PR 与 release.yml 的 test job 里——让漂移在合并前就红,而不是等到发布。
|
|
9
|
+
//
|
|
10
|
+
// 用法:node scripts/check-sync.js (CI 与 release.yml 调用,非 prepack)
|
|
9
11
|
import { existsSync, readFileSync, readdirSync } from "node:fs";
|
|
10
12
|
import { join, relative } from "node:path";
|
|
11
13
|
import { fileURLToPath } from "node:url";
|