@modusensus/dsh-mneme 0.8.5 → 0.8.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +45 -147
- package/bin/cli.mjs +689 -603
- package/lib/api-standalone.js +57 -1
- package/lib/api.js +34 -11
- package/lib/client.js +289 -59
- package/lib/config.js +78 -11
- package/lib/content-hash.js +54 -0
- package/lib/document-index.js +94 -0
- package/lib/document.js +93 -11
- package/lib/dream/sleep.js +122 -8
- package/lib/dream.js +1354 -1347
- package/lib/index.js +115 -14
- package/lib/inject.js +6 -2
- package/lib/lang.js +15 -4
- package/lib/local-embedder.js +44 -6
- package/lib/maintenance.js +142 -0
- package/lib/mirror.js +147 -15
- package/lib/recall-stats.js +52 -2
- package/lib/scope.js +34 -6
- package/lib/service.js +86 -7
- package/lib/settings.js +6 -3
- package/lib/store.js +248 -21
- package/lib/tools.js +27 -9
- package/lib/write-admission.js +259 -0
- package/package.json +4 -4
- package/src/api-standalone.js +57 -1
- package/src/api.js +34 -11
- package/src/config.js +78 -11
- package/src/content-hash.js +54 -0
- package/src/document-index.js +94 -0
- package/src/document.js +93 -11
- package/src/dream/sleep.js +122 -8
- package/src/dream.js +1354 -1347
- package/src/index.js +115 -14
- package/src/inject.js +6 -2
- package/src/lang.js +15 -4
- package/src/local-embedder.js +44 -6
- package/src/maintenance.js +142 -0
- package/src/mirror.js +147 -15
- package/src/recall-stats.js +52 -2
- package/src/scope.js +34 -6
- package/src/service.js +86 -7
- package/src/settings.js +6 -3
- package/src/store.js +248 -21
- package/src/tools.js +27 -9
- package/src/write-admission.js +259 -0
- package/test/api.test.js +15 -2
- package/test/client.test.js +90 -5
- package/test/content-hash.test.js +204 -0
- package/test/document-dir.test.js +233 -0
- package/test/document.test.js +151 -3
- package/test/inject-parent-gate.test.js +106 -0
- package/test/inject-pools.test.js +8 -3
- package/test/lastrunat-persist.test.js +101 -0
- package/test/llm-audit.test.js +136 -0
- package/test/local-embedder.test.js +45 -2
- package/test/maintenance.test.js +391 -0
- package/test/mirror-types.test.js +208 -0
- package/test/recall-evals.test.js +18 -0
- package/test/recall-stats.test.js +66 -0
- package/test/sleep.test.js +117 -0
- package/test/write-admission.test.js +345 -0
package/lib/index.js
CHANGED
|
@@ -1,6 +1,10 @@
|
|
|
1
1
|
import { createStore } from "./store.js";
|
|
2
2
|
import { createMirror, TYPE_FILE } from "./mirror.js";
|
|
3
|
+
import { createDocumentIndex } from "./document-index.js";
|
|
4
|
+
import { resolveDocumentDir } from "./document.js";
|
|
3
5
|
import { createService } from "./service.js";
|
|
6
|
+
// #254 写入准入(第一阶段只计量,不拦截):见 src/write-admission.js 的文件头。
|
|
7
|
+
import { createWriteAdmission } from "./write-admission.js";
|
|
4
8
|
import { createTools } from "./tools.js";
|
|
5
9
|
import { createInjector } from "./inject.js";
|
|
6
10
|
import { createSummarizer } from "./summarize.js";
|
|
@@ -8,6 +12,8 @@ import { createDreamScheduler } from "./dream.js";
|
|
|
8
12
|
import { createSleepScheduler, runSleep } from "./dream/sleep.js";
|
|
9
13
|
import { createApi } from "./api.js";
|
|
10
14
|
import { createStandaloneApi } from "./api-standalone.js";
|
|
15
|
+
// #275 存储生命周期第一批:无损回收(手动入口,不挂启动路径)。
|
|
16
|
+
import { createMaintenance } from "./maintenance.js";
|
|
11
17
|
import { createSettings } from "./settings.js";
|
|
12
18
|
import { createCommandManager } from "./commands.js";
|
|
13
19
|
import { createEmbedder } from "./embedding.js";
|
|
@@ -43,7 +49,15 @@ export { Config };
|
|
|
43
49
|
// unit-testable; the extractor only ever sees a callLLM(messages, options)
|
|
44
50
|
// => Promise<string>. The route always carries a real provider/model (dsh-llm
|
|
45
51
|
// GenerateOptions requires both) — never a bare stream.
|
|
46
|
-
|
|
52
|
+
//
|
|
53
|
+
// Issue #250: it also accounts for itself. Entity extraction is the background
|
|
54
|
+
// LLM path that runs on every memory write, yet it never reached
|
|
55
|
+
// llm_audit_logs — the adapter only ever held an llm handle, so it had no way
|
|
56
|
+
// to call service.saveLlmAudit. It now takes service (+ config for the shared
|
|
57
|
+
// llmAudit.enabled gate) and writes one row per stream attempt, matching the
|
|
58
|
+
// contract of runAuditedLlm in dream.js: a rejected effort attempt records its
|
|
59
|
+
// own error row and the retry records its own success row.
|
|
60
|
+
export function createEntityStreamAdapter({ llm, agentDefaultModel, logger, service, config }) {
|
|
47
61
|
return async function streamEntityText(messages, options = {}) {
|
|
48
62
|
let route = {};
|
|
49
63
|
if (options.provider) route.provider = options.provider;
|
|
@@ -58,8 +72,38 @@ export function createEntityStreamAdapter({ llm, agentDefaultModel, logger }) {
|
|
|
58
72
|
} catch { /* fall through to whatever route we already have */ }
|
|
59
73
|
}
|
|
60
74
|
const effort = options.reasoningEffort;
|
|
75
|
+
// 没有解析出 provider/model 就没有可归属的模型——与 dream 的 resolveRoute
|
|
76
|
+
// 无路由早退同口径,那种情况不写审计行。
|
|
77
|
+
const modelId = route.provider && route.model ? `${route.provider}:${route.model}` : "";
|
|
61
78
|
const tryStream = (withEffort) => {
|
|
62
79
|
let text = "";
|
|
80
|
+
let inputTokens = 0;
|
|
81
|
+
let outputTokens = 0;
|
|
82
|
+
const startedAt = Date.now();
|
|
83
|
+
const timestamp = new Date(startedAt).toISOString();
|
|
84
|
+
// 记账是 best-effort:写审计行失败只 warn,绝不反噬抽取本身(CONTRIBUTING
|
|
85
|
+
// 的 fail-safe 硬约定)。
|
|
86
|
+
const writeAudit = (status, errorMessage) => {
|
|
87
|
+
if (config?.llmAudit?.enabled === false || !modelId || typeof service?.saveLlmAudit !== "function") return;
|
|
88
|
+
try {
|
|
89
|
+
service.saveLlmAudit({
|
|
90
|
+
timestamp,
|
|
91
|
+
trigger_source: "entityExtract",
|
|
92
|
+
operation_type: "entity_extract",
|
|
93
|
+
model_id: modelId,
|
|
94
|
+
input_tokens: inputTokens,
|
|
95
|
+
output_tokens: outputTokens,
|
|
96
|
+
total_tokens: inputTokens + outputTokens,
|
|
97
|
+
cost_usd: 0,
|
|
98
|
+
duration_ms: Date.now() - startedAt,
|
|
99
|
+
status,
|
|
100
|
+
error_message: errorMessage,
|
|
101
|
+
related_memory_ids: []
|
|
102
|
+
});
|
|
103
|
+
} catch (auditError) {
|
|
104
|
+
logger?.warn?.(`dsh-mneme: entity extraction llm audit write failed: ${String(auditError)}`);
|
|
105
|
+
}
|
|
106
|
+
};
|
|
63
107
|
return (async () => {
|
|
64
108
|
for await (const chunk of llm.stream({
|
|
65
109
|
...route,
|
|
@@ -68,11 +112,26 @@ export function createEntityStreamAdapter({ llm, agentDefaultModel, logger }) {
|
|
|
68
112
|
messages
|
|
69
113
|
})) {
|
|
70
114
|
if (chunk.type === "text-delta" && typeof chunk.text === "string") text += chunk.text;
|
|
71
|
-
|
|
115
|
+
// DSH 的 StreamChunk 契约把用量嵌在 chunk.usage(TokenUsage);兼容直接
|
|
116
|
+
// 平铺在 chunk 上的旧形态。归一放在这里,下面只需认平铺形状——与
|
|
117
|
+
// dream.js 的 streamText 同一处理。
|
|
118
|
+
if (chunk.type === "usage") {
|
|
119
|
+
const u = chunk.usage ?? chunk;
|
|
120
|
+
const i = u.input_tokens ?? u.inputTokens ?? u.prompt_tokens ?? u.promptTokens;
|
|
121
|
+
const o = u.output_tokens ?? u.outputTokens ?? u.completion_tokens ?? u.completionTokens;
|
|
122
|
+
if (Number.isFinite(i)) inputTokens = i;
|
|
123
|
+
if (Number.isFinite(o)) outputTokens = o;
|
|
124
|
+
}
|
|
125
|
+
if (chunk.type === "finish" && (chunk.reason?.kind === "error" || chunk.reason?.kind === "aborted")) {
|
|
126
|
+
writeAudit("error", "llm stream aborted or errored");
|
|
127
|
+
return undefined;
|
|
128
|
+
}
|
|
72
129
|
}
|
|
130
|
+
writeAudit("success", null);
|
|
73
131
|
return text;
|
|
74
132
|
})().catch((err) => {
|
|
75
133
|
logger?.warn?.(`[dsh-mneme] entity extraction llm stream failed: ${String(err)}`);
|
|
134
|
+
writeAudit("error", String(err?.message ?? err));
|
|
76
135
|
return undefined;
|
|
77
136
|
});
|
|
78
137
|
};
|
|
@@ -140,15 +199,6 @@ export const apply = (ctx, config) => {
|
|
|
140
199
|
try {
|
|
141
200
|
store.deleteOldFailures(new Date(Date.now() - 90 * 86400000).toISOString());
|
|
142
201
|
} catch { /* non-fatal */ }
|
|
143
|
-
// Bug8: enforce llm_audit_logs retention on boot (config.llmAudit.retentionDays,
|
|
144
|
-
// default 90). Best-effort like the failure prune — the audit trail is
|
|
145
|
-
// bookkeeping and a failed purge must never block plugin boot.
|
|
146
|
-
try {
|
|
147
|
-
if (rawCfg.llmAudit?.enabled !== false) {
|
|
148
|
-
const retentionMs = Number.isInteger(rawCfg.llmAudit?.retentionDays) ? rawCfg.llmAudit.retentionDays : 90;
|
|
149
|
-
store.deleteOldLlmAudits(new Date(Date.now() - retentionMs * 86400000).toISOString());
|
|
150
|
-
}
|
|
151
|
-
} catch { /* non-fatal */ }
|
|
152
202
|
|
|
153
203
|
// User-configurable settings (profile, rules, panel mode, standalone API
|
|
154
204
|
// token) share the same SQLite file in dedicated tables, isolated from
|
|
@@ -187,10 +237,46 @@ export const apply = (ctx, config) => {
|
|
|
187
237
|
cfg[objKey] = { ...(cfg[objKey] ?? {}), ...sub };
|
|
188
238
|
}
|
|
189
239
|
|
|
240
|
+
// Bug8 的启动期清理放在装配之后:面板把 llmAudit.* 写进 kv、经 nestedFlags 合进
|
|
241
|
+
// cfg,而写入侧(dream / summarize / 写入准入)读的都是装配后的 cfg——清理若读
|
|
242
|
+
// rawCfg,面板改了保留期它不认,更糟的是「开不开审计」与写入侧可能取到不同的值
|
|
243
|
+
// (raw 说关 → 不清理,cfg 说开 → 照写,审计表就无保留期地长)。保留期默认 90 天、
|
|
244
|
+
// 失败只 warn,与失败表清理同款:账本清理绝不许挡住插件启动。
|
|
245
|
+
try {
|
|
246
|
+
if (cfg.llmAudit?.enabled !== false) {
|
|
247
|
+
const retentionMs = Number.isInteger(cfg.llmAudit?.retentionDays) ? cfg.llmAudit.retentionDays : 90;
|
|
248
|
+
store.deleteOldLlmAudits(new Date(Date.now() - retentionMs * 86400000).toISOString());
|
|
249
|
+
}
|
|
250
|
+
} catch { /* non-fatal */ }
|
|
251
|
+
|
|
190
252
|
// 记忆语言(memory.language):本实例逐层传入 inject / summarize / dream /
|
|
191
253
|
// sleep / mirror,多实例(如 agent preset 内挂载)互不影响。
|
|
192
254
|
const mirror = createMirror(memoryDir, langOf(cfg));
|
|
193
|
-
|
|
255
|
+
|
|
256
|
+
// #296 第二批:managed 文档目录。解析与 memoryDir 同一套(空 = 跟随 memoryDir 的
|
|
257
|
+
// <memoryDir>/documents/,~ 展开,相对路径落在 memoryDir 下)。索引对象总是建:
|
|
258
|
+
// 闸关时它只承担「清掉陈旧 index.md」这一件事,与 documents.md 在闸关时被镜像删
|
|
259
|
+
// 掉同一口径。建目录与写索引失败都只 warn,不阻断插件加载(它们是机器产物)。
|
|
260
|
+
const documentDir = resolveDocumentDir(memoryDir, cfg.documentDir);
|
|
261
|
+
const documentIndex = createDocumentIndex(documentDir, langOf(cfg));
|
|
262
|
+
if (cfg.documentMemoryEnabled === true) {
|
|
263
|
+
const ensured = documentIndex.ensure();
|
|
264
|
+
if (!ensured.ok) {
|
|
265
|
+
ctx.logger?.warn?.(`[dsh-mneme] documentDir is not writable: ${documentDir}: ${ensured.error}`);
|
|
266
|
+
}
|
|
267
|
+
// 启动即渲染一次:索引不该等到第一次业务写才存在(删掉它之后重启也能自愈)。
|
|
268
|
+
const synced = documentIndex.sync(store.list({ type: "document", limit: null }));
|
|
269
|
+
if (!synced.ok) ctx.logger?.warn?.(`[dsh-mneme] document index sync failed: ${synced.error}`);
|
|
270
|
+
} else {
|
|
271
|
+
documentIndex.remove();
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
// 写入准入实例:本批次只做计量(决策恒放行、写审计行),所以不需要新开关——它
|
|
275
|
+
// 不改变任何写入行为,也不新增拦截分支;既有的 llmAudit.enabled 关掉时它同样
|
|
276
|
+
// 不写(那个开关连审计行的启动期清理一起关掉)。第二阶段把拦截打开时才按仓库
|
|
277
|
+
// 惯例引入 opt-in 默认关的配置键,届时只改这一个实例的构造与 service 的调用点。
|
|
278
|
+
const writeAdmission = createWriteAdmission({ store, config: cfg, logger: ctx.logger });
|
|
279
|
+
const service = createService({ store, mirror, config: cfg, logger: ctx.logger, documentIndex, writeAdmission });
|
|
194
280
|
|
|
195
281
|
// F-NEW-03: if the mirror sync failed last run (persisted dirty state), retry
|
|
196
282
|
// a safe re-render at boot so a stale mirror converges without needing a
|
|
@@ -273,6 +359,9 @@ export const apply = (ctx, config) => {
|
|
|
273
359
|
dimension: cfg.localEmbedDimension,
|
|
274
360
|
device: cfg.localEmbedDevice,
|
|
275
361
|
batchSize: cfg.localEmbedBatchSize,
|
|
362
|
+
// 池化方式必须与模型的训练口径一致(BGE 系 = CLS)。它既进 embed() 的调用,
|
|
363
|
+
// 也进 modelHash —— 池化改了就是换向量空间,既有索引会被判失配并重建。
|
|
364
|
+
pooling: cfg.localEmbedPooling,
|
|
276
365
|
cacheDir: cfg.embedModelCacheDir,
|
|
277
366
|
runtimeDir: cfg.runtimeDir,
|
|
278
367
|
// #188:embedModelMirror 接成 transformers 的下载镜像(此前死配置)。
|
|
@@ -403,6 +492,7 @@ export const apply = (ctx, config) => {
|
|
|
403
492
|
minIntervalMs: (cfg.dreamMinIntervalMinutes ?? 0) * 60000,
|
|
404
493
|
logger: ctx.logger,
|
|
405
494
|
semantic: { embedder, vectorIndex },
|
|
495
|
+
lastRunAtSeed: store.lastDreamRunAt("auto"),
|
|
406
496
|
onRun: () => (dream ? dream.runDream(ctx, service, cfg) : Promise.resolve({ ok: true, skipped: true }))
|
|
407
497
|
});
|
|
408
498
|
service.setDreamHook(() => dream.maybeSchedule(service));
|
|
@@ -419,6 +509,7 @@ export const apply = (ctx, config) => {
|
|
|
419
509
|
service,
|
|
420
510
|
config: cfg,
|
|
421
511
|
logger: ctx.logger,
|
|
512
|
+
lastRunAtSeed: store.lastDreamRunAt("sleep"),
|
|
422
513
|
onRun: (signal) => (sleep ? runSleep(ctx, service, cfg, ctx.logger, { embedder, vectorIndex }, signal) : Promise.resolve({ ok: true, skipped: true }))
|
|
423
514
|
});
|
|
424
515
|
service.setSleepHook(() => sleep.noteWrite());
|
|
@@ -442,7 +533,11 @@ export const apply = (ctx, config) => {
|
|
|
442
533
|
const streamEntityText = createEntityStreamAdapter({
|
|
443
534
|
llm: ctx.llm,
|
|
444
535
|
agentDefaultModel: ctx.agentDefaultModel,
|
|
445
|
-
logger: ctx.logger
|
|
536
|
+
logger: ctx.logger,
|
|
537
|
+
// Issue #250: the adapter needs service to write its llm_audit_logs row
|
|
538
|
+
// (it never had it) and config for the shared llmAudit.enabled gate.
|
|
539
|
+
service,
|
|
540
|
+
config: cfg
|
|
446
541
|
});
|
|
447
542
|
service.setEntityExtractor((memory) =>
|
|
448
543
|
extractEntities(memory, { store, config: cfg, callLLM: streamEntityText, logger: ctx.logger })
|
|
@@ -487,13 +582,19 @@ export const apply = (ctx, config) => {
|
|
|
487
582
|
disposers.push(api.dispose);
|
|
488
583
|
}
|
|
489
584
|
|
|
585
|
+
// #275 存储生命周期第一批:无损回收的手动入口。刻意不挂启动路径、不接定时器——
|
|
586
|
+
// 这一步是不可逆的内容丢弃,只由人显式触发(`dsh-mneme reclaim`)。实例在这里建、
|
|
587
|
+
// 在 standalone API 上暴露,是为了让 CLI 能在插件进程内跑(VACUUM 要排他锁,跟宿主
|
|
588
|
+
// 抢锁的那条路走不通)。
|
|
589
|
+
const maintenance = createMaintenance({ store, config: cfg, logger: ctx.logger });
|
|
590
|
+
|
|
490
591
|
// Standalone external API (v0.7.12): plain node:http server for ecosystem
|
|
491
592
|
// integrations outside the DSH host. Persisted external_api settings win
|
|
492
593
|
// over the bundle config (enabled/port); the Bearer token lives in the same
|
|
493
594
|
// kv and is auto-generated on first boot by createStandaloneApi. Binding a
|
|
494
595
|
// non-loopback host is the operator's documented responsibility.
|
|
495
596
|
if ((settings.getExternalApi?.()?.enabled ?? cfg.externalApiEnabled) === true) {
|
|
496
|
-
const standalone = createStandaloneApi({ service, store, config: cfg, logger: ctx.logger, settings });
|
|
597
|
+
const standalone = createStandaloneApi({ service, store, config: cfg, logger: ctx.logger, settings, maintenance });
|
|
497
598
|
disposers.push(() => standalone.server.close());
|
|
498
599
|
standalone.ready.catch((error) => {
|
|
499
600
|
ctx.logger?.warn?.(`[dsh-mneme] standalone API failed to start: ${String(error)}`);
|
package/lib/inject.js
CHANGED
|
@@ -3,6 +3,7 @@ import { createHotMemory } from "./hot-memory.js";
|
|
|
3
3
|
import { STR, langOf } from "./lang.js";
|
|
4
4
|
import { MEMORY_GUIDE_SECTION } from "./guide.js";
|
|
5
5
|
import { adaptiveInjectBudget } from "./search/adaptive.js";
|
|
6
|
+
import { injectChildEnabled } from "./config.js";
|
|
6
7
|
|
|
7
8
|
// Issue #179:注入预览的数据底座。systemPrompt 渲染是同步回调,面板只能事后
|
|
8
9
|
// 拉取,所以在这里旁路缓存「最近一帧组装」——快照就是本次渲染用过的同一份
|
|
@@ -283,12 +284,15 @@ export function createInjector(ctx, service, settings, config) {
|
|
|
283
284
|
return escapePromptVars(lines.join("\n"));
|
|
284
285
|
}
|
|
285
286
|
|
|
286
|
-
// #249
|
|
287
|
+
// #249 能力说明的系统提示段。order 150 = 插件指引段的既有惯例
|
|
287
288
|
// (ACP 的 ACP_SYSTEM_PROMPT 与 mnemon 的 routing 段都在 150)。文本是常量、
|
|
288
289
|
// 同会话内不随轮次变化,故不影响其后的前缀缓存,也不逐轮复读。宿主若不提供
|
|
289
290
|
// section seam 就静默跳过——#249 §3 的口径是「只用现成 seam,拿不到位的语义
|
|
290
291
|
// 降级处理」:能力说明仍落在工具描述上,不算失败。
|
|
291
|
-
|
|
292
|
+
// 走父/子闸门(#249 第二批):注入器本身只在 `autoInject` 开时挂载,这里是
|
|
293
|
+
// 第二道、也是同一个判据的显式落点——两处消费点共用 injectChildEnabled,
|
|
294
|
+
// 不在各自的地方重写「父关则子不生效」。
|
|
295
|
+
const guideSection = injectChildEnabled(config, "injectGuidanceEnabled") && typeof ctx.systemPrompt?.section === "function"
|
|
292
296
|
? ctx.systemPrompt.section({ name: "memory-guide", order: 150, text: MEMORY_GUIDE_SECTION })
|
|
293
297
|
: null;
|
|
294
298
|
|
package/lib/lang.js
CHANGED
|
@@ -196,12 +196,23 @@ export const STR = {
|
|
|
196
196
|
|
|
197
197
|
// --- mirror.js:镜像文件标签(渲染随实例语言;解析两种语言都认) ----------------
|
|
198
198
|
mirrorLabel: {
|
|
199
|
-
zh: { type: "类型", importance: "重要性", tags: "标签", updated: "更新时间", source: "来源" },
|
|
200
|
-
en: { type: "Type", importance: "Importance", tags: "Tags", updated: "Updated", source: "Source" }
|
|
199
|
+
zh: { type: "类型", importance: "重要性", tags: "标签", updated: "更新时间", source: "来源", scope: "作用域", sensitivity: "敏感度" },
|
|
200
|
+
en: { type: "Type", importance: "Importance", tags: "Tags", updated: "Updated", source: "Source", scope: "Scope", sensitivity: "Sensitivity" }
|
|
201
201
|
},
|
|
202
202
|
mirrorHeader: {
|
|
203
|
-
zh: (name) => `# ${name} — dsh-mneme 镜像\n\n<!--
|
|
204
|
-
en: (name) => `# ${name} — dsh-mneme mirror\n\n<!--
|
|
203
|
+
zh: (name) => `# ${name} — dsh-mneme 镜像\n\n<!-- 条目标题与正文可编辑,会被合并回记忆库(人工优先);文件头与条目元数据行由机器维护,改动会在下次同步时被覆盖。 -->\n\n`,
|
|
204
|
+
en: (name) => `# ${name} — dsh-mneme mirror\n\n<!-- Entry titles and bodies are editable and merged back into the memory store (human edits win); the file header and entry metadata lines are machine-owned and get overwritten on the next sync. -->\n\n`
|
|
205
|
+
},
|
|
206
|
+
// document 的镜像只有指针行(#296 第二批):没有可编辑的正文,手工改动一律被
|
|
207
|
+
// 下次同步覆盖——所以不能复用上面那句「可编辑、会被合并回记忆库」。
|
|
208
|
+
mirrorReadonlyHeader: {
|
|
209
|
+
zh: (name) => `# ${name} — dsh-mneme 只读视图\n\n<!-- 只含指针行:id + 标题 + 摘要首句 + 文件路径,不含正文。正文在路径指向的文件里,批注请写进记忆库;本文件由机器维护,手工改动会在下次同步时被覆盖。 -->\n\n`,
|
|
210
|
+
en: (name) => `# ${name} — dsh-mneme read-only view\n\n<!-- Pointer rows only: id + title + first sentence of the summary + file path, never the full text. The document itself lives at that path, annotations belong in the memory store, and this file is machine-owned: hand edits are overwritten on the next sync. -->\n\n`
|
|
211
|
+
},
|
|
212
|
+
// documentDir 的 index.md(#296 第二批):整文件机器所有、可从库重建。
|
|
213
|
+
documentIndexHeader: {
|
|
214
|
+
zh: () => "# document 索引 — dsh-mneme\n\n<!-- 整文件机器所有,可从记忆库随时重建(所以这里不写生成时间);要批注请写进记忆库(memory_save)。managed = 文件在 documentDir 内。 -->\n\n",
|
|
215
|
+
en: () => "# document index — dsh-mneme\n\n<!-- Machine-owned as a whole and rebuildable from the memory store at any time (which is why it carries no generation timestamp). Annotations belong in the memory store (memory_save). managed = the file sits inside documentDir. -->\n\n"
|
|
205
216
|
},
|
|
206
217
|
|
|
207
218
|
prompts: PROMPTS
|
package/lib/local-embedder.js
CHANGED
|
@@ -21,6 +21,32 @@ function modelHash(model) {
|
|
|
21
21
|
return `${model}#${hashString(model)}`;
|
|
22
22
|
}
|
|
23
23
|
|
|
24
|
+
/**
|
|
25
|
+
* Model families whose embedding is the CLS token rather than a mean over tokens.
|
|
26
|
+
*
|
|
27
|
+
* BGE (BAAI/bge-*) is trained that way: the model's own `1_Pooling/config.json`
|
|
28
|
+
* declares `pooling_mode_cls_token: true` / `pooling_mode_mean_tokens: false`, and
|
|
29
|
+
* the official README says "select the last hidden state of the first token" plus
|
|
30
|
+
* L2 normalization. transformers.js's `feature-extraction` defaults to `mean`, so
|
|
31
|
+
* feeding it a BGE model silently produces slightly-off vectors — no error, just a
|
|
32
|
+
* worse ranking. This table is the extension point: a family goes in only once its
|
|
33
|
+
* pooling is confirmed against the model's own `1_Pooling` (or equivalent) config.
|
|
34
|
+
*/
|
|
35
|
+
const CLS_POOLED_MODEL_PATTERNS = [/bge/i];
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* Resolve the pooling to embed with.
|
|
39
|
+
*
|
|
40
|
+
* `auto` (default) picks by model family — BGE → `cls`, everything else → `mean`,
|
|
41
|
+
* so models that were never affected keep their exact previous behavior. An
|
|
42
|
+
* explicit `cls` / `mean` wins over the family table.
|
|
43
|
+
*/
|
|
44
|
+
export function resolvePooling(pooling, model) {
|
|
45
|
+
const mode = String(pooling ?? "auto").trim().toLowerCase();
|
|
46
|
+
if (mode === "cls" || mode === "mean") return mode;
|
|
47
|
+
return CLS_POOLED_MODEL_PATTERNS.some((re) => re.test(String(model ?? ""))) ? "cls" : "mean";
|
|
48
|
+
}
|
|
49
|
+
|
|
24
50
|
/**
|
|
25
51
|
* Lazy default loader: dynamic import keeps module load cheap.
|
|
26
52
|
*
|
|
@@ -71,8 +97,9 @@ function tensorToRows(tensor) {
|
|
|
71
97
|
|
|
72
98
|
/**
|
|
73
99
|
* ONNX text embedder backed by transformers.js (onnxruntime-node underneath).
|
|
74
|
-
* Runs fully offline
|
|
75
|
-
*
|
|
100
|
+
* Runs fully offline; pooling is resolved per model family (BGE → CLS, else mean —
|
|
101
|
+
* see resolvePooling) and vectors are L2-normalized. `engineFactory` is injectable
|
|
102
|
+
* for tests.
|
|
76
103
|
*/
|
|
77
104
|
export class LocalEmbedder {
|
|
78
105
|
constructor(opts = {}) {
|
|
@@ -87,6 +114,10 @@ export class LocalEmbedder {
|
|
|
87
114
|
// (含测试)保持裸 fetch 的现状,配置默认值由 index.js 传进来。
|
|
88
115
|
this.resilientModelDownload = opts.resilientModelDownload === true;
|
|
89
116
|
this.useDtype = opts.useDtype || "q8";
|
|
117
|
+
// 池化:配置值(auto/cls/mean)→ 生效值(cls/mean)。解析结果既进 embed(),
|
|
118
|
+
// 也进 modelHash —— 换池化就是换向量空间,旧索引必须被判失配后重建。
|
|
119
|
+
this.poolingMode = String(opts.pooling ?? "auto").trim().toLowerCase();
|
|
120
|
+
this.pooling = resolvePooling(this.poolingMode, this.model);
|
|
90
121
|
// 模型镜像下载源(embedModelMirror,#188 补充):空 = transformers 默认。
|
|
91
122
|
this.remoteHost = String(opts.remoteHost ?? "").trim();
|
|
92
123
|
this.logger = opts.logger ?? null;
|
|
@@ -114,19 +145,19 @@ export class LocalEmbedder {
|
|
|
114
145
|
this.extractor = await this.engineFactory("feature-extraction", this.model, options);
|
|
115
146
|
this.ready = true; // service reads this to flush queued re-embeds
|
|
116
147
|
this.logger?.info?.(
|
|
117
|
-
`[dsh-mneme] local embedder ready: ${this.model} (dim=${this._dimension}, device=${this.device})`
|
|
148
|
+
`[dsh-mneme] local embedder ready: ${this.model} (dim=${this._dimension}, device=${this.device}, pooling=${this.pooling})`
|
|
118
149
|
);
|
|
119
150
|
return this;
|
|
120
151
|
}
|
|
121
152
|
|
|
122
|
-
/** Embed many texts with
|
|
153
|
+
/** Embed many texts with the model's pooling; chunks at batchSize. */
|
|
123
154
|
async embed(texts) {
|
|
124
155
|
if (!Array.isArray(texts)) throw new TypeError("embed expects an array of strings");
|
|
125
156
|
if (!this.extractor) throw new Error("LocalEmbedder not initialized");
|
|
126
157
|
const out = [];
|
|
127
158
|
for (let i = 0; i < texts.length; i += this.batchSize) {
|
|
128
159
|
const chunk = texts.slice(i, i + this.batchSize);
|
|
129
|
-
const tensor = await this.extractor(chunk, { pooling:
|
|
160
|
+
const tensor = await this.extractor(chunk, { pooling: this.pooling, normalize: true });
|
|
130
161
|
out.push(...tensorToRows(tensor));
|
|
131
162
|
}
|
|
132
163
|
return out;
|
|
@@ -142,7 +173,14 @@ export class LocalEmbedder {
|
|
|
142
173
|
}
|
|
143
174
|
|
|
144
175
|
get modelHash() {
|
|
145
|
-
|
|
176
|
+
// Pooling is part of the vector space: a CLS index cannot be queried with mean
|
|
177
|
+
// vectors (or vice versa). Keep the historical `model#hash` shape for `mean` so
|
|
178
|
+
// indexes that were never affected are not invalidated needlessly; every other
|
|
179
|
+
// pooling gets a fingerprint of its own, which makes the index-consistency gate
|
|
180
|
+
// flag the stale mean-space index and rebuild it.
|
|
181
|
+
return this.pooling === "mean"
|
|
182
|
+
? modelHash(this.model)
|
|
183
|
+
: modelHash(`${this.model}@${this.pooling}`);
|
|
146
184
|
}
|
|
147
185
|
|
|
148
186
|
dispose() {
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
// #275 存储生命周期第一批:无损回收的维护入口。
|
|
2
|
+
//
|
|
3
|
+
// 为什么是「手动入口 + dry-run + receipt」而不是后台任务:这两步都是不可逆的内容
|
|
4
|
+
// 丢弃(历史 run 的输入快照置空、归档行的向量置空),该由人在看得见数字的时候按下
|
|
5
|
+
// (维护者 2026-09-23 拍板:不挂启动自动跑,VACUUM 约 0.8s 说明它就该是个随手跑的
|
|
6
|
+
// 东西)。所以模块里没有任何定时器,挂点只有两个:standalone 数据面的
|
|
7
|
+
// `POST /maintenance/reclaim` 与它上面的 `dsh-mneme reclaim` 子命令。
|
|
8
|
+
//
|
|
9
|
+
// 两项都零价值判断、零条数变化:
|
|
10
|
+
// A `dream_runs.input` 置空。这一列的含义按 run_type 分叉:auto / sleep 的 run 存的
|
|
11
|
+
// 是当时的记忆库快照,可由记忆库重建;**organize 的 run 存的是 apply 的重放载荷**,
|
|
12
|
+
// 置空会让那次 apply 静默空转并把报告锁死,所以 organize 行不在可清范围内(谓词
|
|
13
|
+
// 在 store.dreamRunInputStats / clearDreamRunInputs 里)。run 的骨架、LLM 决策原文
|
|
14
|
+
// 与 receipt 一律保留,行永不删(拍板 3:裁列同意、删行不同意)。
|
|
15
|
+
// B 归档行向量置空:检索 SQL 恒带 `archived = 0`,这部分向量按定义不可达。它不是
|
|
16
|
+
// 单程票——取消归档时 service 会重新排队嵌入(archiveMemory 的还原分支)。
|
|
17
|
+
//
|
|
18
|
+
// 报告口径(拍板 2):收益按 VACUUM 前后体积量,不按列字节估——实际释放来自溢出页与
|
|
19
|
+
// 索引页回收,列文本大小只是上界,所以两个数分开报,不混成一个「省了多少」。
|
|
20
|
+
//
|
|
21
|
+
// 与 #254 的分界(拍板 4):本入口止体积、不止重复。归档区里同主题堆积是出口问题,
|
|
22
|
+
// 归 #254 的写入准入与后续的整理动作面,这里一条记忆都不碰。
|
|
23
|
+
export const RECLAIM_TRIGGER_SOURCE = "maintenance";
|
|
24
|
+
export const RECLAIM_OPERATION_TYPE = "storage_reclaim";
|
|
25
|
+
// 输入快照的默认保留窗口:7 天内仍可能被离线回放(近期 run 的决策上下文),更早的
|
|
26
|
+
// 重建成本低于留存成本。窗口由调用方显式给,不做成配置键——配置键意味着有个周期性
|
|
27
|
+
// 消费者,而本批次刻意不接周期动作。
|
|
28
|
+
export const DEFAULT_INPUT_RETENTION_DAYS = 7;
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* 无损回收(#275 第一批)。返回一个函数:同一条代码路径跑 dry-run 与执行,dry-run
|
|
32
|
+
* 只是不落 UPDATE/VACUUM,所以「报出来的数字」与「执行时的数字」出自同一处统计。
|
|
33
|
+
*
|
|
34
|
+
* @param {object} deps
|
|
35
|
+
* @param {object} deps.store 存储层(dreamRunInputStats / clearDreamRunInputs /
|
|
36
|
+
* archivedEmbeddingStats / clearArchivedEmbeddings / storageStats / vacuum / saveLlmAudit)
|
|
37
|
+
* @param {object} [deps.config] 已解析配置(读 llmAudit.enabled:关掉 audit 时不落 receipt)
|
|
38
|
+
* @param {object} [deps.logger]
|
|
39
|
+
* @param {() => number} [deps.now] 时钟注入(测试用)
|
|
40
|
+
*/
|
|
41
|
+
export function createMaintenance({ store, config, logger, now = Date.now } = {}) {
|
|
42
|
+
const warn = (msg) => {
|
|
43
|
+
try {
|
|
44
|
+
logger?.warn?.(msg);
|
|
45
|
+
} catch { /* 日志故障不能影响回收本身 */ }
|
|
46
|
+
};
|
|
47
|
+
|
|
48
|
+
/** receipt 落到 llm_audit_logs(既有 bookkeeping 表,不触发写钩子)。 */
|
|
49
|
+
function writeReceipt(report) {
|
|
50
|
+
if (config?.llmAudit?.enabled === false) return null;
|
|
51
|
+
try {
|
|
52
|
+
const row = store.saveLlmAudit({
|
|
53
|
+
trigger_source: RECLAIM_TRIGGER_SOURCE,
|
|
54
|
+
operation_type: RECLAIM_OPERATION_TYPE,
|
|
55
|
+
// 不是模型调用,但该列 NOT NULL——用显式占位串,免得混进按路由分组的花费统计。
|
|
56
|
+
model_id: "-",
|
|
57
|
+
status: "success",
|
|
58
|
+
metadata: {
|
|
59
|
+
older_than_days: report.olderThanDays,
|
|
60
|
+
cleared: report.cleared,
|
|
61
|
+
size_before: report.size.before_bytes,
|
|
62
|
+
size_after: report.size.after_bytes,
|
|
63
|
+
vacuum: report.vacuum.ran,
|
|
64
|
+
duration_ms: report.duration_ms
|
|
65
|
+
}
|
|
66
|
+
});
|
|
67
|
+
return row?.id ?? null;
|
|
68
|
+
} catch (e) {
|
|
69
|
+
// 回执失败只 warn:清理已经发生了,不能因为记不上账就当作没发生(也不能反噬)。
|
|
70
|
+
warn(`[dsh-mneme] storage reclaim receipt failed: ${String(e)}`);
|
|
71
|
+
return null;
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* @param {object} [opts]
|
|
77
|
+
* @param {number} [opts.olderThanDays] 输入快照的保留窗口(天);0 = 清掉全部快照
|
|
78
|
+
* @param {boolean} [opts.vacuum] 是否在清理后 VACUUM(默认否:它要排他锁,由人决定)
|
|
79
|
+
* @param {boolean} [opts.dryRun] 默认 true——只报数字,不改一个字节
|
|
80
|
+
* @returns {object} 报告(dry-run 与执行同形状,执行时多 receipt 字段)
|
|
81
|
+
*/
|
|
82
|
+
function reclaim({ olderThanDays = DEFAULT_INPUT_RETENTION_DAYS, vacuum = false, dryRun = true } = {}) {
|
|
83
|
+
const days = Number.isFinite(olderThanDays) && olderThanDays >= 0 ? olderThanDays : DEFAULT_INPUT_RETENTION_DAYS;
|
|
84
|
+
const cutoff = new Date(now() - days * 86400000).toISOString();
|
|
85
|
+
const started = now();
|
|
86
|
+
const before = store.storageStats();
|
|
87
|
+
const inputs = store.dreamRunInputStats(cutoff);
|
|
88
|
+
const embeddings = store.archivedEmbeddingStats();
|
|
89
|
+
const report = {
|
|
90
|
+
dryRun,
|
|
91
|
+
olderThanDays: days,
|
|
92
|
+
cutoff,
|
|
93
|
+
// planned = 这次会动几行;cleared = 实际动了几行(dry-run 恒 0)
|
|
94
|
+
planned: { dream_run_inputs: inputs.runs, archived_embeddings: embeddings.rows },
|
|
95
|
+
// 列文本大小:上界,不是回收量(真实释放看下面的 size 差值)
|
|
96
|
+
column_bytes: { dream_run_inputs: inputs.bytes, archived_embeddings: embeddings.bytes },
|
|
97
|
+
cleared: { dream_run_inputs: 0, archived_embeddings: 0 },
|
|
98
|
+
size: {
|
|
99
|
+
before_bytes: before.fileBytes,
|
|
100
|
+
after_bytes: before.fileBytes,
|
|
101
|
+
// 空闲页就是「现在 VACUUM 能立刻收回来的那部分」,带上它 dry-run 才有个能
|
|
102
|
+
// 参照的量(真实数字仍然只有做完才知道)
|
|
103
|
+
freelist_pages: before.freelistCount,
|
|
104
|
+
freelist_bytes: before.freelistCount * before.pageSize,
|
|
105
|
+
page_count: before.pageCount
|
|
106
|
+
},
|
|
107
|
+
vacuum: { requested: vacuum === true, ran: false, duration_ms: null, error: null },
|
|
108
|
+
receipt: null,
|
|
109
|
+
duration_ms: 0
|
|
110
|
+
};
|
|
111
|
+
if (dryRun) {
|
|
112
|
+
report.duration_ms = now() - started;
|
|
113
|
+
return report;
|
|
114
|
+
}
|
|
115
|
+
report.cleared.dream_run_inputs = store.clearDreamRunInputs(cutoff);
|
|
116
|
+
report.cleared.archived_embeddings = store.clearArchivedEmbeddings();
|
|
117
|
+
if (report.vacuum.requested) {
|
|
118
|
+
// VACUUM 失败不能把前面的清理一起吞掉:两个 UPDATE 已经各自提交了,此时抛出去
|
|
119
|
+
// 就是「数据已经不可逆地清了,但没有任何回执」。所以失败记进报告、回执照写,
|
|
120
|
+
// 让调用方看到「清了 N 行,VACUUM 没跑成」——锁被别的进程占着(SQLITE_BUSY)是
|
|
121
|
+
// 最常见的成因,重跑一次 --apply --vacuum 即可(清理是幂等的)。
|
|
122
|
+
try {
|
|
123
|
+
const result = store.vacuum();
|
|
124
|
+
report.vacuum.ran = true;
|
|
125
|
+
report.vacuum.duration_ms = result.duration_ms;
|
|
126
|
+
} catch (e) {
|
|
127
|
+
report.vacuum.error = String(e);
|
|
128
|
+
warn(`[dsh-mneme] storage reclaim vacuum failed: ${String(e)}`);
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
const after = store.storageStats();
|
|
132
|
+
report.size.after_bytes = after.fileBytes;
|
|
133
|
+
report.size.freelist_pages = after.freelistCount;
|
|
134
|
+
report.size.freelist_bytes = after.freelistCount * after.pageSize;
|
|
135
|
+
report.size.page_count = after.pageCount;
|
|
136
|
+
report.duration_ms = now() - started;
|
|
137
|
+
report.receipt = writeReceipt(report);
|
|
138
|
+
return report;
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
return { reclaim };
|
|
142
|
+
}
|