@modusensus/dsh-mneme 0.8.4 → 0.8.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +45 -8
- package/bin/dsh-mneme-mcp.mjs +11 -3
- package/lib/api-standalone.js +11 -0
- package/lib/api.js +21 -0
- package/lib/client.js +213 -31
- package/lib/config.js +106 -5
- package/lib/document.js +221 -0
- package/lib/dream/sleep.js +128 -12
- package/lib/dream.js +46 -11
- package/lib/guide.js +44 -0
- package/lib/heat.js +2 -0
- package/lib/index.js +65 -3
- package/lib/inject.js +91 -15
- package/lib/lang.js +6 -0
- package/lib/local-embedder.js +44 -6
- package/lib/organize.js +450 -0
- package/lib/recall-stats.js +23 -1
- package/lib/runtime/layout.js +30 -0
- package/lib/runtime/loader.js +4 -1
- package/lib/runtime/verify.js +19 -6
- package/lib/search/adaptive.js +38 -0
- package/lib/service.js +165 -12
- package/lib/settings.js +35 -4
- package/lib/store.js +126 -8
- package/lib/summarize.js +253 -8
- package/lib/tools.js +136 -8
- package/package.json +6 -6
- package/runtime-manifest.json +104 -174
- package/scripts/build-runtime-manifest.mjs +8 -1
- package/scripts/sync-test-badge.mjs +18 -12
- package/src/api-standalone.js +11 -0
- package/src/api.js +21 -0
- package/src/config.js +106 -5
- package/src/document.js +221 -0
- package/src/dream/sleep.js +128 -12
- package/src/dream.js +46 -11
- package/src/guide.js +44 -0
- package/src/heat.js +2 -0
- package/src/index.js +65 -3
- package/src/inject.js +91 -15
- package/src/lang.js +6 -0
- package/src/local-embedder.js +44 -6
- package/src/organize.js +450 -0
- package/src/recall-stats.js +23 -1
- package/src/runtime/layout.js +30 -0
- package/src/runtime/loader.js +4 -1
- package/src/runtime/verify.js +19 -6
- package/src/search/adaptive.js +38 -0
- package/src/service.js +165 -12
- package/src/settings.js +35 -4
- package/src/store.js +126 -8
- package/src/summarize.js +253 -8
- package/src/tools.js +136 -8
- package/test/adaptive-inject-budget.test.js +85 -0
- package/test/api.test.js +66 -3
- package/test/client.test.js +63 -2
- package/test/document.test.js +523 -0
- package/test/dream.test.js +51 -0
- package/test/inject-content-cap.test.js +5 -3
- package/test/inject-pools.test.js +170 -0
- package/test/inject-recall.test.js +105 -0
- package/test/inject.test.js +35 -1
- package/test/lastrunat-persist.test.js +101 -0
- package/test/llm-audit.test.js +247 -0
- package/test/local-embedder.test.js +45 -2
- package/test/organize.test.js +418 -0
- package/test/recall-stats.test.js +7 -1
- package/test/runtime-loader.test.js +18 -0
- package/test/runtime-manifest.test.js +2 -1
- package/test/runtime-verify.test.js +65 -0
- package/test/sleep.test.js +154 -0
- package/test/store.test.js +13 -0
- package/test/summarize.test.js +407 -3
- package/test/tools-dto-schema-matrix.test.js +325 -0
- package/test/tools.test.js +36 -3
package/src/dream/sleep.js
CHANGED
|
@@ -49,12 +49,22 @@ function parseJsonArray(text) {
|
|
|
49
49
|
}
|
|
50
50
|
}
|
|
51
51
|
|
|
52
|
-
/**
|
|
53
|
-
|
|
52
|
+
/**
|
|
53
|
+
* Same stream consumption contract as dream.js.
|
|
54
|
+
*
|
|
55
|
+
* `onUsage`(optional, issue #250)receives any usage chunk for token accounting.
|
|
56
|
+
* dream.js 那份一直有它;sleep 这份副本没有,导致 conflict / pattern 两条链路的
|
|
57
|
+
* token 从未进 llm_audit_logs(面板因此只看到 autoDream 与 autoSummarize)。
|
|
58
|
+
*/
|
|
59
|
+
async function streamText(ctx, options, onUsage, onStreamError) {
|
|
54
60
|
if (!ctx?.llm?.stream) return undefined;
|
|
55
61
|
let text = "";
|
|
56
62
|
for await (const chunk of ctx.llm.stream(options)) {
|
|
57
63
|
if (chunk.type === "text-delta" && typeof chunk.text === "string") text += chunk.text;
|
|
64
|
+
// DSH 的 StreamChunk 契约把用量嵌在 chunk.usage(TokenUsage:inputTokens /
|
|
65
|
+
// outputTokens / totalTokens);兼容直接平铺在 chunk 上的旧形态。归一放在
|
|
66
|
+
// 这里,调用侧的 reporter 只需认平铺形状——与 dream.js:239 同一处理。
|
|
67
|
+
if (chunk.type === "usage" && typeof onUsage === "function") onUsage(chunk.usage ?? chunk);
|
|
58
68
|
if (chunk.type === "finish" && (chunk.reason?.kind === "error" || chunk.reason?.kind === "aborted")) {
|
|
59
69
|
// Same rc.1 error-as-finish-chunk behavior as dream.js — surface the
|
|
60
70
|
// cause instead of discarding it.
|
|
@@ -67,6 +77,79 @@ async function streamText(ctx, options, onStreamError) {
|
|
|
67
77
|
return text;
|
|
68
78
|
}
|
|
69
79
|
|
|
80
|
+
/**
|
|
81
|
+
* Issue #250:给 sleep 的一条 LLM 调用记一笔 llm_audit_logs(token / 时长 / 状态 /
|
|
82
|
+
* 触发源)。与 dream.js 的 runAuditedLlm 同口径但各自独立一份——dream 那两个
|
|
83
|
+
* checker 的调用形状绑定着它自己的排程细节,硬抽共享抽象只会收敛回 dream 的形态。
|
|
84
|
+
*
|
|
85
|
+
* 记账是 best-effort:写审计行失败只 warn,绝不反噬 sleep 本体(CONTRIBUTING 的
|
|
86
|
+
* fail-safe 硬约定)。`body(reportUsage)` 负责真实消费流,usage chunk 经
|
|
87
|
+
* reportUsage 上报。
|
|
88
|
+
*
|
|
89
|
+
* 审计诚实性:流被中止/报错时 body 返回 undefined,记 status='error';流式成功但
|
|
90
|
+
* 输出不可用时 `spec.auditError` 给出原因,同样记 error——否则同一轮调用会在
|
|
91
|
+
* 「phase 报 failed」与「audit 报 success」之间自相矛盾(CONTRIBUTING 的审计诚实性
|
|
92
|
+
* 硬约定)。`auditError` 必须是**纯判据函数**:调用方的控制流不能依赖它的副作用,
|
|
93
|
+
* 因为审计关闭时这里整段早退、钩子根本不会被调用(dream 正是那样踩的,见 PR 正文)。
|
|
94
|
+
*/
|
|
95
|
+
async function auditedSleepLlm(ctx, service, config, logger, spec, body) {
|
|
96
|
+
if (config?.llmAudit?.enabled === false || typeof service?.saveLlmAudit !== "function") {
|
|
97
|
+
return body(() => {});
|
|
98
|
+
}
|
|
99
|
+
const startedAt = Date.now();
|
|
100
|
+
const timestamp = new Date(startedAt).toISOString();
|
|
101
|
+
let inputTokens = 0;
|
|
102
|
+
let outputTokens = 0;
|
|
103
|
+
let status = "success";
|
|
104
|
+
let errorMessage = null;
|
|
105
|
+
try {
|
|
106
|
+
const result = await body((usage) => {
|
|
107
|
+
if (!usage) return;
|
|
108
|
+
const i = usage.input_tokens ?? usage.inputTokens ?? usage.prompt_tokens ?? usage.promptTokens;
|
|
109
|
+
const o = usage.output_tokens ?? usage.outputTokens ?? usage.completion_tokens ?? usage.completionTokens;
|
|
110
|
+
if (Number.isFinite(i)) inputTokens = i;
|
|
111
|
+
if (Number.isFinite(o)) outputTokens = o;
|
|
112
|
+
});
|
|
113
|
+
if (result === undefined) {
|
|
114
|
+
status = "error";
|
|
115
|
+
const streamErr = typeof spec.streamError === "function" ? String(spec.streamError() ?? "") : "";
|
|
116
|
+
errorMessage = streamErr ? `llm stream aborted or errored (${streamErr})` : "llm stream aborted or errored";
|
|
117
|
+
} else if (typeof spec.auditError === "function") {
|
|
118
|
+
// 流式成功但输出不可用,仍然是一次失败的调用——记 error,不与调用方的
|
|
119
|
+
// failed 返回值打架。
|
|
120
|
+
const message = spec.auditError(result);
|
|
121
|
+
if (message) {
|
|
122
|
+
status = "error";
|
|
123
|
+
errorMessage = message;
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
return result;
|
|
127
|
+
} catch (error) {
|
|
128
|
+
status = "error";
|
|
129
|
+
errorMessage = String(error?.message ?? error);
|
|
130
|
+
throw error;
|
|
131
|
+
} finally {
|
|
132
|
+
try {
|
|
133
|
+
service.saveLlmAudit({
|
|
134
|
+
timestamp,
|
|
135
|
+
trigger_source: spec.triggerSource,
|
|
136
|
+
operation_type: spec.operationType,
|
|
137
|
+
model_id: spec.modelId,
|
|
138
|
+
input_tokens: inputTokens,
|
|
139
|
+
output_tokens: outputTokens,
|
|
140
|
+
total_tokens: inputTokens + outputTokens,
|
|
141
|
+
cost_usd: 0,
|
|
142
|
+
duration_ms: Date.now() - startedAt,
|
|
143
|
+
status,
|
|
144
|
+
error_message: errorMessage,
|
|
145
|
+
related_memory_ids: spec.relatedMemoryIds ?? []
|
|
146
|
+
});
|
|
147
|
+
} catch (auditError) {
|
|
148
|
+
logger?.warn?.(`dsh-mneme sleep: llm audit write failed: ${String(auditError)}`);
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
|
|
70
153
|
/** LLM route (Issue #25): explicit sleepProvider/Model wins, then the dream
|
|
71
154
|
* route as a shared explicit fallback, then the agent default model. Sleep
|
|
72
155
|
* can pin a cheaper model for its bulk passes without disturbing the dream
|
|
@@ -106,7 +189,7 @@ async function phaseConflicts(ctx, service, config, logger, runId, semantic = nu
|
|
|
106
189
|
}
|
|
107
190
|
const strictness = config.sleepConflictStrictness ?? "normal";
|
|
108
191
|
const threshold = CONFLICT_THRESHOLDS[strictness] ?? CONFLICT_THRESHOLDS.normal;
|
|
109
|
-
const memories = service.all().filter((m) => !m.archived && !m.forgotten && m.type !== "summary");
|
|
192
|
+
const memories = service.all().filter((m) => !m.archived && !m.forgotten && m.type !== "summary" && m.type !== "document");
|
|
110
193
|
if (memories.length < 2) return { status: "skipped", reason: "too few memories" };
|
|
111
194
|
if (signal?.aborted) return { status: "aborted", reason: "user activity" };
|
|
112
195
|
|
|
@@ -232,17 +315,29 @@ async function phaseConflicts(ctx, service, config, logger, runId, semantic = nu
|
|
|
232
315
|
const conflictPrompt = actionSet === "full" ? STR.prompts.conflictFull[language] : STR.prompts.conflict[language];
|
|
233
316
|
const runConflict = (withEffort) => {
|
|
234
317
|
conflictStreamFailure = "";
|
|
235
|
-
return
|
|
318
|
+
return auditedSleepLlm(ctx, service, config, logger, {
|
|
319
|
+
triggerSource: "sleep",
|
|
320
|
+
operationType: "sleep_conflict",
|
|
321
|
+
modelId: `${route.provider}:${route.model}`,
|
|
322
|
+
// 候选对的双方 id:面板的活动流用 related_memory_ids 解析「沉淀条数」。
|
|
323
|
+
relatedMemoryIds: samePairs.flatMap((p) => [p.a.id, p.b.id]),
|
|
324
|
+
streamError: () => conflictStreamFailure,
|
|
325
|
+
// 纯判据:输出里没有 JSON 数组 ⇒ 这轮是失败的调用,审计记 error。**不要**把
|
|
326
|
+
// 解析结果存进闭包给下面的控制流用——审计关闭时 auditedSleepLlm 整段早退、
|
|
327
|
+
// 这个钩子不会被调用,控制流会跟着失效(dream 的既有坑)。代价是审计开启时
|
|
328
|
+
// 多解析一次,换来两条路径彻底解耦。
|
|
329
|
+
auditError: (text) => (parseJsonArray(text) ? null : "no json array in llm output")
|
|
330
|
+
}, (reportUsage) => streamText(ctx, {
|
|
236
331
|
provider: route.provider,
|
|
237
332
|
model: route.model,
|
|
238
333
|
purpose: "sleep-conflict",
|
|
239
|
-
maxTokens: 2048,
|
|
334
|
+
maxTokens: config.sleepMaxTokens ?? 2048,
|
|
240
335
|
...(withEffort && sleepEffort ? { reasoningEffort: sleepEffort } : {}),
|
|
241
336
|
messages: [
|
|
242
337
|
{ role: "system", content: [{ type: "text", text: conflictPrompt }], source: { kind: "plugin", plugin: "dsh-mneme" } },
|
|
243
338
|
{ role: "user", content: [{ type: "text", text: listText }], source: { kind: "plugin", plugin: "dsh-mneme" } }
|
|
244
339
|
]
|
|
245
|
-
}, (reason) => { conflictStreamFailure = describeStreamFailure(reason); });
|
|
340
|
+
}, reportUsage, (reason) => { conflictStreamFailure = describeStreamFailure(reason); }));
|
|
246
341
|
};
|
|
247
342
|
const text = await withEffortFallback(ctx, sleepEffort, () => runConflict(true), () => runConflict(false), () => conflictStreamFailure);
|
|
248
343
|
if (text === undefined) {
|
|
@@ -368,8 +463,10 @@ async function phasePatterns(ctx, service, config, logger, runId, signal = null)
|
|
|
368
463
|
const route = resolveSleepRoute(ctx, config, logger);
|
|
369
464
|
if (!route) return { status: "skipped", reason: "no llm route" };
|
|
370
465
|
const limit = config.sleepPatternMinMemories ?? 100;
|
|
466
|
+
// #230:document 在查询层排除(excludeTypes)而不是截断后过滤——LIMIT 200
|
|
467
|
+
// 先生效的话,指针行一多就会把普通记忆挤出扫描窗,池子饿到门槛以下。
|
|
371
468
|
const memories = service
|
|
372
|
-
.list({ limit: 200, includeForgotten: false })
|
|
469
|
+
.list({ limit: 200, includeForgotten: false, excludeTypes: ["document"] })
|
|
373
470
|
.filter((m) => !m.archived && m.type !== "summary" && m.type !== "pattern")
|
|
374
471
|
.sort((a, b) => (a.updated_at < b.updated_at ? 1 : -1))
|
|
375
472
|
.slice(0, limit);
|
|
@@ -383,17 +480,26 @@ async function phasePatterns(ctx, service, config, logger, runId, signal = null)
|
|
|
383
480
|
let patternStreamFailure = "";
|
|
384
481
|
const runPattern = (withEffort) => {
|
|
385
482
|
patternStreamFailure = "";
|
|
386
|
-
return
|
|
483
|
+
return auditedSleepLlm(ctx, service, config, logger, {
|
|
484
|
+
triggerSource: "sleep",
|
|
485
|
+
operationType: "sleep_pattern",
|
|
486
|
+
modelId: `${route.provider}:${route.model}`,
|
|
487
|
+
// 本轮扫描的记忆 id:与 dream 的 relatedMemoryIds 同义,供面板解析沉淀条数。
|
|
488
|
+
relatedMemoryIds: memories.map((m) => m.id),
|
|
489
|
+
streamError: () => patternStreamFailure,
|
|
490
|
+
// 纯判据,同 conflict 阶段:不把解析结果存进闭包供控制流使用。
|
|
491
|
+
auditError: (text) => (parseJsonArray(text) ? null : "no json array in llm output")
|
|
492
|
+
}, (reportUsage) => streamText(ctx, {
|
|
387
493
|
provider: route.provider,
|
|
388
494
|
model: route.model,
|
|
389
495
|
purpose: "sleep-pattern",
|
|
390
|
-
maxTokens: 2048,
|
|
496
|
+
maxTokens: config.sleepMaxTokens ?? 2048,
|
|
391
497
|
...(withEffort && sleepEffort ? { reasoningEffort: sleepEffort } : {}),
|
|
392
498
|
messages: [
|
|
393
499
|
{ role: "system", content: [{ type: "text", text: STR.prompts.pattern[language].replace("N", String(maxPatterns)) }], source: { kind: "plugin", plugin: "dsh-mneme" } },
|
|
394
500
|
{ role: "user", content: [{ type: "text", text: listText }], source: { kind: "plugin", plugin: "dsh-mneme" } }
|
|
395
501
|
]
|
|
396
|
-
}, (reason) => { patternStreamFailure = describeStreamFailure(reason); });
|
|
502
|
+
}, reportUsage, (reason) => { patternStreamFailure = describeStreamFailure(reason); }));
|
|
397
503
|
};
|
|
398
504
|
const text = await withEffortFallback(ctx, sleepEffort, () => runPattern(true), () => runPattern(false), () => patternStreamFailure);
|
|
399
505
|
if (text === undefined) {
|
|
@@ -505,6 +611,9 @@ function deriveStatus(phases) {
|
|
|
505
611
|
*/
|
|
506
612
|
export async function runSleep(ctx, service, config, logger, semantic = null, signal = null) {
|
|
507
613
|
const runId = randomUUID();
|
|
614
|
+
// Issue #89:开跑时刻——审计行 created_at 的基准(= 冷却种子),run 耗时
|
|
615
|
+
// (LLM 各 phase)不应计入重启后的冷却窗口。
|
|
616
|
+
const sleepStartedAt = Date.now();
|
|
508
617
|
const phases = {};
|
|
509
618
|
const attempt = async (name, fn) => {
|
|
510
619
|
if (signal?.aborted) return; // user resumed activity — stop before next phase
|
|
@@ -541,6 +650,9 @@ export async function runSleep(ctx, service, config, logger, semantic = null, si
|
|
|
541
650
|
try {
|
|
542
651
|
service.saveDreamRun({
|
|
543
652
|
id: runId,
|
|
653
|
+
// Issue #89:审计行记开跑时刻而非完成时刻——run 耗时(含 LLM 各 phase)
|
|
654
|
+
// 不该计入重启后的 sleep 冷却种子。
|
|
655
|
+
created_at: new Date(sleepStartedAt).toISOString(),
|
|
544
656
|
status,
|
|
545
657
|
provider: route?.provider,
|
|
546
658
|
model: route?.model,
|
|
@@ -580,10 +692,14 @@ export function createSleepScheduler({
|
|
|
580
692
|
onRun = null,
|
|
581
693
|
now = () => Date.now(),
|
|
582
694
|
setTimeoutFn = setTimeout,
|
|
583
|
-
clearTimeoutFn = clearTimeout
|
|
695
|
+
clearTimeoutFn = clearTimeout,
|
|
696
|
+
// Issue #89 同族:冷却时刻由调用方从 dream_runs 审计表恢复(run_type='sleep')——
|
|
697
|
+
// 内存变量进程重启即归零,冷却闸对新实例放行 → 重启后立即进入 sleep 冷却盲区。
|
|
698
|
+
// 测试注入 0(默认)即保持旧行为。
|
|
699
|
+
lastRunAtSeed = 0
|
|
584
700
|
}) {
|
|
585
701
|
let lastWriteAt = now();
|
|
586
|
-
let lastRunAt =
|
|
702
|
+
let lastRunAt = lastRunAtSeed;
|
|
587
703
|
let running = false;
|
|
588
704
|
let disposed = false;
|
|
589
705
|
let idleTimer = null;
|
package/src/dream.js
CHANGED
|
@@ -6,7 +6,6 @@ import { createHash, randomUUID } from "node:crypto";
|
|
|
6
6
|
import { STR, langOf } from "./lang.js";
|
|
7
7
|
export { validateDecisions, applyDecisions, withEffortFallback, describeStreamFailure, resolveDreamEffort, resolveRoute };
|
|
8
8
|
|
|
9
|
-
|
|
10
9
|
// Extract the first JSON array from LLM output, tolerating markdown fences,
|
|
11
10
|
// leading/trailing prose, and common wrapper noise. Returns an array or null.
|
|
12
11
|
function extractJsonArray(text) {
|
|
@@ -232,7 +231,12 @@ async function streamText(ctx, options, onUsage, onStreamError) {
|
|
|
232
231
|
let text = "";
|
|
233
232
|
for await (const chunk of ctx.llm.stream(options)) {
|
|
234
233
|
if (chunk.type === "text-delta" && typeof chunk.text === "string") text += chunk.text;
|
|
235
|
-
|
|
234
|
+
// DSH 的 StreamChunk 契约把用量嵌在 chunk.usage(TokenUsage:inputTokens /
|
|
235
|
+
// outputTokens / …),chunk 顶层没有 token 字段——把整个 chunk 当用量对象交给
|
|
236
|
+
// reporter,input/output 恒为 undefined,审计行落 0(v0.8.4 实测:7 天 49 次
|
|
237
|
+
// success 调用 token 全 0,面板「LLM 消耗」长期显示 0)。`?? chunk` 兜底把用量
|
|
238
|
+
// 平铺在顶层的替身。
|
|
239
|
+
if (chunk.type === "usage" && typeof onUsage === "function") onUsage(chunk.usage ?? chunk);
|
|
236
240
|
if (chunk.type === "finish" && (chunk.reason?.kind === "error" || chunk.reason?.kind === "aborted")) {
|
|
237
241
|
// dsh-llm rc.1 turns adapter-stage failures (unknown provider route,
|
|
238
242
|
// UNSUPPORTED_REASONING_EFFORT from resolveCallWithInfo, …) into a
|
|
@@ -382,7 +386,7 @@ async function reEmbedMemory(semantic, memory, logger) {
|
|
|
382
386
|
* 任何失败降级为 0 条,绝不反噬 dream 主流程。
|
|
383
387
|
*/
|
|
384
388
|
async function generateNarratives({ ctx, service, config, route, language, effort, semantic, logger }) {
|
|
385
|
-
const inputs = service.all().filter((m) => !m.archived && !m.forgotten && m.type !== "summary");
|
|
389
|
+
const inputs = service.all().filter((m) => !m.archived && !m.forgotten && m.type !== "summary" && m.type !== "document");
|
|
386
390
|
const clusters = clusterByTag(inputs, { minCluster: config.dreamNarrativeMinCluster ?? 3 });
|
|
387
391
|
if (!clusters.length) return 0;
|
|
388
392
|
|
|
@@ -740,7 +744,7 @@ export async function maintainIndexAfterDream(decisions, service, semantic) {
|
|
|
740
744
|
if (embedder.modelHash) vectorIndex.markModel?.(embedder.modelHash, embedder.dimension);
|
|
741
745
|
}
|
|
742
746
|
|
|
743
|
-
export function createDreamScheduler({ onRun, thresholdCount = 10, thresholdChars = 5000, delayMs = 2000, minIntervalMs = 0, logger, semantic = null }) {
|
|
747
|
+
export function createDreamScheduler({ onRun, thresholdCount = 10, thresholdChars = 5000, delayMs = 2000, minIntervalMs = 0, logger, semantic = null, lastRunAtSeed = 0 }) {
|
|
744
748
|
let pendingTimer = null;
|
|
745
749
|
let running = false;
|
|
746
750
|
let disposed = false;
|
|
@@ -748,10 +752,12 @@ export function createDreamScheduler({ onRun, thresholdCount = 10, thresholdChar
|
|
|
748
752
|
let inFlight = null;
|
|
749
753
|
// Issue #89(请求 2):上一次实际开跑时刻。失败/degraded 的 run 也占用
|
|
750
754
|
// 最小间隔——节流的目的正是防止失败调用连发;间隔内的触发静默跳过。
|
|
751
|
-
|
|
755
|
+
// lastRunAtSeed:调用方从 dream_runs 审计表读出的上次开跑时刻——
|
|
756
|
+
// 内存变量进程重启即归零,闸门对新实例放行 → 重启后立刻连发(#89 根因)。
|
|
757
|
+
let lastRunAt = lastRunAtSeed;
|
|
752
758
|
|
|
753
759
|
function shouldTrigger(service) {
|
|
754
|
-
const memories = service.all().filter((m) => !m.archived && m.type !== "summary");
|
|
760
|
+
const memories = service.all().filter((m) => !m.archived && m.type !== "summary" && m.type !== "document");
|
|
755
761
|
const count = memories.length;
|
|
756
762
|
const chars = totalChars(memories);
|
|
757
763
|
const overBase = count >= baseline.count + thresholdCount || chars >= baseline.chars + thresholdChars;
|
|
@@ -814,8 +820,10 @@ export function createDreamScheduler({ onRun, thresholdCount = 10, thresholdChar
|
|
|
814
820
|
async function runDream(ctx, service, config) {
|
|
815
821
|
const language = langOf(config);
|
|
816
822
|
const logger = ctx.logger;
|
|
817
|
-
let memories = service.all().filter((m) => !m.archived && m.type !== "summary");
|
|
823
|
+
let memories = service.all().filter((m) => !m.archived && m.type !== "summary" && m.type !== "document");
|
|
818
824
|
if (memories.length === 0) return { ok: true, applied: 0, skipped: true, summary: false };
|
|
825
|
+
// Issue #89:开跑时刻既喂给审计行(created_at),也是调度器闸门的时间基准。
|
|
826
|
+
const dreamStartedAt = Date.now();
|
|
819
827
|
// v0.4.4 滑动窗口:只 consolidation 最近 dreamMaxSnapshotSize 条记忆,
|
|
820
828
|
// 窗口外的旧记忆不进 snapshot(大记忆量下全量快照会撑爆 LLM 输入,配合
|
|
821
829
|
// 隐式 keep 让 run 始终可收敛)。按 updated_at 倒序取前 maxSize 条。
|
|
@@ -882,6 +890,9 @@ export function createDreamScheduler({ onRun, thresholdCount = 10, thresholdChar
|
|
|
882
890
|
try {
|
|
883
891
|
service.saveDreamRun({
|
|
884
892
|
id: runId,
|
|
893
|
+
// Issue #89:审计行记开跑时刻而非完成时刻——lastDreamRunAt 以它做
|
|
894
|
+
// 重启后的间隔种子,落完成时刻会让 run 耗时白白计入下一轮最小间隔。
|
|
895
|
+
created_at: new Date(dreamStartedAt).toISOString(),
|
|
885
896
|
status,
|
|
886
897
|
error: result.error,
|
|
887
898
|
provider: route?.provider,
|
|
@@ -1206,15 +1217,21 @@ export function createDreamScheduler({ onRun, thresholdCount = 10, thresholdChar
|
|
|
1206
1217
|
let summaryStreamFailure = "";
|
|
1207
1218
|
const runSummary = (withEffort) => {
|
|
1208
1219
|
summaryStreamFailure = "";
|
|
1220
|
+
// Issue #258:总览阶段可用 dreamSummaryProvider/dreamSummaryModel 走独立路由
|
|
1221
|
+
//(默认空 = 沿用 dream 路由);输入侧有 dreamSummaryMaxInputs 硬上限,
|
|
1222
|
+
// 按窗口同款 updated_at 倒序保留最新的(见下方 summaryInputs)。
|
|
1223
|
+
const summaryRoute = (config.dreamSummaryProvider && config.dreamSummaryModel)
|
|
1224
|
+
? { provider: config.dreamSummaryProvider, model: config.dreamSummaryModel }
|
|
1225
|
+
: route;
|
|
1209
1226
|
return runAuditedLlm(ctx, service, config, {
|
|
1210
1227
|
triggerSource: "autoDream",
|
|
1211
1228
|
operationType: "dream_summarize",
|
|
1212
|
-
modelId: `${
|
|
1229
|
+
modelId: `${summaryRoute.provider}:${summaryRoute.model}`,
|
|
1213
1230
|
relatedMemoryIds: [],
|
|
1214
1231
|
streamError: () => summaryStreamFailure
|
|
1215
1232
|
}, (reportUsage) => streamText(ctx, {
|
|
1216
|
-
provider:
|
|
1217
|
-
model:
|
|
1233
|
+
provider: summaryRoute.provider,
|
|
1234
|
+
model: summaryRoute.model,
|
|
1218
1235
|
purpose: "compaction",
|
|
1219
1236
|
maxTokens: config.dreamMaxTokens ?? 2048,
|
|
1220
1237
|
...(withEffort && effort ? { reasoningEffort: effort } : {}),
|
|
@@ -1229,7 +1246,25 @@ export function createDreamScheduler({ onRun, thresholdCount = 10, thresholdChar
|
|
|
1229
1246
|
// v1, content_history traceable). Its 口径 (what snapshot produced it) is
|
|
1230
1247
|
// stamped into the content so the standing answer is always auditable; the
|
|
1231
1248
|
// formal evidence column lands with the narrative-bars batch.
|
|
1232
|
-
|
|
1249
|
+
// Issue #258:输入硬上限(条数)——原全库无界,把 dream 路由指向小 ctx
|
|
1250
|
+
// 模型时 120k token 当场 CONTEXT_WINDOW_EXCEEDED(#258 实测),且随库增长
|
|
1251
|
+
// 渐进恶化。倒序排序与 consolidate 窗口同款(updated_at desc, id tiebreak),
|
|
1252
|
+
// 保留最新的;0 = 关闭上限(回归全库行为,调用方自担 ctx)。
|
|
1253
|
+
// #230:document 指针行不进总览(与 dream/sleep 五个候选池的同款排除)。
|
|
1254
|
+
const summaryAll = service.all().filter((m) => !m.archived && m.type !== "summary" && m.type !== "document");
|
|
1255
|
+
const summaryMaxInputs = Number.isInteger(config.dreamSummaryMaxInputs) ? config.dreamSummaryMaxInputs : 0;
|
|
1256
|
+
const summaryInputs = summaryMaxInputs > 0 && summaryAll.length > summaryMaxInputs
|
|
1257
|
+
? [...summaryAll]
|
|
1258
|
+
.sort((a, b) => {
|
|
1259
|
+
const ta = String(a.updated_at ?? "");
|
|
1260
|
+
const tb = String(b.updated_at ?? "");
|
|
1261
|
+
if (ta < tb) return 1;
|
|
1262
|
+
if (ta > tb) return -1;
|
|
1263
|
+
return a.id < b.id ? -1 : a.id > b.id ? 1 : 0;
|
|
1264
|
+
})
|
|
1265
|
+
.slice(0, summaryMaxInputs)
|
|
1266
|
+
: summaryAll;
|
|
1267
|
+
|
|
1233
1268
|
const summaryScope = STR.summaryScope[language](
|
|
1234
1269
|
summaryInputs.length,
|
|
1235
1270
|
runId.slice(0, 8),
|
package/src/guide.js
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
// #249(第一批):能力说明——写给模型的「怎么用这套记忆」,不是给用户的文案。
|
|
2
|
+
//
|
|
3
|
+
// 为什么英文、且只留一份正本:注入指引的参照实现全部用英文(ACP 的
|
|
4
|
+
// ACP_SYSTEM_PROMPT + HOW_TO_COMPRESS_RULES、mnemon 的 ROUTING_GUIDANCE、
|
|
5
|
+
// 宿主压缩摘要规则),仓库既有先例也是 src/tools.js 的工具描述硬编码英文。
|
|
6
|
+
// src/lang.js 的 memory.language 管的是「生成出来的记忆内容与块内标题」,与本
|
|
7
|
+
// 模块是两件事,故不并入 STR、不做 zh/en 双写(双写只会让两份文本日后漂移)。
|
|
8
|
+
//
|
|
9
|
+
// 两个承载位(#249 §5):①工具描述——常驻、零注入成本,放「何时用这个工具」
|
|
10
|
+
// 这类单工具指引(TOOL_GUIDE);②系统提示段 order 150——放工具描述装不下的
|
|
11
|
+
// 总则(优先序、何时查 / 何时写 / 何时 no-op)。该段必须同会话内稳定,否则
|
|
12
|
+
// 每轮变化会作废其后的前缀缓存,因此这里全是常量,不含任何运行时插值。
|
|
13
|
+
//
|
|
14
|
+
// 前缀 `[dsh-mneme memory]` 是机械校验用的:测试按它断言「只在开时出现、且不
|
|
15
|
+
// 逐轮复读」,也让维护者能一眼认出这段文本的归属。
|
|
16
|
+
|
|
17
|
+
const SECTION_LINES = [
|
|
18
|
+
"[dsh-mneme memory] Read once; it applies to every turn of this session.",
|
|
19
|
+
"1. Precedence: the current instruction and the repository's actual state outrank any stored memory. " +
|
|
20
|
+
"When a memory contradicts either, check the instruction or the repository itself before relying on it — memory_search " +
|
|
21
|
+
"searches stored memories only, so it finds earlier context, never the present state. Never treat an old memory as current fact.",
|
|
22
|
+
"2. Recall on demand: call memory_search when the task depends on earlier decisions, user preferences, or project history " +
|
|
23
|
+
"that is not already in context. Do not search for facts you can read directly from the repository.",
|
|
24
|
+
"3. Write back sparingly: use memory_save for durable, cross-session value — a preference, a decision with its rationale, " +
|
|
25
|
+
"an engineering constraint, a pitfall with its root cause. Trivial single-turn work is not worth a memory.",
|
|
26
|
+
"4. When unsure, do nothing. Not acting is a valid outcome: a useless memory is paid for by every future session.",
|
|
27
|
+
"5. Prefer the reversible tools. memory_archive hides an entry from lists, search, injection and consolidation, and " +
|
|
28
|
+
"memory_forget suppresses it from injection, search results and lists — both are recoverable. memory_delete is permanent, " +
|
|
29
|
+
"so reach for it only when an entry is wrong or unwanted, not merely stale."
|
|
30
|
+
];
|
|
31
|
+
|
|
32
|
+
/** 系统提示段(order 150)的总则文本。常量:同会话内稳定是硬约束。 */
|
|
33
|
+
export const MEMORY_GUIDE_SECTION = SECTION_LINES.join("\n");
|
|
34
|
+
|
|
35
|
+
/** 单工具判断指引,追加到对应工具描述尾部(`injectGuidanceEnabled` 开启时)。 */
|
|
36
|
+
export const TOOL_GUIDE = {
|
|
37
|
+
memory_search:
|
|
38
|
+
" Use this when the task depends on earlier decisions, preferences, or project history that is not already in context, " +
|
|
39
|
+
"or to look for a newer memory behind one that seems stale. " +
|
|
40
|
+
"Skip it for facts you can read directly from the repository.",
|
|
41
|
+
memory_save:
|
|
42
|
+
" Save only durable, cross-session value (a preference, a decision with its rationale, an engineering constraint, " +
|
|
43
|
+
"a pitfall with its root cause). Trivial single-turn work does not belong here, and when unsure, do not save."
|
|
44
|
+
};
|
package/src/heat.js
CHANGED
|
@@ -16,6 +16,8 @@ export const TYPE_DECAY_DEFAULTS = Object.freeze({
|
|
|
16
16
|
preference: 0, // 免疫:用户画像需长期保持
|
|
17
17
|
pattern: 0, // 免疫:发现型稳定规律
|
|
18
18
|
summary: 0, // 免疫:已是压缩产物
|
|
19
|
+
document: 0, // 免疫:指针行(#230)——全文在文件系统,行失效靠 supersede
|
|
20
|
+
// 记账而非遗忘;遗忘曲线灭掉指针 = 按需读全文的入口消失
|
|
19
21
|
project: 0.0008, // 慢衰减
|
|
20
22
|
decision: 0.002, // 中速衰减
|
|
21
23
|
history: 0.006, // 较快(会话摘要不断被合并)
|
package/src/index.js
CHANGED
|
@@ -43,7 +43,15 @@ export { Config };
|
|
|
43
43
|
// unit-testable; the extractor only ever sees a callLLM(messages, options)
|
|
44
44
|
// => Promise<string>. The route always carries a real provider/model (dsh-llm
|
|
45
45
|
// GenerateOptions requires both) — never a bare stream.
|
|
46
|
-
|
|
46
|
+
//
|
|
47
|
+
// Issue #250: it also accounts for itself. Entity extraction is the background
|
|
48
|
+
// LLM path that runs on every memory write, yet it never reached
|
|
49
|
+
// llm_audit_logs — the adapter only ever held an llm handle, so it had no way
|
|
50
|
+
// to call service.saveLlmAudit. It now takes service (+ config for the shared
|
|
51
|
+
// llmAudit.enabled gate) and writes one row per stream attempt, matching the
|
|
52
|
+
// contract of runAuditedLlm in dream.js: a rejected effort attempt records its
|
|
53
|
+
// own error row and the retry records its own success row.
|
|
54
|
+
export function createEntityStreamAdapter({ llm, agentDefaultModel, logger, service, config }) {
|
|
47
55
|
return async function streamEntityText(messages, options = {}) {
|
|
48
56
|
let route = {};
|
|
49
57
|
if (options.provider) route.provider = options.provider;
|
|
@@ -58,8 +66,38 @@ export function createEntityStreamAdapter({ llm, agentDefaultModel, logger }) {
|
|
|
58
66
|
} catch { /* fall through to whatever route we already have */ }
|
|
59
67
|
}
|
|
60
68
|
const effort = options.reasoningEffort;
|
|
69
|
+
// 没有解析出 provider/model 就没有可归属的模型——与 dream 的 resolveRoute
|
|
70
|
+
// 无路由早退同口径,那种情况不写审计行。
|
|
71
|
+
const modelId = route.provider && route.model ? `${route.provider}:${route.model}` : "";
|
|
61
72
|
const tryStream = (withEffort) => {
|
|
62
73
|
let text = "";
|
|
74
|
+
let inputTokens = 0;
|
|
75
|
+
let outputTokens = 0;
|
|
76
|
+
const startedAt = Date.now();
|
|
77
|
+
const timestamp = new Date(startedAt).toISOString();
|
|
78
|
+
// 记账是 best-effort:写审计行失败只 warn,绝不反噬抽取本身(CONTRIBUTING
|
|
79
|
+
// 的 fail-safe 硬约定)。
|
|
80
|
+
const writeAudit = (status, errorMessage) => {
|
|
81
|
+
if (config?.llmAudit?.enabled === false || !modelId || typeof service?.saveLlmAudit !== "function") return;
|
|
82
|
+
try {
|
|
83
|
+
service.saveLlmAudit({
|
|
84
|
+
timestamp,
|
|
85
|
+
trigger_source: "entityExtract",
|
|
86
|
+
operation_type: "entity_extract",
|
|
87
|
+
model_id: modelId,
|
|
88
|
+
input_tokens: inputTokens,
|
|
89
|
+
output_tokens: outputTokens,
|
|
90
|
+
total_tokens: inputTokens + outputTokens,
|
|
91
|
+
cost_usd: 0,
|
|
92
|
+
duration_ms: Date.now() - startedAt,
|
|
93
|
+
status,
|
|
94
|
+
error_message: errorMessage,
|
|
95
|
+
related_memory_ids: []
|
|
96
|
+
});
|
|
97
|
+
} catch (auditError) {
|
|
98
|
+
logger?.warn?.(`dsh-mneme: entity extraction llm audit write failed: ${String(auditError)}`);
|
|
99
|
+
}
|
|
100
|
+
};
|
|
63
101
|
return (async () => {
|
|
64
102
|
for await (const chunk of llm.stream({
|
|
65
103
|
...route,
|
|
@@ -68,11 +106,26 @@ export function createEntityStreamAdapter({ llm, agentDefaultModel, logger }) {
|
|
|
68
106
|
messages
|
|
69
107
|
})) {
|
|
70
108
|
if (chunk.type === "text-delta" && typeof chunk.text === "string") text += chunk.text;
|
|
71
|
-
|
|
109
|
+
// DSH 的 StreamChunk 契约把用量嵌在 chunk.usage(TokenUsage);兼容直接
|
|
110
|
+
// 平铺在 chunk 上的旧形态。归一放在这里,下面只需认平铺形状——与
|
|
111
|
+
// dream.js 的 streamText 同一处理。
|
|
112
|
+
if (chunk.type === "usage") {
|
|
113
|
+
const u = chunk.usage ?? chunk;
|
|
114
|
+
const i = u.input_tokens ?? u.inputTokens ?? u.prompt_tokens ?? u.promptTokens;
|
|
115
|
+
const o = u.output_tokens ?? u.outputTokens ?? u.completion_tokens ?? u.completionTokens;
|
|
116
|
+
if (Number.isFinite(i)) inputTokens = i;
|
|
117
|
+
if (Number.isFinite(o)) outputTokens = o;
|
|
118
|
+
}
|
|
119
|
+
if (chunk.type === "finish" && (chunk.reason?.kind === "error" || chunk.reason?.kind === "aborted")) {
|
|
120
|
+
writeAudit("error", "llm stream aborted or errored");
|
|
121
|
+
return undefined;
|
|
122
|
+
}
|
|
72
123
|
}
|
|
124
|
+
writeAudit("success", null);
|
|
73
125
|
return text;
|
|
74
126
|
})().catch((err) => {
|
|
75
127
|
logger?.warn?.(`[dsh-mneme] entity extraction llm stream failed: ${String(err)}`);
|
|
128
|
+
writeAudit("error", String(err?.message ?? err));
|
|
76
129
|
return undefined;
|
|
77
130
|
});
|
|
78
131
|
};
|
|
@@ -273,6 +326,9 @@ export const apply = (ctx, config) => {
|
|
|
273
326
|
dimension: cfg.localEmbedDimension,
|
|
274
327
|
device: cfg.localEmbedDevice,
|
|
275
328
|
batchSize: cfg.localEmbedBatchSize,
|
|
329
|
+
// 池化方式必须与模型的训练口径一致(BGE 系 = CLS)。它既进 embed() 的调用,
|
|
330
|
+
// 也进 modelHash —— 池化改了就是换向量空间,既有索引会被判失配并重建。
|
|
331
|
+
pooling: cfg.localEmbedPooling,
|
|
276
332
|
cacheDir: cfg.embedModelCacheDir,
|
|
277
333
|
runtimeDir: cfg.runtimeDir,
|
|
278
334
|
// #188:embedModelMirror 接成 transformers 的下载镜像(此前死配置)。
|
|
@@ -403,6 +459,7 @@ export const apply = (ctx, config) => {
|
|
|
403
459
|
minIntervalMs: (cfg.dreamMinIntervalMinutes ?? 0) * 60000,
|
|
404
460
|
logger: ctx.logger,
|
|
405
461
|
semantic: { embedder, vectorIndex },
|
|
462
|
+
lastRunAtSeed: store.lastDreamRunAt("auto"),
|
|
406
463
|
onRun: () => (dream ? dream.runDream(ctx, service, cfg) : Promise.resolve({ ok: true, skipped: true }))
|
|
407
464
|
});
|
|
408
465
|
service.setDreamHook(() => dream.maybeSchedule(service));
|
|
@@ -419,6 +476,7 @@ export const apply = (ctx, config) => {
|
|
|
419
476
|
service,
|
|
420
477
|
config: cfg,
|
|
421
478
|
logger: ctx.logger,
|
|
479
|
+
lastRunAtSeed: store.lastDreamRunAt("sleep"),
|
|
422
480
|
onRun: (signal) => (sleep ? runSleep(ctx, service, cfg, ctx.logger, { embedder, vectorIndex }, signal) : Promise.resolve({ ok: true, skipped: true }))
|
|
423
481
|
});
|
|
424
482
|
service.setSleepHook(() => sleep.noteWrite());
|
|
@@ -442,7 +500,11 @@ export const apply = (ctx, config) => {
|
|
|
442
500
|
const streamEntityText = createEntityStreamAdapter({
|
|
443
501
|
llm: ctx.llm,
|
|
444
502
|
agentDefaultModel: ctx.agentDefaultModel,
|
|
445
|
-
logger: ctx.logger
|
|
503
|
+
logger: ctx.logger,
|
|
504
|
+
// Issue #250: the adapter needs service to write its llm_audit_logs row
|
|
505
|
+
// (it never had it) and config for the shared llmAudit.enabled gate.
|
|
506
|
+
service,
|
|
507
|
+
config: cfg
|
|
446
508
|
});
|
|
447
509
|
service.setEntityExtractor((memory) =>
|
|
448
510
|
extractEntities(memory, { store, config: cfg, callLLM: streamEntityText, logger: ctx.logger })
|