@modusensus/dsh-mneme 0.9.0 → 0.9.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +82 -19
- package/bin/cli.mjs +69 -9
- package/docs/CONFIGURATION.md +48 -0
- package/docs/SEMANTIC.md +3 -3
- package/lib/api-standalone.js +78 -12
- package/lib/api.js +339 -59
- package/lib/candidates.js +454 -454
- package/lib/client.js +669 -256
- package/lib/config.js +50 -2
- package/lib/dream/sleep.js +35 -12
- package/lib/dream.js +86 -14
- package/lib/embedding.js +153 -22
- package/lib/index.js +32 -0
- package/lib/inject.js +34 -3
- package/lib/lang.js +124 -0
- package/lib/pipeline-activities.js +178 -0
- package/lib/pipeline-attribution.js +165 -0
- package/lib/pipeline-command.js +279 -0
- package/lib/pipeline-confirm.js +142 -0
- package/lib/pipeline-exec.js +174 -0
- package/lib/pipeline-nudge.js +1281 -0
- package/lib/pipeline-observations.js +181 -0
- package/lib/pipeline-observe.js +133 -0
- package/lib/pipeline-runner.js +346 -0
- package/lib/pipeline-runs.js +442 -0
- package/lib/pipeline.js +128 -7
- package/lib/scope-state.js +277 -0
- package/lib/semantic.js +66 -10
- package/lib/service.js +80 -3
- package/lib/settings.js +139 -1
- package/lib/store.js +216 -3
- package/lib/summarize.js +2 -3
- package/lib/tools.js +71 -0
- package/package.json +3 -3
- package/scripts/changelog-prep.mjs +62 -7
- package/scripts/check-sync.js +8 -6
- package/scripts/release-docs.mjs +204 -0
- package/scripts/release-notes.mjs +162 -0
- package/src/api-standalone.js +78 -12
- package/src/api.js +339 -59
- package/src/candidates.js +454 -454
- package/src/config.js +50 -2
- package/src/dream/sleep.js +35 -12
- package/src/dream.js +86 -14
- package/src/embedding.js +153 -22
- package/src/index.js +32 -0
- package/src/inject.js +34 -3
- package/src/lang.js +124 -0
- package/src/pipeline-activities.js +178 -0
- package/src/pipeline-attribution.js +165 -0
- package/src/pipeline-command.js +279 -0
- package/src/pipeline-confirm.js +142 -0
- package/src/pipeline-exec.js +174 -0
- package/src/pipeline-nudge.js +1281 -0
- package/src/pipeline-observations.js +181 -0
- package/src/pipeline-observe.js +133 -0
- package/src/pipeline-runner.js +346 -0
- package/src/pipeline-runs.js +442 -0
- package/src/pipeline.js +128 -7
- package/src/scope-state.js +277 -0
- package/src/semantic.js +66 -10
- package/src/service.js +80 -3
- package/src/settings.js +139 -1
- package/src/store.js +216 -3
- package/src/summarize.js +2 -3
- package/src/tools.js +71 -0
- package/test/api-hardening.test.js +173 -0
- package/test/api.test.js +210 -1
- package/test/changelog-prep.test.js +74 -8
- package/test/cli-enums.test.js +37 -0
- package/test/cli-flag-value.test.js +86 -0
- package/test/client.test.js +216 -6
- package/test/distill-tx-embed-gap.test.js +110 -0
- package/test/docs-config-keys.test.js +65 -0
- package/test/dream-maxtokens-fallback.test.js +132 -0
- package/test/embed-text-reasons.test.js +112 -0
- package/test/helpers/write-admission-samples.js +6 -2
- package/test/lib-smoke.test.js +20 -0
- package/test/persona-api.test.js +172 -0
- package/test/persona-flag.test.js +36 -0
- package/test/persona-inject.test.js +145 -0
- package/test/persona-settings.test.js +145 -0
- package/test/pipeline-activities.test.js +155 -0
- package/test/pipeline-attribution.test.js +175 -0
- package/test/pipeline-command.test.js +243 -0
- package/test/pipeline-confirm.test.js +148 -0
- package/test/pipeline-exec.test.js +172 -0
- package/test/pipeline-nudge.test.js +1036 -0
- package/test/pipeline-observations.test.js +157 -0
- package/test/pipeline-observe.test.js +127 -0
- package/test/pipeline-r1r2.test.js +149 -0
- package/test/pipeline-runner.test.js +442 -0
- package/test/pipeline-runs.test.js +767 -0
- package/test/pipeline.test.js +137 -6
- package/test/readme-claims.test.js +68 -0
- package/test/reindex-backfill.test.js +83 -4
- package/test/release-docs.test.js +130 -0
- package/test/release-notes.test.js +240 -0
- package/test/release-workflow.test.js +16 -0
- package/test/route-policy.test.js +113 -0
- package/test/scope-state.test.js +481 -0
- package/test/security-doc.test.js +139 -0
- package/test/sensitive-scan.test.js +44 -0
- package/test/tools-dto-schema-matrix.test.js +12 -3
- package/test/tools.test.js +6 -6
- package/test/workflow-guards.test.js +156 -0
package/lib/config.js
CHANGED
|
@@ -415,6 +415,13 @@ export const Config = z.object({
|
|
|
415
415
|
// in the background after a short delay, rate-limited in batches. On by
|
|
416
416
|
// default; set false to keep the backfill manual only.
|
|
417
417
|
autoReindexOnBoot: z.boolean().default(true),
|
|
418
|
+
// Backfill pacing (previously hardcoded 500 / 10 / 200 in semantic.js). A
|
|
419
|
+
// fixed 500-per-boot cap meant a 4k-row backlog needed nine restarts to
|
|
420
|
+
// clear, and nothing reported that the run had been capped.
|
|
421
|
+
// reindexMaxPerBoot = 0 means "no cap, drain the whole backlog this run".
|
|
422
|
+
reindexMaxPerBoot: z.natural().default(500),
|
|
423
|
+
reindexBatchSize: z.natural().min(1).max(64).default(10),
|
|
424
|
+
reindexRateLimitMs: z.natural().default(200),
|
|
418
425
|
// Semantic-first injection (Bug4): when enabled, injectCandidates with a
|
|
419
426
|
// non-empty query recalls via the vector index first and falls back to the
|
|
420
427
|
// rule-based pick to fill/dedupe. Empty query / no vector → legacy behavior.
|
|
@@ -856,6 +863,23 @@ export const Config = z.object({
|
|
|
856
863
|
// 数据、观察 A2 加权质量后再切硬过滤。
|
|
857
864
|
strictScope: z.boolean().default(false),
|
|
858
865
|
|
|
866
|
+
// scopeState(issue #471):政策轴——作用域级可逆闸门。与上面两个键是**两条不同
|
|
867
|
+
// 的轴**:scopeEnabled/strictScope 管「这条记忆属于谁」(事实轴:持久、跟着记忆
|
|
868
|
+
// 行走、改的是归属数据),本键管「这个作用域现在通不通」(政策轴:暂时、由人单
|
|
869
|
+
// 方面按下、松开即恢复、不改任何记忆行)。
|
|
870
|
+
// 两级语义同 writeAdmission/preInjectGate——enabled 打开时被按下的作用域其候选
|
|
871
|
+
// 照常注入,但留一条审计行说清「本应被拦下几条」(观察档,先在真实流量看会少掉
|
|
872
|
+
// 多少再决定真按);enforce 打开才真的不注入(拦截档)。两个都默认关,打开前
|
|
873
|
+
// 行为与今天逐字段一致。
|
|
874
|
+
// 不进 LIGHT_MODE_OFF:它只是查一张小表的判定,不引入重型后台路径、不加常驻
|
|
875
|
+
// 文本(同 actionPipelineEnabled 的定性)——轻量档预设该关的是重型能力,不该替
|
|
876
|
+
// 用户关掉一个纯判定入口,用户显式勾选仍然赢。
|
|
877
|
+
// 也走 feature_flags(FEATURE_FLAG_BOOLEANS 白名单),面板可启停=线上回滚开关。
|
|
878
|
+
scopeState: z.object({
|
|
879
|
+
enabled: z.boolean().default(false),
|
|
880
|
+
enforce: z.boolean().default(false)
|
|
881
|
+
}).default({}),
|
|
882
|
+
|
|
859
883
|
// --- v0.9.0 M.O.(Modus Operandi · 手法,beta)--------------------------
|
|
860
884
|
// 命名:M.O. = Modus Operandi,拉丁语「一个人一贯的做事方式」,缩写 M.O.;
|
|
861
885
|
// 也正是 org 词根 modus(modusensus)。界面主显示 M.O.,设置项 hint 第一句
|
|
@@ -867,7 +891,27 @@ export const Config = z.object({
|
|
|
867
891
|
// 它只是只读视图、不引入任何重型后台路径,轻量档用户显式勾选仍然赢(预设
|
|
868
892
|
// 该关的是重型能力,不该替用户关掉一个纯 UI 入口)。
|
|
869
893
|
// 也走 feature_flags(FEATURE_FLAG_BOOLEANS 白名单),面板可启停=线上回滚开关。
|
|
870
|
-
actionPipelineEnabled: z.boolean().default(false)
|
|
894
|
+
actionPipelineEnabled: z.boolean().default(false),
|
|
895
|
+
|
|
896
|
+
// M.O. 执行记账(v0.9.0 第二批):`/mo-run` 显式开一次执行窗口,窗口内的命令执行
|
|
897
|
+
// 才可能被记成一条 run。与上面那个键**分开**:上面管「看图的座位」,这个管
|
|
898
|
+
// 「往账本里写」——看图和写账是两件事,混成一个键会让「只想看图」的人不知不觉
|
|
899
|
+
// 获得写入能力。
|
|
900
|
+
//
|
|
901
|
+
// 默认关(beta 必须 opt-in)。它挂在热路径上(每个会话事件都过一次监听),
|
|
902
|
+
// 所以轻量档要关掉:`LIGHT_MODE_OFF` 里有它。
|
|
903
|
+
actionRunEnabled: z.boolean().default(false),
|
|
904
|
+
|
|
905
|
+
// --- 人格定义页(beta)---------------------------------------------------
|
|
906
|
+
// 注入器渲染人格段的顶层总闸(双闸门的 config 半边;user_settings "persona"
|
|
907
|
+
// 键内的 persona.enabled 是 per-agent 半边,两闸同真才渲染)。API 的 effective
|
|
908
|
+
// 必须含这个键——schema 里没有它 effective 就静默少键,面板开关形同虚设
|
|
909
|
+
// (#254 漏登记 NESTED_FLAG_PATHS 是同一个坑,计数锁 test/api.test.js 钉住)。
|
|
910
|
+
// 默认关:beta 功能必须 opt-in,老用户行为逐字节不变。不进 LIGHT_MODE_OFF——
|
|
911
|
+
// 人格段是用户自写的一小段轻量常驻文本,不引入任何重型后台路径,轻量档用户
|
|
912
|
+
// 显式勾选仍然赢(同 actionPipelineEnabled 的定性)。
|
|
913
|
+
// 也走 feature_flags(FEATURE_FLAG_BOOLEANS 白名单),面板可启停=线上回滚开关。
|
|
914
|
+
personaEnabled: z.boolean().default(false)
|
|
871
915
|
});
|
|
872
916
|
|
|
873
917
|
// Fields forced to false by the light-mode preset. Everything not listed here
|
|
@@ -914,7 +958,11 @@ const LIGHT_MODE_OFF = [
|
|
|
914
958
|
// #399:轻量档不开蒸馏原文留存——它是一份对话原文副本(磁盘 + 暴露面),
|
|
915
959
|
// 与轻量档「小模型 / 小上下文 / 少留痕」的取向相反。默认关,这里只是
|
|
916
960
|
// 轻量预设再确认一次;用户显式勾选仍然赢(合并顺序同 continuityRescueEnabled)。
|
|
917
|
-
"distillRecords.enabled"
|
|
961
|
+
"distillRecords.enabled",
|
|
962
|
+
// M.O. 执行记账:它挂在**热路径**上(每个会话事件都过一次监听)。轻量档的取向
|
|
963
|
+
// 是「小模型 / 小上下文 / 少常驻路径」,这与一个每事件都过一遍的监听器相冲突。
|
|
964
|
+
// 默认关,这里只是轻量预设再确认一次;用户显式勾选仍然赢(合并顺序同上面两条)。
|
|
965
|
+
"actionRunEnabled"
|
|
918
966
|
];
|
|
919
967
|
|
|
920
968
|
/**
|
package/lib/dream/sleep.js
CHANGED
|
@@ -21,7 +21,7 @@ import { randomUUID, createHash } from "node:crypto";
|
|
|
21
21
|
import { validateDecisions, applyDecisions, ACTIONS } from "./decisions.js";
|
|
22
22
|
import { findPotentialConflicts, cosineSimilarity } from "./clustering.js";
|
|
23
23
|
import { scopeKeyOf } from "../scope.js";
|
|
24
|
-
import { buildReceipt, describeStreamFailure, resolveDreamEffort, withEffortFallback, maintainIndexAfterDream } from "../dream.js";
|
|
24
|
+
import { buildReceipt, describeStreamFailure, resolveDreamEffort, withEffortFallback, withMaxTokensFallback, maintainIndexAfterDream } from "../dream.js";
|
|
25
25
|
import { computeHeat } from "../heat.js";
|
|
26
26
|
|
|
27
27
|
// Issue #126 review(Copilot):默认档(sleepActionSet="conflict")允许的动作 = 全局
|
|
@@ -172,6 +172,22 @@ function makeSummary(m) {
|
|
|
172
172
|
return text.length <= SUMMARY_MAX ? text : `${text.slice(0, SUMMARY_MAX)}…`;
|
|
173
173
|
}
|
|
174
174
|
|
|
175
|
+
// issue #471 政策轴:被按下的作用域也不参与睡眠整理。与 dream.js 的 withoutPaused
|
|
176
|
+
// 同源(那里记着「判定只许有一份实现」的理由),但这里是**单独一处**而不是复用:
|
|
177
|
+
// 两个模块各被独立装配(runSleep / runDream 各自接 service),任何一个被换进只
|
|
178
|
+
// 实现了一半 service 的夹具时都该各自降级,而不是因为对方模块没装而双双失效。
|
|
179
|
+
//
|
|
180
|
+
// 睡眠侧比日间加固侧更要紧:这几个相位不只读——phaseDemotion 会 setArchived 与
|
|
181
|
+
// demoteToSummary(真改归档面),phasePatterns 会 mint 出 type=pattern 的新行
|
|
182
|
+
// (凭空给被按下的作用域生出衍生记忆)。只挡日间那几处读入口的话,闸门开着而每
|
|
183
|
+
// 6 小时一次的睡眠照样动手,那等于「按下」只是让它在白天看不见。
|
|
184
|
+
//
|
|
185
|
+
// service 上取不到 filterPaused 时(老装配、测试夹具)原样返回,与改动前一致。
|
|
186
|
+
function withoutPaused(service, rows) {
|
|
187
|
+
if (typeof service?.filterPaused !== "function") return rows;
|
|
188
|
+
return service.filterPaused(rows);
|
|
189
|
+
}
|
|
190
|
+
|
|
175
191
|
// ---------------------------------------------------------------- phases
|
|
176
192
|
|
|
177
193
|
/**
|
|
@@ -189,7 +205,7 @@ async function phaseConflicts(ctx, service, config, logger, runId, semantic = nu
|
|
|
189
205
|
}
|
|
190
206
|
const strictness = config.sleepConflictStrictness ?? "normal";
|
|
191
207
|
const threshold = CONFLICT_THRESHOLDS[strictness] ?? CONFLICT_THRESHOLDS.normal;
|
|
192
|
-
const memories = service.all().filter((m) => !m.archived && !m.forgotten && m.type !== "summary" && m.type !== "document");
|
|
208
|
+
const memories = withoutPaused(service, service.all().filter((m) => !m.archived && !m.forgotten && m.type !== "summary" && m.type !== "document"));
|
|
193
209
|
if (memories.length < 2) return { status: "skipped", reason: "too few memories" };
|
|
194
210
|
if (signal?.aborted) return { status: "aborted", reason: "user activity" };
|
|
195
211
|
|
|
@@ -313,7 +329,7 @@ async function phaseConflicts(ctx, service, config, logger, runId, semantic = nu
|
|
|
313
329
|
// actionSet 已在候选构造前解析(跨类型候选需要它),这里只按档位选 prompt:
|
|
314
330
|
// 默认档保持只有 conflict/keep 的窄 prompt,"full" 才开放六分支。
|
|
315
331
|
const conflictPrompt = actionSet === "full" ? STR.prompts.conflictFull[language] : STR.prompts.conflict[language];
|
|
316
|
-
const runConflict = (withEffort) => {
|
|
332
|
+
const runConflict = (withEffort, tokens) => {
|
|
317
333
|
conflictStreamFailure = "";
|
|
318
334
|
return auditedSleepLlm(ctx, service, config, logger, {
|
|
319
335
|
triggerSource: "sleep",
|
|
@@ -331,7 +347,7 @@ async function phaseConflicts(ctx, service, config, logger, runId, semantic = nu
|
|
|
331
347
|
provider: route.provider,
|
|
332
348
|
model: route.model,
|
|
333
349
|
purpose: "sleep-conflict",
|
|
334
|
-
maxTokens:
|
|
350
|
+
maxTokens: tokens,
|
|
335
351
|
...(withEffort && sleepEffort ? { reasoningEffort: sleepEffort } : {}),
|
|
336
352
|
messages: [
|
|
337
353
|
{ role: "system", content: [{ type: "text", text: conflictPrompt }], source: { kind: "plugin:dsh-mneme", plugin: "dsh-mneme" } },
|
|
@@ -339,7 +355,9 @@ async function phaseConflicts(ctx, service, config, logger, runId, semantic = nu
|
|
|
339
355
|
]
|
|
340
356
|
}, reportUsage, (reason) => { conflictStreamFailure = describeStreamFailure(reason); }));
|
|
341
357
|
};
|
|
342
|
-
|
|
358
|
+
// Issue #478:睡眠侧同样接 max_tokens 降级——面板 hint 已承诺「超上限同样自动降级」,
|
|
359
|
+
// 只挂 dream 三个点会让 sleepMaxTokens 的 131072 上限继续原样发。
|
|
360
|
+
const text = await withMaxTokensFallback(ctx, config.sleepMaxTokens ?? 2048, (tokens) => withEffortFallback(ctx, sleepEffort, () => runConflict(true, tokens), () => runConflict(false, tokens), () => conflictStreamFailure), () => conflictStreamFailure);
|
|
343
361
|
if (text === undefined) {
|
|
344
362
|
if (conflictStreamFailure) ctx.logger?.warn?.(`dsh-mneme sleep: conflict stream aborted or errored (${conflictStreamFailure})`);
|
|
345
363
|
return { status: "failed", error: "llm failed" };
|
|
@@ -423,7 +441,10 @@ function phaseDemotion(service, config, logger, runId, signal = null) {
|
|
|
423
441
|
const compressCut = Date.now() - compressDays * 86400000;
|
|
424
442
|
const demoted = [];
|
|
425
443
|
const archived = [];
|
|
426
|
-
|
|
444
|
+
// 政策轴在这里尤其要紧:下面两行会真改归档面(setArchived / demoteToSummary)。
|
|
445
|
+
// 把过滤放在循环之外、而不是在循环体里 continue:一旦将来有人在循环体里加
|
|
446
|
+
// 副作用(现已有),漏掉那条 continue 就是静默改到被按下的行。
|
|
447
|
+
for (const m of withoutPaused(service, service.all())) {
|
|
427
448
|
if (signal?.aborted) break;
|
|
428
449
|
if (m.archived || m.forgotten) continue;
|
|
429
450
|
const ref = m.last_accessed_at ?? m.created_at;
|
|
@@ -472,9 +493,11 @@ async function phasePatterns(ctx, service, config, logger, runId, signal = null)
|
|
|
472
493
|
const limit = config.sleepPatternMinMemories ?? 100;
|
|
473
494
|
// #230:document 在查询层排除(excludeTypes)而不是截断后过滤——LIMIT 200
|
|
474
495
|
// 先生效的话,指针行一多就会把普通记忆挤出扫描窗,池子饿到门槛以下。
|
|
475
|
-
|
|
496
|
+
// 政策轴同理必须排在 `.slice(0, limit)` **之前**,否则被按下的行照样占掉
|
|
497
|
+
// sleepPatternMinMemories 的名额,而它们既不会被扫、也不该挤掉别人(#471)。
|
|
498
|
+
const memories = withoutPaused(service, service
|
|
476
499
|
.list({ limit: 200, includeForgotten: false, excludeTypes: ["document"] })
|
|
477
|
-
.filter((m) => !m.archived && m.type !== "summary" && m.type !== "pattern")
|
|
500
|
+
.filter((m) => !m.archived && m.type !== "summary" && m.type !== "pattern"))
|
|
478
501
|
.sort((a, b) => (a.updated_at < b.updated_at ? 1 : -1))
|
|
479
502
|
.slice(0, limit);
|
|
480
503
|
if (memories.length === 0) return { status: "skipped", reason: "no memories to scan" };
|
|
@@ -485,7 +508,7 @@ async function phasePatterns(ctx, service, config, logger, runId, signal = null)
|
|
|
485
508
|
const maxPatterns = config.sleepMaxPatternPerRun ?? 3;
|
|
486
509
|
const sleepEffort = await resolveDreamEffort(ctx, route, config.sleepReasoningEffort, logger);
|
|
487
510
|
let patternStreamFailure = "";
|
|
488
|
-
const runPattern = (withEffort) => {
|
|
511
|
+
const runPattern = (withEffort, tokens) => {
|
|
489
512
|
patternStreamFailure = "";
|
|
490
513
|
return auditedSleepLlm(ctx, service, config, logger, {
|
|
491
514
|
triggerSource: "sleep",
|
|
@@ -500,7 +523,7 @@ async function phasePatterns(ctx, service, config, logger, runId, signal = null)
|
|
|
500
523
|
provider: route.provider,
|
|
501
524
|
model: route.model,
|
|
502
525
|
purpose: "sleep-pattern",
|
|
503
|
-
maxTokens:
|
|
526
|
+
maxTokens: tokens,
|
|
504
527
|
...(withEffort && sleepEffort ? { reasoningEffort: sleepEffort } : {}),
|
|
505
528
|
messages: [
|
|
506
529
|
{ role: "system", content: [{ type: "text", text: STR.prompts.pattern[language].replace("N", String(maxPatterns)) }], source: { kind: "plugin:dsh-mneme", plugin: "dsh-mneme" } },
|
|
@@ -508,7 +531,7 @@ async function phasePatterns(ctx, service, config, logger, runId, signal = null)
|
|
|
508
531
|
]
|
|
509
532
|
}, reportUsage, (reason) => { patternStreamFailure = describeStreamFailure(reason); }));
|
|
510
533
|
};
|
|
511
|
-
const text = await withEffortFallback(ctx, sleepEffort, () => runPattern(true), () => runPattern(false), () => patternStreamFailure);
|
|
534
|
+
const text = await withMaxTokensFallback(ctx, config.sleepMaxTokens ?? 2048, (tokens) => withEffortFallback(ctx, sleepEffort, () => runPattern(true, tokens), () => runPattern(false, tokens), () => patternStreamFailure), () => patternStreamFailure);
|
|
512
535
|
if (text === undefined) {
|
|
513
536
|
if (patternStreamFailure) ctx.logger?.warn?.(`dsh-mneme sleep: pattern stream aborted or errored (${patternStreamFailure})`);
|
|
514
537
|
return { status: "failed", error: "llm failed" };
|
|
@@ -563,7 +586,7 @@ function phaseRelations(service, config, logger, runId, signal = null) {
|
|
|
563
586
|
if (entities.length < 2) return { status: "skipped", reason: "too few entities" };
|
|
564
587
|
const orphans = entities.filter((e) => (service.getRelations(e.id) ?? []).length === 0);
|
|
565
588
|
if (orphans.length === 0) return { status: "skipped", reason: "no orphan entities" };
|
|
566
|
-
const memories = service.all().filter((m) => !m.archived && !m.forgotten);
|
|
589
|
+
const memories = withoutPaused(service, service.all().filter((m) => !m.archived && !m.forgotten));
|
|
567
590
|
const seen = new Set();
|
|
568
591
|
const related = [];
|
|
569
592
|
const MAX_RELATIONS_PER_ORPHAN = 3;
|
package/lib/dream.js
CHANGED
|
@@ -10,7 +10,7 @@ import { scopeKeyOf } from "./scope.js";
|
|
|
10
10
|
import { isInPeakWindow, nextOffPeakAt } from "./peak-hours.js";
|
|
11
11
|
import { createHash, randomUUID } from "node:crypto";
|
|
12
12
|
import { STR, langOf } from "./lang.js";
|
|
13
|
-
export { validateDecisions, applyDecisions, withEffortFallback, describeStreamFailure, resolveDreamEffort, resolveRoute };
|
|
13
|
+
export { validateDecisions, applyDecisions, withEffortFallback, withMaxTokensFallback, parseMaxTokensCap, describeStreamFailure, resolveDreamEffort, resolveRoute };
|
|
14
14
|
|
|
15
15
|
// Extract the first JSON array from LLM output, tolerating markdown fences,
|
|
16
16
|
// leading/trailing prose, and common wrapper noise. Returns an array or null.
|
|
@@ -57,10 +57,24 @@ function totalChars(memories) {
|
|
|
57
57
|
// 两处若各写一份过滤条件,重启播种读到的基线就会和运行期判定口径漂移(一个把
|
|
58
58
|
// document 算进去、一个不算),阈值随之失真,且这种漂移只会在重启后才显形。
|
|
59
59
|
function activeStoreSize(service) {
|
|
60
|
-
const memories = service.all().filter((m) => !m.archived && m.type !== "summary" && m.type !== "document");
|
|
60
|
+
const memories = withoutPaused(service, service.all().filter((m) => !m.archived && m.type !== "summary" && m.type !== "document"));
|
|
61
61
|
return { count: memories.length, chars: totalChars(memories) };
|
|
62
62
|
}
|
|
63
63
|
|
|
64
|
+
// issue #471 政策轴:被按下的作用域不参与巩固,也就不该再撑着阈值把它自己触发起来。
|
|
65
|
+
// 三个读入口(本函数、generateNarratives 的 inputs、runDream 的 memories)各自的附加
|
|
66
|
+
// 条件不同——有的排 forgotten、有的不排——但「政策轴这一层」对三处是一样的,故收成
|
|
67
|
+
// 一个助手而不是各写一遍过滤,理由与 activeStoreSize 开头记着的口径漂移同源:两处
|
|
68
|
+
// 各写一份过滤条件,漂移只会在重启后才显形。
|
|
69
|
+
//
|
|
70
|
+
// 走 service.filterPaused 而不是自己读状态表:判定只许有一份实现(见
|
|
71
|
+
// src/scope-state.js 的模块注释)。service 上取不到这个函数时(老装配、测试夹具)
|
|
72
|
+
// 原样返回,行为与改动前一致。
|
|
73
|
+
function withoutPaused(service, rows) {
|
|
74
|
+
if (typeof service?.filterPaused !== "function") return rows;
|
|
75
|
+
return service.filterPaused(rows);
|
|
76
|
+
}
|
|
77
|
+
|
|
64
78
|
// 基线的唯一合法形状:两个非负整数;其余(缺字段、负数、非整数、整个对象缺失)
|
|
65
79
|
// 一律视为「没有基线」→ 调用方退回 {0,0}。播种值来自持久化列,可能是老库的 NULL
|
|
66
80
|
// 或手工改坏的库,判定必须比调用点更严格,而不是相信数据库里存的一定合法。
|
|
@@ -399,6 +413,60 @@ async function withEffortFallback(ctx, effort, attempt, fallback, getStreamError
|
|
|
399
413
|
}
|
|
400
414
|
}
|
|
401
415
|
|
|
416
|
+
// Issue #478:max_tokens 超服务商硬上限的拒收判别式与上限提取的单一来源。
|
|
417
|
+
// 实测失败样例(dream_runs 180 连败):"max_tokens 不能超过 65536"——131072 的
|
|
418
|
+
// 默认值原样发出,网关 400 直接拒,推理还没开始。与 EFFORT_REJECT_RE 同款思路:
|
|
419
|
+
// 各处手写会漂移,导出供测试与其它调用点复用。
|
|
420
|
+
export const MAX_TOKENS_REJECT_RE = /max[\s_-]*tokens?/i;
|
|
421
|
+
|
|
422
|
+
/**
|
|
423
|
+
* 从拒收报错里提取服务商声明的 max_tokens 上限。只认「显式给数」的说法
|
|
424
|
+
* (不能超过 / at most / maximum value is / <= 65536 …),认不出就返回 null
|
|
425
|
+
* ——宁可不降级重试,也不能把报错里那个被拒的请求值(131072)误当上限。
|
|
426
|
+
* @returns {number|null}
|
|
427
|
+
*/
|
|
428
|
+
function parseMaxTokensCap(message) {
|
|
429
|
+
const m = String(message ?? "").match(
|
|
430
|
+
/(?:不能超过|不得超过|超出上限|至多|最多(?:只能|仅)?|at\s+most|no\s+more\s+than|maximum(?:\s+value)?(?:\s+is)?|max(?:\.)?(?:\s+value)?(?:\s+is)?|less\s+than\s+or\s+equal\s+to|<=?)\s*[::]?\s*(\d{2,10})/i
|
|
431
|
+
);
|
|
432
|
+
const cap = m ? Number(m[1]) : null;
|
|
433
|
+
return cap && Number.isFinite(cap) && cap > 0 ? cap : null;
|
|
434
|
+
}
|
|
435
|
+
|
|
436
|
+
/**
|
|
437
|
+
* Issue #478:max_tokens 被服务商 400 拒收时,按报错里的上限自动降级重试一次。
|
|
438
|
+
* 与 withEffortFallback 同构:stream 路径的结果折叠成 undefined、真实原因在
|
|
439
|
+
* getStreamError() 里;throw 路径直接看 message。命中拒收且提取到比当前值小的
|
|
440
|
+
* 上限才重试(attempt 接收 tokens 值),否则原样返回/抛出。单次重试,与
|
|
441
|
+
* effort 的先例一致——重试后若仍被拒(同上限再报),cap >= tokens 的守卫让
|
|
442
|
+
* 它自然终止,不会循环。
|
|
443
|
+
*/
|
|
444
|
+
async function withMaxTokensFallback(ctx, maxTokens, attempt, getStreamError) {
|
|
445
|
+
const retry = (tokens, reason) => {
|
|
446
|
+
ctx.logger?.warn?.(`dsh-mneme dream: max_tokens ${maxTokens} rejected by provider (${reason}); retrying with ${tokens}`);
|
|
447
|
+
return attempt(tokens);
|
|
448
|
+
};
|
|
449
|
+
try {
|
|
450
|
+
const result = await attempt(maxTokens);
|
|
451
|
+
if (result === undefined) {
|
|
452
|
+
const reason = String(getStreamError?.() ?? "");
|
|
453
|
+
if (MAX_TOKENS_REJECT_RE.test(reason)) {
|
|
454
|
+
const cap = parseMaxTokensCap(reason);
|
|
455
|
+
if (cap && cap < maxTokens) return retry(cap, reason);
|
|
456
|
+
}
|
|
457
|
+
}
|
|
458
|
+
return result;
|
|
459
|
+
} catch (error) {
|
|
460
|
+
// AbortError 直接放行(取消意图绝不能被降级重试吞掉,与 effort 同款守卫)。
|
|
461
|
+
if (error?.name === "AbortError") throw error;
|
|
462
|
+
const message = String(error?.message ?? error);
|
|
463
|
+
if (!MAX_TOKENS_REJECT_RE.test(message)) throw error;
|
|
464
|
+
const cap = parseMaxTokensCap(message);
|
|
465
|
+
if (!cap || cap >= maxTokens) throw error;
|
|
466
|
+
return retry(cap, message);
|
|
467
|
+
}
|
|
468
|
+
}
|
|
469
|
+
|
|
402
470
|
/** 向量索引同步(总览与叙述条共用):fail-safe,失败只 warn 不反噬主流程。 */
|
|
403
471
|
async function reEmbedMemory(semantic, memory, logger) {
|
|
404
472
|
if (!semantic?.embedder || !semantic?.vectorIndex || !memory) return;
|
|
@@ -418,7 +486,7 @@ async function reEmbedMemory(semantic, memory, logger) {
|
|
|
418
486
|
* 任何失败降级为 0 条,绝不反噬 dream 主流程。
|
|
419
487
|
*/
|
|
420
488
|
async function generateNarratives({ ctx, service, config, route, language, effort, semantic, logger }) {
|
|
421
|
-
const inputs = service.all().filter((m) => !m.archived && !m.forgotten && m.type !== "summary" && m.type !== "document");
|
|
489
|
+
const inputs = withoutPaused(service, service.all().filter((m) => !m.archived && !m.forgotten && m.type !== "summary" && m.type !== "document"));
|
|
422
490
|
const clusters = clusterByTag(inputs, { minCluster: config.dreamNarrativeMinCluster ?? 3 });
|
|
423
491
|
if (!clusters.length) return 0;
|
|
424
492
|
|
|
@@ -427,7 +495,7 @@ async function generateNarratives({ ctx, service, config, route, language, effor
|
|
|
427
495
|
[`# ${tag}`, ...members.map((m) => `- id=${m.id} | title=${m.title} | ${m.content}`)].join("\n"))
|
|
428
496
|
.join("\n\n");
|
|
429
497
|
let streamFailure = "";
|
|
430
|
-
const runNarratives = (withEffort) => {
|
|
498
|
+
const runNarratives = (withEffort, tokens) => {
|
|
431
499
|
streamFailure = "";
|
|
432
500
|
return runAuditedLlm(ctx, service, config, {
|
|
433
501
|
triggerSource: "autoDream",
|
|
@@ -439,7 +507,7 @@ async function generateNarratives({ ctx, service, config, route, language, effor
|
|
|
439
507
|
provider: route.provider,
|
|
440
508
|
model: route.model,
|
|
441
509
|
purpose: "compaction",
|
|
442
|
-
maxTokens:
|
|
510
|
+
maxTokens: tokens,
|
|
443
511
|
...(withEffort && effort ? { reasoningEffort: effort } : {}),
|
|
444
512
|
messages: [
|
|
445
513
|
{ role: "system", content: [{ type: "text", text: STR.prompts.narrative[language] }], source: { kind: "plugin:dsh-mneme", plugin: "dsh-mneme" } },
|
|
@@ -447,7 +515,7 @@ async function generateNarratives({ ctx, service, config, route, language, effor
|
|
|
447
515
|
]
|
|
448
516
|
}, reportUsage, (reason) => { streamFailure = describeStreamFailure(reason); }));
|
|
449
517
|
};
|
|
450
|
-
const text = await withEffortFallback(ctx, effort, () => runNarratives(true), () => runNarratives(false), () => streamFailure);
|
|
518
|
+
const text = await withMaxTokensFallback(ctx, config.dreamMaxTokens ?? 2048, (tokens) => withEffortFallback(ctx, effort, () => runNarratives(true, tokens), () => runNarratives(false, tokens), () => streamFailure), () => streamFailure);
|
|
451
519
|
|
|
452
520
|
const entries = extractJsonArray(text);
|
|
453
521
|
if (!Array.isArray(entries)) return 0;
|
|
@@ -946,7 +1014,7 @@ export function createDreamScheduler({ onRun, thresholdCount = 10, thresholdChar
|
|
|
946
1014
|
async function runDream(ctx, service, config) {
|
|
947
1015
|
const language = langOf(config);
|
|
948
1016
|
const logger = ctx.logger;
|
|
949
|
-
let memories = service.all().filter((m) => !m.archived && m.type !== "summary" && m.type !== "document");
|
|
1017
|
+
let memories = withoutPaused(service, service.all().filter((m) => !m.archived && m.type !== "summary" && m.type !== "document"));
|
|
950
1018
|
if (memories.length === 0) return { ok: true, applied: 0, skipped: true, summary: false };
|
|
951
1019
|
// Issue #89:开跑时刻既喂给审计行(created_at),也是调度器闸门的时间基准。
|
|
952
1020
|
const dreamStartedAt = Date.now();
|
|
@@ -1127,7 +1195,7 @@ export function createDreamScheduler({ onRun, thresholdCount = 10, thresholdChar
|
|
|
1127
1195
|
const effort = await resolveDreamEffort(ctx, route, config.dreamReasoningEffort, logger);
|
|
1128
1196
|
let decisions = null;
|
|
1129
1197
|
let streamFailure = "";
|
|
1130
|
-
const runConsolidation = (withEffort) => {
|
|
1198
|
+
const runConsolidation = (withEffort, tokens) => {
|
|
1131
1199
|
streamFailure = "";
|
|
1132
1200
|
return runAuditedLlm(ctx, service, config, {
|
|
1133
1201
|
triggerSource: "autoDream",
|
|
@@ -1143,7 +1211,7 @@ export function createDreamScheduler({ onRun, thresholdCount = 10, thresholdChar
|
|
|
1143
1211
|
provider: route.provider,
|
|
1144
1212
|
model: route.model,
|
|
1145
1213
|
purpose: "compaction",
|
|
1146
|
-
maxTokens:
|
|
1214
|
+
maxTokens: tokens,
|
|
1147
1215
|
...(withEffort && effort ? { reasoningEffort: effort } : {}),
|
|
1148
1216
|
messages: [
|
|
1149
1217
|
{ role: "system", content: [{ type: "text", text: consolidationPrompt }], source: { kind: "plugin:dsh-mneme", plugin: "dsh-mneme" } },
|
|
@@ -1155,7 +1223,7 @@ export function createDreamScheduler({ onRun, thresholdCount = 10, thresholdChar
|
|
|
1155
1223
|
// Bug8: the consolidation call is audited (tokens/time/status). A throw
|
|
1156
1224
|
// re-propagates to the catch below; an aborted stream returns undefined
|
|
1157
1225
|
// and is treated as a failed run after the check below.
|
|
1158
|
-
decisionText = await withEffortFallback(ctx, effort, () => runConsolidation(true), () => runConsolidation(false), () => streamFailure);
|
|
1226
|
+
decisionText = await withMaxTokensFallback(ctx, config.dreamMaxTokens ?? 4096, (tokens) => withEffortFallback(ctx, effort, () => runConsolidation(true, tokens), () => runConsolidation(false, tokens), () => streamFailure), () => streamFailure);
|
|
1159
1227
|
} catch (error) {
|
|
1160
1228
|
logger?.warn?.(`dsh-mneme dream: consolidation llm call failed: ${String(error)}`);
|
|
1161
1229
|
return finish({ ok: false, error: "llm failed", summary: false });
|
|
@@ -1368,7 +1436,7 @@ export function createDreamScheduler({ onRun, thresholdCount = 10, thresholdChar
|
|
|
1368
1436
|
// a failed run; summary:false marks a run that produced no summary.
|
|
1369
1437
|
let summaryText;
|
|
1370
1438
|
let summaryStreamFailure = "";
|
|
1371
|
-
const runSummary = (withEffort) => {
|
|
1439
|
+
const runSummary = (withEffort, tokens) => {
|
|
1372
1440
|
summaryStreamFailure = "";
|
|
1373
1441
|
// Issue #258:总览阶段可用 dreamSummaryProvider/dreamSummaryModel 走独立路由
|
|
1374
1442
|
//(默认空 = 沿用 dream 路由);输入侧有 dreamSummaryMaxInputs 硬上限,
|
|
@@ -1386,7 +1454,7 @@ export function createDreamScheduler({ onRun, thresholdCount = 10, thresholdChar
|
|
|
1386
1454
|
provider: summaryRoute.provider,
|
|
1387
1455
|
model: summaryRoute.model,
|
|
1388
1456
|
purpose: "compaction",
|
|
1389
|
-
maxTokens:
|
|
1457
|
+
maxTokens: tokens,
|
|
1390
1458
|
...(withEffort && effort ? { reasoningEffort: effort } : {}),
|
|
1391
1459
|
messages: [
|
|
1392
1460
|
{ role: "system", content: [{ type: "text", text: STR.prompts.dreamSummary[language] }], source: { kind: "plugin:dsh-mneme", plugin: "dsh-mneme" } },
|
|
@@ -1404,7 +1472,11 @@ export function createDreamScheduler({ onRun, thresholdCount = 10, thresholdChar
|
|
|
1404
1472
|
// 渐进恶化。倒序排序与 consolidate 窗口同款(updated_at desc, id tiebreak),
|
|
1405
1473
|
// 保留最新的;0 = 关闭上限(回归全库行为,调用方自担 ctx)。
|
|
1406
1474
|
// #230:document 指针行不进总览(与 dream/sleep 五个候选池的同款排除)。
|
|
1407
|
-
|
|
1475
|
+
// issue #471:这一处尤其不能漏——总览落库时是**无作用域**的 summary 行
|
|
1476
|
+
// (saveWithDedupe 不带 agent_scope/workspace_scope),所以被按下作用域的内容
|
|
1477
|
+
// 一旦进了总览,注入侧按作用域过滤就再也拦不住它:内容已经洗成了全局行。
|
|
1478
|
+
// 别的入口漏了只是「那一条漏过去」,这里漏了是「洗一遍之后一直漏」。
|
|
1479
|
+
const summaryAll = withoutPaused(service, service.all().filter((m) => !m.archived && m.type !== "summary" && m.type !== "document"));
|
|
1408
1480
|
const summaryMaxInputs = Number.isInteger(config.dreamSummaryMaxInputs) ? config.dreamSummaryMaxInputs : 0;
|
|
1409
1481
|
const summaryInputs = summaryMaxInputs > 0 && summaryAll.length > summaryMaxInputs
|
|
1410
1482
|
? [...summaryAll]
|
|
@@ -1425,7 +1497,7 @@ export function createDreamScheduler({ onRun, thresholdCount = 10, thresholdChar
|
|
|
1425
1497
|
);
|
|
1426
1498
|
try {
|
|
1427
1499
|
// Bug8: the summary call is audited too (operation dream_summarize).
|
|
1428
|
-
summaryText = await withEffortFallback(ctx, effort, () => runSummary(true), () => runSummary(false), () => summaryStreamFailure);
|
|
1500
|
+
summaryText = await withMaxTokensFallback(ctx, config.dreamMaxTokens ?? 2048, (tokens) => withEffortFallback(ctx, effort, () => runSummary(true, tokens), () => runSummary(false, tokens), () => summaryStreamFailure), () => summaryStreamFailure);
|
|
1429
1501
|
} catch (error) {
|
|
1430
1502
|
logger?.warn?.(`dsh-mneme dream: summary llm call failed: ${String(error)}`);
|
|
1431
1503
|
return finish({ ok: false, error: "llm failed", applied, decisions: auditDecisions, outcome, frozen: frozenCount, summary: false });
|