@modusensus/dsh-mneme 0.8.3 → 0.8.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (205) hide show
  1. package/LICENSE +21 -21
  2. package/README.md +727 -636
  3. package/bin/cli.mjs +0 -0
  4. package/bin/dsh-mneme-mcp.mjs +510 -0
  5. package/cordis.patch.yml +21 -21
  6. package/lib/api-standalone.js +506 -316
  7. package/lib/api.js +1403 -1316
  8. package/lib/bootstrap.js +211 -0
  9. package/lib/client.js +4657 -4366
  10. package/lib/commands.js +95 -95
  11. package/lib/config.js +615 -510
  12. package/lib/document.js +221 -0
  13. package/lib/dream/clustering.js +118 -118
  14. package/lib/dream/decisions.js +658 -658
  15. package/lib/dream/narratives.js +44 -0
  16. package/lib/dream/sleep.js +702 -700
  17. package/lib/dream.js +147 -16
  18. package/lib/embedding.js +177 -177
  19. package/lib/entities/extractor.js +249 -249
  20. package/lib/guide.js +44 -0
  21. package/lib/heat.js +144 -136
  22. package/lib/hot-memory.js +46 -46
  23. package/lib/index.js +515 -515
  24. package/lib/inject.js +375 -295
  25. package/lib/lang.js +208 -179
  26. package/lib/local-embedder.js +320 -320
  27. package/lib/mirror.js +198 -198
  28. package/lib/organize.js +450 -0
  29. package/lib/quality-filter.js +126 -126
  30. package/lib/recall-stats.js +125 -0
  31. package/lib/reranker.js +269 -269
  32. package/lib/runtime/adopt.js +176 -176
  33. package/lib/runtime/closure.js +172 -172
  34. package/lib/runtime/download.js +247 -247
  35. package/lib/runtime/layout.js +265 -235
  36. package/lib/runtime/loader.js +252 -249
  37. package/lib/runtime/model-fetch.js +361 -361
  38. package/lib/runtime/provision.js +237 -237
  39. package/lib/runtime/tarball.js +130 -130
  40. package/lib/runtime/verify.js +204 -191
  41. package/lib/scope.js +118 -118
  42. package/lib/search/adaptive.js +60 -22
  43. package/lib/search/bm25.js +96 -96
  44. package/lib/service.js +2324 -2087
  45. package/lib/settings.js +459 -420
  46. package/lib/store.js +2549 -2367
  47. package/lib/summarize.js +820 -449
  48. package/lib/tools.js +702 -574
  49. package/lib/vector-index.js +110 -110
  50. package/lib/version-check.js +122 -122
  51. package/package.json +109 -108
  52. package/runtime-manifest.json +526 -596
  53. package/scripts/benchmark-embed.js +201 -201
  54. package/scripts/benchmark-recall.js +199 -199
  55. package/scripts/benchmark-rerank.js +166 -166
  56. package/scripts/build-runtime-manifest.mjs +224 -217
  57. package/scripts/check-sync.js +42 -42
  58. package/scripts/e2e-dsh.js +220 -220
  59. package/scripts/mneme-runtime.mjs +277 -277
  60. package/scripts/stress-dsh.js +255 -255
  61. package/scripts/sync-lib.js +47 -47
  62. package/scripts/sync-test-badge.mjs +67 -61
  63. package/src/api-standalone.js +506 -316
  64. package/src/api.js +1403 -1316
  65. package/src/bootstrap.js +211 -0
  66. package/src/commands.js +95 -95
  67. package/src/config.js +615 -510
  68. package/src/document.js +221 -0
  69. package/src/dream/clustering.js +118 -118
  70. package/src/dream/decisions.js +658 -658
  71. package/src/dream/narratives.js +44 -0
  72. package/src/dream/sleep.js +702 -700
  73. package/src/dream.js +147 -16
  74. package/src/embedding.js +177 -177
  75. package/src/entities/extractor.js +249 -249
  76. package/src/guide.js +44 -0
  77. package/src/heat.js +144 -136
  78. package/src/hot-memory.js +46 -46
  79. package/src/index.js +515 -515
  80. package/src/inject.js +375 -295
  81. package/src/lang.js +208 -179
  82. package/src/local-embedder.js +320 -320
  83. package/src/mirror.js +198 -198
  84. package/src/organize.js +450 -0
  85. package/src/quality-filter.js +126 -126
  86. package/src/recall-stats.js +125 -0
  87. package/src/reranker.js +269 -269
  88. package/src/runtime/adopt.js +176 -176
  89. package/src/runtime/closure.js +172 -172
  90. package/src/runtime/download.js +247 -247
  91. package/src/runtime/layout.js +265 -235
  92. package/src/runtime/loader.js +252 -249
  93. package/src/runtime/model-fetch.js +361 -361
  94. package/src/runtime/provision.js +237 -237
  95. package/src/runtime/tarball.js +130 -130
  96. package/src/runtime/verify.js +204 -191
  97. package/src/scope.js +118 -118
  98. package/src/search/adaptive.js +60 -22
  99. package/src/search/bm25.js +96 -96
  100. package/src/service.js +2324 -2087
  101. package/src/settings.js +459 -420
  102. package/src/store.js +2549 -2367
  103. package/src/summarize.js +820 -449
  104. package/src/tools.js +702 -574
  105. package/src/vector-index.js +110 -110
  106. package/src/version-check.js +122 -122
  107. package/test/adaptive-inject-budget.test.js +85 -0
  108. package/test/api-routes-gaps.test.js +218 -218
  109. package/test/api.test.js +1640 -1482
  110. package/test/audit.test.js +448 -448
  111. package/test/benchmark.test.js +94 -94
  112. package/test/bootstrap.test.js +220 -0
  113. package/test/client.test.js +860 -736
  114. package/test/clustering.test.js +100 -100
  115. package/test/commands.test.js +106 -106
  116. package/test/config.test.js +66 -66
  117. package/test/conflict-freeze.test.js +300 -300
  118. package/test/document.test.js +523 -0
  119. package/test/dream-archive-guardrail.test.js +116 -116
  120. package/test/dream-candidate-hybrid.test.js +281 -281
  121. package/test/dream-coverage-degrade.test.js +86 -86
  122. package/test/dream.test.js +1308 -1254
  123. package/test/embedding-ready.test.js +196 -196
  124. package/test/entities.test.js +547 -547
  125. package/test/entity-recall.test.js +179 -0
  126. package/test/epistemic.test.js +298 -298
  127. package/test/fnew-0112.test.js +311 -311
  128. package/test/fnew-03.test.js +422 -422
  129. package/test/graph-api.test.js +175 -175
  130. package/test/heat.test.js +175 -148
  131. package/test/helpers/dream-mock.js +82 -82
  132. package/test/helpers/peer-worker.mjs +25 -25
  133. package/test/helpers/runtime-source.js +60 -60
  134. package/test/helpers/tar-builder.js +80 -80
  135. package/test/hot-memory.test.js +145 -145
  136. package/test/inject-content-cap.test.js +91 -0
  137. package/test/inject-first-round-bm25.test.js +89 -89
  138. package/test/inject-heat.test.js +61 -0
  139. package/test/inject-pools.test.js +170 -0
  140. package/test/inject-recall.test.js +105 -0
  141. package/test/inject-rotation.test.js +161 -161
  142. package/test/inject.test.js +283 -249
  143. package/test/lang.test.js +87 -87
  144. package/test/lib-smoke.test.js +96 -96
  145. package/test/llm-audit.test.js +462 -309
  146. package/test/llm-message-source.test.js +177 -177
  147. package/test/local-embedder.test.js +249 -249
  148. package/test/mcp-stdio.test.js +385 -0
  149. package/test/mirror-dirty.test.js +424 -424
  150. package/test/mirror-edit-digest.test.js +187 -187
  151. package/test/mirror-generation.test.js +499 -499
  152. package/test/mirror.test.js +249 -249
  153. package/test/narrative-bars.test.js +162 -0
  154. package/test/organize.test.js +418 -0
  155. package/test/peer-blockers.test.js +193 -193
  156. package/test/policy-epoch.test.js +259 -259
  157. package/test/quality-filter.test.js +186 -186
  158. package/test/reasoning-effort.test.js +746 -746
  159. package/test/recall-evals.test.js +235 -235
  160. package/test/recall-layer.test.js +326 -326
  161. package/test/recall-stats.test.js +123 -0
  162. package/test/receipt-chain.test.js +451 -451
  163. package/test/reflection.test.js +226 -226
  164. package/test/reindex-backfill.test.js +62 -62
  165. package/test/reranker-single-logit.test.js +73 -73
  166. package/test/reranker.test.js +197 -197
  167. package/test/resident-status.test.js +83 -0
  168. package/test/runtime-adopt.test.js +206 -206
  169. package/test/runtime-cli.test.js +89 -89
  170. package/test/runtime-closure.test.js +193 -193
  171. package/test/runtime-download.test.js +302 -302
  172. package/test/runtime-layout.test.js +163 -163
  173. package/test/runtime-loader.test.js +255 -237
  174. package/test/runtime-manifest.test.js +159 -158
  175. package/test/runtime-model-fetch.test.js +348 -348
  176. package/test/runtime-provision.test.js +101 -101
  177. package/test/runtime-tarball.test.js +81 -81
  178. package/test/runtime-verify.test.js +279 -214
  179. package/test/scope-candidates.test.js +239 -239
  180. package/test/scope-explicit.test.js +452 -452
  181. package/test/scope-retrieval.test.js +197 -197
  182. package/test/scope-storage.test.js +255 -255
  183. package/test/scope-strict.test.js +231 -231
  184. package/test/search-fusion.test.js +90 -90
  185. package/test/semantic.test.js +124 -124
  186. package/test/service-search.test.js +174 -174
  187. package/test/service.test.js +405 -405
  188. package/test/settings.test.js +261 -261
  189. package/test/skip-invalid-restore.test.js +262 -262
  190. package/test/sleep-action-set.test.js +422 -422
  191. package/test/sleep-heat.test.js +126 -125
  192. package/test/sleep-scheduler-rearm.test.js +89 -89
  193. package/test/sleep.test.js +410 -373
  194. package/test/standalone-api.test.js +544 -359
  195. package/test/store-vector-cache.test.js +98 -98
  196. package/test/store.test.js +396 -383
  197. package/test/stress.test.js +209 -209
  198. package/test/summarize.test.js +1288 -575
  199. package/test/tools-dto-schema-matrix.test.js +325 -0
  200. package/test/tools-memory-get-schema.test.js +65 -65
  201. package/test/tools.test.js +374 -341
  202. package/test/updated-at-semantics.test.js +113 -113
  203. package/test/vector-index.test.js +221 -221
  204. package/test/version-check.test.js +152 -152
  205. package/test/webServer-optional.test.js +82 -82
package/src/dream.js CHANGED
@@ -1,11 +1,11 @@
1
1
  import { validateDecisions, applyDecisions } from "./dream/decisions.js";
2
2
  import { clusterMemories, findPotentialConflicts, cosineSimilarity } from "./dream/clustering.js";
3
+ import { clusterByTag, intersectEvidence } from "./dream/narratives.js";
3
4
  import { scopeKeyOf } from "./scope.js";
4
5
  import { createHash, randomUUID } from "node:crypto";
5
6
  import { STR, langOf } from "./lang.js";
6
7
  export { validateDecisions, applyDecisions, withEffortFallback, describeStreamFailure, resolveDreamEffort, resolveRoute };
7
8
 
8
-
9
9
  // Extract the first JSON array from LLM output, tolerating markdown fences,
10
10
  // leading/trailing prose, and common wrapper noise. Returns an array or null.
11
11
  function extractJsonArray(text) {
@@ -231,7 +231,12 @@ async function streamText(ctx, options, onUsage, onStreamError) {
231
231
  let text = "";
232
232
  for await (const chunk of ctx.llm.stream(options)) {
233
233
  if (chunk.type === "text-delta" && typeof chunk.text === "string") text += chunk.text;
234
- if (chunk.type === "usage" && typeof onUsage === "function") onUsage(chunk);
234
+ // DSH 的 StreamChunk 契约把用量嵌在 chunk.usage(TokenUsage:inputTokens /
235
+ // outputTokens / …),chunk 顶层没有 token 字段——把整个 chunk 当用量对象交给
236
+ // reporter,input/output 恒为 undefined,审计行落 0(v0.8.4 实测:7 天 49 次
237
+ // success 调用 token 全 0,面板「LLM 消耗」长期显示 0)。`?? chunk` 兜底把用量
238
+ // 平铺在顶层的替身。
239
+ if (chunk.type === "usage" && typeof onUsage === "function") onUsage(chunk.usage ?? chunk);
235
240
  if (chunk.type === "finish" && (chunk.reason?.kind === "error" || chunk.reason?.kind === "aborted")) {
236
241
  // dsh-llm rc.1 turns adapter-stage failures (unknown provider route,
237
242
  // UNSUPPORTED_REASONING_EFFORT from resolveCallWithInfo, …) into a
@@ -362,6 +367,85 @@ async function withEffortFallback(ctx, effort, attempt, fallback, getStreamError
362
367
  }
363
368
  }
364
369
 
370
+ /** 向量索引同步(总览与叙述条共用):fail-safe,失败只 warn 不反噬主流程。 */
371
+ async function reEmbedMemory(semantic, memory, logger) {
372
+ if (!semantic?.embedder || !semantic?.vectorIndex || !memory) return;
373
+ try {
374
+ const v = await semantic.embedder.embedSingle([memory.title, memory.content].filter(Boolean).join("\n"));
375
+ if (v?.length) semantic.vectorIndex.saveEmbedding(memory.id, v);
376
+ if (semantic.embedder.modelHash) semantic.vectorIndex.markModel?.(semantic.embedder.modelHash, semantic.embedder.dimension);
377
+ } catch (error) {
378
+ logger?.warn?.(`dsh-mneme dream: re-embed failed: ${String(error)}`);
379
+ }
380
+ }
381
+
382
+ /**
383
+ * 叙述条阶段(#164 对齐,opt-in dreamNarrativeEnabled):共享 tag 主题聚类 →
384
+ * 单次 LLM 调用按簇合成叙述 → 证据求交(防捏造,dream/narratives.js 纯函数)
385
+ * → saveWithDedupe(source="narrative", _overwrite) 落库。按需检索不常驻注入。
386
+ * 任何失败降级为 0 条,绝不反噬 dream 主流程。
387
+ */
388
+ async function generateNarratives({ ctx, service, config, route, language, effort, semantic, logger }) {
389
+ const inputs = service.all().filter((m) => !m.archived && !m.forgotten && m.type !== "summary" && m.type !== "document");
390
+ const clusters = clusterByTag(inputs, { minCluster: config.dreamNarrativeMinCluster ?? 3 });
391
+ if (!clusters.length) return 0;
392
+
393
+ const listing = clusters
394
+ .map(({ tag, members }) =>
395
+ [`# ${tag}`, ...members.map((m) => `- id=${m.id} | title=${m.title} | ${m.content}`)].join("\n"))
396
+ .join("\n\n");
397
+ let streamFailure = "";
398
+ const runNarratives = (withEffort) => {
399
+ streamFailure = "";
400
+ return runAuditedLlm(ctx, service, config, {
401
+ triggerSource: "autoDream",
402
+ operationType: "dream_narrative",
403
+ modelId: `${route.provider}:${route.model}`,
404
+ relatedMemoryIds: clusters.flatMap((c) => c.members.map((m) => m.id)),
405
+ streamError: () => streamFailure
406
+ }, (reportUsage) => streamText(ctx, {
407
+ provider: route.provider,
408
+ model: route.model,
409
+ purpose: "compaction",
410
+ maxTokens: config.dreamMaxTokens ?? 2048,
411
+ ...(withEffort && effort ? { reasoningEffort: effort } : {}),
412
+ messages: [
413
+ { role: "system", content: [{ type: "text", text: STR.prompts.narrative[language] }], source: { kind: "plugin", plugin: "dsh-mneme" } },
414
+ { role: "user", content: [{ type: "text", text: listing }], source: { kind: "plugin", plugin: "dsh-mneme" } }
415
+ ]
416
+ }, reportUsage, (reason) => { streamFailure = describeStreamFailure(reason); }));
417
+ };
418
+ const text = await withEffortFallback(ctx, effort, () => runNarratives(true), () => runNarratives(false), () => streamFailure);
419
+
420
+ const entries = extractJsonArray(text);
421
+ if (!Array.isArray(entries)) return 0;
422
+ const clusterByOfferedTag = new Map(clusters.map((c) => [c.tag, c]));
423
+ const now = new Date().toISOString();
424
+ let stored = 0;
425
+ for (const entry of entries) {
426
+ if (!entry || typeof entry !== "object") continue;
427
+ const cluster = clusterByOfferedTag.get(typeof entry.tag === "string" ? entry.tag : "");
428
+ const content = String(entry.content ?? "").trim();
429
+ if (!cluster || !content) continue; // 未提供的 tag / 空叙述 → 跳过
430
+ const evidenceIds = intersectEvidence(entry.evidence, cluster.members);
431
+ const { memory } = service.saveWithDedupe({
432
+ type: "summary",
433
+ title: STR.narrativeTitle[language](cluster.tag),
434
+ content,
435
+ importance: 3,
436
+ tags: [cluster.tag],
437
+ source: "narrative",
438
+ evidence: evidenceIds.map((id) => ({ memory_id: id, op: "support", at: now })),
439
+ _overwrite: true
440
+ });
441
+ if (!memory) continue;
442
+ stored++;
443
+ // 叙述条按需可检索:向量索引同步(与总览共用 reEmbedMemory,fail-safe)。
444
+ await reEmbedMemory(semantic, memory, logger);
445
+ }
446
+ return stored;
447
+ }
448
+
365
449
  /**
366
450
  * Resolve the reasoning effort to actually send for a dream/sleep route.
367
451
  *
@@ -671,7 +755,7 @@ export function createDreamScheduler({ onRun, thresholdCount = 10, thresholdChar
671
755
  let lastRunAt = 0;
672
756
 
673
757
  function shouldTrigger(service) {
674
- const memories = service.all().filter((m) => !m.archived && m.type !== "summary");
758
+ const memories = service.all().filter((m) => !m.archived && m.type !== "summary" && m.type !== "document");
675
759
  const count = memories.length;
676
760
  const chars = totalChars(memories);
677
761
  const overBase = count >= baseline.count + thresholdCount || chars >= baseline.chars + thresholdChars;
@@ -734,7 +818,7 @@ export function createDreamScheduler({ onRun, thresholdCount = 10, thresholdChar
734
818
  async function runDream(ctx, service, config) {
735
819
  const language = langOf(config);
736
820
  const logger = ctx.logger;
737
- let memories = service.all().filter((m) => !m.archived && m.type !== "summary");
821
+ let memories = service.all().filter((m) => !m.archived && m.type !== "summary" && m.type !== "document");
738
822
  if (memories.length === 0) return { ok: true, applied: 0, skipped: true, summary: false };
739
823
  // v0.4.4 滑动窗口:只 consolidation 最近 dreamMaxSnapshotSize 条记忆,
740
824
  // 窗口外的旧记忆不进 snapshot(大记忆量下全量快照会撑爆 LLM 输入,配合
@@ -1126,24 +1210,59 @@ export function createDreamScheduler({ onRun, thresholdCount = 10, thresholdChar
1126
1210
  let summaryStreamFailure = "";
1127
1211
  const runSummary = (withEffort) => {
1128
1212
  summaryStreamFailure = "";
1213
+ // Issue #258:总览阶段可用 dreamSummaryProvider/dreamSummaryModel 走独立路由
1214
+ //(默认空 = 沿用 dream 路由);输入侧有 dreamSummaryMaxInputs 硬上限,
1215
+ // 按窗口同款 updated_at 倒序保留最新的(见下方 summaryInputs)。
1216
+ const summaryRoute = (config.dreamSummaryProvider && config.dreamSummaryModel)
1217
+ ? { provider: config.dreamSummaryProvider, model: config.dreamSummaryModel }
1218
+ : route;
1129
1219
  return runAuditedLlm(ctx, service, config, {
1130
1220
  triggerSource: "autoDream",
1131
1221
  operationType: "dream_summarize",
1132
- modelId: `${route.provider}:${route.model}`,
1222
+ modelId: `${summaryRoute.provider}:${summaryRoute.model}`,
1133
1223
  relatedMemoryIds: [],
1134
1224
  streamError: () => summaryStreamFailure
1135
1225
  }, (reportUsage) => streamText(ctx, {
1136
- provider: route.provider,
1137
- model: route.model,
1226
+ provider: summaryRoute.provider,
1227
+ model: summaryRoute.model,
1138
1228
  purpose: "compaction",
1139
1229
  maxTokens: config.dreamMaxTokens ?? 2048,
1140
1230
  ...(withEffort && effort ? { reasoningEffort: effort } : {}),
1141
1231
  messages: [
1142
1232
  { role: "system", content: [{ type: "text", text: STR.prompts.dreamSummary[language] }], source: { kind: "plugin", plugin: "dsh-mneme" } },
1143
- { role: "user", content: [{ type: "text", text: service.all().filter((m) => !m.archived && m.type !== "summary").map((m) => `- ${m.title}: ${m.content}`).join("\n") }], source: { kind: "plugin", plugin: "dsh-mneme" } }
1233
+ { role: "user", content: [{ type: "text", text: summaryInputs.map((m) => `- ${m.title}: ${m.content}`).join("\n") }], source: { kind: "plugin", plugin: "dsh-mneme" } }
1144
1234
  ]
1145
1235
  }, reportUsage, (reason) => { summaryStreamFailure = describeStreamFailure(reason); }));
1146
1236
  };
1237
+ // Resident status bar (#164 alignment): the overview is THE one resident
1238
+ // narrative — summary tier 0, refreshed every run via _overwrite (supersede
1239
+ // v1, content_history traceable). Its 口径 (what snapshot produced it) is
1240
+ // stamped into the content so the standing answer is always auditable; the
1241
+ // formal evidence column lands with the narrative-bars batch.
1242
+ // Issue #258:输入硬上限(条数)——原全库无界,把 dream 路由指向小 ctx
1243
+ // 模型时 120k token 当场 CONTEXT_WINDOW_EXCEEDED(#258 实测),且随库增长
1244
+ // 渐进恶化。倒序排序与 consolidate 窗口同款(updated_at desc, id tiebreak),
1245
+ // 保留最新的;0 = 关闭上限(回归全库行为,调用方自担 ctx)。
1246
+ // #230:document 指针行不进总览(与 dream/sleep 五个候选池的同款排除)。
1247
+ const summaryAll = service.all().filter((m) => !m.archived && m.type !== "summary" && m.type !== "document");
1248
+ const summaryMaxInputs = Number.isInteger(config.dreamSummaryMaxInputs) ? config.dreamSummaryMaxInputs : 0;
1249
+ const summaryInputs = summaryMaxInputs > 0 && summaryAll.length > summaryMaxInputs
1250
+ ? [...summaryAll]
1251
+ .sort((a, b) => {
1252
+ const ta = String(a.updated_at ?? "");
1253
+ const tb = String(b.updated_at ?? "");
1254
+ if (ta < tb) return 1;
1255
+ if (ta > tb) return -1;
1256
+ return a.id < b.id ? -1 : a.id > b.id ? 1 : 0;
1257
+ })
1258
+ .slice(0, summaryMaxInputs)
1259
+ : summaryAll;
1260
+
1261
+ const summaryScope = STR.summaryScope[language](
1262
+ summaryInputs.length,
1263
+ runId.slice(0, 8),
1264
+ new Date().toISOString().slice(0, 10)
1265
+ );
1147
1266
  try {
1148
1267
  // Bug8: the summary call is audited too (operation dream_summarize).
1149
1268
  summaryText = await withEffortFallback(ctx, effort, () => runSummary(true), () => runSummary(false), () => summaryStreamFailure);
@@ -1156,21 +1275,32 @@ export function createDreamScheduler({ onRun, thresholdCount = 10, thresholdChar
1156
1275
  // Bug5 carve-out: the library overview is regenerated every run, so it
1157
1276
  // must REPLACE the previous overview (not append — that would grow the
1158
1277
  // summary unboundedly). `_overwrite` still archives the old overview into
1159
- // content_history before replacing it.
1160
- service.saveWithDedupe({ type: "summary", title: STR.summaryTitle[language], content: summaryText.trim(), importance: 5, source: "dream", _overwrite: true });
1278
+ // content_history before replacing it. Content carries the snapshot-scope
1279
+ // footer computed above — the resident bar states its own 口径.
1280
+ service.saveWithDedupe({ type: "summary", title: STR.summaryTitle[language], content: `${summaryText.trim()}${summaryScope}`, importance: 5, source: "dream", _overwrite: true });
1161
1281
  summaryStored = true;
1162
1282
  // Re-embed the fresh summary so the index stays in sync with the store.
1163
1283
  if (semantic?.embedder && semantic?.vectorIndex) {
1164
1284
  try {
1165
- const summary = service.all().find((m) => m.type === "summary");
1166
- if (summary) {
1167
- const v = await semantic.embedder.embedSingle([summary.title, summary.content].filter(Boolean).join("\n"));
1168
- if (v?.length) semantic.vectorIndex.saveEmbedding(summary.id, v);
1169
- if (semantic.embedder.modelHash) semantic.vectorIndex.markModel?.(semantic.embedder.modelHash, semantic.embedder.dimension);
1170
- }
1285
+ // source="dream" 限定:叙述条(source=narrative)同为 summary 类型,
1286
+ // 不加限定时 find 可能截胡,把总览的 re-embed 花在叙述条上。
1287
+ const summary = service.all().find((m) => m.type === "summary" && m.source === "dream");
1288
+ if (summary) await reEmbedMemory(semantic, summary, logger);
1171
1289
  } catch { /* best-effort */ }
1172
1290
  }
1173
1291
  }
1292
+ // Narrative bars (#164 alignment, opt-in dreamNarrativeEnabled): per-topic
1293
+ // on-demand narratives after consolidation + summary. Additive phase —
1294
+ // failures degrade to zero narratives, never break the run.
1295
+ let narrativesStored = 0;
1296
+ if (config.dreamNarrativeEnabled === true) {
1297
+ try {
1298
+ narrativesStored = await generateNarratives({ ctx, service, config, route, language, effort, semantic, logger });
1299
+ } catch (error) {
1300
+ logger?.warn?.(`dsh-mneme dream: narrative phase failed: ${String(error)}`);
1301
+ }
1302
+ }
1303
+
1174
1304
  // Honest status assignment (never a fake ok):
1175
1305
  // reconcile — some decisions validated but did not commit (CAS/rollback).
1176
1306
  // noop — nothing changed and no summary persisted: truly an empty
@@ -1206,6 +1336,7 @@ export function createDreamScheduler({ onRun, thresholdCount = 10, thresholdChar
1206
1336
  failures,
1207
1337
  frozen: frozenCount,
1208
1338
  summary: summaryStored,
1339
+ narratives: narrativesStored,
1209
1340
  // Issue #104:只有真发生跳过时才落库(degraded 也可能仅因 summary 为空),
1210
1341
  // ok 轮留 NULL,避免「空数组」与「无此字段」两种假明细占据审计行。
1211
1342
  skipped: skippedInvalid ? skipped : undefined
package/src/embedding.js CHANGED
@@ -1,177 +1,177 @@
1
- // OpenAI-compatible embedding client for vector search. DSH's LLM service is
2
- // chat-only, so dsh-mneme calls an external `/embeddings` endpoint itself.
3
- // Works with OpenAI, SiliconFlow, Zhipu, local Ollama (via OpenAI-compatible
4
- // proxy) and any provider exposing the standard embeddings API.
5
- const DEFAULT_TIMEOUT_MS = 15000;
6
-
7
- /** djb2 — stable, fast fingerprint for a provider/model string. Mirrors the
8
- * hash used by the local embedders so all backends share one fingerprint
9
- * format (model#hex) for vector_meta consistency checks. */
10
- function hashString(s) {
11
- let h = 5381;
12
- for (let i = 0; i < s.length; i++) h = ((h << 5) + h + s.charCodeAt(i)) >>> 0;
13
- return h.toString(16);
14
- }
15
-
16
- /** Full provider+model fingerprint used for index-consistency checks. */
17
- function modelHashOf(model) {
18
- return `${model}#${hashString(model)}`;
19
- }
20
-
21
- /** Normalize a configured baseUrl into the full embeddings endpoint URL. */
22
- function embeddingsUrl(baseUrl) {
23
- const base = String(baseUrl ?? "").trim().replace(/\/+$/, "");
24
- if (!base) return "";
25
- // Accept both "https://host/v1" and a full path ending in /embeddings.
26
- if (/\/embeddings$/i.test(base)) return base;
27
- return `${base}/embeddings`;
28
- }
29
-
30
- /**
31
- * Call the embeddings API for one text. Resolves to a Float64 array, or null
32
- * when the provider is not configured, the call fails, or the response is
33
- * unusable. Never throws: failures degrade to keyword search.
34
- */
35
- export async function embedText({ baseUrl, apiKey, model }, text) {
36
- const url = embeddingsUrl(baseUrl);
37
- if (!url || !apiKey || !model || !text) return null;
38
- let res;
39
- try {
40
- res = await fetch(url, {
41
- method: "POST",
42
- headers: {
43
- "Content-Type": "application/json",
44
- "Authorization": `Bearer ${apiKey}`
45
- },
46
- body: JSON.stringify({ model, input: String(text).slice(0, 8000) }),
47
- signal: AbortSignal.timeout(DEFAULT_TIMEOUT_MS)
48
- });
49
- } catch {
50
- return null;
51
- }
52
- if (!res.ok) return null;
53
- let body;
54
- try {
55
- body = await res.json();
56
- } catch {
57
- return null;
58
- }
59
- const vec = body?.data?.[0]?.embedding;
60
- return Array.isArray(vec) && vec.length ? Array.from(vec) : null;
61
- }
62
-
63
- /**
64
- * Embedder bound to the current settings + store: on each write it re-embeds
65
- * the row's title+content and stores the vector. Failures are swallowed so a
66
- * flaky embedding endpoint never breaks memory writes.
67
- *
68
- * `vectorIndex` (optional) is the vector_meta fingerprint holder: after any
69
- * successful embed the model that produced the vectors is recorded, so the
70
- * index can detect drift and the auto-reindex backfill knows what to rebuild.
71
- */
72
- export function createEmbedder({ store, settings, logger, vectorIndex }) {
73
- // Dimension of the most recent successful embed, exposed for fingerprinting.
74
- let _dimension = 0;
75
-
76
- /** Record the producing model fingerprint in vector_meta (best-effort). */
77
- function markModel(cfg, dimension) {
78
- if (!vectorIndex || typeof vectorIndex.markModel !== "function") return;
79
- try {
80
- vectorIndex.markModel(modelHashOf(cfg.model), dimension);
81
- } catch { /* metadata write is best-effort */ }
82
- }
83
-
84
- // issue #135: 「配置是否齐备」的唯一判据 —— embed()/embedFor() 的守卫与下面
85
- // ready/configured 两个 getter 共用,避免以后改一处漏一处。
86
- const configComplete = (cfg) => !!(cfg?.enabled && cfg.baseUrl && cfg.apiKey && cfg.model);
87
-
88
- async function embedFor(id, title, content) {
89
- const cfg = settings.getVectorConfig();
90
- if (!configComplete(cfg)) return;
91
- const text = [title, content].filter(Boolean).join("\n");
92
- const vector = await embedText(cfg, text);
93
- if (vector) {
94
- store.setEmbedding(id, vector);
95
- _dimension = vector.length;
96
- // Bug3: record which model produced the current vectors so the index can
97
- // detect drift and skip a redundant backfill when nothing changed.
98
- markModel(cfg, vector.length);
99
- logger?.info?.(`[dsh-mneme] embedded memory ${id} (dim=${vector.length})`);
100
- }
101
- }
102
-
103
- return {
104
- // Display name for /semantic: a literal's constructor.name is "Object",
105
- // which the status card would render verbatim.
106
- name: "OpenAI",
107
- /**
108
- * issue #135: 这个 embedder 没有异步初始化,`"ready" in embedder` 的通用
109
- * 分支会把它恒判为可用 —— 而 vector-config 四项有缺时它一条都嵌不出来。
110
- * 对外报真实可用性,让 /semantic 的 ready 不再是「绿色假阳性」。
111
- * 用 getter 而非快照字段:设置面板改完配置无需重启即生效(与 embed() 同源读取)。
112
- */
113
- get ready() {
114
- return configComplete(settings.getVectorConfig());
115
- },
116
- /**
117
- * 与 ready 同源,分开两个名字是给 /semantic 用:local/ollama 的
118
- * ready=false 表示「还在初始化」,而这里的 false 表示「根本没配」。
119
- */
120
- get configured() {
121
- return configComplete(settings.getVectorConfig());
122
- },
123
- /** Fire-and-forget re-embed of a memory after any write. */
124
- schedule(memory) {
125
- if (!memory?.id) return;
126
- embedFor(memory.id, memory.title, memory.content).catch(() => {});
127
- },
128
-
129
- /** Embed one text and return its vector (null on failure/disabled). */
130
- async embed(query) {
131
- const cfg = settings.getVectorConfig();
132
- if (!configComplete(cfg)) return null;
133
- const vector = await embedText(cfg, query);
134
- if (vector) _dimension = vector.length;
135
- return vector;
136
- },
137
-
138
- // Bug1: single-text adapter. Local/ollama embedders expose embedSingle
139
- // natively; the legacy OpenAI-compatible client only has embed. This
140
- // adapter unifies the interface so vector-index rebuildIndex (which guards
141
- // on `typeof embedder.embedSingle === "function"`) accepts this embedder.
142
- async embedSingle(text) {
143
- if (typeof this.embed === "function") return this.embed(text);
144
- return null;
145
- },
146
-
147
- /** Model fingerprint (model#hex), or undefined when not configured. */
148
- get modelHash() {
149
- const cfg = settings.getVectorConfig();
150
- return cfg?.enabled && cfg.model ? modelHashOf(cfg.model) : undefined;
151
- },
152
-
153
- /** Dimension of the last successful embed (0 when never embedded). */
154
- get dimension() {
155
- return _dimension || undefined;
156
- },
157
-
158
- /** Batch re-index rows still missing an embedding. */
159
- async reindexMissing(limit = 50) {
160
- const cfg = settings.getVectorConfig();
161
- if (!cfg?.enabled) return { indexed: 0, skipped: 0 };
162
- const rows = store.needsEmbedding(limit);
163
- let indexed = 0;
164
- for (const row of rows) {
165
- const text = [row.title, row.content].filter(Boolean).join("\n");
166
- const vector = await embedText(cfg, text);
167
- if (vector) {
168
- store.setEmbedding(row.id, vector);
169
- _dimension = vector.length;
170
- indexed++;
171
- }
172
- }
173
- if (indexed > 0) markModel(cfg, _dimension || undefined);
174
- return { indexed, skipped: rows.length - indexed };
175
- }
176
- };
177
- }
1
+ // OpenAI-compatible embedding client for vector search. DSH's LLM service is
2
+ // chat-only, so dsh-mneme calls an external `/embeddings` endpoint itself.
3
+ // Works with OpenAI, SiliconFlow, Zhipu, local Ollama (via OpenAI-compatible
4
+ // proxy) and any provider exposing the standard embeddings API.
5
+ const DEFAULT_TIMEOUT_MS = 15000;
6
+
7
+ /** djb2 — stable, fast fingerprint for a provider/model string. Mirrors the
8
+ * hash used by the local embedders so all backends share one fingerprint
9
+ * format (model#hex) for vector_meta consistency checks. */
10
+ function hashString(s) {
11
+ let h = 5381;
12
+ for (let i = 0; i < s.length; i++) h = ((h << 5) + h + s.charCodeAt(i)) >>> 0;
13
+ return h.toString(16);
14
+ }
15
+
16
+ /** Full provider+model fingerprint used for index-consistency checks. */
17
+ function modelHashOf(model) {
18
+ return `${model}#${hashString(model)}`;
19
+ }
20
+
21
+ /** Normalize a configured baseUrl into the full embeddings endpoint URL. */
22
+ function embeddingsUrl(baseUrl) {
23
+ const base = String(baseUrl ?? "").trim().replace(/\/+$/, "");
24
+ if (!base) return "";
25
+ // Accept both "https://host/v1" and a full path ending in /embeddings.
26
+ if (/\/embeddings$/i.test(base)) return base;
27
+ return `${base}/embeddings`;
28
+ }
29
+
30
+ /**
31
+ * Call the embeddings API for one text. Resolves to a Float64 array, or null
32
+ * when the provider is not configured, the call fails, or the response is
33
+ * unusable. Never throws: failures degrade to keyword search.
34
+ */
35
+ export async function embedText({ baseUrl, apiKey, model }, text) {
36
+ const url = embeddingsUrl(baseUrl);
37
+ if (!url || !apiKey || !model || !text) return null;
38
+ let res;
39
+ try {
40
+ res = await fetch(url, {
41
+ method: "POST",
42
+ headers: {
43
+ "Content-Type": "application/json",
44
+ "Authorization": `Bearer ${apiKey}`
45
+ },
46
+ body: JSON.stringify({ model, input: String(text).slice(0, 8000) }),
47
+ signal: AbortSignal.timeout(DEFAULT_TIMEOUT_MS)
48
+ });
49
+ } catch {
50
+ return null;
51
+ }
52
+ if (!res.ok) return null;
53
+ let body;
54
+ try {
55
+ body = await res.json();
56
+ } catch {
57
+ return null;
58
+ }
59
+ const vec = body?.data?.[0]?.embedding;
60
+ return Array.isArray(vec) && vec.length ? Array.from(vec) : null;
61
+ }
62
+
63
+ /**
64
+ * Embedder bound to the current settings + store: on each write it re-embeds
65
+ * the row's title+content and stores the vector. Failures are swallowed so a
66
+ * flaky embedding endpoint never breaks memory writes.
67
+ *
68
+ * `vectorIndex` (optional) is the vector_meta fingerprint holder: after any
69
+ * successful embed the model that produced the vectors is recorded, so the
70
+ * index can detect drift and the auto-reindex backfill knows what to rebuild.
71
+ */
72
+ export function createEmbedder({ store, settings, logger, vectorIndex }) {
73
+ // Dimension of the most recent successful embed, exposed for fingerprinting.
74
+ let _dimension = 0;
75
+
76
+ /** Record the producing model fingerprint in vector_meta (best-effort). */
77
+ function markModel(cfg, dimension) {
78
+ if (!vectorIndex || typeof vectorIndex.markModel !== "function") return;
79
+ try {
80
+ vectorIndex.markModel(modelHashOf(cfg.model), dimension);
81
+ } catch { /* metadata write is best-effort */ }
82
+ }
83
+
84
+ // issue #135: 「配置是否齐备」的唯一判据 —— embed()/embedFor() 的守卫与下面
85
+ // ready/configured 两个 getter 共用,避免以后改一处漏一处。
86
+ const configComplete = (cfg) => !!(cfg?.enabled && cfg.baseUrl && cfg.apiKey && cfg.model);
87
+
88
+ async function embedFor(id, title, content) {
89
+ const cfg = settings.getVectorConfig();
90
+ if (!configComplete(cfg)) return;
91
+ const text = [title, content].filter(Boolean).join("\n");
92
+ const vector = await embedText(cfg, text);
93
+ if (vector) {
94
+ store.setEmbedding(id, vector);
95
+ _dimension = vector.length;
96
+ // Bug3: record which model produced the current vectors so the index can
97
+ // detect drift and skip a redundant backfill when nothing changed.
98
+ markModel(cfg, vector.length);
99
+ logger?.info?.(`[dsh-mneme] embedded memory ${id} (dim=${vector.length})`);
100
+ }
101
+ }
102
+
103
+ return {
104
+ // Display name for /semantic: a literal's constructor.name is "Object",
105
+ // which the status card would render verbatim.
106
+ name: "OpenAI",
107
+ /**
108
+ * issue #135: 这个 embedder 没有异步初始化,`"ready" in embedder` 的通用
109
+ * 分支会把它恒判为可用 —— 而 vector-config 四项有缺时它一条都嵌不出来。
110
+ * 对外报真实可用性,让 /semantic 的 ready 不再是「绿色假阳性」。
111
+ * 用 getter 而非快照字段:设置面板改完配置无需重启即生效(与 embed() 同源读取)。
112
+ */
113
+ get ready() {
114
+ return configComplete(settings.getVectorConfig());
115
+ },
116
+ /**
117
+ * 与 ready 同源,分开两个名字是给 /semantic 用:local/ollama 的
118
+ * ready=false 表示「还在初始化」,而这里的 false 表示「根本没配」。
119
+ */
120
+ get configured() {
121
+ return configComplete(settings.getVectorConfig());
122
+ },
123
+ /** Fire-and-forget re-embed of a memory after any write. */
124
+ schedule(memory) {
125
+ if (!memory?.id) return;
126
+ embedFor(memory.id, memory.title, memory.content).catch(() => {});
127
+ },
128
+
129
+ /** Embed one text and return its vector (null on failure/disabled). */
130
+ async embed(query) {
131
+ const cfg = settings.getVectorConfig();
132
+ if (!configComplete(cfg)) return null;
133
+ const vector = await embedText(cfg, query);
134
+ if (vector) _dimension = vector.length;
135
+ return vector;
136
+ },
137
+
138
+ // Bug1: single-text adapter. Local/ollama embedders expose embedSingle
139
+ // natively; the legacy OpenAI-compatible client only has embed. This
140
+ // adapter unifies the interface so vector-index rebuildIndex (which guards
141
+ // on `typeof embedder.embedSingle === "function"`) accepts this embedder.
142
+ async embedSingle(text) {
143
+ if (typeof this.embed === "function") return this.embed(text);
144
+ return null;
145
+ },
146
+
147
+ /** Model fingerprint (model#hex), or undefined when not configured. */
148
+ get modelHash() {
149
+ const cfg = settings.getVectorConfig();
150
+ return cfg?.enabled && cfg.model ? modelHashOf(cfg.model) : undefined;
151
+ },
152
+
153
+ /** Dimension of the last successful embed (0 when never embedded). */
154
+ get dimension() {
155
+ return _dimension || undefined;
156
+ },
157
+
158
+ /** Batch re-index rows still missing an embedding. */
159
+ async reindexMissing(limit = 50) {
160
+ const cfg = settings.getVectorConfig();
161
+ if (!cfg?.enabled) return { indexed: 0, skipped: 0 };
162
+ const rows = store.needsEmbedding(limit);
163
+ let indexed = 0;
164
+ for (const row of rows) {
165
+ const text = [row.title, row.content].filter(Boolean).join("\n");
166
+ const vector = await embedText(cfg, text);
167
+ if (vector) {
168
+ store.setEmbedding(row.id, vector);
169
+ _dimension = vector.length;
170
+ indexed++;
171
+ }
172
+ }
173
+ if (indexed > 0) markModel(cfg, _dimension || undefined);
174
+ return { indexed, skipped: rows.length - indexed };
175
+ }
176
+ };
177
+ }