@modusensus/dsh-mneme 0.6.7 → 0.6.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (139) hide show
  1. package/.github/workflows/test.yml +32 -0
  2. package/.release-notes-v0.6.9.md +13 -0
  3. package/CHANGELOG.md +89 -0
  4. package/LICENSE +21 -21
  5. package/README.md +219 -463
  6. package/SECURITY.md +544 -0
  7. package/docs/devlog/2026-08-14-dsh-mneme-dev-log.md +247 -0
  8. package/docs/devlog/2026-08-15-dsh-mneme-audit-stress-dev-log.md +145 -0
  9. package/docs/devlog/2026-08-15-dsh-mneme-pipeline-dev-log.md +56 -0
  10. package/docs/devlog/2026-08-15-dsh-mneme-reflection-dev-log.md +77 -0
  11. package/docs/devlog/2026-08-15-dsh-mneme-review-fixes-dev-log.md +64 -0
  12. package/docs/devlog/2026-08-15-dsh-mneme-semantic-dev-log.md +90 -0
  13. package/dsh-mneme/CHANGELOG.md +248 -0
  14. package/dsh-mneme/LICENSE +21 -0
  15. package/dsh-mneme/README.md +465 -0
  16. package/{cordis.patch.yml → dsh-mneme/cordis.patch.yml} +15 -15
  17. package/dsh-mneme/docs/AGENT_MEMORY_RESEARCH.md +183 -0
  18. package/dsh-mneme/docs/ENTITIES.md +245 -0
  19. package/dsh-mneme/docs/LOCAL_MODEL.md +141 -0
  20. package/dsh-mneme/docs/MIGRATION.md +127 -0
  21. package/dsh-mneme/docs/SEMANTIC.md +256 -0
  22. package/dsh-mneme/docs/SLEEP.md +163 -0
  23. package/{lib → dsh-mneme/lib}/api.js +783 -783
  24. package/{lib → dsh-mneme/lib}/client.js +1757 -1757
  25. package/{src → dsh-mneme/lib}/commands.js +64 -64
  26. package/{lib → dsh-mneme/lib}/config.js +298 -288
  27. package/{lib → dsh-mneme/lib}/dream/clustering.js +118 -118
  28. package/{src → dsh-mneme/lib}/dream/decisions.js +488 -439
  29. package/{lib → dsh-mneme/lib}/dream/sleep.js +561 -554
  30. package/{src → dsh-mneme/lib}/dream/tag-extractor.js +156 -156
  31. package/{lib → dsh-mneme/lib}/dream.js +958 -929
  32. package/{src → dsh-mneme/lib}/embedding.js +154 -154
  33. package/{src → dsh-mneme/lib}/entities/extractor.js +242 -242
  34. package/{src → dsh-mneme/lib}/hot-memory.js +53 -53
  35. package/{lib → dsh-mneme/lib}/index.js +361 -361
  36. package/{src → dsh-mneme/lib}/inject.js +208 -208
  37. package/{lib → dsh-mneme/lib}/local-embedder.js +282 -282
  38. package/{lib → dsh-mneme/lib}/mirror.js +170 -170
  39. package/{lib → dsh-mneme/lib}/parser/tag.js +59 -59
  40. package/{lib → dsh-mneme/lib}/parser/wiki-link.js +38 -38
  41. package/{src → dsh-mneme/lib}/quality-filter.js +123 -123
  42. package/{lib → dsh-mneme/lib}/reranker.js +218 -218
  43. package/{lib → dsh-mneme/lib}/search/adaptive.js +22 -22
  44. package/{lib → dsh-mneme/lib}/search/bm25.js +96 -96
  45. package/{src → dsh-mneme/lib}/search/tag-boost.js +61 -61
  46. package/{src → dsh-mneme/lib}/service.js +1726 -1726
  47. package/{lib → dsh-mneme/lib}/settings.js +172 -172
  48. package/{lib → dsh-mneme/lib}/store.js +2238 -2238
  49. package/{lib → dsh-mneme/lib}/summarize.js +236 -236
  50. package/{lib → dsh-mneme/lib}/tools.js +290 -290
  51. package/{lib → dsh-mneme/lib}/vector-index.js +116 -116
  52. package/dsh-mneme/package-lock.json +1936 -0
  53. package/dsh-mneme/package.json +80 -0
  54. package/{scripts → dsh-mneme/scripts}/benchmark-embed.js +201 -201
  55. package/{scripts → dsh-mneme/scripts}/benchmark-recall.js +133 -133
  56. package/{scripts → dsh-mneme/scripts}/benchmark-rerank.js +166 -166
  57. package/{scripts → dsh-mneme/scripts}/e2e-dsh.js +218 -218
  58. package/{scripts → dsh-mneme/scripts}/stress-dsh.js +255 -255
  59. package/{scripts → dsh-mneme/scripts}/sync-lib.js +52 -52
  60. package/{src → dsh-mneme/src}/api.js +783 -783
  61. package/{lib → dsh-mneme/src}/commands.js +64 -64
  62. package/{src → dsh-mneme/src}/config.js +298 -288
  63. package/{src → dsh-mneme/src}/dream/clustering.js +118 -118
  64. package/{lib → dsh-mneme/src}/dream/decisions.js +488 -439
  65. package/{src → dsh-mneme/src}/dream/sleep.js +561 -554
  66. package/{lib → dsh-mneme/src}/dream/tag-extractor.js +156 -156
  67. package/{src → dsh-mneme/src}/dream.js +958 -929
  68. package/{lib → dsh-mneme/src}/embedding.js +154 -154
  69. package/{lib → dsh-mneme/src}/entities/extractor.js +242 -242
  70. package/{lib → dsh-mneme/src}/hot-memory.js +53 -53
  71. package/{src → dsh-mneme/src}/index.js +361 -361
  72. package/{lib → dsh-mneme/src}/inject.js +208 -208
  73. package/{src → dsh-mneme/src}/local-embedder.js +282 -282
  74. package/{src → dsh-mneme/src}/mirror.js +170 -170
  75. package/{src → dsh-mneme/src}/parser/tag.js +59 -59
  76. package/{src → dsh-mneme/src}/parser/wiki-link.js +38 -38
  77. package/{lib → dsh-mneme/src}/quality-filter.js +123 -123
  78. package/{src → dsh-mneme/src}/reranker.js +218 -218
  79. package/{src → dsh-mneme/src}/search/adaptive.js +22 -22
  80. package/{src → dsh-mneme/src}/search/bm25.js +96 -96
  81. package/{lib → dsh-mneme/src}/search/tag-boost.js +61 -61
  82. package/{lib → dsh-mneme/src}/service.js +1726 -1726
  83. package/{src → dsh-mneme/src}/settings.js +172 -172
  84. package/{src → dsh-mneme/src}/store.js +2238 -2238
  85. package/{src → dsh-mneme/src}/summarize.js +236 -236
  86. package/{src → dsh-mneme/src}/tools.js +290 -290
  87. package/{src → dsh-mneme/src}/vector-index.js +116 -116
  88. package/{test → dsh-mneme/test}/api.test.js +594 -594
  89. package/{test → dsh-mneme/test}/audit.test.js +448 -448
  90. package/{test → dsh-mneme/test}/benchmark.test.js +35 -35
  91. package/{test → dsh-mneme/test}/boundary-v0625.test.js +82 -82
  92. package/{test → dsh-mneme/test}/client.test.js +368 -368
  93. package/{test → dsh-mneme/test}/clustering.test.js +100 -100
  94. package/{test → dsh-mneme/test}/commands.test.js +69 -69
  95. package/{test → dsh-mneme/test}/config.test.js +50 -50
  96. package/{test → dsh-mneme/test}/conflict-freeze.test.js +290 -290
  97. package/{test → dsh-mneme/test}/directory.test.js +134 -134
  98. package/{test → dsh-mneme/test}/dream.test.js +1060 -901
  99. package/{test → dsh-mneme/test}/entities.test.js +522 -522
  100. package/{test → dsh-mneme/test}/epistemic.test.js +298 -298
  101. package/{test → dsh-mneme/test}/fnew-0112.test.js +311 -311
  102. package/{test → dsh-mneme/test}/fnew-03.test.js +422 -422
  103. package/{test → dsh-mneme/test}/graph-api.test.js +175 -175
  104. package/{test → dsh-mneme/test}/helpers/dream-mock.js +82 -82
  105. package/{test → dsh-mneme/test}/hot-memory.test.js +174 -174
  106. package/{test → dsh-mneme/test}/inject.test.js +103 -103
  107. package/{test → dsh-mneme/test}/llm-audit.test.js +279 -279
  108. package/{test → dsh-mneme/test}/local-embedder.test.js +227 -227
  109. package/{test → dsh-mneme/test}/mirror-dirty.test.js +424 -424
  110. package/{test → dsh-mneme/test}/mirror-edit-digest.test.js +187 -187
  111. package/{test → dsh-mneme/test}/mirror-generation.test.js +499 -499
  112. package/{test → dsh-mneme/test}/mirror.test.js +249 -249
  113. package/{test → dsh-mneme/test}/normalize-decisions.test.js +120 -120
  114. package/{test → dsh-mneme/test}/peer-blockers.test.js +190 -190
  115. package/{test → dsh-mneme/test}/policy-epoch.test.js +259 -259
  116. package/{test → dsh-mneme/test}/provenance.test.js +103 -103
  117. package/{test → dsh-mneme/test}/quality-filter.test.js +118 -118
  118. package/{test → dsh-mneme/test}/reasoning-effort.test.js +199 -199
  119. package/{test → dsh-mneme/test}/recall-evals.test.js +235 -235
  120. package/{test → dsh-mneme/test}/recall-layer.test.js +315 -315
  121. package/{test → dsh-mneme/test}/receipt-chain.test.js +451 -451
  122. package/{test → dsh-mneme/test}/reflection.test.js +226 -226
  123. package/{test → dsh-mneme/test}/reranker.test.js +240 -240
  124. package/{test → dsh-mneme/test}/search-fusion.test.js +90 -90
  125. package/{test → dsh-mneme/test}/semantic.test.js +124 -124
  126. package/{test → dsh-mneme/test}/service-search.test.js +199 -199
  127. package/{test → dsh-mneme/test}/service.test.js +435 -435
  128. package/{test → dsh-mneme/test}/settings.test.js +118 -118
  129. package/{test → dsh-mneme/test}/sleep.test.js +365 -365
  130. package/{test → dsh-mneme/test}/store.test.js +436 -436
  131. package/{test → dsh-mneme/test}/stress.test.js +209 -209
  132. package/{test → dsh-mneme/test}/summarize.test.js +191 -191
  133. package/{test → dsh-mneme/test}/tag-boost.test.js +125 -125
  134. package/{test → dsh-mneme/test}/tag.test.js +312 -312
  135. package/{test → dsh-mneme/test}/tools.test.js +285 -285
  136. package/{test → dsh-mneme/test}/vector-index.test.js +221 -221
  137. package/{test → dsh-mneme/test}/wiki-link.test.js +332 -332
  138. package/package.json +18 -40
  139. package//346/250/252/345/271/205.png +0 -0
@@ -1,361 +1,361 @@
1
- import { createStore } from "./store.js";
2
- import { createMirror, TYPE_FILE } from "./mirror.js";
3
- import { createService } from "./service.js";
4
- import { createTools } from "./tools.js";
5
- import { createInjector } from "./inject.js";
6
- import { createSummarizer } from "./summarize.js";
7
- import { createDreamScheduler } from "./dream.js";
8
- import { createSleepScheduler, runSleep } from "./dream/sleep.js";
9
- import { createApi } from "./api.js";
10
- import { createSettings } from "./settings.js";
11
- import { createCommandManager } from "./commands.js";
12
- import { createEmbedder } from "./embedding.js";
13
- import { createEmbedderByProvider } from "./local-embedder.js";
14
- import { LocalReranker } from "./reranker.js";
15
- import { createVectorIndex } from "./vector-index.js";
16
- import { Config } from "./config.js";
17
- import { extractEntities } from "./entities/extractor.js";
18
- import { mkdirSync } from "node:fs";
19
- import { join } from "node:path";
20
- import { homedir } from "node:os";
21
-
22
- export const name = "dsh-mneme";
23
- export const inject = ["tools", "systemPrompt", "webServer", "llm", "agentDefaultModel", "commands"];
24
- export { Config };
25
-
26
- // Arrow (not function declaration): cordis 4 treats any apply with a
27
- // prototype as a class constructor (`new apply(...)`) and discards its return
28
- // value, so a `function apply` disposer would never run on unload. An arrow
29
- // has no prototype, is called normally, and its returned disposer is collected
30
- // and run by the fiber on unload.
31
- export const apply = (ctx, config) => {
32
- const cfg = Config(config);
33
-
34
- // Resolve memoryDir: expand leading "~"
35
- const memoryDir = cfg.memoryDir.startsWith("~")
36
- ? join(homedir(), cfg.memoryDir.slice(1))
37
- : cfg.memoryDir;
38
- mkdirSync(memoryDir, { recursive: true });
39
-
40
- const store = createStore(join(memoryDir, "memory.db"));
41
- // Prune reflection failure rows older than 90 days on boot (best-effort, so
42
- // the failure table never grows unbounded).
43
- try {
44
- store.deleteOldFailures(new Date(Date.now() - 90 * 86400000).toISOString());
45
- } catch { /* non-fatal */ }
46
- // Bug8: enforce llm_audit_logs retention on boot (config.llmAudit.retentionDays,
47
- // default 90). Best-effort like the failure prune — the audit trail is
48
- // bookkeeping and a failed purge must never block plugin boot.
49
- try {
50
- if (cfg.llmAudit?.enabled !== false) {
51
- const retentionMs = Number.isInteger(cfg.llmAudit?.retentionDays) ? cfg.llmAudit.retentionDays : 90;
52
- store.deleteOldLlmAudits(new Date(Date.now() - retentionMs * 86400000).toISOString());
53
- }
54
- } catch { /* non-fatal */ }
55
- const mirror = createMirror(memoryDir);
56
- const service = createService({ store, mirror, config: cfg, logger: ctx.logger });
57
-
58
- // F-NEW-03: if the mirror sync failed last run (persisted dirty state), retry
59
- // a safe re-render at boot so a stale mirror converges without needing a
60
- // business write. Bounded: single attempt; on failure dirty stays for the
61
- // next boot. Never throws.
62
- service.recoverMirror();
63
-
64
- // Recall-layer receipt: when searchMemories runs with recordRecall=true, the
65
- // retrieval scene (query/mode/topK/threshold + candidates) is persisted to
66
- // recall_runs for audit/replay — the sibling of the dream_runs judgment trail.
67
- // Best-effort: a failed recall write must never break the search.
68
- service.setRecallRecorder((recall) => {
69
- try {
70
- store.saveRecallRun({
71
- query: recall.query,
72
- mode: recall.mode,
73
- topK: recall.topK,
74
- threshold: recall.threshold ?? null,
75
- candidates: recall.candidates ?? [],
76
- created_at: recall.createdAt
77
- });
78
- } catch { /* non-fatal: recall recording is bookkeeping */ }
79
- });
80
-
81
- // User-configurable settings (profile, rules) and custom commands share the
82
- // same SQLite file but live in dedicated tables, isolated from memories.
83
- const settings = createSettings(store.db);
84
-
85
- // Semantic pipeline: a local/ollama embedder when configured, otherwise the
86
- // legacy OpenAI-compatible embedder (settings-driven). The vector index wraps
87
- // the store's embedding column and tracks the active model fingerprint. A
88
- // slow embedder init (model download) never blocks plugin boot — failures
89
- // degrade to keyword search.
90
- const vectorIndex = createVectorIndex({ store, logger: ctx.logger });
91
- service.setVectorIndex(vectorIndex);
92
-
93
- // Human edits in mirror files win on every sync; merge them back first.
94
- // TYPE_FILE maps each memory type to its mirror filename. Read every type's
95
- // edits up front: mergeHumanEdits re-renders ALL mirror files on success, so
96
- // a per-type read-then-merge loop would overwrite edits in files not yet read
97
- // (e.g. preferences.md merging would clobber unsynced projects.md edits).
98
- const humanEdits = new Map();
99
- for (const type of Object.keys(TYPE_FILE)) {
100
- humanEdits.set(type, mirror.readHumanEdits(type));
101
- }
102
- const applyHumanEdits = () => {
103
- for (const [type, edits] of humanEdits) {
104
- if (edits.length) service.mergeHumanEdits(type, edits);
105
- }
106
- };
107
-
108
- let embedder = null;
109
- let reranker = null;
110
- if (cfg.embedProvider === "openai") {
111
- // vectorIndex is passed so the legacy OpenAI embedder records the producing
112
- // model fingerprint after each successful embed (Bug3).
113
- embedder = createEmbedder({ store, settings, logger: ctx.logger, vectorIndex });
114
- service.setEmbedder(embedder);
115
- // legacy OpenAI embedder is immediately usable
116
- applyHumanEdits();
117
- } else {
118
- try {
119
- embedder = createEmbedderByProvider(cfg.embedProvider, {
120
- model: cfg.embedProvider === "ollama" ? cfg.ollamaModel : cfg.localEmbedModel,
121
- dimension: cfg.localEmbedDimension,
122
- device: cfg.localEmbedDevice,
123
- batchSize: cfg.localEmbedBatchSize,
124
- cacheDir: cfg.embedModelCacheDir,
125
- baseUrl: cfg.ollamaBaseUrl,
126
- logger: ctx.logger
127
- });
128
- service.setEmbedder(embedder);
129
- // issue #6: wait for extractor init before applying human edits, so
130
- // scheduled embeddings see a ready embedder.
131
- embedder.init()
132
- .then(() => applyHumanEdits())
133
- .catch((error) => {
134
- ctx.logger?.warn?.(`[dsh-mneme] embedder init failed, search degrades to keyword: ${String(error)}`);
135
- service.setEmbedder(null);
136
- applyHumanEdits();
137
- });
138
- } catch (error) {
139
- ctx.logger?.warn?.(`[dsh-mneme] embedder unavailable, search degrades to keyword: ${String(error)}`);
140
- applyHumanEdits();
141
- }
142
- }
143
-
144
- // Cross-encoder rerank over recall candidates. Best-effort: a failed model
145
- // load only disables reranking, never search itself. Explicit opt-in only
146
- // (rerankEnabled defaults to false): constructing LocalReranker is what pulls
147
- // in onnxruntime, so the default config never loads it (item ⑥).
148
- if (cfg.rerankEnabled && cfg.rerankProvider === "local") {
149
- try {
150
- reranker = new LocalReranker({
151
- model: cfg.rerankModel,
152
- batchSize: cfg.rerankBatchSize,
153
- maxCandidates: cfg.rerankMaxCandidates,
154
- scoreThreshold: cfg.rerankScoreThreshold,
155
- device: cfg.localEmbedDevice,
156
- cacheDir: cfg.embedModelCacheDir,
157
- logger: ctx.logger
158
- });
159
- service.setReranker(reranker);
160
- reranker.init().catch((error) => {
161
- ctx.logger?.warn?.(`[dsh-mneme] reranker init failed, rerank disabled: ${String(error)}`);
162
- service.setReranker(null);
163
- });
164
- } catch (error) {
165
- ctx.logger?.warn?.(`[dsh-mneme] reranker unavailable, rerank disabled: ${String(error)}`);
166
- }
167
- }
168
-
169
- // Bug2: lazy auto-backfill of missing embeddings on boot. When the vector API
170
- // is configured and rows still lack an embedding (e.g. written before vector
171
- // search was enabled) AND the vector_meta fingerprint is absent or stale, the
172
- // index is rebuilt in the background after a short delay. Gated on
173
- // cfg.autoReindexOnBoot; rate-limited in small batches so a large backlog
174
- // never floods the provider. Failures degrade silently — search stays keyword.
175
- function scheduleAutoReindex() {
176
- if (cfg.autoReindexOnBoot === false) return;
177
- const attempt = (tries) => {
178
- try {
179
- if (!embedder || typeof embedder.embedSingle !== "function") return;
180
- if ("ready" in embedder && embedder.ready !== true) {
181
- // Local/ollama embedders init asynchronously; give them a moment
182
- // before giving up on this boot (next boot retries).
183
- if (tries > 0) setTimeout(() => attempt(tries - 1), 2000);
184
- return;
185
- }
186
- if (!store.needsEmbedding(1).length) return; // nothing to backfill
187
- // Model fingerprint gate: vectors already produced by the same model
188
- // mean there is no drift and no rebuild needed.
189
- const current = embedder.modelHash;
190
- if (current && vectorIndex.modelHash?.() === current) return;
191
- const BATCH = 10;
192
- const MAX_TOTAL = 500; // bound boot-time work
193
- (async () => {
194
- let indexed = 0;
195
- for (let done = 0; done < MAX_TOTAL;) {
196
- const rows = store.needsEmbedding(BATCH);
197
- if (!rows.length) break;
198
- for (const row of rows) {
199
- try {
200
- const text = [row.title, row.content].filter(Boolean).join("\n");
201
- const vector = await embedder.embedSingle(text);
202
- if (vector?.length) {
203
- store.setEmbedding(row.id, vector);
204
- indexed++;
205
- }
206
- } catch { /* skip the bad row */ }
207
- }
208
- done += rows.length;
209
- // Rate limit: space out batches so the provider is not hammered.
210
- if (store.needsEmbedding(1).length) await new Promise((r) => setTimeout(r, 200));
211
- }
212
- if (indexed > 0 && current) vectorIndex.markModel?.(current, embedder.dimension);
213
- ctx.logger?.info?.(`[dsh-mneme] auto-reindex backfilled ${indexed} embeddings on boot`);
214
- })().catch((error) => {
215
- ctx.logger?.warn?.(`[dsh-mneme] auto-reindex failed: ${String(error)}`);
216
- });
217
- } catch (error) {
218
- ctx.logger?.warn?.(`[dsh-mneme] auto-reindex failed: ${String(error)}`);
219
- }
220
- };
221
- setTimeout(() => attempt(5), 5000);
222
- }
223
- scheduleAutoReindex();
224
-
225
- // Custom commands: register persisted commands into the DSH command registry
226
- // on boot; add/remove re-register live through the API.
227
- let commands = null;
228
- if (ctx.commands) {
229
- commands = createCommandManager({ ctx, settings, logger: ctx.logger });
230
- commands.sync();
231
- }
232
-
233
- // Dream scheduler: automatic consolidation + summary runs, triggered by
234
- // store growth. Writes through the service fire the dream hook, which asks
235
- // the scheduler to (re)schedule a run once absolute and since-last-run
236
- // thresholds are both exceeded. onRun is deferred through `dream` so the
237
- // closure sees the assigned scheduler; the null guard keeps a run safe even
238
- // if the hook fires before assignment or after dispose.
239
- let dream = null;
240
- if (cfg.autoDream) {
241
- dream = createDreamScheduler({
242
- thresholdCount: cfg.dreamThresholdCount,
243
- thresholdChars: cfg.dreamThresholdChars,
244
- delayMs: cfg.dreamDelayMs,
245
- logger: ctx.logger,
246
- semantic: { embedder, vectorIndex },
247
- onRun: () => (dream ? dream.runDream(ctx, service, cfg) : Promise.resolve({ ok: true, skipped: true }))
248
- });
249
- service.setDreamHook(() => dream.maybeSchedule(service));
250
- }
251
-
252
- // Sleep scheduler (v0.4.0): idle-triggered deep maintenance. Fires when the
253
- // store has been quiet for sleepIdleMinutes and re-arms on every write via
254
- // noteWrite (hooked to the service's write path). Runs go through
255
- // service.enqueue so they serialize with autoDream — the two never overlap.
256
- // Abortable on user activity; audited into dream_runs with run_type='sleep'.
257
- let sleep = null;
258
- if (cfg.sleepModeEnabled) {
259
- sleep = createSleepScheduler({
260
- service,
261
- config: cfg,
262
- logger: ctx.logger,
263
- onRun: (signal) => (sleep ? runSleep(ctx, service, cfg, ctx.logger, { embedder, vectorIndex }, signal) : Promise.resolve({ ok: true, skipped: true }))
264
- });
265
- service.setSleepHook(() => sleep.noteWrite());
266
- }
267
-
268
- // Entity gene extraction (v0.3.0): wire the extractor into the service as a
269
- // hook so saveWithDedupe can fire-and-forget an extraction pass on fresh
270
- // writes. The service never sees ctx.llm — index.js adapts it here into the
271
- // callLLM(messages, options) => Promise<string> contract the extractor
272
- // expects, reusing the same ctx.llm.stream consumption pattern as dream.js.
273
- // Explicit opt-in only (entityExtractionEnabled defaults to false); any LLM
274
- // failure degrades inside the extractor to { ok:false }, never a write error.
275
- if (cfg.entityExtractionEnabled && ctx.llm) {
276
- const streamEntityText = async (messages, options = {}) => {
277
- let route = null;
278
- if (options.model) {
279
- route = { model: options.model };
280
- } else {
281
- try {
282
- const sel = ctx.agentDefaultModel?.currentSelection?.();
283
- if (sel?.provider && sel?.model) route = sel;
284
- } catch { /* fall through to no route */ }
285
- }
286
- let text = "";
287
- for await (const chunk of ctx.llm.stream({
288
- ...(route ?? {}),
289
- purpose: "entity-extract",
290
- maxTokens: 4096,
291
- messages
292
- })) {
293
- if (chunk.type === "text-delta" && typeof chunk.text === "string") text += chunk.text;
294
- if (chunk.type === "finish" && (chunk.reason?.kind === "error" || chunk.reason?.kind === "aborted")) return undefined;
295
- }
296
- return text;
297
- };
298
- service.setEntityExtractor((memory) =>
299
- extractEntities(memory, { store, config: cfg, callLLM: streamEntityText, logger: ctx.logger })
300
- .catch((err) => {
301
- ctx.logger?.warn?.(`[dsh-mneme] entity extraction failed: ${String(err)}`);
302
- return { ok: false, error: String(err) };
303
- })
304
- );
305
- }
306
-
307
- const disposers = [];
308
-
309
- ctx.inject(["systemPrompt"], (promptCtx) => {
310
- if (cfg.autoInject) disposers.push(createInjector(promptCtx, service, settings, cfg));
311
- });
312
-
313
- ctx.inject(["tools"], (toolsCtx) => {
314
- disposers.push(createTools(toolsCtx, service, cfg, embedder));
315
- });
316
-
317
- const summarizer = createSummarizer(ctx, service, cfg);
318
- disposers.push(summarizer.dispose);
319
-
320
- // Session lifecycle (v0.6.0): when a session leaves the store and the toggle
321
- // is enabled, mark every memory born in it as session-disposed (hidden from
322
- // injection/search/dream but never destroyed — recoverable via
323
- // restoreBySession). Default off, so a disposed session leaves its memories
324
- // active (legacy behavior). Every path is guarded: a failure inside the
325
- // callback must never propagate into DSH's session teardown (that would crash
326
- // the plugin on the very delete action it serves).
327
- if (cfg.sessionLifecycleEnabled) {
328
- disposers.push(ctx.on("session/disposed", (session) => {
329
- const sessionId = session?.id;
330
- if (!sessionId) return;
331
- try {
332
- const { disposed } = service.disposeBySession(sessionId);
333
- ctx.logger?.info?.(`[dsh-mneme] session disposed, hid ${disposed} memory(s) for ${sessionId}`);
334
- } catch (error) {
335
- ctx.logger?.warn?.(`[dsh-mneme] session dispose failed for ${sessionId}: ${String(error)}`);
336
- }
337
- }));
338
- }
339
-
340
- if (ctx.webServer) {
341
- const api = createApi(ctx, service, settings, commands ?? {
342
- add: () => { throw new Error("commands unavailable"); },
343
- remove: () => false,
344
- list: () => []
345
- }, embedder, { vectorIndex, reranker }, cfg.apiToken);
346
- disposers.push(api.dispose);
347
- }
348
-
349
- // Async disposer: cordis awaits the returned promise on unload (runDisposable),
350
- // so an in-flight dream run is allowed to finish before the SQLite store is
351
- // closed — dream.dispose() resolves only after its current run settles.
352
- return async () => {
353
- for (const dispose of disposers) {
354
- if (typeof dispose === "function") dispose();
355
- }
356
- commands?.dispose();
357
- if (dream) await dream.dispose();
358
- if (sleep) sleep.dispose();
359
- store.close();
360
- };
361
- };
1
+ import { createStore } from "./store.js";
2
+ import { createMirror, TYPE_FILE } from "./mirror.js";
3
+ import { createService } from "./service.js";
4
+ import { createTools } from "./tools.js";
5
+ import { createInjector } from "./inject.js";
6
+ import { createSummarizer } from "./summarize.js";
7
+ import { createDreamScheduler } from "./dream.js";
8
+ import { createSleepScheduler, runSleep } from "./dream/sleep.js";
9
+ import { createApi } from "./api.js";
10
+ import { createSettings } from "./settings.js";
11
+ import { createCommandManager } from "./commands.js";
12
+ import { createEmbedder } from "./embedding.js";
13
+ import { createEmbedderByProvider } from "./local-embedder.js";
14
+ import { LocalReranker } from "./reranker.js";
15
+ import { createVectorIndex } from "./vector-index.js";
16
+ import { Config } from "./config.js";
17
+ import { extractEntities } from "./entities/extractor.js";
18
+ import { mkdirSync } from "node:fs";
19
+ import { join } from "node:path";
20
+ import { homedir } from "node:os";
21
+
22
+ export const name = "dsh-mneme";
23
+ export const inject = ["tools", "systemPrompt", "webServer", "llm", "agentDefaultModel", "commands"];
24
+ export { Config };
25
+
26
+ // Arrow (not function declaration): cordis 4 treats any apply with a
27
+ // prototype as a class constructor (`new apply(...)`) and discards its return
28
+ // value, so a `function apply` disposer would never run on unload. An arrow
29
+ // has no prototype, is called normally, and its returned disposer is collected
30
+ // and run by the fiber on unload.
31
+ export const apply = (ctx, config) => {
32
+ const cfg = Config(config);
33
+
34
+ // Resolve memoryDir: expand leading "~"
35
+ const memoryDir = cfg.memoryDir.startsWith("~")
36
+ ? join(homedir(), cfg.memoryDir.slice(1))
37
+ : cfg.memoryDir;
38
+ mkdirSync(memoryDir, { recursive: true });
39
+
40
+ const store = createStore(join(memoryDir, "memory.db"));
41
+ // Prune reflection failure rows older than 90 days on boot (best-effort, so
42
+ // the failure table never grows unbounded).
43
+ try {
44
+ store.deleteOldFailures(new Date(Date.now() - 90 * 86400000).toISOString());
45
+ } catch { /* non-fatal */ }
46
+ // Bug8: enforce llm_audit_logs retention on boot (config.llmAudit.retentionDays,
47
+ // default 90). Best-effort like the failure prune — the audit trail is
48
+ // bookkeeping and a failed purge must never block plugin boot.
49
+ try {
50
+ if (cfg.llmAudit?.enabled !== false) {
51
+ const retentionMs = Number.isInteger(cfg.llmAudit?.retentionDays) ? cfg.llmAudit.retentionDays : 90;
52
+ store.deleteOldLlmAudits(new Date(Date.now() - retentionMs * 86400000).toISOString());
53
+ }
54
+ } catch { /* non-fatal */ }
55
+ const mirror = createMirror(memoryDir);
56
+ const service = createService({ store, mirror, config: cfg, logger: ctx.logger });
57
+
58
+ // F-NEW-03: if the mirror sync failed last run (persisted dirty state), retry
59
+ // a safe re-render at boot so a stale mirror converges without needing a
60
+ // business write. Bounded: single attempt; on failure dirty stays for the
61
+ // next boot. Never throws.
62
+ service.recoverMirror();
63
+
64
+ // Recall-layer receipt: when searchMemories runs with recordRecall=true, the
65
+ // retrieval scene (query/mode/topK/threshold + candidates) is persisted to
66
+ // recall_runs for audit/replay — the sibling of the dream_runs judgment trail.
67
+ // Best-effort: a failed recall write must never break the search.
68
+ service.setRecallRecorder((recall) => {
69
+ try {
70
+ store.saveRecallRun({
71
+ query: recall.query,
72
+ mode: recall.mode,
73
+ topK: recall.topK,
74
+ threshold: recall.threshold ?? null,
75
+ candidates: recall.candidates ?? [],
76
+ created_at: recall.createdAt
77
+ });
78
+ } catch { /* non-fatal: recall recording is bookkeeping */ }
79
+ });
80
+
81
+ // User-configurable settings (profile, rules) and custom commands share the
82
+ // same SQLite file but live in dedicated tables, isolated from memories.
83
+ const settings = createSettings(store.db);
84
+
85
+ // Semantic pipeline: a local/ollama embedder when configured, otherwise the
86
+ // legacy OpenAI-compatible embedder (settings-driven). The vector index wraps
87
+ // the store's embedding column and tracks the active model fingerprint. A
88
+ // slow embedder init (model download) never blocks plugin boot — failures
89
+ // degrade to keyword search.
90
+ const vectorIndex = createVectorIndex({ store, logger: ctx.logger });
91
+ service.setVectorIndex(vectorIndex);
92
+
93
+ // Human edits in mirror files win on every sync; merge them back first.
94
+ // TYPE_FILE maps each memory type to its mirror filename. Read every type's
95
+ // edits up front: mergeHumanEdits re-renders ALL mirror files on success, so
96
+ // a per-type read-then-merge loop would overwrite edits in files not yet read
97
+ // (e.g. preferences.md merging would clobber unsynced projects.md edits).
98
+ const humanEdits = new Map();
99
+ for (const type of Object.keys(TYPE_FILE)) {
100
+ humanEdits.set(type, mirror.readHumanEdits(type));
101
+ }
102
+ const applyHumanEdits = () => {
103
+ for (const [type, edits] of humanEdits) {
104
+ if (edits.length) service.mergeHumanEdits(type, edits);
105
+ }
106
+ };
107
+
108
+ let embedder = null;
109
+ let reranker = null;
110
+ if (cfg.embedProvider === "openai") {
111
+ // vectorIndex is passed so the legacy OpenAI embedder records the producing
112
+ // model fingerprint after each successful embed (Bug3).
113
+ embedder = createEmbedder({ store, settings, logger: ctx.logger, vectorIndex });
114
+ service.setEmbedder(embedder);
115
+ // legacy OpenAI embedder is immediately usable
116
+ applyHumanEdits();
117
+ } else {
118
+ try {
119
+ embedder = createEmbedderByProvider(cfg.embedProvider, {
120
+ model: cfg.embedProvider === "ollama" ? cfg.ollamaModel : cfg.localEmbedModel,
121
+ dimension: cfg.localEmbedDimension,
122
+ device: cfg.localEmbedDevice,
123
+ batchSize: cfg.localEmbedBatchSize,
124
+ cacheDir: cfg.embedModelCacheDir,
125
+ baseUrl: cfg.ollamaBaseUrl,
126
+ logger: ctx.logger
127
+ });
128
+ service.setEmbedder(embedder);
129
+ // issue #6: wait for extractor init before applying human edits, so
130
+ // scheduled embeddings see a ready embedder.
131
+ embedder.init()
132
+ .then(() => applyHumanEdits())
133
+ .catch((error) => {
134
+ ctx.logger?.warn?.(`[dsh-mneme] embedder init failed, search degrades to keyword: ${String(error)}`);
135
+ service.setEmbedder(null);
136
+ applyHumanEdits();
137
+ });
138
+ } catch (error) {
139
+ ctx.logger?.warn?.(`[dsh-mneme] embedder unavailable, search degrades to keyword: ${String(error)}`);
140
+ applyHumanEdits();
141
+ }
142
+ }
143
+
144
+ // Cross-encoder rerank over recall candidates. Best-effort: a failed model
145
+ // load only disables reranking, never search itself. Explicit opt-in only
146
+ // (rerankEnabled defaults to false): constructing LocalReranker is what pulls
147
+ // in onnxruntime, so the default config never loads it (item ⑥).
148
+ if (cfg.rerankEnabled && cfg.rerankProvider === "local") {
149
+ try {
150
+ reranker = new LocalReranker({
151
+ model: cfg.rerankModel,
152
+ batchSize: cfg.rerankBatchSize,
153
+ maxCandidates: cfg.rerankMaxCandidates,
154
+ scoreThreshold: cfg.rerankScoreThreshold,
155
+ device: cfg.localEmbedDevice,
156
+ cacheDir: cfg.embedModelCacheDir,
157
+ logger: ctx.logger
158
+ });
159
+ service.setReranker(reranker);
160
+ reranker.init().catch((error) => {
161
+ ctx.logger?.warn?.(`[dsh-mneme] reranker init failed, rerank disabled: ${String(error)}`);
162
+ service.setReranker(null);
163
+ });
164
+ } catch (error) {
165
+ ctx.logger?.warn?.(`[dsh-mneme] reranker unavailable, rerank disabled: ${String(error)}`);
166
+ }
167
+ }
168
+
169
+ // Bug2: lazy auto-backfill of missing embeddings on boot. When the vector API
170
+ // is configured and rows still lack an embedding (e.g. written before vector
171
+ // search was enabled) AND the vector_meta fingerprint is absent or stale, the
172
+ // index is rebuilt in the background after a short delay. Gated on
173
+ // cfg.autoReindexOnBoot; rate-limited in small batches so a large backlog
174
+ // never floods the provider. Failures degrade silently — search stays keyword.
175
+ function scheduleAutoReindex() {
176
+ if (cfg.autoReindexOnBoot === false) return;
177
+ const attempt = (tries) => {
178
+ try {
179
+ if (!embedder || typeof embedder.embedSingle !== "function") return;
180
+ if ("ready" in embedder && embedder.ready !== true) {
181
+ // Local/ollama embedders init asynchronously; give them a moment
182
+ // before giving up on this boot (next boot retries).
183
+ if (tries > 0) setTimeout(() => attempt(tries - 1), 2000);
184
+ return;
185
+ }
186
+ if (!store.needsEmbedding(1).length) return; // nothing to backfill
187
+ // Model fingerprint gate: vectors already produced by the same model
188
+ // mean there is no drift and no rebuild needed.
189
+ const current = embedder.modelHash;
190
+ if (current && vectorIndex.modelHash?.() === current) return;
191
+ const BATCH = 10;
192
+ const MAX_TOTAL = 500; // bound boot-time work
193
+ (async () => {
194
+ let indexed = 0;
195
+ for (let done = 0; done < MAX_TOTAL;) {
196
+ const rows = store.needsEmbedding(BATCH);
197
+ if (!rows.length) break;
198
+ for (const row of rows) {
199
+ try {
200
+ const text = [row.title, row.content].filter(Boolean).join("\n");
201
+ const vector = await embedder.embedSingle(text);
202
+ if (vector?.length) {
203
+ store.setEmbedding(row.id, vector);
204
+ indexed++;
205
+ }
206
+ } catch { /* skip the bad row */ }
207
+ }
208
+ done += rows.length;
209
+ // Rate limit: space out batches so the provider is not hammered.
210
+ if (store.needsEmbedding(1).length) await new Promise((r) => setTimeout(r, 200));
211
+ }
212
+ if (indexed > 0 && current) vectorIndex.markModel?.(current, embedder.dimension);
213
+ ctx.logger?.info?.(`[dsh-mneme] auto-reindex backfilled ${indexed} embeddings on boot`);
214
+ })().catch((error) => {
215
+ ctx.logger?.warn?.(`[dsh-mneme] auto-reindex failed: ${String(error)}`);
216
+ });
217
+ } catch (error) {
218
+ ctx.logger?.warn?.(`[dsh-mneme] auto-reindex failed: ${String(error)}`);
219
+ }
220
+ };
221
+ setTimeout(() => attempt(5), 5000);
222
+ }
223
+ scheduleAutoReindex();
224
+
225
+ // Custom commands: register persisted commands into the DSH command registry
226
+ // on boot; add/remove re-register live through the API.
227
+ let commands = null;
228
+ if (ctx.commands) {
229
+ commands = createCommandManager({ ctx, settings, logger: ctx.logger });
230
+ commands.sync();
231
+ }
232
+
233
+ // Dream scheduler: automatic consolidation + summary runs, triggered by
234
+ // store growth. Writes through the service fire the dream hook, which asks
235
+ // the scheduler to (re)schedule a run once absolute and since-last-run
236
+ // thresholds are both exceeded. onRun is deferred through `dream` so the
237
+ // closure sees the assigned scheduler; the null guard keeps a run safe even
238
+ // if the hook fires before assignment or after dispose.
239
+ let dream = null;
240
+ if (cfg.autoDream) {
241
+ dream = createDreamScheduler({
242
+ thresholdCount: cfg.dreamThresholdCount,
243
+ thresholdChars: cfg.dreamThresholdChars,
244
+ delayMs: cfg.dreamDelayMs,
245
+ logger: ctx.logger,
246
+ semantic: { embedder, vectorIndex },
247
+ onRun: () => (dream ? dream.runDream(ctx, service, cfg) : Promise.resolve({ ok: true, skipped: true }))
248
+ });
249
+ service.setDreamHook(() => dream.maybeSchedule(service));
250
+ }
251
+
252
+ // Sleep scheduler (v0.4.0): idle-triggered deep maintenance. Fires when the
253
+ // store has been quiet for sleepIdleMinutes and re-arms on every write via
254
+ // noteWrite (hooked to the service's write path). Runs go through
255
+ // service.enqueue so they serialize with autoDream — the two never overlap.
256
+ // Abortable on user activity; audited into dream_runs with run_type='sleep'.
257
+ let sleep = null;
258
+ if (cfg.sleepModeEnabled) {
259
+ sleep = createSleepScheduler({
260
+ service,
261
+ config: cfg,
262
+ logger: ctx.logger,
263
+ onRun: (signal) => (sleep ? runSleep(ctx, service, cfg, ctx.logger, { embedder, vectorIndex }, signal) : Promise.resolve({ ok: true, skipped: true }))
264
+ });
265
+ service.setSleepHook(() => sleep.noteWrite());
266
+ }
267
+
268
+ // Entity gene extraction (v0.3.0): wire the extractor into the service as a
269
+ // hook so saveWithDedupe can fire-and-forget an extraction pass on fresh
270
+ // writes. The service never sees ctx.llm — index.js adapts it here into the
271
+ // callLLM(messages, options) => Promise<string> contract the extractor
272
+ // expects, reusing the same ctx.llm.stream consumption pattern as dream.js.
273
+ // Explicit opt-in only (entityExtractionEnabled defaults to false); any LLM
274
+ // failure degrades inside the extractor to { ok:false }, never a write error.
275
+ if (cfg.entityExtractionEnabled && ctx.llm) {
276
+ const streamEntityText = async (messages, options = {}) => {
277
+ let route = null;
278
+ if (options.model) {
279
+ route = { model: options.model };
280
+ } else {
281
+ try {
282
+ const sel = ctx.agentDefaultModel?.currentSelection?.();
283
+ if (sel?.provider && sel?.model) route = sel;
284
+ } catch { /* fall through to no route */ }
285
+ }
286
+ let text = "";
287
+ for await (const chunk of ctx.llm.stream({
288
+ ...(route ?? {}),
289
+ purpose: "entity-extract",
290
+ maxTokens: 4096,
291
+ messages
292
+ })) {
293
+ if (chunk.type === "text-delta" && typeof chunk.text === "string") text += chunk.text;
294
+ if (chunk.type === "finish" && (chunk.reason?.kind === "error" || chunk.reason?.kind === "aborted")) return undefined;
295
+ }
296
+ return text;
297
+ };
298
+ service.setEntityExtractor((memory) =>
299
+ extractEntities(memory, { store, config: cfg, callLLM: streamEntityText, logger: ctx.logger })
300
+ .catch((err) => {
301
+ ctx.logger?.warn?.(`[dsh-mneme] entity extraction failed: ${String(err)}`);
302
+ return { ok: false, error: String(err) };
303
+ })
304
+ );
305
+ }
306
+
307
+ const disposers = [];
308
+
309
+ ctx.inject(["systemPrompt"], (promptCtx) => {
310
+ if (cfg.autoInject) disposers.push(createInjector(promptCtx, service, settings, cfg));
311
+ });
312
+
313
+ ctx.inject(["tools"], (toolsCtx) => {
314
+ disposers.push(createTools(toolsCtx, service, cfg, embedder));
315
+ });
316
+
317
+ const summarizer = createSummarizer(ctx, service, cfg);
318
+ disposers.push(summarizer.dispose);
319
+
320
+ // Session lifecycle (v0.6.0): when a session leaves the store and the toggle
321
+ // is enabled, mark every memory born in it as session-disposed (hidden from
322
+ // injection/search/dream but never destroyed — recoverable via
323
+ // restoreBySession). Default off, so a disposed session leaves its memories
324
+ // active (legacy behavior). Every path is guarded: a failure inside the
325
+ // callback must never propagate into DSH's session teardown (that would crash
326
+ // the plugin on the very delete action it serves).
327
+ if (cfg.sessionLifecycleEnabled) {
328
+ disposers.push(ctx.on("session/disposed", (session) => {
329
+ const sessionId = session?.id;
330
+ if (!sessionId) return;
331
+ try {
332
+ const { disposed } = service.disposeBySession(sessionId);
333
+ ctx.logger?.info?.(`[dsh-mneme] session disposed, hid ${disposed} memory(s) for ${sessionId}`);
334
+ } catch (error) {
335
+ ctx.logger?.warn?.(`[dsh-mneme] session dispose failed for ${sessionId}: ${String(error)}`);
336
+ }
337
+ }));
338
+ }
339
+
340
+ if (ctx.webServer) {
341
+ const api = createApi(ctx, service, settings, commands ?? {
342
+ add: () => { throw new Error("commands unavailable"); },
343
+ remove: () => false,
344
+ list: () => []
345
+ }, embedder, { vectorIndex, reranker }, cfg.apiToken);
346
+ disposers.push(api.dispose);
347
+ }
348
+
349
+ // Async disposer: cordis awaits the returned promise on unload (runDisposable),
350
+ // so an in-flight dream run is allowed to finish before the SQLite store is
351
+ // closed — dream.dispose() resolves only after its current run settles.
352
+ return async () => {
353
+ for (const dispose of disposers) {
354
+ if (typeof dispose === "function") dispose();
355
+ }
356
+ commands?.dispose();
357
+ if (dream) await dream.dispose();
358
+ if (sleep) sleep.dispose();
359
+ store.close();
360
+ };
361
+ };