@modusensus/dsh-mneme 0.6.7 → 0.6.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (117) hide show
  1. package/LICENSE +21 -21
  2. package/README.md +463 -463
  3. package/cordis.patch.yml +15 -15
  4. package/lib/api.js +783 -783
  5. package/lib/client.js +1757 -1757
  6. package/lib/commands.js +64 -64
  7. package/lib/config.js +288 -288
  8. package/lib/dream/clustering.js +118 -118
  9. package/lib/dream/decisions.js +439 -439
  10. package/lib/dream/sleep.js +561 -554
  11. package/lib/dream/tag-extractor.js +156 -156
  12. package/lib/dream.js +935 -929
  13. package/lib/embedding.js +154 -154
  14. package/lib/entities/extractor.js +242 -242
  15. package/lib/hot-memory.js +53 -53
  16. package/lib/index.js +361 -361
  17. package/lib/inject.js +208 -208
  18. package/lib/local-embedder.js +282 -282
  19. package/lib/mirror.js +170 -170
  20. package/lib/parser/tag.js +59 -59
  21. package/lib/parser/wiki-link.js +38 -38
  22. package/lib/quality-filter.js +123 -123
  23. package/lib/reranker.js +218 -218
  24. package/lib/search/adaptive.js +22 -22
  25. package/lib/search/bm25.js +96 -96
  26. package/lib/search/tag-boost.js +61 -61
  27. package/lib/service.js +1726 -1726
  28. package/lib/settings.js +172 -172
  29. package/lib/store.js +2238 -2238
  30. package/lib/summarize.js +236 -236
  31. package/lib/tools.js +290 -290
  32. package/lib/vector-index.js +116 -116
  33. package/package.json +80 -80
  34. package/scripts/benchmark-embed.js +201 -201
  35. package/scripts/benchmark-recall.js +133 -133
  36. package/scripts/benchmark-rerank.js +166 -166
  37. package/scripts/e2e-dsh.js +218 -218
  38. package/scripts/stress-dsh.js +255 -255
  39. package/scripts/sync-lib.js +52 -52
  40. package/src/api.js +783 -783
  41. package/src/commands.js +64 -64
  42. package/src/config.js +288 -288
  43. package/src/dream/clustering.js +118 -118
  44. package/src/dream/decisions.js +439 -439
  45. package/src/dream/sleep.js +561 -554
  46. package/src/dream/tag-extractor.js +156 -156
  47. package/src/dream.js +935 -929
  48. package/src/embedding.js +154 -154
  49. package/src/entities/extractor.js +242 -242
  50. package/src/hot-memory.js +53 -53
  51. package/src/index.js +361 -361
  52. package/src/inject.js +208 -208
  53. package/src/local-embedder.js +282 -282
  54. package/src/mirror.js +170 -170
  55. package/src/parser/tag.js +59 -59
  56. package/src/parser/wiki-link.js +38 -38
  57. package/src/quality-filter.js +123 -123
  58. package/src/reranker.js +218 -218
  59. package/src/search/adaptive.js +22 -22
  60. package/src/search/bm25.js +96 -96
  61. package/src/search/tag-boost.js +61 -61
  62. package/src/service.js +1726 -1726
  63. package/src/settings.js +172 -172
  64. package/src/store.js +2238 -2238
  65. package/src/summarize.js +236 -236
  66. package/src/tools.js +290 -290
  67. package/src/vector-index.js +116 -116
  68. package/test/api.test.js +594 -594
  69. package/test/audit.test.js +448 -448
  70. package/test/benchmark.test.js +35 -35
  71. package/test/boundary-v0625.test.js +82 -82
  72. package/test/client.test.js +368 -368
  73. package/test/clustering.test.js +100 -100
  74. package/test/commands.test.js +69 -69
  75. package/test/config.test.js +50 -50
  76. package/test/conflict-freeze.test.js +290 -290
  77. package/test/directory.test.js +134 -134
  78. package/test/dream.test.js +903 -901
  79. package/test/entities.test.js +522 -522
  80. package/test/epistemic.test.js +298 -298
  81. package/test/fnew-0112.test.js +311 -311
  82. package/test/fnew-03.test.js +422 -422
  83. package/test/graph-api.test.js +175 -175
  84. package/test/helpers/dream-mock.js +82 -82
  85. package/test/hot-memory.test.js +174 -174
  86. package/test/inject.test.js +103 -103
  87. package/test/llm-audit.test.js +279 -279
  88. package/test/local-embedder.test.js +227 -227
  89. package/test/mirror-dirty.test.js +424 -424
  90. package/test/mirror-edit-digest.test.js +187 -187
  91. package/test/mirror-generation.test.js +499 -499
  92. package/test/mirror.test.js +249 -249
  93. package/test/normalize-decisions.test.js +120 -120
  94. package/test/peer-blockers.test.js +190 -190
  95. package/test/policy-epoch.test.js +259 -259
  96. package/test/provenance.test.js +103 -103
  97. package/test/quality-filter.test.js +118 -118
  98. package/test/reasoning-effort.test.js +199 -199
  99. package/test/recall-evals.test.js +235 -235
  100. package/test/recall-layer.test.js +315 -315
  101. package/test/receipt-chain.test.js +451 -451
  102. package/test/reflection.test.js +226 -226
  103. package/test/reranker.test.js +240 -240
  104. package/test/search-fusion.test.js +90 -90
  105. package/test/semantic.test.js +124 -124
  106. package/test/service-search.test.js +199 -199
  107. package/test/service.test.js +435 -435
  108. package/test/settings.test.js +118 -118
  109. package/test/sleep.test.js +365 -365
  110. package/test/store.test.js +436 -436
  111. package/test/stress.test.js +209 -209
  112. package/test/summarize.test.js +191 -191
  113. package/test/tag-boost.test.js +125 -125
  114. package/test/tag.test.js +312 -312
  115. package/test/tools.test.js +285 -285
  116. package/test/vector-index.test.js +221 -221
  117. package/test/wiki-link.test.js +332 -332
package/src/index.js CHANGED
@@ -1,361 +1,361 @@
1
- import { createStore } from "./store.js";
2
- import { createMirror, TYPE_FILE } from "./mirror.js";
3
- import { createService } from "./service.js";
4
- import { createTools } from "./tools.js";
5
- import { createInjector } from "./inject.js";
6
- import { createSummarizer } from "./summarize.js";
7
- import { createDreamScheduler } from "./dream.js";
8
- import { createSleepScheduler, runSleep } from "./dream/sleep.js";
9
- import { createApi } from "./api.js";
10
- import { createSettings } from "./settings.js";
11
- import { createCommandManager } from "./commands.js";
12
- import { createEmbedder } from "./embedding.js";
13
- import { createEmbedderByProvider } from "./local-embedder.js";
14
- import { LocalReranker } from "./reranker.js";
15
- import { createVectorIndex } from "./vector-index.js";
16
- import { Config } from "./config.js";
17
- import { extractEntities } from "./entities/extractor.js";
18
- import { mkdirSync } from "node:fs";
19
- import { join } from "node:path";
20
- import { homedir } from "node:os";
21
-
22
- export const name = "dsh-mneme";
23
- export const inject = ["tools", "systemPrompt", "webServer", "llm", "agentDefaultModel", "commands"];
24
- export { Config };
25
-
26
- // Arrow (not function declaration): cordis 4 treats any apply with a
27
- // prototype as a class constructor (`new apply(...)`) and discards its return
28
- // value, so a `function apply` disposer would never run on unload. An arrow
29
- // has no prototype, is called normally, and its returned disposer is collected
30
- // and run by the fiber on unload.
31
- export const apply = (ctx, config) => {
32
- const cfg = Config(config);
33
-
34
- // Resolve memoryDir: expand leading "~"
35
- const memoryDir = cfg.memoryDir.startsWith("~")
36
- ? join(homedir(), cfg.memoryDir.slice(1))
37
- : cfg.memoryDir;
38
- mkdirSync(memoryDir, { recursive: true });
39
-
40
- const store = createStore(join(memoryDir, "memory.db"));
41
- // Prune reflection failure rows older than 90 days on boot (best-effort, so
42
- // the failure table never grows unbounded).
43
- try {
44
- store.deleteOldFailures(new Date(Date.now() - 90 * 86400000).toISOString());
45
- } catch { /* non-fatal */ }
46
- // Bug8: enforce llm_audit_logs retention on boot (config.llmAudit.retentionDays,
47
- // default 90). Best-effort like the failure prune — the audit trail is
48
- // bookkeeping and a failed purge must never block plugin boot.
49
- try {
50
- if (cfg.llmAudit?.enabled !== false) {
51
- const retentionMs = Number.isInteger(cfg.llmAudit?.retentionDays) ? cfg.llmAudit.retentionDays : 90;
52
- store.deleteOldLlmAudits(new Date(Date.now() - retentionMs * 86400000).toISOString());
53
- }
54
- } catch { /* non-fatal */ }
55
- const mirror = createMirror(memoryDir);
56
- const service = createService({ store, mirror, config: cfg, logger: ctx.logger });
57
-
58
- // F-NEW-03: if the mirror sync failed last run (persisted dirty state), retry
59
- // a safe re-render at boot so a stale mirror converges without needing a
60
- // business write. Bounded: single attempt; on failure dirty stays for the
61
- // next boot. Never throws.
62
- service.recoverMirror();
63
-
64
- // Recall-layer receipt: when searchMemories runs with recordRecall=true, the
65
- // retrieval scene (query/mode/topK/threshold + candidates) is persisted to
66
- // recall_runs for audit/replay — the sibling of the dream_runs judgment trail.
67
- // Best-effort: a failed recall write must never break the search.
68
- service.setRecallRecorder((recall) => {
69
- try {
70
- store.saveRecallRun({
71
- query: recall.query,
72
- mode: recall.mode,
73
- topK: recall.topK,
74
- threshold: recall.threshold ?? null,
75
- candidates: recall.candidates ?? [],
76
- created_at: recall.createdAt
77
- });
78
- } catch { /* non-fatal: recall recording is bookkeeping */ }
79
- });
80
-
81
- // User-configurable settings (profile, rules) and custom commands share the
82
- // same SQLite file but live in dedicated tables, isolated from memories.
83
- const settings = createSettings(store.db);
84
-
85
- // Semantic pipeline: a local/ollama embedder when configured, otherwise the
86
- // legacy OpenAI-compatible embedder (settings-driven). The vector index wraps
87
- // the store's embedding column and tracks the active model fingerprint. A
88
- // slow embedder init (model download) never blocks plugin boot — failures
89
- // degrade to keyword search.
90
- const vectorIndex = createVectorIndex({ store, logger: ctx.logger });
91
- service.setVectorIndex(vectorIndex);
92
-
93
- // Human edits in mirror files win on every sync; merge them back first.
94
- // TYPE_FILE maps each memory type to its mirror filename. Read every type's
95
- // edits up front: mergeHumanEdits re-renders ALL mirror files on success, so
96
- // a per-type read-then-merge loop would overwrite edits in files not yet read
97
- // (e.g. preferences.md merging would clobber unsynced projects.md edits).
98
- const humanEdits = new Map();
99
- for (const type of Object.keys(TYPE_FILE)) {
100
- humanEdits.set(type, mirror.readHumanEdits(type));
101
- }
102
- const applyHumanEdits = () => {
103
- for (const [type, edits] of humanEdits) {
104
- if (edits.length) service.mergeHumanEdits(type, edits);
105
- }
106
- };
107
-
108
- let embedder = null;
109
- let reranker = null;
110
- if (cfg.embedProvider === "openai") {
111
- // vectorIndex is passed so the legacy OpenAI embedder records the producing
112
- // model fingerprint after each successful embed (Bug3).
113
- embedder = createEmbedder({ store, settings, logger: ctx.logger, vectorIndex });
114
- service.setEmbedder(embedder);
115
- // legacy OpenAI embedder is immediately usable
116
- applyHumanEdits();
117
- } else {
118
- try {
119
- embedder = createEmbedderByProvider(cfg.embedProvider, {
120
- model: cfg.embedProvider === "ollama" ? cfg.ollamaModel : cfg.localEmbedModel,
121
- dimension: cfg.localEmbedDimension,
122
- device: cfg.localEmbedDevice,
123
- batchSize: cfg.localEmbedBatchSize,
124
- cacheDir: cfg.embedModelCacheDir,
125
- baseUrl: cfg.ollamaBaseUrl,
126
- logger: ctx.logger
127
- });
128
- service.setEmbedder(embedder);
129
- // issue #6: wait for extractor init before applying human edits, so
130
- // scheduled embeddings see a ready embedder.
131
- embedder.init()
132
- .then(() => applyHumanEdits())
133
- .catch((error) => {
134
- ctx.logger?.warn?.(`[dsh-mneme] embedder init failed, search degrades to keyword: ${String(error)}`);
135
- service.setEmbedder(null);
136
- applyHumanEdits();
137
- });
138
- } catch (error) {
139
- ctx.logger?.warn?.(`[dsh-mneme] embedder unavailable, search degrades to keyword: ${String(error)}`);
140
- applyHumanEdits();
141
- }
142
- }
143
-
144
- // Cross-encoder rerank over recall candidates. Best-effort: a failed model
145
- // load only disables reranking, never search itself. Explicit opt-in only
146
- // (rerankEnabled defaults to false): constructing LocalReranker is what pulls
147
- // in onnxruntime, so the default config never loads it (item ⑥).
148
- if (cfg.rerankEnabled && cfg.rerankProvider === "local") {
149
- try {
150
- reranker = new LocalReranker({
151
- model: cfg.rerankModel,
152
- batchSize: cfg.rerankBatchSize,
153
- maxCandidates: cfg.rerankMaxCandidates,
154
- scoreThreshold: cfg.rerankScoreThreshold,
155
- device: cfg.localEmbedDevice,
156
- cacheDir: cfg.embedModelCacheDir,
157
- logger: ctx.logger
158
- });
159
- service.setReranker(reranker);
160
- reranker.init().catch((error) => {
161
- ctx.logger?.warn?.(`[dsh-mneme] reranker init failed, rerank disabled: ${String(error)}`);
162
- service.setReranker(null);
163
- });
164
- } catch (error) {
165
- ctx.logger?.warn?.(`[dsh-mneme] reranker unavailable, rerank disabled: ${String(error)}`);
166
- }
167
- }
168
-
169
- // Bug2: lazy auto-backfill of missing embeddings on boot. When the vector API
170
- // is configured and rows still lack an embedding (e.g. written before vector
171
- // search was enabled) AND the vector_meta fingerprint is absent or stale, the
172
- // index is rebuilt in the background after a short delay. Gated on
173
- // cfg.autoReindexOnBoot; rate-limited in small batches so a large backlog
174
- // never floods the provider. Failures degrade silently — search stays keyword.
175
- function scheduleAutoReindex() {
176
- if (cfg.autoReindexOnBoot === false) return;
177
- const attempt = (tries) => {
178
- try {
179
- if (!embedder || typeof embedder.embedSingle !== "function") return;
180
- if ("ready" in embedder && embedder.ready !== true) {
181
- // Local/ollama embedders init asynchronously; give them a moment
182
- // before giving up on this boot (next boot retries).
183
- if (tries > 0) setTimeout(() => attempt(tries - 1), 2000);
184
- return;
185
- }
186
- if (!store.needsEmbedding(1).length) return; // nothing to backfill
187
- // Model fingerprint gate: vectors already produced by the same model
188
- // mean there is no drift and no rebuild needed.
189
- const current = embedder.modelHash;
190
- if (current && vectorIndex.modelHash?.() === current) return;
191
- const BATCH = 10;
192
- const MAX_TOTAL = 500; // bound boot-time work
193
- (async () => {
194
- let indexed = 0;
195
- for (let done = 0; done < MAX_TOTAL;) {
196
- const rows = store.needsEmbedding(BATCH);
197
- if (!rows.length) break;
198
- for (const row of rows) {
199
- try {
200
- const text = [row.title, row.content].filter(Boolean).join("\n");
201
- const vector = await embedder.embedSingle(text);
202
- if (vector?.length) {
203
- store.setEmbedding(row.id, vector);
204
- indexed++;
205
- }
206
- } catch { /* skip the bad row */ }
207
- }
208
- done += rows.length;
209
- // Rate limit: space out batches so the provider is not hammered.
210
- if (store.needsEmbedding(1).length) await new Promise((r) => setTimeout(r, 200));
211
- }
212
- if (indexed > 0 && current) vectorIndex.markModel?.(current, embedder.dimension);
213
- ctx.logger?.info?.(`[dsh-mneme] auto-reindex backfilled ${indexed} embeddings on boot`);
214
- })().catch((error) => {
215
- ctx.logger?.warn?.(`[dsh-mneme] auto-reindex failed: ${String(error)}`);
216
- });
217
- } catch (error) {
218
- ctx.logger?.warn?.(`[dsh-mneme] auto-reindex failed: ${String(error)}`);
219
- }
220
- };
221
- setTimeout(() => attempt(5), 5000);
222
- }
223
- scheduleAutoReindex();
224
-
225
- // Custom commands: register persisted commands into the DSH command registry
226
- // on boot; add/remove re-register live through the API.
227
- let commands = null;
228
- if (ctx.commands) {
229
- commands = createCommandManager({ ctx, settings, logger: ctx.logger });
230
- commands.sync();
231
- }
232
-
233
- // Dream scheduler: automatic consolidation + summary runs, triggered by
234
- // store growth. Writes through the service fire the dream hook, which asks
235
- // the scheduler to (re)schedule a run once absolute and since-last-run
236
- // thresholds are both exceeded. onRun is deferred through `dream` so the
237
- // closure sees the assigned scheduler; the null guard keeps a run safe even
238
- // if the hook fires before assignment or after dispose.
239
- let dream = null;
240
- if (cfg.autoDream) {
241
- dream = createDreamScheduler({
242
- thresholdCount: cfg.dreamThresholdCount,
243
- thresholdChars: cfg.dreamThresholdChars,
244
- delayMs: cfg.dreamDelayMs,
245
- logger: ctx.logger,
246
- semantic: { embedder, vectorIndex },
247
- onRun: () => (dream ? dream.runDream(ctx, service, cfg) : Promise.resolve({ ok: true, skipped: true }))
248
- });
249
- service.setDreamHook(() => dream.maybeSchedule(service));
250
- }
251
-
252
- // Sleep scheduler (v0.4.0): idle-triggered deep maintenance. Fires when the
253
- // store has been quiet for sleepIdleMinutes and re-arms on every write via
254
- // noteWrite (hooked to the service's write path). Runs go through
255
- // service.enqueue so they serialize with autoDream — the two never overlap.
256
- // Abortable on user activity; audited into dream_runs with run_type='sleep'.
257
- let sleep = null;
258
- if (cfg.sleepModeEnabled) {
259
- sleep = createSleepScheduler({
260
- service,
261
- config: cfg,
262
- logger: ctx.logger,
263
- onRun: (signal) => (sleep ? runSleep(ctx, service, cfg, ctx.logger, { embedder, vectorIndex }, signal) : Promise.resolve({ ok: true, skipped: true }))
264
- });
265
- service.setSleepHook(() => sleep.noteWrite());
266
- }
267
-
268
- // Entity gene extraction (v0.3.0): wire the extractor into the service as a
269
- // hook so saveWithDedupe can fire-and-forget an extraction pass on fresh
270
- // writes. The service never sees ctx.llm — index.js adapts it here into the
271
- // callLLM(messages, options) => Promise<string> contract the extractor
272
- // expects, reusing the same ctx.llm.stream consumption pattern as dream.js.
273
- // Explicit opt-in only (entityExtractionEnabled defaults to false); any LLM
274
- // failure degrades inside the extractor to { ok:false }, never a write error.
275
- if (cfg.entityExtractionEnabled && ctx.llm) {
276
- const streamEntityText = async (messages, options = {}) => {
277
- let route = null;
278
- if (options.model) {
279
- route = { model: options.model };
280
- } else {
281
- try {
282
- const sel = ctx.agentDefaultModel?.currentSelection?.();
283
- if (sel?.provider && sel?.model) route = sel;
284
- } catch { /* fall through to no route */ }
285
- }
286
- let text = "";
287
- for await (const chunk of ctx.llm.stream({
288
- ...(route ?? {}),
289
- purpose: "entity-extract",
290
- maxTokens: 4096,
291
- messages
292
- })) {
293
- if (chunk.type === "text-delta" && typeof chunk.text === "string") text += chunk.text;
294
- if (chunk.type === "finish" && (chunk.reason?.kind === "error" || chunk.reason?.kind === "aborted")) return undefined;
295
- }
296
- return text;
297
- };
298
- service.setEntityExtractor((memory) =>
299
- extractEntities(memory, { store, config: cfg, callLLM: streamEntityText, logger: ctx.logger })
300
- .catch((err) => {
301
- ctx.logger?.warn?.(`[dsh-mneme] entity extraction failed: ${String(err)}`);
302
- return { ok: false, error: String(err) };
303
- })
304
- );
305
- }
306
-
307
- const disposers = [];
308
-
309
- ctx.inject(["systemPrompt"], (promptCtx) => {
310
- if (cfg.autoInject) disposers.push(createInjector(promptCtx, service, settings, cfg));
311
- });
312
-
313
- ctx.inject(["tools"], (toolsCtx) => {
314
- disposers.push(createTools(toolsCtx, service, cfg, embedder));
315
- });
316
-
317
- const summarizer = createSummarizer(ctx, service, cfg);
318
- disposers.push(summarizer.dispose);
319
-
320
- // Session lifecycle (v0.6.0): when a session leaves the store and the toggle
321
- // is enabled, mark every memory born in it as session-disposed (hidden from
322
- // injection/search/dream but never destroyed — recoverable via
323
- // restoreBySession). Default off, so a disposed session leaves its memories
324
- // active (legacy behavior). Every path is guarded: a failure inside the
325
- // callback must never propagate into DSH's session teardown (that would crash
326
- // the plugin on the very delete action it serves).
327
- if (cfg.sessionLifecycleEnabled) {
328
- disposers.push(ctx.on("session/disposed", (session) => {
329
- const sessionId = session?.id;
330
- if (!sessionId) return;
331
- try {
332
- const { disposed } = service.disposeBySession(sessionId);
333
- ctx.logger?.info?.(`[dsh-mneme] session disposed, hid ${disposed} memory(s) for ${sessionId}`);
334
- } catch (error) {
335
- ctx.logger?.warn?.(`[dsh-mneme] session dispose failed for ${sessionId}: ${String(error)}`);
336
- }
337
- }));
338
- }
339
-
340
- if (ctx.webServer) {
341
- const api = createApi(ctx, service, settings, commands ?? {
342
- add: () => { throw new Error("commands unavailable"); },
343
- remove: () => false,
344
- list: () => []
345
- }, embedder, { vectorIndex, reranker }, cfg.apiToken);
346
- disposers.push(api.dispose);
347
- }
348
-
349
- // Async disposer: cordis awaits the returned promise on unload (runDisposable),
350
- // so an in-flight dream run is allowed to finish before the SQLite store is
351
- // closed — dream.dispose() resolves only after its current run settles.
352
- return async () => {
353
- for (const dispose of disposers) {
354
- if (typeof dispose === "function") dispose();
355
- }
356
- commands?.dispose();
357
- if (dream) await dream.dispose();
358
- if (sleep) sleep.dispose();
359
- store.close();
360
- };
361
- };
1
+ import { createStore } from "./store.js";
2
+ import { createMirror, TYPE_FILE } from "./mirror.js";
3
+ import { createService } from "./service.js";
4
+ import { createTools } from "./tools.js";
5
+ import { createInjector } from "./inject.js";
6
+ import { createSummarizer } from "./summarize.js";
7
+ import { createDreamScheduler } from "./dream.js";
8
+ import { createSleepScheduler, runSleep } from "./dream/sleep.js";
9
+ import { createApi } from "./api.js";
10
+ import { createSettings } from "./settings.js";
11
+ import { createCommandManager } from "./commands.js";
12
+ import { createEmbedder } from "./embedding.js";
13
+ import { createEmbedderByProvider } from "./local-embedder.js";
14
+ import { LocalReranker } from "./reranker.js";
15
+ import { createVectorIndex } from "./vector-index.js";
16
+ import { Config } from "./config.js";
17
+ import { extractEntities } from "./entities/extractor.js";
18
+ import { mkdirSync } from "node:fs";
19
+ import { join } from "node:path";
20
+ import { homedir } from "node:os";
21
+
22
+ export const name = "dsh-mneme";
23
+ export const inject = ["tools", "systemPrompt", "webServer", "llm", "agentDefaultModel", "commands"];
24
+ export { Config };
25
+
26
+ // Arrow (not function declaration): cordis 4 treats any apply with a
27
+ // prototype as a class constructor (`new apply(...)`) and discards its return
28
+ // value, so a `function apply` disposer would never run on unload. An arrow
29
+ // has no prototype, is called normally, and its returned disposer is collected
30
+ // and run by the fiber on unload.
31
+ export const apply = (ctx, config) => {
32
+ const cfg = Config(config);
33
+
34
+ // Resolve memoryDir: expand leading "~"
35
+ const memoryDir = cfg.memoryDir.startsWith("~")
36
+ ? join(homedir(), cfg.memoryDir.slice(1))
37
+ : cfg.memoryDir;
38
+ mkdirSync(memoryDir, { recursive: true });
39
+
40
+ const store = createStore(join(memoryDir, "memory.db"));
41
+ // Prune reflection failure rows older than 90 days on boot (best-effort, so
42
+ // the failure table never grows unbounded).
43
+ try {
44
+ store.deleteOldFailures(new Date(Date.now() - 90 * 86400000).toISOString());
45
+ } catch { /* non-fatal */ }
46
+ // Bug8: enforce llm_audit_logs retention on boot (config.llmAudit.retentionDays,
47
+ // default 90). Best-effort like the failure prune — the audit trail is
48
+ // bookkeeping and a failed purge must never block plugin boot.
49
+ try {
50
+ if (cfg.llmAudit?.enabled !== false) {
51
+ const retentionMs = Number.isInteger(cfg.llmAudit?.retentionDays) ? cfg.llmAudit.retentionDays : 90;
52
+ store.deleteOldLlmAudits(new Date(Date.now() - retentionMs * 86400000).toISOString());
53
+ }
54
+ } catch { /* non-fatal */ }
55
+ const mirror = createMirror(memoryDir);
56
+ const service = createService({ store, mirror, config: cfg, logger: ctx.logger });
57
+
58
+ // F-NEW-03: if the mirror sync failed last run (persisted dirty state), retry
59
+ // a safe re-render at boot so a stale mirror converges without needing a
60
+ // business write. Bounded: single attempt; on failure dirty stays for the
61
+ // next boot. Never throws.
62
+ service.recoverMirror();
63
+
64
+ // Recall-layer receipt: when searchMemories runs with recordRecall=true, the
65
+ // retrieval scene (query/mode/topK/threshold + candidates) is persisted to
66
+ // recall_runs for audit/replay — the sibling of the dream_runs judgment trail.
67
+ // Best-effort: a failed recall write must never break the search.
68
+ service.setRecallRecorder((recall) => {
69
+ try {
70
+ store.saveRecallRun({
71
+ query: recall.query,
72
+ mode: recall.mode,
73
+ topK: recall.topK,
74
+ threshold: recall.threshold ?? null,
75
+ candidates: recall.candidates ?? [],
76
+ created_at: recall.createdAt
77
+ });
78
+ } catch { /* non-fatal: recall recording is bookkeeping */ }
79
+ });
80
+
81
+ // User-configurable settings (profile, rules) and custom commands share the
82
+ // same SQLite file but live in dedicated tables, isolated from memories.
83
+ const settings = createSettings(store.db);
84
+
85
+ // Semantic pipeline: a local/ollama embedder when configured, otherwise the
86
+ // legacy OpenAI-compatible embedder (settings-driven). The vector index wraps
87
+ // the store's embedding column and tracks the active model fingerprint. A
88
+ // slow embedder init (model download) never blocks plugin boot — failures
89
+ // degrade to keyword search.
90
+ const vectorIndex = createVectorIndex({ store, logger: ctx.logger });
91
+ service.setVectorIndex(vectorIndex);
92
+
93
+ // Human edits in mirror files win on every sync; merge them back first.
94
+ // TYPE_FILE maps each memory type to its mirror filename. Read every type's
95
+ // edits up front: mergeHumanEdits re-renders ALL mirror files on success, so
96
+ // a per-type read-then-merge loop would overwrite edits in files not yet read
97
+ // (e.g. preferences.md merging would clobber unsynced projects.md edits).
98
+ const humanEdits = new Map();
99
+ for (const type of Object.keys(TYPE_FILE)) {
100
+ humanEdits.set(type, mirror.readHumanEdits(type));
101
+ }
102
+ const applyHumanEdits = () => {
103
+ for (const [type, edits] of humanEdits) {
104
+ if (edits.length) service.mergeHumanEdits(type, edits);
105
+ }
106
+ };
107
+
108
+ let embedder = null;
109
+ let reranker = null;
110
+ if (cfg.embedProvider === "openai") {
111
+ // vectorIndex is passed so the legacy OpenAI embedder records the producing
112
+ // model fingerprint after each successful embed (Bug3).
113
+ embedder = createEmbedder({ store, settings, logger: ctx.logger, vectorIndex });
114
+ service.setEmbedder(embedder);
115
+ // legacy OpenAI embedder is immediately usable
116
+ applyHumanEdits();
117
+ } else {
118
+ try {
119
+ embedder = createEmbedderByProvider(cfg.embedProvider, {
120
+ model: cfg.embedProvider === "ollama" ? cfg.ollamaModel : cfg.localEmbedModel,
121
+ dimension: cfg.localEmbedDimension,
122
+ device: cfg.localEmbedDevice,
123
+ batchSize: cfg.localEmbedBatchSize,
124
+ cacheDir: cfg.embedModelCacheDir,
125
+ baseUrl: cfg.ollamaBaseUrl,
126
+ logger: ctx.logger
127
+ });
128
+ service.setEmbedder(embedder);
129
+ // issue #6: wait for extractor init before applying human edits, so
130
+ // scheduled embeddings see a ready embedder.
131
+ embedder.init()
132
+ .then(() => applyHumanEdits())
133
+ .catch((error) => {
134
+ ctx.logger?.warn?.(`[dsh-mneme] embedder init failed, search degrades to keyword: ${String(error)}`);
135
+ service.setEmbedder(null);
136
+ applyHumanEdits();
137
+ });
138
+ } catch (error) {
139
+ ctx.logger?.warn?.(`[dsh-mneme] embedder unavailable, search degrades to keyword: ${String(error)}`);
140
+ applyHumanEdits();
141
+ }
142
+ }
143
+
144
+ // Cross-encoder rerank over recall candidates. Best-effort: a failed model
145
+ // load only disables reranking, never search itself. Explicit opt-in only
146
+ // (rerankEnabled defaults to false): constructing LocalReranker is what pulls
147
+ // in onnxruntime, so the default config never loads it (item ⑥).
148
+ if (cfg.rerankEnabled && cfg.rerankProvider === "local") {
149
+ try {
150
+ reranker = new LocalReranker({
151
+ model: cfg.rerankModel,
152
+ batchSize: cfg.rerankBatchSize,
153
+ maxCandidates: cfg.rerankMaxCandidates,
154
+ scoreThreshold: cfg.rerankScoreThreshold,
155
+ device: cfg.localEmbedDevice,
156
+ cacheDir: cfg.embedModelCacheDir,
157
+ logger: ctx.logger
158
+ });
159
+ service.setReranker(reranker);
160
+ reranker.init().catch((error) => {
161
+ ctx.logger?.warn?.(`[dsh-mneme] reranker init failed, rerank disabled: ${String(error)}`);
162
+ service.setReranker(null);
163
+ });
164
+ } catch (error) {
165
+ ctx.logger?.warn?.(`[dsh-mneme] reranker unavailable, rerank disabled: ${String(error)}`);
166
+ }
167
+ }
168
+
169
+ // Bug2: lazy auto-backfill of missing embeddings on boot. When the vector API
170
+ // is configured and rows still lack an embedding (e.g. written before vector
171
+ // search was enabled) AND the vector_meta fingerprint is absent or stale, the
172
+ // index is rebuilt in the background after a short delay. Gated on
173
+ // cfg.autoReindexOnBoot; rate-limited in small batches so a large backlog
174
+ // never floods the provider. Failures degrade silently — search stays keyword.
175
+ function scheduleAutoReindex() {
176
+ if (cfg.autoReindexOnBoot === false) return;
177
+ const attempt = (tries) => {
178
+ try {
179
+ if (!embedder || typeof embedder.embedSingle !== "function") return;
180
+ if ("ready" in embedder && embedder.ready !== true) {
181
+ // Local/ollama embedders init asynchronously; give them a moment
182
+ // before giving up on this boot (next boot retries).
183
+ if (tries > 0) setTimeout(() => attempt(tries - 1), 2000);
184
+ return;
185
+ }
186
+ if (!store.needsEmbedding(1).length) return; // nothing to backfill
187
+ // Model fingerprint gate: vectors already produced by the same model
188
+ // mean there is no drift and no rebuild needed.
189
+ const current = embedder.modelHash;
190
+ if (current && vectorIndex.modelHash?.() === current) return;
191
+ const BATCH = 10;
192
+ const MAX_TOTAL = 500; // bound boot-time work
193
+ (async () => {
194
+ let indexed = 0;
195
+ for (let done = 0; done < MAX_TOTAL;) {
196
+ const rows = store.needsEmbedding(BATCH);
197
+ if (!rows.length) break;
198
+ for (const row of rows) {
199
+ try {
200
+ const text = [row.title, row.content].filter(Boolean).join("\n");
201
+ const vector = await embedder.embedSingle(text);
202
+ if (vector?.length) {
203
+ store.setEmbedding(row.id, vector);
204
+ indexed++;
205
+ }
206
+ } catch { /* skip the bad row */ }
207
+ }
208
+ done += rows.length;
209
+ // Rate limit: space out batches so the provider is not hammered.
210
+ if (store.needsEmbedding(1).length) await new Promise((r) => setTimeout(r, 200));
211
+ }
212
+ if (indexed > 0 && current) vectorIndex.markModel?.(current, embedder.dimension);
213
+ ctx.logger?.info?.(`[dsh-mneme] auto-reindex backfilled ${indexed} embeddings on boot`);
214
+ })().catch((error) => {
215
+ ctx.logger?.warn?.(`[dsh-mneme] auto-reindex failed: ${String(error)}`);
216
+ });
217
+ } catch (error) {
218
+ ctx.logger?.warn?.(`[dsh-mneme] auto-reindex failed: ${String(error)}`);
219
+ }
220
+ };
221
+ setTimeout(() => attempt(5), 5000);
222
+ }
223
+ scheduleAutoReindex();
224
+
225
+ // Custom commands: register persisted commands into the DSH command registry
226
+ // on boot; add/remove re-register live through the API.
227
+ let commands = null;
228
+ if (ctx.commands) {
229
+ commands = createCommandManager({ ctx, settings, logger: ctx.logger });
230
+ commands.sync();
231
+ }
232
+
233
+ // Dream scheduler: automatic consolidation + summary runs, triggered by
234
+ // store growth. Writes through the service fire the dream hook, which asks
235
+ // the scheduler to (re)schedule a run once absolute and since-last-run
236
+ // thresholds are both exceeded. onRun is deferred through `dream` so the
237
+ // closure sees the assigned scheduler; the null guard keeps a run safe even
238
+ // if the hook fires before assignment or after dispose.
239
+ let dream = null;
240
+ if (cfg.autoDream) {
241
+ dream = createDreamScheduler({
242
+ thresholdCount: cfg.dreamThresholdCount,
243
+ thresholdChars: cfg.dreamThresholdChars,
244
+ delayMs: cfg.dreamDelayMs,
245
+ logger: ctx.logger,
246
+ semantic: { embedder, vectorIndex },
247
+ onRun: () => (dream ? dream.runDream(ctx, service, cfg) : Promise.resolve({ ok: true, skipped: true }))
248
+ });
249
+ service.setDreamHook(() => dream.maybeSchedule(service));
250
+ }
251
+
252
+ // Sleep scheduler (v0.4.0): idle-triggered deep maintenance. Fires when the
253
+ // store has been quiet for sleepIdleMinutes and re-arms on every write via
254
+ // noteWrite (hooked to the service's write path). Runs go through
255
+ // service.enqueue so they serialize with autoDream — the two never overlap.
256
+ // Abortable on user activity; audited into dream_runs with run_type='sleep'.
257
+ let sleep = null;
258
+ if (cfg.sleepModeEnabled) {
259
+ sleep = createSleepScheduler({
260
+ service,
261
+ config: cfg,
262
+ logger: ctx.logger,
263
+ onRun: (signal) => (sleep ? runSleep(ctx, service, cfg, ctx.logger, { embedder, vectorIndex }, signal) : Promise.resolve({ ok: true, skipped: true }))
264
+ });
265
+ service.setSleepHook(() => sleep.noteWrite());
266
+ }
267
+
268
+ // Entity gene extraction (v0.3.0): wire the extractor into the service as a
269
+ // hook so saveWithDedupe can fire-and-forget an extraction pass on fresh
270
+ // writes. The service never sees ctx.llm — index.js adapts it here into the
271
+ // callLLM(messages, options) => Promise<string> contract the extractor
272
+ // expects, reusing the same ctx.llm.stream consumption pattern as dream.js.
273
+ // Explicit opt-in only (entityExtractionEnabled defaults to false); any LLM
274
+ // failure degrades inside the extractor to { ok:false }, never a write error.
275
+ if (cfg.entityExtractionEnabled && ctx.llm) {
276
+ const streamEntityText = async (messages, options = {}) => {
277
+ let route = null;
278
+ if (options.model) {
279
+ route = { model: options.model };
280
+ } else {
281
+ try {
282
+ const sel = ctx.agentDefaultModel?.currentSelection?.();
283
+ if (sel?.provider && sel?.model) route = sel;
284
+ } catch { /* fall through to no route */ }
285
+ }
286
+ let text = "";
287
+ for await (const chunk of ctx.llm.stream({
288
+ ...(route ?? {}),
289
+ purpose: "entity-extract",
290
+ maxTokens: 4096,
291
+ messages
292
+ })) {
293
+ if (chunk.type === "text-delta" && typeof chunk.text === "string") text += chunk.text;
294
+ if (chunk.type === "finish" && (chunk.reason?.kind === "error" || chunk.reason?.kind === "aborted")) return undefined;
295
+ }
296
+ return text;
297
+ };
298
+ service.setEntityExtractor((memory) =>
299
+ extractEntities(memory, { store, config: cfg, callLLM: streamEntityText, logger: ctx.logger })
300
+ .catch((err) => {
301
+ ctx.logger?.warn?.(`[dsh-mneme] entity extraction failed: ${String(err)}`);
302
+ return { ok: false, error: String(err) };
303
+ })
304
+ );
305
+ }
306
+
307
+ const disposers = [];
308
+
309
+ ctx.inject(["systemPrompt"], (promptCtx) => {
310
+ if (cfg.autoInject) disposers.push(createInjector(promptCtx, service, settings, cfg));
311
+ });
312
+
313
+ ctx.inject(["tools"], (toolsCtx) => {
314
+ disposers.push(createTools(toolsCtx, service, cfg, embedder));
315
+ });
316
+
317
+ const summarizer = createSummarizer(ctx, service, cfg);
318
+ disposers.push(summarizer.dispose);
319
+
320
+ // Session lifecycle (v0.6.0): when a session leaves the store and the toggle
321
+ // is enabled, mark every memory born in it as session-disposed (hidden from
322
+ // injection/search/dream but never destroyed — recoverable via
323
+ // restoreBySession). Default off, so a disposed session leaves its memories
324
+ // active (legacy behavior). Every path is guarded: a failure inside the
325
+ // callback must never propagate into DSH's session teardown (that would crash
326
+ // the plugin on the very delete action it serves).
327
+ if (cfg.sessionLifecycleEnabled) {
328
+ disposers.push(ctx.on("session/disposed", (session) => {
329
+ const sessionId = session?.id;
330
+ if (!sessionId) return;
331
+ try {
332
+ const { disposed } = service.disposeBySession(sessionId);
333
+ ctx.logger?.info?.(`[dsh-mneme] session disposed, hid ${disposed} memory(s) for ${sessionId}`);
334
+ } catch (error) {
335
+ ctx.logger?.warn?.(`[dsh-mneme] session dispose failed for ${sessionId}: ${String(error)}`);
336
+ }
337
+ }));
338
+ }
339
+
340
+ if (ctx.webServer) {
341
+ const api = createApi(ctx, service, settings, commands ?? {
342
+ add: () => { throw new Error("commands unavailable"); },
343
+ remove: () => false,
344
+ list: () => []
345
+ }, embedder, { vectorIndex, reranker }, cfg.apiToken);
346
+ disposers.push(api.dispose);
347
+ }
348
+
349
+ // Async disposer: cordis awaits the returned promise on unload (runDisposable),
350
+ // so an in-flight dream run is allowed to finish before the SQLite store is
351
+ // closed — dream.dispose() resolves only after its current run settles.
352
+ return async () => {
353
+ for (const dispose of disposers) {
354
+ if (typeof dispose === "function") dispose();
355
+ }
356
+ commands?.dispose();
357
+ if (dream) await dream.dispose();
358
+ if (sleep) sleep.dispose();
359
+ store.close();
360
+ };
361
+ };