@seanhogg/builderforce-memory-mcp 2026.7.12 → 2026.9.27

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. package/README.md +47 -5
  2. package/dist/backend.d.ts +32 -0
  3. package/dist/backend.d.ts.map +1 -1
  4. package/dist/backends/memory-store.d.ts +32 -9
  5. package/dist/backends/memory-store.d.ts.map +1 -1
  6. package/dist/backends/memory-store.js +155 -33
  7. package/dist/backends/memory-store.js.map +1 -1
  8. package/dist/bin/http.d.ts +5 -0
  9. package/dist/bin/http.d.ts.map +1 -1
  10. package/dist/bin/http.js +11 -0
  11. package/dist/bin/http.js.map +1 -1
  12. package/dist/bin/install.js +3 -2
  13. package/dist/bin/install.js.map +1 -1
  14. package/dist/bin/stdio.d.ts +17 -7
  15. package/dist/bin/stdio.d.ts.map +1 -1
  16. package/dist/bin/stdio.js +24 -8
  17. package/dist/bin/stdio.js.map +1 -1
  18. package/dist/compaction.d.ts +76 -0
  19. package/dist/compaction.d.ts.map +1 -0
  20. package/dist/compaction.js +77 -0
  21. package/dist/compaction.js.map +1 -0
  22. package/dist/embedding/evermind-embedder.d.ts +119 -0
  23. package/dist/embedding/evermind-embedder.d.ts.map +1 -0
  24. package/dist/embedding/evermind-embedder.js +201 -0
  25. package/dist/embedding/evermind-embedder.js.map +1 -0
  26. package/dist/embedding/index.d.ts +38 -0
  27. package/dist/embedding/index.d.ts.map +1 -0
  28. package/dist/embedding/index.js +44 -0
  29. package/dist/embedding/index.js.map +1 -0
  30. package/dist/index.d.ts +8 -4
  31. package/dist/index.d.ts.map +1 -1
  32. package/dist/index.js +12 -2
  33. package/dist/index.js.map +1 -1
  34. package/dist/install/claude-hooks.d.ts +42 -2
  35. package/dist/install/claude-hooks.d.ts.map +1 -1
  36. package/dist/install/claude-hooks.js +111 -16
  37. package/dist/install/claude-hooks.js.map +1 -1
  38. package/dist/install/server-spec.d.ts +12 -0
  39. package/dist/install/server-spec.d.ts.map +1 -1
  40. package/dist/install/server-spec.js +18 -0
  41. package/dist/install/server-spec.js.map +1 -1
  42. package/dist/tools.d.ts.map +1 -1
  43. package/dist/tools.js +117 -6
  44. package/dist/tools.js.map +1 -1
  45. package/package.json +15 -4
  46. package/server.json +47 -0
  47. package/src/backend.ts +36 -0
  48. package/src/backends/memory-store.ts +184 -35
  49. package/src/bin/http.ts +12 -0
  50. package/src/bin/install.ts +3 -2
  51. package/src/bin/stdio.ts +25 -8
  52. package/src/compaction.ts +126 -0
  53. package/src/embedding/evermind-embedder.ts +276 -0
  54. package/src/embedding/index.ts +54 -0
  55. package/src/index.ts +18 -4
  56. package/src/install/claude-hooks.ts +116 -15
  57. package/src/install/server-spec.ts +25 -0
  58. package/src/tools.ts +133 -12
@@ -2,13 +2,19 @@
2
2
  * MemoryStoreBackend — local adapter mapping @seanhogg/builderforce-memory's MemoryStore
3
3
  * onto the MemoryBackend seam.
4
4
  *
5
- * Recall quality: when an SSM runtime is supplied it is forwarded to
6
- * `recallSimilar`, so recall uses SSM-embedding cosine similarity and improves
7
- * as the model is adapted/distilled. With no runtime it transparently falls
8
- * back to Jaccard word-overlap (still useful, just lexical).
5
+ * Recall quality: when a text embedder is supplied it is forwarded to the store's
6
+ * `recallRanked`, so the ordering is the shared hybrid retrieval — the SSM
7
+ * embedding's cosine ranking FUSED with BM25 by Reciprocal Rank Fusion — and it
8
+ * sharpens as the model is adapted/distilled. With no embedder the same call
9
+ * degrades to the lexical ranking alone. Either way the backend reports which of
10
+ * the two produced the ordering (`method`), so a silent degrade is visible.
11
+ *
12
+ * The embedder does NOT have to be a GPU runtime: `EvermindTextEmbedder` from the
13
+ * engine is a pure-CPU `EvermindLM`, which is what lets the headless stdio server
14
+ * rank recall semantically (see ./evermind-embedder.ts).
9
15
  */
10
16
 
11
- import type { MemoryBackend, RecallHit, RememberInput } from "../backend.js";
17
+ import type { MemoryBackend, RankedRecall, RecallHit, RememberInput } from "../backend.js";
12
18
 
13
19
  // Structural views of the @seanhogg/builderforce-memory surface we use, so this package
14
20
  // type-checks without a hard dependency on the runtime package.
@@ -25,14 +31,20 @@ interface MemoryStoreLike {
25
31
  recall(key: string): Promise<MemoryEntryLike | undefined>;
26
32
  recallAll(): Promise<MemoryEntryLike[]>;
27
33
  recallByTag(tag: string): Promise<MemoryEntryLike[]>;
28
- recallSimilar(query: string, topK: number, runtime?: unknown): Promise<MemoryEntryLike[]>;
34
+ /** Hybrid (dense⊕BM25) recall with each hit's fused score and the ranker used. */
35
+ recallRanked(
36
+ query: string,
37
+ topK: number,
38
+ runtime?: unknown,
39
+ ): Promise<{ hits: Array<{ entry: MemoryEntryLike; score: number }>; method: "embedding" | "lexical" }>;
29
40
  forget(key: string): Promise<void>;
30
41
  }
31
42
 
32
- function toHit(e: MemoryEntryLike): RecallHit {
43
+ function toHit(e: MemoryEntryLike, score?: number): RecallHit {
33
44
  return {
34
45
  key: e.key,
35
46
  content: e.content,
47
+ ...(score === undefined ? {} : { score }),
36
48
  tags: e.tags,
37
49
  importance: e.importance,
38
50
  timestamp: e.timestamp,
@@ -46,9 +58,14 @@ export class MemoryStoreBackend implements MemoryBackend {
46
58
  private readonly runtime?: unknown,
47
59
  ) {}
48
60
 
61
+ /** Plain recall — {@link recallRanked} with the ranker's name dropped. ONE path. */
49
62
  async recall(query: string, topK: number): Promise<RecallHit[]> {
50
- const entries = await this.store.recallSimilar(query, topK, this.runtime);
51
- return entries.map(toHit);
63
+ return (await this.recallRanked(query, topK)).hits;
64
+ }
65
+
66
+ async recallRanked(query: string, topK: number): Promise<RankedRecall> {
67
+ const ranked = await this.store.recallRanked(query, topK, this.runtime);
68
+ return { hits: ranked.hits.map((h) => toHit(h.entry, h.score)), method: ranked.method };
52
69
  }
53
70
 
54
71
  async get(key: string): Promise<RecallHit | undefined> {
@@ -69,6 +86,11 @@ export class MemoryStoreBackend implements MemoryBackend {
69
86
  });
70
87
  }
71
88
 
89
+ /** The store has no batch write; sequencing here keeps ONE definition of a write. */
90
+ async rememberMany(inputs: RememberInput[]): Promise<void> {
91
+ for (const input of inputs) await this.remember(input);
92
+ }
93
+
72
94
  async forget(key: string): Promise<void> {
73
95
  await this.store.forget(key);
74
96
  }
@@ -79,9 +101,11 @@ export interface LocalBackendOptions {
79
101
  /** IndexedDB database name. Defaults to MemoryStore's own default ('ssmjs'). */
80
102
  dbName?: string;
81
103
  /**
82
- * Optional SSMRuntime for embedding-based recall. Omit for lexical (Jaccard)
83
- * recall. In the agent-runtime, pass `ssmMemoryService.runtime` here to reuse
84
- * the already-loaded Evermind runtime instead of standing up a second model.
104
+ * Optional text embedder — anything with `embed(text): Promise<Float32Array>`.
105
+ * Omit for lexical-only recall. Pass the agent-runtime's
106
+ * `ssmMemoryService.runtime` to reuse an already-loaded GPU Evermind, or an
107
+ * `EvermindTextEmbedder` (pure CPU) via {@link createEvermindEmbedder} on a
108
+ * headless host.
85
109
  */
86
110
  runtime?: unknown;
87
111
  /**
@@ -93,6 +117,10 @@ export interface LocalBackendOptions {
93
117
  * When set, the store is hydrated from the file on creation and re-snapshotted
94
118
  * after every remember/forget, giving durable cross-process memory. TTLs are
95
119
  * dropped on persist: the snapshot is the durable long-term tier.
120
+ *
121
+ * The file is also WATCHED: it is a shared artifact (other processes compact it
122
+ * in place), so the store re-hydrates from disk whenever it changed underneath —
123
+ * see {@link DiskPersistedBackend} for the loop guard.
96
124
  */
97
125
  persistFile?: string;
98
126
  }
@@ -110,16 +138,70 @@ type FsLike = {
110
138
  writeFileSync(path: string, data: string): void;
111
139
  mkdirSync(path: string, opts: { recursive: boolean }): void;
112
140
  existsSync(path: string): boolean;
141
+ statSync(path: string): { mtimeMs: number; size: number };
113
142
  };
114
143
  type PathLike = { dirname(p: string): string };
115
144
 
145
+ /** Identity of the snapshot file as this process last left it. */
146
+ interface FileStamp {
147
+ mtimeMs: number;
148
+ size: number;
149
+ /** Digest of the exact bytes — settles the case where a file is touched but unchanged. */
150
+ hash: string;
151
+ }
152
+
153
+ /**
154
+ * FNV-1a over the snapshot text. Non-cryptographic on purpose: this only has to
155
+ * separate "someone rewrote the file" from "the mtime moved but the bytes are
156
+ * ours", and node:crypto is deliberately not imported here (this module must stay
157
+ * bundleable for the browser, where the disk path is never taken).
158
+ */
159
+ function contentHash(text: string): string {
160
+ let h = 0x811c9dc5;
161
+ for (let i = 0; i < text.length; i++) {
162
+ h ^= text.charCodeAt(i);
163
+ h = Math.imul(h, 0x01000193);
164
+ }
165
+ return (h >>> 0).toString(16);
166
+ }
167
+
168
+ /** Parse a snapshot's text into its durable entries, or null when unusable. */
169
+ function parseSnapshot(text: string): SnapshotEntry[] | null {
170
+ let parsed: unknown;
171
+ try {
172
+ parsed = JSON.parse(text);
173
+ } catch {
174
+ return null;
175
+ }
176
+ if (!Array.isArray(parsed)) return null;
177
+ return (parsed as SnapshotEntry[]).filter(
178
+ (raw) => !!raw && typeof raw.key === "string" && typeof raw.content === "string",
179
+ );
180
+ }
181
+
116
182
  /**
117
183
  * Wraps a MemoryStoreBackend so every write is mirrored to a JSON file, and
118
184
  * hydrates that file back into the store on boot. This is what turns a respawned
119
185
  * stdio subprocess into a persistent memory: the store itself is in-memory, the
120
186
  * file is the source of truth across process lifetimes.
187
+ *
188
+ * It also WATCHES the file for external edits. The snapshot is a shared artifact —
189
+ * the BuilderForce VS Code extension compacts absorbed entries by rewriting it
190
+ * directly, and a second server instance may be mirroring the same path. Without a
191
+ * watch this server, still holding the pre-edit bodies, would silently re-snapshot
192
+ * over that work on its very next remember/forget; the edit would only stick once
193
+ * the client respawned the subprocess. So every operation first calls
194
+ * {@link DiskPersistedBackend.ensureFresh}, which re-hydrates the store from disk
195
+ * when — and only when — the file changed underneath us.
196
+ *
197
+ * The loop guard is the {@link FileStamp} recorded immediately AFTER each of our own
198
+ * writes: the next check stats the file, sees the same mtime+size, and returns
199
+ * without reading a byte. A stat per call is the entire steady-state cost.
121
200
  */
122
201
  class DiskPersistedBackend implements MemoryBackend {
202
+ /** How the file looked when this process last wrote or read it. */
203
+ private stamp: FileStamp | null = null;
204
+
123
205
  constructor(
124
206
  private readonly inner: MemoryStoreBackend,
125
207
  private readonly store: MemoryStoreLike,
@@ -127,26 +209,99 @@ class DiskPersistedBackend implements MemoryBackend {
127
209
  private readonly fs: FsLike,
128
210
  ) {}
129
211
 
130
- recall(query: string, topK: number): Promise<RecallHit[]> {
212
+ async recall(query: string, topK: number): Promise<RecallHit[]> {
213
+ await this.ensureFresh();
131
214
  return this.inner.recall(query, topK);
132
215
  }
133
- get(key: string): Promise<RecallHit | undefined> {
216
+ async recallRanked(query: string, topK: number): Promise<RankedRecall> {
217
+ await this.ensureFresh();
218
+ return this.inner.recallRanked(query, topK);
219
+ }
220
+ async get(key: string): Promise<RecallHit | undefined> {
221
+ await this.ensureFresh();
134
222
  return this.inner.get(key);
135
223
  }
136
- recallByTag(tag: string, limit: number): Promise<RecallHit[]> {
224
+ async recallByTag(tag: string, limit: number): Promise<RecallHit[]> {
225
+ await this.ensureFresh();
137
226
  return this.inner.recallByTag(tag, limit);
138
227
  }
139
228
 
140
229
  async remember(input: RememberInput): Promise<void> {
230
+ await this.ensureFresh();
141
231
  await this.inner.remember(input);
142
232
  await this.snapshot();
143
233
  }
144
234
 
235
+ /** Batch write — ONE snapshot for the whole set, so compacting N keys is not N full rewrites. */
236
+ async rememberMany(inputs: RememberInput[]): Promise<void> {
237
+ await this.ensureFresh();
238
+ await this.inner.rememberMany(inputs);
239
+ await this.snapshot();
240
+ }
241
+
145
242
  async forget(key: string): Promise<void> {
243
+ await this.ensureFresh();
146
244
  await this.inner.forget(key);
147
245
  await this.snapshot();
148
246
  }
149
247
 
248
+ /**
249
+ * Re-read the snapshot into the store when the file changed since we last
250
+ * touched it. Also the BOOT hydration: with no stamp yet, an existing file is
251
+ * always read in — one routine, so disk and memory can never diverge by taking
252
+ * two different paths.
253
+ */
254
+ async ensureFresh(): Promise<void> {
255
+ if (!this.fs.existsSync(this.file)) return;
256
+
257
+ let st: { mtimeMs: number; size: number };
258
+ try {
259
+ st = this.fs.statSync(this.file);
260
+ } catch {
261
+ return;
262
+ }
263
+ // Fast path: byte-for-byte what we last wrote. No read, no parse, no writes.
264
+ if (this.stamp && st.mtimeMs === this.stamp.mtimeMs && st.size === this.stamp.size) return;
265
+
266
+ let text: string;
267
+ try {
268
+ text = this.fs.readFileSync(this.file, "utf8");
269
+ } catch {
270
+ return;
271
+ }
272
+ const hash = contentHash(text);
273
+ // Touched (mtime moved) but identical content — re-stamp, don't re-hydrate.
274
+ if (this.stamp && hash === this.stamp.hash) {
275
+ this.stamp = { mtimeMs: st.mtimeMs, size: st.size, hash };
276
+ return;
277
+ }
278
+
279
+ const entries = parseSnapshot(text);
280
+ // Corrupt/partial snapshot (a half-written file, say): keep what we have
281
+ // rather than wiping the live store, and leave the stamp alone so the next
282
+ // call re-checks.
283
+ if (!entries) return;
284
+
285
+ await this.replaceStore(entries);
286
+ this.stamp = { mtimeMs: st.mtimeMs, size: st.size, hash };
287
+ }
288
+
289
+ /**
290
+ * Make the in-memory store equal the snapshot: drop keys the file no longer has,
291
+ * then replay every entry (a re-remember overwrites, so a body shortened to a
292
+ * stub on disk becomes the stub in memory). Replayed as durable — TTLs are
293
+ * dropped on persist; the snapshot is the long-term tier.
294
+ */
295
+ private async replaceStore(entries: SnapshotEntry[]): Promise<void> {
296
+ const wanted = new Set(entries.map((e) => e.key));
297
+ for (const existing of await this.store.recallAll()) {
298
+ if (!wanted.has(existing.key)) await this.store.forget(existing.key);
299
+ }
300
+ for (const raw of entries) {
301
+ await this.store.remember(raw.key, raw.content, { tags: raw.tags, importance: raw.importance });
302
+ }
303
+ }
304
+
150
305
  private async snapshot(): Promise<void> {
151
306
  const entries = await this.store.recallAll();
152
307
  const out: SnapshotEntry[] = entries.map((e) => ({
@@ -155,24 +310,16 @@ class DiskPersistedBackend implements MemoryBackend {
155
310
  tags: e.tags,
156
311
  importance: e.importance,
157
312
  }));
158
- this.fs.writeFileSync(this.file, JSON.stringify(out, null, 2));
159
- }
160
- }
161
-
162
- /** Reads a snapshot file and replays it into the store as durable (no-TTL) entries. */
163
- async function hydrateFromDisk(store: MemoryStoreLike, file: string, fs: FsLike): Promise<void> {
164
- if (!fs.existsSync(file)) return;
165
- let parsed: unknown;
166
- try {
167
- parsed = JSON.parse(fs.readFileSync(file, "utf8"));
168
- } catch {
169
- // Corrupt/partial snapshot — start clean rather than crash the server.
170
- return;
171
- }
172
- if (!Array.isArray(parsed)) return;
173
- for (const raw of parsed as SnapshotEntry[]) {
174
- if (!raw || typeof raw.key !== "string" || typeof raw.content !== "string") continue;
175
- await store.remember(raw.key, raw.content, { tags: raw.tags, importance: raw.importance });
313
+ const json = JSON.stringify(out, null, 2);
314
+ this.fs.writeFileSync(this.file, json);
315
+ // Stamp OUR write immediately — this is what stops the watch above from
316
+ // treating our own output as an external edit (and looping).
317
+ try {
318
+ const st = this.fs.statSync(this.file);
319
+ this.stamp = { mtimeMs: st.mtimeMs, size: st.size, hash: contentHash(json) };
320
+ } catch {
321
+ this.stamp = null;
322
+ }
176
323
  }
177
324
  }
178
325
 
@@ -212,6 +359,8 @@ export async function createLocalMemoryStoreBackend(opts: LocalBackendOptions =
212
359
  const dir = path.dirname(opts.persistFile);
213
360
  if (dir && !fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
214
361
 
215
- await hydrateFromDisk(store, opts.persistFile, fs);
216
- return new DiskPersistedBackend(backend, store, opts.persistFile, fs);
362
+ const persisted = new DiskPersistedBackend(backend, store, opts.persistFile, fs);
363
+ // Boot hydration IS the freshness check with no stamp yet — see ensureFresh().
364
+ await persisted.ensureFresh();
365
+ return persisted;
217
366
  }
package/src/bin/http.ts CHANGED
@@ -9,16 +9,28 @@
9
9
  * BUILDERFORCE_MEMORY_TOKEN Bearer token required on every request (recommended).
10
10
  * BUILDERFORCE_MEMORY_DB IndexedDB database name.
11
11
  * BUILDERFORCE_MEMORY_READONLY '1' to disable remember/forget tools.
12
+ * BUILDERFORCE_MEMORY_FILE JSON snapshot path (also sites the vector cache).
13
+ * BUILDERFORCE_MEMORY_MODEL `.evermind` package to rank recall by SSM embedding
14
+ * (pure CPU, no GPU). Unset = lexical ranking.
15
+ * BUILDERFORCE_MEMORY_TOKENIZER Tokenizer JSON for that checkpoint.
16
+ * BUILDERFORCE_MEMORY_VECTORS Persistent vector-cache path.
12
17
  * BUILDERFORCE_GATEWAY_URL Gateway base URL (default https://api.builderforce.ai).
13
18
  * BUILDERFORCE_API_KEY `bfk_*` tenant key. When set, exposes the cost tools.
14
19
  */
15
20
 
16
21
  import http from "node:http";
17
22
  import { createLocalMemoryStoreBackend } from "../backends/memory-store.js";
23
+ import { createRecallEmbedder } from "../embedding/index.js";
24
+ import { resolveMemoryFile } from "../install/server-spec.js";
18
25
  import { createMemoryHttpHandler } from "../transports/http.js";
19
26
 
27
+ const memoryFile = resolveMemoryFile();
28
+
20
29
  const backend = await createLocalMemoryStoreBackend({
21
30
  dbName: process.env["BUILDERFORCE_MEMORY_DB"],
31
+ // Same embedder decision as the stdio bin, from the same helper — the two
32
+ // servers must not rank recall differently.
33
+ runtime: (await createRecallEmbedder({ memoryFile })) ?? undefined,
22
34
  });
23
35
 
24
36
  const handler = createMemoryHttpHandler(backend, {
@@ -22,6 +22,7 @@ import nodeFs from "node:fs";
22
22
  import { HOSTS } from "../install/hosts.js";
23
23
  import { installMemoryServer, type HostSelector } from "../install/install.js";
24
24
  import { installClaudeCombo } from "../install/claude-hooks.js";
25
+ import { defaultMemoryFile } from "../install/server-spec.js";
25
26
 
26
27
  function argValue(name: string): string | undefined {
27
28
  const hit = process.argv.find((a) => a.startsWith(`--${name}=`));
@@ -54,7 +55,7 @@ const hosts: HostSelector =
54
55
  ? "all"
55
56
  : hostArg.split(",").map((s) => s.trim()).filter(Boolean);
56
57
 
57
- const memoryFile = argValue("memory-file") ?? path.join(os.homedir(), ".builderforce-memory", "memory.json");
58
+ const memoryFile = argValue("memory-file") ?? defaultMemoryFile();
58
59
 
59
60
  let results;
60
61
  try {
@@ -111,6 +112,6 @@ if (results.length === 0) {
111
112
  );
112
113
  } else {
113
114
  process.stdout.write(
114
- `\nDone — ${wrote} config(s) written. Store: ${memoryFile}\nRestart the agent(s) to connect. Tools: memory_recall · memory_remember · memory_get · memory_recall_by_tag · memory_forget\n`,
115
+ `\nDone — ${wrote} config(s) written. Store: ${memoryFile}\nRestart the agent(s) to connect. Tools: memory_recall · memory_remember · memory_get · memory_recall_by_tag · memory_forget · memory_compact\n`,
115
116
  );
116
117
  }
package/src/bin/stdio.ts CHANGED
@@ -5,25 +5,42 @@
5
5
  * Env:
6
6
  * BUILDERFORCE_MEMORY_DB IndexedDB database name (default: MemoryStore default).
7
7
  * BUILDERFORCE_MEMORY_READONLY '1' to disable remember/forget tools.
8
- * BUILDERFORCE_MEMORY_FILE Absolute path to a JSON snapshot. When set, memory
9
- * persists across process restarts — REQUIRED for an
10
- * MCP client that respawns this server each session
11
- * (otherwise fake-indexeddb loses everything on exit).
8
+ * BUILDERFORCE_MEMORY_FILE Absolute path to a JSON snapshot. Defaults to the
9
+ * shared per-machine store (~/.builderforce-memory/
10
+ * memory.json) so memory survives the respawn every
11
+ * MCP client does between sessions — without a
12
+ * snapshot fake-indexeddb loses everything on exit.
13
+ * BUILDERFORCE_MEMORY_MODEL Absolute path to a `.evermind` package (an
14
+ * `evermind-lm`). When set, recall is ranked by SSM
15
+ * embedding cosine FUSED with the lexical ranking
16
+ * instead of lexically alone. Pure CPU — no GPU is
17
+ * involved; see src/embedding/evermind-embedder.ts.
18
+ * BUILDERFORCE_MEMORY_TOKENIZER Tokenizer JSON matching that checkpoint. Defaults
19
+ * to `<model>.tokenizer.json`.
20
+ * BUILDERFORCE_MEMORY_VECTORS Persistent vector-cache path. Defaults beside the
21
+ * snapshot, so a respawned subprocess does not
22
+ * re-embed the whole store on its first recall.
12
23
  * BUILDERFORCE_GATEWAY_URL Gateway base URL (default https://api.builderforce.ai).
13
24
  * BUILDERFORCE_API_KEY `bfk_*` tenant key. When set, exposes the cost tools
14
25
  * (token_usage, model_efficiency).
15
26
  *
16
- * Recall is lexical (Jaccard) here — this headless binary does not stand up the
17
- * SSM runtime/GPU. For SSM-embedding recall, embed the package in-process and
18
- * pass `runtime` to createLocalMemoryStoreBackend (see README).
27
+ * With no model configured recall ranks lexically (BM25) and the tool result SAYS
28
+ * so ("Ranked by: lexical recall (fallback …)"), so a degrade is never silent.
19
29
  */
20
30
 
21
31
  import { createLocalMemoryStoreBackend } from "../backends/memory-store.js";
32
+ import { createRecallEmbedder } from "../embedding/index.js";
33
+ import { resolveMemoryFile } from "../install/server-spec.js";
22
34
  import { runStdio } from "../transports/stdio.js";
23
35
 
36
+ const memoryFile = resolveMemoryFile();
37
+
24
38
  const backend = await createLocalMemoryStoreBackend({
25
39
  dbName: process.env["BUILDERFORCE_MEMORY_DB"],
26
- persistFile: process.env["BUILDERFORCE_MEMORY_FILE"],
40
+ persistFile: memoryFile,
41
+ // Null when no model is configured (or it fails to load) — recall then stays
42
+ // lexical rather than the server refusing to start over an optional model.
43
+ runtime: (await createRecallEmbedder({ memoryFile })) ?? undefined,
27
44
  });
28
45
 
29
46
  await runStdio(backend, {
@@ -0,0 +1,126 @@
1
+ /**
2
+ * Compaction — turning an ABSORBED memory into a terse pointer stub.
3
+ *
4
+ * Once a fact has been folded into an Evermind (or any downstream learner) the
5
+ * full body has done its job: keeping it verbatim in the store means every
6
+ * subsequent recall pays for text the model already carries in its weights.
7
+ * Compaction replaces the body with a one-line `[absorbed→Evermind vN] <first
8
+ * line>` pointer, so the fact stops filling context while it stays findable.
9
+ *
10
+ * The rules live here — not in the tool handler and not in a transport — because
11
+ * BOTH the in-process `memory_compact` tool and any external compactor (the
12
+ * BuilderForce VS Code extension rewrites the snapshot directly) must produce
13
+ * byte-identical stubs; otherwise re-running either one re-stubs the other's work.
14
+ *
15
+ * Dependency-free and published as its own subpath
16
+ * (`@seanhogg/builderforce-memory-mcp/compaction`), so an external compactor imports
17
+ * these rules instead of re-implementing them — the marker below is the shared
18
+ * idempotency contract between the two processes.
19
+ */
20
+
21
+ /**
22
+ * Marker that opens every compacted stub — also the idempotency guard. An entry
23
+ * whose content already starts with this is skipped, so compacting twice (or
24
+ * from two processes) never double-stubs.
25
+ */
26
+ export const STUB_PREFIX = "[absorbed→Evermind";
27
+
28
+ /** Default cap on the pointer line carried inside a stub. */
29
+ export const DEFAULT_STUB_CHARS = 140;
30
+
31
+ /** True when a body is already a compaction stub. */
32
+ export function isStub(content: string): boolean {
33
+ return content.trimStart().startsWith(STUB_PREFIX);
34
+ }
35
+
36
+ /** The first non-empty line of a body, trimmed to `max` chars — the stub's pointer text. */
37
+ export function firstLine(content: string, max = DEFAULT_STUB_CHARS): string {
38
+ const line =
39
+ content
40
+ .split(/\r?\n/)
41
+ .map((l) => l.trim())
42
+ .find((l) => l.length > 0) ?? "";
43
+ return line.length > max ? `${line.slice(0, max - 1).trimEnd()}…` : line;
44
+ }
45
+
46
+ /** Build the terse stub that replaces an absorbed entry's body. */
47
+ export function memoryStub(content: string, version: number, maxChars = DEFAULT_STUB_CHARS): string {
48
+ return `${STUB_PREFIX} v${version}] ${firstLine(content, maxChars)}`;
49
+ }
50
+
51
+ /** One entry considered for compaction. */
52
+ export interface CompactionCandidate {
53
+ key: string;
54
+ content: string;
55
+ }
56
+
57
+ /** A body rewrite the caller should apply. */
58
+ export interface CompactionWrite {
59
+ key: string;
60
+ /** The stub that replaces the entry's body. */
61
+ content: string;
62
+ /** Characters recovered by this rewrite. */
63
+ bytesSaved: number;
64
+ }
65
+
66
+ /** Why a candidate was left alone. */
67
+ export interface CompactionSkip {
68
+ key: string;
69
+ reason: "not_found" | "already_compacted" | "not_smaller";
70
+ }
71
+
72
+ /** The decided plan: what to write, what to leave, and the total recovered. */
73
+ export interface CompactionPlan {
74
+ writes: CompactionWrite[];
75
+ skipped: CompactionSkip[];
76
+ bytesSaved: number;
77
+ }
78
+
79
+ /** Options controlling how a stub body is produced. */
80
+ export interface CompactionOptions {
81
+ /** Explicit replacement body. Omit to generate a pointer from each entry's own first line. */
82
+ stub?: string;
83
+ /** Evermind version that absorbed the facts; appears in a generated stub. Default 0. */
84
+ version?: number;
85
+ /** Max characters of a generated pointer line. Default {@link DEFAULT_STUB_CHARS}. */
86
+ maxChars?: number;
87
+ }
88
+
89
+ /**
90
+ * Decide the compaction for a set of candidates. Pure — no I/O — so the same
91
+ * rules are unit-testable and reusable by any caller that can read and write
92
+ * memories. `undefined` content means the key was not found.
93
+ */
94
+ export function planCompaction(
95
+ candidates: Array<CompactionCandidate | { key: string; content: undefined }>,
96
+ opts: CompactionOptions = {},
97
+ ): CompactionPlan {
98
+ const version = opts.version ?? 0;
99
+ const maxChars = Math.max(20, opts.maxChars ?? DEFAULT_STUB_CHARS);
100
+ const writes: CompactionWrite[] = [];
101
+ const skipped: CompactionSkip[] = [];
102
+ let bytesSaved = 0;
103
+
104
+ for (const c of candidates) {
105
+ const content = c.content;
106
+ if (typeof content !== "string" || !content.trim()) {
107
+ skipped.push({ key: c.key, reason: "not_found" });
108
+ continue;
109
+ }
110
+ if (isStub(content)) {
111
+ skipped.push({ key: c.key, reason: "already_compacted" });
112
+ continue;
113
+ }
114
+ const stub = opts.stub ?? memoryStub(content, version, maxChars);
115
+ // Never grow an entry — a "compaction" that costs tokens is a regression.
116
+ if (stub.length >= content.length) {
117
+ skipped.push({ key: c.key, reason: "not_smaller" });
118
+ continue;
119
+ }
120
+ const saved = content.length - stub.length;
121
+ writes.push({ key: c.key, content: stub, bytesSaved: saved });
122
+ bytesSaved += saved;
123
+ }
124
+
125
+ return { writes, skipped, bytesSaved };
126
+ }