@seanhogg/builderforce-memory-mcp 2026.7.12 → 2026.9.27
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +47 -5
- package/dist/backend.d.ts +32 -0
- package/dist/backend.d.ts.map +1 -1
- package/dist/backends/memory-store.d.ts +32 -9
- package/dist/backends/memory-store.d.ts.map +1 -1
- package/dist/backends/memory-store.js +155 -33
- package/dist/backends/memory-store.js.map +1 -1
- package/dist/bin/http.d.ts +5 -0
- package/dist/bin/http.d.ts.map +1 -1
- package/dist/bin/http.js +11 -0
- package/dist/bin/http.js.map +1 -1
- package/dist/bin/install.js +3 -2
- package/dist/bin/install.js.map +1 -1
- package/dist/bin/stdio.d.ts +17 -7
- package/dist/bin/stdio.d.ts.map +1 -1
- package/dist/bin/stdio.js +24 -8
- package/dist/bin/stdio.js.map +1 -1
- package/dist/compaction.d.ts +76 -0
- package/dist/compaction.d.ts.map +1 -0
- package/dist/compaction.js +77 -0
- package/dist/compaction.js.map +1 -0
- package/dist/embedding/evermind-embedder.d.ts +119 -0
- package/dist/embedding/evermind-embedder.d.ts.map +1 -0
- package/dist/embedding/evermind-embedder.js +201 -0
- package/dist/embedding/evermind-embedder.js.map +1 -0
- package/dist/embedding/index.d.ts +38 -0
- package/dist/embedding/index.d.ts.map +1 -0
- package/dist/embedding/index.js +44 -0
- package/dist/embedding/index.js.map +1 -0
- package/dist/index.d.ts +8 -4
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +12 -2
- package/dist/index.js.map +1 -1
- package/dist/install/claude-hooks.d.ts +42 -2
- package/dist/install/claude-hooks.d.ts.map +1 -1
- package/dist/install/claude-hooks.js +111 -16
- package/dist/install/claude-hooks.js.map +1 -1
- package/dist/install/server-spec.d.ts +12 -0
- package/dist/install/server-spec.d.ts.map +1 -1
- package/dist/install/server-spec.js +18 -0
- package/dist/install/server-spec.js.map +1 -1
- package/dist/tools.d.ts.map +1 -1
- package/dist/tools.js +117 -6
- package/dist/tools.js.map +1 -1
- package/package.json +15 -4
- package/server.json +47 -0
- package/src/backend.ts +36 -0
- package/src/backends/memory-store.ts +184 -35
- package/src/bin/http.ts +12 -0
- package/src/bin/install.ts +3 -2
- package/src/bin/stdio.ts +25 -8
- package/src/compaction.ts +126 -0
- package/src/embedding/evermind-embedder.ts +276 -0
- package/src/embedding/index.ts +54 -0
- package/src/index.ts +18 -4
- package/src/install/claude-hooks.ts +116 -15
- package/src/install/server-spec.ts +25 -0
- package/src/tools.ts +133 -12
|
@@ -0,0 +1,276 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* evermind-embedder — SSM-embedding recall for a HEADLESS host, with no GPU.
|
|
3
|
+
*
|
|
4
|
+
* The stdio/HTTP servers used to rank recall by word overlap because standing up
|
|
5
|
+
* the embedding model was assumed to mean standing up WebGPU. It does not:
|
|
6
|
+
* `EvermindLM` is a pure-CPU model and `EvermindLM.embed()` mean-pools + L2-
|
|
7
|
+
* normalises the same hidden state the GPU `HybridMambaModel.embed()` does, to the
|
|
8
|
+
* same contract. So a headless Node process can embed — it just needs the
|
|
9
|
+
* checkpoint and the tokenizer that produced its token ids.
|
|
10
|
+
*
|
|
11
|
+
* This module is the loader for that pair, plus the ONE thing a per-session
|
|
12
|
+
* subprocess needs to make embedding affordable: a vector cache that OUTLIVES the
|
|
13
|
+
* process. Recall embeds the query plus every stored memory; recomputing 200
|
|
14
|
+
* memory embeddings on the first recall of every session would cost more than the
|
|
15
|
+
* lexical ranking it replaces. The cache is stamped with the model's fingerprint,
|
|
16
|
+
* so an adapted checkpoint invalidates it wholesale rather than silently mixing
|
|
17
|
+
* vectors from two different models.
|
|
18
|
+
*
|
|
19
|
+
* It mirrors how this package already persists memories: a JSON file alongside the
|
|
20
|
+
* memory snapshot, written by the process that owns it, tolerant of a corrupt or
|
|
21
|
+
* absent file (drop it and recompute — a cache is never the source of truth).
|
|
22
|
+
*/
|
|
23
|
+
|
|
24
|
+
/** The seam `MemoryStore.recallRanked(query, k, runtime)` consumes. */
|
|
25
|
+
export interface TextEmbedderLike {
|
|
26
|
+
embed(text: string): Promise<Float32Array>;
|
|
27
|
+
/** Vector width. */
|
|
28
|
+
readonly dimensions: number;
|
|
29
|
+
/** Model identity — vectors are only comparable within one fingerprint. */
|
|
30
|
+
readonly fingerprint: string;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/** A text embedder that also owns a persistent cache. */
|
|
34
|
+
export interface PersistentTextEmbedder extends TextEmbedderLike {
|
|
35
|
+
/** Write any pending vectors to disk now. Safe to call repeatedly. */
|
|
36
|
+
flush(): void;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export interface EvermindEmbedderOptions {
|
|
40
|
+
/**
|
|
41
|
+
* Absolute path to a `.evermind` package holding an `evermind-lm` checkpoint.
|
|
42
|
+
* This is the model whose hidden state becomes the embedding.
|
|
43
|
+
*/
|
|
44
|
+
modelFile: string;
|
|
45
|
+
/**
|
|
46
|
+
* Absolute path to the tokenizer JSON that produced the checkpoint's token ids
|
|
47
|
+
* (`BPETokenizer.toObject()`, or a Hugging Face `tokenizer.json`).
|
|
48
|
+
*
|
|
49
|
+
* Only needed for a package that does NOT embed its own tokenizer. Token ids are
|
|
50
|
+
* meaningless without the exact vocabulary behind them, so an `.evermind` built
|
|
51
|
+
* with `PackageMeta.tokenizer` carries one and is self-contained — that is
|
|
52
|
+
* preferred over any file, because a file can be the WRONG vocabulary while the
|
|
53
|
+
* embedded one is checksummed and vocab-size-checked against the checkpoint.
|
|
54
|
+
* Resolution order: the package's own tokenizer, then this path, then a sibling
|
|
55
|
+
* `<modelFile>.tokenizer.json`.
|
|
56
|
+
*/
|
|
57
|
+
tokenizerFile?: string;
|
|
58
|
+
/**
|
|
59
|
+
* Absolute path for the persistent vector cache. Omit to disable persistence
|
|
60
|
+
* (vectors are still cached in memory for the life of the process).
|
|
61
|
+
*/
|
|
62
|
+
cacheFile?: string;
|
|
63
|
+
/** Max vectors retained in the cache (LRU past this). Default 4000. */
|
|
64
|
+
maxEntries?: number;
|
|
65
|
+
/** Debounce before a dirty cache is written back, ms. Default 500. */
|
|
66
|
+
flushDelayMs?: number;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/** Minimal structural view of the optional engine package. */
|
|
70
|
+
interface EmbedEngine {
|
|
71
|
+
EvermindModelPackage: {
|
|
72
|
+
fromBlob(blob: ArrayBuffer): {
|
|
73
|
+
manifest: { modelType?: string; tokenizerFormat?: string };
|
|
74
|
+
loadLM(): unknown;
|
|
75
|
+
/** Present on engines that support the embedded-tokenizer section. */
|
|
76
|
+
loadTokenizer?(): { encode(t: string): number[] } | null;
|
|
77
|
+
};
|
|
78
|
+
};
|
|
79
|
+
BPETokenizer: {
|
|
80
|
+
new (): { loadFromSpec(spec: unknown): void; loadHuggingFace(spec: unknown): void; encode(t: string): number[] };
|
|
81
|
+
};
|
|
82
|
+
EvermindTextEmbedder: new (model: unknown, codec: unknown) => TextEmbedderLike;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
type FsLike = {
|
|
86
|
+
readFileSync(path: string, enc: "utf8"): string;
|
|
87
|
+
readFileSync(path: string): Uint8Array;
|
|
88
|
+
writeFileSync(path: string, data: string): void;
|
|
89
|
+
existsSync(path: string): boolean;
|
|
90
|
+
};
|
|
91
|
+
|
|
92
|
+
const DEFAULT_MAX_ENTRIES = 4000;
|
|
93
|
+
const DEFAULT_FLUSH_DELAY_MS = 500;
|
|
94
|
+
|
|
95
|
+
/** The on-disk cache shape: a fingerprint stamp plus text→vector rows. */
|
|
96
|
+
interface VectorCacheFile {
|
|
97
|
+
fingerprint: string;
|
|
98
|
+
dimensions: number;
|
|
99
|
+
/** [text, vector] pairs, least-recently-used first (insertion order is the LRU). */
|
|
100
|
+
vectors: Array<[string, number[]]>;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/** Indirect import so the optional engine peer is never statically resolved. */
|
|
104
|
+
function dynamicImport(m: string): Promise<unknown> {
|
|
105
|
+
// eslint-disable-next-line @typescript-eslint/no-implied-eval, no-new-func
|
|
106
|
+
return new Function("m", "return import(m)")(m) as Promise<unknown>;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/**
|
|
110
|
+
* Load the tokenizer spec from `file`, accepting either `BPETokenizer.toObject()`
|
|
111
|
+
* output or a Hugging Face `tokenizer.json`.
|
|
112
|
+
*/
|
|
113
|
+
function loadCodec(engine: EmbedEngine, fs: FsLike, file: string): { encode(t: string): number[] } {
|
|
114
|
+
const spec = JSON.parse(fs.readFileSync(file, "utf8")) as { vocab?: unknown; merges?: unknown; model?: unknown };
|
|
115
|
+
const tok = new engine.BPETokenizer();
|
|
116
|
+
if (spec.vocab && spec.merges) tok.loadFromSpec(spec);
|
|
117
|
+
else tok.loadHuggingFace(spec);
|
|
118
|
+
return tok;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
/**
|
|
122
|
+
* Build a CPU text embedder from a `.evermind` package + its tokenizer, wrapped in
|
|
123
|
+
* a persistent vector cache.
|
|
124
|
+
*
|
|
125
|
+
* Returns `null` — never throws — when the engine peer is absent, a file is
|
|
126
|
+
* missing, or the package is not an `evermind-lm`. Recall then stays on the
|
|
127
|
+
* lexical ranking and says so, which is the correct degrade: a memory server that
|
|
128
|
+
* refuses to start because an OPTIONAL model is unavailable is worse than one that
|
|
129
|
+
* ranks by word overlap.
|
|
130
|
+
*/
|
|
131
|
+
export async function createEvermindEmbedder(
|
|
132
|
+
opts: EvermindEmbedderOptions,
|
|
133
|
+
): Promise<PersistentTextEmbedder | null> {
|
|
134
|
+
try {
|
|
135
|
+
const engine = (await dynamicImport("@seanhogg/builderforce-memory-engine")) as Partial<EmbedEngine>;
|
|
136
|
+
if (!engine?.EvermindModelPackage || !engine.BPETokenizer || !engine.EvermindTextEmbedder) return null;
|
|
137
|
+
|
|
138
|
+
const fs = (await dynamicImport("node:fs")) as FsLike;
|
|
139
|
+
if (!fs.existsSync(opts.modelFile)) return null;
|
|
140
|
+
|
|
141
|
+
const bytes = fs.readFileSync(opts.modelFile);
|
|
142
|
+
const blob = bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength) as ArrayBuffer;
|
|
143
|
+
const pkg = (engine as EmbedEngine).EvermindModelPackage.fromBlob(blob);
|
|
144
|
+
if (pkg.manifest.modelType !== "evermind-lm") return null;
|
|
145
|
+
|
|
146
|
+
// Prefer the package's OWN tokenizer. It is checksummed and cross-checked
|
|
147
|
+
// against the checkpoint's vocab size at package time, so it cannot be the
|
|
148
|
+
// wrong vocabulary — which a separate file silently can be. Falling back to a
|
|
149
|
+
// file keeps every package written before the section existed working.
|
|
150
|
+
const embedded = pkg.loadTokenizer?.() ?? null;
|
|
151
|
+
let codec: { encode(t: string): number[] };
|
|
152
|
+
if (embedded) {
|
|
153
|
+
codec = embedded;
|
|
154
|
+
} else {
|
|
155
|
+
const tokenizerFile = opts.tokenizerFile ?? `${opts.modelFile}.tokenizer.json`;
|
|
156
|
+
if (!fs.existsSync(tokenizerFile)) return null;
|
|
157
|
+
codec = loadCodec(engine as EmbedEngine, fs, tokenizerFile);
|
|
158
|
+
}
|
|
159
|
+
const inner = new (engine as EmbedEngine).EvermindTextEmbedder(pkg.loadLM(), codec);
|
|
160
|
+
return new CachedTextEmbedder(inner, fs, opts);
|
|
161
|
+
} catch {
|
|
162
|
+
// Any failure to stand the model up is a degrade, not a fault.
|
|
163
|
+
return null;
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
/**
|
|
168
|
+
* Wraps a text embedder with an LRU vector cache mirrored to a JSON file.
|
|
169
|
+
*
|
|
170
|
+
* Exported for hosts that already hold an embedder (a GPU runtime, say) and want
|
|
171
|
+
* the same cross-process cache — there is one implementation of "cache these
|
|
172
|
+
* vectors on disk", not one per embedder.
|
|
173
|
+
*/
|
|
174
|
+
export class CachedTextEmbedder implements PersistentTextEmbedder {
|
|
175
|
+
readonly dimensions: number;
|
|
176
|
+
readonly fingerprint: string;
|
|
177
|
+
|
|
178
|
+
/** text → vector. Map insertion order is the LRU order. */
|
|
179
|
+
private readonly cache = new Map<string, Float32Array>();
|
|
180
|
+
private readonly file: string | undefined;
|
|
181
|
+
private readonly maxEntries: number;
|
|
182
|
+
private readonly flushDelayMs: number;
|
|
183
|
+
private dirty = false;
|
|
184
|
+
private timer: ReturnType<typeof setTimeout> | null = null;
|
|
185
|
+
|
|
186
|
+
constructor(
|
|
187
|
+
private readonly inner: TextEmbedderLike,
|
|
188
|
+
private readonly fs: FsLike,
|
|
189
|
+
opts: Pick<EvermindEmbedderOptions, "cacheFile" | "maxEntries" | "flushDelayMs">,
|
|
190
|
+
) {
|
|
191
|
+
this.dimensions = inner.dimensions;
|
|
192
|
+
this.fingerprint = inner.fingerprint;
|
|
193
|
+
this.file = opts.cacheFile;
|
|
194
|
+
this.maxEntries = Math.max(1, opts.maxEntries ?? DEFAULT_MAX_ENTRIES);
|
|
195
|
+
this.flushDelayMs = Math.max(0, opts.flushDelayMs ?? DEFAULT_FLUSH_DELAY_MS);
|
|
196
|
+
this.hydrate();
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
async embed(text: string): Promise<Float32Array> {
|
|
200
|
+
const hit = this.cache.get(text);
|
|
201
|
+
if (hit) {
|
|
202
|
+
// LRU touch — re-insert so this entry becomes most-recently-used.
|
|
203
|
+
this.cache.delete(text);
|
|
204
|
+
this.cache.set(text, hit);
|
|
205
|
+
return hit;
|
|
206
|
+
}
|
|
207
|
+
const vec = await this.inner.embed(text);
|
|
208
|
+
this.put(text, vec);
|
|
209
|
+
return vec;
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
/** Write pending vectors now, cancelling any scheduled write. */
|
|
213
|
+
flush(): void {
|
|
214
|
+
if (this.timer) {
|
|
215
|
+
clearTimeout(this.timer);
|
|
216
|
+
this.timer = null;
|
|
217
|
+
}
|
|
218
|
+
if (!this.dirty || !this.file) return;
|
|
219
|
+
const payload: VectorCacheFile = {
|
|
220
|
+
fingerprint: this.fingerprint,
|
|
221
|
+
dimensions: this.dimensions,
|
|
222
|
+
vectors: [...this.cache.entries()].map(([text, v]) => [text, Array.from(v)]),
|
|
223
|
+
};
|
|
224
|
+
try {
|
|
225
|
+
this.fs.writeFileSync(this.file, JSON.stringify(payload));
|
|
226
|
+
this.dirty = false;
|
|
227
|
+
} catch {
|
|
228
|
+
// A cache that cannot be written is still a working in-memory cache.
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
private put(text: string, vec: Float32Array): void {
|
|
233
|
+
if (this.cache.size >= this.maxEntries) {
|
|
234
|
+
const lru = this.cache.keys().next().value;
|
|
235
|
+
if (lru !== undefined) this.cache.delete(lru);
|
|
236
|
+
}
|
|
237
|
+
this.cache.set(text, vec);
|
|
238
|
+
this.dirty = true;
|
|
239
|
+
this.schedule();
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
/**
|
|
243
|
+
* Debounce the write-back: one recall embeds the query plus every new memory,
|
|
244
|
+
* and that burst should cost ONE file write, not one per vector.
|
|
245
|
+
*/
|
|
246
|
+
private schedule(): void {
|
|
247
|
+
if (!this.file || this.timer) return;
|
|
248
|
+
this.timer = setTimeout(() => {
|
|
249
|
+
this.timer = null;
|
|
250
|
+
this.flush();
|
|
251
|
+
}, this.flushDelayMs);
|
|
252
|
+
// Never hold the process open for a cache write.
|
|
253
|
+
(this.timer as unknown as { unref?: () => void }).unref?.();
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
/**
|
|
257
|
+
* Read the cache back. A cache stamped with a DIFFERENT fingerprint is from a
|
|
258
|
+
* different (or adapted) model, so its vectors are not comparable with this
|
|
259
|
+
* model's — it is dropped whole rather than mixed in.
|
|
260
|
+
*/
|
|
261
|
+
private hydrate(): void {
|
|
262
|
+
if (!this.file || !this.fs.existsSync(this.file)) return;
|
|
263
|
+
let parsed: VectorCacheFile;
|
|
264
|
+
try {
|
|
265
|
+
parsed = JSON.parse(this.fs.readFileSync(this.file, "utf8")) as VectorCacheFile;
|
|
266
|
+
} catch {
|
|
267
|
+
return;
|
|
268
|
+
}
|
|
269
|
+
if (!parsed || parsed.fingerprint !== this.fingerprint || !Array.isArray(parsed.vectors)) return;
|
|
270
|
+
for (const row of parsed.vectors) {
|
|
271
|
+
if (!Array.isArray(row) || typeof row[0] !== "string" || !Array.isArray(row[1])) continue;
|
|
272
|
+
if (row[1].length !== this.dimensions) continue;
|
|
273
|
+
this.cache.set(row[0], Float32Array.from(row[1]));
|
|
274
|
+
}
|
|
275
|
+
}
|
|
276
|
+
}
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* embedding — stand up the recall embedder for a headless server.
|
|
3
|
+
*
|
|
4
|
+
* Both out-of-process bins (stdio and HTTP) need the SAME decision: is an
|
|
5
|
+
* embedding model configured, and if so where do its vectors get cached? That
|
|
6
|
+
* decision lives here once, so the two servers cannot drift into ranking recall
|
|
7
|
+
* differently from one another.
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
import { createEvermindEmbedder, type PersistentTextEmbedder } from "./evermind-embedder.js";
|
|
11
|
+
|
|
12
|
+
export { createEvermindEmbedder, CachedTextEmbedder } from "./evermind-embedder.js";
|
|
13
|
+
export type { EvermindEmbedderOptions, PersistentTextEmbedder, TextEmbedderLike } from "./evermind-embedder.js";
|
|
14
|
+
|
|
15
|
+
/** Absolute path to a `.evermind` package (an `evermind-lm`) to embed recall with. */
|
|
16
|
+
export const MODEL_FILE_ENV = "BUILDERFORCE_MEMORY_MODEL";
|
|
17
|
+
/** Absolute path to the tokenizer JSON matching that checkpoint. */
|
|
18
|
+
export const TOKENIZER_FILE_ENV = "BUILDERFORCE_MEMORY_TOKENIZER";
|
|
19
|
+
/** Absolute path for the persistent vector cache; defaults beside the snapshot. */
|
|
20
|
+
export const VECTOR_CACHE_ENV = "BUILDERFORCE_MEMORY_VECTORS";
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* Where vectors are cached by default: beside the memory snapshot, under the same
|
|
24
|
+
* name. They belong together — the cache is only meaningful for the memories in
|
|
25
|
+
* that snapshot, and pointing a second store at the same vectors would be a cache
|
|
26
|
+
* for the wrong corpus.
|
|
27
|
+
*/
|
|
28
|
+
export function defaultVectorCacheFile(memoryFile: string): string {
|
|
29
|
+
return memoryFile.replace(/\.json$/i, "") + ".vectors.json";
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
export interface RecallEmbedderEnv {
|
|
33
|
+
env?: Record<string, string | undefined>;
|
|
34
|
+
/** The snapshot path in effect — used to site the default vector cache. */
|
|
35
|
+
memoryFile: string;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* The embedder this server should rank recall with, or `null` when none is
|
|
40
|
+
* configured (recall then ranks lexically and says so).
|
|
41
|
+
*
|
|
42
|
+
* Opt-in by design: no model path, no model. A memory server must start and serve
|
|
43
|
+
* recall on a host that has never downloaded a checkpoint.
|
|
44
|
+
*/
|
|
45
|
+
export async function createRecallEmbedder(opts: RecallEmbedderEnv): Promise<PersistentTextEmbedder | null> {
|
|
46
|
+
const env = opts.env ?? process.env;
|
|
47
|
+
const modelFile = env[MODEL_FILE_ENV];
|
|
48
|
+
if (!modelFile) return null;
|
|
49
|
+
return createEvermindEmbedder({
|
|
50
|
+
modelFile,
|
|
51
|
+
tokenizerFile: env[TOKENIZER_FILE_ENV],
|
|
52
|
+
cacheFile: env[VECTOR_CACHE_ENV] || defaultVectorCacheFile(opts.memoryFile),
|
|
53
|
+
});
|
|
54
|
+
}
|
package/src/index.ts
CHANGED
|
@@ -8,7 +8,14 @@
|
|
|
8
8
|
*/
|
|
9
9
|
|
|
10
10
|
// ── Seam ────────────────────────────────────────────────────────────────────
|
|
11
|
-
export type { MemoryBackend, RecallHit, RememberInput } from "./backend.js";
|
|
11
|
+
export type { MemoryBackend, RankedRecall, RecallHit, RecallMethod, RememberInput } from "./backend.js";
|
|
12
|
+
|
|
13
|
+
// -- CPU SSM-embedding recall (no GPU) --------------------------------------
|
|
14
|
+
// The headless servers rank recall by EvermindLM embedding cosine fused with the
|
|
15
|
+
// lexical ranking whenever a checkpoint is configured; exported so an embedding
|
|
16
|
+
// host can build the same embedder (and share its persistent vector cache).
|
|
17
|
+
export { createRecallEmbedder, createEvermindEmbedder, CachedTextEmbedder, defaultVectorCacheFile, MODEL_FILE_ENV, TOKENIZER_FILE_ENV, VECTOR_CACHE_ENV } from "./embedding/index.js";
|
|
18
|
+
export type { EvermindEmbedderOptions, PersistentTextEmbedder, TextEmbedderLike, RecallEmbedderEnv } from "./embedding/index.js";
|
|
12
19
|
|
|
13
20
|
// ── Local backend (IndexedDB via @seanhogg/builderforce-memory) ────────────────────────
|
|
14
21
|
export { MemoryStoreBackend, createLocalMemoryStoreBackend } from "./backends/memory-store.js";
|
|
@@ -18,6 +25,13 @@ export type { LocalBackendOptions } from "./backends/memory-store.js";
|
|
|
18
25
|
export { buildMemoryTools } from "./tools.js";
|
|
19
26
|
export type { MemoryTool, MemoryToolsOptions, ToolResult } from "./tools.js";
|
|
20
27
|
|
|
28
|
+
// ── Compaction (absorbed fact → one-line pointer stub) ───────────────────────
|
|
29
|
+
// Exported so an EXTERNAL compactor (the BuilderForce VS Code extension rewrites
|
|
30
|
+
// the snapshot file directly) can produce byte-identical stubs, keeping the two
|
|
31
|
+
// paths idempotent with respect to each other.
|
|
32
|
+
export { STUB_PREFIX, DEFAULT_STUB_CHARS, isStub, firstLine, memoryStub, planCompaction } from "./compaction.js";
|
|
33
|
+
export type { CompactionCandidate, CompactionWrite, CompactionSkip, CompactionPlan, CompactionOptions } from "./compaction.js";
|
|
34
|
+
|
|
21
35
|
// ── Transports ──────────────────────────────────────────────────────────────
|
|
22
36
|
export { createMemoryMcpServer } from "./transports/sdk.js";
|
|
23
37
|
export type { SdkServerOptions, SdkMcpServerConfig } from "./transports/sdk.js";
|
|
@@ -32,7 +46,7 @@ export type { HttpHandlerOptions } from "./transports/http.js";
|
|
|
32
46
|
export { hashToken, timingSafeEqualStr, buildTenantIndex, bearerToken, RateLimiter } from "./transports/auth.js";
|
|
33
47
|
|
|
34
48
|
// ── Multi-host installer (wire the stdio server into any MCP-capable agent) ───
|
|
35
|
-
export { buildServerSpec, MCP_PACKAGE, MCP_BIN, RUNTIME_PEERS } from "./install/server-spec.js";
|
|
49
|
+
export { buildServerSpec, defaultMemoryFile, resolveMemoryFile, MCP_PACKAGE, MCP_BIN, MEMORY_FILE_ENV, RUNTIME_PEERS } from "./install/server-spec.js";
|
|
36
50
|
export type { StdioServerSpec, ServerSpecOptions } from "./install/server-spec.js";
|
|
37
51
|
export { installMemoryServer } from "./install/install.js";
|
|
38
52
|
export type { InstallOptions, InstallResult, InstallStatus, HostSelector, FsLike } from "./install/install.js";
|
|
@@ -40,5 +54,5 @@ export { HOSTS, SERVER_KEY, findHost } from "./install/hosts.js";
|
|
|
40
54
|
export type { HostAdapter, HostEnv, ConfigFormat } from "./install/hosts.js";
|
|
41
55
|
// Claude Code "memory combo" — hooks (SessionStart/PreCompact/UserPromptSubmit/Stop)
|
|
42
56
|
// + companion skill that make memory self-driving (contextual recall + autonomous capture).
|
|
43
|
-
export { installClaudeCombo, bfmemHookSource, companionSkillMd, claudeComboPaths } from "./install/claude-hooks.js";
|
|
44
|
-
export type { ClaudeComboPaths } from "./install/claude-hooks.js";
|
|
57
|
+
export { installClaudeCombo, bfmemHookSource, companionSkillMd, claudeComboPaths, pluginHooksConfig, HOOK_EVENTS } from "./install/claude-hooks.js";
|
|
58
|
+
export type { ClaudeComboPaths, HookConfig } from "./install/claude-hooks.js";
|
|
@@ -22,9 +22,21 @@
|
|
|
22
22
|
|
|
23
23
|
import nodePath from "node:path";
|
|
24
24
|
import type { FsLike } from "./install.js";
|
|
25
|
+
import { MEMORY_FILE_ENV } from "./server-spec.js";
|
|
25
26
|
|
|
26
|
-
/**
|
|
27
|
-
|
|
27
|
+
/**
|
|
28
|
+
* The single hook script, four modes.
|
|
29
|
+
*
|
|
30
|
+
* `memoryFile` is the JSON snapshot path to bake in. Pass `null` to emit a
|
|
31
|
+
* script that resolves the path at RUN time instead — required for the Claude
|
|
32
|
+
* Code plugin, whose files are static and machine-independent. The runtime
|
|
33
|
+
* branch must stay in lockstep with `resolveMemoryFile`; `install.test.mjs`
|
|
34
|
+
* asserts it does.
|
|
35
|
+
*/
|
|
36
|
+
export function bfmemHookSource(memoryFile: string | null): string {
|
|
37
|
+
const fileExpr = memoryFile === null
|
|
38
|
+
? `process.env[${JSON.stringify(MEMORY_FILE_ENV)}] || path.join(os.homedir(), ".builderforce-memory", "memory.json")`
|
|
39
|
+
: JSON.stringify(memoryFile);
|
|
28
40
|
return `#!/usr/bin/env node
|
|
29
41
|
/**
|
|
30
42
|
* builderforce-memory hook — generated by builderforce-memory-install. Dependency-free:
|
|
@@ -36,8 +48,10 @@ export function bfmemHookSource(memoryFile: string): string {
|
|
|
36
48
|
* durable correction is persisted via memory_remember.
|
|
37
49
|
*/
|
|
38
50
|
import fs from "node:fs";
|
|
51
|
+
import os from "node:os";
|
|
52
|
+
import path from "node:path";
|
|
39
53
|
|
|
40
|
-
const FILE = ${
|
|
54
|
+
const FILE = ${fileExpr};
|
|
41
55
|
const mode = process.argv[2] || "--session";
|
|
42
56
|
|
|
43
57
|
function readStdin() {
|
|
@@ -51,6 +65,27 @@ function tokens(s) {
|
|
|
51
65
|
return String(s || "").toLowerCase().split(/[^a-z0-9]+/).filter((t) => t.length >= 3);
|
|
52
66
|
}
|
|
53
67
|
|
|
68
|
+
// ── Per-session injection ledger ─────────────────────────────────────────────
|
|
69
|
+
// The SessionStart digest and the per-prompt recall both push memory CONTENT
|
|
70
|
+
// into the context window, where it stays for the rest of the session. Without
|
|
71
|
+
// dedup, a long session re-injects the same memories on every turn — unbounded
|
|
72
|
+
// growth that is exactly what pushes a session past 150k tokens. The ledger
|
|
73
|
+
// records which keys have already been surfaced THIS session so we inject each
|
|
74
|
+
// memory at most once. Keyed by the host-provided session_id; degrades to the
|
|
75
|
+
// old always-inject behaviour when no id is available.
|
|
76
|
+
function ledgerFile(sid) {
|
|
77
|
+
const safe = String(sid || "").replace(/[^a-zA-Z0-9_.-]/g, "").slice(0, 80) || "nosession";
|
|
78
|
+
return path.join(os.tmpdir(), "bfmem-injected-" + safe + ".json");
|
|
79
|
+
}
|
|
80
|
+
function loadLedger(sid) {
|
|
81
|
+
try { const a = JSON.parse(fs.readFileSync(ledgerFile(sid), "utf8")); return new Set(Array.isArray(a) ? a : []); }
|
|
82
|
+
catch { return new Set(); }
|
|
83
|
+
}
|
|
84
|
+
function saveLedger(sid, set) {
|
|
85
|
+
if (!sid) return; // no id → no dedup state to persist (safe fallback)
|
|
86
|
+
try { fs.writeFileSync(ledgerFile(sid), JSON.stringify([...set].slice(-500))); } catch { /* best effort */ }
|
|
87
|
+
}
|
|
88
|
+
|
|
54
89
|
if (mode === "--precompact") {
|
|
55
90
|
process.stdout.write(
|
|
56
91
|
"[builderforce-memory] Context is about to be compacted. Persist any durable learnings from this session — decisions, fixes, project facts, user preferences — by calling memory_remember with concise one-line facts. Recall them later with memory_recall instead of re-reading files.",
|
|
@@ -59,13 +94,18 @@ if (mode === "--precompact") {
|
|
|
59
94
|
}
|
|
60
95
|
|
|
61
96
|
if (mode === "--recall") {
|
|
62
|
-
let prompt = "";
|
|
63
|
-
try {
|
|
97
|
+
let prompt = "", sid = "";
|
|
98
|
+
try {
|
|
99
|
+
const j = JSON.parse(readStdin() || "{}");
|
|
100
|
+
prompt = typeof j.prompt === "string" ? j.prompt : "";
|
|
101
|
+
sid = typeof j.session_id === "string" ? j.session_id : "";
|
|
102
|
+
} catch { /* no input */ }
|
|
64
103
|
if (!prompt) process.exit(0);
|
|
65
104
|
const entries = loadEntries();
|
|
66
105
|
if (entries.length === 0) process.exit(0);
|
|
67
106
|
const q = new Set(tokens(prompt));
|
|
68
107
|
if (q.size === 0) process.exit(0);
|
|
108
|
+
const led = loadLedger(sid);
|
|
69
109
|
const scored = entries
|
|
70
110
|
.map((e) => {
|
|
71
111
|
const hay = new Set(tokens([e.key, e.content, (e.tags || []).join(" ")].join(" ")));
|
|
@@ -74,10 +114,13 @@ if (mode === "--recall") {
|
|
|
74
114
|
return { e, score };
|
|
75
115
|
})
|
|
76
116
|
.filter((x) => x.score >= 2)
|
|
117
|
+
.filter((x) => !led.has(x.e.key)) // already surfaced this session → don't re-inject
|
|
77
118
|
.sort((a, b) => b.score - a.score)
|
|
78
119
|
.slice(0, 3);
|
|
79
120
|
if (scored.length === 0) process.exit(0);
|
|
80
|
-
const
|
|
121
|
+
for (const x of scored) led.add(x.e.key);
|
|
122
|
+
saveLedger(sid, led);
|
|
123
|
+
const lines = scored.map((x) => "• " + x.e.key + ": " + String(x.e.content || "").slice(0, 200)).join("\\n");
|
|
81
124
|
process.stdout.write(
|
|
82
125
|
"[builderforce-memory] Relevant stored memories for this request — use these instead of re-deriving or re-reading:\\n" + lines,
|
|
83
126
|
);
|
|
@@ -148,19 +191,37 @@ if (mode === "--capture") {
|
|
|
148
191
|
process.exit(0);
|
|
149
192
|
}
|
|
150
193
|
|
|
194
|
+
// SessionStart (default mode). Read the source + session_id so we can (a) seed
|
|
195
|
+
// the per-session ledger with the digest keys and (b) skip re-injecting on
|
|
196
|
+
// 'resume', where the earlier context — including this digest — is retained and
|
|
197
|
+
// a second copy would be pure waste.
|
|
198
|
+
let src = "startup", sessionId = "";
|
|
199
|
+
try {
|
|
200
|
+
const j = JSON.parse(readStdin() || "{}");
|
|
201
|
+
if (typeof j.source === "string") src = j.source;
|
|
202
|
+
if (typeof j.session_id === "string") sessionId = j.session_id;
|
|
203
|
+
} catch { /* no input */ }
|
|
204
|
+
|
|
151
205
|
const entries = loadEntries();
|
|
152
206
|
if (entries.length === 0) {
|
|
153
|
-
process.stdout.write(
|
|
207
|
+
if (src !== "resume") process.stdout.write(
|
|
154
208
|
"[builderforce-memory] Active (memory empty). Store durable facts with memory_remember and retrieve them with memory_recall instead of re-reading files or chat history — that is how token usage stays low.",
|
|
155
209
|
);
|
|
156
210
|
process.exit(0);
|
|
157
211
|
}
|
|
158
|
-
const
|
|
212
|
+
const ranked = entries
|
|
159
213
|
.map((e) => ({ key: e.key, content: String(e.content ?? ""), importance: typeof e.importance === "number" ? e.importance : 0.5 }))
|
|
160
214
|
.sort((a, b) => b.importance - a.importance)
|
|
161
|
-
.slice(0,
|
|
162
|
-
|
|
163
|
-
|
|
215
|
+
.slice(0, 6);
|
|
216
|
+
|
|
217
|
+
// Record the digest keys so per-prompt recall won't re-surface what's already
|
|
218
|
+
// on screen. Fresh window (startup/clear) → new ledger; otherwise merge.
|
|
219
|
+
const led = (src === "startup" || src === "clear") ? new Set() : loadLedger(sessionId);
|
|
220
|
+
for (const e of ranked) led.add(e.key);
|
|
221
|
+
saveLedger(sessionId, led);
|
|
222
|
+
|
|
223
|
+
if (src === "resume") process.exit(0); // context retained across resume — nothing to re-inject
|
|
224
|
+
const top = ranked.map((e) => "• " + e.key + ": " + e.content.slice(0, 180)).join("\\n");
|
|
164
225
|
process.stdout.write(
|
|
165
226
|
"[builderforce-memory] Active — token-saving memory loaded. Before re-reading files or history, call memory_recall. Store durable facts with memory_remember.\\nTop memories:\\n" + top,
|
|
166
227
|
);
|
|
@@ -213,13 +274,37 @@ export function claudeComboPaths(claudeDir: string): ClaudeComboPaths {
|
|
|
213
274
|
};
|
|
214
275
|
}
|
|
215
276
|
|
|
216
|
-
|
|
277
|
+
/**
|
|
278
|
+
* The four events that make memory self-driving, and the hook-script mode each
|
|
279
|
+
* one runs. Single source of truth for BOTH delivery shapes: the settings.json
|
|
280
|
+
* merge performed by {@link installClaudeCombo}, and the static `hooks.json` a
|
|
281
|
+
* Claude Code plugin ships ({@link pluginHooksConfig}).
|
|
282
|
+
*/
|
|
283
|
+
export const HOOK_EVENTS = [
|
|
217
284
|
{ event: "SessionStart", mode: "--session" },
|
|
218
285
|
{ event: "PreCompact", mode: "--precompact" },
|
|
219
286
|
{ event: "UserPromptSubmit", mode: "--recall" },
|
|
220
287
|
{ event: "Stop", mode: "--capture" },
|
|
221
288
|
] as const;
|
|
222
289
|
|
|
290
|
+
/** Hook-config shape shared by `settings.json` and a plugin's `hooks/hooks.json`. */
|
|
291
|
+
export type HookConfig = Record<string, Array<{ hooks: Array<{ type: "command"; command: string }> }>>;
|
|
292
|
+
|
|
293
|
+
/**
|
|
294
|
+
* The plugin form of the combo: every event pointed at the plugin-local hook
|
|
295
|
+
* script. `${CLAUDE_PLUGIN_ROOT}` is expanded by Claude Code to the installed
|
|
296
|
+
* plugin directory, so the script needs no install step and no baked-in paths.
|
|
297
|
+
*/
|
|
298
|
+
export function pluginHooksConfig(scriptRelPath = "hooks/bfmem-hook.mjs"): HookConfig {
|
|
299
|
+
const config: HookConfig = {};
|
|
300
|
+
for (const { event, mode } of HOOK_EVENTS) {
|
|
301
|
+
config[event] = [{
|
|
302
|
+
hooks: [{ type: "command", command: `node "\${CLAUDE_PLUGIN_ROOT}/${scriptRelPath}" ${mode}` }],
|
|
303
|
+
}];
|
|
304
|
+
}
|
|
305
|
+
return config;
|
|
306
|
+
}
|
|
307
|
+
|
|
223
308
|
/**
|
|
224
309
|
* Write the hook script + companion skill and register all four hooks in
|
|
225
310
|
* settings.json. Idempotent (skips a hook command already present) and
|
|
@@ -251,12 +336,28 @@ export function installClaudeCombo(opts: {
|
|
|
251
336
|
fs.copyFileSync(paths.settingsFile, `${paths.settingsFile}.bak`);
|
|
252
337
|
}
|
|
253
338
|
const hooks = (settings.hooks && typeof settings.hooks === "object" ? settings.hooks : {}) as Record<string, unknown>;
|
|
339
|
+
// Compare commands by a normalized form (path separators unified, whitespace
|
|
340
|
+
// collapsed, case-folded) so a re-install can't register a second copy of the
|
|
341
|
+
// same hook just because an earlier run wrote "C:/…" and this one writes
|
|
342
|
+
// "C:\…". Without this, Windows accumulates duplicate SessionStart/PreCompact
|
|
343
|
+
// hooks that each fire — doubling the memory the combo injects every session.
|
|
344
|
+
const normCmd = (c: string) => String(c ?? "").replace(/\\/g, "/").replace(/\s+/g, " ").trim().toLowerCase();
|
|
345
|
+
const hookRef = normCmd(paths.hookScript);
|
|
254
346
|
const added: string[] = [];
|
|
255
347
|
for (const { event, mode } of HOOK_EVENTS) {
|
|
256
348
|
const command = `node "${paths.hookScript}" ${mode}`;
|
|
257
|
-
const arr = (Array.isArray(hooks[event]) ? hooks[event] :
|
|
258
|
-
|
|
259
|
-
|
|
349
|
+
const arr = (Array.isArray(hooks[event]) ? hooks[event] : []) as Array<{ hooks?: Array<{ type?: string; command?: string }> }>;
|
|
350
|
+
// Drop any existing group that points at OUR hook script for THIS mode
|
|
351
|
+
// (any path-separator variant), then re-add exactly one canonical entry.
|
|
352
|
+
// This both dedupes accidental duplicates and refreshes the command.
|
|
353
|
+
const kept = arr.filter((g) => !(g.hooks || []).some((h) => {
|
|
354
|
+
const n = normCmd(h.command ?? "");
|
|
355
|
+
return n.includes(hookRef) && n.endsWith(mode);
|
|
356
|
+
}));
|
|
357
|
+
const wasPresent = kept.length !== arr.length;
|
|
358
|
+
kept.push({ hooks: [{ type: "command", command }] });
|
|
359
|
+
hooks[event] = kept;
|
|
360
|
+
if (!wasPresent) added.push(event);
|
|
260
361
|
}
|
|
261
362
|
settings.hooks = hooks;
|
|
262
363
|
const settingsDir = nodePath.dirname(paths.settingsFile);
|
|
@@ -7,12 +7,37 @@
|
|
|
7
7
|
* server is launched, so the per-host adapters never re-derive the command.
|
|
8
8
|
*/
|
|
9
9
|
|
|
10
|
+
import nodeOs from "node:os";
|
|
11
|
+
import nodePath from "node:path";
|
|
12
|
+
|
|
10
13
|
export interface StdioServerSpec {
|
|
11
14
|
command: string;
|
|
12
15
|
args: string[];
|
|
13
16
|
env?: Record<string, string>;
|
|
14
17
|
}
|
|
15
18
|
|
|
19
|
+
/**
|
|
20
|
+
* The default JSON snapshot path: ONE store per machine, so every MCP host
|
|
21
|
+
* (Claude Code, Cursor, Claude Desktop, …) shares the same memory.
|
|
22
|
+
*
|
|
23
|
+
* Single source of truth for the installer, the stdio bin's fallback, and the
|
|
24
|
+
* Claude Code plugin's hook script — a memory file the writer and the reader
|
|
25
|
+
* disagree about is memory that silently disappears.
|
|
26
|
+
*/
|
|
27
|
+
export const MEMORY_FILE_ENV = "BUILDERFORCE_MEMORY_FILE";
|
|
28
|
+
|
|
29
|
+
export function defaultMemoryFile(homedir: string = nodeOs.homedir()): string {
|
|
30
|
+
return nodePath.join(homedir, ".builderforce-memory", "memory.json");
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/** The snapshot path in effect: explicit env wins, else the shared default. */
|
|
34
|
+
export function resolveMemoryFile(
|
|
35
|
+
env: Record<string, string | undefined> = process.env,
|
|
36
|
+
homedir?: string,
|
|
37
|
+
): string {
|
|
38
|
+
return env[MEMORY_FILE_ENV] || defaultMemoryFile(homedir);
|
|
39
|
+
}
|
|
40
|
+
|
|
16
41
|
export interface ServerSpecOptions {
|
|
17
42
|
/** Absolute path to the JSON snapshot that makes memory survive restarts. */
|
|
18
43
|
memoryFile?: string;
|