@seanhogg/builderforce-memory-mcp 2026.10.2 → 2026.10.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -5
- package/dist/backend.d.ts +26 -0
- package/dist/backend.d.ts.map +1 -1
- package/dist/backends/memory-store.d.ts +50 -2
- package/dist/backends/memory-store.d.ts.map +1 -1
- package/dist/backends/memory-store.js +88 -10
- package/dist/backends/memory-store.js.map +1 -1
- package/dist/bin/http.js +7 -1
- package/dist/bin/http.js.map +1 -1
- package/dist/bin/install.js +6 -3
- package/dist/bin/install.js.map +1 -1
- package/dist/bin/stdio.d.ts +9 -4
- package/dist/bin/stdio.d.ts.map +1 -1
- package/dist/bin/stdio.js +20 -5
- package/dist/bin/stdio.js.map +1 -1
- package/dist/compaction-tools.d.ts +15 -0
- package/dist/compaction-tools.d.ts.map +1 -0
- package/dist/compaction-tools.js +192 -0
- package/dist/compaction-tools.js.map +1 -0
- package/dist/compaction.d.ts +114 -8
- package/dist/compaction.d.ts.map +1 -1
- package/dist/compaction.js +161 -7
- package/dist/compaction.js.map +1 -1
- package/dist/experience/file-store.d.ts +2 -1
- package/dist/experience/file-store.d.ts.map +1 -1
- package/dist/experience/file-store.js +4 -0
- package/dist/experience/file-store.js.map +1 -1
- package/dist/experience/train.d.ts +13 -4
- package/dist/experience/train.d.ts.map +1 -1
- package/dist/experience/train.js +14 -5
- package/dist/experience/train.js.map +1 -1
- package/dist/index.d.ts +13 -6
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +11 -7
- package/dist/index.js.map +1 -1
- package/dist/install/claude-hooks.d.ts +5 -0
- package/dist/install/claude-hooks.d.ts.map +1 -1
- package/dist/install/claude-hooks.js +69 -9
- package/dist/install/claude-hooks.js.map +1 -1
- package/dist/install/server-spec.d.ts +33 -7
- package/dist/install/server-spec.d.ts.map +1 -1
- package/dist/install/server-spec.js +59 -9
- package/dist/install/server-spec.js.map +1 -1
- package/dist/model/recall-probe.d.ts +29 -0
- package/dist/model/recall-probe.d.ts.map +1 -0
- package/dist/model/recall-probe.js +72 -0
- package/dist/model/recall-probe.js.map +1 -0
- package/dist/persistence/fact-archive.d.ts +47 -0
- package/dist/persistence/fact-archive.d.ts.map +1 -0
- package/dist/persistence/fact-archive.js +85 -0
- package/dist/persistence/fact-archive.js.map +1 -0
- package/dist/persistence/shared-json-file.d.ts +24 -1
- package/dist/persistence/shared-json-file.d.ts.map +1 -1
- package/dist/persistence/shared-json-file.js +46 -1
- package/dist/persistence/shared-json-file.js.map +1 -1
- package/dist/restore.d.ts +56 -0
- package/dist/restore.d.ts.map +1 -0
- package/dist/restore.js +118 -0
- package/dist/restore.js.map +1 -0
- package/dist/tools.d.ts +13 -0
- package/dist/tools.d.ts.map +1 -1
- package/dist/tools.js +11 -90
- package/dist/tools.js.map +1 -1
- package/package.json +6 -6
- package/server.json +2 -2
- package/src/backend.ts +29 -0
- package/src/backends/memory-store.ts +141 -11
- package/src/bin/http.ts +7 -1
- package/src/bin/install.ts +7 -3
- package/src/bin/stdio.ts +20 -5
- package/src/compaction-tools.ts +206 -0
- package/src/compaction.ts +228 -9
- package/src/experience/file-store.ts +5 -0
- package/src/experience/train.ts +26 -9
- package/src/index.ts +55 -10
- package/src/install/claude-hooks.ts +72 -9
- package/src/install/server-spec.ts +74 -8
- package/src/model/recall-probe.ts +81 -0
- package/src/persistence/fact-archive.ts +101 -0
- package/src/persistence/shared-json-file.ts +54 -1
- package/src/restore.ts +144 -0
- package/src/tools.ts +24 -93
package/src/index.ts
CHANGED
|
@@ -18,8 +18,12 @@ export { createRecallEmbedder, createEvermindEmbedder, CachedTextEmbedder, defau
|
|
|
18
18
|
export type { EvermindEmbedderOptions, PersistentTextEmbedder, TextEmbedderLike, RecallEmbedderEnv } from "./embedding/index.js";
|
|
19
19
|
|
|
20
20
|
// ── Local backend (IndexedDB via @seanhogg/builderforce-memory) ────────────────────────
|
|
21
|
-
export { MemoryStoreBackend, createLocalMemoryStoreBackend } from "./backends/memory-store.js";
|
|
22
|
-
export type { LocalBackendOptions } from "./backends/memory-store.js";
|
|
21
|
+
export { MemoryStoreBackend, createLocalMemoryStoreBackend, DEFAULT_RECALL_RANKING } from "./backends/memory-store.js";
|
|
22
|
+
export type { LocalBackendOptions, MemoryStoreBackendOptions, RecallRanking } from "./backends/memory-store.js";
|
|
23
|
+
export { FileFactArchive, MemoryFactArchive } from "./persistence/fact-archive.js";
|
|
24
|
+
export type { FactArchive } from "./persistence/fact-archive.js";
|
|
25
|
+
export { SharedJsonFile, DEFAULT_BACKUPS, backupFile } from "./persistence/shared-json-file.js";
|
|
26
|
+
export type { SharedJsonFileOptions } from "./persistence/shared-json-file.js";
|
|
23
27
|
|
|
24
28
|
// ── Tool core ─────────────────────────────────────────────────────────────────
|
|
25
29
|
export { buildMemoryTools } from "./tools.js";
|
|
@@ -32,12 +36,42 @@ export { createExperienceHost, EXPERIENCE_ENV } from "./experience/host.js";
|
|
|
32
36
|
export type { ExperienceHost, ExperienceHostOptions, ExperienceEngine } from "./experience/host.js";
|
|
33
37
|
export { nextExperienceVersion } from "./experience/train.js";
|
|
34
38
|
|
|
35
|
-
// ── Compaction (absorbed fact → one-line pointer stub)
|
|
36
|
-
// Exported so an EXTERNAL compactor (the BuilderForce VS Code extension
|
|
37
|
-
//
|
|
38
|
-
// paths idempotent with respect to each other.
|
|
39
|
-
export {
|
|
40
|
-
|
|
39
|
+
// ── Compaction (absorbed fact → one-line pointer stub, LOSSLESS) ─────────────
|
|
40
|
+
// Exported so an EXTERNAL compactor (the BuilderForce VS Code extension) produces
|
|
41
|
+
// byte-identical stubs AND follows the same archive + recall-probe rules, keeping
|
|
42
|
+
// the two paths idempotent with respect to each other.
|
|
43
|
+
export {
|
|
44
|
+
STUB_PREFIX,
|
|
45
|
+
DEFAULT_STUB_CHARS,
|
|
46
|
+
CONTENT_LOST_LABEL,
|
|
47
|
+
isStub,
|
|
48
|
+
isContentLost,
|
|
49
|
+
firstLine,
|
|
50
|
+
memoryStub,
|
|
51
|
+
stubPointer,
|
|
52
|
+
planCompaction,
|
|
53
|
+
planLosslessCompaction,
|
|
54
|
+
probeQuestionFor,
|
|
55
|
+
probePasses,
|
|
56
|
+
defaultArchiveFile,
|
|
57
|
+
parseArchive,
|
|
58
|
+
mergeArchive,
|
|
59
|
+
} from "./compaction.js";
|
|
60
|
+
export type {
|
|
61
|
+
ArchivedFact,
|
|
62
|
+
ArchiveSnapshot,
|
|
63
|
+
CompactionCandidate,
|
|
64
|
+
CompactionWrite,
|
|
65
|
+
CompactionSkip,
|
|
66
|
+
CompactionPlan,
|
|
67
|
+
CompactionOptions,
|
|
68
|
+
LosslessCompactionOptions,
|
|
69
|
+
ProbeQuestion,
|
|
70
|
+
RecallProbe,
|
|
71
|
+
} from "./compaction.js";
|
|
72
|
+
export { createRecallProbe, lazyRecallProbe, generatorProbe, PROBE_MAX_NEW_TOKENS } from "./model/recall-probe.js";
|
|
73
|
+
export { parseRestoreSource, planRestore, restoredArchive, stubVersion, MIN_POINTER_CHARS } from "./restore.js";
|
|
74
|
+
export type { RestoreRecord, RestoreTarget, RestoreMatch, RestorePlan } from "./restore.js";
|
|
41
75
|
|
|
42
76
|
// ── Transports ──────────────────────────────────────────────────────────────
|
|
43
77
|
export { createMemoryMcpServer } from "./transports/sdk.js";
|
|
@@ -53,8 +87,19 @@ export type { HttpHandlerOptions } from "./transports/http.js";
|
|
|
53
87
|
export { hashToken, timingSafeEqualStr, buildTenantIndex, bearerToken, RateLimiter } from "./transports/auth.js";
|
|
54
88
|
|
|
55
89
|
// ── Multi-host installer (wire the stdio server into any MCP-capable agent) ───
|
|
56
|
-
export {
|
|
57
|
-
|
|
90
|
+
export {
|
|
91
|
+
buildServerSpec,
|
|
92
|
+
defaultMemoryFile,
|
|
93
|
+
memoryFileCandidates,
|
|
94
|
+
resolveMemoryFile,
|
|
95
|
+
CANONICAL_MEMORY_SEGMENTS,
|
|
96
|
+
LEGACY_MEMORY_SEGMENTS,
|
|
97
|
+
MCP_PACKAGE,
|
|
98
|
+
MCP_BIN,
|
|
99
|
+
MEMORY_FILE_ENV,
|
|
100
|
+
RUNTIME_PEERS,
|
|
101
|
+
} from "./install/server-spec.js";
|
|
102
|
+
export type { StdioServerSpec, ServerSpecOptions, MemoryFileProbe, ResolveMemoryFileOptions } from "./install/server-spec.js";
|
|
58
103
|
export { installMemoryServer } from "./install/install.js";
|
|
59
104
|
export type { InstallOptions, InstallResult, InstallStatus, HostSelector, FsLike } from "./install/install.js";
|
|
60
105
|
export { HOSTS, SERVER_KEY, findHost } from "./install/hosts.js";
|
|
@@ -21,8 +21,33 @@
|
|
|
21
21
|
*/
|
|
22
22
|
|
|
23
23
|
import nodePath from "node:path";
|
|
24
|
+
import { CONTENT_LOST_LABEL, STUB_PREFIX } from "../compaction.js";
|
|
24
25
|
import type { FsLike } from "./install.js";
|
|
25
|
-
import { MEMORY_FILE_ENV } from "./server-spec.js";
|
|
26
|
+
import { CANONICAL_MEMORY_SEGMENTS, LEGACY_MEMORY_SEGMENTS, MEMORY_FILE_ENV } from "./server-spec.js";
|
|
27
|
+
|
|
28
|
+
/** `path.join(os.homedir(), "<seg>", …)` as hook-script source. */
|
|
29
|
+
const homeJoin = (segments: readonly string[]) =>
|
|
30
|
+
`path.join(os.homedir(), ${segments.map((s) => JSON.stringify(s)).join(", ")})`;
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* The run-time twin of `resolveMemoryFile`, as hook-script source: the env
|
|
34
|
+
* override, else whichever candidate holds a memory.json (the most recently
|
|
35
|
+
* written when both do), else the canonical default. Built from the SAME segment
|
|
36
|
+
* constants, so the two cannot name different files.
|
|
37
|
+
*/
|
|
38
|
+
function runtimeMemoryFileExpr(): string {
|
|
39
|
+
return `(() => {
|
|
40
|
+
const explicit = process.env[${JSON.stringify(MEMORY_FILE_ENV)}];
|
|
41
|
+
if (explicit) return explicit;
|
|
42
|
+
const candidates = [${homeJoin(CANONICAL_MEMORY_SEGMENTS)}, ${homeJoin(LEGACY_MEMORY_SEGMENTS)}];
|
|
43
|
+
let best = null, bestMtime = -Infinity;
|
|
44
|
+
for (const c of candidates) {
|
|
45
|
+
try { const st = fs.statSync(c); if (st.isFile() && st.mtimeMs > bestMtime) { best = c; bestMtime = st.mtimeMs; } }
|
|
46
|
+
catch { /* absent */ }
|
|
47
|
+
}
|
|
48
|
+
return best || candidates[0];
|
|
49
|
+
})()`;
|
|
50
|
+
}
|
|
26
51
|
|
|
27
52
|
/**
|
|
28
53
|
* The single hook script, four modes.
|
|
@@ -32,11 +57,14 @@ import { MEMORY_FILE_ENV } from "./server-spec.js";
|
|
|
32
57
|
* Code plugin, whose files are static and machine-independent. The runtime
|
|
33
58
|
* branch must stay in lockstep with `resolveMemoryFile`; `install.test.mjs`
|
|
34
59
|
* asserts it does.
|
|
60
|
+
*
|
|
61
|
+
* Compaction stubs whose original was never archived (`contentLost`) are still
|
|
62
|
+
* injected — the pointer is a real lead — but labelled
|
|
63
|
+
* {@link CONTENT_LOST_LABEL} and importance-demoted, so they rank below intact
|
|
64
|
+
* memories and are never presented as verified fact.
|
|
35
65
|
*/
|
|
36
66
|
export function bfmemHookSource(memoryFile: string | null): string {
|
|
37
|
-
const fileExpr = memoryFile === null
|
|
38
|
-
? `process.env[${JSON.stringify(MEMORY_FILE_ENV)}] || path.join(os.homedir(), ".builderforce-memory", "memory.json")`
|
|
39
|
-
: JSON.stringify(memoryFile);
|
|
67
|
+
const fileExpr = memoryFile === null ? runtimeMemoryFileExpr() : JSON.stringify(memoryFile);
|
|
40
68
|
return `#!/usr/bin/env node
|
|
41
69
|
/**
|
|
42
70
|
* builderforce-memory hook — generated by builderforce-memory-install. Dependency-free:
|
|
@@ -61,6 +89,29 @@ function loadEntries() {
|
|
|
61
89
|
try { const p = JSON.parse(fs.readFileSync(FILE, "utf8")); return Array.isArray(p) ? p : []; }
|
|
62
90
|
catch { return []; }
|
|
63
91
|
}
|
|
92
|
+
|
|
93
|
+
// ── Compaction stubs ─────────────────────────────────────────────────────────
|
|
94
|
+
// A compacted memory's full original lives in the archive beside the snapshot
|
|
95
|
+
// (memory.json → memory.archive.json). A stub with NO archived original is lost
|
|
96
|
+
// content: still injected (the pointer is a real lead) but labelled and demoted.
|
|
97
|
+
const ARCHIVE = FILE.replace(/\\.json$/i, "") + ".archive.json";
|
|
98
|
+
const STUB_PREFIX = ${JSON.stringify(STUB_PREFIX)};
|
|
99
|
+
const LOST_LABEL = ${JSON.stringify(CONTENT_LOST_LABEL)};
|
|
100
|
+
const LOST_DEMOTION = 0.5;
|
|
101
|
+
let archivedKeys = null;
|
|
102
|
+
function archived() {
|
|
103
|
+
if (archivedKeys) return archivedKeys;
|
|
104
|
+
try { const a = JSON.parse(fs.readFileSync(ARCHIVE, "utf8")); archivedKeys = new Set(Object.keys((a && a.facts) || {})); }
|
|
105
|
+
catch { archivedKeys = new Set(); }
|
|
106
|
+
return archivedKeys;
|
|
107
|
+
}
|
|
108
|
+
function isLost(e) {
|
|
109
|
+
// The archive decides (a snapshot's contentLost flag can trail a restore).
|
|
110
|
+
return String(e.content || "").trimStart().startsWith(STUB_PREFIX) && !archived().has(e.key);
|
|
111
|
+
}
|
|
112
|
+
function render(e, max) {
|
|
113
|
+
return "• " + e.key + ": " + String(e.content || "").slice(0, max) + (isLost(e) ? " " + LOST_LABEL : "");
|
|
114
|
+
}
|
|
64
115
|
function tokens(s) {
|
|
65
116
|
return String(s || "").toLowerCase().split(/[^a-z0-9]+/).filter((t) => t.length >= 3);
|
|
66
117
|
}
|
|
@@ -111,16 +162,17 @@ if (mode === "--recall") {
|
|
|
111
162
|
const hay = new Set(tokens([e.key, e.content, (e.tags || []).join(" ")].join(" ")));
|
|
112
163
|
let score = 0;
|
|
113
164
|
for (const t of q) if (hay.has(t)) score++;
|
|
114
|
-
|
|
165
|
+
// Lost content still qualifies on overlap, but ranks below intact memories.
|
|
166
|
+
return { e, score, rank: isLost(e) ? score * LOST_DEMOTION : score };
|
|
115
167
|
})
|
|
116
168
|
.filter((x) => x.score >= 2)
|
|
117
169
|
.filter((x) => !led.has(x.e.key)) // already surfaced this session → don't re-inject
|
|
118
|
-
.sort((a, b) => b.
|
|
170
|
+
.sort((a, b) => b.rank - a.rank)
|
|
119
171
|
.slice(0, 3);
|
|
120
172
|
if (scored.length === 0) process.exit(0);
|
|
121
173
|
for (const x of scored) led.add(x.e.key);
|
|
122
174
|
saveLedger(sid, led);
|
|
123
|
-
const lines = scored.map((x) =>
|
|
175
|
+
const lines = scored.map((x) => render(x.e, 200)).join("\\n");
|
|
124
176
|
process.stdout.write(
|
|
125
177
|
"[builderforce-memory] Relevant stored memories for this request — use these instead of re-deriving or re-reading:\\n" + lines,
|
|
126
178
|
);
|
|
@@ -210,7 +262,11 @@ if (entries.length === 0) {
|
|
|
210
262
|
process.exit(0);
|
|
211
263
|
}
|
|
212
264
|
const ranked = entries
|
|
213
|
-
.map((e) =>
|
|
265
|
+
.map((e) => {
|
|
266
|
+
const importance = typeof e.importance === "number" ? e.importance : 0.5;
|
|
267
|
+
// Lost content is importance-demoted so intact memories lead the digest.
|
|
268
|
+
return { key: e.key, content: String(e.content ?? ""), importance: isLost(e) ? importance * LOST_DEMOTION : importance };
|
|
269
|
+
})
|
|
214
270
|
.sort((a, b) => b.importance - a.importance)
|
|
215
271
|
.slice(0, 6);
|
|
216
272
|
|
|
@@ -221,7 +277,7 @@ for (const e of ranked) led.add(e.key);
|
|
|
221
277
|
saveLedger(sessionId, led);
|
|
222
278
|
|
|
223
279
|
if (src === "resume") process.exit(0); // context retained across resume — nothing to re-inject
|
|
224
|
-
const top = ranked.map((e) =>
|
|
280
|
+
const top = ranked.map((e) => render(e, 180)).join("\\n");
|
|
225
281
|
process.stdout.write(
|
|
226
282
|
"[builderforce-memory] Active — token-saving memory loaded. Before re-reading files or history, call memory_recall. Store durable facts with memory_remember.\\nTop memories:\\n" + top,
|
|
227
283
|
);
|
|
@@ -249,6 +305,13 @@ capture that won't let a durable correction go unrecorded.
|
|
|
249
305
|
- \`memory_recall_by_tag(tag)\` — everything under a tag.
|
|
250
306
|
- \`memory_remember(key, content, [tags], [importance])\` — store ONE durable fact.
|
|
251
307
|
- \`memory_forget(key)\` — drop a superseded fact.
|
|
308
|
+
- \`memory_compact(keys)\` — stub facts the local Evermind has absorbed (proven by a
|
|
309
|
+
recall probe; the original is archived, so recall still returns it).
|
|
310
|
+
- \`memory_restore(file|json)\` — bring back the text of memories marked
|
|
311
|
+
"${CONTENT_LOST_LABEL}" from an export.
|
|
312
|
+
|
|
313
|
+
A memory labelled "${CONTENT_LOST_LABEL}" is a one-line pointer whose full
|
|
314
|
+
text is gone: treat it as a lead, verify it before acting on it.
|
|
252
315
|
|
|
253
316
|
## Discipline
|
|
254
317
|
Reuse a stable key to REPLACE a fact (write-through), don't append duplicates.
|
|
@@ -7,6 +7,7 @@
|
|
|
7
7
|
* server is launched, so the per-host adapters never re-derive the command.
|
|
8
8
|
*/
|
|
9
9
|
|
|
10
|
+
import nodeFs from "node:fs";
|
|
10
11
|
import nodeOs from "node:os";
|
|
11
12
|
import nodePath from "node:path";
|
|
12
13
|
|
|
@@ -17,25 +18,90 @@ export interface StdioServerSpec {
|
|
|
17
18
|
}
|
|
18
19
|
|
|
19
20
|
/**
|
|
20
|
-
* The
|
|
21
|
-
* (Claude Code, Cursor, Claude Desktop, …) shares the same memory.
|
|
21
|
+
* The snapshot path override every reader and writer honours first.
|
|
22
22
|
*
|
|
23
|
-
*
|
|
24
|
-
* Claude Code
|
|
25
|
-
*
|
|
23
|
+
* ONE resolver ({@link resolveMemoryFile}) serves the installer, both server bins
|
|
24
|
+
* and the Claude Code hook script (which inlines the same rules — see
|
|
25
|
+
* `bfmemHookSource`) — a memory file the writer and the reader disagree about is
|
|
26
|
+
* memory that silently disappears.
|
|
26
27
|
*/
|
|
27
28
|
export const MEMORY_FILE_ENV = "BUILDERFORCE_MEMORY_FILE";
|
|
28
29
|
|
|
30
|
+
/** The canonical per-machine store, relative to the home directory. */
|
|
31
|
+
export const CANONICAL_MEMORY_SEGMENTS = [".builderforce-memory", "memory.json"] as const;
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* Where earlier installers put the store (`~/.claude/builderforce-memory/memory.json`).
|
|
35
|
+
* Still honoured: a machine whose live memory sits here keeps using it.
|
|
36
|
+
*/
|
|
37
|
+
export const LEGACY_MEMORY_SEGMENTS = [".claude", "builderforce-memory", "memory.json"] as const;
|
|
38
|
+
|
|
39
|
+
/** The default JSON snapshot path: ONE store per machine, shared by every MCP host. */
|
|
29
40
|
export function defaultMemoryFile(homedir: string = nodeOs.homedir()): string {
|
|
30
|
-
return nodePath.join(homedir,
|
|
41
|
+
return nodePath.join(homedir, ...CANONICAL_MEMORY_SEGMENTS);
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/** Every location a store may live in, canonical first. */
|
|
45
|
+
export function memoryFileCandidates(homedir: string = nodeOs.homedir()): string[] {
|
|
46
|
+
return [defaultMemoryFile(homedir), nodePath.join(homedir, ...LEGACY_MEMORY_SEGMENTS)];
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/** File probe the resolver uses — injectable for tests. */
|
|
50
|
+
export interface MemoryFileProbe {
|
|
51
|
+
/** Unix-ms modification time of `path` when it is a file, else null. */
|
|
52
|
+
fileMtime(path: string): number | null;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
const nodeProbe: MemoryFileProbe = {
|
|
56
|
+
fileMtime(path) {
|
|
57
|
+
try {
|
|
58
|
+
const st = nodeFs.statSync(path);
|
|
59
|
+
return st.isFile() ? st.mtimeMs : null;
|
|
60
|
+
} catch {
|
|
61
|
+
return null;
|
|
62
|
+
}
|
|
63
|
+
},
|
|
64
|
+
};
|
|
65
|
+
|
|
66
|
+
export interface ResolveMemoryFileOptions {
|
|
67
|
+
probe?: MemoryFileProbe;
|
|
68
|
+
/** Told which store won when more than one location is in play. Default: silent. */
|
|
69
|
+
log?: (line: string) => void;
|
|
31
70
|
}
|
|
32
71
|
|
|
33
|
-
/**
|
|
72
|
+
/**
|
|
73
|
+
* The snapshot path in effect:
|
|
74
|
+
* 1. `BUILDERFORCE_MEMORY_FILE`, when set;
|
|
75
|
+
* 2. else whichever candidate location actually holds a `memory.json` — the
|
|
76
|
+
* most recently written one if both do (logged; neither is ever deleted);
|
|
77
|
+
* 3. else the canonical default.
|
|
78
|
+
*/
|
|
34
79
|
export function resolveMemoryFile(
|
|
35
80
|
env: Record<string, string | undefined> = process.env,
|
|
36
81
|
homedir?: string,
|
|
82
|
+
opts: ResolveMemoryFileOptions = {},
|
|
37
83
|
): string {
|
|
38
|
-
|
|
84
|
+
const explicit = env[MEMORY_FILE_ENV];
|
|
85
|
+
if (explicit) return explicit;
|
|
86
|
+
const probe = opts.probe ?? nodeProbe;
|
|
87
|
+
const candidates = memoryFileCandidates(homedir);
|
|
88
|
+
const present = candidates
|
|
89
|
+
.map((path) => ({ path, mtime: probe.fileMtime(path) }))
|
|
90
|
+
.filter((c): c is { path: string; mtime: number } => c.mtime !== null);
|
|
91
|
+
if (present.length === 0) return candidates[0]!;
|
|
92
|
+
if (present.length === 1) {
|
|
93
|
+
if (present[0]!.path !== candidates[0]) {
|
|
94
|
+
opts.log?.(`[builderforce-memory] using the store at ${present[0]!.path} (no memory.json at ${candidates[0]}).`);
|
|
95
|
+
}
|
|
96
|
+
return present[0]!.path;
|
|
97
|
+
}
|
|
98
|
+
const winner = present.reduce((a, b) => (b.mtime > a.mtime ? b : a));
|
|
99
|
+
const others = present.filter((c) => c !== winner).map((c) => c.path);
|
|
100
|
+
opts.log?.(
|
|
101
|
+
`[builderforce-memory] memory.json exists in ${present.length} locations; using ${winner.path} ` +
|
|
102
|
+
`(most recently written). Left untouched: ${others.join(", ")}.`,
|
|
103
|
+
);
|
|
104
|
+
return winner.path;
|
|
39
105
|
}
|
|
40
106
|
|
|
41
107
|
export interface ServerSpecOptions {
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The recall probe compaction runs before it stubs a fact: the person's local
|
|
3
|
+
* Evermind model — the same `.evermind` package recall embeds with and experience
|
|
4
|
+
* training adapts (`BUILDERFORCE_MEMORY_MODEL`) — is asked the question generated
|
|
5
|
+
* from the fact (`probeQuestionFor`), and the fact is stubbed only when the answer
|
|
6
|
+
* carries it. No model configured, or one that cannot generate, means no probe —
|
|
7
|
+
* and no probe means nothing is compacted.
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
import type { RecallProbe } from "../compaction.js";
|
|
11
|
+
import { MODEL_FILE_ENV, TOKENIZER_FILE_ENV } from "../embedding/index.js";
|
|
12
|
+
import { loadEvermindPackage } from "./evermind-package.js";
|
|
13
|
+
|
|
14
|
+
/** Structural view of an `EvermindLM` that can generate text. */
|
|
15
|
+
interface GeneratingLM {
|
|
16
|
+
generateText(prompt: string, codec: unknown, opts: { maxNewTokens: number; temperature?: number }): string;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
/** Tokens the model may spend answering one probe — a cloze answer is a word or two. */
|
|
20
|
+
export const PROBE_MAX_NEW_TOKENS = 16;
|
|
21
|
+
|
|
22
|
+
/** Wrap any text generator as a probe. Greedy, short, never throws. */
|
|
23
|
+
export function generatorProbe(generate: (prompt: string) => string | Promise<string>): RecallProbe {
|
|
24
|
+
return {
|
|
25
|
+
async answer(question: string): Promise<string | null> {
|
|
26
|
+
try {
|
|
27
|
+
const out = await generate(question);
|
|
28
|
+
return typeof out === "string" ? out : null;
|
|
29
|
+
} catch {
|
|
30
|
+
return null;
|
|
31
|
+
}
|
|
32
|
+
},
|
|
33
|
+
};
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export interface RecallProbeEnv {
|
|
37
|
+
env?: Record<string, string | undefined>;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* The probe for this server, or null when no model is configured, the package is
|
|
42
|
+
* not a text LM, or its tokenizer cannot decode (no answer text to check).
|
|
43
|
+
*/
|
|
44
|
+
export async function createRecallProbe(opts: RecallProbeEnv = {}): Promise<RecallProbe | null> {
|
|
45
|
+
const env = opts.env ?? process.env;
|
|
46
|
+
const modelFile = env[MODEL_FILE_ENV];
|
|
47
|
+
if (!modelFile) return null;
|
|
48
|
+
const loaded = await loadEvermindPackage(modelFile, env[TOKENIZER_FILE_ENV] || undefined);
|
|
49
|
+
if (!loaded) return null;
|
|
50
|
+
const codec = loaded.codec as unknown as { decode?: unknown };
|
|
51
|
+
if (typeof codec.decode !== "function") return null;
|
|
52
|
+
let lm: GeneratingLM;
|
|
53
|
+
try {
|
|
54
|
+
lm = loaded.pkg.loadLM() as GeneratingLM;
|
|
55
|
+
} catch {
|
|
56
|
+
return null;
|
|
57
|
+
}
|
|
58
|
+
if (typeof lm?.generateText !== "function") return null;
|
|
59
|
+
return generatorProbe((prompt) =>
|
|
60
|
+
lm.generateText(prompt, loaded.codec, { maxNewTokens: PROBE_MAX_NEW_TOKENS, temperature: 0 }),
|
|
61
|
+
);
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* {@link createRecallProbe}, deferred to the first compaction — a server should not
|
|
66
|
+
* load a checkpoint at boot for a tool most sessions never call. Null when no model
|
|
67
|
+
* is configured; a model that then fails to load answers nothing (every probe fails,
|
|
68
|
+
* so nothing is stubbed).
|
|
69
|
+
*/
|
|
70
|
+
export function lazyRecallProbe(opts: RecallProbeEnv = {}): RecallProbe | null {
|
|
71
|
+
const env = opts.env ?? process.env;
|
|
72
|
+
if (!env[MODEL_FILE_ENV]) return null;
|
|
73
|
+
let probe: Promise<RecallProbe | null> | null = null;
|
|
74
|
+
return {
|
|
75
|
+
async answer(question: string): Promise<string | null> {
|
|
76
|
+
probe ??= createRecallProbe({ env });
|
|
77
|
+
const p = await probe;
|
|
78
|
+
return p ? p.answer(question) : null;
|
|
79
|
+
},
|
|
80
|
+
};
|
|
81
|
+
}
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* FactArchive — where a compacted fact's full original lives once its body is a stub.
|
|
3
|
+
*
|
|
4
|
+
* The archive is deliberately NOT a field on the stored entry: the store snapshot is
|
|
5
|
+
* rebuilt from the store on every write, rewritten in place by other processes, and
|
|
6
|
+
* capped/evicted at {@link DEFAULT_MAX_ENTRIES}; an archived original riding inside
|
|
7
|
+
* it would be lost to any one of those. A separate sidecar (`memory.archive.json`,
|
|
8
|
+
* see `defaultArchiveFile`) is only ever merged into, never pruned, so an evicted or
|
|
9
|
+
* forgotten stub still has its original.
|
|
10
|
+
*
|
|
11
|
+
* Two implementations, one contract: in-memory (a backend with no disk mirror) and
|
|
12
|
+
* file-backed (shared across processes through {@link SharedJsonFile}, with its
|
|
13
|
+
* rotating backups and freshness check).
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
import { mergeArchive, parseArchive, type ArchivedFact } from "../compaction.js";
|
|
17
|
+
import { SharedJsonFile, type SharedFileFs } from "./shared-json-file.js";
|
|
18
|
+
|
|
19
|
+
function isJson(text: string): boolean {
|
|
20
|
+
try {
|
|
21
|
+
JSON.parse(text);
|
|
22
|
+
return true;
|
|
23
|
+
} catch {
|
|
24
|
+
return false;
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
export interface FactArchive {
|
|
29
|
+
/** The archived original for `key`, if one exists. */
|
|
30
|
+
get(key: string): ArchivedFact | undefined;
|
|
31
|
+
/** Every row, read once — for callers resolving many keys in one pass. */
|
|
32
|
+
all(): Readonly<Record<string, ArchivedFact>>;
|
|
33
|
+
/** Add or replace archive rows. Durable before it returns. */
|
|
34
|
+
putMany(facts: ArchivedFact[]): void;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/** Process-local archive — the default for a backend with no disk mirror. */
|
|
38
|
+
export class MemoryFactArchive implements FactArchive {
|
|
39
|
+
private readonly rows: Record<string, ArchivedFact> = Object.create(null) as Record<string, ArchivedFact>;
|
|
40
|
+
|
|
41
|
+
get(key: string): ArchivedFact | undefined {
|
|
42
|
+
return this.rows[key];
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
all(): Readonly<Record<string, ArchivedFact>> {
|
|
46
|
+
return this.rows;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
putMany(facts: ArchivedFact[]): void {
|
|
50
|
+
for (const f of facts) this.rows[f.key] = f;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* Archive mirrored to a JSON file several processes share. Re-reads the file when it
|
|
56
|
+
* changed underneath this process; every write merges into the latest on-disk text,
|
|
57
|
+
* so two writers add to one archive rather than overwriting each other's rows.
|
|
58
|
+
*/
|
|
59
|
+
export class FileFactArchive implements FactArchive {
|
|
60
|
+
private rows: Record<string, ArchivedFact> = {};
|
|
61
|
+
private text: string | null = null;
|
|
62
|
+
private readonly file: SharedJsonFile;
|
|
63
|
+
|
|
64
|
+
constructor(path: string, fs: SharedFileFs) {
|
|
65
|
+
this.file = new SharedJsonFile(path, fs);
|
|
66
|
+
this.refresh();
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
get(key: string): ArchivedFact | undefined {
|
|
70
|
+
this.refresh();
|
|
71
|
+
return this.rows[key];
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
all(): Readonly<Record<string, ArchivedFact>> {
|
|
75
|
+
this.refresh();
|
|
76
|
+
return this.rows;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
putMany(facts: ArchivedFact[]): void {
|
|
80
|
+
if (facts.length === 0) return;
|
|
81
|
+
this.refresh();
|
|
82
|
+
// An archive that exists but will not parse is refused, never overwritten:
|
|
83
|
+
// merging into "{}" would drop every row it holds. The throw reaches the
|
|
84
|
+
// compactor BEFORE any stub is written, so nothing is compacted either.
|
|
85
|
+
if (this.text && this.text.trim() && !isJson(this.text)) {
|
|
86
|
+
throw new Error(`fact archive ${this.file.path} is unreadable; refusing to overwrite it`);
|
|
87
|
+
}
|
|
88
|
+
const next = mergeArchive(this.text, facts);
|
|
89
|
+
this.file.write(next);
|
|
90
|
+
this.text = next;
|
|
91
|
+
this.rows = parseArchive(next);
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
private refresh(): void {
|
|
95
|
+
const changed = this.file.readIfChanged();
|
|
96
|
+
if (!changed) return;
|
|
97
|
+
this.text = changed.text;
|
|
98
|
+
this.rows = parseArchive(changed.text);
|
|
99
|
+
changed.accept();
|
|
100
|
+
}
|
|
101
|
+
}
|
|
@@ -10,6 +10,11 @@
|
|
|
10
10
|
* The loop guard is the stamp recorded immediately AFTER each of our own writes: the
|
|
11
11
|
* next check stats the file, sees the same mtime+size, and returns without reading a
|
|
12
12
|
* byte. A stat per call is the entire steady-state cost.
|
|
13
|
+
*
|
|
14
|
+
* Every overwrite first rotates the previous file into `<file>.bak.1` (shifting
|
|
15
|
+
* `.bak.1`→`.bak.2`→`.bak.3`, the oldest dropping off), so the last
|
|
16
|
+
* {@link DEFAULT_BACKUPS} versions of a shared store are always recoverable — a
|
|
17
|
+
* bad rewrite (from any process) is never the only copy.
|
|
13
18
|
*/
|
|
14
19
|
|
|
15
20
|
export type SharedFileFs = {
|
|
@@ -18,6 +23,8 @@ export type SharedFileFs = {
|
|
|
18
23
|
/** When present, writes land atomically (temp file + rename). */
|
|
19
24
|
renameSync?(from: string, to: string): void;
|
|
20
25
|
unlinkSync?(path: string): void;
|
|
26
|
+
/** Used to snapshot the current file into a backup before an overwrite. */
|
|
27
|
+
copyFileSync?(from: string, to: string): void;
|
|
21
28
|
existsSync(path: string): boolean;
|
|
22
29
|
statSync(path: string): { mtimeMs: number; size: number };
|
|
23
30
|
};
|
|
@@ -55,14 +62,31 @@ export interface ChangedFile {
|
|
|
55
62
|
accept(): void;
|
|
56
63
|
}
|
|
57
64
|
|
|
65
|
+
/** Rotating backups kept beside a shared file by default (`.bak.1` newest … `.bak.3`). */
|
|
66
|
+
export const DEFAULT_BACKUPS = 3;
|
|
67
|
+
|
|
68
|
+
/** The path of the `n`-th backup of `file` (1 = newest). */
|
|
69
|
+
export function backupFile(file: string, n: number): string {
|
|
70
|
+
return `${file}.bak.${n}`;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
export interface SharedJsonFileOptions {
|
|
74
|
+
/** Rotating backups to keep before each overwrite. Default {@link DEFAULT_BACKUPS}; 0 disables. */
|
|
75
|
+
backups?: number;
|
|
76
|
+
}
|
|
77
|
+
|
|
58
78
|
export class SharedJsonFile {
|
|
59
79
|
/** How the file looked when this process last wrote or absorbed it. */
|
|
60
80
|
private stamp: FileStamp | null = null;
|
|
81
|
+
private readonly backups: number;
|
|
61
82
|
|
|
62
83
|
constructor(
|
|
63
84
|
readonly path: string,
|
|
64
85
|
private readonly fs: SharedFileFs,
|
|
65
|
-
|
|
86
|
+
opts: SharedJsonFileOptions = {},
|
|
87
|
+
) {
|
|
88
|
+
this.backups = Math.max(0, Math.floor(opts.backups ?? DEFAULT_BACKUPS));
|
|
89
|
+
}
|
|
66
90
|
|
|
67
91
|
/**
|
|
68
92
|
* The file's text when it changed since we last touched it — and on first use,
|
|
@@ -104,6 +128,7 @@ export class SharedJsonFile {
|
|
|
104
128
|
* file open), the write falls back to writing in place.
|
|
105
129
|
*/
|
|
106
130
|
write(text: string): void {
|
|
131
|
+
this.rotateBackups();
|
|
107
132
|
const rename = this.fs.renameSync?.bind(this.fs);
|
|
108
133
|
let written = false;
|
|
109
134
|
if (rename) {
|
|
@@ -128,4 +153,32 @@ export class SharedJsonFile {
|
|
|
128
153
|
this.stamp = null;
|
|
129
154
|
}
|
|
130
155
|
}
|
|
156
|
+
|
|
157
|
+
/**
|
|
158
|
+
* Shift `.bak.(n-1)` → `.bak.n` down to `.bak.1`, then copy the current file to
|
|
159
|
+
* `.bak.1`. Best effort: a backup that cannot be written never blocks the write
|
|
160
|
+
* it protects.
|
|
161
|
+
*/
|
|
162
|
+
private rotateBackups(): void {
|
|
163
|
+
if (this.backups === 0 || !this.fs.existsSync(this.path)) return;
|
|
164
|
+
try {
|
|
165
|
+
for (let i = this.backups; i > 1; i--) {
|
|
166
|
+
const from = backupFile(this.path, i - 1);
|
|
167
|
+
if (this.fs.existsSync(from)) this.moveFile(from, backupFile(this.path, i));
|
|
168
|
+
}
|
|
169
|
+
const newest = backupFile(this.path, 1);
|
|
170
|
+
if (this.fs.copyFileSync) this.fs.copyFileSync(this.path, newest);
|
|
171
|
+
else this.fs.writeFileSync(newest, this.fs.readFileSync(this.path, "utf8"));
|
|
172
|
+
} catch {
|
|
173
|
+
/* best effort */
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
private moveFile(from: string, to: string): void {
|
|
178
|
+
if (this.fs.renameSync) {
|
|
179
|
+
this.fs.renameSync(from, to);
|
|
180
|
+
return;
|
|
181
|
+
}
|
|
182
|
+
this.fs.writeFileSync(to, this.fs.readFileSync(from, "utf8"));
|
|
183
|
+
}
|
|
131
184
|
}
|