@seanhogg/builderforce-memory-mcp 2026.9.27 → 2026.9.29

package/src/model/evermind-package.ts

@@ -0,0 +1,127 @@
+/**
+ * Load a local `.evermind` text model and the tokenizer behind its token ids.
+ *
+ * Two things on a headless host read the person's private model: recall (which
+ * embeds with it) and experience training (which adapts it). They must agree on
+ * which vocabulary a checkpoint speaks, so the resolution lives here once.
+ */
+
+import { dynamicImport } from "../dynamic-import.js";
+
+/** The tokenizer surface both embedding and adaptation need. */
+export interface TextCodec {
+    encode(text: string): number[];
+}
+
+/** Structural view of the engine's package — enough to embed with or adapt. */
+export interface EvermindPackageLike {
+    manifest: { modelType?: string; version?: string; name?: string };
+    checkpoint: ArrayBuffer;
+    loadLM(): unknown;
+    /** Present on engines that support the embedded-tokenizer section. */
+    loadTokenizer?(): TextCodec | null;
+    toBlob(): ArrayBuffer;
+}
+
+/** Minimal structural view of the optional engine package. */
+interface PackageEngine {
+    EvermindModelPackage: {
+        fromBlob(blob: ArrayBuffer): EvermindPackageLike;
+        manifestEnd(header: ArrayBuffer): number;
+        readManifest(prefix: ArrayBuffer): EvermindPackageLike["manifest"];
+    };
+    PACKAGE_HEADER_BYTES: number;
+    BPETokenizer: new () => { loadFromSpec(spec: unknown): void; loadHuggingFace(spec: unknown): void } & TextCodec;
+}
+
+export type ModelFs = {
+    readFileSync(path: string, enc: "utf8"): string;
+    readFileSync(path: string): Uint8Array;
+    writeFileSync(path: string, data: string | Uint8Array): void;
+    existsSync(path: string): boolean;
+    copyFileSync(src: string, dest: string): void;
+    renameSync(from: string, to: string): void;
+};
+
+export interface LoadedEvermind {
+    pkg: EvermindPackageLike;
+    codec: TextCodec;
+    fs: ModelFs;
+}
+
+type PrefixFs = {
+    openSync(path: string, flags: "r"): number;
+    readSync(fd: number, buffer: Uint8Array, offset: number, length: number, position: number): number;
+    closeSync(fd: number): void;
+};
+
+/**
+ * The version in a `.evermind` file's manifest, reading only the header and manifest —
+ * never the checkpoint — so a caller can ask on every poll, and always sees the file
+ * as it is now (a model restored from `.prev` keeps the size and, on Windows, the
+ * modification time of the one it replaces). Null when there is no readable package.
+ */
+export async function readEvermindVersion(modelFile: string): Promise<string | null> {
+    let fd: number | undefined;
+    const fs = (await dynamicImport("node:fs")) as PrefixFs;
+    try {
+        const engine = (await dynamicImport("@seanhogg/builderforce-memory-engine")) as Partial<PackageEngine>;
+        if (!engine?.EvermindModelPackage || !engine.PACKAGE_HEADER_BYTES) return null;
+        fd = fs.openSync(modelFile, "r");
+        const read = (n: number) => {
+            const buf = new Uint8Array(n);
+            const got = fs.readSync(fd!, buf, 0, n, 0);
+            return buf.buffer.slice(0, got) as ArrayBuffer;
+        };
+        const end = engine.EvermindModelPackage.manifestEnd(read(engine.PACKAGE_HEADER_BYTES));
+        return engine.EvermindModelPackage.readManifest(read(end)).version ?? "0";
+    } catch {
+        return null;
+    } finally {
+        if (fd !== undefined) fs.closeSync(fd);
+    }
+}
+
+/**
+ * Load the tokenizer spec from `file`, accepting either `BPETokenizer.toObject()`
+ * output or a Hugging Face `tokenizer.json`.
+ */
+function loadCodec(engine: PackageEngine, fs: ModelFs, file: string): TextCodec {
+    const spec = JSON.parse(fs.readFileSync(file, "utf8")) as { vocab?: unknown; merges?: unknown };
+    const tok = new engine.BPETokenizer();
+    if (spec.vocab && spec.merges) tok.loadFromSpec(spec);
+    else tok.loadHuggingFace(spec);
+    return tok;
+}
+
+/**
+ * The `evermind-lm` package at `modelFile` with its tokenizer, or `null` — never a
+ * throw — when the engine peer is absent, a file is missing, or the package is not a
+ * text LM. Both callers treat a missing model as a degrade, not a fault.
+ *
+ * Tokenizer resolution: the package's OWN tokenizer (checksummed and vocab-checked
+ * at package time, so it cannot be the wrong vocabulary — which a separate file
+ * silently can be), then `tokenizerFile`, then a sibling `<modelFile>.tokenizer.json`.
+ */
+export async function loadEvermindPackage(modelFile: string, tokenizerFile?: string): Promise<LoadedEvermind | null> {
+    try {
+        const engine = (await dynamicImport("@seanhogg/builderforce-memory-engine")) as Partial<PackageEngine>;
+        if (!engine?.EvermindModelPackage || !engine.BPETokenizer) return null;
+
+        const fs = (await dynamicImport("node:fs")) as ModelFs;
+        if (!fs.existsSync(modelFile)) return null;
+
+        const bytes = fs.readFileSync(modelFile);
+        const blob = bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength) as ArrayBuffer;
+        const pkg = engine.EvermindModelPackage.fromBlob(blob);
+        if (pkg.manifest.modelType !== "evermind-lm") return null;
+
+        const embedded = pkg.loadTokenizer?.() ?? null;
+        if (embedded) return { pkg, codec: embedded, fs };
+        const file = tokenizerFile ?? `${modelFile}.tokenizer.json`;
+        if (!fs.existsSync(file)) return null;
+        return { pkg, codec: loadCodec(engine as PackageEngine, fs, file), fs };
+    } catch {
+        return null;
+    }
+}

package/src/persistence/shared-json-file.ts

@@ -0,0 +1,131 @@
+/**
+ * A JSON file several processes share — the durable tier of every local store here.
+ *
+ * Each MCP client spawns its own server, and other tools (the BuilderForce VS Code
+ * extension's compactor, Synapse) rewrite the same files directly. So a store never
+ * trusts its in-memory copy blindly: before each operation it asks
+ * {@link SharedJsonFile.readIfChanged}, which re-reads the file only when it changed
+ * underneath this process.
+ *
+ * The loop guard is the stamp recorded immediately AFTER each of our own writes: the
+ * next check stats the file, sees the same mtime+size, and returns without reading a
+ * byte. A stat per call is the entire steady-state cost.
+ */
+
+export type SharedFileFs = {
+    readFileSync(path: string, enc: "utf8"): string;
+    writeFileSync(path: string, data: string): void;
+    /** When present, writes land atomically (temp file + rename). */
+    renameSync?(from: string, to: string): void;
+    unlinkSync?(path: string): void;
+    existsSync(path: string): boolean;
+    statSync(path: string): { mtimeMs: number; size: number };
+};
+
+/** Identity of the file as this process last left it. */
+interface FileStamp {
+    mtimeMs: number;
+    size: number;
+    /** Digest of the exact bytes — settles the case where a file is touched but unchanged. */
+    hash: string;
+}
+
+/**
+ * FNV-1a over the file text. Non-cryptographic on purpose: this only has to separate
+ * "someone rewrote the file" from "the mtime moved but the bytes are ours", and
+ * node:crypto is deliberately not imported (these modules stay bundleable for the
+ * browser, where the disk path is never taken).
+ */
+export function contentHash(text: string): string {
+    let h = 0x811c9dc5;
+    for (let i = 0; i < text.length; i++) {
+        h ^= text.charCodeAt(i);
+        h = Math.imul(h, 0x01000193);
+    }
+    return (h >>> 0).toString(16);
+}
+
+/** A changed file's text, and the call that marks it as absorbed. */
+export interface ChangedFile {
+    text: string;
+    /**
+     * Call once the text was parsed and applied. Not calling it (a half-written file
+     * that failed to parse) leaves the stamp alone, so the next check reads again.
+     */
+    accept(): void;
+}
+
+export class SharedJsonFile {
+    /** How the file looked when this process last wrote or absorbed it. */
+    private stamp: FileStamp | null = null;
+
+    constructor(
+        readonly path: string,
+        private readonly fs: SharedFileFs,
+    ) {}
+
+    /**
+     * The file's text when it changed since we last touched it — and on first use,
+     * which IS the boot hydration (one routine, so disk and memory never diverge by
+     * taking two paths). Null when unchanged, absent or unreadable.
+     */
+    readIfChanged(): ChangedFile | null {
+        if (!this.fs.existsSync(this.path)) return null;
+        let st: { mtimeMs: number; size: number };
+        try {
+            st = this.fs.statSync(this.path);
+        } catch {
+            return null;
+        }
+        // Fast path: byte-for-byte what we last wrote. No read, no parse.
+        if (this.stamp && st.mtimeMs === this.stamp.mtimeMs && st.size === this.stamp.size) return null;
+        let text: string;
+        try {
+            text = this.fs.readFileSync(this.path, "utf8");
+        } catch {
+            return null;
+        }
+        const hash = contentHash(text);
+        // Touched (mtime moved) but identical content — re-stamp, nothing to absorb.
+        if (this.stamp && hash === this.stamp.hash) {
+            this.stamp = { mtimeMs: st.mtimeMs, size: st.size, hash };
+            return null;
+        }
+        return { text, accept: () => (this.stamp = { mtimeMs: st.mtimeMs, size: st.size, hash }) };
+    }
+
+    /**
+     * Write, and stamp our own write so the next check does not read it back as foreign.
+     *
+     * Atomic where the filesystem allows: the text goes to a sibling temp file that is
+     * then renamed over the target, so a crash mid-write leaves the previous complete
+     * file, never a truncated one (which would read as empty and be overwritten by the
+     * next write). If the rename is refused (Windows, while another process holds the
+     * file open), the write falls back to writing in place.
+     */
+    write(text: string): void {
+        const rename = this.fs.renameSync?.bind(this.fs);
+        let written = false;
+        if (rename) {
+            const tmp = `${this.path}.${Date.now().toString(36)}${Math.random().toString(36).slice(2, 8)}.tmp`;
+            try {
+                this.fs.writeFileSync(tmp, text);
+                rename(tmp, this.path);
+                written = true;
+            } catch {
+                try {
+                    if (this.fs.existsSync(tmp)) this.fs.unlinkSync?.(tmp);
+                } catch {
+                    /* best effort: a stray temp file is harmless */
+                }
+            }
+        }
+        if (!written) this.fs.writeFileSync(this.path, text);
+        try {
+            const st = this.fs.statSync(this.path);
+            this.stamp = { mtimeMs: st.mtimeMs, size: st.size, hash: contentHash(text) };
+        } catch {
+            this.stamp = null;
+        }
+    }
+}

package/src/tool-core.ts

@@ -0,0 +1,39 @@
+/**
+ * The framework-neutral tool shape every tool family here is written in (memory,
+ * gateway cost, experience), and the result helpers they share. Each transport maps
+ * a {@link MemoryTool} 1:1 onto Agent-SDK `tool()` or MCP `registerTool()`.
+ */
+
+import type { z } from "zod";
+
+/** The MCP CallToolResult shape both server frameworks expect. */
+export interface ToolResult {
+    content: Array<{ type: "text"; text: string }>;
+    isError?: boolean;
+}
+
+/** A framework-neutral tool: maps 1:1 onto Agent-SDK `tool()` and MCP `registerTool()`. */
+export interface MemoryTool {
+    name: string;
+    description: string;
+    /** Zod *raw shape* (e.g. `{ query: z.string() }`), not a ZodObject. */
+    inputSchema: z.ZodRawShape;
+    handler: (args: Record<string, unknown>) => Promise<ToolResult>;
+}
+
+export function clip(s: string, n: number): string {
+    return s.length <= n ? s : `${s.slice(0, n)}…`;
+}
+
+export function ok(text: string): ToolResult {
+    return { content: [{ type: "text", text }] };
+}
+
+export function fail(text: string): ToolResult {
+    return { content: [{ type: "text", text }], isError: true };
+}
+
+/** A structured result, as indented JSON — for tools a program reads as well as a model. */
+export function okJson(value: unknown): ToolResult {
+    return ok(JSON.stringify(value, null, 2));
+}

package/src/tools.ts

@@ -17,21 +17,12 @@
 import { z } from "zod";
 import type { MemoryBackend, RankedRecall, RecallHit, RecallMethod, RememberInput } from "./backend.js";
 import { DEFAULT_STUB_CHARS, planCompaction, type CompactionSkip } from "./compaction.js";
+import { buildExperienceTools } from "./experience/tools.js";
+import type { ExperienceHost } from "./experience/host.js";
+import { appendGatewayCostTools } from "./gateway-tools.js";
+import { clip, fail, ok, type MemoryTool, type ToolResult } from "./tool-core.js";
 
-/** The MCP CallToolResult shape both server frameworks expect. */
-export interface ToolResult {
-    content: Array<{ type: "text"; text: string }>;
-    isError?: boolean;
-}
-
-/** A framework-neutral tool: maps 1:1 onto Agent-SDK `tool()` and MCP `registerTool()`. */
-export interface MemoryTool {
-    name: string;
-    description: string;
-    /** Zod *raw shape* (e.g. `{ query: z.string() }`), not a ZodObject. */
-    inputSchema: z.ZodRawShape;
-    handler: (args: Record<string, unknown>) => Promise<ToolResult>;
-}
+export type { MemoryTool, ToolResult } from "./tool-core.js";
 
 export interface MemoryToolsOptions {
     /** Hard cap on entries any recall tool returns to the model. Default 5. */
@@ -47,23 +38,17 @@ export interface MemoryToolsOptions {
     gatewayUrl?: string;
     /** A `bfk_*` tenant API key. Required (with {@link gatewayUrl}) for cost tools. */
     gatewayApiKey?: string;
+    /**
+     * Experience (demonstrations, Train Once skills, runs, training the private model).
+     * When present and the server is writable, the experience tools are registered
+     * beside the memory tools. See {@link createExperienceHost}.
+     */
+    experience?: ExperienceHost | null;
 }
 
 const DEFAULT_MAX_RESULTS = 5;
 const DEFAULT_MAX_CONTENT = 500;
 
-function clip(s: string, n: number): string {
-    return s.length <= n ? s : `${s.slice(0, n)}…`;
-}
-
-function ok(text: string): ToolResult {
-    return { content: [{ type: "text", text }] };
-}
-
-function fail(text: string): ToolResult {
-    return { content: [{ type: "text", text }], isError: true };
-}
-
 /**
  * How the ordering was produced, in one line the model can read.
  *
@@ -338,6 +323,8 @@ export function buildMemoryTools(backend: MemoryBackend, opts: MemoryToolsOption
         });
     }
 
+    if (writable && opts.experience) tools.push(...buildExperienceTools(opts.experience));
+
     // Gateway-backed cost tools — only when BOTH a gateway URL and key are
     // present, so memory-only deployments are unchanged.
     if (opts.gatewayUrl && opts.gatewayApiKey) {
@@ -346,93 +333,3 @@ export function buildMemoryTools(backend: MemoryBackend, opts: MemoryToolsOption
 
     return tools;
 }
-
-/** Snapshot shape returned by GET /llm/v1/builder-insights. */
-interface BuilderInsightsSnapshot {
-    windowLabel?: string;
-    todayTokens?: number;
-    todayCostUsd?: number;
-    dailyCapTokens?: number | null;
-    pctOfDailyCap?: number | null;
-    topModel?: { model: string; tokens: number } | null;
-    costPerMergedPrUsd?: number | null;
-    tip?: string | null;
-}
-
-/** Ranking shape returned by GET /llm/v1/model-analytics. */
-interface ModelAnalytics {
-    byAction?: Array<{
-        actionType?: string;
-        label?: string;
-        models?: Array<{
-            model: string;
-            samples?: number;
-            avgScore?: number;
-            mergeRate?: number;
-            avgCostMillicents?: number;
-        }>;
-    }>;
-}
-
-async function gatewayGet<T>(gatewayUrl: string, path: string, apiKey: string): Promise<T> {
-    const res = await fetch(`${gatewayUrl}${path}`, {
-        headers: { authorization: `Bearer ${apiKey}` },
-    });
-    if (!res.ok) throw new Error(`gateway ${path} → HTTP ${res.status}`);
-    return (await res.json()) as T;
-}
-
-function appendGatewayCostTools(tools: MemoryTool[], gatewayUrl: string, apiKey: string): void {
-    tools.push({
-        name: "token_usage",
-        description: "Current token spend + budget for the workspace (today).",
-        inputSchema: {},
-        handler: async () => {
-            try {
-                const s = await gatewayGet<BuilderInsightsSnapshot>(gatewayUrl, "/llm/v1/builder-insights", apiKey);
-                const lines = [
-                    `Token usage (${s.windowLabel ?? "today"}):`,
-                    `• Tokens: ${(s.todayTokens ?? 0).toLocaleString()}`,
-                    `• Cost: $${(s.todayCostUsd ?? 0).toFixed(2)}`,
-                    `• % of daily cap: ${
-                        s.pctOfDailyCap == null
-                            ? "no cap"
-                            : `${s.pctOfDailyCap}%${s.dailyCapTokens ? ` of ${s.dailyCapTokens.toLocaleString()}` : ""}`
-                    }`,
-                    `• Top model: ${s.topModel ? `${s.topModel.model} (${s.topModel.tokens.toLocaleString()} tok)` : "—"}`,
-                ];
-                if (s.costPerMergedPrUsd != null) lines.push(`• Cost / merged PR: $${s.costPerMergedPrUsd.toFixed(2)}`);
-                if (s.tip) lines.push(`• Tip: ${s.tip}`);
-                return ok(lines.join("\n"));
-            } catch (err) {
-                return fail(`token_usage failed: ${String(err)}`);
-            }
-        },
-    });
-
-    tools.push({
-        name: "model_efficiency",
-        description: "Which models performed best/cheapest for this workspace's recent work.",
-        inputSchema: {},
-        handler: async () => {
-            try {
-                const a = await gatewayGet<ModelAnalytics>(gatewayUrl, "/llm/v1/model-analytics", apiKey);
-                const groups = a.byAction ?? [];
-                if (groups.length === 0) return ok("No model efficiency data yet for this workspace.");
-                const out: string[] = ["Model efficiency by action type (best first):"];
-                for (const g of groups) {
-                    out.push(`\n${g.label ?? g.actionType ?? "action"}:`);
-                    for (const m of (g.models ?? []).slice(0, 3)) {
-                        const cost = m.avgCostMillicents != null ? ` $${(m.avgCostMillicents / 100_000).toFixed(4)}/call` : "";
-                        const merge = m.mergeRate != null ? ` merge=${Math.round(m.mergeRate * 100)}%` : "";
-                        const score = m.avgScore != null ? ` score=${m.avgScore}` : "";
-                        out.push(`  • ${m.model}${score}${merge}${cost} (n=${m.samples ?? 0})`);
-                    }
-                }
-                return ok(out.join("\n"));
-            } catch (err) {
-                return fail(`model_efficiency failed: ${String(err)}`);
-            }
-        },
-    });
-}

package/src/transports/http.ts

@@ -89,6 +89,8 @@ export function createMemoryHttpHandler(
         // ── Resolve the per-request backend + a rate-limit key ────────────────
         let resolved: MemoryBackend | undefined;
         let rateKey: string;
+        // True only when the caller proved the owner's shared secret (see below).
+        let ownerAuthed = false;
 
         if (tenantIndex) {
             // Multi-tenant: token → that tenant's backend (hash-keyed lookup).
@@ -109,6 +111,7 @@ export function createMemoryHttpHandler(
             }
             resolved = backend;
             rateKey = `t:${hashToken(token)}`;
+            ownerAuthed = true;
         } else {
             // Open (trusted gateway): rate-limit by client IP.
             resolved = backend;
@@ -121,7 +124,11 @@ export function createMemoryHttpHandler(
             return;
         }
 
-        const server = buildMcpServer(resolved, opts);
+        // Experience is ONE person's local store (it can delete their recordings and
+        // rewrite their model), so it is served only to a caller holding the owner's
+        // shared secret — never to a token-selected tenant, and never in open mode,
+        // where anyone who can reach the port would get it.
+        const server = buildMcpServer(resolved, ownerAuthed ? opts : { ...opts, experience: null });
         // Stateless: no session id generator → a fresh transport per request.
         const transport = new StreamableHTTPServerTransport({ sessionIdGenerator: undefined });
 

package/src/transports/sdk.ts

@@ -12,6 +12,7 @@
  */
 
 import type { MemoryBackend } from "../backend.js";
+import { dynamicImport } from "../dynamic-import.js";
 import { buildMemoryTools, type MemoryToolsOptions } from "../tools.js";
 
 export interface SdkServerOptions extends MemoryToolsOptions {
@@ -49,11 +50,7 @@ export async function createMemoryMcpServer(
     backend: MemoryBackend,
     opts: SdkServerOptions = {},
 ): Promise<SdkMcpServerConfig> {
-    const _import = (m: string): Promise<unknown> =>
-        // eslint-disable-next-line @typescript-eslint/no-implied-eval, no-new-func
-        new Function("m", "return import(m)")(m) as Promise<unknown>;
-
-    const sdk = (await _import("@anthropic-ai/claude-agent-sdk")) as {
+    const sdk = (await dynamicImport("@anthropic-ai/claude-agent-sdk")) as {
         tool: (
             name: string,
             description: string,