@appsoftwareltd/etherpk-mcp 0.4.0 → 0.4.2

package/README.md

@@ -97,6 +97,10 @@ model files into `~/.cache/etherpk/mcp/models/all-MiniLM-L6-v2-int8/`.
   after that only edits are processed. `semantic status` lists each cached graph with how many
   passages are done; every semantic result says `embedded`/`total` and `complete`; run by
   hand, `serve` prints a line every 30 seconds. `semantic remove` deletes runtime and model.
+- **Load on the machine.** The first pass uses a quarter of the cores, at most four, and pauses
+  between model calls; `ETHERPK_MCP_SEMANTIC_THREADS=8` in the registration's `env` makes it
+  faster and hotter. `ETHERPK_MCP_DEBUG_MEMORY=1` logs the process's memory if you ever need
+  to see it.
 - **One cache directory for both.** Setup and `serve` must see the same cache root - by default
   `~/.cache/etherpk/mcp` (`C:\Users\<you>\.cache\etherpk\mcp` on Windows). If you set
   `ETHERPK_MCP_CACHE_DIR` in your shell, put it in the agent registration's `env` too, since

package/dist/main.js

@@ -25,7 +25,7 @@ import { parse, stringify } from "yaml";
 import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
 var package_default = {
 	name: "@appsoftwareltd/etherpk-mcp",
-	version: "0.4.0",
+	version: "0.4.2",
 	license: "Elastic-2.0",
 	description: "EtherPK Headless Client: an MCP server over a synced knowledge graph, run beside the agent on the user's own machine.",
 	type: "module",
@@ -741,8 +741,11 @@ function normaliseSyncServer(value) {
 }
 /**
 * Tolerant of a hand-edited file: keeps the entries it knows, refuses the rest. A file in the
-* single-login shape written before 0.3.0 reads as null, so its commands say "not logged in"
-* and one `login` rewrites it; the token in it was never lost, only its shape.
+* single-login shape written before 0.3.0 - `{ syncServer, pat, vaultKey }` at the top level -
+* is read as that one login under its server: an agent registered under 0.2.0 must keep
+* working when npx pulls a newer version, and refusing the file showed the user only
+* "Connection closed" in Claude Code (2026-09-17). The next `login` or `logout` rewrites it in
+* the current shape.
 */
 function parseConfig(raw) {
 	let parsed;
@@ -752,6 +755,14 @@ function parseConfig(raw) {
 		return null;
 	}
 	if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) return null;
+	const legacy = parsed;
+	if (typeof legacy.syncServer === "string" && typeof legacy.pat === "string" && legacy.pat !== "" && !("servers" in legacy)) {
+		const syncServer = normaliseSyncServer(legacy.syncServer);
+		if (!/^https?:\/\//.test(syncServer)) return null;
+		const login = { pat: legacy.pat };
+		if (typeof legacy.vaultKey === "string" && legacy.vaultKey !== "") login.vaultKey = legacy.vaultKey;
+		return { servers: { [syncServer]: login } };
+	}
 	const servers = parsed.servers;
 	if (typeof servers !== "object" || servers === null || Array.isArray(servers)) return null;
 	const out = emptyConfig();
@@ -3338,13 +3349,38 @@ async function openCacheDb() {
 	}
 	return db;
 }
+/**
+* Give every typed array in a row its own exact-size buffer.
+*
+* Node's v8 deserialiser hands typed arrays back as zero-copy VIEWS on the buffer it was given
+* - here the whole snapshot file - and IndexedDB's structured clone copies a view's entire
+* underlying buffer, not the view. Restoring 2,474 rows that all pointed into one 11 MB file
+* therefore cloned 11 MB per row: 28 GB resident, the machine in swap, a thermal shutdown
+* (2026-09-17). Capture copies too, so a snapshot never carries more than the bytes it means
+* whatever the store handed back. Plain Uint8Arrays, which is what the engine reads.
+*/
+function withStandaloneBuffers(value) {
+	if (ArrayBuffer.isView(value)) {
+		const view = value;
+		const copy = new Uint8Array(view.byteLength);
+		copy.set(new Uint8Array(view.buffer, view.byteOffset, view.byteLength));
+		return copy;
+	}
+	if (Array.isArray(value)) return value.map(withStandaloneBuffers);
+	if (value !== null && typeof value === "object" && Object.getPrototypeOf(value) === Object.prototype) {
+		const out = {};
+		for (const [key, entry] of Object.entries(value)) out[key] = withStandaloneBuffers(entry);
+		return out;
+	}
+	return value;
+}
 /** Every row of every cache store that belongs to `graphId`, as one serialisable snapshot. */
 async function captureLocalCache(graphId) {
 	const db = await openCacheDb();
 	try {
 		const tx = db.transaction([...CACHE_STORES], "readonly");
 		const rows = {};
-		for (const store of CACHE_STORES) rows[store] = (await request(tx.objectStore(store).getAll())).filter((row) => row.graphId === graphId);
+		for (const store of CACHE_STORES) rows[store] = (await request(tx.objectStore(store).getAll())).filter((row) => row.graphId === graphId).map(withStandaloneBuffers);
 		await done(tx);
 		return {
 			version: CACHE_DB_VERSION,
@@ -3362,7 +3398,7 @@ async function restoreLocalCache(snapshot) {
 		const tx = db.transaction([...CACHE_STORES], "readwrite");
 		let count = 0;
 		for (const store of CACHE_STORES) for (const row of snapshot.rows[store]) {
-			tx.objectStore(store).put(row);
+			tx.objectStore(store).put(withStandaloneBuffers(row));
 			count++;
 		}
 		await done(tx);
@@ -3409,9 +3445,38 @@ async function loadLocalCache(dir, graphId) {
 		return 0;
 	}
 }
+/** A `.tmp` younger than this may be a snapshot another process is still writing. */
+var ABANDONED_TMP_AFTER_MS = 10 * 6e4;
+/**
+* Remove what an older build or a killed write left in a graph's directory: index and vectors
+* files stamped with another version (a bump renames them, so they are never opened again and
+* were 52 MB of litter per graph), and `.tmp` files from an atomic write that never reached its
+* rename - only once they are old enough that no live `serve` can be writing them. The files
+* this build reads are left alone.
+*/
+async function removeStaleFiles(dir, now = Date.now()) {
+	const keep = new Set([
+		cacheFile(dir),
+		indexFile(dir),
+		vectorsFile(dir)
+	].map((f) => f.split("/").pop()));
+	const removed = [];
+	for (const name of await readdir(dir).catch(() => [])) {
+		const stale = /^(index|vectors)\.v\d+\.sqlite$/.test(name) && !keep.has(name);
+		const temp = /\.\d+\.tmp$/.test(name);
+		if (!stale && !temp) continue;
+		if (temp) {
+			if (now - ((await stat(join(dir, name)).catch(() => null))?.mtimeMs ?? now) < ABANDONED_TMP_AFTER_MS) continue;
+		}
+		await rm(join(dir, name), { force: true });
+		removed.push(name);
+	}
+	return removed;
+}
 /** Hosts opened in this process, so each gets its own paths in sqlite-wasm's filesystem. */
 var hostSerial = 0;
-function nodeIndexHost(dir) {
+function nodeIndexHost(dir, options = {}) {
+	const tidy = options.tidy ?? true;
 	let sqlite3;
 	let live;
 	const path = indexFile(dir);
@@ -3444,7 +3509,7 @@ function nodeIndexHost(dir) {
 			}
 		} catch (error) {
 			console.error(`etherpk-mcp: ignoring an unreadable vectors file ${vectorsPath}: ${error instanceof Error ? error.message : String(error)}`);
-			await rm(vectorsPath, { force: true });
+			if (tidy) await rm(vectorsPath, { force: true });
 			s.capi.sqlite3_js_posix_create_file(vectorsVfsName, new Uint8Array(0));
 		}
 	}
@@ -3471,7 +3536,7 @@ function nodeIndexHost(dir) {
 			const db = wrapOo1Db(oo1);
 			if (!isUsableIndex(db)) {
 				db.close();
-				await rm(path, { force: true });
+				if (tidy) await rm(path, { force: true });
 				return null;
 			}
 			attachVectors(db);
@@ -3481,13 +3546,14 @@ function nodeIndexHost(dir) {
 			};
 		} catch (error) {
 			console.error(`etherpk-mcp: ignoring an unreadable index file ${path}: ${error instanceof Error ? error.message : String(error)}`);
-			await rm(path, { force: true });
+			if (tidy) await rm(path, { force: true });
 			return null;
 		}
 	}
 	return {
 		async open() {
 			const s = await init();
+			if (tidy) await removeStaleFiles(dir);
 			await importVectors(s);
 			live = await openSaved(s) ?? openFresh(s);
 			return {
@@ -3534,7 +3600,7 @@ async function listGraphStores(env, model) {
 		for (const graphId of await readdir(hostDir).catch(() => [])) {
 			const dir = join(hostDir, graphId);
 			if (!await exists$1(indexFile(dir))) continue;
-			const opened = await nodeIndexHost(dir).open(graphId);
+			const opened = await nodeIndexHost(dir, { tidy: false }).open(graphId);
 			try {
 				prepareEmbeddingStore(opened.db);
 				const status = semanticStatus(opened.db, model);
@@ -3553,6 +3619,22 @@ async function listGraphStores(env, model) {
 	}
 	return stores;
 }
+/**
+* A store's state as one line a person can act on: a word first, then the counts, then how
+* fresh they are. A building `serve` refreshes the snapshot every 30 seconds, so a snapshot
+* under a minute and a half old means "building now" and an older one means nothing is - the
+* distinction the raw count and timestamp left the reader to infer (2026-09-17).
+*/
+function describeGraphStore(store, now = Date.now()) {
+	const { embedded, total } = store.status;
+	const ageMs = Math.max(0, now - store.updatedAt.getTime());
+	const age = ageMs < 9e4 ? `${Math.round(ageMs / 1e3)} s ago` : ageMs < 36e5 ? `${Math.round(ageMs / 6e4)} min ago` : `${Math.round(ageMs / 36e5)} h ago`;
+	const count = `${embedded.toLocaleString("en-GB")} of ${total.toLocaleString("en-GB")} passages`;
+	if (total === 0) return `empty - no passages indexed here yet (snapshot ${age})`;
+	if (embedded >= total) return `up to date - ${count} embedded (snapshot ${age})`;
+	const percent = Math.floor(embedded / total * 100);
+	return ageMs < 9e4 ? `building - ${count} (${percent}%) embedded, snapshot ${age}, refreshed every 30 s while it builds` : `paused - ${count} (${percent}%) embedded, last snapshot ${age}; nothing is building it now, it continues when serve next runs`;
+}
 /** Remove every persisted graph under the cache root (logout of the last server). */
 async function removeCacheRoot(env) {
 	await rm(cacheRoot(env), {
@@ -3785,6 +3867,13 @@ var SemanticUnavailable = class extends Error {
 	}
 };
 var SETUP_HINT = "run `npx @appsoftwareltd/etherpk-mcp semantic setup` on this computer once (it installs a ~300 MB runtime and a 23 MB model into the cache directory); the next semantic search will use it, no restart needed.";
+/** The thread count `serve` uses: the option, else the environment, else the conservative default. */
+function semanticThreads(env, requested) {
+	if (requested !== void 0) return Math.max(1, Math.floor(requested));
+	const fromEnv = Number(env.ETHERPK_MCP_SEMANTIC_THREADS);
+	if (Number.isInteger(fromEnv) && fromEnv >= 1) return fromEnv;
+	return Math.max(1, Math.min(4, Math.floor(availableParallelism() / 4)));
+}
 /**
 * Load the runtime from the cache directory and open the model. Resident memory is a few
 * hundred megabytes, so `serve` calls this only when semantic mode is set up on the machine.
@@ -3801,7 +3890,7 @@ async function loadEmbeddingModel(env, options = {}) {
 		throw new SemanticUnavailable(`The embedding runtime in ${status.runtimeDir} failed to load (${error instanceof Error ? error.message : String(error)}); ${SETUP_HINT}`);
 	}
 	const tokenizer = new Tokenizer(JSON.parse(await readFile(join(status.modelDir, "tokenizer.json"), "utf8")), JSON.parse(await readFile(join(status.modelDir, "tokenizer_config.json"), "utf8")));
-	const threads = options.threads ?? Math.max(1, Math.min(8, Math.floor(availableParallelism() / 2)));
+	const threads = semanticThreads(env, options.threads);
 	const session = await ort.InferenceSession.create(join(status.modelDir, "model.onnx"), {
 		intraOpNumThreads: threads,
 		interOpNumThreads: 1,
@@ -7492,6 +7581,7 @@ function createSemanticIndex(options) {
 	const { index, model } = options;
 	const floor = options.floor ?? .25;
 	const settleMs = options.settleMs ?? 1500;
+	const pauseMs = options.pauseMs ?? 100;
 	let disposed = false;
 	let running;
 	let again = false;
@@ -7512,6 +7602,7 @@ function createSemanticIndex(options) {
 					hash: passage.hash,
 					...quantise(vectors[n])
 				}));
+				if (pauseMs > 0) await new Promise((resolve) => setTimeout(resolve, pauseMs));
 			}
 			await index.semantic.put(model.id, model.dims, rows);
 			if (options.onProgress) options.onProgress(await index.semantic.status(model.id));
@@ -9773,11 +9864,8 @@ async function semanticCommand(what) {
 			const stores = await listGraphStores(process.env, MODEL.id);
 			if (stores.length === 0) return;
 			console.log("");
-			console.log("Cached graphs, as of the last snapshot each serve wrote (a running build refreshes it every 30 seconds):");
-			for (const store of stores) {
-				const done = store.status.embedded >= store.status.total;
-				console.log(`  ${store.host}  ${store.graphId}  ${store.status.embedded} of ${store.status.total} passages embedded${done ? " - up to date" : ""}  (${store.updatedAt.toISOString()})`);
-			}
+			console.log("Cached graphs:");
+			for (const store of stores) console.log(`  ${store.host}  ${store.graphId}\n    ${describeGraphStore(store)}`);
 			return;
 		}
 		case "remove":
@@ -9796,8 +9884,21 @@ function reportSemanticProgress(status) {
 	if (!upToDate && Date.now() - lastProgressLog < 3e4) return;
 	lastProgressLog = Date.now();
 	announcedUpToDate = upToDate;
-	console.error(`etherpk-mcp: semantic: ${status.embedded} of ${status.total} passages embedded${upToDate ? " - up to date" : ""}.`);
+	console.error(`etherpk-mcp: semantic: ${status.embedded} of ${status.total} passages embedded${upToDate ? " - up to date" : ""}.${memoryNote()}`);
+}
+/**
+* `ETHERPK_MCP_DEBUG_MEMORY=1` adds the process's memory to every progress line and logs it every
+* ten seconds: the numbers that separate the JavaScript heap, the buffers outside it and the
+* native runtime when a serve grows without reason (2026-09-17).
+*/
+var debugMemory = process.env.ETHERPK_MCP_DEBUG_MEMORY === "1";
+function memoryNote() {
+	if (!debugMemory) return "";
+	const m = process.memoryUsage();
+	const mb = (n) => Math.round(n / 1e6);
+	return ` [rss ${mb(m.rss)} MB, heap ${mb(m.heapUsed)} MB, external ${mb(m.external)} MB, arrayBuffers ${mb(m.arrayBuffers)} MB]`;
 }
+if (debugMemory) setInterval(() => console.error(`etherpk-mcp: memory${memoryNote()}`), 1e4).unref();
 async function serve(args) {
 	const wanted = args.graph?.trim();
 	if (!wanted) fail("serve needs --graph <id or name>.");