@compr/opscontext-mcp 2.5.8 → 2.6.0

package/CHANGELOG.md

@@ -4,6 +4,58 @@ All notable changes to OpsContext for AI Agents (previously ContextEngine — MC
 
 > Entries for 2.2.0 through 2.4.0 were not backfilled here; see `docs/sessions/SESSION_19` through `SESSION_21` for those releases.
 
+## [2.6.0] 2026-09-05: One indexer, many readers
+
+Every Claude Code chat spawns its own MCP server, plus launchd, plus VS Code. Measured on
+2026-09-05 (SESSION_26): eleven servers, 9.3 CPU-hours in 1.4 h, load average 230, each one
+parsing the same ~820 sources and re-embedding every chunk after every save. The embedding cache
+had hit 2 times in 51 starts, because its key was one hash over the whole corpus, git status
+included; and a hit meant the model never loaded, so no semantic search and no re-embed, ever.
+
+### Added
+
+- **Content-addressed vector store** (`[EMBEDDINGS-ARE-CONTENT-ADDRESSED]`,
+  `src/embedding-store.ts`): one vector per distinct embedded text in
+  `~/.contextengine/embeddings.bin`, shared by every server on the machine. A doc change embeds
+  only its new chunks; a cold start embeds only texts never seen before. Always on. The old
+  `embedding-cache.json` is no longer read.
+- **Shared index with one writer per corpus** (`[ONE-INDEXER-MANY-READERS]`,
+  `src/shared-index.ts`), behind `CONTEXTENGINE_SHARED_INDEX=1`: the indexer is elected from the
+  server registry (a build equal to the file on disk first, then the earliest start), parses,
+  imports learnings, embeds, watches the files and writes `~/.contextengine/index/<corpus>.json`;
+  every other server of that corpus loads it, reloads within 3 s of a change, never watches files,
+  never sweep-imports learnings, and loads the model only on its first semantic query. A reader
+  with no index yet builds once locally. Off, every server indexes on its own as before.
+- `contextengine servers` shows `indexer` / `reader` and the corpus per server; audit events
+  `server.role` and `index.write`; `scripts/trial-shared-index.mjs` proves the behaviour on
+  three servers from three cwds (one re-index, readers current within 5 s, reader CPU flat).
+
+### Changed
+
+- The `reindex` tool on a reader reloads the shared index and names the indexer instead of
+  rebuilding on its own.
+- The "too many servers" warning counts only servers that index on their own.
+
+## [2.5.9] — 2026-09-05 — The servers inventory themselves; growth is a tripwire too
+
+The evening 2.5.7 shipped, two MCP servers that had started before the build kept the old
+importer for two hours and re-imported 1,766 records the owner had just had deleted. `ps` found
+them on the second look only. Nine more servers, one per open chat, were each re-embedding the
+corpus after every doc change (load average 230). Nothing in the product could say any of this.
+
+### Added
+
+- **Server registry** (`[SERVERS-ARE-INVENTORIED]`, `src/server-registry.ts`): every server writes
+  `~/.contextengine/servers/<pid>.json` at the top of `main()` with pid, parent process, start time,
+  version, a sha256 of the script it loaded and its cwd; heartbeats every 60 s; removes the record on
+  exit; emits `server.start` to the audit log. `contextengine servers` and the end-session checklist
+  (§ 3b) list live servers, drop dead records, flag **STALE BUILD** when the script on disk no longer
+  matches the hash a server loaded, and warn above 3 concurrent servers. Exit 1 on any warning.
+- **Growth tripwire** (`[STORE-GROWTH-IS-A-TRIPWIRE-TOO]`): a write that adds more than 200 records
+  to the learnings store is refused (`learning.store_growth_refused`), the mirror of the shrink guard.
+  The auto-import reports `refused` and the server keeps running; `import_learnings` and the CLI
+  print the refusal. `CONTEXTENGINE_ALLOW_BULK=1` for one deliberate bulk import.
+
 ## [2.5.8] — 2026-09-05 — "rules" is not a learnings heading
 
 ### Fixed

package/dist/audit.d.ts

@@ -1,4 +1,4 @@
-export type AuditEvent = "learning.save" | "learning.delete" | "learning.store_unreadable" | "learning.store_shrink_refused" | "learning.import" | "learning.export" | "session.save" | "session.delete" | "activation.activate" | "activation.deactivate" | "activation.heartbeat" | "activation.signature_reject" | "activation.legacy_signature" | "firewall.escalate" | "hook.block" | "hook.bypass" | "policy.skipped" | "browser.prompt" | "browser.response" | "browser.tool_call" | "browser.session_start" | "browser.session_end" | "browser.capture_miss" | "vscode.prompt_submit" | "vscode.tool_call" | "vscode.session_start" | "drift.detected" | "notification.fired" | "community.sync_ok" | "community.sync_error" | "audit.rotate" | "audit.redact";
+export type AuditEvent = "learning.save" | "learning.delete" | "learning.store_unreadable" | "learning.store_shrink_refused" | "learning.store_growth_refused" | "server.start" | "server.role" | "index.write" | "learning.import" | "learning.export" | "session.save" | "session.delete" | "activation.activate" | "activation.deactivate" | "activation.heartbeat" | "activation.signature_reject" | "activation.legacy_signature" | "firewall.escalate" | "hook.block" | "hook.bypass" | "policy.skipped" | "browser.prompt" | "browser.response" | "browser.tool_call" | "browser.session_start" | "browser.session_end" | "browser.capture_miss" | "vscode.prompt_submit" | "vscode.tool_call" | "vscode.session_start" | "drift.detected" | "notification.fired" | "community.sync_ok" | "community.sync_error" | "audit.rotate" | "audit.redact";
 export interface AuditRecord {
     ts: string;
     event: AuditEvent;

package/dist/cli-commands.js

@@ -27,6 +27,7 @@ export const KNOWN_COMMANDS = [
     "audit-rotate",
     "audit-redact-ack",
     "audit-verify",
+    "servers",
     "autostart-status",
     "cost",
     "deactivate",

package/dist/cli.js

@@ -655,6 +655,7 @@ import { loadRepoPolicy, parsePolicy, formatPolicySummary, formatValidationError
 import { buildCostReport } from "./cost-report.js";
 import { getStagedFiles, runSecretScan, runDocCoverage, runCommitMessageRequired, runRuleParity, formatSecretViolations, formatDocCoverageViolations, formatSecretViolationsJson, formatDocCoverageViolationsJson, formatCommitMessageViolations, formatCommitMessageViolationsJson, formatRuleParityViolations, formatRuleParityViolationsJson, } from "./hooks.js";
 import { safeAppend } from "./audit.js";
+import { listServers, formatServers } from "./server-registry.js";
 import { installSkill, locateBundledSkill, buildManagedBlock, syncClaudeMd, } from "./claude-integration.js";
 import { fileURLToPath } from "url";
 // ---------------------------------------------------------------------------
@@ -2208,6 +2209,10 @@ async function cliEndSession() {
     if (autoImport.imported > 0) {
         checks.push(`📥 Auto-imported ${autoImport.imported} new learnings from ${autoImport.total} doc sources\n`);
     }
+    if (autoImport.refused) {
+        checks.push(`- ⛔ Auto-import write refused: ${autoImport.refused}`);
+        failCount++;
+    }
     // --- Check 3: Learnings Store ---
     checks.push("## 3. Learnings Store\n");
     const stats = learningsStats();
@@ -2225,6 +2230,13 @@ async function cliEndSession() {
     passCount++;
     checks.push("");
     // --- Check 4: Sessions ---
+    // --- Check 3b: running servers ([LOCK] [SERVERS-ARE-INVENTORIED]) ---
+    checks.push("## 3b. Running servers\n");
+    const fleet = listServers();
+    checks.push("```\n" + formatServers(fleet) + "\n```");
+    if (fleet.warnings.length > 0)
+        failCount += fleet.warnings.length;
+    checks.push("");
     checks.push("## 4. Sessions\n");
     const sessions = listSessions();
     if (sessions.length > 0) {
@@ -2290,7 +2302,14 @@ async function cliImportLearnings(args) {
         console.error("  --permissive: every H3 heading, bold bullet and table row too.");
         process.exit(1);
     }
-    const result = importLearningsFromFile(filePath, category, project, { permissive });
+    let result;
+    try {
+        result = importLearningsFromFile(filePath, category, project, { permissive });
+    }
+    catch (e) {
+        console.error(`⛔ Import refused: ${e?.message || e}`);
+        process.exit(1);
+    }
     console.log(`\n📥 Import Results:`);
     console.log(`   Imported: ${result.imported}`);
     console.log(`   Updated:  ${result.updated}`);
@@ -2508,6 +2527,7 @@ Usage:
                                        Export hash-chained audit log (evidence aligned with
                                        SOC 2 CC7.2 + ISO 27001 A.12.4.1 — not a certification)
   contextengine audit-verify           Verify audit log chain integrity (tamper detection)
+  contextengine servers                List running MCP servers, their build vs the file on disk
   contextengine audit-redact-ack       Acknowledge deliberately redacted records on the chain (--index i,j --reason "...")
   contextengine audit-rotate [--keep-days N] [--max-records N] [--dry-run]
                                        Move old history into an archive segment. Archives
@@ -2719,6 +2739,11 @@ else if (command === "audit-redact-ack") {
 else if (command === "audit-rotate") {
     cliAuditRotate(process.argv.slice(3));
 }
+else if (command === "servers") {
+    const fleet = listServers();
+    console.log(formatServers(fleet));
+    process.exit(fleet.warnings.length > 0 ? 1 : 0);
+}
 else if (command === "audit-verify") {
     cliAuditVerify().catch((err) => {
         console.error("Error:", err);

package/dist/config.d.ts

@@ -49,6 +49,11 @@ export interface ContextEngineConfig {
         enabled?: boolean;
     }>;
 }
+/**
+ * Look for contextengine.json in standard locations.
+ * Priority: env var > CWD > home dir
+ */
+export declare function findConfigFile(): string | null;
 /**
  * Load knowledge sources.
  *

package/dist/config.js

@@ -28,7 +28,7 @@ const DEFAULT_PATTERNS = [
  * Look for contextengine.json in standard locations.
  * Priority: env var > CWD > home dir
  */
-function findConfigFile() {
+export function findConfigFile() {
     const candidates = [];
     const envPath = process.env.CONTEXTENGINE_CONFIG;
     if (envPath) {

package/dist/embedding-store.d.ts

@@ -0,0 +1,46 @@
+export declare const EMBED_DIM = 384;
+export declare function embeddingStoreDir(): string;
+export declare function embeddingStorePath(): string;
+/** The key of one embedded text: 16 bytes (32 hex) of SHA-256 over model + text. */
+export declare function embedKey(model: string, text: string): string;
+export interface StoreLoad {
+    vectors: Map<string, Float32Array>;
+    /** Records on disk, duplicates included (the same key appended twice by two servers). */
+    records: number;
+    /** Bytes ignored at the tail (a partial record from a concurrent appender). */
+    partialBytes: number;
+    /** True when the file exists but its header is not ours; nothing is read from it. */
+    foreign: boolean;
+}
+/** Read the whole store. Missing file = empty store. ~7 MB for 4,400 vectors, tens of ms. */
+export declare function loadEmbeddingStore(path?: string): StoreLoad;
+/**
+ * Append vectors. One write syscall for the batch; the header is written with the first batch.
+ * Two servers creating the file at the same instant both write header + batch and the last
+ * truncating write wins; the loser's vectors are simply embedded again later.
+ */
+export declare function appendEmbeddings(entries: Array<[string, Float32Array]>, path?: string): {
+    written: number;
+    bytes: number;
+};
+/**
+ * Rewrite the store with only the live keys, temp file + rename. Called by an indexer when the
+ * file holds far more records than the corpus needs (every save of an edited doc appends its
+ * changed chunks again). A record appended by another server between the read and the rename
+ * is lost and re-embedded later; nothing else can be.
+ */
+export declare function compactEmbeddingStore(liveKeys: Set<string>, path?: string, opts?: {
+    minRecords?: number;
+    ratio?: number;
+}): {
+    compacted: boolean;
+    before: number;
+    after: number;
+};
+export declare const _internal: {
+    HEADER_BYTES: number;
+    RECORD_BYTES: number;
+    KEY_BYTES: number;
+    MAGIC: string;
+};
+//# sourceMappingURL=embedding-store.d.ts.map

package/dist/embedding-store.js

@@ -0,0 +1,141 @@
+// [LOCKED] [EMBEDDINGS-ARE-CONTENT-ADDRESSED] 2026-09-05
+// [NEVER] key the embedding cache on the whole corpus again (one hash over every chunk), and
+//         [NEVER] skip loading the model because "the cache hit".
+// WHY: the previous cache (src/cache.ts, removed in this commit) used one SHA-256 over all
+//      ~4,400 chunks as its key, and the corpus contains a `git diff --stat HEAD~1..HEAD` ops
+//      chunk per project, the learnings and the last session. Any commit in any of 40 projects,
+//      any saved learning, or another server with a different cwd writing the same single-slot
+//      file made it stale: measured 2 hits in 51 server starts (SESSION_26). Every start and
+//      every doc change then re-embedded every chunk, about 2 min unloaded and 29 min under the
+//      load those re-embeds themselves created (nine chats open, load average 230). And on the
+//      rare hit, initEmbeddings() was never called, so that server had no query pipeline: no
+//      semantic search, and reindex() never re-embedded again for its whole life.
+// FIX: one vector per distinct embedded text, keyed by SHA-256(model + text), in an
+//      append-only binary file shared by every server on the machine. A doc change embeds
+//      only its new chunks; a cold start embeds only texts never seen before, whatever the cwd.
+//      Whoever embeds loads the model at start; a reader of the shared index loads it on its
+//      first semantic query, never "skipped because the cache hit". Appends are one write
+//      syscall each; a torn tail is cut before the next append and ignored by the loader.
+import { appendFileSync, existsSync, mkdirSync, readFileSync, renameSync, statSync, truncateSync, writeFileSync } from "fs";
+import { join } from "path";
+import { homedir } from "os";
+import { createHash } from "crypto";
+export const EMBED_DIM = 384;
+const MAGIC = "CEEMB001"; // 8 bytes
+const HEADER_BYTES = 16; // magic (8) + dims uint32 LE (4) + reserved (4)
+const KEY_BYTES = 16;
+const RECORD_BYTES = KEY_BYTES + EMBED_DIM * 4;
+export function embeddingStoreDir() {
+    return process.env.CONTEXTENGINE_HOME || join(homedir(), ".contextengine");
+}
+export function embeddingStorePath() {
+    return join(embeddingStoreDir(), "embeddings.bin");
+}
+/** The key of one embedded text: 16 bytes (32 hex) of SHA-256 over model + text. */
+export function embedKey(model, text) {
+    return createHash("sha256").update(model).update("\0").update(text).digest("hex").slice(0, KEY_BYTES * 2);
+}
+function header() {
+    const b = Buffer.alloc(HEADER_BYTES);
+    b.write(MAGIC, 0, "ascii");
+    b.writeUInt32LE(EMBED_DIM, 8);
+    return b;
+}
+function recordOf(key, vec) {
+    const b = Buffer.alloc(RECORD_BYTES);
+    Buffer.from(key, "hex").copy(b, 0, 0, KEY_BYTES);
+    for (let i = 0; i < EMBED_DIM; i++)
+        b.writeFloatLE(vec[i] ?? 0, KEY_BYTES + i * 4);
+    return b;
+}
+/** Read the whole store. Missing file = empty store. ~7 MB for 4,400 vectors, tens of ms. */
+export function loadEmbeddingStore(path = embeddingStorePath()) {
+    const out = { vectors: new Map(), records: 0, partialBytes: 0, foreign: false };
+    if (!existsSync(path))
+        return out;
+    let buf;
+    try {
+        buf = readFileSync(path);
+    }
+    catch {
+        return out;
+    }
+    if (buf.length < HEADER_BYTES || buf.toString("ascii", 0, 8) !== MAGIC || buf.readUInt32LE(8) !== EMBED_DIM) {
+        out.foreign = true;
+        return out;
+    }
+    const body = buf.length - HEADER_BYTES;
+    const n = Math.floor(body / RECORD_BYTES);
+    out.partialBytes = body - n * RECORD_BYTES;
+    for (let r = 0; r < n; r++) {
+        const off = HEADER_BYTES + r * RECORD_BYTES;
+        const key = buf.toString("hex", off, off + KEY_BYTES);
+        // A copy, not a view: the file buffer must be collectable.
+        const vec = new Float32Array(EMBED_DIM);
+        for (let i = 0; i < EMBED_DIM; i++)
+            vec[i] = buf.readFloatLE(off + KEY_BYTES + i * 4);
+        out.vectors.set(key, vec);
+    }
+    out.records = n;
+    return out;
+}
+/**
+ * Append vectors. One write syscall for the batch; the header is written with the first batch.
+ * Two servers creating the file at the same instant both write header + batch and the last
+ * truncating write wins; the loser's vectors are simply embedded again later.
+ */
+export function appendEmbeddings(entries, path = embeddingStorePath()) {
+    if (entries.length === 0)
+        return { written: 0, bytes: 0 };
+    const recs = Buffer.concat(entries.map(([k, v]) => recordOf(k, v)));
+    mkdirSync(join(path, ".."), { recursive: true });
+    let size = 0;
+    try {
+        size = statSync(path).size;
+    }
+    catch {
+        size = 0;
+    }
+    if (size < HEADER_BYTES) {
+        writeFileSync(path, Buffer.concat([header(), recs]));
+        return { written: entries.length, bytes: recs.length };
+    }
+    // A torn tail (a writer that died mid-record) would shift the frame of every record appended
+    // after it; cut back to the last whole record before adding ours.
+    const torn = (size - HEADER_BYTES) % RECORD_BYTES;
+    if (torn !== 0)
+        truncateSync(path, size - torn);
+    appendFileSync(path, recs);
+    return { written: entries.length, bytes: recs.length };
+}
+/**
+ * Rewrite the store with only the live keys, temp file + rename. Called by an indexer when the
+ * file holds far more records than the corpus needs (every save of an edited doc appends its
+ * changed chunks again). A record appended by another server between the read and the rename
+ * is lost and re-embedded later; nothing else can be.
+ */
+export function compactEmbeddingStore(liveKeys, path = embeddingStorePath(), opts = {}) {
+    const minRecords = opts.minRecords ?? 10_000;
+    const ratio = opts.ratio ?? 2;
+    // Decide from the size first: below the floor there is nothing to read.
+    let onDisk = 0;
+    try {
+        onDisk = Math.max(0, Math.floor((statSync(path).size - HEADER_BYTES) / RECORD_BYTES));
+    }
+    catch {
+        onDisk = 0;
+    }
+    if (onDisk < minRecords)
+        return { compacted: false, before: onDisk, after: onDisk };
+    const load = loadEmbeddingStore(path);
+    const live = [...load.vectors.entries()].filter(([k]) => liveKeys.has(k));
+    if (load.records < minRecords || load.records < ratio * Math.max(live.length, 1)) {
+        return { compacted: false, before: load.records, after: load.records };
+    }
+    const tmp = `${path}.tmp-${process.pid}`;
+    writeFileSync(tmp, Buffer.concat([header(), ...live.map(([k, v]) => recordOf(k, v))]));
+    renameSync(tmp, path);
+    return { compacted: true, before: load.records, after: live.length };
+}
+export const _internal = { HEADER_BYTES, RECORD_BYTES, KEY_BYTES, MAGIC };
+//# sourceMappingURL=embedding-store.js.map

package/dist/embeddings.d.ts

@@ -1,4 +1,5 @@
 import type { Chunk } from "./ingest.js";
+export declare const MODEL_NAME = "Xenova/all-MiniLM-L6-v2";
 /**
  * Initialize the embedding pipeline (downloads model on first run, ~22MB).
  * Subsequent calls use cached model.
@@ -16,11 +17,22 @@ export interface EmbeddedChunk {
     chunk: Chunk;
     vector: Float32Array;
 }
+/** The exact text that is embedded for a chunk; the store key is derived from it. */
+export declare function embedInputOf(chunk: Chunk): string;
+export declare function embedKeyOf(chunk: Chunk): string;
+export interface EmbedChunksResult {
+    embedded: EmbeddedChunk[];
+    /** Chunks whose vector came from the shared store. */
+    reused: number;
+    /** Chunks embedded now and appended to the store. */
+    fresh: number;
+}
 /**
- * Embed all chunks. Returns the chunks with their vectors.
- * Shows progress on stderr.
+ * Embed all chunks, reusing the content-addressed store for every text it already holds and
+ * embedding only the rest. [LOCK] [EMBEDDINGS-ARE-CONTENT-ADDRESSED]
+ * Returns the chunks in input order. Shows progress on stderr.
  */
-export declare function embedChunks(chunks: Chunk[]): Promise<EmbeddedChunk[]>;
+export declare function embedChunks(chunks: Chunk[], store?: Map<string, Float32Array>, embedFn?: (text: string) => Promise<Float32Array>, storePath?: string): Promise<EmbedChunksResult>;
 export interface VectorSearchResult {
     chunk: Chunk;
     score: number;

package/dist/embeddings.js

@@ -14,10 +14,11 @@
 //    a whole class of buyers. A static import would silently re-break this.
 // FIX: If you need to take a dependency on a transformer feature, add it
 //    behind the same dynamic-import + isEmbeddingsReady() check pattern.
+import { embedKey, appendEmbeddings } from "./embedding-store.js";
 // We dynamically import @huggingface/transformers to keep startup fast
 // and handle the case where it fails gracefully.
 let embedPipeline = null;
-const MODEL_NAME = "Xenova/all-MiniLM-L6-v2";
+export const MODEL_NAME = "Xenova/all-MiniLM-L6-v2";
 /**
  * Initialize the embedding pipeline (downloads model on first run, ~22MB).
  * Subsequent calls use cached model.
@@ -77,29 +78,58 @@ function cosineSimilarity(a, b) {
     }
     return dot;
 }
+/** The exact text that is embedded for a chunk; the store key is derived from it. */
+export function embedInputOf(chunk) {
+    // Embed section + content together for context
+    return `${chunk.section}\n${chunk.content}`.slice(0, 512);
+}
+export function embedKeyOf(chunk) {
+    return embedKey(MODEL_NAME, embedInputOf(chunk));
+}
 /**
- * Embed all chunks. Returns the chunks with their vectors.
- * Shows progress on stderr.
+ * Embed all chunks, reusing the content-addressed store for every text it already holds and
+ * embedding only the rest. [LOCK] [EMBEDDINGS-ARE-CONTENT-ADDRESSED]
+ * Returns the chunks in input order. Shows progress on stderr.
  */
-export async function embedChunks(chunks) {
-    const results = [];
-    const total = chunks.length;
+export async function embedChunks(chunks, store, embedFn = embedText, storePath) {
+    const known = store ?? new Map();
+    const keys = chunks.map(embedKeyOf);
+    const results = new Array(chunks.length);
+    const todo = [];
+    for (let i = 0; i < chunks.length; i++) {
+        const v = known.get(keys[i]);
+        if (v)
+            results[i] = { chunk: chunks[i], vector: v };
+        else
+            todo.push(i);
+    }
+    const total = todo.length;
     const batchSize = 10;
-    for (let i = 0; i < total; i += batchSize) {
-        const batch = chunks.slice(i, i + batchSize);
-        const batchResults = await Promise.all(batch.map(async (chunk) => {
-            // Embed section + content together for context
-            const text = `${chunk.section}\n${chunk.content}`.slice(0, 512);
-            const vector = await embedText(text);
-            return { chunk, vector };
-        }));
-        results.push(...batchResults);
-        const done = Math.min(i + batchSize, total);
+    const appended = [];
+    for (let b = 0; b < total; b += batchSize) {
+        const batch = todo.slice(b, b + batchSize);
+        const vectors = await Promise.all(batch.map((i) => embedFn(embedInputOf(chunks[i]))));
+        batch.forEach((i, j) => {
+            results[i] = { chunk: chunks[i], vector: vectors[j] };
+            if (!known.has(keys[i])) {
+                known.set(keys[i], vectors[j]);
+                appended.push([keys[i], vectors[j]]);
+            }
+        });
+        const done = Math.min(b + batchSize, total);
         if (done % 50 === 0 || done === total) {
-            console.error(`[ContextEngine] 📊 Embedded ${done}/${total} chunks`);
+            console.error(`[ContextEngine] 📊 Embedded ${done}/${total} new chunks`);
+        }
+    }
+    if (appended.length > 0) {
+        try {
+            appendEmbeddings(appended, storePath);
+        }
+        catch (err) {
+            console.error(`[ContextEngine] ⚠ embedding store append failed: ${err.message}`);
         }
     }
-    return results;
+    return { embedded: results, reused: chunks.length - total, fresh: total };
 }
 /**
  * Semantic search: embed the query, then find most similar chunks.