@compr/opscontext-mcp 2.5.9 → 2.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -4,6 +4,38 @@ All notable changes to OpsContext for AI Agents (previously ContextEngine — MC
4
4
 
5
5
  > Entries for 2.2.0 through 2.4.0 were not backfilled here; see `docs/sessions/SESSION_19` through `SESSION_21` for those releases.
6
6
 
7
+ ## [2.6.0] 2026-09-05: One indexer, many readers
8
+
9
+ Every Claude Code chat spawns its own MCP server, plus launchd, plus VS Code. Measured on
10
+ 2026-09-05 (SESSION_26): eleven servers, 9.3 CPU-hours in 1.4 h, load average 230, each one
11
+ parsing the same ~820 sources and re-embedding every chunk after every save. The embedding cache
12
+ had hit 2 times in 51 starts, because its key was one hash over the whole corpus, git status
13
+ included; and a hit meant the model never loaded, so no semantic search and no re-embed, ever.
14
+
15
+ ### Added
16
+
17
+ - **Content-addressed vector store** (`[EMBEDDINGS-ARE-CONTENT-ADDRESSED]`,
18
+ `src/embedding-store.ts`): one vector per distinct embedded text in
19
+ `~/.contextengine/embeddings.bin`, shared by every server on the machine. A doc change embeds
20
+ only its new chunks; a cold start embeds only texts never seen before. Always on. The old
21
+ `embedding-cache.json` is no longer read.
22
+ - **Shared index with one writer per corpus** (`[ONE-INDEXER-MANY-READERS]`,
23
+ `src/shared-index.ts`), behind `CONTEXTENGINE_SHARED_INDEX=1`: the indexer is elected from the
24
+ server registry (a build equal to the file on disk first, then the earliest start), parses,
25
+ imports learnings, embeds, watches the files and writes `~/.contextengine/index/<corpus>.json`;
26
+ every other server of that corpus loads it, reloads within 3 s of a change, never watches files,
27
+ never sweep-imports learnings, and loads the model only on its first semantic query. A reader
28
+ with no index yet builds once locally. Off, every server indexes on its own as before.
29
+ - `contextengine servers` shows `indexer` / `reader` and the corpus per server; audit events
30
+ `server.role` and `index.write`; `scripts/trial-shared-index.mjs` proves the behaviour on
31
+ three servers from three cwds (one re-index, readers current within 5 s, reader CPU flat).
32
+
33
+ ### Changed
34
+
35
+ - The `reindex` tool on a reader reloads the shared index and names the indexer instead of
36
+ rebuilding on its own.
37
+ - The "too many servers" warning counts only servers that index on their own.
38
+
7
39
  ## [2.5.9] — 2026-09-05 — The servers inventory themselves; growth is a tripwire too
8
40
 
9
41
  The evening 2.5.7 shipped, two MCP servers that had started before the build kept the old
package/dist/audit.d.ts CHANGED
@@ -1,4 +1,4 @@
1
- export type AuditEvent = "learning.save" | "learning.delete" | "learning.store_unreadable" | "learning.store_shrink_refused" | "learning.store_growth_refused" | "server.start" | "learning.import" | "learning.export" | "session.save" | "session.delete" | "activation.activate" | "activation.deactivate" | "activation.heartbeat" | "activation.signature_reject" | "activation.legacy_signature" | "firewall.escalate" | "hook.block" | "hook.bypass" | "policy.skipped" | "browser.prompt" | "browser.response" | "browser.tool_call" | "browser.session_start" | "browser.session_end" | "browser.capture_miss" | "vscode.prompt_submit" | "vscode.tool_call" | "vscode.session_start" | "drift.detected" | "notification.fired" | "community.sync_ok" | "community.sync_error" | "audit.rotate" | "audit.redact";
1
+ export type AuditEvent = "learning.save" | "learning.delete" | "learning.store_unreadable" | "learning.store_shrink_refused" | "learning.store_growth_refused" | "server.start" | "server.role" | "index.write" | "learning.import" | "learning.export" | "session.save" | "session.delete" | "activation.activate" | "activation.deactivate" | "activation.heartbeat" | "activation.signature_reject" | "activation.legacy_signature" | "firewall.escalate" | "hook.block" | "hook.bypass" | "policy.skipped" | "browser.prompt" | "browser.response" | "browser.tool_call" | "browser.session_start" | "browser.session_end" | "browser.capture_miss" | "vscode.prompt_submit" | "vscode.tool_call" | "vscode.session_start" | "drift.detected" | "notification.fired" | "community.sync_ok" | "community.sync_error" | "audit.rotate" | "audit.redact";
2
2
  export interface AuditRecord {
3
3
  ts: string;
4
4
  event: AuditEvent;
package/dist/config.d.ts CHANGED
@@ -49,6 +49,11 @@ export interface ContextEngineConfig {
49
49
  enabled?: boolean;
50
50
  }>;
51
51
  }
52
+ /**
53
+ * Look for contextengine.json in standard locations.
54
+ * Priority: env var > CWD > home dir
55
+ */
56
+ export declare function findConfigFile(): string | null;
52
57
  /**
53
58
  * Load knowledge sources.
54
59
  *
package/dist/config.js CHANGED
@@ -28,7 +28,7 @@ const DEFAULT_PATTERNS = [
28
28
  * Look for contextengine.json in standard locations.
29
29
  * Priority: env var > CWD > home dir
30
30
  */
31
- function findConfigFile() {
31
+ export function findConfigFile() {
32
32
  const candidates = [];
33
33
  const envPath = process.env.CONTEXTENGINE_CONFIG;
34
34
  if (envPath) {
@@ -0,0 +1,46 @@
1
+ export declare const EMBED_DIM = 384;
2
+ export declare function embeddingStoreDir(): string;
3
+ export declare function embeddingStorePath(): string;
4
+ /** The key of one embedded text: 16 bytes (32 hex) of SHA-256 over model + text. */
5
+ export declare function embedKey(model: string, text: string): string;
6
+ export interface StoreLoad {
7
+ vectors: Map<string, Float32Array>;
8
+ /** Records on disk, duplicates included (the same key appended twice by two servers). */
9
+ records: number;
10
+ /** Bytes ignored at the tail (a partial record from a concurrent appender). */
11
+ partialBytes: number;
12
+ /** True when the file exists but its header is not ours; nothing is read from it. */
13
+ foreign: boolean;
14
+ }
15
+ /** Read the whole store. Missing file = empty store. ~7 MB for 4,400 vectors, tens of ms. */
16
+ export declare function loadEmbeddingStore(path?: string): StoreLoad;
17
+ /**
18
+ * Append vectors. One write syscall for the batch; the header is written with the first batch.
19
+ * Two servers creating the file at the same instant both write header + batch and the last
20
+ * truncating write wins; the loser's vectors are simply embedded again later.
21
+ */
22
+ export declare function appendEmbeddings(entries: Array<[string, Float32Array]>, path?: string): {
23
+ written: number;
24
+ bytes: number;
25
+ };
26
+ /**
27
+ * Rewrite the store with only the live keys, temp file + rename. Called by an indexer when the
28
+ * file holds far more records than the corpus needs (every save of an edited doc appends its
29
+ * changed chunks again). A record appended by another server between the read and the rename
30
+ * is lost and re-embedded later; nothing else can be.
31
+ */
32
+ export declare function compactEmbeddingStore(liveKeys: Set<string>, path?: string, opts?: {
33
+ minRecords?: number;
34
+ ratio?: number;
35
+ }): {
36
+ compacted: boolean;
37
+ before: number;
38
+ after: number;
39
+ };
40
+ export declare const _internal: {
41
+ HEADER_BYTES: number;
42
+ RECORD_BYTES: number;
43
+ KEY_BYTES: number;
44
+ MAGIC: string;
45
+ };
46
+ //# sourceMappingURL=embedding-store.d.ts.map
@@ -0,0 +1,141 @@
1
+ // [LOCKED] [EMBEDDINGS-ARE-CONTENT-ADDRESSED] 2026-09-05
2
+ // [NEVER] key the embedding cache on the whole corpus again (one hash over every chunk), and
3
+ // [NEVER] skip loading the model because "the cache hit".
4
+ // WHY: the previous cache (src/cache.ts, removed in this commit) used one SHA-256 over all
5
+ // ~4,400 chunks as its key, and the corpus contains a `git diff --stat HEAD~1..HEAD` ops
6
+ // chunk per project, the learnings and the last session. Any commit in any of 40 projects,
7
+ // any saved learning, or another server with a different cwd writing the same single-slot
8
+ // file made it stale: measured 2 hits in 51 server starts (SESSION_26). Every start and
9
+ // every doc change then re-embedded every chunk, about 2 min unloaded and 29 min under the
10
+ // load those re-embeds themselves created (nine chats open, load average 230). And on the
11
+ // rare hit, initEmbeddings() was never called, so that server had no query pipeline: no
12
+ // semantic search, and reindex() never re-embedded again for its whole life.
13
+ // FIX: one vector per distinct embedded text, keyed by SHA-256(model + text), in an
14
+ // append-only binary file shared by every server on the machine. A doc change embeds
15
+ // only its new chunks; a cold start embeds only texts never seen before, whatever the cwd.
16
+ // Whoever embeds loads the model at start; a reader of the shared index loads it on its
17
+ // first semantic query, never "skipped because the cache hit". Appends are one write
18
+ // syscall each; a torn tail is cut before the next append and ignored by the loader.
19
+ import { appendFileSync, existsSync, mkdirSync, readFileSync, renameSync, statSync, truncateSync, writeFileSync } from "fs";
20
+ import { join } from "path";
21
+ import { homedir } from "os";
22
+ import { createHash } from "crypto";
23
+ export const EMBED_DIM = 384;
24
+ const MAGIC = "CEEMB001"; // 8 bytes
25
+ const HEADER_BYTES = 16; // magic (8) + dims uint32 LE (4) + reserved (4)
26
+ const KEY_BYTES = 16;
27
+ const RECORD_BYTES = KEY_BYTES + EMBED_DIM * 4;
28
+ export function embeddingStoreDir() {
29
+ return process.env.CONTEXTENGINE_HOME || join(homedir(), ".contextengine");
30
+ }
31
+ export function embeddingStorePath() {
32
+ return join(embeddingStoreDir(), "embeddings.bin");
33
+ }
34
+ /** The key of one embedded text: 16 bytes (32 hex) of SHA-256 over model + text. */
35
+ export function embedKey(model, text) {
36
+ return createHash("sha256").update(model).update("\0").update(text).digest("hex").slice(0, KEY_BYTES * 2);
37
+ }
38
+ function header() {
39
+ const b = Buffer.alloc(HEADER_BYTES);
40
+ b.write(MAGIC, 0, "ascii");
41
+ b.writeUInt32LE(EMBED_DIM, 8);
42
+ return b;
43
+ }
44
+ function recordOf(key, vec) {
45
+ const b = Buffer.alloc(RECORD_BYTES);
46
+ Buffer.from(key, "hex").copy(b, 0, 0, KEY_BYTES);
47
+ for (let i = 0; i < EMBED_DIM; i++)
48
+ b.writeFloatLE(vec[i] ?? 0, KEY_BYTES + i * 4);
49
+ return b;
50
+ }
51
+ /** Read the whole store. Missing file = empty store. ~7 MB for 4,400 vectors, tens of ms. */
52
+ export function loadEmbeddingStore(path = embeddingStorePath()) {
53
+ const out = { vectors: new Map(), records: 0, partialBytes: 0, foreign: false };
54
+ if (!existsSync(path))
55
+ return out;
56
+ let buf;
57
+ try {
58
+ buf = readFileSync(path);
59
+ }
60
+ catch {
61
+ return out;
62
+ }
63
+ if (buf.length < HEADER_BYTES || buf.toString("ascii", 0, 8) !== MAGIC || buf.readUInt32LE(8) !== EMBED_DIM) {
64
+ out.foreign = true;
65
+ return out;
66
+ }
67
+ const body = buf.length - HEADER_BYTES;
68
+ const n = Math.floor(body / RECORD_BYTES);
69
+ out.partialBytes = body - n * RECORD_BYTES;
70
+ for (let r = 0; r < n; r++) {
71
+ const off = HEADER_BYTES + r * RECORD_BYTES;
72
+ const key = buf.toString("hex", off, off + KEY_BYTES);
73
+ // A copy, not a view: the file buffer must be collectable.
74
+ const vec = new Float32Array(EMBED_DIM);
75
+ for (let i = 0; i < EMBED_DIM; i++)
76
+ vec[i] = buf.readFloatLE(off + KEY_BYTES + i * 4);
77
+ out.vectors.set(key, vec);
78
+ }
79
+ out.records = n;
80
+ return out;
81
+ }
82
+ /**
83
+ * Append vectors. One write syscall for the batch; the header is written with the first batch.
84
+ * Two servers creating the file at the same instant both write header + batch and the last
85
+ * truncating write wins; the loser's vectors are simply embedded again later.
86
+ */
87
+ export function appendEmbeddings(entries, path = embeddingStorePath()) {
88
+ if (entries.length === 0)
89
+ return { written: 0, bytes: 0 };
90
+ const recs = Buffer.concat(entries.map(([k, v]) => recordOf(k, v)));
91
+ mkdirSync(join(path, ".."), { recursive: true });
92
+ let size = 0;
93
+ try {
94
+ size = statSync(path).size;
95
+ }
96
+ catch {
97
+ size = 0;
98
+ }
99
+ if (size < HEADER_BYTES) {
100
+ writeFileSync(path, Buffer.concat([header(), recs]));
101
+ return { written: entries.length, bytes: recs.length };
102
+ }
103
+ // A torn tail (a writer that died mid-record) would shift the frame of every record appended
104
+ // after it; cut back to the last whole record before adding ours.
105
+ const torn = (size - HEADER_BYTES) % RECORD_BYTES;
106
+ if (torn !== 0)
107
+ truncateSync(path, size - torn);
108
+ appendFileSync(path, recs);
109
+ return { written: entries.length, bytes: recs.length };
110
+ }
111
+ /**
112
+ * Rewrite the store with only the live keys, temp file + rename. Called by an indexer when the
113
+ * file holds far more records than the corpus needs (every save of an edited doc appends its
114
+ * changed chunks again). A record appended by another server between the read and the rename
115
+ * is lost and re-embedded later; nothing else can be.
116
+ */
117
+ export function compactEmbeddingStore(liveKeys, path = embeddingStorePath(), opts = {}) {
118
+ const minRecords = opts.minRecords ?? 10_000;
119
+ const ratio = opts.ratio ?? 2;
120
+ // Decide from the size first: below the floor there is nothing to read.
121
+ let onDisk = 0;
122
+ try {
123
+ onDisk = Math.max(0, Math.floor((statSync(path).size - HEADER_BYTES) / RECORD_BYTES));
124
+ }
125
+ catch {
126
+ onDisk = 0;
127
+ }
128
+ if (onDisk < minRecords)
129
+ return { compacted: false, before: onDisk, after: onDisk };
130
+ const load = loadEmbeddingStore(path);
131
+ const live = [...load.vectors.entries()].filter(([k]) => liveKeys.has(k));
132
+ if (load.records < minRecords || load.records < ratio * Math.max(live.length, 1)) {
133
+ return { compacted: false, before: load.records, after: load.records };
134
+ }
135
+ const tmp = `${path}.tmp-${process.pid}`;
136
+ writeFileSync(tmp, Buffer.concat([header(), ...live.map(([k, v]) => recordOf(k, v))]));
137
+ renameSync(tmp, path);
138
+ return { compacted: true, before: load.records, after: live.length };
139
+ }
140
+ export const _internal = { HEADER_BYTES, RECORD_BYTES, KEY_BYTES, MAGIC };
141
+ //# sourceMappingURL=embedding-store.js.map
@@ -1,4 +1,5 @@
1
1
  import type { Chunk } from "./ingest.js";
2
+ export declare const MODEL_NAME = "Xenova/all-MiniLM-L6-v2";
2
3
  /**
3
4
  * Initialize the embedding pipeline (downloads model on first run, ~22MB).
4
5
  * Subsequent calls use cached model.
@@ -16,11 +17,22 @@ export interface EmbeddedChunk {
16
17
  chunk: Chunk;
17
18
  vector: Float32Array;
18
19
  }
20
+ /** The exact text that is embedded for a chunk; the store key is derived from it. */
21
+ export declare function embedInputOf(chunk: Chunk): string;
22
+ export declare function embedKeyOf(chunk: Chunk): string;
23
+ export interface EmbedChunksResult {
24
+ embedded: EmbeddedChunk[];
25
+ /** Chunks whose vector came from the shared store. */
26
+ reused: number;
27
+ /** Chunks embedded now and appended to the store. */
28
+ fresh: number;
29
+ }
19
30
  /**
20
- * Embed all chunks. Returns the chunks with their vectors.
21
- * Shows progress on stderr.
31
+ * Embed all chunks, reusing the content-addressed store for every text it already holds and
32
+ * embedding only the rest. [LOCK] [EMBEDDINGS-ARE-CONTENT-ADDRESSED]
33
+ * Returns the chunks in input order. Shows progress on stderr.
22
34
  */
23
- export declare function embedChunks(chunks: Chunk[]): Promise<EmbeddedChunk[]>;
35
+ export declare function embedChunks(chunks: Chunk[], store?: Map<string, Float32Array>, embedFn?: (text: string) => Promise<Float32Array>, storePath?: string): Promise<EmbedChunksResult>;
24
36
  export interface VectorSearchResult {
25
37
  chunk: Chunk;
26
38
  score: number;
@@ -14,10 +14,11 @@
14
14
  // a whole class of buyers. A static import would silently re-break this.
15
15
  // FIX: If you need to take a dependency on a transformer feature, add it
16
16
  // behind the same dynamic-import + isEmbeddingsReady() check pattern.
17
+ import { embedKey, appendEmbeddings } from "./embedding-store.js";
17
18
  // We dynamically import @huggingface/transformers to keep startup fast
18
19
  // and handle the case where it fails gracefully.
19
20
  let embedPipeline = null;
20
- const MODEL_NAME = "Xenova/all-MiniLM-L6-v2";
21
+ export const MODEL_NAME = "Xenova/all-MiniLM-L6-v2";
21
22
  /**
22
23
  * Initialize the embedding pipeline (downloads model on first run, ~22MB).
23
24
  * Subsequent calls use cached model.
@@ -77,29 +78,58 @@ function cosineSimilarity(a, b) {
77
78
  }
78
79
  return dot;
79
80
  }
81
+ /** The exact text that is embedded for a chunk; the store key is derived from it. */
82
+ export function embedInputOf(chunk) {
83
+ // Embed section + content together for context
84
+ return `${chunk.section}\n${chunk.content}`.slice(0, 512);
85
+ }
86
+ export function embedKeyOf(chunk) {
87
+ return embedKey(MODEL_NAME, embedInputOf(chunk));
88
+ }
80
89
  /**
81
- * Embed all chunks. Returns the chunks with their vectors.
82
- * Shows progress on stderr.
90
+ * Embed all chunks, reusing the content-addressed store for every text it already holds and
91
+ * embedding only the rest. [LOCK] [EMBEDDINGS-ARE-CONTENT-ADDRESSED]
92
+ * Returns the chunks in input order. Shows progress on stderr.
83
93
  */
84
- export async function embedChunks(chunks) {
85
- const results = [];
86
- const total = chunks.length;
94
+ export async function embedChunks(chunks, store, embedFn = embedText, storePath) {
95
+ const known = store ?? new Map();
96
+ const keys = chunks.map(embedKeyOf);
97
+ const results = new Array(chunks.length);
98
+ const todo = [];
99
+ for (let i = 0; i < chunks.length; i++) {
100
+ const v = known.get(keys[i]);
101
+ if (v)
102
+ results[i] = { chunk: chunks[i], vector: v };
103
+ else
104
+ todo.push(i);
105
+ }
106
+ const total = todo.length;
87
107
  const batchSize = 10;
88
- for (let i = 0; i < total; i += batchSize) {
89
- const batch = chunks.slice(i, i + batchSize);
90
- const batchResults = await Promise.all(batch.map(async (chunk) => {
91
- // Embed section + content together for context
92
- const text = `${chunk.section}\n${chunk.content}`.slice(0, 512);
93
- const vector = await embedText(text);
94
- return { chunk, vector };
95
- }));
96
- results.push(...batchResults);
97
- const done = Math.min(i + batchSize, total);
108
+ const appended = [];
109
+ for (let b = 0; b < total; b += batchSize) {
110
+ const batch = todo.slice(b, b + batchSize);
111
+ const vectors = await Promise.all(batch.map((i) => embedFn(embedInputOf(chunks[i]))));
112
+ batch.forEach((i, j) => {
113
+ results[i] = { chunk: chunks[i], vector: vectors[j] };
114
+ if (!known.has(keys[i])) {
115
+ known.set(keys[i], vectors[j]);
116
+ appended.push([keys[i], vectors[j]]);
117
+ }
118
+ });
119
+ const done = Math.min(b + batchSize, total);
98
120
  if (done % 50 === 0 || done === total) {
99
- console.error(`[ContextEngine] 📊 Embedded ${done}/${total} chunks`);
121
+ console.error(`[ContextEngine] 📊 Embedded ${done}/${total} new chunks`);
122
+ }
123
+ }
124
+ if (appended.length > 0) {
125
+ try {
126
+ appendEmbeddings(appended, storePath);
127
+ }
128
+ catch (err) {
129
+ console.error(`[ContextEngine] ⚠ embedding store append failed: ${err.message}`);
100
130
  }
101
131
  }
102
- return results;
132
+ return { embedded: results, reused: chunks.length - total, fresh: total };
103
133
  }
104
134
  /**
105
135
  * Semantic search: embed the query, then find most similar chunks.
package/dist/index.js CHANGED
@@ -5,9 +5,10 @@ import { z } from "zod";
5
5
  import { loadSources, loadProjectDirs, loadConfig, resolveProjectDir } from "./config.js";
6
6
  import { ingestSources } from "./ingest.js";
7
7
  import { searchChunks } from "./search.js";
8
- import { initEmbeddings, embedChunks, vectorSearch, isEmbeddingsReady, } from "./embeddings.js";
8
+ import { initEmbeddings, embedChunks, embedKeyOf, vectorSearch, isEmbeddingsReady, } from "./embeddings.js";
9
9
  import { collectProjectOps, collectSystemOps } from "./collectors.js";
10
- import { loadCache, saveCache } from "./cache.js";
10
+ import { loadEmbeddingStore, compactEmbeddingStore } from "./embedding-store.js";
11
+ import { sharedIndexEnabled, corpusId, electIndexer, writeSharedIndex, readSharedIndex, sharedIndexMtime, } from "./shared-index.js";
11
12
  import { listProjects, checkPorts, runComplianceAudit, formatProjectList, formatPortMap, formatPlan, scoreProject, formatScoreReport, runScoreCanary, } from "./agents.js";
12
13
  import { saveSession, loadSession, listSessions, deleteSession, formatSession, formatSessionList, } from "./sessions.js";
13
14
  import { verifyChain, readAuditLog, filterByRange, autoRotateAuditLog, safeAppend } from "./audit.js";
@@ -44,6 +45,18 @@ let chunks = [];
44
45
  let embeddedChunks = [];
45
46
  let activeProjectNames = [];
46
47
  const firewall = new ProtocolFirewall();
48
+ // One indexer, many readers. [LOCK] [ONE-INDEXER-MANY-READERS]
49
+ // With the shared index off (default until the trial), every server is its own indexer, exactly
50
+ // as before, and only the content-addressed vector store below is new.
51
+ let role = "indexer";
52
+ let corpus;
53
+ let indexerPid = null;
54
+ let setRegistryRole = null;
55
+ /** key -> vector, loaded from ~/.contextengine/embeddings.bin and grown by what we embed. */
56
+ let vectorStore = new Map();
57
+ let indexSeq = 0;
58
+ let lastIndexMtime = null;
59
+ let modelInit = null;
47
60
  // Wire up learning search for auto-injection (avoids circular import)
48
61
  firewall.setLearningSearchFn((query, projects) => {
49
62
  return searchLearnings(query)
@@ -59,9 +72,11 @@ firewall.setLearningSearchFn((query, projects) => {
59
72
  .map((l) => ({ rule: l.rule, project: l.project, category: l.category }));
60
73
  });
61
74
  /**
62
- * (Re-)ingest all sources. Called at startup and on file changes.
75
+ * Parse every source, collect ops and code, import learnings (indexer only), inject learnings,
76
+ * community rules and adapters. Sets `sources`, `chunks`, `activeProjectNames`. No embedding
77
+ * here. One body for startup and for every reindex; the two used to be separate copies.
63
78
  */
64
- async function reindex() {
79
+ async function buildIndex(opts) {
65
80
  sources = loadSources();
66
81
  chunks = ingestSources(sources);
67
82
  // Collect operational data from project directories
@@ -105,14 +120,16 @@ async function reindex() {
105
120
  console.error(`[ContextEngine] 💻 Parsed ${codeChunks} code chunks from source files`);
106
121
  }
107
122
  }
108
- // Auto-import learnings from discovered doc sources
109
- // Dedup is built-in — safe to call on every reindex, no duplicates created
110
- const autoImport = autoImportFromSources(sources.map((s) => ({ path: s.path, name: s.name })));
111
- if (autoImport.imported > 0) {
112
- console.error(`[ContextEngine] 📥 Auto-imported ${autoImport.imported} new learnings from ${autoImport.total} doc sources (${autoImport.updated} updated)`);
113
- }
114
- if (autoImport.refused) {
115
- console.error(`[ContextEngine] ⛔ Auto-import write refused: ${autoImport.refused}`);
123
+ // Auto-import learnings from discovered doc sources. Dedup is built-in. Only the indexer of a
124
+ // corpus writes the store from a sweep; a reader leaves that to it (one writer, not N).
125
+ if (opts.importLearnings) {
126
+ const autoImport = autoImportFromSources(sources.map((s) => ({ path: s.path, name: s.name })));
127
+ if (autoImport.imported > 0) {
128
+ console.error(`[ContextEngine] 📥 Auto-imported ${autoImport.imported} new learnings from ${autoImport.total} doc sources (${autoImport.updated} updated)`);
129
+ }
130
+ if (autoImport.refused) {
131
+ console.error(`[ContextEngine] ⛔ Auto-import write refused: ${autoImport.refused}`);
132
+ }
116
133
  }
117
134
  // Inject learnings as searchable chunks (project-scoped to prevent IP leakage)
118
135
  const learningChunks = learningsToChunks(activeProjectNames);
@@ -135,18 +152,110 @@ async function reindex() {
135
152
  }
136
153
  // Collect from plugin adapters
137
154
  if (config.adapters && config.adapters.length > 0) {
155
+ if (opts.loadAdapters) {
156
+ const adapterCount = await loadAdapters(config.adapters);
157
+ if (adapterCount === 0)
158
+ return;
159
+ }
138
160
  const adapterChunks = await collectFromAdapters(config.adapters);
139
161
  if (adapterChunks.length > 0) {
140
162
  chunks.push(...adapterChunks);
141
163
  console.error(`[ContextEngine] 🔌 Adapters contributed ${adapterChunks.length} chunks`);
142
164
  }
143
165
  }
144
- if (isEmbeddingsReady()) {
145
- console.error(`[ContextEngine] 🧠 Re-embedding ${chunks.length} chunks...`);
146
- embeddedChunks = await embedChunks(chunks);
147
- saveCache(chunks, embeddedChunks);
166
+ }
167
+ /** Load the model once; every caller shares the same promise. */
168
+ function ensureModel() {
169
+ if (!modelInit)
170
+ modelInit = initEmbeddings();
171
+ return modelInit;
172
+ }
173
+ /**
174
+ * Vectors for the current chunks: from the store for every text it holds, embedded now for
175
+ * the rest. [LOCK] [EMBEDDINGS-ARE-CONTENT-ADDRESSED]
176
+ */
177
+ async function embedAll() {
178
+ if (!isEmbeddingsReady())
179
+ return;
180
+ const snapshot = chunks;
181
+ const r = await embedChunks(snapshot, vectorStore);
182
+ if (snapshot !== chunks)
183
+ return; // a newer build replaced these chunks meanwhile; its own embed follows
184
+ embeddedChunks = r.embedded;
185
+ console.error(`[ContextEngine] ✅ Semantic search ready: ${r.reused} vectors from the store, ${r.fresh} embedded now`);
186
+ if (role === "indexer") {
187
+ try {
188
+ const c = compactEmbeddingStore(new Set(snapshot.map(embedKeyOf)));
189
+ if (c.compacted)
190
+ console.error(`[ContextEngine] 🧹 Embedding store compacted: ${c.before} -> ${c.after} records`);
191
+ }
192
+ catch (err) {
193
+ console.error(`[ContextEngine] ⚠ embedding store compaction failed: ${err.message}`);
194
+ }
195
+ }
196
+ }
197
+ /** Indexer only: write the shared index for readers of this corpus. No-op otherwise. */
198
+ function publishIndex() {
199
+ if (!corpus || role !== "indexer")
200
+ return;
201
+ try {
202
+ indexSeq++;
203
+ const r = writeSharedIndex({
204
+ corpus,
205
+ seq: indexSeq,
206
+ writer: process.pid,
207
+ sources,
208
+ activeProjectNames,
209
+ chunks,
210
+ keys: chunks.map(embedKeyOf),
211
+ });
212
+ lastIndexMtime = sharedIndexMtime(corpus);
213
+ safeAppend("index.write", { corpus, seq: indexSeq, chunks: chunks.length, vectors: embeddedChunks.length, bytes: r.bytes, ms: r.ms });
214
+ console.error(`[ContextEngine] 📤 Shared index written: seq ${indexSeq}, ${chunks.length} chunks, ${Math.round(r.bytes / 1024)} KB, ${r.ms} ms`);
215
+ }
216
+ catch (err) {
217
+ console.error(`[ContextEngine] ⚠ shared index write failed: ${err.message}`);
148
218
  }
149
219
  }
220
+ /** Reader: take the indexer's chunks and resolve their vectors from the store. */
221
+ function adoptSharedIndex() {
222
+ if (!corpus)
223
+ return false;
224
+ const f = readSharedIndex(corpus);
225
+ if (!f)
226
+ return false;
227
+ sources = f.sources;
228
+ chunks = f.chunks;
229
+ activeProjectNames = f.activeProjectNames;
230
+ try {
231
+ firewall.setProjectDirs(loadProjectDirs());
232
+ }
233
+ catch { /* scoping keeps its last value */ }
234
+ vectorStore = loadEmbeddingStore().vectors;
235
+ const vecs = [];
236
+ let missing = 0;
237
+ f.chunks.forEach((c, i) => {
238
+ const v = vectorStore.get(f.keys[i]);
239
+ if (v)
240
+ vecs.push({ chunk: c, vector: v });
241
+ else
242
+ missing++;
243
+ });
244
+ embeddedChunks = vecs;
245
+ indexSeq = f.seq;
246
+ lastIndexMtime = sharedIndexMtime(corpus);
247
+ console.error(`[ContextEngine] 📥 Shared index loaded: seq ${f.seq} from pid ${f.writer}, ${chunks.length} chunks, ${vecs.length} vectors${missing ? `, ${missing} not embedded yet` : ""}`);
248
+ return true;
249
+ }
250
+ /**
251
+ * (Re-)ingest all sources. Called at startup and on file changes by the indexer; a reader
252
+ * never calls it on its own except as the fallback when no shared index exists yet.
253
+ */
254
+ async function reindex() {
255
+ await buildIndex({ importLearnings: role === "indexer" });
256
+ await embedAll();
257
+ publishIndex();
258
+ }
150
259
  // ---------------------------------------------------------------------------
151
260
  // Hybrid Search: combine keyword + vector scores with temporal decay
152
261
  // ---------------------------------------------------------------------------
@@ -221,8 +330,11 @@ function hybridSearch(query, keywordResults, vectorResults, topK) {
221
330
  // File Watching
222
331
  // ---------------------------------------------------------------------------
223
332
  const watchers = [];
224
- function startWatching() {
225
- // Clean up old watchers
333
+ let indexPoll = null;
334
+ let rolePoll = null;
335
+ const INDEX_POLL_MS = 3_000;
336
+ const ROLE_POLL_MS = 15_000;
337
+ function stopWatching() {
226
338
  for (const w of watchers) {
227
339
  try {
228
340
  w.close();
@@ -232,6 +344,10 @@ function startWatching() {
232
344
  }
233
345
  }
234
346
  watchers.length = 0;
347
+ }
348
+ /** Indexer only: one fs.watch per source; a change rebuilds, embeds the new chunks, publishes. */
349
+ function startWatching() {
350
+ stopWatching();
235
351
  let debounceTimer = null;
236
352
  for (const source of sources) {
237
353
  if (!existsSync(source.path))
@@ -242,6 +358,8 @@ function startWatching() {
242
358
  if (debounceTimer)
243
359
  clearTimeout(debounceTimer);
244
360
  debounceTimer = setTimeout(async () => {
361
+ if (role !== "indexer")
362
+ return; // demoted while the timer ran
245
363
  console.error(`[ContextEngine] 📝 File changed: ${basename(source.path)} — re-indexing...`);
246
364
  await reindex();
247
365
  console.error(`[ContextEngine] ✅ Re-indexed: ${chunks.length} chunks from ${sources.length} sources`);
@@ -255,6 +373,61 @@ function startWatching() {
255
373
  }
256
374
  console.error(`[ContextEngine] 👁 Watching ${watchers.length} source files for changes`);
257
375
  }
376
+ /** Reader only: reload the shared index when its stamp moves. A stat every 3 s, nothing else. */
377
+ function startIndexPolling() {
378
+ if (indexPoll || !corpus)
379
+ return;
380
+ indexPoll = setInterval(() => {
381
+ if (role !== "reader" || !corpus)
382
+ return;
383
+ const m = sharedIndexMtime(corpus);
384
+ if (m !== null && m !== lastIndexMtime)
385
+ adoptSharedIndex();
386
+ }, INDEX_POLL_MS);
387
+ indexPoll.unref();
388
+ }
389
+ function stopIndexPolling() {
390
+ if (indexPoll)
391
+ clearInterval(indexPoll);
392
+ indexPoll = null;
393
+ }
394
+ /** Re-run the election; on a change of role, switch what this server does. */
395
+ function evaluateRole(reason) {
396
+ if (!corpus)
397
+ return;
398
+ let e;
399
+ try {
400
+ e = electIndexer(corpus, listServers().servers, process.pid);
401
+ }
402
+ catch (err) {
403
+ console.error(`[ContextEngine] ⚠ election failed, staying ${role}: ${err.message}`);
404
+ return;
405
+ }
406
+ indexerPid = e.indexer;
407
+ if (e.role === role)
408
+ return;
409
+ const was = role;
410
+ role = e.role;
411
+ setRegistryRole?.(role);
412
+ safeAppend("server.role", { pid: process.pid, corpus, role, indexer: indexerPid, reason });
413
+ console.error(`[ContextEngine] 🧭 Role ${was} -> ${role} (${reason}; indexer pid ${indexerPid ?? process.pid})`);
414
+ if (role === "indexer") {
415
+ stopIndexPolling();
416
+ ensureModel().then(() => reindex()).then(() => startWatching()).catch((err) => {
417
+ console.error(`[ContextEngine] ⚠ taking over as indexer failed: ${err.message}`);
418
+ });
419
+ }
420
+ else {
421
+ stopWatching();
422
+ startIndexPolling();
423
+ }
424
+ }
425
+ function startRolePolling() {
426
+ if (rolePoll || !corpus)
427
+ return;
428
+ rolePoll = setInterval(() => evaluateRole("periodic"), ROLE_POLL_MS);
429
+ rolePoll.unref();
430
+ }
258
431
  // ---------------------------------------------------------------------------
259
432
  // MCP Server
260
433
  // ---------------------------------------------------------------------------
@@ -293,6 +466,10 @@ server.tool("search_context", "Search across all indexed project knowledge (copi
293
466
  .describe("Search mode: hybrid (default), keyword-only, or semantic-only"),
294
467
  }, async ({ query, top_k, mode }) => {
295
468
  let results = [];
469
+ // A reader keeps its 300 MB model unloaded until someone asks for semantics; the first such
470
+ // query is answered by keyword while the model loads. [LOCK] [EMBEDDINGS-ARE-CONTENT-ADDRESSED]
471
+ if (mode !== "keyword" && !isEmbeddingsReady())
472
+ void ensureModel();
296
473
  if (mode === "keyword" || mode === "hybrid") {
297
474
  const kwResults = searchChunks(chunks, query, top_k * 2);
298
475
  if (mode === "keyword" || !isEmbeddingsReady()) {
@@ -428,6 +605,10 @@ server.tool("read_source", "Read the full content of a specific knowledge source
428
605
  // Tool: reindex
429
606
  // ---------------------------------------------------------------------------
430
607
  server.tool("reindex", "Force a full re-index of all knowledge sources. Use after adding new files or changing contextengine.json.", {}, async () => {
608
+ if (role === "reader" && corpus) {
609
+ adoptSharedIndex();
610
+ return respond("reindex", `This server reads the shared index of corpus ${corpus}, written by pid ${indexerPid ?? "?"}: reloaded seq ${indexSeq}, ${chunks.length} chunks, ${embeddedChunks.length} vectors. Saving a doc makes the indexer rebuild; every reader picks it up within ${INDEX_POLL_MS / 1000} s.`);
611
+ }
431
612
  await reindex();
432
613
  return respond("reindex", `Re-indexed: ${chunks.length} chunks from ${sources.length} sources. Embeddings: ${embeddedChunks.length} vectors.`);
433
614
  });
@@ -1089,83 +1270,44 @@ function registerResources() {
1089
1270
  // ---------------------------------------------------------------------------
1090
1271
  async function main() {
1091
1272
  // 0. Inventory this server FIRST, before indexing takes minutes: a server exists the moment it
1092
- // starts. [LOCK] [SERVERS-ARE-INVENTORIED]
1273
+ // starts. [LOCK] [SERVERS-ARE-INVENTORIED]. With the shared index on, the registry is also
1274
+ // the electorate: the record carries the corpus and the role. [LOCK] [ONE-INDEXER-MANY-READERS]
1275
+ if (sharedIndexEnabled()) {
1276
+ try {
1277
+ corpus = corpusId();
1278
+ }
1279
+ catch (err) {
1280
+ console.error("[ContextEngine] ⚠ corpus id failed, shared index off for this server:", err);
1281
+ }
1282
+ }
1093
1283
  try {
1094
- const { record } = registerServer({ version: PKG_VERSION, script: fileURLToPath(import.meta.url) });
1284
+ const reg = registerServer({ version: PKG_VERSION, script: fileURLToPath(import.meta.url), corpus, role: corpus ? "reader" : undefined });
1285
+ setRegistryRole = reg.setRole;
1095
1286
  const fleet = listServers();
1287
+ if (corpus) {
1288
+ const e = electIndexer(corpus, fleet.servers, process.pid);
1289
+ role = e.role;
1290
+ indexerPid = e.indexer;
1291
+ reg.setRole(role);
1292
+ safeAppend("server.role", { pid: process.pid, corpus, role, indexer: indexerPid, reason: "start" });
1293
+ }
1096
1294
  console.error(`[ContextEngine] 🧭 ${formatServers(fleet)}`);
1097
- safeAppend("server.start", { pid: record.pid, parent: record.parent, version: record.version, build: record.build, cwd: record.cwd, servers_running: fleet.servers.length, stale_builds: fleet.servers.filter((x) => x.staleBuild).length });
1295
+ if (corpus)
1296
+ console.error(`[ContextEngine] 🧭 This server: ${role} of corpus ${corpus}${role === "reader" ? ` (indexer pid ${indexerPid})` : ""}`);
1297
+ safeAppend("server.start", { pid: reg.record.pid, parent: reg.record.parent, version: reg.record.version, build: reg.record.build, cwd: reg.record.cwd, servers_running: fleet.servers.length, stale_builds: fleet.servers.filter((x) => x.staleBuild).length });
1098
1298
  }
1099
1299
  catch (err) {
1100
1300
  console.error("[ContextEngine] ⚠ Server registry failed:", err);
1101
1301
  }
1102
- // 1. Ingest all sources (fast — keyword search available immediately)
1103
- sources = loadSources();
1104
- chunks = ingestSources(sources);
1105
- // 1b. Collect operational data (git, deps, env, docker, pm2, etc.)
1106
- const config = loadConfig();
1107
- const projectDirs = loadProjectDirs();
1108
- activeProjectNames = projectDirs.map((d) => d.name);
1109
- firewall.setProjectDirs(projectDirs);
1110
- if (config.collectOps !== false) {
1111
- let opsChunks = 0;
1112
- for (const dir of projectDirs) {
1113
- const ops = collectProjectOps(dir.path, dir.name);
1114
- chunks.push(...ops);
1115
- opsChunks += ops.length;
1116
- }
1117
- if (opsChunks > 0) {
1118
- console.error(`[ContextEngine] ⚙ Collected ${opsChunks} operational chunks from ${projectDirs.length} projects`);
1119
- }
1120
- }
1121
- if (config.collectSystemOps !== false) {
1122
- const sysOps = collectSystemOps();
1123
- if (sysOps.length > 0) {
1124
- chunks.push(...sysOps);
1125
- console.error(`[ContextEngine] 🖥 Collected ${sysOps.length} system operational chunks`);
1126
- }
1127
- }
1128
- // 1c. Scan code files (TS/JS/Python) if configured
1129
- if (config.codeDirs && config.codeDirs.length > 0) {
1130
- let codeChunks = 0;
1131
- for (const dir of projectDirs) {
1132
- for (const codeDir of config.codeDirs) {
1133
- const codePath = join(dir.path, codeDir);
1134
- if (existsSync(codePath)) {
1135
- const codeResults = scanCodeDir(codePath, dir.name);
1136
- chunks.push(...codeResults);
1137
- codeChunks += codeResults.length;
1138
- }
1139
- }
1140
- }
1141
- if (codeChunks > 0) {
1142
- console.error(`[ContextEngine] 💻 Parsed ${codeChunks} code chunks from source files`);
1143
- }
1144
- }
1145
- // 1d. Auto-import learnings from discovered doc sources
1146
- const autoImport = autoImportFromSources(sources.map((s) => ({ path: s.path, name: s.name })));
1147
- if (autoImport.imported > 0) {
1148
- console.error(`[ContextEngine] 📥 Auto-imported ${autoImport.imported} new learnings from ${autoImport.total} doc sources (${autoImport.updated} updated)`);
1149
- }
1150
- if (autoImport.refused) {
1151
- console.error(`[ContextEngine] ⛔ Auto-import write refused: ${autoImport.refused}`);
1152
- }
1153
- // 1e. Inject learnings into search index (project-scoped)
1154
- const learningChunks = learningsToChunks(activeProjectNames);
1155
- if (learningChunks.length > 0) {
1156
- chunks.push(...learningChunks);
1157
- console.error(`[ContextEngine] 💡 Injected ${learningChunks.length} learning chunks into search index (scoped)`);
1158
- }
1159
- // 1f. Load and collect from plugin adapters
1160
- if (config.adapters && config.adapters.length > 0) {
1161
- const adapterCount = await loadAdapters(config.adapters);
1162
- if (adapterCount > 0) {
1163
- const adapterChunks = await collectFromAdapters(config.adapters);
1164
- if (adapterChunks.length > 0) {
1165
- chunks.push(...adapterChunks);
1166
- console.error(`[ContextEngine] 🔌 Adapters contributed ${adapterChunks.length} chunks from ${adapterCount} adapters`);
1167
- }
1168
- }
1302
+ // 1. The index: a reader takes the indexer's; everyone else, or a reader with nothing to
1303
+ // take yet, builds it (fast, keyword search available immediately).
1304
+ let adopted = false;
1305
+ if (role === "reader")
1306
+ adopted = adoptSharedIndex();
1307
+ if (!adopted) {
1308
+ if (role === "reader")
1309
+ console.error("[ContextEngine] 📥 No shared index yet; building locally once, without importing learnings");
1310
+ await buildIndex({ importLearnings: role === "indexer", loadAdapters: true });
1169
1311
  }
1170
1312
  // 2. Register MCP resources
1171
1313
  registerResources();
@@ -1232,24 +1374,28 @@ async function main() {
1232
1374
  catch (err) {
1233
1375
  console.error("[ContextEngine] ⚠ Failed to write server-meta.json:", err);
1234
1376
  }
1235
- // 4. Load embeddings — try cache first, then model (non-blocking)
1236
- const cached = loadCache(chunks);
1237
- if (cached) {
1238
- embeddedChunks = cached;
1239
- console.error(`[ContextEngine] ✅ Semantic search ready from cache (${embeddedChunks.length} vectors)`);
1240
- }
1241
- else {
1242
- initEmbeddings().then(async (ready) => {
1243
- if (ready) {
1244
- console.error(`[ContextEngine] 🧠 Embedding ${chunks.length} chunks...`);
1245
- embeddedChunks = await embedChunks(chunks);
1246
- saveCache(chunks, embeddedChunks);
1247
- console.error(`[ContextEngine] ✅ Semantic search ready (${embeddedChunks.length} vectors)`);
1248
- }
1377
+ // 4. Vectors. The store holds one vector per text ever embedded on this machine; the model
1378
+ // always loads for whoever embeds or answers queries. [LOCK] [EMBEDDINGS-ARE-CONTENT-ADDRESSED]
1379
+ // A reader that adopted the index already has its vectors and loads the model on its first
1380
+ // semantic query, not before: 300 MB per process is worth waiting for.
1381
+ if (!adopted) {
1382
+ vectorStore = loadEmbeddingStore().vectors;
1383
+ if (vectorStore.size > 0)
1384
+ console.error(`[ContextEngine] 💾 Embedding store: ${vectorStore.size} vectors`);
1385
+ publishIndex(); // keyword-searchable index for readers now; vectors follow
1386
+ ensureModel().then(async (ready) => {
1387
+ if (!ready)
1388
+ return;
1389
+ await embedAll();
1390
+ publishIndex();
1249
1391
  });
1250
1392
  }
1251
- // 5. Start file watchers
1252
- startWatching();
1393
+ // 5. Watch (indexer) or poll (reader), and keep the election running
1394
+ if (role === "indexer")
1395
+ startWatching();
1396
+ else
1397
+ startIndexPolling();
1398
+ startRolePolling();
1253
1399
  // 6. Boot the local HTTP event-ingest endpoint for the browser extension.
1254
1400
  // Local 127.0.0.1:7842 only; auth via shared secret at
1255
1401
  // ~/.contextengine/extension-secret (see init-extension-secret CLI).
@@ -9,6 +9,10 @@ export interface ServerRecord {
9
9
  build: string;
10
10
  cwd: string;
11
11
  node: string;
12
+ /** Since 2.6.0: what this server indexes (see shared-index.ts corpusId) and whether it is
13
+ * the one writing the shared index for it, or a reader of it. Absent on older builds. */
14
+ corpus?: string;
15
+ role?: "indexer" | "reader";
12
16
  }
13
17
  export interface ServerReport {
14
18
  servers: Array<ServerRecord & {
@@ -30,9 +34,12 @@ export declare function isAlive(pid: number): boolean;
30
34
  export declare function registerServer(opts: {
31
35
  version: string;
32
36
  script: string;
37
+ corpus?: string;
38
+ role?: "indexer" | "reader";
33
39
  }): {
34
40
  record: ServerRecord;
35
41
  stop: () => void;
42
+ setRole: (role: "indexer" | "reader") => void;
36
43
  };
37
44
  /** Read every record, drop the dead ones, compare builds with the files on disk now. */
38
45
  export declare function listServers(): ServerReport;
@@ -69,6 +69,8 @@ export function registerServer(opts) {
69
69
  build: buildHashOf(opts.script) || "unknown",
70
70
  cwd: process.cwd(),
71
71
  node: process.version,
72
+ ...(opts.corpus ? { corpus: opts.corpus } : {}),
73
+ ...(opts.role ? { role: opts.role } : {}),
72
74
  };
73
75
  const file = join(dir, `${process.pid}.json`);
74
76
  const write = () => { try {
@@ -93,7 +95,8 @@ export function registerServer(opts) {
93
95
  for (const sig of ["SIGTERM", "SIGINT", "SIGHUP"]) {
94
96
  process.on(sig, () => { stop(); process.exit(0); });
95
97
  }
96
- return { record, stop };
98
+ const setRole = (role) => { record.role = role; write(); };
99
+ return { record, stop, setRole };
97
100
  }
98
101
  /** Read every record, drop the dead ones, compare builds with the files on disk now. */
99
102
  export function listServers() {
@@ -134,8 +137,11 @@ export function listServers() {
134
137
  if (stale.length > 0) {
135
138
  report.warnings.push(`${stale.length} server(s) run a build older than the file on disk (pid ${stale.map((s) => s.pid).join(", ")}): restart them or they keep the old behaviour`);
136
139
  }
137
- if (report.servers.length > SERVER_COUNT_WARN) {
138
- report.warnings.push(`${report.servers.length} servers run at once; every doc change makes each of them re-index and re-embed the corpus (${SERVER_COUNT_WARN} is the comfortable ceiling)`);
140
+ // Only servers that index on their own cost a re-index per doc change; readers of a shared
141
+ // index do not. [LOCK] [ONE-INDEXER-MANY-READERS]
142
+ const indexing = report.servers.filter((s) => s.role !== "reader");
143
+ if (indexing.length > SERVER_COUNT_WARN) {
144
+ report.warnings.push(`${indexing.length} of ${report.servers.length} servers index on their own; every doc change makes each of them re-index the corpus (${SERVER_COUNT_WARN} is the comfortable ceiling; CONTEXTENGINE_SHARED_INDEX=1 makes all but one per corpus readers)`);
139
145
  }
140
146
  return report;
141
147
  }
@@ -146,7 +152,8 @@ export function formatServers(report, home = homedir()) {
146
152
  for (const s of report.servers) {
147
153
  const t = s.started.slice(11, 19) + "Z";
148
154
  const flag = s.staleBuild ? `STALE BUILD (disk ${s.currentBuild})` : s.currentBuild === null ? "script missing on disk" : "current";
149
- lines.push(` pid ${String(s.pid).padEnd(6)} ${t} v${s.version} build ${s.build} ${flag} parent ${s.parent} cwd ${short(s.cwd)}`);
155
+ const role = s.role ? ` ${s.role.padEnd(7)} corpus ${s.corpus ?? "?"}` : "";
156
+ lines.push(` pid ${String(s.pid).padEnd(6)} ${t} v${s.version} build ${s.build} ${flag}${role} parent ${s.parent} cwd ${short(s.cwd)}`);
150
157
  }
151
158
  for (const w of report.warnings)
152
159
  lines.push(` ⚠ ${w}`);
@@ -0,0 +1,45 @@
1
+ import { type KnowledgeSource } from "./config.js";
2
+ import type { Chunk } from "./ingest.js";
3
+ import type { ServerReport } from "./server-registry.js";
4
+ export type ServerRole = "indexer" | "reader";
5
+ export declare const SHARED_INDEX_VERSION = 1;
6
+ export declare function sharedIndexEnabled(): boolean;
7
+ /**
8
+ * What a server's corpus is made of, as a short id: the config file it resolved (path and
9
+ * content) or the discovery fallback, and the env flags that change discovery. Two servers with
10
+ * the same id index the same thing and can share one index; different ids get their own writer.
11
+ */
12
+ export declare function corpusId(): string;
13
+ export interface SharedIndexFile {
14
+ version: number;
15
+ corpus: string;
16
+ seq: number;
17
+ stamp: string;
18
+ writer: number;
19
+ sources: KnowledgeSource[];
20
+ activeProjectNames: string[];
21
+ chunks: Chunk[];
22
+ /** One embedding-store key per chunk, same order. */
23
+ keys: string[];
24
+ }
25
+ export declare function sharedIndexPath(corpus: string): string;
26
+ /** Temp file + rename: a reader never sees a half-written index. */
27
+ export declare function writeSharedIndex(data: Omit<SharedIndexFile, "version" | "stamp">): {
28
+ path: string;
29
+ bytes: number;
30
+ ms: number;
31
+ };
32
+ export declare function readSharedIndex(corpus: string): SharedIndexFile | null;
33
+ /** Cheap change detector for readers: the file's mtime, or null when absent. */
34
+ export declare function sharedIndexMtime(corpus: string): number | null;
35
+ /**
36
+ * Who indexes this corpus. Among the live registered servers of the corpus: a server whose
37
+ * build equals the file on disk beats a stale one, then the earliest start, then the lowest pid.
38
+ * A server that does not find itself in the registry indexes on its own: never wait on a
39
+ * registry that failed.
40
+ */
41
+ export declare function electIndexer(corpus: string, servers: ServerReport["servers"], myPid: number): {
42
+ indexer: number | null;
43
+ role: ServerRole;
44
+ };
45
+ //# sourceMappingURL=shared-index.d.ts.map
@@ -0,0 +1,113 @@
1
+ // [LOCKED] [ONE-INDEXER-MANY-READERS] 2026-09-05
2
+ // [NEVER] let every MCP server watch the corpus and re-index it on its own again once the
3
+ // shared index is on, and [NEVER] let a server whose build is older than the file on
4
+ // disk win the election while a current one is alive.
5
+ // WHY: every Claude Code chat spawns its own MCP server (`.mcp.json` per project, the user-scope
6
+ // entry elsewhere), plus launchd, plus VS Code. Each one parsed the same ~820 doc sources,
7
+ // collected ops from 40 projects, watched the same files and re-embedded the whole corpus
8
+ // on every save. Measured 2026-09-05 (SESSION_26): eleven servers, 9.3 CPU-hours in 1.4 h
9
+ // of wall clock, load average 230, a test suite that timed out, and until 2.5.6 nine
10
+ // writers racing on one learnings.json. A stale-build server that kept indexing after a
11
+ // rebuild re-imported 1,766 records the owner had just had deleted (SESSION_25).
12
+ // FIX: one writer per corpus, chosen from the server registry (current build first, then the
13
+ // earliest start, then the lowest pid), parses, embeds and writes this file with a stamp;
14
+ // every other server of that corpus loads it read-only and reloads when the stamp moves.
15
+ // Off unless CONTEXTENGINE_SHARED_INDEX=1 until the trial has run on a real fleet. A reader
16
+ // that finds no index runs the old pipeline once, without importing learnings: the fallback
17
+ // is today's code path, not a second one.
18
+ import { existsSync, mkdirSync, readFileSync, renameSync, statSync, writeFileSync } from "fs";
19
+ import { join } from "path";
20
+ import { homedir } from "os";
21
+ import { createHash } from "crypto";
22
+ import { findConfigFile } from "./config.js";
23
+ export const SHARED_INDEX_VERSION = 1;
24
+ export function sharedIndexEnabled() {
25
+ return process.env.CONTEXTENGINE_SHARED_INDEX === "1";
26
+ }
27
+ function ceHome() {
28
+ return process.env.CONTEXTENGINE_HOME || join(homedir(), ".contextengine");
29
+ }
30
+ /**
31
+ * What a server's corpus is made of, as a short id: the config file it resolved (path and
32
+ * content) or the discovery fallback, and the env flags that change discovery. Two servers with
33
+ * the same id index the same thing and can share one index; different ids get their own writer.
34
+ */
35
+ export function corpusId() {
36
+ const h = createHash("sha256");
37
+ const cfg = findConfigFile();
38
+ h.update(`config=${cfg ?? "none"}\0`);
39
+ if (cfg) {
40
+ try {
41
+ h.update(readFileSync(cfg));
42
+ }
43
+ catch {
44
+ h.update("unreadable");
45
+ }
46
+ }
47
+ h.update(`\0ws=${process.env.CONTEXTENGINE_WORKSPACES ?? ""}`);
48
+ h.update(`\0skipmem=${process.env.OPSCONTEXT_SKIP_CLAUDE_MEMORY ?? ""}`);
49
+ h.update(`\0home=${homedir()}`);
50
+ return h.digest("hex").slice(0, 12);
51
+ }
52
+ export function sharedIndexPath(corpus) {
53
+ return join(ceHome(), "index", `${corpus}.json`);
54
+ }
55
+ /** Temp file + rename: a reader never sees a half-written index. */
56
+ export function writeSharedIndex(data) {
57
+ const t0 = Date.now();
58
+ const path = sharedIndexPath(data.corpus);
59
+ mkdirSync(join(path, ".."), { recursive: true });
60
+ const file = { version: SHARED_INDEX_VERSION, stamp: new Date().toISOString(), ...data };
61
+ const json = JSON.stringify(file);
62
+ const tmp = `${path}.tmp-${process.pid}`;
63
+ writeFileSync(tmp, json);
64
+ renameSync(tmp, path);
65
+ return { path, bytes: Buffer.byteLength(json), ms: Date.now() - t0 };
66
+ }
67
+ export function readSharedIndex(corpus) {
68
+ const path = sharedIndexPath(corpus);
69
+ if (!existsSync(path))
70
+ return null;
71
+ try {
72
+ const f = JSON.parse(readFileSync(path, "utf8"));
73
+ if (f.version !== SHARED_INDEX_VERSION || f.corpus !== corpus || !Array.isArray(f.chunks) || !Array.isArray(f.keys))
74
+ return null;
75
+ if (f.keys.length !== f.chunks.length)
76
+ return null;
77
+ return f;
78
+ }
79
+ catch {
80
+ return null;
81
+ }
82
+ }
83
+ /** Cheap change detector for readers: the file's mtime, or null when absent. */
84
+ export function sharedIndexMtime(corpus) {
85
+ try {
86
+ return statSync(sharedIndexPath(corpus)).mtimeMs;
87
+ }
88
+ catch {
89
+ return null;
90
+ }
91
+ }
92
+ /**
93
+ * Who indexes this corpus. Among the live registered servers of the corpus: a server whose
94
+ * build equals the file on disk beats a stale one, then the earliest start, then the lowest pid.
95
+ * A server that does not find itself in the registry indexes on its own: never wait on a
96
+ * registry that failed.
97
+ */
98
+ export function electIndexer(corpus, servers, myPid) {
99
+ const mine = servers.filter((s) => s.corpus === corpus);
100
+ if (!mine.some((s) => s.pid === myPid))
101
+ return { indexer: null, role: "indexer" };
102
+ mine.sort((a, b) => {
103
+ if (a.staleBuild !== b.staleBuild)
104
+ return a.staleBuild ? 1 : -1;
105
+ const t = a.started.localeCompare(b.started);
106
+ if (t !== 0)
107
+ return t;
108
+ return a.pid - b.pid;
109
+ });
110
+ const winner = mine[0];
111
+ return { indexer: winner.pid, role: winner.pid === myPid ? "indexer" : "reader" };
112
+ }
113
+ //# sourceMappingURL=shared-index.js.map
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@compr/opscontext-mcp",
3
- "version": "2.5.9",
3
+ "version": "2.6.0",
4
4
  "description": "OpsContext for AI Agents — read-only fleet visibility (PM2/nginx/Docker/git/cron) + tamper-evident audit log + policy-as-code hooks. The ops + compliance layer Claude Code can't grow natively.",
5
5
  "type": "module",
6
6
  "main": "dist/index.js",