@compr/opscontext-mcp 2.5.9 → 2.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +32 -0
- package/dist/audit.d.ts +1 -1
- package/dist/config.d.ts +5 -0
- package/dist/config.js +1 -1
- package/dist/embedding-store.d.ts +46 -0
- package/dist/embedding-store.js +141 -0
- package/dist/embeddings.d.ts +15 -3
- package/dist/embeddings.js +48 -18
- package/dist/index.js +250 -104
- package/dist/server-registry.d.ts +7 -0
- package/dist/server-registry.js +11 -4
- package/dist/shared-index.d.ts +45 -0
- package/dist/shared-index.js +113 -0
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -4,6 +4,38 @@ All notable changes to OpsContext for AI Agents (previously ContextEngine — MC
|
|
|
4
4
|
|
|
5
5
|
> Entries for 2.2.0 through 2.4.0 were not backfilled here; see `docs/sessions/SESSION_19` through `SESSION_21` for those releases.
|
|
6
6
|
|
|
7
|
+
## [2.6.0] 2026-09-05: One indexer, many readers
|
|
8
|
+
|
|
9
|
+
Every Claude Code chat spawns its own MCP server, plus launchd, plus VS Code. Measured on
|
|
10
|
+
2026-09-05 (SESSION_26): eleven servers, 9.3 CPU-hours in 1.4 h, load average 230, each one
|
|
11
|
+
parsing the same ~820 sources and re-embedding every chunk after every save. The embedding cache
|
|
12
|
+
had hit 2 times in 51 starts, because its key was one hash over the whole corpus, git status
|
|
13
|
+
included; and a hit meant the model never loaded, so no semantic search and no re-embed, ever.
|
|
14
|
+
|
|
15
|
+
### Added
|
|
16
|
+
|
|
17
|
+
- **Content-addressed vector store** (`[EMBEDDINGS-ARE-CONTENT-ADDRESSED]`,
|
|
18
|
+
`src/embedding-store.ts`): one vector per distinct embedded text in
|
|
19
|
+
`~/.contextengine/embeddings.bin`, shared by every server on the machine. A doc change embeds
|
|
20
|
+
only its new chunks; a cold start embeds only texts never seen before. Always on. The old
|
|
21
|
+
`embedding-cache.json` is no longer read.
|
|
22
|
+
- **Shared index with one writer per corpus** (`[ONE-INDEXER-MANY-READERS]`,
|
|
23
|
+
`src/shared-index.ts`), behind `CONTEXTENGINE_SHARED_INDEX=1`: the indexer is elected from the
|
|
24
|
+
server registry (a build equal to the file on disk first, then the earliest start), parses,
|
|
25
|
+
imports learnings, embeds, watches the files and writes `~/.contextengine/index/<corpus>.json`;
|
|
26
|
+
every other server of that corpus loads it, reloads within 3 s of a change, never watches files,
|
|
27
|
+
never sweep-imports learnings, and loads the model only on its first semantic query. A reader
|
|
28
|
+
with no index yet builds once locally. Off, every server indexes on its own as before.
|
|
29
|
+
- `contextengine servers` shows `indexer` / `reader` and the corpus per server; audit events
|
|
30
|
+
`server.role` and `index.write`; `scripts/trial-shared-index.mjs` proves the behaviour on
|
|
31
|
+
three servers from three cwds (one re-index, readers current within 5 s, reader CPU flat).
|
|
32
|
+
|
|
33
|
+
### Changed
|
|
34
|
+
|
|
35
|
+
- The `reindex` tool on a reader reloads the shared index and names the indexer instead of
|
|
36
|
+
rebuilding on its own.
|
|
37
|
+
- The "too many servers" warning counts only servers that index on their own.
|
|
38
|
+
|
|
7
39
|
## [2.5.9] — 2026-09-05 — The servers inventory themselves; growth is a tripwire too
|
|
8
40
|
|
|
9
41
|
The evening 2.5.7 shipped, two MCP servers that had started before the build kept the old
|
package/dist/audit.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export type AuditEvent = "learning.save" | "learning.delete" | "learning.store_unreadable" | "learning.store_shrink_refused" | "learning.store_growth_refused" | "server.start" | "learning.import" | "learning.export" | "session.save" | "session.delete" | "activation.activate" | "activation.deactivate" | "activation.heartbeat" | "activation.signature_reject" | "activation.legacy_signature" | "firewall.escalate" | "hook.block" | "hook.bypass" | "policy.skipped" | "browser.prompt" | "browser.response" | "browser.tool_call" | "browser.session_start" | "browser.session_end" | "browser.capture_miss" | "vscode.prompt_submit" | "vscode.tool_call" | "vscode.session_start" | "drift.detected" | "notification.fired" | "community.sync_ok" | "community.sync_error" | "audit.rotate" | "audit.redact";
|
|
1
|
+
export type AuditEvent = "learning.save" | "learning.delete" | "learning.store_unreadable" | "learning.store_shrink_refused" | "learning.store_growth_refused" | "server.start" | "server.role" | "index.write" | "learning.import" | "learning.export" | "session.save" | "session.delete" | "activation.activate" | "activation.deactivate" | "activation.heartbeat" | "activation.signature_reject" | "activation.legacy_signature" | "firewall.escalate" | "hook.block" | "hook.bypass" | "policy.skipped" | "browser.prompt" | "browser.response" | "browser.tool_call" | "browser.session_start" | "browser.session_end" | "browser.capture_miss" | "vscode.prompt_submit" | "vscode.tool_call" | "vscode.session_start" | "drift.detected" | "notification.fired" | "community.sync_ok" | "community.sync_error" | "audit.rotate" | "audit.redact";
|
|
2
2
|
export interface AuditRecord {
|
|
3
3
|
ts: string;
|
|
4
4
|
event: AuditEvent;
|
package/dist/config.d.ts
CHANGED
|
@@ -49,6 +49,11 @@ export interface ContextEngineConfig {
|
|
|
49
49
|
enabled?: boolean;
|
|
50
50
|
}>;
|
|
51
51
|
}
|
|
52
|
+
/**
|
|
53
|
+
* Look for contextengine.json in standard locations.
|
|
54
|
+
* Priority: env var > CWD > home dir
|
|
55
|
+
*/
|
|
56
|
+
export declare function findConfigFile(): string | null;
|
|
52
57
|
/**
|
|
53
58
|
* Load knowledge sources.
|
|
54
59
|
*
|
package/dist/config.js
CHANGED
|
@@ -28,7 +28,7 @@ const DEFAULT_PATTERNS = [
|
|
|
28
28
|
* Look for contextengine.json in standard locations.
|
|
29
29
|
* Priority: env var > CWD > home dir
|
|
30
30
|
*/
|
|
31
|
-
function findConfigFile() {
|
|
31
|
+
export function findConfigFile() {
|
|
32
32
|
const candidates = [];
|
|
33
33
|
const envPath = process.env.CONTEXTENGINE_CONFIG;
|
|
34
34
|
if (envPath) {
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
export declare const EMBED_DIM = 384;
|
|
2
|
+
export declare function embeddingStoreDir(): string;
|
|
3
|
+
export declare function embeddingStorePath(): string;
|
|
4
|
+
/** The key of one embedded text: 16 bytes (32 hex) of SHA-256 over model + text. */
|
|
5
|
+
export declare function embedKey(model: string, text: string): string;
|
|
6
|
+
export interface StoreLoad {
|
|
7
|
+
vectors: Map<string, Float32Array>;
|
|
8
|
+
/** Records on disk, duplicates included (the same key appended twice by two servers). */
|
|
9
|
+
records: number;
|
|
10
|
+
/** Bytes ignored at the tail (a partial record from a concurrent appender). */
|
|
11
|
+
partialBytes: number;
|
|
12
|
+
/** True when the file exists but its header is not ours; nothing is read from it. */
|
|
13
|
+
foreign: boolean;
|
|
14
|
+
}
|
|
15
|
+
/** Read the whole store. Missing file = empty store. ~7 MB for 4,400 vectors, tens of ms. */
|
|
16
|
+
export declare function loadEmbeddingStore(path?: string): StoreLoad;
|
|
17
|
+
/**
|
|
18
|
+
* Append vectors. One write syscall for the batch; the header is written with the first batch.
|
|
19
|
+
* Two servers creating the file at the same instant both write header + batch and the last
|
|
20
|
+
* truncating write wins; the loser's vectors are simply embedded again later.
|
|
21
|
+
*/
|
|
22
|
+
export declare function appendEmbeddings(entries: Array<[string, Float32Array]>, path?: string): {
|
|
23
|
+
written: number;
|
|
24
|
+
bytes: number;
|
|
25
|
+
};
|
|
26
|
+
/**
|
|
27
|
+
* Rewrite the store with only the live keys, temp file + rename. Called by an indexer when the
|
|
28
|
+
* file holds far more records than the corpus needs (every save of an edited doc appends its
|
|
29
|
+
* changed chunks again). A record appended by another server between the read and the rename
|
|
30
|
+
* is lost and re-embedded later; nothing else can be.
|
|
31
|
+
*/
|
|
32
|
+
export declare function compactEmbeddingStore(liveKeys: Set<string>, path?: string, opts?: {
|
|
33
|
+
minRecords?: number;
|
|
34
|
+
ratio?: number;
|
|
35
|
+
}): {
|
|
36
|
+
compacted: boolean;
|
|
37
|
+
before: number;
|
|
38
|
+
after: number;
|
|
39
|
+
};
|
|
40
|
+
export declare const _internal: {
|
|
41
|
+
HEADER_BYTES: number;
|
|
42
|
+
RECORD_BYTES: number;
|
|
43
|
+
KEY_BYTES: number;
|
|
44
|
+
MAGIC: string;
|
|
45
|
+
};
|
|
46
|
+
//# sourceMappingURL=embedding-store.d.ts.map
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
// [LOCKED] [EMBEDDINGS-ARE-CONTENT-ADDRESSED] 2026-09-05
|
|
2
|
+
// [NEVER] key the embedding cache on the whole corpus again (one hash over every chunk), and
|
|
3
|
+
// [NEVER] skip loading the model because "the cache hit".
|
|
4
|
+
// WHY: the previous cache (src/cache.ts, removed in this commit) used one SHA-256 over all
|
|
5
|
+
// ~4,400 chunks as its key, and the corpus contains a `git diff --stat HEAD~1..HEAD` ops
|
|
6
|
+
// chunk per project, the learnings and the last session. Any commit in any of 40 projects,
|
|
7
|
+
// any saved learning, or another server with a different cwd writing the same single-slot
|
|
8
|
+
// file made it stale: measured 2 hits in 51 server starts (SESSION_26). Every start and
|
|
9
|
+
// every doc change then re-embedded every chunk, about 2 min unloaded and 29 min under the
|
|
10
|
+
// load those re-embeds themselves created (nine chats open, load average 230). And on the
|
|
11
|
+
// rare hit, initEmbeddings() was never called, so that server had no query pipeline: no
|
|
12
|
+
// semantic search, and reindex() never re-embedded again for its whole life.
|
|
13
|
+
// FIX: one vector per distinct embedded text, keyed by SHA-256(model + text), in an
|
|
14
|
+
// append-only binary file shared by every server on the machine. A doc change embeds
|
|
15
|
+
// only its new chunks; a cold start embeds only texts never seen before, whatever the cwd.
|
|
16
|
+
// Whoever embeds loads the model at start; a reader of the shared index loads it on its
|
|
17
|
+
// first semantic query, never "skipped because the cache hit". Appends are one write
|
|
18
|
+
// syscall each; a torn tail is cut before the next append and ignored by the loader.
|
|
19
|
+
import { appendFileSync, existsSync, mkdirSync, readFileSync, renameSync, statSync, truncateSync, writeFileSync } from "fs";
|
|
20
|
+
import { join } from "path";
|
|
21
|
+
import { homedir } from "os";
|
|
22
|
+
import { createHash } from "crypto";
|
|
23
|
+
export const EMBED_DIM = 384;
|
|
24
|
+
const MAGIC = "CEEMB001"; // 8 bytes
|
|
25
|
+
const HEADER_BYTES = 16; // magic (8) + dims uint32 LE (4) + reserved (4)
|
|
26
|
+
const KEY_BYTES = 16;
|
|
27
|
+
const RECORD_BYTES = KEY_BYTES + EMBED_DIM * 4;
|
|
28
|
+
export function embeddingStoreDir() {
|
|
29
|
+
return process.env.CONTEXTENGINE_HOME || join(homedir(), ".contextengine");
|
|
30
|
+
}
|
|
31
|
+
export function embeddingStorePath() {
|
|
32
|
+
return join(embeddingStoreDir(), "embeddings.bin");
|
|
33
|
+
}
|
|
34
|
+
/** The key of one embedded text: 16 bytes (32 hex) of SHA-256 over model + text. */
|
|
35
|
+
export function embedKey(model, text) {
|
|
36
|
+
return createHash("sha256").update(model).update("\0").update(text).digest("hex").slice(0, KEY_BYTES * 2);
|
|
37
|
+
}
|
|
38
|
+
function header() {
|
|
39
|
+
const b = Buffer.alloc(HEADER_BYTES);
|
|
40
|
+
b.write(MAGIC, 0, "ascii");
|
|
41
|
+
b.writeUInt32LE(EMBED_DIM, 8);
|
|
42
|
+
return b;
|
|
43
|
+
}
|
|
44
|
+
function recordOf(key, vec) {
|
|
45
|
+
const b = Buffer.alloc(RECORD_BYTES);
|
|
46
|
+
Buffer.from(key, "hex").copy(b, 0, 0, KEY_BYTES);
|
|
47
|
+
for (let i = 0; i < EMBED_DIM; i++)
|
|
48
|
+
b.writeFloatLE(vec[i] ?? 0, KEY_BYTES + i * 4);
|
|
49
|
+
return b;
|
|
50
|
+
}
|
|
51
|
+
/** Read the whole store. Missing file = empty store. ~7 MB for 4,400 vectors, tens of ms. */
|
|
52
|
+
export function loadEmbeddingStore(path = embeddingStorePath()) {
|
|
53
|
+
const out = { vectors: new Map(), records: 0, partialBytes: 0, foreign: false };
|
|
54
|
+
if (!existsSync(path))
|
|
55
|
+
return out;
|
|
56
|
+
let buf;
|
|
57
|
+
try {
|
|
58
|
+
buf = readFileSync(path);
|
|
59
|
+
}
|
|
60
|
+
catch {
|
|
61
|
+
return out;
|
|
62
|
+
}
|
|
63
|
+
if (buf.length < HEADER_BYTES || buf.toString("ascii", 0, 8) !== MAGIC || buf.readUInt32LE(8) !== EMBED_DIM) {
|
|
64
|
+
out.foreign = true;
|
|
65
|
+
return out;
|
|
66
|
+
}
|
|
67
|
+
const body = buf.length - HEADER_BYTES;
|
|
68
|
+
const n = Math.floor(body / RECORD_BYTES);
|
|
69
|
+
out.partialBytes = body - n * RECORD_BYTES;
|
|
70
|
+
for (let r = 0; r < n; r++) {
|
|
71
|
+
const off = HEADER_BYTES + r * RECORD_BYTES;
|
|
72
|
+
const key = buf.toString("hex", off, off + KEY_BYTES);
|
|
73
|
+
// A copy, not a view: the file buffer must be collectable.
|
|
74
|
+
const vec = new Float32Array(EMBED_DIM);
|
|
75
|
+
for (let i = 0; i < EMBED_DIM; i++)
|
|
76
|
+
vec[i] = buf.readFloatLE(off + KEY_BYTES + i * 4);
|
|
77
|
+
out.vectors.set(key, vec);
|
|
78
|
+
}
|
|
79
|
+
out.records = n;
|
|
80
|
+
return out;
|
|
81
|
+
}
|
|
82
|
+
/**
|
|
83
|
+
* Append vectors. One write syscall for the batch; the header is written with the first batch.
|
|
84
|
+
* Two servers creating the file at the same instant both write header + batch and the last
|
|
85
|
+
* truncating write wins; the loser's vectors are simply embedded again later.
|
|
86
|
+
*/
|
|
87
|
+
export function appendEmbeddings(entries, path = embeddingStorePath()) {
|
|
88
|
+
if (entries.length === 0)
|
|
89
|
+
return { written: 0, bytes: 0 };
|
|
90
|
+
const recs = Buffer.concat(entries.map(([k, v]) => recordOf(k, v)));
|
|
91
|
+
mkdirSync(join(path, ".."), { recursive: true });
|
|
92
|
+
let size = 0;
|
|
93
|
+
try {
|
|
94
|
+
size = statSync(path).size;
|
|
95
|
+
}
|
|
96
|
+
catch {
|
|
97
|
+
size = 0;
|
|
98
|
+
}
|
|
99
|
+
if (size < HEADER_BYTES) {
|
|
100
|
+
writeFileSync(path, Buffer.concat([header(), recs]));
|
|
101
|
+
return { written: entries.length, bytes: recs.length };
|
|
102
|
+
}
|
|
103
|
+
// A torn tail (a writer that died mid-record) would shift the frame of every record appended
|
|
104
|
+
// after it; cut back to the last whole record before adding ours.
|
|
105
|
+
const torn = (size - HEADER_BYTES) % RECORD_BYTES;
|
|
106
|
+
if (torn !== 0)
|
|
107
|
+
truncateSync(path, size - torn);
|
|
108
|
+
appendFileSync(path, recs);
|
|
109
|
+
return { written: entries.length, bytes: recs.length };
|
|
110
|
+
}
|
|
111
|
+
/**
|
|
112
|
+
* Rewrite the store with only the live keys, temp file + rename. Called by an indexer when the
|
|
113
|
+
* file holds far more records than the corpus needs (every save of an edited doc appends its
|
|
114
|
+
* changed chunks again). A record appended by another server between the read and the rename
|
|
115
|
+
* is lost and re-embedded later; nothing else can be.
|
|
116
|
+
*/
|
|
117
|
+
export function compactEmbeddingStore(liveKeys, path = embeddingStorePath(), opts = {}) {
|
|
118
|
+
const minRecords = opts.minRecords ?? 10_000;
|
|
119
|
+
const ratio = opts.ratio ?? 2;
|
|
120
|
+
// Decide from the size first: below the floor there is nothing to read.
|
|
121
|
+
let onDisk = 0;
|
|
122
|
+
try {
|
|
123
|
+
onDisk = Math.max(0, Math.floor((statSync(path).size - HEADER_BYTES) / RECORD_BYTES));
|
|
124
|
+
}
|
|
125
|
+
catch {
|
|
126
|
+
onDisk = 0;
|
|
127
|
+
}
|
|
128
|
+
if (onDisk < minRecords)
|
|
129
|
+
return { compacted: false, before: onDisk, after: onDisk };
|
|
130
|
+
const load = loadEmbeddingStore(path);
|
|
131
|
+
const live = [...load.vectors.entries()].filter(([k]) => liveKeys.has(k));
|
|
132
|
+
if (load.records < minRecords || load.records < ratio * Math.max(live.length, 1)) {
|
|
133
|
+
return { compacted: false, before: load.records, after: load.records };
|
|
134
|
+
}
|
|
135
|
+
const tmp = `${path}.tmp-${process.pid}`;
|
|
136
|
+
writeFileSync(tmp, Buffer.concat([header(), ...live.map(([k, v]) => recordOf(k, v))]));
|
|
137
|
+
renameSync(tmp, path);
|
|
138
|
+
return { compacted: true, before: load.records, after: live.length };
|
|
139
|
+
}
|
|
140
|
+
export const _internal = { HEADER_BYTES, RECORD_BYTES, KEY_BYTES, MAGIC };
|
|
141
|
+
//# sourceMappingURL=embedding-store.js.map
|
package/dist/embeddings.d.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { Chunk } from "./ingest.js";
|
|
2
|
+
export declare const MODEL_NAME = "Xenova/all-MiniLM-L6-v2";
|
|
2
3
|
/**
|
|
3
4
|
* Initialize the embedding pipeline (downloads model on first run, ~22MB).
|
|
4
5
|
* Subsequent calls use cached model.
|
|
@@ -16,11 +17,22 @@ export interface EmbeddedChunk {
|
|
|
16
17
|
chunk: Chunk;
|
|
17
18
|
vector: Float32Array;
|
|
18
19
|
}
|
|
20
|
+
/** The exact text that is embedded for a chunk; the store key is derived from it. */
|
|
21
|
+
export declare function embedInputOf(chunk: Chunk): string;
|
|
22
|
+
export declare function embedKeyOf(chunk: Chunk): string;
|
|
23
|
+
export interface EmbedChunksResult {
|
|
24
|
+
embedded: EmbeddedChunk[];
|
|
25
|
+
/** Chunks whose vector came from the shared store. */
|
|
26
|
+
reused: number;
|
|
27
|
+
/** Chunks embedded now and appended to the store. */
|
|
28
|
+
fresh: number;
|
|
29
|
+
}
|
|
19
30
|
/**
|
|
20
|
-
* Embed all chunks
|
|
21
|
-
*
|
|
31
|
+
* Embed all chunks, reusing the content-addressed store for every text it already holds and
|
|
32
|
+
* embedding only the rest. [LOCK] [EMBEDDINGS-ARE-CONTENT-ADDRESSED]
|
|
33
|
+
* Returns the chunks in input order. Shows progress on stderr.
|
|
22
34
|
*/
|
|
23
|
-
export declare function embedChunks(chunks: Chunk[]): Promise<
|
|
35
|
+
export declare function embedChunks(chunks: Chunk[], store?: Map<string, Float32Array>, embedFn?: (text: string) => Promise<Float32Array>, storePath?: string): Promise<EmbedChunksResult>;
|
|
24
36
|
export interface VectorSearchResult {
|
|
25
37
|
chunk: Chunk;
|
|
26
38
|
score: number;
|
package/dist/embeddings.js
CHANGED
|
@@ -14,10 +14,11 @@
|
|
|
14
14
|
// a whole class of buyers. A static import would silently re-break this.
|
|
15
15
|
// FIX: If you need to take a dependency on a transformer feature, add it
|
|
16
16
|
// behind the same dynamic-import + isEmbeddingsReady() check pattern.
|
|
17
|
+
import { embedKey, appendEmbeddings } from "./embedding-store.js";
|
|
17
18
|
// We dynamically import @huggingface/transformers to keep startup fast
|
|
18
19
|
// and handle the case where it fails gracefully.
|
|
19
20
|
let embedPipeline = null;
|
|
20
|
-
const MODEL_NAME = "Xenova/all-MiniLM-L6-v2";
|
|
21
|
+
export const MODEL_NAME = "Xenova/all-MiniLM-L6-v2";
|
|
21
22
|
/**
|
|
22
23
|
* Initialize the embedding pipeline (downloads model on first run, ~22MB).
|
|
23
24
|
* Subsequent calls use cached model.
|
|
@@ -77,29 +78,58 @@ function cosineSimilarity(a, b) {
|
|
|
77
78
|
}
|
|
78
79
|
return dot;
|
|
79
80
|
}
|
|
81
|
+
/** The exact text that is embedded for a chunk; the store key is derived from it. */
|
|
82
|
+
export function embedInputOf(chunk) {
|
|
83
|
+
// Embed section + content together for context
|
|
84
|
+
return `${chunk.section}\n${chunk.content}`.slice(0, 512);
|
|
85
|
+
}
|
|
86
|
+
export function embedKeyOf(chunk) {
|
|
87
|
+
return embedKey(MODEL_NAME, embedInputOf(chunk));
|
|
88
|
+
}
|
|
80
89
|
/**
|
|
81
|
-
* Embed all chunks
|
|
82
|
-
*
|
|
90
|
+
* Embed all chunks, reusing the content-addressed store for every text it already holds and
|
|
91
|
+
* embedding only the rest. [LOCK] [EMBEDDINGS-ARE-CONTENT-ADDRESSED]
|
|
92
|
+
* Returns the chunks in input order. Shows progress on stderr.
|
|
83
93
|
*/
|
|
84
|
-
export async function embedChunks(chunks) {
|
|
85
|
-
const
|
|
86
|
-
const
|
|
94
|
+
export async function embedChunks(chunks, store, embedFn = embedText, storePath) {
|
|
95
|
+
const known = store ?? new Map();
|
|
96
|
+
const keys = chunks.map(embedKeyOf);
|
|
97
|
+
const results = new Array(chunks.length);
|
|
98
|
+
const todo = [];
|
|
99
|
+
for (let i = 0; i < chunks.length; i++) {
|
|
100
|
+
const v = known.get(keys[i]);
|
|
101
|
+
if (v)
|
|
102
|
+
results[i] = { chunk: chunks[i], vector: v };
|
|
103
|
+
else
|
|
104
|
+
todo.push(i);
|
|
105
|
+
}
|
|
106
|
+
const total = todo.length;
|
|
87
107
|
const batchSize = 10;
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
const
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
108
|
+
const appended = [];
|
|
109
|
+
for (let b = 0; b < total; b += batchSize) {
|
|
110
|
+
const batch = todo.slice(b, b + batchSize);
|
|
111
|
+
const vectors = await Promise.all(batch.map((i) => embedFn(embedInputOf(chunks[i]))));
|
|
112
|
+
batch.forEach((i, j) => {
|
|
113
|
+
results[i] = { chunk: chunks[i], vector: vectors[j] };
|
|
114
|
+
if (!known.has(keys[i])) {
|
|
115
|
+
known.set(keys[i], vectors[j]);
|
|
116
|
+
appended.push([keys[i], vectors[j]]);
|
|
117
|
+
}
|
|
118
|
+
});
|
|
119
|
+
const done = Math.min(b + batchSize, total);
|
|
98
120
|
if (done % 50 === 0 || done === total) {
|
|
99
|
-
console.error(`[ContextEngine] 📊 Embedded ${done}/${total} chunks`);
|
|
121
|
+
console.error(`[ContextEngine] 📊 Embedded ${done}/${total} new chunks`);
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
if (appended.length > 0) {
|
|
125
|
+
try {
|
|
126
|
+
appendEmbeddings(appended, storePath);
|
|
127
|
+
}
|
|
128
|
+
catch (err) {
|
|
129
|
+
console.error(`[ContextEngine] ⚠ embedding store append failed: ${err.message}`);
|
|
100
130
|
}
|
|
101
131
|
}
|
|
102
|
-
return results;
|
|
132
|
+
return { embedded: results, reused: chunks.length - total, fresh: total };
|
|
103
133
|
}
|
|
104
134
|
/**
|
|
105
135
|
* Semantic search: embed the query, then find most similar chunks.
|
package/dist/index.js
CHANGED
|
@@ -5,9 +5,10 @@ import { z } from "zod";
|
|
|
5
5
|
import { loadSources, loadProjectDirs, loadConfig, resolveProjectDir } from "./config.js";
|
|
6
6
|
import { ingestSources } from "./ingest.js";
|
|
7
7
|
import { searchChunks } from "./search.js";
|
|
8
|
-
import { initEmbeddings, embedChunks, vectorSearch, isEmbeddingsReady, } from "./embeddings.js";
|
|
8
|
+
import { initEmbeddings, embedChunks, embedKeyOf, vectorSearch, isEmbeddingsReady, } from "./embeddings.js";
|
|
9
9
|
import { collectProjectOps, collectSystemOps } from "./collectors.js";
|
|
10
|
-
import {
|
|
10
|
+
import { loadEmbeddingStore, compactEmbeddingStore } from "./embedding-store.js";
|
|
11
|
+
import { sharedIndexEnabled, corpusId, electIndexer, writeSharedIndex, readSharedIndex, sharedIndexMtime, } from "./shared-index.js";
|
|
11
12
|
import { listProjects, checkPorts, runComplianceAudit, formatProjectList, formatPortMap, formatPlan, scoreProject, formatScoreReport, runScoreCanary, } from "./agents.js";
|
|
12
13
|
import { saveSession, loadSession, listSessions, deleteSession, formatSession, formatSessionList, } from "./sessions.js";
|
|
13
14
|
import { verifyChain, readAuditLog, filterByRange, autoRotateAuditLog, safeAppend } from "./audit.js";
|
|
@@ -44,6 +45,18 @@ let chunks = [];
|
|
|
44
45
|
let embeddedChunks = [];
|
|
45
46
|
let activeProjectNames = [];
|
|
46
47
|
const firewall = new ProtocolFirewall();
|
|
48
|
+
// One indexer, many readers. [LOCK] [ONE-INDEXER-MANY-READERS]
|
|
49
|
+
// With the shared index off (default until the trial), every server is its own indexer, exactly
|
|
50
|
+
// as before, and only the content-addressed vector store below is new.
|
|
51
|
+
let role = "indexer";
|
|
52
|
+
let corpus;
|
|
53
|
+
let indexerPid = null;
|
|
54
|
+
let setRegistryRole = null;
|
|
55
|
+
/** key -> vector, loaded from ~/.contextengine/embeddings.bin and grown by what we embed. */
|
|
56
|
+
let vectorStore = new Map();
|
|
57
|
+
let indexSeq = 0;
|
|
58
|
+
let lastIndexMtime = null;
|
|
59
|
+
let modelInit = null;
|
|
47
60
|
// Wire up learning search for auto-injection (avoids circular import)
|
|
48
61
|
firewall.setLearningSearchFn((query, projects) => {
|
|
49
62
|
return searchLearnings(query)
|
|
@@ -59,9 +72,11 @@ firewall.setLearningSearchFn((query, projects) => {
|
|
|
59
72
|
.map((l) => ({ rule: l.rule, project: l.project, category: l.category }));
|
|
60
73
|
});
|
|
61
74
|
/**
|
|
62
|
-
*
|
|
75
|
+
* Parse every source, collect ops and code, import learnings (indexer only), inject learnings,
|
|
76
|
+
* community rules and adapters. Sets `sources`, `chunks`, `activeProjectNames`. No embedding
|
|
77
|
+
* here. One body for startup and for every reindex; the two used to be separate copies.
|
|
63
78
|
*/
|
|
64
|
-
async function
|
|
79
|
+
async function buildIndex(opts) {
|
|
65
80
|
sources = loadSources();
|
|
66
81
|
chunks = ingestSources(sources);
|
|
67
82
|
// Collect operational data from project directories
|
|
@@ -105,14 +120,16 @@ async function reindex() {
|
|
|
105
120
|
console.error(`[ContextEngine] 💻 Parsed ${codeChunks} code chunks from source files`);
|
|
106
121
|
}
|
|
107
122
|
}
|
|
108
|
-
// Auto-import learnings from discovered doc sources
|
|
109
|
-
//
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
123
|
+
// Auto-import learnings from discovered doc sources. Dedup is built-in. Only the indexer of a
|
|
124
|
+
// corpus writes the store from a sweep; a reader leaves that to it (one writer, not N).
|
|
125
|
+
if (opts.importLearnings) {
|
|
126
|
+
const autoImport = autoImportFromSources(sources.map((s) => ({ path: s.path, name: s.name })));
|
|
127
|
+
if (autoImport.imported > 0) {
|
|
128
|
+
console.error(`[ContextEngine] 📥 Auto-imported ${autoImport.imported} new learnings from ${autoImport.total} doc sources (${autoImport.updated} updated)`);
|
|
129
|
+
}
|
|
130
|
+
if (autoImport.refused) {
|
|
131
|
+
console.error(`[ContextEngine] ⛔ Auto-import write refused: ${autoImport.refused}`);
|
|
132
|
+
}
|
|
116
133
|
}
|
|
117
134
|
// Inject learnings as searchable chunks (project-scoped to prevent IP leakage)
|
|
118
135
|
const learningChunks = learningsToChunks(activeProjectNames);
|
|
@@ -135,18 +152,110 @@ async function reindex() {
|
|
|
135
152
|
}
|
|
136
153
|
// Collect from plugin adapters
|
|
137
154
|
if (config.adapters && config.adapters.length > 0) {
|
|
155
|
+
if (opts.loadAdapters) {
|
|
156
|
+
const adapterCount = await loadAdapters(config.adapters);
|
|
157
|
+
if (adapterCount === 0)
|
|
158
|
+
return;
|
|
159
|
+
}
|
|
138
160
|
const adapterChunks = await collectFromAdapters(config.adapters);
|
|
139
161
|
if (adapterChunks.length > 0) {
|
|
140
162
|
chunks.push(...adapterChunks);
|
|
141
163
|
console.error(`[ContextEngine] 🔌 Adapters contributed ${adapterChunks.length} chunks`);
|
|
142
164
|
}
|
|
143
165
|
}
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
166
|
+
}
|
|
167
|
+
/** Load the model once; every caller shares the same promise. */
|
|
168
|
+
function ensureModel() {
|
|
169
|
+
if (!modelInit)
|
|
170
|
+
modelInit = initEmbeddings();
|
|
171
|
+
return modelInit;
|
|
172
|
+
}
|
|
173
|
+
/**
|
|
174
|
+
* Vectors for the current chunks: from the store for every text it holds, embedded now for
|
|
175
|
+
* the rest. [LOCK] [EMBEDDINGS-ARE-CONTENT-ADDRESSED]
|
|
176
|
+
*/
|
|
177
|
+
async function embedAll() {
|
|
178
|
+
if (!isEmbeddingsReady())
|
|
179
|
+
return;
|
|
180
|
+
const snapshot = chunks;
|
|
181
|
+
const r = await embedChunks(snapshot, vectorStore);
|
|
182
|
+
if (snapshot !== chunks)
|
|
183
|
+
return; // a newer build replaced these chunks meanwhile; its own embed follows
|
|
184
|
+
embeddedChunks = r.embedded;
|
|
185
|
+
console.error(`[ContextEngine] ✅ Semantic search ready: ${r.reused} vectors from the store, ${r.fresh} embedded now`);
|
|
186
|
+
if (role === "indexer") {
|
|
187
|
+
try {
|
|
188
|
+
const c = compactEmbeddingStore(new Set(snapshot.map(embedKeyOf)));
|
|
189
|
+
if (c.compacted)
|
|
190
|
+
console.error(`[ContextEngine] 🧹 Embedding store compacted: ${c.before} -> ${c.after} records`);
|
|
191
|
+
}
|
|
192
|
+
catch (err) {
|
|
193
|
+
console.error(`[ContextEngine] ⚠ embedding store compaction failed: ${err.message}`);
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
/** Indexer only: write the shared index for readers of this corpus. No-op otherwise. */
|
|
198
|
+
function publishIndex() {
|
|
199
|
+
if (!corpus || role !== "indexer")
|
|
200
|
+
return;
|
|
201
|
+
try {
|
|
202
|
+
indexSeq++;
|
|
203
|
+
const r = writeSharedIndex({
|
|
204
|
+
corpus,
|
|
205
|
+
seq: indexSeq,
|
|
206
|
+
writer: process.pid,
|
|
207
|
+
sources,
|
|
208
|
+
activeProjectNames,
|
|
209
|
+
chunks,
|
|
210
|
+
keys: chunks.map(embedKeyOf),
|
|
211
|
+
});
|
|
212
|
+
lastIndexMtime = sharedIndexMtime(corpus);
|
|
213
|
+
safeAppend("index.write", { corpus, seq: indexSeq, chunks: chunks.length, vectors: embeddedChunks.length, bytes: r.bytes, ms: r.ms });
|
|
214
|
+
console.error(`[ContextEngine] 📤 Shared index written: seq ${indexSeq}, ${chunks.length} chunks, ${Math.round(r.bytes / 1024)} KB, ${r.ms} ms`);
|
|
215
|
+
}
|
|
216
|
+
catch (err) {
|
|
217
|
+
console.error(`[ContextEngine] ⚠ shared index write failed: ${err.message}`);
|
|
148
218
|
}
|
|
149
219
|
}
|
|
220
|
+
/** Reader: take the indexer's chunks and resolve their vectors from the store. */
|
|
221
|
+
function adoptSharedIndex() {
|
|
222
|
+
if (!corpus)
|
|
223
|
+
return false;
|
|
224
|
+
const f = readSharedIndex(corpus);
|
|
225
|
+
if (!f)
|
|
226
|
+
return false;
|
|
227
|
+
sources = f.sources;
|
|
228
|
+
chunks = f.chunks;
|
|
229
|
+
activeProjectNames = f.activeProjectNames;
|
|
230
|
+
try {
|
|
231
|
+
firewall.setProjectDirs(loadProjectDirs());
|
|
232
|
+
}
|
|
233
|
+
catch { /* scoping keeps its last value */ }
|
|
234
|
+
vectorStore = loadEmbeddingStore().vectors;
|
|
235
|
+
const vecs = [];
|
|
236
|
+
let missing = 0;
|
|
237
|
+
f.chunks.forEach((c, i) => {
|
|
238
|
+
const v = vectorStore.get(f.keys[i]);
|
|
239
|
+
if (v)
|
|
240
|
+
vecs.push({ chunk: c, vector: v });
|
|
241
|
+
else
|
|
242
|
+
missing++;
|
|
243
|
+
});
|
|
244
|
+
embeddedChunks = vecs;
|
|
245
|
+
indexSeq = f.seq;
|
|
246
|
+
lastIndexMtime = sharedIndexMtime(corpus);
|
|
247
|
+
console.error(`[ContextEngine] 📥 Shared index loaded: seq ${f.seq} from pid ${f.writer}, ${chunks.length} chunks, ${vecs.length} vectors${missing ? `, ${missing} not embedded yet` : ""}`);
|
|
248
|
+
return true;
|
|
249
|
+
}
|
|
250
|
+
/**
|
|
251
|
+
* (Re-)ingest all sources. Called at startup and on file changes by the indexer; a reader
|
|
252
|
+
* never calls it on its own except as the fallback when no shared index exists yet.
|
|
253
|
+
*/
|
|
254
|
+
async function reindex() {
|
|
255
|
+
await buildIndex({ importLearnings: role === "indexer" });
|
|
256
|
+
await embedAll();
|
|
257
|
+
publishIndex();
|
|
258
|
+
}
|
|
150
259
|
// ---------------------------------------------------------------------------
|
|
151
260
|
// Hybrid Search: combine keyword + vector scores with temporal decay
|
|
152
261
|
// ---------------------------------------------------------------------------
|
|
@@ -221,8 +330,11 @@ function hybridSearch(query, keywordResults, vectorResults, topK) {
|
|
|
221
330
|
// File Watching
|
|
222
331
|
// ---------------------------------------------------------------------------
|
|
223
332
|
const watchers = [];
|
|
224
|
-
|
|
225
|
-
|
|
333
|
+
let indexPoll = null;
|
|
334
|
+
let rolePoll = null;
|
|
335
|
+
const INDEX_POLL_MS = 3_000;
|
|
336
|
+
const ROLE_POLL_MS = 15_000;
|
|
337
|
+
function stopWatching() {
|
|
226
338
|
for (const w of watchers) {
|
|
227
339
|
try {
|
|
228
340
|
w.close();
|
|
@@ -232,6 +344,10 @@ function startWatching() {
|
|
|
232
344
|
}
|
|
233
345
|
}
|
|
234
346
|
watchers.length = 0;
|
|
347
|
+
}
|
|
348
|
+
/** Indexer only: one fs.watch per source; a change rebuilds, embeds the new chunks, publishes. */
|
|
349
|
+
function startWatching() {
|
|
350
|
+
stopWatching();
|
|
235
351
|
let debounceTimer = null;
|
|
236
352
|
for (const source of sources) {
|
|
237
353
|
if (!existsSync(source.path))
|
|
@@ -242,6 +358,8 @@ function startWatching() {
|
|
|
242
358
|
if (debounceTimer)
|
|
243
359
|
clearTimeout(debounceTimer);
|
|
244
360
|
debounceTimer = setTimeout(async () => {
|
|
361
|
+
if (role !== "indexer")
|
|
362
|
+
return; // demoted while the timer ran
|
|
245
363
|
console.error(`[ContextEngine] 📝 File changed: ${basename(source.path)} — re-indexing...`);
|
|
246
364
|
await reindex();
|
|
247
365
|
console.error(`[ContextEngine] ✅ Re-indexed: ${chunks.length} chunks from ${sources.length} sources`);
|
|
@@ -255,6 +373,61 @@ function startWatching() {
|
|
|
255
373
|
}
|
|
256
374
|
console.error(`[ContextEngine] 👁 Watching ${watchers.length} source files for changes`);
|
|
257
375
|
}
|
|
376
|
+
/** Reader only: reload the shared index when its stamp moves. A stat every 3 s, nothing else. */
|
|
377
|
+
function startIndexPolling() {
|
|
378
|
+
if (indexPoll || !corpus)
|
|
379
|
+
return;
|
|
380
|
+
indexPoll = setInterval(() => {
|
|
381
|
+
if (role !== "reader" || !corpus)
|
|
382
|
+
return;
|
|
383
|
+
const m = sharedIndexMtime(corpus);
|
|
384
|
+
if (m !== null && m !== lastIndexMtime)
|
|
385
|
+
adoptSharedIndex();
|
|
386
|
+
}, INDEX_POLL_MS);
|
|
387
|
+
indexPoll.unref();
|
|
388
|
+
}
|
|
389
|
+
function stopIndexPolling() {
|
|
390
|
+
if (indexPoll)
|
|
391
|
+
clearInterval(indexPoll);
|
|
392
|
+
indexPoll = null;
|
|
393
|
+
}
|
|
394
|
+
/** Re-run the election; on a change of role, switch what this server does. */
|
|
395
|
+
function evaluateRole(reason) {
|
|
396
|
+
if (!corpus)
|
|
397
|
+
return;
|
|
398
|
+
let e;
|
|
399
|
+
try {
|
|
400
|
+
e = electIndexer(corpus, listServers().servers, process.pid);
|
|
401
|
+
}
|
|
402
|
+
catch (err) {
|
|
403
|
+
console.error(`[ContextEngine] ⚠ election failed, staying ${role}: ${err.message}`);
|
|
404
|
+
return;
|
|
405
|
+
}
|
|
406
|
+
indexerPid = e.indexer;
|
|
407
|
+
if (e.role === role)
|
|
408
|
+
return;
|
|
409
|
+
const was = role;
|
|
410
|
+
role = e.role;
|
|
411
|
+
setRegistryRole?.(role);
|
|
412
|
+
safeAppend("server.role", { pid: process.pid, corpus, role, indexer: indexerPid, reason });
|
|
413
|
+
console.error(`[ContextEngine] 🧭 Role ${was} -> ${role} (${reason}; indexer pid ${indexerPid ?? process.pid})`);
|
|
414
|
+
if (role === "indexer") {
|
|
415
|
+
stopIndexPolling();
|
|
416
|
+
ensureModel().then(() => reindex()).then(() => startWatching()).catch((err) => {
|
|
417
|
+
console.error(`[ContextEngine] ⚠ taking over as indexer failed: ${err.message}`);
|
|
418
|
+
});
|
|
419
|
+
}
|
|
420
|
+
else {
|
|
421
|
+
stopWatching();
|
|
422
|
+
startIndexPolling();
|
|
423
|
+
}
|
|
424
|
+
}
|
|
425
|
+
function startRolePolling() {
|
|
426
|
+
if (rolePoll || !corpus)
|
|
427
|
+
return;
|
|
428
|
+
rolePoll = setInterval(() => evaluateRole("periodic"), ROLE_POLL_MS);
|
|
429
|
+
rolePoll.unref();
|
|
430
|
+
}
|
|
258
431
|
// ---------------------------------------------------------------------------
|
|
259
432
|
// MCP Server
|
|
260
433
|
// ---------------------------------------------------------------------------
|
|
@@ -293,6 +466,10 @@ server.tool("search_context", "Search across all indexed project knowledge (copi
|
|
|
293
466
|
.describe("Search mode: hybrid (default), keyword-only, or semantic-only"),
|
|
294
467
|
}, async ({ query, top_k, mode }) => {
|
|
295
468
|
let results = [];
|
|
469
|
+
// A reader keeps its 300 MB model unloaded until someone asks for semantics; the first such
|
|
470
|
+
// query is answered by keyword while the model loads. [LOCK] [EMBEDDINGS-ARE-CONTENT-ADDRESSED]
|
|
471
|
+
if (mode !== "keyword" && !isEmbeddingsReady())
|
|
472
|
+
void ensureModel();
|
|
296
473
|
if (mode === "keyword" || mode === "hybrid") {
|
|
297
474
|
const kwResults = searchChunks(chunks, query, top_k * 2);
|
|
298
475
|
if (mode === "keyword" || !isEmbeddingsReady()) {
|
|
@@ -428,6 +605,10 @@ server.tool("read_source", "Read the full content of a specific knowledge source
|
|
|
428
605
|
// Tool: reindex
|
|
429
606
|
// ---------------------------------------------------------------------------
|
|
430
607
|
server.tool("reindex", "Force a full re-index of all knowledge sources. Use after adding new files or changing contextengine.json.", {}, async () => {
|
|
608
|
+
if (role === "reader" && corpus) {
|
|
609
|
+
adoptSharedIndex();
|
|
610
|
+
return respond("reindex", `This server reads the shared index of corpus ${corpus}, written by pid ${indexerPid ?? "?"}: reloaded seq ${indexSeq}, ${chunks.length} chunks, ${embeddedChunks.length} vectors. Saving a doc makes the indexer rebuild; every reader picks it up within ${INDEX_POLL_MS / 1000} s.`);
|
|
611
|
+
}
|
|
431
612
|
await reindex();
|
|
432
613
|
return respond("reindex", `Re-indexed: ${chunks.length} chunks from ${sources.length} sources. Embeddings: ${embeddedChunks.length} vectors.`);
|
|
433
614
|
});
|
|
@@ -1089,83 +1270,44 @@ function registerResources() {
|
|
|
1089
1270
|
// ---------------------------------------------------------------------------
|
|
1090
1271
|
async function main() {
|
|
1091
1272
|
// 0. Inventory this server FIRST, before indexing takes minutes: a server exists the moment it
|
|
1092
|
-
// starts. [LOCK] [SERVERS-ARE-INVENTORIED]
|
|
1273
|
+
// starts. [LOCK] [SERVERS-ARE-INVENTORIED]. With the shared index on, the registry is also
|
|
1274
|
+
// the electorate: the record carries the corpus and the role. [LOCK] [ONE-INDEXER-MANY-READERS]
|
|
1275
|
+
if (sharedIndexEnabled()) {
|
|
1276
|
+
try {
|
|
1277
|
+
corpus = corpusId();
|
|
1278
|
+
}
|
|
1279
|
+
catch (err) {
|
|
1280
|
+
console.error("[ContextEngine] ⚠ corpus id failed, shared index off for this server:", err);
|
|
1281
|
+
}
|
|
1282
|
+
}
|
|
1093
1283
|
try {
|
|
1094
|
-
const
|
|
1284
|
+
const reg = registerServer({ version: PKG_VERSION, script: fileURLToPath(import.meta.url), corpus, role: corpus ? "reader" : undefined });
|
|
1285
|
+
setRegistryRole = reg.setRole;
|
|
1095
1286
|
const fleet = listServers();
|
|
1287
|
+
if (corpus) {
|
|
1288
|
+
const e = electIndexer(corpus, fleet.servers, process.pid);
|
|
1289
|
+
role = e.role;
|
|
1290
|
+
indexerPid = e.indexer;
|
|
1291
|
+
reg.setRole(role);
|
|
1292
|
+
safeAppend("server.role", { pid: process.pid, corpus, role, indexer: indexerPid, reason: "start" });
|
|
1293
|
+
}
|
|
1096
1294
|
console.error(`[ContextEngine] 🧭 ${formatServers(fleet)}`);
|
|
1097
|
-
|
|
1295
|
+
if (corpus)
|
|
1296
|
+
console.error(`[ContextEngine] 🧭 This server: ${role} of corpus ${corpus}${role === "reader" ? ` (indexer pid ${indexerPid})` : ""}`);
|
|
1297
|
+
safeAppend("server.start", { pid: reg.record.pid, parent: reg.record.parent, version: reg.record.version, build: reg.record.build, cwd: reg.record.cwd, servers_running: fleet.servers.length, stale_builds: fleet.servers.filter((x) => x.staleBuild).length });
|
|
1098
1298
|
}
|
|
1099
1299
|
catch (err) {
|
|
1100
1300
|
console.error("[ContextEngine] ⚠ Server registry failed:", err);
|
|
1101
1301
|
}
|
|
1102
|
-
// 1.
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
let opsChunks = 0;
|
|
1112
|
-
for (const dir of projectDirs) {
|
|
1113
|
-
const ops = collectProjectOps(dir.path, dir.name);
|
|
1114
|
-
chunks.push(...ops);
|
|
1115
|
-
opsChunks += ops.length;
|
|
1116
|
-
}
|
|
1117
|
-
if (opsChunks > 0) {
|
|
1118
|
-
console.error(`[ContextEngine] ⚙ Collected ${opsChunks} operational chunks from ${projectDirs.length} projects`);
|
|
1119
|
-
}
|
|
1120
|
-
}
|
|
1121
|
-
if (config.collectSystemOps !== false) {
|
|
1122
|
-
const sysOps = collectSystemOps();
|
|
1123
|
-
if (sysOps.length > 0) {
|
|
1124
|
-
chunks.push(...sysOps);
|
|
1125
|
-
console.error(`[ContextEngine] 🖥 Collected ${sysOps.length} system operational chunks`);
|
|
1126
|
-
}
|
|
1127
|
-
}
|
|
1128
|
-
// 1c. Scan code files (TS/JS/Python) if configured
|
|
1129
|
-
if (config.codeDirs && config.codeDirs.length > 0) {
|
|
1130
|
-
let codeChunks = 0;
|
|
1131
|
-
for (const dir of projectDirs) {
|
|
1132
|
-
for (const codeDir of config.codeDirs) {
|
|
1133
|
-
const codePath = join(dir.path, codeDir);
|
|
1134
|
-
if (existsSync(codePath)) {
|
|
1135
|
-
const codeResults = scanCodeDir(codePath, dir.name);
|
|
1136
|
-
chunks.push(...codeResults);
|
|
1137
|
-
codeChunks += codeResults.length;
|
|
1138
|
-
}
|
|
1139
|
-
}
|
|
1140
|
-
}
|
|
1141
|
-
if (codeChunks > 0) {
|
|
1142
|
-
console.error(`[ContextEngine] 💻 Parsed ${codeChunks} code chunks from source files`);
|
|
1143
|
-
}
|
|
1144
|
-
}
|
|
1145
|
-
// 1d. Auto-import learnings from discovered doc sources
|
|
1146
|
-
const autoImport = autoImportFromSources(sources.map((s) => ({ path: s.path, name: s.name })));
|
|
1147
|
-
if (autoImport.imported > 0) {
|
|
1148
|
-
console.error(`[ContextEngine] 📥 Auto-imported ${autoImport.imported} new learnings from ${autoImport.total} doc sources (${autoImport.updated} updated)`);
|
|
1149
|
-
}
|
|
1150
|
-
if (autoImport.refused) {
|
|
1151
|
-
console.error(`[ContextEngine] ⛔ Auto-import write refused: ${autoImport.refused}`);
|
|
1152
|
-
}
|
|
1153
|
-
// 1e. Inject learnings into search index (project-scoped)
|
|
1154
|
-
const learningChunks = learningsToChunks(activeProjectNames);
|
|
1155
|
-
if (learningChunks.length > 0) {
|
|
1156
|
-
chunks.push(...learningChunks);
|
|
1157
|
-
console.error(`[ContextEngine] 💡 Injected ${learningChunks.length} learning chunks into search index (scoped)`);
|
|
1158
|
-
}
|
|
1159
|
-
// 1f. Load and collect from plugin adapters
|
|
1160
|
-
if (config.adapters && config.adapters.length > 0) {
|
|
1161
|
-
const adapterCount = await loadAdapters(config.adapters);
|
|
1162
|
-
if (adapterCount > 0) {
|
|
1163
|
-
const adapterChunks = await collectFromAdapters(config.adapters);
|
|
1164
|
-
if (adapterChunks.length > 0) {
|
|
1165
|
-
chunks.push(...adapterChunks);
|
|
1166
|
-
console.error(`[ContextEngine] 🔌 Adapters contributed ${adapterChunks.length} chunks from ${adapterCount} adapters`);
|
|
1167
|
-
}
|
|
1168
|
-
}
|
|
1302
|
+
// 1. The index: a reader takes the indexer's; everyone else, or a reader with nothing to
|
|
1303
|
+
// take yet, builds it (fast, keyword search available immediately).
|
|
1304
|
+
let adopted = false;
|
|
1305
|
+
if (role === "reader")
|
|
1306
|
+
adopted = adoptSharedIndex();
|
|
1307
|
+
if (!adopted) {
|
|
1308
|
+
if (role === "reader")
|
|
1309
|
+
console.error("[ContextEngine] 📥 No shared index yet; building locally once, without importing learnings");
|
|
1310
|
+
await buildIndex({ importLearnings: role === "indexer", loadAdapters: true });
|
|
1169
1311
|
}
|
|
1170
1312
|
// 2. Register MCP resources
|
|
1171
1313
|
registerResources();
|
|
@@ -1232,24 +1374,28 @@ async function main() {
|
|
|
1232
1374
|
catch (err) {
|
|
1233
1375
|
console.error("[ContextEngine] ⚠ Failed to write server-meta.json:", err);
|
|
1234
1376
|
}
|
|
1235
|
-
// 4.
|
|
1236
|
-
|
|
1237
|
-
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
|
|
1241
|
-
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
|
|
1377
|
+
// 4. Vectors. The store holds one vector per text ever embedded on this machine; the model
|
|
1378
|
+
// always loads for whoever embeds or answers queries. [LOCK] [EMBEDDINGS-ARE-CONTENT-ADDRESSED]
|
|
1379
|
+
// A reader that adopted the index already has its vectors and loads the model on its first
|
|
1380
|
+
// semantic query, not before: 300 MB per process is worth waiting for.
|
|
1381
|
+
if (!adopted) {
|
|
1382
|
+
vectorStore = loadEmbeddingStore().vectors;
|
|
1383
|
+
if (vectorStore.size > 0)
|
|
1384
|
+
console.error(`[ContextEngine] 💾 Embedding store: ${vectorStore.size} vectors`);
|
|
1385
|
+
publishIndex(); // keyword-searchable index for readers now; vectors follow
|
|
1386
|
+
ensureModel().then(async (ready) => {
|
|
1387
|
+
if (!ready)
|
|
1388
|
+
return;
|
|
1389
|
+
await embedAll();
|
|
1390
|
+
publishIndex();
|
|
1249
1391
|
});
|
|
1250
1392
|
}
|
|
1251
|
-
// 5.
|
|
1252
|
-
|
|
1393
|
+
// 5. Watch (indexer) or poll (reader), and keep the election running
|
|
1394
|
+
if (role === "indexer")
|
|
1395
|
+
startWatching();
|
|
1396
|
+
else
|
|
1397
|
+
startIndexPolling();
|
|
1398
|
+
startRolePolling();
|
|
1253
1399
|
// 6. Boot the local HTTP event-ingest endpoint for the browser extension.
|
|
1254
1400
|
// Local 127.0.0.1:7842 only; auth via shared secret at
|
|
1255
1401
|
// ~/.contextengine/extension-secret (see init-extension-secret CLI).
|
|
@@ -9,6 +9,10 @@ export interface ServerRecord {
|
|
|
9
9
|
build: string;
|
|
10
10
|
cwd: string;
|
|
11
11
|
node: string;
|
|
12
|
+
/** Since 2.6.0: what this server indexes (see shared-index.ts corpusId) and whether it is
|
|
13
|
+
* the one writing the shared index for it, or a reader of it. Absent on older builds. */
|
|
14
|
+
corpus?: string;
|
|
15
|
+
role?: "indexer" | "reader";
|
|
12
16
|
}
|
|
13
17
|
export interface ServerReport {
|
|
14
18
|
servers: Array<ServerRecord & {
|
|
@@ -30,9 +34,12 @@ export declare function isAlive(pid: number): boolean;
|
|
|
30
34
|
export declare function registerServer(opts: {
|
|
31
35
|
version: string;
|
|
32
36
|
script: string;
|
|
37
|
+
corpus?: string;
|
|
38
|
+
role?: "indexer" | "reader";
|
|
33
39
|
}): {
|
|
34
40
|
record: ServerRecord;
|
|
35
41
|
stop: () => void;
|
|
42
|
+
setRole: (role: "indexer" | "reader") => void;
|
|
36
43
|
};
|
|
37
44
|
/** Read every record, drop the dead ones, compare builds with the files on disk now. */
|
|
38
45
|
export declare function listServers(): ServerReport;
|
package/dist/server-registry.js
CHANGED
|
@@ -69,6 +69,8 @@ export function registerServer(opts) {
|
|
|
69
69
|
build: buildHashOf(opts.script) || "unknown",
|
|
70
70
|
cwd: process.cwd(),
|
|
71
71
|
node: process.version,
|
|
72
|
+
...(opts.corpus ? { corpus: opts.corpus } : {}),
|
|
73
|
+
...(opts.role ? { role: opts.role } : {}),
|
|
72
74
|
};
|
|
73
75
|
const file = join(dir, `${process.pid}.json`);
|
|
74
76
|
const write = () => { try {
|
|
@@ -93,7 +95,8 @@ export function registerServer(opts) {
|
|
|
93
95
|
for (const sig of ["SIGTERM", "SIGINT", "SIGHUP"]) {
|
|
94
96
|
process.on(sig, () => { stop(); process.exit(0); });
|
|
95
97
|
}
|
|
96
|
-
|
|
98
|
+
const setRole = (role) => { record.role = role; write(); };
|
|
99
|
+
return { record, stop, setRole };
|
|
97
100
|
}
|
|
98
101
|
/** Read every record, drop the dead ones, compare builds with the files on disk now. */
|
|
99
102
|
export function listServers() {
|
|
@@ -134,8 +137,11 @@ export function listServers() {
|
|
|
134
137
|
if (stale.length > 0) {
|
|
135
138
|
report.warnings.push(`${stale.length} server(s) run a build older than the file on disk (pid ${stale.map((s) => s.pid).join(", ")}): restart them or they keep the old behaviour`);
|
|
136
139
|
}
|
|
137
|
-
|
|
138
|
-
|
|
140
|
+
// Only servers that index on their own cost a re-index per doc change; readers of a shared
|
|
141
|
+
// index do not. [LOCK] [ONE-INDEXER-MANY-READERS]
|
|
142
|
+
const indexing = report.servers.filter((s) => s.role !== "reader");
|
|
143
|
+
if (indexing.length > SERVER_COUNT_WARN) {
|
|
144
|
+
report.warnings.push(`${indexing.length} of ${report.servers.length} servers index on their own; every doc change makes each of them re-index the corpus (${SERVER_COUNT_WARN} is the comfortable ceiling; CONTEXTENGINE_SHARED_INDEX=1 makes all but one per corpus readers)`);
|
|
139
145
|
}
|
|
140
146
|
return report;
|
|
141
147
|
}
|
|
@@ -146,7 +152,8 @@ export function formatServers(report, home = homedir()) {
|
|
|
146
152
|
for (const s of report.servers) {
|
|
147
153
|
const t = s.started.slice(11, 19) + "Z";
|
|
148
154
|
const flag = s.staleBuild ? `STALE BUILD (disk ${s.currentBuild})` : s.currentBuild === null ? "script missing on disk" : "current";
|
|
149
|
-
|
|
155
|
+
const role = s.role ? ` ${s.role.padEnd(7)} corpus ${s.corpus ?? "?"}` : "";
|
|
156
|
+
lines.push(` pid ${String(s.pid).padEnd(6)} ${t} v${s.version} build ${s.build} ${flag}${role} parent ${s.parent} cwd ${short(s.cwd)}`);
|
|
150
157
|
}
|
|
151
158
|
for (const w of report.warnings)
|
|
152
159
|
lines.push(` ⚠ ${w}`);
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import { type KnowledgeSource } from "./config.js";
|
|
2
|
+
import type { Chunk } from "./ingest.js";
|
|
3
|
+
import type { ServerReport } from "./server-registry.js";
|
|
4
|
+
export type ServerRole = "indexer" | "reader";
|
|
5
|
+
export declare const SHARED_INDEX_VERSION = 1;
|
|
6
|
+
export declare function sharedIndexEnabled(): boolean;
|
|
7
|
+
/**
|
|
8
|
+
* What a server's corpus is made of, as a short id: the config file it resolved (path and
|
|
9
|
+
* content) or the discovery fallback, and the env flags that change discovery. Two servers with
|
|
10
|
+
* the same id index the same thing and can share one index; different ids get their own writer.
|
|
11
|
+
*/
|
|
12
|
+
export declare function corpusId(): string;
|
|
13
|
+
export interface SharedIndexFile {
|
|
14
|
+
version: number;
|
|
15
|
+
corpus: string;
|
|
16
|
+
seq: number;
|
|
17
|
+
stamp: string;
|
|
18
|
+
writer: number;
|
|
19
|
+
sources: KnowledgeSource[];
|
|
20
|
+
activeProjectNames: string[];
|
|
21
|
+
chunks: Chunk[];
|
|
22
|
+
/** One embedding-store key per chunk, same order. */
|
|
23
|
+
keys: string[];
|
|
24
|
+
}
|
|
25
|
+
export declare function sharedIndexPath(corpus: string): string;
|
|
26
|
+
/** Temp file + rename: a reader never sees a half-written index. */
|
|
27
|
+
export declare function writeSharedIndex(data: Omit<SharedIndexFile, "version" | "stamp">): {
|
|
28
|
+
path: string;
|
|
29
|
+
bytes: number;
|
|
30
|
+
ms: number;
|
|
31
|
+
};
|
|
32
|
+
export declare function readSharedIndex(corpus: string): SharedIndexFile | null;
|
|
33
|
+
/** Cheap change detector for readers: the file's mtime, or null when absent. */
|
|
34
|
+
export declare function sharedIndexMtime(corpus: string): number | null;
|
|
35
|
+
/**
|
|
36
|
+
* Who indexes this corpus. Among the live registered servers of the corpus: a server whose
|
|
37
|
+
* build equals the file on disk beats a stale one, then the earliest start, then the lowest pid.
|
|
38
|
+
* A server that does not find itself in the registry indexes on its own: never wait on a
|
|
39
|
+
* registry that failed.
|
|
40
|
+
*/
|
|
41
|
+
export declare function electIndexer(corpus: string, servers: ServerReport["servers"], myPid: number): {
|
|
42
|
+
indexer: number | null;
|
|
43
|
+
role: ServerRole;
|
|
44
|
+
};
|
|
45
|
+
//# sourceMappingURL=shared-index.d.ts.map
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
// [LOCKED] [ONE-INDEXER-MANY-READERS] 2026-09-05
|
|
2
|
+
// [NEVER] let every MCP server watch the corpus and re-index it on its own again once the
|
|
3
|
+
// shared index is on, and [NEVER] let a server whose build is older than the file on
|
|
4
|
+
// disk win the election while a current one is alive.
|
|
5
|
+
// WHY: every Claude Code chat spawns its own MCP server (`.mcp.json` per project, the user-scope
|
|
6
|
+
// entry elsewhere), plus launchd, plus VS Code. Each one parsed the same ~820 doc sources,
|
|
7
|
+
// collected ops from 40 projects, watched the same files and re-embedded the whole corpus
|
|
8
|
+
// on every save. Measured 2026-09-05 (SESSION_26): eleven servers, 9.3 CPU-hours in 1.4 h
|
|
9
|
+
// of wall clock, load average 230, a test suite that timed out, and until 2.5.6 nine
|
|
10
|
+
// writers racing on one learnings.json. A stale-build server that kept indexing after a
|
|
11
|
+
// rebuild re-imported 1,766 records the owner had just had deleted (SESSION_25).
|
|
12
|
+
// FIX: one writer per corpus, chosen from the server registry (current build first, then the
|
|
13
|
+
// earliest start, then the lowest pid), parses, embeds and writes this file with a stamp;
|
|
14
|
+
// every other server of that corpus loads it read-only and reloads when the stamp moves.
|
|
15
|
+
// Off unless CONTEXTENGINE_SHARED_INDEX=1 until the trial has run on a real fleet. A reader
|
|
16
|
+
// that finds no index runs the old pipeline once, without importing learnings: the fallback
|
|
17
|
+
// is today's code path, not a second one.
|
|
18
|
+
import { existsSync, mkdirSync, readFileSync, renameSync, statSync, writeFileSync } from "fs";
|
|
19
|
+
import { join } from "path";
|
|
20
|
+
import { homedir } from "os";
|
|
21
|
+
import { createHash } from "crypto";
|
|
22
|
+
import { findConfigFile } from "./config.js";
|
|
23
|
+
export const SHARED_INDEX_VERSION = 1;
|
|
24
|
+
export function sharedIndexEnabled() {
|
|
25
|
+
return process.env.CONTEXTENGINE_SHARED_INDEX === "1";
|
|
26
|
+
}
|
|
27
|
+
function ceHome() {
|
|
28
|
+
return process.env.CONTEXTENGINE_HOME || join(homedir(), ".contextengine");
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* What a server's corpus is made of, as a short id: the config file it resolved (path and
|
|
32
|
+
* content) or the discovery fallback, and the env flags that change discovery. Two servers with
|
|
33
|
+
* the same id index the same thing and can share one index; different ids get their own writer.
|
|
34
|
+
*/
|
|
35
|
+
export function corpusId() {
|
|
36
|
+
const h = createHash("sha256");
|
|
37
|
+
const cfg = findConfigFile();
|
|
38
|
+
h.update(`config=${cfg ?? "none"}\0`);
|
|
39
|
+
if (cfg) {
|
|
40
|
+
try {
|
|
41
|
+
h.update(readFileSync(cfg));
|
|
42
|
+
}
|
|
43
|
+
catch {
|
|
44
|
+
h.update("unreadable");
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
h.update(`\0ws=${process.env.CONTEXTENGINE_WORKSPACES ?? ""}`);
|
|
48
|
+
h.update(`\0skipmem=${process.env.OPSCONTEXT_SKIP_CLAUDE_MEMORY ?? ""}`);
|
|
49
|
+
h.update(`\0home=${homedir()}`);
|
|
50
|
+
return h.digest("hex").slice(0, 12);
|
|
51
|
+
}
|
|
52
|
+
export function sharedIndexPath(corpus) {
|
|
53
|
+
return join(ceHome(), "index", `${corpus}.json`);
|
|
54
|
+
}
|
|
55
|
+
/** Temp file + rename: a reader never sees a half-written index. */
|
|
56
|
+
export function writeSharedIndex(data) {
|
|
57
|
+
const t0 = Date.now();
|
|
58
|
+
const path = sharedIndexPath(data.corpus);
|
|
59
|
+
mkdirSync(join(path, ".."), { recursive: true });
|
|
60
|
+
const file = { version: SHARED_INDEX_VERSION, stamp: new Date().toISOString(), ...data };
|
|
61
|
+
const json = JSON.stringify(file);
|
|
62
|
+
const tmp = `${path}.tmp-${process.pid}`;
|
|
63
|
+
writeFileSync(tmp, json);
|
|
64
|
+
renameSync(tmp, path);
|
|
65
|
+
return { path, bytes: Buffer.byteLength(json), ms: Date.now() - t0 };
|
|
66
|
+
}
|
|
67
|
+
export function readSharedIndex(corpus) {
|
|
68
|
+
const path = sharedIndexPath(corpus);
|
|
69
|
+
if (!existsSync(path))
|
|
70
|
+
return null;
|
|
71
|
+
try {
|
|
72
|
+
const f = JSON.parse(readFileSync(path, "utf8"));
|
|
73
|
+
if (f.version !== SHARED_INDEX_VERSION || f.corpus !== corpus || !Array.isArray(f.chunks) || !Array.isArray(f.keys))
|
|
74
|
+
return null;
|
|
75
|
+
if (f.keys.length !== f.chunks.length)
|
|
76
|
+
return null;
|
|
77
|
+
return f;
|
|
78
|
+
}
|
|
79
|
+
catch {
|
|
80
|
+
return null;
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
/** Cheap change detector for readers: the file's mtime, or null when absent. */
|
|
84
|
+
export function sharedIndexMtime(corpus) {
|
|
85
|
+
try {
|
|
86
|
+
return statSync(sharedIndexPath(corpus)).mtimeMs;
|
|
87
|
+
}
|
|
88
|
+
catch {
|
|
89
|
+
return null;
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
/**
|
|
93
|
+
* Who indexes this corpus. Among the live registered servers of the corpus: a server whose
|
|
94
|
+
* build equals the file on disk beats a stale one, then the earliest start, then the lowest pid.
|
|
95
|
+
* A server that does not find itself in the registry indexes on its own: never wait on a
|
|
96
|
+
* registry that failed.
|
|
97
|
+
*/
|
|
98
|
+
export function electIndexer(corpus, servers, myPid) {
|
|
99
|
+
const mine = servers.filter((s) => s.corpus === corpus);
|
|
100
|
+
if (!mine.some((s) => s.pid === myPid))
|
|
101
|
+
return { indexer: null, role: "indexer" };
|
|
102
|
+
mine.sort((a, b) => {
|
|
103
|
+
if (a.staleBuild !== b.staleBuild)
|
|
104
|
+
return a.staleBuild ? 1 : -1;
|
|
105
|
+
const t = a.started.localeCompare(b.started);
|
|
106
|
+
if (t !== 0)
|
|
107
|
+
return t;
|
|
108
|
+
return a.pid - b.pid;
|
|
109
|
+
});
|
|
110
|
+
const winner = mine[0];
|
|
111
|
+
return { indexer: winner.pid, role: winner.pid === myPid ? "indexer" : "reader" };
|
|
112
|
+
}
|
|
113
|
+
//# sourceMappingURL=shared-index.js.map
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@compr/opscontext-mcp",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.6.0",
|
|
4
4
|
"description": "OpsContext for AI Agents — read-only fleet visibility (PM2/nginx/Docker/git/cron) + tamper-evident audit log + policy-as-code hooks. The ops + compliance layer Claude Code can't grow natively.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "dist/index.js",
|