open-context-engine 0.1.2 → 0.1.4

package/src/service.mjs

@@ -5,21 +5,22 @@ import { homedir } from 'node:os';
 import { resolve, dirname, delimiter } from 'node:path';
 import { createInterface } from 'node:readline';
 import { projectRoot, loadEnvironment } from './config.mjs';
+import { defaultPython, venvPython } from './runtime.mjs';
 import { embeddingTransportConfig, remoteRerankerConfig, rerankerExecutionTransport } from './eval/remote-models.mjs';
 
 export function serviceConfig({root, state, port = 0} = {}, environment = process.env) {
   const env = loadEnvironment(environment);
   const embedding = embeddingTransportConfig(env);
   const reranker = remoteRerankerConfig(env), runtime = rerankerExecutionTransport(env);
-  const repository = root ? realpathSync(resolve(root)) : undefined;
+  const repository = root ? realpathSync.native(resolve(root)) : undefined;
   const workspaceId = repository && createHash('sha256').update(repository).digest('hex').slice(0, 24);
-  const candidates = ['.venv/bin/python', '.pilot-state/baselines/cocoindex-venv/bin/python',
-    '.pilot-state/language-adapters-venv/bin/python'].map(path => resolve(projectRoot, path));
+  const candidates = ['.venv', '.pilot-state/baselines/cocoindex-venv',
+    '.pilot-state/language-adapters-venv'].map(path => venvPython(resolve(projectRoot, path)));
   const currentState = repository && resolve(homedir(), '.cache/opencontextengine', workspaceId);
   const previousState = repository && resolve(homedir(), '.cache/reponerve', workspaceId);
   const defaultState = repository && (existsSync(currentState) ? currentState : existsSync(previousState) ? previousState : currentState);
   return {
-    python: env.OCE_PYTHON || candidates.find(existsSync) || 'python3',
+    python: env.OCE_PYTHON || candidates.find(existsSync) || defaultPython(),
     workerEnv: {...(env.OCE_GO_BINARY ? {OCE_GO_BINARY:env.OCE_GO_BINARY} : {}),
       ...(env.TIKTOKEN_CACHE_DIR ? {TIKTOKEN_CACHE_DIR:env.TIKTOKEN_CACHE_DIR} : {})},
     config: {root: repository, state: state ? resolve(state) : repository
@@ -29,6 +30,7 @@ export function serviceConfig({root, state, port = 0} = {}, environment = proces
     embeddingKey: env.EMBEDDING_API_KEY,
     embeddingModel: env.EMBEDDING_MODEL || 'Qwen3-Embedding-4B',
     embeddingDimensions: Number(env.OCE_EMBEDDING_DIMENSIONS || 1024),
+    embeddingBatchSize: Number(env.OCE_EMBEDDING_BATCH_SIZE ?? 64),
     embeddingRevision: env.OCE_EMBEDDING_REVISION || '1',
     reranker: {...reranker, baseUrl: runtime.requestBaseUrl},
     languageOptions: env.OCE_LANGUAGE_OPTIONS ? JSON.parse(env.OCE_LANGUAGE_OPTIONS) : {},
@@ -37,12 +39,13 @@ export function serviceConfig({root, state, port = 0} = {}, environment = proces
   };
 }
 
-export function startService(settings, {log = line => process.stderr.write(line + '\n')} = {}) {
+export function startService(settings, {log = line => process.stderr.write(line + '\n'), startupMs = 15000} = {}) {
   const {python, config} = settings;
   const child = spawn(python, [resolve(projectRoot, 'scripts/retrieval-server.py')], {
     cwd: projectRoot, env: {...process.env, ...settings.workerEnv,
-      PATH:dirname(process.execPath)+delimiter+(process.env.PATH || ''), OPENBLAS_NUM_THREADS:'2', OMP_NUM_THREADS:'2'},
-    stdio:['pipe', 'pipe', 'pipe'],
+      PATH:dirname(process.execPath)+delimiter+(process.env.PATH || ''),
+      PYTHONUTF8:'1', PYTHONIOENCODING:'utf-8', OPENBLAS_NUM_THREADS:'2', OMP_NUM_THREADS:'2'},
+    stdio:['pipe', 'pipe', 'pipe'], detached:Boolean(config.shared), windowsHide:true,
   });
   const lines = createInterface({input: child.stdout});
   const errors = createInterface({input: child.stderr});
@@ -50,14 +53,29 @@ export function startService(settings, {log = line => process.stderr.write(line
   child.stdin.on('error', () => {}); // Spawn/exit handlers report early failures.
   child.stdin.end(JSON.stringify(config) + '\n');
   const ready = new Promise((resolveReady, reject) => {
-    const timer = setTimeout(() => {child.kill(); reject(new Error('Retrieval worker startup timed out'));}, 15000);
+    const timer = setTimeout(() => {
+      child.kill();
+      reject(Object.assign(new Error('Retrieval worker startup timed out'), {code:'STARTUP_TIMEOUT'}));
+    }, startupMs);
     child.once('error', error => {clearTimeout(timer); reject(error);});
     child.once('exit', code => {clearTimeout(timer); reject(new Error(`Retrieval worker exited (${code})`));});
     lines.on('line', line => {
       try {
         const value = JSON.parse(line);
+        if (value.startupError) {
+          clearTimeout(timer);
+          const error = new Error(`Retrieval worker failed to start: ${value.startupError.message}`);
+          error.code = value.startupError.code;
+          reject(error);
+          return;
+        }
         if (value.listening) {
           clearTimeout(timer);
+          if (config.shared) {
+            lines.close(); errors.close();
+            child.stdout.destroy(); child.stderr.destroy();
+            child.unref();
+          }
           resolveReady({baseUrl:value.listening, apiKey:config.serviceKey});
           return;
         }

package/src/setup.mjs

@@ -2,7 +2,7 @@ import { createInterface } from 'node:readline/promises';
 import { Writable } from 'node:stream';
 import { configDirectory, loadEnvironment, saveUserConfig } from './config.mjs';
 import { remoteRerankerConfig, embeddingTransportConfig, rerankerExecutionTransport } from './eval/remote-models.mjs';
-import { ensureRuntime, run } from './runtime.mjs';
+import { ensureRuntime, run, defaultPython } from './runtime.mjs';
 
 export function validateModels(env) {
   embeddingTransportConfig(env);
@@ -13,6 +13,8 @@ export function validateModels(env) {
   }
   const dimensions = Number(env.OCE_EMBEDDING_DIMENSIONS);
   if (!Number.isInteger(dimensions) || dimensions < 1) throw new Error('Embedding dimensions must be a positive integer');
+  const batchSize = Number(env.OCE_EMBEDDING_BATCH_SIZE ?? 64);
+  if (!Number.isInteger(batchSize) || batchSize < 1 || batchSize > 64) throw new Error('Embedding batch size must be an integer from 1 to 64');
 }
 
 export function terminalPrompt(input = process.stdin, output = process.stderr) {
@@ -37,8 +39,9 @@ export function terminalPrompt(input = process.stdin, output = process.stderr) {
 }
 
 export async function setup({environment = process.env, nonInteractive = false, python,
-  prompt, install = ensureRuntime, execute = run, log = message => process.stderr.write(message+'\n')} = {}) {
-  if (!['darwin','linux'].includes(process.platform)) throw new Error('OpenContextEngine currently supports macOS and Linux');
+  platform = process.platform, prompt, install = ensureRuntime, execute = run,
+  log = message => process.stderr.write(message+'\n')} = {}) {
+  if (!['darwin','linux','win32'].includes(platform)) throw new Error('OpenContextEngine supports macOS, Linux and Windows');
   let env = {EMBEDDING_MODEL:'Qwen3-Embedding-4B',RERANK_MODEL:'Qwen3-Reranker-4B',
     OCE_EMBEDDING_DIMENSIONS:'1024',...loadEnvironment(environment)};
   log(`Configuration: ${configDirectory(environment)}`);
@@ -57,10 +60,10 @@ export async function setup({environment = process.env, nonInteractive = false,
   validateModels(env);
   await execute('git',['--version'],{timeout:10000});
   if (python) delete env.OCE_PYTHON;
-  const runtime = await install(env,{python:python || 'python3',execute,log});
+  const runtime = await install(env,{platform,python:python || defaultPython(platform),execute,log});
   env = {...env,...runtime};
   const path = saveUserConfig(env,environment);
-  log(`Saved configuration to ${path}. API keys are stored with owner-only file permissions.`);
+  log(`Saved configuration to ${path}.`);
   log('Setup complete. Go projects additionally require Go 1.22+ on PATH or OCE_GO_BINARY.');
   return {path,env};
 }

package/src/shared-service.mjs

@@ -0,0 +1,142 @@
+import { createHash, createHmac, randomBytes, randomUUID } from 'node:crypto';
+import { readFile, mkdir, realpath } from 'node:fs/promises';
+import { readFileSync, readdirSync } from 'node:fs';
+import { join } from 'node:path';
+import { setTimeout as delay } from 'node:timers/promises';
+import { projectRoot } from './config.mjs';
+import { startService } from './service.mjs';
+
+const protocol = 1;
+function stable(value) {
+  if (Array.isArray(value)) return value.map(stable);
+  if (value && typeof value === 'object') return Object.fromEntries(Object.keys(value).sort()
+    .filter(key => value[key] !== undefined).map(key => [key, stable(value[key])]));
+  return value;
+}
+function runtimeIdentity() {
+  const hash = createHash('sha256');
+  function add(path) {
+    for (const entry of readdirSync(join(projectRoot, path), {withFileTypes:true}).sort((a,b) => a.name.localeCompare(b.name))) {
+      const name = `${path}/${entry.name}`;
+      if (entry.isDirectory() && entry.name !== '__pycache__') add(name);
+      else if (entry.isFile() && /\.(py|mjs|go)$/.test(name)) hash.update(name).update(readFileSync(join(projectRoot, name)));
+    }
+  }
+  add('src/retrieval');
+  for (const name of ['scripts/retrieval-server.py', 'requirements.txt']) hash.update(readFileSync(join(projectRoot, name)));
+  hash.update(JSON.parse(readFileSync(join(projectRoot, 'package.json'))).version);
+  return hash.digest('hex');
+}
+
+// A handle owns a renewable lease, never the lifetime of the shared child process.
+export function startSharedService(settings, {startupMs = 15000, leaseSeconds = 15, idleSeconds = 30, start = startService} = {}) {
+  const runtime = runtimeIdentity();
+  let configured, connection, pending, timer, closed = false, closePromise;
+  const leaseId = randomUUID();
+  const initialized = (async () => {
+    await mkdir(settings.config.state, {recursive:true, mode:0o700});
+    const root = await realpath(settings.config.root), state = await realpath(settings.config.state);
+    const config = {...settings.config, root, state};
+    const {serviceKey, port, shared, ...behavior} = config;
+    const identity = JSON.stringify(stable({protocol, runtime, config:behavior,
+      python:settings.python, workerEnv:settings.workerEnv || {}}));
+    configured = {config, identity, record:join(state, 'worker.json')};
+  })();
+
+  function fingerprint(key) {
+    // Credentials participate in compatibility without a reusable plaintext/hash in discovery files.
+    return createHmac('sha256', key).update(configured.identity).digest('hex');
+  }
+  async function readRecord() {
+    let record;
+    try {record = JSON.parse(await readFile(configured.record, 'utf8'));}
+    catch (error) {
+      if (error.code === 'ENOENT' || error instanceof SyntaxError) return;
+      throw error;
+    }
+    if (!record || !Number.isInteger(record.port) || record.port < 1 || record.port > 65535 ||
+        typeof record.apiKey !== 'string' || !/^[a-f0-9]{64}$/.test(record.apiKey) ||
+        typeof record.instanceId !== 'string') return;
+    // Discovery cannot redirect credentials or requests off this machine.
+    return {...record, baseUrl:`http://127.0.0.1:${record.port}`};
+  }
+  async function lease(record, release = false) {
+    const response = await fetch(record.baseUrl + (release ? '/release' : '/lease'), {
+      method:'POST', redirect:'error', signal:AbortSignal.timeout(1500),
+      headers:{'content-type':'application/json', authorization:`Bearer ${record.apiKey}`},
+      body:JSON.stringify({instanceId:record.instanceId, fingerprint:fingerprint(record.apiKey), leaseId}),
+    });
+    const body = await response.json();
+    if (response.status === 409 && body.code === 'CONFIG_MISMATCH') {
+      throw Object.assign(new Error(body.error), {code:'CONFIG_MISMATCH'});
+    }
+    if (!response.ok || body.instanceId !== record.instanceId || record.protocol !== protocol ||
+        !Number.isFinite(body.leaseSeconds) || body.leaseSeconds <= 0) throw new Error('Shared worker handshake failed');
+    return {...record, leaseSeconds:body.leaseSeconds};
+  }
+  function schedule(record) {
+    clearTimeout(timer);
+    timer = setTimeout(() => {void get().catch(() => {});}, record.leaseSeconds * 1000 / 3);
+    timer.unref();
+  }
+  async function connect() {
+    await initialized;
+    const deadline = performance.now() + startupMs;
+    let nextSpawn = 0, lastBusy;
+    while (!closed && performance.now() < deadline) {
+      const record = connection || await readRecord();
+      if (record) {
+        try {
+          connection = await lease(record);
+          if (closed) {await lease(connection, true).catch(() => {}); break;}
+          schedule(connection);
+          return {baseUrl:connection.baseUrl, apiKey:connection.apiKey};
+        } catch (error) {
+          connection = undefined;
+          if (error.code === 'CONFIG_MISMATCH') throw error;
+        }
+      }
+      if (performance.now() >= nextSpawn) {
+        if (runtimeIdentity() !== runtime) throw new Error('Worker runtime changed on disk; restart this MCP client');
+        const key = randomBytes(32).toString('hex');
+        const worker = start({...settings, config:{...configured.config, port:0, serviceKey:key,
+          shared:{fingerprint:fingerprint(key), leaseSeconds, idleSeconds}}}, {log:() => {}, startupMs:Math.max(1, deadline - performance.now())});
+        try {await worker.ready;}
+        catch (error) {
+          await worker.close();
+          // A retry near the overall deadline may have too little time to import
+          // Python. Preserve the writer conflict already established by an earlier attempt.
+          if (error.code === 'STARTUP_TIMEOUT' && lastBusy) break;
+          if (error.code !== 'INDEX_LOCKED') throw error;
+          lastBusy = error;
+        }
+        nextSpawn = performance.now() + 1000;
+      }
+      await delay(100);
+    }
+    if (closed) throw new Error('MCP workspace manager is shutting down');
+    throw new Error(lastBusy
+      ? `${lastBusy.message}. No compatible shared worker became available. An older or manually started worker may own this index; close its clients or connect to it explicitly.`
+      : 'Shared retrieval worker startup or connection timed out');
+  }
+  function get() {
+    if (closed) return Promise.reject(new Error('MCP workspace manager is shutting down'));
+    if (!pending) {
+      pending = connect().finally(() => {pending = undefined;});
+    }
+    return pending;
+  }
+  function close() {
+    if (!closePromise) {
+      closed = true;
+      clearTimeout(timer);
+      closePromise = (async () => {
+        await pending?.catch(() => {});
+        if (connection) await lease(connection, true).catch(() => {});
+        connection = undefined;
+      })();
+    }
+    return closePromise;
+  }
+  return {ready:get(), get, close};
+}

package/src/workspaces.mjs

@@ -1,16 +1,17 @@
 import { realpathSync, statSync } from 'node:fs';
 import { isAbsolute, resolve, join } from 'node:path';
 import { createHash } from 'node:crypto';
-import { serviceConfig, startService } from './service.mjs';
+import { serviceConfig } from './service.mjs';
+import { startSharedService } from './shared-service.mjs';
 
 function directory(path) {
-  const canonical = realpathSync(path);
+  const canonical = realpathSync.native(path);
   if (!statSync(canonical).isDirectory()) throw new Error('directory_path must point to a directory');
   return canonical;
 }
 
-// Each canonical repository owns one worker, including while it is starting.
-export function createWorkspaceManager({root, state} = {}, {configure = serviceConfig, start = startService} = {}) {
+// Each manager holds one lease per canonical repository, including while connecting.
+export function createWorkspaceManager({root, state} = {}, {configure = serviceConfig, start = startSharedService} = {}) {
   const fixedRoot = root ? directory(resolve(root)) : undefined;
   const workers = new Map();
   let closing = false, closed;
@@ -35,13 +36,14 @@ export function createWorkspaceManager({root, state} = {}, {configure = serviceC
       entry = {worker};
       workers.set(repository, entry);
       const remove = () => {if (workers.get(repository) === entry) workers.delete(repository);};
-      worker.child.once('exit', remove);
+      worker.child?.once('exit', remove);
       entry.ready = worker.ready.catch(async error => {
         try {await worker.close();} finally {remove();}
         throw error;
       });
     }
-    return entry.ready;
+    await entry.ready;
+    return entry.worker.get ? entry.worker.get() : entry.ready;
   }
 
   function close() {