open-context-engine 0.1.0

package/src/retrieval/live.py

@@ -0,0 +1,273 @@
+"""Single-workspace index: durable vector reuse and atomic searchable generations."""
+from collections import Counter
+from contextlib import closing
+from dataclasses import dataclass
+import fcntl
+import hashlib
+import json
+import os
+from pathlib import Path
+import shutil
+import sqlite3
+import threading
+import time
+import uuid
+
+import numpy as np
+
+from engine import document, post
+from languages import adapter_manifest, source_units
+from languages.files import discover_snapshot
+from routed import RoutedEngine
+
+
+def digest(value):
+    return hashlib.sha256(json.dumps(value, sort_keys=True).encode()).hexdigest()
+
+
+class IndexUnavailable(Exception):
+    """The requested working tree has no complete, current searchable generation."""
+
+
+class SourceChanged(Exception):
+    pass
+
+
+@dataclass(frozen=True)
+class Generation:
+    identity: str
+    snapshot: dict
+    info: dict
+    engine: object
+
+
+class LiveIndex:
+    def __init__(self, config, *, embed=post, engine_factory=RoutedEngine):
+        self.config = config
+        self.root = Path(config['root']).resolve()
+        self.state = Path(config['state']).resolve()
+        if not self.root.is_dir():
+            raise ValueError('Repository root must be a directory')
+        if self.state.is_relative_to(self.root):
+            raise ValueError('Live index state must be outside the repository root')
+        self.options = config.get('languageOptions', {})
+        self.poll = float(config.get('pollSeconds', 1))
+        self.debounce = float(config.get('debounceSeconds', .3))
+        if not .05 <= self.poll <= 60 or not 0 <= self.debounce <= 10:
+            raise ValueError('Invalid live index polling or debounce interval')
+        self.embedding = {'provider': config['embeddingIdentity'],
+                          'model': config.get('embeddingModel', 'Qwen3-Embedding-4B'),
+                          'dimensions': config.get('embeddingDimensions', 1024),
+                          'revision': config.get('embeddingRevision', '1')}
+        if not isinstance(self.embedding['model'], str) or not self.embedding['model']:
+            raise ValueError('Invalid embedding model')
+        self.dimensions = self.embedding['dimensions']
+        if type(self.dimensions) is not int or not 1 <= self.dimensions <= 65536:
+            raise ValueError('Invalid embedding dimensions')
+        self.embed, self.engine_factory = embed, engine_factory
+        self.condition = threading.Condition()
+        self.stop_event = threading.Event()
+        self.generation = None
+        self.error = None
+        self.phase = 'starting'
+        self.parse_cache = {}
+        self.state.mkdir(parents=True, exist_ok=True, mode=0o700)
+        self.lock_file = (self.state/'writer.lock').open('a')
+        try:
+            fcntl.flock(self.lock_file, fcntl.LOCK_EX | fcntl.LOCK_NB)
+        except OSError:
+            self.lock_file.close()
+            raise ValueError('This index directory already has a running writer') from None
+        self.thread = threading.Thread(target=self._run, name='repository-index', daemon=True)
+
+    def scan(self):
+        snapshot = discover_snapshot(self.root)
+        # Ignore diagnostic exclusions and mtime: identities bind actual inputs.
+        snapshot = {'files': [{'path': f['path'], 'sha256': f['sha256']} for f in snapshot['files']],
+                    'languageOptions': self.options}
+        return snapshot
+
+    def identity(self, snapshot):
+        return digest({'schema': 'live-index-v1', 'root': str(self.root), 'snapshot': snapshot,
+                       'adapters': adapter_manifest(snapshot['files'], self.options),
+                       'embedding': self.embedding})
+
+    def start(self):
+        self.thread.start()
+        return self
+
+    def close(self):
+        self.stop_event.set()
+        with self.condition:
+            self.condition.notify_all()
+        if self.thread.is_alive():
+            self.thread.join(timeout=5)
+        # An in-flight model call may finish later. Keep its writer lock until
+        # the worker exits, rather than allowing another process to overlap it.
+        if not self.thread.is_alive() and not self.lock_file.closed:
+            self.lock_file.close()
+
+    def status(self):
+        with self.condition:
+            return {'status': self.phase, 'mode': 'live', 'root': str(self.root),
+                    'generation': self.generation.info if self.generation else None,
+                    'error': self.error, 'pollSeconds': self.poll}
+
+    def current(self, timeout=30):
+        deadline = time.monotonic() + timeout
+        while not self.stop_event.is_set():
+            try:
+                target = self.identity(self.scan())
+            except (OSError, ValueError) as error:
+                raise IndexUnavailable('Cannot read current source: ' + type(error).__name__) from None
+            with self.condition:
+                if self.generation and self.generation.identity == target:
+                    return self.generation
+                if self.error and self.error['identity'] == target:
+                    raise IndexUnavailable('Index update failed: ' + self.error['type'])
+                remaining = deadline - time.monotonic()
+                if remaining <= 0:
+                    raise IndexUnavailable('Index update pending; retry after synchronization')
+                self.condition.wait(min(remaining, self.poll))
+        raise IndexUnavailable('Index is stopping')
+
+    def verify(self, generation):
+        try:
+            current = self.identity(self.scan()) == generation.identity
+        except (OSError, ValueError):
+            current = False
+        if not current:
+            raise IndexUnavailable('Source changed during search; retry for current code')
+
+    def _engine(self, units, vectors):
+        if not units:
+            return None
+        return self.engine_factory(units, vectors, self.config['embeddingUrl'],
+                                   self.config['reranker'], self.config.get('embeddingKey', 'local-only'),
+                                   embedding_model=self.embedding['model'])
+
+    def _restore(self, snapshot, identity):
+        pointer = self.state/'current.json'
+        if not pointer.exists():
+            return None
+        try:
+            saved = json.loads(pointer.read_text())
+            name = saved['directory']
+            if name != Path(name).name or not name.startswith('generation-'):
+                return None
+            folder = self.state/name
+            info = json.loads((folder/'metadata.json').read_text())
+            if info['identity'] != identity:
+                return None
+            units = json.loads((folder/'units.json').read_text())
+            vectors = np.load(folder/'vectors.npy', allow_pickle=False)
+            if vectors.shape != (len(units), self.dimensions) or not np.isfinite(vectors).all():
+                return None
+            return Generation(identity, snapshot, info, self._engine(units, vectors))
+        except (OSError, ValueError, KeyError):
+            return None
+
+    def _build(self, snapshot, identity):
+        start = time.monotonic()
+        units = source_units(self.root, snapshot['files'], language_options=self.options, cache=self.parse_cache)
+        documents = [document(unit) for unit in units]
+        keys = [digest([self.embedding, text]) for text in documents]
+        vectors, missing = {}, {}
+        with closing(sqlite3.connect(self.state/'embeddings.sqlite')) as db:
+            db.execute('CREATE TABLE IF NOT EXISTS vectors (key TEXT PRIMARY KEY, value BLOB NOT NULL)')
+            for key, text in zip(keys, documents):
+                row = db.execute('SELECT value FROM vectors WHERE key=?', (key,)).fetchone()
+                if row:
+                    value = np.frombuffer(row[0], dtype=np.float32)
+                    if value.shape == (self.dimensions,) and np.isfinite(value).all():
+                        vectors[key] = value
+                        continue
+                missing[key] = text
+            entries = list(missing.items())
+            for offset in range(0, len(entries), 64):
+                if self.stop_event.is_set():
+                    raise SourceChanged()
+                batch = entries[offset:offset+64]
+                result = self.embed(self.config['embeddingUrl']+'/embeddings',
+                    {'model': self.embedding['model'], 'input': [text for _, text in batch]},
+                    self.config.get('embeddingKey', 'local-only'), timeout=60)
+                rows = sorted(result['data'], key=lambda row: row['index'])
+                if [row['index'] for row in rows] != list(range(len(batch))):
+                    raise ValueError('Invalid embedding response indices')
+                matrix = np.asarray([row['embedding'] for row in rows], dtype=np.float32)
+                if matrix.shape != (len(batch), self.dimensions) or not np.isfinite(matrix).all():
+                    raise ValueError('Invalid embedding vectors')
+                for (key, _), vector in zip(batch, matrix):
+                    vectors[key] = vector
+                    db.execute('INSERT OR REPLACE INTO vectors VALUES (?, ?)', (key, vector.tobytes()))
+                db.commit()  # Reuse successful batches even after a concurrent edit.
+        matrix = np.asarray([vectors[key] for key in keys], dtype=np.float32).reshape(len(keys), self.dimensions)
+        engine = self._engine(units, matrix)
+        if self.stop_event.is_set() or self.scan() != snapshot:
+            raise SourceChanged()
+        previous = {f['path']: f['sha256'] for f in self.generation.snapshot['files']} if self.generation else {}
+        now = {f['path']: f['sha256'] for f in snapshot['files']}
+        info = {'identity': identity, 'files': len(now), 'units': len(units),
+                'embeddedDocuments': len(missing), 'reusedUnits': sum(key not in missing for key in keys),
+                'changedFiles': sum(previous.get(path) != sha for path, sha in now.items()),
+                'deletedFiles': len(previous.keys() - now.keys()), 'embedding': self.embedding,
+                'languageUnits': dict(Counter(unit['language'] for unit in units)),
+                'indexingMs': round((time.monotonic()-start)*1000), 'completedAt': time.time()}
+        folder = self.state/('generation-'+uuid.uuid4().hex)
+        folder.mkdir(mode=0o700)
+        try:
+            (folder/'units.json').write_text(json.dumps(units))
+            np.save(folder/'vectors.npy', matrix)
+            (folder/'metadata.json').write_text(json.dumps(info))
+            pointer = self.state/'current.tmp'
+            pointer.write_text(json.dumps({'directory': folder.name}))
+            os.replace(pointer, self.state/'current.json')
+        except BaseException:
+            shutil.rmtree(folder)
+            raise
+        # Search retains its own immutable in-memory generation.
+        for old in self.state.glob('generation-*'):
+            if old != folder and old.is_dir():
+                shutil.rmtree(old)
+        return Generation(identity, snapshot, info, engine)
+
+    def _run(self):
+        failed_identity, retry_at = None, 0
+        try:
+            while not self.stop_event.is_set():
+                identity = None
+                try:
+                    snapshot = self.scan()
+                    identity = self.identity(snapshot)
+                    if self.generation and self.generation.identity == identity:
+                        with self.condition:
+                            self.phase, self.error = 'ready', None
+                        self.stop_event.wait(self.poll)
+                        continue
+                    if failed_identity == identity and time.monotonic() < retry_at:
+                        self.stop_event.wait(self.poll)
+                        continue
+                    with self.condition:
+                        self.phase, self.error = 'updating', None
+                    if self.stop_event.wait(self.debounce):
+                        break
+                    if self.scan() != snapshot:
+                        continue
+                    generation = self._restore(snapshot, identity) or self._build(snapshot, identity)
+                    if self.scan() != snapshot:
+                        raise SourceChanged()
+                    with self.condition:
+                        self.generation, self.phase, self.error = generation, 'ready', None
+                        failed_identity = None
+                        self.condition.notify_all()
+                except SourceChanged:
+                    continue
+                except Exception as error:
+                    failed_identity, retry_at = identity, time.monotonic() + max(2, self.poll)
+                    with self.condition:
+                        self.phase = 'failed'
+                        self.error = {'identity': identity, 'type': type(error).__name__}
+                        self.condition.notify_all()
+                    self.stop_event.wait(self.poll)
+        finally:
+            self.lock_file.close()

package/src/retrieval/reranker.py

@@ -0,0 +1,83 @@
+"""Normalize ordinary rerank and optional multi-query batch responses.
+
+Only requested pairs are sent. Result indices always refer to the submitted
+documents, never their relevance-sorted position in a provider response.
+"""
+from collections import defaultdict
+from concurrent.futures import ThreadPoolExecutor
+import math
+
+
+def bounded_integer(value, name, maximum):
+    if type(value) is not int or not 1 <= value <= maximum:
+        raise ValueError(f'{name} must be an integer from 1 to {maximum}')
+    return value
+
+
+def checked_rows(data, count):
+    rows = data.get('results') if isinstance(data, dict) else None
+    if not isinstance(rows, list) or len(rows) != count:
+        raise ValueError('Incomplete rerank response')
+    indexed = {}
+    for row in rows:
+        if not isinstance(row, dict):
+            raise ValueError('Invalid rerank result')
+        index, score = row.get('index'), row.get('relevance_score')
+        if type(index) is not int or not 0 <= index < count or index in indexed:
+            raise ValueError('Invalid rerank result index')
+        if type(score) not in (int, float) or not math.isfinite(score) or not 0 <= score <= 1:
+            raise ValueError('Invalid rerank relevance score')
+        indexed[index] = row
+    return [indexed[i] for i in range(count)]
+
+
+def rerank_pairs(config, queries, documents, pairs, post):
+    api = config.get('api', 'rerank')
+    if api not in ('rerank', 'rerank-batch'):
+        raise ValueError('Rerank API must be rerank or rerank-batch')
+    concurrency = bounded_integer(config.get('concurrency', 2), 'Rerank concurrency', 8)
+    limit = bounded_integer(config.get('maxDocuments', 128), 'Rerank document limit', 1024)
+    for q, d in pairs:
+        if type(q) is not int or type(d) is not int or not 0 <= q < len(queries) or not 0 <= d < len(documents):
+            raise ValueError('Invalid requested rerank pair')
+    if not pairs:
+        return {'results': [], 'meta': {'request_count': 0}}
+    url = config['baseUrl'].rstrip('/') + '/' + api
+    if api == 'rerank-batch':
+        data = post(url, {'model': config['model'], 'queries': queries,
+                         'documents': documents, 'pairs': pairs}, config['apiKey'])
+        rows = checked_rows(data, len(pairs))
+        for row, (q, d) in zip(rows, pairs):
+            if (type(row.get('query_index')) is not int or type(row.get('document_index')) is not int
+                    or (row['query_index'], row['document_index']) != (q, d)):
+                raise ValueError('Invalid neural pair mapping')
+        return {**data, 'results': rows, 'meta': {**data.get('meta', {}), 'request_count': 1}}
+
+    grouped = defaultdict(dict)
+    for q, d in pairs:
+        grouped[q][d] = None
+    jobs = [(q, ids[start:start + limit]) for q, group in grouped.items()
+            for ids in [list(group)] for start in range(0, len(ids), limit)]
+
+    def request(job):
+        q, ids = job
+        data = post(url, {'model': config['model'], 'query': queries[q],
+                         'documents': [documents[d] for d in ids], 'top_n': len(ids)}, config['apiKey'])
+        rows = checked_rows(data, len(ids))
+        return {(q, ids[row['index']]): row['relevance_score'] for row in rows}, data
+
+    # Fail the whole wave on any error; never silently drop a query or switch API.
+    if len(jobs) == 1:
+        responses = [request(jobs[0])]
+    else:
+        with ThreadPoolExecutor(max_workers=min(concurrency, len(jobs))) as pool:
+            responses = list(pool.map(request, jobs))
+    scores = {pair: score for mapping, _ in responses for pair, score in mapping.items()}
+    def total(section, key):
+        values = [data[section].get(key) if isinstance(data.get(section), dict) else None
+                  for _, data in responses]
+        return sum(values) if all(type(v) in (int, float) and math.isfinite(v) and v >= 0 for v in values) else None
+    return {'results': [{'index': i, 'query_index': q, 'document_index': d,
+                         'relevance_score': scores[q, d]} for i, (q, d) in enumerate(pairs)],
+            'usage': {'input_tokens': total('usage', 'input_tokens')},
+            'meta': {'request_count': len(jobs), 'elapsed_ms': total('meta', 'elapsed_ms')}}

package/src/retrieval/routed.py

@@ -0,0 +1,35 @@
+"""Shared intent scoring with dense facet affinity and scored graph expansion.
+
+Every candidate is scored against the complete question, so fragments containing
+pronouns never lose their context. Facet affinity distributes its packing value.
+"""
+import numpy as np
+from batched import BatchedEngine
+
+VERSION = 'shared-intent-v4'
+
+
+class RoutedEngine(BatchedEngine):
+    version = VERSION
+    # Shared-intent values are lower than independently maximized facet scores.
+    # Only fill the remaining budget after all v2 selections; never displace them.
+    min_gain = .005
+
+    def scoring_policy(self, cache, queries, second_wave):
+        confidence = max((value for (query, _), value in cache.items() if query == queries[0]), default=0)
+        return {'fullFacets': second_wave and len(set(queries)) > 1 and confidence < .1}
+
+    def scoring_pairs(self, requested, queries, dense, policy):
+        if policy['fullFacets']:
+            return requested
+        return [(queries[0], uid) for _, uid in requested]
+
+    def pair_score(self, cache, query, uid, queries, dense, policy):
+        if policy['fullFacets']:
+            return cache[(query, uid)]
+        score = cache[(queries[0], uid)]
+        if query == queries[0]:
+            return score
+        col = queries.index(query)
+        affinity = np.exp(min(0, dense[uid, col] - max(dense[uid, 1:])) / .06)
+        return score * (.35 + .65 * float(affinity))

package/src/runtime.mjs

@@ -0,0 +1,60 @@
+import { execFile } from 'node:child_process';
+import { promisify } from 'node:util';
+import { readFile, mkdir, mkdtemp, writeFile, rm } from 'node:fs/promises';
+import { join, dirname, relative, isAbsolute } from 'node:path';
+import { createHash } from 'node:crypto';
+import { configDirectory, projectRoot } from './config.mjs';
+
+const exec = promisify(execFile);
+export async function run(command, args, options = {}) {
+  try {return await exec(command,args,{timeout:600000,maxBuffer:4*1024*1024,...options});}
+  catch (error) {
+    // Do not echo pip output: configured package indexes may contain credentials.
+    throw new Error(`${command} failed (${error.code || 'unknown'}). Check the executable, network, and Python venv/pip support.`);
+  }
+}
+const probe = 'import sys, importlib.metadata as m; assert sys.version_info >= (3,10); import numpy,tiktoken; '
+  + 'n=tuple(map(int,m.version("numpy").split(".")[:2])); t=tuple(map(int,m.version("tiktoken").split(".")[:2])); '
+  + 'assert (2,2)<=n<(3,0) and (0,9)<=t<(1,0); tiktoken.get_encoding("cl100k_base"); print(sys.version.split()[0])';
+
+export async function verifyPython(python, environment = process.env, execute = run) {
+  return (await execute(python,['-c',probe],{env:environment,timeout:120000})).stdout.trim();
+}
+
+export async function ensureRuntime(environment, {python = 'python3', execute = run, log = () => {}} = {}) {
+  const runtimes = join(configDirectory(environment),'runtimes');
+  const requirements = await readFile(join(projectRoot,'requirements.txt'));
+  const fingerprint = createHash('sha256').update(requirements).digest('hex');
+  const current = environment.OCE_PYTHON;
+  const child = current && relative(runtimes,current);
+  if (current && child && !child.startsWith('..') && !isAbsolute(child)) {
+    try {
+      const directory = dirname(dirname(current));
+      const manifest = JSON.parse(await readFile(join(directory,'ready.json'),'utf8'));
+      if (manifest.requirementsSha256 === fingerprint) {
+        const env = {...environment,TIKTOKEN_CACHE_DIR:join(directory,'tokenizer')};
+        await verifyPython(current,env,execute);
+        log('Reusing the installed Python runtime.');
+        return {OCE_PYTHON:current,TIKTOKEN_CACHE_DIR:env.TIKTOKEN_CACHE_DIR};
+      }
+    } catch { /* Rebuild separately; the existing runtime and config remain intact. */ }
+  }
+  await execute(python,['-c','import sys; assert sys.version_info >= (3,10), "Python 3.10+ required"'],{timeout:10000});
+  await mkdir(runtimes,{recursive:true,mode:0o700});
+  // Virtual environments cannot be relocated. Each installation is built in its final directory.
+  const directory = await mkdtemp(join(runtimes,'python-'));
+  const executable = join(directory,'bin','python');
+  const env = {...environment,TIKTOKEN_CACHE_DIR:join(directory,'tokenizer')};
+  try {
+    log('Creating an isolated Python environment...');
+    await execute(python,['-m','venv',directory]);
+    log('Installing NumPy and tiktoken (no model weights)...');
+    await execute(executable,['-m','pip','install','--disable-pip-version-check','--no-input','-r',join(projectRoot,'requirements.txt')]);
+    await verifyPython(executable,env,execute);
+    await writeFile(join(directory,'ready.json'),JSON.stringify({requirementsSha256:fingerprint})+'\n',{mode:0o600});
+    return {OCE_PYTHON:executable,TIKTOKEN_CACHE_DIR:env.TIKTOKEN_CACHE_DIR};
+  } catch (error) {
+    await rm(directory,{recursive:true,force:true});
+    throw error;
+  }
+}

package/src/service.mjs

@@ -0,0 +1,77 @@
+import { spawn } from 'node:child_process';
+import { existsSync, realpathSync } from 'node:fs';
+import { createHash, randomBytes } from 'node:crypto';
+import { homedir } from 'node:os';
+import { resolve, dirname, delimiter } from 'node:path';
+import { createInterface } from 'node:readline';
+import { projectRoot, loadEnvironment } from './config.mjs';
+import { embeddingTransportConfig, remoteRerankerConfig, rerankerExecutionTransport } from './eval/remote-models.mjs';
+
+export function serviceConfig({root, state, port = 0} = {}, environment = process.env) {
+  const env = loadEnvironment(environment);
+  const embedding = embeddingTransportConfig(env);
+  const reranker = remoteRerankerConfig(env), runtime = rerankerExecutionTransport(env);
+  const repository = root ? realpathSync(resolve(root)) : undefined;
+  const workspaceId = repository && createHash('sha256').update(repository).digest('hex').slice(0, 24);
+  const candidates = ['.venv/bin/python', '.pilot-state/baselines/cocoindex-venv/bin/python',
+    '.pilot-state/language-adapters-venv/bin/python'].map(path => resolve(projectRoot, path));
+  const currentState = repository && resolve(homedir(), '.cache/opencontextengine', workspaceId);
+  const previousState = repository && resolve(homedir(), '.cache/reponerve', workspaceId);
+  const defaultState = repository && (existsSync(currentState) ? currentState : existsSync(previousState) ? previousState : currentState);
+  return {
+    python: env.OCE_PYTHON || candidates.find(existsSync) || 'python3',
+    workerEnv: {...(env.OCE_GO_BINARY ? {OCE_GO_BINARY:env.OCE_GO_BINARY} : {}),
+      ...(env.TIKTOKEN_CACHE_DIR ? {TIKTOKEN_CACHE_DIR:env.TIKTOKEN_CACHE_DIR} : {})},
+    config: {root: repository, state: state ? resolve(state) : repository
+      ? defaultState : resolve(projectRoot, '.pilot-state/reponerve/index'),
+    serviceKey: env.OCE_API_KEY || randomBytes(32).toString('hex'), port,
+    embeddingUrl: embedding.requestBaseUrl, embeddingIdentity: embedding.baseUrl,
+    embeddingKey: env.EMBEDDING_API_KEY,
+    embeddingModel: env.EMBEDDING_MODEL || 'Qwen3-Embedding-4B',
+    embeddingDimensions: Number(env.OCE_EMBEDDING_DIMENSIONS || 1024),
+    embeddingRevision: env.OCE_EMBEDDING_REVISION || '1',
+    reranker: {...reranker, baseUrl: runtime.requestBaseUrl},
+    languageOptions: env.OCE_LANGUAGE_OPTIONS ? JSON.parse(env.OCE_LANGUAGE_OPTIONS) : {},
+    pollSeconds: Number(env.OCE_POLL_SECONDS || 1),
+    debounceSeconds: Number(env.OCE_DEBOUNCE_SECONDS || .3)},
+  };
+}
+
+export function startService(settings, {log = line => process.stderr.write(line + '\n')} = {}) {
+  const {python, config} = settings;
+  const child = spawn(python, [resolve(projectRoot, 'scripts/retrieval-server.py')], {
+    cwd: projectRoot, env: {...process.env, ...settings.workerEnv,
+      PATH:dirname(process.execPath)+delimiter+(process.env.PATH || ''), OPENBLAS_NUM_THREADS:'2', OMP_NUM_THREADS:'2'},
+    stdio:['pipe', 'pipe', 'pipe'],
+  });
+  const lines = createInterface({input: child.stdout});
+  const errors = createInterface({input: child.stderr});
+  errors.on('line', log);
+  child.stdin.on('error', () => {}); // Spawn/exit handlers report early failures.
+  child.stdin.end(JSON.stringify(config) + '\n');
+  const ready = new Promise((resolveReady, reject) => {
+    const timer = setTimeout(() => {child.kill(); reject(new Error('Retrieval worker startup timed out'));}, 15000);
+    child.once('error', error => {clearTimeout(timer); reject(error);});
+    child.once('exit', code => {clearTimeout(timer); reject(new Error(`Retrieval worker exited (${code})`));});
+    lines.on('line', line => {
+      try {
+        const value = JSON.parse(line);
+        if (value.listening) {
+          clearTimeout(timer);
+          resolveReady({baseUrl:value.listening, apiKey:config.serviceKey});
+          return;
+        }
+      } catch { /* Non-protocol worker logs belong on stderr. */ }
+      log(line);
+    });
+  });
+  async function close() {
+    if (!child.pid || child.exitCode !== null || child.signalCode !== null) return;
+    await new Promise(resolveClosed => {
+      const timer = setTimeout(() => child.kill('SIGKILL'), 5000);
+      child.once('exit', () => {clearTimeout(timer); resolveClosed();});
+      child.kill('SIGTERM');
+    });
+  }
+  return {child, ready, close};
+}

package/src/setup.mjs

@@ -0,0 +1,66 @@
+import { createInterface } from 'node:readline/promises';
+import { Writable } from 'node:stream';
+import { configDirectory, loadEnvironment, saveUserConfig } from './config.mjs';
+import { remoteRerankerConfig, embeddingTransportConfig, rerankerExecutionTransport } from './eval/remote-models.mjs';
+import { ensureRuntime, run } from './runtime.mjs';
+
+export function validateModels(env) {
+  embeddingTransportConfig(env);
+  remoteRerankerConfig(env);
+  rerankerExecutionTransport(env);
+  for (const key of ['EMBEDDING_API_KEY','EMBEDDING_MODEL']) {
+    if (!env[key]?.trim()) throw new Error(`Set ${key}`);
+  }
+  const dimensions = Number(env.OCE_EMBEDDING_DIMENSIONS);
+  if (!Number.isInteger(dimensions) || dimensions < 1) throw new Error('Embedding dimensions must be a positive integer');
+}
+
+export function terminalPrompt(input = process.stdin, output = process.stderr) {
+  if (!input.isTTY || !output.isTTY) throw new Error('Run setup in a terminal, or use --non-interactive with model environment variables');
+  let muted = false;
+  const sink = new Writable({write(chunk,encoding,callback) {if (!muted) output.write(chunk,encoding); callback();}});
+  sink.isTTY = true; sink.columns = output.columns;
+  const terminal = createInterface({input,output:sink,terminal:true});
+  const abort = new AbortController();
+  terminal.on('SIGINT',() => abort.abort());
+  return {
+    async ask(label,current = '',secret = false) {
+      const hint = current ? (secret ? ' [saved; Enter to keep]' : ` [${current}]`) : '';
+      if (!secret) return (await terminal.question(`${label}${hint}: `,{signal:abort.signal})).trim() || current;
+      output.write(`${label}${hint}: `);
+      muted = true;
+      try {return (await terminal.question('',{signal:abort.signal})).trim() || current;}
+      finally {muted = false; output.write('\n');}
+    },
+    close() {terminal.close();},
+  };
+}
+
+export async function setup({environment = process.env, nonInteractive = false, python,
+  prompt, install = ensureRuntime, execute = run, log = message => process.stderr.write(message+'\n')} = {}) {
+  if (!['darwin','linux'].includes(process.platform)) throw new Error('OpenContextEngine currently supports macOS and Linux');
+  let env = {EMBEDDING_MODEL:'Qwen3-Embedding-4B',RERANK_MODEL:'Qwen3-Reranker-4B',
+    OCE_EMBEDDING_DIMENSIONS:'1024',...loadEnvironment(environment)};
+  log(`Configuration: ${configDirectory(environment)}`);
+  if (!nonInteractive) {
+    const questions = prompt || terminalPrompt();
+    try {
+      for (const [key,label,secret] of [
+        ['EMBEDDING_BASE_URL','Embedding base URL (including /v1)'],
+        ['EMBEDDING_API_KEY','Embedding API key',true],['EMBEDDING_MODEL','Embedding model'],
+        ['OCE_EMBEDDING_DIMENSIONS','Embedding dimensions'],
+        ['RERANK_BASE_URL','Rerank base URL (before /rerank; include a version prefix if required)'],
+        ['RERANK_API_KEY','Rerank API key',true],['RERANK_MODEL','Rerank model'],
+      ]) env[key] = await questions.ask(label,env[key] || '',Boolean(secret));
+    } finally {questions.close();}
+  }
+  validateModels(env);
+  await execute('git',['--version'],{timeout:10000});
+  if (python) delete env.OCE_PYTHON;
+  const runtime = await install(env,{python:python || 'python3',execute,log});
+  env = {...env,...runtime};
+  const path = saveUserConfig(env,environment);
+  log(`Saved configuration to ${path}. API keys are stored with owner-only file permissions.`);
+  log('Setup complete. Go projects additionally require Go 1.22+ on PATH or OCE_GO_BINARY.');
+  return {path,env};
+}

package/src/workspaces.mjs

@@ -0,0 +1,59 @@
+import { realpathSync, statSync } from 'node:fs';
+import { isAbsolute, resolve, join } from 'node:path';
+import { createHash } from 'node:crypto';
+import { serviceConfig, startService } from './service.mjs';
+
+function directory(path) {
+  const canonical = realpathSync(path);
+  if (!statSync(canonical).isDirectory()) throw new Error('directory_path must point to a directory');
+  return canonical;
+}
+
+// Each canonical repository owns one worker, including while it is starting.
+export function createWorkspaceManager({root, state} = {}, {configure = serviceConfig, start = startService} = {}) {
+  const fixedRoot = root ? directory(resolve(root)) : undefined;
+  const workers = new Map();
+  let closing = false, closed;
+
+  async function get(directoryPath) {
+    if (closing) throw new Error('MCP workspace manager is shutting down');
+    if (directoryPath !== undefined && (typeof directoryPath !== 'string' || !isAbsolute(directoryPath))) {
+      throw new Error('directory_path must be an absolute project directory');
+    }
+    if (!directoryPath && !fixedRoot) {
+      throw new Error('Pass directory_path with the absolute path of the project to search, or start MCP with --root');
+    }
+    const repository = directoryPath ? directory(directoryPath) : fixedRoot;
+    if (fixedRoot && repository !== fixedRoot) {
+      throw new Error('This MCP server is fixed to --root; omit --root at startup to search multiple projects');
+    }
+    let entry = workers.get(repository);
+    if (!entry) {
+      const workspaceState = state && (fixedRoot ? state : join(resolve(state),
+        createHash('sha256').update(repository).digest('hex').slice(0, 24)));
+      const worker = start(configure({root:repository, state:workspaceState}));
+      entry = {worker};
+      workers.set(repository, entry);
+      const remove = () => {if (workers.get(repository) === entry) workers.delete(repository);};
+      worker.child.once('exit', remove);
+      entry.ready = worker.ready.catch(async error => {
+        try {await worker.close();} finally {remove();}
+        throw error;
+      });
+    }
+    return entry.ready;
+  }
+
+  function close() {
+    if (!closed) {
+      closing = true;
+      closed = Promise.allSettled([...workers.values()].map(entry => entry.worker.close())).then(results => {
+        workers.clear();
+        const errors = results.filter(result => result.status === 'rejected').map(result => result.reason);
+        if (errors.length) throw new AggregateError(errors, 'Failed to close repository workers');
+      });
+    }
+    return closed;
+  }
+  return {get, close};
+}