graph-indexer 2.1.0 → 3.0.0

package/daemon-ctl.mjs

@@ -1,175 +0,0 @@
-#!/usr/bin/env node
-/**
- * @file daemon-ctl.mjs
- * @description `idx-daemon` — control the graph-indexer watch daemon from the
- *              command line (and therefore from npm scripts): start, stop,
- *              restart, status, logs. Exactly one daemon runs per project; the
- *              daemon self-guards via an atomic PID lock (daemon-lock.mjs), so
- *              `start` is safe to call repeatedly and from any launcher.
- *
- *                idx-daemon start      Start it (no-op if already running)
- *                idx-daemon stop       Gracefully stop it
- *                idx-daemon restart    Stop then start
- *                idx-daemon status     Show daemon + index state (default)
- *                idx-daemon logs [-f]  Print recent daemon logs (-f to follow)
- * @author MaquinaTech <https://github.com/MaquinaTech>
- * @copyright (c) 2026 MaquinaTech. All rights reserved.
- * @license MIT
- */
-import fs from 'fs';
-import path from 'path';
-import { fileURLToPath } from 'url';
-import { spawn } from 'child_process';
-import { resolveConfig } from './config.mjs';
-import { resolveBackend } from './storage.mjs';
-import { ensureDataDir, migrateLegacyLayout } from './layout.mjs';
-import { daemonStatus, readPid, isAlive } from './daemon-lock.mjs';
-import { c, glyph, log } from './cli-ui.mjs';
-
-const __dirname = path.dirname(fileURLToPath(import.meta.url));
-const DAEMON_PATH = path.join(__dirname, 'watch-daemon.mjs');
-
-const config = resolveConfig();
-const ROOT = config.projectRoot;
-const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
-
-function relArtifact() {
-    const p = resolveBackend(config) === 'sqlite' ? config.sqlitePath : config.indexPath;
-    return path.relative(ROOT, p) || p;
-}
-
-function freshness(file) {
-    try {
-        const sec = Math.floor((Date.now() - fs.statSync(file).mtimeMs) / 1000);
-        if (sec < 60) return `${sec}s ago`;
-        if (sec < 3600) return `${Math.floor(sec / 60)}m ago`;
-        if (sec < 86400) return `${Math.floor(sec / 3600)}h ago`;
-        return `${Math.floor(sec / 86400)}d ago`;
-    } catch { return 'never indexed'; }
-}
-
-function spawnDaemon() {
-    ensureDataDir(ROOT);
-    let logFd = null;
-    try { logFd = fs.openSync(config.logFile, 'a'); } catch { }
-    const child = spawn(process.execPath, [DAEMON_PATH], {
-        detached: true,
-        stdio: logFd !== null ? ['ignore', logFd, logFd] : 'ignore',
-        env: { ...process.env, MCP_PROJECT_ROOT: ROOT },
-    });
-    child.unref();
-    if (logFd !== null) fs.closeSync(logFd);
-    return child.pid;
-}
-
-function cmdStatus() {
-    const { running, pid } = daemonStatus(config.pidFile);
-    const backend = resolveBackend(config);
-    const artifact = backend === 'sqlite' ? config.sqlitePath : config.indexPath;
-    log('');
-    log(`  ${c.bold('graph-indexer')} ${c.dim('· daemon status')}`);
-    log(`  ${running ? glyph.run : glyph.stop} Daemon       ${running ? c.green(`running (PID ${pid})`) : c.dim('stopped')}`);
-    log(`  ${glyph.arrow} Project      ${c.dim(ROOT)}`);
-    log(`  ${glyph.arrow} Backend      ${backend === 'sqlite' ? 'SQLite (disk-backed)' : 'in-memory'}${config.storage === 'auto' ? c.dim(' · auto') : ''}`);
-    log(`  ${glyph.arrow} Index        ${c.dim(relArtifact())} ${c.dim('· ' + freshness(artifact))}`);
-    log(`  ${glyph.arrow} Logs         ${c.dim(path.relative(ROOT, config.logFile) || config.logFile)}`);
-    log('');
-    if (!running) log(c.dim(`  Start it with: ${c.cyan('npm run mcp:daemon:start')}\n`));
-    return 0;
-}
-
-function cmdStart() {
-    ensureDataDir(ROOT);
-    migrateLegacyLayout(ROOT);
-    const { running, pid } = daemonStatus(config.pidFile);
-    if (running) {
-        log(`${glyph.keep} Daemon already running ${c.dim(`(PID ${pid})`)} — only one runs per project.`);
-        return 0;
-    }
-    const newPid = spawnDaemon();
-    log(`${glyph.ok} Watch daemon started ${c.dim(`(PID ${newPid})`)}`);
-    log(c.dim(`  Logs: ${path.relative(ROOT, config.logFile) || config.logFile}`));
-    return 0;
-}
-
-async function cmdStop() {
-    const pid = readPid(config.pidFile);
-    if (!isAlive(pid)) {
-        log(`${glyph.keep} No daemon running.`);
-        try { if (pid) fs.unlinkSync(config.pidFile); } catch { }
-        return 0;
-    }
-    try { process.kill(pid, 'SIGTERM'); } catch { /* already gone */ }
-    for (let i = 0; i < 50 && isAlive(readPid(config.pidFile) ?? pid); i++) await sleep(100);
-    if (isAlive(pid)) {
-        log(`${glyph.warn} Daemon ${c.dim(`(PID ${pid})`)} did not stop within 5s.`);
-        return 1;
-    }
-    log(`${glyph.ok} Daemon stopped ${c.dim(`(PID ${pid})`)}.`);
-    return 0;
-}
-
-async function cmdRestart() {
-    await cmdStop();
-    await sleep(200);
-    return cmdStart();
-}
-
-function tail(file, n) {
-    try {
-        const size = fs.statSync(file).size;
-        const start = Math.max(0, size - 256 * 1024); // last 256 KB is plenty
-        const fd = fs.openSync(file, 'r');
-        try {
-            const buf = Buffer.alloc(size - start);
-            fs.readSync(fd, buf, 0, buf.length, start);
-            const lines = buf.toString('utf-8').split('\n');
-            return { lines: lines.slice(-n - 1), size };
-        } finally { fs.closeSync(fd); }
-    } catch { return { lines: [], size: 0 }; }
-}
-
-async function cmdLogs(args) {
-    const follow = args.includes('-f') || args.includes('--follow');
-    const nArg = args.find((a) => /^\d+$/.test(a));
-    const n = nArg ? Number(nArg) : 50;
-    if (!fs.existsSync(config.logFile)) {
-        log(c.dim(`No daemon log yet (${path.relative(ROOT, config.logFile) || config.logFile}).`));
-        return 0;
-    }
-    const { lines, size } = tail(config.logFile, n);
-    process.stdout.write(lines.join('\n'));
-    if (!follow) { log(''); return 0; }
-
-    let pos = size;
-    log(c.dim(`\n— following ${path.relative(ROOT, config.logFile)} (Ctrl-C to stop) —`));
-    fs.watchFile(config.logFile, { interval: 500 }, (cur) => {
-        if (cur.size < pos) { pos = 0; return; } // truncated/rotated
-        if (cur.size === pos) return;
-        const fd = fs.openSync(config.logFile, 'r');
-        try {
-            const buf = Buffer.alloc(cur.size - pos);
-            fs.readSync(fd, buf, 0, buf.length, pos);
-            process.stdout.write(buf.toString('utf-8'));
-            pos = cur.size;
-        } finally { fs.closeSync(fd); }
-    });
-    return new Promise(() => { /* runs until Ctrl-C */ });
-}
-
-const [cmd = 'status', ...rest] = process.argv.slice(2);
-const run = {
-    start: cmdStart,
-    stop: cmdStop,
-    restart: cmdRestart,
-    status: cmdStatus,
-    logs: () => cmdLogs(rest),
-}[cmd];
-
-if (!run) {
-    log(`${glyph.err} Unknown command: ${c.bold(cmd)}`);
-    log(c.dim('  Usage: idx-daemon [start|stop|restart|status|logs]'));
-    process.exit(2);
-}
-
-Promise.resolve(run()).then((code) => process.exit(code ?? 0));

package/daemon-lock.mjs

@@ -1,69 +0,0 @@
-/**
- * @file daemon-lock.mjs
- * @description Single-instance guard for the watch daemon. One — and only one —
- *              daemon may run per project, regardless of how it was launched
- *              (auto-spawned by the MCP server, `idx-daemon start`, or `idx-watch`
- *              directly). The guard is a PID lock file owned exclusively by the
- *              live daemon: acquisition is atomic (O_EXCL create) and tolerant of
- *              stale locks left by a crashed process.
- * @author MaquinaTech <https://github.com/MaquinaTech>
- * @copyright (c) 2026 MaquinaTech. All rights reserved.
- * @license MIT
- */
-import fs from 'fs';
-
-/** Read a PID from a lock file, or null if absent/garbage. */
-export function readPid(pidFile) {
-    try {
-        const pid = parseInt(fs.readFileSync(pidFile, 'utf-8').trim(), 10);
-        return Number.isInteger(pid) && pid > 0 ? pid : null;
-    } catch { return null; }
-}
-
-/** True if a process with this PID currently exists. */
-export function isAlive(pid) {
-    if (!pid) return false;
-    try { process.kill(pid, 0); return true; }
-    catch (err) { return err.code === 'EPERM'; } // exists but owned by another user
-}
-
-/**
- * Inspect the lock without acquiring it.
- * @returns {{running:boolean, pid:number|null}}
- */
-export function daemonStatus(pidFile) {
-    const pid = readPid(pidFile);
-    return { running: isAlive(pid), pid: isAlive(pid) ? pid : null };
-}
-
-/**
- * Atomically claim the daemon lock for the current process. A stale lock (PID no
- * longer alive) is cleared and retried; a lock held by a *live* daemon is left
- * untouched.
- *
- * @returns {boolean} true if this process now owns the lock; false if another
- *                    live daemon already holds it (caller should exit).
- */
-export function acquireLock(pidFile) {
-    for (let attempt = 0; attempt < 3; attempt++) {
-        try {
-            const fd = fs.openSync(pidFile, 'wx'); // O_EXCL: fails if the file exists
-            try { fs.writeSync(fd, String(process.pid)); } finally { fs.closeSync(fd); }
-            return true;
-        } catch (err) {
-            if (err.code !== 'EEXIST') throw err;
-            const pid = readPid(pidFile);
-            if (pid === process.pid) return true;
-            if (isAlive(pid)) return false;
-            try { fs.unlinkSync(pidFile); } catch { /* another racer cleared it — retry */ }
-        }
-    }
-    return false;
-}
-
-/** Release the lock, but only if this process owns it (never steal another's). */
-export function releaseLock(pidFile) {
-    if (readPid(pidFile) === process.pid) {
-        try { fs.unlinkSync(pidFile); } catch { }
-    }
-}

package/embedders/python/mlx_embed_server.py

@@ -1,61 +0,0 @@
-#!/usr/bin/env python3
-"""
-MLX Embedder Server — stdin/stdout JSON Lines protocol.
-
-Input:  {"texts": ["text1", "text2", ...]} (batch, up to 32)
-Output: {"embeddings": [[...], [...]]} or {"error": "message"}
-
-Prints "READY\n" to stdout once the model is loaded.
-Logs to stderr only (stdout is reserved for the JSON protocol).
-
-The model id is passed by the Node side as argv[1] (default below). Whatever id is
-loaded here is also what the Node side stamps into the index meta sidecar, so the
-{provider, model, dim} always reflects the exact model that produced the vectors.
-Override it with --mlx-embed-model / INDEXER_MLX_EMBED_MODEL on the graph-indexer side.
-"""
-import sys, json
-import numpy as np
-
-DEFAULT_MODEL_ID = "mlx-community/all-MiniLM-L6-v2-4bit"
-MODEL_ID = sys.argv[1] if len(sys.argv) > 1 and sys.argv[1].strip() else DEFAULT_MODEL_ID
-
-
-def load_model():
-    try:
-        from mlx_embeddings.utils import load
-        model, tokenizer = load(MODEL_ID)
-        return model, tokenizer
-    except ImportError as e:
-        print(json.dumps({"error": f"mlx_embeddings not installed: {e}"}), flush=True)
-        sys.exit(1)
-    except Exception as e:
-        print(json.dumps({"error": f"model load failed: {e}"}), flush=True)
-        sys.exit(1)
-
-
-def embed_batch(model, tokenizer, texts):
-    # Empty / falsy texts → "" so the tokenizer never sees None.
-    texts = [t[:8000] if t else "" for t in texts]
-    inputs = tokenizer.batch_encode_plus(
-        texts, return_tensors="mlx",
-        padding=True, truncation=True, max_length=512
-    )
-    out = model(inputs["input_ids"], attention_mask=inputs.get("attention_mask"))
-    vecs = out.text_embeds  # already mean-pooled and L2-normalized
-    return np.asarray(vecs, dtype=np.float32).tolist()
-
-
-model, tokenizer = load_model()
-print("READY", flush=True)
-
-for line in sys.stdin:
-    line = line.strip()
-    if not line:
-        continue
-    try:
-        req = json.loads(line)
-        texts = req.get("texts", [])
-        embeddings = embed_batch(model, tokenizer, texts)
-        print(json.dumps({"embeddings": embeddings}), flush=True)
-    except Exception as e:
-        print(json.dumps({"error": str(e)}), flush=True)

package/embedders/python/requirements-mlx.txt

@@ -1,5 +0,0 @@
-# Apple Silicon only (macOS darwin).
-# Install:  pip install -r requirements-mlx.txt --break-system-packages
-mlx>=0.21.0
-mlx-embeddings>=0.0.3
-numpy>=1.24.0

package/embedders/setup-mlx.mjs

@@ -1,110 +0,0 @@
-/**
- * @file embedders/setup-mlx.mjs
- * @description Lifecycle for the optional MLX (Apple Metal) embedder's dedicated Python
- *              virtualenv. graph-indexer runs the MLX server under `embedders/venv-mlx`
- *              rather than the system Python so its (large) deps never pollute the host
- *              and the interpreter is found by absolute path regardless of `$PATH`.
- *
- *              Exposes the venv interpreter path + a readiness probe to embeddings.mjs,
- *              and a one-shot `ensureMlxEnv()` that creates the venv and pip-installs
- *              requirements-mlx.txt into it. Runnable as a CLI (`npm run embed:setup:mlx`)
- *              so users can pre-provision the environment with a single command. This
- *              module intentionally does NOT import embeddings.mjs (avoids a cycle).
- *
- * @author MaquinaTech <https://github.com/MaquinaTech>
- * @copyright (c) 2026 MaquinaTech. All rights reserved.
- * @license MIT
- */
-import fs from 'fs';
-import { spawnSync } from 'child_process';
-import { fileURLToPath } from 'url';
-import { dirname, join } from 'path';
-
-const __dirname = dirname(fileURLToPath(import.meta.url)); // embedders/
-export const mlxVenvDir = join(__dirname, 'venv-mlx');
-const REQUIREMENTS = join(__dirname, 'python', 'requirements-mlx.txt');
-// Candidate base interpreters to bootstrap the venv, newest first. MLX needs Python
-// 3.9+; preferring a modern one avoids a stale system python (e.g. macOS 3.9).
-const BASE_PYTHON_CANDIDATES = ['python3.12', 'python3.11', 'python3.10', 'python3'];
-
-// venv layout is POSIX (bin/python3). MLX is macOS-only, so we never need Windows Scripts/.
-function venvPythonPath() { return join(mlxVenvDir, 'bin', 'python3'); }
-
-/** Absolute path to the venv interpreter, or bare 'python3' if the venv is absent. */
-export function mlxVenvPython() {
-    const p = venvPythonPath();
-    return fs.existsSync(p) ? p : 'python3';
-}
-
-/** True iff the venv exists AND `import mlx, mlx_embeddings` succeeds under it. */
-export function mlxEnvReady() {
-    const p = venvPythonPath();
-    if (!fs.existsSync(p)) return false;
-    const r = spawnSync(p, ['-c', 'import mlx, mlx_embeddings'], { stdio: 'ignore', timeout: 30000 });
-    return r.status === 0;
-}
-
-/** First base interpreter on PATH that runs, to seed the venv (newest preferred). */
-function pickBasePython() {
-    for (const c of BASE_PYTHON_CANDIDATES) {
-        const r = spawnSync(c, ['--version'], { stdio: 'ignore' });
-        if (r.status === 0) return c;
-    }
-    return null;
-}
-
-/**
- * Ensure the MLX embedder venv exists and its deps are importable.
- *
- * @param {object} [opts]
- * @param {boolean} [opts.autoInstall=true]  When false, only report readiness (no side effects).
- * @param {(s:string)=>void} [opts.log]      Progress sink (defaults to no-op).
- * @returns {{ready:boolean, created:boolean, error:(string|null)}}
- */
-export function ensureMlxEnv({ autoInstall = true, log = () => {} } = {}) {
-    if (process.platform !== 'darwin') {
-        return { ready: false, created: false, error: 'MLX requires macOS (Apple Silicon).' };
-    }
-    if (mlxEnvReady()) { log('MLX environment already present.'); return { ready: true, created: false, error: null }; }
-    if (!autoInstall) {
-        return { ready: false, created: false, error: 'MLX environment not ready — run `npm run embed:setup:mlx`.' };
-    }
-
-    let created = false;
-    const venvPy = venvPythonPath();
-    if (!fs.existsSync(venvPy)) {
-        const base = pickBasePython();
-        if (!base) return { ready: false, created: false, error: 'No python3 found on PATH to create the venv.' };
-        log(`creating virtualenv  (${base} -m venv embedders/venv-mlx)`);
-        const mk = spawnSync(base, ['-m', 'venv', mlxVenvDir], { stdio: 'inherit' });
-        if (mk.status !== 0) return { ready: false, created: false, error: 'virtualenv creation failed.' };
-        created = true;
-    }
-
-    log('installing MLX deps  (pip install -r embedders/python/requirements-mlx.txt)');
-    // Upgrade pip first so wheels resolve cleanly on older venv seeds; ignore its exit
-    // code (a failed upgrade is non-fatal — the install below is what matters).
-    spawnSync(venvPy, ['-m', 'pip', 'install', '-q', '--upgrade', 'pip'], { stdio: 'inherit' });
-    const pip = spawnSync(venvPy, ['-m', 'pip', 'install', '-r', REQUIREMENTS], { stdio: 'inherit' });
-    if (pip.status !== 0) return { ready: false, created, error: 'pip install failed.' };
-
-    const ready = mlxEnvReady();
-    return { ready, created, error: ready ? null : 'deps installed but `import mlx` still fails.' };
-}
-
-// ── CLI: `node embedders/setup-mlx.mjs` (npm run embed:setup:mlx) ─────────────
-const _invokedDirectly = process.argv[1]
-    && (() => { try { return fs.realpathSync(process.argv[1]) === fs.realpathSync(fileURLToPath(import.meta.url)); } catch { return false; } })();
-
-if (_invokedDirectly) {
-    console.log('Setting up the MLX (Apple Metal) embedder…\n');
-    const res = ensureMlxEnv({ autoInstall: true, log: (s) => console.log('  • ' + s) });
-    if (res.ready) {
-        console.log(`\n✓ MLX embedder ready — vectors will run under embedders/venv-mlx.`);
-        console.log(`  Use it:  npx idx-index --repo . --embeddings --embed-provider mlx`);
-        process.exit(0);
-    }
-    console.error(`\n✗ MLX setup failed: ${res.error}`);
-    if (process.platform !== 'darwin') console.error('  MLX is macOS-only; use --embed-provider local or ollama instead.');
-    process.exit(1);
-}