graph-indexer 1.2.0 → 2.0.0

package/daemon-lock.mjs

@@ -0,0 +1,69 @@
+/**
+ * @file daemon-lock.mjs
+ * @description Single-instance guard for the watch daemon. One — and only one —
+ *              daemon may run per project, regardless of how it was launched
+ *              (auto-spawned by the MCP server, `idx-daemon start`, or `idx-watch`
+ *              directly). The guard is a PID lock file owned exclusively by the
+ *              live daemon: acquisition is atomic (O_EXCL create) and tolerant of
+ *              stale locks left by a crashed process.
+ * @author MaquinaTech <https://github.com/MaquinaTech>
+ * @copyright (c) 2026 MaquinaTech. All rights reserved.
+ * @license MIT
+ */
+import fs from 'fs';
+
+/** Read a PID from a lock file, or null if absent/garbage. */
+export function readPid(pidFile) {
+    try {
+        const pid = parseInt(fs.readFileSync(pidFile, 'utf-8').trim(), 10);
+        return Number.isInteger(pid) && pid > 0 ? pid : null;
+    } catch { return null; }
+}
+
+/** True if a process with this PID currently exists. */
+export function isAlive(pid) {
+    if (!pid) return false;
+    try { process.kill(pid, 0); return true; }
+    catch (err) { return err.code === 'EPERM'; } // exists but owned by another user
+}
+
+/**
+ * Inspect the lock without acquiring it.
+ * @returns {{running:boolean, pid:number|null}}
+ */
+export function daemonStatus(pidFile) {
+    const pid = readPid(pidFile);
+    return { running: isAlive(pid), pid: isAlive(pid) ? pid : null };
+}
+
+/**
+ * Atomically claim the daemon lock for the current process. A stale lock (PID no
+ * longer alive) is cleared and retried; a lock held by a *live* daemon is left
+ * untouched.
+ *
+ * @returns {boolean} true if this process now owns the lock; false if another
+ *                    live daemon already holds it (caller should exit).
+ */
+export function acquireLock(pidFile) {
+    for (let attempt = 0; attempt < 3; attempt++) {
+        try {
+            const fd = fs.openSync(pidFile, 'wx'); // O_EXCL: fails if the file exists
+            try { fs.writeSync(fd, String(process.pid)); } finally { fs.closeSync(fd); }
+            return true;
+        } catch (err) {
+            if (err.code !== 'EEXIST') throw err;
+            const pid = readPid(pidFile);
+            if (pid === process.pid) return true;
+            if (isAlive(pid)) return false;
+            try { fs.unlinkSync(pidFile); } catch { /* another racer cleared it — retry */ }
+        }
+    }
+    return false;
+}
+
+/** Release the lock, but only if this process owns it (never steal another's). */
+export function releaseLock(pidFile) {
+    if (readPid(pidFile) === process.pid) {
+        try { fs.unlinkSync(pidFile); } catch { }
+    }
+}

package/embedders/python/mlx_embed_server.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python3
+"""
+MLX Embedder Server — stdin/stdout JSON Lines protocol.
+
+Input:  {"texts": ["text1", "text2", ...]} (batch, up to 32)
+Output: {"embeddings": [[...], [...]]} or {"error": "message"}
+
+Prints "READY\n" to stdout once the model is loaded.
+Logs to stderr only (stdout is reserved for the JSON protocol).
+
+The model id is passed by the Node side as argv[1] (default below). Whatever id is
+loaded here is also what the Node side stamps into the index meta sidecar, so the
+{provider, model, dim} always reflects the exact model that produced the vectors.
+Override it with --mlx-embed-model / INDEXER_MLX_EMBED_MODEL on the graph-indexer side.
+"""
+import sys, json
+import numpy as np
+
+DEFAULT_MODEL_ID = "mlx-community/all-MiniLM-L6-v2-4bit"
+MODEL_ID = sys.argv[1] if len(sys.argv) > 1 and sys.argv[1].strip() else DEFAULT_MODEL_ID
+
+
+def load_model():
+    try:
+        from mlx_embeddings.utils import load
+        model, tokenizer = load(MODEL_ID)
+        return model, tokenizer
+    except ImportError as e:
+        print(json.dumps({"error": f"mlx_embeddings not installed: {e}"}), flush=True)
+        sys.exit(1)
+    except Exception as e:
+        print(json.dumps({"error": f"model load failed: {e}"}), flush=True)
+        sys.exit(1)
+
+
+def embed_batch(model, tokenizer, texts):
+    # Empty / falsy texts → "" so the tokenizer never sees None.
+    texts = [t[:8000] if t else "" for t in texts]
+    inputs = tokenizer.batch_encode_plus(
+        texts, return_tensors="mlx",
+        padding=True, truncation=True, max_length=512
+    )
+    out = model(inputs["input_ids"], attention_mask=inputs.get("attention_mask"))
+    vecs = out.text_embeds  # already mean-pooled and L2-normalized
+    return np.asarray(vecs, dtype=np.float32).tolist()
+
+
+model, tokenizer = load_model()
+print("READY", flush=True)
+
+for line in sys.stdin:
+    line = line.strip()
+    if not line:
+        continue
+    try:
+        req = json.loads(line)
+        texts = req.get("texts", [])
+        embeddings = embed_batch(model, tokenizer, texts)
+        print(json.dumps({"embeddings": embeddings}), flush=True)
+    except Exception as e:
+        print(json.dumps({"error": str(e)}), flush=True)

package/embedders/python/requirements-mlx.txt

@@ -0,0 +1,5 @@
+# Apple Silicon only (macOS darwin).
+# Install:  pip install -r requirements-mlx.txt --break-system-packages
+mlx>=0.21.0
+mlx-embeddings>=0.0.3
+numpy>=1.24.0

package/embedders/setup-mlx.mjs

@@ -0,0 +1,110 @@
+/**
+ * @file embedders/setup-mlx.mjs
+ * @description Lifecycle for the optional MLX (Apple Metal) embedder's dedicated Python
+ *              virtualenv. graph-indexer runs the MLX server under `embedders/venv-mlx`
+ *              rather than the system Python so its (large) deps never pollute the host
+ *              and the interpreter is found by absolute path regardless of `$PATH`.
+ *
+ *              Exposes the venv interpreter path + a readiness probe to embeddings.mjs,
+ *              and a one-shot `ensureMlxEnv()` that creates the venv and pip-installs
+ *              requirements-mlx.txt into it. Runnable as a CLI (`npm run embed:setup:mlx`)
+ *              so users can pre-provision the environment with a single command. This
+ *              module intentionally does NOT import embeddings.mjs (avoids a cycle).
+ *
+ * @author MaquinaTech <https://github.com/MaquinaTech>
+ * @copyright (c) 2026 MaquinaTech. All rights reserved.
+ * @license MIT
+ */
+import fs from 'fs';
+import { spawnSync } from 'child_process';
+import { fileURLToPath } from 'url';
+import { dirname, join } from 'path';
+
+const __dirname = dirname(fileURLToPath(import.meta.url)); // embedders/
+export const mlxVenvDir = join(__dirname, 'venv-mlx');
+const REQUIREMENTS = join(__dirname, 'python', 'requirements-mlx.txt');
+// Candidate base interpreters to bootstrap the venv, newest first. MLX needs Python
+// 3.9+; preferring a modern one avoids a stale system python (e.g. macOS 3.9).
+const BASE_PYTHON_CANDIDATES = ['python3.12', 'python3.11', 'python3.10', 'python3'];
+
+// venv layout is POSIX (bin/python3). MLX is macOS-only, so we never need Windows Scripts/.
+function venvPythonPath() { return join(mlxVenvDir, 'bin', 'python3'); }
+
+/** Absolute path to the venv interpreter, or bare 'python3' if the venv is absent. */
+export function mlxVenvPython() {
+    const p = venvPythonPath();
+    return fs.existsSync(p) ? p : 'python3';
+}
+
+/** True iff the venv exists AND `import mlx, mlx_embeddings` succeeds under it. */
+export function mlxEnvReady() {
+    const p = venvPythonPath();
+    if (!fs.existsSync(p)) return false;
+    const r = spawnSync(p, ['-c', 'import mlx, mlx_embeddings'], { stdio: 'ignore', timeout: 30000 });
+    return r.status === 0;
+}
+
+/** First base interpreter on PATH that runs, to seed the venv (newest preferred). */
+function pickBasePython() {
+    for (const c of BASE_PYTHON_CANDIDATES) {
+        const r = spawnSync(c, ['--version'], { stdio: 'ignore' });
+        if (r.status === 0) return c;
+    }
+    return null;
+}
+
+/**
+ * Ensure the MLX embedder venv exists and its deps are importable.
+ *
+ * @param {object} [opts]
+ * @param {boolean} [opts.autoInstall=true]  When false, only report readiness (no side effects).
+ * @param {(s:string)=>void} [opts.log]      Progress sink (defaults to no-op).
+ * @returns {{ready:boolean, created:boolean, error:(string|null)}}
+ */
+export function ensureMlxEnv({ autoInstall = true, log = () => {} } = {}) {
+    if (process.platform !== 'darwin') {
+        return { ready: false, created: false, error: 'MLX requires macOS (Apple Silicon).' };
+    }
+    if (mlxEnvReady()) { log('MLX environment already present.'); return { ready: true, created: false, error: null }; }
+    if (!autoInstall) {
+        return { ready: false, created: false, error: 'MLX environment not ready — run `npm run embed:setup:mlx`.' };
+    }
+
+    let created = false;
+    const venvPy = venvPythonPath();
+    if (!fs.existsSync(venvPy)) {
+        const base = pickBasePython();
+        if (!base) return { ready: false, created: false, error: 'No python3 found on PATH to create the venv.' };
+        log(`creating virtualenv  (${base} -m venv embedders/venv-mlx)`);
+        const mk = spawnSync(base, ['-m', 'venv', mlxVenvDir], { stdio: 'inherit' });
+        if (mk.status !== 0) return { ready: false, created: false, error: 'virtualenv creation failed.' };
+        created = true;
+    }
+
+    log('installing MLX deps  (pip install -r embedders/python/requirements-mlx.txt)');
+    // Upgrade pip first so wheels resolve cleanly on older venv seeds; ignore its exit
+    // code (a failed upgrade is non-fatal — the install below is what matters).
+    spawnSync(venvPy, ['-m', 'pip', 'install', '-q', '--upgrade', 'pip'], { stdio: 'inherit' });
+    const pip = spawnSync(venvPy, ['-m', 'pip', 'install', '-r', REQUIREMENTS], { stdio: 'inherit' });
+    if (pip.status !== 0) return { ready: false, created, error: 'pip install failed.' };
+
+    const ready = mlxEnvReady();
+    return { ready, created, error: ready ? null : 'deps installed but `import mlx` still fails.' };
+}
+
+// ── CLI: `node embedders/setup-mlx.mjs` (npm run embed:setup:mlx) ─────────────
+const _invokedDirectly = process.argv[1]
+    && (() => { try { return fs.realpathSync(process.argv[1]) === fs.realpathSync(fileURLToPath(import.meta.url)); } catch { return false; } })();
+
+if (_invokedDirectly) {
+    console.log('Setting up the MLX (Apple Metal) embedder…\n');
+    const res = ensureMlxEnv({ autoInstall: true, log: (s) => console.log('  • ' + s) });
+    if (res.ready) {
+        console.log(`\n✓ MLX embedder ready — vectors will run under embedders/venv-mlx.`);
+        console.log(`  Use it:  npx idx-index --repo . --embeddings --embed-provider mlx`);
+        process.exit(0);
+    }
+    console.error(`\n✗ MLX setup failed: ${res.error}`);
+    if (process.platform !== 'darwin') console.error('  MLX is macOS-only; use --embed-provider local or ollama instead.');
+    process.exit(1);
+}