@jafreck/lore 0.3.3 → 0.3.5

package/dist/indexer/embedder.js

@@ -2,16 +2,14 @@
  * @module indexer/embedder
  *
  * Provides an `EmbeddingProvider` abstraction for generating dense vector
- * embeddings from text. The primary implementation (`SentenceTransformersProvider`)
- * delegates to a Python subprocess that runs a sentence-transformers model,
- * communicating over stdin/stdout using newline-delimited JSON (NDJSON).
+ * embeddings from text. The primary implementation (`TransformersJsProvider`)
+ * uses the `@huggingface/transformers` library to run ONNX models natively
+ * in Node.js — no Python or external processes required.
  *
- * The model's embedding dimensionality is auto-detected at startup — the
- * Python script writes a `{"dims": N}` line before entering the request loop.
+ * The model's embedding dimensionality is auto-detected at startup by
+ * embedding a probe sentence and inspecting the output shape.
  */
-import { spawn } from 'node:child_process';
-import * as readline from 'node:readline';
-import { trackProcess } from '../process-tracker.js';
+import { createHash } from 'node:crypto';
 export function buildStructuralEmbeddingText(input) {
     const parts = [
         input.signature?.trim() ?? '',
@@ -22,51 +20,79 @@ export function buildStructuralEmbeddingText(input) {
     const uniqueParts = [...new Set(parts)];
     return uniqueParts.join('\n');
 }
-// ─── Python bootstrap script ──────────────────────────────────────────────────
+/** SHA-256 hex hash of an embedding input text (used for skip-unchanged logic). */
+export function hashEmbeddingText(text) {
+    return createHash('sha256').update(text).digest('hex');
+}
+// ─── Token-aware batching ─────────────────────────────────────────────────────
+/**
+ * Approximate token count for a text string.
+ * Uses the ~4 chars/token heuristic (reasonable for code/English).
+ */
+function estimateTokens(text) {
+    return Math.ceil(text.length / 4);
+}
+/**
+ * Maximum total tokens per embedding batch.  Larger batches amortise
+ * call overhead but Transformers.js pads every item to the longest in the
+ * batch, so one very long text inflates memory for the whole batch.
+ *
+ * 32 768 tokens ≈ 128 KB of text — keeps peak memory reasonable while
+ * avoiding pathological padding waste.
+ */
+const MAX_BATCH_TOKENS = 32_768;
+/** Absolute cap on items per batch (avoids degenerate cases with many tiny texts). */
+const MAX_BATCH_ITEMS = 512;
 /**
- * Inline Python script that:
- *   1. Loads the model.
- *   2. Prints `{"dims": <N>}` on the first stdout line.
- *   3. Enters an NDJSON request loop (stdin → stdout).
+ * Split `items` into token-budget-aware batches.
  *
- * Model name is received as sys.argv[1].
+ * Each batch stays within `MAX_BATCH_TOKENS` total estimated tokens and
+ * `MAX_BATCH_ITEMS` items.  An individual item that exceeds the token
+ * budget gets its own single-item batch.
  */
-const BOOTSTRAP_SCRIPT = `
-import sys, json
-from sentence_transformers import SentenceTransformer
-model = SentenceTransformer(sys.argv[1], trust_remote_code=True)
-dims = model.get_sentence_embedding_dimension()
-sys.stdout.write(json.dumps({"dims": dims}) + "\\n")
-sys.stdout.flush()
-for line in sys.stdin:
-    line = line.strip()
-    if not line:
-        continue
-    req = json.loads(line)
-    vecs = model.encode(req.get('texts', []), normalize_embeddings=True).tolist()
-    sys.stdout.write(json.dumps({'embeddings': vecs}) + '\\n')
-    sys.stdout.flush()
-`.trimStart();
+export function tokenAwareBatch(items, getText) {
+    const batches = [];
+    let current = [];
+    let currentTokens = 0;
+    for (const item of items) {
+        const tokens = estimateTokens(getText(item));
+        if (current.length > 0 && (currentTokens + tokens > MAX_BATCH_TOKENS || current.length >= MAX_BATCH_ITEMS)) {
+            batches.push(current);
+            current = [];
+            currentTokens = 0;
+        }
+        current.push(item);
+        currentTokens += tokens;
+    }
+    if (current.length > 0)
+        batches.push(current);
+    return batches;
+}
 /**
- * Communicates with a Python subprocess via stdin/stdout NDJSON to produce
- * embeddings using a sentence-transformers compatible model.
+ * Generates embeddings using `@huggingface/transformers` (Transformers.js).
+ * Runs ONNX models natively in Node — zero Python dependency.
+ *
+ * Auto-detects the best available ONNX execution provider:
+ *   - `webgpu` when available (Linux/Windows with GPU)
+ *   - `coreml` on Apple Silicon (macOS arm64)
+ *   - `cpu` everywhere else (always available)
+ *
+ * Override via `LORE_EMBED_DEVICE` env var (e.g. `cpu`, `coreml`, `webgpu`).
+ * Override quantization via `LORE_EMBED_DTYPE` env var (e.g. `q8`, `q4`, `fp16`).
  *
- * Call `init()` first to spawn the process and detect the model's embedding
- * dimensionality.  The subprocess is kept alive for the lifetime of the
- * provider for efficiency.
+ * Call `init()` first to download/load the model and detect its embedding
+ * dimensionality. The model is kept in memory for the provider's lifetime.
  */
-export class SentenceTransformersProvider {
+export class TransformersJsProvider {
     modelName;
+    dtype;
     _dims = null;
-    proc = null;
-    rl = null;
-    pendingRequests = [];
-    pythonBin;
-    /** Whether the first stdout line (dims handshake) has been consumed. */
+    _device = null;
+    _pipeline = null;
     initialized = false;
-    constructor(modelName, pythonBin = 'python3') {
+    constructor(modelName, dtype) {
         this.modelName = modelName;
-        this.pythonBin = pythonBin;
+        this.dtype = dtype ?? process.env['LORE_EMBED_DTYPE'] ?? 'fp32';
     }
     /** Embedding dimensionality — available only after `init()`. */
     get dims() {
@@ -75,129 +101,108 @@ export class SentenceTransformersProvider {
         }
         return this._dims;
     }
+    /** ONNX execution provider selected during init (cpu/coreml/webgpu). */
+    get device() {
+        return this._device ?? 'unknown';
+    }
     /**
-     * Spawn the Python subprocess, load the model, and read the `{"dims": N}`
-     * handshake line.  This may take a while on first run (model download).
+     * Load the model via Transformers.js and detect embedding dimensionality
+     * by running a single probe sentence.
      */
     async init() {
         if (this.initialized)
             return;
-        this.spawnProcess();
-        // The very first line from the subprocess is the dims handshake.
-        return new Promise((resolve, reject) => {
-            const onLine = (line) => {
-                try {
-                    const msg = JSON.parse(line);
-                    if (typeof msg.dims === 'number') {
-                        this._dims = msg.dims;
-                        this.initialized = true;
-                        resolve();
-                    }
-                    else {
-                        reject(new Error(`Unexpected handshake from embedding subprocess: ${line}`));
-                    }
-                }
-                catch (err) {
-                    reject(new Error(`Failed to parse embedding handshake: ${line}`));
-                }
-            };
-            // Read exactly one line for the handshake, then re-wire for embed requests.
-            this.rl.once('line', onLine);
-            // If the process dies before the handshake, reject.
-            this.proc.once('exit', (code) => {
-                if (!this.initialized) {
-                    reject(new Error(`Embedding subprocess exited with code ${code} before handshake`));
-                }
-            });
+        const device = this.detectDevice();
+        this._device = device;
+        const { pipeline } = await import('@huggingface/transformers');
+        this._pipeline = await pipeline('feature-extraction', this.modelName, {
+            device: device,
+            ...(this.dtype !== 'fp32' && { dtype: this.dtype }),
         });
+        // Probe the model to detect dimensionality.
+        const probe = await this._pipeline('dimensionality probe', { pooling: 'mean', normalize: true });
+        const probeList = probe.tolist();
+        this._dims = probeList[0].length;
+        this.initialized = true;
     }
     async embed(texts) {
         if (texts.length === 0)
             return [];
-        if (!this.initialized) {
+        if (!this.initialized || !this._pipeline) {
             throw new Error('EmbeddingProvider not initialised — call init() first');
         }
-        return new Promise((resolve, reject) => {
-            this.pendingRequests.push({ resolve, reject });
-            this.proc.stdin.write(JSON.stringify({ texts }) + '\n');
-        });
+        const output = await this._pipeline(texts, { pooling: 'mean', normalize: true });
+        return output.tolist();
     }
     async dispose() {
-        if (!this.proc)
-            return;
-        const proc = this.proc;
-        // Reject all pending requests before tearing down.
-        const pending = this.pendingRequests.splice(0);
-        const err = new Error('EmbeddingProvider disposed');
-        for (const r of pending)
-            r.reject(err);
-        this.proc = null;
-        this.rl?.close();
-        this.rl = null;
-        proc.stdin?.end();
-        await new Promise(resolve => {
-            const timeout = setTimeout(() => { proc.kill(); resolve(); }, 5_000);
-            proc.once('close', () => { clearTimeout(timeout); resolve(); });
-        });
-    }
-    /** Spawn the Python subprocess and wire up the readline interface. */
-    spawnProcess() {
-        if (this.proc)
-            return;
-        this.proc = spawn(this.pythonBin, ['-c', BOOTSTRAP_SCRIPT, this.modelName], {
-            stdio: ['pipe', 'pipe', 'inherit'],
-        });
-        trackProcess(this.proc);
-        this.rl = readline.createInterface({ input: this.proc.stdout });
-        // After init(), all subsequent lines are embed responses.
-        this.rl.on('line', (line) => {
-            // Skip lines until init handshake is done (handled by init's once listener).
-            if (!this.initialized)
-                return;
-            const pending = this.pendingRequests.shift();
-            if (!pending)
-                return;
+        if (this._pipeline) {
             try {
-                const { embeddings } = JSON.parse(line);
-                pending.resolve(embeddings);
-            }
-            catch (err) {
-                pending.reject(err);
-            }
-        });
-        this.proc.on('error', (err) => {
-            const reqs = this.pendingRequests.splice(0);
-            this.proc = null;
-            this.rl = null;
-            for (const r of reqs)
-                r.reject(err);
-        });
-        this.proc.on('exit', (code) => {
-            if (code !== 0 && this.pendingRequests.length > 0) {
-                const err = new Error(`Embedding subprocess exited with code ${code}`);
-                const reqs = this.pendingRequests.splice(0);
-                for (const r of reqs)
-                    r.reject(err);
+                await this._pipeline.dispose?.();
             }
-            if (this.proc) {
-                this.proc = null;
-                this.rl = null;
-            }
-        });
+            catch { /* best-effort cleanup */ }
+            this._pipeline = null;
+        }
+    }
+    /**
+     * Detect the best available ONNX execution provider.
+     * Respects `LORE_EMBED_DEVICE` env var for explicit override.
+     */
+    detectDevice() {
+        const envDevice = process.env['LORE_EMBED_DEVICE'];
+        if (envDevice)
+            return envDevice;
+        // CoreML is available on Apple Silicon via onnxruntime-node
+        if (process.platform === 'darwin' && process.arch === 'arm64') {
+            return 'coreml';
+        }
+        return 'cpu';
     }
 }
-// ─── Qwen3 factory ────────────────────────────────────────────────────────────
-/** Default embedding model used when no model is explicitly specified. */
-export const DEFAULT_EMBEDDING_MODEL = 'Qwen/Qwen3-Embedding-4B';
+// ─── Lazy embedding provider ──────────────────────────────────────────────────
 /**
- * Creates a `SentenceTransformersProvider` pre-configured for the specified
- * Qwen3-Embedding model size.
+ * Wraps a `TransformersJsProvider` with deferred initialisation.
+ *
+ * The model is only downloaded and loaded on the first call to `embed()` or
+ * an explicit `init()`.  This allows `lore index` to complete faster when
+ * embeddings are configured but the user primarily uses structural search.
  *
- * @param size      Model size variant: `'0.6B'`, `'4B'`, or `'8B'`.
- * @param pythonBin Path to the Python executable (default: `'python3'`).
+ * The MCP server can pass a `LazyEmbeddingProvider` so semantic search
+ * triggers model loading on-demand rather than at startup.
  */
-export function Qwen3EmbeddingProvider(size, pythonBin = 'python3') {
-    const modelName = `Qwen/Qwen3-Embedding-${size}`;
-    return new SentenceTransformersProvider(modelName, pythonBin);
+export class LazyEmbeddingProvider {
+    inner;
+    _initPromise = null;
+    constructor(modelName, dtype) {
+        this.inner = new TransformersJsProvider(modelName, dtype);
+    }
+    get modelName() { return this.inner.modelName; }
+    get dims() { return this.inner.dims; }
+    async init() {
+        if (!this._initPromise) {
+            this._initPromise = this.inner.init();
+        }
+        return this._initPromise;
+    }
+    async embed(texts) {
+        if (texts.length === 0)
+            return [];
+        await this.init();
+        return this.inner.embed(texts);
+    }
+    async dispose() {
+        if (this._initPromise) {
+            try {
+                await this._initPromise;
+            }
+            catch { /* init may have failed */ }
+        }
+        return this.inner.dispose();
+    }
 }
+// ─── Default model ────────────────────────────────────────────────────────────
+/**
+ * Default embedding model — `Qwen/Qwen3-Embedding-0.6B` is a compact,
+ * 1024-dim model with strong multilingual and code understanding.
+ */
+export const DEFAULT_EMBEDDING_MODEL = 'Qwen/Qwen3-Embedding-0.6B';
 //# sourceMappingURL=embedder.js.map

package/dist/indexer/embedder.js.map

@@ -1 +1 @@
-{"version":3,"file":"embedder.js","sourceRoot":"","sources":["../../src/indexer/embedder.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EAAE,KAAK,EAAqB,MAAM,oBAAoB,CAAC;AAC9D,OAAO,KAAK,QAAQ,MAAM,eAAe,CAAC;AAC1C,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AA8BrD,MAAM,UAAU,4BAA4B,CAAC,KAA+B;IAC1E,MAAM,KAAK,GAAG;QACZ,KAAK,CAAC,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE;QAC7B,KAAK,CAAC,qBAAqB,EAAE,IAAI,EAAE,IAAI,EAAE;QACzC,KAAK,CAAC,kBAAkB,EAAE,IAAI,EAAE,IAAI,EAAE;QACtC,KAAK,CAAC,IAAI,CAAC,IAAI,EAAE;KAClB,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACpC,MAAM,WAAW,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;IACxC,OAAO,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAChC,CAAC;AAED,iFAAiF;AAEjF;;;;;;;GAOG;AACH,MAAM,gBAAgB,GAAG;;;;;;;;;;;;;;;CAexB,CAAC,SAAS,EAAE,CAAC;AASd;;;;;;;GAOG;AACH,MAAM,OAAO,4BAA4B;IAC9B,SAAS,CAAS;IACnB,KAAK,GAAkB,IAAI,CAAC;IAE5B,IAAI,GAAwB,IAAI,CAAC;IACjC,EAAE,GAA8B,IAAI,CAAC;IAC5B,eAAe,GAAqB,EAAE,CAAC;IACvC,SAAS,CAAS;IACnC,wEAAwE;IAChE,WAAW,GAAG,KAAK,CAAC;IAE5B,YAAY,SAAiB,EAAE,SAAS,GAAG,SAAS;QAClD,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;IAC7B,CAAC;IAED,gEAAgE;IAChE,IAAI,IAAI;QACN,IAAI,IAAI,CAAC,KAAK,KAAK,IAAI,EAAE,CAAC;YACxB,MAAM,IAAI,KAAK,CAAC,uDAAuD,CAAC,CAAC;QAC3E,CAAC;QACD,OAAO,IAAI,CAAC,KAAK,CAAC;IACpB,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,IAAI;QACR,IAAI,IAAI,CAAC,WAAW;YAAE,OAAO;QAC7B,IAAI,CAAC,YAAY,EAAE,CAAC;QAEpB,iEAAiE;QACjE,OAAO,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC3C,MAAM,MAAM,GAAG,CAAC,IAAY,EAAE,EAAE;gBAC9B,IAAI,CAAC;oBACH,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAsB,CAAC;oBAClD,IAAI,OAAO,GAAG,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;wBACjC,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC,IAAI,CAAC;wBACtB,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;wBACxB,OAAO,EAAE,CAAC;oBACZ,CAAC;yBAAM,CAAC;wBACN,MAAM,CAAC,IAAI,KAAK,CAAC,mDAAmD,IAAI,EAAE,CAAC,CAAC,CAAC;oBAC/E,CAAC;gBACH,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,MAAM,CAAC,IAAI,KAAK,CAAC,wCAAwC,IAAI,EAAE,CAAC,CAAC,CAAC;gBACpE,CAAC;YACH,CAAC,CAAC;YACF,4EAA4E;YAC5E,IAAI,CAAC,EAAG,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;YAE9B,oDAAoD;YACpD,IAAI,CAAC,IAAK,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;gBAC/B,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;oBACtB,MAAM,CAAC,IAAI,KAAK,CAAC,yCAAyC,IAAI,mBAAmB,CAAC,CAAC,CAAC;gBACtF,CAAC;YACH,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,KAAe;QACzB,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAClC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACtB,MAAM,IAAI,KAAK,CAAC,uDAAuD,CAAC,CAAC;QAC3E,CAAC;QACD,OAAO,IAAI,OAAO,CAAa,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACjD,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;YAC/C,IAAI,CAAC,IAAK,CAAC,KAAM,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;QAC5D,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,OAAO;QACX,IAAI,CAAC,IAAI,CAAC,IAAI;YAAE,OAAO;QACvB,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;QACvB,mDAAmD;QACnD,MAAM,OAAO,GAAG,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAC/C,MAAM,GAAG,GAAG,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC;QACpD,KAAK,MAAM,CAAC,IAAI,OAAO;YAAE,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACvC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC;QACjB,IAAI,CAAC,EAAE,GAAG,IAAI,CAAC;QACf,IAAI,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC;QAClB,MAAM,IAAI,OAAO,CAAO,OAAO,CAAC,EAAE;YAChC,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;YACrE,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,EAAE,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAClE,CAAC,CAAC,CAAC;IACL,CAAC;IAED,sEAAsE;IAC9D,YAAY;QAClB,IAAI,IAAI,CAAC,IAAI;YAAE,OAAO;QAEtB,IAAI,CAAC,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC,IAAI,EAAE,gBAAgB,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE;YAC1E,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,SAAS,CAAC;SACnC,CAAC,CAAC;QACH,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAExB,IAAI,CAAC,EAAE,GAAG,QAAQ,CAAC,eAAe,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,MAAO,EAAE,CAAC,CAAC;QAEjE,0DAA0D;QAC1D,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;YAC1B,6EAA6E;YAC7E,IAAI,CAAC,IAAI,CAAC,WAAW;gBAAE,OAAO;YAC9B,MAAM,OAAO,GAAG,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;YAC7C,IAAI,CAAC,OAAO;gBAAE,OAAO;YACrB,IAAI,CAAC;gBACH,MAAM,EAAE,UAAU,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAA+B,CAAC;gBACtE,OAAO,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;YAC9B,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACtB,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;YAC5B,MAAM,IAAI,GAAG,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;YAC5C,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;YACjB,IAAI,CAAC,EAAE,GAAG,IAAI,CAAC;YACf,KAAK,MAAM,CAAC,IAAI,IAAI;gBAAE,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACtC,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;YAC5B,IAAI,IAAI,KAAK,CAAC,IAAI,IAAI,CAAC,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAClD,MAAM,GAAG,GAAG,IAAI,KAAK,CAAC,yCAAyC,IAAI,EAAE,CAAC,CAAC;gBACvE,MAAM,IAAI,GAAG,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;gBAC5C,KAAK,MAAM,CAAC,IAAI,IAAI;oBAAE,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACtC,CAAC;YACD,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;gBAAC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;gBAAC,IAAI,CAAC,EAAE,GAAG,IAAI,CAAC;YAAC,CAAC;QACtD,CAAC,CAAC,CAAC;IACL,CAAC;CACF;AAED,iFAAiF;AAEjF,0EAA0E;AAC1E,MAAM,CAAC,MAAM,uBAAuB,GAAG,yBAAyB,CAAC;AAEjE;;;;;;GAMG;AACH,MAAM,UAAU,sBAAsB,CACpC,IAA0B,EAC1B,SAAS,GAAG,SAAS;IAErB,MAAM,SAAS,GAAG,wBAAwB,IAAI,EAAE,CAAC;IACjD,OAAO,IAAI,4BAA4B,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;AAChE,CAAC"}
+{"version":3,"file":"embedder.js","sourceRoot":"","sources":["../../src/indexer/embedder.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AA8BzC,MAAM,UAAU,4BAA4B,CAAC,KAA+B;IAC1E,MAAM,KAAK,GAAG;QACZ,KAAK,CAAC,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE;QAC7B,KAAK,CAAC,qBAAqB,EAAE,IAAI,EAAE,IAAI,EAAE;QACzC,KAAK,CAAC,kBAAkB,EAAE,IAAI,EAAE,IAAI,EAAE;QACtC,KAAK,CAAC,IAAI,CAAC,IAAI,EAAE;KAClB,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACpC,MAAM,WAAW,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;IACxC,OAAO,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAChC,CAAC;AAED,mFAAmF;AACnF,MAAM,UAAU,iBAAiB,CAAC,IAAY;IAC5C,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AACzD,CAAC;AAED,iFAAiF;AAEjF;;;GAGG;AACH,SAAS,cAAc,CAAC,IAAY;IAClC,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AACpC,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,gBAAgB,GAAG,MAAM,CAAC;AAEhC,sFAAsF;AACtF,MAAM,eAAe,GAAG,GAAG,CAAC;AAE5B;;;;;;GAMG;AACH,MAAM,UAAU,eAAe,CAAI,KAAU,EAAE,OAA4B;IACzE,MAAM,OAAO,GAAU,EAAE,CAAC;IAC1B,IAAI,OAAO,GAAQ,EAAE,CAAC;IACtB,IAAI,aAAa,GAAG,CAAC,CAAC;IAEtB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,MAAM,GAAG,cAAc,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;QAC7C,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,aAAa,GAAG,MAAM,GAAG,gBAAgB,IAAI,OAAO,CAAC,MAAM,IAAI,eAAe,CAAC,EAAE,CAAC;YAC3G,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACtB,OAAO,GAAG,EAAE,CAAC;YACb,aAAa,GAAG,CAAC,CAAC;QACpB,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACnB,aAAa,IAAI,MAAM,CAAC;IAC1B,CAAC;IACD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAC9C,OAAO,OAAO,CAAC;AACjB,CAAC;AAYD;;;;;;;;;;;;;;GAcG;AACH,MAAM,OAAO,sBAAsB;IACxB,SAAS,CAAS;IAClB,KAAK,CAAY;IAClB,KAAK,GAAkB,IAAI,CAAC;IAC5B,OAAO,GAAkB,IAAI,CAAC;IAC9B,SAAS,GAAqC,IAAI,CAAC;IACnD,WAAW,GAAG,KAAK,CAAC;IAE5B,YAAY,SAAiB,EAAE,KAAiB;QAC9C,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,KAAK,GAAG,KAAK,IAAK,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAA2B,IAAI,MAAM,CAAC;IAC7F,CAAC;IAED,gEAAgE;IAChE,IAAI,IAAI;QACN,IAAI,IAAI,CAAC,KAAK,KAAK,IAAI,EAAE,CAAC;YACxB,MAAM,IAAI,KAAK,CAAC,uDAAuD,CAAC,CAAC;QAC3E,CAAC;QACD,OAAO,IAAI,CAAC,KAAK,CAAC;IACpB,CAAC;IAED,wEAAwE;IACxE,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,OAAO,IAAI,SAAS,CAAC;IACnC,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,IAAI;QACR,IAAI,IAAI,CAAC,WAAW;YAAE,OAAO;QAE7B,MAAM,MAAM,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;QACnC,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;QAEtB,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,MAAM,CAAC,2BAA2B,CAAC,CAAC;QAC/D,IAAI,CAAC,SAAS,GAAG,MAAO,QAAuB,CAC7C,oBAAoB,EACpB,IAAI,CAAC,SAAS,EACd;YACE,MAAM,EAAE,MAAe;YACvB,GAAG,CAAC,IAAI,CAAC,KAAK,KAAK,MAAM,IAAI,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC;SACpD,CACF,CAAC;QAEF,4CAA4C;QAC5C,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,sBAAsB,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACjG,MAAM,SAAS,GAAI,KAAkC,CAAC,MAAM,EAAE,CAAC;QAC/D,IAAI,CAAC,KAAK,GAAG,SAAS,CAAC,CAAC,CAAE,CAAC,MAAM,CAAC;QAClC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;IAC1B,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,KAAe;QACzB,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAClC,IAAI,CAAC,IAAI,CAAC,WAAW,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;YACzC,MAAM,IAAI,KAAK,CAAC,uDAAuD,CAAC,CAAC;QAC3E,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACjF,OAAQ,MAAmC,CAAC,MAAM,EAAE,CAAC;IACvD,CAAC;IAED,KAAK,CAAC,OAAO;QACX,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACnB,IAAI,CAAC;gBACH,MAAO,IAAI,CAAC,SAA0D,CAAC,OAAO,EAAE,EAAE,CAAC;YACrF,CAAC;YAAC,MAAM,CAAC,CAAC,yBAAyB,CAAC,CAAC;YACrC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACxB,CAAC;IACH,CAAC;IAED;;;OAGG;IACK,YAAY;QAClB,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;QACnD,IAAI,SAAS;YAAE,OAAO,SAAS,CAAC;QAEhC,4DAA4D;QAC5D,IAAI,OAAO,CAAC,QAAQ,KAAK,QAAQ,IAAI,OAAO,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;YAC9D,OAAO,QAAQ,CAAC;QAClB,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;CACF;AAED,iFAAiF;AAEjF;;;;;;;;;GASG;AACH,MAAM,OAAO,qBAAqB;IACf,KAAK,CAAyB;IACvC,YAAY,GAAyB,IAAI,CAAC;IAElD,YAAY,SAAiB,EAAE,KAAiB;QAC9C,IAAI,CAAC,KAAK,GAAG,IAAI,sBAAsB,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;IAC5D,CAAC;IAED,IAAI,SAAS,KAAa,OAAO,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC;IAExD,IAAI,IAAI,KAAa,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;IAE9C,KAAK,CAAC,IAAI;QACR,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;YACvB,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;QACxC,CAAC;QACD,OAAO,IAAI,CAAC,YAAY,CAAC;IAC3B,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,KAAe;QACzB,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAClC,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAClB,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACjC,CAAC;IAED,KAAK,CAAC,OAAO;QACX,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACtB,IAAI,CAAC;gBAAC,MAAM,IAAI,CAAC,YAAY,CAAC;YAAC,CAAC;YAAC,MAAM,CAAC,CAAC,0BAA0B,CAAC,CAAC;QACvE,CAAC;QACD,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;IAC9B,CAAC;CACF;AAED,iFAAiF;AAEjF;;;GAGG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAG,2BAA2B,CAAC"}

package/dist/indexer/graph-analysis.d.ts

@@ -0,0 +1,115 @@
+/**
+ * @module indexer/graph-analysis
+ *
+ * Higher-level graph analysis primitives operating on the SQLite
+ * knowledge-base:
+ *
+ *  - `detectSymbolCycles(db, opts)` — Tarjan's SCC on the symbol adjacency
+ *    graph (call_refs, type_refs, or both).
+ *  - `findConnectedComponents(db, opts)` — union-find connected components
+ *    at file or symbol scope.
+ *  - `clusterSymbols(db, opts)` — partitions the call graph into bounded-
+ *    size coherent chunks via SCC contraction, same-file merge, greedy
+ *    edge-weight consolidation, and affinity folding.
+ *  - `buildCodebaseSummary(db, opts)` — condensed dependency summary with
+ *    per-module files, symbol counts, line spans, and SCCs.
+ */
+import type { Database } from './db.js';
+import type { ResolutionMethod } from './resolution-method.js';
+export type EdgeKind = 'call' | 'type' | 'both';
+export interface GraphAnalysisOptions {
+    /** Which edge kinds to traverse. Default: 'both'. */
+    edgeKinds?: EdgeKind;
+    /** Resolution methods to include. Default: RESOLVED_METHODS. */
+    methods?: ResolutionMethod[];
+    /** Branch filter. */
+    branch?: string;
+}
+/**
+ * Detects strongly connected components (mutual recursion, circular type
+ * dependencies) in the **symbol** adjacency graph using Tarjan's algorithm.
+ *
+ * Returns arrays of symbol IDs where each SCC has 2+ members (or a single
+ * member with a self-edge).
+ */
+export declare function detectSymbolCycles(db: Database.Database, options?: GraphAnalysisOptions): number[][];
+export interface ConnectedComponentsOptions extends GraphAnalysisOptions {
+    /** Scope of the analysis. Default: 'symbol'. */
+    scope?: 'file' | 'symbol';
+}
+/**
+ * Finds connected components in the **undirected** graph of files or symbols
+ * using a union-find (disjoint set) data structure.
+ *
+ * Returns arrays of IDs where each component has 2+ members.
+ */
+export declare function findConnectedComponents(db: Database.Database, options?: ConnectedComponentsOptions): number[][];
+export interface ClusterOptions extends GraphAnalysisOptions {
+    /** Maximum total line span per cluster. Default: 500. */
+    maxLinesPerCluster?: number;
+}
+export interface SymbolCluster {
+    /** Cluster index (0-based). */
+    id: number;
+    /** Symbol IDs belonging to this cluster. */
+    symbolIds: number[];
+    /** Total line count across all symbols. */
+    totalLines: number;
+    /** File IDs that have at least one symbol in this cluster. */
+    fileIds: number[];
+    /** Number of internal edges (edges within the cluster). */
+    internalEdges: number;
+    /** Number of external edges (edges crossing cluster boundaries). */
+    externalEdges: number;
+}
+/**
+ * Partition the symbol call graph into bounded-size coherent clusters.
+ *
+ * Algorithm:
+ * 1. Contract SCCs (mutually dependent symbols → same cluster)
+ * 2. Merge same-file symbols into one cluster per file
+ * 3. Greedy merge by edge weight (respecting maxLines)
+ * 4. Fold undersized clusters into their heaviest-edge neighbor
+ */
+export declare function clusterSymbols(db: Database.Database, options?: ClusterOptions): SymbolCluster[];
+export interface CodebaseSummaryOptions extends GraphAnalysisOptions {
+    /** Maximum lines per module for clustering. Default: 500. */
+    maxLinesPerModule?: number;
+}
+export interface ModuleSummary {
+    /** Module index. */
+    id: number;
+    /** File paths in this module. */
+    files: string[];
+    /** Total symbol count. */
+    symbolCount: number;
+    /** Total line span. */
+    totalLines: number;
+    /** IDs of modules this module depends on. */
+    dependsOn: number[];
+    /** IDs of modules that depend on this module. */
+    dependedOnBy: number[];
+}
+export interface CodebaseSummary {
+    /** Total indexed files. */
+    totalFiles: number;
+    /** Total indexed symbols. */
+    totalSymbols: number;
+    /** Total resolved edges. */
+    totalEdges: number;
+    /** Modules grouped by clustering. */
+    modules: ModuleSummary[];
+    /** Connected component groups (module IDs). */
+    connectedComponents: number[][];
+    /** Strongly connected component groups (module IDs). */
+    cyclicGroups: number[][];
+}
+/**
+ * Produces a condensed dependency summary of the codebase — the "30-second
+ * architecture overview."
+ *
+ * Combines symbol clustering with inter-module edge analysis and SCC/CC
+ * detection at the module level.
+ */
+export declare function buildCodebaseSummary(db: Database.Database, options?: CodebaseSummaryOptions): CodebaseSummary;
+//# sourceMappingURL=graph-analysis.d.ts.map

package/dist/indexer/graph-analysis.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"graph-analysis.d.ts","sourceRoot":"","sources":["../../src/indexer/graph-analysis.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AACxC,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAK/D,MAAM,MAAM,QAAQ,GAAG,MAAM,GAAG,MAAM,GAAG,MAAM,CAAC;AAEhD,MAAM,WAAW,oBAAoB;IACnC,qDAAqD;IACrD,SAAS,CAAC,EAAE,QAAQ,CAAC;IACrB,gEAAgE;IAChE,OAAO,CAAC,EAAE,gBAAgB,EAAE,CAAC;IAC7B,qBAAqB;IACrB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAqED;;;;;;GAMG;AACH,wBAAgB,kBAAkB,CAChC,EAAE,EAAE,QAAQ,CAAC,QAAQ,EACrB,OAAO,GAAE,oBAAyB,GACjC,MAAM,EAAE,EAAE,CAoEZ;AAID,MAAM,WAAW,0BAA2B,SAAQ,oBAAoB;IACtE,gDAAgD;IAChD,KAAK,CAAC,EAAE,MAAM,GAAG,QAAQ,CAAC;CAC3B;AAED;;;;;GAKG;AACH,wBAAgB,uBAAuB,CACrC,EAAE,EAAE,QAAQ,CAAC,QAAQ,EACrB,OAAO,GAAE,0BAA+B,GACvC,MAAM,EAAE,EAAE,CAOZ;AAkHD,MAAM,WAAW,cAAe,SAAQ,oBAAoB;IAC1D,yDAAyD;IACzD,kBAAkB,CAAC,EAAE,MAAM,CAAC;CAC7B;AAED,MAAM,WAAW,aAAa;IAC5B,+BAA+B;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,4CAA4C;IAC5C,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,2CAA2C;IAC3C,UAAU,EAAE,MAAM,CAAC;IACnB,8DAA8D;IAC9D,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,2DAA2D;IAC3D,aAAa,EAAE,MAAM,CAAC;IACtB,oEAAoE;IACpE,aAAa,EAAE,MAAM,CAAC;CACvB;AAQD;;;;;;;;GAQG;AACH,wBAAgB,cAAc,CAC5B,EAAE,EAAE,QAAQ,CAAC,QAAQ,EACrB,OAAO,GAAE,cAAmB,GAC3B,aAAa,EAAE,CA+LjB;AAmBD,MAAM,WAAW,sBAAuB,SAAQ,oBAAoB;IAClE,6DAA6D;IAC7D,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED,MAAM,WAAW,aAAa;IAC5B,oBAAoB;IACpB,EAAE,EAAE,MAAM,CAAC;IACX,iCAAiC;IACjC,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,0BAA0B;IAC1B,WAAW,EAAE,MAAM,CAAC;IACpB,uBAAuB;IACvB,UAAU,EAAE,MAAM,CAAC;IACnB,6CAA6C;IAC7C,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,iDAAiD;IACjD,YAAY,EAAE,MAAM,EAAE,CAAC;CACxB;AAED,MAAM,WAAW,eAAe;IAC9B,2BAA2B;IAC3B,UAAU,EAAE,MAAM,CAAC;IACnB,6BAA6B;IAC7B,YAAY,EAAE,MAAM,CAAC;IACrB,4BAA4B;IAC5B,UAAU,EAAE,MAAM,CAAC;IACnB,qCAAqC;IACrC,OAAO,EAAE,aAAa,EAAE,CAAC;IACzB,+CAA+C;IAC/C,mBAAmB,EAAE,MAAM,EAAE,EAAE,CAAC;IAChC,wDAAwD;IACxD,YAAY,EAAE,MAAM,EAAE,EAAE,CAAC;CAC1B;AAED;;;;;;GAMG;AACH,wBAAgB,oBAAoB,CAClC,EAAE,EAAE,QAAQ,CAAC,QAAQ,EACrB,OAAO,GAAE,sBAA2B,GACnC,eAAe,CAqHjB"}