gitnexus 1.6.4-rc.2 → 1.6.4-rc.21

package/dist/core/ingestion/workers/parse-worker.js

@@ -15,7 +15,7 @@ import Ruby from 'tree-sitter-ruby';
 import { createRequire } from 'node:module';
 import { SupportedLanguages } from '../../../_shared/index.js';
 import { getProvider } from '../languages/index.js';
-import { getTreeSitterBufferSize, TREE_SITTER_MAX_BUFFER } from '../constants.js';
+import { getTreeSitterBufferSize, getTreeSitterContentByteLength, TREE_SITTER_MAX_BUFFER, } from '../constants.js';
 // tree-sitter-swift is an optionalDependency — may not be installed
 const _require = createRequire(import.meta.url);
 let Swift = null;
@@ -452,7 +452,7 @@ const processBatch = (files, onProgress) => {
     }
     let totalProcessed = 0;
     let lastReported = 0;
-    const PROGRESS_INTERVAL = 100; // report every 100 files
+    const PROGRESS_INTERVAL = Math.max(1, Math.min(100, Math.ceil(files.length / 10)));
     const onFileProcessed = onProgress
         ? () => {
             totalProcessed++;
@@ -519,6 +519,9 @@ const processBatch = (files, onProgress) => {
             }
         }
     }
+    if (onProgress && totalProcessed !== lastReported) {
+        onProgress(totalProcessed);
+    }
     return result;
 };
 const ROUTE_HTTP_METHODS = new Set([
@@ -1001,7 +1004,7 @@ const processFileGroup = (files, language, queryString, result, onFileProcessed)
     }
     for (const file of files) {
         // Skip files larger than the max tree-sitter buffer (32 MB)
-        if (file.content.length > TREE_SITTER_MAX_BUFFER)
+        if (getTreeSitterContentByteLength(file.content) > TREE_SITTER_MAX_BUFFER)
             continue;
         // Vue SFC preprocessing: extract <script> block content
         let parseContent = file.content;
@@ -1019,7 +1022,7 @@ const processFileGroup = (files, language, queryString, result, onFileProcessed)
         let tree;
         try {
             tree = parser.parse(parseContent, undefined, {
-                bufferSize: getTreeSitterBufferSize(parseContent.length),
+                bufferSize: getTreeSitterBufferSize(parseContent),
             });
         }
         catch (err) {
@@ -1048,7 +1051,7 @@ const processFileGroup = (files, language, queryString, result, onFileProcessed)
                 parentPort.postMessage({ type: 'warning', message });
             else
                 console.warn(message);
-        });
+        }, tree);
         if (parsedFile !== undefined)
             result.parsedFiles.push(parsedFile);
         // Pre-pass: extract heritage from query matches to build parentMap for buildTypeEnv.
@@ -1612,10 +1615,12 @@ const processFileGroup = (files, language, queryString, result, onFileProcessed)
                     }
                 }
             }
-            // Append #<paramCount> to Method/Constructor IDs to disambiguate overloads.
-            // Functions are not suffixed — they don't overload by name in the same scope.
+            // Append #<paramCount> to owned callable IDs to disambiguate overloads.
+            // Top-level Function IDs stay stable; functions inside an owner may overload.
             // When same-arity collisions exist, append ~type1,type2 for further disambiguation.
-            const needsAritySuffix = nodeLabel === 'Method' || nodeLabel === 'Constructor';
+            const needsAritySuffix = nodeLabel === 'Method' ||
+                nodeLabel === 'Constructor' ||
+                (nodeLabel === 'Function' && enclosingClassId !== null);
             let arityTag = needsAritySuffix && arityForId !== undefined ? `#${arityForId}` : '';
             if (arityTag && defMethodMap && defMethodInfo) {
                 const groups = buildCollisionGroups(defMethodMap);
@@ -1668,8 +1673,9 @@ const processFileGroup = (files, language, queryString, result, onFileProcessed)
                             result.toolDefs.push({
                                 filePath: file.path,
                                 toolName: nodeName,
-                                description: dec.arg || '',
+                                description: (dec.arg || description || '').slice(0, 200),
                                 lineNumber: definitionNode.startPosition.row + lineOffset,
+                                handlerNodeId: nodeId,
                             });
                         }
                         fileDecorators.delete(checkLine);

package/dist/core/ingestion/workers/worker-pool.d.ts

@@ -1,8 +1,7 @@
 export interface WorkerPool {
     /**
-     * Dispatch items across workers. Items are split into chunks (one per worker),
-     * each worker processes its chunk via sub-batches to limit peak memory,
-     * and results are concatenated back in order.
+     * Dispatch items across workers. Items are split into bounded jobs, each job
+     * is committed independently, and stalled jobs are split/retried locally.
      */
     dispatch<TInput, TResult>(items: TInput[], onProgress?: (filesProcessed: number) => void): Promise<TResult[]>;
     /** Terminate all workers. Must be called when done. */
@@ -10,7 +9,15 @@ export interface WorkerPool {
     /** Number of workers in the pool */
     readonly size: number;
 }
+export interface WorkerPoolOptions {
+    subBatchSize?: number;
+    subBatchMaxBytes?: number;
+    subBatchIdleTimeoutMs?: number;
+    maxTimeoutRetries?: number;
+    timeoutBackoffFactor?: number;
+}
+export declare function resolveWorkerPoolOptions(options?: WorkerPoolOptions): Required<WorkerPoolOptions>;
 /**
  * Create a pool of worker threads.
  */
-export declare const createWorkerPool: (workerUrl: URL, poolSize?: number) => WorkerPool;
+export declare const createWorkerPool: (workerUrl: URL, poolSize?: number, options?: WorkerPoolOptions) => WorkerPool;

package/dist/core/ingestion/workers/worker-pool.js

@@ -7,13 +7,83 @@ import { fileURLToPath } from 'node:url';
  * Keeps structured-clone memory bounded per sub-batch.
  */
 const SUB_BATCH_SIZE = 1500;
-/** Per sub-batch timeout. If a single sub-batch takes longer than this,
- *  likely a pathological file (e.g. minified 50MB JS). Fail fast. */
-const SUB_BATCH_TIMEOUT_MS = 30_000;
+const SUB_BATCH_MAX_BYTES = 8 * 1024 * 1024;
+const DEFAULT_SUB_BATCH_IDLE_TIMEOUT_MS = 30_000;
+const DEFAULT_TIMEOUT_RETRIES = 1;
+const DEFAULT_TIMEOUT_BACKOFF_FACTOR = 2;
+function positiveInteger(value) {
+    const parsed = typeof value === 'string' ? Number(value) : value;
+    return typeof parsed === 'number' && Number.isFinite(parsed) && parsed > 0
+        ? Math.floor(parsed)
+        : undefined;
+}
+function nonNegativeInteger(value) {
+    const parsed = typeof value === 'string' ? Number(value) : value;
+    return typeof parsed === 'number' && Number.isFinite(parsed) && parsed >= 0
+        ? Math.floor(parsed)
+        : undefined;
+}
+export function resolveWorkerPoolOptions(options = {}) {
+    return {
+        subBatchSize: positiveInteger(options.subBatchSize) ?? SUB_BATCH_SIZE,
+        subBatchMaxBytes: positiveInteger(options.subBatchMaxBytes) ??
+            positiveInteger(process.env.GITNEXUS_WORKER_SUB_BATCH_MAX_BYTES) ??
+            SUB_BATCH_MAX_BYTES,
+        subBatchIdleTimeoutMs: positiveInteger(options.subBatchIdleTimeoutMs) ??
+            positiveInteger(process.env.GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS) ??
+            DEFAULT_SUB_BATCH_IDLE_TIMEOUT_MS,
+        maxTimeoutRetries: nonNegativeInteger(options.maxTimeoutRetries) ?? DEFAULT_TIMEOUT_RETRIES,
+        timeoutBackoffFactor: positiveInteger(options.timeoutBackoffFactor) ?? DEFAULT_TIMEOUT_BACKOFF_FACTOR,
+    };
+}
+function estimateItemBytes(item) {
+    if (typeof item !== 'object' || item === null)
+        return 0;
+    const content = item.content;
+    return typeof content === 'string' ? Buffer.byteLength(content, 'utf8') : 0;
+}
+function itemPath(item) {
+    if (typeof item !== 'object' || item === null)
+        return undefined;
+    const path = item.path;
+    return typeof path === 'string' ? path : undefined;
+}
+function createJobs(items, maxItems, maxBytes, timeoutMs) {
+    const jobs = [];
+    let startIndex = 0;
+    let batch = [];
+    let batchBytes = 0;
+    const flush = () => {
+        if (batch.length === 0)
+            return;
+        jobs.push({
+            startIndex,
+            items: batch,
+            estimatedBytes: batchBytes,
+            attempt: 0,
+            splitDepth: 0,
+            timeoutMs,
+        });
+        startIndex += batch.length;
+        batch = [];
+        batchBytes = 0;
+    };
+    for (const item of items) {
+        const itemBytes = estimateItemBytes(item);
+        const wouldExceedItems = batch.length >= maxItems;
+        const wouldExceedBytes = batch.length > 0 && batchBytes + itemBytes > maxBytes;
+        if (wouldExceedItems || wouldExceedBytes)
+            flush();
+        batch.push(item);
+        batchBytes += itemBytes;
+    }
+    flush();
+    return jobs;
+}
 /**
  * Create a pool of worker threads.
  */
-export const createWorkerPool = (workerUrl, poolSize) => {
+export const createWorkerPool = (workerUrl, poolSize, options) => {
     // Validate worker script exists before spawning to prevent uncaught
     // MODULE_NOT_FOUND crashes in worker threads (e.g. when running from src/ via vitest)
     const workerPath = fileURLToPath(workerUrl);
@@ -21,102 +91,228 @@ export const createWorkerPool = (workerUrl, poolSize) => {
         throw new Error(`Worker script not found: ${workerPath}`);
     }
     const size = poolSize ?? Math.min(8, Math.max(1, os.cpus().length - 1));
+    const poolOptions = resolveWorkerPoolOptions(options);
     const workers = [];
+    let poolBroken = false;
+    let poolFailure;
     for (let i = 0; i < size; i++) {
         workers.push(new Worker(workerUrl));
     }
     const dispatch = (items, onProgress) => {
+        if (poolBroken) {
+            const reason = poolFailure ? `: ${poolFailure.message}` : '';
+            return Promise.reject(new Error(`Worker pool is unavailable after a previous failure${reason}`));
+        }
         if (items.length === 0)
             return Promise.resolve([]);
-        const chunkSize = Math.ceil(items.length / size);
-        const chunks = [];
-        for (let i = 0; i < items.length; i += chunkSize) {
-            chunks.push(items.slice(i, i + chunkSize));
-        }
-        const workerProgress = new Array(chunks.length).fill(0);
-        const promises = chunks.map((chunk, i) => {
-            const worker = workers[i];
-            return new Promise((resolve, reject) => {
+        if (workers.length === 0)
+            return Promise.reject(new Error('Worker pool has no active workers'));
+        const jobs = createJobs(items, poolOptions.subBatchSize, poolOptions.subBatchMaxBytes, poolOptions.subBatchIdleTimeoutMs);
+        return new Promise((resolve, reject) => {
+            const results = [];
+            const inFlightProgress = new Array(workers.length).fill(0);
+            let completedFiles = 0;
+            let activeWorkers = 0;
+            let stopped = false;
+            let maxReported = 0;
+            const reportProgress = () => {
+                if (!onProgress)
+                    return;
+                const inFlight = inFlightProgress.reduce((sum, value) => sum + value, 0);
+                const next = Math.min(items.length, Math.max(maxReported, completedFiles + inFlight));
+                if (next === maxReported)
+                    return;
+                maxReported = next;
+                onProgress(next);
+            };
+            const replaceWorker = async (workerIndex) => {
+                const worker = workers[workerIndex];
+                await worker?.terminate().catch(() => undefined);
+                if (!stopped)
+                    workers[workerIndex] = new Worker(workerUrl);
+            };
+            const fail = async (err) => {
+                poolBroken = true;
+                poolFailure = err;
+                if (stopped)
+                    return;
+                stopped = true;
+                await Promise.all(workers.map((worker) => worker.terminate().catch(() => undefined)));
+                reject(err);
+            };
+            const maybeDone = () => {
+                if (stopped)
+                    return;
+                if (jobs.length === 0 && activeWorkers === 0) {
+                    stopped = true;
+                    results.sort((a, b) => a.startIndex - b.startIndex);
+                    if (onProgress && maxReported < items.length)
+                        onProgress(items.length);
+                    resolve(results.map((result) => result.data));
+                }
+            };
+            const requeueAfterTimeout = (workerIndex, job, lastProgress) => {
+                const nextTimeout = Math.ceil(job.timeoutMs * poolOptions.timeoutBackoffFactor);
+                if (job.items.length > 1) {
+                    const midpoint = Math.ceil(job.items.length / 2);
+                    const firstItems = job.items.slice(0, midpoint);
+                    const secondItems = job.items.slice(midpoint);
+                    const first = {
+                        startIndex: job.startIndex,
+                        items: firstItems,
+                        estimatedBytes: firstItems.reduce((sum, item) => sum + estimateItemBytes(item), 0),
+                        attempt: job.attempt,
+                        splitDepth: job.splitDepth + 1,
+                        timeoutMs: nextTimeout,
+                    };
+                    const second = {
+                        startIndex: job.startIndex + midpoint,
+                        items: secondItems,
+                        estimatedBytes: secondItems.reduce((sum, item) => sum + estimateItemBytes(item), 0),
+                        attempt: job.attempt,
+                        splitDepth: job.splitDepth + 1,
+                        timeoutMs: nextTimeout,
+                    };
+                    console.warn(`Worker ${workerIndex} parse job idle timeout after ${job.timeoutMs / 1000}s ` +
+                        `(${job.items.length} items, ${job.estimatedBytes} bytes, last progress: ${lastProgress}). ` +
+                        `Splitting into ${first.items.length}/${second.items.length} item jobs with ` +
+                        `${nextTimeout / 1000}s timeout.`);
+                    // Preserve intuitive retry order; final result order is still enforced by startIndex sort.
+                    jobs.unshift(first, second);
+                    return true;
+                }
+                const nextAttempt = job.attempt + 1;
+                if (nextAttempt <= poolOptions.maxTimeoutRetries) {
+                    console.warn(`Worker ${workerIndex} parse job idle timeout after ${job.timeoutMs / 1000}s ` +
+                        `(single item, attempt ${nextAttempt}/${poolOptions.maxTimeoutRetries + 1}). ` +
+                        `Retrying with ${nextTimeout / 1000}s timeout.`);
+                    jobs.unshift({
+                        ...job,
+                        attempt: nextAttempt,
+                        timeoutMs: nextTimeout,
+                    });
+                    return true;
+                }
+                void fail(new Error(`Worker ${workerIndex} parse job idle timeout after ${job.timeoutMs / 1000}s ` +
+                    `(single item${itemPath(job.items[0]) ? `: ${itemPath(job.items[0])}` : ''}, ` +
+                    `${job.estimatedBytes} bytes, last progress: ${lastProgress}). ` +
+                    `Analyze will retry through sequential fallback. Increase with ` +
+                    `--worker-timeout or GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS.`));
+                return false;
+            };
+            const runWorker = (workerIndex) => {
+                if (stopped)
+                    return;
+                const job = jobs.shift();
+                if (!job) {
+                    maybeDone();
+                    return;
+                }
+                activeWorkers++;
+                inFlightProgress[workerIndex] = 0;
+                const worker = workers[workerIndex];
                 let settled = false;
-                let subBatchTimer = null;
+                let waitingForFlush = false;
+                let idleTimer = null;
+                let lastProgress = 0;
                 const cleanup = () => {
-                    if (subBatchTimer)
-                        clearTimeout(subBatchTimer);
+                    if (idleTimer)
+                        clearTimeout(idleTimer);
                     worker.removeListener('message', handler);
                     worker.removeListener('error', errorHandler);
                     worker.removeListener('exit', exitHandler);
                 };
-                const resetSubBatchTimer = () => {
-                    if (subBatchTimer)
-                        clearTimeout(subBatchTimer);
-                    subBatchTimer = setTimeout(() => {
+                const finishJob = () => {
+                    activeWorkers--;
+                    inFlightProgress[workerIndex] = 0;
+                    runWorker(workerIndex);
+                    maybeDone();
+                };
+                const resetIdleTimer = () => {
+                    if (idleTimer)
+                        clearTimeout(idleTimer);
+                    idleTimer = setTimeout(async () => {
                         if (!settled) {
                             settled = true;
                             cleanup();
-                            reject(new Error(`Worker ${i} sub-batch timed out after ${SUB_BATCH_TIMEOUT_MS / 1000}s (chunk: ${chunk.length} items).`));
+                            activeWorkers--;
+                            inFlightProgress[workerIndex] = 0;
+                            const shouldContinue = requeueAfterTimeout(workerIndex, job, lastProgress);
+                            if (!shouldContinue)
+                                return;
+                            await replaceWorker(workerIndex);
+                            reportProgress();
+                            runWorker(workerIndex);
+                            maybeDone();
                         }
-                    }, SUB_BATCH_TIMEOUT_MS);
-                };
-                let subBatchIdx = 0;
-                const sendNextSubBatch = () => {
-                    const start = subBatchIdx * SUB_BATCH_SIZE;
-                    if (start >= chunk.length) {
-                        worker.postMessage({ type: 'flush' });
-                        return;
-                    }
-                    const subBatch = chunk.slice(start, start + SUB_BATCH_SIZE);
-                    subBatchIdx++;
-                    resetSubBatchTimer();
-                    worker.postMessage({ type: 'sub-batch', files: subBatch });
+                    }, job.timeoutMs);
                 };
                 const handler = (msg) => {
-                    if (settled)
+                    if (settled || stopped)
                         return;
                     if (msg.type === 'progress') {
-                        workerProgress[i] = msg.filesProcessed;
-                        if (onProgress) {
-                            const total = workerProgress.reduce((a, b) => a + b, 0);
-                            onProgress(total);
-                        }
+                        const bounded = Math.min(job.items.length, Math.max(0, msg.filesProcessed));
+                        inFlightProgress[workerIndex] = bounded;
+                        lastProgress = bounded;
+                        resetIdleTimer();
+                        reportProgress();
                     }
                     else if (msg.type === 'warning') {
+                        resetIdleTimer();
                         console.warn(msg.message);
                     }
                     else if (msg.type === 'sub-batch-done') {
-                        sendNextSubBatch();
+                        waitingForFlush = true;
+                        resetIdleTimer();
+                        worker.postMessage({ type: 'flush' });
                     }
                     else if (msg.type === 'error') {
                         settled = true;
                         cleanup();
-                        reject(new Error(`Worker ${i} error: ${msg.error}`));
+                        void fail(new Error(`Worker ${workerIndex} error: ${msg.error}`));
                     }
                     else if (msg.type === 'result') {
+                        if (!waitingForFlush) {
+                            settled = true;
+                            cleanup();
+                            void fail(new Error(`Worker ${workerIndex} protocol error: result before flush`));
+                            return;
+                        }
                         settled = true;
                         cleanup();
-                        resolve(msg.data);
+                        results.push({ startIndex: job.startIndex, data: msg.data });
+                        completedFiles += job.items.length;
+                        reportProgress();
+                        finishJob();
                     }
                 };
                 const errorHandler = (err) => {
                     if (!settled) {
                         settled = true;
                         cleanup();
-                        reject(err);
+                        void fail(err);
                     }
                 };
                 const exitHandler = (code) => {
                     if (!settled) {
                         settled = true;
                         cleanup();
-                        reject(new Error(`Worker ${i} exited with code ${code}. Likely OOM or native addon failure.`));
+                        void fail(new Error(`Worker ${workerIndex} exited with code ${code}. Likely OOM or native addon failure.`));
                     }
                 };
                 worker.on('message', handler);
                 worker.once('error', errorHandler);
                 worker.once('exit', exitHandler);
-                sendNextSubBatch();
-            });
+                resetIdleTimer();
+                if (stopped) {
+                    cleanup();
+                    return;
+                }
+                worker.postMessage({ type: 'sub-batch', files: job.items });
+            };
+            for (let i = 0; i < workers.length; i++)
+                runWorker(i);
         });
-        return Promise.all(promises);
     };
     const terminate = async () => {
         await Promise.all(workers.map((w) => w.terminate()));

package/dist/core/lbug/extension-loader.d.ts

@@ -0,0 +1,86 @@
+/**
+ * Lifecycle policy for an optional DuckDB extension.
+ *
+ * - `auto`     — try `LOAD`, fall back to one bounded out-of-process `INSTALL`
+ *                attempt per process if `LOAD` fails. Default for analyze.
+ * - `load-only`— try `LOAD` only; never spawn an installer. Used by serve/MCP
+ *                read paths so user queries never block on a network install.
+ * - `never`    — skip the extension entirely. Operators can use this to
+ *                forcibly disable optional search features.
+ */
+export type ExtensionInstallPolicy = 'auto' | 'load-only' | 'never';
+export interface ExtensionInstallResult {
+    success: boolean;
+    timedOut: boolean;
+    message: string;
+}
+/** Snapshot of one optional extension's resolved capability state. */
+export interface ExtensionCapability {
+    name: string;
+    loaded: boolean;
+    /** Human-readable reason when `loaded` is false. */
+    reason?: string;
+}
+/** Per-call overrides applied on top of `ExtensionManager` defaults. */
+export interface ExtensionEnsureOptions {
+    policy?: ExtensionInstallPolicy;
+    installTimeoutMs?: number;
+}
+export interface ExtensionManagerOptions {
+    policy?: ExtensionInstallPolicy;
+    installTimeoutMs?: number;
+    installExtension?: (extensionName: string, timeoutMs: number) => Promise<ExtensionInstallResult>;
+    warn?: (message: string) => void;
+}
+export declare const getExtensionInstallTimeoutMs: () => number;
+export declare const getExtensionInstallChildProcessArgs: (extensionName: string) => string[];
+/**
+ * Run `INSTALL <extension>` in a short-lived child Node process so the parent
+ * event loop is never blocked by DuckDB's synchronous network call.
+ *
+ * The child opens its own scratch LadybugDB, executes the install, and exits.
+ * If the child exceeds `timeoutMs` the parent kills it with SIGKILL and
+ * resolves with `timedOut: true`.
+ */
+export declare const installDuckDbExtensionOutOfProcess: (extensionName: string, timeoutMs?: number) => Promise<ExtensionInstallResult>;
+/**
+ * Centralized lifecycle manager for optional LadybugDB extensions.
+ *
+ * Always tries `LOAD EXTENSION <name>` first — it is per-connection,
+ * idempotent, and never touches the network. If `LOAD` fails and the active
+ * policy permits, the manager runs a single bounded out-of-process `INSTALL`
+ * attempt per process and retries `LOAD`. Capability outcomes are cached so
+ * unavailable extensions degrade search features without ever blocking
+ * subsequent analyze or query calls.
+ *
+ * Policy precedence (most specific wins):
+ *   per-call `opts.policy` → constructor `options.policy` → env → `auto`
+ */
+export declare class ExtensionManager {
+    private readonly options;
+    private readonly capabilities;
+    private readonly installAttempted;
+    private readonly warnedKeys;
+    constructor(options?: ExtensionManagerOptions);
+    /** Reset cached capability and install state. Test-only. */
+    reset(): void;
+    /** Snapshot of currently-known optional extension capabilities. */
+    getCapabilities(): ExtensionCapability[];
+    /**
+     * Ensure an optional extension is loaded on the supplied connection.
+     *
+     * Returns `true` when the extension is usable on `query`, `false` when it
+     * is unavailable. Never throws on install failure — analyze and query
+     * paths are expected to degrade gracefully.
+     */
+    ensure(query: (sql: string) => Promise<unknown>, name: string, label: string, opts?: ExtensionEnsureOptions): Promise<boolean>;
+    private tryLoad;
+    private markLoaded;
+    private markUnavailable;
+}
+/** Process-wide singleton shared by core and pool adapters. */
+export declare const extensionManager: ExtensionManager;
+/** Snapshot of which optional DuckDB extensions are loaded in this process. */
+export declare const getExtensionCapabilities: () => ExtensionCapability[];
+/** Test-only: clear the singleton's cached capability and install state. */
+export declare const resetExtensionState: () => void;