gitnexus 1.2.9 → 1.3.0

package/dist/core/ingestion/workers/worker-pool.d.ts

@@ -1,22 +1,16 @@
 export interface WorkerPool {
     /**
      * Dispatch items across workers. Items are split into chunks (one per worker),
-     * each worker processes its chunk, and results are concatenated back in order.
-     *
-     * @param onProgress - Called with cumulative files processed across all workers
+     * each worker processes its chunk via sub-batches to limit peak memory,
+     * and results are concatenated back in order.
      */
     dispatch<TInput, TResult>(items: TInput[], onProgress?: (filesProcessed: number) => void): Promise<TResult[]>;
-    /**
-     * Terminate all workers. Must be called when done.
-     */
+    /** Terminate all workers. Must be called when done. */
     terminate(): Promise<void>;
     /** Number of workers in the pool */
     readonly size: number;
 }
 /**
  * Create a pool of worker threads.
- *
- * @param workerUrl - URL to the worker script (use `new URL('./parse-worker.js', import.meta.url)`)
- * @param poolSize - Number of workers (defaults to cpus - 1, minimum 1)
  */
 export declare const createWorkerPool: (workerUrl: URL, poolSize?: number) => WorkerPool;

package/dist/core/ingestion/workers/worker-pool.js

@@ -1,13 +1,18 @@
 import { Worker } from 'node:worker_threads';
 import os from 'node:os';
+/**
+ * Max files to send to a worker in a single postMessage.
+ * Keeps structured-clone memory bounded per sub-batch.
+ */
+const SUB_BATCH_SIZE = 1500;
+/** Per sub-batch timeout. If a single sub-batch takes longer than this,
+ *  likely a pathological file (e.g. minified 50MB JS). Fail fast. */
+const SUB_BATCH_TIMEOUT_MS = 30_000;
 /**
  * Create a pool of worker threads.
- *
- * @param workerUrl - URL to the worker script (use `new URL('./parse-worker.js', import.meta.url)`)
- * @param poolSize - Number of workers (defaults to cpus - 1, minimum 1)
  */
 export const createWorkerPool = (workerUrl, poolSize) => {
-    const size = poolSize ?? Math.max(1, os.cpus().length - 1);
+    const size = poolSize ?? Math.min(8, Math.max(1, os.cpus().length - 1));
     const workers = [];
     for (let i = 0; i < size; i++) {
         workers.push(new Worker(workerUrl));
@@ -15,32 +20,47 @@ export const createWorkerPool = (workerUrl, poolSize) => {
     const dispatch = (items, onProgress) => {
         if (items.length === 0)
             return Promise.resolve([]);
-        // Split items into one chunk per worker
         const chunkSize = Math.ceil(items.length / size);
         const chunks = [];
         for (let i = 0; i < items.length; i += chunkSize) {
             chunks.push(items.slice(i, i + chunkSize));
         }
-        // Track per-worker progress for cumulative reporting
         const workerProgress = new Array(chunks.length).fill(0);
-        // Send one chunk to each worker, collect results
         const promises = chunks.map((chunk, i) => {
             const worker = workers[i];
             return new Promise((resolve, reject) => {
                 let settled = false;
+                let subBatchTimer = null;
                 const cleanup = () => {
-                    clearTimeout(timer);
+                    if (subBatchTimer)
+                        clearTimeout(subBatchTimer);
                     worker.removeListener('message', handler);
                     worker.removeListener('error', errorHandler);
                     worker.removeListener('exit', exitHandler);
                 };
-                const timer = setTimeout(() => {
-                    if (!settled) {
-                        settled = true;
-                        cleanup();
-                        reject(new Error(`Worker ${i} timed out after 5 minutes (chunk: ${chunk.length} items). Worker may have crashed or is processing too much data.`));
+                const resetSubBatchTimer = () => {
+                    if (subBatchTimer)
+                        clearTimeout(subBatchTimer);
+                    subBatchTimer = setTimeout(() => {
+                        if (!settled) {
+                            settled = true;
+                            cleanup();
+                            reject(new Error(`Worker ${i} sub-batch timed out after ${SUB_BATCH_TIMEOUT_MS / 1000}s (chunk: ${chunk.length} items).`));
+                        }
+                    }, SUB_BATCH_TIMEOUT_MS);
+                };
+                let subBatchIdx = 0;
+                const sendNextSubBatch = () => {
+                    const start = subBatchIdx * SUB_BATCH_SIZE;
+                    if (start >= chunk.length) {
+                        worker.postMessage({ type: 'flush' });
+                        return;
                     }
-                }, 5 * 60 * 1000);
+                    const subBatch = chunk.slice(start, start + SUB_BATCH_SIZE);
+                    subBatchIdx++;
+                    resetSubBatchTimer();
+                    worker.postMessage({ type: 'sub-batch', files: subBatch });
+                };
                 const handler = (msg) => {
                     if (settled)
                         return;
@@ -51,8 +71,10 @@ export const createWorkerPool = (workerUrl, poolSize) => {
                             onProgress(total);
                         }
                     }
+                    else if (msg && msg.type === 'sub-batch-done') {
+                        sendNextSubBatch();
+                    }
                     else if (msg && msg.type === 'error') {
-                        // Error reported by worker via postMessage
                         settled = true;
                         cleanup();
                         reject(new Error(`Worker ${i} error: ${msg.error}`));
@@ -63,7 +85,6 @@ export const createWorkerPool = (workerUrl, poolSize) => {
                         resolve(msg.data);
                     }
                     else {
-                        // Legacy: treat any non-typed message as result
                         settled = true;
                         cleanup();
                         resolve(msg);
@@ -80,13 +101,13 @@ export const createWorkerPool = (workerUrl, poolSize) => {
                     if (!settled) {
                         settled = true;
                         cleanup();
-                        reject(new Error(`Worker ${i} exited unexpectedly with code ${code}. This usually indicates an out-of-memory crash or native addon failure.`));
+                        reject(new Error(`Worker ${i} exited with code ${code}. Likely OOM or native addon failure.`));
                     }
                 };
                 worker.on('message', handler);
                 worker.once('error', errorHandler);
                 worker.once('exit', exitHandler);
-                worker.postMessage(chunk);
+                sendNextSubBatch();
             });
         });
         return Promise.all(promises);

package/dist/core/kuzu/csv-generator.d.ts

@@ -1,8 +1,10 @@
 /**
  * CSV Generator for KuzuDB Hybrid Schema
  *
- * Generates separate CSV files for each node table and one relation CSV.
- * This enables efficient bulk loading via COPY FROM for hybrid schema.
+ * Streams CSV rows directly to disk files in a single pass over graph nodes.
+ * File contents are lazy-read from disk per-node to avoid holding the entire
+ * repo in RAM. Rows are buffered (FLUSH_EVERY) before writing to minimize
+ * per-row Promise overhead.
  *
  * RFC 4180 Compliant:
  * - Fields containing commas, double quotes, or newlines are enclosed in double quotes
@@ -11,12 +13,17 @@
  */
 import { KnowledgeGraph } from '../graph/types.js';
 import { NodeTableName } from './schema.js';
-export interface CSVData {
-    nodes: Map<NodeTableName, string>;
-    relCSV: string;
+export interface StreamedCSVResult {
+    nodeFiles: Map<NodeTableName, {
+        csvPath: string;
+        rows: number;
+    }>;
+    relCsvPath: string;
+    relRows: number;
 }
 /**
- * Generate all CSV data for hybrid schema bulk loading
- * Returns Maps of node table name -> CSV content, and single relation CSV
+ * Stream all CSV data directly to disk files.
+ * Iterates graph nodes exactly ONCE — routes each node to the right writer.
+ * File contents are lazy-read from disk with a generous LRU cache.
  */
-export declare const generateAllCSVs: (graph: KnowledgeGraph, fileContents: Map<string, string>) => CSVData;
+export declare const streamAllCSVsToDisk: (graph: KnowledgeGraph, repoPath: string, csvDir: string) => Promise<StreamedCSVResult>;