@grafema/core 0.1.0-alpha.1

package/dist/core/ASTWorkerPool.js

@@ -0,0 +1,207 @@
+/**
+ * ASTWorkerPool - pool of worker_threads for parallel AST parsing
+ *
+ * Uses actual OS threads via Node.js worker_threads module
+ * for true parallel CPU-intensive parsing with Babel.
+ */
+import { Worker } from 'worker_threads';
+import { EventEmitter } from 'events';
+import { fileURLToPath } from 'url';
+import { dirname, join } from 'path';
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+const WORKER_SCRIPT = join(__dirname, 'ASTWorker.js');
+export class ASTWorkerPool extends EventEmitter {
+    workerCount;
+    workers;
+    taskQueue;
+    pendingTasks;
+    taskIdCounter;
+    readyWorkers;
+    initialized;
+    constructor(workerCount = 4) {
+        super();
+        this.workerCount = Math.min(workerCount, 8); // Cap at 8 threads
+        this.workers = [];
+        this.taskQueue = [];
+        this.pendingTasks = new Map();
+        this.taskIdCounter = 0;
+        this.readyWorkers = [];
+        this.initialized = false;
+    }
+    /**
+     * Initialize the worker pool
+     */
+    async init() {
+        if (this.initialized)
+            return;
+        const initPromises = [];
+        for (let i = 0; i < this.workerCount; i++) {
+            const worker = new Worker(WORKER_SCRIPT);
+            const initPromise = new Promise((resolve, reject) => {
+                const timeout = setTimeout(() => {
+                    reject(new Error(`Worker ${i} initialization timeout`));
+                }, 10000);
+                worker.once('message', (msg) => {
+                    if (msg.type === 'ready') {
+                        clearTimeout(timeout);
+                        resolve();
+                    }
+                });
+                worker.once('error', (err) => {
+                    clearTimeout(timeout);
+                    reject(err);
+                });
+            });
+            worker.on('message', (msg) => this._handleMessage(worker, msg));
+            worker.on('error', (err) => this._handleError(worker, err));
+            worker.on('exit', (code) => this._handleExit(worker, code));
+            this.workers.push(worker);
+            initPromises.push(initPromise);
+        }
+        await Promise.all(initPromises);
+        // All workers ready
+        this.readyWorkers = [...this.workers];
+        this.initialized = true;
+        this.emit('pool:ready', { workerCount: this.workerCount });
+    }
+    /**
+     * Parse a module using a worker thread
+     * Returns promise that resolves to collections
+     */
+    parseModule(filePath, moduleId, moduleName) {
+        return new Promise((resolve, reject) => {
+            const taskId = this.taskIdCounter++;
+            const task = { taskId, filePath, moduleId, moduleName, resolve, reject };
+            this.pendingTasks.set(taskId, task);
+            // Try to dispatch immediately
+            this._dispatchNext(task);
+        });
+    }
+    /**
+     * Parse multiple modules in parallel
+     */
+    async parseModules(modules) {
+        await this.init();
+        const promises = modules.map(m => this.parseModule(m.file, m.id, m.name)
+            .then(collections => ({ module: m, collections, error: null }))
+            .catch(error => ({ module: m, collections: null, error: error })));
+        return Promise.all(promises);
+    }
+    /**
+     * Dispatch task to available worker or queue it
+     */
+    _dispatchNext(task) {
+        if (this.readyWorkers.length > 0) {
+            const worker = this.readyWorkers.pop();
+            worker.postMessage({
+                type: 'parse',
+                taskId: task.taskId,
+                filePath: task.filePath,
+                moduleId: task.moduleId,
+                moduleName: task.moduleName
+            });
+            this.emit('task:started', { taskId: task.taskId, filePath: task.filePath });
+        }
+        else {
+            // Queue the task
+            this.taskQueue.push(task);
+        }
+    }
+    /**
+     * Handle message from worker
+     */
+    _handleMessage(worker, msg) {
+        if (msg.type === 'result') {
+            const task = this.pendingTasks.get(msg.taskId);
+            if (task) {
+                this.pendingTasks.delete(msg.taskId);
+                task.resolve(msg.collections);
+                this.emit('task:completed', { taskId: msg.taskId });
+            }
+            // Worker is ready for more work
+            this._workerReady(worker);
+        }
+        else if (msg.type === 'error') {
+            const task = this.pendingTasks.get(msg.taskId);
+            if (task) {
+                this.pendingTasks.delete(msg.taskId);
+                task.reject(new Error(msg.error));
+                this.emit('task:failed', { taskId: msg.taskId, error: msg.error });
+            }
+            // Worker is still ready
+            this._workerReady(worker);
+        }
+    }
+    /**
+     * Mark worker as ready and dispatch queued task
+     */
+    _workerReady(worker) {
+        if (this.taskQueue.length > 0) {
+            const nextTask = this.taskQueue.shift();
+            worker.postMessage({
+                type: 'parse',
+                taskId: nextTask.taskId,
+                filePath: nextTask.filePath,
+                moduleId: nextTask.moduleId,
+                moduleName: nextTask.moduleName
+            });
+            this.emit('task:started', { taskId: nextTask.taskId, filePath: nextTask.filePath });
+        }
+        else {
+            this.readyWorkers.push(worker);
+        }
+    }
+    /**
+     * Handle worker error
+     */
+    _handleError(worker, error) {
+        console.error('[ASTWorkerPool] Worker error:', error);
+        this.emit('worker:error', { error });
+    }
+    /**
+     * Handle worker exit
+     */
+    _handleExit(worker, code) {
+        if (code !== 0) {
+            console.error(`[ASTWorkerPool] Worker exited with code ${code}`);
+        }
+        // Remove from workers list
+        const index = this.workers.indexOf(worker);
+        if (index !== -1) {
+            this.workers.splice(index, 1);
+        }
+        // Remove from ready workers
+        const readyIndex = this.readyWorkers.indexOf(worker);
+        if (readyIndex !== -1) {
+            this.readyWorkers.splice(readyIndex, 1);
+        }
+    }
+    /**
+     * Terminate all workers
+     */
+    async terminate() {
+        const terminatePromises = this.workers.map(worker => {
+            return new Promise((resolve) => {
+                worker.once('exit', () => resolve());
+                worker.postMessage({ type: 'exit' });
+            });
+        });
+        await Promise.all(terminatePromises);
+        this.workers = [];
+        this.readyWorkers = [];
+        this.initialized = false;
+        this.emit('pool:terminated');
+    }
+    /**
+     * Get pool stats
+     */
+    getStats() {
+        return {
+            workerCount: this.workerCount,
+            activeWorkers: this.workerCount - this.readyWorkers.length,
+            queuedTasks: this.taskQueue.length,
+            pendingTasks: this.pendingTasks.size
+        };
+    }
+}

package/dist/core/AnalysisQueue.d.ts

@@ -0,0 +1,104 @@
+/**
+ * AnalysisQueue - Queue-based parallel analysis orchestration
+ *
+ * Architecture:
+ * - Tasks are file + applicable plugins
+ * - Workers pick tasks from queue, run plugins, write to RFDB
+ * - Barrier waits for all tasks before ENRICHMENT phase
+ *
+ * Usage:
+ *   const queue = new AnalysisQueue({
+ *     socketPath: '/tmp/rfdb.sock',
+ *     maxWorkers: 4,
+ *     plugins: [JSASTAnalyzer, ExpressRouteAnalyzer, ...]
+ *   });
+ *
+ *   await queue.start();
+ *   queue.addTask({ file: '/path/to/file.js', moduleId: '...', plugins: ['JSASTAnalyzer'] });
+ *   await queue.waitForCompletion();
+ *   await queue.stop();
+ */
+import { EventEmitter } from 'events';
+/**
+ * Queue options
+ */
+export interface AnalysisQueueOptions {
+    socketPath?: string;
+    maxWorkers?: number;
+    plugins?: unknown[];
+}
+/**
+ * Task definition
+ */
+export interface AnalysisTask {
+    file: string;
+    moduleId: string;
+    moduleName: string;
+    plugins?: string[];
+}
+/**
+ * Queue statistics
+ */
+export interface QueueStats {
+    tasksTotal: number;
+    tasksCompleted: number;
+    tasksFailed: number;
+    nodesCreated: number;
+    edgesCreated: number;
+    startTime: number | null;
+    errors: Array<{
+        file: string;
+        error: string;
+    }>;
+    pending?: number;
+    active?: number;
+    elapsed?: number;
+}
+export declare class AnalysisQueue extends EventEmitter {
+    private socketPath;
+    private maxWorkers;
+    private pluginConfigs;
+    private workers;
+    private pendingTasks;
+    private activeTasks;
+    private completedCount;
+    private failedCount;
+    private taskIdCounter;
+    private running;
+    private draining;
+    private completionPromise;
+    private completionResolve;
+    private stats;
+    constructor(options?: AnalysisQueueOptions);
+    /**
+     * Start the queue (spawn workers)
+     */
+    start(): Promise<void>;
+    /**
+     * Stop the queue (terminate workers)
+     */
+    stop(): Promise<void>;
+    /**
+     * Add a task to the queue
+     */
+    addTask(task: AnalysisTask): number;
+    /**
+     * Add multiple tasks at once
+     */
+    addTasks(tasks: AnalysisTask[]): void;
+    /**
+     * Wait for all tasks to complete
+     */
+    waitForCompletion(): Promise<QueueStats>;
+    /**
+     * Get current statistics
+     */
+    getStats(): QueueStats;
+    private _dispatchNext;
+    private _handleWorkerMessage;
+    private _handleWorkerError;
+    private _handleWorkerExit;
+    private _checkCompletion;
+}
+export default AnalysisQueue;
+//# sourceMappingURL=AnalysisQueue.d.ts.map

package/dist/core/AnalysisQueue.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"AnalysisQueue.d.ts","sourceRoot":"","sources":["../../src/core/AnalysisQueue.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAGH,OAAO,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAC;AAStC;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,OAAO,EAAE,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;CACpB;AA0BD;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,UAAU,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,MAAM,CAAC;IACvB,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,MAAM,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC/C,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAoCD,qBAAa,aAAc,SAAQ,YAAY;IAC7C,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,aAAa,CAAY;IACjC,OAAO,CAAC,OAAO,CAAe;IAC9B,OAAO,CAAC,YAAY,CAAa;IACjC,OAAO,CAAC,WAAW,CAA8B;IACjD,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,OAAO,CAAU;IACzB,OAAO,CAAC,QAAQ,CAAU;IAC1B,OAAO,CAAC,iBAAiB,CAAuB;IAChD,OAAO,CAAC,iBAAiB,CAAsB;IAC/C,OAAO,CAAC,KAAK,CAAa;gBAEd,OAAO,GAAE,oBAAyB;IAkC9C;;OAEG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAyD5B;;OAEG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAqB3B;;OAEG;IACH,OAAO,CAAC,IAAI,EAAE,YAAY,GAAG,MAAM;IAmBnC;;OAEG;IACH,QAAQ,CAAC,KAAK,EAAE,YAAY,EAAE,GAAG,IAAI;IAMrC;;OAEG;IACG,iBAAiB,IAAI,OAAO,CAAC,UAAU,CAAC;IAiB9C;;OAEG;IACH,QAAQ,IAAI,UAAU;IAatB,OAAO,CAAC,aAAa;IAgCrB,OAAO,CAAC,oBAAoB;IAsD5B,OAAO,CAAC,kBAAkB;IAK1B,OAAO,CAAC,iBAAiB;IAYzB,OAAO,CAAC,gBAAgB;CAgBzB;AAED,eAAe,aAAa,CAAC"}

package/dist/core/AnalysisQueue.js

@@ -0,0 +1,299 @@
+/**
+ * AnalysisQueue - Queue-based parallel analysis orchestration
+ *
+ * Architecture:
+ * - Tasks are file + applicable plugins
+ * - Workers pick tasks from queue, run plugins, write to RFDB
+ * - Barrier waits for all tasks before ENRICHMENT phase
+ *
+ * Usage:
+ *   const queue = new AnalysisQueue({
+ *     socketPath: '/tmp/rfdb.sock',
+ *     maxWorkers: 4,
+ *     plugins: [JSASTAnalyzer, ExpressRouteAnalyzer, ...]
+ *   });
+ *
+ *   await queue.start();
+ *   queue.addTask({ file: '/path/to/file.js', moduleId: '...', plugins: ['JSASTAnalyzer'] });
+ *   await queue.waitForCompletion();
+ *   await queue.stop();
+ */
+import { Worker } from 'worker_threads';
+import { EventEmitter } from 'events';
+import { cpus } from 'os';
+import { fileURLToPath } from 'url';
+import { dirname, join } from 'path';
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+const WORKER_SCRIPT = join(__dirname, 'QueueWorker.ts');
+export class AnalysisQueue extends EventEmitter {
+    socketPath;
+    maxWorkers;
+    pluginConfigs;
+    workers;
+    pendingTasks;
+    activeTasks;
+    completedCount;
+    failedCount;
+    taskIdCounter;
+    running;
+    draining;
+    completionPromise;
+    completionResolve;
+    stats;
+    constructor(options = {}) {
+        super();
+        // Configuration
+        this.socketPath = options.socketPath || '/tmp/rfdb.sock';
+        this.maxWorkers = Math.min(options.maxWorkers || cpus().length, 16);
+        this.pluginConfigs = options.plugins || [];
+        // State
+        this.workers = [];
+        this.pendingTasks = [];
+        this.activeTasks = new Map();
+        this.completedCount = 0;
+        this.failedCount = 0;
+        this.taskIdCounter = 0;
+        this.running = false;
+        this.draining = false;
+        // Completion tracking
+        this.completionPromise = null;
+        this.completionResolve = null;
+        // Stats
+        this.stats = {
+            tasksTotal: 0,
+            tasksCompleted: 0,
+            tasksFailed: 0,
+            nodesCreated: 0,
+            edgesCreated: 0,
+            startTime: null,
+            errors: []
+        };
+    }
+    /**
+     * Start the queue (spawn workers)
+     */
+    async start() {
+        if (this.running)
+            return;
+        console.log(`[AnalysisQueue] Starting ${this.maxWorkers} workers...`);
+        const initPromises = [];
+        for (let i = 0; i < this.maxWorkers; i++) {
+            const worker = new Worker(WORKER_SCRIPT, {
+                workerData: {
+                    workerId: i,
+                    socketPath: this.socketPath,
+                    pluginConfigs: this.pluginConfigs
+                }
+            });
+            const initPromise = new Promise((resolve, reject) => {
+                const timeout = setTimeout(() => {
+                    reject(new Error(`Worker ${i} initialization timeout`));
+                }, 30000);
+                const onMessage = (msg) => {
+                    if (msg.type === 'ready') {
+                        clearTimeout(timeout);
+                        worker.removeListener('message', onMessage);
+                        resolve();
+                    }
+                    else if (msg.type === 'error' && !this.running) {
+                        clearTimeout(timeout);
+                        worker.removeListener('message', onMessage);
+                        reject(new Error(msg.error));
+                    }
+                };
+                worker.on('message', onMessage);
+                worker.once('error', (err) => {
+                    clearTimeout(timeout);
+                    reject(err);
+                });
+            });
+            worker.on('message', (msg) => this._handleWorkerMessage(worker, msg));
+            worker.on('error', (err) => this._handleWorkerError(worker, err));
+            worker.on('exit', (code) => this._handleWorkerExit(worker, code));
+            this.workers.push({ worker, busy: false });
+            initPromises.push(initPromise);
+        }
+        await Promise.all(initPromises);
+        this.running = true;
+        this.stats.startTime = Date.now();
+        console.log(`[AnalysisQueue] ${this.maxWorkers} workers ready`);
+        this.emit('started', { workerCount: this.maxWorkers });
+    }
+    /**
+     * Stop the queue (terminate workers)
+     */
+    async stop() {
+        if (!this.running)
+            return;
+        console.log('[AnalysisQueue] Stopping workers...');
+        const terminatePromises = this.workers.map(({ worker }) => {
+            return new Promise((resolve) => {
+                worker.once('exit', () => resolve());
+                worker.postMessage({ type: 'exit' });
+            });
+        });
+        await Promise.all(terminatePromises);
+        this.workers = [];
+        this.running = false;
+        console.log('[AnalysisQueue] Stopped');
+        this.emit('stopped');
+    }
+    /**
+     * Add a task to the queue
+     */
+    addTask(task) {
+        const taskId = this.taskIdCounter++;
+        const fullTask = {
+            taskId,
+            file: task.file,
+            moduleId: task.moduleId,
+            moduleName: task.moduleName,
+            plugins: task.plugins || []
+        };
+        this.pendingTasks.push(fullTask);
+        this.stats.tasksTotal++;
+        // Try to dispatch immediately if workers available
+        this._dispatchNext();
+        return taskId;
+    }
+    /**
+     * Add multiple tasks at once
+     */
+    addTasks(tasks) {
+        for (const task of tasks) {
+            this.addTask(task);
+        }
+    }
+    /**
+     * Wait for all tasks to complete
+     */
+    async waitForCompletion() {
+        if (this.pendingTasks.length === 0 && this.activeTasks.size === 0) {
+            return this.getStats();
+        }
+        this.draining = true;
+        this.completionPromise = new Promise((resolve) => {
+            this.completionResolve = resolve;
+        });
+        await this.completionPromise;
+        this.draining = false;
+        return this.getStats();
+    }
+    /**
+     * Get current statistics
+     */
+    getStats() {
+        return {
+            ...this.stats,
+            pending: this.pendingTasks.length,
+            active: this.activeTasks.size,
+            elapsed: this.stats.startTime ? Date.now() - this.stats.startTime : 0
+        };
+    }
+    // ===========================================================================
+    // Internal methods
+    // ===========================================================================
+    _dispatchNext() {
+        if (!this.running)
+            return;
+        if (this.pendingTasks.length === 0)
+            return;
+        // Find an idle worker
+        const idleWorker = this.workers.find(w => !w.busy);
+        if (!idleWorker)
+            return;
+        // Get next task
+        const task = this.pendingTasks.shift();
+        // Mark worker as busy
+        idleWorker.busy = true;
+        this.activeTasks.set(task.taskId, {
+            file: task.file,
+            worker: idleWorker,
+            startTime: Date.now()
+        });
+        // Send to worker
+        idleWorker.worker.postMessage({
+            type: 'process',
+            taskId: task.taskId,
+            file: task.file,
+            moduleId: task.moduleId,
+            moduleName: task.moduleName,
+            plugins: task.plugins
+        });
+        this.emit('taskStarted', { taskId: task.taskId, file: task.file });
+    }
+    _handleWorkerMessage(worker, msg) {
+        switch (msg.type) {
+            case 'done': {
+                const taskInfo = this.activeTasks.get(msg.taskId);
+                if (taskInfo) {
+                    this.activeTasks.delete(msg.taskId);
+                    taskInfo.worker.busy = false;
+                    // Update stats
+                    this.stats.tasksCompleted++;
+                    this.stats.nodesCreated += msg.stats?.nodes || 0;
+                    this.stats.edgesCreated += msg.stats?.edges || 0;
+                    this.emit('taskCompleted', {
+                        taskId: msg.taskId,
+                        file: taskInfo.file,
+                        stats: msg.stats,
+                        duration: Date.now() - taskInfo.startTime
+                    });
+                }
+                this._dispatchNext();
+                this._checkCompletion();
+                break;
+            }
+            case 'error': {
+                const taskInfo = this.activeTasks.get(msg.taskId);
+                if (taskInfo) {
+                    this.activeTasks.delete(msg.taskId);
+                    taskInfo.worker.busy = false;
+                    this.stats.tasksFailed++;
+                    this.stats.errors.push({ file: taskInfo.file, error: msg.error });
+                    this.emit('taskFailed', {
+                        taskId: msg.taskId,
+                        file: taskInfo.file,
+                        error: msg.error
+                    });
+                }
+                this._dispatchNext();
+                this._checkCompletion();
+                break;
+            }
+            case 'progress': {
+                this.emit('progress', msg);
+                break;
+            }
+        }
+    }
+    _handleWorkerError(worker, error) {
+        console.error('[AnalysisQueue] Worker error:', error);
+        this.emit('workerError', { error });
+    }
+    _handleWorkerExit(worker, code) {
+        if (code !== 0 && this.running) {
+            console.error(`[AnalysisQueue] Worker exited with code ${code}`);
+        }
+        // Remove from workers list
+        const idx = this.workers.findIndex(w => w.worker === worker);
+        if (idx !== -1) {
+            this.workers.splice(idx, 1);
+        }
+    }
+    _checkCompletion() {
+        if (this.draining &&
+            this.pendingTasks.length === 0 &&
+            this.activeTasks.size === 0) {
+            const duration = Date.now() - this.stats.startTime;
+            console.log(`[AnalysisQueue] All tasks complete in ${duration}ms`);
+            console.log(`[AnalysisQueue] ${this.stats.tasksCompleted} succeeded, ${this.stats.tasksFailed} failed`);
+            if (this.completionResolve) {
+                this.completionResolve();
+                this.completionResolve = null;
+            }
+        }
+    }
+}
+export default AnalysisQueue;

package/dist/core/AnalysisWorker.d.ts

@@ -0,0 +1,14 @@
+/**
+ * AnalysisWorker - worker thread for parallel AST analysis
+ *
+ * Each worker:
+ * 1. Connects to RFDB server
+ * 2. Receives file paths to analyze
+ * 3. Parses AST and writes nodes/edges directly to RFDB
+ *
+ * Communication:
+ *   Main -> Worker: { type: 'analyze', file, moduleId, moduleName }
+ *   Worker -> Main: { type: 'done', file, stats } | { type: 'error', file, error }
+ */
+export {};
+//# sourceMappingURL=AnalysisWorker.d.ts.map

package/dist/core/AnalysisWorker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"AnalysisWorker.d.ts","sourceRoot":"","sources":["../../src/core/AnalysisWorker.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG"}