@ninebix/nmt-system 1.0.0

package/dist/extensions/distributed/coordinator.js

@@ -0,0 +1,319 @@
+/**
+ * Coordinator - Distributed worker coordination
+ * @module extensions/distributed/coordinator
+ */
+import { EventEmitter } from 'events';
+import { generateUUID } from '../../utils/uuid.js';
+/**
+ * Distributed Coordinator
+ * Manages workers, queues, and job distribution
+ */
+export class Coordinator extends EventEmitter {
+    id;
+    queues;
+    workers;
+    schedulers;
+    config;
+    startTime;
+    completedJobs;
+    failedJobs;
+    healthCheckTimer;
+    rebalanceTimer;
+    running;
+    constructor(config = {}) {
+        super();
+        this.id = generateUUID();
+        this.queues = new Map();
+        this.workers = new Map();
+        this.schedulers = new Map();
+        this.startTime = Date.now();
+        this.completedJobs = 0;
+        this.failedJobs = 0;
+        this.running = false;
+        this.config = {
+            heartbeatTimeout: config.heartbeatTimeout ?? 60000,
+            rebalanceInterval: config.rebalanceInterval ?? 30000,
+            maxWorkersPerJob: config.maxWorkersPerJob ?? 1
+        };
+    }
+    /**
+     * Start the coordinator
+     */
+    start() {
+        if (this.running)
+            return;
+        this.running = true;
+        this.startTime = Date.now();
+        // Start health check timer
+        this.healthCheckTimer = setInterval(() => {
+            this.checkWorkerHealth();
+        }, this.config.heartbeatTimeout / 2);
+        // Start rebalance timer
+        this.rebalanceTimer = setInterval(() => {
+            this.rebalanceLoad();
+        }, this.config.rebalanceInterval);
+        this.emit('started', { id: this.id });
+    }
+    /**
+     * Stop the coordinator
+     */
+    async stop() {
+        this.running = false;
+        // Stop timers
+        if (this.healthCheckTimer) {
+            clearInterval(this.healthCheckTimer);
+            this.healthCheckTimer = undefined;
+        }
+        if (this.rebalanceTimer) {
+            clearInterval(this.rebalanceTimer);
+            this.rebalanceTimer = undefined;
+        }
+        // Stop all schedulers
+        for (const scheduler of this.schedulers.values()) {
+            scheduler.stop();
+        }
+        // Stop all workers
+        for (const { worker } of this.workers.values()) {
+            await worker.stop();
+        }
+        // Close all queues
+        for (const queue of this.queues.values()) {
+            await queue.close();
+        }
+        this.emit('stopped', { id: this.id });
+    }
+    /**
+     * Register a queue
+     */
+    registerQueue(name, queue) {
+        this.queues.set(name, queue);
+        // Listen to queue events
+        queue.on('completed', (job) => {
+            this.completedJobs++;
+            this.emit('job:completed', job);
+        });
+        queue.on('failed', (job, error) => {
+            this.failedJobs++;
+            this.emit('job:failed', job, error);
+        });
+        this.emit('queue:registered', { name });
+    }
+    /**
+     * Unregister a queue
+     */
+    async unregisterQueue(name) {
+        const queue = this.queues.get(name);
+        if (!queue)
+            return false;
+        await queue.close();
+        this.queues.delete(name);
+        this.emit('queue:unregistered', { name });
+        return true;
+    }
+    /**
+     * Register a worker
+     */
+    registerWorker(worker) {
+        const registration = {
+            worker,
+            stats: worker.getStats(),
+            lastHeartbeat: Date.now(),
+            healthy: true
+        };
+        this.workers.set(worker.id, registration);
+        // Listen to worker events
+        worker.on('heartbeat', (stats) => {
+            const reg = this.workers.get(worker.id);
+            if (reg) {
+                reg.stats = stats;
+                reg.lastHeartbeat = Date.now();
+                reg.healthy = true;
+            }
+        });
+        worker.on('stopped', () => {
+            this.workers.delete(worker.id);
+            this.emit('worker:unregistered', { id: worker.id });
+        });
+        this.emit('worker:registered', { id: worker.id, name: worker.name });
+    }
+    /**
+     * Unregister a worker
+     */
+    async unregisterWorker(workerId) {
+        const registration = this.workers.get(workerId);
+        if (!registration)
+            return false;
+        await registration.worker.stop();
+        this.workers.delete(workerId);
+        this.emit('worker:unregistered', { id: workerId });
+        return true;
+    }
+    /**
+     * Register a scheduler
+     */
+    registerScheduler(name, scheduler) {
+        this.schedulers.set(name, scheduler);
+        scheduler.on('job:executed', (job) => {
+            this.emit('scheduled:executed', job);
+        });
+        this.emit('scheduler:registered', { name });
+    }
+    /**
+     * Get cluster status
+     */
+    getStatus() {
+        const queueStats = new Map();
+        // This would be async in production
+        for (const [name] of this.queues) {
+            queueStats.set(name, {
+                waiting: 0,
+                active: 0,
+                completed: 0,
+                failed: 0,
+                delayed: 0,
+                paused: false
+            });
+        }
+        let scheduledJobCount = 0;
+        for (const scheduler of this.schedulers.values()) {
+            scheduledJobCount += scheduler.getAllScheduledJobs().length;
+        }
+        return {
+            coordinatorId: this.id,
+            uptime: Date.now() - this.startTime,
+            workers: Array.from(this.workers.values()).map(r => r.stats),
+            queues: queueStats,
+            activeJobs: Array.from(this.workers.values())
+                .reduce((sum, r) => sum + r.worker.getActiveJobs().length, 0),
+            completedJobs: this.completedJobs,
+            failedJobs: this.failedJobs,
+            scheduledJobs: scheduledJobCount
+        };
+    }
+    /**
+     * Get healthy workers
+     */
+    getHealthyWorkers() {
+        return Array.from(this.workers.values())
+            .filter(r => r.healthy)
+            .map(r => r.worker);
+    }
+    /**
+     * Get queue by name
+     */
+    getQueue(name) {
+        return this.queues.get(name);
+    }
+    /**
+     * Get all queues
+     */
+    getQueues() {
+        return new Map(this.queues);
+    }
+    /**
+     * Get scheduler by name
+     */
+    getScheduler(name) {
+        return this.schedulers.get(name);
+    }
+    /**
+     * Scale workers
+     */
+    async scaleWorkers(createWorker, targetCount) {
+        const currentCount = this.workers.size;
+        if (targetCount > currentCount) {
+            // Scale up
+            for (let i = 0; i < targetCount - currentCount; i++) {
+                const worker = createWorker();
+                this.registerWorker(worker);
+                await worker.start();
+            }
+            this.emit('scaled:up', { from: currentCount, to: targetCount });
+        }
+        else if (targetCount < currentCount) {
+            // Scale down
+            const workersToRemove = Array.from(this.workers.values())
+                .sort((a, b) => a.stats.processedJobs - b.stats.processedJobs)
+                .slice(0, currentCount - targetCount);
+            for (const reg of workersToRemove) {
+                await this.unregisterWorker(reg.worker.id);
+            }
+            this.emit('scaled:down', { from: currentCount, to: targetCount });
+        }
+    }
+    /**
+     * Pause all queues
+     */
+    async pauseAll() {
+        for (const queue of this.queues.values()) {
+            await queue.pause();
+        }
+        this.emit('paused:all');
+    }
+    /**
+     * Resume all queues
+     */
+    async resumeAll() {
+        for (const queue of this.queues.values()) {
+            await queue.resume();
+        }
+        this.emit('resumed:all');
+    }
+    // ==================== Private Methods ====================
+    checkWorkerHealth() {
+        const now = Date.now();
+        for (const [id, registration] of this.workers) {
+            const timeSinceHeartbeat = now - registration.lastHeartbeat;
+            if (timeSinceHeartbeat > this.config.heartbeatTimeout) {
+                if (registration.healthy) {
+                    registration.healthy = false;
+                    this.emit('worker:unhealthy', {
+                        id,
+                        name: registration.worker.name,
+                        lastHeartbeat: registration.lastHeartbeat
+                    });
+                }
+            }
+            else if (!registration.healthy) {
+                registration.healthy = true;
+                this.emit('worker:healthy', {
+                    id,
+                    name: registration.worker.name
+                });
+            }
+        }
+    }
+    async rebalanceLoad() {
+        const healthyWorkers = this.getHealthyWorkers();
+        if (healthyWorkers.length === 0)
+            return;
+        // Calculate load per worker
+        const loads = healthyWorkers.map(w => ({
+            worker: w,
+            load: w.getActiveJobs().length
+        }));
+        const avgLoad = loads.reduce((sum, l) => sum + l.load, 0) / loads.length;
+        const maxLoad = Math.max(...loads.map(l => l.load));
+        const minLoad = Math.min(...loads.map(l => l.load));
+        // Check if rebalancing is needed
+        if (maxLoad - minLoad > avgLoad * 0.5) {
+            this.emit('rebalancing', {
+                avgLoad,
+                maxLoad,
+                minLoad,
+                workers: loads.length
+            });
+            // In a real implementation, we would:
+            // 1. Pause overloaded workers
+            // 2. Let jobs complete
+            // 3. Resume with better distribution
+        }
+    }
+}
+/**
+ * Create a coordinator
+ */
+export function createCoordinator(config) {
+    return new Coordinator(config);
+}
+//# sourceMappingURL=coordinator.js.map

package/dist/extensions/distributed/coordinator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"coordinator.js","sourceRoot":"","sources":["../../../src/extensions/distributed/coordinator.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAC;AAEtC,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAsCnD;;;GAGG;AACH,MAAM,OAAO,WAAY,SAAQ,YAAY;IAClC,EAAE,CAAS;IAEZ,MAAM,CAAwB;IAC9B,OAAO,CAAgC;IACvC,UAAU,CAAyB;IACnC,MAAM,CAA8B;IACpC,SAAS,CAAS;IAClB,aAAa,CAAS;IACtB,UAAU,CAAS;IACnB,gBAAgB,CAAkB;IAClC,cAAc,CAAkB;IAChC,OAAO,CAAU;IAEzB,YAAY,SAA4B,EAAE;QACxC,KAAK,EAAE,CAAC;QAER,IAAI,CAAC,EAAE,GAAG,YAAY,EAAE,CAAC;QACzB,IAAI,CAAC,MAAM,GAAG,IAAI,GAAG,EAAE,CAAC;QACxB,IAAI,CAAC,OAAO,GAAG,IAAI,GAAG,EAAE,CAAC;QACzB,IAAI,CAAC,UAAU,GAAG,IAAI,GAAG,EAAE,CAAC;QAC5B,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC5B,IAAI,CAAC,aAAa,GAAG,CAAC,CAAC;QACvB,IAAI,CAAC,UAAU,GAAG,CAAC,CAAC;QACpB,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC;QAErB,IAAI,CAAC,MAAM,GAAG;YACZ,gBAAgB,EAAE,MAAM,CAAC,gBAAgB,IAAI,KAAK;YAClD,iBAAiB,EAAE,MAAM,CAAC,iBAAiB,IAAI,KAAK;YACpD,gBAAgB,EAAE,MAAM,CAAC,gBAAgB,IAAI,CAAC;SAC/C,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,KAAK;QACH,IAAI,IAAI,CAAC,OAAO;YAAE,OAAO;QAEzB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpB,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAE5B,2BAA2B;QAC3B,IAAI,CAAC,gBAAgB,GAAG,WAAW,CAAC,GAAG,EAAE;YACvC,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAC3B,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,gBAAgB,GAAG,CAAC,CAAC,CAAC;QAErC,wBAAwB;QACxB,IAAI,CAAC,cAAc,GAAG,WAAW,CAAC,GAAG,EAAE;YACrC,IAAI,CAAC,aAAa,EAAE,CAAC;QACvB,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC;QAElC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,EAAE,EAAE,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;IACxC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,IAAI;QACR,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC;QAErB,cAAc;QACd,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC1B,aAAa,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;YACrC,IAAI,CAAC,gBAAgB,GAAG,SAAS,CAAC;QACpC,CAAC;QAED,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACxB,aAAa,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;YACnC,IAAI,CAAC,cAAc,GAAG,SAAS,CAAC;QAClC,CAAC;QAED,sBAAsB;QACtB,KAAK,MAAM,SAAS,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,EAAE,CAAC;YACjD,SAAS,CAAC,IAAI,EAAE,CAAC;QACnB,CAAC;QAED,mBAAmB;QACnB,KAAK,MAAM,EAAE,MAAM,EAAE,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;YAC/C,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;QACtB,CAAC;QAED,mBAAmB;QACnB,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,EAAE,CAAC;YACzC,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;QACtB,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,EAAE,EAAE,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;IACxC,CAAC;IAED;;OAEG;IACH,aAAa,CAAC,IAAY,EAAE,KAAe;QACzC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QAE7B,yBAAyB;QACzB,KAAK,CAAC,EAAE,CAAC,WAAW,EAAE,CAAC,GAAQ,EAAE,EAAE;YACjC,IAAI,CAAC,aAAa,EAAE,CAAC;YACrB,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,GAAG,CAAC,CAAC;QAClC,CAAC,CAAC,CAAC;QAEH,KAAK,CAAC,EAAE,CAAC,QAAQ,EAAE,CAAC,GAAQ,EAAE,KAAa,EAAE,EAAE;YAC7C,IAAI,CAAC,UAAU,EAAE,CAAC;YAClB,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC;QACtC,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,IAAI,CAAC,kBAAkB,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;IAC1C,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,eAAe,CAAC,IAAY;QAChC,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACpC,IAAI,CAAC,KAAK;YAAE,OAAO,KAAK,CAAC;QAEzB,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;QACpB,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAEzB,IAAI,CAAC,IAAI,CAAC,oBAAoB,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;QAC1C,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;OAEG;IACH,cAAc,CAAC,MAAc;QAC3B,MAAM,YAAY,GAAqB;YACrC,MAAM;YACN,KAAK,EAAE,MAAM,CAAC,QAAQ,EAAE;YACxB,aAAa,EAAE,IAAI,CAAC,GAAG,EAAE;YACzB,OAAO,EAAE,IAAI;SACd,CAAC;QAEF,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,EAAE,YAAY,CAAC,CAAC;QAE1C,0BAA0B;QAC1B,MAAM,CAAC,EAAE,CAAC,WAAW,EAAE,CAAC,KAAkB,EAAE,EAAE;YAC5C,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YACxC,IAAI,GAAG,EAAE,CAAC;gBACR,GAAG,CAAC,KAAK,GAAG,KAAK,CAAC;gBAClB,GAAG,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;gBAC/B,GAAG,CAAC,OAAO,GAAG,IAAI,CAAC;YACrB,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,MAAM,CAAC,EAAE,CAAC,SAAS,EAAE,GAAG,EAAE;YACxB,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YAC/B,IAAI,CAAC,IAAI,CAAC,qBAAqB,EAAE,EAAE,EAAE,EAAE,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC;QACtD,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,IAAI,CAAC,mBAAmB,EAAE,EAAE,EAAE,EAAE,MAAM,CAAC,EAAE,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;IACvE,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,gBAAgB,CAAC,QAAgB;QACrC,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAChD,IAAI,CAAC,YAAY;YAAE,OAAO,KAAK,CAAC;QAEhC,MAAM,YAAY,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QACjC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAE9B,IAAI,CAAC,IAAI,CAAC,qBAAqB,EAAE,EAAE,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC;QACnD,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;OAEG;IACH,iBAAiB,CAAC,IAAY,EAAE,SAAoB;QAClD,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;QAErC,SAAS,CAAC,EAAE,CAAC,cAAc,EAAE,CAAC,GAAiB,EAAE,EAAE;YACjD,IAAI,CAAC,IAAI,CAAC,oBAAoB,EAAE,GAAG,CAAC,CAAC;QACvC,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,IAAI,CAAC,sBAAsB,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;IAC9C,CAAC;IAED;;OAEG;IACH,SAAS;QACP,MAAM,UAAU,GAAG,IAAI,GAAG,EAAsB,CAAC;QAEjD,oCAAoC;QACpC,KAAK,MAAM,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YACjC,UAAU,CAAC,GAAG,CAAC,IAAI,EAAE;gBACnB,OAAO,EAAE,CAAC;gBACV,MAAM,EAAE,CAAC;gBACT,SAAS,EAAE,CAAC;gBACZ,MAAM,EAAE,CAAC;gBACT,OAAO,EAAE,CAAC;gBACV,MAAM,EAAE,KAAK;aACd,CAAC,CAAC;QACL,CAAC;QAED,IAAI,iBAAiB,GAAG,CAAC,CAAC;QAC1B,KAAK,MAAM,SAAS,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,EAAE,CAAC;YACjD,iBAAiB,IAAI,SAAS,CAAC,mBAAmB,EAAE,CAAC,MAAM,CAAC;QAC9D,CAAC;QAED,OAAO;YACL,aAAa,EAAE,IAAI,CAAC,EAAE;YACtB,MAAM,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,SAAS;YACnC,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;YAC5D,MAAM,EAAE,UAAU;YAClB,UAAU,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;iBAC1C,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,MAAM,CAAC,aAAa,EAAE,CAAC,MAAM,EAAE,CAAC,CAAC;YAC/D,aAAa,EAAE,IAAI,CAAC,aAAa;YACjC,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,aAAa,EAAE,iBAAiB;SACjC,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,iBAAiB;QACf,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;aACrC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC;aACtB,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IACxB,CAAC;IAED;;OAEG;IACH,QAAQ,CAAC,IAAY;QACnB,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAC/B,CAAC;IAED;;OAEG;IACH,SAAS;QACP,OAAO,IAAI,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC9B,CAAC;IAED;;OAEG;IACH,YAAY,CAAC,IAAY;QACvB,OAAO,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACnC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,YAAY,CAChB,YAA0B,EAC1B,WAAmB;QAEnB,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;QAEvC,IAAI,WAAW,GAAG,YAAY,EAAE,CAAC;YAC/B,WAAW;YACX,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,GAAG,YAAY,EAAE,CAAC,EAAE,EAAE,CAAC;gBACpD,MAAM,MAAM,GAAG,YAAY,EAAE,CAAC;gBAC9B,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;gBAC5B,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;YACvB,CAAC;YACD,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,EAAE,IAAI,EAAE,YAAY,EAAE,EAAE,EAAE,WAAW,EAAE,CAAC,CAAC;QAClE,CAAC;aAAM,IAAI,WAAW,GAAG,YAAY,EAAE,CAAC;YACtC,aAAa;YACb,MAAM,eAAe,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;iBACtD,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC;iBAC7D,KAAK,CAAC,CAAC,EAAE,YAAY,GAAG,WAAW,CAAC,CAAC;YAExC,KAAK,MAAM,GAAG,IAAI,eAAe,EAAE,CAAC;gBAClC,MAAM,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YAC7C,CAAC;YACD,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,EAAE,IAAI,EAAE,YAAY,EAAE,EAAE,EAAE,WAAW,EAAE,CAAC,CAAC;QACpE,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,QAAQ;QACZ,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,EAAE,CAAC;YACzC,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;QACtB,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IAC1B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,SAAS;QACb,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,EAAE,CAAC;YACzC,MAAM,KAAK,CAAC,MAAM,EAAE,CAAC;QACvB,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IAC3B,CAAC;IAED,4DAA4D;IAEpD,iBAAiB;QACvB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAEvB,KAAK,MAAM,CAAC,EAAE,EAAE,YAAY,CAAC,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YAC9C,MAAM,kBAAkB,GAAG,GAAG,GAAG,YAAY,CAAC,aAAa,CAAC;YAE5D,IAAI,kBAAkB,GAAG,IAAI,CAAC,MAAM,CAAC,gBAAgB,EAAE,CAAC;gBACtD,IAAI,YAAY,CAAC,OAAO,EAAE,CAAC;oBACzB,YAAY,CAAC,OAAO,GAAG,KAAK,CAAC;oBAC7B,IAAI,CAAC,IAAI,CAAC,kBAAkB,EAAE;wBAC5B,EAAE;wBACF,IAAI,EAAE,YAAY,CAAC,MAAM,CAAC,IAAI;wBAC9B,aAAa,EAAE,YAAY,CAAC,aAAa;qBAC1C,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;iBAAM,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC;gBACjC,YAAY,CAAC,OAAO,GAAG,IAAI,CAAC;gBAC5B,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE;oBAC1B,EAAE;oBACF,IAAI,EAAE,YAAY,CAAC,MAAM,CAAC,IAAI;iBAC/B,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,aAAa;QACzB,MAAM,cAAc,GAAG,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAChD,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAExC,4BAA4B;QAC5B,MAAM,KAAK,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACrC,MAAM,EAAE,CAAC;YACT,IAAI,EAAE,CAAC,CAAC,aAAa,EAAE,CAAC,MAAM;SAC/B,CAAC,CAAC,CAAC;QAEJ,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC;QACzE,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QACpD,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QAEpD,iCAAiC;QACjC,IAAI,OAAO,GAAG,OAAO,GAAG,OAAO,GAAG,GAAG,EAAE,CAAC;YACtC,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE;gBACvB,OAAO;gBACP,OAAO;gBACP,OAAO;gBACP,OAAO,EAAE,KAAK,CAAC,MAAM;aACtB,CAAC,CAAC;YAEH,sCAAsC;YACtC,8BAA8B;YAC9B,uBAAuB;YACvB,qCAAqC;QACvC,CAAC;IACH,CAAC;CACF;AAED;;GAEG;AACH,MAAM,UAAU,iBAAiB,CAAC,MAA0B;IAC1D,OAAO,IAAI,WAAW,CAAC,MAAM,CAAC,CAAC;AACjC,CAAC"}

package/dist/extensions/distributed/index.d.ts

@@ -0,0 +1,10 @@
+/**
+ * Distributed Processing Module
+ * Redis/Bull based worker queue for large-scale processing
+ * @module extensions/distributed
+ */
+export * from './queue.js';
+export * from './worker.js';
+export * from './scheduler.js';
+export * from './coordinator.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/extensions/distributed/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/extensions/distributed/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,cAAc,YAAY,CAAC;AAC3B,cAAc,aAAa,CAAC;AAC5B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,kBAAkB,CAAC"}

package/dist/extensions/distributed/index.js

@@ -0,0 +1,10 @@
+/**
+ * Distributed Processing Module
+ * Redis/Bull based worker queue for large-scale processing
+ * @module extensions/distributed
+ */
+export * from './queue.js';
+export * from './worker.js';
+export * from './scheduler.js';
+export * from './coordinator.js';
+//# sourceMappingURL=index.js.map

package/dist/extensions/distributed/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/extensions/distributed/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,cAAc,YAAY,CAAC;AAC3B,cAAc,aAAa,CAAC;AAC5B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,kBAAkB,CAAC"}

package/dist/extensions/distributed/queue.d.ts

@@ -0,0 +1,157 @@
+/**
+ * Job Queue - Bull-based distributed job queue
+ * @module extensions/distributed/queue
+ */
+import type { UUID, JobStatus } from '../../types/index.js';
+import { EventEmitter } from 'events';
+/**
+ * Job definition
+ */
+export interface Job<T = unknown> {
+    id: UUID;
+    type: JobType;
+    data: T;
+    status: JobStatus;
+    priority: number;
+    attempts: number;
+    maxAttempts: number;
+    progress: number;
+    result?: unknown;
+    error?: string;
+    createdAt: string;
+    startedAt?: string;
+    completedAt?: string;
+    workerId?: string;
+}
+/**
+ * Job types supported
+ */
+export type JobType = 'INGEST_TEXT' | 'INGEST_BATCH' | 'GENERATE_EMBEDDINGS' | 'BUILD_INDEX' | 'CLUSTER' | 'ANALYZE_GRAPH' | 'VERIFY_INTEGRITY' | 'EXPORT_DATA' | 'IMPORT_DATA' | 'CUSTOM';
+/**
+ * Queue options
+ */
+export interface QueueOptions {
+    redis?: {
+        host: string;
+        port: number;
+        password?: string;
+        db?: number;
+    };
+    defaultJobOptions?: {
+        priority?: number;
+        maxAttempts?: number;
+        backoffDelay?: number;
+    };
+    concurrency?: number;
+}
+/**
+ * Job options
+ */
+export interface JobOptions {
+    priority?: number;
+    maxAttempts?: number;
+    delay?: number;
+    timeout?: number;
+}
+/**
+ * Queue statistics
+ */
+export interface QueueStats {
+    waiting: number;
+    active: number;
+    completed: number;
+    failed: number;
+    delayed: number;
+    paused: boolean;
+}
+/**
+ * In-memory job queue (can be replaced with Bull for production)
+ * This implementation provides the same interface as Bull
+ */
+export declare class JobQueue extends EventEmitter {
+    private name;
+    private jobs;
+    private waitingQueue;
+    private activeJobs;
+    private completedJobs;
+    private failedJobs;
+    private delayedJobs;
+    private paused;
+    private concurrency;
+    private defaultOptions;
+    private processors;
+    private processingPromises;
+    constructor(name: string, options?: QueueOptions);
+    /**
+     * Add a job to the queue
+     */
+    add<T>(type: JobType, data: T, options?: JobOptions): Promise<Job<T>>;
+    /**
+     * Add multiple jobs
+     */
+    addBulk<T>(jobs: Array<{
+        type: JobType;
+        data: T;
+        options?: JobOptions;
+    }>): Promise<Job<T>[]>;
+    /**
+     * Process jobs of a specific type
+     */
+    process(type: JobType, handler: (job: Job) => Promise<unknown>): void;
+    /**
+     * Process all job types with a single handler
+     */
+    processAll(handler: (job: Job) => Promise<unknown>): void;
+    /**
+     * Get job by ID
+     */
+    getJob(id: UUID): Promise<Job | undefined>;
+    /**
+     * Get jobs by status
+     */
+    getJobs(status: JobStatus[]): Promise<Job[]>;
+    /**
+     * Get queue statistics
+     */
+    getStats(): Promise<QueueStats>;
+    /**
+     * Pause the queue
+     */
+    pause(): Promise<void>;
+    /**
+     * Resume the queue
+     */
+    resume(): Promise<void>;
+    /**
+     * Retry a failed job
+     */
+    retry(jobId: UUID): Promise<Job | undefined>;
+    /**
+     * Remove a job
+     */
+    remove(jobId: UUID): Promise<boolean>;
+    /**
+     * Clean completed/failed jobs
+     */
+    clean(olderThanMs: number, status: 'completed' | 'failed' | 'all'): Promise<number>;
+    /**
+     * Drain the queue (remove all waiting jobs)
+     */
+    drain(): Promise<void>;
+    /**
+     * Close the queue
+     */
+    close(): Promise<void>;
+    /**
+     * Update job progress
+     */
+    updateProgress(jobId: UUID, progress: number): void;
+    private enqueue;
+    private tryProcess;
+    private processJob;
+}
+/**
+ * Create a job queue
+ */
+export declare function createJobQueue(name: string, options?: QueueOptions): JobQueue;
+//# sourceMappingURL=queue.d.ts.map

package/dist/extensions/distributed/queue.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"queue.d.ts","sourceRoot":"","sources":["../../../src/extensions/distributed/queue.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,sBAAsB,CAAC;AAE5D,OAAO,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAC;AAEtC;;GAEG;AACH,MAAM,WAAW,GAAG,CAAC,CAAC,GAAG,OAAO;IAC9B,EAAE,EAAE,IAAI,CAAC;IACT,IAAI,EAAE,OAAO,CAAC;IACd,IAAI,EAAE,CAAC,CAAC;IACR,MAAM,EAAE,SAAS,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,MAAM,OAAO,GACf,aAAa,GACb,cAAc,GACd,qBAAqB,GACrB,aAAa,GACb,SAAS,GACT,eAAe,GACf,kBAAkB,GAClB,aAAa,GACb,aAAa,GACb,QAAQ,CAAC;AAEb;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,KAAK,CAAC,EAAE;QACN,IAAI,EAAE,MAAM,CAAC;QACb,IAAI,EAAE,MAAM,CAAC;QACb,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,EAAE,CAAC,EAAE,MAAM,CAAC;KACb,CAAC;IACF,iBAAiB,CAAC,EAAE;QAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,YAAY,CAAC,EAAE,MAAM,CAAC;KACvB,CAAC;IACF,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,OAAO,CAAC;CACjB;AAED;;;GAGG;AACH,qBAAa,QAAS,SAAQ,YAAY;IAa5B,OAAO,CAAC,IAAI;IAZxB,OAAO,CAAC,IAAI,CAAiB;IAC7B,OAAO,CAAC,YAAY,CAAS;IAC7B,OAAO,CAAC,UAAU,CAAY;IAC9B,OAAO,CAAC,aAAa,CAAY;IACjC,OAAO,CAAC,UAAU,CAAY;IAC9B,OAAO,CAAC,WAAW,CAA4B;IAC/C,OAAO,CAAC,MAAM,CAAU;IACxB,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,cAAc,CAAuB;IAC7C,OAAO,CAAC,UAAU,CAA+C;IACjE,OAAO,CAAC,kBAAkB,CAA2B;gBAEjC,IAAI,EAAE,MAAM,EAAE,OAAO,GAAE,YAAiB;IAyB5D;;OAEG;IACG,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,EAAE,OAAO,GAAE,UAAe,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IA+B/E;;OAEG;IACG,OAAO,CAAC,CAAC,EACb,IAAI,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,OAAO,CAAC;QAAC,IAAI,EAAE,CAAC,CAAC;QAAC,OAAO,CAAC,EAAE,UAAU,CAAA;KAAE,CAAC,GAC5D,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;IAWpB;;OAEG;IACH,OAAO,CAAC,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,GAAG,EAAE,GAAG,KAAK,OAAO,CAAC,OAAO,CAAC,GAAG,IAAI;IAKrE;;OAEG;IACH,UAAU,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,GAAG,KAAK,OAAO,CAAC,OAAO,CAAC,GAAG,IAAI;IAczD;;OAEG;IACG,MAAM,CAAC,EAAE,EAAE,IAAI,GAAG,OAAO,CAAC,GAAG,GAAG,SAAS,CAAC;IAIhD;;OAEG;IACG,OAAO,CAAC,MAAM,EAAE,SAAS,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;IAKlD;;OAEG;IACG,QAAQ,IAAI,OAAO,CAAC,UAAU,CAAC;IAWrC;;OAEG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAK5B;;OAEG;IACG,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC;IAM7B;;OAEG;IACG,KAAK,CAAC,KAAK,EAAE,IAAI,GAAG,OAAO,CAAC,GAAG,GAAG,SAAS,CAAC;IAclD;;OAEG;IACG,MAAM,CAAC,KAAK,EAAE,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC;IAoB3C;;OAEG;IACG,KAAK,CAAC,WAAW,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,GAAG,QAAQ,GAAG,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC;IAoBzF;;OAEG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAO5B;;OAEG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAe5B;;OAEG;IACH,cAAc,CAAC,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,GAAG,IAAI;IAUnD,OAAO,CAAC,OAAO;IAsBf,OAAO,CAAC,UAAU;YAwBJ,UAAU;CA0DzB;AAED;;GAEG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,YAAY,GAAG,QAAQ,CAE7E"}