jtcsv 2.2.8 → 3.1.0

package/dist/src/workers/worker-pool.js

@@ -0,0 +1,228 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.AggregateError = exports.WorkerPool = void 0;
+exports.createWorkerTask = createWorkerTask;
+exports.createWorkerResult = createWorkerResult;
+exports.getWorkerPool = getWorkerPool;
+exports.shutdownWorkerPool = shutdownWorkerPool;
+const worker_threads_1 = require("worker_threads");
+const events_1 = require("events");
+class WorkerPool extends events_1.EventEmitter {
+    constructor(workerCount = Math.max(1, require('os').cpus().length - 1), workerScript = __dirname + '/csv-parser.worker.js') {
+        super();
+        this.workers = [];
+        this.idleWorkers = [];
+        this.taskQueue = [];
+        this.stats = {
+            totalWorkers: 0,
+            activeWorkers: 0,
+            idleWorkers: 0,
+            totalTasks: 0,
+            completedTasks: 0,
+            failedTasks: 0,
+            averageTaskDuration: 0
+        };
+        this.taskDurations = [];
+        this.stats.totalWorkers = workerCount;
+        for (let i = 0; i < workerCount; i++) {
+            this.createWorker(workerScript, i);
+        }
+        setInterval(() => this.updateStats(), 5000);
+    }
+    createWorker(workerScript, id) {
+        const worker = new worker_threads_1.Worker(workerScript, {
+            workerData: { workerId: id }
+        });
+        worker.on('message', (result) => {
+            this.handleWorkerResult(worker, result);
+        });
+        worker.on('error', (error) => {
+            this.handleWorkerError(worker, error);
+        });
+        worker.on('exit', (code) => {
+            this.handleWorkerExit(worker, code);
+        });
+        this.workers.push(worker);
+        this.idleWorkers.push(worker);
+        this.stats.idleWorkers++;
+        this.emit('workerCreated', { workerId: id });
+    }
+    handleWorkerResult(worker, result) {
+        const taskIndex = this.taskQueue.findIndex(item => item.task.id === result.id);
+        if (taskIndex !== -1) {
+            const { task, resolve, reject } = this.taskQueue[taskIndex];
+            this.taskQueue.splice(taskIndex, 1);
+            this.stats.completedTasks++;
+            this.taskDurations.push(result.duration);
+            this.updateAverageDuration();
+            if (result.error) {
+                this.stats.failedTasks++;
+                reject(result.error);
+                this.emit('taskFailed', { task, error: result.error });
+            }
+            else {
+                resolve(result.result);
+                this.emit('taskCompleted', { task, result: result.result, duration: result.duration });
+            }
+        }
+        this.idleWorkers.push(worker);
+        this.stats.activeWorkers--;
+        this.stats.idleWorkers++;
+        this.processNextTask();
+    }
+    handleWorkerError(worker, error) {
+        const workerIndex = this.workers.indexOf(worker);
+        const idleIndex = this.idleWorkers.indexOf(worker);
+        if (idleIndex !== -1) {
+            this.idleWorkers.splice(idleIndex, 1);
+        }
+        if (workerIndex !== -1) {
+            this.workers.splice(workerIndex, 1);
+            this.stats.totalWorkers--;
+            this.stats.idleWorkers--;
+        }
+        this.emit('workerError', { worker, error });
+        setTimeout(() => {
+            this.createWorker(worker.threadId.toString(), this.workers.length);
+        }, 1000);
+    }
+    handleWorkerExit(worker, code) {
+        const workerIndex = this.workers.indexOf(worker);
+        const idleIndex = this.idleWorkers.indexOf(worker);
+        if (idleIndex !== -1) {
+            this.idleWorkers.splice(idleIndex, 1);
+        }
+        if (workerIndex !== -1) {
+            this.workers.splice(workerIndex, 1);
+            this.stats.totalWorkers--;
+            this.stats.idleWorkers--;
+        }
+        this.emit('workerExit', { worker, code });
+    }
+    processNextTask() {
+        if (this.taskQueue.length === 0 || this.idleWorkers.length === 0) {
+            return;
+        }
+        const nextTask = this.taskQueue[0];
+        const worker = this.idleWorkers.shift();
+        this.taskQueue.shift();
+        worker.postMessage(nextTask.task);
+        this.stats.activeWorkers++;
+        this.stats.idleWorkers--;
+        this.stats.totalTasks++;
+        this.emit('taskStarted', { task: nextTask.task, workerId: worker.threadId });
+    }
+    updateAverageDuration() {
+        if (this.taskDurations.length === 0) {
+            this.stats.averageTaskDuration = 0;
+            return;
+        }
+        const sum = this.taskDurations.reduce((a, b) => a + b, 0);
+        this.stats.averageTaskDuration = sum / this.taskDurations.length;
+        if (this.taskDurations.length > 100) {
+            this.taskDurations = this.taskDurations.slice(-100);
+        }
+    }
+    updateStats() {
+        if (typeof process !== 'undefined' && process.memoryUsage) {
+            this.stats.memoryUsage = process.memoryUsage();
+        }
+        this.emit('statsUpdated', this.stats);
+    }
+    async executeTask(task) {
+        return new Promise((resolve, reject) => {
+            if (!task.id) {
+                task.id = `task_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
+            }
+            this.taskQueue.push({ task, resolve, reject });
+            this.processNextTask();
+        });
+    }
+    async executeTasks(tasks, concurrency) {
+        const results = [];
+        const errors = [];
+        const maxConcurrency = concurrency || this.workers.length;
+        const taskChunks = this.chunkArray(tasks, maxConcurrency);
+        for (const chunk of taskChunks) {
+            const chunkPromises = chunk.map(task => this.executeTask(task));
+            try {
+                const chunkResults = await Promise.all(chunkPromises);
+                results.push(...chunkResults);
+            }
+            catch (error) {
+                errors.push(error);
+            }
+        }
+        if (errors.length > 0) {
+            throw new AggregateError(errors, `Failed to execute ${errors.length} tasks`);
+        }
+        return results;
+    }
+    chunkArray(array, chunkSize) {
+        const chunks = [];
+        for (let i = 0; i < array.length; i += chunkSize) {
+            chunks.push(array.slice(i, i + chunkSize));
+        }
+        return chunks;
+    }
+    getStats() {
+        return { ...this.stats };
+    }
+    async shutdown() {
+        const terminationPromises = this.workers.map(worker => {
+            return worker.terminate();
+        });
+        await Promise.all(terminationPromises);
+        this.workers = [];
+        this.idleWorkers = [];
+        this.taskQueue = [];
+        this.emit('shutdown');
+    }
+    async restart() {
+        await this.shutdown();
+        const workerCount = this.stats.totalWorkers;
+        for (let i = 0; i < workerCount; i++) {
+            this.createWorker(__dirname + '/csv-parser.worker.js', i);
+        }
+        this.emit('restart');
+    }
+}
+exports.WorkerPool = WorkerPool;
+function createWorkerTask(type, data, options) {
+    return {
+        id: `task_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`,
+        type,
+        data,
+        options
+    };
+}
+function createWorkerResult(id, result, duration, error) {
+    return {
+        id,
+        result,
+        error,
+        duration
+    };
+}
+let globalWorkerPool = null;
+function getWorkerPool(workerCount, workerScript) {
+    if (!globalWorkerPool) {
+        globalWorkerPool = new WorkerPool(workerCount, workerScript);
+    }
+    return globalWorkerPool;
+}
+async function shutdownWorkerPool() {
+    if (globalWorkerPool) {
+        await globalWorkerPool.shutdown();
+        globalWorkerPool = null;
+    }
+}
+class AggregateError extends Error {
+    constructor(errors, message) {
+        super(message);
+        this.name = 'AggregateError';
+        this.errors = errors;
+    }
+}
+exports.AggregateError = AggregateError;
+//# sourceMappingURL=worker-pool.js.map

package/dist/src/workers/worker-pool.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"worker-pool.js","sourceRoot":"","sources":["../../../src/workers/worker-pool.ts"],"names":[],"mappings":";;;AAkVA,4CAWC;AAKD,gDAYC;AAUD,sCASC;AAKD,gDAKC;AArYD,mDAA8E;AAC9E,mCAAsC;AAMtC,MAAa,UAAW,SAAQ,qBAAY;IA2B1C,YACE,cAAsB,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,EAClE,eAAuB,SAAS,GAAG,uBAAuB;QAE1D,KAAK,EAAE,CAAC;QA9BF,YAAO,GAAa,EAAE,CAAC;QACvB,gBAAW,GAAa,EAAE,CAAC;QAC3B,cAAS,GAIZ,EAAE,CAAC;QAEA,UAAK,GAAoB;YAC/B,YAAY,EAAE,CAAC;YACf,aAAa,EAAE,CAAC;YAChB,WAAW,EAAE,CAAC;YACd,UAAU,EAAE,CAAC;YACb,cAAc,EAAE,CAAC;YACjB,WAAW,EAAE,CAAC;YACd,mBAAmB,EAAE,CAAC;SACvB,CAAC;QAEM,kBAAa,GAAa,EAAE,CAAC;QAcnC,IAAI,CAAC,KAAK,CAAC,YAAY,GAAG,WAAW,CAAC;QAGtC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,EAAE,CAAC,EAAE,EAAE,CAAC;YACrC,IAAI,CAAC,YAAY,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC;QACrC,CAAC;QAGD,WAAW,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,WAAW,EAAE,EAAE,IAAI,CAAC,CAAC;IAC9C,CAAC;IAKO,YAAY,CAAC,YAAoB,EAAE,EAAU;QACnD,MAAM,MAAM,GAAG,IAAI,uBAAM,CAAC,YAAY,EAAE;YACtC,UAAU,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE;SAC7B,CAAC,CAAC;QAEH,MAAM,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,MAAoB,EAAE,EAAE;YAC5C,IAAI,CAAC,kBAAkB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC1C,CAAC,CAAC,CAAC;QAEH,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAY,EAAE,EAAE;YAClC,IAAI,CAAC,iBAAiB,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;QACxC,CAAC,CAAC,CAAC;QAEH,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAY,EAAE,EAAE;YACjC,IAAI,CAAC,gBAAgB,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QACtC,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC1B,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC9B,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC;QAEzB,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;IAC/C,CAAC;IAKO,kBAAkB,CAAC,MAAc,EAAE,MAAoB;QAE7D,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,KAAK,MAAM,CAAC,EAAE,CAAC,CAAC;QAE/E,IAAI,SAAS,KAAK,CAAC,CAAC,EAAE,CAAC;YACrB,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;YAG5D,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;YAGpC,IAAI,CAAC,KAAK,CAAC,cAAc,EAAE,CAAC;YAC5B,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YACzC,IAAI,CAAC,qBAAqB,EAAE,CAAC;YAE7B,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;gBACjB,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC;gBACzB,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBACrB,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;YACzD,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;gBACvB,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;YACzF,CAAC;QACH,CAAC;QAGD,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC9B,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE,CAAC;QAC3B,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC;QAGzB,IAAI,CAAC,eAAe,EAAE,CAAC;IACzB,CAAC;IAKO,iBAAiB,CAAC,MAAc,EAAE,KAAY;QACpD,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QACjD,MAAM,SAAS,GAAG,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAEnD,IAAI,SAAS,KAAK,CAAC,CAAC,EAAE,CAAC;YACrB,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;QACxC,CAAC;QAED,IAAI,WAAW,KAAK,CAAC,CAAC,EAAE,CAAC;YACvB,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;YACpC,IAAI,CAAC,KAAK,CAAC,YAAY,EAAE,CAAC;YAC1B,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC;QAC3B,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;QAG5C,UAAU,CAAC,GAAG,EAAE;YACd,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QACrE,CAAC,EAAE,IAAI,CAAC,CAAC;IACX,CAAC;IAKO,gBAAgB,CAAC,MAAc,EAAE,IAAY;QACnD,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QACjD,MAAM,SAAS,GAAG,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAEnD,IAAI,SAAS,KAAK,CAAC,CAAC,EAAE,CAAC;YACrB,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;QACxC,CAAC;QAED,IAAI,WAAW,KAAK,CAAC,CAAC,EAAE,CAAC;YACvB,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;YACpC,IAAI,CAAC,KAAK,CAAC,YAAY,EAAE,CAAC;YAC1B,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC;QAC3B,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;IAC5C,CAAC;IAKO,eAAe;QACrB,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACjE,OAAO;QACT,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;QACnC,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,EAAG,CAAC;QAGzC,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;QAGvB,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAElC,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE,CAAC;QAC3B,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC;QACzB,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,CAAC;QAExB,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;IAC/E,CAAC;IAKO,qBAAqB;QAC3B,IAAI,IAAI,CAAC,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACpC,IAAI,CAAC,KAAK,CAAC,mBAAmB,GAAG,CAAC,CAAC;YACnC,OAAO;QACT,CAAC;QAED,MAAM,GAAG,GAAG,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;QAC1D,IAAI,CAAC,KAAK,CAAC,mBAAmB,GAAG,GAAG,GAAG,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC;QAGjE,IAAI,IAAI,CAAC,aAAa,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;YACpC,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC;QACtD,CAAC;IACH,CAAC;IAKO,WAAW;QAEjB,IAAI,OAAO,OAAO,KAAK,WAAW,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC;YAC1D,IAAI,CAAC,KAAK,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;QACjD,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;IACxC,CAAC;IAQD,KAAK,CAAC,WAAW,CAAmB,IAAsB;QACxD,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAErC,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC;gBACb,IAAI,CAAC,EAAE,GAAG,QAAQ,IAAI,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;YAC5E,CAAC;YAGD,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;YAG/C,IAAI,CAAC,eAAe,EAAE,CAAC;QACzB,CAAC,CAAC,CAAC;IACL,CAAC;IASD,KAAK,CAAC,YAAY,CAChB,KAAyB,EACzB,WAAoB;QAEpB,MAAM,OAAO,GAAQ,EAAE,CAAC;QACxB,MAAM,MAAM,GAAY,EAAE,CAAC;QAG3B,MAAM,cAAc,GAAG,WAAW,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;QAC1D,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,cAAc,CAAC,CAAC;QAE1D,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE,CAAC;YAC/B,MAAM,aAAa,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC;YAEhE,IAAI,CAAC;gBACH,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;gBACtD,OAAO,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,CAAC;YAChC,CAAC;YAAC,OAAO,KAAU,EAAE,CAAC;gBACpB,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACrB,CAAC;QACH,CAAC;QAED,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtB,MAAM,IAAI,cAAc,CAAC,MAAM,EAAE,qBAAqB,MAAM,CAAC,MAAM,QAAQ,CAAC,CAAC;QAC/E,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAKO,UAAU,CAAI,KAAU,EAAE,SAAiB;QACjD,MAAM,MAAM,GAAU,EAAE,CAAC;QAEzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;YACjD,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC;QAC7C,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAKD,QAAQ;QACN,OAAO,EAAE,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC;IAC3B,CAAC;IAKD,KAAK,CAAC,QAAQ;QAEZ,MAAM,mBAAmB,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE;YACpD,OAAO,MAAM,CAAC,SAAS,EAAE,CAAC;QAC5B,CAAC,CAAC,CAAC;QAEH,MAAM,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;QAEvC,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;QAClB,IAAI,CAAC,WAAW,GAAG,EAAE,CAAC;QACtB,IAAI,CAAC,SAAS,GAAG,EAAE,CAAC;QAEpB,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IACxB,CAAC;IAKD,KAAK,CAAC,OAAO;QACX,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;QAGtB,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC;QAC5C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,EAAE,CAAC,EAAE,EAAE,CAAC;YACrC,IAAI,CAAC,YAAY,CAAC,SAAS,GAAG,uBAAuB,EAAE,CAAC,CAAC,CAAC;QAC5D,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACvB,CAAC;CACF;AA5TD,gCA4TC;AASD,SAAgB,gBAAgB,CAC9B,IAAY,EACZ,IAAO,EACP,OAA6B;IAE7B,OAAO;QACL,EAAE,EAAE,QAAQ,IAAI,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE;QACnE,IAAI;QACJ,IAAI;QACJ,OAAO;KACR,CAAC;AACJ,CAAC;AAKD,SAAgB,kBAAkB,CAChC,EAAU,EACV,MAAS,EACT,QAAgB,EAChB,KAAa;IAEb,OAAO;QACL,EAAE;QACF,MAAM;QACN,KAAK;QACL,QAAQ;KACT,CAAC;AACJ,CAAC;AAKD,IAAI,gBAAgB,GAAsB,IAAI,CAAC;AAK/C,SAAgB,aAAa,CAC3B,WAAoB,EACpB,YAAqB;IAErB,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACtB,gBAAgB,GAAG,IAAI,UAAU,CAAC,WAAW,EAAE,YAAY,CAAC,CAAC;IAC/D,CAAC;IAED,OAAO,gBAAgB,CAAC;AAC1B,CAAC;AAKM,KAAK,UAAU,kBAAkB;IACtC,IAAI,gBAAgB,EAAE,CAAC;QACrB,MAAM,gBAAgB,CAAC,QAAQ,EAAE,CAAC;QAClC,gBAAgB,GAAG,IAAI,CAAC;IAC1B,CAAC;AACH,CAAC;AAKD,MAAa,cAAe,SAAQ,KAAK;IAGvC,YAAY,MAAe,EAAE,OAAe;QAC1C,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,gBAAgB,CAAC;QAC7B,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;CACF;AARD,wCAQC","sourcesContent":["/**\r\n * Worker Pool для многопоточной обработки данных\r\n * \r\n * Система управления воркерами для параллельной обработки больших наборов данных\r\n */\r\n\r\nimport { Worker, isMainThread, parentPort, workerData } from 'worker_threads';\r\nimport { EventEmitter } from 'events';\r\nimport { WorkerTask, WorkerResult, WorkerPoolStats } from '../types';\r\n\r\n/**\r\n * Класс для управления пулом воркеров\r\n */\r\nexport class WorkerPool extends EventEmitter {\r\n  private workers: Worker[] = [];\r\n  private idleWorkers: Worker[] = [];\r\n  private taskQueue: Array<{\r\n    task: WorkerTask;\r\n    resolve: (result: any) => void;\r\n    reject: (error: Error) => void;\r\n  }> = [];\r\n  \r\n  private stats: WorkerPoolStats = {\r\n    totalWorkers: 0,\r\n    activeWorkers: 0,\r\n    idleWorkers: 0,\r\n    totalTasks: 0,\r\n    completedTasks: 0,\r\n    failedTasks: 0,\r\n    averageTaskDuration: 0\r\n  };\r\n  \r\n  private taskDurations: number[] = [];\r\n  \r\n  /**\r\n   * Создает пул воркеров\r\n   * \r\n   * @param workerCount - Количество воркеров (по умолчанию: количество ядер CPU - 1)\r\n   * @param workerScript - Путь к скрипту воркера\r\n   */\r\n  constructor(\r\n    workerCount: number = Math.max(1, require('os').cpus().length - 1),\r\n    workerScript: string = __dirname + '/csv-parser.worker.js'\r\n  ) {\r\n    super();\r\n    \r\n    this.stats.totalWorkers = workerCount;\r\n    \r\n    // Создаем воркеры\r\n    for (let i = 0; i < workerCount; i++) {\r\n      this.createWorker(workerScript, i);\r\n    }\r\n    \r\n    // Обновляем статистику каждые 5 секунд\r\n    setInterval(() => this.updateStats(), 5000);\r\n  }\r\n  \r\n  /**\r\n   * Создает нового воркера\r\n   */\r\n  private createWorker(workerScript: string, id: number): void {\r\n    const worker = new Worker(workerScript, {\r\n      workerData: { workerId: id }\r\n    });\r\n    \r\n    worker.on('message', (result: WorkerResult) => {\r\n      this.handleWorkerResult(worker, result);\r\n    });\r\n    \r\n    worker.on('error', (error: Error) => {\r\n      this.handleWorkerError(worker, error);\r\n    });\r\n    \r\n    worker.on('exit', (code: number) => {\r\n      this.handleWorkerExit(worker, code);\r\n    });\r\n    \r\n    this.workers.push(worker);\r\n    this.idleWorkers.push(worker);\r\n    this.stats.idleWorkers++;\r\n    \r\n    this.emit('workerCreated', { workerId: id });\r\n  }\r\n  \r\n  /**\r\n   * Обрабатывает результат выполнения задачи воркером\r\n   */\r\n  private handleWorkerResult(worker: Worker, result: WorkerResult): void {\r\n    // Находим задачу в очереди\r\n    const taskIndex = this.taskQueue.findIndex(item => item.task.id === result.id);\r\n    \r\n    if (taskIndex !== -1) {\r\n      const { task, resolve, reject } = this.taskQueue[taskIndex];\r\n      \r\n      // Удаляем задачу из очереди\r\n      this.taskQueue.splice(taskIndex, 1);\r\n      \r\n      // Обновляем статистику\r\n      this.stats.completedTasks++;\r\n      this.taskDurations.push(result.duration);\r\n      this.updateAverageDuration();\r\n      \r\n      if (result.error) {\r\n        this.stats.failedTasks++;\r\n        reject(result.error);\r\n        this.emit('taskFailed', { task, error: result.error });\r\n      } else {\r\n        resolve(result.result);\r\n        this.emit('taskCompleted', { task, result: result.result, duration: result.duration });\r\n      }\r\n    }\r\n    \r\n    // Возвращаем воркер в пул ожидания\r\n    this.idleWorkers.push(worker);\r\n    this.stats.activeWorkers--;\r\n    this.stats.idleWorkers++;\r\n    \r\n    // Обрабатываем следующую задачу из очереди\r\n    this.processNextTask();\r\n  }\r\n  \r\n  /**\r\n   * Обрабатывает ошибку воркера\r\n   */\r\n  private handleWorkerError(worker: Worker, error: Error): void {\r\n    const workerIndex = this.workers.indexOf(worker);\r\n    const idleIndex = this.idleWorkers.indexOf(worker);\r\n    \r\n    if (idleIndex !== -1) {\r\n      this.idleWorkers.splice(idleIndex, 1);\r\n    }\r\n    \r\n    if (workerIndex !== -1) {\r\n      this.workers.splice(workerIndex, 1);\r\n      this.stats.totalWorkers--;\r\n      this.stats.idleWorkers--;\r\n    }\r\n    \r\n    this.emit('workerError', { worker, error });\r\n    \r\n    // Перезапускаем воркер\r\n    setTimeout(() => {\r\n      this.createWorker(worker.threadId.toString(), this.workers.length);\r\n    }, 1000);\r\n  }\r\n  \r\n  /**\r\n   * Обрабатывает завершение работы воркера\r\n   */\r\n  private handleWorkerExit(worker: Worker, code: number): void {\r\n    const workerIndex = this.workers.indexOf(worker);\r\n    const idleIndex = this.idleWorkers.indexOf(worker);\r\n    \r\n    if (idleIndex !== -1) {\r\n      this.idleWorkers.splice(idleIndex, 1);\r\n    }\r\n    \r\n    if (workerIndex !== -1) {\r\n      this.workers.splice(workerIndex, 1);\r\n      this.stats.totalWorkers--;\r\n      this.stats.idleWorkers--;\r\n    }\r\n    \r\n    this.emit('workerExit', { worker, code });\r\n  }\r\n  \r\n  /**\r\n   * Обрабатывает следующую задачу из очереди\r\n   */\r\n  private processNextTask(): void {\r\n    if (this.taskQueue.length === 0 || this.idleWorkers.length === 0) {\r\n      return;\r\n    }\r\n    \r\n    const nextTask = this.taskQueue[0];\r\n    const worker = this.idleWorkers.shift()!;\r\n    \r\n    // Удаляем задачу из начала очереди\r\n    this.taskQueue.shift();\r\n    \r\n    // Отправляем задачу воркеру\r\n    worker.postMessage(nextTask.task);\r\n    \r\n    this.stats.activeWorkers++;\r\n    this.stats.idleWorkers--;\r\n    this.stats.totalTasks++;\r\n    \r\n    this.emit('taskStarted', { task: nextTask.task, workerId: worker.threadId });\r\n  }\r\n  \r\n  /**\r\n   * Обновляет среднюю продолжительность выполнения задач\r\n   */\r\n  private updateAverageDuration(): void {\r\n    if (this.taskDurations.length === 0) {\r\n      this.stats.averageTaskDuration = 0;\r\n      return;\r\n    }\r\n    \r\n    const sum = this.taskDurations.reduce((a, b) => a + b, 0);\r\n    this.stats.averageTaskDuration = sum / this.taskDurations.length;\r\n    \r\n    // Ограничиваем историю до последних 100 задач\r\n    if (this.taskDurations.length > 100) {\r\n      this.taskDurations = this.taskDurations.slice(-100);\r\n    }\r\n  }\r\n  \r\n  /**\r\n   * Обновляет статистику\r\n   */\r\n  private updateStats(): void {\r\n    // Обновляем использование памяти\r\n    if (typeof process !== 'undefined' && process.memoryUsage) {\r\n      this.stats.memoryUsage = process.memoryUsage();\r\n    }\r\n    \r\n    this.emit('statsUpdated', this.stats);\r\n  }\r\n  \r\n  /**\r\n   * Выполняет задачу через пул воркеров\r\n   * \r\n   * @param task - Задача для выполнения\r\n   * @returns Promise с результатом\r\n   */\r\n  async executeTask<T = any, R = any>(task: WorkerTask<T, R>): Promise<R> {\r\n    return new Promise((resolve, reject) => {\r\n      // Добавляем ID задачи если не указан\r\n      if (!task.id) {\r\n        task.id = `task_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;\r\n      }\r\n      \r\n      // Добавляем задачу в очередь\r\n      this.taskQueue.push({ task, resolve, reject });\r\n      \r\n      // Пытаемся обработать задачу сразу\r\n      this.processNextTask();\r\n    });\r\n  }\r\n  \r\n  /**\r\n   * Выполняет несколько задач параллельно\r\n   * \r\n   * @param tasks - Массив задач\r\n   * @param concurrency - Максимальное количество параллельных задач\r\n   * @returns Promise с результатами\r\n   */\r\n  async executeTasks<T = any, R = any>(\r\n    tasks: WorkerTask<T, R>[],\r\n    concurrency?: number\r\n  ): Promise<R[]> {\r\n    const results: R[] = [];\r\n    const errors: Error[] = [];\r\n    \r\n    // Ограничиваем параллелизм если указано\r\n    const maxConcurrency = concurrency || this.workers.length;\r\n    const taskChunks = this.chunkArray(tasks, maxConcurrency);\r\n    \r\n    for (const chunk of taskChunks) {\r\n      const chunkPromises = chunk.map(task => this.executeTask(task));\r\n      \r\n      try {\r\n        const chunkResults = await Promise.all(chunkPromises);\r\n        results.push(...chunkResults);\r\n      } catch (error: any) {\r\n        errors.push(error);\r\n      }\r\n    }\r\n    \r\n    if (errors.length > 0) {\r\n      throw new AggregateError(errors, `Failed to execute ${errors.length} tasks`);\r\n    }\r\n    \r\n    return results;\r\n  }\r\n  \r\n  /**\r\n   * Разделяет массив на чанки\r\n   */\r\n  private chunkArray<T>(array: T[], chunkSize: number): T[][] {\r\n    const chunks: T[][] = [];\r\n    \r\n    for (let i = 0; i < array.length; i += chunkSize) {\r\n      chunks.push(array.slice(i, i + chunkSize));\r\n    }\r\n    \r\n    return chunks;\r\n  }\r\n  \r\n  /**\r\n   * Возвращает статистику пула\r\n   */\r\n  getStats(): WorkerPoolStats {\r\n    return { ...this.stats };\r\n  }\r\n  \r\n  /**\r\n   * Останавливает пул воркеров\r\n   */\r\n  async shutdown(): Promise<void> {\r\n    // Завершаем все воркеры\r\n    const terminationPromises = this.workers.map(worker => {\r\n      return worker.terminate();\r\n    });\r\n    \r\n    await Promise.all(terminationPromises);\r\n    \r\n    this.workers = [];\r\n    this.idleWorkers = [];\r\n    this.taskQueue = [];\r\n    \r\n    this.emit('shutdown');\r\n  }\r\n  \r\n  /**\r\n   * Перезапускает пул воркеров\r\n   */\r\n  async restart(): Promise<void> {\r\n    await this.shutdown();\r\n    \r\n    // Создаем новых воркеров\r\n    const workerCount = this.stats.totalWorkers;\r\n    for (let i = 0; i < workerCount; i++) {\r\n      this.createWorker(__dirname + '/csv-parser.worker.js', i);\r\n    }\r\n    \r\n    this.emit('restart');\r\n  }\r\n}\r\n\r\n/**\r\n * Утилитарные функции для работы с воркерами\r\n */\r\n\r\n/**\r\n * Создает задачу для воркера\r\n */\r\nexport function createWorkerTask<T = any, R = any>(\r\n  type: string,\r\n  data: T,\r\n  options?: Record<string, any>\r\n): WorkerTask<T, R> {\r\n  return {\r\n    id: `task_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`,\r\n    type,\r\n    data,\r\n    options\r\n  };\r\n}\r\n\r\n/**\r\n * Создает результат выполнения задачи\r\n */\r\nexport function createWorkerResult<R = any>(\r\n  id: string,\r\n  result: R,\r\n  duration: number,\r\n  error?: Error\r\n): WorkerResult<R> {\r\n  return {\r\n    id,\r\n    result,\r\n    error,\r\n    duration\r\n  };\r\n}\r\n\r\n/**\r\n * Глобальный экземпляр пула воркеров (синглтон)\r\n */\r\nlet globalWorkerPool: WorkerPool | null = null;\r\n\r\n/**\r\n * Возвращает глобальный экземпляр пула воркеров\r\n */\r\nexport function getWorkerPool(\r\n  workerCount?: number,\r\n  workerScript?: string\r\n): WorkerPool {\r\n  if (!globalWorkerPool) {\r\n    globalWorkerPool = new WorkerPool(workerCount, workerScript);\r\n  }\r\n  \r\n  return globalWorkerPool;\r\n}\r\n\r\n/**\r\n * Останавливает глобальный пул воркеров\r\n */\r\nexport async function shutdownWorkerPool(): Promise<void> {\r\n  if (globalWorkerPool) {\r\n    await globalWorkerPool.shutdown();\r\n    globalWorkerPool = null;\r\n  }\r\n}\r\n\r\n/**\r\n * Класс для агрегированных ошибок\r\n */\r\nexport class AggregateError extends Error {\r\n  errors: Error[];\r\n  \r\n  constructor(errors: Error[], message: string) {\r\n    super(message);\r\n    this.name = 'AggregateError';\r\n    this.errors = errors;\r\n  }\r\n}"]}