od-temp 1.0.4 → 1.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/HealthCheck-6-mYE1uY.cjs +252 -0
- package/dist/HealthCheck-B79o7xg2.mjs +280 -0
- package/dist/HealthCheck-B79o7xg2.mjs.map +1 -0
- package/dist/HealthCheck-CCC7Wgoe.mjs +3 -0
- package/dist/HealthCheck-CFX1wPqE.js +298 -0
- package/dist/HealthCheck-CFX1wPqE.js.map +1 -0
- package/dist/HealthCheck-CNRn3PcM.js +4 -0
- package/dist/document-C4T2JLdu.js +1552 -0
- package/dist/document-C4T2JLdu.js.map +1 -0
- package/dist/document-CaOtDvq_.js +4 -0
- package/dist/document-ChTIy4sC.mjs +1531 -0
- package/dist/document-ChTIy4sC.mjs.map +1 -0
- package/dist/document-DBn-i6L4.cjs +1470 -0
- package/dist/document-DxUjbEOE.mjs +3 -0
- package/dist/index.cli.cjs +15504 -0
- package/dist/index.d.mts.map +1 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js.map +1 -0
- package/dist/index.mjs.map +1 -0
- package/dist/workers-B2xx_81m.js +4 -0
- package/dist/workers-BGF3tjiF.mjs +3 -0
- package/dist/workers-BmzAqLSu.js +167 -0
- package/dist/workers-BmzAqLSu.js.map +1 -0
- package/dist/workers-DSxzXbrR.cjs +156 -0
- package/dist/workers-vfrB0Vzh.mjs +155 -0
- package/dist/workers-vfrB0Vzh.mjs.map +1 -0
- package/package.json +2 -7
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
const require_index = require('./index.js');
|
|
2
|
+
let path = require("path");
|
|
3
|
+
let worker_threads = require("worker_threads");
|
|
4
|
+
let os = require("os");
|
|
5
|
+
|
|
6
|
+
//#region src/workers/WorkerPool.ts
|
|
7
|
+
/**
|
|
8
|
+
* Worker thread pool for parallel processing
|
|
9
|
+
*/
|
|
10
|
+
/**
|
|
11
|
+
* Worker pool for parallel text detection and document processing
|
|
12
|
+
*/
|
|
13
|
+
var WorkerPool = class {
|
|
14
|
+
constructor(config = {}) {
|
|
15
|
+
this.workers = [];
|
|
16
|
+
this.availableWorkers = [];
|
|
17
|
+
this.taskQueue = [];
|
|
18
|
+
this.totalProcessingTime = 0;
|
|
19
|
+
this.config = {
|
|
20
|
+
numWorkers: config.numWorkers || (0, os.cpus)().length,
|
|
21
|
+
maxQueueSize: config.maxQueueSize || 100,
|
|
22
|
+
idleTimeout: config.idleTimeout || 3e4
|
|
23
|
+
};
|
|
24
|
+
this.stats = {
|
|
25
|
+
activeWorkers: 0,
|
|
26
|
+
idleWorkers: 0,
|
|
27
|
+
queueSize: 0,
|
|
28
|
+
totalProcessed: 0,
|
|
29
|
+
totalErrors: 0,
|
|
30
|
+
avgProcessingTime: 0
|
|
31
|
+
};
|
|
32
|
+
this.workerPath = (0, path.join)(__dirname, "worker.js");
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* Initialize worker pool
|
|
36
|
+
*/
|
|
37
|
+
async initialize() {
|
|
38
|
+
for (let i = 0; i < this.config.numWorkers; i++) await this.createWorker();
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* Create a new worker
|
|
42
|
+
*/
|
|
43
|
+
async createWorker() {
|
|
44
|
+
const worker = new worker_threads.Worker(this.workerPath);
|
|
45
|
+
worker.on("message", (result) => {
|
|
46
|
+
this.handleWorkerResult(worker, result);
|
|
47
|
+
});
|
|
48
|
+
worker.on("error", (error) => {
|
|
49
|
+
console.error("[WorkerPool] Worker error:", error);
|
|
50
|
+
this.stats.totalErrors++;
|
|
51
|
+
this.removeWorker(worker);
|
|
52
|
+
this.createWorker();
|
|
53
|
+
});
|
|
54
|
+
worker.on("exit", (code) => {
|
|
55
|
+
if (code !== 0) console.error(`[WorkerPool] Worker exited with code ${code}`);
|
|
56
|
+
this.removeWorker(worker);
|
|
57
|
+
});
|
|
58
|
+
this.workers.push(worker);
|
|
59
|
+
this.availableWorkers.push(worker);
|
|
60
|
+
this.stats.idleWorkers++;
|
|
61
|
+
return worker;
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* Execute a task on the worker pool
|
|
65
|
+
*/
|
|
66
|
+
async execute(task) {
|
|
67
|
+
if (this.taskQueue.length >= this.config.maxQueueSize) throw new Error(`[WorkerPool] Queue is full (max: ${this.config.maxQueueSize})`);
|
|
68
|
+
return new Promise((resolve, reject) => {
|
|
69
|
+
this.taskQueue.push({
|
|
70
|
+
task,
|
|
71
|
+
resolve,
|
|
72
|
+
reject
|
|
73
|
+
});
|
|
74
|
+
this.stats.queueSize = this.taskQueue.length;
|
|
75
|
+
this.processQueue();
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
/**
|
|
79
|
+
* Process task queue
|
|
80
|
+
*/
|
|
81
|
+
processQueue() {
|
|
82
|
+
while (this.taskQueue.length > 0 && this.availableWorkers.length > 0) {
|
|
83
|
+
const worker = this.availableWorkers.shift();
|
|
84
|
+
const { task, resolve, reject } = this.taskQueue.shift();
|
|
85
|
+
this.stats.idleWorkers--;
|
|
86
|
+
this.stats.activeWorkers++;
|
|
87
|
+
this.stats.queueSize = this.taskQueue.length;
|
|
88
|
+
worker.__currentTask = {
|
|
89
|
+
resolve,
|
|
90
|
+
reject,
|
|
91
|
+
startTime: Date.now()
|
|
92
|
+
};
|
|
93
|
+
worker.postMessage(task);
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
/**
|
|
97
|
+
* Handle worker result
|
|
98
|
+
*/
|
|
99
|
+
handleWorkerResult(worker, result) {
|
|
100
|
+
const currentTask = worker.__currentTask;
|
|
101
|
+
if (!currentTask) return;
|
|
102
|
+
this.stats.activeWorkers--;
|
|
103
|
+
this.stats.idleWorkers++;
|
|
104
|
+
this.stats.totalProcessed++;
|
|
105
|
+
this.totalProcessingTime += result.processingTime;
|
|
106
|
+
this.stats.avgProcessingTime = this.totalProcessingTime / this.stats.totalProcessed;
|
|
107
|
+
this.availableWorkers.push(worker);
|
|
108
|
+
delete worker.__currentTask;
|
|
109
|
+
if (result.error) {
|
|
110
|
+
this.stats.totalErrors++;
|
|
111
|
+
currentTask.reject(new Error(result.error));
|
|
112
|
+
} else currentTask.resolve(result.result);
|
|
113
|
+
this.processQueue();
|
|
114
|
+
}
|
|
115
|
+
/**
|
|
116
|
+
* Remove worker from pool
|
|
117
|
+
*/
|
|
118
|
+
removeWorker(worker) {
|
|
119
|
+
const index = this.workers.indexOf(worker);
|
|
120
|
+
if (index !== -1) this.workers.splice(index, 1);
|
|
121
|
+
const availableIndex = this.availableWorkers.indexOf(worker);
|
|
122
|
+
if (availableIndex !== -1) {
|
|
123
|
+
this.availableWorkers.splice(availableIndex, 1);
|
|
124
|
+
this.stats.idleWorkers--;
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
/**
|
|
128
|
+
* Get pool statistics
|
|
129
|
+
*/
|
|
130
|
+
getStats() {
|
|
131
|
+
return { ...this.stats };
|
|
132
|
+
}
|
|
133
|
+
/**
|
|
134
|
+
* Terminate all workers
|
|
135
|
+
*/
|
|
136
|
+
async terminate() {
|
|
137
|
+
const terminatePromises = this.workers.map((worker) => worker.terminate());
|
|
138
|
+
await Promise.all(terminatePromises);
|
|
139
|
+
this.workers = [];
|
|
140
|
+
this.availableWorkers = [];
|
|
141
|
+
this.taskQueue = [];
|
|
142
|
+
this.stats.activeWorkers = 0;
|
|
143
|
+
this.stats.idleWorkers = 0;
|
|
144
|
+
this.stats.queueSize = 0;
|
|
145
|
+
}
|
|
146
|
+
};
|
|
147
|
+
/**
|
|
148
|
+
* Create a worker pool instance
|
|
149
|
+
*/
|
|
150
|
+
function createWorkerPool(config) {
|
|
151
|
+
return new WorkerPool(config);
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
//#endregion
|
|
155
|
+
Object.defineProperty(exports, 'WorkerPool', {
|
|
156
|
+
enumerable: true,
|
|
157
|
+
get: function () {
|
|
158
|
+
return WorkerPool;
|
|
159
|
+
}
|
|
160
|
+
});
|
|
161
|
+
Object.defineProperty(exports, 'createWorkerPool', {
|
|
162
|
+
enumerable: true,
|
|
163
|
+
get: function () {
|
|
164
|
+
return createWorkerPool;
|
|
165
|
+
}
|
|
166
|
+
});
|
|
167
|
+
//# sourceMappingURL=workers-BmzAqLSu.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"workers-BmzAqLSu.js","names":["Worker"],"sources":["../src/workers/WorkerPool.ts"],"sourcesContent":["/**\n * Worker thread pool for parallel processing\n */\n\nimport { Worker } from 'worker_threads';\nimport { cpus } from 'os';\nimport { join } from 'path';\nimport type { WorkerTask, WorkerResult, WorkerPoolConfig, WorkerPoolStats } from './types';\n\n/**\n * Worker pool for parallel text detection and document processing\n */\nexport class WorkerPool {\n private workers: Worker[] = [];\n private availableWorkers: Worker[] = [];\n private taskQueue: Array<{ task: WorkerTask; resolve: Function; reject: Function }> = [];\n private config: Required<WorkerPoolConfig>;\n private stats: WorkerPoolStats;\n private workerPath: string;\n private totalProcessingTime: number = 0;\n\n constructor(config: WorkerPoolConfig = {}) {\n this.config = {\n numWorkers: config.numWorkers || cpus().length,\n maxQueueSize: config.maxQueueSize || 100,\n idleTimeout: config.idleTimeout || 30000\n };\n\n this.stats = {\n activeWorkers: 0,\n idleWorkers: 0,\n queueSize: 0,\n totalProcessed: 0,\n totalErrors: 0,\n avgProcessingTime: 0\n };\n\n // Worker script path (will be in dist after build)\n this.workerPath = join(__dirname, 'worker.js');\n }\n\n /**\n * Initialize worker pool\n */\n async initialize(): Promise<void> {\n for (let i = 0; i < this.config.numWorkers; i++) {\n await this.createWorker();\n }\n }\n\n /**\n * Create a new worker\n */\n private async createWorker(): Promise<Worker> {\n const worker = new Worker(this.workerPath);\n\n worker.on('message', (result: WorkerResult) => {\n this.handleWorkerResult(worker, result);\n });\n\n worker.on('error', (error) => {\n console.error('[WorkerPool] Worker error:', error);\n this.stats.totalErrors++;\n this.removeWorker(worker);\n // Create replacement worker\n this.createWorker();\n });\n\n worker.on('exit', (code) => {\n if (code !== 0) {\n console.error(`[WorkerPool] Worker exited with code ${code}`);\n }\n this.removeWorker(worker);\n });\n\n this.workers.push(worker);\n this.availableWorkers.push(worker);\n this.stats.idleWorkers++;\n\n return worker;\n }\n\n /**\n * Execute a task on the worker pool\n */\n async execute<T = any>(task: WorkerTask): Promise<T> {\n // Check queue size\n if (this.taskQueue.length >= this.config.maxQueueSize) {\n throw new Error(`[WorkerPool] Queue is full (max: ${this.config.maxQueueSize})`);\n }\n\n return new Promise((resolve, reject) => {\n this.taskQueue.push({ task, resolve, reject });\n this.stats.queueSize = this.taskQueue.length;\n this.processQueue();\n });\n }\n\n /**\n * Process task queue\n */\n private processQueue(): void {\n while (this.taskQueue.length > 0 && this.availableWorkers.length > 0) {\n const worker = this.availableWorkers.shift()!;\n const { task, resolve, reject } = this.taskQueue.shift()!;\n\n this.stats.idleWorkers--;\n this.stats.activeWorkers++;\n this.stats.queueSize = this.taskQueue.length;\n\n // Store resolve/reject for this task\n (worker as any).__currentTask = { resolve, reject, startTime: Date.now() };\n\n // Send task to worker\n worker.postMessage(task);\n }\n }\n\n /**\n * Handle worker result\n */\n private handleWorkerResult(worker: Worker, result: WorkerResult): void {\n const currentTask = (worker as any).__currentTask;\n if (!currentTask) return;\n\n this.stats.activeWorkers--;\n this.stats.idleWorkers++;\n this.stats.totalProcessed++;\n\n // Update processing time stats\n this.totalProcessingTime += result.processingTime;\n this.stats.avgProcessingTime = this.totalProcessingTime / this.stats.totalProcessed;\n\n // Return worker to available pool\n this.availableWorkers.push(worker);\n delete (worker as any).__currentTask;\n\n // Resolve or reject the promise\n if (result.error) {\n this.stats.totalErrors++;\n currentTask.reject(new Error(result.error));\n } else {\n currentTask.resolve(result.result);\n }\n\n // Process next task in queue\n this.processQueue();\n }\n\n /**\n * Remove worker from pool\n */\n private removeWorker(worker: Worker): void {\n const index = this.workers.indexOf(worker);\n if (index !== -1) {\n this.workers.splice(index, 1);\n }\n\n const availableIndex = this.availableWorkers.indexOf(worker);\n if (availableIndex !== -1) {\n this.availableWorkers.splice(availableIndex, 1);\n this.stats.idleWorkers--;\n }\n }\n\n /**\n * Get pool statistics\n */\n getStats(): WorkerPoolStats {\n return { ...this.stats };\n }\n\n /**\n * Terminate all workers\n */\n async terminate(): Promise<void> {\n const terminatePromises = this.workers.map(worker => worker.terminate());\n await Promise.all(terminatePromises);\n this.workers = [];\n this.availableWorkers = [];\n this.taskQueue = [];\n this.stats.activeWorkers = 0;\n this.stats.idleWorkers = 0;\n this.stats.queueSize = 0;\n }\n}\n\n/**\n * Create a worker pool instance\n */\nexport function createWorkerPool(config?: WorkerPoolConfig): WorkerPool {\n return new WorkerPool(config);\n}\n"],"mappings":";;;;;;;;;;;;AAYA,IAAa,aAAb,MAAwB;CAStB,YAAY,SAA2B,EAAE,EAAE;iBARf,EAAE;0BACO,EAAE;mBAC+C,EAAE;6BAIlD;AAGpC,OAAK,SAAS;GACZ,YAAY,OAAO,4BAAoB,CAAC;GACxC,cAAc,OAAO,gBAAgB;GACrC,aAAa,OAAO,eAAe;GACpC;AAED,OAAK,QAAQ;GACX,eAAe;GACf,aAAa;GACb,WAAW;GACX,gBAAgB;GAChB,aAAa;GACb,mBAAmB;GACpB;AAGD,OAAK,4BAAkB,WAAW,YAAY;;;;;CAMhD,MAAM,aAA4B;AAChC,OAAK,IAAI,IAAI,GAAG,IAAI,KAAK,OAAO,YAAY,IAC1C,OAAM,KAAK,cAAc;;;;;CAO7B,MAAc,eAAgC;EAC5C,MAAM,SAAS,IAAIA,sBAAO,KAAK,WAAW;AAE1C,SAAO,GAAG,YAAY,WAAyB;AAC7C,QAAK,mBAAmB,QAAQ,OAAO;IACvC;AAEF,SAAO,GAAG,UAAU,UAAU;AAC5B,WAAQ,MAAM,8BAA8B,MAAM;AAClD,QAAK,MAAM;AACX,QAAK,aAAa,OAAO;AAEzB,QAAK,cAAc;IACnB;AAEF,SAAO,GAAG,SAAS,SAAS;AAC1B,OAAI,SAAS,EACX,SAAQ,MAAM,wCAAwC,OAAO;AAE/D,QAAK,aAAa,OAAO;IACzB;AAEF,OAAK,QAAQ,KAAK,OAAO;AACzB,OAAK,iBAAiB,KAAK,OAAO;AAClC,OAAK,MAAM;AAEX,SAAO;;;;;CAMT,MAAM,QAAiB,MAA8B;AAEnD,MAAI,KAAK,UAAU,UAAU,KAAK,OAAO,aACvC,OAAM,IAAI,MAAM,oCAAoC,KAAK,OAAO,aAAa,GAAG;AAGlF,SAAO,IAAI,SAAS,SAAS,WAAW;AACtC,QAAK,UAAU,KAAK;IAAE;IAAM;IAAS;IAAQ,CAAC;AAC9C,QAAK,MAAM,YAAY,KAAK,UAAU;AACtC,QAAK,cAAc;IACnB;;;;;CAMJ,AAAQ,eAAqB;AAC3B,SAAO,KAAK,UAAU,SAAS,KAAK,KAAK,iBAAiB,SAAS,GAAG;GACpE,MAAM,SAAS,KAAK,iBAAiB,OAAO;GAC5C,MAAM,EAAE,MAAM,SAAS,WAAW,KAAK,UAAU,OAAO;AAExD,QAAK,MAAM;AACX,QAAK,MAAM;AACX,QAAK,MAAM,YAAY,KAAK,UAAU;AAGtC,GAAC,OAAe,gBAAgB;IAAE;IAAS;IAAQ,WAAW,KAAK,KAAK;IAAE;AAG1E,UAAO,YAAY,KAAK;;;;;;CAO5B,AAAQ,mBAAmB,QAAgB,QAA4B;EACrE,MAAM,cAAe,OAAe;AACpC,MAAI,CAAC,YAAa;AAElB,OAAK,MAAM;AACX,OAAK,MAAM;AACX,OAAK,MAAM;AAGX,OAAK,uBAAuB,OAAO;AACnC,OAAK,MAAM,oBAAoB,KAAK,sBAAsB,KAAK,MAAM;AAGrE,OAAK,iBAAiB,KAAK,OAAO;AAClC,SAAQ,OAAe;AAGvB,MAAI,OAAO,OAAO;AAChB,QAAK,MAAM;AACX,eAAY,OAAO,IAAI,MAAM,OAAO,MAAM,CAAC;QAE3C,aAAY,QAAQ,OAAO,OAAO;AAIpC,OAAK,cAAc;;;;;CAMrB,AAAQ,aAAa,QAAsB;EACzC,MAAM,QAAQ,KAAK,QAAQ,QAAQ,OAAO;AAC1C,MAAI,UAAU,GACZ,MAAK,QAAQ,OAAO,OAAO,EAAE;EAG/B,MAAM,iBAAiB,KAAK,iBAAiB,QAAQ,OAAO;AAC5D,MAAI,mBAAmB,IAAI;AACzB,QAAK,iBAAiB,OAAO,gBAAgB,EAAE;AAC/C,QAAK,MAAM;;;;;;CAOf,WAA4B;AAC1B,SAAO,EAAE,GAAG,KAAK,OAAO;;;;;CAM1B,MAAM,YAA2B;EAC/B,MAAM,oBAAoB,KAAK,QAAQ,KAAI,WAAU,OAAO,WAAW,CAAC;AACxE,QAAM,QAAQ,IAAI,kBAAkB;AACpC,OAAK,UAAU,EAAE;AACjB,OAAK,mBAAmB,EAAE;AAC1B,OAAK,YAAY,EAAE;AACnB,OAAK,MAAM,gBAAgB;AAC3B,OAAK,MAAM,cAAc;AACzB,OAAK,MAAM,YAAY;;;;;;AAO3B,SAAgB,iBAAiB,QAAuC;AACtE,QAAO,IAAI,WAAW,OAAO"}
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
const require_index_cli = require('./index.cli.cjs');
|
|
3
|
+
let path = require("path");
|
|
4
|
+
let worker_threads = require("worker_threads");
|
|
5
|
+
let os = require("os");
|
|
6
|
+
|
|
7
|
+
//#region src/workers/WorkerPool.ts
|
|
8
|
+
/**
|
|
9
|
+
* Worker thread pool for parallel processing
|
|
10
|
+
*/
|
|
11
|
+
/**
|
|
12
|
+
* Worker pool for parallel text detection and document processing
|
|
13
|
+
*/
|
|
14
|
+
var WorkerPool = class {
|
|
15
|
+
constructor(config = {}) {
|
|
16
|
+
this.workers = [];
|
|
17
|
+
this.availableWorkers = [];
|
|
18
|
+
this.taskQueue = [];
|
|
19
|
+
this.totalProcessingTime = 0;
|
|
20
|
+
this.config = {
|
|
21
|
+
numWorkers: config.numWorkers || (0, os.cpus)().length,
|
|
22
|
+
maxQueueSize: config.maxQueueSize || 100,
|
|
23
|
+
idleTimeout: config.idleTimeout || 3e4
|
|
24
|
+
};
|
|
25
|
+
this.stats = {
|
|
26
|
+
activeWorkers: 0,
|
|
27
|
+
idleWorkers: 0,
|
|
28
|
+
queueSize: 0,
|
|
29
|
+
totalProcessed: 0,
|
|
30
|
+
totalErrors: 0,
|
|
31
|
+
avgProcessingTime: 0
|
|
32
|
+
};
|
|
33
|
+
this.workerPath = (0, path.join)(__dirname, "worker.js");
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* Initialize worker pool
|
|
37
|
+
*/
|
|
38
|
+
async initialize() {
|
|
39
|
+
for (let i = 0; i < this.config.numWorkers; i++) await this.createWorker();
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* Create a new worker
|
|
43
|
+
*/
|
|
44
|
+
async createWorker() {
|
|
45
|
+
const worker = new worker_threads.Worker(this.workerPath);
|
|
46
|
+
worker.on("message", (result) => {
|
|
47
|
+
this.handleWorkerResult(worker, result);
|
|
48
|
+
});
|
|
49
|
+
worker.on("error", (error) => {
|
|
50
|
+
console.error("[WorkerPool] Worker error:", error);
|
|
51
|
+
this.stats.totalErrors++;
|
|
52
|
+
this.removeWorker(worker);
|
|
53
|
+
this.createWorker();
|
|
54
|
+
});
|
|
55
|
+
worker.on("exit", (code) => {
|
|
56
|
+
if (code !== 0) console.error(`[WorkerPool] Worker exited with code ${code}`);
|
|
57
|
+
this.removeWorker(worker);
|
|
58
|
+
});
|
|
59
|
+
this.workers.push(worker);
|
|
60
|
+
this.availableWorkers.push(worker);
|
|
61
|
+
this.stats.idleWorkers++;
|
|
62
|
+
return worker;
|
|
63
|
+
}
|
|
64
|
+
/**
|
|
65
|
+
* Execute a task on the worker pool
|
|
66
|
+
*/
|
|
67
|
+
async execute(task) {
|
|
68
|
+
if (this.taskQueue.length >= this.config.maxQueueSize) throw new Error(`[WorkerPool] Queue is full (max: ${this.config.maxQueueSize})`);
|
|
69
|
+
return new Promise((resolve, reject) => {
|
|
70
|
+
this.taskQueue.push({
|
|
71
|
+
task,
|
|
72
|
+
resolve,
|
|
73
|
+
reject
|
|
74
|
+
});
|
|
75
|
+
this.stats.queueSize = this.taskQueue.length;
|
|
76
|
+
this.processQueue();
|
|
77
|
+
});
|
|
78
|
+
}
|
|
79
|
+
/**
|
|
80
|
+
* Process task queue
|
|
81
|
+
*/
|
|
82
|
+
processQueue() {
|
|
83
|
+
while (this.taskQueue.length > 0 && this.availableWorkers.length > 0) {
|
|
84
|
+
const worker = this.availableWorkers.shift();
|
|
85
|
+
const { task, resolve, reject } = this.taskQueue.shift();
|
|
86
|
+
this.stats.idleWorkers--;
|
|
87
|
+
this.stats.activeWorkers++;
|
|
88
|
+
this.stats.queueSize = this.taskQueue.length;
|
|
89
|
+
worker.__currentTask = {
|
|
90
|
+
resolve,
|
|
91
|
+
reject,
|
|
92
|
+
startTime: Date.now()
|
|
93
|
+
};
|
|
94
|
+
worker.postMessage(task);
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
/**
|
|
98
|
+
* Handle worker result
|
|
99
|
+
*/
|
|
100
|
+
handleWorkerResult(worker, result) {
|
|
101
|
+
const currentTask = worker.__currentTask;
|
|
102
|
+
if (!currentTask) return;
|
|
103
|
+
this.stats.activeWorkers--;
|
|
104
|
+
this.stats.idleWorkers++;
|
|
105
|
+
this.stats.totalProcessed++;
|
|
106
|
+
this.totalProcessingTime += result.processingTime;
|
|
107
|
+
this.stats.avgProcessingTime = this.totalProcessingTime / this.stats.totalProcessed;
|
|
108
|
+
this.availableWorkers.push(worker);
|
|
109
|
+
delete worker.__currentTask;
|
|
110
|
+
if (result.error) {
|
|
111
|
+
this.stats.totalErrors++;
|
|
112
|
+
currentTask.reject(new Error(result.error));
|
|
113
|
+
} else currentTask.resolve(result.result);
|
|
114
|
+
this.processQueue();
|
|
115
|
+
}
|
|
116
|
+
/**
|
|
117
|
+
* Remove worker from pool
|
|
118
|
+
*/
|
|
119
|
+
removeWorker(worker) {
|
|
120
|
+
const index = this.workers.indexOf(worker);
|
|
121
|
+
if (index !== -1) this.workers.splice(index, 1);
|
|
122
|
+
const availableIndex = this.availableWorkers.indexOf(worker);
|
|
123
|
+
if (availableIndex !== -1) {
|
|
124
|
+
this.availableWorkers.splice(availableIndex, 1);
|
|
125
|
+
this.stats.idleWorkers--;
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
/**
|
|
129
|
+
* Get pool statistics
|
|
130
|
+
*/
|
|
131
|
+
getStats() {
|
|
132
|
+
return { ...this.stats };
|
|
133
|
+
}
|
|
134
|
+
/**
|
|
135
|
+
* Terminate all workers
|
|
136
|
+
*/
|
|
137
|
+
async terminate() {
|
|
138
|
+
const terminatePromises = this.workers.map((worker) => worker.terminate());
|
|
139
|
+
await Promise.all(terminatePromises);
|
|
140
|
+
this.workers = [];
|
|
141
|
+
this.availableWorkers = [];
|
|
142
|
+
this.taskQueue = [];
|
|
143
|
+
this.stats.activeWorkers = 0;
|
|
144
|
+
this.stats.idleWorkers = 0;
|
|
145
|
+
this.stats.queueSize = 0;
|
|
146
|
+
}
|
|
147
|
+
};
|
|
148
|
+
/**
|
|
149
|
+
* Create a worker pool instance
|
|
150
|
+
*/
|
|
151
|
+
function createWorkerPool(config) {
|
|
152
|
+
return new WorkerPool(config);
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
//#endregion
|
|
156
|
+
exports.createWorkerPool = createWorkerPool;
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
import { join } from "path";
|
|
2
|
+
import { Worker } from "worker_threads";
|
|
3
|
+
import { cpus } from "os";
|
|
4
|
+
|
|
5
|
+
//#region src/workers/WorkerPool.ts
|
|
6
|
+
/**
|
|
7
|
+
* Worker thread pool for parallel processing
|
|
8
|
+
*/
|
|
9
|
+
/**
|
|
10
|
+
* Worker pool for parallel text detection and document processing
|
|
11
|
+
*/
|
|
12
|
+
var WorkerPool = class {
|
|
13
|
+
constructor(config = {}) {
|
|
14
|
+
this.workers = [];
|
|
15
|
+
this.availableWorkers = [];
|
|
16
|
+
this.taskQueue = [];
|
|
17
|
+
this.totalProcessingTime = 0;
|
|
18
|
+
this.config = {
|
|
19
|
+
numWorkers: config.numWorkers || cpus().length,
|
|
20
|
+
maxQueueSize: config.maxQueueSize || 100,
|
|
21
|
+
idleTimeout: config.idleTimeout || 3e4
|
|
22
|
+
};
|
|
23
|
+
this.stats = {
|
|
24
|
+
activeWorkers: 0,
|
|
25
|
+
idleWorkers: 0,
|
|
26
|
+
queueSize: 0,
|
|
27
|
+
totalProcessed: 0,
|
|
28
|
+
totalErrors: 0,
|
|
29
|
+
avgProcessingTime: 0
|
|
30
|
+
};
|
|
31
|
+
this.workerPath = join(__dirname, "worker.js");
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Initialize worker pool
|
|
35
|
+
*/
|
|
36
|
+
async initialize() {
|
|
37
|
+
for (let i = 0; i < this.config.numWorkers; i++) await this.createWorker();
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* Create a new worker
|
|
41
|
+
*/
|
|
42
|
+
async createWorker() {
|
|
43
|
+
const worker = new Worker(this.workerPath);
|
|
44
|
+
worker.on("message", (result) => {
|
|
45
|
+
this.handleWorkerResult(worker, result);
|
|
46
|
+
});
|
|
47
|
+
worker.on("error", (error) => {
|
|
48
|
+
console.error("[WorkerPool] Worker error:", error);
|
|
49
|
+
this.stats.totalErrors++;
|
|
50
|
+
this.removeWorker(worker);
|
|
51
|
+
this.createWorker();
|
|
52
|
+
});
|
|
53
|
+
worker.on("exit", (code) => {
|
|
54
|
+
if (code !== 0) console.error(`[WorkerPool] Worker exited with code ${code}`);
|
|
55
|
+
this.removeWorker(worker);
|
|
56
|
+
});
|
|
57
|
+
this.workers.push(worker);
|
|
58
|
+
this.availableWorkers.push(worker);
|
|
59
|
+
this.stats.idleWorkers++;
|
|
60
|
+
return worker;
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* Execute a task on the worker pool
|
|
64
|
+
*/
|
|
65
|
+
async execute(task) {
|
|
66
|
+
if (this.taskQueue.length >= this.config.maxQueueSize) throw new Error(`[WorkerPool] Queue is full (max: ${this.config.maxQueueSize})`);
|
|
67
|
+
return new Promise((resolve, reject) => {
|
|
68
|
+
this.taskQueue.push({
|
|
69
|
+
task,
|
|
70
|
+
resolve,
|
|
71
|
+
reject
|
|
72
|
+
});
|
|
73
|
+
this.stats.queueSize = this.taskQueue.length;
|
|
74
|
+
this.processQueue();
|
|
75
|
+
});
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* Process task queue
|
|
79
|
+
*/
|
|
80
|
+
processQueue() {
|
|
81
|
+
while (this.taskQueue.length > 0 && this.availableWorkers.length > 0) {
|
|
82
|
+
const worker = this.availableWorkers.shift();
|
|
83
|
+
const { task, resolve, reject } = this.taskQueue.shift();
|
|
84
|
+
this.stats.idleWorkers--;
|
|
85
|
+
this.stats.activeWorkers++;
|
|
86
|
+
this.stats.queueSize = this.taskQueue.length;
|
|
87
|
+
worker.__currentTask = {
|
|
88
|
+
resolve,
|
|
89
|
+
reject,
|
|
90
|
+
startTime: Date.now()
|
|
91
|
+
};
|
|
92
|
+
worker.postMessage(task);
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
/**
|
|
96
|
+
* Handle worker result
|
|
97
|
+
*/
|
|
98
|
+
handleWorkerResult(worker, result) {
|
|
99
|
+
const currentTask = worker.__currentTask;
|
|
100
|
+
if (!currentTask) return;
|
|
101
|
+
this.stats.activeWorkers--;
|
|
102
|
+
this.stats.idleWorkers++;
|
|
103
|
+
this.stats.totalProcessed++;
|
|
104
|
+
this.totalProcessingTime += result.processingTime;
|
|
105
|
+
this.stats.avgProcessingTime = this.totalProcessingTime / this.stats.totalProcessed;
|
|
106
|
+
this.availableWorkers.push(worker);
|
|
107
|
+
delete worker.__currentTask;
|
|
108
|
+
if (result.error) {
|
|
109
|
+
this.stats.totalErrors++;
|
|
110
|
+
currentTask.reject(new Error(result.error));
|
|
111
|
+
} else currentTask.resolve(result.result);
|
|
112
|
+
this.processQueue();
|
|
113
|
+
}
|
|
114
|
+
/**
|
|
115
|
+
* Remove worker from pool
|
|
116
|
+
*/
|
|
117
|
+
removeWorker(worker) {
|
|
118
|
+
const index = this.workers.indexOf(worker);
|
|
119
|
+
if (index !== -1) this.workers.splice(index, 1);
|
|
120
|
+
const availableIndex = this.availableWorkers.indexOf(worker);
|
|
121
|
+
if (availableIndex !== -1) {
|
|
122
|
+
this.availableWorkers.splice(availableIndex, 1);
|
|
123
|
+
this.stats.idleWorkers--;
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
/**
|
|
127
|
+
* Get pool statistics
|
|
128
|
+
*/
|
|
129
|
+
getStats() {
|
|
130
|
+
return { ...this.stats };
|
|
131
|
+
}
|
|
132
|
+
/**
|
|
133
|
+
* Terminate all workers
|
|
134
|
+
*/
|
|
135
|
+
async terminate() {
|
|
136
|
+
const terminatePromises = this.workers.map((worker) => worker.terminate());
|
|
137
|
+
await Promise.all(terminatePromises);
|
|
138
|
+
this.workers = [];
|
|
139
|
+
this.availableWorkers = [];
|
|
140
|
+
this.taskQueue = [];
|
|
141
|
+
this.stats.activeWorkers = 0;
|
|
142
|
+
this.stats.idleWorkers = 0;
|
|
143
|
+
this.stats.queueSize = 0;
|
|
144
|
+
}
|
|
145
|
+
};
|
|
146
|
+
/**
|
|
147
|
+
* Create a worker pool instance
|
|
148
|
+
*/
|
|
149
|
+
function createWorkerPool(config) {
|
|
150
|
+
return new WorkerPool(config);
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
//#endregion
|
|
154
|
+
export { createWorkerPool as n, WorkerPool as t };
|
|
155
|
+
//# sourceMappingURL=workers-vfrB0Vzh.mjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"workers-vfrB0Vzh.mjs","names":[],"sources":["../src/workers/WorkerPool.ts"],"sourcesContent":["/**\n * Worker thread pool for parallel processing\n */\n\nimport { Worker } from 'worker_threads';\nimport { cpus } from 'os';\nimport { join } from 'path';\nimport type { WorkerTask, WorkerResult, WorkerPoolConfig, WorkerPoolStats } from './types';\n\n/**\n * Worker pool for parallel text detection and document processing\n */\nexport class WorkerPool {\n private workers: Worker[] = [];\n private availableWorkers: Worker[] = [];\n private taskQueue: Array<{ task: WorkerTask; resolve: Function; reject: Function }> = [];\n private config: Required<WorkerPoolConfig>;\n private stats: WorkerPoolStats;\n private workerPath: string;\n private totalProcessingTime: number = 0;\n\n constructor(config: WorkerPoolConfig = {}) {\n this.config = {\n numWorkers: config.numWorkers || cpus().length,\n maxQueueSize: config.maxQueueSize || 100,\n idleTimeout: config.idleTimeout || 30000\n };\n\n this.stats = {\n activeWorkers: 0,\n idleWorkers: 0,\n queueSize: 0,\n totalProcessed: 0,\n totalErrors: 0,\n avgProcessingTime: 0\n };\n\n // Worker script path (will be in dist after build)\n this.workerPath = join(__dirname, 'worker.js');\n }\n\n /**\n * Initialize worker pool\n */\n async initialize(): Promise<void> {\n for (let i = 0; i < this.config.numWorkers; i++) {\n await this.createWorker();\n }\n }\n\n /**\n * Create a new worker\n */\n private async createWorker(): Promise<Worker> {\n const worker = new Worker(this.workerPath);\n\n worker.on('message', (result: WorkerResult) => {\n this.handleWorkerResult(worker, result);\n });\n\n worker.on('error', (error) => {\n console.error('[WorkerPool] Worker error:', error);\n this.stats.totalErrors++;\n this.removeWorker(worker);\n // Create replacement worker\n this.createWorker();\n });\n\n worker.on('exit', (code) => {\n if (code !== 0) {\n console.error(`[WorkerPool] Worker exited with code ${code}`);\n }\n this.removeWorker(worker);\n });\n\n this.workers.push(worker);\n this.availableWorkers.push(worker);\n this.stats.idleWorkers++;\n\n return worker;\n }\n\n /**\n * Execute a task on the worker pool\n */\n async execute<T = any>(task: WorkerTask): Promise<T> {\n // Check queue size\n if (this.taskQueue.length >= this.config.maxQueueSize) {\n throw new Error(`[WorkerPool] Queue is full (max: ${this.config.maxQueueSize})`);\n }\n\n return new Promise((resolve, reject) => {\n this.taskQueue.push({ task, resolve, reject });\n this.stats.queueSize = this.taskQueue.length;\n this.processQueue();\n });\n }\n\n /**\n * Process task queue\n */\n private processQueue(): void {\n while (this.taskQueue.length > 0 && this.availableWorkers.length > 0) {\n const worker = this.availableWorkers.shift()!;\n const { task, resolve, reject } = this.taskQueue.shift()!;\n\n this.stats.idleWorkers--;\n this.stats.activeWorkers++;\n this.stats.queueSize = this.taskQueue.length;\n\n // Store resolve/reject for this task\n (worker as any).__currentTask = { resolve, reject, startTime: Date.now() };\n\n // Send task to worker\n worker.postMessage(task);\n }\n }\n\n /**\n * Handle worker result\n */\n private handleWorkerResult(worker: Worker, result: WorkerResult): void {\n const currentTask = (worker as any).__currentTask;\n if (!currentTask) return;\n\n this.stats.activeWorkers--;\n this.stats.idleWorkers++;\n this.stats.totalProcessed++;\n\n // Update processing time stats\n this.totalProcessingTime += result.processingTime;\n this.stats.avgProcessingTime = this.totalProcessingTime / this.stats.totalProcessed;\n\n // Return worker to available pool\n this.availableWorkers.push(worker);\n delete (worker as any).__currentTask;\n\n // Resolve or reject the promise\n if (result.error) {\n this.stats.totalErrors++;\n currentTask.reject(new Error(result.error));\n } else {\n currentTask.resolve(result.result);\n }\n\n // Process next task in queue\n this.processQueue();\n }\n\n /**\n * Remove worker from pool\n */\n private removeWorker(worker: Worker): void {\n const index = this.workers.indexOf(worker);\n if (index !== -1) {\n this.workers.splice(index, 1);\n }\n\n const availableIndex = this.availableWorkers.indexOf(worker);\n if (availableIndex !== -1) {\n this.availableWorkers.splice(availableIndex, 1);\n this.stats.idleWorkers--;\n }\n }\n\n /**\n * Get pool statistics\n */\n getStats(): WorkerPoolStats {\n return { ...this.stats };\n }\n\n /**\n * Terminate all workers\n */\n async terminate(): Promise<void> {\n const terminatePromises = this.workers.map(worker => worker.terminate());\n await Promise.all(terminatePromises);\n this.workers = [];\n this.availableWorkers = [];\n this.taskQueue = [];\n this.stats.activeWorkers = 0;\n this.stats.idleWorkers = 0;\n this.stats.queueSize = 0;\n }\n}\n\n/**\n * Create a worker pool instance\n */\nexport function createWorkerPool(config?: WorkerPoolConfig): WorkerPool {\n return new WorkerPool(config);\n}\n"],"mappings":";;;;;;;;;;;AAYA,IAAa,aAAb,MAAwB;CAStB,YAAY,SAA2B,EAAE,EAAE;iBARf,EAAE;0BACO,EAAE;mBAC+C,EAAE;6BAIlD;AAGpC,OAAK,SAAS;GACZ,YAAY,OAAO,cAAc,MAAM,CAAC;GACxC,cAAc,OAAO,gBAAgB;GACrC,aAAa,OAAO,eAAe;GACpC;AAED,OAAK,QAAQ;GACX,eAAe;GACf,aAAa;GACb,WAAW;GACX,gBAAgB;GAChB,aAAa;GACb,mBAAmB;GACpB;AAGD,OAAK,aAAa,KAAK,WAAW,YAAY;;;;;CAMhD,MAAM,aAA4B;AAChC,OAAK,IAAI,IAAI,GAAG,IAAI,KAAK,OAAO,YAAY,IAC1C,OAAM,KAAK,cAAc;;;;;CAO7B,MAAc,eAAgC;EAC5C,MAAM,SAAS,IAAI,OAAO,KAAK,WAAW;AAE1C,SAAO,GAAG,YAAY,WAAyB;AAC7C,QAAK,mBAAmB,QAAQ,OAAO;IACvC;AAEF,SAAO,GAAG,UAAU,UAAU;AAC5B,WAAQ,MAAM,8BAA8B,MAAM;AAClD,QAAK,MAAM;AACX,QAAK,aAAa,OAAO;AAEzB,QAAK,cAAc;IACnB;AAEF,SAAO,GAAG,SAAS,SAAS;AAC1B,OAAI,SAAS,EACX,SAAQ,MAAM,wCAAwC,OAAO;AAE/D,QAAK,aAAa,OAAO;IACzB;AAEF,OAAK,QAAQ,KAAK,OAAO;AACzB,OAAK,iBAAiB,KAAK,OAAO;AAClC,OAAK,MAAM;AAEX,SAAO;;;;;CAMT,MAAM,QAAiB,MAA8B;AAEnD,MAAI,KAAK,UAAU,UAAU,KAAK,OAAO,aACvC,OAAM,IAAI,MAAM,oCAAoC,KAAK,OAAO,aAAa,GAAG;AAGlF,SAAO,IAAI,SAAS,SAAS,WAAW;AACtC,QAAK,UAAU,KAAK;IAAE;IAAM;IAAS;IAAQ,CAAC;AAC9C,QAAK,MAAM,YAAY,KAAK,UAAU;AACtC,QAAK,cAAc;IACnB;;;;;CAMJ,AAAQ,eAAqB;AAC3B,SAAO,KAAK,UAAU,SAAS,KAAK,KAAK,iBAAiB,SAAS,GAAG;GACpE,MAAM,SAAS,KAAK,iBAAiB,OAAO;GAC5C,MAAM,EAAE,MAAM,SAAS,WAAW,KAAK,UAAU,OAAO;AAExD,QAAK,MAAM;AACX,QAAK,MAAM;AACX,QAAK,MAAM,YAAY,KAAK,UAAU;AAGtC,GAAC,OAAe,gBAAgB;IAAE;IAAS;IAAQ,WAAW,KAAK,KAAK;IAAE;AAG1E,UAAO,YAAY,KAAK;;;;;;CAO5B,AAAQ,mBAAmB,QAAgB,QAA4B;EACrE,MAAM,cAAe,OAAe;AACpC,MAAI,CAAC,YAAa;AAElB,OAAK,MAAM;AACX,OAAK,MAAM;AACX,OAAK,MAAM;AAGX,OAAK,uBAAuB,OAAO;AACnC,OAAK,MAAM,oBAAoB,KAAK,sBAAsB,KAAK,MAAM;AAGrE,OAAK,iBAAiB,KAAK,OAAO;AAClC,SAAQ,OAAe;AAGvB,MAAI,OAAO,OAAO;AAChB,QAAK,MAAM;AACX,eAAY,OAAO,IAAI,MAAM,OAAO,MAAM,CAAC;QAE3C,aAAY,QAAQ,OAAO,OAAO;AAIpC,OAAK,cAAc;;;;;CAMrB,AAAQ,aAAa,QAAsB;EACzC,MAAM,QAAQ,KAAK,QAAQ,QAAQ,OAAO;AAC1C,MAAI,UAAU,GACZ,MAAK,QAAQ,OAAO,OAAO,EAAE;EAG/B,MAAM,iBAAiB,KAAK,iBAAiB,QAAQ,OAAO;AAC5D,MAAI,mBAAmB,IAAI;AACzB,QAAK,iBAAiB,OAAO,gBAAgB,EAAE;AAC/C,QAAK,MAAM;;;;;;CAOf,WAA4B;AAC1B,SAAO,EAAE,GAAG,KAAK,OAAO;;;;;CAM1B,MAAM,YAA2B;EAC/B,MAAM,oBAAoB,KAAK,QAAQ,KAAI,WAAU,OAAO,WAAW,CAAC;AACxE,QAAM,QAAQ,IAAI,kBAAkB;AACpC,OAAK,UAAU,EAAE;AACjB,OAAK,mBAAmB,EAAE;AAC1B,OAAK,YAAY,EAAE;AACnB,OAAK,MAAM,gBAAgB;AAC3B,OAAK,MAAM,cAAc;AACzB,OAAK,MAAM,YAAY;;;;;;AAO3B,SAAgB,iBAAiB,QAAuC;AACtE,QAAO,IAAI,WAAW,OAAO"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "od-temp",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.5",
|
|
4
4
|
"main": "./dist/index.js",
|
|
5
5
|
"module": "./dist/index.mjs",
|
|
6
6
|
"types": "./dist/index.d.ts",
|
|
@@ -17,12 +17,7 @@
|
|
|
17
17
|
}
|
|
18
18
|
},
|
|
19
19
|
"files": [
|
|
20
|
-
"dist
|
|
21
|
-
"dist/index.mjs",
|
|
22
|
-
"dist/index.d.ts",
|
|
23
|
-
"dist/index.d.mts",
|
|
24
|
-
"dist/index.cli.js",
|
|
25
|
-
"dist/cli/"
|
|
20
|
+
"dist"
|
|
26
21
|
],
|
|
27
22
|
"scripts": {
|
|
28
23
|
"build": "tsdown && node scripts/dual-types.js",
|