flexbiz-server 12.5.18 → 12.5.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json
CHANGED
|
@@ -190,7 +190,7 @@ class controllerRPT {
|
|
|
190
190
|
const result = returnvalue.data;
|
|
191
191
|
switch(ext){
|
|
192
192
|
case "xlsx":{
|
|
193
|
-
console.log("[controllerRPT] send file from worker...",ext,
|
|
193
|
+
console.log("[controllerRPT] send file from worker...",ext,",is buffer: ",Buffer.isBuffer(result));
|
|
194
194
|
res.setHeader('Content-Type', 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet');
|
|
195
195
|
res.setHeader('Content-Disposition', 'attachment; filename="' + rptId + '".xlsx');
|
|
196
196
|
//convert sang buffer nếu data là một object có type ==Buffer
|
|
@@ -230,6 +230,7 @@ class controllerRPT {
|
|
|
230
230
|
data = Buffer.from(data, 'base64');
|
|
231
231
|
}
|
|
232
232
|
else {
|
|
233
|
+
console.error("[controllerRPT] Dữ liệu không hợp lệ. Data type:", typeof(data))
|
|
233
234
|
return res.status(400).send({error:'Dữ liệu không hợp lệ'});
|
|
234
235
|
}
|
|
235
236
|
|
|
@@ -1,16 +1,287 @@
|
|
|
1
|
-
const {Worker}=require(
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
(
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
1
|
+
const { Worker } = require('worker_threads');
|
|
2
|
+
const fs = require('fs');
|
|
3
|
+
const _ = require("lodash");
|
|
4
|
+
const crypto = require("crypto");
|
|
5
|
+
const moment = require("moment");
|
|
6
|
+
const redisCache = require("./redis-cache");
|
|
7
|
+
|
|
8
|
+
class WorkerPool {
|
|
9
|
+
constructor(filename,maxQueue = 0,maxWorkers = 1,defaultTimeout=5*60*1000,pre_created_workers=0,poolName="WorkerPool") {
|
|
10
|
+
if (!fs.existsSync(filename)) throw new Error('Worker file does not exist');
|
|
11
|
+
this.filename = filename;
|
|
12
|
+
this.maxWorkers = maxWorkers;
|
|
13
|
+
this.maxQueue = maxQueue;
|
|
14
|
+
if(this.maxQueue && this.maxQueue<maxWorkers){
|
|
15
|
+
this.maxQueue = maxWorkers;
|
|
16
|
+
}
|
|
17
|
+
this.defaultTimeout = defaultTimeout
|
|
18
|
+
this.pre_created_workers = pre_created_workers;
|
|
19
|
+
this.queue = [];
|
|
20
|
+
this.queue_running = [];
|
|
21
|
+
this.workers ={};
|
|
22
|
+
this.poolName = poolName;
|
|
23
|
+
console.log(this.poolName,"đang tạo sẵn", pre_created_workers,"workes/",maxWorkers);
|
|
24
|
+
for (let i = 0; i < Math.min(pre_created_workers,maxWorkers); i++) {
|
|
25
|
+
const worker = this.createWorker();
|
|
26
|
+
this.workers[worker.id] = worker;
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
workerIsAlive(wk){
|
|
30
|
+
if(!wk.worker) return false;
|
|
31
|
+
if(wk.worker.threadId<=0) return false;
|
|
32
|
+
if(moment().diff(moment(wk.time_responded),"seconds")>((wk.task||{}).timeout||10000)/1000){
|
|
33
|
+
return false;
|
|
34
|
+
}
|
|
35
|
+
return true;
|
|
36
|
+
}
|
|
37
|
+
createWorker() {
|
|
38
|
+
const wk = {id:`worker-${crypto.randomBytes(20).toString('hex')}`,isBusy:false,time_responded:new Date()};
|
|
39
|
+
wk.worker = new Worker(this.filename);
|
|
40
|
+
wk.worker.on('message', async (message) => {
|
|
41
|
+
wk.time_responded = new Date();
|
|
42
|
+
//Nhận tín hiệu phản hổi ping
|
|
43
|
+
if (message === 'pong') {
|
|
44
|
+
return;
|
|
45
|
+
}
|
|
46
|
+
//
|
|
47
|
+
let task;
|
|
48
|
+
try{
|
|
49
|
+
if(message.id_task){
|
|
50
|
+
task = this.queue_running.find(q=>q.id==message.id_task);
|
|
51
|
+
}else{
|
|
52
|
+
task = wk.task;
|
|
53
|
+
}
|
|
54
|
+
if(task) clearTimeout(task.timeoutId); // Clear timeout
|
|
55
|
+
|
|
56
|
+
if (task && task.callback) {
|
|
57
|
+
if(message.id_task && message.id_task!==task.id){
|
|
58
|
+
console.error("[",this.poolName,"] Kết quả trả về không đúng cho task",task.id,message);
|
|
59
|
+
}else{
|
|
60
|
+
//update status
|
|
61
|
+
task.finished = new Date();
|
|
62
|
+
task.status = 5;
|
|
63
|
+
task.message = "Đã thực hiện xong";
|
|
64
|
+
task.result = message && message.map?undefined:message;
|
|
65
|
+
if(task.save_log){
|
|
66
|
+
try{
|
|
67
|
+
let {finished,status,message,result} = task;
|
|
68
|
+
if(task.save_log) await redisCache.updateObject(task.id,{finished,status,message,result});
|
|
69
|
+
}catch(e){
|
|
70
|
+
console.error("[",this.poolName,"] can't update task status",task.id,e.message);
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
this.queue_running = this.queue_running.filter(q=>q.id!==task.id);
|
|
75
|
+
|
|
76
|
+
console.log("[WorkerStaticPool] Trả kết quả...",Object.keys(message));
|
|
77
|
+
console.log("[WorkerStaticPool] data type is buffer",Buffer.isBuffer(message?.data));
|
|
78
|
+
|
|
79
|
+
task.callback(message,task.id);
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
delete wk.task; // Clear the task
|
|
83
|
+
wk.isBusy = false;
|
|
84
|
+
this.processQueue(); // Process next task
|
|
85
|
+
}catch(e){
|
|
86
|
+
console.error("[",this.poolName,"]",e);
|
|
87
|
+
try{
|
|
88
|
+
delete wk.task; // Clear the task
|
|
89
|
+
wk.isBusy = false;
|
|
90
|
+
this.processQueue(); // Process next task
|
|
91
|
+
}catch(e){
|
|
92
|
+
console.error("[",this.poolName,"] cancel current task",e);
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
});
|
|
96
|
+
|
|
97
|
+
wk.worker.on('error', (error) => {
|
|
98
|
+
console.error('Worker error:', error);
|
|
99
|
+
wk.worker.terminate();
|
|
100
|
+
});
|
|
101
|
+
|
|
102
|
+
wk.worker.on('exit', async (code) => {
|
|
103
|
+
console.error("[",this.poolName,"]",`: worker exited with code ${code}. Creating new worker...`);
|
|
104
|
+
//xử lý gửi lỗi thông báo cho task nếu worker này đang chạy một task
|
|
105
|
+
if(wk.task){
|
|
106
|
+
let task = this.queue_running.find(q=>q.id==wk.task.id);
|
|
107
|
+
//console.log("xử lý task đang chạy",wk.task.id,task,this.queue_running.map(q=>q.id));
|
|
108
|
+
if(task){
|
|
109
|
+
clearTimeout(task.timeoutId);
|
|
110
|
+
task.finished = new Date();
|
|
111
|
+
task.status = 9;
|
|
112
|
+
task.message = "Đã có lỗi xảy ra trong hệ thống. Hãy thử lại";
|
|
113
|
+
let {finished,status,message,result} = task;
|
|
114
|
+
if(task.save_log){
|
|
115
|
+
try{
|
|
116
|
+
if(task.save_log) await redisCache.updateObject(task.id,{finished,status,message,result});
|
|
117
|
+
}catch(e){
|
|
118
|
+
console.error("[",this.poolName,"] can't update task status",task.id,e.message);
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
if(task.callback) task.callback({error:message},task.id);
|
|
123
|
+
this.queue_running = this.queue_running.filter(q=>q.id!==task.id);
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
//Tạo worker mới thay thế
|
|
127
|
+
delete this.workers[wk.id];
|
|
128
|
+
const newWorker = this.createWorker();
|
|
129
|
+
this.workers[newWorker.id] = newWorker;
|
|
130
|
+
});
|
|
131
|
+
return wk;
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
async processQueue(id_worker) {
|
|
135
|
+
for (let i =0;i<this.maxWorkers;i++) {
|
|
136
|
+
if(this.queue.length==0) break;
|
|
137
|
+
let wk = Object.values(this.workers).find(w=>!w.isBusy && !w.task && (!id_worker || id_worker==w.id))
|
|
138
|
+
//Nếu không có worker sẵn sàng thì tạo một worker mới nếu số lượng worker < maxWorker
|
|
139
|
+
if(!wk){
|
|
140
|
+
if (Object.keys(this.workers).length < this.maxWorkers) {
|
|
141
|
+
console.log("[",this.poolName,"] tạo thêm worker mới...",", số lượng workers hiện tại",Object.values(this.workers).length,", max:" ,this.maxWorkers)
|
|
142
|
+
wk = this.createWorker();
|
|
143
|
+
this.workers[wk.id] = wk;
|
|
144
|
+
} else {
|
|
145
|
+
break;
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
if(wk){
|
|
149
|
+
//kiểm tra xem worker còn hoạt động không. nếu không còn hoạt động thì khởi tạo lại
|
|
150
|
+
if(!this.workerIsAlive(wk)){
|
|
151
|
+
console.error("[",this.poolName,"]: Worker",wk.id, "không còn hoạt động. Tạo lại...");
|
|
152
|
+
delete this.workers[wk.id];
|
|
153
|
+
wk = this.createWorker();
|
|
154
|
+
this.workers[wk.id] = wk;
|
|
155
|
+
}
|
|
156
|
+
//
|
|
157
|
+
const task = this.queue.shift();
|
|
158
|
+
if (task) {
|
|
159
|
+
task.status = 2;
|
|
160
|
+
task.message = "Đang thực hiện";
|
|
161
|
+
this.queue_running.push(task);
|
|
162
|
+
//save status of task
|
|
163
|
+
if(task.save_log){
|
|
164
|
+
try{
|
|
165
|
+
let {finished,status,message,result} = task;
|
|
166
|
+
if(task.save_log) await redisCache.updateObject(task.id,{finished,status,message,result});
|
|
167
|
+
}catch(e){
|
|
168
|
+
console.error("[",this.poolName,"] can't update task status",task.id,e.message);
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
wk.isBusy = true;
|
|
173
|
+
wk.task = task;
|
|
174
|
+
task.params.id_worker = wk.id;
|
|
175
|
+
wk.worker.postMessage(task.params);
|
|
176
|
+
|
|
177
|
+
// Set a timeout for the task
|
|
178
|
+
task.timeoutId = setTimeout(async () => {
|
|
179
|
+
this.queue_running = this.queue_running.filter(q=>q.id!==task.id);
|
|
180
|
+
task.finished = new Date();
|
|
181
|
+
task.status = 9;
|
|
182
|
+
task.message = `Đã hết thời gian thực hiện (Time out: ${task.timeout} ms)`;
|
|
183
|
+
let {finished,status,message,result} = task;
|
|
184
|
+
if(task.save_log){
|
|
185
|
+
//save status
|
|
186
|
+
try{
|
|
187
|
+
if(task.save_log) await redisCache.updateObject(task.id,{finished,status,message,result});
|
|
188
|
+
}catch(e){
|
|
189
|
+
console.error("[",this.poolName,"] can't update task status",task.id,e.message);
|
|
190
|
+
}
|
|
191
|
+
//
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
if (task.callback) {
|
|
195
|
+
task.callback({error:message},task.id);
|
|
196
|
+
}
|
|
197
|
+
delete wk.task;
|
|
198
|
+
wk.isBusy = false;
|
|
199
|
+
this.processQueue(); // Process next task
|
|
200
|
+
}, task.timeout || this.defaultTimeout);
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
async pushToQueue(params,callback,timeout=0,id_worker=undefined){
|
|
206
|
+
let save_log = true;
|
|
207
|
+
if(!params.id_task){
|
|
208
|
+
params.id_task = `task-${crypto.randomBytes(20).toString('hex')}`;
|
|
209
|
+
save_log = false;
|
|
210
|
+
}
|
|
211
|
+
//
|
|
212
|
+
const status =0;
|
|
213
|
+
timeout = timeout||this.defaultTimeout;
|
|
214
|
+
const task = {
|
|
215
|
+
id:params.id_task,
|
|
216
|
+
start:new Date(),
|
|
217
|
+
params:_.cloneDeep(params),
|
|
218
|
+
timeout,
|
|
219
|
+
status,
|
|
220
|
+
save_log
|
|
221
|
+
}
|
|
222
|
+
delete task.params.configs
|
|
223
|
+
delete task.params.user
|
|
224
|
+
delete task.params.configs
|
|
225
|
+
delete task.params.data
|
|
226
|
+
if(task.params.req){
|
|
227
|
+
delete task.params.req.user;
|
|
228
|
+
delete task.params.req.body;
|
|
229
|
+
delete (task.params.req.query||{}).access_token;
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
try{
|
|
234
|
+
//save task to redis
|
|
235
|
+
try{
|
|
236
|
+
if(save_log){
|
|
237
|
+
await redisCache.setObject(params.id_task,task);
|
|
238
|
+
//console.log("[",this.poolName,"]: Đã lưu task tới redis",params.id_task);
|
|
239
|
+
}
|
|
240
|
+
}catch(e){
|
|
241
|
+
console.error("[",this.poolName,"] Không thể lưu task tới redis:",e.message);
|
|
242
|
+
}
|
|
243
|
+
//
|
|
244
|
+
this.queue.push({id:params.id_task,params,callback,timeout,status,save_log });
|
|
245
|
+
this.processQueue(id_worker);
|
|
246
|
+
}catch(e){
|
|
247
|
+
console.error("[",this.poolName,"] : lỗi đẩy task tới queue",e);
|
|
248
|
+
callback({error:e.error||e.message||e});
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
}
|
|
252
|
+
async exec(args, callback = () => {}, timeout =0) {
|
|
253
|
+
if (this.maxQueue > 0 && this.queue.length >= this.maxQueue) {
|
|
254
|
+
console.error("[",this.poolName,"]",': queue is full');
|
|
255
|
+
callback({error:'Queue is full'});
|
|
256
|
+
return;
|
|
257
|
+
}
|
|
258
|
+
if(args.load){
|
|
259
|
+
for(let id_worker of Object.keys(this.workers)){
|
|
260
|
+
//console.error("worrker service loading...",i)
|
|
261
|
+
let params = _.cloneDeep(args);
|
|
262
|
+
this.pushToQueue(params,callback,timeout,id_worker);
|
|
263
|
+
}
|
|
264
|
+
}else{
|
|
265
|
+
let params = _.cloneDeep(args);
|
|
266
|
+
this.pushToQueue(params,callback,timeout);
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
fullQueue(){
|
|
270
|
+
return this.maxQueue > 0 && this.queue.length >= this.maxQueue;
|
|
271
|
+
}
|
|
272
|
+
busy() {
|
|
273
|
+
// Check if all workers are busy
|
|
274
|
+
return Object.values(this.workers).filter(w=>!w.isBusy).length==0
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
destroy() {
|
|
278
|
+
for (const worker of Object.values(this.workers)) {
|
|
279
|
+
worker.terminate();
|
|
280
|
+
}
|
|
281
|
+
this.workers={};
|
|
282
|
+
this.queue = [];
|
|
283
|
+
}
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
// Export WorkerPool
|
|
287
|
+
module.exports = WorkerPool;
|
|
@@ -37,11 +37,15 @@ const handleReport =async (msg)=>{
|
|
|
37
37
|
result = JSON.parse(JSON.stringify(result));
|
|
38
38
|
}
|
|
39
39
|
|
|
40
|
-
console.log("[reportWorker] result type",typeof(result));
|
|
41
|
-
console.log("[reportWorker] data type is buffer",Buffer.isBuffer(result?.data))
|
|
40
|
+
//console.log("[reportWorker] result type",typeof(result));
|
|
41
|
+
console.log("[reportWorker] data type is buffer",Buffer.isBuffer(result?.data));//true
|
|
42
42
|
}
|
|
43
43
|
|
|
44
|
-
parentPort.postMessage({
|
|
44
|
+
parentPort.postMessage({
|
|
45
|
+
error:error?.message||error?.error||error,
|
|
46
|
+
result,
|
|
47
|
+
id_task:msg.id_task
|
|
48
|
+
});
|
|
45
49
|
}catch(e){
|
|
46
50
|
console.error("[reportWorker] Error post mesage to parent report processs",{e,result,type:typeof(result)});
|
|
47
51
|
parentPort.postMessage({error:e.message,id_task:msg.id_task});
|