@scz/queue-job-worker-pool 0.0.1

package/package.json

@@ -0,0 +1,16 @@
+{
+  "name": "@scz/queue-job-worker-pool",
+  "version": "0.0.1",
+  "description": "",
+  "main": "src/index.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1",
+    "pub": "npm publish --access public --registry=https://registry.npmjs.org"
+  },
+  "author": "Sunny",
+  "license": "ISC",
+  "dependencies": {
+    "@scz/worker-threads": "^1.0.0",
+    "deepmerge": "^4.3.1"
+  }
+}

package/src/JobQueue.js

@@ -0,0 +1,50 @@
+/**
+ * 
+ */
+const JobQueueTool = require('./tools/JobQueueTool');
+
+class JobQueue {
+    #jobs = [];
+    #job_null_count = 0;
+    #is_init = true;
+
+    constructor(options, poolthis) {
+        this.#jobs = [];
+        this.options = options;
+        this.pool = poolthis;
+
+        this.#is_init = true;
+    }
+
+    addJob(jobData) {
+        if (this.#jobs.length >= this.options.jobQueueSize) {
+            throw new Error('Job queue is full');
+        }
+        let job = JobQueueTool.makeJob(jobData);
+        this.#jobs.push(job);
+
+        this.#job_null_count = 0;
+        this.#is_init = false;
+    }
+
+    getJob() {
+        let job = this.#jobs.shift();
+        if (!job && this.#jobs.length === 0 && !this.#is_init) {
+            this.#job_null_count++;
+        }
+        this.#checkJobQueueDrained();
+        return job;
+    }
+
+    getJobQueueLength() {
+        return this.#jobs.length;
+    }
+
+    #checkJobQueueDrained() {
+        if (this.#job_null_count >= this.pool.getWorkerCount()) {
+            this.#job_null_count = 0;
+            this.pool.emit('job:drained');
+        }
+    }
+}
+module.exports = JobQueue;

package/src/JobQueueThreadPool.js

@@ -0,0 +1,139 @@
+/**
+ * 
+ */
+const EventEmitter = require('node:events');
+
+const deepmerge = require('deepmerge');
+const threads = require('@scz/worker-threads');
+
+const JobQueue = require('./JobQueue');
+const JobQueueTool = require('./tools/JobQueueTool');
+
+class JobQueueThreadPool extends EventEmitter {
+    #jobs_queue = null;
+    #workers = {};
+    #workerPath;
+    #options;
+    #workerOptions = {};
+
+    constructor(workerPath, options) {
+        super();
+        let defaultOptions = {
+            threadsPoolSize: 4,
+            jobQueueSize: 1000,
+            workerOptions: {
+                workerData: {},
+                //如果你要给a.js传递参数，要使用此选项。
+                //请参考官方文档对选项的描述。
+                argv: [],
+                // restart: 'fail',
+                // restartDelay: 1000,
+
+            }
+        };
+        this.#workerPath = workerPath;
+        this.#options = deepmerge(defaultOptions, options);
+        if (options.workerOptions && options.workerOptions.events) {
+            delete options.workerOptions.events;
+        }
+        this.#workerOptions = deepmerge(this.#workerOptions, this.#options.workerOptions);
+
+        this.#jobs_queue = new JobQueue(this.#options, this);
+        this.#createThreadPool();
+    }
+
+    #createThread() {
+        let self = this;
+        let worker = threads(this.#workerPath, {
+            ...this.#workerOptions,
+            events: {
+                message: async (w, msg) => {
+                    switch (msg.type) {
+                        case 'job:completed':
+                            self.emit('job:completed', {
+                                worker: w,
+                                job: msg.job,
+                            });
+                            break;
+
+                        case 'job:failed':
+                            self.emit('job:failed', {
+                                worker: w,
+                                job: msg.job,
+                                error: msg.error,
+                            });
+                            break;
+
+                        default:
+                            break;
+                    }
+
+                    let job = self.#getJob();
+                    if (job) {
+                        self.#sendJobToWorker(w, job);
+                    } else {
+                        w._status = 'waiting';
+                    }
+                },
+                'online': (w) => {
+                    let job2 = self.#getJob();
+                    if (job2) {
+                        w._status = 'running';
+                        self.#sendJobToWorker(w, job2);
+                    }
+                    self.emit('worker:online', w);
+                },
+                'error': (w, error) => {
+                    self.emit('worker:error', { worker: w, error });
+                },
+                // w就是worker，扩展做了包装处理，
+                // 这样方便拿到worker对象。
+                exit: (w, code) => {
+                    self.emit('worker:exit', { worker: w, code });
+                    delete this.#workers[w.threadId];
+                },
+            },
+        });
+
+        this.#workers[worker.threadId] = worker;
+        this.#workers[worker.threadId]._status = 'waiting';
+    }
+
+    #createThreadPool() {
+        for (let i = 0; i < this.#options.threadsPoolSize; i++) {
+            this.#createThread();
+        }
+    }
+
+    #sendJobToWorker(worker, job) {
+        worker.postMessage({
+            type: 'job:data',
+            job,
+        });
+    }
+
+    #getJob() {
+        return this.#jobs_queue.getJob();
+    }
+
+    addJob(jobData) {
+        this.#jobs_queue.addJob(jobData);
+
+        let w_keys = Object.keys(this.#workers);
+        w: for (const key of w_keys) {
+            let w = this.#workers[key];
+            if (w._status === 'waiting') {
+                let job = this.#getJob();
+                this.#sendJobToWorker(w, job);
+                w._status = 'running';
+                return;
+            }
+        }
+    }
+
+    getWorkerCount() {
+        return Object.keys(this.#workers).length;
+    }
+}
+
+module.exports = JobQueueThreadPool;

package/src/WorkerThreadControl.js

@@ -0,0 +1,52 @@
+/**
+ * 
+ */
+
+const EventEmitter = require('node:events');
+const { parentPort } = require('node:worker_threads');
+
+
+
+class WorkerThreadControl extends EventEmitter {
+    constructor() {
+        super();
+    }
+
+    sendJobCompleted(job) {
+        parentPort.postMessage({
+            type: 'job:completed',
+            job,
+        });
+    }
+
+    sendJobFailed(job, error) {
+        parentPort.postMessage({
+            type: 'job:failed',
+            job,
+            error,
+        });
+    }
+}
+
+let wtc = null;
+
+
+module.exports = function () {
+    if (wtc) {
+        return wtc;
+    }
+    wtc = new WorkerThreadControl();
+
+    parentPort.on('message', (msg) => {
+        switch (msg.type) {
+            case 'job:data':
+                wtc.emit('data', msg.job);
+                break;
+
+            default:
+                break;
+        }
+
+    });
+    return wtc;
+};

package/src/index.js

@@ -0,0 +1,10 @@
+/**
+ * 
+ */
+const JobQueueThreadPool = require('./JobQueueThreadPool');
+const WorkerThreadControl = require('./WorkerThreadControl');
+
+module.exports = {
+    JobQueueThreadPool,
+    WorkerThreadControl,
+};

package/src/tools/JobQueueTool.js

@@ -0,0 +1,23 @@
+/**
+ * 
+ */
+const crypto = require('crypto');
+
+function md5(content) {
+    const hash = crypto.createHash('md5'); // 创建一个MD5哈希对象
+    hash.update(content); // 对输入文本进行更新
+    return hash.digest('hex'); // 返回十六进制格式的哈希值
+}
+
+function makeJob(jobData) {
+    let str = JSON.stringify(jobData);
+    let jobId = md5(str);
+    return {
+        id: jobId,
+        data: jobData,
+    };
+}
+
+module.exports = {
+    makeJob,
+};