@rpcbase/server 0.174.0 → 0.176.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rpcbase/server",
-  "version": "0.174.0",
+  "version": "0.176.0",
   "license": "SSPL-1.0",
   "main": "./index.js",
   "bin": {

package/queue/dispatch_queue.js

@@ -18,7 +18,11 @@ const dispatch_queue = (queue, model_name, op, doc) => {
 
   // console.log("handler_name", handler_name)
 
-  queue.add(handler_name, {doc})
+  queue.add(handler_name, {doc}, {
+    jobId: `${op}-${doc._id}`,
+    removeOnComplete: true,
+    removeOnFail: true,
+  })
 }
 
 

package/queue/index.js

@@ -5,13 +5,22 @@ const {WORKER_QUEUE_PORT, CONTAINER_MODE} = process.env
 const hostname = CONTAINER_MODE === "native" ? "127.0.0.1" : "worker-queue"
 const worker_queue_url = `redis://${hostname}:${WORKER_QUEUE_PORT}`
 
+const QUEUE_NAME = "worker-queue"
+
+
 const tasks_list = Object.create(null)
 
 let worker_queue
 
+const get_instance = () => {
+  if (!worker_queue) worker_queue = new Queue(QUEUE_NAME, worker_queue_url)
+  return worker_queue
+}
+
+
 const worker_add = async(task_name, payload, options) => {
   if (!worker_queue) {
-    worker_queue = new Queue("worker-queue", worker_queue_url)
+    worker_queue = new Queue(QUEUE_NAME, worker_queue_url)
   }
   const res = await worker_queue.add({task_name, payload}, options)
   console.log("worker_add:created task:", task_name, res.id)
@@ -20,6 +29,10 @@ const worker_add = async(task_name, payload, options) => {
   return res
 }
 
+const get_job = async(job_id) => {
+  const queue = get_instance()
+  return queue.getJob(job_id)
+}
 
 const register_task = (name, fn) => {
   // append module to tasks list
@@ -31,7 +44,7 @@ const get_tasks = () => tasks_list
 const start_worker = async() => {
   console.log("start worker queue", worker_queue_url)
 
-  worker_queue = new Queue("worker-queue", worker_queue_url)
+  worker_queue = new Queue(QUEUE_NAME, worker_queue_url)
 
   // error handler
   worker_queue.on("error", (err) => {
@@ -48,7 +61,9 @@ const start_worker = async() => {
   })
 
   // processor function
-  worker_queue.process(async(job) => {
+  // default concurrency is 2, in practice we will implement this with multiple workers
+  // to take advantage of multi cpu machines
+  worker_queue.process(2, async(job) => {
     try {
       const {task_name, payload} = job.data
 
@@ -67,11 +82,6 @@ const start_worker = async() => {
 }
 
 
-const get_instance = () => {
-  if (!worker_queue) worker_queue = new Queue("worker-queue", worker_queue_url)
-  return worker_queue
-}
-
 // TMP used to create additional queues, should be implemented with a better API in the future
 const get_url = () => {
   return worker_queue_url
@@ -81,6 +91,7 @@ module.exports = {
   start: start_worker,
   register_task,
   get_tasks,
+  get_job,
   add: worker_add,
   instance: get_instance,
   get_url,