@abtnode/queue 1.8.20 → 1.8.22

package/lib/index.js

@@ -3,12 +3,14 @@ const uuid = require('uuid');
 const Queue = require('fastq');
 const EventEmitter = require('events');
 const tryWithTimeout = require('@abtnode/util/lib/try-with-timeout');
+const sleep = require('@abtnode/util/lib/sleep');
 const debug = require('debug')('core/queue');
 
 const JobStore = require('./store');
 const logger = require('./logger');
 
 const CANCELLED = '__CANCELLED__';
+const MIN_DELAY = 5;
 
 /**
  *
@@ -21,6 +23,7 @@ const CANCELLED = '__CANCELLED__';
  * @param {number} Options.options.maxRetries [param=1] number of max retries, default 1
  * @param {number} Options.options.maxTimeout [param=86400000] max timeout, in ms, default 86400000ms(1d)
  * @param {number} Options.options.retryDelay [param=0] retry delay, in ms, default 0ms
+ * @param {boolean} Options.options.enableScheduledJob [param=false] enable scheduled job or not, default is false
  */
 module.exports = function createQueue({ file, onJob, options = {} }) {
   if (typeof onJob !== 'function') {
@@ -38,6 +41,7 @@ module.exports = function createQueue({ file, onJob, options = {} }) {
   const concurrency = Math.max(options.concurrency || defaults.concurrency, 1);
   const maxTimeout = Math.max(options.maxTimeout || defaults.maxTimeout, 0);
   const retryDelay = Math.max(options.retryDelay || defaults.retryDelay, 0);
+  const enableScheduledJob = typeof options.enableScheduledJob === 'boolean' ? options.enableScheduledJob : false;
 
   if (typeof options.maxRetries === 'number' && options.maxRetries >= 0) {
     maxRetries = options.maxRetries;
@@ -46,6 +50,9 @@ module.exports = function createQueue({ file, onJob, options = {} }) {
   const store = new JobStore(file);
   const queueEvents = new EventEmitter();
 
+  const getJobId = (jobId, job) =>
+    jobId || (typeof options.id === 'function' ? options.id(job) : uuid.v4()) || uuid.v4();
+
   const queue = Queue(async ({ job, id, persist }, cb) => {
     if (persist) {
       try {
@@ -68,7 +75,32 @@ module.exports = function createQueue({ file, onJob, options = {} }) {
     }
   }, concurrency);
 
-  const push = (job, jobId, persist = true) => {
+  /**
+   * Push job to the queue, the old way of calling `push(job, jobId, persist)` will be deprecated
+   * @param {object} args
+   * @param {object} args.job The data of the job
+   * @param {string} args.jobId Optional, custom jobId
+   * @param {boolean} args.persist [persist=true] Persisting the job to the database
+   * @param {number} args.delay Optional, default with no delay, unit is second, for example: 10 will run after 10s
+   * @returns
+   */
+  const push = (...args) => {
+    let job;
+    let jobId;
+    let persist;
+    let delay;
+
+    if (
+      args.length === 1 &&
+      args[0] &&
+      typeof args[0] === 'object' &&
+      (args[0].job || args[0].jobId || args[0].persist)
+    ) {
+      [{ job, jobId, persist = true, delay }] = args;
+    } else {
+      [job, jobId, persist = true] = args;
+    }
+
     const jobEvents = new EventEmitter();
     const emit = (e, data) => {
       queueEvents.emit(e, data);
@@ -79,10 +111,46 @@ module.exports = function createQueue({ file, onJob, options = {} }) {
       throw new Error('Can not queue empty job');
     }
 
-    const id = jobId || (typeof options.id === 'function' ? options.id(job) : uuid.v4()) || uuid.v4();
+    const id = getJobId(jobId, job);
+
+    if (delay) {
+      if (!enableScheduledJob) {
+        throw new Error('The queue must have options.enableScheduledJob set to true to run delay jobs');
+      }
+
+      // 这里不是精确的 delay, 延迟的时间太短没有意义，所以这里限制了最小 delay
+      if (delay < MIN_DELAY) {
+        throw new Error(`minimum delay is ${MIN_DELAY}s`);
+      }
+
+      store.addJob(
+        id,
+        job,
+        (err) => {
+          if (err) {
+            logger.error('failed to add job', err);
+            emit('failed', { id, job, error: err });
+            return;
+          }
+
+          emit('queued', { id, job });
+        },
+        { delay, willRunAt: Date.now() + delay * 1000 }
+      );
+
+      jobEvents.id = id;
+
+      return jobEvents;
+    }
+
     // eslint-disable-next-line no-shadow
-    const clearJob = (id) => {
+    const clearJob = (id, cb) => {
       store.deleteJob(id, (e) => {
+        if (typeof cb === 'function') {
+          cb(e);
+          return;
+        }
+
         if (e) {
           logger.error('Failed to delete job from persistent store', { error: e, id });
         }
@@ -111,8 +179,9 @@ module.exports = function createQueue({ file, onJob, options = {} }) {
 
         if (attrs.retryCount >= maxRetries) {
           logger.info('fail job', { id });
-          emit('failed', { id, job, error: err });
-          clearJob(id);
+          clearJob(id, () => {
+            emit('failed', { id, job, error: err });
+          });
           return;
         }
 
@@ -205,6 +274,26 @@ module.exports = function createQueue({ file, onJob, options = {} }) {
     });
   });
 
+  const loop = async () => {
+    if (enableScheduledJob === true) {
+      while (true) {
+        /* eslint-disable no-await-in-loop */
+        const jobs = await store.getScheduledJobs();
+        jobs.forEach((x) => {
+          if (x.job && x._id) {
+            push(x.job, x._id, false);
+          } else {
+            logger.info('skip invalid job from db', { job: x });
+          }
+        });
+
+        await sleep(2000);
+      }
+    }
+  };
+
+  loop();
+
   return Object.assign(queueEvents, {
     store,
     push,
@@ -215,6 +304,7 @@ module.exports = function createQueue({ file, onJob, options = {} }) {
       maxRetries,
       maxTimeout,
       retryDelay,
+      enableScheduledJob,
     },
   });
 };

package/lib/store.js

@@ -27,7 +27,7 @@ class JobStore {
 
   getJobs(cb) {
     this.db
-      .cursor({})
+      .cursor({ delay: { $exists: false } })
       .sort({ createdAt: 1 })
       .exec((err, result) => {
         if (err) {
@@ -37,6 +37,13 @@ class JobStore {
       });
   }
 
+  getScheduledJobs() {
+    return this.db
+      .cursor({ delay: { $exists: true }, willRunAt: { $lte: Date.now() } })
+      .sort({ createdAt: 1 })
+      .exec();
+  }
+
   updateJob(id, updates, cb) {
     this.db.findOne({ _id: id }, (err, exist) => {
       if (err) return cb(err);
@@ -55,7 +62,7 @@ class JobStore {
         },
         (e) => {
           if (e) {
-            console.error('error updating job', err);
+            console.error('error updating job', e);
             return cb(e, null);
           }
           return cb(null, Object.assign(exist, update));

package/package.json

@@ -3,7 +3,7 @@
   "publishConfig": {
     "access": "public"
   },
-  "version": "1.8.20",
+  "version": "1.8.22",
   "description": "",
   "main": "lib/index.js",
   "files": [
@@ -19,13 +19,13 @@
   "author": "wangshijun <wangshijun2010@gmail.com> (http://github.com/wangshijun)",
   "license": "MIT",
   "dependencies": {
-    "@abtnode/db": "1.8.20",
-    "@abtnode/util": "1.8.20",
+    "@abtnode/db": "1.8.22",
+    "@abtnode/util": "1.8.22",
     "debug": "^4.3.4",
     "fastq": "^1.13.0",
     "uuid": "7.0.3"
   },
-  "gitHead": "abe47e26c9583bfe5c4969e19cd36574f436a515",
+  "gitHead": "b0b31c2a0d29de79b29591f020f577d6c3806248",
   "devDependencies": {
     "jest": "^27.5.1"
   }