@abtnode/queue 1.15.17 → 1.16.0-beta-8ee536d7

package/lib/index.js

@@ -3,12 +3,14 @@ const uuid = require('uuid');
 const Queue = require('fastq');
 const EventEmitter = require('events');
 const tryWithTimeout = require('@abtnode/util/lib/try-with-timeout');
+const sleep = require('@abtnode/util/lib/sleep');
 const debug = require('debug')('core/queue');
 
 const JobStore = require('./store');
 const logger = require('./logger');
 
 const CANCELLED = '__CANCELLED__';
+const MIN_DELAY = 5;
 
 /**
  *
@@ -21,6 +23,7 @@ const CANCELLED = '__CANCELLED__';
  * @param {number} Options.options.maxRetries [param=1] number of max retries, default 1
  * @param {number} Options.options.maxTimeout [param=86400000] max timeout, in ms, default 86400000ms(1d)
  * @param {number} Options.options.retryDelay [param=0] retry delay, in ms, default 0ms
+ * @param {boolean} Options.options.enableScheduledJob [param=false] enable scheduled job or not, default is false
  */
 module.exports = function createQueue({ file, onJob, options = {} }) {
   if (typeof onJob !== 'function') {
@@ -38,6 +41,7 @@ module.exports = function createQueue({ file, onJob, options = {} }) {
   const concurrency = Math.max(options.concurrency || defaults.concurrency, 1);
   const maxTimeout = Math.max(options.maxTimeout || defaults.maxTimeout, 0);
   const retryDelay = Math.max(options.retryDelay || defaults.retryDelay, 0);
+  const enableScheduledJob = typeof options.enableScheduledJob === 'boolean' ? options.enableScheduledJob : false;
 
   if (typeof options.maxRetries === 'number' && options.maxRetries >= 0) {
     maxRetries = options.maxRetries;
@@ -46,6 +50,9 @@ module.exports = function createQueue({ file, onJob, options = {} }) {
   const store = new JobStore(file);
   const queueEvents = new EventEmitter();
 
+  const getJobId = (jobId, job) =>
+    jobId || (typeof options.id === 'function' ? options.id(job) : uuid.v4()) || uuid.v4();
+
   const queue = Queue(async ({ job, id, persist }, cb) => {
     if (persist) {
       try {
@@ -68,7 +75,32 @@ module.exports = function createQueue({ file, onJob, options = {} }) {
     }
   }, concurrency);
 
-  const push = (job, jobId, persist = true) => {
+  /**
+   * Push job to the queue, the old way of calling `push(job, jobId, persist)` will be deprecated
+   * @param {object} args
+   * @param {object} args.job The data of the job
+   * @param {string} args.jobId Optional, custom jobId
+   * @param {boolean} args.persist [persist=true] Persisting the job to the database
+   * @param {number} args.delay Optional, default with no delay, unit is second, for example: 10 will run after 10s
+   * @returns
+   */
+  const push = (...args) => {
+    let job;
+    let jobId;
+    let persist;
+    let delay;
+
+    if (
+      args.length === 1 &&
+      args[0] &&
+      typeof args[0] === 'object' &&
+      (args[0].job || args[0].jobId || args[0].persist)
+    ) {
+      [{ job, jobId, persist = true, delay }] = args;
+    } else {
+      [job, jobId, persist = true] = args;
+    }
+
     const jobEvents = new EventEmitter();
     const emit = (e, data) => {
       queueEvents.emit(e, data);
@@ -79,11 +111,46 @@ module.exports = function createQueue({ file, onJob, options = {} }) {
       throw new Error('Can not queue empty job');
     }
 
-    const id = jobId || (typeof options.id === 'function' ? options.id(job) : uuid.v4()) || uuid.v4();
+    const id = getJobId(jobId, job);
+
+    if (delay) {
+      if (!enableScheduledJob) {
+        throw new Error('The queue must have options.enableScheduledJob set to true to run delay jobs');
+      }
+
+      // 这里不是精确的 delay, 延迟的时间太短没有意义，所以这里限制了最小 delay
+      if (delay < MIN_DELAY) {
+        throw new Error(`minimum delay is ${MIN_DELAY}s`);
+      }
+
+      store.addJob(
+        id,
+        job,
+        (err) => {
+          if (err) {
+            logger.error('failed to add job', err);
+            emit('failed', { id, job, error: err });
+            return;
+          }
+
+          emit('queued', { id, job });
+        },
+        { delay, willRunAt: Date.now() + delay * 1000 }
+      );
+
+      jobEvents.id = id;
+
+      return jobEvents;
+    }
 
     // eslint-disable-next-line no-shadow
-    const clearJob = (id) => {
+    const clearJob = (id, cb) => {
       store.deleteJob(id, (e) => {
+        if (typeof cb === 'function') {
+          cb(e);
+          return;
+        }
+
         if (e) {
           logger.error('Failed to delete job from persistent store', { error: e, id });
         }
@@ -112,8 +179,9 @@ module.exports = function createQueue({ file, onJob, options = {} }) {
 
         if (attrs.retryCount >= maxRetries) {
           logger.info('fail job', { id });
-          emit('failed', { id, job, error: err });
-          clearJob(id);
+          clearJob(id, () => {
+            emit('failed', { id, job, error: err });
+          });
           return;
         }
 
@@ -162,11 +230,22 @@ module.exports = function createQueue({ file, onJob, options = {} }) {
 
   const cancel = async (id) =>
     new Promise((resolve, reject) => {
-      store.updateJob(id, { cancelled: true }, (e, _job) => {
+      store.updateJob(id, { cancelled: true }, (e, doc) => {
         if (e) {
           return reject(e);
         }
-        return resolve(_job);
+        return resolve(doc ? doc.job : null);
+      });
+    });
+
+  const getJob = async (id) =>
+    new Promise((resolve, reject) => {
+      store.getJob(id, (e, doc) => {
+        if (e) {
+          return reject(e);
+        }
+
+        return resolve(doc ? doc.job : null);
       });
     });
 
@@ -195,15 +274,38 @@ module.exports = function createQueue({ file, onJob, options = {} }) {
     });
   });
 
+  const loop = async () => {
+    if (enableScheduledJob === true) {
+      // eslint-disable-next-line no-constant-condition
+      while (true) {
+        /* eslint-disable no-await-in-loop */
+        const jobs = await store.getScheduledJobs();
+        jobs.forEach((x) => {
+          if (x.job && x._id) {
+            push(x.job, x._id, false);
+          } else {
+            logger.info('skip invalid job from db', { job: x });
+          }
+        });
+
+        await sleep(2000);
+      }
+    }
+  };
+
+  loop();
+
   return Object.assign(queueEvents, {
     store,
     push,
+    get: getJob,
     cancel,
     options: {
       concurrency,
       maxRetries,
       maxTimeout,
       retryDelay,
+      enableScheduledJob,
     },
   });
 };

package/lib/logger.js

@@ -1,9 +1,9 @@
-/* eslint-disable no-console */
+const debug = require('debug')(require('../package.json').name);
 
 const noop = () => {};
-const isTest = process.env.NODE_ENV === 'test';
+const isTest = !!process.env.NODE_ENV && process.env.NODE_ENV.startsWith('test');
 
-module.exports = {
-  info: isTest ? noop : (...args) => console.log(...args),
-  error: isTest ? noop : (...args) => console.error(...args),
-};
+module.exports = ['error', 'warn', 'info', 'debug', 'verbose', 'silly'].reduce((acc, fn) => {
+  acc[fn] = isTest ? noop : debug;
+  return acc;
+}, {});

package/lib/store.js

@@ -1,10 +1,7 @@
 /* eslint-disable no-underscore-dangle */
 /* eslint-disable func-names */
 /* eslint-disable consistent-return */
-const DataStore =
-  process.env.NODE_ENV === 'test' || !process.env.NEDB_MULTI_PORT
-    ? require('@nedb/core')
-    : require('@nedb/multi')(Number(process.env.NEDB_MULTI_PORT));
+const { DataStore } = require('@abtnode/db/lib/base');
 
 class JobStore {
   constructor(dbPath) {
@@ -17,9 +14,9 @@ class JobStore {
 
   isCancelled(id) {
     return new Promise((resolve, reject) => {
-      this.db.findOne({ _id: id }, (e, attrs) => {
+      this.db.findOne({ _id: id }, (e, job) => {
         if (e) return reject(e);
-        return resolve(!!attrs.cancelled);
+        return resolve(!!job && !!job.cancelled);
       });
     });
   }
@@ -30,7 +27,7 @@ class JobStore {
 
   getJobs(cb) {
     this.db
-      .find({})
+      .cursor({ delay: { $exists: false } })
       .sort({ createdAt: 1 })
       .exec((err, result) => {
         if (err) {
@@ -40,6 +37,13 @@ class JobStore {
       });
   }
 
+  getScheduledJobs() {
+    return this.db
+      .cursor({ delay: { $exists: true }, willRunAt: { $lte: Date.now() } })
+      .sort({ createdAt: 1 })
+      .exec();
+  }
+
   updateJob(id, updates, cb) {
     this.db.findOne({ _id: id }, (err, exist) => {
       if (err) return cb(err);
@@ -58,7 +62,7 @@ class JobStore {
         },
         (e) => {
           if (e) {
-            console.error('error updating job', err);
+            console.error('error updating job', e);
             return cb(e, null);
           }
           return cb(null, Object.assign(exist, update));

package/package.json

@@ -3,7 +3,7 @@
   "publishConfig": {
     "access": "public"
   },
-  "version": "1.15.17",
+  "version": "1.16.0-beta-8ee536d7",
   "description": "",
   "main": "lib/index.js",
   "files": [
@@ -19,15 +19,14 @@
   "author": "wangshijun <wangshijun2010@gmail.com> (http://github.com/wangshijun)",
   "license": "MIT",
   "dependencies": {
-    "@abtnode/util": "1.15.17",
-    "@nedb/core": "^1.2.2",
-    "@nedb/multi": "^1.2.2",
-    "debug": "^4.3.2",
-    "fastq": "^1.8.0",
+    "@abtnode/db": "1.16.0-beta-8ee536d7",
+    "@abtnode/util": "1.16.0-beta-8ee536d7",
+    "debug": "^4.3.4",
+    "fastq": "^1.13.0",
     "uuid": "7.0.3"
   },
-  "gitHead": "22715c3ea74d0230f3413162a17f491614b6735a",
+  "gitHead": "57d0c45be311a5fbc1c0fffa2814b62c1a3ee34c",
   "devDependencies": {
-    "jest": "^27.3.1"
+    "jest": "^27.5.1"
   }
 }