@abtnode/queue 1.16.8-beta-186fd5aa → 1.16.8-next-d1e52353

package/lib/index.js

@@ -4,28 +4,31 @@ const Queue = require('fastq');
 const EventEmitter = require('events');
 const tryWithTimeout = require('@abtnode/util/lib/try-with-timeout');
 const sleep = require('@abtnode/util/lib/sleep');
-const debug = require('debug')('core/queue');
 
-const JobStore = require('./store');
+const NedbStore = require('./store/nedb');
 const logger = require('./logger');
 
 const CANCELLED = '__CANCELLED__';
-const MIN_DELAY = 5;
+const MIN_DELAY = process.env.NODE_ENV === 'test' ? 2 : 5;
 
 /**
  *
- * @param {Object} Options
- * @param {string} Options.file filepath of JobStore
- * @param {function} Options.onJob called on receives job
- * @param {Object} Options.options options
- * @param {string} Options.options.id id of the job
- * @param {number} Options.options.concurrency [param=1] number of concurrent jobs
- * @param {number} Options.options.maxRetries [param=1] number of max retries, default 1
- * @param {number} Options.options.maxTimeout [param=86400000] max timeout, in ms, default 86400000ms(1d)
- * @param {number} Options.options.retryDelay [param=0] retry delay, in ms, default 0ms
- * @param {boolean} Options.options.enableScheduledJob [param=false] enable scheduled job or not, default is false
+ * @param {Object} config
+ * @param {string} [config.file] filepath of JobStore
+ * @param {object} [config.store] queue store instance
+ * @param {function} config.onJob called on receives job
+ * @param {Object} [config.options] options
+ * @param {string} [config.options.id] id of the job
+ * @param {number} [config.options.concurrency] [param=1] number of concurrent jobs
+ * @param {number} [config.options.maxRetries] [param=1] number of max retries, default 1
+ * @param {number} [config.options.maxTimeout] [param=86400000] max timeout, in ms, default 86400000ms(1d)
+ * @param {number} [config.options.retryDelay] [param=0] retry delay, in ms, default 0ms
+ * @param {boolean} [config.options.enableScheduledJob] [param=false] enable scheduled job or not, default is false
  */
-module.exports = function createQueue({ file, onJob, options = {} }) {
+module.exports = function createQueue({ file, store, onJob, options = {} }) {
+  if (!file && !store) {
+    throw new Error('Either nedb file path or store instance must be provided to create a queue');
+  }
   if (typeof onJob !== 'function') {
     throw new Error('onJob must be a function to create a queue');
   }
@@ -42,13 +45,16 @@ module.exports = function createQueue({ file, onJob, options = {} }) {
   const maxTimeout = Math.max(options.maxTimeout || defaults.maxTimeout, 0);
   const retryDelay = Math.max(options.retryDelay || defaults.retryDelay, 0);
   const enableScheduledJob = typeof options.enableScheduledJob === 'boolean' ? options.enableScheduledJob : false;
+  const queueEvents = new EventEmitter();
 
   if (typeof options.maxRetries === 'number' && options.maxRetries >= 0) {
     maxRetries = options.maxRetries;
   }
 
-  const store = new JobStore(file);
-  const queueEvents = new EventEmitter();
+  if (file) {
+    // eslint-disable-next-line no-param-reassign
+    store = new NedbStore(file);
+  }
 
   const getJobId = (jobId, job) =>
     jobId || (typeof options.id === 'function' ? options.id(job) : uuid.v4()) || uuid.v4();
@@ -77,11 +83,11 @@ module.exports = function createQueue({ file, onJob, options = {} }) {
 
   /**
    * Push job to the queue, the old way of calling `push(job, jobId, persist)` will be deprecated
-   * @param {object} args
-   * @param {object} args.job The data of the job
-   * @param {string} args.jobId Optional, custom jobId
-   * @param {boolean} args.persist [persist=true] Persisting the job to the database
-   * @param {number} args.delay Optional, default with no delay, unit is second, for example: 10 will run after 10s
+   * @param {object} params
+   * @param {object} params.job The data of the job
+   * @param {string} params.jobId Optional, custom jobId
+   * @param {boolean} params.persist [persist=true] Persisting the job to the database
+   * @param {number} params.delay Optional, default with no delay, unit is second, for example: 10 will run after 10s
    * @returns
    */
   const push = (...args) => {
@@ -101,10 +107,10 @@ module.exports = function createQueue({ file, onJob, options = {} }) {
       [job, jobId, persist = true] = args;
     }
 
-    const jobEvents = new EventEmitter();
+    const events = new EventEmitter();
     const emit = (e, data) => {
       queueEvents.emit(e, data);
-      jobEvents.emit(e, data);
+      events.emit(e, data);
     };
 
     if (!job) {
@@ -123,41 +129,18 @@ module.exports = function createQueue({ file, onJob, options = {} }) {
         throw new Error(`minimum delay is ${MIN_DELAY}s`);
       }
 
-      store.addJob(
-        id,
-        job,
-        (err) => {
-          if (err) {
-            logger.error('failed to add job', err);
-            emit('failed', { id, job, error: err });
-            return;
-          }
-
-          emit('queued', { id, job });
-        },
-        { delay, willRunAt: Date.now() + delay * 1000 }
-      );
-
-      jobEvents.id = id;
+      store.addJob(id, job, { delay, willRunAt: Date.now() + delay * 1000 }).then(() => {
+        emit('queued', { id, job });
+      });
 
-      return jobEvents;
+      events.id = id;
+      return events;
     }
 
     // eslint-disable-next-line no-shadow
-    const clearJob = (id, cb) => {
-      store.deleteJob(id, (e) => {
-        if (typeof cb === 'function') {
-          cb(e);
-          return;
-        }
-
-        if (e) {
-          logger.error('Failed to delete job from persistent store', { error: e, id });
-        }
-      });
-    };
+    const clearJob = (id) => store.deleteJob(id);
 
-    const onJobComplete = (err, result) => {
+    const onJobComplete = async (err, result) => {
       if (result === CANCELLED) {
         emit('cancelled', { id, job, result });
         clearJob(id);
@@ -170,125 +153,99 @@ module.exports = function createQueue({ file, onJob, options = {} }) {
         return;
       }
 
-      store.getJob(id, (e, attrs) => {
-        if (e) {
-          logger.error('Failed to get job for retry', { error: e, id });
-          emit('failed', { id, job, error: err });
-          return;
-        }
-
-        if (attrs.retryCount >= maxRetries) {
+      try {
+        const doc = await store.getJob(id);
+        if (doc.retryCount >= maxRetries) {
           logger.info('fail job', { id });
-          clearJob(id, () => {
-            emit('failed', { id, job, error: err });
-          });
+          await clearJob(id);
+          emit('failed', { id, job, error: err });
           return;
         }
 
-        store.updateJob(id, { retryCount: attrs.retryCount + 1 }, (_e) => {
-          if (_e) {
-            logger.error('Failed to update job for retry', { error: _e, id });
-            emit('failed', { id, job, error: _e });
-            clearJob(id);
-            return;
-          }
-
-          logger.info('retry job', { id, count: attrs.retryCount + 1 });
-          setTimeout(() => {
-            queue.unshift({ id, job }, onJobComplete);
-          }, retryDelay);
-        });
-      });
+        await store.updateJob(id, { retryCount: doc.retryCount + 1 });
+        logger.info('retry job', { id, count: doc.retryCount + 1 });
+        setTimeout(() => {
+          queue.unshift({ id, job }, onJobComplete);
+        }, retryDelay);
+        // eslint-disable-next-line no-shadow
+      } catch (err) {
+        console.error(err);
+        await clearJob(id);
+        emit('failed', { id, job, error: err });
+      }
     };
 
-    if (persist) {
-      store.addJob(id, job, (err) => {
-        if (err) {
-          logger.error('failed to add job', err);
-          emit('failed', { id, job, error: err });
-          return;
-        }
-
-        setImmediate(() => {
-          emit('queued', { id, job });
-          logger.info('queue job', { id, job, persist });
-          queue.push({ id, job, persist }, onJobComplete);
-        });
-      });
-    } else {
+    const queueJob = () =>
       setImmediate(() => {
         emit('queued', { id, job, persist });
         logger.info('queue job', { id, job });
         queue.push({ id, job, persist }, onJobComplete);
       });
-    }
 
-    jobEvents.id = id;
+    if (persist) {
+      store.addJob(id, job).then(queueJob);
+    } else {
+      queueJob();
+    }
 
-    return jobEvents;
+    events.id = id;
+    return events;
   };
 
-  const cancel = async (id) =>
-    new Promise((resolve, reject) => {
-      store.updateJob(id, { cancelled: true }, (e, doc) => {
-        if (e) {
-          return reject(e);
-        }
-        return resolve(doc ? doc.job : null);
-      });
-    });
-
-  const getJob = async (id) =>
-    new Promise((resolve, reject) => {
-      store.getJob(id, (e, doc) => {
-        if (e) {
-          return reject(e);
-        }
+  const cancel = async (id) => {
+    const doc = await store.updateJob(id, { cancelled: true });
+    return doc ? doc.job : null;
+  };
 
-        return resolve(doc ? doc.job : null);
-      });
-    });
+  const getJob = async (id) => {
+    const doc = await store.getJob(id);
+    return doc ? doc.job : null;
+  };
 
   // Populate the queue on startup
-  store.db.loadDatabase((err) => {
+  store.loadDatabase(async (err) => {
     if (err) {
-      logger.error('Can not load job database from disk', { error: err });
+      logger.error('Can not load job database', { error: err });
       throw err;
     }
 
-    debug('queue database loaded', { file });
-
-    store.getJobs((e, jobs) => {
-      if (err) {
-        logger.error('Can not find old jobs from db', { error: e });
-      } else {
-        debug('queue jobs to populate', { count: jobs.length });
-        jobs.forEach((x) => {
-          if (x.job && x._id) {
-            push(x.job, x._id, false);
-          } else {
-            logger.info('skip invalid job from db', { job: x });
-          }
-        });
-      }
-    });
+    try {
+      const jobs = await store.getJobs();
+      logger.info('queue jobs to populate', { count: jobs.length });
+      jobs.forEach((x) => {
+        if (x.job && x.id) {
+          push(x.job, x.id, false);
+        } else {
+          logger.info('skip invalid job from db', { job: x });
+        }
+      });
+      // eslint-disable-next-line no-shadow
+    } catch (err) {
+      console.error(err);
+      logger.error('Can not load existing jobs', { error: err });
+    }
   });
 
   const loop = async () => {
     if (enableScheduledJob === true) {
       // eslint-disable-next-line no-constant-condition
       while (true) {
-        /* eslint-disable no-await-in-loop */
-        const jobs = await store.getScheduledJobs();
-        jobs.forEach((x) => {
-          if (x.job && x._id) {
-            push(x.job, x._id, false);
-          } else {
-            logger.info('skip invalid job from db', { job: x });
-          }
-        });
-
-        await sleep(2000);
+        try {
+          /* eslint-disable no-await-in-loop */
+          const jobs = await store.getScheduledJobs();
+          jobs.forEach((x) => {
+            if (x.job && x.id) {
+              push(x.job, x.id, false);
+            } else {
+              logger.info('skip invalid job from db', { job: x });
+            }
+          });
+        } catch (err) {
+          console.error(err);
+          logger.error('Can not load scheduled jobs', { error: err });
+        }
+
+        await sleep(MIN_DELAY / 2);
       }
     }
   };

package/lib/logger.js

@@ -1,9 +1,10 @@
 const debug = require('debug')(require('../package.json').name);
 
-const noop = () => {};
-const isTest = !!process.env.NODE_ENV && process.env.NODE_ENV.startsWith('test');
-
-module.exports = ['error', 'warn', 'info', 'debug', 'verbose', 'silly'].reduce((acc, fn) => {
-  acc[fn] = isTest ? noop : debug;
-  return acc;
-}, {});
+module.exports = {
+  error: debug,
+  warn: debug,
+  info: debug,
+  debug,
+  verbose: debug,
+  silly: debug,
+};

package/lib/store/nedb.js

@@ -0,0 +1,74 @@
+/* eslint-disable no-underscore-dangle */
+/* eslint-disable func-names */
+/* eslint-disable consistent-return */
+const { DataStore } = require('@abtnode/db/lib/base');
+
+class NedbStore {
+  constructor(dbPath) {
+    this.dbPath = dbPath;
+    this.db = new DataStore({
+      filename: this.dbPath,
+      autoload: true,
+    });
+  }
+
+  async isCancelled(id) {
+    const job = await this.db.findOne({ id });
+    return !!job && !!job.cancelled;
+  }
+
+  getJob(id) {
+    return this.db.findOne({ id });
+  }
+
+  getJobs() {
+    return this.db
+      .cursor({ delay: { $exists: false } })
+      .sort({ createdAt: 1 })
+      .exec();
+  }
+
+  getScheduledJobs() {
+    return this.db
+      .cursor({ delay: { $exists: true }, willRunAt: { $lte: Date.now() } })
+      .sort({ createdAt: 1 })
+      .exec();
+  }
+
+  async updateJob(id, updates) {
+    const job = await this.db.findOne({ id });
+    if (!job) {
+      throw new Error(`Job ${id} does not exist`);
+    }
+
+    const update = { ...updates, updatedAt: Date.now() };
+    await this.db.update({ id }, { $set: update });
+    return Object.assign(job, update);
+  }
+
+  async addJob(id, job, attrs = {}) {
+    const exist = await this.db.findOne({ id });
+    if (exist) {
+      throw new Error(`Job ${id} already exist`);
+    }
+    return this.db.insert({
+      id,
+      job,
+      retryCount: 1,
+      cancelled: false,
+      ...attrs,
+      createdAt: Date.now(),
+      updatedAt: Date.now(),
+    });
+  }
+
+  async deleteJob(id) {
+    return this.db.remove({ id });
+  }
+
+  loadDatabase(cb) {
+    this.db.loadDatabase(cb);
+  }
+}
+
+module.exports = NedbStore;

package/lib/store/sequelize.js

@@ -0,0 +1,65 @@
+class SequelizeStore {
+  constructor(db, queue) {
+    this.db = db;
+    this.queue = queue;
+  }
+
+  async isCancelled(id) {
+    const job = await this.db.findOne({ queue: this.queue, id });
+    return !!job && !!job.cancelled;
+  }
+
+  getJob(id) {
+    return this.db.findOne({ queue: this.queue, id });
+  }
+
+  getJobs() {
+    return this.db.find({ queue: this.queue, delay: { $exists: false } }, {}, { createdAt: 1 });
+  }
+
+  getScheduledJobs() {
+    return this.db.find(
+      { queue: this.queue, delay: { $exists: true }, willRunAt: { $lte: Date.now() } },
+      {},
+      { createdAt: 1 }
+    );
+  }
+
+  async updateJob(id, updates) {
+    const job = await this.db.findOne({ queue: this.queue, id });
+    if (!job) {
+      throw new Error(`Job ${id} does not exist`);
+    }
+
+    const update = { ...updates, updatedAt: Date.now() };
+    await this.db.update({ queue: this.queue, id }, { $set: update });
+    return Object.assign(job, update);
+  }
+
+  async addJob(id, job, attrs = {}) {
+    const exist = await this.db.findOne({ queue: this.queue, id });
+    if (exist) {
+      throw new Error(`Job ${id} already exist`);
+    }
+    return this.db.insert({
+      id,
+      job,
+      queue: this.queue,
+      retryCount: 1,
+      cancelled: false,
+      ...attrs,
+      createdAt: Date.now(),
+      updatedAt: Date.now(),
+    });
+  }
+
+  async deleteJob(id) {
+    return this.db.remove({ queue: this.queue, id });
+  }
+
+  loadDatabase(cb) {
+    return this.db.loadDatabase(cb);
+  }
+}
+
+module.exports = SequelizeStore;

package/package.json

@@ -3,7 +3,7 @@
   "publishConfig": {
     "access": "public"
   },
-  "version": "1.16.8-beta-186fd5aa",
+  "version": "1.16.8-next-d1e52353",
   "description": "",
   "main": "lib/index.js",
   "files": [
@@ -13,19 +13,19 @@
     "lint": "eslint tests lib",
     "lint:fix": "eslint --fix tests lib",
     "test": "node tools/jest.js",
-    "coverage": "npm run test -- --coverage"
+    "coverage": "npm run test -- --coverage --runInBand"
   },
   "keywords": [],
   "author": "wangshijun <wangshijun2010@gmail.com> (http://github.com/wangshijun)",
   "license": "MIT",
   "dependencies": {
-    "@abtnode/db": "1.16.8-beta-186fd5aa",
-    "@abtnode/util": "1.16.8-beta-186fd5aa",
+    "@abtnode/db": "1.16.8-next-d1e52353",
+    "@abtnode/util": "1.16.8-next-d1e52353",
     "debug": "^4.3.4",
     "fastq": "^1.13.0",
-    "uuid": "7.0.3"
+    "uuid": "^8.3.2"
   },
-  "gitHead": "3150c3a5c3e041fe7f5a3902418d9d62adee3173",
+  "gitHead": "d357376aa3df9ef789befc7f548629deecb04d96",
   "devDependencies": {
     "jest": "^27.5.1"
   }

package/lib/store.js

@@ -1,104 +0,0 @@
-/* eslint-disable no-underscore-dangle */
-/* eslint-disable func-names */
-/* eslint-disable consistent-return */
-const { DataStore } = require('@abtnode/db/lib/base');
-
-class JobStore {
-  constructor(dbPath) {
-    this.dbPath = dbPath;
-    this.db = new DataStore({
-      filename: this.dbPath,
-      autoload: true,
-    });
-  }
-
-  isCancelled(id) {
-    return new Promise((resolve, reject) => {
-      this.db.findOne({ _id: id }, (e, job) => {
-        if (e) return reject(e);
-        return resolve(!!job && !!job.cancelled);
-      });
-    });
-  }
-
-  getJob(id, cb) {
-    this.db.findOne({ _id: id }, cb);
-  }
-
-  getJobs(cb) {
-    this.db
-      .cursor({ delay: { $exists: false } })
-      .sort({ createdAt: 1 })
-      .exec((err, result) => {
-        if (err) {
-          console.error('error find jobs', err);
-        }
-        cb(err, result);
-      });
-  }
-
-  getScheduledJobs() {
-    return this.db
-      .cursor({ delay: { $exists: true }, willRunAt: { $lte: Date.now() } })
-      .sort({ createdAt: 1 })
-      .exec();
-  }
-
-  updateJob(id, updates, cb) {
-    this.db.findOne({ _id: id }, (err, exist) => {
-      if (err) return cb(err);
-      if (!exist) return cb(new Error(`Job ${id} does not exist`));
-
-      const update = {
-        ...updates,
-        updatedAt: Date.now(),
-      };
-      this.db.update(
-        {
-          _id: id,
-        },
-        {
-          $set: update,
-        },
-        (e) => {
-          if (e) {
-            console.error('error updating job', e);
-            return cb(e, null);
-          }
-          return cb(null, Object.assign(exist, update));
-        }
-      );
-    });
-  }
-
-  addJob(id, job, cb, attrs = {}) {
-    this.db.findOne({ _id: id }, (err, exist) => {
-      if (err) return cb(err);
-      if (exist) return new Error(`Job ${id} already exist`);
-      this.db.insert(
-        {
-          _id: id,
-          job,
-          retryCount: 1,
-          cancelled: false,
-          ...attrs,
-          createdAt: Date.now(),
-          updatedAt: Date.now(),
-        },
-        (e) => {
-          if (e) console.error('error inserting job', e);
-          return cb(e);
-        }
-      );
-    });
-  }
-
-  deleteJob(id, cb) {
-    this.db.findOne({ _id: id }, (err) => {
-      if (err) return cb(err);
-      this.db.remove({ _id: id }, {}, (e) => cb(e));
-    });
-  }
-}
-
-module.exports = JobStore;