agenda 4.3.0 → 5.0.0

package/lib/agenda/index.ts

@@ -0,0 +1,227 @@
+import { EventEmitter } from "events";
+import humanInterval from "human-interval";
+import {
+  AnyError,
+  Collection,
+  Db as MongoDb,
+  MongoClient,
+  MongoClientOptions,
+} from "mongodb";
+import { Job } from "../job";
+import { cancel } from "./cancel";
+import { close } from "./close";
+import { create } from "./create";
+import { database } from "./database";
+import { dbInit } from "./db-init";
+import { defaultConcurrency } from "./default-concurrency";
+import { defaultLockLifetime } from "./default-lock-lifetime";
+import { defaultLockLimit } from "./default-lock-limit";
+import { define } from "./define";
+import { disable } from "./disable";
+import { enable } from "./enable";
+import { every } from "./every";
+import { findAndLockNextJob } from "./find-and-lock-next-job";
+import { JobProcessingQueue } from "./job-processing-queue";
+import { jobs } from "./jobs";
+import { lockLimit } from "./lock-limit";
+import { maxConcurrency } from "./max-concurrency";
+import { mongo } from "./mongo";
+import { name } from "./name";
+import { now } from "./now";
+import { processEvery } from "./process-every";
+import { purge } from "./purge";
+import { saveJob } from "./save-job";
+import { schedule } from "./schedule";
+import { sort } from "./sort";
+import { start } from "./start";
+import { stop } from "./stop";
+import { drain } from './drain';
+
+export interface AgendaConfig {
+  name?: string;
+  processEvery?: string;
+  maxConcurrency?: number;
+  defaultConcurrency?: number;
+  lockLimit?: number;
+  defaultLockLimit?: number;
+  defaultLockLifetime?: number;
+  sort?: any;
+  mongo?: MongoDb;
+  db?: {
+    address: string;
+    collection?: string;
+    options?: MongoClientOptions;
+  };
+  disableAutoIndex?: boolean;
+}
+
+/**
+ * @class Agenda
+ * @param {Object} config - Agenda Config
+ * @param {Function} cb - Callback after Agenda has started and connected to mongo
+ * @property {Object} _name - Name of the current Agenda queue
+ * @property {Number} _processEvery
+ * @property {Number} _defaultConcurrency
+ * @property {Number} _maxConcurrency
+ * @property {Number} _defaultLockLimit
+ * @property {Number} _lockLimit
+ * @property {Object} _definitions
+ * @property {Object} _runningJobs
+ * @property {Object} _lockedJobs
+ * @property {Object} _jobQueue
+ * @property {Number} _defaultLockLifetime
+ * @property {Object} _sort
+ * @property {Object} _indices
+ * @property {Boolean} _isLockingOnTheFly - true if 'lockingOnTheFly' is currently running. Prevent concurrent execution of this method.
+ * @property {Map} _isJobQueueFilling - A map of jobQueues and if the 'jobQueueFilling' method is currently running for a given map. 'lockingOnTheFly' and 'jobQueueFilling' should not run concurrently for the same jobQueue. It can cause that lock limits aren't honored.
+ * @property {Array} _jobsToLock
+ */
+class Agenda extends EventEmitter {
+  _defaultConcurrency: any;
+  _defaultLockLifetime: any;
+  _defaultLockLimit: any;
+  _definitions: any;
+  _findAndLockNextJob = findAndLockNextJob;
+  _indices: any;
+  _disableAutoIndex: boolean;
+  _isLockingOnTheFly: boolean;
+  _isJobQueueFilling: Map<string, boolean>;
+  _jobQueue: JobProcessingQueue;
+  _jobsToLock: Job[];
+  _lockedJobs: Job[];
+  _runningJobs: Job[];
+  _lockLimit: any;
+  _maxConcurrency: any;
+  _mongoUseUnifiedTopology?: boolean;
+  _name: any;
+  _processEvery: number;
+  _ready: Promise<unknown>;
+  _sort: any;
+  _db!: MongoClient;
+  _mdb!: MongoDb;
+  _collection!: Collection;
+  _nextScanAt: any;
+  _processInterval: any;
+
+  cancel!: typeof cancel;
+  close!: typeof close;
+  create!: typeof create;
+  database!: typeof database;
+  db_init!: typeof dbInit;
+  defaultConcurrency!: typeof defaultConcurrency;
+  defaultLockLifetime!: typeof defaultLockLifetime;
+  defaultLockLimit!: typeof defaultLockLimit;
+  define!: typeof define;
+  disable!: typeof disable;
+  enable!: typeof enable;
+  every!: typeof every;
+  jobs!: typeof jobs;
+  lockLimit!: typeof lockLimit;
+  maxConcurrency!: typeof maxConcurrency;
+  mongo!: typeof mongo;
+  name!: typeof name;
+  now!: typeof now;
+  processEvery!: typeof processEvery;
+  purge!: typeof purge;
+  saveJob!: typeof saveJob;
+  schedule!: typeof schedule;
+  sort!: typeof sort;
+  start!: typeof start;
+  stop!: typeof stop;
+  drain!: typeof drain;
+
+  /**
+   * Constructs a new Agenda object.
+   * @param config Optional configuration to initialize the Agenda.
+   * @param cb Optional callback called with the MongoDB collection.
+   */
+  constructor(
+    config: AgendaConfig = {},
+    cb?: (
+      error: AnyError | undefined,
+      collection: Collection<any> | null
+    ) => void
+  ) {
+    super();
+
+    this._name = config.name;
+    this._processEvery = (humanInterval(config.processEvery) ??
+      humanInterval("5 seconds")) as number; // eslint-disable-line @typescript-eslint/non-nullable-type-assertion-style
+    this._defaultConcurrency = config.defaultConcurrency || 5;
+    this._maxConcurrency = config.maxConcurrency || 20;
+    this._defaultLockLimit = config.defaultLockLimit || 0;
+    this._lockLimit = config.lockLimit || 0;
+    this._definitions = {};
+    this._runningJobs = [];
+    this._lockedJobs = [];
+    this._jobQueue = new JobProcessingQueue();
+    this._defaultLockLifetime = config.defaultLockLifetime || 10 * 60 * 1000; // 10 minute default lockLifetime
+    this._sort = config.sort || { nextRunAt: 1, priority: -1 };
+    this._indices = {
+      name: 1,
+      ...this._sort,
+      priority: -1,
+      lockedAt: 1,
+      nextRunAt: 1,
+      disabled: 1,
+    };
+    this._disableAutoIndex = config.disableAutoIndex === true;
+
+    this._isLockingOnTheFly = false;
+    this._isJobQueueFilling = new Map<string, boolean>();
+    this._jobsToLock = [];
+    this._ready = new Promise((resolve) => {
+      this.once("ready", resolve);
+    });
+
+    if (config.mongo) {
+      this.mongo(
+        config.mongo,
+        config.db ? config.db.collection : undefined,
+        cb
+      ); // @ts-expect-error // the documentation shows it should be correct: http://mongodb.github.io/node-mongodb-native/3.6/api/Db.html
+      if (config.mongo.s && config.mongo.topology && config.mongo.topology.s) {
+        this._mongoUseUnifiedTopology = Boolean(
+          // @ts-expect-error
+          config.mongo.topology.s.options.useUnifiedTopology
+        );
+      }
+    } else if (config.db) {
+      this.database(
+        config.db.address,
+        config.db.collection,
+        config.db.options,
+        cb
+      );
+    }
+  }
+}
+
+Agenda.prototype.cancel = cancel;
+Agenda.prototype.close = close;
+Agenda.prototype.create = create;
+Agenda.prototype.database = database;
+Agenda.prototype.db_init = dbInit;
+Agenda.prototype.defaultConcurrency = defaultConcurrency;
+Agenda.prototype.defaultLockLifetime = defaultLockLifetime;
+Agenda.prototype.defaultLockLimit = defaultLockLimit;
+Agenda.prototype.define = define;
+Agenda.prototype.disable = disable;
+Agenda.prototype.enable = enable;
+Agenda.prototype.every = every;
+Agenda.prototype.jobs = jobs;
+Agenda.prototype.lockLimit = lockLimit;
+Agenda.prototype.maxConcurrency = maxConcurrency;
+Agenda.prototype.mongo = mongo;
+Agenda.prototype.name = name;
+Agenda.prototype.now = now;
+Agenda.prototype.processEvery = processEvery;
+Agenda.prototype.purge = purge;
+Agenda.prototype.saveJob = saveJob;
+Agenda.prototype.schedule = schedule;
+Agenda.prototype.sort = sort;
+Agenda.prototype.start = start;
+Agenda.prototype.stop = stop;
+Agenda.prototype.drain = drain;
+
+export { Agenda };

package/lib/agenda/job-processing-queue.ts

@@ -0,0 +1,99 @@
+import { Job } from "../job";
+
+/**
+ * @class
+ * @param {Object} args - Job Options
+ * @property {Object} agenda - The Agenda instance
+ * @property {Object} attrs
+ */
+class JobProcessingQueue {
+  pop!: () => Job | undefined;
+  push!: (job: Job) => void;
+  insert!: (job: Job) => void;
+  returnNextConcurrencyFreeJob!: (agendaDefinitions: any) => Job;
+
+  protected _queue: Job[];
+
+  constructor() {
+    this._queue = [];
+  }
+
+  get length(): number {
+    return this._queue.length;
+  }
+}
+
+/**
+ * Pops and returns last queue element (next job to be processed) without checking concurrency.
+ * @returns Next Job to be processed
+ */
+JobProcessingQueue.prototype.pop = function (this: JobProcessingQueue) {
+  return this._queue.pop();
+};
+
+/**
+ * Inserts job in first queue position
+ * @param job job to add to queue
+ */
+JobProcessingQueue.prototype.push = function (
+  this: JobProcessingQueue,
+  job: Job
+) {
+  this._queue.push(job);
+};
+
+/**
+ * Inserts job in queue where it will be order from left to right in decreasing
+ * order of nextRunAt and priority (in case of same nextRunAt), if all values
+ * are even the first jobs to be introduced will have priority
+ * @param job job to add to queue
+ */
+JobProcessingQueue.prototype.insert = function (
+  this: JobProcessingQueue,
+  job: Job
+) {
+  const matchIndex = this._queue.findIndex((element) => {
+    if (element.attrs.nextRunAt!.getTime() <= job.attrs.nextRunAt!.getTime()) {
+      if (
+        element.attrs.nextRunAt!.getTime() === job.attrs.nextRunAt!.getTime()
+      ) {
+        if (element.attrs.priority >= job.attrs.priority) {
+          return true;
+        }
+      } else {
+        return true;
+      }
+    }
+
+    return false;
+  });
+
+  if (matchIndex === -1) {
+    this._queue.push(job);
+  } else {
+    this._queue.splice(matchIndex, 0, job);
+  }
+};
+
+/**
+ * Returns (does not pop, element remains in queue) first element (always from the right)
+ * that can be processed (not blocked by concurrency execution)
+ * @param agendaDefinitions job to add to queue
+ * @returns Next Job to be processed
+ */
+JobProcessingQueue.prototype.returnNextConcurrencyFreeJob = function (
+  this: JobProcessingQueue,
+  agendaDefinitions: any
+) {
+  let next;
+  for (next = this._queue.length - 1; next > 0; next -= 1) {
+    const def = agendaDefinitions[this._queue[next].attrs.name];
+    if (def.concurrency > def.running) {
+      break;
+    }
+  }
+
+  return this._queue[next];
+};
+
+export { JobProcessingQueue };

package/lib/agenda/jobs.ts

@@ -0,0 +1,31 @@
+import { Filter } from "mongodb";
+import { Agenda } from ".";
+import { Job } from "../job";
+import { createJob } from "../utils";
+
+/**
+ * Finds all jobs matching 'query'
+ * @name Agenda#jobs
+ * @function
+ * @param [query] object for MongoDB
+ * @param [sort] object for MongoDB
+ * @param [limit] number of documents to return from MongoDB
+ * @param [number] of documents to skip in MongoDB
+ * @returns resolves when fails or passes
+ */
+export const jobs = async function (
+  this: Agenda,
+  query: Filter<any> = {},
+  sort = {},
+  limit = 0,
+  skip = 0
+): Promise<Job[]> {
+  const result = await this._collection
+    .find(query) // eslint-disable-line
+    .sort(sort)
+    .limit(limit)
+    .skip(skip)
+    .toArray();
+
+  return result.map((job: any) => createJob(this, job));
+};

package/lib/agenda/lock-limit.ts

@@ -0,0 +1,17 @@
+import createDebugger from "debug";
+import { Agenda } from ".";
+
+const debug = createDebugger("agenda:locklimit");
+
+/**
+ * Set the default amount jobs that are allowed to be locked at one time (GLOBAL)
+ * @name Agenda#locklimit
+ * @function
+ * @param limit num Lock limit
+ */
+export const lockLimit = function (this: Agenda, limit: number): Agenda {
+  // @NOTE: Is this different than max concurrency?
+  debug("Agenda.lockLimit(%d)", limit);
+  this._lockLimit = limit;
+  return this;
+};

package/lib/agenda/max-concurrency.ts

@@ -0,0 +1,20 @@
+import createDebugger from "debug";
+import { Agenda } from ".";
+
+const debug = createDebugger("agenda:maxConcurrency");
+
+/**
+ * Set the concurrency for jobs (globally), type does not matter
+ * @name Agenda#maxConcurrency
+ * @function
+ * @param concurrency max concurrency value
+ * @returns agenda instance
+ */
+export const maxConcurrency = function (
+  this: Agenda,
+  concurrency: number
+): Agenda {
+  debug("Agenda.maxConcurrency(%d)", concurrency);
+  this._maxConcurrency = concurrency;
+  return this;
+};

package/lib/agenda/mongo.ts

@@ -0,0 +1,21 @@
+import { AnyError, Collection, Db } from "mongodb";
+import { Agenda } from ".";
+
+/**
+ * Build method used to add MongoDB connection details
+ * @name Agenda#mongo
+ * @function
+ * @param mdb instance of MongoClient to use
+ * @param [collection] name collection we want to use ('agendaJobs')
+ * @param [cb] called when MongoDB connection fails or passes
+ */
+export const mongo = function (
+  this: Agenda,
+  mdb: Db,
+  collection?: string,
+  cb?: (error: AnyError | undefined, collection: Collection<any> | null) => void
+): Agenda {
+  this._mdb = mdb;
+  this.db_init(collection, cb);
+  return this;
+};

package/lib/agenda/name.ts

@@ -0,0 +1,16 @@
+import createDebugger from "debug";
+import { Agenda } from ".";
+
+const debug = createDebugger("agenda:name");
+
+/**
+ * Set name of queue
+ * @name Agenda#name
+ * @function
+ * @param name name of agenda instance
+ */
+export const name = function (this: Agenda, name: string): Agenda {
+  debug("Agenda.name(%s)", name);
+  this._name = name;
+  return this;
+};

package/lib/agenda/now.ts

@@ -0,0 +1,31 @@
+import createDebugger from "debug";
+import { Agenda } from ".";
+import { Job, JobAttributesData } from "../job";
+
+const debug = createDebugger("agenda:now");
+
+/**
+ * Create a job for this exact moment
+ * @name Agenda#now
+ * @function
+ * @param name name of job to schedule
+ * @param data data to pass to job
+ */
+export const now = async function<T extends JobAttributesData> (
+  this: Agenda,
+  name: string,
+  data: T
+): Promise<Job> {
+  debug("Agenda.now(%s, [Object])", name);
+  try {
+    const job = this.create(name, data);
+
+    job.schedule(new Date());
+    await job.save();
+
+    return job;
+  } catch (error) {
+    debug("error trying to create a job for this exact moment");
+    throw error;
+  }
+};

package/lib/agenda/process-every.ts

@@ -0,0 +1,18 @@
+import createDebugger from "debug";
+import humanInterval from "human-interval";
+import { Agenda } from ".";
+
+const debug = createDebugger("agenda:processEvery");
+
+/**
+ * Set the default process interval
+ * @name Agenda#processEvery
+ * @function
+ * @param time - time to process, expressed in human interval
+ */
+export const processEvery = function (this: Agenda, time: string): Agenda {
+  debug("Agenda.processEvery(%d)", time);
+  // @ts-expect-error
+  this._processEvery = humanInterval(time);
+  return this;
+};

package/lib/agenda/purge.ts

@@ -0,0 +1,19 @@
+import createDebugger from "debug";
+import { Agenda } from ".";
+
+const debug = createDebugger("agenda:purge");
+
+/**
+ * Removes all jobs from queue
+ * @name Agenda#purge
+ * @function
+ * @returns resolved when job cancelling fails or passes
+ */
+export const purge = async function (
+  this: Agenda
+): Promise<number | undefined> {
+  // @NOTE: Only use after defining your jobs
+  const definedNames = Object.keys(this._definitions);
+  debug("Agenda.purge(%o)", definedNames);
+  return this.cancel({ name: { $not: { $in: definedNames } } });
+};

package/lib/agenda/save-job.ts

@@ -0,0 +1,185 @@
+import createDebugger from "debug";
+import { ObjectId } from "mongodb";
+import { Agenda } from ".";
+import { Job } from "../job";
+import { processJobs } from "../utils";
+
+const debug = createDebugger("agenda:saveJob");
+
+/**
+ * Given a result for findOneAndUpdate() or insert() above, determine whether to process
+ * the job immediately or to let the processJobs() interval pick it up later
+ * @param job job instance
+ * @param result the data returned from the findOneAndUpdate() call or insertOne() call
+ * @access private
+ */
+const processDbResult = async function (this: Agenda, job: Job, result: any) {
+  debug(
+    "processDbResult() called with success, checking whether to process job immediately or not"
+  );
+
+  // We have a result from the above calls
+  // findOneAndUpdate() returns different results than insertOne() so check for that
+  let resultValue = result.insertedId ? result.insertedId : result.value;
+
+  if (resultValue) {
+    let _id: ObjectId;
+    let nextRunAt: Date | null | undefined;
+
+    if (result.insertedId) {
+      _id = result.insertedId;
+      // find the doc using _id
+      const _job = await this._collection.findOne({ _id });
+
+      if (_job) {
+        nextRunAt = _job.nextRunAt;
+      }
+    } else {
+      // If it is an array, grab the first job
+      if (Array.isArray(resultValue)) {
+        resultValue = resultValue[0];
+      }
+
+      _id = resultValue._id;
+      nextRunAt = resultValue.nextRunAt;
+    }
+
+    // Grab ID and nextRunAt from MongoDB and store it as an attribute on Job
+    job.attrs._id = _id;
+    job.attrs.nextRunAt = nextRunAt;
+
+    // If the current job would have been processed in an older scan, process the job immediately
+    if (job.attrs.nextRunAt && job.attrs.nextRunAt < this._nextScanAt) {
+      debug(
+        "[%s:%s] job would have ran by nextScanAt, processing the job immediately",
+        job.attrs.name,
+        resultValue._id
+      );
+      await processJobs.call(this, job);
+    }
+  }
+
+  // Return the Job instance
+  return job;
+};
+
+/**
+ * Save the properties on a job to MongoDB
+ * @name Agenda#saveJob
+ * @function
+ * @param job job to save into MongoDB
+ * @returns resolves when job is saved or errors
+ */
+export const saveJob = async function (this: Agenda, job: Job): Promise<Job> {
+  try {
+    debug("attempting to save a job into Agenda instance");
+
+    // Grab information needed to save job but that we don't want to persist in MongoDB
+    const id = job.attrs._id;
+    const { unique, uniqueOpts } = job.attrs;
+
+    // Store job as JSON and remove props we don't want to store from object
+    const props = job.toJSON();
+    delete props._id;
+    delete props.unique;
+    delete props.uniqueOpts;
+
+    // Store name of agenda queue as last modifier in job data
+    props.lastModifiedBy = this._name;
+    debug("[job %s] set job props: \n%O", id, props);
+
+    // Grab current time and set default query options for MongoDB
+    const now = new Date();
+    const protect = {};
+    let update = { $set: props };
+    debug("current time stored as %s", now.toISOString());
+
+    // If the job already had an ID, then update the properties of the job
+    // i.e, who last modified it, etc
+    if (id) {
+      // Update the job and process the resulting data'
+      debug(
+        "job already has _id, calling findOneAndUpdate() using _id as query"
+      );
+      const result = await this._collection.findOneAndUpdate(
+        { _id: id },
+        update,
+        { returnDocument: "after" }
+      );
+      return await processDbResult.call(this, job, result);
+    }
+
+    if (props.type === "single") {
+      // Job type set to 'single' so...
+      // NOTE: Again, not sure about difference between 'single' here and 'once' in job.js
+      debug('job with type of "single" found');
+
+      // If the nextRunAt time is older than the current time, "protect" that property, meaning, don't change
+      // a scheduled job's next run time!
+      if (props.nextRunAt && props.nextRunAt <= now) {
+        debug(
+          "job has a scheduled nextRunAt time, protecting that field from upsert"
+        );
+        // @ts-expect-error
+        protect.nextRunAt = props.nextRunAt;
+        delete props.nextRunAt;
+      }
+
+      // If we have things to protect, set them in MongoDB using $setOnInsert
+      if (Object.keys(protect).length > 0) {
+        // @ts-expect-error
+        update.$setOnInsert = protect;
+      }
+
+      // Try an upsert
+      // NOTE: 'single' again, not exactly sure what it means
+      debug(
+        'calling findOneAndUpdate() with job name and type of "single" as query'
+      );
+      const result = await this._collection.findOneAndUpdate(
+        {
+          name: props.name,
+          type: "single",
+        },
+        update,
+        {
+          upsert: true,
+          returnDocument: "after",
+        }
+      );
+      return await processDbResult.call(this, job, result);
+    }
+
+    if (unique) {
+      // If we want the job to be unique, then we can upsert based on the 'unique' query object that was passed in
+      const query = job.attrs.unique;
+      query.name = props.name;
+      if (uniqueOpts?.insertOnly) {
+        // @ts-expect-error
+        update = { $setOnInsert: props };
+      }
+
+      // Use the 'unique' query object to find an existing job or create a new one
+      debug(
+        "calling findOneAndUpdate() with unique object as query: \n%O",
+        query
+      );
+      const result = await this._collection.findOneAndUpdate(query, update, {
+        upsert: true,
+        returnDocument: "after",
+      });
+      return await processDbResult.call(this, job, result);
+    }
+
+    // If all else fails, the job does not exist yet so we just insert it into MongoDB
+    debug(
+      "using default behavior, inserting new job via insertOne() with props that were set: \n%O",
+      props
+    );
+    const result = await this._collection.insertOne(props);
+    return await processDbResult.call(this, job, result);
+  } catch (error) {
+    debug("processDbResult() received an error, job was not updated/created");
+    throw error;
+  }
+};