agenda 4.4.0 → 5.0.0

package/lib/agenda/define.ts

@@ -0,0 +1,85 @@
+import { Agenda } from ".";
+import createDebugger from "debug";
+import { Job } from "../job";
+
+const debug = createDebugger("agenda:define");
+
+export enum JobPriority {
+  highest = 20,
+  high = 10,
+  normal = 0,
+  low = -10,
+  lowest = -20,
+}
+
+export interface DefineOptions {
+  /**
+   * Maximum number of that job that can be running at once (per instance of agenda)
+   */
+  concurrency?: number;
+
+  /**
+   * Maximum number of that job that can be locked at once (per instance of agenda)
+   */
+  lockLimit?: number;
+
+  /**
+   * Interval in ms of how long the job stays locked for (see multiple job processors for more info). A job will
+   * automatically unlock if done() is called.
+   */
+  lockLifetime?: number;
+
+  /**
+   * (lowest|low|normal|high|highest|number) specifies the priority of the job. Higher priority jobs will run
+   * first.
+   */
+  priority?: JobPriority;
+
+  /**
+   * Should the return value of the job be persisted
+   */
+  shouldSaveResult?: boolean;
+}
+
+export type Processor<T> =
+  | ((job: Job<T>) => Promise<void>)
+  | ((job: Job<T>, done: () => void) => void);
+
+/**
+ * Setup definition for job
+ * Method is used by consumers of lib to setup their functions
+ * @name Agenda#define
+ * @function
+ * @param name name of job
+ * @param options options for job to run
+ * @param [processor] function to be called to run actual job
+ */
+export const define = function<T> (
+  this: Agenda,
+  name: string,
+  options: DefineOptions | Processor<T>,
+  processor?: Processor<T>
+): void {
+  if (processor === undefined) {
+    processor = options as Processor<T>;
+    options = {};
+  }
+
+  this._definitions[name] = {
+    fn: processor,
+    concurrency:
+      (options as DefineOptions).concurrency || this._defaultConcurrency, // `null` is per interface definition of DefineOptions not valid
+    lockLimit: (options as DefineOptions).lockLimit || this._defaultLockLimit,
+    priority: (options as DefineOptions).priority || JobPriority.normal,
+    lockLifetime:
+      (options as DefineOptions).lockLifetime || this._defaultLockLifetime,
+    running: 0,
+    locked: 0,
+    shouldSaveResult: (options as DefineOptions).shouldSaveResult || false
+  };
+  debug(
+    "job [%s] defined with following options: \n%O",
+    name,
+    this._definitions[name]
+  );
+};

package/lib/agenda/disable.ts

@@ -0,0 +1,28 @@
+import createDebugger from "debug";
+import { Filter } from "mongodb";
+import { Agenda } from ".";
+const debug = createDebugger("agenda:disable");
+
+/**
+ * Disables any jobs matching the passed MongoDB query by setting the `disabled` flag to `true`
+ * @name Agenda#disable
+ * @function
+ * @param query MongoDB query to use when enabling
+ * @returns {Promise<number>} Resolved with the number of disabled job instances.
+ */
+export const disable = async function (
+  this: Agenda,
+  query: Filter<unknown> = {}
+): Promise<number> {
+  debug("attempting to disable all jobs matching query", query);
+  try {
+    const { modifiedCount } = await this._collection.updateMany(query, {
+      $set: { disabled: true },
+    });
+    debug("%s jobs disabled");
+    return modifiedCount;
+  } catch (error) {
+    debug("error trying to mark jobs as `disabled`");
+    throw error;
+  }
+};

package/lib/agenda/drain.ts

@@ -0,0 +1,30 @@
+import createDebugger from "debug";
+import { Agenda } from ".";
+
+const debug = createDebugger("agenda:drain");
+
+/**
+ * Clear the interval that processes the jobs
+ * @name Agenda#drain
+ * @function
+ * @returns resolves when all running jobs completes
+ */
+export const drain = async function (this: Agenda): Promise<void> {
+    return new Promise((resolve) => {
+        debug("Agenda.drain called, clearing interval for processJobs()");
+        clearInterval(this._processInterval);
+        this._processInterval = undefined;
+
+        if (this._runningJobs.length === 0) {
+            resolve();
+        } else {
+            debug("Agenda.drain waiting for jobs to finish");
+            this.on('complete', () => {
+                // running jobs are removed after the event
+                if (this._runningJobs.length === 1) {
+                    resolve();
+                }
+            });
+        }
+    });
+};

package/lib/agenda/enable.ts

@@ -0,0 +1,29 @@
+import createDebugger from "debug";
+import { Filter } from "mongodb";
+import { Agenda } from ".";
+const debug = createDebugger("agenda:enable");
+
+/**
+ * Enables any jobs matching the passed MongoDB query by setting the `disabled` flag to `false`
+ * @name Agenda#enable
+ * @function
+ * @param query MongoDB query to use when enabling
+ * @caller client code, Agenda.purge(), Job.remove()
+ * @returns {Promise<Number>} A promise that contains the number of removed documents when fulfilled.
+ */
+export const enable = async function (
+  this: Agenda,
+  query: Filter<unknown> = {}
+): Promise<number> {
+  debug("attempting to enable all jobs matching query", query);
+  try {
+    const { modifiedCount } = await this._collection.updateMany(query, {
+      $set: { disabled: false },
+    });
+    debug("%s jobs enabled", modifiedCount);
+    return modifiedCount;
+  } catch (error) {
+    debug("error trying to mark jobs as `enabled`");
+    throw error;
+  }
+};

package/lib/agenda/every.ts

@@ -0,0 +1,86 @@
+import createDebugger from "debug";
+import { Agenda } from ".";
+import { Job, JobAttributesData } from "../job";
+import { JobOptions } from "../job/repeat-every";
+
+const debug = createDebugger("agenda:every");
+
+/**
+ * Creates a scheduled job with given interval and name/names of the job to run
+ * @name Agenda#every
+ * @function
+ * @param interval - run every X interval
+ * @param names - String or strings of jobs to schedule
+ * @param data - data to run for job
+ * @param options - options to run job for
+ * @returns Job/s created. Resolves when schedule fails or passes
+ */
+export const every = async function<T extends JobAttributesData> (
+  this: Agenda,
+  interval: string,
+  names: string | string[],
+  data?: T,
+  options?: JobOptions
+): Promise<any> {
+  /**
+   * Internal method to setup job that gets run every interval
+   * @param interval run every X interval
+   * @param name String job to schedule
+   * @param [data] data to run for job
+   * @param [options] options to run job for
+   * @returns instance of job
+   */
+  const createJob = async<T extends JobAttributesData> (
+    interval: string,
+    name: string,
+    data?: T,
+    options?: JobOptions
+  ): Promise<Job> => {
+    const job = this.create(name, data || {});
+
+    job.attrs.type = "single";
+    job.repeatEvery(interval, options);
+    return job.save();
+  };
+
+  /**
+   * Internal helper method that uses createJob to create jobs for an array of names
+   * @param interval run every X interval
+   * @param names Strings of jobs to schedule
+   * @param [data] data to run for job
+   * @param [options] options to run job for
+   * @return array of jobs created
+   */
+  const createJobs = async (
+    interval: string,
+    names: string[],
+    data?: T,
+    options?: JobOptions
+  ): Promise<Job[] | undefined> => {
+    try {
+      const jobs: Array<Promise<Job>> = [];
+      names.map((name) => jobs.push(createJob(interval, name, data, options)));
+
+      debug("every() -> all jobs created successfully");
+
+      return Promise.all(jobs);
+    } catch (error) {
+      // @TODO: catch - ignore :O
+      debug("every() -> error creating one or more of the jobs", error);
+    }
+  };
+
+  if (typeof names === "string") {
+    debug("Agenda.every(%s, %O, %O)", interval, names, options);
+    const jobs = await createJob(interval, names, data, options);
+
+    return jobs;
+  }
+
+  if (Array.isArray(names)) {
+    debug("Agenda.every(%s, %s, %O)", interval, names, options);
+    const jobs = await createJobs(interval, names, data, options);
+
+    return jobs;
+  }
+};

package/lib/agenda/find-and-lock-next-job.ts

@@ -0,0 +1,78 @@
+import createDebugger from "debug";
+import { createJob } from "../utils";
+import { Agenda } from ".";
+import { Job } from "../job";
+import { ReturnDocument } from "mongodb";
+
+const debug = createDebugger("agenda:internal:_findAndLockNextJob");
+
+/**
+ * Find and lock jobs
+ * @name Agenda#findAndLockNextJob
+ * @function
+ * @param jobName name of job to try to lock
+ * @param definition definition used to tell how job is run
+ * @access protected
+ * @caller jobQueueFilling() only
+ */
+export const findAndLockNextJob = async function (
+  this: Agenda,
+  jobName: string,
+  definition: any
+): Promise<Job | undefined> {
+  const now = new Date();
+  const lockDeadline = new Date(Date.now().valueOf() - definition.lockLifetime);
+  debug("_findAndLockNextJob(%s, [Function])", jobName);
+
+  const JOB_PROCESS_WHERE_QUERY = {
+    $and: [
+      {
+        name: jobName,
+        disabled: { $ne: true },
+      },
+      {
+        $or: [
+          {
+            lockedAt: { $eq: null },
+            nextRunAt: { $lte: this._nextScanAt },
+          },
+          {
+            lockedAt: { $lte: lockDeadline },
+          },
+        ],
+      },
+    ],
+  };
+
+  /**
+   * Query used to set a job as locked
+   * @type {{$set: {lockedAt: Date}}}
+   */
+  const JOB_PROCESS_SET_QUERY = { $set: { lockedAt: now } };
+
+  /**
+   * Query used to affect what gets returned
+   * @type {{returnOriginal: boolean, sort: object}}
+   */
+  const JOB_RETURN_QUERY = { returnDocument: ReturnDocument.AFTER, sort: this._sort };
+
+  // Find ONE and ONLY ONE job and set the 'lockedAt' time so that job begins to be processed
+  const result = await this._collection.findOneAndUpdate(
+    JOB_PROCESS_WHERE_QUERY,
+    JOB_PROCESS_SET_QUERY,
+    JOB_RETURN_QUERY
+  );
+
+  let job: Job | undefined = undefined;
+  if (result.value) {
+    debug(
+      "found a job available to lock, creating a new job on Agenda with id [%s]",
+      result.value._id
+    );
+
+    // @ts-ignore
+    job = createJob(this, result.value);
+  }
+
+  return job;
+};

package/lib/agenda/has-mongo-protocol.ts

@@ -0,0 +1,8 @@
+/**
+ * Given a mongo connection url will check if it contains the mongo
+ * @param url URL to be tested
+ * @returns whether or not the url is a valid mongo URL
+ */
+export const hasMongoProtocol = function (url: string): boolean {
+  return /mongodb(?:\+srv)?:\/\/.*/.exec(url) !== null;
+};

package/lib/agenda/index.ts

@@ -0,0 +1,227 @@
+import { EventEmitter } from "events";
+import humanInterval from "human-interval";
+import {
+  AnyError,
+  Collection,
+  Db as MongoDb,
+  MongoClient,
+  MongoClientOptions,
+} from "mongodb";
+import { Job } from "../job";
+import { cancel } from "./cancel";
+import { close } from "./close";
+import { create } from "./create";
+import { database } from "./database";
+import { dbInit } from "./db-init";
+import { defaultConcurrency } from "./default-concurrency";
+import { defaultLockLifetime } from "./default-lock-lifetime";
+import { defaultLockLimit } from "./default-lock-limit";
+import { define } from "./define";
+import { disable } from "./disable";
+import { enable } from "./enable";
+import { every } from "./every";
+import { findAndLockNextJob } from "./find-and-lock-next-job";
+import { JobProcessingQueue } from "./job-processing-queue";
+import { jobs } from "./jobs";
+import { lockLimit } from "./lock-limit";
+import { maxConcurrency } from "./max-concurrency";
+import { mongo } from "./mongo";
+import { name } from "./name";
+import { now } from "./now";
+import { processEvery } from "./process-every";
+import { purge } from "./purge";
+import { saveJob } from "./save-job";
+import { schedule } from "./schedule";
+import { sort } from "./sort";
+import { start } from "./start";
+import { stop } from "./stop";
+import { drain } from './drain';
+
+export interface AgendaConfig {
+  name?: string;
+  processEvery?: string;
+  maxConcurrency?: number;
+  defaultConcurrency?: number;
+  lockLimit?: number;
+  defaultLockLimit?: number;
+  defaultLockLifetime?: number;
+  sort?: any;
+  mongo?: MongoDb;
+  db?: {
+    address: string;
+    collection?: string;
+    options?: MongoClientOptions;
+  };
+  disableAutoIndex?: boolean;
+}
+
+/**
+ * @class Agenda
+ * @param {Object} config - Agenda Config
+ * @param {Function} cb - Callback after Agenda has started and connected to mongo
+ * @property {Object} _name - Name of the current Agenda queue
+ * @property {Number} _processEvery
+ * @property {Number} _defaultConcurrency
+ * @property {Number} _maxConcurrency
+ * @property {Number} _defaultLockLimit
+ * @property {Number} _lockLimit
+ * @property {Object} _definitions
+ * @property {Object} _runningJobs
+ * @property {Object} _lockedJobs
+ * @property {Object} _jobQueue
+ * @property {Number} _defaultLockLifetime
+ * @property {Object} _sort
+ * @property {Object} _indices
+ * @property {Boolean} _isLockingOnTheFly - true if 'lockingOnTheFly' is currently running. Prevent concurrent execution of this method.
+ * @property {Map} _isJobQueueFilling - A map of jobQueues and if the 'jobQueueFilling' method is currently running for a given map. 'lockingOnTheFly' and 'jobQueueFilling' should not run concurrently for the same jobQueue. It can cause that lock limits aren't honored.
+ * @property {Array} _jobsToLock
+ */
+class Agenda extends EventEmitter {
+  _defaultConcurrency: any;
+  _defaultLockLifetime: any;
+  _defaultLockLimit: any;
+  _definitions: any;
+  _findAndLockNextJob = findAndLockNextJob;
+  _indices: any;
+  _disableAutoIndex: boolean;
+  _isLockingOnTheFly: boolean;
+  _isJobQueueFilling: Map<string, boolean>;
+  _jobQueue: JobProcessingQueue;
+  _jobsToLock: Job[];
+  _lockedJobs: Job[];
+  _runningJobs: Job[];
+  _lockLimit: any;
+  _maxConcurrency: any;
+  _mongoUseUnifiedTopology?: boolean;
+  _name: any;
+  _processEvery: number;
+  _ready: Promise<unknown>;
+  _sort: any;
+  _db!: MongoClient;
+  _mdb!: MongoDb;
+  _collection!: Collection;
+  _nextScanAt: any;
+  _processInterval: any;
+
+  cancel!: typeof cancel;
+  close!: typeof close;
+  create!: typeof create;
+  database!: typeof database;
+  db_init!: typeof dbInit;
+  defaultConcurrency!: typeof defaultConcurrency;
+  defaultLockLifetime!: typeof defaultLockLifetime;
+  defaultLockLimit!: typeof defaultLockLimit;
+  define!: typeof define;
+  disable!: typeof disable;
+  enable!: typeof enable;
+  every!: typeof every;
+  jobs!: typeof jobs;
+  lockLimit!: typeof lockLimit;
+  maxConcurrency!: typeof maxConcurrency;
+  mongo!: typeof mongo;
+  name!: typeof name;
+  now!: typeof now;
+  processEvery!: typeof processEvery;
+  purge!: typeof purge;
+  saveJob!: typeof saveJob;
+  schedule!: typeof schedule;
+  sort!: typeof sort;
+  start!: typeof start;
+  stop!: typeof stop;
+  drain!: typeof drain;
+
+  /**
+   * Constructs a new Agenda object.
+   * @param config Optional configuration to initialize the Agenda.
+   * @param cb Optional callback called with the MongoDB collection.
+   */
+  constructor(
+    config: AgendaConfig = {},
+    cb?: (
+      error: AnyError | undefined,
+      collection: Collection<any> | null
+    ) => void
+  ) {
+    super();
+
+    this._name = config.name;
+    this._processEvery = (humanInterval(config.processEvery) ??
+      humanInterval("5 seconds")) as number; // eslint-disable-line @typescript-eslint/non-nullable-type-assertion-style
+    this._defaultConcurrency = config.defaultConcurrency || 5;
+    this._maxConcurrency = config.maxConcurrency || 20;
+    this._defaultLockLimit = config.defaultLockLimit || 0;
+    this._lockLimit = config.lockLimit || 0;
+    this._definitions = {};
+    this._runningJobs = [];
+    this._lockedJobs = [];
+    this._jobQueue = new JobProcessingQueue();
+    this._defaultLockLifetime = config.defaultLockLifetime || 10 * 60 * 1000; // 10 minute default lockLifetime
+    this._sort = config.sort || { nextRunAt: 1, priority: -1 };
+    this._indices = {
+      name: 1,
+      ...this._sort,
+      priority: -1,
+      lockedAt: 1,
+      nextRunAt: 1,
+      disabled: 1,
+    };
+    this._disableAutoIndex = config.disableAutoIndex === true;
+
+    this._isLockingOnTheFly = false;
+    this._isJobQueueFilling = new Map<string, boolean>();
+    this._jobsToLock = [];
+    this._ready = new Promise((resolve) => {
+      this.once("ready", resolve);
+    });
+
+    if (config.mongo) {
+      this.mongo(
+        config.mongo,
+        config.db ? config.db.collection : undefined,
+        cb
+      ); // @ts-expect-error // the documentation shows it should be correct: http://mongodb.github.io/node-mongodb-native/3.6/api/Db.html
+      if (config.mongo.s && config.mongo.topology && config.mongo.topology.s) {
+        this._mongoUseUnifiedTopology = Boolean(
+          // @ts-expect-error
+          config.mongo.topology.s.options.useUnifiedTopology
+        );
+      }
+    } else if (config.db) {
+      this.database(
+        config.db.address,
+        config.db.collection,
+        config.db.options,
+        cb
+      );
+    }
+  }
+}
+
+Agenda.prototype.cancel = cancel;
+Agenda.prototype.close = close;
+Agenda.prototype.create = create;
+Agenda.prototype.database = database;
+Agenda.prototype.db_init = dbInit;
+Agenda.prototype.defaultConcurrency = defaultConcurrency;
+Agenda.prototype.defaultLockLifetime = defaultLockLifetime;
+Agenda.prototype.defaultLockLimit = defaultLockLimit;
+Agenda.prototype.define = define;
+Agenda.prototype.disable = disable;
+Agenda.prototype.enable = enable;
+Agenda.prototype.every = every;
+Agenda.prototype.jobs = jobs;
+Agenda.prototype.lockLimit = lockLimit;
+Agenda.prototype.maxConcurrency = maxConcurrency;
+Agenda.prototype.mongo = mongo;
+Agenda.prototype.name = name;
+Agenda.prototype.now = now;
+Agenda.prototype.processEvery = processEvery;
+Agenda.prototype.purge = purge;
+Agenda.prototype.saveJob = saveJob;
+Agenda.prototype.schedule = schedule;
+Agenda.prototype.sort = sort;
+Agenda.prototype.start = start;
+Agenda.prototype.stop = stop;
+Agenda.prototype.drain = drain;
+
+export { Agenda };

package/lib/agenda/job-processing-queue.ts

@@ -0,0 +1,99 @@
+import { Job } from "../job";
+
+/**
+ * @class
+ * @param {Object} args - Job Options
+ * @property {Object} agenda - The Agenda instance
+ * @property {Object} attrs
+ */
+class JobProcessingQueue {
+  pop!: () => Job | undefined;
+  push!: (job: Job) => void;
+  insert!: (job: Job) => void;
+  returnNextConcurrencyFreeJob!: (agendaDefinitions: any) => Job;
+
+  protected _queue: Job[];
+
+  constructor() {
+    this._queue = [];
+  }
+
+  get length(): number {
+    return this._queue.length;
+  }
+}
+
+/**
+ * Pops and returns last queue element (next job to be processed) without checking concurrency.
+ * @returns Next Job to be processed
+ */
+JobProcessingQueue.prototype.pop = function (this: JobProcessingQueue) {
+  return this._queue.pop();
+};
+
+/**
+ * Inserts job in first queue position
+ * @param job job to add to queue
+ */
+JobProcessingQueue.prototype.push = function (
+  this: JobProcessingQueue,
+  job: Job
+) {
+  this._queue.push(job);
+};
+
+/**
+ * Inserts job in queue where it will be order from left to right in decreasing
+ * order of nextRunAt and priority (in case of same nextRunAt), if all values
+ * are even the first jobs to be introduced will have priority
+ * @param job job to add to queue
+ */
+JobProcessingQueue.prototype.insert = function (
+  this: JobProcessingQueue,
+  job: Job
+) {
+  const matchIndex = this._queue.findIndex((element) => {
+    if (element.attrs.nextRunAt!.getTime() <= job.attrs.nextRunAt!.getTime()) {
+      if (
+        element.attrs.nextRunAt!.getTime() === job.attrs.nextRunAt!.getTime()
+      ) {
+        if (element.attrs.priority >= job.attrs.priority) {
+          return true;
+        }
+      } else {
+        return true;
+      }
+    }
+
+    return false;
+  });
+
+  if (matchIndex === -1) {
+    this._queue.push(job);
+  } else {
+    this._queue.splice(matchIndex, 0, job);
+  }
+};
+
+/**
+ * Returns (does not pop, element remains in queue) first element (always from the right)
+ * that can be processed (not blocked by concurrency execution)
+ * @param agendaDefinitions job to add to queue
+ * @returns Next Job to be processed
+ */
+JobProcessingQueue.prototype.returnNextConcurrencyFreeJob = function (
+  this: JobProcessingQueue,
+  agendaDefinitions: any
+) {
+  let next;
+  for (next = this._queue.length - 1; next > 0; next -= 1) {
+    const def = agendaDefinitions[this._queue[next].attrs.name];
+    if (def.concurrency > def.running) {
+      break;
+    }
+  }
+
+  return this._queue[next];
+};
+
+export { JobProcessingQueue };