0z2i6v3u5t 1.0.0

package/src/modules/task.ts

@@ -0,0 +1,527 @@
+import { api, config, utils, log } from "./../index";
+import { Task, TaskInputs } from "./../classes/task";
+import { ErrorPayload } from "node-resque";
+
+export namespace task {
+  /**
+ * An example middleware
+ * ```js
+ * const middleware = {
+ *   name: 'timer',
+ *   global: true,
+ *   priority: 90,
+ *   preProcessor: async function () {
+ *     const worker = this.worker
+ *     worker.startTime = process.hrtime()
+ *   },
+ *   postProcessor: async function () {
+ *     const worker = this.worker
+ *     const elapsed = process.hrtime(worker.startTime)
+ *     const seconds = elapsed[0]
+ *     const millis = elapsed[1] / 1000000
+ *     log(worker.job.class + ' done in ' + seconds + ' s and ' + millis + ' ms.', 'info')
+ *   },
+ *   preEnqueue: async function () {
+ *     return true // returning `false` will prevent the task from enqueueing
+ *   },
+ *   postEnqueue: async function () {
+ *     log("Task successfully enqueued!")
+ *   }
+ * }
+ * api.tasks.addMiddleware(middleware)
+```
+ */
+  export interface TaskMiddleware {
+    /**Unique name for the middleware. */
+    name: string;
+    /**Is this middleware applied to all tasks? */
+    global: boolean;
+    /**Module load order. Defaults to `api.config.general.defaultMiddlewarePriority`. */
+    priority?: number;
+    /**Called berore the task runs.  Has access to all params, before sanitization.  Can modify the data object for use in tasks. */
+    preProcessor?: () => boolean;
+    /**Called after the task runs.*/
+    postProcessor?: () => boolean;
+    /**Called before a task using this middleware is enqueued. */
+    preEnqueue?: () => Promise<boolean>;
+    /**Called after a task using this middleware is enqueued. */
+    postEnqueue?: () => boolean;
+  }
+
+  /**
+   * Enqueue a task to be performed in the background.
+   * Will throw an error if redis cannot be reached.
+   */
+  export async function enqueue(
+    taskName: string,
+    inputs?: TaskInputs,
+    queue: string = api.tasks.tasks[taskName].queue,
+  ) {
+    await validateInput(taskName, inputs);
+    return api.resque.queue.enqueue(queue, taskName, [inputs]);
+  }
+
+  /**
+   * Enqueue a task to be performed in the background, at a certain time in the future.
+   * Will throw an error if redis cannot be reached.
+   *
+   * Inputs:
+   * * taskName: The name of the task.
+   * * inputs: inputs to pass to the task.
+   * * queue: (Optional) Which queue/priority to run this instance of the task on.
+   * * suppressDuplicateTaskError: (optional) Suppress errors when the same task with the same arguments are double-enqueued for the same time
+   */
+  export async function enqueueAt(
+    timestamp: number,
+    taskName: string,
+    inputs?: TaskInputs,
+    queue: string = api.tasks.tasks[taskName].queue,
+    suppressDuplicateTaskError = false,
+  ) {
+    await validateInput(taskName, inputs);
+    return api.resque.queue.enqueueAt(
+      timestamp,
+      queue,
+      taskName,
+      [inputs],
+      suppressDuplicateTaskError,
+    );
+  }
+
+  /**
+   * Enqueue a task to be performed in the background, at a certain number of ms from now.
+   * Will throw an error if redis cannot be reached.
+   *
+   * Inputs:
+   * * timestamp: At what time the task is able to be run.  Does not guarantee that the task will be run at this time. (in ms)
+   * * taskName: The name of the task.
+   * * inputs: inputs to pass to the task.
+   * * queue: (Optional) Which queue/priority to run this instance of the task on.
+   * * suppressDuplicateTaskError: (optional) Suppress errors when the same task with the same arguments are double-enqueued for the same time
+   */
+  export async function enqueueIn(
+    time: number,
+    taskName: string,
+    inputs?: TaskInputs,
+    queue: string = api.tasks.tasks[taskName].queue,
+    suppressDuplicateTaskError = false,
+  ) {
+    await validateInput(taskName, inputs);
+    return api.resque.queue.enqueueIn(
+      time,
+      queue,
+      taskName,
+      [inputs],
+      suppressDuplicateTaskError,
+    );
+  }
+
+  /**
+   * Delete a previously enqueued task, which hasn't been run yet, from a queue.
+   * Will throw an error if redis cannot be reached.
+   *
+   * Inputs:
+   * * q: Which queue/priority is the task stored on?
+   * * taskName: The name of the job, likely to be the same name as a tak.
+   * * args: The arguments of the job.  Note, arguments passed to a Task initially may be modified when enqueuing.  It is best to read job properties first via `api.tasks.queued` or similar method.
+   * * count: Of the jobs that match q, taskName, and args, up to what position should we delete? (Default 0; this command is 0-indexed)
+   */
+  export async function del(
+    q: string,
+    taskName: string,
+    args?: TaskInputs,
+    count?: number,
+  ) {
+    return api.resque.queue.del(q, taskName, [args], count);
+  }
+
+  /**
+   * * will delete all jobs in the given queue of the named function/class
+   * * will not prevent new jobs from being added as this method is running
+   * * will not delete jobs in the delayed queues
+   *
+   * Inputs:
+   * * q: Which queue/priority is to run on?
+   * * taskName: The name of the job, likely to be the same name as a tak.
+   * * start? - starting position of task count to remove
+   * * stop? - stop position of task count to remove
+   */
+  export async function delByFunction(
+    q: string,
+    taskName: string,
+    start?: number,
+    stop?: number,
+  ) {
+    return api.resque.queue.delByFunction(q, taskName, start, stop);
+  }
+
+  /**
+   * Delete all previously enqueued tasks, which haven't been run yet, from all possible delayed timestamps.
+   * Will throw an error if redis cannot be reached.
+   *
+   * Inputs:
+   * * q: Which queue/priority is to run on?
+   * * taskName: The name of the job, likely to be the same name as a tak.
+   * * inputs  The arguments of the job.  Note, arguments passed to a Task initially may be modified when enqueuing. It is best to read job properties first via `api.tasks.delayedAt` or similar method.
+   */
+  export async function delDelayed(
+    q: string,
+    taskName: string,
+    inputs?: TaskInputs,
+  ) {
+    return api.resque.queue.delDelayed(q, taskName, [inputs]);
+  }
+
+  /**
+   * Return the timestamps a task is scheduled for.
+   * Will throw an error if redis cannot be reached.
+   *
+   * Inputs:
+   * * q: Which queue/priority is to run on?
+   * * taskName: The name of the job, likely to be the same name as a tak.
+   * * inputs: The arguments of the job.  Note, arguments passed to a Task initially may be modified when enqueuing.  It is best to read job properties first via `api.tasks.delayedAt` or similar method.
+   */
+  export async function scheduledAt(
+    q: string,
+    taskName: string,
+    inputs: TaskInputs,
+  ): Promise<Array<number>> {
+    return api.resque.queue.scheduledAt(q, taskName, [inputs]);
+  }
+
+  /**
+   * Return all resque stats for this namespace (how jobs failed, jobs succeeded, etc)
+   * Will throw an error if redis cannot be reached.
+   */
+  export async function stats() {
+    return api.resque.queue.stats();
+  }
+
+  /**
+   * Retrieve the details of jobs enqueued on a certain queue between start and stop (0-indexed)
+   * Will throw an error if redis cannot be reached.
+   *
+   * Inputs:
+   * * q      The name of the queue.
+   * * start  The index of the first job to return.
+   * * stop   The index of the last job to return.
+   */
+  export async function queued(
+    q: string,
+    start: number,
+    stop: number,
+  ): Promise<Array<TaskInputs>> {
+    return api.resque.queue.queued(q, start, stop);
+  }
+
+  /**
+   * Delete a queue in redis, and all jobs stored on it.
+   * Will throw an error if redis cannot be reached.
+   */
+  export async function delQueue(q: string) {
+    return api.resque.queue.delQueue(q);
+  }
+
+  /**
+   * Return any locks, as created by resque plugins or task middleware, in this redis namespace.
+   * Will contain locks with keys like `resque:lock:{job}` and `resque:workerslock:{workerId}`
+   * Will throw an error if redis cannot be reached.
+   */
+  export async function locks() {
+    return api.resque.queue.locks();
+  }
+
+  /**
+   * Delete a lock on a job or worker.  Locks can be found via `api.tasks.locks`
+   * Will throw an error if redis cannot be reached.
+   */
+  export async function delLock(lock: string) {
+    return api.resque.queue.delLock(lock);
+  }
+
+  /**
+   * List all timestamps for which tasks are enqueued in the future, via `api.tasks.enqueueIn` or `api.tasks.enqueueAt`
+   * Will throw an error if redis cannot be reached.
+   */
+  export async function timestamps(): Promise<Array<number>> {
+    return api.resque.queue.timestamps();
+  }
+
+  /**
+   * Return all jobs which have been enqueued to run at a certain timestamp.
+   * Will throw an error if redis cannot be reached.
+   */
+  export async function delayedAt(timestamp: number): Promise<any> {
+    return api.resque.queue.delayedAt(timestamp);
+  }
+
+  /**
+   * Return all delayed jobs, organized by the timestamp at where they are to run at.
+   * Note: This is a very slow command.
+   * Will throw an error if redis cannot be reached.
+   */
+  export async function allDelayed(): Promise<{ [timestamp: string]: any[] }> {
+    return api.resque.queue.allDelayed();
+  }
+
+  /**
+   * Return all workers registered by all members of this cluster.
+   * Note: MultiWorker processors each register as a unique worker.
+   * Will throw an error if redis cannot be reached.
+   */
+  export async function workers() {
+    return api.resque.queue.workers();
+  }
+
+  /**
+   * What is a given worker working on?  If the worker is idle, 'started' will be returned.
+   * Will throw an error if redis cannot be reached.
+   */
+  export async function workingOn(
+    workerName: string,
+    queues: string,
+  ): Promise<any> {
+    return api.resque.queue.workingOn(workerName, queues);
+  }
+
+  /**
+   * Return all workers and what job they might be working on.
+   * Will throw an error if redis cannot be reached.
+   */
+  export async function allWorkingOn() {
+    return api.resque.queue.allWorkingOn();
+  }
+
+  /**
+   * How many jobs are in the failed queue.
+   * Will throw an error if redis cannot be reached.
+   */
+  export async function failedCount(): Promise<number> {
+    return api.resque.queue.failedCount();
+  }
+
+  /**
+   * Retrieve the details of failed jobs between start and stop (0-indexed).
+   * Will throw an error if redis cannot be reached.
+   */
+  export async function failed(start: number, stop: number) {
+    return api.resque.queue.failed(start, stop);
+  }
+
+  /**
+   * Remove a specific job from the failed queue.
+   * Will throw an error if redis cannot be reached.
+   */
+  export async function removeFailed(failedJob: ErrorPayload) {
+    return api.resque.queue.removeFailed(failedJob);
+  }
+
+  /**
+   * Remove a specific job from the failed queue, and retry it by placing it back into its original queue.
+   * Will throw an error if redis cannot be reached.
+   */
+  export async function retryAndRemoveFailed(failedJob: ErrorPayload) {
+    return api.resque.queue.retryAndRemoveFailed(failedJob);
+  }
+
+  /**
+   * If a worker process crashes, it will leave its state in redis as "working".
+   * You can remove workers from redis you know to be over, by specificizing an age which would make them too old to exist.
+   * This method will remove the data created by a 'stuck' worker and move the payload to the error queue.
+   * However, it will not actually remove any processes which may be running.  A job *may* be running that you have removed.
+   * Will throw an error if redis cannot be reached.
+   */
+  export async function cleanOldWorkers(age: number) {
+    return api.resque.queue.cleanOldWorkers(age);
+  }
+
+  /**
+   * Ensures that a task which has a frequency is either running, or already enqueued.
+   * This is run automatically at boot for all tasks which have a frequency, via `api.tasks.enqueueAllRecurrentTasks`.
+   * Will throw an error if redis cannot be reached.
+   */
+  export async function enqueueRecurrentTask(taskName: string) {
+    const thisTask = api.tasks.tasks[taskName];
+
+    if (thisTask.frequency > 0) {
+      await task.del(thisTask.queue, taskName);
+      await task.delDelayed(thisTask.queue, taskName);
+      await task.enqueueIn(thisTask.frequency, taskName, {}, undefined, true);
+      log(
+        `re-enqueued recurrent job ${taskName}`,
+        config.tasks.schedulerLogging.reEnqueue,
+      );
+    }
+  }
+
+  /**
+   * This is run automatically at boot for all tasks which have a frequency, calling `api.tasks.enqueueRecurrentTask`
+   * Will throw an error if redis cannot be reached.
+   */
+  export async function enqueueAllRecurrentTasks() {
+    const enqueuedTasks: string[] = [];
+
+    for (const thisTask of Object.values(api.tasks.tasks)) {
+      if (thisTask.frequency > 0) {
+        try {
+          const toRun = await task.enqueue(thisTask.name, {});
+          if (toRun === true) {
+            log(
+              `enqueuing periodic task: ${thisTask.name}`,
+              config.tasks.schedulerLogging.enqueue,
+            );
+            enqueuedTasks.push(thisTask.name);
+          }
+        } catch (error) {
+          checkForRepeatRecurringTaskEnqueue(thisTask.name, error);
+        }
+      }
+    }
+
+    return enqueuedTasks;
+  }
+
+  /**
+   * Stop a task with a frequency by removing it from all possible queues.
+   * Will throw an error if redis cannot be reached.
+   */
+  export async function stopRecurrentTask(taskName: string): Promise<number> {
+    // find the jobs in either the normal queue or delayed queues
+    const thisTask = api.tasks.tasks[taskName];
+    if (thisTask.frequency > 0) {
+      let removedCount = 0;
+      const count = await task.del(thisTask.queue, thisTask.name, null, 1);
+      removedCount = removedCount + count;
+      const timestamps = await task.delDelayed(
+        thisTask.queue,
+        thisTask.name,
+        null,
+      );
+      removedCount = removedCount + timestamps.length;
+      return removedCount;
+    }
+  }
+
+  /**
+   * Return wholistic details about the task system, including failures, queues, and workers.
+   * Will throw an error if redis cannot be reached.
+   */
+  export async function details() {
+    const details: {
+      queues: { [key: string]: any };
+      workers: { [key: string]: any };
+      stats: { [key: string]: any };
+      leader: string;
+    } = { queues: {}, workers: {}, stats: null, leader: null };
+
+    details.workers = await task.allWorkingOn();
+    details.stats = await task.stats();
+    const queues = await api.resque.queue.queues();
+
+    for (const i in queues) {
+      const queue = queues[i];
+      const length = await api.resque.queue.length(queue);
+      details.queues[queue] = { length: length };
+    }
+
+    details.leader = await api.resque.queue.leader();
+
+    return details;
+  }
+
+  export async function addMiddleware(middleware: TaskMiddleware) {
+    if (!middleware.name) {
+      throw new Error("middleware.name is required");
+    }
+    if (!middleware.priority) {
+      middleware.priority = config.general.defaultMiddlewarePriority;
+    }
+    middleware.priority = Number(middleware.priority);
+    api.tasks.middleware[middleware.name] = middleware;
+    if (middleware.global === true) {
+      api.tasks.globalMiddleware.push(middleware.name);
+      utils.sortGlobalMiddleware(
+        api.tasks.globalMiddleware,
+        api.tasks.middleware,
+      );
+    }
+    await api.tasks.loadTasks(true);
+  }
+
+  async function validateInput(taskName: string, inputs: TaskInputs) {
+    const task: Task = api.tasks.tasks[taskName];
+
+    if (!task) {
+      throw new Error(`task ${taskName} not found`);
+    }
+
+    for (const key in task.inputs) {
+      // default
+      if (inputs[key] === undefined && task.inputs[key].default !== undefined) {
+        if (typeof task.inputs[key].default === "function") {
+          inputs[key] = await task.inputs[key].default.call(
+            api,
+            inputs[key],
+            this,
+          );
+        } else {
+          inputs[key] = task.inputs[key].default;
+        }
+      }
+
+      // validator
+      if (
+        inputs[key] !== undefined &&
+        task.inputs[key].validator !== undefined
+      ) {
+        let validatorResponse;
+        if (typeof task.inputs[key].validator === "function") {
+          // allowed to throw too
+          const method = task.inputs[key].validator as Function;
+          validatorResponse = await method.call(api, inputs[key], this);
+        } else {
+          const method = this.prepareStringMethod(task.inputs[key].validator);
+          validatorResponse = await method.call(api, inputs[key], this);
+        }
+
+        // validator function returned nothing; assume param is OK
+        if (validatorResponse === null || validatorResponse === undefined) {
+          // ok
+        }
+
+        // validator returned something that was not `true`
+        else if (validatorResponse !== true) {
+          if (validatorResponse === false) {
+            throw new Error(
+              `${inputs[key]} is not a valid value for ${key} in task ${taskName}`,
+            );
+          } else {
+            throw new Error(validatorResponse);
+          }
+        }
+      }
+
+      // required
+      if (task.inputs[key].required === true) {
+        if (config.general.missingParamChecks.indexOf(inputs[key]) >= 0) {
+          throw new Error(`${key} is a required input for task ${taskName}`);
+        }
+      }
+    }
+  }
+
+  function checkForRepeatRecurringTaskEnqueue(
+    taskName: string,
+    error: NodeJS.ErrnoException,
+  ) {
+    if (error.toString().match(/already enqueued at this time/)) {
+      // this is OK, the job was enqueued by another process as this method was running
+      log(
+        `not enqueuing periodic task ${taskName} - error.toString()`,
+        "warning",
+      );
+    } else {
+      throw error;
+    }
+  }
+}

package/src/modules/utils/argv.ts

@@ -0,0 +1,3 @@
+const yargs = require("yargs/yargs"); // cannot be partially imported
+const { hideBin } = require("yargs/helpers");
+export const argv: { [key: string]: string } = yargs(hideBin(process.argv));

package/src/modules/utils/arrayStartingMatch.ts

@@ -0,0 +1,21 @@
+/**
+ Compare the first n elements of an array with another, longer array
+*/
+export function arrayStartingMatch(a: Array<any>, b: Array<any>): boolean {
+  if (a.length === 0) {
+    return false;
+  }
+  if (b.length === 0) {
+    return false;
+  }
+
+  let matching = true;
+  let i = 0;
+  while (i < a.length) {
+    if (a[i] !== b[i]) {
+      matching = false;
+    }
+    i++;
+  }
+  return matching;
+}

package/src/modules/utils/arrayUnique.ts

@@ -0,0 +1,15 @@
+/**
+ * Return only the unique values in an Array.
+ */
+export function arrayUnique(arr: Array<any>): Array<any> {
+  const a = [];
+  for (let i = 0; i < arr.length; i++) {
+    for (let j = i + 1; j < arr.length; j++) {
+      if (arr[i] === arr[j]) {
+        j = ++i;
+      }
+    }
+    a.push(arr[i]);
+  }
+  return a;
+}

package/src/modules/utils/collapseObjectToArray.ts

@@ -0,0 +1,33 @@
+/**
+ * Collapses an Object with numerical keys (like `arguments` in a function) to an Array
+ */
+export function collapseObjectToArray(
+  obj: Record<string, any>,
+): Array<any> | boolean {
+  try {
+    const keys = Object.keys(obj);
+    if (keys.length < 1) {
+      return false;
+    }
+    if (keys[0] !== "0") {
+      return false;
+    }
+    if (keys[keys.length - 1] !== String(keys.length - 1)) {
+      return false;
+    }
+
+    const arr = [];
+    for (const i in keys) {
+      const key = keys[i];
+      if (String(parseInt(key)) !== key) {
+        return false;
+      } else {
+        arr.push(obj[key]);
+      }
+    }
+
+    return arr;
+  } catch (e) {
+    return false;
+  }
+}

package/src/modules/utils/deepCopy.ts

@@ -0,0 +1,3 @@
+export function deepCopy<T>(obj: T) {
+  return JSON.parse(JSON.stringify(obj)) as T;
+}

package/src/modules/utils/ensureNoTsHeaderOrSpecFiles.ts

@@ -0,0 +1,19 @@
+/**
+ *swap out d.ts files for the JS versions when running with ts-node
+ * also filter out *.test. and *.spec. js|ts files
+ */
+export function ensureNoTsHeaderOrSpecFiles(
+  files: Array<string>,
+): Array<string> {
+  return files.filter((f) => {
+    if (f.match(/.*\.d\.ts$/)) {
+      return false;
+    }
+
+    if (f.match(/.*\.(?:spec|test)\.[tj]s$/)) {
+      return false;
+    }
+
+    return true;
+  });
+}

package/src/modules/utils/eventLoopDelay.ts

@@ -0,0 +1,34 @@
+/**
+ * Returns the average delay between a tick of the node.js event loop, as measured for N calls of `process.nextTick`
+ */
+export async function eventLoopDelay(
+  iterations: number = 10000,
+): Promise<number> {
+  const jobs: Array<() => Promise<number>> = [];
+
+  async function sleepyFunc(): Promise<number> {
+    return new Promise((resolve) => {
+      const start = process.hrtime();
+      process.nextTick(() => {
+        const delta = process.hrtime(start);
+        const ms = delta[0] * 1000 + delta[1] / 1000000;
+        resolve(ms);
+      });
+    });
+  }
+
+  let i = 0;
+  while (i < iterations) {
+    jobs.push(sleepyFunc);
+    i++;
+  }
+
+  const results = [];
+  for (const job of jobs) results.push(await job());
+
+  let sum = 0;
+  results.forEach((t) => (sum += t));
+
+  const avg = Math.round((sum / results.length) * 10000) / 1000;
+  return avg;
+}