bullmq 4.0.0-beta.1 → 4.0.0

package/src/classes/worker.ts

@@ -1,288 +0,0 @@
-import * as Bluebird from 'bluebird';
-import fs from 'fs';
-import path from 'path';
-import { Processor, WorkerOptions } from '../interfaces';
-import { QueueBase, Repeat } from './';
-import { ChildPool, pool } from './child-pool';
-import { Job } from './job';
-import sandbox from './sandbox';
-import { Scripts } from './scripts';
-
-// note: sandboxed processors would also like to define concurrency per process
-// for better resource utilization.
-
-export const clientCommandMessageReg = /ERR unknown command '\s*client\s*'/;
-
-export class Worker extends QueueBase {
-  private drained: boolean;
-  private waiting = false;
-  private processFn: Processor;
-
-  private resumeWorker: () => void;
-  private paused: Promise<void>;
-  private repeat: Repeat;
-  private childPool: ChildPool;
-  public opts: WorkerOptions;
-
-  private processing: Set<Promise<Job | void>>; // { [index: number]: Promise<Job | void> } = {};
-  constructor(
-    name: string,
-    processor: string | Processor,
-    opts: WorkerOptions = {},
-  ) {
-    super(name, opts);
-
-    this.opts = {
-      // settings: {},
-      drainDelay: 5,
-      concurrency: 1,
-      ...this.opts,
-    };
-
-    if (typeof processor === 'function') {
-      this.processFn = processor;
-    } else {
-      // SANDBOXED
-      const supportedFileTypes = ['.js', '.ts', '.flow'];
-      const processorFile =
-        processor +
-        (supportedFileTypes.includes(path.extname(processor)) ? '' : '.js');
-
-      if (!fs.existsSync(processorFile)) {
-        // TODO are we forced to use sync api here?
-        throw new Error(`File ${processorFile} does not exist`);
-      }
-
-      this.childPool = this.childPool || pool;
-      this.processFn = sandbox(processor, this.childPool).bind(this);
-    }
-
-    this.repeat = new Repeat(name, opts);
-
-    /* tslint:disable: no-floating-promises */
-    this.run();
-  }
-
-  private async run() {
-    await this.waitUntilReady();
-
-    // IDEA, How to store metadata associated to a worker.
-    // create a key from the worker ID associated to the given name.
-    // We keep a hash table bull:myqueue:workers where every worker is a hash key workername:workerId with json holding
-    // metadata of the worker. The worker key gets expired every 30 seconds or so, we renew the worker metadata.
-    //
-    try {
-      await this.client.client('setname', this.clientName());
-    } catch (err) {
-      if (!clientCommandMessageReg.test(err.message)) {
-        throw err;
-      }
-    }
-
-    const opts: WorkerOptions = <WorkerOptions>this.opts;
-
-    const processing = (this.processing = new Set());
-
-    while (!this.closing) {
-      if (processing.size < opts.concurrency) {
-        processing.add(this.getNextJob());
-      }
-
-      //
-      // Get the first promise that completes
-      //
-      const [completed] = await Promise.race(
-        [...processing].map(p => p.then(res => [p])),
-      );
-
-      processing.delete(completed);
-
-      const job = await completed;
-
-      if (job) {
-        processing.add(this.processJob(job));
-      }
-    }
-    return Promise.all(processing);
-  }
-
-  /**
-    Returns a promise that resolves to the next job in queue.
-  */
-  async getNextJob() {
-    if (this.paused) {
-      await this.paused;
-    }
-
-    if (this.closing) {
-      return;
-    }
-
-    if (this.drained) {
-      try {
-        const jobId = await this.waitForJob();
-
-        if (jobId) {
-          return this.moveToActive(jobId);
-        }
-      } catch (err) {
-        // Swallow error
-        if (err.message !== 'Connection is closed.') {
-          console.error('BRPOPLPUSH', err);
-        }
-      }
-    } else {
-      return this.moveToActive();
-    }
-  }
-
-  private async moveToActive(jobId?: string) {
-    const [jobData, id] = await Scripts.moveToActive(this, jobId);
-    return this.nextJobFromJobData(jobData, id);
-  }
-
-  private async waitForJob() {
-    let jobId;
-    const opts: WorkerOptions = <WorkerOptions>this.opts;
-
-    try {
-      this.waiting = true;
-      jobId = await this.client.brpoplpush(
-        this.keys.wait,
-        this.keys.active,
-        opts.drainDelay,
-      );
-    } finally {
-      this.waiting = false;
-    }
-    return jobId;
-  }
-
-  private async nextJobFromJobData(jobData: any, jobId: string) {
-    if (jobData) {
-      this.drained = false;
-      const job = Job.fromJSON(this, jobData, jobId);
-      if (job.opts.repeat) {
-        await this.repeat.addNextRepeatableJob(job.name, job.data, job.opts);
-      }
-      return job;
-    } else if (!this.drained) {
-      this.emit('drained');
-      this.drained = true;
-    }
-  }
-
-  async processJob(job: Job) {
-    if (!job || this.closing || this.paused) {
-      return;
-    }
-    const handleCompleted = async (result: any) => {
-      const jobData = await job.moveToCompleted(
-        result,
-        !(this.closing || this.paused),
-      );
-      this.emit('completed', job, result, 'active');
-      return jobData ? this.nextJobFromJobData(jobData[0], jobData[1]) : null;
-    };
-
-    const handleFailed = async (err: Error) => {
-      let error = err;
-      if (
-        error instanceof Bluebird.OperationalError &&
-        (<any>error).cause instanceof Error
-      ) {
-        error = (<any>error).cause; // Handle explicit rejection
-      }
-
-      await job.moveToFailed(err);
-      this.emit('failed', job, error, 'active');
-    };
-
-    // TODO: how to cancel the processing? (null -> job.cancel() => throw CancelError()void)
-    this.emit('active', job, null, 'waiting');
-
-    try {
-      const result = await this.processFn(job);
-      const nextJob = await handleCompleted(result);
-      return nextJob;
-    } catch (err) {
-      return handleFailed(err);
-    }
-
-    /*
-      var timeoutMs = job.opts.timeout;
-
-      if (timeoutMs) {
-        jobPromise = jobPromise.timeout(timeoutMs);
-      }
-    */
-    // Local event with jobPromise so that we can cancel job.
-    // this.emit('active', job, jobPromise, 'waiting');
-
-    // return jobPromise.then(handleCompleted).catch(handleFailed);
-  }
-
-  /**
-    Pauses the processing of this queue only for this worker.
-  */
-  async pause(doNotWaitActive?: boolean) {
-    if (!this.paused) {
-      this.paused = new Promise(resolve => {
-        this.resumeWorker = function() {
-          resolve();
-          this.paused = null; // Allow pause to be checked externally for paused state.
-          this.resumeWorker = null;
-        };
-      });
-      await (!doNotWaitActive && this.whenCurrentJobsFinished());
-      this.emit('paused');
-    }
-  }
-
-  resume() {
-    if (this.resumeWorker) {
-      this.resumeWorker();
-      this.emit('resumed');
-    }
-  }
-
-  isPaused() {
-    return !!this.paused;
-  }
-
-  /**
-   * Returns a promise that resolves when active jobs are cleared
-   *
-   * @returns {Promise}
-   */
-  private async whenCurrentJobsFinished(reconnect = true) {
-    //
-    // Force reconnection of blocking connection to abort blocking redis call immediately.
-    //
-    this.waiting && (await this.client.disconnect());
-
-    // If we are disconnected, how are we going to update the completed/failed sets?
-    if (this.processing) {
-      await Promise.all(this.processing);
-    }
-
-    this.waiting && reconnect && (await this.client.connect());
-  }
-
-  async close(force = false) {
-    this.emit('closing', 'closing queue');
-    await super.close();
-
-    try {
-      await this.resume();
-      if (!force) {
-        await this.whenCurrentJobsFinished(false);
-      } else {
-        await this.disconnect();
-      }
-    } finally {
-      this.childPool && this.childPool.clean();
-    }
-    this.emit('closed');
-  }
-}

package/src/commands/addJob-8.lua

@@ -1,106 +0,0 @@
---[[
-  Adds a job to the queue by doing the following:
-    - Increases the job counter if needed.
-    - Creates a new job key with the job data.
-
-    - if delayed:
-      - computes timestamp.
-      - adds to delayed zset.
-      - Emits a global event 'delayed' if the job is delayed.
-    - if not delayed
-      - Adds the jobId to the wait/paused list in one of three ways:
-         - LIFO
-         - FIFO
-         - prioritized.
-      - Adds the job to the "added" list so that workers gets notified.
-
-    Input:
-      KEYS[1] 'wait',
-      KEYS[2] 'paused'
-      KEYS[3] 'meta'
-      KEYS[4] 'id'
-      KEYS[5] 'delayed'
-      KEYS[6] 'priority'
-      KEYS[7] events stream key
-      KEYS[8] delay stream key
-
-      ARGV[1]  key prefix,
-      ARGV[2]  custom id (will not generate one automatically)
-      ARGV[3]  name
-      ARGV[4]  data (json stringified job data)
-      ARGV[5]  opts (json stringified job opts)
-      ARGV[6]  timestamp
-      ARGV[7]  delay
-      ARGV[8]  delayedTimestamp
-      ARGV[9]  priority
-      ARGV[10] LIFO
-]]
-local jobId
-local jobIdKey
-local rcall = redis.call
-
-local jobCounter = rcall("INCR", KEYS[4])
-
-if ARGV[2] == "" then
-    jobId = jobCounter
-    jobIdKey = ARGV[1] .. jobId
-else
-    jobId = ARGV[2]
-    jobIdKey = ARGV[1] .. jobId
-    if rcall("EXISTS", jobIdKey) == 1 then
-        return jobId .. "" -- convert to string
-    end
-end
-
--- Store the job.
-rcall("HMSET", jobIdKey, "name", ARGV[3], "data", ARGV[4], "opts", ARGV[5],
-      "timestamp", ARGV[6], "delay", ARGV[7], "priority", ARGV[9])
-
--- Check if job is delayed
-local delayedTimestamp = tonumber(ARGV[8])
-if (delayedTimestamp ~= 0) then
-    local timestamp = delayedTimestamp * 0x1000 + bit.band(jobCounter, 0xfff)
-    rcall("ZADD", KEYS[5], timestamp, jobId)
-    rcall("XADD", KEYS[7], "*", "event", "delayed", "jobId", jobId, "delay",
-          delayedTimestamp)
-    rcall("XADD", KEYS[8], "*", "nextTimestamp", delayedTimestamp)
-else
-    local target
-
-    -- Whe check for the meta.paused key to decide if we are paused or not
-    -- (since an empty list and !EXISTS are not really the same)
-    local paused
-    if rcall("HEXISTS", KEYS[3], "paused") ~= 1 then
-        target = KEYS[1]
-        paused = false
-    else
-        target = KEYS[2]
-        paused = true
-    end
-
-    -- Standard or priority add
-    local priority = tonumber(ARGV[9])
-    if priority == 0 then
-        -- LIFO or FIFO
-        rcall(ARGV[10], target, jobId)
-    else
-        -- Priority add
-        rcall("ZADD", KEYS[6], priority, jobId)
-        local count = rcall("ZCOUNT", KEYS[6], 0, priority)
-
-        local len = rcall("LLEN", target)
-        local id = rcall("LINDEX", target, len - (count - 1))
-        if id then
-            rcall("LINSERT", target, "BEFORE", id, jobId)
-        else
-            rcall("RPUSH", target, jobId)
-        end
-    end
-    -- Emit waiting event
-    rcall("XADD", KEYS[7], "*", "event", "waiting", "jobId", jobId)
-end
-
-local maxEvents = rcall("HGET", KEYS[3], "opts.maxLenEvents")
-rcall("XTRIM", KEYS[7], "MAXLEN", "~", maxEvents)
-
-return jobId .. "" -- convert to string

package/src/commands/cleanJobsInSet-1.lua

@@ -1,48 +0,0 @@
---[[
-  Remove jobs from the specific set.
-
-  Input:
-    KEYS[1]  set key,
-
-    ARGV[1]  jobId
-    ARGV[2]  timestamp
-    ARGV[3]  limit the number of jobs to be removed. 0 is unlimited
-    ARGV[4]  set name, can be any of 'wait', 'active', 'paused', 'delayed', 'completed', or 'failed'
-
-    TODO: emit a "cleaned" global event.
-]]
-local command = "ZRANGE"
-local isList = false
-
-if ARGV[4] == "wait" or ARGV[4] == "active" or ARGV[4] == "paused" then
-  command = "LRANGE"
-  isList = true
-end
-
-local jobs = redis.call(command, KEYS[1], 0, -1)
-local deleted = {}
-local deletedCount = 0
-local limit = tonumber(ARGV[3])
-local jobTS
-for _, job in ipairs(jobs) do
-  if limit > 0 and deletedCount >= limit then
-    break
-  end
-
-  local jobKey = ARGV[1] .. job
-  if (redis.call("EXISTS", jobKey .. ":lock") == 0) then
-    jobTS = redis.call("HGET", jobKey, "timestamp")
-    if (not jobTS or jobTS < ARGV[2]) then
-      if isList then
-        redis.call("LREM", KEYS[1], 0, job)
-      else
-        redis.call("ZREM", KEYS[1], job)
-      end
-      redis.call("DEL", jobKey)
-      deletedCount = deletedCount + 1
-      table.insert(deleted, job)
-    end
-  end
-end
-
-return deleted

package/src/commands/index.ts

@@ -1,62 +0,0 @@
-/**
- * Load redis lua scripts.
- * The name of the script must have the following format:
- *
- * cmdName-numKeys.lua
- *
- * cmdName must be in camel case format.
- *
- * For example:
- * moveToFinish-3.lua
- *
- */
-'use strict';
-
-import IORedis from 'ioredis';
-
-const path = require('path');
-const util = require('util');
-
-const fs = require('fs');
-
-const readdir = util.promisify(fs.readdir);
-const readFile = util.promisify(fs.readFile);
-
-interface Command {
-  name: string;
-  options: {
-    numberOfKeys: number;
-    lua: string;
-  };
-}
-
-export const load = async function(client: IORedis.Redis) {
-  const scripts = await loadScripts(__dirname);
-
-  scripts.forEach((command: Command) => {
-    client.defineCommand(command.name, command.options);
-  });
-};
-
-async function loadScripts(dir: string): Promise<Command[]> {
-  const files = await readdir(dir);
-
-  const commands = await Promise.all<Command>(
-    files
-      .filter((file: string) => path.extname(file) === '.lua')
-      .map(async (file: string) => {
-        const longName = path.basename(file, '.lua');
-        const name = longName.split('-')[0];
-        const numberOfKeys = parseInt(longName.split('-')[1]);
-
-        const lua = await readFile(path.join(dir, file));
-
-        return {
-          name,
-          options: { numberOfKeys, lua: lua.toString() },
-        };
-      }),
-  );
-
-  return commands;
-}

package/src/commands/isFinished-2.lua

@@ -1,22 +0,0 @@
---[[
-  Checks if a job is finished (.i.e. is in the completed or failed set)
-
-  Input: 
-    KEYS[1] completed key
-    KEYS[2] failed key
-
-    ARGV[1] job id
-  Output:
-    0 - not finished.
-    1 - completed.
-    2 - failed.
-]]
-if redis.call("ZSCORE", KEYS[1], ARGV[1]) ~= false then
-  return 1
-end
-
-if redis.call("ZSCORE", KEYS[2], ARGV[1]) ~= false then
-  return 2
-end
-
-return 0

package/src/commands/isJobInList-1.lua

@@ -1,20 +0,0 @@
---[[
-      Checks if job is in a given list.
-
-      Input:
-        KEYS[1]
-        ARGV[1]
-
-      Output:
-        1 if element found in the list.
-]]
-local function item_in_list (list, item)
-  for _, v in pairs(list) do
-    if v == item then
-      return 1
-    end
-  end
-  return nil
-end
-local items = redis.call("LRANGE", KEYS[1] , 0, -1)
-return item_in_list(items, ARGV[1])

package/src/commands/moveStalledJobsToWait-8.lua

@@ -1,91 +0,0 @@
---[[
-  Move stalled jobs to wait.
-
-    Input:
-      KEYS[1] 'stalled' (SET)
-      KEYS[2] 'wait',   (LIST)
-      KEYS[3] 'active', (LIST)
-      KEYS[4] 'failed', (ZSET)
-      KEYS[5] 'stalled-check', (KEY)
-
-      KEYS[6] 'meta', (KEY)
-      KEYS[7] 'paused', (LIST)
-
-      KEYS[8] 'event stream' (STREAM)
-
-      ARGV[1]  Max stalled job count
-      ARGV[2]  queue.toKey('')
-      ARGV[3]  timestamp
-      ARGV[4]  max check time
-
-    Events:
-      'stalled' with stalled job id.
-]] local rcall = redis.call
-
--- Check if we need to check for stalled jobs now.
-if rcall("EXISTS", KEYS[5]) == 1 then return {{}, {}} end
-
-rcall("SET", KEYS[5], ARGV[3], "PX", ARGV[4])
-
--- Move all stalled jobs to wait
-local stalling = rcall('SMEMBERS', KEYS[1])
-local stalled = {}
-local failed = {}
-if (#stalling > 0) then
-
-    local dst
-    -- wait or paused destination
-    if rcall("HEXISTS", KEYS[6], "paused") ~= 1 then
-        dst = KEYS[2]
-    else
-        dst = KEYS[7]
-    end
-
-    rcall('DEL', KEYS[1])
-
-    local MAX_STALLED_JOB_COUNT = tonumber(ARGV[1])
-
-    -- Remove from active list
-    for i, jobId in ipairs(stalling) do
-        local jobKey = ARGV[2] .. jobId
-
-        -- Check that the lock is also missing, then we can handle this job as really stalled.
-        if (rcall("EXISTS", jobKey .. ":lock") == 0) then
-            --  Remove from the active queue.
-            local removed = rcall("LREM", KEYS[3], 1, jobId)
-
-            if (removed > 0) then
-                -- If this job has been stalled too many times, such as if it crashes the worker, then fail it.
-                local stalledCount = rcall("HINCRBY", jobKey, "stalledCounter",
-                                           1)
-                if (stalledCount > MAX_STALLED_JOB_COUNT) then
-                    rcall("ZADD", KEYS[4], ARGV[3], jobId)
-                    rcall("HSET", jobKey, "failedReason",
-                          "job stalled more than allowable limit")
-                    rcall("XADD", KEYS[8], "*", "event", "failed", "jobId",
-                          jobId, 'prev', 'active')
-                    table.insert(failed, jobId)
-                else
-                    -- Move the job back to the wait queue, to immediately be picked up by a waiting worker.
-                    rcall("RPUSH", dst, jobId)
-                    rcall("XADD", KEYS[8], "*", "event", "waiting", "jobId",
-                          jobId, 'prev', 'active')
-
-                    -- Emit the stalled event
-                    rcall("XADD", KEYS[8], "*", "event", "stalled", "jobId",
-                          jobId)
-                    table.insert(stalled, jobId)
-                end
-            end
-        end
-    end
-end
-
--- Mark potentially stalled jobs
-local active = rcall('LRANGE', KEYS[3], 0, -1)
-if (#active > 0) then rcall('SADD', KEYS[1], unpack(active)) end
-
-local maxEvents = rcall("HGET", KEYS[6], "opts.maxLenEvents")
-rcall("XTRIM", KEYS[8], "MAXLEN", "~", maxEvents)
-
-return {failed, stalled}