bullmq 4.0.0-beta.1 → 4.0.0

package/src/classes/queue-events.ts

@@ -1,78 +0,0 @@
-import { delay } from 'bluebird';
-import { QueueEventsOptions } from '../interfaces';
-import { array2obj } from '../utils';
-import { QueueBase } from './queue-base';
-
-export class QueueEvents extends QueueBase {
-  consuming: Promise<void>;
-
-  constructor(name: string, opts?: QueueEventsOptions) {
-    super(name, opts);
-
-    this.opts = Object.assign(
-      {
-        blockingTimeout: 10000,
-      },
-      this.opts,
-    );
-
-    // tslint:disable: no-floating-promises
-    this.consumeEvents().catch(err => this.emit('error'));
-  }
-
-  private async consumeEvents() {
-    await this.waitUntilReady();
-
-    const opts: QueueEventsOptions = this.opts;
-
-    const key = this.keys.events;
-    let id = opts.lastEventId || '0-0';
-
-    while (!this.closing) {
-      try {
-        const data = await this.client.xread(
-          'BLOCK',
-          opts.blockingTimeout,
-          'STREAMS',
-          key,
-          id,
-        );
-
-        if (data) {
-          const stream = data[0];
-          const events = stream[1];
-
-          for (let i = 0; i < events.length; i++) {
-            id = events[i][0];
-            const args = array2obj(events[i][1]);
-
-            //
-            // TODO: we may need to have a separate xtream for progress data
-            // to avoid this hack.
-            switch (args.event) {
-              case 'progress':
-                args.data = JSON.parse(args.data);
-                break;
-              case 'completed':
-                args.returnvalue = JSON.parse(args.returnvalue);
-                break;
-            }
-
-            this.emit(args.event, args, id);
-            this.emit(`${args.event}:${args.jobId}`, args, id);
-          }
-        }
-      } catch (err) {
-        if (err.message !== 'Connection is closed.') {
-          throw err;
-        }
-        await delay(5000);
-      }
-    }
-  }
-
-  async close() {
-    await super.close();
-    return this.disconnect();
-  }
-}

package/src/classes/queue-getters.ts

@@ -1,213 +0,0 @@
-/*eslint-env node */
-'use strict';
-
-import { QueueBase } from './queue-base';
-import { Job } from './job';
-import { clientCommandMessageReg } from './worker';
-
-export class QueueGetters extends QueueBase {
-  getJob(jobId: string) {
-    return Job.fromId(this, jobId);
-  }
-
-  private commandByType(
-    types: string[],
-    count: boolean,
-    callback: (key: string, dataType: string) => void,
-  ) {
-    return types.map((type: string) => {
-      type = type === 'waiting' ? 'wait' : type; // alias
-
-      const key = this.toKey(type);
-
-      switch (type) {
-        case 'completed':
-        case 'failed':
-        case 'delayed':
-        case 'repeat':
-          return callback(key, count ? 'zcard' : 'zrange');
-        case 'active':
-        case 'wait':
-        case 'paused':
-          return callback(key, count ? 'llen' : 'lrange');
-      }
-    });
-  }
-
-  /**
-    Returns the number of jobs waiting to be processed.
-  */
-  count() {
-    return this.getJobCountByTypes('waiting', 'paused', 'delayed');
-  }
-
-  // Job counts by type
-  // Queue#getJobCountByTypes('completed') => completed count
-  // Queue#getJobCountByTypes('completed,failed') => completed + failed count
-  // Queue#getJobCountByTypes('completed', 'failed') => completed + failed count
-  // Queue#getJobCountByTypes('completed', 'waiting', 'failed') => completed + waiting + failed count
-  async getJobCountByTypes(...types: string[]) {
-    const result = await this.getJobCounts(...types);
-    return Object.values(result).reduce((sum, count) => sum + count);
-  }
-
-  /**
-   * Returns the job counts for each type specified or every list/set in the queue by default.
-   *
-   */
-  async getJobCounts(...types: string[]) {
-    await this.waitUntilReady();
-
-    const multi = this.client.multi();
-
-    this.commandByType(types, true, function(key, command) {
-      (<any>multi)[command](key);
-    });
-
-    const res = await multi.exec();
-    const counts: { [index: string]: number } = {};
-    res.forEach((res: number[], index: number) => {
-      counts[types[index]] = res[1] || 0;
-    });
-    return counts;
-  }
-
-  getCompletedCount() {
-    return this.getJobCountByTypes('completed');
-  }
-
-  getFailedCount() {
-    return this.getJobCountByTypes('failed');
-  }
-
-  getDelayedCount() {
-    return this.getJobCountByTypes('delayed');
-  }
-
-  getActiveCount() {
-    return this.getJobCountByTypes('active');
-  }
-
-  getWaitingCount() {
-    return this.getJobCountByTypes('waiting', 'paused');
-  }
-
-  getWaiting(start = 0, end = 1) {
-    return this.getJobs(['waiting'], start, end, true);
-  }
-
-  getActive(start = 0, end = 1) {
-    return this.getJobs(['active'], start, end, true);
-  }
-
-  getDelayed(start = 0, end = 1) {
-    return this.getJobs(['delayed'], start, end, true);
-  }
-
-  getCompleted(start = 0, end = 1) {
-    return this.getJobs(['completed'], start, end, false);
-  }
-
-  getFailed(start = 0, end = 1) {
-    return this.getJobs(['failed'], start, end, false);
-  }
-
-  async getRanges(types: string[], start = 0, end = 1, asc = false) {
-    const multi = this.client.multi();
-    const multiCommands: string[] = [];
-
-    this.commandByType(types, false, (key, command) => {
-      switch (command) {
-        case 'lrange':
-          if (asc) {
-            multiCommands.push('lrange');
-            multi.lrange(key, -(end + 1), -(start + 1));
-          } else {
-            multi.lrange(key, start, end);
-          }
-          break;
-        case 'zrange':
-          multiCommands.push('zrange');
-          if (asc) {
-            multi.zrange(key, start, end);
-          } else {
-            multi.zrevrange(key, start, end);
-          }
-          break;
-      }
-    });
-
-    const responses = await multi.exec();
-    let results: any[] = [];
-
-    responses.forEach((response: any[], index: number) => {
-      const result = response[1] || [];
-
-      if (asc && multiCommands[index] === 'lrange') {
-        results = results.concat(result.reverse());
-      } else {
-        results = results.concat(result);
-      }
-    });
-    return results;
-  }
-
-  async getJobs(types: string[] | string, start = 0, end = -1, asc = false) {
-    await this.waitUntilReady();
-    types = Array.isArray(types) ? types : [types];
-
-    if (types.indexOf('waiting') !== -1) {
-      types = types.concat(['paused']);
-    }
-    const jobIds = await this.getRanges(types, start, end, asc);
-
-    return Promise.all(jobIds.map(jobId => Job.fromId(this, jobId)));
-  }
-
-  async getJobLogs(jobId: string, start = 0, end = -1) {
-    const multi = this.client.multi();
-
-    const logsKey = this.toKey(jobId + ':logs');
-    multi.lrange(logsKey, -(end + 1), -(start + 1));
-    multi.llen(logsKey);
-    return multi.exec().then(result => ({
-      logs: result[0][1],
-      count: result[1][1],
-    }));
-  }
-
-  async getWorkers() {
-    await this.waitUntilReady();
-    const clients = await this.client.client('list');
-    try {
-      const list = await this.parseClientList(clients);
-      return list;
-    } catch (err) {
-      if (!clientCommandMessageReg.test(err.message)) {
-        throw err;
-      }
-    }
-  }
-
-  private parseClientList(list: string) {
-    const lines = list.split('\n');
-    const clients: { [index: string]: string }[] = [];
-
-    lines.forEach((line: string) => {
-      const client: { [index: string]: string } = {};
-      const keyValues = line.split(' ');
-      keyValues.forEach(function(keyValue) {
-        const index = keyValue.indexOf('=');
-        const key = keyValue.substring(0, index);
-        const value = keyValue.substring(index + 1);
-        client[key] = value;
-      });
-      const name = client['name'];
-      if (name && name.startsWith(this.clientName())) {
-        client['name'] = this.name;
-        clients.push(client);
-      }
-    });
-    return clients;
-  }
-}

package/src/classes/queue-scheduler.ts

@@ -1,125 +0,0 @@
-import { QueueSchedulerOptions } from '../interfaces';
-import { array2obj } from '../utils';
-import { QueueBase } from './';
-import { Scripts } from './scripts';
-
-const MAX_TIMEOUT_MS = Math.pow(2, 31) - 1; // 32 bit signed
-
-/**
- * This class is just used for some automatic bookkeeping of the queue,
- * such as updating the delay set as well as moving stalled jobs back
- * to the waiting list.
- *
- * Jobs are checked for stallness once every "visibility window" seconds.
- * Jobs are then marked as candidates for being stalled, in the next check,
- * the candidates are marked as stalled and moved to wait.
- * Workers need to clean the candidate list with the jobs that they are working
- * on, failing to update the list results in the job ending being stalled.
- *
- * This class requires a dedicated redis connection, and at least one is needed
- * to be running at a given time, otherwise delays, stalled jobs, retries, repeatable
- * jobs, etc, will not work correctly or at all.
- *
- */
-export class QueueScheduler extends QueueBase {
-  private nextTimestamp = Number.MAX_VALUE;
-
-  constructor(protected name: string, opts: QueueSchedulerOptions = {}) {
-    super(name, { maxStalledCount: 1, stalledInterval: 30000, ...opts });
-
-    // tslint:disable: no-floating-promises
-    this.run();
-  }
-
-  private async run() {
-    await this.waitUntilReady();
-
-    const key = this.keys.delay;
-    const opts = this.opts as QueueSchedulerOptions;
-    const delaySet = await Scripts.updateDelaySet(this, Date.now());
-
-    const [nextTimestamp] = delaySet;
-    let streamLastId = delaySet[1] || '0-0';
-
-    if (nextTimestamp) {
-      this.nextTimestamp = nextTimestamp;
-    }
-
-    while (!this.closing) {
-      // Check if at least the min stalled check time has passed.
-      await this.moveStalledJobsToWait();
-
-      // Listen to the delay event stream from lastDelayStreamTimestamp
-      // Can we use XGROUPS to reduce redundancy?
-      const nextDelay = this.nextTimestamp - Date.now();
-      const blockTime = Math.round(
-        Math.min(opts.stalledInterval, Math.max(nextDelay, 0)),
-      );
-
-      let data;
-      if (blockTime) {
-        data = await this.client.xread(
-          'BLOCK',
-          blockTime,
-          'STREAMS',
-          key,
-          streamLastId,
-        );
-      } else {
-        data = await this.client.xread('STREAMS', key, streamLastId);
-      }
-
-      if (data && data[0]) {
-        const stream = data[0];
-        const events = stream[1];
-
-        for (let i = 0; i < events.length; i++) {
-          streamLastId = events[i][0];
-          const args = array2obj(events[i][1]);
-          const nextTimestamp: number = parseInt(args.nextTimestamp);
-
-          if (nextTimestamp < this.nextTimestamp) {
-            this.nextTimestamp = nextTimestamp;
-          }
-        }
-
-        //
-        // We trim to a length of 100, which should be a very safe value
-        // for all kind of scenarios.
-        //
-        this.client.xtrim(key, 'MAXLEN', '~', 100);
-      }
-
-      const now = Date.now();
-      const delay = this.nextTimestamp - now;
-
-      if (delay <= 0) {
-        const [nextTimestamp, id] = await Scripts.updateDelaySet(this, now);
-        if (nextTimestamp) {
-          this.nextTimestamp = nextTimestamp / 4096;
-          streamLastId = id;
-        } else {
-          this.nextTimestamp = Number.MAX_VALUE;
-        }
-      }
-    }
-  }
-
-  private async moveStalledJobsToWait() {
-    if (this.closing) {
-      return;
-    }
-
-    const [failed, stalled] = await Scripts.moveStalledJobsToWait(this);
-
-    failed.forEach((jobId: string) =>
-      this.emit(
-        'failed',
-        jobId,
-        new Error('job stalled more than allowable limit'),
-        'active',
-      ),
-    );
-    stalled.forEach((jobId: string) => this.emit('stalled', jobId, 'active'));
-  }
-}

package/src/classes/queue.ts

@@ -1,180 +0,0 @@
-import { get } from 'lodash';
-import { v4 } from 'node-uuid';
-import {
-  JobsOptions,
-  QueueOptions,
-  RateLimiterOptions,
-  RepeatOptions,
-} from '../interfaces';
-import { Job, QueueGetters, Repeat } from './';
-import { Scripts } from './scripts';
-
-export class Queue extends QueueGetters {
-  token = v4();
-  limiter: RateLimiterOptions = null;
-  repeat: Repeat;
-  jobsOpts: JobsOptions;
-
-  constructor(name: string, opts?: QueueOptions) {
-    super(name, opts);
-
-    this.repeat = new Repeat(name, {
-      ...opts,
-      connection: this.client,
-    });
-
-    this.jobsOpts = get(opts, 'defaultJobOptions');
-
-    // tslint:disable: no-floating-promises
-    this.waitUntilReady().then(() => {
-      this.client.hset(
-        this.keys.meta,
-        'opts.maxLenEvents',
-        get(opts, 'streams.events.maxLen', 10000),
-      );
-    });
-  }
-
-  get defaultJobOptions() {
-    return this.jobsOpts;
-  }
-
-  async add(jobName: string, data: any, opts?: JobsOptions) {
-    if (opts && opts.repeat) {
-      return this.repeat.addNextRepeatableJob(
-        jobName,
-        data,
-        { ...opts, ...this.jobsOpts },
-        true,
-      );
-    } else {
-      const job = await Job.create(this, jobName, data, {
-        ...opts,
-        ...this.jobsOpts,
-      });
-      this.emit('waiting', job);
-      return job;
-    }
-  }
-
-  /**
-  Adds an array of jobs to the queue.
-  @method add
-  @param jobs: [] The array of jobs to add to the queue. Each job is defined by 3 
-  properties, 'name', 'data' and 'opts'. They follow the same signature as 'Queue.add'.
-*/
-  async addBulk(jobs: { name: string; data: any; opts?: JobsOptions }[]) {
-    return Job.createBulk(
-      this,
-      jobs.map(job => ({
-        name: job.name,
-        data: job.data,
-        opts: { ...job.opts, ...this.jobsOpts },
-      })),
-    );
-  }
-
-  /**
-    Pauses the processing of this queue globally.
-
-    We use an atomic RENAME operation on the wait queue. Since
-    we have blocking calls with BRPOPLPUSH on the wait queue, as long as the queue
-    is renamed to 'paused', no new jobs will be processed (the current ones
-    will run until finalized).
-
-    Adding jobs requires a LUA script to check first if the paused list exist
-    and in that case it will add it there instead of the wait list.
-  */
-  async pause() {
-    await this.waitUntilReady();
-    await Scripts.pause(this, true);
-    this.emit('paused');
-  }
-
-  async resume() {
-    await this.waitUntilReady();
-    await Scripts.pause(this, false);
-    this.emit('resumed');
-  }
-
-  removeRepeatable(name: string, repeatOpts: RepeatOptions, jobId?: string) {
-    return this.repeat.removeRepeatable(name, repeatOpts, jobId);
-  }
-
-  removeRepeatableByKey(key: string) {
-    return this.repeat.removeRepeatableByKey(key);
-  }
-
-  /**
-   * Drains the queue, i.e., removes all jobs that are waiting
-   * or delayed, but not active, completed or failed.
-   *
-   * TODO: Convert to an atomic LUA script.
-   */
-  async drain(delayed = false) {
-    // Get all jobids and empty all lists atomically.
-    let multi = this.client.multi();
-
-    multi.lrange(this.toKey('wait'), 0, -1);
-    multi.lrange(this.toKey('paused'), 0, -1);
-    if (delayed) {
-      // TODO: get delayed jobIds too!
-      multi.del(this.toKey('delayed'));
-    }
-    multi.del(this.toKey('wait'));
-    multi.del(this.toKey('paused'));
-    multi.del(this.toKey('priority'));
-
-    const [waiting, paused] = await multi.exec();
-    const waitingjobs = waiting[1];
-    const pausedJobs = paused[1];
-
-    const jobKeys = pausedJobs.concat(waitingjobs).map(this.toKey, this);
-
-    if (jobKeys.length) {
-      multi = this.client.multi();
-
-      multi.del.apply(multi, jobKeys);
-      return multi.exec();
-    }
-  }
-
-  /*@function clean
-   *
-   * Cleans jobs from a queue. Similar to remove but keeps jobs within a certain
-   * grace period.
-   *
-   * @param {number} grace - The grace period
-   * @param {number} The max number of jobs to clean
-   * @param {string} [type=completed] - The type of job to clean
-   * Possible values are completed, wait, active, paused, delayed, failed. Defaults to completed.
-   */
-  async clean(
-    grace: number,
-    limit: number,
-    type:
-      | 'completed'
-      | 'wait'
-      | 'active'
-      | 'paused'
-      | 'delayed'
-      | 'failed' = 'completed',
-  ) {
-    await this.waitUntilReady();
-
-    const jobs = await Scripts.cleanJobsInSet(
-      this,
-      type,
-      Date.now() - grace,
-      limit,
-    );
-
-    this.emit('cleaned', jobs, type);
-    return jobs;
-  }
-
-  async trimEvents(maxLength: number) {
-    await this.waitUntilReady();
-    return this.client.xtrim(this.keys.events, 'MAXLEN', '~', maxLength);
-  }
-}

package/src/classes/redis-connection.ts

@@ -1,111 +0,0 @@
-import IORedis from 'ioredis';
-import * as semver from 'semver';
-import { load } from '../commands';
-import { ConnectionOptions, RedisOptions } from '../interfaces';
-
-export class RedisConnection {
-  static minimumVersion = '5.0.0';
-  client: IORedis.Redis;
-
-  constructor(private opts?: ConnectionOptions | IORedis.Redis) {
-    if (!(opts instanceof IORedis)) {
-      this.opts = Object.assign(
-        {
-          port: 6379,
-          host: '127.0.0.1',
-          retryStrategy: function(times: number) {
-            return Math.min(Math.exp(times), 20000);
-          },
-        },
-        opts,
-      );
-    } else {
-      this.client = opts;
-    }
-  }
-
-  /**
-   * Waits for a redis client to be ready.
-   * @param {Redis} redis client
-   */
-  static async waitUntilReady(client: IORedis.Redis) {
-    return new Promise(function(resolve, reject) {
-      if (client.status === 'ready') {
-        resolve();
-      } else {
-        async function handleReady() {
-          client.removeListener('error', handleError);
-          await load(client);
-          resolve();
-        }
-
-        function handleError(err: Error) {
-          client.removeListener('ready', handleReady);
-          reject(err);
-        }
-
-        client.once('ready', handleReady);
-        client.once('error', handleError);
-      }
-    });
-  }
-
-  async init() {
-    if (!this.client) {
-      this.client = new IORedis(<RedisOptions>this.opts);
-    }
-
-    await RedisConnection.waitUntilReady(this.client);
-
-    this.client.on('error', (err: Error) => {
-      console.error(err);
-    });
-
-    if ((<RedisOptions>this.opts).skipVersionCheck !== true) {
-      const version = await this.getRedisVersion();
-      if (semver.lt(version, RedisConnection.minimumVersion)) {
-        throw new Error(
-          `Redis version needs to be greater than ${RedisConnection.minimumVersion} Current: ${version}`,
-        );
-      }
-    }
-    return this.client;
-  }
-
-  async disconnect() {
-    const client = this.client;
-    if (client.status !== 'end') {
-      let _resolve, _reject;
-
-      const disconnecting = new Promise((resolve, reject) => {
-        client.once('end', resolve);
-        client.once('error', reject);
-        _resolve = resolve;
-        _reject = reject;
-      });
-
-      client.disconnect();
-
-      try {
-        await disconnecting;
-      } finally {
-        client.removeListener('end', _resolve);
-        client.removeListener('error', _reject);
-      }
-    }
-  }
-
-  async close() {}
-
-  private async getRedisVersion() {
-    const doc = await this.client.info();
-    const prefix = 'redis_version:';
-    const lines = doc.split('\r\n');
-
-    for (let i = 0; i < lines.length; i++) {
-      if (lines[i].indexOf(prefix) === 0) {
-        return lines[i].substr(prefix.length);
-      }
-    }
-  }
-}