bullmq 4.0.0-beta.2 → 4.0.0

package/src/test/test_sandboxed_process.ts

@@ -1,380 +0,0 @@
-import { expect } from 'chai';
-import IORedis from 'ioredis';
-import { after } from 'lodash';
-import { Queue, QueueEvents, Worker } from '@src/classes';
-import { beforeEach } from 'mocha';
-import { v4 } from 'node-uuid';
-const delay = require('delay');
-const pReflect = require('p-reflect');
-const pool = require('../classes/child-pool').pool;
-
-describe('sandboxed process', () => {
-  let queue: Queue;
-  let queueEvents: QueueEvents;
-  let queueName: string;
-  let client: IORedis.Redis;
-
-  beforeEach(() => {
-    client = new IORedis();
-    return client.flushdb();
-  });
-
-  beforeEach(async () => {
-    queueName = 'test-' + v4();
-    queue = new Queue(queueName);
-    queueEvents = new QueueEvents(queueName);
-    return queueEvents.waitUntilReady();
-  });
-
-  afterEach(async () => {
-    await queue.close();
-    await queueEvents.close();
-    await client.flushall();
-    pool.clean();
-    return client.quit();
-  });
-
-  it('should process and complete', async () => {
-    const processFile = __dirname + '/fixtures/fixture_processor.js';
-
-    const worker = new Worker(queueName, processFile, {
-      drainDelay: 1,
-      settings: {
-        guardInterval: 300000,
-        stalledInterval: 300000,
-      },
-    });
-
-    const completting = new Promise((resolve, reject) => {
-      worker.on('completed', async (job, value) => {
-        try {
-          expect(job.data).to.be.eql({ foo: 'bar' });
-          expect(value).to.be.eql(42);
-          expect(Object.keys(worker['childPool'].retained)).to.have.lengthOf(0);
-          expect(worker['childPool'].free[processFile]).to.have.lengthOf(1);
-          await worker.close();
-          resolve();
-        } catch (err) {
-          await worker.close();
-          reject(err);
-        }
-      });
-    });
-
-    await queue.add('test', { foo: 'bar' });
-
-    await completting;
-
-    await worker.close();
-  });
-
-  it('should process with named processor', async () => {
-    const processFile = __dirname + '/fixtures/fixture_processor.js';
-    const worker = new Worker(queueName, processFile, {
-      drainDelay: 1,
-      settings: {
-        guardInterval: 300000,
-        stalledInterval: 300000,
-      },
-    });
-
-    const completting = new Promise((resolve, reject) => {
-      worker.on('completed', async (job, value) => {
-        try {
-          expect(job.data).to.be.eql({ foo: 'bar' });
-          expect(value).to.be.eql(42);
-          expect(Object.keys(worker['childPool'].retained)).to.have.lengthOf(0);
-          expect(worker['childPool'].free[processFile]).to.have.lengthOf(1);
-          await worker.close();
-          resolve();
-        } catch (err) {
-          await worker.close();
-          reject(err);
-        }
-      });
-    });
-
-    await queue.add('foobar', { foo: 'bar' });
-
-    await completting;
-  });
-
-  it('should process with concurrent processors', async function() {
-    this.timeout(10000);
-
-    let worker: Worker;
-
-    await Promise.all([
-      queue.add('test', { foo: 'bar1' }),
-      queue.add('test', { foo: 'bar2' }),
-      queue.add('test', { foo: 'bar3' }),
-      queue.add('test', { foo: 'bar4' }),
-    ]);
-
-    const processFile = __dirname + '/fixtures/fixture_processor_slow.js';
-    worker = new Worker(queueName, processFile, {
-      concurrency: 4,
-      drainDelay: 1,
-      settings: {
-        guardInterval: 300000,
-        stalledInterval: 300000,
-      },
-    });
-
-    const completing = new Promise((resolve, reject) => {
-      const after4 = after(4, () => {
-        expect(worker['childPool'].getAllFree().length).to.eql(4);
-        resolve();
-      });
-
-      worker.on('completed', async (job, value) => {
-        try {
-          expect(value).to.be.eql(42);
-          expect(
-            Object.keys(worker['childPool'].retained).length +
-              worker['childPool'].getAllFree().length,
-          ).to.eql(4);
-          after4();
-        } catch (err) {
-          await worker.close();
-          reject(err);
-        }
-      });
-    });
-
-    await completing;
-    await worker.close();
-  });
-
-  it('should reuse process with single processors', async function() {
-    this.timeout(30000);
-
-    let worker: Worker;
-    const processFile = __dirname + '/fixtures/fixture_processor_slow.js';
-    worker = new Worker(queueName, processFile, {
-      concurrency: 1,
-      drainDelay: 1,
-      settings: {
-        guardInterval: 300000,
-        stalledInterval: 300000,
-      },
-    });
-
-    await Promise.all([
-      queue.add('1', { foo: 'bar1' }),
-      queue.add('2', { foo: 'bar2' }),
-      queue.add('3', { foo: 'bar3' }),
-      queue.add('4', { foo: 'bar4' }),
-    ]);
-
-    const completting = new Promise((resolve, reject) => {
-      const after4 = after(4, async () => {
-        expect(worker['childPool'].getAllFree().length).to.eql(1);
-        await worker.close();
-        resolve();
-      });
-
-      worker.on('completed', async (job, value) => {
-        try {
-          expect(value).to.be.eql(42);
-          expect(
-            Object.keys(worker['childPool'].retained).length +
-              worker['childPool'].getAllFree().length,
-          ).to.eql(1);
-          await after4();
-        } catch (err) {
-          await worker.close();
-          reject(err);
-        }
-      });
-    });
-
-    await completting;
-  });
-
-  it('should process and update progress', async () => {
-    const processFile = __dirname + '/fixtures/fixture_processor_progress.js';
-
-    const worker = new Worker(queueName, processFile, {
-      drainDelay: 1,
-      settings: {
-        guardInterval: 300000,
-        stalledInterval: 300000,
-      },
-    });
-
-    const progresses: any[] = [];
-
-    const completting = new Promise((resolve, reject) => {
-      worker.on('completed', (job, value) => {
-        try {
-          expect(job.data).to.be.eql({ foo: 'bar' });
-          expect(value).to.be.eql(37);
-          expect(job.progress).to.be.eql(100);
-          expect(progresses).to.be.eql([10, 27, 78, 100]);
-          expect(Object.keys(worker['childPool'].retained)).to.have.lengthOf(0);
-          expect(worker['childPool'].getAllFree()).to.have.lengthOf(1);
-          resolve();
-        } catch (err) {
-          reject(err);
-        }
-      });
-    });
-
-    worker.on('progress', (job, progress) => {
-      progresses.push(progress);
-    });
-
-    await queue.add('test', { foo: 'bar' });
-
-    await completting;
-    await worker.close();
-  });
-
-  it('should process and fail', async () => {
-    const processFile = __dirname + '/fixtures/fixture_processor_fail.js';
-
-    const worker = new Worker(queueName, processFile, {
-      drainDelay: 1,
-      settings: {
-        guardInterval: 300000,
-        stalledInterval: 300000,
-      },
-    });
-
-    const failing = new Promise((resolve, reject) => {
-      worker.on('failed', async (job, err) => {
-        try {
-          expect(job.data).eql({ foo: 'bar' });
-          expect(job.failedReason).eql('Manually failed processor');
-          expect(err.message).eql('Manually failed processor');
-          expect(err.stack).include('fixture_processor_fail.js');
-          expect(Object.keys(worker['childPool'].retained)).to.have.lengthOf(0);
-          expect(worker['childPool'].getAllFree()).to.have.lengthOf(1);
-
-          resolve();
-        } catch (err) {
-          await worker.close();
-          reject(err);
-        }
-      });
-    });
-
-    await queue.add('test', { foo: 'bar' });
-
-    await failing;
-
-    await worker.close();
-  });
-
-  it('should error if processor file is missing', async () => {
-    let worker;
-    let didThrow = false;
-    try {
-      const missingProcessFile = __dirname + '/fixtures/missing_processor.js';
-      worker = new Worker(queueName, missingProcessFile, {});
-    } catch (err) {
-      didThrow = true;
-    }
-
-    worker && (await worker.close());
-
-    if (!didThrow) {
-      throw new Error('did not throw error');
-    }
-  });
-
-  it('should fail if the process crashes', async () => {
-    const processFile = __dirname + '/fixtures/fixture_processor_crash.js';
-
-    const worker = new Worker(queueName, processFile, {
-      drainDelay: 1,
-      settings: {
-        guardInterval: 300000,
-        stalledInterval: 300000,
-      },
-    });
-
-    const job = await queue.add('test', {});
-    const inspection = await pReflect(
-      Promise.resolve(job.waitUntilFinished(queueEvents)),
-    );
-    expect(inspection.isRejected).to.be.eql(true);
-    expect(inspection.reason.message).to.be.eql('boom!');
-  });
-
-  it('should fail if the process exits 0', async () => {
-    const processFile = __dirname + '/fixtures/fixture_processor_crash.js';
-
-    new Worker(queueName, processFile, {
-      drainDelay: 1,
-      settings: {
-        guardInterval: 300000,
-        stalledInterval: 300000,
-      },
-    });
-
-    const job = await queue.add('test', { exitCode: 0 });
-    const inspection = await pReflect(
-      Promise.resolve(job.waitUntilFinished(queueEvents)),
-    );
-    expect(inspection.isRejected).to.be.eql(true);
-    expect(inspection.reason.message).to.be.eql(
-      'Unexpected exit code: 0 signal: null',
-    );
-  });
-
-  it('should fail if the process exits non-0', async () => {
-    const processFile = __dirname + '/fixtures/fixture_processor_crash.js';
-
-    new Worker(queueName, processFile, {
-      drainDelay: 1,
-      settings: {
-        guardInterval: 300000,
-        stalledInterval: 300000,
-      },
-    });
-
-    const job = await queue.add('test', { exitCode: 1 });
-    const inspection = await pReflect(
-      Promise.resolve(job.waitUntilFinished(queueEvents)),
-    );
-    expect(inspection.isRejected).to.be.eql(true);
-    expect(inspection.reason.message).to.be.eql(
-      'Unexpected exit code: 1 signal: null',
-    );
-  });
-
-  it('should remove exited process', async () => {
-    const processFile = __dirname + '/fixtures/fixture_processor_exit.js';
-
-    const worker = new Worker(queueName, processFile, {
-      drainDelay: 1,
-      settings: {
-        guardInterval: 300000,
-        stalledInterval: 300000,
-      },
-    });
-
-    const completting = new Promise((resolve, reject) => {
-      worker.on('completed', async () => {
-        try {
-          expect(Object.keys(worker['childPool'].retained)).to.have.lengthOf(0);
-          expect(worker['childPool'].getAllFree()).to.have.lengthOf(1);
-          await delay(500);
-          expect(Object.keys(worker['childPool'].retained)).to.have.lengthOf(0);
-          expect(worker['childPool'].getAllFree()).to.have.lengthOf(0);
-          resolve();
-        } catch (err) {
-          reject(err);
-        }
-      });
-    });
-
-    await queue.add('test', { foo: 'bar' });
-
-    await completting;
-
-    await worker.close();
-  });
-});

package/src/test/test_stalled_jobs.ts

@@ -1,146 +0,0 @@
-import { Queue, QueueScheduler, Worker, QueueEvents } from '@src/classes';
-import { delay } from 'bluebird';
-import IORedis from 'ioredis';
-import { after } from 'lodash';
-import { beforeEach, describe, it } from 'mocha';
-import { v4 } from 'node-uuid';
-import * as sinon from 'sinon';
-
-describe('stalled jobs', function() {
-  let queue: Queue;
-  let queueName: string;
-  let client: IORedis.Redis;
-
-  beforeEach(function() {
-    client = new IORedis();
-    return client.flushdb();
-  });
-
-  beforeEach(async function() {
-    queueName = 'test-' + v4();
-    queue = new Queue(queueName);
-  });
-
-  afterEach(async function() {
-    await queue.close();
-    return client.quit();
-  });
-
-  it('process stalled jobs when starting a queue', async function() {
-    this.timeout(10000);
-
-    const queueEvents = new QueueEvents(queueName);
-    await queueEvents.waitUntilReady();
-
-    const concurrency = 4;
-
-    const worker = new Worker(
-      queueName,
-      async job => {
-        return delay(10000);
-      },
-      {
-        settings: {
-          stalledInterval: 1000,
-        },
-        concurrency,
-      },
-    );
-
-    const allActive = new Promise(resolve => {
-      worker.on('active', after(concurrency, resolve));
-    });
-
-    await worker.waitUntilReady();
-
-    const jobs = await Promise.all([
-      queue.add('test', { bar: 'baz' }),
-      queue.add('test', { bar1: 'baz1' }),
-      queue.add('test', { bar2: 'baz2' }),
-      queue.add('test', { bar3: 'baz3' }),
-    ]);
-
-    await allActive;
-
-    const queueScheduler = new QueueScheduler(queueName, {
-      stalledInterval: 100,
-    });
-    await queueScheduler.waitUntilReady();
-
-    const allStalled = new Promise(resolve => {
-      queueScheduler.on('stalled', after(concurrency, resolve));
-    });
-
-    const allStalledGlobalEvent = new Promise(resolve => {
-      queueEvents.on('stalled', after(concurrency, resolve));
-    });
-
-    await allStalled;
-    await allStalledGlobalEvent;
-
-    await worker.close(true);
-
-    const worker2 = new Worker(queueName, async job => {}, { concurrency });
-
-    const allCompleted = new Promise(resolve => {
-      worker2.on('completed', after(concurrency, resolve));
-    });
-
-    await allCompleted;
-
-    await queueEvents.close();
-
-    await queueScheduler.close();
-    await worker2.close();
-  });
-
-  it('fail stalled jobs that stall more than allowable stalled limit', async function() {
-    this.timeout(6000);
-
-    const concurrency = 4;
-
-    const worker = new Worker(
-      queueName,
-      async job => {
-        return delay(10000);
-      },
-      {
-        settings: {
-          stalledInterval: 100,
-        },
-        concurrency,
-      },
-    );
-
-    const allActive = new Promise(resolve => {
-      worker.on('active', after(concurrency, resolve));
-    });
-
-    await worker.waitUntilReady();
-
-    const jobs = await Promise.all([
-      queue.add('test', { bar: 'baz' }),
-      queue.add('test', { bar1: 'baz1' }),
-      queue.add('test', { bar2: 'baz2' }),
-      queue.add('test', { bar3: 'baz3' }),
-    ]);
-
-    await allActive;
-
-    const queueScheduler = new QueueScheduler(queueName, {
-      stalledInterval: 100,
-      maxStalledCount: 0,
-    });
-    await queueScheduler.waitUntilReady();
-
-    const allFailed = new Promise(resolve => {
-      queueScheduler.on('failed', after(concurrency, resolve));
-    });
-
-    await allFailed;
-
-    await worker.close(true);
-
-    await queueScheduler.close();
-  });
-});