@usehenri/jobs 0.0.0 → 1.2.0

package/src/store/mongo.js

@@ -0,0 +1,1334 @@
+const debug = require('debug')('henri:jobs:mongo');
+
+const { JobStoreError } = require('../errors');
+const { keep } = require('../keys');
+
+/**
+ * The MongoDB backend of the queue.
+ *
+ * ## Claiming
+ *
+ * MongoDB has no `SELECT ... FOR UPDATE`, but it does not need one here: a
+ * single-document update is atomic, and `findOneAndUpdate` matches and
+ * writes in that one operation. The filter carries `state: 'pending'`, so
+ * two runners racing for the same document are resolved by the server --
+ * one of them writes, the other one's filter no longer matches and it is
+ * handed the next document instead. That is the same guarantee the SQL
+ * backend gets from its single-statement claim, and it holds on a
+ * standalone `mongod` (the `disk` adapter) as much as on a replica set, no
+ * transaction involved.
+ *
+ * The cost is that documents are claimed one at a time rather than in a
+ * batch: a runner asking for eight jobs makes eight round trips.
+ *
+ * Documents use the same field names as the SQL columns so the two backends
+ * hand the queue the same rows.
+ *
+ * ## Concurrency limits
+ *
+ * The same as the SQL backend, for the same reason: a bound on how many of
+ * a job run at once cannot be counted inside the claim, so it is held by a
+ * collection whose `_id` is `<key>#<slot>`. An `insertOne` either writes it
+ * or answers 11000, which is exactly the primitive the SQL side gets from a
+ * primary key -- and the one thing MongoDB has no other way to give, having
+ * neither `SELECT ... FOR UPDATE` nor an advisory lock.
+ *
+ * A collection needs no upgrade: a document written by an older henri
+ * simply has no `concurrency_key`, which is what an unlimited job's row
+ * looks like anyway. The same is true of `tenant`, which is why
+ * `tenanted()` answers yes here and asks nothing.
+ *
+ * ## Batches
+ *
+ * `$inc` is atomic on one document, so the counter of a batch is advanced
+ * the way the SQL backend advances its own: never read into this process to
+ * be written back. What the SQL backend puts in the same statement -- the
+ * `EXISTS` saying the job row still holds this runner's claim token -- is
+ * two operations here, and it is exact all the same: a job document whose
+ * outcome was written under this token is **terminal**, and a terminal
+ * document is never recovered, never re-claimed and never written by anyone
+ * else. So once the check passes, it stays true.
+ */
+
+/** The collection is written with the SQL column names, on purpose */
+const ID = 'id';
+
+/**
+ * The MongoDB store
+ *
+ * @class MongoStore
+ */
+class MongoStore {
+  /**
+   * Creates an instance of MongoStore.
+   *
+   * @param {object} adapter A henri mongoose (or disk) adapter
+   * @param {object} tables `{ jobs, schedules, limits, batches }` collection names
+   * @memberof MongoStore
+   */
+  constructor(adapter, tables) {
+    this.adapter = adapter;
+    this.tables = tables;
+    this.dialect = 'mongodb';
+    this.kind = 'mongo';
+  }
+
+  /**
+   * The database of the store
+   *
+   * @returns {object} A MongoDB database
+   * @throws {JobStoreError} When the store is not connected
+   * @memberof MongoStore
+   */
+  database() {
+    const connection =
+      this.adapter.mongoose && this.adapter.mongoose.connection;
+
+    if (!connection || connection.readyState !== 1 || !connection.db) {
+      throw new JobStoreError(
+        `@usehenri/jobs: the ${this.adapter.name} store is not connected`
+      );
+    }
+
+    return connection.db;
+  }
+
+  /**
+   * The jobs collection
+   *
+   * @returns {object} A MongoDB collection
+   * @memberof MongoStore
+   */
+  jobs() {
+    return this.database().collection(this.tables.jobs);
+  }
+
+  /**
+   * The schedules collection
+   *
+   * @returns {object} A MongoDB collection
+   * @memberof MongoStore
+   */
+  schedules() {
+    return this.database().collection(this.tables.schedules);
+  }
+
+  /**
+   * The concurrency slots collection
+   *
+   * @returns {object} A MongoDB collection
+   * @memberof MongoStore
+   */
+  limits() {
+    return this.database().collection(this.tables.limits);
+  }
+
+  /**
+   * The batches collection
+   *
+   * @returns {object} A MongoDB collection
+   * @memberof MongoStore
+   */
+  batches() {
+    return this.database().collection(this.tables.batches);
+  }
+
+  /**
+   * Whether concurrency limits can be held here; they always can
+   *
+   * A collection has no columns to be missing, so there is nothing for an
+   * installation to upgrade and nothing to refuse.
+   *
+   * @returns {Promise<boolean>} true
+   * @memberof MongoStore
+   */
+  async concurrent() {
+    return true;
+  }
+
+  /**
+   * Whether a batch can be held here; it always can
+   *
+   * A collection appears when something is written into it, so there is
+   * nothing to upgrade and nothing to refuse -- see `concurrent()`.
+   *
+   * @returns {Promise<boolean>} true
+   * @memberof MongoStore
+   */
+  async batched() {
+    return true;
+  }
+
+  /**
+   * Whether a tenant can be stamped here; it always can
+   *
+   * A field appears when it is written, so there is nothing to upgrade and
+   * nothing to refuse -- see `concurrent()`. A document an older henri
+   * wrote has no `tenant`, which is what a job of no tenant looks like
+   * anyway.
+   *
+   * @returns {Promise<boolean>} true
+   * @memberof MongoStore
+   */
+  async tenanted() {
+    return true;
+  }
+
+  /**
+   * A document, as the queue reads rows
+   *
+   * @param {?object} document A stored document
+   * @returns {?object} The row, or null
+   * @memberof MongoStore
+   */
+  row(document) {
+    if (!document) {
+      return null;
+    }
+
+    const { _id, ...rest } = document;
+
+    return { ...rest, [ID]: _id };
+  }
+
+  /**
+   * Creates the collections and their indexes; idempotent
+   *
+   * @returns {Promise<Array<string>>} What was created
+   * @memberof MongoStore
+   */
+  async install() {
+    // The key order of a compound index decides which queries it serves:
+    // these are index specifications, not objects to be sorted
+    /* eslint-disable sort-keys */
+    await this.index(
+      { state: 1, queue: 1, priority: 1, run_at: 1 },
+      { name: `${this.tables.jobs}_claim` }
+    );
+    await this.index(
+      { state: 1, finished_at: 1 },
+      { name: `${this.tables.jobs}_finished` }
+    );
+    /* eslint-enable sort-keys */
+    // A partial index, not a sparse one: a sparse unique index still holds
+    // every document whose `unique_key` is present and null, so the second
+    // job without a unique key would be refused
+    await this.index(
+      { unique_key: 1 },
+      {
+        name: `${this.tables.jobs}_unique`,
+        partialFilterExpression: { unique_key: { $type: 'string' } },
+        unique: true,
+      }
+    );
+
+    await this.index({ batch_id: 1 }, { name: `${this.tables.jobs}_batch` });
+
+    /* eslint-disable sort-keys */
+    await this.index(
+      { tenant: 1, state: 1, run_at: 1 },
+      { name: `${this.tables.jobs}_tenant` }
+    );
+    /* eslint-enable sort-keys */
+
+    await this.limits().createIndex(
+      { heartbeat_at: 1 },
+      { name: `${this.tables.limits}_stale` }
+    );
+
+    await this.batches().createIndex(
+      { finished_at: 1, updated_at: 1 },
+      { name: `${this.tables.batches}_open` }
+    );
+
+    return [
+      this.tables.jobs,
+      this.tables.schedules,
+      this.tables.limits,
+      this.tables.batches,
+    ];
+  }
+
+  /**
+   * Creates an index, replacing one an older version left with other options
+   *
+   * @param {object} keys The index keys
+   * @param {object} options The index options (`name` is required)
+   * @returns {Promise<void>} Resolves when the index is there
+   * @memberof MongoStore
+   */
+  async index(keys, options) {
+    try {
+      await this.jobs().createIndex(keys, options);
+    } catch (error) {
+      // IndexOptionsConflict / IndexKeySpecsConflict
+      if (error.code !== 85 && error.code !== 86) {
+        throw error;
+      }
+
+      debug('replacing the index %s', options.name);
+      await this.jobs().dropIndex(options.name);
+      await this.jobs().createIndex(keys, options);
+    }
+  }
+
+  /**
+   * Drops the collections
+   *
+   * @returns {Promise<Array<string>>} What was dropped
+   * @memberof MongoStore
+   */
+  async uninstall() {
+    for (const name of [
+      this.tables.jobs,
+      this.tables.schedules,
+      this.tables.limits,
+      this.tables.batches,
+    ]) {
+      await this.database()
+        .collection(name)
+        .drop()
+        .catch(() => null);
+    }
+
+    return [
+      this.tables.batches,
+      this.tables.limits,
+      this.tables.schedules,
+      this.tables.jobs,
+    ];
+  }
+
+  /**
+   * Whether the collections can be read
+   *
+   * @returns {Promise<boolean>} true when the database answers
+   * @memberof MongoStore
+   */
+  async installed() {
+    try {
+      await this.jobs().estimatedDocumentCount();
+
+      return true;
+    } catch (error) {
+      return false;
+    }
+  }
+
+  /**
+   * Inserts a job
+   *
+   * @param {object} job A row
+   * @returns {Promise<object>} The job, read back
+   * @memberof MongoStore
+   */
+  async insert(job) {
+    const { id, ...rest } = job;
+
+    // A job without a unique key has no `unique_key` field at all, so the
+    // partial unique index never sees it
+    if (rest.unique_key === null || typeof rest.unique_key === 'undefined') {
+      delete rest.unique_key;
+    }
+
+    // Likewise: an unlimited job has no `concurrency_key` field at all, and
+    // that is what a document an older henri wrote looks like
+    if (
+      rest.concurrency_key === null ||
+      typeof rest.concurrency_key === 'undefined'
+    ) {
+      delete rest.concurrency_key;
+    }
+
+    // And a job that belongs to no batch has no `batch_id`
+    if (rest.batch_id === null || typeof rest.batch_id === 'undefined') {
+      delete rest.batch_id;
+    }
+
+    // And a job of no tenant has no `tenant`, which is what every document
+    // an older henri wrote looks like
+    if (rest.tenant === null || typeof rest.tenant === 'undefined') {
+      delete rest.tenant;
+    }
+
+    try {
+      await this.jobs().insertOne({ _id: id, ...rest });
+    } catch (error) {
+      // Only a duplicate key is answered with the job that holds it. Any
+      // other failure is the caller's to see, or an enqueue would silently
+      // do nothing
+      if (job.unique_key && error.code === 11000) {
+        const existing = await this.findByUniqueKey(job.unique_key);
+
+        if (existing && existing.id !== id) {
+          return existing;
+        }
+      }
+
+      throw error;
+    }
+
+    return this.find(id);
+  }
+
+  /**
+   * One job by id
+   *
+   * @param {string} id The job id
+   * @returns {Promise<?object>} The row, or null
+   * @memberof MongoStore
+   */
+  async find(id) {
+    return this.row(await this.jobs().findOne({ _id: id }));
+  }
+
+  /**
+   * One job by unique key
+   *
+   * @param {string} key The unique key
+   * @returns {Promise<?object>} The row, or null
+   * @memberof MongoStore
+   */
+  async findByUniqueKey(key) {
+    return this.row(await this.jobs().findOne({ unique_key: key }));
+  }
+
+  /**
+   * Claims up to `limit` jobs for this runner
+   *
+   * Every document is taken with one atomic `findOneAndUpdate`; the loop
+   * stops as soon as there is nothing left to take.
+   *
+   * @param {object} options Options
+   * @param {Array<string>} [options.queues=[]] The queues to take from
+   * @param {number} [options.limit=1] How many jobs at most
+   * @param {string} options.runner The runner id
+   * @param {string} options.token A token unique to this claim
+   * @param {number} options.now The current time
+   * @param {object} [options.key] The concurrency key a slot is held for
+   * @param {Array<string>} [options.names] Only these job names
+   * @param {Array<string>} [options.except] Every name but these
+   * @returns {Promise<Array<object>>} The rows this runner owns
+   * @memberof MongoStore
+   */
+  async claim({
+    queues = [],
+    limit = 1,
+    runner,
+    token,
+    now,
+    key,
+    names,
+    except,
+  }) {
+    // `includeResultMetadata: false` is the default of the v6+ driver and is
+    // named here on purpose: under an older one findOneAndUpdate answers
+    // `{ value }`, which would read as a win for every runner
+
+    const filter = { run_at: { $lte: now }, state: 'pending' };
+    const claimed = [];
+
+    if (queues.length > 0) {
+      filter.queue = { $in: queues };
+    }
+
+    // The two passes partition the pending documents by **name**, exactly as
+    // the SQL backend does
+    if (except && except.length > 0) {
+      filter.name = { $nin: except };
+    }
+
+    if (names && names.length > 0) {
+      filter.name = { $in: names };
+    }
+
+    if (key) {
+      // A document with no `concurrency_key` field matches `null`, which is
+      // what a job enqueued before the limit was declared looks like
+      filter.concurrency_key = key.own ? { $in: [key.value, null] } : key.value;
+    }
+
+    for (let taken = 0; taken < limit; taken += 1) {
+      const document = await this.jobs().findOneAndUpdate(
+        filter,
+        {
+          $inc: { attempts: 1 },
+          $set: {
+            claim_token: token,
+            claimed_at: now,
+            claimed_by: runner,
+            heartbeat_at: now,
+            started_at: now,
+            state: 'running',
+            updated_at: now,
+          },
+        },
+        {
+          includeResultMetadata: false,
+          returnDocument: 'after',
+          // Ordered, like an ORDER BY
+          // eslint-disable-next-line sort-keys
+          sort: { priority: 1, run_at: 1, _id: 1 },
+        }
+      );
+
+      if (!document) {
+        break;
+      }
+
+      claimed.push(this.row(document));
+    }
+
+    debug('claimed %d job(s) for %s', claimed.length, runner);
+
+    return claimed;
+  }
+
+  /**
+   * The concurrency keys with work waiting, the most urgent first
+   *
+   * @param {object} options Options
+   * @param {number} options.now The current time
+   * @param {Array<string>} options.names The names of the limited jobs
+   * @param {Array<string>} [options.queues=[]] The queues to look at
+   * @param {number} [options.limit=100] How many keys at most
+   * @returns {Promise<Array<object>>} `{ key, name, total }` rows
+   * @memberof MongoStore
+   */
+  async waiting({ now, names, queues = [], limit = 100 }) {
+    if (!names || names.length === 0) {
+      return [];
+    }
+
+    const match = {
+      name: { $in: names },
+      run_at: { $lte: now },
+      state: 'pending',
+    };
+
+    if (queues.length > 0) {
+      match.queue = { $in: queues };
+    }
+
+    const rows = await this.jobs()
+      .aggregate([
+        { $match: match },
+        {
+          $group: {
+            _id: { key: '$concurrency_key', name: '$name' },
+            due: { $min: '$run_at' },
+            priority: { $min: '$priority' },
+            total: { $sum: 1 },
+          },
+        },
+        // Ordered, like an ORDER BY
+        // eslint-disable-next-line sort-keys
+        { $sort: { priority: 1, due: 1 } },
+        { $limit: Math.max(1, Number(limit) || 100) },
+      ])
+      .toArray();
+
+    return rows.map((row) => ({
+      key: row._id.key || null,
+      name: row._id.name,
+      total: row.total || 0,
+    }));
+  }
+
+  /**
+   * The `_id` of one slot
+   *
+   * The slot is an integer and it is always last, so the split on the final
+   * `#` gives the key back whatever the key holds.
+   *
+   * @param {string} key The concurrency key
+   * @param {number} slot The slot
+   * @returns {string} The document id
+   * @memberof MongoStore
+   */
+  slotId(key, slot) {
+    return `${key}#${slot}`;
+  }
+
+  /**
+   * Takes one of a key's slots, or answers null when they are all held
+   *
+   * **This is the bound.** An `insertOne` on a taken `_id` answers 11000 and
+   * writes nothing, which is the same refusal a primary key gives the SQL
+   * backend.
+   *
+   * @param {object} options Options
+   * @param {string} options.key The concurrency key
+   * @param {number} options.limit How many may run at once
+   * @param {string} options.runner The runner id
+   * @param {number} options.now The current time
+   * @returns {Promise<?number>} The slot this runner holds, or null
+   * @memberof MongoStore
+   */
+  async takeSlot({ key, limit, runner, now }) {
+    for (let slot = 0; slot < limit; slot += 1) {
+      try {
+        await this.limits().insertOne({
+          _id: this.slotId(key, slot),
+          heartbeat_at: now,
+          job_id: null,
+          limit_key: key,
+          runner,
+          slot,
+          taken_at: now,
+        });
+
+        return slot;
+      } catch (error) {
+        if (error.code !== 11000) {
+          throw error;
+        }
+
+        debug('slot %d of %s was taken first', slot, key);
+      }
+    }
+
+    return null;
+  }
+
+  /**
+   * Says which job a slot is being held for
+   *
+   * @param {string} key The concurrency key
+   * @param {number} slot The slot
+   * @param {?string} id The job id
+   * @param {number} now The current time
+   * @returns {Promise<void>} Resolves when written
+   * @memberof MongoStore
+   */
+  async holdSlot(key, slot, id, now) {
+    await this.limits().updateOne(
+      { _id: this.slotId(key, slot) },
+      { $set: { heartbeat_at: now, job_id: id } }
+    );
+  }
+
+  /**
+   * Gives a slot back
+   *
+   * @param {string} key The concurrency key
+   * @param {number} slot The slot
+   * @param {string} [runner] Only when this runner still holds it
+   * @returns {Promise<void>} Resolves when written
+   * @memberof MongoStore
+   */
+  async releaseSlot(key, slot, runner) {
+    const filter = { _id: this.slotId(key, slot) };
+
+    if (runner) {
+      filter.runner = runner;
+    }
+
+    await this.limits().deleteOne(filter);
+  }
+
+  /**
+   * Tells the database this runner still holds these slots
+   *
+   * @param {Array<object>} slots `{ key, slot }` entries
+   * @param {number} now The current time
+   * @param {string} runner The runner id
+   * @returns {Promise<void>} Resolves when written
+   * @memberof MongoStore
+   */
+  async heartbeatSlots(slots, now, runner) {
+    if (slots.length === 0) {
+      return;
+    }
+
+    await this.limits().updateMany(
+      {
+        _id: { $in: slots.map((held) => this.slotId(held.key, held.slot)) },
+        runner,
+      },
+      { $set: { heartbeat_at: now } }
+    );
+  }
+
+  /**
+   * Frees the slots of runners that stopped answering
+   *
+   * @param {object} options `now`, `stuckAfter` and `limit`
+   * @returns {Promise<Array<object>>} The slots that were freed
+   * @memberof MongoStore
+   */
+  async sweepSlots({ now, stuckAfter, limit = 100 }) {
+    const documents = await this.limits()
+      .find({ heartbeat_at: { $lt: now - stuckAfter } })
+      .sort({ heartbeat_at: 1 })
+      .limit(limit)
+      .toArray();
+
+    for (const document of documents) {
+      await this.limits().deleteOne({
+        _id: document._id,
+        heartbeat_at: document.heartbeat_at,
+      });
+    }
+
+    return documents.map((document) => ({
+      job: document.job_id || null,
+      key: document.limit_key,
+      runner: document.runner,
+      slot: document.slot,
+      takenAt: document.taken_at,
+    }));
+  }
+
+  /**
+   * Every slot being held right now
+   *
+   * @param {number} [limit=200] How many at most
+   * @returns {Promise<Array<object>>} The held slots
+   * @memberof MongoStore
+   */
+  async slots(limit = 200) {
+    const documents = await this.limits()
+      .find({})
+      .sort({ limit_key: 1, slot: 1 })
+      .limit(limit)
+      .toArray();
+
+    return documents.map((document) => ({
+      heartbeatAt: document.heartbeat_at,
+      job: document.job_id || null,
+      key: document.limit_key,
+      runner: document.runner,
+      slot: document.slot,
+      takenAt: document.taken_at,
+    }));
+  }
+
+  /**
+   * Writes the outcome of an attempt
+   *
+   * With a token the write only lands while this runner still owns the
+   * document, so a runner that was recovered from cannot write over the new
+   * owner's outcome.
+   *
+   * @param {string} id The job id
+   * @param {object} changes The fields to set
+   * @param {string} [token] The claim token this runner holds
+   * @returns {Promise<void>} Resolves when written
+   * @memberof MongoStore
+   */
+  async update(id, changes, token) {
+    if (Object.keys(changes).length === 0) {
+      return;
+    }
+
+    const filter = token
+      ? { _id: id, claim_token: token, state: 'running' }
+      : { _id: id };
+
+    await this.jobs().updateOne(filter, { $set: changes });
+  }
+
+  /**
+   * Puts back the jobs of runners that stopped answering
+   *
+   * @param {object} options `now`, `stuckAfter` and `limit`
+   * @returns {Promise<Array<object>>} The rows that were recovered
+   * @memberof MongoStore
+   */
+  async recover({ now, stuckAfter, limit = 100 }) {
+    const documents = await this.jobs()
+      .find({ heartbeat_at: { $lt: now - stuckAfter }, state: 'running' })
+      .sort({ heartbeat_at: 1 })
+      .limit(limit)
+      .toArray();
+    const rows = documents.map((document) => this.row(document));
+
+    for (const row of rows) {
+      const dead = (row.attempts || 0) >= (row.max_attempts || 0);
+      // A dead job holds its unique key no longer, unless the queue wrote it
+      // for itself (see ../keys.js)
+      const held = dead ? keep(row.unique_key) : row.unique_key;
+
+      await this.jobs().updateOne(
+        { _id: row.id, claim_token: row.claim_token, state: 'running' },
+        {
+          $set: {
+            error_message: `the runner ${row.claimed_by} stopped answering while performing this job`,
+            finished_at: dead ? now : null,
+            run_at: now,
+            state: dead ? 'dead' : 'pending',
+            updated_at: now,
+          },
+          $unset: held
+            ? { claim_token: '' }
+            : { claim_token: '', unique_key: '' },
+        }
+      );
+    }
+
+    return rows;
+  }
+
+  /**
+   * Tells the database this runner is still on these jobs
+   *
+   * A runner that was already recovered from no longer owns these documents,
+   * so the token is part of the filter.
+   *
+   * @param {Array<string>} ids The job ids
+   * @param {number} now The current time
+   * @param {string} [token] The claim token this runner holds
+   * @returns {Promise<void>} Resolves when written
+   * @memberof MongoStore
+   */
+  async heartbeat(ids, now, token) {
+    if (ids.length === 0) {
+      return;
+    }
+
+    const filter = { _id: { $in: ids } };
+
+    if (token) {
+      filter.claim_token = token;
+    }
+
+    await this.jobs().updateMany(filter, { $set: { heartbeat_at: now } });
+  }
+
+  /**
+   * Deletes the finished jobs older than a moment
+   *
+   * @param {number} before A timestamp
+   * @param {number} [limit=1000] How many documents one pass deletes
+   * @returns {Promise<number>} How many documents were deleted
+   * @memberof MongoStore
+   */
+  async prune(before, limit = 1000) {
+    const documents = await this.jobs()
+      .find({ finished_at: { $lt: before }, state: 'done' })
+      .sort({ finished_at: 1 })
+      .limit(limit)
+      .project({ _id: 1 })
+      .toArray();
+
+    if (documents.length === 0) {
+      return 0;
+    }
+
+    const result = await this.jobs().deleteMany({
+      _id: { $in: documents.map((document) => document._id) },
+    });
+
+    return result.deletedCount || 0;
+  }
+
+  /**
+   * Lists jobs
+   *
+   * @param {object} [options={}] `state`, `queue`, `name`, `batch`,
+   *   `tenant`, `limit`, `offset`
+   * @returns {Promise<Array<object>>} The rows
+   * @memberof MongoStore
+   */
+  async list({
+    state,
+    queue,
+    name,
+    batch,
+    tenant,
+    limit = 50,
+    offset = 0,
+  } = {}) {
+    const filter = {};
+
+    if (state) {
+      filter.state = state;
+    }
+
+    if (queue) {
+      filter.queue = queue;
+    }
+
+    if (name) {
+      filter.name = name;
+    }
+
+    if (batch) {
+      filter.batch_id = batch;
+    }
+
+    if (tenant) {
+      filter.tenant = tenant;
+    }
+
+    const documents = await this.jobs()
+      .find(filter)
+      // eslint-disable-next-line sort-keys
+      .sort({ updated_at: -1, _id: 1 })
+      .skip(offset)
+      .limit(limit)
+      .toArray();
+
+    return documents.map((document) => this.row(document));
+  }
+
+  /**
+   * Deletes jobs
+   *
+   * @param {object} [options={}] `id`, `state`, `queue`, `name`, `tenant`
+   * @returns {Promise<number>} How many documents were deleted
+   * @memberof MongoStore
+   */
+  async remove({ id, state, queue, name, tenant } = {}) {
+    const filter = {};
+
+    if (id) {
+      filter._id = id;
+    }
+
+    if (state) {
+      filter.state = state;
+    }
+
+    if (queue) {
+      filter.queue = queue;
+    }
+
+    if (name) {
+      filter.name = name;
+    }
+
+    if (tenant) {
+      filter.tenant = tenant;
+    }
+
+    if (Object.keys(filter).length === 0) {
+      return 0;
+    }
+
+    const result = await this.jobs().deleteMany(filter);
+
+    return result.deletedCount || 0;
+  }
+
+  /**
+   * Counts the jobs of every queue and state
+   *
+   * @returns {Promise<Array<object>>} `{ queue, state, total }` rows
+   * @memberof MongoStore
+   */
+  async counts() {
+    const rows = await this.jobs()
+      .aggregate([
+        {
+          $group: {
+            _id: { queue: '$queue', state: '$state' },
+            total: { $sum: 1 },
+          },
+        },
+      ])
+      .toArray();
+
+    return rows.map((row) => ({
+      queue: row._id.queue,
+      state: row._id.state,
+      total: row.total,
+    }));
+  }
+
+  /**
+   * How long the finished jobs of every queue took
+   *
+   * @returns {Promise<Array<object>>} `{ queue, runs, shortest, longest, average }` rows
+   * @memberof MongoStore
+   */
+  async timings() {
+    const rows = await this.jobs()
+      .aggregate([
+        { $match: { duration_ms: { $ne: null }, state: 'done' } },
+        {
+          $group: {
+            _id: '$queue',
+            average: { $avg: '$duration_ms' },
+            longest: { $max: '$duration_ms' },
+            runs: { $sum: 1 },
+            shortest: { $min: '$duration_ms' },
+          },
+        },
+      ])
+      .toArray();
+
+    return rows.map((row) => ({
+      average: Math.round(row.average || 0),
+      longest: row.longest || 0,
+      queue: row._id,
+      runs: row.runs || 0,
+      shortest: row.shortest || 0,
+    }));
+  }
+
+  /**
+   * The moment the oldest job waiting in every queue was due
+   *
+   * @param {number} now The current time
+   * @returns {Promise<Array<object>>} `{ queue, waiting }` rows
+   * @memberof MongoStore
+   */
+  async oldest(now) {
+    const rows = await this.jobs()
+      .aggregate([
+        { $match: { run_at: { $lte: now }, state: 'pending' } },
+        { $group: { _id: '$queue', due: { $min: '$run_at' } } },
+      ])
+      .toArray();
+
+    return rows.map((row) => ({
+      queue: row._id,
+      waiting: Math.max(0, now - (row.due || now)),
+    }));
+  }
+
+  /**
+   * The schedule of a recurring job
+   *
+   * @param {string} name The schedule name
+   * @returns {Promise<?object>} The row, or null
+   * @memberof MongoStore
+   */
+  async schedule(name) {
+    const document = await this.schedules().findOne({ _id: name });
+
+    return document ? { ...document, name: document._id } : null;
+  }
+
+  /**
+   * Records a schedule that has none yet
+   *
+   * @param {object} row The schedule row
+   * @returns {Promise<?object>} The schedule
+   * @memberof MongoStore
+   */
+  async addSchedule(row) {
+    const { name, ...rest } = row;
+
+    try {
+      await this.schedules().insertOne({
+        _id: name,
+        ...rest,
+        last_run_at: null,
+        token: null,
+      });
+    } catch (error) {
+      // Another runner recording it first is the expected failure; anything
+      // else is answered with null, and the runner says so
+      debug('schedule %s not recorded (%s)', name, error.message);
+    }
+
+    return this.schedule(name).catch(() => null);
+  }
+
+  /**
+   * Moves a schedule forward, if this runner is the one that got there first
+   *
+   * @param {object} options `name`, `spec`, `due`, `next`, `token`, `now`
+   * @returns {Promise<boolean>} Whether this runner won the slot
+   * @memberof MongoStore
+   */
+  async advanceSchedule({ name, spec, due, next, token, now }) {
+    const document = await this.schedules().findOneAndUpdate(
+      { _id: name, next_run_at: due },
+      {
+        $set: {
+          last_run_at: due,
+          next_run_at: next,
+          spec,
+          token,
+          updated_at: now,
+        },
+      },
+      { includeResultMetadata: false, returnDocument: 'after' }
+    );
+
+    return Boolean(document);
+  }
+
+  /**
+   * Points a schedule at a new moment
+   *
+   * @param {object} options `name`, `spec`, `next` and `now`
+   * @returns {Promise<void>} Resolves when written
+   * @memberof MongoStore
+   */
+  async resetSchedule({ name, spec, next, now }) {
+    await this.schedules().updateOne(
+      { _id: name },
+      { $set: { next_run_at: next, spec, updated_at: now } }
+    );
+  }
+
+  /**
+   * Forgets the schedules the configuration no longer declares
+   *
+   * @param {Array<string>} names The schedules to keep
+   * @returns {Promise<void>} Resolves when done
+   * @memberof MongoStore
+   */
+  async pruneSchedules(names) {
+    await this.schedules().deleteMany({ _id: { $nin: names } });
+  }
+
+  /**
+   * Records a batch
+   *
+   * @param {object} batch A batch row
+   * @returns {Promise<object>} The batch, read back
+   * @memberof MongoStore
+   */
+  async createBatch(batch) {
+    const { id, ...rest } = batch;
+
+    await this.batches().insertOne({ _id: id, ...rest });
+
+    return this.findBatch(id);
+  }
+
+  /**
+   * One batch by id
+   *
+   * @param {string} id The batch id
+   * @returns {Promise<?object>} The row, or null
+   * @memberof MongoStore
+   */
+  async findBatch(id) {
+    return this.row(await this.batches().findOne({ _id: id }));
+  }
+
+  /**
+   * Closes a batch to new jobs and writes down how many it holds
+   *
+   * @param {object} options `id`, `total` and `now`
+   * @returns {Promise<?object>} The batch, or null when it was sealed
+   *   already
+   * @memberof MongoStore
+   */
+  async sealBatch({ id, total, now }) {
+    const document = await this.batches().findOneAndUpdate(
+      { _id: id, sealed_at: null },
+      { $set: { sealed_at: now, total, updated_at: now } },
+      { includeResultMetadata: false, returnDocument: 'after' }
+    );
+
+    return this.row(document);
+  }
+
+  /**
+   * Counts one terminal outcome into its batch
+   *
+   * The check and the `$inc` are two operations and the pair is still
+   * exact: a job document that is terminal **and** still holds this
+   * runner's claim token was written by this runner and can never be
+   * claimed, recovered or written again, so nothing can make the check stale
+   * between here and the increment. `$inc` itself is atomic on the document.
+   *
+   * @param {object} options `id`, `job`, `token`, `failed` and `now`
+   * @returns {Promise<?object>} The batch as it is now, or null
+   * @memberof MongoStore
+   */
+  async advanceBatch({ id, job, token, failed, now }) {
+    const owned = await this.jobs().findOne({
+      _id: job,
+      batch_id: id,
+      claim_token: token,
+      state: { $in: ['done', 'dead'] },
+    });
+
+    if (!owned) {
+      return this.findBatch(id);
+    }
+
+    const document = await this.batches().findOneAndUpdate(
+      { _id: id, finished_at: null },
+      {
+        $inc: { done: 1, failed: failed ? 1 : 0 },
+        $set: { updated_at: now },
+      },
+      { includeResultMetadata: false, returnDocument: 'after' }
+    );
+
+    return this.row(document) || this.findBatch(id);
+  }
+
+  /**
+   * Gives a batch its slot back, when a job of it is put back in the queue
+   *
+   * @param {object} options `id`, `failed` and `now`
+   * @returns {Promise<void>} Resolves when written
+   * @memberof MongoStore
+   */
+  async releaseBatch({ id, failed, now }) {
+    await this.batches().updateOne(
+      { _id: id, done: { $gt: 0 }, finished_at: null },
+      { $inc: { done: -1, failed: failed ? -1 : 0 }, $set: { updated_at: now } }
+    );
+  }
+
+  /**
+   * Says a batch has finished, and what enqueued its callback
+   *
+   * @param {object} options `id`, `callback` and `now`
+   * @returns {Promise<void>} Resolves when written
+   * @memberof MongoStore
+   */
+  async finishBatch({ id, callback, now }) {
+    await this.batches().updateOne(
+      { _id: id, finished_at: null },
+      {
+        $set: {
+          callback_id: callback || null,
+          finished_at: now,
+          updated_at: now,
+        },
+      }
+    );
+  }
+
+  /**
+   * What a batch's jobs actually say, read from the queue itself
+   *
+   * @param {string} id The batch id
+   * @returns {Promise<object>} `{ done, failed }`
+   * @memberof MongoStore
+   */
+  async countBatch(id) {
+    const rows = await this.jobs()
+      .aggregate([
+        { $match: { batch_id: id, state: { $in: ['done', 'dead'] } } },
+        { $group: { _id: '$state', total: { $sum: 1 } } },
+      ])
+      .toArray();
+    const counted = { done: 0, failed: 0 };
+
+    for (const row of rows) {
+      counted.done += row.total;
+
+      if (row._id === 'dead') {
+        counted.failed += row.total;
+      }
+    }
+
+    return counted;
+  }
+
+  /**
+   * Moves a batch's counters up to what its jobs say, never backwards
+   *
+   * @param {object} options `id`, `done`, `failed` and `now`
+   * @returns {Promise<?object>} The batch as it is now
+   * @memberof MongoStore
+   */
+  async syncBatch({ id, done, failed, now }) {
+    await this.batches().updateOne(
+      { _id: id, done: { $lt: done }, finished_at: null },
+      { $set: { done, failed, updated_at: now } }
+    );
+
+    return this.findBatch(id);
+  }
+
+  /**
+   * The batches that were sealed and have not finished
+   *
+   * @param {object} options `before` and `limit`
+   * @returns {Promise<Array<object>>} The rows
+   * @memberof MongoStore
+   */
+  async openBatches({ before, limit = 50 }) {
+    const documents = await this.batches()
+      .find({
+        finished_at: null,
+        sealed_at: { $ne: null },
+        updated_at: { $lt: before },
+      })
+      .sort({ updated_at: 1 })
+      .limit(Math.max(1, Number(limit) || 50))
+      .toArray();
+
+    return documents.map((document) => this.row(document));
+  }
+
+  /**
+   * Lists batches, the newest first
+   *
+   * @param {object} [options={}] `finished`, `limit`, `offset`
+   * @returns {Promise<Array<object>>} The rows
+   * @memberof MongoStore
+   */
+  async listBatches({ finished, limit = 50, offset = 0 } = {}) {
+    const filter =
+      typeof finished === 'boolean'
+        ? { finished_at: finished ? { $ne: null } : null }
+        : {};
+    const documents = await this.batches()
+      .find(filter)
+      // eslint-disable-next-line sort-keys
+      .sort({ created_at: -1, _id: 1 })
+      .skip(offset)
+      .limit(limit)
+      .toArray();
+
+    return documents.map((document) => this.row(document));
+  }
+
+  /**
+   * Forgets a batch
+   *
+   * @param {string} id The batch id
+   * @returns {Promise<boolean>} Whether there was one
+   * @memberof MongoStore
+   */
+  async removeBatch(id) {
+    const result = await this.batches().deleteOne({ _id: id });
+
+    return (result.deletedCount || 0) > 0;
+  }
+
+  /**
+   * Deletes the batches that finished before a moment
+   *
+   * @param {number} before A timestamp
+   * @param {number} [limit=1000] How many one pass deletes
+   * @returns {Promise<number>} How many were deleted
+   * @memberof MongoStore
+   */
+  async pruneBatches(before, limit = 1000) {
+    const documents = await this.batches()
+      .find({ finished_at: { $lt: before, $ne: null } })
+      .sort({ finished_at: 1 })
+      .limit(limit)
+      .project({ _id: 1 })
+      .toArray();
+
+    if (documents.length === 0) {
+      return 0;
+    }
+
+    const ids = documents.map((document) => document._id);
+
+    await this.batches().deleteMany({ _id: { $in: ids } });
+
+    return ids.length;
+  }
+}
+
+/**
+ * Builds the MongoDB store of an adapter
+ *
+ * @param {object} adapter A henri mongoose (or disk) adapter
+ * @param {object} tables `{ jobs, schedules, limits, batches }` collection
+ *   names
+ * @returns {MongoStore} The store
+ */
+const create = (adapter, tables) => new MongoStore(adapter, tables);
+
+module.exports = { MongoStore, create };