@alexify/migronaut 2.2.0 → 2.3.0

package/src/core/background-partition.js

@@ -0,0 +1,298 @@
+const { mapLimit } = require('../utils/concurrency.js');
+const { sameValue } = require('../versioning/internal.js');
+const { ID_BRACKETS, bracketOfType, keysetFilter, scopeFilter } = require('./background-spec.js');
+const { READ_CONCURRENCY, READ_OPTIONS } = require('./server-info.js');
+
+/**
+ * The default partitioner: a background migration's documents split into
+ * `_id` ranges, so several lanes can work one collection at once.
+ *
+ * A partitioner has one job — spread the work — and nothing rests on it being
+ * exact: a document no range covers is found by the final count of what is
+ * left and picked up by the next pass, and two lanes that meet over a document
+ * are kept apart by the optimistic guard on every write. So the planner may
+ * sample, round and degrade freely; it only has to stay cheap.
+ *
+ * Every partitioner has the same shape (the shard-aware one too):
+ * `{ id, plan(ctx), batchQuery(scope, cursor, opts), advance(cursor, docs, opts),
+ * past(cursor, docs), writeFilter(prev), checkTransform(prev, next), stale(ctx, epoch) }`.
+ */
+
+/** Server error: the operation ran out of its `maxTimeMS` */
+const MAX_TIME_EXPIRED = 50;
+
+/** How long the sample may take before the plan falls back to one partition per bracket */
+const SAMPLE_TIMEOUT_MS = 30_000;
+
+/** Ids sampled at most, whatever the settings say */
+const MAX_SAMPLE = 100_000;
+
+/** The number of partitions to aim for: one per lane with no parallelism, a few per lane otherwise */
+function targetPartitions(settings, maxParallel) {
+  if (maxParallel === 1) return 1;
+  const { overPartition, maxPartitions } = settings;
+  return Math.max(1, Math.min(overPartition * maxParallel, maxPartitions));
+}
+
+const withHint = (hint) => (hint === undefined ? {} : { hint });
+
+/**
+ * The brackets the matched documents' `_id`s fall in — one indexed `findOne`
+ * each (`_id` is the second key of the version index), no admin rights.
+ */
+async function presentBrackets(collection, match, hint) {
+  const found = await mapLimit(ID_BRACKETS, READ_CONCURRENCY, async (bracket) => {
+    const doc = await collection.findOne(
+      { $and: [match, { _id: { $type: bracket.aliases } }] },
+      { projection: { _id: 1 }, ...withHint(hint), ...READ_OPTIONS },
+    );
+    return doc === null ? null : bracket.name;
+  });
+  const names = [];
+  for (const name of found) if (name !== null) names.push(name);
+  return names;
+}
+
+/**
+ * `$sample` walks a random cursor only while it asks for less than 5% of the
+ * collection; at 5% or more it scans the collection and sorts it at random —
+ * every document. A sample-first request stays under it.
+ */
+const RANDOM_CURSOR_SHARE = 0.04;
+
+/** How many documents a sample-first pass asks for: `size` matches' worth, within the random cursor */
+function sampleRequest(size, ratio, total) {
+  const oversampled = Math.ceil(size / Math.max(ratio, 0.01));
+  return Math.max(1, Math.min(MAX_SAMPLE, oversampled, Math.floor(total * RANDOM_CURSOR_SHARE)));
+}
+
+/**
+ * A sorted sample of the matched `_id`s with their `$type`: the server sorts
+ * them (JavaScript cannot compare BSON across types the way the server does).
+ * A large match samples the collection first (a random cursor) and filters
+ * after, oversampling by how little of it matches; a small one filters first.
+ */
+async function sampleIds(collection, match, { size, large, hint, maxTimeMS }) {
+  const project = { $project: { _id: 1, t: { $type: '$_id' } } };
+  const sort = { $sort: { _id: 1 } };
+  const pipeline = large
+    ? [{ $sample: { size } }, { $match: match }, project, sort]
+    : [{ $match: match }, { $sample: { size } }, project, sort];
+  return collection
+    .aggregate(pipeline, {
+      maxTimeMS,
+      allowDiskUse: true,
+      ...(large ? {} : withHint(hint)),
+      ...READ_OPTIONS,
+    })
+    .toArray();
+}
+
+/**
+ * Cut one bracket's sorted sample into `parts` ranges: boundaries at
+ * `sample[floor(i·m/parts)]`, duplicates dropped, the outer ranges open-ended.
+ * Returns `[{ scope, count }]` — `count` being the samples the range holds.
+ */
+function sliceBracket(bracket, ids, parts) {
+  if (parts <= 1 || ids.length < 2) {
+    return [{ scope: { kind: 'id-range', bracket }, count: ids.length }];
+  }
+  const bounds = [];
+  for (let i = 1; i < parts; i++) {
+    const position = Math.floor((i * ids.length) / parts);
+    const candidate = ids[position];
+    const previous = bounds.length > 0 ? bounds[bounds.length - 1].id : ids[0];
+    // A boundary equal to the one before (or to the very first id) would
+    // make an empty range — skip it.
+    if (!sameValue(candidate, previous)) bounds.push({ id: candidate, position });
+  }
+  const ranges = [];
+  let start = 0;
+  let lower;
+  for (const bound of bounds) {
+    ranges.push({
+      scope: {
+        kind: 'id-range',
+        bracket,
+        ...(lower !== undefined ? { gte: lower } : {}),
+        lt: bound.id,
+      },
+      count: bound.position - start,
+    });
+    lower = bound.id;
+    start = bound.position;
+  }
+  ranges.push({
+    scope: { kind: 'id-range', bracket, ...(lower !== undefined ? { gte: lower } : {}) },
+    count: ids.length - start,
+  });
+  return ranges;
+}
+
+/** One partition per bracket present, no bounds — the plan when sampling is not worth it */
+const perBracket = (brackets, estimate) =>
+  brackets.map((bracket) => ({
+    scope: { kind: 'id-range', bracket },
+    estimate: Math.round(estimate / brackets.length),
+  }));
+
+/** Largest first: a lane starts on the work that would otherwise finish last */
+function largestFirst(partitions) {
+  return partitions.sort((a, b) => b.estimate - a.estimate);
+}
+
+/**
+ * Plan the partitions of one pass. `ctx`:
+ * `{ collection, match, hint?, maxParallel, settings: spec.partitions, maxTimeMS? }`.
+ * Returns `{ epoch, method, estimate, partitions: [{ scope, estimate }], degraded? }`
+ * — `method` says how it was planned (`empty`, `single`, `brackets`,
+ * `match-first`, `sample-first`), `degraded: 'sample-timeout'` that the sample
+ * ran out of time.
+ */
+async function planIdRanges({ collection, match, hint, maxParallel, settings, maxTimeMS }) {
+  const target = targetPartitions(settings, maxParallel);
+  const cap = target * settings.minPartitionDocs;
+  const count = await collection.countDocuments(match, {
+    limit: cap,
+    ...withHint(hint),
+    ...READ_OPTIONS,
+  });
+  if (count === 0) return { epoch: null, method: 'empty', estimate: 0, partitions: [] };
+  const parts = Math.max(1, Math.min(target, Math.ceil(count / settings.minPartitionDocs)));
+  const brackets = await presentBrackets(collection, match, hint);
+  if (brackets.length === 0) {
+    // Matched a moment ago, gone now — the next pass will know.
+    return { epoch: null, method: 'empty', estimate: 0, partitions: [] };
+  }
+  if (parts === 1) {
+    return {
+      epoch: null,
+      method: brackets.length === 1 ? 'single' : 'brackets',
+      estimate: count,
+      // The count stopped at its limit: there are at least that many.
+      ...(count >= cap ? { atLeast: true } : {}),
+      partitions: perBracket(brackets, count),
+    };
+  }
+
+  const large = count >= cap;
+  let total = count;
+  let ratio = 1;
+  if (large) {
+    total = Math.max(count, await collection.estimatedDocumentCount());
+    ratio = count / total;
+  }
+  const size = Math.min(settings.sampleSize, 100 * parts);
+  // A large match samples the whole collection first, oversampled by how
+  // little of it matches — but below the share past which `$sample` stops
+  // walking a random cursor and scans and sorts every document instead.
+  const requested = large ? sampleRequest(size, ratio, total) : size;
+  let sample;
+  try {
+    sample = await sampleIds(collection, match, {
+      size: requested,
+      large,
+      hint,
+      maxTimeMS: maxTimeMS ?? SAMPLE_TIMEOUT_MS,
+    });
+  } catch (error) {
+    if (error?.code !== MAX_TIME_EXPIRED) throw error;
+    return {
+      epoch: null,
+      method: 'brackets',
+      estimate: count,
+      ...(count >= cap ? { atLeast: true } : {}),
+      degraded: 'sample-timeout',
+      partitions: perBracket(brackets, count),
+    };
+  }
+  // How many documents match, as far as the sample can tell: the bounded
+  // count when it was not capped, the matched share of the sampled ones when
+  // it was.
+  const estimate = large
+    ? Math.max(count, Math.round(total * Math.min(1, sample.length / requested)))
+    : count;
+
+  // The sample grouped by bracket, in server order — it arrives sorted.
+  const groups = new Map();
+  for (const { _id: id, t } of sample) {
+    const bracket = bracketOfType(t);
+    if (!groups.has(bracket)) groups.set(bracket, []);
+    groups.get(bracket).push(id);
+  }
+  const partitions = [];
+  const sampled = Math.max(1, sample.length);
+  const present = new Set(brackets);
+  for (const bracket of ID_BRACKETS) {
+    const ids = groups.get(bracket.name);
+    if (ids === undefined) {
+      // In the collection but not in the sample: one open partition keeps it covered.
+      if (present.has(bracket.name)) {
+        partitions.push({ scope: { kind: 'id-range', bracket: bracket.name }, estimate: 0 });
+      }
+      continue;
+    }
+    const share = Math.max(1, Math.round((parts * ids.length) / sampled));
+    for (const range of sliceBracket(bracket.name, ids, bracket.splittable ? share : 1)) {
+      partitions.push({
+        scope: range.scope,
+        estimate: Math.round((estimate * range.count) / sampled),
+      });
+    }
+  }
+  return {
+    epoch: null,
+    method: large ? 'sample-first' : 'match-first',
+    estimate,
+    partitions: largestFirst(partitions),
+  };
+}
+
+/**
+ * The batch query of an `_id`-range partition: its scope and the keyset after
+ * the last id, sorted by `_id` through the version index.
+ */
+function idRangeBatchQuery(scope, cursor, { limit, match, hint }) {
+  const conditions = [match, scopeFilter(scope)];
+  if (cursor?.lastId !== undefined) conditions.push(keysetFilter(cursor.lastId));
+  return {
+    filter: { $and: conditions },
+    options: { sort: { _id: 1 }, limit, ...withHint(hint), ...READ_OPTIONS },
+  };
+}
+
+/** The cursor after a batch: the last id read — `null` when the partition is done */
+function idRangeAdvance(cursor, docs, { limit }) {
+  if (docs.length < limit) return null;
+  return idRangePast(cursor, docs);
+}
+
+/** The cursor just past `docs` — where a batch cut short (or a document stepped over) ends */
+function idRangePast(cursor, docs) {
+  return { ...cursor, lastId: docs[docs.length - 1]._id };
+}
+
+const idRangePartitioner = Object.freeze({
+  id: 'id',
+  plan: planIdRanges,
+  batchQuery: idRangeBatchQuery,
+  advance: idRangeAdvance,
+  past: idRangePast,
+  /** Nothing to add to the optimistic filter — `_id` is already in it */
+  writeFilter: () => ({}),
+  /** Every transformation is fine: `_id` is checked by stampedDiff itself */
+  checkTransform: () => null,
+  /** An `_id` range never goes stale */
+  stale: () => false,
+});
+
+module.exports = {
+  MAX_TIME_EXPIRED,
+  RANDOM_CURSOR_SHARE,
+  SAMPLE_TIMEOUT_MS,
+  idRangePartitioner,
+  largestFirst,
+  sampleRequest,
+  sliceBracket,
+  targetPartitions,
+};

package/src/core/background-runner.js

@@ -0,0 +1,305 @@
+const { ConfigInvalidError, RunAbortedError } = require('../errors/index.js');
+const { errorText } = require('../utils/error.js');
+const { jitter, sleep } = require('./background-throttle.js');
+const { assertSliceMs } = require('./background-spec.js');
+const { watchOptions } = require('./background-watch.js');
+const { MigratorKit } = require('./migrator.js');
+
+/**
+ * The in-process runner: background migrations driven from inside the
+ * application, with no queue — `concurrency` lane loops shared by every
+ * runnable background migration, round-robin, each taking a slice at a time
+ * (never more than a migration's `maxParallel` here; the leases cap it
+ * across every process). Run several application instances and they share
+ * the work through the same leases.
+ *
+ * A lane loop never throws: a failed slice goes to `onError` and that
+ * migration backs off. The drift watch runs every `verifyIntervalMs`; the
+ * live drift watcher (change streams) runs alongside when `watch` says so —
+ * by default when `backgroundDrift` is `'stream'` or `'both'`.
+ */
+
+const DEFAULTS = Object.freeze({
+  concurrency: 1,
+  pollIntervalMs: 5_000,
+  verifyIntervalMs: 600_000,
+});
+
+/** The longest a failing migration backs off for */
+const MAX_BACKOFF_MS = 60_000;
+
+function readOptions(options) {
+  const concurrency = options.concurrency ?? DEFAULTS.concurrency;
+  if (!Number.isSafeInteger(concurrency) || concurrency < 1 || concurrency > 64) {
+    throw new ConfigInvalidError('concurrency must be an integer from 1 to 64', { concurrency });
+  }
+  const pollIntervalMs = options.pollIntervalMs ?? DEFAULTS.pollIntervalMs;
+  if (!Number.isSafeInteger(pollIntervalMs) || pollIntervalMs < 10) {
+    throw new ConfigInvalidError('pollIntervalMs must be an integer ≥ 10', { pollIntervalMs });
+  }
+  const verifyIntervalMs = options.verifyIntervalMs ?? DEFAULTS.verifyIntervalMs;
+  if (
+    verifyIntervalMs !== false &&
+    (!Number.isSafeInteger(verifyIntervalMs) || verifyIntervalMs < 100)
+  ) {
+    throw new ConfigInvalidError('verifyIntervalMs must be false or an integer ≥ 100', {
+      verifyIntervalMs,
+    });
+  }
+  if (options.sliceMs !== undefined) assertSliceMs(options.sliceMs);
+  if (options.onError !== undefined && typeof options.onError !== 'function') {
+    throw new ConfigInvalidError('onError must be a function');
+  }
+  if (options.signal !== undefined && !(options.signal instanceof AbortSignal)) {
+    throw new ConfigInvalidError('signal must be an AbortSignal');
+  }
+  if (options.kit !== undefined && !(options.kit instanceof MigratorKit)) {
+    throw new ConfigInvalidError('kit must be a MigratorKit');
+  }
+  const { watch } = options;
+  if (watch !== undefined && typeof watch !== 'boolean') {
+    if (watch === null || typeof watch !== 'object') {
+      throw new ConfigInvalidError('watch must be a boolean or the watcher options');
+    }
+    watchOptions(watch);
+  }
+  return { concurrency, pollIntervalMs, verifyIntervalMs };
+}
+
+/**
+ * Start the runner. `options`: `{ kit | config, kitOptions?, concurrency?,
+ * pollIntervalMs?, sliceMs?, verifyIntervalMs?, signal?, onError? }`.
+ * Returns `{ kit, running, stop() }` — `stop()` resolves once every lane
+ * has released its lease (at the next batch) and, when the runner made the
+ * kit, the kit is disconnected.
+ */
+function startBackgroundRunner(options = {}) {
+  const { concurrency, pollIntervalMs, verifyIntervalMs } = readOptions(options);
+  const kit = options.kit ?? new MigratorKit(options.config ?? {}, options.kitOptions ?? {});
+  const ownsKit = options.kit === undefined;
+  const controller = new AbortController();
+  const signal = controller.signal;
+  const onOuterAbort = () => controller.abort(options.signal.reason);
+  options.signal?.addEventListener('abort', onOuterAbort, { once: true });
+  if (options.signal?.aborted) controller.abort(options.signal.reason);
+
+  const report = (error, migration) => {
+    try {
+      options.onError?.(error, migration);
+    } catch {
+      // A throwing onError is its own problem.
+    }
+    kit.logger.warn(
+      `⚠ Background runner${migration ? ` (${migration})` : ''}: ${errorText(error)}`,
+      { ...(migration ? { background: migration } : {}), error: errorText(error) },
+    );
+  };
+
+  const shared = {
+    runnable: [],
+    refreshedAt: -Infinity,
+    refreshing: undefined,
+    next: 0,
+    active: new Map(),
+    backoff: new Map(),
+    failures: new Map(),
+  };
+
+  /** The runnable list, refreshed at most every poll interval — by one lane at a time */
+  async function runnable() {
+    if (Date.now() - shared.refreshedAt < pollIntervalMs) return shared.runnable;
+    shared.refreshing ??= kit
+      .runnableBackground()
+      .then((list) => {
+        shared.runnable = list;
+        shared.refreshedAt = Date.now();
+      })
+      .finally(() => {
+        shared.refreshing = undefined;
+      });
+    await shared.refreshing;
+    return shared.runnable;
+  }
+
+  const forget = () => {
+    shared.refreshedAt = -Infinity;
+  };
+
+  /** The next background migration a lane may take, round-robin, or `undefined` */
+  function pick(list) {
+    const now = Date.now();
+    for (let i = 0; i < list.length; i++) {
+      const entry = list[(shared.next + i) % list.length];
+      if ((shared.backoff.get(entry.migration) ?? 0) > now) continue;
+      if ((shared.active.get(entry.migration) ?? 0) >= entry.maxParallel) continue;
+      shared.next = (shared.next + i + 1) % list.length;
+      return entry;
+    }
+    return undefined;
+  }
+
+  const backOff = (migration, ms) => shared.backoff.set(migration, Date.now() + jitter(ms));
+
+  async function lane() {
+    while (!signal.aborted) {
+      let entry;
+      try {
+        entry = pick(await runnable());
+      } catch (error) {
+        report(error);
+        await sleep(pollIntervalMs, signal).catch(() => undefined);
+        continue;
+      }
+      if (entry === undefined) {
+        await sleep(pollIntervalMs, signal).catch(() => undefined);
+        continue;
+      }
+      const name = entry.migration;
+      shared.active.set(name, (shared.active.get(name) ?? 0) + 1);
+      try {
+        await turn(name);
+        shared.failures.delete(name);
+      } catch (error) {
+        if (signal.aborted) break;
+        const failures = (shared.failures.get(name) ?? 0) + 1;
+        shared.failures.set(name, failures);
+        report(error, name);
+        backOff(name, Math.min(MAX_BACKOFF_MS, 1000 * 2 ** failures));
+      } finally {
+        shared.active.set(name, shared.active.get(name) - 1);
+      }
+    }
+  }
+
+  /** One turn on one background migration: a slice, and the coordinator when it is needed */
+  async function turn(name) {
+    const slice = await kit.runBackgroundSlice(name, {
+      signal,
+      ...(options.sliceMs !== undefined ? { sliceMs: options.sliceMs } : {}),
+    });
+    switch (slice.outcome) {
+      case 'yielded':
+      case 'lost':
+      case 'stopped':
+        return;
+      case 'busy':
+        backOff(name, slice.retryAfterMs ?? pollIntervalMs);
+        return;
+      case 'paused':
+      case 'cancelled':
+      case 'failed':
+        forget();
+        return;
+      default: {
+        // Stale (no plan yet) or exhausted (nothing left to claim): the coordinator decides.
+        const answer = await kit.coordinateBackground(name, { signal, driver: { kind: 'runner' } });
+        if (answer.next === 'done') forget();
+        else if (answer.next !== 'process') backOff(name, answer.retryAfterMs ?? pollIntervalMs);
+        // Every partition is held by a lane elsewhere: look again later, not at once.
+        else if (slice.outcome === 'exhausted') backOff(name, pollIntervalMs);
+      }
+    }
+  }
+
+  async function verifier() {
+    if (verifyIntervalMs === false) return;
+    while (!signal.aborted) {
+      try {
+        await sleep(verifyIntervalMs, signal);
+      } catch {
+        return;
+      }
+      try {
+        const result = await kit.verifyBackground();
+        if (result.drift.length > 0) forget();
+      } catch (error) {
+        report(error);
+      }
+    }
+  }
+
+  /** The live drift watcher, when this runner hosts one — it never throws either */
+  async function watcher() {
+    let wanted = options.watch;
+    try {
+      wanted ??= (await kit.driftMode()) !== 'poll';
+      if (wanted === false || signal.aborted) return undefined;
+      return await kit.watchBackground({
+        ...(typeof wanted === 'object' ? wanted : {}),
+        signal,
+        onError: (error, collection) => report(error, collection),
+      });
+    } catch (error) {
+      if (!signal.aborted) report(error);
+      return undefined;
+    }
+  }
+
+  const watching = watcher();
+  const work = Promise.all([...Array.from({ length: concurrency }, () => lane()), verifier()]);
+  let stopping;
+  let hosted;
+  watching.then((started) => {
+    hosted = started;
+  });
+  return {
+    kit,
+    get running() {
+      return !signal.aborted;
+    },
+    /** The live drift watcher this runner hosts, once started — or undefined */
+    get watcher() {
+      return hosted;
+    },
+    /**
+     * Stop: each lane at its next batch boundary, releasing its lease.
+     * `timeoutMs`: stop waiting for a lane stuck in its transformation (its
+     * lease expires on its own, and the next lane resumes from the last
+     * checkpoint) — so a shutdown is never held for good.
+     */
+    stop({ timeoutMs } = {}) {
+      if (timeoutMs !== undefined && (!Number.isSafeInteger(timeoutMs) || timeoutMs < 0)) {
+        return Promise.reject(
+          new ConfigInvalidError('timeoutMs must be an integer ≥ 0', { timeoutMs }),
+        );
+      }
+      stopping ??= (async () => {
+        if (!signal.aborted) {
+          controller.abort(
+            new RunAbortedError('Background runner stopped', {
+              reason: 'Background runner stopped',
+            }),
+          );
+        }
+        const settled = (async () => {
+          await (await watching)?.stop();
+          await work;
+        })();
+        if (timeoutMs === undefined) {
+          await settled;
+        } else {
+          let timer;
+          const timedOut = await Promise.race([
+            settled.then(() => false),
+            new Promise((resolve) => {
+              timer = setTimeout(() => resolve(true), timeoutMs);
+            }),
+          ]);
+          clearTimeout(timer);
+          if (timedOut) {
+            kit.logger.warn(
+              `⚠ Background runner: lanes still at work after ${timeoutMs}ms — not waiting ` +
+                'for them (their leases expire, and the work resumes from the last checkpoint)',
+              { timeoutMs },
+            );
+          }
+        }
+        options.signal?.removeEventListener('abort', onOuterAbort);
+        if (ownsKit) await kit.disconnect().catch(() => undefined);
+      })();
+      return stopping;
+    },
+  };
+}
+
+module.exports = { BACKGROUND_RUNNER_DEFAULTS: DEFAULTS, startBackgroundRunner };