@alexify/migronaut 2.2.0 → 2.3.0

package/src/core/background-throttle.js

@@ -0,0 +1,269 @@
+const { errorText } = require('../utils/error.js');
+const { isUnauthorized } = require('./shard-info.js');
+
+/**
+ * Pacing a background migration: it rewrites a production collection while
+ * the application uses it, so it must yield to it — before every batch:
+ *
+ * - the author's pause between batches (`pauseMs`);
+ * - the author's own `throttle(ctx)` hook (a returned number is extra pause);
+ * - replication lag: on a replica set, wait while a secondary that counts
+ *   (not hidden, not delayed) lags more than `maxReplicationLagMs` — the
+ *   status read at most every 5 s, and switched off quietly where it cannot
+ *   be read (standalone, mongos) or with one warning where it may not be
+ *   (no `clusterMonitor`).
+ *
+ * The latency-driven batch sizing (AIMD) lives here too.
+ * Clock and sleep are injected, so every rule is tested without waiting.
+ */
+
+/** How often the replication status is read, at most */
+const LAG_CHECK_INTERVAL_MS = 5_000;
+
+/** How often the replica set config (hidden and delayed members) is read, at most */
+const CONFIG_CHECK_INTERVAL_MS = 60_000;
+
+/** How long to wait between two looks at a lag that is too high */
+const LAG_WAIT_MS = 1_000;
+
+/** Errors that mean "no replication status here" — not a replica set member */
+const NO_REPLICATION = new Set([
+  59, // CommandNotFound (mongos)
+  76, // NoReplicationEnabled (standalone)
+  115, // CommandNotSupported
+]);
+
+/** `ms`, jittered down to half of it — so processes that start together drift apart */
+const jitter = (ms) => Math.round(ms * (0.5 + Math.random() / 2));
+
+/** Sleep `ms`, cut short (rejecting with the reason) when `signal` aborts */
+function sleep(ms, signal) {
+  return new Promise((resolve, reject) => {
+    if (signal?.aborted) {
+      reject(signal.reason);
+      return;
+    }
+    if (ms <= 0) {
+      resolve();
+      return;
+    }
+    const onAbort = () => {
+      clearTimeout(timer);
+      reject(signal.reason);
+    };
+    const timer = setTimeout(() => {
+      signal?.removeEventListener('abort', onAbort);
+      resolve();
+    }, ms);
+    signal?.addEventListener('abort', onAbort, { once: true });
+  });
+}
+
+/**
+ * The worst lag of a secondary that counts, in ms — or 0. Hidden and delayed
+ * members are left out: a delayed one lags on purpose, and a hidden one
+ * serves no reads.
+ */
+function replicationLagMs(status, excluded) {
+  const members = Array.isArray(status?.members) ? status.members : [];
+  let primary;
+  for (const member of members) {
+    if (member.stateStr === 'PRIMARY') primary = member;
+  }
+  if (primary?.optimeDate === undefined) return 0;
+  const primaryTime = new Date(primary.optimeDate).getTime();
+  let worst = 0;
+  for (const member of members) {
+    if (member.stateStr !== 'SECONDARY' || excluded.has(member.name)) continue;
+    if (member.optimeDate === undefined) continue;
+    const lag = primaryTime - new Date(member.optimeDate).getTime();
+    if (lag > worst) worst = lag;
+  }
+  return worst;
+}
+
+/** Hidden or delayed member hosts, from the replica set config */
+function excludedMembers(config) {
+  const hosts = new Set();
+  for (const member of config?.config?.members ?? []) {
+    const delay = member.secondaryDelaySecs ?? member.slaveDelay ?? 0;
+    if (member.hidden === true || Number(delay) > 0) hosts.add(member.host);
+  }
+  return hosts;
+}
+
+/**
+ * A throttle for one lane. `options`: `{ spec, db, logger, name, now?,
+ * sleep?, signal? }`. `beforeBatch(ctx)` waits as the rules say; it rejects
+ * only when `signal` aborts.
+ */
+function createThrottle({ spec, db, logger, name, now = Date.now, wait = sleep, warned }) {
+  const maxLag = spec.maxReplicationLagMs;
+  let lagEnabled = maxLag !== false && typeof db?.admin === 'function';
+  let lastLagCheck = -Infinity;
+  let lastConfigCheck = -Infinity;
+  let excluded = new Set();
+  let first = true;
+  // Set when the last wait was for a lagging secondary — an overload signal.
+  let lagged = false;
+  // One warning per process for a lag that cannot be read for lack of rights.
+  const warnings = warned ?? new Set();
+
+  async function currentLag() {
+    const admin = db.admin();
+    if (now() - lastConfigCheck >= CONFIG_CHECK_INTERVAL_MS) {
+      lastConfigCheck = now();
+      try {
+        excluded = excludedMembers(await admin.command({ replSetGetConfig: 1 }));
+      } catch {
+        // Without the config every secondary counts — the cautious reading.
+      }
+    }
+    return replicationLagMs(await admin.command({ replSetGetStatus: 1 }), excluded);
+  }
+
+  async function waitForLag(signal) {
+    if (!lagEnabled || now() - lastLagCheck < LAG_CHECK_INTERVAL_MS) return;
+    for (;;) {
+      lastLagCheck = now();
+      let lag;
+      try {
+        lag = await currentLag();
+      } catch (error) {
+        lagEnabled = false;
+        if (isUnauthorized(error) && !warnings.has('lag')) {
+          warnings.add('lag');
+          logger.warn(
+            '⚠ Background migrations cannot read the replication lag (replSetGetStatus needs ' +
+              'the clusterMonitor role) — they no longer wait for secondaries',
+            { background: name, error: errorText(error) },
+          );
+        } else if (!NO_REPLICATION.has(error?.code)) {
+          logger.debug(`Replication lag unreadable — not waiting for it: ${errorText(error)}`, {
+            background: name,
+          });
+        }
+        return;
+      }
+      if (lag <= maxLag) return;
+      lagged = true;
+      logger.debug(`Replication lag ${lag}ms > ${maxLag}ms — waiting`, {
+        background: name,
+        lagMs: lag,
+      });
+      await wait(LAG_WAIT_MS, signal);
+    }
+  }
+
+  return {
+    /** Whether the last `beforeBatch` waited for a lagging secondary — read once */
+    takeLagged() {
+      const was = lagged;
+      lagged = false;
+      return was;
+    },
+    /** Everything a lane waits for before reading its next batch */
+    async beforeBatch({ signal, generation, partition, batchSize, extraPauseMs = 0 }) {
+      const pause = (first ? 0 : spec.pauseMs) + extraPauseMs;
+      if (pause > 0) await wait(pause, signal);
+      first = false;
+      if (typeof spec.throttle === 'function') {
+        const extra = await spec.throttle({
+          name,
+          ...(spec.collection !== undefined ? { collection: spec.collection } : {}),
+          generation,
+          partition,
+          batchSize,
+          signal,
+        });
+        if (typeof extra === 'number' && extra > 0) await wait(extra, signal);
+      }
+      await waitForLag(signal);
+    },
+  };
+}
+
+// ─── Adaptive batch sizing (AIMD) ─────────────────────────────────────────────
+
+/** How often a throttle change is reported, at most, per controller */
+const THROTTLE_REPORT_INTERVAL_MS = 10_000;
+
+/** The first extra pause once the batch is as small as it gets */
+const FIRST_BACKOFF_MS = 100;
+
+/**
+ * A latency-driven batch size, additive-increase / multiplicative-decrease —
+ * TCP's answer to "how fast may I go without knowing the capacity": every
+ * batch written within `targetLatencyMs` grows the next one a little; one
+ * slower, or a sign of overload (a write concern timeout, a transient error,
+ * a lagging secondary), halves it — at most once per round trip, so one slow
+ * moment is not punished twice. At `minBatchSize` it backs off in time
+ * instead: an extra pause that doubles up to `maxPauseMs` and halves again as
+ * things recover. The first batch only warms up.
+ *
+ * Works on every topology — behind a mongos and on a standalone it is the
+ * only signal there is. `settings` is the resolved `spec.adaptive`;
+ * `initial` the state saved on the partition (`{ batchSize, pauseMs }`).
+ */
+function createAdaptive(settings, { initial, now = Date.now } = {}) {
+  const { targetLatencyMs, minBatchSize, maxBatchSize, maxPauseMs } = settings;
+  const increase = Math.max(1, Math.floor(maxBatchSize / 20));
+  let size = clamp(initial?.batchSize ?? maxBatchSize, minBatchSize, maxBatchSize);
+  let pause = clamp(initial?.pauseMs ?? 0, 0, maxPauseMs);
+  let warm = initial !== undefined;
+  let lastDecrease = -Infinity;
+  let lastReport = -Infinity;
+  let pendingReason;
+
+  return {
+    batchSize: () => size,
+    pauseMs: () => pause,
+    state: () => ({ batchSize: size, pauseMs: pause }),
+    /**
+     * Judge one batch: `{ latencyMs, overloaded? }`. Returns the change to
+     * report — `{ reason, batchSize, pauseMs }` with `reason` `'slow'`,
+     * `'overload'` or `'recover'` — at most every 10 s, else `undefined`.
+     */
+    record({ latencyMs, overloaded = false }) {
+      if (!warm) {
+        warm = true;
+        return undefined;
+      }
+      const slow = latencyMs > targetLatencyMs;
+      let reason;
+      if (overloaded || slow) {
+        // Once per round trip: the batch that saw the last decrease does not count twice.
+        if (now() - lastDecrease < Math.max(latencyMs, 1)) return undefined;
+        lastDecrease = now();
+        reason = overloaded ? 'overload' : 'slow';
+        if (size > minBatchSize) size = Math.max(minBatchSize, Math.floor(size / 2));
+        else pause = Math.min(maxPauseMs, Math.max(FIRST_BACKOFF_MS, pause * 2));
+      } else if (pause > 0 || size < maxBatchSize) {
+        reason = 'recover';
+        if (pause > 0) pause = Math.floor(pause / 2);
+        else size = Math.min(maxBatchSize, size + increase);
+      }
+      if (reason === undefined) return undefined;
+      pendingReason =
+        pendingReason === 'recover' || pendingReason === undefined ? reason : pendingReason;
+      if (now() - lastReport < THROTTLE_REPORT_INTERVAL_MS) return undefined;
+      lastReport = now();
+      const report = { reason: pendingReason, batchSize: size, pauseMs: pause };
+      pendingReason = undefined;
+      return report;
+    },
+  };
+}
+
+const clamp = (value, min, max) => Math.min(max, Math.max(min, value));
+
+module.exports = {
+  LAG_CHECK_INTERVAL_MS,
+  THROTTLE_REPORT_INTERVAL_MS,
+  createAdaptive,
+  createThrottle,
+  excludedMembers,
+  jitter,
+  replicationLagMs,
+  sleep,
+};

package/src/core/background-watch-plan.js

@@ -0,0 +1,164 @@
+/**
+ * The live drift watcher's decisions, kept pure: which background migrations
+ * a collection's watcher upgrades with, what its change stream asks the
+ * server for, how a stream error is read, and when a resume token is due.
+ * background-watch.js does the I/O.
+ */
+
+/** Server errors that mean the stream cannot resume from where it was */
+const HISTORY_LOST = new Set([
+  286, // ChangeStreamHistoryLost — the oplog moved past the token
+  280, // ChangeStreamFatalError
+  136, // CappedPositionLost
+]);
+/** Not authorized to open (or keep) a change stream there */
+const UNAUTHORIZED = 13;
+/** "The $changeStream stage is only supported on replica sets" — a standalone */
+const UNSUPPORTED = 40573;
+
+/** Events that end a stream for good: the collection is gone or renamed */
+const ENDING = new Set(['drop', 'rename', 'dropDatabase', 'invalidate']);
+
+/**
+ * The background migrations a collection's watcher may upgrade with — its
+ * edges: completed, forward, declarative ones (a step migration cannot run
+ * on one document). Returns `{ edges, target }`: `edges` by the version they
+ * start from (the first registered wins a tie), `target` the highest
+ * version any reaches — the shape a document should have at least.
+ */
+function edgesOf(states, collection) {
+  const edges = new Map();
+  let target;
+  for (const state of states) {
+    const spec = state.spec;
+    if (spec?.collection !== collection || spec.mode !== 'declarative') continue;
+    if (state.status !== 'completed' || state.direction === 'revert') continue;
+    if (!edges.has(spec.from)) {
+      edges.set(spec.from, { name: state._id, from: spec.from, to: spec.to });
+    }
+    if (target === undefined || spec.to > target) target = spec.to;
+  }
+  return { edges, target };
+}
+
+/** Statuses after which a background migration does nothing more */
+const SETTLED = new Set(['completed', 'failed', 'cancelled']);
+
+/**
+ * The revert that makes a collection's watcher stand aside, if any: one not
+ * settled yet — the watcher would upgrade straight back what it rewrites.
+ */
+function suspendedBy(states, collection) {
+  for (const state of states) {
+    if (state.spec?.collection !== collection || state.direction !== 'revert') continue;
+    if (!SETTLED.has(state.status)) return state._id;
+  }
+  return undefined;
+}
+
+/**
+ * The change stream's pipeline: only the writes that can leave a document
+ * below `target` — an insert or replace of an old shape, an update that sets
+ * the version field below it or to null, one that removes it — plus the
+ * events that end the stream. No `updateLookup`: the watcher reads the one
+ * document an event names, never one per update of the collection. An update
+ * that does not touch the version field matches nothing (`$exists` guards
+ * the null test, which would otherwise match every such update).
+ */
+function watchPipeline(field, target) {
+  const full = `fullDocument.${field}`;
+  const updated = `updateDescription.updatedFields.${field}`;
+  return [
+    {
+      $match: {
+        $or: [
+          {
+            operationType: { $in: ['insert', 'replace'] },
+            $or: [{ [full]: { $lt: target } }, { [full]: { $exists: false } }, { [full]: null }],
+          },
+          {
+            operationType: 'update',
+            $or: [
+              { [updated]: { $lt: target } },
+              { [updated]: { $exists: true, $type: 'null' } },
+              { 'updateDescription.removedFields': field },
+            ],
+          },
+          { operationType: { $in: [...ENDING] } },
+        ],
+      },
+    },
+    { $project: { operationType: 1, documentKey: 1, clusterTime: 1 } },
+  ];
+}
+
+/**
+ * How a stream error is met: `history-lost` (start over from now, after a
+ * drift probe of the collection), `unauthorized` (leave the collection to
+ * the polling watch), `unsupported` (no change streams here at all), or
+ * `retry` (reopen from the last token, after a backoff).
+ */
+function classifyStreamError(error) {
+  const code = error?.code;
+  if (HISTORY_LOST.has(code)) return 'history-lost';
+  if (code === UNAUTHORIZED) return 'unauthorized';
+  if (code === UNSUPPORTED) return 'unsupported';
+  if (error?.hasErrorLabel?.('NonResumableChangeStreamError')) return 'history-lost';
+  return 'retry';
+}
+
+/** Whether an event ends the stream (the collection dropped or renamed) */
+const isEnding = (event) => ENDING.has(event?.operationType);
+
+/** Milliseconds since the event happened — the stream's lag — or `undefined` without a time */
+function lagOf(event, now) {
+  const time = event?.clusterTime;
+  if (time === undefined || time === null) return undefined;
+  // A BSON Timestamp: seconds in `t` (the high 32 bits), an ordinal in `i`.
+  const seconds = time.t;
+  return typeof seconds === 'number' ? Math.max(0, now - seconds * 1000) : undefined;
+}
+
+/** Whether a resume token is due: at most every `checkpointMs`, and always when told to */
+function tokenDue(lastSavedAt, now, checkpointMs, { force = false } = {}) {
+  return force || lastSavedAt === undefined || now - lastSavedAt >= checkpointMs;
+}
+
+/**
+ * A leader's view of its collection, in one pass over the states: the revert
+ * it stands aside for (`suspended`), its `edges` and `target` (as
+ * {@link edgesOf}), and every state by name.
+ */
+function watchView(states, collection) {
+  const edges = new Map();
+  const byName = new Map();
+  let target;
+  let suspended;
+  for (const state of states) {
+    byName.set(state._id, state);
+    const spec = state.spec;
+    if (spec?.collection !== collection) continue;
+    if (state.direction === 'revert') {
+      if (suspended === undefined && !SETTLED.has(state.status)) suspended = state._id;
+      continue;
+    }
+    if (spec.mode !== 'declarative' || state.status !== 'completed') continue;
+    if (!edges.has(spec.from)) {
+      edges.set(spec.from, { name: state._id, from: spec.from, to: spec.to });
+    }
+    if (target === undefined || spec.to > target) target = spec.to;
+  }
+  return { suspended, edges, target, byName };
+}
+
+module.exports = {
+  HISTORY_LOST,
+  classifyStreamError,
+  edgesOf,
+  isEnding,
+  lagOf,
+  suspendedBy,
+  tokenDue,
+  watchPipeline,
+  watchView,
+};

package/src/core/background-watch-store.js

@@ -0,0 +1,78 @@
+const os = require('node:os');
+const { backgroundCollectionNames } = require('./config.js');
+const { READ_OPTIONS } = require('./server-info.js');
+
+/**
+ * Where the live drift watcher keeps what outlives a process — one document
+ * per watched collection in `<backgroundCollection>_watch`:
+ * `{ _id: collection, resumeToken?, target, edges, state, leader,
+ * counters, lastEventAt, lastError, updatedAt }`.
+ *
+ * Only the leader writes, and every write is fenced by `leader.owner`: a
+ * watcher that lost its lock (and so its leadership) without knowing it
+ * cannot move the token back. The token never leaves this module's status
+ * reads — it is the stream's position, of no use to anyone else.
+ */
+class BackgroundWatchStore {
+  #collection;
+
+  constructor(db, backgroundCollection) {
+    this.#collection = db.collection(backgroundCollectionNames(backgroundCollection).watch);
+  }
+
+  /** The stored document of a collection's watcher, token included, or null */
+  async get(collection) {
+    return this.#collection.findOne({ _id: collection }, READ_OPTIONS);
+  }
+
+  /** Become the writer of a collection's document — called while holding its watcher lock */
+  async lead(collection, owner) {
+    await this.#collection.updateOne(
+      { _id: collection },
+      {
+        $set: {
+          leader: { owner, host: os.hostname(), pid: process.pid, at: new Date() },
+          updatedAt: new Date(),
+        },
+        $setOnInsert: { counters: {} },
+      },
+      { upsert: true },
+    );
+  }
+
+  /**
+   * Save what the leader knows: `fields` set (a `resumeToken`, a `state`, …),
+   * `counters` added, `unset` removed. Resolves whether `owner` still leads.
+   */
+  async save(collection, owner, { fields = {}, counters = {}, unset = [] } = {}) {
+    const update = { $set: { ...fields, updatedAt: new Date() } };
+    const inc = {};
+    for (const [key, value] of Object.entries(counters)) {
+      if (value !== 0) inc[`counters.${key}`] = value;
+    }
+    if (Object.keys(inc).length > 0) update.$inc = inc;
+    if (unset.length > 0) {
+      update.$unset = {};
+      for (const key of unset) update.$unset[key] = '';
+    }
+    const result = await this.#collection.updateOne(
+      { _id: collection, 'leader.owner': owner },
+      update,
+    );
+    return result.matchedCount === 1;
+  }
+
+  /** Every watched collection's document — or one — without its resume token */
+  async status(collection) {
+    const projection = { resumeToken: 0 };
+    if (collection !== undefined) {
+      return this.#collection.findOne({ _id: collection }, { projection, ...READ_OPTIONS });
+    }
+    return this.#collection
+      .find({}, { projection, ...READ_OPTIONS })
+      .sort({ _id: 1 })
+      .toArray();
+  }
+}
+
+module.exports = { BackgroundWatchStore };