@alexify/migronaut 2.0.0 → 2.2.0

package/src/core/sequence.js

@@ -0,0 +1,134 @@
+const fs = require('node:fs/promises');
+const { MigrationBlockedError, MigrationFileNotFoundError } = require('../errors/index.js');
+
+/**
+ * The migration sequence: the files on disk, in name order, measured against
+ * the names the changelog holds as applied — which are pending, which arrived
+ * late, which stand in the way of an ordered step, and in what order applied
+ * records are reverted. Pure apart from reading the directory; what to do
+ * with each answer (refuse, warn, wait) is the kit's decision.
+ */
+
+/** Declaration files sit next to TypeScript migrations and are never one */
+const DECLARATION_SUFFIXES = ['.d.ts', '.d.mts', '.d.cts'];
+
+/** The migration files in `dir`, sorted ascending — empty when the directory does not exist */
+async function listMigrationFiles(dir, extensions) {
+  let entries;
+  try {
+    entries = await fs.readdir(dir, { withFileTypes: true });
+  } catch (error) {
+    if (error.code === 'ENOENT') return [];
+    throw error;
+  }
+  const matches = [];
+  for (const entry of entries) {
+    // A directory named `foo.js`, a dotfile, or a `types.d.ts` sitting next
+    // to the migrations is not a migration — including it would hard-fail
+    // the whole run with MigrationInvalidExportError.
+    if (!entry.isFile()) continue;
+    const file = entry.name;
+    if (file.startsWith('.')) continue;
+    if (DECLARATION_SUFFIXES.some((suffix) => file.endsWith(suffix))) continue;
+    for (const ext of extensions) {
+      if (file.endsWith(ext)) {
+        matches.push(file);
+        break;
+      }
+    }
+  }
+  return matches.sort();
+}
+
+/**
+ * The files of `sequence` with no applied name — all of them, or only those
+ * sorting before `before`. Pending means "no applied record": a `'failed'`
+ * trace counts, so a migration that failed stops the line exactly like one
+ * that never ran.
+ */
+function pendingIn(sequence, appliedNames, before) {
+  const pending = [];
+  for (const file of sequence) {
+    if (before !== undefined && file >= before) break;
+    if (!appliedNames.has(file)) pending.push(file);
+  }
+  return pending;
+}
+
+/**
+ * Keep only the pending migrations up to and including `to`.
+ *
+ * `to` must name a migration that exists; it may already be applied (then
+ * nothing before it is pending either, and the result is empty), which is
+ * what makes `up --to X` idempotent — running it twice is a no-op rather
+ * than an error.
+ */
+function truncateAtTarget(pending, allFiles, to) {
+  if (!allFiles.includes(to)) {
+    throw new MigrationFileNotFoundError('Migration file not found', { to });
+  }
+  const kept = [];
+  for (const file of pending) {
+    if (file > to) break;
+    kept.push(file);
+  }
+  return kept;
+}
+
+/** The name that sorts last — '' for none */
+function newestOf(names) {
+  let newest = '';
+  for (const name of names) {
+    if (name > newest) newest = name;
+  }
+  return newest;
+}
+
+/**
+ * Out-of-order arrivals among `targets`: pending files that sort before the
+ * newest applied name — migrations merged late from a parallel branch, which
+ * will run after migrations authored later. `null` when there are none.
+ */
+function lateArrivals(targets, appliedNames) {
+  if (targets.length === 0 || appliedNames.size === 0) return null;
+  const newestApplied = newestOf(appliedNames);
+  const late = [];
+  for (const target of targets) {
+    if (!appliedNames.has(target) && target < newestApplied) late.push(target);
+  }
+  return late.length > 0 ? { late, newestApplied } : null;
+}
+
+/** The names of the records to revert, newest first unless already in revert order */
+function revertOrder(records, preserveOrder) {
+  const names = [];
+  for (const record of records) names.push(record.name);
+  if (!preserveOrder) {
+    names.sort();
+    names.reverse();
+  }
+  return names;
+}
+
+/**
+ * The refusal of an ordered step: "`subject` is blocked: N `what`: a, b".
+ * `context` carries `blockedBy` (and `failed`, the blockers with a failed
+ * trace — a stopped line rather than one still on its way).
+ */
+function blockedError(subject, what, context) {
+  const { blockedBy } = context;
+  return new MigrationBlockedError(
+    `${subject} is blocked: ${blockedBy.length} ${what}: ${blockedBy.join(', ')}`,
+    context,
+  );
+}
+
+module.exports = {
+  blockedError,
+  lateArrivals,
+  listMigrationFiles,
+  newestOf,
+  pendingIn,
+  revertOrder,
+  truncateAtTarget,
+};

package/src/core/server-info.js

@@ -0,0 +1,63 @@
+/**
+ * What converge and audit read about the server, and how: the read options
+ * forced onto every read, the pace of reading many collections, the server's
+ * version and topology, and the error codes for a namespace or an index that
+ * is not there. Mechanism only — no logger, no decisions.
+ */
+
+/**
+ * Read options forced onto both reads: the primary (a secondary may not have
+ * an index build yet), and BSON values as plain JavaScript — an injected
+ * client configured with `promoteValues: false` or `useBigInt64: true` would
+ * otherwise hand back `Int32` objects or `1n`, and everything would compare as
+ * changed.
+ */
+const READ_OPTIONS = Object.freeze({
+  readPreference: 'primary',
+  promoteLongs: true,
+  promoteValues: true,
+  useBigInt64: false,
+  bsonRegExp: false,
+});
+
+/** listIndexes calls in flight while reading many collections — a pace, not a pool */
+const READ_CONCURRENCY = 8;
+
+/**
+ * What the server is: a mongos in front of shards (its shard keys matter to
+ * prune), and its version (what it can change in place). Best-effort — a
+ * server that refuses to say gets the conservative answer: no in-place
+ * extras, no shard-key handling.
+ */
+async function readServer(db) {
+  const server = { mongos: false, version: undefined };
+  if (typeof db.admin !== 'function') return server;
+  try {
+    const hello = await db.admin().command({ hello: 1 });
+    server.mongos = hello?.msg === 'isdbgrid';
+  } catch {
+    // Unknown — treated as a replica set or standalone.
+  }
+  try {
+    const info = await db.admin().command({ buildInfo: 1 });
+    const [major, minor, patch] = Array.isArray(info?.versionArray) ? info.versionArray : [];
+    if (Number.isInteger(major) && Number.isInteger(minor)) {
+      server.version = { major, minor, ...(Number.isInteger(patch) ? { patch } : {}) };
+    }
+  } catch {
+    // Unknown version: only the always-available in-place changes.
+  }
+  return server;
+}
+
+const NAMESPACE_NOT_FOUND = 26;
+
+const INDEX_NOT_FOUND = 27;
+
+module.exports = {
+  INDEX_NOT_FOUND,
+  NAMESPACE_NOT_FOUND,
+  READ_CONCURRENCY,
+  READ_OPTIONS,
+  readServer,
+};

package/src/errors/index.js

@@ -204,6 +204,62 @@ class OutOfOrderMigrationError extends MigronautError {
   }
 }
 
+/**
+ * Thrown by an `ordered` single-file run that would apply or revert out of
+ * sequence: an earlier migration is still pending (`up`), or one applied later
+ * is still applied (`down`). `context.blockedBy` names what must go first.
+ */
+class MigrationBlockedError extends MigronautError {
+  constructor(message, context, options) {
+    super('MIGRATION_BLOCKED', message, context, options);
+    this.name = 'MigrationBlockedError';
+  }
+}
+
+/**
+ * Thrown by the queue adapter when a job's payload fails the contract check.
+ * Job data comes back from Redis, so it is untrusted input — never a config
+ * mistake of the process that reads it.
+ */
+class QueueJobInvalidError extends MigronautError {
+  constructor(message, context, options) {
+    super('QUEUE_JOB_INVALID', message, context, options);
+    this.name = 'QueueJobInvalidError';
+  }
+}
+
+/**
+ * Thrown by a queue group's `wait()` when one of its jobs failed or the wait
+ * timed out. The worker's typed error does not cross the queue — only its
+ * message does — so `context.failedReason` carries it and `context.results`
+ * lists the jobs that finished before it.
+ */
+class QueueJobFailedError extends MigronautError {
+  constructor(message, context, options) {
+    super('QUEUE_JOB_FAILED', message, context, options);
+    this.name = 'QueueJobFailedError';
+  }
+}
+
+/**
+ * Thrown by `converge` when the database cannot be brought to the declared
+ * state. `context.phase` says where it stopped: `'plan'` (a conflict refused
+ * the run before any write), `'replan'` (a collection changed while the run
+ * was under way), `'apply'` (a step failed) or `'wait'` (the search index
+ * builds did not finish — `context.reason`: `'failed'`, `'timeout'` or
+ * `'unreadable'`). A search index list that could not be read is reported in
+ * the phase that read it. `context.converge` is the converge result so far —
+ * which steps were applied, which failed, which were never reached — and
+ * `context.hint`, when present, says what usually fixes the server error
+ * behind it.
+ */
+class ConvergeFailedError extends MigronautError {
+  constructor(message, context, options) {
+    super('CONVERGE_FAILED', message, context, options);
+    this.name = 'ConvergeFailedError';
+  }
+}
+
 module.exports = {
   MigronautError,
   LockAlreadyHeldError,
@@ -226,4 +282,8 @@ module.exports = {
   ImportTargetNotEmptyError,
   IrreversibleMigrationError,
   OutOfOrderMigrationError,
+  MigrationBlockedError,
+  QueueJobInvalidError,
+  QueueJobFailedError,
+  ConvergeFailedError,
 };

package/src/index.js

@@ -7,12 +7,14 @@ const {
   ConfigFileExistsError,
   ConfigInvalidError,
   ConnectionFailedError,
+  ConvergeFailedError,
   HookFailedError,
   ImportTargetNotEmptyError,
   IrreversibleMigrationError,
   LockAlreadyHeldError,
   LockLostError,
   LockReleaseFailedError,
+  MigrationBlockedError,
   MigrationExecutionFailedError,
   MigrationFileExistsError,
   MigrationFileNotFoundError,
@@ -23,6 +25,8 @@ const {
   MigronautError,
   NotAppliedError,
   OutOfOrderMigrationError,
+  QueueJobFailedError,
+  QueueJobInvalidError,
   RunAbortedError,
 } = require('./errors/index.js');
 
@@ -47,12 +51,14 @@ module.exports = {
   ConfigFileExistsError,
   ConfigInvalidError,
   ConnectionFailedError,
+  ConvergeFailedError,
   HookFailedError,
   ImportTargetNotEmptyError,
   IrreversibleMigrationError,
   LockAlreadyHeldError,
   LockLostError,
   LockReleaseFailedError,
+  MigrationBlockedError,
   MigrationExecutionFailedError,
   MigrationFileExistsError,
   MigrationFileNotFoundError,
@@ -63,5 +69,7 @@ module.exports = {
   MigronautError,
   NotAppliedError,
   OutOfOrderMigrationError,
+  QueueJobFailedError,
+  QueueJobInvalidError,
   RunAbortedError,
 };

package/src/utils/actor.js

@@ -0,0 +1,48 @@
+/**
+ * Who asked for a run, and why: `requestedBy` and `reason`, stamped on what
+ * the run writes to the changelog (and on a converge's history entry). Kept
+ * apart from `executedBy` — the OS user that ran it, which on a queue worker
+ * is the container's, not the person behind the request.
+ *
+ * The one definition of the two fields' limits, shared by the kit's options
+ * and the queue's job contract, so a producer can never send what its worker
+ * refuses.
+ */
+const ACTOR_LIMITS = Object.freeze({ requestedBy: 128, reason: 512 });
+
+/** The problem with an actor field, or null when it is absent or valid */
+function actorIssue(key, value) {
+  if (value === undefined) return null;
+  const max = ACTOR_LIMITS[key];
+  if (typeof value !== 'string' || value.length === 0 || value.length > max) {
+    return `${key} must be a non-empty string of at most ${max} characters`;
+  }
+  return null;
+}
+
+/** Just the actor fields of `source` that are set */
+function pickActor(source) {
+  const actor = {};
+  for (const key of Object.keys(ACTOR_LIMITS)) {
+    if (source?.[key] !== undefined) actor[key] = source[key];
+  }
+  return actor;
+}
+
+/**
+ * The actor fields of `options` as changelog fields: `requestedBy` / `reason`
+ * for what a run applies, `<prefix>RequestedBy` / `<prefix>Reason` for what
+ * it reverts (`prefix: 'revert'`), so a revert never overwrites who applied.
+ */
+function actorFields(options, prefix) {
+  const fields = {};
+  if (options?.requestedBy !== undefined) {
+    fields[prefix ? `${prefix}RequestedBy` : 'requestedBy'] = options.requestedBy;
+  }
+  if (options?.reason !== undefined) {
+    fields[prefix ? `${prefix}Reason` : 'reason'] = options.reason;
+  }
+  return fields;
+}
+
+module.exports = { ACTOR_LIMITS, actorFields, actorIssue, pickActor };

package/src/utils/canonical.js

@@ -0,0 +1,212 @@
+/**
+ * Value comparison for declared state against what the server returns.
+ *
+ * The server stores a validator or a partial filter exactly as it was sent,
+ * but "as sent" and "as declared" are not the same JavaScript value: object
+ * keys may come back in another order, an `Int32` or a `Long` may stand where
+ * the declaration had a plain number, and an `undefined` property is either
+ * dropped or stored as `null` depending on the client's `ignoreUndefined`.
+ * `canonical` maps both sides onto one JSON-safe shape so that a plain string
+ * comparison decides equality.
+ *
+ * Arrays keep their order — `required: ['a', 'b']` and `['b', 'a']` are
+ * different documents to the server, and treating them as equal would hide a
+ * real change. A false "changed" costs one idempotent command; a false "same"
+ * would leave the database out of step with the declaration forever.
+ */
+
+const isPlainObject = (value) => {
+  if (value === null || typeof value !== 'object') return false;
+  const proto = Object.getPrototypeOf(value);
+  return proto === Object.prototype || proto === null;
+};
+
+/** A BSON value class from the driver (`Int32`, `Long`, `ObjectId`, …) */
+const bsonType = (value) =>
+  value !== null && typeof value === 'object' && typeof value._bsontype === 'string'
+    ? value._bsontype
+    : undefined;
+
+function canonicalNumber(number) {
+  if (Number.isNaN(number)) return { $number: 'NaN' };
+  if (!Number.isFinite(number)) return { $number: number > 0 ? 'Infinity' : '-Infinity' };
+  // -0 and 0 are the same BSON value for every purpose a validator has.
+  return number === 0 ? 0 : number;
+}
+
+function canonicalBson(type, value) {
+  if (type === 'Int32' || type === 'Double') return canonicalNumber(Number(value.valueOf()));
+  if (type === 'Long') {
+    const number = value.toNumber();
+    return Number.isSafeInteger(number) ? number : { $long: value.toString() };
+  }
+  if (type === 'ObjectId' || type === 'ObjectID') return { $oid: value.toHexString() };
+  if (type === 'Decimal128') return { $decimal: value.toString() };
+  if (type === 'BSONRegExp') return { $regex: value.pattern, $options: sortFlags(value.options) };
+  // Binary, Timestamp, MinKey, … — rare in a validator. Their JSON form is
+  // stable and type-tagged, which is all equality needs.
+  const json = typeof value.toJSON === 'function' ? value.toJSON() : String(value);
+  return { $bson: type, value: canonical(json) };
+}
+
+function sortFlags(flags) {
+  return [...String(flags)].sort().join('');
+}
+
+/**
+ * A JavaScript RegExp's flags as the server stores them. The driver writes
+ * `i` and `m` as they are, `g` as the server's `s` (dotAll) — and drops every
+ * other flag. Comparing in this form is what lets a declared `/x/i` and the
+ * `/x/i` read back compare equal, and a `BSONRegExp('x', 's')` match too.
+ */
+const DRIVER_REGEXP_FLAGS = { i: 'i', m: 'm', g: 's' };
+function storedFlags(flags) {
+  let out = '';
+  for (const flag of String(flags)) out += DRIVER_REGEXP_FLAGS[flag] ?? '';
+  return out;
+}
+
+/** JavaScript RegExp flags that do not survive the trip to the server as written */
+const UNSTORABLE_FLAGS = /[^im]/g;
+
+/**
+ * Why `value` cannot be stored as declared — a RegExp whose flags the driver
+ * changes (`g` becomes dotAll) or drops (`s`, `u`, `y`, `d`, `v`) — or null.
+ * A `BSONRegExp` states server options directly and is always fine.
+ */
+function regExpIssue(value, seen = new Set()) {
+  if (value instanceof RegExp) {
+    const bad = value.flags.match(UNSTORABLE_FLAGS);
+    return bad
+      ? `regular expression /${value.source}/${value.flags}: flag(s) ${bad.join('')} cannot be ` +
+          'stored as written (the driver keeps only i and m, and turns g into dotAll) — use ' +
+          "BSONRegExp from 'bson' for server options"
+      : null;
+  }
+  if (value === null || typeof value !== 'object' || seen.has(value)) return null;
+  seen.add(value);
+  const items =
+    value instanceof Map
+      ? [...value.values()]
+      : Array.isArray(value)
+        ? value
+        : isPlainObject(value)
+          ? Object.values(value)
+          : [];
+  for (const item of items) {
+    const issue = regExpIssue(item, seen);
+    if (issue) return issue;
+  }
+  return null;
+}
+
+/**
+ * Why `value` cannot be sent as declared — a function, a symbol, a cycle, or a
+ * RegExp the driver would change (see {@link regExpIssue}) — or null. Walks
+ * plain objects and arrays, the shapes a declaration is written in.
+ */
+function unsendable(value, seen = new Set()) {
+  if (seen.size === 0) {
+    const issue = regExpIssue(value);
+    if (issue) return issue;
+  }
+  const type = typeof value;
+  if (type === 'function') return 'must not contain functions';
+  if (type === 'symbol') return 'must not contain symbols';
+  if (value === null || type !== 'object') return null;
+  if (seen.has(value)) return 'must not contain circular references';
+  seen.add(value);
+  const items = Array.isArray(value) ? value : isPlainObject(value) ? Object.values(value) : [];
+  for (const item of items) {
+    const reason = unsendable(item, seen);
+    if (reason) return reason;
+  }
+  seen.delete(value);
+  return null;
+}
+
+/**
+ * Assign without invoking setters: a key named `__proto__` (JSON.parse makes
+ * one an own property) must stay a key, not replace the object's prototype —
+ * otherwise it vanishes from what is sent and what is compared.
+ */
+function assign(target, key, value) {
+  Object.defineProperty(target, key, {
+    value,
+    enumerable: true,
+    writable: true,
+    configurable: true,
+  });
+}
+
+/** A JSON-safe, key-sorted stand-in for `value` (see the module comment) */
+function canonical(value) {
+  if (value === undefined || value === null) return null;
+  const type = typeof value;
+  if (type === 'string' || type === 'boolean') return value;
+  if (type === 'number') return canonicalNumber(value);
+  if (type === 'bigint') {
+    const number = Number(value);
+    return Number.isSafeInteger(number) ? number : { $long: value.toString() };
+  }
+  if (type !== 'object') return { $opaque: type };
+  if (Array.isArray(value)) {
+    const out = new Array(value.length);
+    for (let i = 0; i < value.length; i++) out[i] = canonical(value[i]);
+    return out;
+  }
+  if (value instanceof Date) {
+    const time = value.getTime();
+    return { $date: Number.isNaN(time) ? 'invalid' : value.toISOString() };
+  }
+  if (value instanceof RegExp) {
+    return { $regex: value.source, $options: sortFlags(storedFlags(value.flags)) };
+  }
+  const bson = bsonType(value);
+  if (bson !== undefined) return canonicalBson(bson, value);
+  const entries = value instanceof Map ? [...value.entries()] : Object.entries(value);
+  if (!(value instanceof Map) && !isPlainObject(value)) return { $opaque: 'object' };
+  const out = {};
+  const keys = [];
+  for (const [key, item] of entries) {
+    // Dropped, not nulled: that is what the declaration means, and what a
+    // client with `ignoreUndefined` stores. toWire() makes sure it is also
+    // what migronaut itself sends.
+    if (item !== undefined) keys.push([String(key), item]);
+  }
+  keys.sort((a, b) => (a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0));
+  for (const [key, item] of keys) assign(out, key, canonical(item));
+  return out;
+}
+
+/** Deep equality under {@link canonical} */
+function deepEqual(a, b) {
+  return JSON.stringify(canonical(a)) === JSON.stringify(canonical(b));
+}
+
+/**
+ * A copy of a declared value fit to send: plain objects lose their
+ * `undefined` properties, everything else (arrays, Dates, RegExps, BSON
+ * values) is kept as is. Without it a declared `undefined` would be stored as
+ * `null` by a client with the default `ignoreUndefined: false`, and the next
+ * comparison would report a change that can never converge.
+ */
+function toWire(value) {
+  if (Array.isArray(value)) return value.map((item) => toWire(item));
+  if (!isPlainObject(value)) return value;
+  const out = {};
+  for (const [key, item] of Object.entries(value)) {
+    if (item !== undefined) assign(out, key, toWire(item));
+  }
+  return out;
+}
+
+module.exports = {
+  assign,
+  canonical,
+  deepEqual,
+  isPlainObject,
+  regExpIssue,
+  toWire,
+  unsendable,
+};

package/src/utils/collection-name.js

@@ -0,0 +1,21 @@
+/**
+ * Collection names we accept for the changelog/lock collections,
+ * `import --from/--to` and declared collections: non-empty, no `$` or NUL
+ * (invalid server-side), and outside the reserved `system.` namespace — so no
+ * config value can ever point a read or write at a system collection.
+ *
+ * Lives here rather than in config.js because the collection-definition
+ * validator (core/collections.js) needs it too, and config.js requires that
+ * validator — keeping the predicate in config.js would make the two a cycle.
+ */
+function isCollectionName(value) {
+  return (
+    typeof value === 'string' &&
+    value.length > 0 &&
+    !value.includes('$') &&
+    !value.includes('\0') &&
+    !value.startsWith('system.')
+  );
+}
+
+module.exports = { isCollectionName };

package/src/utils/error.js

@@ -1,3 +1,4 @@
+const { MigronautError } = require('../errors/index.js');
 const { redactUris } = require('./redact.js');
 
 /**
@@ -8,4 +9,20 @@ const { redactUris } = require('./redact.js');
  */
 const errorText = (error) => redactUris(error instanceof Error ? error.message : String(error));
 
-module.exports = { errorText };
+/**
+ * {@link errorText} joined with the wrapped cause: "Migration up failed: X"
+ * says WHICH migration failed, and its `context.cause` says WHY — the half
+ * forensics actually needs. Both halves are redacted (the cause is the raw
+ * thrown message). What the changelog's failure trace and a failed span's
+ * status message both carry.
+ */
+function errorWithCause(error) {
+  const message = errorText(error);
+  const cause =
+    error instanceof MigronautError && typeof error.context?.cause === 'string'
+      ? errorText(error.context.cause)
+      : undefined;
+  return cause ? `${message} — ${cause}` : message;
+}
+
+module.exports = { errorText, errorWithCause };