@alexify/migronaut 2.0.0 → 2.1.0

package/src/core/sequence.js

@@ -0,0 +1,134 @@
+const fs = require('node:fs/promises');
+const { MigrationBlockedError, MigrationFileNotFoundError } = require('../errors/index.js');
+
+/**
+ * The migration sequence: the files on disk, in name order, measured against
+ * the names the changelog holds as applied — which are pending, which arrived
+ * late, which stand in the way of an ordered step, and in what order applied
+ * records are reverted. Pure apart from reading the directory; what to do
+ * with each answer (refuse, warn, wait) is the kit's decision.
+ */
+
+/** Declaration files sit next to TypeScript migrations and are never one */
+const DECLARATION_SUFFIXES = ['.d.ts', '.d.mts', '.d.cts'];
+
+/** The migration files in `dir`, sorted ascending — empty when the directory does not exist */
+async function listMigrationFiles(dir, extensions) {
+  let entries;
+  try {
+    entries = await fs.readdir(dir, { withFileTypes: true });
+  } catch (error) {
+    if (error.code === 'ENOENT') return [];
+    throw error;
+  }
+  const matches = [];
+  for (const entry of entries) {
+    // A directory named `foo.js`, a dotfile, or a `types.d.ts` sitting next
+    // to the migrations is not a migration — including it would hard-fail
+    // the whole run with MigrationInvalidExportError.
+    if (!entry.isFile()) continue;
+    const file = entry.name;
+    if (file.startsWith('.')) continue;
+    if (DECLARATION_SUFFIXES.some((suffix) => file.endsWith(suffix))) continue;
+    for (const ext of extensions) {
+      if (file.endsWith(ext)) {
+        matches.push(file);
+        break;
+      }
+    }
+  }
+  return matches.sort();
+}
+
+/**
+ * The files of `sequence` with no applied name — all of them, or only those
+ * sorting before `before`. Pending means "no applied record": a `'failed'`
+ * trace counts, so a migration that failed stops the line exactly like one
+ * that never ran.
+ */
+function pendingIn(sequence, appliedNames, before) {
+  const pending = [];
+  for (const file of sequence) {
+    if (before !== undefined && file >= before) break;
+    if (!appliedNames.has(file)) pending.push(file);
+  }
+  return pending;
+}
+
+/**
+ * Keep only the pending migrations up to and including `to`.
+ *
+ * `to` must name a migration that exists; it may already be applied (then
+ * nothing before it is pending either, and the result is empty), which is
+ * what makes `up --to X` idempotent — running it twice is a no-op rather
+ * than an error.
+ */
+function truncateAtTarget(pending, allFiles, to) {
+  if (!allFiles.includes(to)) {
+    throw new MigrationFileNotFoundError('Migration file not found', { to });
+  }
+  const kept = [];
+  for (const file of pending) {
+    if (file > to) break;
+    kept.push(file);
+  }
+  return kept;
+}
+
+/** The name that sorts last — '' for none */
+function newestOf(names) {
+  let newest = '';
+  for (const name of names) {
+    if (name > newest) newest = name;
+  }
+  return newest;
+}
+
+/**
+ * Out-of-order arrivals among `targets`: pending files that sort before the
+ * newest applied name — migrations merged late from a parallel branch, which
+ * will run after migrations authored later. `null` when there are none.
+ */
+function lateArrivals(targets, appliedNames) {
+  if (targets.length === 0 || appliedNames.size === 0) return null;
+  const newestApplied = newestOf(appliedNames);
+  const late = [];
+  for (const target of targets) {
+    if (!appliedNames.has(target) && target < newestApplied) late.push(target);
+  }
+  return late.length > 0 ? { late, newestApplied } : null;
+}
+
+/** The names of the records to revert, newest first unless already in revert order */
+function revertOrder(records, preserveOrder) {
+  const names = [];
+  for (const record of records) names.push(record.name);
+  if (!preserveOrder) {
+    names.sort();
+    names.reverse();
+  }
+  return names;
+}
+
+/**
+ * The refusal of an ordered step: "`subject` is blocked: N `what`: a, b".
+ * `context` carries `blockedBy` (and `failed`, the blockers with a failed
+ * trace — a stopped line rather than one still on its way).
+ */
+function blockedError(subject, what, context) {
+  const { blockedBy } = context;
+  return new MigrationBlockedError(
+    `${subject} is blocked: ${blockedBy.length} ${what}: ${blockedBy.join(', ')}`,
+    context,
+  );
+}
+
+module.exports = {
+  blockedError,
+  lateArrivals,
+  listMigrationFiles,
+  newestOf,
+  pendingIn,
+  revertOrder,
+  truncateAtTarget,
+};

package/src/errors/index.js

@@ -204,6 +204,58 @@ class OutOfOrderMigrationError extends MigronautError {
   }
 }
 
+/**
+ * Thrown by an `ordered` single-file run that would apply or revert out of
+ * sequence: an earlier migration is still pending (`up`), or one applied later
+ * is still applied (`down`). `context.blockedBy` names what must go first.
+ */
+class MigrationBlockedError extends MigronautError {
+  constructor(message, context, options) {
+    super('MIGRATION_BLOCKED', message, context, options);
+    this.name = 'MigrationBlockedError';
+  }
+}
+
+/**
+ * Thrown by the queue adapter when a job's payload fails the contract check.
+ * Job data comes back from Redis, so it is untrusted input — never a config
+ * mistake of the process that reads it.
+ */
+class QueueJobInvalidError extends MigronautError {
+  constructor(message, context, options) {
+    super('QUEUE_JOB_INVALID', message, context, options);
+    this.name = 'QueueJobInvalidError';
+  }
+}
+
+/**
+ * Thrown by a queue group's `wait()` when one of its jobs failed or the wait
+ * timed out. The worker's typed error does not cross the queue — only its
+ * message does — so `context.failedReason` carries it and `context.results`
+ * lists the jobs that finished before it.
+ */
+class QueueJobFailedError extends MigronautError {
+  constructor(message, context, options) {
+    super('QUEUE_JOB_FAILED', message, context, options);
+    this.name = 'QueueJobFailedError';
+  }
+}
+
+/**
+ * Thrown by `converge` when the database cannot be brought to the declared
+ * state: the plan has a conflict (refused before any write — `context.phase`
+ * is `'plan'`), or a step failed (`'apply'`). `context.converge` is the
+ * converge result so far — which steps were applied, which failed, which were
+ * never reached — and `context.hint`, when present, says what usually fixes
+ * the server error behind it.
+ */
+class ConvergeFailedError extends MigronautError {
+  constructor(message, context, options) {
+    super('CONVERGE_FAILED', message, context, options);
+    this.name = 'ConvergeFailedError';
+  }
+}
+
 module.exports = {
   MigronautError,
   LockAlreadyHeldError,
@@ -226,4 +278,8 @@ module.exports = {
   ImportTargetNotEmptyError,
   IrreversibleMigrationError,
   OutOfOrderMigrationError,
+  MigrationBlockedError,
+  QueueJobInvalidError,
+  QueueJobFailedError,
+  ConvergeFailedError,
 };

package/src/index.js

@@ -7,12 +7,14 @@ const {
   ConfigFileExistsError,
   ConfigInvalidError,
   ConnectionFailedError,
+  ConvergeFailedError,
   HookFailedError,
   ImportTargetNotEmptyError,
   IrreversibleMigrationError,
   LockAlreadyHeldError,
   LockLostError,
   LockReleaseFailedError,
+  MigrationBlockedError,
   MigrationExecutionFailedError,
   MigrationFileExistsError,
   MigrationFileNotFoundError,
@@ -23,6 +25,8 @@ const {
   MigronautError,
   NotAppliedError,
   OutOfOrderMigrationError,
+  QueueJobFailedError,
+  QueueJobInvalidError,
   RunAbortedError,
 } = require('./errors/index.js');
 
@@ -47,12 +51,14 @@ module.exports = {
   ConfigFileExistsError,
   ConfigInvalidError,
   ConnectionFailedError,
+  ConvergeFailedError,
   HookFailedError,
   ImportTargetNotEmptyError,
   IrreversibleMigrationError,
   LockAlreadyHeldError,
   LockLostError,
   LockReleaseFailedError,
+  MigrationBlockedError,
   MigrationExecutionFailedError,
   MigrationFileExistsError,
   MigrationFileNotFoundError,
@@ -63,5 +69,7 @@ module.exports = {
   MigronautError,
   NotAppliedError,
   OutOfOrderMigrationError,
+  QueueJobFailedError,
+  QueueJobInvalidError,
   RunAbortedError,
 };

package/src/utils/actor.js

@@ -0,0 +1,48 @@
+/**
+ * Who asked for a run, and why: `requestedBy` and `reason`, stamped on what
+ * the run writes to the changelog (and on a converge's history entry). Kept
+ * apart from `executedBy` — the OS user that ran it, which on a queue worker
+ * is the container's, not the person behind the request.
+ *
+ * The one definition of the two fields' limits, shared by the kit's options
+ * and the queue's job contract, so a producer can never send what its worker
+ * refuses.
+ */
+const ACTOR_LIMITS = Object.freeze({ requestedBy: 128, reason: 512 });
+
+/** The problem with an actor field, or null when it is absent or valid */
+function actorIssue(key, value) {
+  if (value === undefined) return null;
+  const max = ACTOR_LIMITS[key];
+  if (typeof value !== 'string' || value.length === 0 || value.length > max) {
+    return `${key} must be a non-empty string of at most ${max} characters`;
+  }
+  return null;
+}
+
+/** Just the actor fields of `source` that are set */
+function pickActor(source) {
+  const actor = {};
+  for (const key of Object.keys(ACTOR_LIMITS)) {
+    if (source?.[key] !== undefined) actor[key] = source[key];
+  }
+  return actor;
+}
+
+/**
+ * The actor fields of `options` as changelog fields: `requestedBy` / `reason`
+ * for what a run applies, `<prefix>RequestedBy` / `<prefix>Reason` for what
+ * it reverts (`prefix: 'revert'`), so a revert never overwrites who applied.
+ */
+function actorFields(options, prefix) {
+  const fields = {};
+  if (options?.requestedBy !== undefined) {
+    fields[prefix ? `${prefix}RequestedBy` : 'requestedBy'] = options.requestedBy;
+  }
+  if (options?.reason !== undefined) {
+    fields[prefix ? `${prefix}Reason` : 'reason'] = options.reason;
+  }
+  return fields;
+}
+
+module.exports = { ACTOR_LIMITS, actorFields, actorIssue, pickActor };

package/src/utils/canonical.js

@@ -0,0 +1,179 @@
+/**
+ * Value comparison for declared state against what the server returns.
+ *
+ * The server stores a validator or a partial filter exactly as it was sent,
+ * but "as sent" and "as declared" are not the same JavaScript value: object
+ * keys may come back in another order, an `Int32` or a `Long` may stand where
+ * the declaration had a plain number, and an `undefined` property is either
+ * dropped or stored as `null` depending on the client's `ignoreUndefined`.
+ * `canonical` maps both sides onto one JSON-safe shape so that a plain string
+ * comparison decides equality.
+ *
+ * Arrays keep their order — `required: ['a', 'b']` and `['b', 'a']` are
+ * different documents to the server, and treating them as equal would hide a
+ * real change. A false "changed" costs one idempotent command; a false "same"
+ * would leave the database out of step with the declaration forever.
+ */
+
+const isPlainObject = (value) => {
+  if (value === null || typeof value !== 'object') return false;
+  const proto = Object.getPrototypeOf(value);
+  return proto === Object.prototype || proto === null;
+};
+
+/** A BSON value class from the driver (`Int32`, `Long`, `ObjectId`, …) */
+const bsonType = (value) =>
+  value !== null && typeof value === 'object' && typeof value._bsontype === 'string'
+    ? value._bsontype
+    : undefined;
+
+function canonicalNumber(number) {
+  if (Number.isNaN(number)) return { $number: 'NaN' };
+  if (!Number.isFinite(number)) return { $number: number > 0 ? 'Infinity' : '-Infinity' };
+  // -0 and 0 are the same BSON value for every purpose a validator has.
+  return number === 0 ? 0 : number;
+}
+
+function canonicalBson(type, value) {
+  if (type === 'Int32' || type === 'Double') return canonicalNumber(Number(value.valueOf()));
+  if (type === 'Long') {
+    const number = value.toNumber();
+    return Number.isSafeInteger(number) ? number : { $long: value.toString() };
+  }
+  if (type === 'ObjectId' || type === 'ObjectID') return { $oid: value.toHexString() };
+  if (type === 'Decimal128') return { $decimal: value.toString() };
+  if (type === 'BSONRegExp') return { $regex: value.pattern, $options: sortFlags(value.options) };
+  // Binary, Timestamp, MinKey, … — rare in a validator. Their JSON form is
+  // stable and type-tagged, which is all equality needs.
+  const json = typeof value.toJSON === 'function' ? value.toJSON() : String(value);
+  return { $bson: type, value: canonical(json) };
+}
+
+function sortFlags(flags) {
+  return [...String(flags)].sort().join('');
+}
+
+/**
+ * A JavaScript RegExp's flags as the server stores them. The driver writes
+ * `i` and `m` as they are, `g` as the server's `s` (dotAll) — and drops every
+ * other flag. Comparing in this form is what lets a declared `/x/i` and the
+ * `/x/i` read back compare equal, and a `BSONRegExp('x', 's')` match too.
+ */
+const DRIVER_REGEXP_FLAGS = { i: 'i', m: 'm', g: 's' };
+function storedFlags(flags) {
+  let out = '';
+  for (const flag of String(flags)) out += DRIVER_REGEXP_FLAGS[flag] ?? '';
+  return out;
+}
+
+/** JavaScript RegExp flags that do not survive the trip to the server as written */
+const UNSTORABLE_FLAGS = /[^im]/g;
+
+/**
+ * Why `value` cannot be stored as declared — a RegExp whose flags the driver
+ * changes (`g` becomes dotAll) or drops (`s`, `u`, `y`, `d`, `v`) — or null.
+ * A `BSONRegExp` states server options directly and is always fine.
+ */
+function regExpIssue(value, seen = new Set()) {
+  if (value instanceof RegExp) {
+    const bad = value.flags.match(UNSTORABLE_FLAGS);
+    return bad
+      ? `regular expression /${value.source}/${value.flags}: flag(s) ${bad.join('')} cannot be ` +
+          'stored as written (the driver keeps only i and m, and turns g into dotAll) — use ' +
+          "BSONRegExp from 'bson' for server options"
+      : null;
+  }
+  if (value === null || typeof value !== 'object' || seen.has(value)) return null;
+  seen.add(value);
+  const items =
+    value instanceof Map
+      ? [...value.values()]
+      : Array.isArray(value)
+        ? value
+        : isPlainObject(value)
+          ? Object.values(value)
+          : [];
+  for (const item of items) {
+    const issue = regExpIssue(item, seen);
+    if (issue) return issue;
+  }
+  return null;
+}
+
+/**
+ * Assign without invoking setters: a key named `__proto__` (JSON.parse makes
+ * one an own property) must stay a key, not replace the object's prototype —
+ * otherwise it vanishes from what is sent and what is compared.
+ */
+function assign(target, key, value) {
+  Object.defineProperty(target, key, {
+    value,
+    enumerable: true,
+    writable: true,
+    configurable: true,
+  });
+}
+
+/** A JSON-safe, key-sorted stand-in for `value` (see the module comment) */
+function canonical(value) {
+  if (value === undefined || value === null) return null;
+  const type = typeof value;
+  if (type === 'string' || type === 'boolean') return value;
+  if (type === 'number') return canonicalNumber(value);
+  if (type === 'bigint') {
+    const number = Number(value);
+    return Number.isSafeInteger(number) ? number : { $long: value.toString() };
+  }
+  if (type !== 'object') return { $opaque: type };
+  if (Array.isArray(value)) {
+    const out = new Array(value.length);
+    for (let i = 0; i < value.length; i++) out[i] = canonical(value[i]);
+    return out;
+  }
+  if (value instanceof Date) {
+    const time = value.getTime();
+    return { $date: Number.isNaN(time) ? 'invalid' : value.toISOString() };
+  }
+  if (value instanceof RegExp) {
+    return { $regex: value.source, $options: sortFlags(storedFlags(value.flags)) };
+  }
+  const bson = bsonType(value);
+  if (bson !== undefined) return canonicalBson(bson, value);
+  const entries = value instanceof Map ? [...value.entries()] : Object.entries(value);
+  if (!(value instanceof Map) && !isPlainObject(value)) return { $opaque: 'object' };
+  const out = {};
+  const keys = [];
+  for (const [key, item] of entries) {
+    // Dropped, not nulled: that is what the declaration means, and what a
+    // client with `ignoreUndefined` stores. toWire() makes sure it is also
+    // what migronaut itself sends.
+    if (item !== undefined) keys.push([String(key), item]);
+  }
+  keys.sort((a, b) => (a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0));
+  for (const [key, item] of keys) assign(out, key, canonical(item));
+  return out;
+}
+
+/** Deep equality under {@link canonical} */
+function deepEqual(a, b) {
+  return JSON.stringify(canonical(a)) === JSON.stringify(canonical(b));
+}
+
+/**
+ * A copy of a declared value fit to send: plain objects lose their
+ * `undefined` properties, everything else (arrays, Dates, RegExps, BSON
+ * values) is kept as is. Without it a declared `undefined` would be stored as
+ * `null` by a client with the default `ignoreUndefined: false`, and the next
+ * comparison would report a change that can never converge.
+ */
+function toWire(value) {
+  if (Array.isArray(value)) return value.map((item) => toWire(item));
+  if (!isPlainObject(value)) return value;
+  const out = {};
+  for (const [key, item] of Object.entries(value)) {
+    if (item !== undefined) assign(out, key, toWire(item));
+  }
+  return out;
+}
+
+module.exports = { canonical, deepEqual, isPlainObject, regExpIssue, toWire };

package/src/utils/collection-name.js

@@ -0,0 +1,21 @@
+/**
+ * Collection names we accept for the changelog/lock collections,
+ * `import --from/--to` and declared collections: non-empty, no `$` or NUL
+ * (invalid server-side), and outside the reserved `system.` namespace — so no
+ * config value can ever point a read or write at a system collection.
+ *
+ * Lives here rather than in config.js because the collection-definition
+ * validator (core/collections.js) needs it too, and config.js requires that
+ * validator — keeping the predicate in config.js would make the two a cycle.
+ */
+function isCollectionName(value) {
+  return (
+    typeof value === 'string' &&
+    value.length > 0 &&
+    !value.includes('$') &&
+    !value.includes('\0') &&
+    !value.startsWith('system.')
+  );
+}
+
+module.exports = { isCollectionName };

package/src/utils/error.js

@@ -1,3 +1,4 @@
+const { MigronautError } = require('../errors/index.js');
 const { redactUris } = require('./redact.js');
 
 /**
@@ -8,4 +9,20 @@ const { redactUris } = require('./redact.js');
  */
 const errorText = (error) => redactUris(error instanceof Error ? error.message : String(error));
 
-module.exports = { errorText };
+/**
+ * {@link errorText} joined with the wrapped cause: "Migration up failed: X"
+ * says WHICH migration failed, and its `context.cause` says WHY — the half
+ * forensics actually needs. Both halves are redacted (the cause is the raw
+ * thrown message). What the changelog's failure trace and a failed span's
+ * status message both carry.
+ */
+function errorWithCause(error) {
+  const message = errorText(error);
+  const cause =
+    error instanceof MigronautError && typeof error.context?.cause === 'string'
+      ? errorText(error.context.cause)
+      : undefined;
+  return cause ? `${message} — ${cause}` : message;
+}
+
+module.exports = { errorText, errorWithCause };

package/src/utils/id.js

@@ -0,0 +1,77 @@
+const { randomUUID } = require('node:crypto');
+const { ConfigInvalidError } = require('../errors/index.js');
+const { errorText } = require('./error.js');
+
+/**
+ * Longest id migronaut accepts. Also the limit a queue worker enforces on a
+ * job's group id — one constant, so a producer can never mint an id its own
+ * worker would reject.
+ */
+const MAX_ID_LENGTH = 128;
+
+/**
+ * The default id: a random (v4) UUID. This module is the only place in `src/`
+ * that mints one — everything else asks for an id through it, which is what
+ * lets the `generateId` config option replace the format everywhere at once.
+ */
+const randomId = () => randomUUID();
+
+/**
+ * Return `value` when it is usable as an id, throw otherwise. An id is stored
+ * as a string field (changelog `runId`, lock `owner`, a job's `groupId`) and
+ * gated on by truthiness in the kit, so an empty or non-string value would
+ * silently switch off the reentrancy guard and the owner-scoped lock release.
+ */
+function assertId(value) {
+  if (typeof value !== 'string' || value.length === 0 || value.length > MAX_ID_LENGTH) {
+    throw new ConfigInvalidError(
+      `generateId must return a non-empty string of at most ${MAX_ID_LENGTH} characters`,
+      typeof value === 'string' ? { length: value.length } : { returned: typeof value },
+    );
+  }
+  return value;
+}
+
+/**
+ * Turn the `generateId` config option into the function the kit mints ids
+ * with. `undefined` keeps the default (`randomId`); anything else must be a
+ * function, and every id it returns is checked.
+ *
+ * The user's function is called bare — no arguments, no receiver — so a
+ * third-party generator passes straight through (`generateId: ulid`,
+ * `generateId: nanoid`): their first parameter means something of its own
+ * (a seed time, a size), and any argument migronaut passed would be read as it.
+ *
+ * It must be synchronous: the run id is minted in the same tick as the
+ * reentrancy guard that checks it, and a promise would be stored as a truthy
+ * non-id.
+ */
+function createIdGenerator(generateId) {
+  if (generateId === undefined) return randomId;
+  if (typeof generateId !== 'function') {
+    throw new ConfigInvalidError('generateId must be a function', {
+      generateId: typeof generateId,
+    });
+  }
+  return () => {
+    let value;
+    try {
+      value = generateId();
+    } catch (error) {
+      throw new ConfigInvalidError(
+        'generateId threw',
+        { cause: errorText(error) },
+        { cause: error },
+      );
+    }
+    if (typeof value?.then === 'function') {
+      // The promise is dropped, so its rejection must not surface as an
+      // unhandled one on top of the error below.
+      value.then(undefined, () => {});
+      throw new ConfigInvalidError('generateId must be synchronous — it returned a promise');
+    }
+    return assertId(value);
+  };
+}
+
+module.exports = { MAX_ID_LENGTH, assertId, createIdGenerator, randomId };