@alexify/migronaut 2.1.0 → 2.3.0

package/src/core/converge-plan.js

@@ -1,5 +1,20 @@
 const { deepEqual } = require('../utils/canonical.js');
-const { compareIndex, normalizeLiveIndex, restoreSpec, sameSignature } = require('./index-spec.js');
+const { shardedVersionIndexKey, versionFloorConflict } = require('./versioning-spec.js');
+const {
+  compareIndex,
+  normalizeDeclaredIndex,
+  normalizeLiveIndex,
+  restoreSpec,
+  sameSignature,
+} = require('./index-spec.js');
+const {
+  compareSearchIndex,
+  isBeingRemoved,
+  normalizeLiveSearchIndex,
+  searchBuild,
+  searchIndexSpec,
+  searchIndexValue,
+} = require('./search-index-spec.js');
 
 /**
  * The converge planner: one declared collection against its live state, as
@@ -17,17 +32,35 @@ const { compareIndex, normalizeLiveIndex, restoreSpec, sameSignature } = require
  * - undeclared indexes are kept (and reported) unless `prune` is on;
  * - a rebuild that drops a unique index to build a unique one back is a
  *   conflict unless `rebuildUnique` is on — see UNIQUE_REBUILD_REASON;
- * - `_id_` and a clustered index are the collection's own and never listed.
+ * - `_id_` and a clustered index are the collection's own and never listed;
+ * - a search index is created or updated in place — never dropped to be built
+ *   again: a search against a missing index returns nothing rather than fail,
+ *   so a rebuild would be a silent outage. What no update can change (the
+ *   type, an autoEmbed field's model or size) is a conflict instead.
  */
 
 /** Actions that change the database — what a plan "would do" and a run "did" */
 const CHANGE_ACTIONS = new Set(['create', 'modify', 'recreate', 'drop']);
 
+/**
+ * Actions of a declared search index that leaves it on the server — the
+ * ones whose build is worth reporting, or waiting for
+ */
+const SERVED_ACTIONS = new Set(['create', 'modify', 'unchanged']);
+
+/** A row's target as people read it — where the code's name is not plain English */
+const TARGET_LABELS = Object.freeze({ searchIndex: 'search index' });
+
 /** Server defaults for a collection that has a validator but did not say how to apply it */
 const VALIDATOR_DEFAULTS = { validationLevel: 'strict', validationAction: 'error' };
 
-/** Whether a row drops or rebuilds an index */
+/**
+ * Whether a row drops or rebuilds an index — or drops a search index (a
+ * search index is updated in place, never rebuilt: the old one serves until
+ * the new definition is built).
+ */
 function isDestructive(action) {
+  if (action.target === 'searchIndex') return action.action === 'drop';
   return action.target === 'index' && (action.action === 'drop' || action.action === 'recreate');
 }
 
@@ -140,12 +173,63 @@ function dropsUniqueConstraint(declared, drops) {
  */
 function backsShardKey(index, shardKey) {
   if (!shardKey) return false;
-  const shardFields = Object.keys(shardKey);
-  const fields = index.serverKey.map(([field]) => field);
-  return shardFields.every((field, position) => fields[position] === field);
+  let position = 0;
+  for (const field of Object.keys(shardKey)) {
+    if (index.serverKey[position]?.[0] !== field) return false;
+    position += 1;
+  }
+  return true;
 }
 
-function planIndexes(declaredIndexes, live, { prune, rebuildUnique, capabilities }, row, steps) {
+/** `first; second`, or `second` alone when there is no first */
+function joinReasons(first, second) {
+  return first ? `${first}; ${second}` : second;
+}
+
+/** Why the ordinary version index stays on a sharded collection whose version index replaced it */
+const DISPLACED_REASON =
+  'replaced by the shard-key-prefixed version index — drop it once nothing hints it ' +
+  '(prune does, when the indexes are declared)';
+
+/**
+ * The declared indexes as they apply to this live collection: on a sharded
+ * one, the version index takes the shard key between the version field and
+ * `_id` (versioning-spec.js). `displaced` names the ordinary version index it
+ * replaces, when the two differ.
+ */
+function indexesFor(definition, live) {
+  const indexes = definition.indexes;
+  if (indexes === undefined || !definition.versioning || !live.shardKey) return { indexes };
+  const sharded = normalizeDeclaredIndex({
+    key: shardedVersionIndexKey(definition.versioning, live.shardKey),
+  });
+  const result = [];
+  let displaced;
+  for (const index of indexes) {
+    if (index.versionIndex === true && index.name !== sharded.name) {
+      displaced = index.name;
+      result.push({ ...sharded, versionIndex: true });
+    } else {
+      result.push(index);
+    }
+  }
+  return { indexes: result, displaced };
+}
+
+/**
+ * Plan the declared indexes of one collection against its live ones. With
+ * `partial` (only the version index of `versioning` is declared, not the
+ * collection's own list) the other live indexes are not managed at all: never
+ * listed, never dropped — prune does not reach them.
+ */
+function planIndexes(
+  declaredIndexes,
+  live,
+  { prune: pruneOption, rebuildUnique, capabilities, partial = false, displaced },
+  row,
+  steps,
+) {
+  const prune = pruneOption && !partial;
   const defaultCollation = live.options?.collation;
   const liveIndexes = [];
   for (const raw of live.indexes) {
@@ -267,9 +351,7 @@ function planIndexes(declaredIndexes, live, { prune, rebuildUnique, capabilities
       }
       drops.push(...blockers);
       action.action = 'recreate';
-      action.reason = identical
-        ? 'name'
-        : [item.reason, `replaces ${names}`].filter(Boolean).join('; ');
+      action.reason = identical ? 'name' : joinReasons(item.reason, `replaces ${names}`);
     }
 
     if (drops.length === 0) {
@@ -278,7 +360,7 @@ function planIndexes(declaredIndexes, live, { prune, rebuildUnique, capabilities
       // Not done unasked: the CLI asks with --rebuild-unique, while the paths
       // nobody watches (after up, a queue job) never get this far on their own.
       action.action = 'conflict';
-      action.reason = [action.reason, UNIQUE_REBUILD_REASON].filter(Boolean).join('; ');
+      action.reason = joinReasons(action.reason, UNIQUE_REBUILD_REASON);
     } else {
       rebuilds.push({ declared, action, drops });
     }
@@ -287,16 +369,16 @@ function planIndexes(declaredIndexes, live, { prune, rebuildUnique, capabilities
   // A create that collides with an index another rebuild is about to drop has
   // to wait for that drop — and two rebuilds that swap keys each wait for the
   // other. Such entangled ones run as one group: every drop, then every create.
-  const doomed = new Map();
+  // Each doomed index keeps its rebuild and itself, so no lookup is needed below.
+  const doomed = [];
   for (const rebuild of rebuilds) {
-    for (const index of rebuild.drops) doomed.set(index.name, rebuild);
+    for (const index of rebuild.drops) doomed.push({ owner: rebuild, index });
   }
   const entangled = new Set();
-  for (const item of [...creates, ...rebuilds]) {
-    for (const [name, owner] of doomed) {
-      if (owner === item) continue;
-      const index = owner.drops.find((drop) => drop.name === name);
-      if (collides(item.declared, index)) {
+  for (const list of [creates, rebuilds]) {
+    for (const item of list) {
+      for (const { owner, index } of doomed) {
+        if (owner === item || !collides(item.declared, index)) continue;
         entangled.add(item);
         entangled.add(owner);
       }
@@ -313,20 +395,37 @@ function planIndexes(declaredIndexes, live, { prune, rebuildUnique, capabilities
   }
   steps.push(...modifies);
   const group = { op: 'rebuild', drops: [], creates: [], actions: [] };
-  for (const item of [...rebuilds, ...waiting]) {
-    const step = entangled.has(item)
-      ? group
-      : { op: 'rebuild', drops: [], creates: [], actions: [] };
-    for (const index of item.drops ?? []) {
-      step.drops.push({ name: index.name, restore: restoreSpec(index.raw) });
+  for (const list of [rebuilds, waiting]) {
+    for (const item of list) {
+      const step = entangled.has(item)
+        ? group
+        : { op: 'rebuild', drops: [], creates: [], actions: [] };
+      for (const index of item.drops ?? []) {
+        step.drops.push({ name: index.name, restore: restoreSpec(index.raw) });
+      }
+      step.creates.push({ spec: item.declared.spec, action: item.action });
+      step.actions.push(item.action);
+      if (step !== group) steps.push(step);
     }
-    step.creates.push({ spec: item.declared.spec, action: item.action });
-    step.actions.push(item.action);
-    if (step !== group) steps.push(step);
   }
   if (group.actions.length > 0) steps.push(group);
 
+  // The ordinary version index a sharded one replaced: kept, and said so —
+  // a background migration may still be hinting it.
+  const old = displaced !== undefined ? byName.get(displaced) : undefined;
+  if (old !== undefined && !declaredNames.has(displaced) && !prune) {
+    consumed.add(displaced);
+    row({
+      target: 'index',
+      name: displaced,
+      action: 'keep',
+      reason: DISPLACED_REASON,
+      from: indexValue(old.raw),
+    });
+  }
+
   // Last, so an index is only ever removed once everything declared exists.
+  if (partial) return;
   for (const index of liveIndexes) {
     if (consumed.has(index.name) || declaredNames.has(index.name)) continue;
     if (prune && backsShardKey(index, live.shardKey)) {
@@ -358,12 +457,173 @@ function planIndexes(declaredIndexes, live, { prune, rebuildUnique, capabilities
   }
 }
 
+/** The way out of a change a search index cannot make in place */
+const NEW_NAME_RECIPE =
+  'declare it under a new name, converge, then remove the old declaration and converge with prune';
+
+/** Why a declared search index is not planned on a server without Atlas Search */
+const SEARCH_UNAVAILABLE_REASON = 'Atlas Search is not available on this server';
+
+function typeChangeReason(from, to) {
+  return `the type cannot change in place (${from} → ${to}) — ${NEW_NAME_RECIPE}`;
+}
+
+function autoEmbedReason(changes) {
+  return `autoEmbed ${changes.join(', ')} cannot change in place — ${NEW_NAME_RECIPE}`;
+}
+
+/** `mappings.fields.title.norms, storedSource (+2 more)` */
+function diffReason(paths, more) {
+  return `${paths.join(', ')}${more > 0 ? ` (+${more} more)` : ''}`;
+}
+
+function deletingReason(status) {
+  return `is being deleted on the server (${status}) — converge again once it is gone`;
+}
+
+/**
+ * Plan the search indexes of one collection. Submissions (creates, then
+ * updates) go to `submit`, drops to `drops` — converge.js sends submissions
+ * before the regular index builds (the server builds a search index in the
+ * background) and drops last of all.
+ */
+function planSearchIndexes(declaredList, live, { prune, search }, row, submit, drops) {
+  if (!search.available) {
+    // Nothing to compare with: every declaration is refused, or skipped when
+    // the configuration says a server without Search is expected.
+    for (const declared of declaredList) {
+      row({
+        target: 'searchIndex',
+        name: declared.name,
+        action: search.onUnavailable === 'skip' ? 'skip' : 'conflict',
+        reason: SEARCH_UNAVAILABLE_REASON,
+        to: searchIndexValue(declared),
+      });
+    }
+    return;
+  }
+  // Normalized and indexed by name in one pass; the map's order is the server's.
+  const byName = new Map();
+  for (const raw of live.searchIndexes ?? []) {
+    const index = normalizeLiveSearchIndex(raw);
+    byName.set(index.name, index);
+  }
+  const declaredNames = new Set();
+
+  const specs = [];
+  const created = [];
+  const updates = [];
+  for (const declared of declaredList) {
+    declaredNames.add(declared.name);
+    const current = byName.get(declared.name);
+    const to = searchIndexValue(declared);
+    if (!current) {
+      specs.push(searchIndexSpec(declared));
+      created.push(row({ target: 'searchIndex', name: declared.name, action: 'create', to }));
+      continue;
+    }
+    const from = searchIndexValue(current);
+    const build = searchBuild(current);
+    if (isBeingRemoved(current)) {
+      row({
+        target: 'searchIndex',
+        name: declared.name,
+        action: 'conflict',
+        reason: deletingReason(current.status),
+        build,
+      });
+      continue;
+    }
+    const { diffs, paths, more, typeChange, immutable, ignored } = compareSearchIndex(
+      declared,
+      current,
+    );
+    // Options only the server reports, left out of the comparison: named on the row.
+    const tolerated = ignored.length > 0 ? { ignored } : {};
+    if (diffs.length === 0) {
+      row({ target: 'searchIndex', name: declared.name, action: 'unchanged', build, ...tolerated });
+    } else if (typeChange || immutable.length > 0) {
+      row({
+        target: 'searchIndex',
+        name: declared.name,
+        action: 'conflict',
+        reason: typeChange
+          ? typeChangeReason(current.type, declared.type)
+          : autoEmbedReason(immutable),
+        from,
+        to,
+        build,
+        ...tolerated,
+      });
+    } else {
+      const action = row({
+        target: 'searchIndex',
+        name: declared.name,
+        action: 'modify',
+        reason: diffReason(paths, more),
+        from,
+        to,
+        build,
+        ...tolerated,
+      });
+      updates.push({
+        op: 'updateSearchIndex',
+        name: declared.name,
+        type: declared.type,
+        definition: declared.definition,
+        ...(current.version !== undefined ? { sinceVersion: current.version } : {}),
+        actions: [action],
+      });
+    }
+  }
+  if (specs.length > 0) submit.push({ op: 'createSearchIndexes', specs, actions: created });
+  submit.push(...updates);
+
+  for (const index of byName.values()) {
+    if (declaredNames.has(index.name)) continue;
+    const from = searchIndexValue(index);
+    const build = searchBuild(index);
+    if (isBeingRemoved(index)) {
+      // On its way out already — dropping it again would only fail.
+      row({
+        target: 'searchIndex',
+        name: index.name,
+        action: 'keep',
+        reason: 'being deleted',
+        from,
+        build,
+      });
+    } else if (prune) {
+      const action = row({
+        target: 'searchIndex',
+        name: index.name,
+        action: 'drop',
+        reason: 'not declared',
+        from,
+        build,
+      });
+      drops.push({ op: 'dropSearchIndex', name: index.name, actions: [action] });
+    } else {
+      row({
+        target: 'searchIndex',
+        name: index.name,
+        action: 'keep',
+        reason: 'not declared',
+        from,
+        build,
+      });
+    }
+  }
+}
+
 /**
  * Plan one collection. `definition` is a normalized definition (see
- * collections.js); `live` is `{ exists, type?, options?, indexes }` as
- * converge.js reads it. Returns `{ name, actions, steps }`: `actions` are the
- * result rows (status `'planned'`), `steps` the operations that carry them
- * out, in execution order, each pointing at the rows it settles.
+ * collections.js); `live` is `{ exists, type?, options?, indexes,
+ * searchIndexes?, shardKey?, versionFloor? }` as converge.js reads it.
+ * Returns `{ name, actions, steps }`: `actions` are the result rows (status `'planned'`), `steps` the operations
+ * that carry them out, in execution order, each pointing at the rows it
+ * settles. `search` is `{ available, onUnavailable }` — whether the server has
+ * Atlas Search, and what a declared search index becomes when it does not.
  */
 /**
  * Fold every run of consecutive `createIndex` steps into one `createIndexes`:
@@ -393,10 +653,12 @@ function planCollection(definition, live, options = {}) {
   return { ...plan, steps: batchCreates(plan.steps) };
 }
 
+const SEARCH_AVAILABLE = Object.freeze({ available: true, onUnavailable: 'fail' });
+
 function planCollectionSteps(
   definition,
   live,
-  { prune = false, rebuildUnique = false, capabilities = {} } = {},
+  { prune = false, rebuildUnique = false, capabilities = {}, search = SEARCH_AVAILABLE } = {},
 ) {
   const name = definition.name;
   const actions = [];
@@ -418,17 +680,28 @@ function planCollectionSteps(
   }
 
   const desired = desiredValidator(definition);
-  const declaredIndexes = definition.indexes;
+  const { indexes: declaredIndexes, displaced } = indexesFor(definition, live);
+  const declaredSearch = definition.searchIndexes;
+  const indexSteps = [];
+  const searchSubmit = [];
+  const searchDrops = [];
 
   if (!live.exists) {
     // Nothing worth creating an empty collection for — a definition that only
-    // says "no validator" is already true of a collection that does not exist.
-    if (!desired && !(declaredIndexes?.length > 0)) return { name, actions, steps };
-    const linked = [row({ target: 'collection', name, action: 'create' })];
-    if (desired) {
-      linked.push(row({ target: 'validator', name, action: 'create', to: desired }));
+    // says "no validator" is already true of a collection that does not exist,
+    // and so is "no search indexes" (or any, on a server without Search).
+    const wantsSearch = search.available && declaredSearch?.length > 0;
+    if (desired || declaredIndexes?.length > 0 || wantsSearch) {
+      const linked = [row({ target: 'collection', name, action: 'create' })];
+      if (desired) {
+        linked.push(row({ target: 'validator', name, action: 'create', to: desired }));
+      }
+      steps.push({
+        op: 'createCollection',
+        options: desired ? { ...desired } : {},
+        actions: linked,
+      });
     }
-    steps.push({ op: 'createCollection', options: desired ? { ...desired } : {}, actions: linked });
     for (const declared of declaredIndexes ?? []) {
       const action = row({
         target: 'index',
@@ -436,40 +709,50 @@ function planCollectionSteps(
         action: 'create',
         to: indexValue(declared.spec),
       });
-      steps.push({ op: 'createIndex', spec: declared.spec, actions: [action] });
+      indexSteps.push({ op: 'createIndex', spec: declared.spec, actions: [action] });
     }
+    if (declaredSearch !== undefined) {
+      const none = { searchIndexes: [] };
+      planSearchIndexes(declaredSearch, none, { prune, search }, row, searchSubmit, searchDrops);
+    }
+    steps.push(...searchSubmit, ...indexSteps);
     return { name, actions, steps };
   }
 
   if (desired !== undefined) {
-    planValidator(name, desired, liveValidator(live.options), row, steps);
+    // Raising versioning.min over documents still below it: refused, so the
+    // whole run stops before its first write (see versionFloorToCheck).
+    const refused = versionFloorConflict(live.versionFloor);
+    if (refused) {
+      row({ target: 'validator', name, action: 'conflict', reason: refused, to: desired });
+    } else {
+      planValidator(name, desired, liveValidator(live.options), row, steps);
+    }
   }
   if (declaredIndexes !== undefined) {
-    planIndexes(declaredIndexes, live, { prune, rebuildUnique, capabilities }, row, steps);
+    const partial = definition.indexesPartial === true;
+    planIndexes(
+      declaredIndexes,
+      live,
+      { prune, rebuildUnique, capabilities, partial, displaced },
+      row,
+      indexSteps,
+    );
   }
-  return { name, actions, steps };
-}
-
-/** Counts over planned (or executed) collections */
-function summarize(collections) {
-  let changes = 0;
-  let applied = 0;
-  let conflicts = 0;
-  let destructive = 0;
-  for (const collection of collections) {
-    for (const action of collection.actions) {
-      if (action.action === 'conflict') conflicts += 1;
-      if (!CHANGE_ACTIONS.has(action.action)) continue;
-      changes += 1;
-      if (action.status === 'applied') applied += 1;
-      if (isDestructive(action)) destructive += 1;
-    }
+  if (declaredSearch !== undefined) {
+    planSearchIndexes(declaredSearch, live, { prune, search }, row, searchSubmit, searchDrops);
   }
-  return { changes, applied, conflicts, destructive };
+  // Search submissions return at once and build in the background, so they go
+  // before the regular index builds; every drop still comes last.
+  steps.push(...searchSubmit, ...indexSteps, ...searchDrops);
+  return { name, actions, steps };
 }
 
 module.exports = {
   CHANGE_ACTIONS,
+  SEARCH_UNAVAILABLE_REASON,
+  SERVED_ACTIONS,
+  TARGET_LABELS,
   UNIQUE_REBUILD_REASON,
   VALIDATOR_DEFAULTS,
   desiredValidator,
@@ -479,5 +762,4 @@ module.exports = {
   needsConfirmation,
   liveValidator,
   planCollection,
-  summarize,
 };