@underpostnet/cyberia 3.3.0 → 3.3.76

package/src/db/mariadb/MariaDB.js

@@ -1,6 +1,7 @@
 import { createPool } from 'mariadb';
 import Underpost from '../../index.js';
-import { loggerFactory } from '../../server/logger.js';
+import { loggerFactory } from '../../server/ops/logger.js';
+import { latchRuntimeError } from '../../server/runtime/runtime-status.js';
 
 /**
  * Module for interacting with MariaDB/MySQL databases using the mariadb connector.
@@ -51,7 +52,7 @@ class MariaDBService {
       console.log(result);
     } catch (error) {
       logger.error('MariaDB query failed', { error: error.message });
-      if (Underpost.env.isInsideContainer()) Underpost.env.set('container-status', 'error');
+      latchRuntimeError();
     } finally {
       if (conn) conn.release(); // release to pool
       await pool.end();

package/src/db/mongo/MongoBootstrap.js

@@ -9,15 +9,18 @@
  */
 
 import fs from 'fs-extra';
-import { loggerFactory } from '../../server/logger.js';
-import { shellExec } from '../../server/process.js';
-import { crictlCommandFactory } from '../../server/cri.js';
-import { resolveReplicaCount } from '../../server/conf.js';
+import { loggerFactory } from '../../server/ops/logger.js';
+import { shellExec } from '../../server/runtime/process.js';
+import { crictlCommandFactory } from '../../server/ops/cri.js';
+import { ensureContainerStorage } from '../../server/security/container-storage.js';
+import { resolveReplicaCount } from '../../server/runtime/conf.js';
 // Cyclic by construction (index -> cluster -> MongoBootstrap -> index), same as cluster.js.
 // Safe because the binding is only dereferenced inside method bodies, never at module scope.
 import Underpost from '../../index.js';
 import {
   MONGODB_DATA_ROOT,
+  MONGODB_DEFAULT_AUTH_SOURCE,
+  MONGODB_DEFAULT_PORT,
   MONGODB_DEFAULT_REPLICA_COUNT,
   MONGODB_DEFAULT_REPLICA_SET,
   MONGODB_SERVICE_NAME,
@@ -29,6 +32,68 @@ import {
 
 const logger = loggerFactory(import.meta);
 
+const MONGODB_PRIMARY_PROBE_URI =
+  `mongodb://127.0.0.1:${MONGODB_DEFAULT_PORT}/${MONGODB_DEFAULT_AUTH_SOURCE}` +
+  `?directConnection=true&serverSelectionTimeoutMS=8000&connectTimeoutMS=8000&socketTimeoutMS=15000`;
+
+const MONGODB_PRIMARY_PROBE_TIMEOUT_SECONDS = 45;
+
+const MONGODB_PRIMARY_EVAL = {
+  hello: 'db.hello().primary',
+  isMaster: 'rs.isMaster().primary',
+  rsStatus: 'rs.status().members.filter(m=>m.stateStr=="PRIMARY").map(m=>m.name)[0]',
+};
+
+// Rejected verbatim probe results: a shell that answered but knows no primary.
+const MONGODB_EMPTY_PRIMARY_TOKENS = ['null', 'undefined', '[]', '[ ]', 'false', ''];
+
+/**
+ * Extracts a pod name from a primary-host probe result.
+ *
+ * Normalises the two shapes the shells emit — a bare `host:port` string from
+ * `hello`, a quoted element from the `rs.status()` projection — and ignores any
+ * banner or warning lines printed alongside them.
+ *
+ * @param {string} output - Raw probe stdout.
+ * @returns {string|null} Pod name (`mongodb-0`), or null when no host was reported.
+ * @memberof MongoBootstrap
+ */
+const parsePrimaryHost = (output) => {
+  if (!output) return null;
+  const lines = `${output}`
+    .split('\n')
+    .map((line) => line.trim())
+    .filter(Boolean);
+
+  for (const line of lines.reverse()) {
+    const quoted = line.match(/['"]([^'"]+)['"]/);
+    const candidate = (quoted ? quoted[1] : line).trim();
+    if (MONGODB_EMPTY_PRIMARY_TOKENS.includes(candidate.toLowerCase())) continue;
+    if (!/^[A-Za-z0-9][A-Za-z0-9._-]*(:\d+)?$/.test(candidate)) continue;
+    return candidate.split(':')[0].split('.')[0];
+  }
+  return null;
+};
+
+/**
+ * Extracts the two replica set identities from a heartbeat rejection.
+ *
+ * mongod reports `Our replica set ID did not match that of our request target` when two
+ * members carry different set IDs — the signature of one member having been re-initiated on
+ * an empty volume while the others kept the original set.
+ *
+ * @param {string} output - Combined stdout/stderr of a mongosh bootstrap attempt.
+ * @returns {{selfSetId: string, target: string, targetSetId: string}|null} Parsed identities, or null.
+ * @memberof MongoBootstrap
+ */
+const parseDivergentSetIds = (output) => {
+  const match = `${output || ''}`.match(
+    /replSetId:\s*([0-9a-f]+),\s*requestTarget:\s*([^,]+),\s*requestTargetReplSetId:\s*([0-9a-f]+)/,
+  );
+  if (!match) return null;
+  return { selfSetId: match[1], target: match[2].trim(), targetSetId: match[3] };
+};
+
 // Teardown sweeps ordinals rather than a live replica count: the deployed count is not knowable
 // during a reset, and a member left behind from a larger previous deployment keeps its volume and
 // stale replica-set config, which a later deploy then trips over.
@@ -90,7 +155,7 @@ class MongoBootstrap {
     rootUser,
     rootPassword,
   }) {
-    const mePort = '27017';
+    const mePort = MONGODB_DEFAULT_PORT;
     const defaultShortHosts = Array.from(
       { length: replicaCount },
       (_, i) => `${statefulSetName}-${i}.${serviceName}:${mePort}`,
@@ -102,8 +167,13 @@ class MongoBootstrap {
     };
 
     return [
-      `const mePort = "27017";`,
       `const desiredConfig = ${JSON.stringify(desiredConfig)};`,
+      // Single-member bootstrap and recovery configs name this node by the same host it
+      // advertises in the final config, never `localhost`. A client that connects while the
+      // set is still one member long copies that member into its topology, and inside any
+      // other pod `localhost:27017` is that pod itself — the client then sits on an
+      // unreachable member and never recovers.
+      `const selfHost = desiredConfig.members[0].host;`,
       `const rootUser = ${JSON.stringify(rootUser)};`,
       `const rootPassword = ${JSON.stringify(rootPassword)};`,
 
@@ -166,7 +236,7 @@ class MongoBootstrap {
       `  const soloConfig = {`,
       `    _id: desiredConfig._id,`,
       `    version: currentConfigVersion() + 1,`,
-      `    members: [{ _id: 0, host: "localhost:" + mePort }],`,
+      `    members: [{ _id: 0, host: selfHost }],`,
       `  };`,
       `  rs.reconfig(soloConfig, { force: true });`,
       `  print("SUCCESS_FORCE_RECONFIGURED");`,
@@ -178,7 +248,12 @@ class MongoBootstrap {
       `  const curHosts = cur.members.map(m => m.host).sort().join(",");`,
       `  const nextHosts = desiredConfig.members.map(m => m.host).sort().join(",");`,
       `  if (curHosts !== nextHosts || cur._id !== desiredConfig._id) {`,
-      `    rs.reconfig({...desiredConfig, version: (cur.version || 1) + 1}, { force: true });`,
+      // Not forced: this runs on a node waitPrimary() just confirmed is writable, so an
+      // ordinary reconfig applies and bumps the version by one. A forced reconfig instead
+      // jumps the config version and election id by a random amount, and drivers that
+      // recorded the higher values reject the legitimate primary as stale for the rest of
+      // the process lifetime — pods come back as `ReplicaSetNoPrimary` and stay there.
+      `    rs.reconfig({...desiredConfig, version: (cur.version || 1) + 1});`,
       `    print("SUCCESS_RECONFIGURED");`,
       `  } else {`,
       `    print("SUCCESS_ALREADY_MATCHES");`,
@@ -212,7 +287,7 @@ class MongoBootstrap {
 
       `if (state === "pristine") {`,
       `  try {`,
-      `    rs.initiate({ _id: desiredConfig._id, members: [{ _id: 0, host: "localhost:" + mePort }] });`,
+      `    rs.initiate({ _id: desiredConfig._id, members: [{ _id: 0, host: selfHost }] });`,
       `  } catch(e) {`,
       `    const msg = String(e);`,
       `    if (!msg.includes("already initialized") && !msg.includes("AlreadyInitialized")) throw e;`,
@@ -301,14 +376,15 @@ class MongoBootstrap {
   }
 
   /**
-   * Reads MongoDB root credentials from engine-private.
-   * @param {string} enginePrivateRoot - Path to the engine-private directory.
+   * Reads MongoDB root credentials from their origin seed files.
+   * Paths come from the workload secret domain, so this never spells out a credential path.
    * @returns {{ username: string, password: string }}
    */
-  static readMongoCredentials(enginePrivateRoot) {
+  static readMongoCredentials() {
+    const sources = Underpost.secret.seedSources('mongodb-secret');
     return {
-      username: MongoBootstrap.readCredential(`${enginePrivateRoot}/mongodb-username`),
-      password: MongoBootstrap.readCredential(`${enginePrivateRoot}/mongodb-password`),
+      username: MongoBootstrap.readCredential(sources.username),
+      password: MongoBootstrap.readCredential(sources.password),
     };
   }
 
@@ -324,32 +400,59 @@ class MongoBootstrap {
    * in the process table and the command log; `disableLog` keeps it out of the log at least. The
    * encrypted path has no such exposure — the value only ever crosses an anonymous pipe.
    * @param {string} namespace - Target namespace.
-   * @param {string} enginePrivateRoot - Path to engine-private directory.
    */
-  static ensureMongoSecrets(namespace, enginePrivateRoot) {
-    if (!Underpost.secret.sops.applyIfPresent('mongodb-keyfile', namespace)) {
-      const keyfile = MongoBootstrap.readCredential(`${enginePrivateRoot}/mongodb-keyfile`);
+  static ensureMongoSecrets(namespace) {
+    const keyfilePath = Underpost.secret.seedSources('mongodb-keyfile')['mongodb-keyfile'];
+    if (!Underpost.secret.applyIfPresent('mongodb-keyfile', namespace)) {
+      const keyfile = MongoBootstrap.readCredential(keyfilePath);
       shellExec(
         `sudo kubectl create secret generic mongodb-keyfile` +
           ` --from-literal=mongodb-keyfile="${keyfile.replace(/'/g, "'\\''")}"` +
           ` --dry-run=client -o yaml | kubectl apply -f - -n ${namespace}`,
         { disableLog: true },
       );
-      logger.info(`Seeded mongodb-keyfile from ${enginePrivateRoot}/mongodb-keyfile`);
+      logger.info(`Seeded mongodb-keyfile from ${keyfilePath}`);
     }
 
-    if (!Underpost.secret.sops.applyIfPresent('mongodb-secret', namespace)) {
-      const { username, password } = MongoBootstrap.readMongoCredentials(enginePrivateRoot);
+    if (!Underpost.secret.applyIfPresent('mongodb-secret', namespace)) {
+      const { username, password } = MongoBootstrap.readMongoCredentials();
       shellExec(
         `sudo kubectl create secret generic mongodb-secret` +
           ` --from-literal=username="${username}" --from-literal=password="${password}"` +
           ` --dry-run=client -o yaml | kubectl apply -f - -n ${namespace}`,
         { disableLog: true },
       );
-      logger.info(`Seeded mongodb-secret from ${enginePrivateRoot}/mongodb-{username,password}`);
+      logger.info('Seeded mongodb-secret from origin seed files');
     }
   }
 
+  /**
+   * Creates the replica data directories and gives the tree the shared container label.
+   *
+   * The PVs are `hostPath` with `type: DirectoryOrCreate`, so kubelet will happily create a
+   * missing member directory — with whatever context its parent carries. `/data` has no mapping
+   * in the base RHEL policy, so that context is `default_t`, which `container_t` cannot write:
+   * mongod's checkpointer and `ftdc` writer then fail on every `WiredTiger.turtle.set` rename and
+   * `metrics.interim.temp` create. Registering the mapping and restoring the tree here makes the
+   * label a property of the directory rather than of whoever happened to create it, and it
+   * survives a policy update or a full relabel.
+   *
+   * Only the directories are created; ownership and mode are left to the StatefulSet's
+   * `data-dir-permissions` init container, which is the single owner of that concern.
+   * @param {number} replicaCount - Number of member directories to prepare.
+   * @param {{execute?: Function}} [options] - Command executor, injectable for tests.
+   * @returns {string[]} Prepared paths.
+   * @memberof MongoBootstrap
+   */
+  static prepareReplicaDataRoot(replicaCount, { execute = shellExec } = {}) {
+    const paths = [
+      MONGODB_DATA_ROOT,
+      ...Array.from({ length: Math.max(0, Number(replicaCount) || 0) }, (_, i) => `${MONGODB_DATA_ROOT}/v${i}`),
+    ];
+    ensureContainerStorage(paths, { execute });
+    return paths;
+  }
+
   /**
    * Renders one hostPath PersistentVolume per replica, each pre-bound to that member's claim.
    *
@@ -484,25 +587,6 @@ class MongoBootstrap {
       );
   }
 
-  /**
-   * Waits for all MongoDB statefulset pods to reach Running state.
-   * @param {string} namespace - Target namespace.
-   * @param {number} replicaCount - Expected number of pods.
-   * @returns {Promise<number>} Number of pods that failed to become ready (0 = all good).
-   */
-  static async waitForPods(namespace, replicaCount) {
-    let failedCount = 0;
-    for (let i = 0; i < replicaCount; i++) {
-      const podName = `${MONGODB_STATEFULSET_NAME}-${i}`;
-      const result = shellExec(`kubectl wait --for=condition=Ready pod/${podName} -n ${namespace} --timeout=60s`);
-      if (result.code !== 0) {
-        logger.error(`Pod ${podName} did not become ready`);
-        failedCount++;
-      }
-    }
-    return failedCount;
-  }
-
   /**
    * Full MongoDB replica set initialization.
    *
@@ -523,7 +607,6 @@ class MongoBootstrap {
       underpostRoot,
     } = options;
 
-    const enginePrivateRoot = `${process.cwd()}/engine-private`;
     // No upward clamp: an explicit `--replicas 2` must deploy two members, not be silently
     // raised to the default.
     const effectiveReplicaCount = resolveReplicaCount(replicaCount, MONGODB_DEFAULT_REPLICA_COUNT);
@@ -533,8 +616,7 @@ class MongoBootstrap {
           `majority when one member is down, so the set becomes read-only on a single failure. ` +
           `Odd counts (3, 5) are recommended.`,
       );
-    const mongoRootUsername = MongoBootstrap.readCredential(`${enginePrivateRoot}/mongodb-username`);
-    const mongoRootPassword = MongoBootstrap.readCredential(`${enginePrivateRoot}/mongodb-password`);
+    const { username: mongoRootUsername, password: mongoRootPassword } = MongoBootstrap.readMongoCredentials();
     const mongoReplicaHosts = resolveMongoReplicaHosts({
       hostList,
       replicaCount: effectiveReplicaCount,
@@ -578,7 +660,7 @@ class MongoBootstrap {
     }
 
     // Secrets
-    MongoBootstrap.ensureMongoSecrets(namespace, enginePrivateRoot);
+    MongoBootstrap.ensureMongoSecrets(namespace);
 
     // Tear down existing statefulset
     shellExec(`kubectl delete statefulset ${MONGODB_STATEFULSET_NAME} -n ${namespace} --ignore-not-found`);
@@ -614,8 +696,11 @@ class MongoBootstrap {
       shellExec(`sudo mkdir -p ${MONGODB_DATA_ROOT}`);
       for (let i = 0; i < effectiveReplicaCount; i++) {
         shellExec(`sudo rm -rf ${MONGODB_DATA_ROOT}/v${i}`);
-        shellExec(`sudo mkdir -p ${MONGODB_DATA_ROOT}/v${i}`);
       }
+      // A directory recreated after a wipe inherits its parent's context, and the WiredTiger
+      // files mongod then creates inherit that in turn — so recreate and relabel here, before the
+      // Kind remount below binds these inodes into the node containers.
+      MongoBootstrap.prepareReplicaDataRoot(effectiveReplicaCount);
       // Fix any stale bind mounts caused by prior deletion of /data/mongodb on the host
       if (isKind) MongoBootstrap.remountKindMongoVolume(kindNodes, effectiveReplicaCount);
     }
@@ -637,6 +722,10 @@ class MongoBootstrap {
       shellExec(`kubectl delete storageclass ${MONGODB_STORAGE_CLASS_NAME} --ignore-not-found`);
     }
     shellExec(`kubectl apply -f ${underpostRoot}/manifests/mongodb/storage-class.yaml -n ${namespace}`);
+    // Unconditional, not only on the reset path: an existing cluster's member directories were
+    // created before the mapping existed (or by kubelet's `DirectoryOrCreate`), so they carry the
+    // policy default until this runs. Idempotent, and it never touches the data itself.
+    MongoBootstrap.prepareReplicaDataRoot(effectiveReplicaCount);
     // One PV per member, generated from the effective replica count. A static manifest can only
     // ever describe a fixed number, so any `--replicas` above it leaves the extra members with no
     // volume to bind and the StatefulSet stalls forever on Pending.
@@ -647,13 +736,16 @@ class MongoBootstrap {
     );
 
     // Wait for all pods
-    const failedCount = await MongoBootstrap.waitForPods(namespace, effectiveReplicaCount);
-    if (failedCount > 0) {
+    const failedPods = await Underpost.kubectl.waitForPodsReady({
+      namespace,
+      podNames: Array.from({ length: effectiveReplicaCount }, (_, i) => `${MONGODB_STATEFULSET_NAME}-${i}`),
+    });
+    if (failedPods.length > 0) {
       // Surface the volume topology before failing: a stalled rollout is far more often a binding
       // problem than a MongoDB one, and the mapping names it immediately.
       MongoBootstrap.reportReplicaVolumeBindings(namespace, effectiveReplicaCount);
       throw new Error(
-        `MongoDB replica pods did not reach Running state in time. ` +
+        `MongoDB replica pods did not reach Running state in time: ${failedPods.join(', ')}. ` +
           `Ensure podManagementPolicy is set to OrderedReady in statefulset.yaml.`,
       );
     }
@@ -663,7 +755,7 @@ class MongoBootstrap {
     // Build the bootstrap script
     const defaultHosts = Array.from(
       { length: effectiveReplicaCount },
-      (_, i) => `${MONGODB_STATEFULSET_NAME}-${i}.${MONGODB_SERVICE_NAME}:27017`,
+      (_, i) => `${MONGODB_STATEFULSET_NAME}-${i}.${MONGODB_SERVICE_NAME}:${MONGODB_DEFAULT_PORT}`,
     );
     const desiredHosts = useExplicitHosts ? mongoReplicaHosts : defaultHosts;
 
@@ -697,32 +789,46 @@ class MongoBootstrap {
       );
     };
 
+    // Two members holding different replica set IDs cannot be merged — every heartbeat
+    // between them is rejected — so retrying only repeats the same failure. Diagnose it once
+    // and stop: choosing which identity survives means discarding the other member's data,
+    // which is never safe to decide automatically.
+    const assertSetIdentityMatch = (result) => {
+      const divergent = parseDivergentSetIds(`${result.stdout || ''}${result.stderr || ''}`);
+      if (!divergent) return;
+      throw new Error(
+        `MongoDB members belong to different replica sets: ${MONGODB_STATEFULSET_NAME}-0 holds replSetId ` +
+          `${divergent.selfSetId}, ${divergent.target} holds ${divergent.targetSetId}. A set cannot merge two ` +
+          `identities. Find which volume under ${MONGODB_DATA_ROOT} holds the data to keep, back it up, then ` +
+          `wipe the other members' volumes so they resync from the survivor.`,
+      );
+    };
+
     let success = false;
     const maxAttempts = 5;
 
     for (let attempt = 1; attempt <= maxAttempts; attempt++) {
       const noAuthResult = execMongoCmd(false);
-      if (noAuthResult.code === 0) {
-        if (mongoRootUsername && mongoRootPassword) {
-          const authResult = execMongoCmd(true);
-          if (authResult.code === 0) {
-            success = true;
-            break;
-          }
-          logger.warn('No-auth bootstrap succeeded but auth verify failed, retrying...', { attempt });
-        } else {
-          success = true;
-          break;
-        }
-      } else {
-        const authResult = execMongoCmd(true);
-        if (authResult.code === 0) {
-          success = true;
-          break;
-        }
-        logger.warn('Both bootstrap paths failed, retrying...', { attempt });
+      assertSetIdentityMatch(noAuthResult);
+
+      if (noAuthResult.code === 0 && !(mongoRootUsername && mongoRootPassword)) {
+        success = true;
+        break;
       }
 
+      const authResult = execMongoCmd(true);
+      assertSetIdentityMatch(authResult);
+      if (authResult.code === 0) {
+        success = true;
+        break;
+      }
+      logger.warn(
+        noAuthResult.code === 0
+          ? 'No-auth bootstrap succeeded but auth verify failed, retrying...'
+          : 'Both bootstrap paths failed, retrying...',
+        { attempt },
+      );
+
       if (attempt < maxAttempts) {
         await new Promise((r) => setTimeout(r, 3000));
       }
@@ -780,7 +886,7 @@ class MongoBootstrap {
       // Also catch any remaining PVs with the app=mongodb label
       shellExec(`kubectl delete pv -l app=mongodb --ignore-not-found`);
       // Wait for PVs to be fully deleted to avoid "object modified" conflict on re-apply
-      shellExec(`kubectl wait --for=delete pv mongodb-pv-0 mongodb-pv-1 mongodb-pv-2 mongodb-pv --timeout=60s`, {
+      shellExec(`kubectl wait --for=delete pv mongodb-pv-0 mongodb-pv-1 mongodb-pv-2 mongodb-pv --timeout=180s`, {
         silentOnError: true,
       });
 
@@ -794,8 +900,8 @@ class MongoBootstrap {
       shellExec(`sudo mkdir -p ${MONGODB_DATA_ROOT}`);
       for (let i = 0; i < MONGODB_ORDINAL_SWEEP; i++) {
         shellExec(`sudo rm -rf ${MONGODB_DATA_ROOT}/v${i}`);
-        shellExec(`sudo mkdir -p ${MONGODB_DATA_ROOT}/v${i}`);
       }
+      MongoBootstrap.prepareReplicaDataRoot(MONGODB_ORDINAL_SWEEP);
       // For Kind: repair any stale bind mounts via nsenter (overmounts with current host inode)
       if (isKind) {
         const nodesRaw = shellExec('kind get nodes', { stdout: true, silent: true, silentOnError: true });
@@ -819,13 +925,22 @@ class MongoBootstrap {
 
   /**
    * Gets the primary MongoDB pod name from replica set status.
+   *
+   * Probing is bounded on every axis, because an unbounded probe is what makes the
+   * backup pipeline stall: the driver connects `directConnection` so it never tries
+   * to reach sibling members by DNS, each shell call carries explicit server-selection
+   * timeouts, and the whole `kubectl exec` is wrapped in a wall-clock budget. The
+   * cheapest probe (`hello`, which the server answers before authentication) runs
+   * first, so a healthy replica set resolves without credentials at all.
+   *
    * @param {object} [options] - Query options.
    * @param {string} [options.namespace='default'] - Kubernetes namespace.
-   * @param {string} [options.podName='mongodb-0'] - Any MongoDB pod to query.
+   * @param {string} [options.podName='mongodb-0'] - Preferred MongoDB pod to query first.
    * @param {string} [options.username] - MongoDB admin username.
    * @param {string} [options.password] - MongoDB admin password.
    * @param {string} [options.authDatabase='admin'] - Auth database.
    * @param {boolean} [options.disableAuth=false] - Whether to disable auth in the query (for testing).
+   * @param {number} [options.probeTimeoutSeconds=45] - Wall-clock budget per shell probe.
    * @returns {string|null} Primary pod name, or null if not found.
    */
   static getPrimaryPodName(options = {}) {
@@ -836,6 +951,7 @@ class MongoBootstrap {
       password,
       authDatabase = 'admin',
       disableAuth = false,
+      probeTimeoutSeconds = MONGODB_PRIMARY_PROBE_TIMEOUT_SECONDS,
     } = options;
 
     const readTrimmedFile = (filePath) => {
@@ -851,48 +967,104 @@ class MongoBootstrap {
       username ||
       process.env.MONGODB_USERNAME ||
       process.env.DB_USER ||
-      readTrimmedFile('./engine-private/mongodb-username');
+      readTrimmedFile(Underpost.secret.seedSources('mongodb-secret').username);
     const mongoPass =
       password ||
       process.env.MONGODB_PASSWORD ||
       process.env.DB_PASSWORD ||
-      readTrimmedFile('./engine-private/mongodb-password');
+      readTrimmedFile(Underpost.secret.seedSources('mongodb-secret').password);
+
+    const candidates = MongoBootstrap.resolvePrimaryProbeCandidates({ namespace, podName });
+    const probes = MongoBootstrap.primaryProbeFactory({
+      disableAuth,
+      username: mongoUser,
+      password: mongoPass,
+      authDatabase,
+    });
 
-    // Ensure the pod is ready before querying
-    shellExec(`kubectl wait --for=condition=Ready pod/${podName} -n ${namespace} --timeout=60s`);
+    for (const candidate of candidates) {
+      if (!Underpost.kubectl.ensureExecReady({ podName: candidate, namespace })) continue;
 
-    const evalExpr = 'rs.status().members.filter(m=>m.stateStr=="PRIMARY").map(m=>m.name)';
+      for (const probe of probes) {
+        const output = shellExec(
+          `timeout --kill-after=10s ${probeTimeoutSeconds}s ` +
+            `sudo kubectl exec -n ${namespace} -i ${candidate} -- ${probe.command}`,
+          { stdout: true, silent: true, silentOnError: true, disableLog: true },
+        );
+        const primary = parsePrimaryHost(output);
+        if (primary) {
+          logger.info('Found MongoDB primary pod', { primary, probedFrom: candidate, probe: probe.id });
+          return primary;
+        }
+      }
 
-    const cli = disableAuth ? 'mongo' : 'mongosh';
+      logger.warn('MongoDB pod answered no primary', { podName: candidate, namespace });
+    }
 
-    let output = shellExec(`sudo kubectl exec -n ${namespace} -i ${podName} -- ${cli} --quiet --eval '${evalExpr}'`, {
-      stdout: true,
-      silent: true,
-      silentOnError: true,
-    });
+    logger.warn('No MongoDB primary pod found.', { namespace, candidates });
+    return null;
+  }
 
-    if (!disableAuth && (!output || output.trim() === '') && mongoUser && mongoPass) {
-      output = shellExec(
-        `sudo kubectl exec -n ${namespace} -i ${podName} -- mongosh --quiet ` +
-          `--authenticationDatabase ${JSON.stringify(authDatabase)} ` +
-          `-u ${JSON.stringify(mongoUser)} -p ${JSON.stringify(mongoPass)} --eval '${evalExpr}'`,
-        { stdout: true, silent: true, silentOnError: true },
-      );
+  /**
+   * Resolves which pods to interrogate for the replica-set primary, preferred pod first.
+   *
+   * Any reachable member knows who the primary is, so a member that is mid-restart
+   * must not end the search — the caller would otherwise fall back to a pod that is
+   * merely first in the list rather than the one accepting writes.
+   *
+   * @param {object} params
+   * @param {string} params.namespace - Kubernetes namespace.
+   * @param {string} params.podName - Preferred pod to probe first.
+   * @returns {string[]} Ordered, de-duplicated pod names.
+   */
+  static resolvePrimaryProbeCandidates({ namespace, podName }) {
+    const candidates = podName ? [podName] : [];
+    try {
+      for (const pod of Underpost.kubectl.get(MONGODB_STATEFULSET_NAME, 'pods', namespace)) {
+        if (!pod.NAME || pod.STATUS !== 'Running') continue;
+        if (!candidates.includes(pod.NAME)) candidates.push(pod.NAME);
+      }
+    } catch (error) {
+      logger.warn('Could not list MongoDB pods, probing the requested pod only', {
+        namespace,
+        error: error.message,
+      });
     }
+    return candidates;
+  }
 
-    if (!output || output.trim() === '') {
-      logger.warn('No MongoDB primary pod found.');
-      return null;
-    }
+  /**
+   * Builds the ordered probe list used to resolve the replica-set primary.
+   * @param {object} params
+   * @param {boolean} params.disableAuth - Legacy `mongo` shell without credentials.
+   * @param {string} params.username - MongoDB username, when available.
+   * @param {string} params.password - MongoDB password, when available.
+   * @param {string} params.authDatabase - Auth database for credentialed probes.
+   * @returns {Array<{id: string, command: string}>} Probes in cheapest-first order.
+   */
+  static primaryProbeFactory({ disableAuth, username, password, authDatabase }) {
+    if (disableAuth)
+      return [
+        { id: 'legacy-isMaster', command: `mongo --quiet --eval '${MONGODB_PRIMARY_EVAL.isMaster}'` },
+        { id: 'legacy-rs-status', command: `mongo --quiet --eval '${MONGODB_PRIMARY_EVAL.rsStatus}'` },
+      ];
+
+    const shell = `mongosh ${JSON.stringify(MONGODB_PRIMARY_PROBE_URI)} --quiet`;
+    const authFlags =
+      username && password
+        ? ` --authenticationDatabase ${JSON.stringify(authDatabase)} -u ${JSON.stringify(username)} -p ${JSON.stringify(password)}`
+        : '';
 
-    const match = output.match(/['"]([^'"]+)['"]/);
-    if (match && match[1]) {
-      const primary = match[1].split(':')[0].split('.')[0];
-      logger.info('Found MongoDB primary pod', { primary });
-      return primary;
+    // `hello` is answered before authentication, so it resolves the primary even when
+    // the credentials at hand are an app user without cluster-monitor rights.
+    const probes = [{ id: 'hello', command: `${shell} --eval '${MONGODB_PRIMARY_EVAL.hello}'` }];
+    if (authFlags) {
+      probes.push({ id: 'hello-auth', command: `${shell}${authFlags} --eval '${MONGODB_PRIMARY_EVAL.hello}'` });
+      probes.push({ id: 'rs-status-auth', command: `${shell}${authFlags} --eval '${MONGODB_PRIMARY_EVAL.rsStatus}'` });
     }
-    return null;
+    probes.push({ id: 'rs-status', command: `${shell} --eval '${MONGODB_PRIMARY_EVAL.rsStatus}'` });
+    return probes;
   }
 }
 
-export { MongoBootstrap };
+export { MongoBootstrap, parsePrimaryHost, parseDivergentSetIds, MONGODB_PRIMARY_PROBE_URI };