@underpostnet/cyberia 3.2.90 → 3.3.73

package/src/cli/ipfs.js

@@ -4,14 +4,19 @@
  * @namespace UnderpostIPFS
  */
 
-import { loggerFactory } from '../server/logger.js';
-import { shellExec } from '../server/process.js';
-import { resolveReplicaCount } from '../server/conf.js';
+import { loggerFactory } from '../server/ops/logger.js';
+import { shellExec } from '../server/runtime/process.js';
+import { resolveReplicaCount } from '../server/runtime/conf.js';
 import fs from 'fs-extra';
+import dir from 'node:path';
 import Underpost from '../index.js';
 
 const logger = loggerFactory(import.meta);
 
+// The managed secret this module owns; its origin, its encrypted manifest and its Kubernetes
+// projection all resolve from this one name.
+const IPFS_SECRET_NAME = 'ipfs-cluster-secret';
+
 const IPFS_DEFAULT_REPLICA_COUNT = 3;
 
 /**
@@ -25,27 +30,32 @@ class UnderpostIPFS {
   static API = {
     /**
      * @method resolveCredentials
-     * @description Resolves the IPFS cluster credentials from engine-private/ if they
-     * already exist, otherwise generates new ones (hex cluster secret + peer identity
-     * via ipfs-cluster-service init) and persists them with mode 0o600.
+     * @description Resolves the IPFS cluster origin credentials, generating them when absent
+     * (hex cluster secret + peer identity via ipfs-cluster-service init) and persisting them
+     * with mode 0o600.
+     *
+     * Paths come from the workload secret domain, so this module names no credential path of its
+     * own and the pair sits in the deploy secret area with every other origin credential.
+     *
      * Idempotent and self-healing: an existing pair is reused untouched, and a missing or
      * unreadable one is regenerated. Both files are always rewritten together — the peer id and
      * the private key are two halves of one identity, so reusing one with a freshly minted other
      * would advertise a peer id that does not match the key.
-     * @param {string} privateDir - Absolute path to the engine-private directory.
      * @returns {{ CLUSTER_SECRET: string, IDENTITY_JSON: { id: string, private_key: string }, generated: boolean }}
      * @memberof UnderpostIPFS
      */
-    resolveCredentials(privateDir) {
-      const secretPath = `${privateDir}/ipfs-cluster-secret`;
-      const identityPath = `${privateDir}/ipfs-cluster-identity.json`;
+    resolveCredentials() {
+      const sources = Underpost.secret.seedSources(IPFS_SECRET_NAME);
+      const secretPath = sources['cluster-secret'];
+      const identityPath = sources['bootstrap-peer-priv-key'];
+      const privateDir = dir.dirname(secretPath);
 
       if (fs.existsSync(secretPath) && fs.existsSync(identityPath)) {
         try {
           const CLUSTER_SECRET = fs.readFileSync(secretPath, 'utf8').trim();
           const IDENTITY_JSON = JSON.parse(fs.readFileSync(identityPath, 'utf8'));
           if (CLUSTER_SECRET && IDENTITY_JSON?.id && IDENTITY_JSON?.private_key) {
-            logger.info('Reusing existing IPFS cluster credentials from engine-private/');
+            logger.info('Reusing existing IPFS cluster credentials', { secretPath, identityPath });
             return { CLUSTER_SECRET, IDENTITY_JSON, generated: false };
           }
           logger.warn('Existing IPFS cluster credentials are incomplete; regenerating');
@@ -54,7 +64,7 @@ class UnderpostIPFS {
         }
       }
 
-      logger.info('Generating new IPFS cluster credentials and persisting to engine-private/');
+      logger.info('Generating new IPFS cluster credentials', { secretPath, identityPath });
 
       // ipfs-cluster-service requires CLUSTER_SECRET as a 64-char hex string.
       // base64 (openssl rand -base64 32) contains '/', '+', '=' which are invalid hex bytes.
@@ -115,7 +125,7 @@ class UnderpostIPFS {
         );
         fs.chmodSync(stagePath, 0o600);
         // encrypt() stages, validates, moves into place, and shreds the plaintext source.
-        Underpost.secret.sops.encrypt(stagePath, options.namespace, { force: true });
+        Underpost.secret.encrypt(stagePath, options.namespace, { force: true });
         logger.info('Re-encrypted regenerated IPFS credentials into the SOPS store');
       } finally {
         fs.removeSync(stageDir);
@@ -148,28 +158,22 @@ class UnderpostIPFS {
      * @method applySecrets
      * @description Creates (or idempotently updates) the Kubernetes Secret and env ConfigMap
      * that the StatefulSet pods read at startup.
-     * - Secret `ipfs-cluster-secret`: cluster-secret + bootstrap-peer-priv-key
+     * - Secret `ipfs-cluster-secret`: cluster-secret + bootstrap-peer-priv-key, projected by the
+     *   secret domain from the encrypted store or, failing that, from the origin credentials
      * - ConfigMap `env-config`: bootstrap-peer-id + CLUSTER_SVC_NAME
-     * @param {{ CLUSTER_SECRET: string, IDENTITY_JSON: { id: string, private_key: string } }} credentials
+     * @param {{ IDENTITY_JSON: { id: string } }} credentials - Only the public peer id is read here; the Secret keys are projected by the secret domain.
      * @param {object} options
      * @param {string} options.namespace - Kubernetes namespace.
      * @memberof UnderpostIPFS
      */
-    applySecrets({ CLUSTER_SECRET, IDENTITY_JSON }, options) {
+    applySecrets({ IDENTITY_JSON }, options) {
       logger.info('Applying IPFS cluster Kubernetes Secret and env ConfigMap');
 
-      // Encrypted store first, origin generate/read logic only when no manifest is stored.
-      // `--from-literal` places the cluster secret and the peer private key in the command
-      // string, so the seed path is kept out of the command log; the encrypted path never
-      // exposes them at all.
-      if (!Underpost.secret.sops.applyIfPresent('ipfs-cluster-secret', options.namespace))
-        shellExec(
-          `kubectl create secret generic ipfs-cluster-secret \
---from-literal=cluster-secret=${CLUSTER_SECRET} \
---from-literal=bootstrap-peer-priv-key=${IDENTITY_JSON.private_key} \
---dry-run=client -o yaml | kubectl apply -f - -n ${options.namespace}`,
-          { disableLog: true },
-        );
+      // Encrypted store first; the origin projection runs only when no manifest is stored. Both
+      // paths keep the credentials out of the command string — the store streams a decrypt into
+      // kubectl, and the origin projection stages the values on tmpfs.
+      if (!Underpost.secret.applyIfPresent(IPFS_SECRET_NAME, options.namespace))
+        Underpost.secret.applyFromOriginSeed(IPFS_SECRET_NAME, options.namespace);
 
       shellExec(
         `kubectl create configmap env-config \
@@ -227,13 +231,13 @@ sudo sysctl -w net.core.wmem_max=7500000`,
         Underpost.cluster.pullImage('ipfs/ipfs-cluster:latest', options);
       }
 
-      const credentials = Underpost.ipfs.resolveCredentials(`${underpostRoot}/engine-private`);
+      const credentials = Underpost.ipfs.resolveCredentials();
 
       // `env-config` advertises `bootstrap-peer-id` from the local identity file — the peer id is
       // not carried in the Secret. So a regeneration invalidates any stored manifest: it would
       // pair the previous private key with the new id and the cluster would never form. Re-encrypt
       // the fresh pair so the store and the ConfigMap stay one identity.
-      if (credentials.generated && Underpost.secret.sops.has('ipfs-cluster-secret', options.namespace))
+      if (credentials.generated && Underpost.secret.has(IPFS_SECRET_NAME, options.namespace))
         Underpost.ipfs.storeCredentials(credentials, options);
 
       const ipfsReplicas = resolveReplicaCount(options.replicas, IPFS_DEFAULT_REPLICA_COUNT);

package/src/cli/kickstart.js

@@ -7,7 +7,7 @@
 import fs from 'fs';
 import path from 'path';
 import { fileURLToPath } from 'url';
-import { loggerFactory } from '../server/logger.js';
+import { loggerFactory } from '../server/ops/logger.js';
 
 const logger = loggerFactory(import.meta);
 const __filename = fileURLToPath(import.meta.url);

package/src/cli/kubectl.js

@@ -6,26 +6,37 @@
  * @namespace UnderpostKubectl
  */
 
-import { loggerFactory } from '../server/logger.js';
-import { shellExec } from '../server/process.js';
+import { loggerFactory, redactSensitiveText } from '../server/ops/logger.js';
+import { shellExec, sleepSync } from '../server/runtime/process.js';
+import { timer } from '../client/components/core/CommonJs.js';
 import Underpost from '../index.js';
 
 const logger = loggerFactory(import.meta);
 
+const DEFAULT_POD_RETRIES = 2;
+const RETRY_BASE_DELAY_MS = 2000;
+const DEFAULT_EXEC_READY_ATTEMPTS = 5;
+const DEFAULT_CREATION_TIMEOUT_SECONDS = 120;
+const DEFAULT_READY_TIMEOUT_SECONDS = 180;
+
+/**
+ * Exponential backoff for attempt `n`: 2s, 4s, 8s, …
+ * @param {number} attempt - 1-based attempt that just failed.
+ * @param {number} [baseDelayMs=2000] - Delay after the first failure.
+ * @returns {number} Milliseconds to wait before the next attempt.
+ * @memberof UnderpostKubectl
+ */
+const backoffDelayMs = (attempt, baseDelayMs = RETRY_BASE_DELAY_MS) => baseDelayMs * 2 ** (attempt - 1);
+
 /**
- * Redacts credentials from shell command strings before logging.
- * Masks passwords in `-p<password>`, `--password=<password>`, and `-P <password>` patterns.
- * @param {string} cmd - The raw command string.
- * @returns {string} The command with credentials replaced by `***`.
+ * Wraps a command in `timeout(1)` so a stalled API-server stream cannot hang a job forever.
+ * @param {string} command - Command to bound.
+ * @param {number} timeoutSeconds - Wall-clock budget; `0`/falsy returns the command unchanged.
+ * @returns {string} The bounded command.
  * @memberof UnderpostKubectl
  */
-const sanitizeCommand = (cmd) => {
-  if (typeof cmd !== 'string') return cmd;
-  return cmd
-    .replace(/-p['"]?[^\s'"]+/g, '-p***')
-    .replace(/--password=['"]?[^\s'"]+/g, '--password=***')
-    .replace(/-P\s+['"]?[^\s'"]+/g, '-P ***');
-};
+const withTimeout = (command, timeoutSeconds) =>
+  timeoutSeconds > 0 ? `timeout --kill-after=10s ${timeoutSeconds}s ${command}` : command;
 
 /**
  * @class UnderpostKubectl
@@ -80,22 +91,187 @@ class UnderpostKubectl {
 
     /**
      * Executes a kubectl command with credential-safe logging and error propagation.
+     * Retries any failure up to `retries` times with exponential backoff; callers whose
+     * command is not safe to replay pass `retries: 0`.
      * @param {string} command - Full kubectl command string.
      * @param {object} [options={}] - Execution options.
      * @param {string} [options.context=''] - Human-readable label for log messages.
+     * @param {number} [options.retries=0] - Extra attempts allowed after a failure.
+     * @param {number} [options.baseDelayMs=2000] - Delay after the first failure; doubles per attempt.
+     * @param {number} [options.timeoutSeconds=0] - Wall-clock budget per attempt; `0` disables the bound.
+     * @param {string} [options.podName=''] - Pod the command targets; retries wait for it to accept exec again.
+     * @param {string} [options.namespace='default'] - Namespace of `podName`.
      * @returns {string} stdout output from the command.
-     * @throws {Error} Re-throws any execution error after logging.
+     * @throws {Error} Re-throws the last execution error after logging.
      * @memberof UnderpostKubectl
      */
     run(command, options = {}) {
-      const { context = '' } = options;
-      try {
-        logger.info(`Executing kubectl command`, { command: sanitizeCommand(command), context });
-        return shellExec(command, { stdout: true, disableLog: true });
-      } catch (error) {
-        logger.error(`kubectl command failed`, { command: sanitizeCommand(command), error: error.message, context });
-        throw error;
+      const {
+        context = '',
+        retries = 0,
+        baseDelayMs = RETRY_BASE_DELAY_MS,
+        timeoutSeconds = 0,
+        podName = '',
+        namespace = 'default',
+      } = options;
+      const attempts = Math.max(0, retries) + 1;
+      const boundedCommand = withTimeout(command, timeoutSeconds);
+
+      for (let attempt = 1; ; attempt++) {
+        try {
+          logger.info(`Executing kubectl command`, {
+            command: redactSensitiveText(command),
+            context,
+            ...(attempts > 1 ? { attempt, attempts } : {}),
+          });
+          return shellExec(boundedCommand, { stdout: true, disableLog: true });
+        } catch (error) {
+          if (attempt === attempts) {
+            logger.error(`kubectl command failed`, {
+              command: redactSensitiveText(command),
+              error: error.message,
+              context,
+              attempts,
+            });
+            throw error;
+          }
+          const delayMs = backoffDelayMs(attempt, baseDelayMs);
+          logger.warn(`kubectl command failed, retrying`, {
+            command: redactSensitiveText(command),
+            context,
+            attempt,
+            attempts,
+            delayMs,
+            error: error.message,
+          });
+          sleepSync(delayMs);
+          // An OOM kill takes the container with it (exit 137), so the replay would land on
+          // a container that is still coming back up. Wait for it rather than burning an attempt.
+          if (podName) Underpost.kubectl.ensureExecReady({ podName, namespace });
+        }
+      }
+    },
+
+    /**
+     * Polls until the pod can actually accept an exec stream.
+     *
+     * `kubectl wait --for=condition=Ready` reads the pod status the API server last
+     * recorded, which stays `Ready` for a window after the container underneath is
+     * gone — the exact gap that makes a dump fail with `container not found`. This
+     * probes the real thing: a no-op exec into the container.
+     *
+     * @param {object} params
+     * @param {string} params.podName - Pod to gate on.
+     * @param {string} [params.namespace='default'] - Pod namespace.
+     * @param {number} [params.attempts=5] - Probe attempts before giving up.
+     * @param {number} [params.baseDelayMs=2000] - Delay after the first probe; doubles per attempt.
+     * @param {number} [params.readyTimeoutSeconds=30] - Budget for the readiness condition wait.
+     * @param {number} [params.probeTimeoutSeconds=15] - Budget for the no-op exec probe.
+     * @returns {boolean} `true` when the pod accepted an exec stream.
+     * @memberof UnderpostKubectl
+     */
+    ensureExecReady({
+      podName,
+      namespace = 'default',
+      attempts = DEFAULT_EXEC_READY_ATTEMPTS,
+      baseDelayMs = RETRY_BASE_DELAY_MS,
+      readyTimeoutSeconds = 30,
+      probeTimeoutSeconds = 15,
+    }) {
+      const probeOptions = { silent: true, silentOnError: true, disableLog: true };
+      for (let attempt = 1; attempt <= attempts; attempt++) {
+        shellExec(
+          `kubectl wait --for=condition=Ready pod/${podName} -n ${namespace} --timeout=${readyTimeoutSeconds}s`,
+          probeOptions,
+        );
+        const probe = shellExec(
+          withTimeout(`sudo kubectl exec -n ${namespace} -i ${podName} -- sh -c "exit 0"`, probeTimeoutSeconds),
+          probeOptions,
+        );
+        if (probe && probe.code === 0) return true;
+        if (attempt < attempts) sleepSync(backoffDelayMs(attempt, baseDelayMs));
+      }
+      logger.error('Pod never became exec-ready', { podName, namespace, attempts });
+      return false;
+    },
+
+    /**
+     * Waits for a pod object to exist.
+     *
+     * `kubectl wait` fails on a missing object rather than waiting for one to appear, so any
+     * readiness wait issued while the controller is still creating the pod returns
+     * `Error from server (NotFound)`. A StatefulSet with `podManagementPolicy: OrderedReady`
+     * hits this every rollout: member N+1 is created only once N reports Ready.
+     *
+     * @param {object} params
+     * @param {string} params.podName - Pod to wait for.
+     * @param {string} [params.namespace='default'] - Pod namespace.
+     * @param {number} [params.timeoutSeconds=120] - Maximum wait window.
+     * @param {number} [params.intervalMs=2000] - Poll interval.
+     * @returns {Promise<boolean>} `true` once the pod object exists.
+     * @memberof UnderpostKubectl
+     */
+    async waitForPodCreation({
+      podName,
+      namespace = 'default',
+      timeoutSeconds = DEFAULT_CREATION_TIMEOUT_SECONDS,
+      intervalMs = 2000,
+    }) {
+      const deadline = Date.now() + timeoutSeconds * 1000;
+      while (Date.now() < deadline) {
+        const found = shellExec(`kubectl get pod ${podName} -n ${namespace} --ignore-not-found -o name`, {
+          stdout: true,
+          silent: true,
+          silentOnError: true,
+          disableLog: true,
+        });
+        if (`${found || ''}`.trim()) return true;
+        await timer(intervalMs);
       }
+      return false;
+    },
+
+    /**
+     * Waits for each named pod to be created and then to report Ready.
+     *
+     * Never throws: a pod that never arrives or never becomes Ready is returned to the caller,
+     * which owns the diagnosis — a stalled rollout usually needs surrounding context (volume
+     * bindings, events) that a bare non-zero exit code cannot carry.
+     *
+     * @param {object} params
+     * @param {string[]} params.podNames - Pods to wait for, in order.
+     * @param {string} [params.namespace='default'] - Pod namespace.
+     * @param {number} [params.creationTimeoutSeconds=120] - Per-pod budget for the pod to appear.
+     * @param {number} [params.readyTimeoutSeconds=180] - Per-pod budget for the Ready condition.
+     * @returns {Promise<string[]>} Names of the pods that did not become ready (empty = all good).
+     * @memberof UnderpostKubectl
+     */
+    async waitForPodsReady({
+      podNames,
+      namespace = 'default',
+      creationTimeoutSeconds = DEFAULT_CREATION_TIMEOUT_SECONDS,
+      readyTimeoutSeconds = DEFAULT_READY_TIMEOUT_SECONDS,
+    }) {
+      const failed = [];
+      for (const podName of podNames) {
+        if (
+          !(await Underpost.kubectl.waitForPodCreation({ podName, namespace, timeoutSeconds: creationTimeoutSeconds }))
+        ) {
+          logger.error('Pod was never created', { podName, namespace, timeoutSeconds: creationTimeoutSeconds });
+          failed.push(podName);
+          continue;
+        }
+
+        const result = shellExec(
+          `kubectl wait --for=condition=Ready pod/${podName} -n ${namespace} --timeout=${readyTimeoutSeconds}s`,
+          { silentOnError: true },
+        );
+        if (result.code !== 0) {
+          logger.error('Pod did not become ready', { podName, namespace, timeoutSeconds: readyTimeoutSeconds });
+          failed.push(podName);
+        }
+      }
+      return failed;
     },
 
     /**
@@ -104,22 +280,21 @@ class UnderpostKubectl {
      * @param {string} params.podName - Target pod name.
      * @param {string} params.namespace - Pod namespace.
      * @param {string} params.command - Shell command to run inside the container.
+     * @param {number} [params.retries=2] - Extra attempts allowed after a failure, with exponential backoff.
+     * @param {number} [params.timeoutSeconds=0] - Wall-clock budget per attempt; `0` disables the bound.
      * @returns {string} stdout output from the in-pod command.
      * @throws {Error} Re-throws any execution error after logging.
      * @memberof UnderpostKubectl
      */
-    exec({ podName, namespace, command }) {
-      try {
-        const kubectlCmd = `sudo kubectl exec -n ${namespace} -i ${podName} -- sh -c "${command}"`;
-        return Underpost.kubectl.run(kubectlCmd, { context: `exec in pod ${podName}` });
-      } catch (error) {
-        logger.error('Failed to execute command in pod', {
-          podName,
-          command: sanitizeCommand(command),
-          error: error.message,
-        });
-        throw error;
-      }
+    exec({ podName, namespace, command, retries = DEFAULT_POD_RETRIES, timeoutSeconds = 0 }) {
+      const kubectlCmd = `sudo kubectl exec -n ${namespace} -i ${podName} -- sh -c "${command}"`;
+      return Underpost.kubectl.run(kubectlCmd, {
+        context: `exec in pod ${podName}`,
+        retries,
+        timeoutSeconds,
+        podName,
+        namespace,
+      });
     },
 
     /**
@@ -129,16 +304,16 @@ class UnderpostKubectl {
      * @param {string} params.podName - Target pod name.
      * @param {string} params.namespace - Pod namespace.
      * @param {string} params.destPath - Destination path inside the container.
+     * @param {number} [params.retries=2] - Extra attempts allowed after a failure, with exponential backoff.
      * @returns {boolean} `true` on success, `false` on error.
      * @memberof UnderpostKubectl
      */
-    cpTo({ sourcePath, podName, namespace, destPath }) {
+    cpTo({ sourcePath, podName, namespace, destPath, retries = DEFAULT_POD_RETRIES }) {
       try {
         const command = `sudo kubectl cp ${sourcePath} ${namespace}/${podName}:${destPath}`;
-        Underpost.kubectl.run(command, { context: `copy to pod ${podName}` });
+        Underpost.kubectl.run(command, { context: `copy to pod ${podName}`, retries, podName, namespace });
         return true;
-      } catch (error) {
-        logger.error('Failed to copy file to pod', { sourcePath, podName, destPath, error: error.message });
+      } catch {
         return false;
       }
     },
@@ -150,16 +325,16 @@ class UnderpostKubectl {
      * @param {string} params.namespace - Pod namespace.
      * @param {string} params.sourcePath - Source path inside the container.
      * @param {string} params.destPath - Local destination path.
+     * @param {number} [params.retries=2] - Extra attempts allowed after a failure, with exponential backoff.
      * @returns {boolean} `true` on success, `false` on error.
      * @memberof UnderpostKubectl
      */
-    cpFrom({ podName, namespace, sourcePath, destPath }) {
+    cpFrom({ podName, namespace, sourcePath, destPath, retries = DEFAULT_POD_RETRIES }) {
       try {
         const command = `sudo kubectl cp ${namespace}/${podName}:${sourcePath} ${destPath}`;
-        Underpost.kubectl.run(command, { context: `copy from pod ${podName}` });
+        Underpost.kubectl.run(command, { context: `copy from pod ${podName}`, retries, podName, namespace });
         return true;
-      } catch (error) {
-        logger.error('Failed to copy file from pod', { podName, sourcePath, destPath, error: error.message });
+      } catch {
         return false;
       }
     },
@@ -212,3 +387,5 @@ class UnderpostKubectl {
 }
 
 export default UnderpostKubectl;
+
+export { backoffDelayMs, withTimeout };

package/src/cli/lxd.js

@@ -5,11 +5,11 @@
  *
  */
 
-import { getNpmRootPath } from '../server/conf.js';
-import { pbcopy, shellExec } from '../server/process.js';
+import { getNpmRootPath } from '../server/runtime/environment.js';
+import { pbcopy, shellExec } from '../server/runtime/process.js';
 import walk from 'ignore-walk';
 import fs from 'fs-extra';
-import { loggerFactory } from '../server/logger.js';
+import { loggerFactory } from '../server/ops/logger.js';
 import Underpost from '../index.js';
 
 const logger = loggerFactory(import.meta);