cyberia 3.2.9 → 3.2.22

package/src/cli/monitor.js

@@ -4,10 +4,25 @@
  * @namespace UnderpostMonitor
  */
 
-import { loadReplicas, pathPortAssignmentFactory, loadConfServerJson, loadCronDeployEnv } from '../server/conf.js';
+import {
+  loadReplicas,
+  pathPortAssignmentFactory,
+  loadConfServerJson,
+  loadCronDeployEnv,
+  etcHostFactory,
+  deployRangePortFactory,
+} from '../server/conf.js';
 import { loggerFactory } from '../server/logger.js';
+import { timer } from '../client/components/core/CommonJs.js';
+import {
+  RUNTIME_STATUS,
+  INTERNAL_STATUS_PATH,
+  normalizeContainerStatus,
+  deployStatusPort,
+} from '../server/runtime-status.js';
 import axios from 'axios';
 import fs from 'fs-extra';
+import net from 'node:net';
 import { shellExec } from '../server/process.js';
 import Underpost from '../index.js';
 
@@ -87,13 +102,13 @@ class UnderpostMonitor {
       }
 
       if (options.readyDeployment) {
-        for (const version of options.versions.split(',')) {
-          (async () => {
-            await Underpost.deploy.monitorReadyRunner(deployId, env, version, [], options.namespace, 'underpost');
+        await Promise.all(
+          options.versions.split(',').map(async (version) => {
+            await Underpost.monitor.monitorReadyRunner(deployId, env, version, [], options.namespace);
             if (options.promote)
               Underpost.deploy.switchTraffic(deployId, env, version, options.replicas, options.namespace, options);
-          })();
-        }
+          }),
+        );
         return;
       }
 
@@ -144,7 +159,7 @@ class UnderpostMonitor {
             if (path.match('peer') || path.match('socket')) continue;
             const urlTest = `http${env === 'development' ? '' : 's'}://${host}${path}`;
             if (env === 'development') {
-              const { renderHosts } = Underpost.deploy.etcHostFactory([host]);
+              const { renderHosts } = etcHostFactory([host]);
               logger.info('renderHosts', renderHosts);
             }
             await axios.get(urlTest, { timeout: 10000 }).catch((error) => {
@@ -200,7 +215,7 @@ class UnderpostMonitor {
       let monitorPodName;
       const monitorCallBack = (resolve, reject) => {
         if (env === 'development') {
-          const { renderHosts } = Underpost.deploy.etcHostFactory([]);
+          const { renderHosts } = etcHostFactory([]);
           logger.info('renderHosts', renderHosts);
         }
         const envMsTimeout = Underpost.env.get(`${deployId}-${env}-monitor-ms`);
@@ -221,7 +236,7 @@ class UnderpostMonitor {
                     monitorPodName = undefined;
                   }
                   const checkDeploymentReadyStatus = async () => {
-                    const { ready, notReadyPods, readyPods } = await Underpost.deploy.checkDeploymentReadyStatus(
+                    const { ready, notReadyPods, readyPods } = await Underpost.monitor.checkDeploymentReadyStatus(
                       deployId,
                       env,
                       traffic,
@@ -266,6 +281,378 @@ class UnderpostMonitor {
       };
       return new Promise((...args) => monitorCallBack(...args));
     },
+    /**
+     * Checks the status of a deployment.
+     * @param {string} deployId - Deployment ID for which the status is being checked.
+     * @param {string} env - Environment for which the status is being checked.
+     * @param {string} traffic - Current traffic status for the deployment.
+     * @param {Array<string>} ignoresNames - List of pod names to ignore.
+     * @param {string} [namespace='default'] - Kubernetes namespace for the deployment.
+     * @returns {object} - Object containing the status of the deployment.
+     * @memberof UnderpostMonitor
+     */
+    async checkDeploymentReadyStatus(deployId, env, traffic, ignoresNames = [], namespace = 'default') {
+      const pods = Underpost.kubectl.get(`${deployId}-${env}-${traffic}`, 'pods', namespace);
+      const readyPods = [];
+      const notReadyPods = [];
+
+      // Readiness signal: the pod's Kubernetes `Ready` condition driven by the
+      // container's readinessProbe (TCP socket, HTTP get, or exec). Set by kubelet
+      // when the probe passes. A failed or crashing runtime never becomes Ready —
+      // kubelet surfaces CrashLoopBackOff and this gate stays closed.
+      for (const pod of pods) {
+        const { NAME } = pod;
+        if (ignoresNames && ignoresNames.find((t) => NAME.trim().toLowerCase().match(t.trim().toLowerCase()))) continue;
+
+        let podJson = null;
+        try {
+          // Pod may not exist yet (between deployment apply and pod
+          // scheduling). silentOnError lets the monitor loop continue
+          // instead of aborting on the transient NotFound exit.
+          const raw = shellExec(`sudo kubectl get pod ${NAME} -n ${namespace} -o json`, {
+            silent: true,
+            disableLog: true,
+            stdout: true,
+            silentOnError: true,
+          });
+          podJson = raw ? JSON.parse(raw) : null;
+        } catch (_) {
+          podJson = null;
+        }
+        const conditions = podJson?.status?.conditions || [];
+        const readyCondition = conditions.find((c) => c.type === 'Ready');
+        const k8sReady = readyCondition?.status === 'True';
+
+        pod.out = JSON.stringify({ k8sReady, condition: readyCondition ?? null });
+
+        if (k8sReady) readyPods.push(pod);
+        else notReadyPods.push(pod);
+      }
+      const consideredCount = readyPods.length + notReadyPods.length;
+      return {
+        ready: consideredCount > 0 && notReadyPods.length === 0,
+        notReadyPods,
+        readyPods,
+      };
+    },
+    /**
+     * Resolves a free ephemeral TCP port on the loopback interface, used as the
+     * local end of the `kubectl port-forward` tunnel so it never collides with
+     * host-local services.
+     * @returns {Promise<number>}
+     * @memberof UnderpostMonitor
+     */
+    findFreePort() {
+      return new Promise((resolve) => {
+        const srv = net.createServer();
+        srv.once('error', () => resolve(20000 + Math.floor(Math.random() * 20000)));
+        srv.listen(0, '127.0.0.1', () => {
+          const { port } = srv.address();
+          srv.close(() => resolve(port));
+        });
+      });
+    },
+    /**
+     * Resolves the deployment's internal status port (Phase-2 transport target).
+     *
+     * Canonical value is `fromPort - 1` from the deployment router — the exact
+     * port `buildManifest` injects into the pod (UNDERPOST_INTERNAL_PORT) and
+     * uses for the probes — so the tunnel target always matches the in-pod bind.
+     * `UNDERPOST_INTERNAL_PORT` overrides; ambient resolution is the last resort.
+     *
+     * @param {string} deployId
+     * @param {string} env
+     * @returns {Promise<number>}
+     * @memberof UnderpostMonitor
+     */
+    async deployInternalPort(deployId, env) {
+      const override = parseInt(process.env.UNDERPOST_INTERNAL_PORT);
+      if (!Number.isNaN(override)) return override;
+      try {
+        const router = await Underpost.deploy.routerFactory(deployId, env);
+        const { fromPort } = deployRangePortFactory(router);
+        if (Number.isFinite(fromPort) && fromPort > 0) return fromPort - 1;
+      } catch (_) {
+        /* fall through to ambient resolution */
+      }
+      return deployStatusPort(deployId, env) ?? 3000;
+    },
+    /**
+     * Reads Phase-2 runtime status from a single pod using the selected transport.
+     *
+     *   - `exec` (default): `kubectl exec … underpost config get container-status`
+     *     reads the env-file value. Synchronous, no background process — required
+     *     for custom instances (cyberia-server/client) and the safe choice for
+     *     CI/SSH. See `Deploy custom instance to K8S.md`.
+     *   - `http`: port-forward to the in-pod `/_internal/status` endpoint served
+     *     by the `underpost start` launcher (dd-* runtime deploys). Opt-in.
+     *
+     * Transport failures are reported as `{ ok: false }` and must never be read
+     * as success — they are retried, not promoted.
+     *
+     * @param {string} podName
+     * @param {string} namespace
+     * @param {number} internalPort
+     * @param {('http'|'exec')} [transport='exec']
+     * @returns {Promise<{ok: boolean, status?: (string|null), transportError?: string}>}
+     * @memberof UnderpostMonitor
+     */
+    async readRuntimeStatus(podName, namespace, internalPort, transport = 'exec') {
+      return transport === 'exec'
+        ? Underpost.monitor.readRuntimeStatusViaExec(podName, namespace)
+        : Underpost.monitor.readRuntimeStatusViaHttp(podName, namespace, internalPort);
+    },
+    /**
+     * Phase-2 read over `kubectl exec` (env-file transport). Works for any pod
+     * whose image bakes the underpost CLI — notably custom instances that stamp
+     * `container-status` from `lifecycle.postStart`/`preStop` hooks.
+     * @param {string} podName
+     * @param {string} namespace
+     * @returns {{ok: boolean, status?: (string|null), transportError?: string}}
+     * @memberof UnderpostMonitor
+     */
+    readRuntimeStatusViaExec(podName, namespace) {
+      try {
+        const raw = shellExec(
+          `sudo kubectl exec ${podName} -n ${namespace} -- sh -c 'underpost config get container-status --plain'`,
+          { silent: true, disableLog: true, stdout: true, silentOnError: true },
+        );
+        const status = normalizeContainerStatus(raw ? raw.toString().trim() : '');
+        return status === undefined ? { ok: false, transportError: 'empty_status' } : { ok: true, status };
+      } catch (error) {
+        return { ok: false, transportError: error?.code || error?.message || 'exec_failed' };
+      }
+    },
+    /**
+     * Phase-2 read over `kubectl port-forward` + HTTP `/_internal/status`.
+     *
+     * The local side of the tunnel MUST be an ephemeral free port: pinning it to
+     * internalPort collides with any host-local service on that number (e.g. a
+     * dev runtime on the same machine as the cluster), making port-forward fail
+     * to bind and every read return a false transport error.
+     *
+     * @param {string} podName
+     * @param {string} namespace
+     * @param {number} internalPort
+     * @returns {Promise<{ok: boolean, status?: (string|null), transportError?: string}>}
+     * @memberof UnderpostMonitor
+     */
+    async readRuntimeStatusViaHttp(podName, namespace, internalPort) {
+      const override = parseInt(process.env.UNDERPOST_PF_LOCAL_PORT);
+      const localPort = Number.isNaN(override) ? await Underpost.monitor.findFreePort() : override;
+      const url = `http://127.0.0.1:${localPort}${INTERNAL_STATUS_PATH}`;
+      let portForward;
+      try {
+        // `exec` makes the tracked child the sudo/kubectl process (so kill
+        // reaches it); stdio is redirected to /dev/null so the tunnel never
+        // inherits — and therefore never holds open — a CI/SSH session's pipes,
+        // which would hang the job after a successful deploy.
+        portForward = shellExec(
+          `exec sudo kubectl port-forward pod/${podName} ${localPort}:${internalPort} -n ${namespace} </dev/null >/dev/null 2>&1`,
+          { async: true, silent: true, disableLog: true, silentOnError: true },
+        );
+      } catch (_) {
+        portForward = undefined;
+      }
+      try {
+        let lastError;
+        const attempts = parseInt(process.env.UNDERPOST_PF_ATTEMPTS) || 20;
+        for (let attempt = 0; attempt < attempts; attempt++) {
+          try {
+            const res = await axios.get(url, { timeout: 2500 });
+            const raw = res?.data?.status ?? null;
+            return { ok: true, status: normalizeContainerStatus(raw) ?? raw, payload: res.data };
+          } catch (error) {
+            lastError = error;
+            await timer(350);
+          }
+        }
+        return { ok: false, transportError: lastError?.code || lastError?.message || 'transport_failed' };
+      } finally {
+        if (portForward && typeof portForward.kill === 'function') {
+          try {
+            portForward.kill('SIGTERM');
+          } catch (_) {
+            /* tunnel already gone */
+          }
+        }
+      }
+    },
+    /**
+     * Monitors a deployment to terminal readiness using a deterministic
+     * two-phase state machine.
+     *
+     *   Phase 1 (Kubernetes): pod `Ready` condition via `checkDeploymentReadyStatus`.
+     *   Phase 2 (Runtime):    `container-status`, read via the selected transport.
+     *
+     * Two deployment shapes are supported via `options`:
+     *   - `runtime` gate (default, dd-* deploys): the `underpost start` launcher
+     *     stamps `running-deployment`. Success requires K8S Ready AND every pod
+     *     reporting `running-deployment`.
+     *   - `kubernetes` gate (custom instances, e.g. cyberia): the runtime is a
+     *     bare binary; K8S `readinessProbe` (TCP) IS the running signal and
+     *     `container-status` is stamped to `initializing`/`stopping` by lifecycle
+     *     hooks. Success requires K8S Ready; the status read is used only for
+     *     fast `error` detection and display.
+     *
+     * Phase-2 transport defaults to `exec` (`kubectl exec`, no background
+     * process). The `http` transport (`kubectl port-forward` → `/_internal/status`)
+     * is opt-in via `options.statusTransport='http'` or
+     * `UNDERPOST_STATUS_TRANSPORT=http`; it must not be used in CI/SSH sessions
+     * where a stray tunnel can hang the job.
+     *
+     * Contract (both shapes):
+     *   - Runtime readiness is never declared before Kubernetes readiness.
+     *   - An explicit runtime `error` (or a fatal pod status) transitions
+     *     immediately to `failed` (throw → CD exit 1).
+     *   - Transport failures never count as success and never advance state.
+     *   - `timeout` is a distinct terminal state from `failed`.
+     *   - Every transition emits a structured, secret-free event.
+     *
+     * @param {string} deployId - Deployment ID for which the ready status is being monitored.
+     * @param {string} env - Environment for which the ready status is being monitored.
+     * @param {string} targetTraffic - Target traffic status for the deployment.
+     * @param {Array<string>} ignorePods - List of pod names to ignore.
+     * @param {string} [namespace='default'] - Kubernetes namespace for the deployment.
+     * @param {object} [options] - Monitoring shape.
+     * @param {('runtime'|'kubernetes')} [options.readyGate='runtime'] - Running-signal owner.
+     * @param {('http'|'exec')} [options.statusTransport='http'] - Phase-2 read transport.
+     * @returns {object} - Object containing the ready status of the deployment.
+     * @memberof UnderpostMonitor
+     */
+    async monitorReadyRunner(deployId, env, targetTraffic, ignorePods = [], namespace = 'default', options = {}) {
+      const delayMs = parseInt(process.env.UNDERPOST_MONITOR_DELAY_MS) || 1000;
+      const maxIterations = parseInt(process.env.UNDERPOST_MONITOR_MAX_ITERATIONS) || 3000;
+      const deploymentId = `${deployId}-${env}-${targetTraffic}`;
+      const tag = `[${deploymentId}]`;
+      const expectedStatus = RUNTIME_STATUS.RUNNING;
+      const readyGate = options.readyGate === 'kubernetes' ? 'kubernetes' : 'runtime';
+      // Default to `exec`: a single synchronous `kubectl exec` read leaves no
+      // background process behind. The `http` transport spawns `kubectl
+      // port-forward` children that, if orphaned, inherit a CI/SSH session's
+      // stdio and hang the job after a successful deploy — opt in explicitly.
+      const statusTransport =
+        (options.statusTransport || process.env.UNDERPOST_STATUS_TRANSPORT) === 'http' ? 'http' : 'exec';
+      const internalPort =
+        statusTransport === 'http' ? await Underpost.monitor.deployInternalPort(deployId, env) : null;
+      const podErrorStates = ['error', 'crashloopbackoff', 'oomkilled', 'imagepullbackoff', 'errimagepull'];
+
+      const emit = (state, status) =>
+        logger.info('deploy-monitor', {
+          deployId: deploymentId,
+          phase: state.startsWith('runtime') ? 'runtime' : 'kubernetes',
+          state,
+          status: status ?? null,
+          timestamp: new Date().toISOString(),
+        });
+
+      logger.info('Deployment init', {
+        deployId,
+        env,
+        targetTraffic,
+        namespace,
+        internalPort,
+        readyGate,
+        statusTransport,
+      });
+      emit('pending');
+
+      const runtimeStatusCache = new Map();
+      const advancedPods = new Set();
+
+      for (let i = 0; i < maxIterations; i++) {
+        const result = await Underpost.monitor.checkDeploymentReadyStatus(
+          deployId,
+          env,
+          targetTraffic,
+          ignorePods,
+          namespace,
+        );
+        const allPods = [...result.readyPods, ...result.notReadyPods];
+
+        if (allPods.length === 0) {
+          emit('pending');
+          await timer(delayMs);
+          continue;
+        }
+        emit('pod_scheduled');
+
+        // Phase 1 fatal: a Kubernetes-level pod failure is terminal (failed,
+        // not timeout) — fail the CD runner immediately instead of waiting out
+        // the full window.
+        for (const pod of allPods) {
+          const podStatus = (pod.STATUS || '').toLowerCase().trim();
+          if (podErrorStates.find((s) => podStatus.includes(s)))
+            throw new Error(`Pod ${pod.NAME} has error pod status: ${pod.STATUS}`);
+        }
+
+        const allPodsK8sReady = result.notReadyPods.length === 0;
+        if (allPodsK8sReady) emit('pod_ready');
+
+        // Phase 2: runtime status via the selected transport. Transport failures
+        // neither advance state nor count as success; explicit `error` is terminal.
+        let allRuntimeRead = true;
+        for (const pod of allPods) {
+          if (!pod?.NAME) continue;
+          const read = await Underpost.monitor.readRuntimeStatus(pod.NAME, namespace, internalPort, statusTransport);
+          if (!read.ok) {
+            allRuntimeRead = false;
+            emit('runtime_booting', `transport:${read.transportError}`);
+            continue;
+          }
+          const status = read.status;
+          if (status === RUNTIME_STATUS.ERROR) throw new Error(`Pod ${pod.NAME} reported runtime status=error`);
+          // Regression (advanced → empty/build) means a pod restarted. Under the
+          // kubernetes gate the runtime never advances past `initializing`, so
+          // only treat a drop to empty/build as a regression there.
+          if (advancedPods.has(pod.NAME) && (!status || status === RUNTIME_STATUS.BUILD))
+            throw new Error(`Pod ${pod.NAME} runtime status regressed (${status ?? 'empty'}) — pod likely restarted`);
+          if (status && status !== RUNTIME_STATUS.BUILD) advancedPods.add(pod.NAME);
+          runtimeStatusCache.set(pod.NAME, status);
+          emit('runtime_booting', status);
+        }
+
+        // Under the kubernetes gate the readinessProbe is the running signal, so
+        // K8S Ready alone confirms Phase 2; the status read above is kept only
+        // for `error` fast-fail and display.
+        const allRuntimeReady =
+          readyGate === 'kubernetes'
+            ? true
+            : allRuntimeRead && allPods.every((pod) => runtimeStatusCache.get(pod.NAME) === expectedStatus);
+
+        for (const pod of allPods) {
+          const status = runtimeStatusCache.get(pod.NAME) || 'waiting for status';
+          const podStatus = pod.STATUS || 'Unknown';
+          const statusDisplay = status === expectedStatus ? status : `${status} (pending)`;
+          console.log(
+            'Target pod:',
+            pod.NAME[pod.NAME.includes('green') ? 'bgGreen' : 'bgBlue'].bold.black,
+            '| Pod status:',
+            podStatus.bold.yellow,
+            '| Runtime status:',
+            statusDisplay.bold.cyan,
+          );
+        }
+
+        // Terminal success requires both phases. runtime_ready cannot precede
+        // Kubernetes readiness.
+        if (allPodsK8sReady && allRuntimeReady) {
+          const readySignal = readyGate === 'kubernetes' ? 'K8S readinessProbe' : `runtime ${expectedStatus}`;
+          emit('runtime_ready', readyGate === 'kubernetes' ? 'k8s-ready' : expectedStatus);
+          logger.info(`${tag} | Deployment ready (K8S Ready + ${readySignal})`);
+          return result;
+        }
+
+        await timer(delayMs);
+        if ((i + 1) % 10 === 0) logger.info(`${tag} | In progress... iteration ${i + 1}`);
+      }
+
+      emit('timeout');
+      logger.error(`${tag} | Deployment timeout after ${maxIterations} iterations`);
+      throw new Error(
+        `monitorReadyRunner timeout: ${deploymentId} did not become Ready within ${maxIterations}*${delayMs}ms`,
+      );
+    },
   };
 }