@leverege/build-tools 2.114.0-DEVOP-578.2 → 2.114.0

package/src/k8s/commands/cryo.mjs

@@ -1,49 +1,138 @@
+// Hibernate and wake phases are inherently sequential — each kubectl/gcloud call
+// must complete before the next begins to maintain dependency order. Every
+// await-in-loop in this file is intentional.
+/* eslint-disable no-await-in-loop */
+
+import fs from 'node:fs/promises'
+
 import { Command } from 'commander'
 import chalk from 'chalk'
 import { $ } from 'zx'
 
-import { log, errorExit, sleep, warning, proceed, parseJsonFile } from '../../Utils.mjs'
+import { log, errorExit, sleep, warning, proceed, parseYamlFile } from '../../Utils.mjs'
 import { overwhelmGate } from '../lib/gate.mjs'
+import { listDeployments, namespaceExists, listStatefulSets, getReplicaCount } from '../lib/kubectl.mjs'
 
 $.verbose = false
 
-// ── Config ────────────────────────────────────────────────────────────────
+// ── Known service categories (from helmup.sh, in hibernate order) ─────────
+// Within each phase, services are listed in the order they should be stopped
+// (reverse of bootstrap/deploy order within that phase).
 
-const loadConfig = async ( configPath ) => {
-  const config = await parseJsonFile( configPath )
-  if ( !config ) errorExit( `Config file not found: ${configPath}` )
+const PLATFORM_TRANSPONDERS = [ 'transponder-tsdb', 'transponder-rt', 'transponder-bq', 'transponder-dh' ]
+const PLATFORM_SUPPORT      = [ 'rule-engine', 'resource-server', 'messenger', 'llm-prompt-server', 'imagine', 'emailer', 'db-curator' ]
+const PLATFORM_CORE         = [ 'scheduler', 'message-processor', 'api-server', 'authz-server' ]
+const AUXILIARY_KNOWN       = [ 'vms-server', 'vin-decoder-server', 'reason', 'pusher', 'push-notifier', 'pubsub-pulse', 'pgbouncer', 'postgres-pgbouncer', 'overdose', 'geotile-server', 'fota-server', 'argocd' ]
+const SYSTEM_DEPLOY         = [ 'reloader-reloader', 'reflector' ]
+
+const ALL_KNOWN = new Set( [
+  ...PLATFORM_TRANSPONDERS, ...PLATFORM_SUPPORT, ...PLATFORM_CORE,
+  ...AUXILIARY_KNOWN, ...SYSTEM_DEPLOY,
+] )
 
-  const { cluster, deploymentPhases, nodePools } = config
-  if ( !cluster?.project || !cluster?.name || !cluster?.region ) {
-    errorExit( 'Config must include cluster.project, cluster.name, and cluster.region' )
+// ── Config ─────────────────────────────────────────────────────────────────
+
+const loadConfig = async ( configPath ) => {
+  let config
+  try {
+    config = await parseYamlFile( configPath )
+  } catch {
+    errorExit( `Config file not found or invalid YAML: ${configPath}` )
   }
-  if ( !deploymentPhases || !nodePools ) {
+  if ( !config?.deploymentPhases || !config?.nodePools ) {
     errorExit( 'Config must include deploymentPhases and nodePools' )
   }
   return config
 }
 
-// ── Display ───────────────────────────────────────────────────────────────
+const overwhelmKey = async key => ( await $`overwhelm -k ${key}` ).stdout.trim()
+
+const getClusterEnv = async () => ( {
+  project : await overwhelmKey( 'PROJECT_ID' ),
+  name    : await overwhelmKey( 'CLUSTER_NAME' ),
+  region  : await overwhelmKey( 'GCE_REGION' ),
+} )
+
+// ── Display ────────────────────────────────────────────────────────────────
 
 const banner = ( text ) => {
   const bar = '─'.repeat( Math.max( 0, 56 - text.length ) )
   log( chalk.cyan.bold( `\n── ${text} ${bar}` ) )
 }
 
-// ── kubectl ───────────────────────────────────────────────────────────────
+// ── kubectl ────────────────────────────────────────────────────────────────
+
+const scaleDeployments = async ( names, replicas, namespace ) => {
+  return $`kubectl scale deployment ${names} --replicas=${replicas} -n ${namespace}`
+}
+
+const scaleStatefulSets = async ( names, replicas, namespace ) => {
+  return $`kubectl scale statefulset ${names} --replicas=${replicas} -n ${namespace}`
+}
 
-const scaleDeployments = async ( names, replicas, namespace ) =>
-  $`kubectl scale deployment ${names} --replicas=${replicas} -n ${namespace}`
+const CNPG_CRD = 'clusters.postgresql.cnpg.io'
 
-const scaleStatefulSets = async ( names, replicas, namespace ) =>
-  $`kubectl scale statefulset ${names} --replicas=${replicas} -n ${namespace}`
+const cnpgHibernate = async ( name, namespace ) => {
+  log( `  cnpg hibernate on: ${name} (${namespace})` )
+  await $`kubectl cnpg hibernate on -n ${namespace} ${name}`
+}
 
-const waitForPodsGone = async ( namespace, timeoutSecs = 300 ) => {
-  log( `\nWaiting for pods to terminate (up to ${timeoutSecs}s)...` )
+const cnpgWake = async ( name, namespace, timeoutSecs = 300 ) => {
+  log( `  cnpg hibernate off: ${name} (${namespace})` )
   try {
-    await $`kubectl wait pod --all -n ${namespace} --for=delete --timeout=${timeoutSecs}s`
+    await $`kubectl cnpg hibernate off -n ${namespace} ${name}`
+  } catch ( err ) {
+    if ( err.stderr?.includes( 'not found' ) ) {
+      log( `  ${name} — cluster not found, skipping` )
+      return
+    }
+    throw err
+  }
+  log( `  waiting for ${name} to be Ready (up to ${timeoutSecs}s)...` )
+  try {
+    await $`kubectl wait ${CNPG_CRD} ${name} -n ${namespace} --for=condition=Ready --timeout=${timeoutSecs}s`
   } catch {
-    warning( `Some pods still terminating after ${timeoutSecs}s — proceeding.` )
+    warning( `${name} not Ready after ${timeoutSecs}s — proceeding, but poolers may fail to connect.` )
+  }
+}
+
+const anyCnpgClusters = async ( namespace, retries = 4, delayMs = 10_000 ) => {
+  for ( let i = 0; i <= retries; i++ ) {
+    try {
+      const result = await $`kubectl get ${CNPG_CRD} -n ${namespace} --no-headers -o name`
+      if ( result.stdout.trim().length > 0 ) return true
+    } catch {
+      // webhook may not be registered yet — retry
+    }
+    if ( i < retries ) {
+      log( `  CNPG webhook not ready, retrying in ${delayMs / 1000}s… (${i + 1}/${retries})` )
+      await sleep( delayMs )
+    }
+  }
+  return false
+}
+
+const filterExisting = async ( kind, names, namespace ) => {
+  try {
+    const result = await $`kubectl get ${kind} -n ${namespace} --no-headers -o name`
+    const existing = new Set(
+      result.stdout.trim().split( '\n' ).filter( Boolean )
+        .map( n => n.replace( /^[^/]+\//, '' ) )
+    )
+    return names.filter( n => existing.has( n ) )
+  } catch {
+    return []
+  }
+}
+
+const waitForPodsGone = async ( namespaces, timeoutSecs = 300 ) => {
+  log( `\nWaiting for pods to terminate in [${namespaces.join( ', ' )}] (up to ${timeoutSecs}s)...` )
+  for ( const ns of namespaces ) {
+    try {
+      await $`kubectl wait pod --all -n ${ns} --for=delete --timeout=${timeoutSecs}s`
+    } catch {
+      warning( `Some pods still terminating in ${ns} after ${timeoutSecs}s — proceeding.` )
+    }
   }
 }
 
@@ -52,26 +141,23 @@ const waitForNodesReady = async ( timeoutSecs = 600 ) => {
   await $`kubectl wait node --all --for=condition=Ready --timeout=${timeoutSecs}s`
 }
 
-// ── gcloud ────────────────────────────────────────────────────────────────
+// ── gcloud ─────────────────────────────────────────────────────────────────
 
 const updateAutoscalerMin = async ( pool, cluster, min ) => {
   log( `  ${pool.name}: autoscaler min → ${min}` )
-  const { name, project, region } = cluster
-  await $`gcloud container node-pools update ${pool.name} --cluster=${name} --project=${project} --region=${region} --enable-autoscaling --min-nodes=${min} --max-nodes=${pool.autoscaler.max} --quiet`
+  await $`gcloud container node-pools update ${pool.name} --cluster=${cluster.name} --project=${cluster.project} --region=${cluster.region} --enable-autoscaling --min-nodes=${min} --max-nodes=${pool.autoscaler.max} --quiet`
 }
 
 const resizePool = async ( pool, cluster, size ) => {
   log( `  ${pool.name} → ${size} node(s)` )
-  const { name, project, region } = cluster
-  await $`gcloud container clusters resize ${name} --node-pool=${pool.name} --num-nodes=${size} --project=${project} --region=${region} --quiet`
+  await $`gcloud container clusters resize ${cluster.name} --node-pool=${pool.name} --num-nodes=${size} --project=${cluster.project} --region=${cluster.region} --quiet`
 }
 
-// ── Hibernate ─────────────────────────────────────────────────────────────
+// ── Hibernate ──────────────────────────────────────────────────────────────
 
 const runHibernate = async ( configPath ) => {
   const config = await loadConfig( configPath )
   const {
-    cluster,
     namespace = 'default',
     phasePauseMs = 10_000,
     deploymentPhases,
@@ -80,27 +166,104 @@ const runHibernate = async ( configPath ) => {
   } = config
 
   await overwhelmGate()
+  const cluster = await getClusterEnv()
+
   await proceed( `Hibernate ${chalk.yellow.bold( cluster.name )}? All workloads scale to 0 and node pools drain.` )
 
+  // deferShutdown phases (e.g. CNPG operator) are held back until after
+  // statefulset phases so their webhooks remain available for cnpg hibernate on.
+  const deferredDeployPhases = []
+
   for ( const group of deploymentPhases ) {
+    if ( group.deferShutdown ) {
+      deferredDeployPhases.push( group )
+      continue
+    }
+    const phaseNamespace = group.namespace ?? namespace
+    const members = group.optional ?
+      await filterExisting( 'deployment', group.members, phaseNamespace ) :
+      group.members
+    if ( !members.length ) {
+      log( `  ${group.name} — not present, skipping` )
+      continue
+    }
     banner( `Deployments: ${group.name}` )
-    await scaleDeployments( group.members, 0, namespace )
+    await scaleDeployments( members, 0, phaseNamespace )
     await sleep( group.pauseMs ?? phasePauseMs )
   }
 
+  // Before statefulset phases, ensure any deferShutdown phases are actually
+  // running — a prior failed run may have left them at 0, which would cause
+  // CNPG webhook calls to fail. Wait for rollout (not just a fixed sleep)
+  // so the webhook is registered before cnpg hibernate on is called.
+  if ( deferredDeployPhases.length && statefulsetPhases.some( p => p.type === 'cnpg' ) ) {
+    banner( 'Ensuring deferred services are running' )
+    for ( const group of deferredDeployPhases ) {
+      const phaseNamespace = group.namespace ?? namespace
+      log( `  ${group.name} → 1` )
+      await scaleDeployments( group.members, 1, phaseNamespace )
+      if ( group.waitForReady ) {
+        log( `  waiting for ${group.name} to be ready...` )
+        for ( const member of group.members ) {
+          try {
+            await $`kubectl rollout status deployment ${member} -n ${phaseNamespace} --timeout=120s`
+          } catch {
+            warning( `${member} not ready after 120s — proceeding` )
+          }
+        }
+      }
+    }
+  }
+
   if ( statefulsetPhases.length ) {
     for ( const group of statefulsetPhases ) {
+      const phaseNamespace = group.namespace ?? namespace
+      const kind = group.type === 'cnpg' ? CNPG_CRD : 'statefulset'
+      const members = group.optional ?
+        await filterExisting( kind, group.members, phaseNamespace ) :
+        group.members
+      if ( !members.length ) {
+        log( `  ${group.name} — not present, skipping` )
+        continue
+      }
       banner( `StatefulSets: ${group.name}` )
-      await scaleStatefulSets( group.members, 0, namespace )
+      if ( group.type === 'cnpg' ) {
+        for ( const member of members ) {
+          await cnpgHibernate( member, phaseNamespace )
+        }
+      } else {
+        await scaleStatefulSets( members, 0, phaseNamespace )
+      }
       await sleep( group.pauseMs ?? phasePauseMs )
     }
   } else {
-    // no ordered phases defined yet — scale all StatefulSets together
-    banner( 'StatefulSets' )
-    await $`kubectl scale statefulset --all -n ${namespace} --replicas=0`
+    warning( 'statefulsetPhases is empty — StatefulSet shutdown skipped.' )
+    warning( 'Define phases in cluster-config.yaml when ready to hibernate database workloads.' )
+    warning( 'Note: CNPG clusters require kubectl cnpg hibernate, not kubectl scale.' )
   }
 
-  await waitForPodsGone( namespace )
+  if ( deferredDeployPhases.length ) {
+    for ( const group of deferredDeployPhases ) {
+      const phaseNamespace = group.namespace ?? namespace
+      const members = group.optional ?
+        await filterExisting( 'deployment', group.members, phaseNamespace ) :
+        group.members
+      if ( !members.length ) {
+        log( `  ${group.name} — not present, skipping` )
+        continue
+      }
+      banner( `Deployments: ${group.name}` )
+      await scaleDeployments( members, 0, phaseNamespace )
+      await sleep( group.pauseMs ?? phasePauseMs )
+    }
+  }
+
+  const namespacesToDrain = [ ...new Set( [
+    namespace,
+    ...deploymentPhases.map( g => g.namespace ?? namespace ),
+    ...statefulsetPhases.map( g => g.namespace ?? namespace ),
+  ] ) ]
+  await waitForPodsGone( namespacesToDrain )
 
   const poolsWithAutoscaler = nodePools.filter( p => p.autoscaler )
   if ( poolsWithAutoscaler.length ) {
@@ -116,19 +279,23 @@ const runHibernate = async ( configPath ) => {
   }
 
   log( chalk.green( '\nCluster hibernated. Control plane (~$0.10/hr) continues running.' ) )
-  log( `To wake: ${chalk.cyan( `k8s wake ${configPath}` )}` )
+  log( `To wake: ${chalk.cyan( 'k8s cryo wake' )}` )
 }
 
-// ── Wake ──────────────────────────────────────────────────────────────────
+// ── Wake ───────────────────────────────────────────────────────────────────
 
 const runWake = async ( configPath ) => {
   const config = await loadConfig( configPath )
-  const { cluster, nodePools } = config
+  const {
+    namespace = 'default',
+    nodePools,
+    deploymentPhases = [],
+    statefulsetPhases = [],
+  } = config
 
   await overwhelmGate()
+  const cluster = await getClusterEnv()
 
-  // Restore autoscaler minimums before resizing so the autoscaler
-  // doesn't immediately scale back down the nodes we're bringing up.
   const poolsWithAutoscaler = nodePools.filter( p => p.autoscaler )
   if ( poolsWithAutoscaler.length ) {
     banner( 'Restoring autoscaler minimums' )
@@ -139,26 +306,552 @@ const runWake = async ( configPath ) => {
 
   banner( 'Restoring node pools' )
   for ( const pool of nodePools ) {
-    if ( pool.restoreSize > 0 ) {
-      await resizePool( pool, cluster, pool.restoreSize )
-    } else {
+    if ( pool.restoreSize <= 0 ) {
       log( `  ${pool.name} → skipped (autoscaler brings up on demand)` )
+      continue
     }
+    // Autoscaled pools start at 1 node per zone — enough for initial scheduling.
+    // The autoscaler brings up additional nodes as workloads are placed.
+    const initialSize = pool.autoscaler ? 1 : pool.restoreSize
+    await resizePool( pool, cluster, initialSize )
   }
 
   await waitForNodesReady()
 
-  log( chalk.green( '\nNodes ready. Run helmup to restore all services.' ) )
+  // Build a unified wake list from both phase arrays, tagged with kind.
+  // Phases at the same wakeOrder run in parallel; levels are processed in order.
+  // Helm won't restore manually-scaled-to-0 resources on its own — it only
+  // patches fields that changed between old and new manifest — so cryo must
+  // restore replica counts explicitly before helmup reconciles.
+  const wakeList = [
+    ...deploymentPhases.map( p => ( { ...p, phaseKind: 'deployment' } ) ),
+    ...statefulsetPhases.map( p => ( { ...p, phaseKind: 'statefulset' } ) ),
+  ]
+
+  const wakeMap = new Map()
+  for ( const group of wakeList ) {
+    const order = group.wakeOrder ?? Infinity
+    if ( !wakeMap.has( order ) ) wakeMap.set( order, [] )
+    wakeMap.get( order ).push( group )
+  }
+
+  const wakeGroup = async ( group ) => {
+    const phaseNamespace = group.namespace ?? namespace
+
+    if ( group.phaseKind === 'statefulset' && group.type === 'cnpg' ) {
+      // Attempt hibernate off unconditionally — cnpgWake handles not-found gracefully.
+      // filterExisting is unreliable here: hibernated clusters are invisible when the
+      // operator is down, and the operator may not have been running during a prior failed run.
+      for ( const member of group.members ) {
+        await cnpgWake( member, phaseNamespace )
+      }
+      return
+    }
+
+    if ( group.requiresCnpg && !await anyCnpgClusters( phaseNamespace ) ) {
+      log( `  ${group.name} — no CNPG clusters in ${phaseNamespace}, skipping` )
+      return
+    }
+
+    const kind = group.phaseKind === 'statefulset' ? 'statefulset' : 'deployment'
+    const members = group.optional ?
+      await filterExisting( kind, group.members, phaseNamespace ) :
+      group.members
+    if ( !members.length ) {
+      log( `  ${group.name} — not present, skipping` )
+      return
+    }
+    const replicas = group.restoreReplicas ?? 1
+    log( `  ${group.name} → ${replicas}` )
+    if ( group.phaseKind === 'statefulset' ) {
+      await scaleStatefulSets( members, replicas, phaseNamespace )
+    } else {
+      await scaleDeployments( members, replicas, phaseNamespace )
+    }
+    if ( group.waitForReady ) {
+      log( `  waiting for ${group.name} to be ready...` )
+      for ( const member of members ) {
+        try {
+          await $`kubectl rollout status deployment ${member} -n ${phaseNamespace} --timeout=120s`
+        } catch {
+          warning( `${member} not ready after 120s — proceeding` )
+        }
+      }
+    }
+  }
+
+  banner( 'Restoring workloads' )
+  const levels = [ ...wakeMap.keys() ].sort( ( a, b ) => a - b )
+  for ( const level of levels ) {
+    await Promise.all( wakeMap.get( level ).map( wakeGroup ) )
+  }
+
+  log( chalk.green( '\nCluster restored. Run helmup to reconcile any config or version changes.' ) )
+}
+
+// ── Init: YAML generation ──────────────────────────────────────────────────
+
+const buildConfigYaml = ( { clusterName, namespace, phasePauseMs, deployPhases, stsPhases, pools } ) => {
+  const bar = label => `  # ── ${label} ${'─'.repeat( Math.max( 0, 57 - label.length ) )}`
+
+  const serializePhase = ( phase ) => {
+    const ns      = phase.namespace && phase.namespace !== namespace ?
+      `\n    namespace: ${phase.namespace}` : ''
+    const type    = phase.type              ? `\n    type: ${phase.type}`               : ''
+    const opt     = phase.optional          ? '\n    optional: true'                    : ''
+    const rcnpg   = phase.requiresCnpg      ? '\n    requiresCnpg: true'                : ''
+    const wo      = phase.wakeOrder != null ? `\n    wakeOrder: ${phase.wakeOrder}`     : ''
+    const defer   = phase.deferShutdown     ? '\n    deferShutdown: true'               : ''
+    const waitRdy = phase.waitForReady      ? '\n    waitForReady: true'                : ''
+    const restore = phase.restoreReplicas != null && phase.restoreReplicas !== 1 ?
+      `\n    restoreReplicas: ${phase.restoreReplicas}` : ''
+    const pauseMs = phase.pauseMs           ? `\n    pauseMs: ${phase.pauseMs}`         : ''
+    const members = phase.members.map( m => `      - ${m}` ).join( '\n' )
+    return `  - name: "${phase.name}"${type}${ns}${opt}${rcnpg}${wo}${defer}${waitRdy}${restore}${pauseMs}\n    members:\n${members}`
+  }
+
+  const phaseBlock = ( phase ) => {
+    const header  = phase.header  ? `\n${bar( phase.header )}\n` : ''
+    const comment = phase.comment ? `  # ${phase.comment}\n`     : ''
+    return `${header}${comment}${serializePhase( phase )}`
+  }
+
+  const deployBlocks = deployPhases.filter( p => p.members?.length ).map( phaseBlock ).join( '\n' )
+
+  const stsContent = stsPhases.filter( p => p.members?.length )
+  const stsBlocks  = stsContent.map( phaseBlock ).join( '\n' )
+  const hasCnpg    = stsContent.some( p => p.type === 'cnpg' )
+
+  const poolBlocks = pools.map( ( pool ) => {
+    const auto = pool.autoscaler ?
+      `\n    autoscaler:\n      min: ${pool.autoscaler.min}\n      max: ${pool.autoscaler.max}` : ''
+    return `  - name: ${pool.name}\n    restoreSize: ${pool.restoreSize}${auto}`
+  } ).join( '\n' )
+
+  const cnpgNote = hasCnpg ?
+    '\n# CNPG clusters use type: cnpg → kubectl cnpg hibernate on/off (not kubectl scale).' : ''
+
+  return `# k8s-cryo.yaml — cluster hibernation / wake configuration for ${clusterName}
+# Cluster identity (project, name, region) is read from overwhelm at runtime.
+# Generated by: k8s cryo init — review before committing.
+#
+# Phases execute top → bottom during hibernate.
+# wakeOrder controls the wake sequence independently of hibernate order.
+
+namespace: ${namespace}
+phasePauseMs: ${phasePauseMs}
+
+deploymentPhases:
+${deployBlocks}
+
+# StatefulSet shutdown phases — executed after all deployments are down.${cnpgNote}
+${stsContent.length ? `statefulsetPhases:\n${stsBlocks}` : 'statefulsetPhases: [] # TODO: add StatefulSet phases before hibernating'}
+
+# Node pools — restoreSize is nodes per zone (regional cluster × 3 zones).
+# Pools with restoreSize: 0 are managed on demand by the cluster autoscaler.
+nodePools:
+${poolBlocks}
+`
+}
+
+// ── Init ───────────────────────────────────────────────────────────────────
+
+const runInit = async ( outputPath, opts ) => {
+  const { namespace, force } = opts
+
+  if ( !force ) {
+    try {
+      await fs.access( outputPath )
+      errorExit( `${outputPath} already exists — use --force to overwrite` )
+    } catch ( err ) {
+      if ( err.code !== 'ENOENT' ) errorExit( `Cannot access ${outputPath}: ${err.message}` )
+    }
+  }
+
+  await overwhelmGate()
+  const cluster = await getClusterEnv()
+
+  log( `\nScanning ${chalk.cyan( cluster.name )} (${cluster.project})…` )
+
+  // ── Default namespace deployments ─────────────────────────────────────────
+
+  banner( 'Listing deployments' )
+  const allDeploys = await listDeployments( namespace )
+  log( `  Found ${allDeploys.length} deployment(s) in ${namespace}` )
+
+  const deployed       = new Set( allDeploys )
+  const transponders   = PLATFORM_TRANSPONDERS.filter( d => deployed.has( d ) )
+  const platformSupport = PLATFORM_SUPPORT.filter( d => deployed.has( d ) )
+  const platformCore   = PLATFORM_CORE.filter( d => deployed.has( d ) )
+  const auxiliary      = AUXILIARY_KNOWN.filter( d => deployed.has( d ) )
+  const systemDeploy   = SYSTEM_DEPLOY.filter( d => deployed.has( d ) )
+  const project        = allDeploys.filter( d => !ALL_KNOWN.has( d ) )
+
+  if ( project.length ) {
+    log( `  → Project / product (review placement): ${project.join( ', ' )}` )
+  }
+
+  // ── System namespace scans ────────────────────────────────────────────────
+
+  banner( 'Scanning system namespaces' )
+
+  // Returns deployments present in ns from candidates; skips if namespace absent.
+  const scanNs = async ( ns, candidates ) => {
+    if ( !await namespaceExists( ns ) ) return []
+    const found = await filterExisting( 'deployment', candidates, ns )
+    log( found.length ?
+      `  ${ns}: ${found.join( ', ' )}` :
+      `  ${ns}: (empty)` )
+    return found
+  }
+
+  // CNPG — operator, poolers, cluster CRDs
+  const cnpgSystemExists = await namespaceExists( 'cnpg-system' )
+  let cnpgOperatorMembers = []
+  let cnpgPoolers         = []
+  let cnpgClusters        = []
+
+  if ( cnpgSystemExists ) {
+    cnpgOperatorMembers = await filterExisting(
+      'deployment', [ 'barman-cloud', 'cnpg-controller-manager' ], 'cnpg-system'
+    )
+    log( `  cnpg-system: ${cnpgOperatorMembers.join( ', ' ) || '(none)'}` )
+
+    if ( await namespaceExists( 'cnpg-operands' ) ) {
+      const operandDeploys = await listDeployments( 'cnpg-operands' )
+      cnpgPoolers = operandDeploys.filter( d => d.endsWith( '-pool-rw' ) || d.endsWith( '-pool-ro' ) )
+      if ( cnpgPoolers.length ) log( `  cnpg-operands poolers: ${cnpgPoolers.join( ', ' )}` )
+
+      try {
+        const r = await $`kubectl get ${CNPG_CRD} -n cnpg-operands --no-headers -o name`
+        cnpgClusters = r.stdout.trim().split( '\n' ).filter( Boolean )
+          .map( n => n.replace( /^[^/]+\//, '' ) )
+        if ( cnpgClusters.length ) log( `  cnpg-operands clusters: ${cnpgClusters.join( ', ' )}` )
+      } catch {
+        warning( 'Could not list CNPG clusters — operator may not be running' )
+      }
+    }
+  }
+
+  const traefikDeploys   = await scanNs( 'traefik',   [ 'traefik' ] )
+  const certDeploys      = await scanNs( 'cert-manager',
+    [ 'cert-manager', 'cert-manager-cainjector', 'cert-manager-webhook' ] )
+  const esoDeploys       = await scanNs( 'external-secrets',
+    [ 'external-secrets', 'external-secrets-cert-controller', 'external-secrets-webhook' ] )
+  const veleroDeploys    = await scanNs( 'velero',     [ 'velero' ] )
+  const estafetteDeploys = await scanNs( 'estafette',  [ 'estafette-gke-preemptible-killer' ] )
+  const promDeploys      = await scanNs( 'prometheus', [
+    'prometheus-stack-elasticsearch8-metrics',
+    'prometheus-stack-grafana',
+    'prometheus-stack-kube-prom-operator',
+    'prometheus-stack-kube-state-metrics',
+    'prometheus-stack-stackdriver-metrics',
+  ] )
+  const monitoringDeploys = await scanNs( 'monitoring', [
+    'grafana',
+    'prometheus-server',
+    'prometheus-kube-state-metrics',
+    'elasticsearch8-exporter',
+    'stackdriver-exporter-prometheus-stackdriver-exporter',
+  ] )
+
+  // ── StatefulSet scans ─────────────────────────────────────────────────────
+
+  banner( 'Scanning StatefulSets' )
+
+  const defaultSts    = new Set( await listStatefulSets( namespace ) )
+  const hasValkey     = defaultSts.has( 'valkey-primary' )
+  const hasRedis      = !hasValkey && defaultSts.has( 'redis-master' )
+  const hasPostgres   = defaultSts.has( 'postgres-postgresql' )
+  const hasTsdbMaster = defaultSts.has( 'timescale-db-postgresql-master' )
+  const hasTsdbSlave  = defaultSts.has( 'timescale-db-postgresql-slave' )
+
+  log( `  ${namespace}: ${[ ...defaultSts ].join( ', ' ) || '(none)'}` )
+
+  const tsdbSlaveReplicas = hasTsdbSlave ?
+    await getReplicaCount( 'statefulset', 'timescale-db-postgresql-slave', namespace ) : 2
+
+  const hasValkeyReplicas = defaultSts.has( 'valkey-replicas' )
+  const valkeyReplicas    = hasValkeyReplicas ?
+    await getReplicaCount( 'statefulset', 'valkey-replicas', namespace ) : 1
+
+  const elasticSts    = await listStatefulSets( 'elastic' )
+  const esKnown       = [ 'elasticsearch8-coordinating', 'elasticsearch8-ingest',
+    'elasticsearch8-data', 'elasticsearch8-master' ]
+  const esMembers     = esKnown.filter( s => elasticSts.includes( s ) )
+  const esDataMembers = esMembers.filter( s => s.includes( 'data' ) )
+  const esOtherMembers = esMembers.filter( s => !s.includes( 'data' ) )
+  const esDataReplicas  = esDataMembers.length ?
+    await getReplicaCount( 'statefulset', esDataMembers[0], 'elastic' ) : 3
+  const esOtherReplicas = esOtherMembers.length ?
+    await getReplicaCount( 'statefulset', esOtherMembers[0], 'elastic' ) : 3
+  if ( esMembers.length ) {
+    log( `  elastic: ${esDataMembers.join( ', ' )} (×${esDataReplicas}), ${esOtherMembers.join( ', ' )} (×${esOtherReplicas})` )
+  }
+
+  const promSts        = new Set( await listStatefulSets( 'prometheus' ) )
+  const promStsMembers = [
+    'prometheus-prometheus-stack-kube-prom-prometheus',
+    'alertmanager-prometheus-stack-kube-prom-alertmanager',
+  ].filter( s => promSts.has( s ) )
+  if ( promStsMembers.length ) log( `  prometheus: ${promStsMembers.join( ', ' )}` )
+
+  const monitoringSts        = new Set( await listStatefulSets( 'monitoring' ) )
+  const monitoringStsMembers = [ 'prometheus-alertmanager' ]
+    .filter( s => monitoringSts.has( s ) )
+  if ( monitoringStsMembers.length ) log( `  monitoring: ${monitoringStsMembers.join( ', ' )}` )
+
+  // ── Node pools ────────────────────────────────────────────────────────────
+
+  banner( 'Listing node pools' )
+  const poolsRaw = JSON.parse(
+    ( await $`gcloud container node-pools list --cluster=${cluster.name} --project=${cluster.project} --region=${cluster.region} --format=json` ).stdout
+  )
+  log( `  Found ${poolsRaw.length} node pool(s)` )
+
+  const pools = poolsRaw.map( ( pool ) => {
+    const auto = pool.autoscaling
+    const isPreemptible = pool.config?.spot || pool.config?.preemptible ||
+      pool.name.includes( 'preemptible' ) || pool.name.includes( 'spot' )
+    const hasAutoscalerMin = auto?.enabled && ( auto.minNodeCount ?? 0 ) > 0
+
+    let restoreSize
+    if ( isPreemptible ) restoreSize = 0
+    else if ( hasAutoscalerMin ) restoreSize = auto.minNodeCount
+    else restoreSize = pool.initialNodeCount ?? 1
+
+    const entry = { name: pool.name, restoreSize }
+    if ( hasAutoscalerMin ) entry.autoscaler = { min: auto.minNodeCount, max: auto.maxNodeCount }
+    return entry
+  } )
+
+  // ── Build phase lists ─────────────────────────────────────────────────────
+
+  const deployPhases = [
+    project.length && {
+      header  : 'PROJECT / PRODUCT',
+      comment : 'Stopped first — depend on the platform but nothing depends on them.',
+      name    : 'Project',
+      pauseMs : 15_000,
+      members : project,
+    },
+    auxiliary.length && {
+      header   : 'AUXILIARY',
+      name     : 'Auxiliary',
+      optional : true,
+      members  : auxiliary,
+    },
+    transponders.length && {
+      header  : 'PLATFORM — transponders',
+      comment : 'Data pipeline — last platform tier deployed, so first to hibernate.',
+      name    : 'Platform — transponders',
+      members : transponders,
+    },
+    platformSupport.length && {
+      header  : 'PLATFORM — support',
+      name    : 'Platform — support',
+      members : platformSupport,
+    },
+    platformCore.length && {
+      header  : 'PLATFORM — core',
+      comment : 'First platform tier deployed — last among deployments to hibernate.',
+      name    : 'Platform — core',
+      members : platformCore,
+    },
+    systemDeploy.length && {
+      header  : 'SYSTEM — deployments',
+      name    : 'System — deployments',
+      members : systemDeploy,
+    },
+    cnpgPoolers.length && {
+      header       : 'SYSTEM — CNPG poolers',
+      comment      : 'Must come down before CNPG databases are hibernated.',
+      name         : 'System — CNPG poolers',
+      namespace    : 'cnpg-operands',
+      optional     : true,
+      requiresCnpg : true,
+      wakeOrder    : 40,
+      members      : cnpgPoolers,
+    },
+    cnpgOperatorMembers.length && {
+      header        : 'SYSTEM — CNPG operator',
+      comment       : 'Must wake before CNPG clusters (webhook dependency).',
+      name          : 'System — CNPG operator',
+      namespace     : 'cnpg-system',
+      wakeOrder     : 30,
+      deferShutdown : true,
+      waitForReady  : true,
+      members       : cnpgOperatorMembers,
+    },
+    veleroDeploys.length && {
+      header    : 'SYSTEM — velero',
+      name      : 'System — velero',
+      namespace : 'velero',
+      optional  : true,
+      members   : veleroDeploys,
+    },
+    estafetteDeploys.length && {
+      header    : 'SYSTEM — estafette',
+      name      : 'System — estafette',
+      namespace : 'estafette',
+      optional  : true,
+      members   : estafetteDeploys,
+    },
+    monitoringDeploys.length && {
+      header    : 'SYSTEM — monitoring (legacy)',
+      comment   : 'Legacy prometheus-community/prometheus + grafana/grafana in monitoring ns.',
+      name      : 'System — monitoring (legacy)',
+      namespace : 'monitoring',
+      optional  : true,
+      members   : monitoringDeploys,
+    },
+    promDeploys.length && {
+      header    : 'SYSTEM — prometheus',
+      name      : 'System — prometheus',
+      namespace : 'prometheus',
+      optional  : true,
+      members   : promDeploys,
+    },
+    traefikDeploys.length && {
+      header    : 'SYSTEM — traefik',
+      name      : 'System — traefik',
+      namespace : 'traefik',
+      optional  : true,
+      members   : traefikDeploys,
+    },
+    certDeploys.length && {
+      header    : 'SYSTEM — cert-manager',
+      name      : 'System — cert-manager',
+      namespace : 'cert-manager',
+      optional  : true,
+      wakeOrder : 20,
+      members   : certDeploys,
+    },
+    esoDeploys.length && {
+      header    : 'SYSTEM — external-secrets',
+      name      : 'System — external-secrets',
+      namespace : 'external-secrets',
+      optional  : true,
+      wakeOrder : 20,
+      members   : esoDeploys,
+    },
+  ].filter( Boolean )
+
+  const stsPhases = [
+    ...cnpgClusters.map( ( name, i ) => ( {
+      header    : i === 0 ? 'CNPG clusters' : undefined,
+      name      : `CNPG — ${name.replace( /^cnpg-db-/, '' )}`,
+      type      : 'cnpg',
+      namespace : 'cnpg-operands',
+      optional  : true,
+      wakeOrder : 35,
+      members   : [ name ],
+    } ) ),
+    hasPostgres && {
+      header    : 'Legacy postgres',
+      name      : 'System — postgres',
+      wakeOrder : 10,
+      members   : [ 'postgres-postgresql' ],
+    },
+    hasTsdbMaster && {
+      header    : 'TimescaleDB',
+      name      : 'System — timescale-db (master)',
+      wakeOrder : 10,
+      members   : [ 'timescale-db-postgresql-master' ],
+    },
+    hasTsdbSlave && {
+      name            : 'System — timescale-db (slave)',
+      wakeOrder       : 10,
+      restoreReplicas : tsdbSlaveReplicas,
+      members         : [ 'timescale-db-postgresql-slave' ],
+    },
+    hasValkey && {
+      header    : 'Valkey',
+      name      : 'System — valkey',
+      wakeOrder : 10,
+      restoreReplicas : valkeyReplicas !== 1 ? valkeyReplicas : undefined,
+      members   : [ 'valkey-primary', ...( hasValkeyReplicas ? [ 'valkey-replicas' ] : [] ) ],
+    },
+    hasRedis && {
+      header    : 'Redis',
+      name      : 'System — redis',
+      wakeOrder : 10,
+      members   : [
+        'redis-master',
+        ...( defaultSts.has( 'redis-replicas' ) ? [ 'redis-replicas' ] : [] ),
+      ],
+    },
+    promStsMembers.length && {
+      header    : 'Prometheus',
+      name      : 'System — prometheus',
+      namespace : 'prometheus',
+      optional  : true,
+      members   : promStsMembers,
+    },
+    monitoringStsMembers.length && {
+      header    : 'Monitoring (legacy)',
+      name      : 'System — monitoring (legacy)',
+      namespace : 'monitoring',
+      optional  : true,
+      members   : monitoringStsMembers,
+    },
+    esDataMembers.length && {
+      header          : 'Elasticsearch',
+      comment         : 'Bitnami chart — no operator, plain kubectl scale.',
+      name            : 'System — elasticsearch (data)',
+      namespace       : 'elastic',
+      wakeOrder       : 10,
+      restoreReplicas : esDataReplicas,
+      members         : esDataMembers,
+    },
+    esOtherMembers.length && {
+      name            : 'System — elasticsearch',
+      namespace       : 'elastic',
+      wakeOrder       : 10,
+      restoreReplicas : esOtherReplicas,
+      members         : esOtherMembers,
+    },
+  ].filter( Boolean )
+
+  // ── Write ─────────────────────────────────────────────────────────────────
+
+  const yaml = buildConfigYaml( {
+    clusterName  : cluster.name,
+    namespace,
+    phasePauseMs : 10_000,
+    deployPhases,
+    stsPhases,
+    pools,
+  } )
+
+  // eslint-disable-next-line security/detect-non-literal-fs-filename
+  await fs.writeFile( outputPath, yaml, 'utf8' )
+
+  log( chalk.green( `\nWrote ${outputPath}` ) )
+  log( 'Review the file — especially the Project phase and StatefulSet replica counts — then commit.' )
 }
 
-// ── Commands ──────────────────────────────────────────────────────────────
+// ── Commands ───────────────────────────────────────────────────────────────
 
-export const hibernateCommand = new Command( 'hibernate' )
+const hibernateCommand = new Command( 'hibernate' )
   .description( 'Scale all workloads to zero and drain all node pools' )
-  .argument( '[config]', 'path to cluster-config.json', './cluster-config.json' )
-  .action( async ( config ) => runHibernate( config ).catch( errorExit ) )
+  .argument( '[config]', 'path to k8s-cryo.yaml', './k8s-cryo.yaml' )
+  .action( async config => runHibernate( config ).catch( errorExit ) )
 
-export const wakeCommand = new Command( 'wake' )
+const wakeCommand = new Command( 'wake' )
   .description( 'Restore node pools from hibernation (then run helmup)' )
-  .argument( '[config]', 'path to cluster-config.json', './cluster-config.json' )
-  .action( async ( config ) => runWake( config ).catch( errorExit ) )
+  .argument( '[config]', 'path to k8s-cryo.yaml', './k8s-cryo.yaml' )
+  .action( async config => runWake( config ).catch( errorExit ) )
+
+const initCommand = new Command( 'init' )
+  .description( 'Generate a starter k8s-cryo.yaml from live cluster state' )
+  .argument( '[output]', 'output file path', './k8s-cryo.yaml' )
+  .option( '-n, --namespace <ns>', 'namespace to scan for deployments', 'default' )
+  .option( '-f, --force', 'overwrite existing file' )
+  .action( async ( output, opts ) => runInit( output, opts ).catch( errorExit ) )
+
+export const cryoCommand = new Command( 'cryo' )
+  .description( 'Hibernate, wake, and initialize cluster configurations' )
+  .addCommand( initCommand )
+  .addCommand( hibernateCommand )
+  .addCommand( wakeCommand )