@leverege/build-tools 2.113.6 → 2.114.0-DEVOP-578.2

package/README.md

@@ -199,21 +199,6 @@ klog <app-name> <ns>     # specify namespace
 
 ---
 
-### Artifact Management
-
-**`pull-artifact`** — Pulls generic artifacts from GCP Artifact Registry based on an
-`artifacts.yaml` spec file.
-```bash
-pull-artifact [-f <spec-file>] [-n <name>] [-v <version>]
-```
-
-**`push-artifact`** — Pushes generic artifacts to GCP Artifact Registry.
-```bash
-push-artifact [-f <spec-file>] [-n <name>] [-v <version>]
-```
-
----
-
 ### Package & Release Management
 
 **`hook-and-release`** — Manages git pre-commit hooks and version consistency for npm

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@leverege/build-tools",
-  "version": "2.113.6",
+  "version": "2.114.0-DEVOP-578.2",
   "description": "A collection of build / support tools for Leverege developers",
   "main": "index.js",
   "repository": {
@@ -12,7 +12,7 @@
   ],
   "scripts": {
     "lint": "eslint",
-    "lint-fix": "eslint --fix",
+    "lint:fix": "eslint --fix",
     "prepack": "npm exec prepack",
     "prepare": "[ $SKIP_PREPARE ] && exit 0 || ./src/hook-and-release.mjs",
     "test": "mocha -t 60000 --exit 'test/**/!(*e2e*).mjs'",
@@ -21,7 +21,6 @@
   "bin": {
     "build-cnpg-image": "src/build-cnpg-image.mjs",
     "build-tools": "src/build-tools.mjs",
-    "change-up": "src/change-up.sh",
     "chart-compass": "src/chart-compass.mjs",
     "chart-to-registry": "src/chart-to-registry.mjs",
     "circle-orb-it": "src/circle-orb-it/circle-orb-it.mjs",
@@ -53,15 +52,14 @@
     "k8scale": "src/k8scale.sh",
     "k8x": "src/k8x.sh",
     "klog": "src/klog.sh",
+    "k8s": "src/k8s/k8s.mjs",
     "npm-link": "src/npm-link.sh",
     "npm-unlink": "src/npm-unlink.sh",
     "overwhelm": "src/overwhelm.mjs",
     "pkglint": "src/pkglint.sh",
     "prepack": "src/prepack.mjs",
     "prune-git": "src/prune-git.sh",
-    "pull-artifact": "src/artifacts/pull-artifact.mjs",
     "pull-git": "src/pull-git.sh",
-    "push-artifact": "src/artifacts/push-artifact.mjs",
     "push-my-chart": "src/push-my-chart.mjs",
     "refresh-npm-token": "src/refresh-npm-token.mjs",
     "refresh-py-idx": "src/refresh-py-idx.sh",
@@ -70,6 +68,7 @@
     "service-man": "src/service-man/service-man.mjs",
     "snapshot-cnpg": "src/snapshot-cnpg.mjs",
     "sync-chart-version": "src/sync-chart-version.mjs",
+    "throwup": "src/helmup.sh",
     "upload-model": "src/upload-model.sh",
     "yarn-build": "src/yarn-build.mjs"
   },
@@ -83,16 +82,22 @@
     "npm": "11.12.0"
   },
   "license": "SEE LICENSE IN LICENSE.md",
+  "overrides": {
+    "diff": "^8.0.3",
+    "mocha": {
+      "serialize-javascript": "^7.0.3"
+    }
+  },
   "dependencies": {
     "@kubernetes/client-node": "^1.4.0",
     "@leverege/jsdoc-template": "^1.0.1",
-    "chalk": "^5.6.2",
+    "chalk": "^6.0.0",
     "command-line-args": "^6.0.2",
     "command-line-usage": "^7.0.4",
     "commander": "^15.0.0",
     "deepmerge": "^4.3.1",
     "enquirer": "^2.4.1",
-    "execa": "^9.6.1",
+    "execa": "^10.0.1",
     "find-yarn-workspace-root": "^2.0.0",
     "glob": "^13.0.6",
     "googleapis": "^173.0.0",
@@ -113,7 +118,7 @@
     "sloc": "^0.3.2",
     "superstruct": "^2.0.2",
     "tmp": "^0.2.7",
-    "toml": "^4.1.1",
+    "toml": "^5.0.0",
     "yaml": "^2.9.0",
     "zx": "^8.8.5"
   },
@@ -121,8 +126,8 @@
     "@leverege/eslint-config-leverege": "^9",
     "chai": "^6.2.2",
     "eslint": "^9",
-    "mocha": "^11.7.6",
-    "npm": "^11.17.0"
+    "mocha": "^11.8.0",
+    "npm": "^12.0.2"
   },
   "allowScripts": {
     "unrs-resolver@1.12.2": true

package/src/bash-funcs

@@ -298,7 +298,7 @@ function ifPluggedIn() {
   local PLUGIN=
   HELMUP="helm upgrade --install $SERVICE $CHARTLOC -f $SERVICE/values.yaml $CHARTVER $HELM_WHAT"
   case "$INVOKED_AS" in
-    "helmup"|"helmwhat"|"helmcycle"|"helminit" )
+    "helmup"|"helmwhat"|"helmcycle"|"helminit"|"throwup" )
       PLUGIN="$1/helmup.plugin"
       ;;
     "helmdn" )

package/src/build-tools.mjs

@@ -9,7 +9,7 @@ import chalk from 'chalk'
 import cliArgs from 'command-line-args'
 import cliHelp from 'command-line-usage'
 import npmRegistryFetch from 'npm-registry-fetch'
-import { lt as semverLt } from 'semver'
+import { lt as semverLt, prerelease } from 'semver'
 
 // because __dirname goes away with ESM
 const getFilename = () => {
@@ -124,7 +124,8 @@ const checkForLatest = async ( pkg = '@leverege/build-tools' ) => {
     const list = await npmRegistryFetch.json(
       pkg, { '//registry.npmjs.org/:_authToken' : getToken() }
     )
-    return list['dist-tags'].latest
+    const latest = list['dist-tags'].latest
+    return prerelease( latest ) ? undefined : latest
   } catch ( err ) {
     console.log( err )
   }

package/src/chart-compass.yaml

@@ -22,6 +22,11 @@
           - transponder-tsdb
 
     leverege:
+      llm-prompt-server: {}
+      overdose: {}
       pubsub-pulse: {}
       pusher: {}
-      overdose: {}
+      vms-server: {}
+
+    system:
+      triton-inference-server: {}

package/src/helm-charts/api-server/values-local.yaml

@@ -15,7 +15,7 @@ config:
   MODEL_SQL_HOST: "cnpg-db-psql-stack-pool-rw.cnpg-operands"
   PG_MODELS_HOST: "cnpg-db-psql-stack-pool-rw.cnpg-operands"
   # timescale
-  PG_HOST: "cnpg-db-tsdb-basic-pool-r.cnpg-operands"
+  PG_HOST: "cnpg-db-tsdb-basic-r.cnpg-operands"
   # imagine db is legacy - newer setups use timescale as the db name
   PG_DATABASE: "timescale"
   PG_DENSE_HISTORY_DATABASE: "timescale"

package/src/helm-charts/cnpg-db-psql-stack/cluster.yaml.ovh

@@ -55,7 +55,7 @@ spec:
     size: 16Gi
 
   resources:
-#   limit:
+#   limits:
 #     cpu: 1000m
 #     memory: 8Gi
     requests:

package/src/helm-charts/cnpg-db-tsdb-basic/cluster.yaml.ovh

@@ -55,7 +55,7 @@ spec:
     size: 16Gi
 
   resources:
-#   limit:
+#   limits:
 #     cpu: 1000m
 #     memory: 8Gi
     requests:

package/src/helm-charts/cnpg-db-tsdb-dense/cluster.yaml.ovh

@@ -54,7 +54,7 @@ spec:
     size: 16Gi
 
   resources:
-#   limit:
+#   limits:
 #     cpu: 1000m
 #     memory: 8Gi
     requests:

package/src/helm-charts/llm-prompt-server/values-local.yaml

@@ -0,0 +1,5 @@
+image:
+  tag:
+
+config:
+  LOG_CONFIG: '{"type":"pino","level":"warn"}'

package/src/helm-charts/prom-operator/helmup.plugin

@@ -8,14 +8,14 @@ showInstalling "The Prometheus Operator (kube-prometheus-stack)"
 
 # https://artifacthub.io/packages/helm/prometheus-community/kube-prometheus-stack
 OCI_CHART="oci://ghcr.io/prometheus-community/charts/kube-prometheus-stack"
-[ -z "$PROMETHEUS_STACK_CHART_VERSION" ] && PROMETHEUS_STACK_CHART_VERSION="86.2.3"
+[ -z "$PROMETHEUS_STACK_CHART_VERSION" ] && PROMETHEUS_STACK_CHART_VERSION="87.21.0"
 helm upgrade $NS --install prometheus-stack $OCI_CHART \
                  --values  prom-operator/prometheus-stack.yaml \
                  --version $PROMETHEUS_STACK_CHART_VERSION $HELM_WHAT
 
 # https://artifacthub.io/packages/helm/prometheus-community/prometheus-elasticsearch-exporter
 showInstalling "The Elasticsearch Exporter (prom-operator ES metrics exports)"
-[ -z "$ELASTICSEARCH_EXPORTER_CHART_VERSION" ] && ELASTICSEARCH_EXPORTER_CHART_VERSION="7.2.1"
+[ -z "$ELASTICSEARCH_EXPORTER_CHART_VERSION" ] && ELASTICSEARCH_EXPORTER_CHART_VERSION="7.4.0"
 OCI_CHART="oci://ghcr.io/prometheus-community/charts/prometheus-elasticsearch-exporter"
 helm upgrade $NS --install elasticsearch8-exporter $OCI_CHART \
                  --values prom-operator/elasticsearch-exporter.yaml \
@@ -23,7 +23,7 @@ helm upgrade $NS --install elasticsearch8-exporter $OCI_CHART \
 
 # https://artifacthub.io/packages/helm/prometheus-community/prometheus-stackdriver-exporter
 showInstalling "The Stackdriver Exporter (prom-operator GKE metrics exports)"
-[ -z "$STACKDRIVER_EXPORTER_CHART_VERSION" ] && STACKDRIVER_EXPORTER_CHART_VERSION="4.12.2"
+[ -z "$STACKDRIVER_EXPORTER_CHART_VERSION" ] && STACKDRIVER_EXPORTER_CHART_VERSION="4.13.0"
 OCI_CHART="oci://ghcr.io/prometheus-community/charts/prometheus-stackdriver-exporter"
 helm upgrade $NS --install stackdriver-exporter $OCI_CHART \
                  --values prom-operator/stackdriver-exporter.yaml \

package/src/helm-charts/prom-operator/prometheus-stack.yaml.ovh

@@ -7,7 +7,7 @@ crds:
   enabled: true
   upgradeJob:
     enabled: true
-    forceConflicts: false
+    forceConflicts: true
 
 kubeControllerManager:
   enabled: false

package/src/helm-charts/triton-inference-server/values-local.yaml

@@ -0,0 +1,28 @@
+image:
+  tag:
+
+modelInit:
+  sources:
+    triton: "gs://${PROJECT_ID}-triton-model-repository/models"
+  targetDir: "/models"
+
+# container args may be tuned as necessary - see the repo README for more info
+#
+# https://bitbucket.org/leverege/triton-inference-server/src/development/
+
+# example of how to add objects
+#extraObjects:
+#  - apiVersion: traefik.io/v1alpha1
+#    kind: IngressRoute
+#    metadata:
+#      name: ${PROJECT_NAME}-extra-api-${HOST}-com
+#      namespace: default
+#    spec:
+#      entryPoints:
+#        - websecure
+#      routes:
+#        - match: Host(`${PROJECT_NAME}-test-api.${HOST}.com`)
+#          kind: Rule
+#          services:
+#            - name: triton-inference-server
+#              port: 8001

package/src/helm-charts/vms-server/values-local.yaml

@@ -0,0 +1,45 @@
+image:
+  tag:
+
+config:
+  LOG_LEVEL: "WARNING"
+  GCP_PROJECT_ID: "${PROJECT_ID}" # needed for python secrets manager?
+  GCP_SECRET_MANAGER_NAMESPACE: "VMS_SERVER"
+  IMAGINE_API_HOST: "http://api-server:8181"
+  IMAGINE_API_PROJECT_ID: "set-in-values-local"
+  IMAGINE_API_SYSTEM_ID: "set-in-values-local"
+  IMAGINE_API_KEY: "set-in-values-local"
+  IMAGINE_API_AUDIENCE: "${PROJECT_ID}-aud"
+  IMAGINE_SERVICE_SYNC_ENABLED: "true"
+  IMAGINE_CAMERA_BLUEPRINT_ID: "set-in-values-local"
+  VMS_SERVER_HOST: "http://vms-server:8080"
+  VIDEO_PLAYBACK_SERVICE_VERSION: "20"
+  VIDEO_MANAGEMENT_SERVICE_VERSION: "2"
+  USE_SHARED_METRICS: "true"
+  VIDEO_SEGMENTS_UPLOAD_TIMEOUT: "60"
+  PLAYLIST_OPEN_ENDED_CACHE_ENABLED: "true"
+  SEGMENT_RECONCILER_ENABLED: "true"
+  SEGMENT_RECONCILER_INTERVAL_SECONDS: "300"
+  SEGMENT_RECONCILER_MIN_AGE_SECONDS: "300"
+  SEGMENT_RECONCILER_LOOKBACK_HOURS: "24"
+
+# run on the preemptible node pool (base chart adds the target-env=preemptible
+# nodeAffinity, toleration and safe-to-evict annotation)
+service:
+  preemptible: true
+
+serviceMonitor:
+  enabled: true
+  interval: "60s"
+  namespace: "default"
+
+resources:
+  requests:
+    cpu: 500m
+    memory: 512Mi
+
+autoscaling:
+  enabled: true
+  minReplicas: 2
+  maxReplicas: 16
+  averageCPU: 75

package/src/helmup.sh

@@ -52,9 +52,6 @@ for arg in "$@"; do
     --mutate)
       MUTATE_DEPLOYMENT=1
       ;;
-    --conflicts)
-      export FORCE_CONFLICTS=1
-      ;;
     --help|-h)
       giveHelp && exit 0
       ;;
@@ -100,6 +97,8 @@ gcloud auth application-default print-access-token | helm registry login \
 refresh-npm-token
 exitOnError $? "refresh-npm-token failed - the rc files were not refreshed"
 
+HELM_EMOJI=$( [ "$INVOKED_AS" == "throwup" ] && echo "🤮" || echo "⎈" )
+
 # removed geotile-server from standard platform 3/20/22 - low usage
 PLATFORM=(
   authz-server
@@ -107,6 +106,7 @@ PLATFORM=(
   db-curator
   emailer
   imagine
+  llm-prompt-server
   message-processor
   messenger
   resource-server
@@ -129,6 +129,7 @@ AUXILIARY=(
   reason
   transponder-dh
   vin-decoder-server
+  vms-server
 )
 
 # There's a chicken / egg situation that arises with new clusters in that
@@ -629,7 +630,7 @@ HELMUP_BANNER
 function doHelmup() {
   local culprit="$(gcloud config get account)"
   local deployT="$(date -u +'%Y-%m-%dT%H:%M:%SZ')"
-  local doing="Helming => `color g \"$*\"`"
+  local doing="Helming => $HELM_EMOJI `color g \"$*\"` $HELM_EMOJI"
   local from=
   [ ! -z "$FROM_VERSIONS" ] && from="`color y \"<= from Versions.json\"`"
   printf "$doing $from\n"

package/src/k8s/commands/completion.mjs

@@ -0,0 +1,100 @@
+import { Command } from 'commander'
+
+import { errorExit } from '../../Utils.mjs'
+
+const SHELLS = [ 'bash', 'zsh', 'fish' ]
+
+const BASH = `\
+_k8s_completions() {
+  local cur prev words
+  COMPREPLY=()
+  cur="\${COMP_WORDS[COMP_CWORD]}"
+  words=("\${COMP_WORDS[@]}")
+
+  case "\${words[1]}" in
+    fwd)
+      COMPREPLY=($(compgen -W "elastic redis valkey prom alert grafana" -- "$cur"))
+      return ;;
+    exe)
+      COMPREPLY=($(compgen -W "cnpg es8 grafana grafold prom-operator psql redis valkey" -- "$cur"))
+      return ;;
+    scale)
+      [ "\${#words[@]}" -eq 3 ] && COMPREPLY=($(compgen -W "up dn" -- "$cur"))
+      return ;;
+    log)
+      COMPREPLY=($(compgen -W "--namespace --container --tail --tee --no-pretty" -- "$cur"))
+      return ;;
+    completion)
+      COMPREPLY=($(compgen -W "bash zsh fish" -- "$cur"))
+      return ;;
+    *)
+      COMPREPLY=($(compgen -W "fwd exe scale roll log completion" -- "$cur"))
+      ;;
+  esac
+}
+complete -F _k8s_completions k8s
+`
+
+const ZSH = `\
+#compdef k8s
+
+_k8s() {
+  local state
+  _arguments '1: :->cmd' '*: :->args'
+
+  case $state in
+    cmd)
+      local -a commands=(
+        'fwd:Port-forward to a cluster endpoint'
+        'exe:Exec into a pod interactively'
+        'scale:Scale a deployment up or down'
+        'roll:Rolling restart of deployments'
+        'log:Stream logs with pino-pretty'
+        'completion:Emit shell completion script'
+      )
+      _describe 'command' commands ;;
+    args)
+      case $words[2] in
+        fwd)        _values target elastic redis valkey prom alert grafana ;;
+        exe)        _values target cnpg es8 grafana grafold prom-operator psql redis valkey ;;
+        scale)      [ $CURRENT -eq 3 ] && _values direction up dn ;;
+        completion) _values shell bash zsh fish ;;
+      esac ;;
+  esac
+}
+
+_k8s
+`
+
+const FISH = `\
+set -l cmds fwd exe scale roll log completion
+complete -c k8s -f
+complete -c k8s -n "not __fish_seen_subcommand_from $cmds" -a fwd        -d 'Port-forward to a cluster endpoint'
+complete -c k8s -n "not __fish_seen_subcommand_from $cmds" -a exe        -d 'Exec into a pod interactively'
+complete -c k8s -n "not __fish_seen_subcommand_from $cmds" -a scale      -d 'Scale a deployment up or down'
+complete -c k8s -n "not __fish_seen_subcommand_from $cmds" -a roll       -d 'Rolling restart of deployments'
+complete -c k8s -n "not __fish_seen_subcommand_from $cmds" -a log        -d 'Stream logs with pino-pretty'
+complete -c k8s -n "not __fish_seen_subcommand_from $cmds" -a completion -d 'Emit shell completion script'
+
+complete -c k8s -n "__fish_seen_subcommand_from fwd"   -a "elastic redis valkey prom alert grafana"
+complete -c k8s -n "__fish_seen_subcommand_from exe"   -a "cnpg es8 grafana grafold prom-operator psql redis valkey"
+complete -c k8s -n "__fish_seen_subcommand_from scale; and not __fish_seen_subcommand_from up dn" -a "up dn"
+complete -c k8s -n "__fish_seen_subcommand_from completion" -a "bash zsh fish"
+`
+
+export const completionCommand = new Command( 'completion' )
+  .description( 'Emit a shell completion script' )
+  .argument( '<shell>', `shell: ${SHELLS.join( ', ' )}` )
+  .addHelpText( 'after', `
+Examples:
+  eval "$(k8s completion bash)"         # add to ~/.bashrc
+  eval "$(k8s completion zsh)"          # add to ~/.zshrc
+  k8s completion fish | source          # fish interactive session
+` )
+  .action( ( shell ) => {
+    const scripts = { bash: BASH, zsh: ZSH, fish: FISH }
+    if ( !scripts[shell] ) {
+      errorExit( `Unknown shell: ${shell}\nValid: ${SHELLS.join( ', ' )}` )
+    }
+    process.stdout.write( scripts[shell] )
+  } )

package/src/k8s/commands/cryo.mjs

@@ -0,0 +1,164 @@
+import { Command } from 'commander'
+import chalk from 'chalk'
+import { $ } from 'zx'
+
+import { log, errorExit, sleep, warning, proceed, parseJsonFile } from '../../Utils.mjs'
+import { overwhelmGate } from '../lib/gate.mjs'
+
+$.verbose = false
+
+// ── Config ────────────────────────────────────────────────────────────────
+
+const loadConfig = async ( configPath ) => {
+  const config = await parseJsonFile( configPath )
+  if ( !config ) errorExit( `Config file not found: ${configPath}` )
+
+  const { cluster, deploymentPhases, nodePools } = config
+  if ( !cluster?.project || !cluster?.name || !cluster?.region ) {
+    errorExit( 'Config must include cluster.project, cluster.name, and cluster.region' )
+  }
+  if ( !deploymentPhases || !nodePools ) {
+    errorExit( 'Config must include deploymentPhases and nodePools' )
+  }
+  return config
+}
+
+// ── Display ───────────────────────────────────────────────────────────────
+
+const banner = ( text ) => {
+  const bar = '─'.repeat( Math.max( 0, 56 - text.length ) )
+  log( chalk.cyan.bold( `\n── ${text} ${bar}` ) )
+}
+
+// ── kubectl ───────────────────────────────────────────────────────────────
+
+const scaleDeployments = async ( names, replicas, namespace ) =>
+  $`kubectl scale deployment ${names} --replicas=${replicas} -n ${namespace}`
+
+const scaleStatefulSets = async ( names, replicas, namespace ) =>
+  $`kubectl scale statefulset ${names} --replicas=${replicas} -n ${namespace}`
+
+const waitForPodsGone = async ( namespace, timeoutSecs = 300 ) => {
+  log( `\nWaiting for pods to terminate (up to ${timeoutSecs}s)...` )
+  try {
+    await $`kubectl wait pod --all -n ${namespace} --for=delete --timeout=${timeoutSecs}s`
+  } catch {
+    warning( `Some pods still terminating after ${timeoutSecs}s — proceeding.` )
+  }
+}
+
+const waitForNodesReady = async ( timeoutSecs = 600 ) => {
+  log( `\nWaiting for nodes to become Ready (up to ${timeoutSecs}s)...` )
+  await $`kubectl wait node --all --for=condition=Ready --timeout=${timeoutSecs}s`
+}
+
+// ── gcloud ────────────────────────────────────────────────────────────────
+
+const updateAutoscalerMin = async ( pool, cluster, min ) => {
+  log( `  ${pool.name}: autoscaler min → ${min}` )
+  const { name, project, region } = cluster
+  await $`gcloud container node-pools update ${pool.name} --cluster=${name} --project=${project} --region=${region} --enable-autoscaling --min-nodes=${min} --max-nodes=${pool.autoscaler.max} --quiet`
+}
+
+const resizePool = async ( pool, cluster, size ) => {
+  log( `  ${pool.name} → ${size} node(s)` )
+  const { name, project, region } = cluster
+  await $`gcloud container clusters resize ${name} --node-pool=${pool.name} --num-nodes=${size} --project=${project} --region=${region} --quiet`
+}
+
+// ── Hibernate ─────────────────────────────────────────────────────────────
+
+const runHibernate = async ( configPath ) => {
+  const config = await loadConfig( configPath )
+  const {
+    cluster,
+    namespace = 'default',
+    phasePauseMs = 10_000,
+    deploymentPhases,
+    statefulsetPhases = [],
+    nodePools,
+  } = config
+
+  await overwhelmGate()
+  await proceed( `Hibernate ${chalk.yellow.bold( cluster.name )}? All workloads scale to 0 and node pools drain.` )
+
+  for ( const group of deploymentPhases ) {
+    banner( `Deployments: ${group.name}` )
+    await scaleDeployments( group.members, 0, namespace )
+    await sleep( group.pauseMs ?? phasePauseMs )
+  }
+
+  if ( statefulsetPhases.length ) {
+    for ( const group of statefulsetPhases ) {
+      banner( `StatefulSets: ${group.name}` )
+      await scaleStatefulSets( group.members, 0, namespace )
+      await sleep( group.pauseMs ?? phasePauseMs )
+    }
+  } else {
+    // no ordered phases defined yet — scale all StatefulSets together
+    banner( 'StatefulSets' )
+    await $`kubectl scale statefulset --all -n ${namespace} --replicas=0`
+  }
+
+  await waitForPodsGone( namespace )
+
+  const poolsWithAutoscaler = nodePools.filter( p => p.autoscaler )
+  if ( poolsWithAutoscaler.length ) {
+    banner( 'Clearing autoscaler minimums' )
+    for ( const pool of poolsWithAutoscaler ) {
+      await updateAutoscalerMin( pool, cluster, 0 )
+    }
+  }
+
+  banner( 'Draining node pools' )
+  for ( const pool of nodePools ) {
+    await resizePool( pool, cluster, 0 )
+  }
+
+  log( chalk.green( '\nCluster hibernated. Control plane (~$0.10/hr) continues running.' ) )
+  log( `To wake: ${chalk.cyan( `k8s wake ${configPath}` )}` )
+}
+
+// ── Wake ──────────────────────────────────────────────────────────────────
+
+const runWake = async ( configPath ) => {
+  const config = await loadConfig( configPath )
+  const { cluster, nodePools } = config
+
+  await overwhelmGate()
+
+  // Restore autoscaler minimums before resizing so the autoscaler
+  // doesn't immediately scale back down the nodes we're bringing up.
+  const poolsWithAutoscaler = nodePools.filter( p => p.autoscaler )
+  if ( poolsWithAutoscaler.length ) {
+    banner( 'Restoring autoscaler minimums' )
+    for ( const pool of poolsWithAutoscaler ) {
+      await updateAutoscalerMin( pool, cluster, pool.autoscaler.min )
+    }
+  }
+
+  banner( 'Restoring node pools' )
+  for ( const pool of nodePools ) {
+    if ( pool.restoreSize > 0 ) {
+      await resizePool( pool, cluster, pool.restoreSize )
+    } else {
+      log( `  ${pool.name} → skipped (autoscaler brings up on demand)` )
+    }
+  }
+
+  await waitForNodesReady()
+
+  log( chalk.green( '\nNodes ready. Run helmup to restore all services.' ) )
+}
+
+// ── Commands ──────────────────────────────────────────────────────────────
+
+export const hibernateCommand = new Command( 'hibernate' )
+  .description( 'Scale all workloads to zero and drain all node pools' )
+  .argument( '[config]', 'path to cluster-config.json', './cluster-config.json' )
+  .action( async ( config ) => runHibernate( config ).catch( errorExit ) )
+
+export const wakeCommand = new Command( 'wake' )
+  .description( 'Restore node pools from hibernation (then run helmup)' )
+  .argument( '[config]', 'path to cluster-config.json', './cluster-config.json' )
+  .action( async ( config ) => runWake( config ).catch( errorExit ) )