@leverege/build-tools 2.114.0-DEVOP-578.1 → 2.114.0-DEVOP-578.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@leverege/build-tools",
-  "version": "2.114.0-DEVOP-578.1",
+  "version": "2.114.0-DEVOP-578.2",
   "description": "A collection of build / support tools for Leverege developers",
   "main": "index.js",
   "repository": {
@@ -68,6 +68,7 @@
     "service-man": "src/service-man/service-man.mjs",
     "snapshot-cnpg": "src/snapshot-cnpg.mjs",
     "sync-chart-version": "src/sync-chart-version.mjs",
+    "throwup": "src/helmup.sh",
     "upload-model": "src/upload-model.sh",
     "yarn-build": "src/yarn-build.mjs"
   },
@@ -96,7 +97,7 @@
     "commander": "^15.0.0",
     "deepmerge": "^4.3.1",
     "enquirer": "^2.4.1",
-    "execa": "^10.0.0",
+    "execa": "^10.0.1",
     "find-yarn-workspace-root": "^2.0.0",
     "glob": "^13.0.6",
     "googleapis": "^173.0.0",
@@ -125,8 +126,8 @@
     "@leverege/eslint-config-leverege": "^9",
     "chai": "^6.2.2",
     "eslint": "^9",
-    "mocha": "^11.7.6",
-    "npm": "^12.0.1"
+    "mocha": "^11.8.0",
+    "npm": "^12.0.2"
   },
   "allowScripts": {
     "unrs-resolver@1.12.2": true

package/src/bash-funcs

@@ -120,6 +120,18 @@ function getManagedSecret() {
   echo "$SECRET"
 }
 
+# Fail fast when gcloud cannot mint tokens. Downstream tools capture gcloud
+# output, so reauthentication cannot prompt there; surfacing it here lets
+# gcloud reauth interactively on the TTY when policy allows. stdout goes to a
+# literal /dev/null (NOT $DEVNULL, which is /dev/stderr in debug mode) so the
+# access tokens are never printed.
+function errorIfStaleGcloudAuth() {
+  gcloud auth print-access-token > /dev/null
+  exitOnError $? "GCP credentials expired. Please run: `color g 'gcloud auth login'`"
+  gcloud auth application-default print-access-token > /dev/null
+  exitOnError $? "GCP application-default credentials expired. Please run: `color g 'gcloud auth application-default login'`"
+}
+
 function getProjectIdFromK8sContext() {
   local currContext=`kubectl config current-context`
 
@@ -286,7 +298,7 @@ function ifPluggedIn() {
   local PLUGIN=
   HELMUP="helm upgrade --install $SERVICE $CHARTLOC -f $SERVICE/values.yaml $CHARTVER $HELM_WHAT"
   case "$INVOKED_AS" in
-    "helmup"|"helmwhat"|"helmcycle"|"helminit" )
+    "helmup"|"helmwhat"|"helmcycle"|"helminit"|"throwup" )
       PLUGIN="$1/helmup.plugin"
       ;;
     "helmdn" )

package/src/circle-orb-it/circle-npm.yml

@@ -6,12 +6,8 @@ orbs:
 workflows:
   build-and-verify:
     jobs:
-      - leverege/install-dependencies:
-          context: npm
       - leverege/code-analysis:
           context: npm
-          requires:
-            - leverege/install-dependencies
       - leverege/code-coverage:
           context:
             - npm
@@ -19,12 +15,8 @@ workflows:
           coverage-threshold: 70          # percentage threshold for functions, lines, and statements
           enable-redis: false             # set to true if tests require a live Redis connection
           cache-config: '{"type":"inMemory"}' # sets CACHE_CONFIG env var - options: inMemory, redis
-          requires:
-            - leverege/install-dependencies
       - leverege/generate-docs:
           context: npm
-          requires:
-            - leverege/install-dependencies
       - leverege/mirror-to-github:
           context: github-mirror
           # github-team: platform   # available teams: platform, utilities, pitcrew, recovr

package/src/helm-charts/vms-server/values-local.yaml

@@ -0,0 +1,45 @@
+image:
+  tag:
+
+config:
+  LOG_LEVEL: "WARNING"
+  GCP_PROJECT_ID: "${PROJECT_ID}" # needed for python secrets manager?
+  GCP_SECRET_MANAGER_NAMESPACE: "VMS_SERVER"
+  IMAGINE_API_HOST: "http://api-server:8181"
+  IMAGINE_API_PROJECT_ID: "set-in-values-local"
+  IMAGINE_API_SYSTEM_ID: "set-in-values-local"
+  IMAGINE_API_KEY: "set-in-values-local"
+  IMAGINE_API_AUDIENCE: "${PROJECT_ID}-aud"
+  IMAGINE_SERVICE_SYNC_ENABLED: "true"
+  IMAGINE_CAMERA_BLUEPRINT_ID: "set-in-values-local"
+  VMS_SERVER_HOST: "http://vms-server:8080"
+  VIDEO_PLAYBACK_SERVICE_VERSION: "20"
+  VIDEO_MANAGEMENT_SERVICE_VERSION: "2"
+  USE_SHARED_METRICS: "true"
+  VIDEO_SEGMENTS_UPLOAD_TIMEOUT: "60"
+  PLAYLIST_OPEN_ENDED_CACHE_ENABLED: "true"
+  SEGMENT_RECONCILER_ENABLED: "true"
+  SEGMENT_RECONCILER_INTERVAL_SECONDS: "300"
+  SEGMENT_RECONCILER_MIN_AGE_SECONDS: "300"
+  SEGMENT_RECONCILER_LOOKBACK_HOURS: "24"
+
+# run on the preemptible node pool (base chart adds the target-env=preemptible
+# nodeAffinity, toleration and safe-to-evict annotation)
+service:
+  preemptible: true
+
+serviceMonitor:
+  enabled: true
+  interval: "60s"
+  namespace: "default"
+
+resources:
+  requests:
+    cpu: 500m
+    memory: 512Mi
+
+autoscaling:
+  enabled: true
+  minReplicas: 2
+  maxReplicas: 16
+  averageCPU: 75

package/src/helmup.sh

@@ -82,6 +82,9 @@ BAD_IDEA
   exit 1
 fi
 
+# stop here rather than half way through a deploy if the creds cannot mint tokens
+errorIfStaleGcloudAuth
+
 # artifact-registry => https://console.cloud.google.com/artifacts?project=leverege-registry
 REGISTRY_DOMAIN="us-docker.pkg.dev"
 # open container initiative (oci) => https://opencontainers.org/
@@ -92,6 +95,9 @@ gcloud auth application-default print-access-token | helm registry login \
 
 # make sure we have a current npmrc file
 refresh-npm-token
+exitOnError $? "refresh-npm-token failed - the rc files were not refreshed"
+
+HELM_EMOJI=$( [ "$INVOKED_AS" == "throwup" ] && echo "🤮" || echo "⎈" )
 
 # removed geotile-server from standard platform 3/20/22 - low usage
 PLATFORM=(
@@ -100,6 +106,7 @@ PLATFORM=(
   db-curator
   emailer
   imagine
+  llm-prompt-server
   message-processor
   messenger
   resource-server
@@ -122,6 +129,7 @@ AUXILIARY=(
   reason
   transponder-dh
   vin-decoder-server
+  vms-server
 )
 
 # There's a chicken / egg situation that arises with new clusters in that
@@ -622,7 +630,7 @@ HELMUP_BANNER
 function doHelmup() {
   local culprit="$(gcloud config get account)"
   local deployT="$(date -u +'%Y-%m-%dT%H:%M:%SZ')"
-  local doing="Helming => `color g \"$*\"`"
+  local doing="Helming => $HELM_EMOJI `color g \"$*\"` $HELM_EMOJI"
   local from=
   [ ! -z "$FROM_VERSIONS" ] && from="`color y \"<= from Versions.json\"`"
   printf "$doing $from\n"

package/src/k8s/commands/cryo.mjs

@@ -0,0 +1,164 @@
+import { Command } from 'commander'
+import chalk from 'chalk'
+import { $ } from 'zx'
+
+import { log, errorExit, sleep, warning, proceed, parseJsonFile } from '../../Utils.mjs'
+import { overwhelmGate } from '../lib/gate.mjs'
+
+$.verbose = false
+
+// ── Config ────────────────────────────────────────────────────────────────
+
+const loadConfig = async ( configPath ) => {
+  const config = await parseJsonFile( configPath )
+  if ( !config ) errorExit( `Config file not found: ${configPath}` )
+
+  const { cluster, deploymentPhases, nodePools } = config
+  if ( !cluster?.project || !cluster?.name || !cluster?.region ) {
+    errorExit( 'Config must include cluster.project, cluster.name, and cluster.region' )
+  }
+  if ( !deploymentPhases || !nodePools ) {
+    errorExit( 'Config must include deploymentPhases and nodePools' )
+  }
+  return config
+}
+
+// ── Display ───────────────────────────────────────────────────────────────
+
+const banner = ( text ) => {
+  const bar = '─'.repeat( Math.max( 0, 56 - text.length ) )
+  log( chalk.cyan.bold( `\n── ${text} ${bar}` ) )
+}
+
+// ── kubectl ───────────────────────────────────────────────────────────────
+
+const scaleDeployments = async ( names, replicas, namespace ) =>
+  $`kubectl scale deployment ${names} --replicas=${replicas} -n ${namespace}`
+
+const scaleStatefulSets = async ( names, replicas, namespace ) =>
+  $`kubectl scale statefulset ${names} --replicas=${replicas} -n ${namespace}`
+
+const waitForPodsGone = async ( namespace, timeoutSecs = 300 ) => {
+  log( `\nWaiting for pods to terminate (up to ${timeoutSecs}s)...` )
+  try {
+    await $`kubectl wait pod --all -n ${namespace} --for=delete --timeout=${timeoutSecs}s`
+  } catch {
+    warning( `Some pods still terminating after ${timeoutSecs}s — proceeding.` )
+  }
+}
+
+const waitForNodesReady = async ( timeoutSecs = 600 ) => {
+  log( `\nWaiting for nodes to become Ready (up to ${timeoutSecs}s)...` )
+  await $`kubectl wait node --all --for=condition=Ready --timeout=${timeoutSecs}s`
+}
+
+// ── gcloud ────────────────────────────────────────────────────────────────
+
+const updateAutoscalerMin = async ( pool, cluster, min ) => {
+  log( `  ${pool.name}: autoscaler min → ${min}` )
+  const { name, project, region } = cluster
+  await $`gcloud container node-pools update ${pool.name} --cluster=${name} --project=${project} --region=${region} --enable-autoscaling --min-nodes=${min} --max-nodes=${pool.autoscaler.max} --quiet`
+}
+
+const resizePool = async ( pool, cluster, size ) => {
+  log( `  ${pool.name} → ${size} node(s)` )
+  const { name, project, region } = cluster
+  await $`gcloud container clusters resize ${name} --node-pool=${pool.name} --num-nodes=${size} --project=${project} --region=${region} --quiet`
+}
+
+// ── Hibernate ─────────────────────────────────────────────────────────────
+
+const runHibernate = async ( configPath ) => {
+  const config = await loadConfig( configPath )
+  const {
+    cluster,
+    namespace = 'default',
+    phasePauseMs = 10_000,
+    deploymentPhases,
+    statefulsetPhases = [],
+    nodePools,
+  } = config
+
+  await overwhelmGate()
+  await proceed( `Hibernate ${chalk.yellow.bold( cluster.name )}? All workloads scale to 0 and node pools drain.` )
+
+  for ( const group of deploymentPhases ) {
+    banner( `Deployments: ${group.name}` )
+    await scaleDeployments( group.members, 0, namespace )
+    await sleep( group.pauseMs ?? phasePauseMs )
+  }
+
+  if ( statefulsetPhases.length ) {
+    for ( const group of statefulsetPhases ) {
+      banner( `StatefulSets: ${group.name}` )
+      await scaleStatefulSets( group.members, 0, namespace )
+      await sleep( group.pauseMs ?? phasePauseMs )
+    }
+  } else {
+    // no ordered phases defined yet — scale all StatefulSets together
+    banner( 'StatefulSets' )
+    await $`kubectl scale statefulset --all -n ${namespace} --replicas=0`
+  }
+
+  await waitForPodsGone( namespace )
+
+  const poolsWithAutoscaler = nodePools.filter( p => p.autoscaler )
+  if ( poolsWithAutoscaler.length ) {
+    banner( 'Clearing autoscaler minimums' )
+    for ( const pool of poolsWithAutoscaler ) {
+      await updateAutoscalerMin( pool, cluster, 0 )
+    }
+  }
+
+  banner( 'Draining node pools' )
+  for ( const pool of nodePools ) {
+    await resizePool( pool, cluster, 0 )
+  }
+
+  log( chalk.green( '\nCluster hibernated. Control plane (~$0.10/hr) continues running.' ) )
+  log( `To wake: ${chalk.cyan( `k8s wake ${configPath}` )}` )
+}
+
+// ── Wake ──────────────────────────────────────────────────────────────────
+
+const runWake = async ( configPath ) => {
+  const config = await loadConfig( configPath )
+  const { cluster, nodePools } = config
+
+  await overwhelmGate()
+
+  // Restore autoscaler minimums before resizing so the autoscaler
+  // doesn't immediately scale back down the nodes we're bringing up.
+  const poolsWithAutoscaler = nodePools.filter( p => p.autoscaler )
+  if ( poolsWithAutoscaler.length ) {
+    banner( 'Restoring autoscaler minimums' )
+    for ( const pool of poolsWithAutoscaler ) {
+      await updateAutoscalerMin( pool, cluster, pool.autoscaler.min )
+    }
+  }
+
+  banner( 'Restoring node pools' )
+  for ( const pool of nodePools ) {
+    if ( pool.restoreSize > 0 ) {
+      await resizePool( pool, cluster, pool.restoreSize )
+    } else {
+      log( `  ${pool.name} → skipped (autoscaler brings up on demand)` )
+    }
+  }
+
+  await waitForNodesReady()
+
+  log( chalk.green( '\nNodes ready. Run helmup to restore all services.' ) )
+}
+
+// ── Commands ──────────────────────────────────────────────────────────────
+
+export const hibernateCommand = new Command( 'hibernate' )
+  .description( 'Scale all workloads to zero and drain all node pools' )
+  .argument( '[config]', 'path to cluster-config.json', './cluster-config.json' )
+  .action( async ( config ) => runHibernate( config ).catch( errorExit ) )
+
+export const wakeCommand = new Command( 'wake' )
+  .description( 'Restore node pools from hibernation (then run helmup)' )
+  .argument( '[config]', 'path to cluster-config.json', './cluster-config.json' )
+  .action( async ( config ) => runWake( config ).catch( errorExit ) )

package/src/k8s/k8s.mjs

@@ -8,6 +8,7 @@ import { scaleCommand } from './commands/scale.mjs'
 import { rollCommand } from './commands/roll.mjs'
 import { logCommand } from './commands/log.mjs'
 import { completionCommand } from './commands/completion.mjs'
+import { hibernateCommand, wakeCommand } from './commands/cryo.mjs'
 
 $.verbose = false
 
@@ -20,4 +21,6 @@ program
   .addCommand( rollCommand )
   .addCommand( logCommand )
   .addCommand( completionCommand )
+  .addCommand( hibernateCommand )
+  .addCommand( wakeCommand )
   .parse()

package/src/service-man/config/service-accounts/vms-server.json

@@ -0,0 +1,8 @@
+{
+  "roles" : [
+    "leveregeBigQueryReader",
+    "leveregePubSub",
+    "leveregeSecretsViewer",
+    "leveregeStorageObjectAdmin"
+  ]
+}