@leverege/build-tools 2.94.0 → 2.96.0-quinn.1

package/.vscode/launch.json

@@ -17,6 +17,20 @@
       "env": {
       },
       "outputCapture": "std"
+    },
+    {
+      "name": "service-man (launch)",
+      "type": "node",
+      "request": "launch",
+      "program": "${workspaceFolder}/src/service-man/service-man.mjs",
+      "args": ["--dump-configs","dbg-cfgs"],
+      "cwd": "/Users/john/Work/leverege/k8s/tst-builder",
+      "runtimeExecutable": "node",
+      "console": "integratedTerminal",
+      "skipFiles": ["<node_internals>/**"],
+      "env": {
+        "NODE_ENV": "development"
+      }
     }
   ]
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@leverege/build-tools",
-  "version": "2.94.0",
+  "version": "2.96.0-quinn.1",
   "description": "A collection of build / support tools for Leverege developers",
   "main": "index.js",
   "repository": {
@@ -40,6 +40,7 @@
     "getjson": "src/getjson.mjs",
     "git-tar": "src/git-tar.sh",
     "helm-audit": "src/helm-audit.mjs",
+    "helmboot": "src/helmup.sh",
     "helmcycle": "src/helmup.sh",
     "helmdn": "src/helmdn.sh",
     "helminit": "src/helmup.sh",
@@ -64,6 +65,7 @@
     "refresh-npm-token": "src/refresh-npm-token.mjs",
     "refresh-py-idx": "src/refresh-py-idx.sh",
     "send-to-slack": "src/send-to-slack.mjs",
+    "service-man": "src/service-man/service-man.mjs",
     "snapshot-cnpg": "src/snapshot-cnpg.mjs",
     "tag-release": "src/tag-release.mjs",
     "unleash": "src/unleash.mjs",
@@ -99,12 +101,13 @@
     "simple-git": "^3.30.0",
     "sloc": "^0.3.2",
     "superstruct": "^2.0.2",
+    "tmp": "^0.2.5",
     "toml": "^3.0.0",
     "zx": "^8.8.5"
   },
   "devDependencies": {
     "@leverege/eslint-config-leverege": "^5.1.1",
-    "chai": "^6.2.1",
+    "chai": "^6.2.2",
     "mocha": "^11.7.5",
     "npm": "^11.7.0"
   }

package/src/Docker.mjs

@@ -1,3 +1,4 @@
+/* eslint-disable security/detect-non-literal-fs-filename */
 import fs from 'node:fs'
 import path from 'node:path'
 import url from 'node:url'
@@ -23,7 +24,7 @@ const DIRNAME = path.dirname( url.fileURLToPath( import.meta.url ) )
 //
 // See the latest docker images here => https://hub.docker.com/_/node
 //
-const dockerfileNodeVersion = process.env.DOCKER_BUILD_NODE_VERSION || '22.21.1-alpine3.23' // TODO: 1/1/26 => '24.12.0-alpine3.23'
+const dockerfileNodeVersion = process.env.DOCKER_BUILD_NODE_VERSION || '24.12.0-alpine3.23'
 const dockerfileNpmVersion = process.env.DOCKER_BUILD_NPM_VERSION || '11.6.0'
 
 // The contents of these variables were initially located in files that live in
@@ -83,7 +84,7 @@ const generateDockerfile = async ( options ) => {
   const gitReference = await shellCmd( 'git rev-parse --short HEAD' )
   const settings = { gitReference, ...defaultSettings, ...options }
   const dockerTemplate = options.isNodeJsProject ? 'dockerfileTemplateNodeJs' : 'dockerfileTemplatePython'
-  const dockerfileTemplate = fs.readFileSync( path.join( DIRNAME, `./templates/${dockerTemplate}.hbs` ), 'utf8' ) // eslint-disable-line security/detect-non-literal-fs-filename
+  const dockerfileTemplate = fs.readFileSync( path.join( DIRNAME, `./templates/${dockerTemplate}.hbs` ), 'utf8' )
   
   // defaults to the current "official" version of node unless it gets
   // overridden in the leverege.nodeimg package.json stanza

package/src/Utils.mjs

@@ -39,11 +39,42 @@ export const warning = ( warningText ) => {
   console.error( `\n${chalk.yellow.bold( fullWarning )}` )
 }
 
-export const errorExit = ( error, opts = {} ) => {
-  const { errorCode = 1 } = opts
-  console.error( `\n${chalk.red.bold( error )}\n` )
-  process.exit( errorCode )
+export const errorExit = ( arg, opts = {} ) => {
+  const exitCode =
+    typeof opts.exitCode === 'number' ? opts.exitCode :
+      ( arg && typeof arg.exitCode === 'number' ) ? arg.exitCode : 1
+
+  // lets figure out what to print
+  let message
+  let details
+
+  if ( arg instanceof Error ) {
+    message = arg.message || 'Error'
+    details = arg.stack
+  } else if ( typeof arg === 'string' ) {
+    message = arg
+  } else if ( arg && typeof arg === 'object' ) {
+    message = 'Error'
+    details = JSON.stringify( arg, null, 2 )
+  } else if ( arg != null ) {
+    message = String( arg )
+  }
+
+  console.error( `\n${chalk.red.bold( message )}\n` )
+  if ( details && details !== message ) {
+    console.error( `${details}\n` )
+  }
+
+  process.exit( exitCode )
 }
+
+/**
+ * Clears the screen and optionally homes the cursor.
+ */
+export const clear = ( home = true ) => {
+  log( `${home ? '\x1B[0f' : ''}\x1B[2J` )
+}
+
 /* eslint-enable no-console */
 
 export const isHex = ( hex ) => {
@@ -54,6 +85,22 @@ export const isString = ( s ) => {
   return typeof s === 'string' || s instanceof String
 }
 
+/**
+  *  * Sets first char to Upper
+  *   */
+export const capitalize = ( norm ) => {
+  return norm.charAt( 0 ).toUpperCase() + norm.slice( 1 )
+}
+
+/**
+ * A simple mS sleep delay.
+ */
+export const sleep = ( ms = 0 ) => {
+  return new Promise( ( resolve ) => {
+    setTimeout( resolve, ms )
+  } )
+}
+
 /**
  * General "yes" to proceed function.
  */
@@ -226,7 +273,7 @@ export const gitRepoIsDirty = async () => {
 
 export const parseYamlFile = async ( filePath ) => {
   try {
-    const fileContent = await fs.promises.readFile( filePath, 'utf8' )
+    const fileContent = await fs.promises.readFile( filePath, 'utf8' ) /* eslint-disable-line security/detect-non-literal-fs-filename */
     return YAML.load( fileContent )
   } catch ( error ) {
     throw new Error( `Failed to parse YAML file ${filePath}: ${error.message}` )
@@ -538,7 +585,7 @@ export const getRepositoryDescr = async () => {
   const helmChart = await parseHelmChart()
   const rootPackageJson = await parseJsonFile( `${gitRoot}/package.json` )
   const rootPyProjectToml = await parsePyProjectToml( `${gitRoot}/pyproject.toml` )
-  const isYarnProject = fs.existsSync( path.join( gitRoot, 'yarn.lock' ) ) || fs.existsSync( path.join( gitRoot, '.yarn' ) )
+  const isYarnProject = getIsYarnProject( gitRoot )
 
   return {
     currentDir : process.cwd(),
@@ -683,3 +730,30 @@ export const analyzeRepository = async ( giveGuidance ) => {
     isYarnProject
   }
 }
+
+export function isDir( path ) {
+  // eslint-disable-next-line security/detect-non-literal-fs-filename
+  const exists = fs.existsSync( path )
+
+  // eslint-disable-next-line security/detect-non-literal-fs-filename
+  return exists && fs.statSync( path ).isDirectory()
+}
+
+export function getSecretsDir() {
+  let secretsDir
+  
+  if ( process.env.SECRETS_DIR ) {
+    secretsDir = process.env.SECRETS_DIR
+  } else if ( isDir( path.join( process.cwd(), 'env' ) ) ) {
+    secretsDir = 'env'
+  } else {
+    secretsDir = 'secrets'
+  }
+
+  return secretsDir
+}
+
+export function getIsYarnProject( gitRoot ) {
+  // eslint-disable-next-line security/detect-non-literal-fs-filename
+  return fs.existsSync( path.join( gitRoot, 'yarn.lock' ) ) || isDir( path.join( gitRoot, '.yarn' ) )
+}

package/src/artifacts/Utils.mjs

@@ -1,6 +1,6 @@
 import fs from 'node:fs'
 import path from 'node:path'
-import crypto from 'crypto'
+import crypto from 'node:crypto'
 
 import { validate } from 'superstruct'
 

package/src/bash-funcs

@@ -322,4 +322,4 @@ function commandChecker() {
   done
 }
 
-commandChecker git helm kubectl service-man
+commandChecker git helm jq kubectl service-man

package/src/firebaseDeploy.mjs

@@ -4,6 +4,8 @@
 import pkg from 'enquirer'
 import { $, fs, argv, path, chalk } from 'zx'
 
+import { getGitRootDirectory, getIsYarnProject, getSecretsDir } from './Utils.mjs'
+
 const { prompt } = pkg
 
 // This will preserve coloration from spawned child processes
@@ -34,8 +36,9 @@ const INTERACTIVE_MODE = !TARGET
 
 let exited = false
 
-const SECRETS_DIR = 'secrets'
+const SECRETS_DIR = await getSecretsDir()
 const EXIT_EVENTS = [ 'SIGINT', 'exit', 'uncaughException', 'unhandledRejection' ]
+const isYarnProject = getIsYarnProject( await getGitRootDirectory() )
 
 const [
   firebaserc,
@@ -211,7 +214,10 @@ try {
   const deployType = CHANNEL ? `hosting:channel:deploy ${CHANNEL.name} ${CHANNEL.expiration ? `--expires ${CHANNEL.expiration}` : ''}`.split( ' ' ) : 'deploy'
   const hostingPrefix = CHANNEL ? '' : 'hosting:'
 
-  await $( { ac } )`npm run clean && DOTENV_CONFIG_PATH=${path.join( process.cwd(), '.env.temp' )} DEPLOYMENT_TARGET=${TARGET.siteId} npm run build && firebase use ${TARGET.projectId} && firebase ${deployType} --only ${hostingPrefix}${TARGET.siteId}`
+  const cleanCommand = isYarnProject ? 'yarn clean' : 'npm run clean'
+  const buildCommand = isYarnProject ? 'yarn build' : 'npm run build'
+
+  await $( { ac } )`${cleanCommand} && DOTENV_CONFIG_PATH=${path.join( process.cwd(), '.env.temp' )} DEPLOYMENT_TARGET=${TARGET.siteId} ${buildCommand} && firebase use ${TARGET.projectId} && firebase ${deployType} --only ${hostingPrefix}${TARGET.siteId}`
 } catch ( err ) {
   console.error( err )
 }

package/src/firebaseServe.mjs

@@ -4,6 +4,8 @@
 import pkg from 'enquirer'
 import { $, fs, argv, path, chalk } from 'zx'
 
+import { getGitRootDirectory, getIsYarnProject, getSecretsDir } from './Utils.mjs'
+
 const { prompt } = pkg
 
 // This will preserve coloration from spawned child processes
@@ -13,9 +15,12 @@ process.env.FORCE_COLOR = '1'
 let TARGET = argv._[0]
 let exited = false
 
-const SECRETS_DIR = 'secrets'
 const EXIT_EVENTS = [ 'SIGINT', 'exit', 'uncaughtException', 'unhandledRejection' ]
 
+const SECRETS_DIR = await getSecretsDir()
+
+const isYarnProject = getIsYarnProject( await getGitRootDirectory() )
+
 const [
   firebaserc,
   firebaseJson,
@@ -142,7 +147,8 @@ EXIT_EVENTS
 await fs.writeFile( path.join( process.cwd(), '.env.temp' ), envFileContent )
 
 try {
-  await $( { ac } )`DOTENV_CONFIG_PATH=${path.join( process.cwd(), '.env.temp' )} DEPLOYMENT_TARGET=${TARGET.siteId} npm run serve --colors`
+  const serveCommand = isYarnProject ? 'yarn serve' : 'npm run serve'
+  await $( { ac } )`DOTENV_CONFIG_PATH=${path.join( process.cwd(), '.env.temp' )} DEPLOYMENT_TARGET=${TARGET.siteId} ${serveCommand} --colors`
 } catch ( err ) {
   console.error( err )
 }

package/src/git-tar.sh

@@ -3,7 +3,7 @@
 # Load the standard helper functions
 . `build-tools --bashfun`
 
-# If it isn't already set use the default ~/work/.gitar
+# If it isn't already set use the default ~/.gitar
 #
 [ -z $GITAR ] && GITAR="${HOME}/.gitar"
 

package/src/helm-charts/emailer/values-local.yaml

@@ -1,12 +1,5 @@
 image:
-  tag: # v3.7.0
+  tag: # v5.1.0
 
 config:
   LOG_CONFIG: '{"type":"pino","level":"warn"}'
-
-  API_UI_URL: "${PROJECT_ID}.web.app"
-  MAIL_GUN_DOMAIN: "${MAIL_GUN_DOMAIN}"
-  MAIL_GUN_DEFAULT_SENDER: "no-reply@${MAIL_GUN_DOMAIN}"
-
-serviceMonitor:
-  enabled: true

package/src/helm-charts/prom-operator/helmup.plugin

@@ -8,7 +8,7 @@ showInstalling "The Prometheus Operator (kube-prometheus-stack)"
 
 # https://artifacthub.io/packages/helm/prometheus-community/kube-prometheus-stack
 OCI_CHART="oci://ghcr.io/prometheus-community/charts/kube-prometheus-stack"
-[ -z "$PROMETHEUS_STACK_CHART_VERSION" ] && PROMETHEUS_STACK_CHART_VERSION="80.4.1"
+[ -z "$PROMETHEUS_STACK_CHART_VERSION" ] && PROMETHEUS_STACK_CHART_VERSION="80.11.0"
 helm upgrade $NS --install prometheus-stack $OCI_CHART \
                  --values  prom-operator/prometheus-stack.yaml \
                  --version $PROMETHEUS_STACK_CHART_VERSION $HELM_WHAT

package/src/helm-charts/traefik/helmup.plugin

@@ -17,7 +17,7 @@ TRAEFIK_NAMESPACE="traefik"
 
 addHelmRepo traefik https://helm.traefik.io/traefik
 
-[ -z "$TRAEFIK_CHART_VERSION" ] && TRAEFIK_CHART_VERSION="37"
+[ -z "$TRAEFIK_CHART_VERSION" ] && TRAEFIK_CHART_VERSION="38"
 helm upgrade --install traefik traefik/traefik \
              --namespace $TRAEFIK_NAMESPACE --create-namespace \
              --values  traefik/values.yaml \

package/src/helm-charts/velero/README.md

@@ -0,0 +1,135 @@
+# Velero Deployment Notes
+
+## Why this document exists
+
+This document explains **why we had to change the Velero deployment process** and how we now safely manage **CRDs and Helm upgrades** without relying on Bitnami images or brittle Helm hook jobs.
+
+These changes were driven by:
+- Removal / breakage of Bitnami `kubectl` images referenced by the `vmware-tanzu/velero` Helm chart
+- Broadcom’s changes to Bitnami image access and licensing
+- The need for **deterministic, version-safe CRD management** on new and existing GKE clusters
+
+---
+
+## Background: What broke
+
+The Velero Helm chart (`vmware-tanzu/velero` >= 11.x) includes a Helm **hook job** that attempts to install or upgrade CRDs using an initContainer based on:
+
+```
+docker.io/bitnamilegacy/kubectl:<k8s-version>
+```
+
+On Kubernetes 1.34+ clusters, this image **no longer exists**, causing installs to fail with errors such as:
+
+```
+ErrImagePull: bitnamilegacy/kubectl:1.34 not found
+```
+
+Attempts to substitute other kubectl images failed due to hard assumptions in the hook job (e.g. `/tmp/sh`, shell paths, filesystem layout).
+
+Because this hook runs **before Helm considers the release successful**, it blocks both fresh installs and upgrades.
+
+---
+
+## Why we disabled Helm-based CRD upgrades
+
+Helm does not natively manage CRD upgrades well:
+- CRDs in `crds/` directories are only installed once
+- Schema upgrades require hooks or out-of-band logic
+- Hook jobs are fragile and opaque when they fail
+
+The Velero chart’s CRD hook job is effectively doing:
+
+```
+velero install --crds-only | kubectl apply
+```
+
+…but wrapped in a container that is now broken due to image dependencies.
+
+**Conclusion:** We disable the Helm CRD hook entirely and manage CRDs explicitly.
+
+---
+
+## Our current approach (high-level)
+
+1. **CRDs are managed explicitly**, outside Helm
+2. **CRDs are generated from the exact Velero version being deployed**
+3. Helm is used only for deploying/upgrading the Velero controller and configuration
+4. Helm CRD hooks are disabled (`upgradeCRDs=false`)
+
+This makes installs:
+- Reproducible
+- Debuggable
+- Independent of third-party image publishing decisions
+
+---
+
+## Version alignment is critical
+
+The command:
+
+```bash
+velero install --crds-only --dry-run -o yaml
+```
+
+**uses the local `velero` CLI version** to generate CRDs.
+
+If the CLI version does not match the Velero server version deployed by Helm, you can:
+- Apply incompatible CRD schemas
+- Introduce validation failures on existing custom resources
+
+### Our safeguard
+
+`helmup.plugin` now:
+- Reads the chart `appVersion` directly from Helm
+- Verifies the local `velero` CLI version matches exactly
+- Fails fast if versions are mismatched
+
+This guarantees CRDs and controllers stay in lockstep.
+
+---
+
+## When CRDs are applied
+
+CRDs are **not applied on every deploy**.
+
+They are applied only when:
+- Bootstrapping a new cluster
+- Upgrading the Velero chart to a version with a new `appVersion`
+
+This is controlled explicitly via:
+
+```bash
+VELERO_APPLY_CRDS=1 helmup velero
+```
+
+Normal configuration-only updates do **not** touch CRDs.
+
+---
+
+## Summary of changes
+
+| Area | Old behavior | New behavior |
+|-----|-------------|-------------|
+| CRD management | Helm hook job | Explicit, pinned CRD apply |
+| kubectl dependency | Bitnami legacy image | None |
+| Version safety | Implicit | Enforced |
+| Failure mode | Late, opaque | Early, explicit |
+| New cluster installs | Unreliable | Deterministic |
+
+---
+
+## Key takeaways
+
+- **CRDs are part of the API, not application config** — they must be versioned and applied intentionally
+- Helm hooks are not a reliable mechanism for critical cluster-scoped resources
+- Pinning tool versions eliminates an entire class of upgrade failures
+- This approach aligns with GitOps and production-safe Kubernetes practices
+
+---
+
+## References
+
+- Velero documentation: https://velero.io/docs/
+- Velero Helm charts: https://github.com/vmware-tanzu/helm-charts
+- Kubernetes CRD management: https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/

package/src/helm-charts/velero/helmup.bootstrap

@@ -4,6 +4,7 @@
 function installVeleroEnvironment() {
   # Follows instructions => https://github.com/vmware-tanzu/velero-plugin-for-gcp
   showInstalling "Velero Backup Management"
+
   createNamespaceIfNeeded velero
 
   # Do we need to => gcloud auth application-default

package/src/helm-charts/velero/helmup.plugin

@@ -2,10 +2,36 @@
 #
 showInstalling "Velero Backup System"
 
+# We use a specific version for the velero chart because we need to control
+# the version of the locally installed velero client application.
+#
+VELERO_CHART_VERSION="11.3.1"
+EXPECTED_CLIENT_VERSION="v1.17.1"
+
 addHelmRepo vmware-tanzu https://vmware-tanzu.github.io/helm-charts
 
-[ -z "$VELERO_CHART_VERSION" ] && VELERO_CHART_VERSION="11"
+commandChecker velero # verify the velero client is installed
+
+local client_version="$(velero version --client-only 2>/dev/null | awk -F': ' '/Version:/ {print $2; exit}')"
+
+if [[ "$client_version" != "$EXPECTED_CLIENT_VERSION" ]]; then
+  cat<<WRONG_VELERO_APP_VERSION
+
+  ERROR: velero CLI version mismatch.
+    Expected: `color g "${EXPECTED_CLIENT_VERSION}"`
+    Found:    `color r "${client_version}"`
+  
+  Fix => `color y "install velero ${EXPECTED_CLIENT_VERSION}"`
+WRONG_VELERO_APP_VERSION
+  exit 1
+fi
+
+# If the CRDs need to be updated set VELERO_APPLY_CRDS!
 #
+if [[ "${VELERO_APPLY_CRDS:-0}" == "1" ]]; then
+  velero install --crds-only --dry-run -o yaml | kubectl apply -f - --validate=false
+fi
+
 # Build the bucket, region and SA email variables and use --set to mod the
 # chart values as opposed to using an ${label} approach. This eliminates
 # the need for an overwhelm generated values.yaml file which also contains
@@ -13,7 +39,7 @@ addHelmRepo vmware-tanzu https://vmware-tanzu.github.io/helm-charts
 #
 # Note: GCP_PROJECT_ID is passed down from helmup's environment.
 #
-GCE_BUCKET="$GCP_PROJECT_ID-velero-$VELERO_CHART_VERSION"
+GCE_BUCKET="$GCP_PROJECT_ID-velero-${VELERO_CHART_VERSION%%.*}"
 GCE_REGION=`overwhelm -k GCE_REGION`
 GCE_SA_EMAIL="velero@${GCP_PROJECT_ID}.iam.gserviceaccount.com"
 
@@ -27,6 +53,7 @@ helm upgrade \
      --namespace velero --create-namespace \
      --values velero/values-local.yaml \
      --version $VELERO_CHART_VERSION $HELM_WHAT \
+     --set upgradeCRDs=false \
      --set configuration.backupStorageLocation[0].bucket="$GCE_BUCKET" \
      --set configuration.backupStorageLocation[0].config.serviceAccount="$GCE_SA_EMAIL" \
      --set configuration.volumeSnapshotLocation[0].config.snapshotLocation="$GCE_REGION"

package/src/helm-charts/velero/values-local.yaml

@@ -137,7 +137,7 @@ nodeSelector: {}
 # See => https://github.com/vmware-tanzu/velero-plugin-for-gcp
 initContainers:
   - name: velero-plugin-for-gcp
-    image: velero/velero-plugin-for-gcp:v1.10.0
+    image: velero/velero-plugin-for-gcp:v1.13.1
     volumeMounts:
       - mountPath: /target
         name: plugins