@leverege/build-tools 2.101.0 → 2.101.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@leverege/build-tools",
-  "version": "2.101.0",
+  "version": "2.101.2",
   "description": "A collection of build / support tools for Leverege developers",
   "main": "index.js",
   "repository": {
@@ -48,7 +48,7 @@
     "helmup": "src/helmup.sh",
     "helmwhat": "src/helmup.sh",
     "hook-and-release": "src/hook-and-release.mjs",
-    "init-my-chart": "src/init-my-chart.mjs",
+    "init-my-chart": "src/init-my-chart/init-my-chart.mjs",
     "k8scale": "src/k8scale.sh",
     "k8x": "src/k8x.sh",
     "klog": "src/klog.sh",
@@ -94,12 +94,12 @@
     "googleapis": "^171.4.0",
     "handlebars": "^4.7.9",
     "ignore": "^7.0.5",
-    "inquirer": "^13.4.1",
+    "inquirer": "^13.4.2",
     "js-yaml": "^4.1.1",
     "jsdoc": "^4.0.5",
     "ms": "^2.1.3",
     "npm-registry-fetch": "^19.1.1",
-    "ora": "^9.3.0",
+    "ora": "^9.4.0",
     "p-limit": "^7.3.0",
     "package-up": "^5.0.0",
     "readline-sync": "^1.4.10",
@@ -117,6 +117,6 @@
     "@leverege/eslint-config-leverege": "^5.1.2",
     "chai": "^6.2.2",
     "mocha": "^11.7.5",
-    "npm": "^11.12.1"
+    "npm": "^11.13.0"
   }
 }

package/src/Utils.mjs

@@ -560,6 +560,11 @@ export const parseHelmChart = async ( helmroot = './helm' ) => {
     } )
     
     depsOutOfSync = mismatches.length > 0
+
+    // One final check - if the helm/charts dir is missing then we also need to update dependencies
+    if ( hasDependencies && !chartFiles.includes( 'charts' ) ) {
+      depsOutOfSync = true
+    }
   }
   /* eslint-enable security/detect-non-literal-fs-filename */
 

package/src/bash-funcs

@@ -229,26 +229,31 @@ function waitForIAM() {
   done
 }
 
-# *** DEPRECATED *** REMOVE AFTER BITNAMI ERADICATED ***
-# *** DEPRECATED *** REMOVE AFTER BITNAMI ERADICATED ***
-# *** DEPRECATED *** REMOVE AFTER BITNAMI ERADICATED ***
-#
-function addBitnamiRepo() {
-  warning '*** the addBitnamiRepo function should be removed - DEPRECATED ***'
-  if [ ! -z "$BITNAMI_LATEST" ] || [ "$1" == "latest" ];
-  then
-    BITNAMI_URL="https://charts.bitnami.com/bitnami"
-  else
-    cat<<CATNAMI
-    Using the pre 2022 legacy charts from bitnami. To use latest run:
-
-        `color y "BITNAMI_LATEST=1 $HELMUP_ARGS"`
-
-CATNAMI
-    BITNAMI_URL="https://raw.githubusercontent.com/bitnami/charts/eb5f9a9513d987b519f0ecd732e7031241c50328/bitnami"
-  fi
+# waitForSAIAM SVC_ACCT PROJECT ROLE MEMBER [TIMEOUT_SECONDS]
+# Like waitForIAM but checks the SA's own IAM policy (for bindings like workloadIdentityUser
+# that are set via `gcloud iam service-accounts add-iam-policy-binding`, not project-level).
+function waitForSAIAM() {
+  local svcAcct="$1" project="$2" role="$3" member="$4" timeout="${5:-120}"
+  local svcAcctEmail="${svcAcct}@${project}.iam.gserviceaccount.com"
+  local start=$(date +%s) delay=2
 
-  addHelmRepo bitnami $BITNAMI_URL
+  echo "⏳ waiting for SA IAM binding ${role} → ${member} on ${svcAcctEmail} (timeout ${timeout}s)..." >&2
+  while true; do
+    local found
+    found="$(gcloud iam service-accounts get-iam-policy "${svcAcctEmail}" \
+              --flatten="bindings[].members" \
+              --filter="bindings.role=${role} AND bindings.members=${member}" \
+              --format="value(bindings.role)" 2> "$DEVNULL")"
+    if [[ -n "$found" ]]; then
+      echo "✅ SA IAM binding active: ${role} → ${member}" >&2
+      return 0
+    fi
+    if (( $(date +%s) - start >= timeout )); then
+      echo "❌ timed out waiting for SA IAM binding ${role} → ${member}" >&2
+      return 1
+    fi
+    sleep "$delay"; (( delay = delay < 20 ? delay * 2 : 20 ))
+  done
 }
 
 function removeHelmRepo(){

package/src/helm-charts/cnpg-operator/helmup.bootstrap

@@ -26,34 +26,33 @@ function installCnpgOperatorEnvironment() {
   #
   # kubectl delete serviceaccounts -n $CNPG_NAMESPACE $CNPG_SVC_ACCT &> $DEVNULL
   # gcloud --quiet iam service-accounts delete $CNPG_SVC_EMAIL  --project $GCP_PROJECT_ID&> $DEVNULL
-  kubectl describe serviceaccounts -n $CNPG_NAMESPACE $CNPG_SVC_ACCT &> $DEVNULL
+
+  createNamespaceIfNeeded $CNPG_NAMESPACE
+
+  # Guard against accidental SA deletion on re-runs. Check the GCP SA (not the
+  # k8s SA) — consistent with ESO bootstrap pattern and correct for new clusters
+  # where the GCP SA exists but the k8s SA does not yet.
+  gcloud iam service-accounts describe $CNPG_SVC_EMAIL --project $GCP_PROJECT_ID &> $DEVNULL
   if [ $? -eq 0 ];
   then
     cat<<SKIP_CNPG_SA_CREATION
 
-  $YELO_WARN The CNPG SA $CNPG_SVC_ACCT already exists
+  $YELO_WARN The CNPG SA `color y $CNPG_SVC_ACCT` already exists - skipping creation
 
-  Deleting the SA for the purposes of upgrading the operator is going to
-  wreck someone's day (probably yours) if the desire is to simply upgrade
-  in place. To force a clean reinstall, delete both the k8s and gcp SA for
-  CNPG and try again. (see comments in helmup for more info)
+  To force a clean reinstall, delete both the GCP and k8s SAs for CNPG and
+  re-run helmup cnpg-operator.
 
 SKIP_CNPG_SA_CREATION
-    return
+  else
+    printf "\nCreating the gcloud `color g $CNPG_SVC_ACCT` service account (SA)\n"
+    gcloud iam service-accounts create $CNPG_SVC_ACCT \
+      --project "$GCP_PROJECT_ID" \
+      --description "CNPG Operands SA" \
+      --display-name "CNPG Operands SA" &> $DEVNULL
+    exitOnError $? "Failed to create GCP SA `color y $CNPG_SVC_ACCT`"
+    waitForSA "$CNPG_SVC_ACCT" "$GCP_PROJECT_ID" 60
   fi
 
-  # Create the operands namespace and annotate to allow redis access
-  createNamespaceIfNeeded $CNPG_NAMESPACE
-
-  # Create GCP SA and bind policies
-  printf "\nCreating the gcloud `color g $CNPG_SVC_ACCT` service account (SA)\n"
-  gcloud iam service-accounts create $CNPG_SVC_ACCT \
-    --project "$GCP_PROJECT_ID" \
-    --description "CNPG Operands SA" \
-    --display-name "CNPG Operands SA" &> $DEVNULL
-  warnOnError $? "the GCP SA `color y $CNPG_SVC_ACCT` may already exist"
-  waitForSA "$CNPG_SVC_ACCT" "$GCP_PROJECT_ID" 60
-
   CNPG_OPS_ROLE="cnpg.operands"
   printf "\nCreating the `color g $CNPG_OPS_ROLE` IAM role\n"
   ## Attach roles
@@ -136,7 +135,6 @@ driver: pd.csi.storage.gke.io
 deletionPolicy: Retain
 EOSNAPSC
 
-  sleep 2 # hold up processing for a moment to allow IAM mods to propagate
 }
 
 # this function is a little convoluted in the way it attempts to install the
@@ -184,7 +182,7 @@ CNPG_FETCH_CLEANUP_CRONJOB_YAML
     --project="$GCP_PROJECT_ID" \
     --description="Used for cronjob cleanup with Workload Identity" \
     --display-name="CNPG Cleanup SA" &> $DEVNULL
-    warnOnError $? "the GCP SA $CNPG_CLEANUP_SA may already exist"
+  warnOnError $? "the GCP SA $CNPG_CLEANUP_SA may already exist"
   waitForSA "$CNPG_CLEANUP_SA" "$GCP_PROJECT_ID" 60
 
   gcloud projects add-iam-policy-binding "$GCP_PROJECT_ID" \

package/src/helm-charts/eso/helmup.bootstrap

@@ -11,6 +11,8 @@ function installEsoEnvironment() {
   ESO_SA="eso-sa"
   ESO_SA_EMAIL="$ESO_SA@$GCP_PROJECT_ID.iam.gserviceaccount.com"
 
+  createNamespaceIfNeeded $ESO_NAMESPACE
+
   # Guard against accidental SA deletion on re-runs (see cnpg-operator for war story)
   gcloud iam service-accounts describe $ESO_SA_EMAIL --project $GCP_PROJECT_ID &> $DEVNULL
   if [ $? -eq 0 ];
@@ -50,8 +52,8 @@ SKIP_ESO_SA_CREATION
     --member "serviceAccount:$GCP_PROJECT_ID.svc.id.goog[$ESO_NAMESPACE/external-secrets]" \
     --role roles/iam.workloadIdentityUser $ESO_SA_EMAIL
   warnOnError $? "gcloud workload identity binding may have failed"
-
-  createNamespaceIfNeeded $ESO_NAMESPACE
+  waitForSAIAM "$ESO_SA" "$GCP_PROJECT_ID" "roles/iam.workloadIdentityUser" \
+    "serviceAccount:$GCP_PROJECT_ID.svc.id.goog[$ESO_NAMESPACE/external-secrets]" 60
 }
 
 installEsoEnvironment

package/src/helm-charts/prom-operator/README-CRD-UPGRADES.md

@@ -0,0 +1,105 @@
+# CRD Upgrades — kube-prometheus-stack
+
+## Why this exists
+
+Helm can install CustomResourceDefinitions (CRDs) but will not upgrade them
+when you bump chart versions. This chart ships a CRD upgrade job that uses
+server-side apply (SSA) to reconcile CRDs to the versions bundled with the
+chart.
+
+## Default: disabled
+
+The CRD upgrade job is disabled by default (`crds.enabled: false`) because it
+requires cluster-admin level permissions to run. Specifically, it creates and
+deletes ClusterRole and ClusterRoleBinding resources as pre-upgrade Helm hooks,
+which requires `container.clusterRoles.delete` in GCP IAM. Only enable it when
+explicitly bumping the chart version, and only a devops engineer should run
+`helmup prom-operator` when it is enabled.
+
+## Settings
+
+**`crds.enabled`** — controls whether the chart manages CRDs at all. When
+`false`, the CRD upgrade job and its associated hook resources are not rendered.
+Default: `false`.
+
+**`crds.upgradeJob.enabled`** — controls whether the SSA upgrade job hook runs.
+Must be `true` for CRDs to be reconciled on upgrade. Default: `false`.
+
+**`crds.upgradeJob.forceConflicts`** — adds `--force-conflicts` to the SSA
+apply, overriding fields owned by another field manager. Use only for a one-time
+conflict resolution. Revert to `false` immediately after. Default: `false`.
+
+## Permissions requirement
+
+The upgrade job runs as a pair of Helm pre-upgrade hooks that create a
+ClusterRole and ClusterRoleBinding. Helm's `before-hook-creation` delete policy
+causes it to attempt deletion of these resources before creating them — even if
+they don't yet exist in the cluster. The GCP IAM permission check fires before
+the existence check, so a 403 is returned for users without
+`container.clusterRoles.delete` regardless of whether the resources are present.
+
+The `admission-webhooks/job-patch` hooks have the same requirement and are
+always present, making `helmup prom-operator` a full-devops-only operation
+regardless of the CRD upgrade setting.
+
+## Hook lifecycle
+
+The ClusterRole and ClusterRoleBinding created by the upgrade job are ephemeral.
+They are deleted automatically after the job succeeds via the
+`hook-delete-policy: hook-succeeded` annotation. You will not see them in
+`kubectl get clusterrole` after a successful run.
+
+## Switching from enabled to disabled
+
+If `crds.enabled` was previously `true` and a successful upgrade ran, the CRD
+hook resources are recorded in the Helm release manifest even though they no
+longer exist in the cluster. Switching to `false` and running `helmup` will
+cause Helm to attempt to delete those orphaned hook resources from the previous
+release — again requiring cluster-admin permissions for that one transition run.
+
+To verify what hooks are currently stored in the release:
+```bash
+helm get hooks prometheus-stack -n prometheus | grep ^"# Source"
+```
+
+After a devops engineer runs `helmup prom-operator` with `crds.enabled: false`,
+the hooks will be cleared from the release manifest and subsequent runs will not
+require cluster-admin permissions (beyond the always-present admission-webhook
+hooks).
+
+## Runbook — bumping the chart version
+
+1. **Preflight** — confirm current CRD state:
+   ```bash
+   kubectl get crd | grep monitoring.coreos.com
+   kubectl apply --server-side --dry-run=server -f <crd-dir/>
+   ```
+
+2. **Enable** — set both flags in `prometheus-stack.yaml.ovh`:
+   ```yaml
+   crds:
+     enabled: true
+     upgradeJob:
+       enabled: true
+       forceConflicts: false
+   ```
+
+3. **Run** — as a devops engineer:
+   ```bash
+   helmup prom-operator
+   ```
+
+4. **If the job fails with field-manager conflicts:**
+   - Confirm no other source manages these CRDs
+   - Temporarily set `forceConflicts: true` for this run only
+   - Revert to `false` immediately after
+
+5. **Post-check:**
+   ```bash
+   kubectl get crd | grep monitoring.coreos.com
+   # verify CRDs show Established: True
+   # verify operator pod is healthy
+   # verify alerts and rules are normal
+   ```
+
+6. **Disable** — revert both flags to `false` before committing.

package/src/helm-charts/prom-operator/helmup.bootstrap

@@ -17,11 +17,14 @@ function installStackdriverExporterEnvironment() {
 
   createNamespaceIfNeeded $SDEXP_NS
 
-  # guarantee any previous remnants of the exporter are gone
-  printf "\n*** Removing the previous $SDEXP_SA installation...\n"
-  helm uninstall -n $SDEXP_NS "stackdriver-exporter"    &> $DEVNULL
-  kubectl delete serviceaccounts -n $SDEXP_NS $SDEXP_SA &> $DEVNULL
-  printf "*** Deleting the gcloud SA $SDEXP_EM\n"
+# used to remove this to avoid weird behavior when attempting to reboot
+# the service with udated configs - no longer doing this (for now)
+#
+# guarantee any previous remnants of the exporter are gone
+# printf "\n*** Removing the previous $SDEXP_SA installation...\n"
+# helm uninstall -n $SDEXP_NS "stackdriver-exporter"    &> $DEVNULL
+# kubectl delete serviceaccounts -n $SDEXP_NS $SDEXP_SA &> $DEVNULL
+# printf "*** Deleting the gcloud SA $SDEXP_EM\n"
 # gcloud --quiet iam service-accounts delete $SDEXP_EM  \
 #                            --project $GCP_PROJECT_ID  &> $DEVNULL
 
@@ -58,18 +61,19 @@ function createGrafanaOAuthSecret() {
     printf "\nCreating placeholder `color g grafana-google-oauth` secret in prometheus namespace\n"
     kubectl create secret generic grafana-google-oauth \
       --namespace prometheus \
-      --from-literal=GF_AUTH_GOOGLE_CLIENT_ID='' \
-      --from-literal=GF_AUTH_GOOGLE_CLIENT_SECRET=''
+      --from-literal=GF_AUTH_GOOGLE_CLIENT_ID='bootstrap id' \
+      --from-literal=GF_AUTH_GOOGLE_CLIENT_SECRET='boostrap secret'
     warnOnError $? "Failed to create grafana-google-oauth placeholder secret"
   else
     printf "\n`color g grafana-google-oauth` secret already exists - skipping\n"
   fi
 
-  # Wire up the ExternalSecret now that ESO is guaranteed to be deployed first
-  # in the SYSTEM array. This syncs credentials from GCP Secret Manager into the
-  # grafana-google-oauth secret once the deployment engineer populates them.
-  showInstalling "Grafana Google OAuth ExternalSecret"
-  cat<<GRAFANA_OAUTH_SECRET | kubectl apply -f -
+  # Wire up the ExternalSecret only if ESO is enabled for this cluster.
+  # Set ESO_ENABLED=true in overwhelm.yaml to activate (default: false).
+  if [ "${ESO_ENABLED}" = "true" ];
+  then
+    showInstalling "Grafana Google OAuth ExternalSecret"
+    cat<<GRAFANA_OAUTH_SECRET | kubectl apply -f -
 apiVersion: external-secrets.io/v1
 kind: ExternalSecret
 metadata:
@@ -86,11 +90,15 @@ spec:
   data:
     - secretKey: GF_AUTH_GOOGLE_CLIENT_ID
       remoteRef:
-        key: grafana-google-client-id
+        key: GF_AUTH_GOOGLE_CLIENT_ID
     - secretKey: GF_AUTH_GOOGLE_CLIENT_SECRET
       remoteRef:
-        key: grafana-google-client-secret
+        key: GF_AUTH_GOOGLE_CLIENT_SECRET
 GRAFANA_OAUTH_SECRET
+  else
+    printf "\n`color y '***WARNING:'` ESO_ENABLED is not set - skipping Grafana OAuth ExternalSecret\n"
+    printf "  Set ESO_ENABLED=true in overwhelm.yaml and re-run helmup prom-operator to wire up the OAuth secret.\n\n"
+  fi
 }
 
 installStackdriverExporterEnvironment

package/src/helm-charts/prom-operator/helmup.plugin

@@ -8,7 +8,7 @@ showInstalling "The Prometheus Operator (kube-prometheus-stack)"
 
 # https://artifacthub.io/packages/helm/prometheus-community/kube-prometheus-stack
 OCI_CHART="oci://ghcr.io/prometheus-community/charts/kube-prometheus-stack"
-[ -z "$PROMETHEUS_STACK_CHART_VERSION" ] && PROMETHEUS_STACK_CHART_VERSION="82.15.0"
+[ -z "$PROMETHEUS_STACK_CHART_VERSION" ] && PROMETHEUS_STACK_CHART_VERSION="83.7.0"
 helm upgrade $NS --install prometheus-stack $OCI_CHART \
                  --values  prom-operator/prometheus-stack.yaml \
                  --version $PROMETHEUS_STACK_CHART_VERSION $HELM_WHAT

package/src/helm-charts/prom-operator/prometheus-stack.yaml.ovh

@@ -1,50 +1,13 @@
+#
+# Apparently you must have devops access in order to deploy the prometheus
+# operator since it attempts to create cluster roles during deployment.
+#
 crds:
-  # ─────────────────────────────────────────────────────────────────────────────
-  # CRD MANAGEMENT — kube-prometheus-stack
-  #
-  # Why this exists
-  #   Helm can install CustomResourceDefinitions (CRDs) but will not upgrade them
-  #   when you bump chart versions. This chart ships a CRD upgrade job that uses
-  #   server-side apply (SSA) to reconcile CRDs to the versions bundled with the
-  #   chart.
-  #
-  # Safe operating model
-  #   ✅ Leave the upgrade job ENABLED if this chart is the sole owner of the
-  #      Prometheus Operator CRDs.
-  #   ⚠️  Keep "forceConflicts" DISABLED unless you hit a specific field-manager
-  #      conflict that you’ve reviewed and want this chart to own.
-  #   ❌ If another source manages CRDs (e.g., a separate CRD chart, GitOps
-  #      bootstrap, platform repo), DISABLE the job to avoid ownership fights.
-  #
-  # Settings
-  #   enabled:        runs the SSA job on chart upgrade to reconcile CRDs.
-  #                   Recommended: true if chart owns the CRDs; false if
-  #                   another tool owns them.
-  #
-  #   forceConflicts: adds `--force-conflicts` to SSA, overriding fields owned by
-  #                   another controller. Use only for a one-time conflict
-  #                   resolution. Can overwrite annotations and mask real
-  #                   incompatibilities if left true.
-  #
-  # Recommended defaults
-  #   enabled: true
-  #   forceConflicts: false
-  #
-  # Mini-runbook
-  #   1) Preflight:
-  #      kubectl get crd | grep monitoring.coreos.com
-  #      kubectl apply --server-side --dry-run=server -f <crd-dir/>
-  #   2) Upgrade with enabled: true, forceConflicts: false.
-  #   3) If job fails with conflicts:
-  #        - confirm no other source manages these CRDs
-  #        - temporarily set forceConflicts: true for this upgrade only
-  #        - revert to false afterwards
-  #   4) Post-check: CRDs Established, operator healthy, alerts/rules normal
-  # ─────────────────────────────────────────────────────────────────────────────
+  # See README-CRD-UPGRADES.md — disabled by default, full-devops only
   enabled: true
   upgradeJob:
     enabled: true
-    forceConflicts: false # See README-CRD-UPGRADES.md
+    forceConflicts: false
 
 kubeControllerManager:
   enabled: false
@@ -186,7 +149,7 @@ grafana:
   envFromSecret: grafana-google-oauth
   env:
     GF_SERVER_ROOT_URL: https://${PROJECT_NAME}-monitoring.${HOST}.com
-    GF_AUTH_GOOGLE_ENABLED: "true"
+    GF_AUTH_GOOGLE_ENABLED: "${ESO_ENABLED}"
     GF_AUTH_GOOGLE_SCOPES: "https://www.googleapis.com/auth/userinfo.profile https://www.googleapis.com/auth/userinfo.email"
     GF_AUTH_GOOGLE_AUTH_URL: "https://accounts.google.com/o/oauth2/auth"
     GF_AUTH_GOOGLE_TOKEN_URL: "https://accounts.google.com/o/oauth2/token"
@@ -284,7 +247,7 @@ alertmanager:
               {{- end }}
 
               {{- if .Labels.pod }}
-              *Pod Link:* *<https://console.cloud.google.com/kubernetes/pod/${GCE_REGION}/${CLUSTER_NAME}/{{ .Labels.namespace }}/{{ .Labels.pod }}?project=${PROJECT_NAME}|`{{ .Labels.pod }}`>*
+              *Pod Link:* *<https://console.cloud.google.com/kubernetes/pod/${GCE_REGION}/${CLUSTER_NAME}/{{ .Labels.namespace }}/{{ .Labels.pod }}?project=${PROJECT_ID}|`{{ .Labels.pod }}`>*
               {{- end }}
 
               *Status:* `{{ .Status }}` | *Severity:* `{{ .Labels.severity }}` {{- if .Labels.container }} *Container:* `{{ .Labels.container }}` {{- end }}

package/src/helm-charts/prom-operator/stackdriver-exporter.yaml.ovh

@@ -5,7 +5,7 @@
 fullnameOverride: "prometheus-stack-stackdriver-metrics"
 
 stackdriver:
-  projectId: "${PROJECT_NAME}"
+  projectId: "${PROJECT_ID}"
   metrics:
     typePrefixes:  >
       agent.googleapis.com/cpu/utilization,

package/src/helm-charts/velero/helmup.bootstrap

@@ -86,6 +86,8 @@ function installVeleroEnvironment() {
       --role roles/iam.workloadIdentityUser \
       $SERVICE_ACCOUNT_EMAIL
   warnOnError $? "gcloud workload identity binding may have failed"
+  waitForSAIAM "velero" "$GCP_PROJECT_ID" "roles/iam.workloadIdentityUser" \
+    "serviceAccount:$GCP_PROJECT_ID.svc.id.goog[velero/velero]" 60
 
   printf "\nCreate and annotate the k8s SA\n"
   kubectl create serviceaccount velero --namespace velero

package/src/helmup.sh

@@ -135,7 +135,7 @@ SYSTEM=(
   cnpg-operator
   preemptible-killer
   zombie-killer
-  reflector
+# reflector
   reloader
   velero
   cnpg-db-psql-stack
@@ -218,8 +218,8 @@ WORKLOAD_ID
       --member "serviceAccount:$GCP_PROJECT_ID.svc.id.goog[${NAMESPACE}/${BOUND_SVC_ACCT}]" \
       --role roles/iam.workloadIdentityUser $SVC_ACCT_EMAIL
   warnOnError $? "gcloud workload identity binding may have failed"
-
-  sleep 2 # give the policy a chance to propagate
+  waitForSAIAM "$SVC_ACCT" "$GCP_PROJECT_ID" "roles/iam.workloadIdentityUser" \
+    "serviceAccount:$GCP_PROJECT_ID.svc.id.goog[${NAMESPACE}/${BOUND_SVC_ACCT}]" 60
  
   # Verify the Kubernetes SA and create it if necessary
   printf "\nChecking for the `color g $NAMESPACE/$SVC_ACCT` k8s SA\n"
@@ -821,6 +821,10 @@ export GCP_PROJECT_ID="$(getProjectIdFromK8sContext)"
 # Same for the region, but not as prevalent
 export GCE_REGION=`overwhelm -k GCE_REGION`
 
+# TODO: Temporary, until ESO is widely deployed, then the ESO_ENABLED logic
+# can be removed from the configs entirely.
+export ESO_ENABLED=`overwhelm -k ESO_ENABLED`
+
 doInstall "${SERVICES[@]}"
 
 displayBanner

package/src/init-my-chart/Chartwright.mjs

@@ -17,7 +17,7 @@ const hbsContext = {
   hbsPartOf : '',
   hbsServicePreemptible : true,
   hbsRedisEnabled : true,
-  hbsAutoScalingEnabled : true
+  hbsAutoscalingEnabled : true,
 }
 
 const PRESERVE_SECTIONS = new Set( [

package/src/init-my-chart/leaf-chart-templates/templates/ingressroute.yaml

@@ -0,0 +1 @@
+{{ include "leverege.ingressroute" . }}

package/src/service-man/config/secrets/npmrc.json

@@ -0,0 +1,13 @@
+{
+  "type": "managed",
+  "name": "NPMRC",
+  "source": "leverege-registry:DEPRECATED_NPMRC",
+  "labels": {
+    "env-type": "string"
+  },
+  "annotations": {
+    "description": "a deprecated npmrc key",
+    "owner": "platform-team",
+    "source": "service-man"
+  }
+}

package/src/DockerPy.mjs

@@ -1,266 +0,0 @@
-/**
- * DEPRECATED - DO NOT MODIFY
- */
-import fs from 'node:fs'
-import path from 'node:path'
-import url from 'node:url'
-
-import chalk from 'chalk'
-import handlebars from 'handlebars'
-
-import {
-  errorExit,
-  getCleanTag,
-  getDateTimestamp,
-  getGcpAccount,
-  log,
-  proceed,
-  shellCmd,
-  warning
-} from './Utils.mjs'
-
-const DIRNAME = path.dirname( url.fileURLToPath( import.meta.url ) )
-
-// The contents of these variables were initially located in files that live in
-// the build-tools repository, but it just became simpler to pull the contents
-// directly into these variables and forego the file loading.
-//
-const pluginTemplate = `
-# Dockerfile.plugin - optionally extend the service Docker image
-#
-# This file may be used to run additional docker commands during the image
-# build process without needing to maintain a local custom Dockerfile. For
-# example, uncommenting the following docker RUN command will cause the
-# curl and vim packages to be added to the deployed image thus making them
-# available from the pod's command line on k8s:
-#
-# RUN apk update && apk add --no-cache curl vim
-#
-# Keep in mind that the Alpine Linux base image is used to keep image
-# footprints small, so adding packages "just because" is not considered
-# a best practice.
-`
-const dockerfileTemplate = fs.readFileSync( path.join( DIRNAME, './templates/dockerfileTemplatePy.hbs' ), 'utf8' )
-const cloudBuildSteps = fs.readFileSync( path.join( DIRNAME, './templates/cloudBuildTargetGKE.hbs' ), 'utf8' )
-let bashrcTemplate = fs.readFileSync( path.join( DIRNAME, './templates/bashrcTemplate.hbs' ), 'utf8' )
-const bashrcPyTemplate = fs.readFileSync( path.join( DIRNAME, './templates/bashrcTemplatePy.hbs' ), 'utf8' )
-bashrcTemplate = `${bashrcTemplate}\n${bashrcPyTemplate}`
-
-const defaultSettings = {
-  date : getDateTimestamp(),
-  imageBase : 'python:3.12.8-slim-bookworm',
-  npmlogging : process.env.VERBOSE_NPM_LOGGING ? '--ddd' : '--silent',
-  npmVersion : 'npm@11',
-  pluginfile : '# NO PLUGIN',
-  preInstallPluginfile : '# NO PRE-INSTALL PLUGIN',
-  regvers : 'package.version',
-  runuser : 'python',
-}
-
-// the previous build version is simply informational for the user
-const previousFile = './docker/.previous'
-
-function getPreviousBuild() {
-  return fs.existsSync( previousFile ) ? fs.readFileSync( previousFile ) : 'FIRST BUILD'
-}
-
-const setPreviousBuild = ( version ) => {
-  fs.writeFileSync( previousFile, version )
-}
-
-// the base image string to be in the form of an image name and a version
-// for example:
-//
-//   python:3.12.8-slim-bookworm becomes python-3.12.8
-//   nvcr.io/nvidia/cuda:12.3.1-base-ubuntu22.04 becomes cuda-12.3.1
-//
-// which will then be used to tag the image for information purposes only
-const extractBaseImageTag = ( imageStr ) => {
-  const match = imageStr.match( /(\w+:\d+(?:\.\d+)*)(?=-)/ ) // eslint-disable-line security/detect-unsafe-regex
-  return match ? match[1].replace( ':', '-' ) : null
-}
-
-function generateDockerfile( options ) {
-
-  const settings = { ...defaultSettings, ...options }
-  settings.image = settings.imageBase
-  // TODO: this is a janky way to pass the image tag back out
-  defaultSettings.buildTag = extractBaseImageTag( settings.image )
-
-  // Ensure the presence of the docker dir with plugin and bashrc - this code
-  // could probably be tightened up a little.
-  try {
-    fs.readdirSync( './docker' )
-  } catch ( err ) {
-    warning( 'creating the missing ./docker directory' )
-    fs.mkdirSync( './docker' ) // TODO: blindly assumes the mkdir succeeds
-  }
-
-  const bashrcFile = './docker/bashrc'
-  let bf = bashrcTemplate
-  const bashrcFilePlugin = './docker/Bashrc.plugin'
-  if ( fs.existsSync( bashrcFilePlugin ) ) {
-    const bashrcAdditions = fs.readFileSync( bashrcFilePlugin )
-    bf = `${bf}\n${bashrcAdditions}`
-  }
-  fs.writeFileSync( bashrcFile, bf )
-
-  const dockerfilePlugin = './docker/Dockerfile.plugin'
-  if ( !fs.existsSync( dockerfilePlugin ) ) {
-    warning( `creating the missing ${dockerfilePlugin} file in the docker directory` )
-    fs.writeFileSync( dockerfilePlugin, pluginTemplate )
-  }
-
-  try {
-    const build = fs.readdirSync( './build' )
-    if ( build.length === 0 ) {
-      errorExit( `***ERROR: the build dir is empty - ${chalk.yellow( 'npm run build' )}` )
-    }
-  } catch ( err ) {
-    errorExit( `***ERROR: expected the ./build directory to exist - ${chalk.yellow( 'npm run build' )}` )
-  }
-
-  // At this point the plugin template better exist.
-  if ( fs.existsSync( dockerfilePlugin ) ) {
-    settings.pluginfile = fs.readFileSync( dockerfilePlugin )
-  } else {
-    errorExit( `***ERROR: something went wrong with ${dockerfilePlugin} plugin creation` )
-  }
-
-  const dockerfilePrePlugin = './docker/DockerfilePreInstall.plugin'
-  if ( fs.existsSync( dockerfilePrePlugin ) ) {
-    settings.preInstallPluginfile = fs.readFileSync( dockerfilePrePlugin )
-  }
-
-  const compiled = handlebars.compile( dockerfileTemplate, { noEscape : true } )
-  const replaced = compiled( settings )
-  // temporarily write the Dockerfile to the docker subdir - the container
-  // image builder will then move it over to the build subdir
-  const dockerfile = './docker/Dockerfile'
-  try {
-    fs.writeFileSync( dockerfile, replaced )
-  } catch ( err ) {
-    errorExit( `***ERROR: failed to write ${dockerfile}\n${err}` )
-  }
-
-  const previousBuild = getPreviousBuild()
-  const dockerInfo = { dockerfile, previousBuild, ...settings }
-  delete dockerInfo.pluginfile
-  return dockerInfo
-}
-
-function formCloudBuildBucketName( artifactProject ) {
-  return `gs://${artifactProject}_cloudbuild`
-}
-
-async function validateCloudBuildBucket( artifactProject ) {
-  const bucketName = formCloudBuildBucketName( artifactProject )
-  log( chalk.green.bold( `Verifying the build bucket exists => ${bucketName}\n` ) )
-  try {
-    await shellCmd( `gsutil ls -p ${artifactProject} -b ${bucketName}` )
-  } catch ( error ) {
-    errorExit( `***Error: from Docker.validateCloudBuildBucket\n\n${chalk.bold.yellow( error )}` )
-  }
-}
-
-async function buildContainerImage( {
-  artifactProject, artifactRegistry, containerName, 
-  imageVersion, isNpmWorkspace, options
-} ) {
-  const buildWorkspace = './build/workspace' // passed to Cloud Build
-  const imageName = `${artifactRegistry}/images/${containerName}`
-
-  // clean up any residual build cruft (should be clean though)
-  if ( fs.existsSync( buildWorkspace ) ) {
-    fs.renameSync( buildWorkspace, `${buildWorkspace}-junk` )
-    fs.rm( `${buildWorkspace}-junk`, { recursive : true }, ( err ) => {} ) // fire and forget
-  }
-
-  // TODO: This is janky - I'll clean it up later
-  // NOTE: this is dealing with raw filesystem interactions, meaning there
-  //   isn't a shell performing file globbing and what not
-  fs.renameSync( './build', './workspace' )
-  fs.mkdirSync( './build' )
-  fs.renameSync( './workspace', buildWorkspace )
-  fs.renameSync( './docker/Dockerfile', './build/Dockerfile' )
-  fs.renameSync( './docker/bashrc', './build/bashrc' )
-  // fs.cpSync( `${process.env.HOME}/.npmrc`, './build/.npmrc' )
-  // fs.cpSync( './package.json', `${buildWorkspace}/package.json` )
-  // fs.cpSync( './package-lock.json', `${buildWorkspace}/package-lock.json` )
-
-  setPreviousBuild( imageVersion ) // update docker/.previous file
-
-  const { fullName } = await getGcpAccount()
-  const imageBuilder = getCleanTag( fullName )
-  const imageArch = defaultSettings.buildTag
-  const imageInfoTag = `zzinfo_${imageArch}_${imageBuilder}_${defaultSettings.date}`
-  const gcloudProjectNumber = await shellCmd( `gcloud projects describe ${artifactProject} --format="value(projectNumber)"` )
-
-  // build the cloudbuild.yaml from the handlebars template
-  const cloudBuildFile = './build/cloudbuild.yaml'
-  const cbst = handlebars.compile( cloudBuildSteps, { noEscape : true } )
-  const cbs = cbst( { artifactProject, artifactRegistry, imageName, imageVersion, imageInfoTag, } )
-  fs.writeFileSync( cloudBuildFile, cbs )
-
-  // build up the cloud build command and options
-  const cloudBuildCmd = `gcloud builds submit --project ${artifactProject}`
-  const cloudBuildLogs = `--gcs-log-dir ${formCloudBuildBucketName( artifactProject )}/log`
-  const cloudBuildCfg = `--config=${cloudBuildFile}`
-  const cloudBuildSA = `--service-account=${gcloudProjectNumber}-compute@cloudbuild.gserviceaccount.com`
-
-  log( chalk.green.bold( `
-  *** Submitting Build as ${imageBuilder} ***
-      ${cloudBuildCmd} \\
-           ${cloudBuildLogs}  \\
-           ${cloudBuildCfg} \\
-           ${cloudBuildSA}
-  ` ) )
-
-  if ( process.env.DRY_RUN === '1' ) {
-    log( `Would have run => ${cloudBuildCmd} ${cloudBuildLogs} ${cloudBuildCfg} ./build` )
-    log( '\n***Exiting from DRY_RUN\n' )
-    process.exit( 0 )
-  }
-
-  await shellCmd( `${cloudBuildCmd} ${cloudBuildLogs} ${cloudBuildCfg} ./build`, { stdio : 'inherit' } )
-}
-
-async function repoCleanup() {
-  // remove garbage from the repo
-  const garbage = [
-    'build',
-    'dist',
-    'docker/.keep',
-    'docker/.npmrc',
-    'docker/.gitignore',
-    'docker/.previous',
-    'docker/Dockerfile',
-    'docker/workspace',
-  ]
-
-  garbage.forEach( trash => fs.rmSync( trash, { recursive : true, force : true } ) )
-
-  fs.writeFileSync( 'docker/.gitignore', '.previous\nDockerfile' )
-
-  await shellCmd( 'git add -f docker' )
-
-  log( `
-  ${chalk.yellow.bold( '***IMPORTANT***' )}
-
-  Please make the following .gitignore modifications:
-    change /dist to ${chalk.green.bold( '/build' )}
-    remove ${chalk.red.bold( '/docker' )}
-` )
-
-  await proceed( 'Acknowledge the .gitignore mods were made?' )
-}
-
-export default {
-  getPreviousBuild,
-  setPreviousBuild,
-  generateDockerfile,
-  formCloudBuildBucketName,
-  validateCloudBuildBucket,
-  buildContainerImage,
-  repoCleanup,
-}

package/src/docker-to-registry-py.mjs

@@ -1,287 +0,0 @@
-#!/usr/bin/env node
-/**
- * DEPRECATED - DO NOT MODIFY
- */
-import fs from 'node:fs'
-
-import toml from 'toml'
-import chalk from 'chalk'
-
-import {
-  debug, log, warning,
-  errorExit,
-  getGitBranchAndUpstream,
-  gitRepoIsDirty,
-  parseHelmChart,
-  proceed,
-  shellCmd } from './Utils.mjs'
-import Docker from './DockerPy.mjs'
-
-// refresh-npm-token does not throw so no need to try, but it emits in debug
-// const refreshErr = await shellCmd( 'refresh-npm-token' )
-// if ( refreshErr && !process.env.BUILD_TOOLS_DEBUG ) {
-//   log( refreshErr ) // most likely the token refreshed message
-// }
-
-warning( 'docker-to-registry-py has been DEPRECATED - use docker-to-registry' )
-
-const options = {}
-let imageVersion = null
-// Expected to be invoked like docker-to-registry-py v1.2.3
-for ( let n = 2; n < process.argv.length; n++ ) {
-  if ( process.argv[n].startsWith( '-' ) ) {
-    const str = process.argv[n].slice( 1 )
-    if ( str.indexOf( '=' ) > 0 ) {
-      const [ key, value ] = str.split( '=' )
-      options[key] = value
-    } else {
-      options[str] = true
-    }
-  } else {
-    imageVersion = process.argv[n]
-  }
-}
-if ( !imageVersion ) {
-  errorExit( '***Error: docker-to-registry-py requires an image version' )
-}
-
-// may use guide-me or guideme to ask for update guidance
-const giveGuidance = imageVersion.match( /^guide-*me$/ ) !== null
-
-const willGitTag = imageVersion.match( /^v\d+\.\d+\.\d+$/ )
-const tagInfo = willGitTag ?
-  chalk.yellow.bold( 'will be git tagged' ) :
-  chalk.red( 'BETA RELEASE WILL NOT BE GIT TAGGED' )
-
-const repoIsDirty = await gitRepoIsDirty()
-
-if ( willGitTag && repoIsDirty ) {
-  log( `
-  ${chalk.red.bold( '***ERROR: attempting to tag a non-beta dirty git repository' )}
-
-  You are attempting to create a tagged release image for pushing to the
-  artifact registry, but there are locally modified files. This is not
-  allowed since the applied tag will not be relevant to the image version
-  due to the pending commits.
-
-  Available options for proceeding:
-
-    ${chalk.green.bold( '1) cleanly commit all local mods assuming relevance' )}
-    ${chalk.yellow( '2) eliminate unwanted local modifications' )}
-    ${chalk.red( '3) stash anything that is irrelevant to this release' )}
-
-  ` )
-  process.exit( 1 )
-}
-
-if ( !fs.existsSync( './helm' ) ) {
-  log( `
-  ${chalk.red.bold( '***Error: docker-to-registry-py requires a helm chart directory' )}
-
-  The docker-to-registry-py script expects to run in the root directory of a k8s
-  service, which requires a helm chart directory. It also expects a docker
-  directory but will build that if it does not exist.
-
-  ` )
-  process.exit( 1 )
-}
-
-const tomlStr = fs.readFileSync( './pyproject.toml' )
-const pyproject = toml.parse( tomlStr )
-const repoDescr = {
-  artifactProject : pyproject.leverege.project || 'leverege-registry',
-  containerName : pyproject.project.name,
-  packageVersion : `v${pyproject.project.version}`,
-  artifactRegistry : pyproject.leverege.registry || 'us-docker.pkg.dev/leverege-registry/stack',
-  registryFolder : `${pyproject.leverege['registry-folder']}/images/${pyproject.project.name}`,
-  imageBase : pyproject.leverege['image-base'],
-  helmChart : await parseHelmChart(),
-}
-
-debug( { repoDescr }, '<==The Repo Description' )
-
-const dockerInfo = await Docker.generateDockerfile( {
-  regvers : repoDescr.packageVersion,
-  imageBase : repoDescr.imageBase,
-} )
-
-debug( { dockerInfo }, '<==The Docker Info' )
-
-const {
-  artifactProject,
-  artifactRegistry,
-  packageVersion,
-  containerName,
-  helmChart,
-} = repoDescr
-
-if ( giveGuidance ) {
-  await Docker.validateCloudBuildBucket( artifactProject )
-}
-
-const registryFolder = `${artifactRegistry}/images/${containerName}`
-
-// Update the dependencies...
-// log( chalk.green.bold( 'Updating dependencies and workspace...' ) )
-// await shellCmd( 'npm install', { stdio : 'inherit' } )
-
-if ( giveGuidance ) {
-  await Docker.repoCleanup() // adjusts gitignore files and removes cruft
-  log( `
-
-
-
-  ${chalk.green.bold( '-------------------- Potential Helm Chart Mods --------------------' )}
-
-  Moving to the new artifact-registry for docker image storage may also require
-  helm chart changes depending on how old the current charts are. Charts that
-  contain helm/values.yaml files resembling this:
-
-  ${chalk.yellow.bold( `serviceConfig:
-    VERSION: v1.2.3
-    PREEMPTIBLE: true` )}
-
-  are pre-ignition era charts and should be replaced with the latest helm chart
-  templates available from ignition. Either dive in or ask devops for a hand
-  when converting these legacy / deprecated charts.
-
-
-  More recently updated helm charts that have already started the migration to
-  the new approach may already have registry entries like this:
-
-  ${chalk.yellow.bold( `image:
-    registry: gcr.io/leverege-docker-images
-    tag: ""` )}
-
-  The above registry statement references the deprecated container registry and
-  must be adjusted accordingly to resemble:
-
-    ${chalk.green.bold( 'registry: us-docker.pkg.dev/leverege-registry/<registry folder>/images' )}
-
-  There may also be a corresponding modification needed in the helm template
-  deployment.yaml file in the container image spec:
-
-    ${chalk.green.bold( 'image: {{ .Values.image.registry }}/{{ .Chart.Name }}:{{ default .Chart.AppVersion .Values.image.tag }}' )}
-
-
-  ${chalk.magenta.bold( `Upgrading all service charts to the latest ignition template chart structure
-  is the preferred approach to making the transition to the artifact registry.` )}
-
-  ` )
-  log( chalk.green.bold( '*** docker-to-registry-py guidance complete - ready for dockerization ***' ) )
-  process.exit( 0 )
-}
-
-// do chart checks too
-let chart
-if ( options?.helmChecks !== 'false' ) {
-  chart = helmChart?.yaml['Chart.yaml']
-  const values = helmChart?.yaml['values.yaml']
-  const expectedChartRegistry = `${artifactRegistry}/images`
-
-  if ( expectedChartRegistry !== values.image?.registry ) {
-    log( `
-  ${chalk.red.bold( '***Error: mismatched package.json registry and helm/values.yaml' )}
-
-  The registry specified in the package.json leverege stanza does not line up
-  with the image registry in helm/values.yaml. The naming convention expects
-  the '/images' suffix to be added to the package.json registry string and then
-  stored as the image.registry in the helm/values.yaml file.
-
-    Current settings:
-      package.json registry => ${chalk.yellow.bold( artifactRegistry )}
-      values.yaml  registry => ${chalk.red.bold( values.image?.registry )}
-
-    Expected settings:
-      values.yaml  registry => ${chalk.green.bold( expectedChartRegistry )}
-  ` )
-
-    if ( !values.image ) {
-      log( `
-  ${chalk.yellow.bold( '***DEPRECATED: legacy helm charts detected' )}
-
-  The helm chart structure appears to be based on the pre-ignition helm chart
-  layouts. The charts must be upgraded before docker-to-registry-py can complete
-  its job.
-
-    ` )
-    }
-
-    process.exit( 1 )
-  }
-}
-
-// Give the summary and the user a chance to proceed or not
-log( `
-  ${chalk.green.bold( 'Build Information:' )}
-    Container:     ${chalk.green.bold( containerName )}
-    Version:       ${chalk.green.bold( imageVersion )}   ${tagInfo}
-    Registry:      ${chalk.green.bold( artifactRegistry )}
-    Image Folder:  ${chalk.yellow.bold( registryFolder )}
-
-  ${chalk.green.bold( 'Docker Information:' )}
-    NodeImage:     ${chalk.green.bold( dockerInfo.imageBase || `node:${dockerInfo.nodeimage}` )}
-    Run User:      ${chalk.green.bold( dockerInfo.runuser )}
-    Previous:      ${chalk.yellow.bold( dockerInfo.previousBuild )}
-    DateStamp:     ${chalk.green.bold( dockerInfo.date )}
-` )
-
-if ( imageVersion !== packageVersion ) {
-  log( `
-  ${chalk.yellow.bold( '***WARNING: specified version does not match project version' )}
-        pyproject.toml => ${chalk.green.bold( packageVersion )}
-        specified    => ${chalk.red.bold( imageVersion )}
-  ` )
-
-  if ( willGitTag ) {
-    log( chalk.red.bold( '... and you are attempting to release so NOPE!\n' ) )
-    process.exit( 1 )
-  }
-}
-
-const repoInfo = await getGitBranchAndUpstream()
-
-if ( willGitTag && !repoInfo.upstream ) {
-  log( `
-  ${chalk.red.bold( '***ERROR: the current branch must have an upstream' )}
-
-  You are attempting to create a tagged release image for pushing to the
-  container registry, but the current branch does not have an upstream to
-  push to. This is not allowed since the applied tag will be stranded here
-  in your local repository, which is what we are trying to avoid.
-
-  Available options for proceeding:` )
-  log( chalk.green.bold( `
-    1) set an upstream for this branch via:
-           git push --set-upstream origin ${repoInfo.branch}` ) )
-  log( chalk.yellow.bold( `
-    2) rebase, squash and merge onto a branch with an upstream` ) )
-
-  process.exit( 1 )
-}
-
-if ( willGitTag ) {
-  if ( chart.appVersion !== imageVersion ) {
-    log( `
-  ${chalk.yellow.bold( '***WARNING: the helm/Chart.yaml appVersion does not align with the build version' )}
-    chart appVersion => ${chalk.yellow.bold( chart.appVersion )}
-    build version    => ${chalk.green.bold( imageVersion )}
-  ` )
-  }
-}
-
-await proceed()
-
-log( chalk.green.bold( 'Building the Docker Image...' ) )
-await Docker.buildContainerImage( { ...repoDescr, options, imageVersion } )
-log( chalk.blue.bold( 'Done Building the Docker Image...' ) )
-
-if ( willGitTag ) {
-  const tagDescription = `docker_${imageVersion}`
-  const fullImageTag = repoDescr.isMonoRepo ? `${repoDescr.containerName}/${imageVersion}` : imageVersion
-  log( chalk.green.bold( `Tagging and Pushing ${fullImageTag}\n` ) )
-  await shellCmd( `git tag -f ${fullImageTag} -m ${tagDescription}` )
-  await shellCmd( 'git push --follow-tags' )
-} else {
-  log( chalk.yellow.bold( '\n*** Skipped git tagging - beta release' ) )
-}