@leverege/build-tools 2.98.1 → 2.99.1

package/.vscode/launch.json

@@ -18,6 +18,16 @@
       },
       "outputCapture": "std"
     },
+    {
+      "name": "overwhelm tst-dense",
+      "type": "node",
+      "request": "launch",
+      "program": "${workspaceFolder}/src/overwhelm.mjs",
+      "args": [],
+      "cwd": "/Users/john/Work/leverege/k8s/tst-dense",
+      "runtimeExecutable": "node",
+      "console": "integratedTerminal"
+    },
     {
       "name": "push-my-chart rsrc-srvr",
       "type": "node",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@leverege/build-tools",
-  "version": "2.98.1",
+  "version": "2.99.1",
   "description": "A collection of build / support tools for Leverege developers",
   "main": "index.js",
   "repository": {
@@ -92,7 +92,7 @@
     "googleapis": "^171.4.0",
     "handlebars": "^4.7.8",
     "ignore": "^7.0.5",
-    "inquirer": "^13.3.0",
+    "inquirer": "^13.3.2",
     "js-yaml": "^4.1.1",
     "jsdoc": "^4.0.5",
     "ms": "^2.1.3",

package/src/Docker.mjs

@@ -24,8 +24,8 @@ const DIRNAME = path.dirname( url.fileURLToPath( import.meta.url ) )
 //
 // See the latest docker images here => https://hub.docker.com/_/node
 //
-const dockerfileNodeVersion = process.env.DOCKER_BUILD_NODE_VERSION || '24.12.0-alpine3.23'
-const dockerfileNpmVersion = process.env.DOCKER_BUILD_NPM_VERSION || '11.6.0'
+const dockerfileNodeVersion = process.env.DOCKER_BUILD_NODE_VERSION || '24.14.0-alpine3.23'
+const dockerfileNpmVersion = process.env.DOCKER_BUILD_NPM_VERSION || '11.9.0'
 
 // The contents of these variables were initially located in files that live in
 // the build-tools repository, but it just became simpler to pull the contents

package/src/bash-funcs

@@ -5,14 +5,22 @@
 # and can now be easily added to a script via bash sourcing:
 #         . `build-tools --bashfun`
 
+# If the script is being run in an interactive terminal, we want to make sure to
+# reset the terminal settings on exit. This is especially important if the script
+# is killed while running a command that changes the terminal settings (like stty raw).
+if [[ -t 0 ]]; then
+  STTY_SAVE=$(stty -g)
+  trap 'stty "$STTY_SAVE"' EXIT
+fi
+
 # Typically /dev/null is used to redirect the noise but sometimes we need to
 # see exactly what certain commands are complaining about. Running any of the
 # build-tools with BUILD_TOOLS_DEBUG=1 will redirect the noise to stdout.
 DEVNULL="/dev/null"
 if [[ -n "${BUILD_TOOLS_DEBUG:-}" ]];
 then
-  DEVNULL="/dev/stdout"
-  printf "\n***BUILD_TOOLS_DEBUG DEVNULL=>$DEVNULL\n"
+  DEVNULL="/dev/stderr"
+  printf "\n***BUILD_TOOLS_DEBUG DEVNULL=>$DEVNULL\n" >&2
 fi
 
 GITTOP=`git rev-parse --show-toplevel`
@@ -35,7 +43,7 @@ YELO_WARN="`color y '***WARNING:'`"
 GREEN_INFO=`color g '***INFO:'`
 
 function clearScreen() {
-  [ "$BUILD_TOOLS_DEBUG" != "1" ] && clear
+  [[ -z "${BUILD_TOOLS_DEBUG:-}" ]] && clear
 }
 
 function clearlyWarn() {
@@ -71,7 +79,7 @@ function exitOnError() {
 
 function yesOrSkipToContinue() {
   printf "\n Enter \"`color g yes`\" ${1}, \"`color y skip`\" to skip this step, or anything else to exit.\n\n"
-  read ANS
+  read -e ANS
   [ "$ANS" == 'yes'  ] && return 0
   [ "$ANS" == 'skip' ] && return 2
   exit 1
@@ -79,12 +87,12 @@ function yesOrSkipToContinue() {
 
 function yesToContinue() {
   printf "\n Enter \"`color g yes`\" ${1}, or anything else to exit.\n\n"
-  read ANS
+  read -e ANS
   [ "$ANS" != 'yes'  ] && exit 1
 }
 
 function proceedOnEnter() {
-  printf " Hit `color g return` to proceed\n\n" && read ANS
+  printf " Hit `color g return` to proceed\n\n" && read -e ANS
 }
 
 function setOrUnset() {
@@ -109,13 +117,13 @@ function getManagedSecret() {
     printf "Returned   =>[$SECRET]\n\n"
     exit 1
   fi
-  echo $SECRET
+  echo "$SECRET"
 }
 
 function getProjectIdFromK8sContext() {
   local currContext=`kubectl config current-context`
 
-  IFS='_'
+  local IFS='_'
   read -ra ADDR <<< "$currContext"
 
   echo ${ADDR[1]} # project id is the second string split by _
@@ -273,7 +281,7 @@ function ifPluggedIn() {
   local PLUGIN=
   HELMUP="helm upgrade --install $SERVICE $CHARTLOC -f $SERVICE/values.yaml $CHARTVER $HELM_WHAT"
   case "$INVOKED_AS" in
-    "helmup"|"helmwhat"|"helmcycle" )
+    "helmup"|"helmwhat"|"helmcycle"|"helminit" )
       PLUGIN="$1/helmup.plugin"
       ;;
     "helmdn" )
@@ -323,4 +331,4 @@ function commandChecker() {
   done
 }
 
-commandChecker git helm jq kubectl service-man
+commandChecker git helm jq kubectl

package/src/circle-orb/circle-orb.mjs

@@ -0,0 +1,55 @@
+#!/usr/bin/env node
+/* eslint-disable */ // for now, we can relax this rule since ...
+/* eslint-disable security/detect-non-literal-fs-filename */
+import fs from 'node:fs'
+
+import { program } from 'commander'
+import chalk from 'chalk'
+
+import {
+  analyzeRepository,
+  condir,
+  enableDebug,
+  errorExit,
+  log, 
+} from '../Utils.mjs'
+
+program
+  .name( 'circle-orb' )
+  .description( 'updates a .circle/circle.yml to use the leverge orb' )
+  .option( '--debug', 'Enable debug logging - use BUILD_TOOLS_DEBUG=1 for global' )
+  .parse()
+  
+const options = program.opts()
+if ( options.debug ) {
+  enableDebug() // enables debugging in the Utils functions
+}
+
+const repo = await analyzeRepository()
+if ( !repo.isGitRepo ) {
+  errorExit( '***ERROR: This command must be run inside a git repository - run "git init" to proceed' )
+}
+const Container = repo.containerName
+const chartwright = new Chartwright( { Container, debug : options.debug } )
+
+// If there isn't an existing helm chart then we just bootstrap the new chart structure
+// from the current helm template and we're done. This will most likely be the case after
+// a new service has been bootstrapped with ignition.
+if ( !repo.helmChart ) {
+  await chartwright.bootstrap( { Container } )
+  process.exit( 0 )
+}
+
+// Safety First: if there is already an older helm chart, we don't want to overwrite it - we
+// want the user to review / intervene before proceeding.
+const packageRoot = repo.currentDir
+if ( fs.existsSync( `${packageRoot}/helm-old` ) ) {
+  errorExit( '***ERROR: An old helm chart structure already exists - cannot proceed' )
+}
+
+const {
+  Chart,
+  Values,
+  valuesText : ValuesText,
+  imageRegistry : Registry,
+} = repo.helmChart

package/src/circle-orb/circle-orb.yml

@@ -0,0 +1,24 @@
+# See => https://bitbucket.org/leverege/circle-nodejs-ci/src/development/
+version: 2.1
+
+orbs:
+  leverege: leverege/circle-nodejs-ci@0.3.0
+
+workflows:
+  build-and-verify:
+    jobs:
+      - leverege/install-dependencies:
+          context: npm
+      - leverege/code-analysis:
+          requires:
+            - leverege/install-dependencies
+      - leverege/code-coverage:
+          coverage-threshold: 70
+          context: leverege-gcp  # 👈 attaches the secure secrets access to GCP_SA_KEY_JSON
+          enable-redis: false
+          cache-config: '{"type":"inMemory"}' # '{"type":"redis"}'
+          requires:
+            - leverege/install-dependencies
+      - leverege/generate-docs:
+          requires:
+            - leverege/install-dependencies

package/src/helm-charts/api-server/values-local.yaml

@@ -12,10 +12,10 @@ config:
 
 # CNPG overrides
   # stock pgsql
-  MODEL_SQL_HOST: "cnpg-db-psql-stack-rw.cnpg-operands"
-  PG_MODELS_HOST: "cnpg-db-psql-stack-rw.cnpg-operands"
+  MODEL_SQL_HOST: "cnpg-db-psql-stack-pool-rw.cnpg-operands"
+  PG_MODELS_HOST: "cnpg-db-psql-stack-pool-rw.cnpg-operands"
   # timescale
-  PG_HOST: "cnpg-db-tsdb-basic-r.cnpg-operands"
+  PG_HOST: "cnpg-db-tsdb-basic-pool-r.cnpg-operands"
   # imagine db is legacy - newer setups use timescale as the db name
   PG_DATABASE: "timescale"
   PG_DENSE_HISTORY_DATABASE: "timescale"

package/src/helm-charts/authz-server/values-local.yaml

@@ -10,7 +10,7 @@ config:
   LOG_CONFIG: '{"type":"pino","level":"warn"}'
 
   # CNPG override
-  SQL_HOST: "cnpg-db-psql-stack-rw.cnpg-operands" # vs postgres-postgresql
+  SQL_HOST: "cnpg-db-psql-stack-pool-rw.cnpg-operands" # vs postgres-postgresql
 
 serviceMonitor:
   enabled: true

package/src/helm-charts/cnpg-db-psql-stack/cluster.yaml.ovh

@@ -118,3 +118,26 @@ spec:
   method: volumeSnapshot # barmanObjectStore
   backupOwnerReference: self
   immediate: true
+---
+apiVersion: postgresql.cnpg.io/v1
+kind: Pooler
+metadata:
+  name: cnpg-db-psql-stack-pool-rw
+  namespace: cnpg-operands
+spec:
+  cluster:
+    name: cnpg-db-psql-stack
+
+  instances: 2
+  type: rw
+  pgbouncer:
+    poolMode: transaction
+    parameters:
+      # https://www.pgbouncer.org/config.html#generic-settings
+      max_client_conn: "500"
+      default_pool_size: "5"
+      # https://www.pgbouncer.org/config.html#log-settings
+      log_connections: "0"
+      log_disconnections: "0"
+  monitoring:
+    enablePodMonitor: true

package/src/helm-charts/cnpg-db-tsdb-basic/cluster.yaml.ovh

@@ -117,3 +117,26 @@ spec:
   method: volumeSnapshot # barmanObjectStore
   backupOwnerReference: self
   immediate: true
+---
+apiVersion: postgresql.cnpg.io/v1
+kind: Pooler
+metadata:
+  name: cnpg-db-tsdb-basic-pool-rw
+  namespace: cnpg-operands
+spec:
+  cluster:
+    name: cnpg-db-tsdb-basic
+
+  instances: 2
+  type: rw
+  pgbouncer:
+    poolMode: transaction
+    parameters:
+      # https://www.pgbouncer.org/config.html#generic-settings
+      max_client_conn: "500"
+      default_pool_size: "5"
+      # https://www.pgbouncer.org/config.html#log-settings
+      log_connections: "0"
+      log_disconnections: "0"
+  monitoring:
+    enablePodMonitor: true

package/src/helm-charts/fota-server/values-local.yaml

@@ -8,7 +8,7 @@ config:
   NETWORK_ID: 'set-me-in-values-local'
 
   # CNPG override
-  SQL_HOST: "cnpg-db-psql-stack-rw.cnpg-operands" # vs postgres-postgresql
+  SQL_HOST: "cnpg-db-psql-stack-pool-rw.cnpg-operands" # vs postgres-postgresql
 
 serviceMonitor:
-  enabled: false
+  enabled: true

package/src/helm-charts/geotile-server/values-local.yaml

@@ -7,7 +7,7 @@ config:
   UPLOADER_BUCKET: "${PROJECT_ID}-geotile-server"
 
   # CNPG override
-  SQL_HOST: "cnpg-db-psql-stack-rw.cnpg-operands" # vs postgres-postgresql
+  SQL_HOST: "cnpg-db-psql-stack-pool-rw.cnpg-operands" # vs postgres-postgresql
   SQL_PASSWORD_SECRET: "cnpg-db-psql-stack-postgres-pw"
   SQL_PASSWORD_KEY: "password"
 

package/src/helm-charts/prom-operator/prometheus-stack.yaml.ovh

@@ -44,12 +44,21 @@ crds:
   enabled: true
   upgradeJob:
     enabled: true
-    forceConflicts: true # See README-CRD-UPGRADES.md
+    forceConflicts: false # See README-CRD-UPGRADES.md
 
 kubeControllerManager:
   enabled: false
 nodeExporter:
   enabled: false
+
+kubelet:
+  serviceMonitor:
+    metricRelabelings:
+      # prober_probe_duration_seconds is high-cardinality (1 histogram per
+      # container probe × node) and not referenced by any alerts or dashboards
+      - sourceLabels: [__name__]
+        regex: prober_probe_duration_seconds.*
+        action: drop
 defaultRules:
   create: true
   rules:
@@ -61,7 +70,7 @@ defaultRules:
     kubeApiserver: true  # GPT Keep an eye on API server availability
     kubeApiserverAvailability: false
     kubeApiserverSlos: false
-    kubelet: true  # GPT Consider monitoring kubelet status, especially on scaled nodes
+    kubelet: false # GKE manages node/kubelet health; no fat nodes so pod limit irrelevant
     kubeProxy: false
     kubePrometheusGeneral: false
     kubePrometheusNodeRecording: false
@@ -109,7 +118,7 @@ prometheus:
           accessModes: ["ReadWriteOnce"]
           resources:
             requests:
-              storage: 50Gi  # Adjust size as needed
+              storage: 64Gi  # Adjust size as needed
 
 # External access to prometheus may be enabled by creating an htpasswd secret
 # by running this example command:

package/src/helm-charts/prom-operator/rules/postgres-rules.yaml.disabled

@@ -90,14 +90,14 @@ spec:
         summary: "Postgresql high rollback rate on {{ $labels.release }} - {{ $labels.datname }}"
         description: "Ratio of transactions being aborted compared to committed is > 2 %"
 
-    - alert: PostgresqlCommitRateLow
-      expr: rate(pg_stat_database_xact_commit[1m]) < 10
-      for: 5m
-      labels:
-        severity: error
-      annotations:
-        summary: "Postgresql commit rate low on {{ $labels.release }} - {{ $labels.datname }}"
-        description: "Postgres seems to be processing very few transactions"
+#   - alert: PostgresqlCommitRateLow
+#     expr: rate(pg_stat_database_xact_commit[1m]) < 10
+#     for: 5m
+#     labels:
+#       severity: error
+#     annotations:
+#       summary: "Postgresql commit rate low on {{ $labels.release }} - {{ $labels.datname }}"
+#       description: "Postgres seems to be processing very few transactions"
 
     - alert: PostgresqlLowXidConsumption
       expr: rate(pg_txid_current[1m]) < 5

package/src/helm-charts/prom-operator/stackdriver-exporter.yaml.ovh

@@ -5,7 +5,7 @@
 fullnameOverride: "prometheus-stack-stackdriver-metrics"
 
 stackdriver:
-  projectId: "${PROJECT_ID}"
+  projectId: "${PROJECT_NAME}"
   metrics:
     typePrefixes:  >
       agent.googleapis.com/cpu/utilization,

package/src/helm-charts/resource-server/values-local.yaml

@@ -15,7 +15,7 @@ config:
   UPLOADER_LOCATION: "${GCE_REGION}"
 
   # CNPG override
-  SQL_HOST: "cnpg-db-psql-stack-rw.cnpg-operands" # vs postgres-postgresql
+  SQL_HOST: "cnpg-db-psql-stack-pool-rw.cnpg-operands" # vs postgres-postgresql
 # SQL_READ_REPLICATION_CONFIG: '[ { "host": "cnpg-db-psql-stack-r.cnpg-operands" } ]'
 # TRANSACTION_MAX_RETRIES: "5"
 

package/src/helm-charts/rule-engine/values-local.yaml

@@ -5,7 +5,7 @@ config:
   LOG_CONFIG: '{"type":"pino","level":"warn"}'
 
   # CNPG override
-  MODELS_SEQUELIZE_HOST: "cnpg-db-psql-stack-rw.cnpg-operands" # vs postgres-postgresql
+  MODELS_SEQUELIZE_HOST: "cnpg-db-psql-stack-pool-rw.cnpg-operands" # vs postgres-postgresql
   # MODELS_SEQUELIZE_HOST: "cnpg-db-psql-stack-pool-rw.cnpg-operands" # cnpg pooled (pgbouncer)
 
   # Limit the SQL connections from the defaults of 10

package/src/helm-charts/scheduler/values-local.yaml

@@ -5,7 +5,7 @@ config:
   LOG_CONFIG: '{"type":"pino","level":"warn"}'
 
   # CNPG override
-  SQL_HOST: "cnpg-db-psql-stack-rw.cnpg-operands" # vs postgres-postgresql
+  SQL_HOST: "cnpg-db-psql-stack-pool-rw.cnpg-operands" # vs postgres-postgresql
 
 serviceMonitor:
   enabled: true

package/src/helm-charts/traefik/helmup.plugin

@@ -15,12 +15,11 @@ showInstalling "Traefik v3 Load Balancer / Router"
 
 TRAEFIK_NAMESPACE="traefik"
 
-addHelmRepo traefik https://helm.traefik.io/traefik
+OCI_CHART="oci://ghcr.io/traefik/helm/traefik"
 
-[ -z "$TRAEFIK_CHART_VERSION" ] && TRAEFIK_CHART_VERSION="38"
-helm upgrade --install traefik traefik/traefik \
+[ -z "$TRAEFIK_CHART_VERSION" ] && TRAEFIK_CHART_VERSION="38.0.2"
+
+helm upgrade --install traefik $OCI_CHART \
              --namespace $TRAEFIK_NAMESPACE --create-namespace \
              --values  traefik/values.yaml \
              --version $TRAEFIK_CHART_VERSION $HELM_WHAT
-
-removeHelmRepo traefik

package/src/helm-charts/traefik/values-local.yaml

@@ -28,7 +28,7 @@ resources:
 metrics:
   prometheus:
     serviceMonitor:
-      enabled: true # initially disabled until prometheus operator is installed
+      enabled: false # initially disabled until prometheus operator is installed
       interval: 30s
       namespace: prometheus
 

package/src/helm-charts/transponder-dh/values-local.yaml

@@ -10,7 +10,7 @@ config:
   DENSE_HISTORY_TDB_CACHE_KEY_PREFIX: "timescale-dh"
   DENSE_HISTORY_MIGRATE_PAGE_SIZE_LIMIT: "50"
 
-  PG_HOST: "cnpg-db-tsdb-dense-rw.cnpg-operands"
+  PG_HOST: "cnpg-db-tsdb-dense-pool-rw.cnpg-operands"
 # PG_HOST: "cnpg-db-tsdb-dense-pool-rw.cnpg-operands"
 # PG_DATABASE: "timescale" # "imagine" is legacy and default
 

package/src/helm-charts/transponder-tsdb/values-local.yaml

@@ -14,8 +14,8 @@ config:
   LOG_CONFIG: '{"type":"pino","level":"warn"}'
 
 # CNPG override
-  PG_HOST: "cnpg-db-tsdb-basic-rw.cnpg-operands"
-# PG_DATABASE: "timescale" # "imagine" is legacy and default
+  PG_HOST: "cnpg-db-tsdb-basic-pool-rw.cnpg-operands"
+  PG_DATABASE: "timescale" # "imagine" is legacy and default
 
   PG_PASSWORD_SECRET: "cnpg-db-tsdb-basic-postgres-pw"
   PG_PASSWORD_KEY: "password"

package/src/helmup.sh

@@ -5,10 +5,62 @@
 # indicator to the bash-funcs ifNotFromHelmup function
 export _FROM_HELMUP_="true"
 
+INVOKED_AS=`basename $0`
+
 # Load the standard helper functions
 . `build-tools --bashfun`
 
-if [[ ! -z "$BUILD_TOOLS_DEBUG" ]]; then
+function giveHelp() {
+  cat<<HELP
+    `color c "Helm Deployment Orchestrator"`
+
+    Usage: `color g "$INVOKED_AS <service> [service...] [options]"`
+
+    Options:
+      `color g "--dry-run"`   Show what helm would do without applying changes
+      `color g "--debug"`     Enable verbose debug output
+      `color g "--help"`      Show this help message
+
+    Invocation Modes:
+      `color g "helmup"`      Deploy / upgrade one or more services
+      `color g "helmdn"`      Tear down one or more services
+      `color g "helmwhat"`    Dry-run a deployment (same as --dry-run)
+      `color g "helminit"`    Run init scripts for a service
+      `color g "helmboot"`    Re-run the boot scripts for a service
+      `color g "helmcycle"`   Cycle (down + up) one or more services
+
+    Examples:
+      `color g "helmup api-server"`              # deploy a single service
+      `color g "helmup api-server scheduler"`    # deploy multiple services
+      `color g "helmup transponder-*"`           # deploy by glob pattern
+      `color g "helmup transponder-rt/5.1.0"`    # deploy a specific chart version
+      `color g "helmup platform"`                # deploy all platform services
+      `color g "helmup api-server --dry-run"`    # dry run without applying
+      `color g "helmdn api-server"`              # tear down a service
+      `color g "helmcycle api-server"`           # helmdn / helmup a service
+HELP
+}
+
+# command line parsing is handled in getServiceAndChartInfo which allows us to pull the chart
+for arg in "$@"; do
+  case "$arg" in
+    --help|-h)
+      giveHelp && exit 0
+      ;;
+    --debug)
+      export BUILD_TOOLS_DEBUG=1
+      ;;
+    --dry-run)
+      export HELM_WHAT="--dry-run --debug"
+      export K8S_WHAT="--dry-run=server -o yaml"
+      ;;
+    *)
+      SERVICES+=("$arg")
+      ;;
+  esac
+done
+  
+if [[ ! -z "$BUILD_TOOLS_DEBUG_XXX" ]]; then
   cat<<BAD_IDEA
 
   `color y "***WARNING: enabling debugging for helmup is a bad idea..."`
@@ -22,8 +74,6 @@ BAD_IDEA
   exit 1
 fi
 
-SHELMUP_ARGS=("$*")
-
 # artifact-registry => https://console.cloud.google.com/artifacts?project=leverege-registry
 REGISTRY_DOMAIN="us-docker.pkg.dev"
 # open container initiative (oci) => https://opencontainers.org/
@@ -44,7 +94,6 @@ PLATFORM=(
   imagine
   message-processor
   messenger
-  reason
   resource-server
   rule-engine
   scheduler
@@ -58,10 +107,11 @@ AUXILIARY=(
   fota-server
   geotile-server
   overdose
-  pgbouncer,
+  pgbouncer
   pubsub-pulse
   push-notifier
   pusher
+  reason
   transponder-dh
   vin-decoder-server
 )
@@ -237,7 +287,7 @@ function bootstrapLocalSetup() {
 
   if [ ! -d "$SERVICE" ];
   then
-    mkdir $SERVICE
+    mkdir "$SERVICE"
   else
     # If the service dir already exists but the helmup.plugin file is missing
     # then this is most likely a legacy setup - for example, an older setup for
@@ -261,13 +311,13 @@ MISSING_HELMUP_PLUGIN
 
   if [ ! -d "$SERVICE_CHART" ];
   then
-    rm -rf $SERVICE
+    rm -rf "$SERVICE"
     errorExit "Missing $SERVICE chart => $(color y $SERVICE_CHART)"
   fi
 
-  cp -a $SERVICE_CHART .
+  cp -a "$SERVICE_CHART" .
   # because npm tends to not allow .gitinore files the easy way...
-  find $SERVICE -type f -name "gitignore" -execdir mv {} .gitignore \;
+  find "$SERVICE" -type f -name "gitignore" -execdir mv {} .gitignore \;
   overwhelm --genvals
   FIRST_RUN="$SERVICE/helmup.bootstrap"
   if [ -f "$FIRST_RUN" ];
@@ -286,8 +336,8 @@ SERVICE_BOOTSTRAP
       exit 1
     fi
   fi
-  git add $SERVICE
-  ifPluggedIn $SERVICE
+  git add "$SERVICE"
+  ifPluggedIn "$SERVICE"
 }
 
 # If the service is already plugged in the go - otherwise if this is a first
@@ -318,17 +368,15 @@ function installDefaultConfigs() {
 INSTALLDEFAULTCONFIGS
   yesToContinue "to apply the default-configs to the cluster"
 
-  kubecmd="kubectl apply -f default-configs.yaml"
-
-  printf "\nRunning => `color y \"$kubecmd\"`\n\n"
-  eval $kubecmd
-  sendToSlack "$kubecmd"
+  printf "\n`color y \"*** Updating the Default Configs!!!\"` \n\n"
+  kubectl apply -f default-configs.yaml
+  sendToSlack "Updated the Default ConfigMap!"
 }
 
 function errorIfMissingSecret() {
   SECRET="$1"
   kubectl get secret $SECRET &> $DEVNULL
-  if [[ $? = 1 || SECRET = "" ]];
+  if [[ $? = 1 || "${SECRET}" = "" ]];
   then
     errorExit "missing secret `color y $SECRET` - did you `color g 'service-man --new-k8s'` ?"
   fi
@@ -563,17 +611,34 @@ HELMUP_BANNER
 }
 
 function doHelmup() {
-  local culprit="--set helmup.culprit=\"$(gcloud config get account)\""
-  local deployT="--set helmup.deployT=\"$(date -u +'%Y-%m-%dT%H:%M:%SZ')\""
-  local helmup="$@"
-  local doing="Helming => `color g \"$helmup\"`"
+  local culprit="$(gcloud config get account)"
+  local deployT="$(date -u +'%Y-%m-%dT%H:%M:%SZ')"
+  local doing="Helming => `color g \"$*\"`"
   local from=
   [ ! -z "$FROM_VERSIONS" ] && from="`color y \"<= from Versions.json\"`"
   printf "$doing $from\n"
-  eval "$helmup $culprit $deployT"
+  # plugins pass HELMUP as a single quoted string - split it into args without using eval
+  [[ $# -eq 1 ]] && set -- $1
+  "$@" --set "helmup.culprit=$culprit" --set "helmup.deployT=$deployT"
+  
+  # usually the helm command will run without error but a common failure is caused
+  # by the lack of use of our protonVPN profile which causes the i/o timeout from helm
+  if [[ $? -ne 0 ]];
+  then
+    cat<<HELMUP_FAIL
+
+  $RED_ERROR helmup failed for => `color y "$SERVICE"`
+
+  `color y "Please investigate the issue, and if it isn't obvious then check your VPN connectivity!"`
+
+HELMUP_FAIL
+
+    exit 1
+  fi
+
   case "$INVOKED_AS" in
     "helmup"|"helmdn")
-      sendToSlack "$helmup"
+      sendToSlack "$*"
       ;;
     *)
       ;;
@@ -588,7 +653,17 @@ FROM_VERSIONS=
 
 function doInstall() {
 
-  helm repo update &> $DEVNULL
+  # we used to do a Chartmuseum update here which means that we incur the cost of
+  # updating the index and pulling the charts down every time we install a service
+  # which is a lot of overhead for a process that is already pretty slow
+  #
+  #    helm repo update &> $DEVNULL <===DEPRECATED
+  #
+  # the chart museum is now updated in the background when helmup runs, just before
+  # the overwhelm step, so we get the benefits of the updated charts without the
+  # overhead on every install - this is, of course, a race condition, but this will
+  # all go away once we move to artifact registry and can pull the charts directly
+  # from there without the need for a local repo update
 
   for PARAM in "$@";
   do
@@ -598,7 +673,6 @@ function doInstall() {
     # slap additional helm syntax in there if the version is set
     [ ! -z "$CHARTVER" ] && CHARTVER="--version $CHARTVER"
 
-    INVOKED_AS=`basename $0`
     # set the dry-run flag on what
     [ "$INVOKED_AS" == "helmwhat" ] && HELM_WHAT="--dry-run --debug" && K8S_WHAT="--dry-run=server -o yaml"
 
@@ -688,7 +762,7 @@ REBOOTSTRAP
         [ ! -d $SERVICE ] && bootstrapLocalSetup $SERVICE
         CHARTLOC="oci://us-docker.pkg.dev/leverege-registry/stack/charts/transponder" # always use art-reg
         HELMUP="helm upgrade --install $SERVICE $CHARTLOC --values $SERVICE/values.yaml $CHARTVER $HELM_WHAT"
-        doHelmup "$HELMUP"
+        doHelmup $HELMUP
         ;;
         
       *)
@@ -711,17 +785,6 @@ REBOOTSTRAP
         fi
 
         printf "\n--------------------------------------------------------------\nCharting $SERVICE\n"
-        if [ ! -x "$(command -v service-man)" ];
-        then
-          cat<<NO_SERVICE_MAN
-
-$RED_ERROR The Leverege service-man utility is missing - install and rerun helmup
-
-    `color g "npm i -g @leverege/service-man"`
-
-NO_SERVICE_MAN
-          exit 1
-        fi
 
         if [[ "$SERVICE" == *-canary ]];
         then
@@ -734,22 +797,7 @@ NO_SERVICE_MAN
 
         HELMUP="helm upgrade --install $SERVICE $CHARTLOC -f $SERVICE/values.yaml $CHARTVER $HELM_WHAT"
 
-        doHelmup $HELMUP
-        SAVERR=$?
-        if [ $SAVERR -ne 0 ];
-        then
-          cat<<NOTRUNNING
-
-$RED_ERROR helm exited with error status: $SAVERR
-
-$RED_ERROR The $SERVICE chart failed to upgrade or install. There may be an
-           issue with the service's helm charts, or you may have to follow
-           the instructions on the wiki under `color g "Helm Charts via Chartmuseum"`
-           in the `color y "Using the Public Endpoint"` section.
-
-NOTRUNNING
-          exit 1
-        fi
+        doHelmup $HELMUP # will exit on failure, otherwise will continue to the next service
         ;;
     esac
   done
@@ -758,7 +806,7 @@ NOTRUNNING
 }
 
 # ---------- helmup MAIN ----------
-GITTOP=`git rev-parse --show-toplevel`
+helm repo update &>/dev/null & # see comments in the doInstall function for rationale
 
 overwhelm
 [ $? -ne 0 ] && printf "\n\n***Aborting helmup...\n\n" && exit 1
@@ -768,6 +816,6 @@ export GCP_PROJECT_ID="$(getProjectIdFromK8sContext)"
 # Same for the region, but not as prevalent
 export GCE_REGION=`overwhelm -k GCE_REGION`
 
-doInstall $@
+doInstall "${SERVICES[@]}"
 
 displayBanner

package/src/overwhelm.mjs

@@ -51,6 +51,15 @@ const deprecated = ( deprecation ) => {
   log( `${chalk.yellow.bold( '***DEPRECATED:' )} ${deprecation}\n` )
 }
 
+if ( !fs.existsSync( 'overwhelm.yaml' ) ) {
+  log( chalk.red.bold( `
+
+***ERROR overwhelm => missing the overwhelm.yaml file - this command expects
+                      to execute from within a platform-k8s branch
+  ` ) )
+  process.exit( 1 )
+}
+
 // See if we are in a monorepo, which changes some of the rules.
 const gitTop = await shellCmd( 'git rev-parse --show-toplevel' )
 
@@ -65,17 +74,8 @@ const gitBranch = await shellCmd( 'git rev-parse --abbrev-ref HEAD' )
 
 const context = args.context || monoTop || gitBranch
 
-if ( context === 'master' ) {
-  log( '***ERROR: master context is forbidden; change to a different branch\n\n' )
-  process.exit( 1 )
-}
-
-if ( !fs.existsSync( 'overwhelm.yaml' ) ) {
-  log( chalk.red.bold( `
-
-***ERROR overwhelm => missing the overwhelm.yaml file - this command expects
-                      to execute from within a platform-k8s branch
-  ` ) )
+if ( context.match( /^(main|master)$/ ) ) {
+  log( `***ERROR: ${context} context is forbidden; change to a different branch\n\n` )
   process.exit( 1 )
 }
 
@@ -92,32 +92,15 @@ if ( !args.key && !args.fetchctx && !args.matchk8s && fs.existsSync( 'values-def
   }
 }
 
-// Avoid race condition when running overwhelm for the first time on a new
-// cluster and the default-configs.yaml file has yet to be created.
-const defaultConfigs = 'default-configs.yaml'
-if ( !fs.existsSync( defaultConfigs ) ) {
-  fs.closeSync( fs.openSync( defaultConfigs, 'w' ) ) // same as unix touch
-}
-
-// Load the default values and then overlay with the project specific yaml
-// values. This supports the notion of this type of approach in the
-// overwhelm yaml settings:
-//   default:
-//     GCE_MACHINE : "n1-standard-2"
-//
-//   my-project:
-//     GCE_MACHINE : "n1-highcpu-8"
-//
-// fashion, but it is advised not to set a default region with this
-// approach in order to keep us cognizant of exactly where a cluster is
-// being formed.
-//
+// Load the overwhelm.yaml file with the project specific settings to be rendered
+// into the helm values.yaml files. The overwhelm.yaml file is expected to have a
+// default section and one or more context-specific sections.
 const yaml = YAML.load( fs.readFileSync( 'overwhelm.yaml', 'utf-8' ) )
 const over = yaml.default
 
 debug( { over, yaml, }, '<==Contents of the over and yaml structs' )
 
-let cfgs = yaml[context]
+const cfgs = yaml[context]
 
 // If we just need to check context return 0 if cfgs exists, else error.
 if ( args.checkctx ) { process.exit( !cfgs ? 1 : 0 ) }
@@ -132,18 +115,7 @@ if ( !cfgs ) {
   process.exit( 1 )
 }
 
-// Check to see if we are deriving from another configuration.
-if ( cfgs.PROJECT_BASE ) {
-  const baseName = cfgs.PROJECT_BASE
-  const tempCfgs = cfgs
-  cfgs = yaml[baseName]
-  Object.keys( tempCfgs ).forEach( ( key ) => { cfgs[key] = tempCfgs[key] } )
-  log( 'PROJECT_BASE still under evaluation' )
-  log( '\n  cfgs=', cfgs )
-  process.exit( 1 ) // XXX HERE
-}
-
-Object.keys( cfgs ).forEach( ( key ) => { over[key] = cfgs[key] || over[key] } )
+Object.keys( cfgs ).forEach( ( key ) => { over[key] = cfgs[key] ?? over[key] } )
 
 // Expand any variables that reference other variable settings. A perfect
 // example of where this is useful is in the setting of a PROJECT_NAME and
@@ -163,6 +135,11 @@ const varErrors = []
 Object.keys( over ).forEach( ( key ) => {
   if ( !over[key] ) return // no value to work with - skip it
 
+  if ( !isString( over[key] ) ) {
+    varErrors.push( `${key} : ${over[key]} is not a string` )
+    return
+  }
+
   // Look for something in the form of "thing-${VAR1}-${VAR2}-${VARn}-stuff"
   const matches = over[key].match( /\$\{\w+\}/g )
   if ( !matches ) return
@@ -182,9 +159,9 @@ Object.keys( over ).forEach( ( key ) => {
 } )
 
 if ( varErrors.length > 0 ) {
-  log( '***ERROR: unkown variable reference(s) in overwhelm.yaml:\n' )
+  log( chalk.red.bold( '\n***ERROR: unkown variable reference(s) in overwhelm.yaml:\n' ) )
   varErrors.forEach( ( error ) => {
-    log( `     ${error}` )
+    log( `     ${chalk.yellow.bold( error )}` )
   } )
   log( '' )
   process.exit( 1 )
@@ -194,8 +171,8 @@ if ( varErrors.length > 0 ) {
 const yams = glob.sync( 'values-*.yaml' )
 
 // A function to apply the replacements to all of the matching yaml which
-// will error out if it runs into an unknown OVH<tag>.
-const doReplacements = ( yamls, replacements, fileName = 'default-config.yaml' ) => {
+// will error out if it runs into an unknown ${TAG}
+const doReplacements = ( yamls, replacements, fileName = 'unknown-file' ) => {
   let replYaml = yamls
 
   // wrap special chars with quotes if needed, like "#---incident-alerts---"
@@ -217,16 +194,23 @@ const doReplacements = ( yamls, replacements, fileName = 'default-config.yaml' )
     }
   } )
 
-  // Verify all replaceables have been handled, or error out.
-  const missed = replYaml.match( /OVH:<(.*)>/g )
-
-  if ( missed ) {
-    log( '\n***ERROR: missing replacements in overwhelm.yaml for these tags:\n' )
-    missed.forEach( ( miss ) => {
-      log( `     TAG: ${miss.match( /OVH:<(.*)>/ )[1]}` )
-    } )
-    log( '' )
-    process.exit( 1 )
+  // this feels brittle because we have to treat the default-configs.yaml.ovh file
+  // differently due to variables like ${shard} which gets expanded by api-server -
+  // it feels brittle because there could be other yaml files that have similar
+  // variables that are not meant to be replaced by overwhelm - I might eventually
+  // just rip this out ...
+  if ( fileName !== 'default-configs.yaml.ovh' ) {
+    // Verify all replaceables have been handled, or error out.
+    const missed = replYaml.match( /OVH:<(.*)>|\${(.*)}/g )
+
+    if ( missed ) {
+      log( chalk.red.bold( `\n***ERROR: missing replacements in ${fileName} for these tags:\n` ) )
+      missed.forEach( ( miss ) => {
+        log( `     TAG: ${chalk.yellow.bold( miss )}` )
+      } )
+      log( '' )
+      process.exit( 1 )
+    }
   }
 
   // Eat any doubly double quoted values such that
@@ -241,8 +225,7 @@ const doReplacements = ( yamls, replacements, fileName = 'default-config.yaml' )
 
 // Collect all of the yaml from the yams...
 const buildYaml = ( yamls, replacements ) => {
-  const always = [ defaultConfigs ]
-  const values = [ ...always, ...yamls ]
+  const values = [ ...yamls ]
   const loaded = []
   let allYamls = ''
 
@@ -260,7 +243,7 @@ const buildYaml = ( yamls, replacements ) => {
   } )
 
   // Now apply all of the replacements from the overwhelm file.
-  return doReplacements( allYamls, replacements )
+  return doReplacements( allYamls, replacements, 'from-buildYaml' )
 }
 
 // Call buildYaml to replace the OVH strings in the yaml files.
@@ -280,7 +263,7 @@ if ( args.key ) {
 }
 
 // Find all of the potential target dirs. Utilize .gitignore to ignore files
-const ahoy = getDirectories( process.cwd() )
+const ahoy = getDirectories( { path : process.cwd(), respectGitignore : true } )
 
 // Verifies that all values in a config block are quoted
 const validValuesLocalConfig = ( vlf ) => {
@@ -307,7 +290,7 @@ const validValuesLocalConfig = ( vlf ) => {
 ahoy.forEach( ( dir ) => {
   if ( !fs.existsSync( `${dir}/.nohelm` ) && !dir.match( /^\..*/ ) ) {
     // debug( { dir }, 'Generating values' ) // eslint-disable-line no-unused-expressions
-    let valuesOut = [] // An accumulator for all of the output values.
+    let valuesOut = '' // An accumulator for all of the output values.
 
     // Add support for localized values for things like keel (or whatever)
     const valuesLocal = `${dir}/values-local.yaml`
@@ -342,7 +325,7 @@ ahoy.forEach( ( dir ) => {
 
 // we can also apply replaceables to all .ovh files in the project
 glob.sync( '**/*.ovh' ).forEach( ( cfgmap ) => {
-  const target = cfgmap.replace( /.ovh$/, '' )
+  const target = cfgmap.replace( /\.ovh$/, '' )
   deprecated( `rename deprecated ${cfgmap} OVH file to ${target}` )
   let valuesOut = fs.readFileSync( cfgmap, { encoding : 'utf-8' } )
   valuesOut = doReplacements( valuesOut, over, cfgmap )

package/src/helm-charts/cnpg-db-psql-stack/pooler.yaml

@@ -1,22 +0,0 @@
-apiVersion: postgresql.cnpg.io/v1
-kind: Pooler
-metadata:
-  name: cnpg-db-psql-stack-pool-rw
-  namespace: cnpg-operands
-spec:
-  cluster:
-    name: cnpg-db-psql-stack
-
-  instances: 1
-  type: rw
-  pgbouncer:
-    poolMode: transaction
-    parameters:
-      # https://www.pgbouncer.org/config.html#generic-settings
-      max_client_conn: "500"
-      default_pool_size: "5"
-      # https://www.pgbouncer.org/config.html#log-settings
-      log_connections: "0"
-      log_disconnections: "0"
-  monitoring:
-    enablePodMonitor: true

package/src/helm-charts/cnpg-db-tsdb-basic/pooler.yaml

@@ -1,22 +0,0 @@
-apiVersion: postgresql.cnpg.io/v1
-kind: Pooler
-metadata:
-  name: cnpg-db-tsdb-basic-pool-rw
-  namespace: cnpg-operands
-spec:
-  cluster:
-    name: cnpg-db-tsdb-basic
-
-  instances: 1
-  type: rw
-  pgbouncer:
-    poolMode: transaction
-    parameters:
-      # https://www.pgbouncer.org/config.html#generic-settings
-      max_client_conn: "500"
-      default_pool_size: "5"
-      # https://www.pgbouncer.org/config.html#log-settings
-      log_connections: "0"
-      log_disconnections: "0"
-  monitoring:
-    enablePodMonitor: true