@leverege/build-tools 2.81.1 → 2.83.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@leverege/build-tools",
-  "version": "2.81.1",
+  "version": "2.83.0",
   "description": "A collection of build / support tools for Leverege developers",
   "main": "index.js",
   "repository": {
@@ -65,7 +65,7 @@
   "license": "SEE LICENSE IN LICENSE.md",
   "dependencies": {
     "@leverege/jsdoc-template": "^1.0.1",
-    "chalk": "^5.4.1",
+    "chalk": "^5.6.0",
     "command-line-args": "^6.0.1",
     "command-line-usage": "^7.0.3",
     "commander": "^14.0.0",
@@ -75,13 +75,13 @@
     "glob": "^11.0.3",
     "handlebars": "^4.7.8",
     "ignore": "^7.0.5",
-    "inquirer": "^12.9.0",
+    "inquirer": "^12.9.3",
     "js-yaml": "^4.1.0",
     "jsdoc": "^4.0.4",
     "ms": "^2.1.3",
-    "npm-registry-fetch": "^18.0.2",
+    "npm-registry-fetch": "^19.0.0",
     "ora": "^8.2.0",
-    "p-limit": "^6.2.0",
+    "p-limit": "^7.1.0",
     "package-up": "^5.0.0",
     "readline-sync": "^1.4.10",
     "semver": "^7.7.2",
@@ -89,10 +89,10 @@
     "simple-git": "^3.28.0",
     "sloc": "^0.3.2",
     "toml": "^3.0.0",
-    "zx": "^8.7.2"
+    "zx": "^8.8.0"
   },
   "devDependencies": {
     "@leverege/eslint-config-leverege": "^5.0.1",
-    "npm": "^11.5.1"
+    "npm": "^11.5.2"
   }
 }

package/src/bash-funcs

@@ -149,6 +149,27 @@ REPOADD
   helm repo update &> $DEVNULL
 }
 
+# *** DEPRECATED *** REMOVE AFTER BITNAMI ERADICATED ***
+# *** DEPRECATED *** REMOVE AFTER BITNAMI ERADICATED ***
+# *** DEPRECATED *** REMOVE AFTER BITNAMI ERADICATED ***
+#
+function addBitnamiRepoDEPRECATED() {
+  if [ ! -z "$BITNAMI_LATEST" ] || [ "$1" == "latest" ];
+  then
+    BITNAMI_URL="https://charts.bitnami.com/bitnami"
+  else
+    cat<<CATNAMI
+    Using the pre 2022 legacy charts from bitnami. To use latest run:
+
+        `color y "BITNAMI_LATEST=1 $HELMUP_ARGS"`
+
+CATNAMI
+    BITNAMI_URL="https://raw.githubusercontent.com/bitnami/charts/eb5f9a9513d987b519f0ecd732e7031241c50328/bitnami"
+  fi
+
+  addHelmRepo bitnami $BITNAMI_URL
+}
+
 function removeHelmRepo(){
   [ ! -z "$SKIP_RM_HELM_REPO" ] && return
   helm repo remove $1 &> $DEVNULL

package/src/bitnami-audit.mjs

@@ -0,0 +1,169 @@
+#!/usr/bin/env node
+//
+// This script is used to scan all of the projects in the auditProject array
+// to look for any deployments that are still using the bitnami images. The
+// output from this script can then be imported / upload / replace existing in
+// the BitnamiImageRegistry gsheet to show progess on the migration effort.
+//
+import fs from 'node:fs'
+import chalk from 'chalk'
+import semver from 'semver'
+
+import {
+  debug,
+  log,  
+  shellCmd,
+} from './Utils.mjs'
+
+const auditProjects = [
+  'dev-controltek',
+  'dev-cox-health',
+  'dev-pitcrew',
+  'xxx-bogus', // expect warning
+  'dev-recovr-auto',
+  'dev-schnucks',
+  'dev-tpi',
+  'discount-tire-prd',
+  'discount-tire-stg',
+  'leverege-cnpg-20240729',
+  'leverege-ops-20250730',
+  'leverege-demo', // expect warning
+  'leverege-elastic',
+  'leverege-onboarding',
+  'leverege-sandbox',
+  'nimbelink-demo',
+  'prd-controltek',
+  'prd-cox-health',
+  'prd-generic-kudelski',
+  'prd-les-schwab',
+  'prd-recovr-auto',
+  'prd-skymirr',
+  'prd-tpi',
+  'prd-valvoline',
+  'qa-cox-health', // expect warning
+  'siren-marine-dev',
+  'siren-marine-prd',
+  'siren-marine-staging',
+  'siren-marine-test',
+  'stg-cox-health',
+  'stg-generic-kudelski',
+  'stg-recovr-auto',
+  'stg-recovr-perf',
+]
+
+const gitCheckoutCleanBranch = async ( branch = 'bootstrap-k8s' ) => {
+  // find and remove cruft, like generated values.yaml files
+  await shellCmd( 'find . -not \\( -path \\*junk -prune -o -path \\*helm-elasticsearch -prune \\) -name values.yaml -exec rm {} \\;' )
+
+  // start with a clean baseline
+  await shellCmd( 'git checkout bootstrap-k8s' )
+  await shellCmd( 'git clean -d --force' )
+
+  // checkout the target branch
+  await shellCmd( `git checkout ${branch}` )
+}
+
+const setK8sContext = async ( project ) => {
+  await shellCmd( 'overwhelm --fetchctx' )
+  await shellCmd( 'overwhelm --matchk8s' )
+}
+
+const prepareContext = async ( project ) => {
+  await gitCheckoutCleanBranch( project )
+  await setK8sContext( project )
+}
+
+const filterBitnamiImages = ( project, imagesJSON ) => {
+  const data = JSON.parse( imagesJSON )
+  const seen = new Set()
+
+  for ( const item of data.items ) {
+    const ns = item.metadata.namespace
+    let service = item.metadata.name
+
+    // Collapse services
+    service = service.replace( /elasticsearch8-(coordinating|data|master|ingest)/, 'elasticsearch' )
+    service = service.replace( /redis-(master|replicas)/, 'redis' )
+    service = service.replace( /valkey-(primary|replicas)/, 'valkey' )
+
+    const containers = item.spec?.template?.spec?.containers || []
+    for ( const c of containers ) {
+      let img = c.image
+
+      // Only keep bitnami images
+      if ( !/bitnami/.test( img ) ) continue
+
+      // Registry decluttering
+      img = img.replace( /^docker\.io\/bitnami/, 'bitnami' )
+      img = img.replace( /^us-docker\.pkg\.dev\/leverege-registry\/bitnami-mirror\/images/, 'leverege' )
+      img = img.replace( /^us-docker\.pkg\.dev\/leverege-registry\/system\/images/, 'leverege/system' )
+
+      const [ image, fullVersion = 'latest' ] = img.split( ':' )
+      const displayVersion = fullVersion.replace( /-debian-.*$/, '' )
+
+      const row = [ project, ns, service, image, displayVersion, fullVersion ].join( ',' )
+      if ( !seen.has( row ) ) {
+        debug( row )
+        seen.add( row )
+      }
+    }
+  }
+  return seen
+}
+
+const imageSummary = new Map()
+
+const runner = async ( projects ) => {
+  const CSV_FILE = '../BitnamiImageRegistry.csv'
+  const fd = fs.openSync( CSV_FILE, 'w' )
+  fs.writeSync( fd, 'Project,Namespace,Service,Image,Version,Tag\n' )
+
+  for ( const project of projects ) {
+    log( `Project [${project}]` )
+    try {
+      await prepareContext( project ) // eslint-disable-line no-await-in-loop
+    } catch ( error ) {
+      log( `Project or context is invalid, skipping [${chalk.redBright( project )}]` )
+      continue
+    }
+
+    // eslint-disable-next-line no-await-in-loop
+    const k8sImageDump = await shellCmd( 'kubectl get deploy,statefulset -A -o json' )
+    const bitnamiImages = filterBitnamiImages( project, k8sImageDump )
+    
+    for ( const line of bitnamiImages ) {
+      fs.writeSync( fd, `${line}\n` )
+
+      const [ , , , image, version, tag ] = line.split( ',' )
+      if ( /^leverege/.test( image ) ) continue // we don't need leverege in the summary
+
+      if ( !imageSummary.has( image ) ) {
+        imageSummary.set( image, { version : new Set(), tag : new Set() } )
+      }
+      imageSummary.get( image ).version.add( version )
+      imageSummary.get( image ).tag.add( tag )
+    }
+  }
+  fs.closeSync( fd )
+
+  log( '\n==== Bitnami Image Summary ====\n' )
+
+  log( 'IMAGES=(' )
+
+  for ( const [ image, { tag, version } ] of imageSummary.entries() ) {
+    const sortedTags = Array.from( tag ).sort( semver.rcompare )
+    for ( const t of sortedTags.slice( 0, 15 ) ) {
+      log( `  "${image}:${t}"` )
+    }
+
+    // Optional: also log a human-friendly display version list
+    const sortedVersions = Array.from( version ).sort( semver.rcompare )
+    log( `  # Display versions: ${sortedVersions.join( ', ' )}` )
+  }
+
+  log( ')' )
+
+  gitCheckoutCleanBranch() // back to clean bootstrap-k8s
+}
+
+await runner( auditProjects )

package/src/bitnami-to-leverege-artifacts.sh

@@ -0,0 +1,140 @@
+#!/bin/bash
+#
+# This script was used to bootstrap the leverege-registry bitnmai-mirror
+# folder with initial images and charts needed for the migration away from
+# bitnami. I left it here as an artifact and can be deleted once the entire
+# migration is complete.
+#
+# CONFIGURATION
+PROJECT_ID="leverege-registry"
+REGION="us" # adjust if needed
+MIRROR_ROOT="${REGION}-docker.pkg.dev/${PROJECT_ID}/bitnami-mirror"
+DOCKER_PREFIX="${MIRROR_ROOT}/images"
+CHARTS_PREFIX="oci://${MIRROR_ROOT}/charts"
+
+# Images to mirror
+IMAGES=(
+  "bitnami/postgresql:11.14.0-debian-10-r28"
+  "bitnami/postgresql:11.7.0-debian-10-r99"
+  # Display versions: 11.14.0, 11.7.0
+  "bitnami/postgres-exporter:0.10.0-debian-10-r172"
+  "bitnami/postgres-exporter:0.8.0-debian-10-r106"
+  # Display versions: 0.10.0, 0.8.0
+  "bitnami/redis:7.4.3-debian-12-r0"
+  "bitnami/redis:7.4.2-debian-12-r2"
+  "bitnami/redis:7.4.1-debian-12-r0"
+  "bitnami/redis:7.2.5-debian-12-r2"
+  "bitnami/redis:7.2.5-debian-12-r0"
+  "bitnami/redis:7.2.4-debian-12-r12"
+  "bitnami/redis:7.2.3-debian-11-r1"
+  "bitnami/redis:7.0.12-debian-11-r34"
+  "bitnami/redis:7.0.11-debian-11-r12"
+  "bitnami/redis:7.0.8-debian-11-r0"
+  "bitnami/redis:6.2.1-debian-10-r36"
+  "bitnami/redis:6.0.3-debian-10-r2"
+  # Display versions: 7.4.3, 7.4.2, 7.4.1, 7.2.5, 7.2.4, 7.2.3, 7.0.12, 7.0.11, 7.0.8, 6.2.1, 6.0.3
+  "bitnami/redis-exporter:1.70.0-debian-12-r2"
+  "bitnami/redis-exporter:1.67.0-debian-12-r7"
+  "bitnami/redis-exporter:1.63.0-debian-12-r1"
+  "bitnami/redis-exporter:1.61.0-debian-12-r2"
+  "bitnami/redis-exporter:1.61.0-debian-12-r0"
+  "bitnami/redis-exporter:1.58.0-debian-12-r7"
+  "bitnami/redis-exporter:1.55.0-debian-11-r2"
+  "bitnami/redis-exporter:1.52.0-debian-11-r25"
+  "bitnami/redis-exporter:1.50.0-debian-11-r13"
+  "bitnami/redis-exporter:1.45.0-debian-11-r26"
+  "bitnami/redis-exporter:1.20.0-debian-10-r16"
+  "bitnami/redis-exporter:1.6.1-debian-10-r7"
+  # Display versions: 1.70.0, 1.67.0, 1.63.0, 1.61.0, 1.58.0, 1.55.0, 1.52.0, 1.50.0, 1.45.0, 1.20.0, 1.6.1
+  "bitnami/elasticsearch:8.18.0-debian-12-r2"
+  "bitnami/elasticsearch:8.17.3-debian-12-r0"
+  "bitnami/elasticsearch:8.16.1-debian-12-r0"
+  "bitnami/elasticsearch:8.15.2-debian-12-r2"
+  "bitnami/elasticsearch:8.12.2-debian-12-r0"
+  "bitnami/elasticsearch:8.12.1-debian-11-r2"
+  "bitnami/elasticsearch:8.12.1-debian-11-r1"
+  "bitnami/elasticsearch:8.12.0-debian-11-r1"
+  "bitnami/elasticsearch:8.11.1-debian-11-r0"
+  "bitnami/elasticsearch:8.10.3-debian-11-r1"
+  "bitnami/elasticsearch:8.9.1-debian-11-r2"
+  # Display versions: 8.18.0, 8.17.3, 8.16.1, 8.15.2, 8.12.2, 8.12.1, 8.12.0, 8.11.1, 8.10.3, 8.9.1
+  "bitnami/sealed-secrets-controller:0.29.0-debian-12-r1"
+  "bitnami/valkey:8.1.3-debian-12-r1"
+  # Display versions: 8.1.30
+)
+
+# Charts to mirror (chart:version)
+CHARTS=(
+  "bitnamicharts/elasticsearch:22.1.0"
+  "bitnamicharts/elasticsearch:21.6.3"
+  "bitnamicharts/elasticsearch:21.3.20"
+  "bitnamicharts/elasticsearch:19.17.6"
+# "bitnamicharts/postgresql:10.16.2" # pulled manually from ancient bitnami source
+  "bitnamicharts/redis:20.13.4"
+  "bitnamicharts/valkey:3.0.22"
+)
+
+echo "🔑 Authenticating to Artifact Registry..."
+gcloud auth configure-docker ${REGION}-docker.pkg.dev -q &> /dev/null
+gcloud auth application-default print-access-token | helm registry login \
+  -u oauth2accesstoken --password-stdin ${REGION}-docker.pkg.dev
+
+echo "📦 Ensuring Artifact Registry repos exist..."
+gcloud artifacts repositories create bitnami-mirror \
+  --repository-format=docker \
+  --location=${REGION} \
+  --project=${PROJECT_ID} \
+  --description="Mirror of Bitnami images/charts" &> /dev/null \
+  || echo "✅ Docker repo already exists."
+
+# Check if crane is available (brew install crane)
+if command -v crane &>/dev/null;
+then
+  echo "🚀 Using crane for image copies (fast + multi-arch safe)"
+  USE_CRANE=true
+else
+  echo "🐳 crane not found, falling back to Docker CLI"
+  USE_CRANE=false
+fi
+
+echo "⬇️ Pulling & pushing container images..."
+for IMAGE in "${IMAGES[@]}"; do
+  SRC_IMAGE="docker.io/${IMAGE}"
+  NAME=$(echo "${IMAGE}" | cut -d: -f1 | awk -F/ '{print $NF}')
+  TAG=$(echo "${IMAGE}" | cut -d: -f2)
+  DEST_IMAGE="${DOCKER_PREFIX}/${NAME}:${TAG}"
+
+  echo "Mirroring ${SRC_IMAGE} → ${DEST_IMAGE}"
+
+  if [ "$USE_CRANE" = true ];
+  then
+    # crane copies the whole manifest list (multi-arch safe)
+    cat<<CRANE_CAT
+    crane copy "${SRC_IMAGE}" "${DEST_IMAGE}"
+CRANE_CAT
+    crane copy "${SRC_IMAGE}" "${DEST_IMAGE}"
+  else
+    docker pull --platform linux/amd64 "${SRC_IMAGE}"
+    docker tag "${SRC_IMAGE}" "${DEST_IMAGE}"
+    docker push "${DEST_IMAGE}"
+
+    # clean up local docker - the || true ensures the script won’t fail if the
+    # image was already removed or the tag isn’t present
+    docker rmi -f "${SRC_IMAGE}" "${DEST_IMAGE}" || true
+  fi
+done
+
+echo "⬇️ Pulling & pushing Helm charts..."
+for ENTRY in "${CHARTS[@]}"; do
+  CHART=$(echo $ENTRY | cut -d: -f1)
+  VERSION=$(echo $ENTRY | cut -d: -f2)
+  echo "Mirroring chart ${CHART}:${VERSION}"
+  #cat<<CHART_MOVER
+  helm pull oci://registry-1.docker.io/${CHART} --version ${VERSION}
+  CHART_TARBALL=$(basename ${CHART})-${VERSION}.tgz
+  helm push "${CHART_TARBALL}" ${CHARTS_PREFIX}
+  rm -f "${CHART_TARBALL}"
+#CHART_MOVER
+done
+
+echo "✅ Mirror complete."

package/src/count-lines-of-code.mjs

@@ -2,6 +2,8 @@
 import path from 'node:path'
 import { $, fs, chalk, argv } from 'zx'
 
+import { log } from './Utils.mjs'
+
 const srcOnly = argv.srcOnly
 
 const $$ = $( {
@@ -21,10 +23,10 @@ if ( !hasWorkspace ) {
     for ( let i = 0; i < dirs.length; i++ ) {
       const dir = dirs[i]
 
-      await getLinesOfCodeForDirectory( path.join( rootDir, 'packages', dir ) )
+      await getLinesOfCodeForDirectory( path.join( rootDir, 'packages', dir ) ) // eslint-disable-line no-await-in-loop
     }
   } else {
-    console.log( chalk.redBright( 'No workspace directories for which to count code' ) )
+    log( chalk.redBright( 'No workspace directories for which to count code' ) )
   }
 }
 
@@ -33,13 +35,13 @@ async function getLinesOfCodeForDirectory( directory ) {
   const hasSrc = await fs.exists( srcDir )
   
   if ( hasSrc ) {
-    console.log( `${chalk.greenBright( srcOnly ? srcDir : directory )}:` )
+    log( `${chalk.greenBright( srcOnly ? srcDir : directory )}:` )
     if ( srcOnly ) {
       await $$`sloc ${srcDir} --format cli-table`
     } else {
       await $$`sloc ${directory} --format cli-table -i '^(src|test)/.*' -e node_modules`
     }
   } else {
-    console.log( chalk.redBright( `No src directory at ${directory}` ) )
+    log( chalk.redBright( `No src directory at ${directory}` ) )
   }
 }

package/src/crane-lift.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+#
+# just a simple wrapper round crane docker lifter
+#
+IMAGE_WITH_TAG="${1}" # like valkey:8.1.3-debian-12-r1
+
+REGISTRY_LEVEREGE="us-docker.pkg.dev/leverege-registry/bitnami-mirror/images"
+REGISTRY_BITNAMI="docker.io/bitnami"
+
+crane copy "${REGISTRY_BITNAMI}/${IMAGE_WITH_TAG}" \
+           "${REGISTRY_LEVEREGE}"/${IMAGE_WITH_TAG}"

package/src/helm-charts/elasticsearch8/elasticsearch-local-dev.yaml

@@ -8,7 +8,16 @@
 # for each of the 4 ES node types in the cluster. See prd-recovr-auto for an
 # example of a big beefy system.
 #
+# Moves us away from bitnami hosted charts/images
+#
+image:
+  registry: us-docker.pkg.dev
+  repository: leverege-registry/bitnami-mirror/images/elasticsearch
+# tag: 8.18.0
+
 global:
+  security:
+    allowInsecureImages: true
   storageClass: "ssd"
 
 fullnameOverride: "elasticsearch8"

package/src/helm-charts/elasticsearch8/elasticsearch-local-prd.yaml

@@ -8,7 +8,16 @@
 # for each of the 4 ES node types in the cluster. See prd-recovr-auto for an
 # example of a big beefy system.
 #
+# Moves us away from bitnami hosted charts/images
+#
+image:
+  registry: us-docker.pkg.dev
+  repository: leverege-registry/bitnami-mirror/images/elasticsearch
+# tag: 8.18.0
+
 global:
+  security:
+    allowInsecureImages: true
   storageClass: "ssd"
 
 fullnameOverride: "elasticsearch8"

package/src/helm-charts/elasticsearch8/helmup.plugin

@@ -2,11 +2,11 @@
 #
 showInstalling "Elastic Search 8"
 
+OCI_CHART="oci://us-docker.pkg.dev/leverege-registry/bitnami-mirror/charts/elasticsearch"
+
 # *** WARNING TO FUTURE SELF *** v22 of the chart is ES v9!!!
 [ -z "$ELASTIC_CHART_VERSION" ] && ELASTIC_CHART_VERSION="21.6.3"
 
-OCI_CHART="oci://registry-1.docker.io/bitnamicharts/elasticsearch"
-
 LOCAL_ES_VALUES="elasticsearch8/elasticsearch-local-dev.yaml"
 #LOCAL_ES_VALUES="elasticsearch8/elasticsearch-local-prd.yaml"
 [ ! -f "$LOCAL_ES_VALUES" ] && exitOnError 1 "Missing the local ES config file => `color y $LOCAL_ES_VALUES`"
@@ -14,4 +14,4 @@ LOCAL_ES_VALUES="elasticsearch8/elasticsearch-local-dev.yaml"
 helm upgrade --install elasticsearch8 $OCI_CHART \
              --namespace elastic --create-namespace \
              --values $LOCAL_ES_VALUES \
-             --version $ELASTIC_CHART_VERSION $HELM_WHAT
+             --version $ELASTIC_CHART_VERSION $HELM_WHAT

package/src/helm-charts/postgres/README.md

@@ -0,0 +1,6 @@
+## Legacy PostgreSQL 11
+
+This configuration was temporarily resurrected for supporting the bitnami
+migrations. There are still quite a few legacy deployments using this
+ancient psql approach. We use this config to point the servers at the Leverege
+artifact registry for the mirrored bitnami images.

package/src/helm-charts/postgres/helmdn.plugin

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+echo 'NO!'

package/src/helm-charts/postgres/helmup.plugin

@@ -0,0 +1,21 @@
+#!/bin/bash
+#
+showInstalling "PostgreSQL Database (legacy)"
+
+errorIfMissingSecret authz-postgres
+
+OCI_CHART="oci://us-docker.pkg.dev/leverege-registry/bitnami-mirror/charts/postgresql"
+
+[ -z "$PSQL_CHART_VERSION" ] && PSQL_CHART_VERSION="10.16.2"
+
+helm upgrade --install postgres $OCI_CHART          \
+             --values  postgres/postgres-local.yaml \
+             --version $PSQL_CHART_VERSION $HELM_WHAT
+
+#
+# legacy
+#helm upgrade --install postgres bitnami/postgresql  \
+#             --values  postgres/postgres-local.yaml \
+#             --version 10 $HELM_WHAT
+#
+#removeHelmRepo bitnami

package/src/helm-charts/postgres/postgres-local.yaml

@@ -0,0 +1,51 @@
+# Moves us away from bitnami hosted charts/images
+#
+image:
+  registry: us-docker.pkg.dev
+  repository: leverege-registry/system/images/leverege-pgsql11-repack-kubectl
+  tag: 11.14-kctl.3
+
+postgresqlExtendedConf:
+  maxWorkerProcesses: 4
+  maxParallelWorkers: 4
+  maxConnections: 500
+  shared_buffers: 1GB
+  effectiveCacheSize: 4GB
+  maintenanceWorkMem: 1GB
+  workMem: 10MB
+
+persistence:
+  storageClass: "ssd"
+# size: 16Gi # default 8Gi
+
+resources:
+  requests:
+    memory: 1Gi
+    cpu: 750m
+
+postgresqlDatabase: authz
+existingSecret: authz-postgres
+
+metrics:
+  enabled: true
+  image:
+    registry: us-docker.pkg.dev
+    repository: leverege-registry/bitnami-mirror/images/postgres-exporter
+#   tag: 0.10.0-debian-10-r172
+
+primary:
+  tolerations:
+  - key: "database"
+    operator: "Equal"
+    value: "true"
+    effect: "NoSchedule"
+
+  affinity:
+    nodeAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+        nodeSelectorTerms:
+        - matchExpressions:
+          - key: target-env
+            operator: In
+            values:
+            - database

package/src/helm-charts/prom-operator/rules/kubestate-rules.yaml

@@ -229,7 +229,7 @@ spec:
                 LABELS = {{ $labels }}
         - alert: KubernetesReplicasetReplicasMismatch
           expr: kube_replicaset_spec_replicas != kube_replicaset_status_ready_replicas
-          for: 10m
+          for: 15m # awesome was => 10m
           labels:
             severity: warning
           annotations:

package/src/helm-charts/prom-operator/stackdriver-exporter.yaml.ovh

@@ -34,7 +34,8 @@ stackdriver:
 serviceMonitor:
   enabled: true
   namespace: prometheus
-  interval: "30s"
+  interval: "60s"        # Align with Stackdriver's 60s data resolution
+# honorTimestamps: false # Avoid duplicate timestamp conflicts (eliminates dup timestamps?)
   relabelings:
     # Legacy labelmap used to preserve compatibility with pre-operator Grafana dashboards
     # Can be removed once legacy Prometheus stack is deprecated

package/src/helm-charts/redis/helmup.plugin

@@ -2,10 +2,10 @@
 
 showInstalling "Redis"
 
-OCI_CHART="oci://registry-1.docker.io/bitnamicharts/redis"
+OCI_CHART="oci://us-docker.pkg.dev/leverege-registry/bitnami-mirror/charts/redis"
 
 [ -z "$REDIS_CHART_VERSION" ] && REDIS_CHART_VERSION="20.13.4"
 
-helm upgrade --install redis $OCI_CHART       \
+helm upgrade --install redis $OCI_CHART        \
              --values  redis/values-local.yaml \
              --version $REDIS_CHART_VERSION $HELM_WHAT

package/src/helm-charts/redis/values-local.yaml

@@ -1,4 +1,14 @@
+#
+# Moves us away from bitnami hosted charts/images
+#
+image:
+  registry: us-docker.pkg.dev
+  repository: leverege-registry/bitnami-mirror/images/redis
+# tag: 7.4.3-debian-12-r0
+
 global:
+  security: # this allows us to use local images
+    allowInsecureImages: true
   storageClass: ssd
 
 auth:
@@ -70,6 +80,10 @@ networkPolicy:
 
 metrics:
   enabled: true
+  image:
+    registry: us-docker.pkg.dev
+    repository: leverege-registry/bitnami-mirror/images/redis-exporter
+#   tag: 1.74.0-debian-12-r2
   serviceMonitor:
     enabled: true # set to true for prometheus operator
     namespace: "prometheus"

package/src/helm-charts/traefik/helmup.plugin

@@ -17,7 +17,7 @@ TRAEFIK_NAMESPACE="traefik"
 
 addHelmRepo traefik https://helm.traefik.io/traefik
 
-[ -z "$TRAEFIK_CHART_VERSION" ] && TRAEFIK_CHART_VERSION="36"
+[ -z "$TRAEFIK_CHART_VERSION" ] && TRAEFIK_CHART_VERSION="37"
 helm upgrade --install traefik traefik/traefik \
              --namespace $TRAEFIK_NAMESPACE --create-namespace \
              --values  traefik/values.yaml \

package/src/helm-charts/valkey/helmup.plugin

@@ -2,10 +2,10 @@
 
 showInstalling "Valkey"
 
-OCI_CHART="oci://registry-1.docker.io/bitnamicharts/valkey"
+OCI_CHART="oci://us-docker.pkg.dev/leverege-registry/bitnami-mirror/charts/valkey"
 
-[ -z "$VALKEY_CHART_VERSION" ] && VALKEY_CHART_VERSION="2.2.4"
+[ -z "$VALKEY_CHART_VERSION" ] && VALKEY_CHART_VERSION="3.0.22"
 
-helm upgrade --install valkey $OCI_CHART       \
+helm upgrade --install valkey $OCI_CHART        \
              --values  valkey/valkey-local.yaml \
              --version $VALKEY_CHART_VERSION $HELM_WHAT

package/src/helm-charts/valkey/valkey-local.yaml

@@ -1,5 +1,11 @@
+image:
+  registry: us-docker.pkg.dev
+  repository: leverege-registry/bitnami-mirror/images/valkey
+# tag: 8.1.3-devian-12-r1
 
 global:
+  security:
+    allowInsecureImages: true
   storageClass: "ssd"
 
 architecture: replication
@@ -11,7 +17,7 @@ auth:
 primary:
   replicaCount: 1
 
-  resourcesPreset: "nano"
+  resourcesPreset: "small"
   resources:
     requests:
       cpu: 250m
@@ -77,6 +83,10 @@ sentinel:
 
 metrics:
   enabled: true
+  image:
+    registry: us-docker.pkg.dev
+    repository: leverege-registry/bitnami-mirror/images/redis-exporter
+#   tag: 1.74.0-debian-12-r2
   serviceMonitor:
     enabled: true
     namespace: "prometheus"

package/src/helm-charts/velero/helmup.plugin

@@ -7,7 +7,7 @@ addHelmRepo vmware-tanzu https://vmware-tanzu.github.io/helm-charts
 [ -z "$VELERO_CHART_VERSION" ] && VELERO_CHART_VERSION="10"
 #
 # Build the bucket, region and SA email variables and use --set to mod the
-# chart values as opposed to using an OVH:<label> approach. This eliminates
+# chart values as opposed to using an ${label} approach. This eliminates
 # the need for an overwhelm generated values.yaml file which also contains
 # all of the contamination from the default-values.yaml file.
 #

package/src/helmup.sh

@@ -83,25 +83,6 @@ function isLeveregeService() {
   echo $answer
 }
 
-# audit trail via slack
-# support for legacy bitnami charts
-function addBitnamiRepo() {
-  if [ ! -z "$BITNAMI_LATEST" ] || [ "$1" == "latest" ];
-  then
-    BITNAMI_URL="https://charts.bitnami.com/bitnami"
-  else
-    cat<<CATNAMI
-    Using the pre 2022 legacy charts from bitnami. To use latest run:
-
-        `color y "BITNAMI_LATEST=1 $HELMUP_ARGS"`
-
-CATNAMI
-    BITNAMI_URL="https://raw.githubusercontent.com/bitnami/charts/eb5f9a9513d987b519f0ecd732e7031241c50328/bitnami"
-  fi
-
-  addHelmRepo bitnami $BITNAMI_URL
-}
-
 # Accepts 3 paramaters - 1 mandatory.
 #   bindWorkloadIdentity <Service Account> <Namespace> <Target Service Account>
 # where:
@@ -265,6 +246,22 @@ SERVICE_BOOTSTRAP
   ifPluggedIn $SERVICE
 }
 
+# If the service is already plugged in the go - otherwise if this is a first
+# time install of a known service then bootstrap it and go. Otherwise return
+# a status of 1 to indicate to continue processing.
+function ifPluggedInOrBootstrapped() {
+  local svc_name="$1"
+
+  ifPluggedIn $svc_name && return 0 # if it's already plugged in then go
+
+  # check to see if build-tools knows about the new service in its configs
+  local svc_plugin="$(build-tools --reporoot)/src/helm-charts/${svc_name}/helmup.plugin"
+  [[ ! -x "$svc_plugin" ]] && return 1 # indicates unknown
+
+  bootstrapLocalSetup $svc_name
+  return 0
+}
+
 function installDefaultConfigs() {
   showInstalling "Default Configs"
 
@@ -387,7 +384,7 @@ spec:
           containers:
           - name: cron-zombie-killer
             imagePullPolicy: IfNotPresent
-            image: bitnami/kubectl
+            image: registry.k8s.io/kubectl:v1.30.3
             command:
               - "/bin/sh"
             args:
@@ -581,8 +578,9 @@ CATBACKUP
       helmdn $SERVICE
     fi
 
-    # If we're referencing a plugin then execute it and continue
-    ifPluggedIn $SERVICE && continue
+    # If the service is already plugged in or can be bootstrapped
+    # during initial install then do so here
+    ifPluggedInOrBootstrapped $SERVICE && continue
 
     case "$SERVICE" in
       "platform")
@@ -594,18 +592,6 @@ CATBACKUP
         doInstall "${SYSTEM[@]}"
         ;;
 
-      "argocd")
-        bootstrapLocalSetup $SERVICE
-        ;;
-
-      cnpg-db-*)
-        bootstrapLocalSetup $SERVICE
-        ;;
-
-      "cnpg-operator")
-        bootstrapLocalSetup $SERVICE
-        ;;
-
       "cronZombieKiller"|"zombie-killer")
         installCronZombieKiller # eradicate the gke 1.20+ zombies
         ;;
@@ -614,10 +600,6 @@ CATBACKUP
         installDefaultConfigs
         ;;
 
-      "elasticsearch8")
-        bootstrapLocalSetup $SERVICE
-        ;;
-
       "estafette"|"preemptible-killer")
         installPreemptibleKiller # the gke preemptible node spreader outer
         ;;
@@ -626,18 +608,6 @@ CATBACKUP
         setupStackBuckets && exit 0
         ;;
 
-      "pgbouncer")
-        bootstrapLocalSetup $SERVICE
-        ;;
-
-      "prom-operator")
-        bootstrapLocalSetup $SERVICE
-        ;;
-
-      "redis"|"valkey")
-        bootstrapLocalSetup $SERVICE
-        ;;
-
       "reflector")
         showInstalling "Secrets/ConfigMap Namespace Reflector"
         addHelmRepo emberstack https://emberstack.github.io/helm-charts
@@ -658,22 +628,7 @@ CATBACKUP
         HELMUP="helm upgrade --install $SERVICE leverege/transponder --values $SERVICE/values.yaml $CHARTVER $HELM_WHAT"
         doHelmup "$HELMUP"
         ;;
-
-      "traefik")
-        # installTraefikEnvironment
-        bootstrapLocalSetup $SERVICE
-        ;;
-
-      "traefik-canary")
-        # only valid if traefik is already installed
-        bootstrapLocalSetup $SERVICE
-        ;;
-
-      "velero")
-        # installVeleroEnvironment
-        bootstrapLocalSetup $SERVICE
-        ;;
-
+        
       *)
         # if the service dir is missing then check to see if it's a platform
         # service and if it is then try to bootstrap from charts in build-tools
@@ -750,7 +705,7 @@ NOTRUNNING
 }
 
 # ---------- helmup MAIN ----------
-GITTOP=`git rev-parse --show-toplevel` # `&& cd $GITTOP
+GITTOP=`git rev-parse --show-toplevel`
 K8S_UTILS="${GITTOP}/k8s-utils"
 
 overwhelm
@@ -764,5 +719,3 @@ GCE_REGION=`overwhelm -k GCE_REGION`
 doInstall $@
 
 displayBanner
-
-cd - &> $DEVNULL

package/src/k8x.sh

@@ -47,6 +47,10 @@ K8X_HELP
     FZFQ="--query=grafana" && ns="prometheus" && cmd="bash"
     ;;
 
+  "grafold")
+    FZFQ="--query=grafana" && ns="monitoring" && cmd="bash"
+    ;;
+
   "prom-operator"|"prometheus")
     ns="prometheus" && cmd="bash"
     ;;

package/src/overwhelm.mjs

@@ -79,7 +79,7 @@ if ( !fs.existsSync( 'overwhelm.yaml' ) ) {
   process.exit( 1 )
 }
 
-if ( !args.key && fs.existsSync( 'values-default.yaml' ) ) {
+if ( !args.key && !args.fetchctx && !args.matchk8s && fs.existsSync( 'values-default.yaml' ) ) {
   log( chalk.red.bold( `
 
 ***ERROR overwhelm => values-default.yaml is no longer used/supported and will

package/src/pg_repack/Dockerfile

@@ -0,0 +1,37 @@
+FROM bitnami/postgresql:11.14.0-debian-10-r28
+
+USER root
+
+# ---- Patch apt to use Debian archive repos since Buster is EOL ----
+RUN sed -i 's|http://deb.debian.org/debian|http://archive.debian.org/debian|g' /etc/apt/sources.list \
+ && sed -i 's|http://security.debian.org|http://archive.debian.org/debian-security|g' /etc/apt/sources.list \
+ && echo 'Acquire::Check-Valid-Until "false";' > /etc/apt/apt.conf.d/99no-check-valid-until \
+ && apt-get update
+
+# ---- Install pg_repack build dependencies ----
+RUN install_packages \
+    git \
+    build-essential \
+    libpq-dev \
+    postgresql-server-dev-11 \
+    libssl-dev \
+    zlib1g-dev \
+    ca-certificates \
+    curl
+
+# ---- Build and install pg_repack ----
+RUN git clone https://github.com/reorg/pg_repack.git /tmp/pg_repack \
+ && cd /tmp/pg_repack \
+ && make \
+ && make install \
+ && rm -rf /tmp/pg_repack
+
+# ---- Install kubectl ----
+ARG KUBECTL_VERSION=1.30.0
+RUN curl -LO "https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl" \
+ && install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl \
+ && rm kubectl
+
+# Drop back to non-root user (Bitnami default)
+USER 1001
+

package/src/pg_repack/README.md

@@ -0,0 +1,6 @@
+## Docker with pg_repack and kubectl
+
+This configuration is needed for providing repack capabilities for legacy
+postgresql 11 systems. Specifically, rule-engine / scheduler on prd-recovr-auto
+cause PVC issues with the constant one-shot timer rescheduling, which gets
+auto vacuumed but the PVC will continue to grow.

package/src/pg_repack/build-repack-kubectl.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+docker buildx build \
+  --platform linux/amd64 \
+  --build-arg KUBECTL_VERSION=1.30.2 \
+  --push \
+  -t us-docker.pkg.dev/leverege-registry/system/images/leverege-pgsql11-repack-kubectl:11.14-kctl.2 \
+  .

package/src/pg_repack/charts/repack/Chart.yaml

@@ -0,0 +1,5 @@
+name: repack
+version: 0.1.0
+appVersion: 0.1.0
+apiVersion: v2
+icon: https://raw.githubusercontent.com/kubernetes/kubernetes/master/logo/logo.svg

package/src/pg_repack/charts/repack/templates/_helpers.tpl

@@ -0,0 +1,46 @@
+{{- define "repackMaintScript" -}}
+#!/usr/bin/env bash
+set -euo pipefail
+
+NAMESPACE="{{ .Values.namespace }}"
+DEPLOY="{{ .Values.deploymentName }}"
+DB_HOST="{{ .Values.db.host }}"
+DB_PORT="{{ .Values.db.port }}"
+DB_USER="{{ .Values.db.user }}"
+DB_NAME="{{ .Values.db.name }}"
+WAIT_TIMEOUT="{{ .Values.repack.waitTimeout }}"
+
+CURRENT=$(kubectl -n "$NAMESPACE" get deploy "$DEPLOY" -o jsonpath='{.spec.replicas}' 2>/dev/null || echo "")
+REPLICAS="${CURRENT:-1}"
+echo "[maint] current replicas for $DEPLOY: $REPLICAS"
+
+scale_up() {
+  echo "[maint] scaling $DEPLOY back to $REPLICAS"
+  kubectl -n "$NAMESPACE" scale deploy "$DEPLOY" --replicas="$REPLICAS" || true
+  echo "[maint] waiting for rollout..."
+  kubectl -n "$NAMESPACE" rollout status deploy/"$DEPLOY" --timeout=5m || true
+}
+trap 'echo "[maint] EXIT trap fired"; scale_up' EXIT
+
+echo "[maint] scaling $DEPLOY to 0"
+kubectl -n "$NAMESPACE" scale deploy "$DEPLOY" --replicas=0
+kubectl -n "$NAMESPACE" rollout status deploy/"$DEPLOY" --timeout=5m || true
+
+echo "[maint] waiting for DB sessions to drain..."
+for i in {1..10}; do
+  ACTIVE=$(psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" -Atc \
+    "SELECT count(*) FROM pg_stat_activity
+     WHERE datname=current_database()
+       AND pid <> pg_backend_pid()
+       AND state IS DISTINCT FROM 'idle';" || echo "999")
+  echo "[maint] active sessions: $ACTIVE"
+  [ "$ACTIVE" -eq 0 ] && break
+  sleep 6
+done
+
+echo "[maint] running pg_repack..."
+pg_repack -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" \
+  --wait-timeout="$WAIT_TIMEOUT" -E DEBUG
+
+echo "[maint] repack finished successfully"
+{{- end -}}

package/src/pg_repack/charts/repack/templates/configmap.yaml

@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: repack-maint-script
+  namespace: {{ .Values.namespace }}
+data:
+  maint.sh: |
+{{- nindent 4 (include "repackMaintScript" .) }}

package/src/pg_repack/charts/repack/templates/cronjob.yaml

@@ -0,0 +1,33 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: scheduler-db-repack
+  namespace: {{ .Values.namespace }}
+spec:
+  schedule: {{ .Values.schedule | quote }}
+  concurrencyPolicy: Forbid
+  successfulJobsHistoryLimit: 1
+  failedJobsHistoryLimit: 2
+  jobTemplate:
+    spec:
+      backoffLimit: 0
+      template:
+        spec:
+          serviceAccountName: {{ .Values.serviceAccountName }}
+          restartPolicy: Never
+          containers:
+            - name: repack
+              image: "{{ .Values.image.repo }}:{{ .Values.image.tag }}"
+              imagePullPolicy: {{ .Values.image.pullPolicy }}
+              envFrom:
+                - secretRef:
+                    name: scheduler-pg-creds
+              volumeMounts:
+                - name: script
+                  mountPath: /opt/maint
+              command: ["/bin/bash","/opt/maint/maint.sh"]
+          volumes:
+            - name: script
+              configMap:
+                name: repack-maint-script
+                defaultMode: 0755

package/src/pg_repack/charts/repack/templates/rbac.yaml

@@ -0,0 +1,25 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: repack-maint-role
+  namespace: {{ .Values.namespace }}
+rules:
+  - apiGroups: ["apps"]
+    resources: ["deployments", "deployments/scale"]
+    verbs: ["get","list","watch","patch","update"]
+  - apiGroups: [""]
+    resources: ["pods"]
+    verbs: ["get","list","watch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: repack-maint-rb
+  namespace: {{ .Values.namespace }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ .Values.serviceAccountName }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: repack-maint-role

package/src/pg_repack/charts/repack/templates/secret.yaml

@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: scheduler-pg-creds
+  namespace: {{ .Values.namespace }}
+type: Opaque
+stringData:
+  PGPASSWORD: {{ .Values.pgPassword | quote }}

package/src/pg_repack/charts/repack/templates/serviceaccount.yaml

@@ -0,0 +1,5 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ .Values.serviceAccountName }}
+  namespace: {{ .Values.namespace }}

package/src/pg_repack/charts/repack/values.yaml

@@ -0,0 +1,23 @@
+namespace: default
+
+image:
+  repo: us-docker.pkg.dev/leverege-registry/system/images/leverege-pgsql11-repack-kubectl
+  tag: 11.14-kctl.2
+  pullPolicy: IfNotPresent
+
+#schedule: "0 3 * * *" # 3 in the morning UTC
+schedule: "*/5 * * * *" # every 5 minutes for testing
+deploymentName: scheduler
+db:
+  host: postgres-scheduler
+  port: "5432"
+  user: postgres
+  name: scheduler
+repack:
+  waitTimeout: "15"
+
+# You can set this via --set-file or external secrets in prod
+pgPassword: "REPLACE_ME"
+
+replicasDefault: "" # leave blank to detect automatically
+serviceAccountName: repack-maint-sa

package/src/pg_repack/deploy.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+helm upgrade -i repack charts/repack -n default

package/src/pg_repack/quick.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: pg-repack-cli
+spec:
+  restartPolicy: Never
+  containers:
+    - name: repack
+      image: us-docker.pkg.dev/leverege-registry/system/images/leverege-pgsql11-repack-kubectl:11.14-kctl.3
+      command: ["sleep","infinity"]