@leverege/build-tools 2.82.0 → 2.83.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@leverege/build-tools",
-  "version": "2.82.0",
+  "version": "2.83.0",
   "description": "A collection of build / support tools for Leverege developers",
   "main": "index.js",
   "repository": {
@@ -65,7 +65,7 @@
   "license": "SEE LICENSE IN LICENSE.md",
   "dependencies": {
     "@leverege/jsdoc-template": "^1.0.1",
-    "chalk": "^5.5.0",
+    "chalk": "^5.6.0",
     "command-line-args": "^6.0.1",
     "command-line-usage": "^7.0.3",
     "commander": "^14.0.0",
@@ -75,13 +75,13 @@
     "glob": "^11.0.3",
     "handlebars": "^4.7.8",
     "ignore": "^7.0.5",
-    "inquirer": "^12.9.0",
+    "inquirer": "^12.9.3",
     "js-yaml": "^4.1.0",
     "jsdoc": "^4.0.4",
     "ms": "^2.1.3",
     "npm-registry-fetch": "^19.0.0",
     "ora": "^8.2.0",
-    "p-limit": "^6.2.0",
+    "p-limit": "^7.1.0",
     "package-up": "^5.0.0",
     "readline-sync": "^1.4.10",
     "semver": "^7.7.2",

package/src/bash-funcs

@@ -153,7 +153,7 @@ REPOADD
 # *** DEPRECATED *** REMOVE AFTER BITNAMI ERADICATED ***
 # *** DEPRECATED *** REMOVE AFTER BITNAMI ERADICATED ***
 #
-function addBitnamiRepo() {
+function addBitnamiRepoDEPRECATED() {
   if [ ! -z "$BITNAMI_LATEST" ] || [ "$1" == "latest" ];
   then
     BITNAMI_URL="https://charts.bitnami.com/bitnami"

package/src/helm-charts/postgres/postgres-local.yaml

@@ -2,8 +2,8 @@
 #
 image:
   registry: us-docker.pkg.dev
-  repository: leverege-registry/bitnami-mirror/images/postgresql
-# tag: 11.14.0-debian-10-r28
+  repository: leverege-registry/system/images/leverege-pgsql11-repack-kubectl
+  tag: 11.14-kctl.3
 
 postgresqlExtendedConf:
   maxWorkerProcesses: 4
@@ -33,7 +33,7 @@ metrics:
     repository: leverege-registry/bitnami-mirror/images/postgres-exporter
 #   tag: 0.10.0-debian-10-r172
 
-master:
+primary:
   tolerations:
   - key: "database"
     operator: "Equal"

package/src/helm-charts/prom-operator/rules/kubestate-rules.yaml

@@ -229,7 +229,7 @@ spec:
                 LABELS = {{ $labels }}
         - alert: KubernetesReplicasetReplicasMismatch
           expr: kube_replicaset_spec_replicas != kube_replicaset_status_ready_replicas
-          for: 10m
+          for: 15m # awesome was => 10m
           labels:
             severity: warning
           annotations:

package/src/helmup.sh

@@ -246,32 +246,20 @@ SERVICE_BOOTSTRAP
   ifPluggedIn $SERVICE
 }
 
-# some initial installs may be performed from the local setups in the
-# build-tools helm-charts directory
-function ifCanBootstrap() {
-  local bootstrap_targets=(
-    argocd
-    cnpg-operator
-    cnpg-db-psql-stack
-    cnpg-db-tsdb-basic
-    elasticsearch8
-    pgbouncer
-    prom-operator
-    redis
-    valkey
-    traefik
-    traefik-canary
-    velero
-  )
+# If the service is already plugged in the go - otherwise if this is a first
+# time install of a known service then bootstrap it and go. Otherwise return
+# a status of 1 to indicate to continue processing.
+function ifPluggedInOrBootstrapped() {
   local svc_name="$1"
 
-  if [[ " ${bootstrap_targets[*]} " == *" $svc_name "* ]];
-  then
-    bootstrapLocalSetup $svc_name
-    return 0
-  else
-    return 1
-  fi
+  ifPluggedIn $svc_name && return 0 # if it's already plugged in then go
+
+  # check to see if build-tools knows about the new service in its configs
+  local svc_plugin="$(build-tools --reporoot)/src/helm-charts/${svc_name}/helmup.plugin"
+  [[ ! -x "$svc_plugin" ]] && return 1 # indicates unknown
+
+  bootstrapLocalSetup $svc_name
+  return 0
 }
 
 function installDefaultConfigs() {
@@ -396,7 +384,7 @@ spec:
           containers:
           - name: cron-zombie-killer
             imagePullPolicy: IfNotPresent
-            image: bitnami/kubectl
+            image: registry.k8s.io/kubectl:v1.30.3
             command:
               - "/bin/sh"
             args:
@@ -590,11 +578,9 @@ CATBACKUP
       helmdn $SERVICE
     fi
 
-    # If we're referencing a plugin then execute it and continue
-    ifPluggedIn $SERVICE && continue
-
-    # If we can bootstrap a service on a new setup
-    ifCanBootstrap $SERVICE && continue
+    # If the service is already plugged in or can be bootstrapped
+    # during initial install then do so here
+    ifPluggedInOrBootstrapped $SERVICE && continue
 
     case "$SERVICE" in
       "platform")

package/src/pg_repack/Dockerfile

@@ -0,0 +1,37 @@
+FROM bitnami/postgresql:11.14.0-debian-10-r28
+
+USER root
+
+# ---- Patch apt to use Debian archive repos since Buster is EOL ----
+RUN sed -i 's|http://deb.debian.org/debian|http://archive.debian.org/debian|g' /etc/apt/sources.list \
+ && sed -i 's|http://security.debian.org|http://archive.debian.org/debian-security|g' /etc/apt/sources.list \
+ && echo 'Acquire::Check-Valid-Until "false";' > /etc/apt/apt.conf.d/99no-check-valid-until \
+ && apt-get update
+
+# ---- Install pg_repack build dependencies ----
+RUN install_packages \
+    git \
+    build-essential \
+    libpq-dev \
+    postgresql-server-dev-11 \
+    libssl-dev \
+    zlib1g-dev \
+    ca-certificates \
+    curl
+
+# ---- Build and install pg_repack ----
+RUN git clone https://github.com/reorg/pg_repack.git /tmp/pg_repack \
+ && cd /tmp/pg_repack \
+ && make \
+ && make install \
+ && rm -rf /tmp/pg_repack
+
+# ---- Install kubectl ----
+ARG KUBECTL_VERSION=1.30.0
+RUN curl -LO "https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl" \
+ && install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl \
+ && rm kubectl
+
+# Drop back to non-root user (Bitnami default)
+USER 1001
+

package/src/pg_repack/README.md

@@ -0,0 +1,6 @@
+## Docker with pg_repack and kubectl
+
+This configuration is needed for providing repack capabilities for legacy
+postgresql 11 systems. Specifically, rule-engine / scheduler on prd-recovr-auto
+cause PVC issues with the constant one-shot timer rescheduling, which gets
+auto vacuumed but the PVC will continue to grow.

package/src/pg_repack/build-repack-kubectl.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+docker buildx build \
+  --platform linux/amd64 \
+  --build-arg KUBECTL_VERSION=1.30.2 \
+  --push \
+  -t us-docker.pkg.dev/leverege-registry/system/images/leverege-pgsql11-repack-kubectl:11.14-kctl.2 \
+  .

package/src/pg_repack/charts/repack/Chart.yaml

@@ -0,0 +1,5 @@
+name: repack
+version: 0.1.0
+appVersion: 0.1.0
+apiVersion: v2
+icon: https://raw.githubusercontent.com/kubernetes/kubernetes/master/logo/logo.svg

package/src/pg_repack/charts/repack/templates/_helpers.tpl

@@ -0,0 +1,46 @@
+{{- define "repackMaintScript" -}}
+#!/usr/bin/env bash
+set -euo pipefail
+
+NAMESPACE="{{ .Values.namespace }}"
+DEPLOY="{{ .Values.deploymentName }}"
+DB_HOST="{{ .Values.db.host }}"
+DB_PORT="{{ .Values.db.port }}"
+DB_USER="{{ .Values.db.user }}"
+DB_NAME="{{ .Values.db.name }}"
+WAIT_TIMEOUT="{{ .Values.repack.waitTimeout }}"
+
+CURRENT=$(kubectl -n "$NAMESPACE" get deploy "$DEPLOY" -o jsonpath='{.spec.replicas}' 2>/dev/null || echo "")
+REPLICAS="${CURRENT:-1}"
+echo "[maint] current replicas for $DEPLOY: $REPLICAS"
+
+scale_up() {
+  echo "[maint] scaling $DEPLOY back to $REPLICAS"
+  kubectl -n "$NAMESPACE" scale deploy "$DEPLOY" --replicas="$REPLICAS" || true
+  echo "[maint] waiting for rollout..."
+  kubectl -n "$NAMESPACE" rollout status deploy/"$DEPLOY" --timeout=5m || true
+}
+trap 'echo "[maint] EXIT trap fired"; scale_up' EXIT
+
+echo "[maint] scaling $DEPLOY to 0"
+kubectl -n "$NAMESPACE" scale deploy "$DEPLOY" --replicas=0
+kubectl -n "$NAMESPACE" rollout status deploy/"$DEPLOY" --timeout=5m || true
+
+echo "[maint] waiting for DB sessions to drain..."
+for i in {1..10}; do
+  ACTIVE=$(psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" -Atc \
+    "SELECT count(*) FROM pg_stat_activity
+     WHERE datname=current_database()
+       AND pid <> pg_backend_pid()
+       AND state IS DISTINCT FROM 'idle';" || echo "999")
+  echo "[maint] active sessions: $ACTIVE"
+  [ "$ACTIVE" -eq 0 ] && break
+  sleep 6
+done
+
+echo "[maint] running pg_repack..."
+pg_repack -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" \
+  --wait-timeout="$WAIT_TIMEOUT" -E DEBUG
+
+echo "[maint] repack finished successfully"
+{{- end -}}

package/src/pg_repack/charts/repack/templates/configmap.yaml

@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: repack-maint-script
+  namespace: {{ .Values.namespace }}
+data:
+  maint.sh: |
+{{- nindent 4 (include "repackMaintScript" .) }}

package/src/pg_repack/charts/repack/templates/cronjob.yaml

@@ -0,0 +1,33 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: scheduler-db-repack
+  namespace: {{ .Values.namespace }}
+spec:
+  schedule: {{ .Values.schedule | quote }}
+  concurrencyPolicy: Forbid
+  successfulJobsHistoryLimit: 1
+  failedJobsHistoryLimit: 2
+  jobTemplate:
+    spec:
+      backoffLimit: 0
+      template:
+        spec:
+          serviceAccountName: {{ .Values.serviceAccountName }}
+          restartPolicy: Never
+          containers:
+            - name: repack
+              image: "{{ .Values.image.repo }}:{{ .Values.image.tag }}"
+              imagePullPolicy: {{ .Values.image.pullPolicy }}
+              envFrom:
+                - secretRef:
+                    name: scheduler-pg-creds
+              volumeMounts:
+                - name: script
+                  mountPath: /opt/maint
+              command: ["/bin/bash","/opt/maint/maint.sh"]
+          volumes:
+            - name: script
+              configMap:
+                name: repack-maint-script
+                defaultMode: 0755

package/src/pg_repack/charts/repack/templates/rbac.yaml

@@ -0,0 +1,25 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: repack-maint-role
+  namespace: {{ .Values.namespace }}
+rules:
+  - apiGroups: ["apps"]
+    resources: ["deployments", "deployments/scale"]
+    verbs: ["get","list","watch","patch","update"]
+  - apiGroups: [""]
+    resources: ["pods"]
+    verbs: ["get","list","watch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: repack-maint-rb
+  namespace: {{ .Values.namespace }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ .Values.serviceAccountName }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: repack-maint-role

package/src/pg_repack/charts/repack/templates/secret.yaml

@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: scheduler-pg-creds
+  namespace: {{ .Values.namespace }}
+type: Opaque
+stringData:
+  PGPASSWORD: {{ .Values.pgPassword | quote }}

package/src/pg_repack/charts/repack/templates/serviceaccount.yaml

@@ -0,0 +1,5 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ .Values.serviceAccountName }}
+  namespace: {{ .Values.namespace }}

package/src/pg_repack/charts/repack/values.yaml

@@ -0,0 +1,23 @@
+namespace: default
+
+image:
+  repo: us-docker.pkg.dev/leverege-registry/system/images/leverege-pgsql11-repack-kubectl
+  tag: 11.14-kctl.2
+  pullPolicy: IfNotPresent
+
+#schedule: "0 3 * * *" # 3 in the morning UTC
+schedule: "*/5 * * * *" # every 5 minutes for testing
+deploymentName: scheduler
+db:
+  host: postgres-scheduler
+  port: "5432"
+  user: postgres
+  name: scheduler
+repack:
+  waitTimeout: "15"
+
+# You can set this via --set-file or external secrets in prod
+pgPassword: "REPLACE_ME"
+
+replicasDefault: "" # leave blank to detect automatically
+serviceAccountName: repack-maint-sa

package/src/pg_repack/deploy.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+helm upgrade -i repack charts/repack -n default

package/src/pg_repack/quick.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: pg-repack-cli
+spec:
+  restartPolicy: Never
+  containers:
+    - name: repack
+      image: us-docker.pkg.dev/leverege-registry/system/images/leverege-pgsql11-repack-kubectl:11.14-kctl.3
+      command: ["sleep","infinity"]

package/src/bitnami-audit.sh

@@ -1,75 +0,0 @@
-#!/bin/bash
-
-BITNAMI_CTX=(
-  dev-controltek
-  dev-cox-health
-  dev-pitcrew
-  dev-recovr-auto
-  dev-schnucks
-  dev-tpi
-  discount-tire-prd
-  discount-tire-stg
-  leverege-cnpg-20240729
-  leverege-ops-20250730
-  leverege-demo
-  leverege-elastic
-  leverege-onboarding
-  leverege-sandbox
-  nimbelink-demo
-  prd-controltek
-  prd-cox-health
-  prd-generic-kudelski
-  prd-les-schwab
-  prd-recovr-auto
-  prd-skymirr
-  prd-tpi
-  prd-valvoline
-  qa-cox-health
-  siren-marine-dev
-  siren-marine-prd
-  siren-marine-staging
-  siren-marine-test
-  stg-cox-health
-  stg-generic-kudelski
-  stg-recovr-auto
-  stg-recovr-perf
-)
-
-function gitCheckoutCleanBranch() {
-  # find the cruft, namely any values.yaml files which tend to block the
-  # cleaning of subdirs that contain them (values.yaml aren't tracked in git)
-  find . -not \( -path \*junk -prune -o -path \*helm-elasticsearch -prune \) -name values.yaml -exec rm {} \;
-  git ko bootstrap-k8s &> /dev/null
-  git clean -d --force &> /dev/null
-
-  local branch=${1:bootstrap-k8s}
-  git ko "$branch" &> /dev/null
-}
-
-# create a clean sheet with headers
-CSV_FILE="../BitnamiImageRegistry.csv"
-rm -f $CSV_FILE
-echo 'project,namespace,service,image,version' > $CSV_FILE
-
-gitCheckoutCleanBranch # start clean
-
-for CTX in "${BITNAMI_CTX[@]}";
-do
-  gitCheckoutCleanBranch $CTX
-
-  if [[ $? -eq 0 ]];
-  then
-    overwhelm -f
-    overwhelm -m
-    if [[ $? -ne 0 ]];
-    then
-      echo "Project [${CTX}] is valid, but the k8s context is not - skipping..."
-      continue
-    fi
-    echo "Context GOOD [$CTX]"
-    kubectl get deploy,statefulset -A -o json | bitnami-k8s-parse $CTX >> $CSV_FILE
-  fi
-  find . -not \( -path \*junk -prune -o -path \*helm-elasticsearch -prune \) -name values.yaml -exec rm {} \;
-done
-
-gitCheckoutCleanBranch # end clean