@leverege/build-tools 2.83.5 → 2.84.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@leverege/build-tools",
-  "version": "2.83.5",
+  "version": "2.84.0",
   "description": "A collection of build / support tools for Leverege developers",
   "main": "index.js",
   "repository": {
@@ -65,7 +65,7 @@
   "license": "SEE LICENSE IN LICENSE.md",
   "dependencies": {
     "@leverege/jsdoc-template": "^1.0.1",
-    "chalk": "^5.6.0",
+    "chalk": "^5.6.2",
     "command-line-args": "^6.0.1",
     "command-line-usage": "^7.0.3",
     "commander": "^14.0.0",

package/src/Utils.mjs

@@ -134,10 +134,10 @@ export const getDateTimestamp = () => {
   return `${fullDTS.slice( 0, 8 )}-${fullDTS.slice( 8, 12 )}`
 }
 
-export const getCleanTag = (str) => {
+export const getCleanTag = ( str ) => {
   // Sanitize string to only include characters allowed in GCP Artifact Registry image tags
   // Allowed characters: abcdefghijklmnopqrstuvwxyz0123456789_-./
-  return str.replace(/[^a-zA-Z0-9_\-\.\/]/g, '')
+  return str.replace( /[^a-zA-Z0-9_\-./]/g, '' )
 }
 
 export const getGcpAccount = async () => {

package/src/chart-to-registry.mjs

@@ -14,11 +14,17 @@ import {
   errorExit,
   log,
   warning,
-  analyzeRepository,
+  parseHelmChart,
   shellCmd,
 } from './Utils.mjs'
 
 const commandLineOptions = [ // Use commandLineOptions to tie into the Usage statements
+  {
+    name : 'chart',
+    type : String,
+    defaultValue : 'helm',
+    description : '{green directory containing the chart - default: helm}',
+  },
   {
     name : 'location',
     type : String,
@@ -83,12 +89,8 @@ if ( semverLt( process.version, minNodejsVersion ) ) {
   errorExit( `\n***ERROR: must be running at least node ${minNodejsVersion}\n` )
 }
 
-// load up all we need to know about the repo
-const repoDescr = await analyzeRepository()
-
-const {
-  helmChart,
-} = repoDescr
+// load up all we need to know about the chart
+const helmChart = await parseHelmChart( args.chart )
 
 // helm push helm-package.tgz oci://REGISTRY_LOCATION/REGISTRY_PROJECT/REGISTRY_REPOSITORY/charts
 //
@@ -133,7 +135,7 @@ try {
 }
 
 try {
-  await shellCmd( 'helm package helm' )
+  await shellCmd( `helm package ${args.chart}` )
   if ( !existsSync( chartPackage ) ) { // eslint-disable-line security/detect-non-literal-fs-filename
     errorExit( `Error: expected helm package to produce ${chartPackage}` )
   }

package/src/custom-images/{bitnami-audit.mjs → bitnami-audits/bitnami-audit.mjs}

@@ -13,7 +13,7 @@ import {
   debug,
   log,  
   shellCmd,
-} from './Utils.mjs'
+} from '../Utils.mjs'
 
 const auditProjects = [
   'dev-controltek',

package/src/custom-images/kubectl/build-kubetcl.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
-
+#
+# builds an alpine image with kubectl installed
+#
 KUBECTL_VERSION="1.30.3"
 
 docker buildx build \

package/src/custom-images/pg_repack/README.md

@@ -3,4 +3,18 @@
 This configuration is needed for providing repack capabilities for legacy
 postgresql 11 systems. Specifically, rule-engine / scheduler on prd-recovr-auto
 cause PVC issues with the constant one-shot timer rescheduling, which gets
-auto vacuumed but the PVC will continue to grow.
+auto vacuumed but the PVC will continue to grow.
+
+### Docker subdir
+
+The Dockerfile in the docker directory is used for building the all-in-one
+image bundle of debian postgres which pg_repack and kubectl. This is the
+image that is used by the cronjob that periodically repacks the configured
+database. This was initially created for managing the scheduler database
+on prd-recovr. Rebuild the image with => build-repack-kubectl.sh
+
+### Charts subdir
+
+The charts directory houses the custom charts that we are using. In particular,
+the cronjob that run the scheduler database repacking resides down in this area.
+

package/src/custom-images/pg_repack/charts/push-charts.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+push-my-chart --chart repack-cron

package/src/custom-images/pg_repack/charts/repack-cron/Chart.yaml

@@ -0,0 +1,7 @@
+apiVersion: v2
+name: repack-cron
+version: 0.5.2
+appVersion: 11.14-kctl.3
+description: Automated pg_repack job with safe scale-down/scale-up
+type: application
+icon: https://raw.githubusercontent.com/kubernetes/kubernetes/master/logo/logo.svg

package/src/custom-images/pg_repack/charts/{repack → repack-cron}/templates/_helpers.tpl

@@ -20,27 +20,17 @@ scale_up() {
   echo "[maint] waiting for rollout..."
   kubectl -n "$NAMESPACE" rollout status deploy/"$DEPLOY" --timeout=5m || true
 }
+
+# this trap ensures the scale_up function always runs on exit
 trap 'echo "[maint] EXIT trap fired"; scale_up' EXIT
 
 echo "[maint] scaling $DEPLOY to 0"
 kubectl -n "$NAMESPACE" scale deploy "$DEPLOY" --replicas=0
 kubectl -n "$NAMESPACE" rollout status deploy/"$DEPLOY" --timeout=5m || true
 
-echo "[maint] waiting for DB sessions to drain..."
-for i in {1..10}; do
-  ACTIVE=$(psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" -Atc \
-    "SELECT count(*) FROM pg_stat_activity
-     WHERE datname=current_database()
-       AND pid <> pg_backend_pid()
-       AND state IS DISTINCT FROM 'idle';" || echo "999")
-  echo "[maint] active sessions: $ACTIVE"
-  [ "$ACTIVE" -eq 0 ] && break
-  sleep 6
-done
-
-echo "[maint] running pg_repack..."
-pg_repack -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" \
-  --wait-timeout="$WAIT_TIMEOUT" -E DEBUG
+PG_REPACK_CMD="pg_repack -h $DB_HOST -p $DB_PORT -U $DB_USER -d $DB_NAME  --wait-timeout=$WAIT_TIMEOUT -E INFO"
+echo "[maint] Running => $PG_REPACK_CMD"
+$PG_REPACK_CMD
 
 echo "[maint] repack finished successfully"
 {{- end -}}

package/src/custom-images/pg_repack/charts/{repack → repack-cron}/templates/configmap.yaml

@@ -1,7 +1,7 @@
 apiVersion: v1
 kind: ConfigMap
 metadata:
-  name: repack-maint-script
+  name: {{ .Values.deploymentName }}-repack-script
   namespace: {{ .Values.namespace }}
 data:
   maint.sh: |

package/src/custom-images/pg_repack/charts/repack-cron/templates/cronjob.yaml

@@ -0,0 +1,54 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: {{ .Values.deploymentName }}-repack-job
+  namespace: {{ .Values.namespace }}
+spec:
+  schedule: {{ .Values.schedule | quote }}
+  concurrencyPolicy: Forbid
+  successfulJobsHistoryLimit: 1
+  failedJobsHistoryLimit: 2
+  jobTemplate:
+    metadata:
+      labels:
+        helm.sh/chart: {{ .Chart.Version }}
+    spec:
+      backoffLimit: 0
+      template:
+        spec:
+          serviceAccountName: {{ .Values.serviceAccountName }}
+          restartPolicy: Never
+          containers:
+            - name: repack
+              imagePullPolicy: {{ .Values.image.pullPolicy }}
+              image: {{ .Values.image.registry }}/{{ .Values.image.repo }}:{{ default .Chart.AppVersion .Values.image.tag }}
+              env:
+                - name: NAMESPACE
+                  value: "{{ .Values.namespace }}"
+                - name: TARGET_DEPLOYMENT
+                  value: "{{ .Values.deploymentName }}"
+                - name: PGHOST
+                  value: "{{ .Values.db.host }}"
+                - name: PGPORT
+                  value: "{{ .Values.db.port }}"
+                - name: PGUSER
+                  value: "{{ .Values.db.user }}"
+                - name: PGDATABASE
+                  value: "{{ .Values.db.name }}"
+                - name: REPACK_WAIT_TIMEOUT
+                  value: "{{ .Values.repack.waitTimeout }}"
+                # password from EXISTING secret
+                - name: PGPASSWORD
+                  valueFrom:
+                    secretKeyRef:
+                      name: {{ .Values.secret.name }}
+                      key: {{ .Values.secret.passwordKey }}
+              volumeMounts:
+                - name: script
+                  mountPath: /opt/maint
+              command: ["/bin/bash","/opt/maint/maint.sh"]
+          volumes:
+            - name: script
+              configMap:
+                name: {{ .Values.deploymentName }}-repack-script
+                defaultMode: 0755

package/src/custom-images/pg_repack/charts/repack-cron/values.yaml

@@ -0,0 +1,33 @@
+namespace: default
+
+image:
+  pullPolicy: IfNotPresent
+  registry: us-docker.pkg.dev/leverege-registry/system/images
+  repo: leverege-pgsql11-repack-kubectl
+  # Overrides the image tag whose default is the chart appVersion.
+  tag: ""
+
+schedule: "0 3 * * *" # 3 in the morning UTC
+#schedule: "*/5 * * * *" # every 5 minutes for testing
+
+serviceAccountName: repack-maint-sa
+deploymentName: scheduler
+
+db:
+  host: postgres-scheduler
+  port: "5432"
+  user: postgres
+  name: scheduler
+
+repack:
+  waitTimeout: "15"
+  allowKillBlockers: "false" # "true" to cancel residual blockers (rarely needed)
+
+# EXISTING secret in the cluster; we do NOT create it here
+# Default assumes the secret has key "password". Change if yours differs.
+secret:
+  name: authz-postgres
+  passwordKey: postgresql-password
+
+# leave blank to auto-detect current replicas before scaling down
+replicasDefault: ""

package/src/custom-images/pg_repack/{Dockerfile → docker/Dockerfile}

@@ -1,3 +1,7 @@
+# Create a legacy psql11 image that includes the pg_repack extension and
+# the k8s kubectl command. This image may be used as either a psql11 pod
+# or as a pod for running remote repacks on a deployed db.
+#
 FROM bitnami/postgresql:11.14.0-debian-10-r28
 
 USER root
@@ -32,6 +36,5 @@ RUN curl -LO "https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kube
  && install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl \
  && rm kubectl
 
-# Drop back to non-root user (Bitnami default)
+# Drop back to non-root user
 USER 1001
-

package/src/custom-images/pg_repack/{build-repack-kubectl.sh → docker/build-repack-kubectl.sh}

@@ -4,5 +4,5 @@ docker buildx build \
   --platform linux/amd64 \
   --build-arg KUBECTL_VERSION=1.30.2 \
   --push \
-  -t us-docker.pkg.dev/leverege-registry/system/images/leverege-pgsql11-repack-kubectl:11.14-kctl.2 \
+  -t us-docker.pkg.dev/leverege-registry/system/images/leverege-pgsql11-repack-kubectl:11.14-kctl.3 \
   .

package/src/helm-charts/pg-repack/helmup.plugin

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+showInstalling "PG Repacking Cronjob"
+
+OCI_CHART="oci://us-docker.pkg.dev/leverege-registry/system/charts/repack-cron"
+
+[ -z "$PG_REPACK_CHART_VERSION" ] && PG_REPACK_CHART_VERSION="0.5.2"
+
+helm upgrade --install repack $OCI_CHART     \
+             --values  pg-repack/values.yaml \
+             --version $PG_REPACK_CHART_VERSION $HELM_WHAT

package/src/helm-charts/pg-repack/values-local.yaml

@@ -0,0 +1,16 @@
+#
+# 0 3 * * *    - 3am UTC
+# */15 * * * * - 15 minute intervals
+#
+schedule: "*/5 * * * *"
+
+db:
+  host: "postgres-scheduler"
+  port: "5432"
+  user: "postgres"
+  name: "scheduler"
+
+# the existing password to use
+secret:
+  name: "authz-postgres"
+  passwordKey: "postgresql-password"

package/src/helm-charts/prom-operator/prometheus-stack.yaml.ovh

@@ -221,7 +221,7 @@ alertmanager:
           continue: true # 👈 KEEP GOING after this match
 
         - matchers:
-            - namespace="kube-system" # 🙈 ignore these alerts - GKE will handle them
+            - namespace=~"kube-system|gmp-system" # 🙈 ignore these alerts - GKE will handle them
           receiver: discard
 
         - matchers:

package/src/helm-charts/prom-operator/rules/kubestate-rules.yaml

@@ -383,7 +383,7 @@ spec:
                 LABELS = {{ $labels }}
         - alert: KubernetesApiServerLatency
           expr: histogram_quantile(0.99, sum(rate(apiserver_request_duration_seconds_bucket{verb!~"(?:CONNECT|WATCHLIST|WATCH|PROXY)"} [10m])) WITHOUT (subresource)) > 1
-          for: 15m
+          for: 30m # awesome was 15m
           labels:
             severity: warning
           annotations:

package/src/push-my-chart.mjs

@@ -8,6 +8,12 @@ import { lt as semverLt } from 'semver'
 import { debug, log, shellCmd } from './Utils.mjs'
 
 const commandLineOptions = [
+  {
+    name : 'chart',
+    type : String,
+    defaultValue : 'helm',
+    description : '{green directory containing the chart - default: helm}',
+  },
   {
     name : 'no-registry',
     type : Boolean,
@@ -49,7 +55,7 @@ const pushItGood = async () => {
   let museumStatus = 0
   try {
     log( chalk.green( 'Pushing chart to the Chartmuseum\n' ) )
-    const museum = await shellCmd( `helm ${pusher} helm leverege` )
+    const museum = await shellCmd( `helm ${pusher} ${args.chart} leverege` )
     debug( { museum }, '<==output from helm push' )
   } catch ( ex ) {
     log( ex.stderr )
@@ -63,7 +69,7 @@ const pushItGood = async () => {
 
   try {
     log( chalk.green( 'Pushing chart to the Artifact Registry\n' ) )
-    const registry = await shellCmd( 'chart-to-registry' )
+    const registry = await shellCmd( `chart-to-registry --chart ${args.chart}` )
     debug( { registry }, '<==chart-to-registry' )
     process.exit( museumStatus )
   } catch ( ex ) {

package/src/custom-images/pg_repack/charts/repack/Chart.yaml

@@ -1,5 +0,0 @@
-name: repack
-version: 0.1.0
-appVersion: 0.1.0
-apiVersion: v2
-icon: https://raw.githubusercontent.com/kubernetes/kubernetes/master/logo/logo.svg

package/src/custom-images/pg_repack/charts/repack/templates/cronjob.yaml

@@ -1,33 +0,0 @@
-apiVersion: batch/v1
-kind: CronJob
-metadata:
-  name: scheduler-db-repack
-  namespace: {{ .Values.namespace }}
-spec:
-  schedule: {{ .Values.schedule | quote }}
-  concurrencyPolicy: Forbid
-  successfulJobsHistoryLimit: 1
-  failedJobsHistoryLimit: 2
-  jobTemplate:
-    spec:
-      backoffLimit: 0
-      template:
-        spec:
-          serviceAccountName: {{ .Values.serviceAccountName }}
-          restartPolicy: Never
-          containers:
-            - name: repack
-              image: "{{ .Values.image.repo }}:{{ .Values.image.tag }}"
-              imagePullPolicy: {{ .Values.image.pullPolicy }}
-              envFrom:
-                - secretRef:
-                    name: scheduler-pg-creds
-              volumeMounts:
-                - name: script
-                  mountPath: /opt/maint
-              command: ["/bin/bash","/opt/maint/maint.sh"]
-          volumes:
-            - name: script
-              configMap:
-                name: repack-maint-script
-                defaultMode: 0755

package/src/custom-images/pg_repack/charts/repack/templates/secret.yaml

@@ -1,8 +0,0 @@
-apiVersion: v1
-kind: Secret
-metadata:
-  name: scheduler-pg-creds
-  namespace: {{ .Values.namespace }}
-type: Opaque
-stringData:
-  PGPASSWORD: {{ .Values.pgPassword | quote }}

package/src/custom-images/pg_repack/charts/repack/values.yaml

@@ -1,23 +0,0 @@
-namespace: default
-
-image:
-  repo: us-docker.pkg.dev/leverege-registry/system/images/leverege-pgsql11-repack-kubectl
-  tag: 11.14-kctl.2
-  pullPolicy: IfNotPresent
-
-#schedule: "0 3 * * *" # 3 in the morning UTC
-schedule: "*/5 * * * *" # every 5 minutes for testing
-deploymentName: scheduler
-db:
-  host: postgres-scheduler
-  port: "5432"
-  user: postgres
-  name: scheduler
-repack:
-  waitTimeout: "15"
-
-# You can set this via --set-file or external secrets in prod
-pgPassword: "REPLACE_ME"
-
-replicasDefault: "" # leave blank to detect automatically
-serviceAccountName: repack-maint-sa

package/src/custom-images/pg_repack/deploy.sh

@@ -1,3 +0,0 @@
-#!/bin/bash
-
-helm upgrade -i repack charts/repack -n default