@leverege/build-tools 2.51.0 → 2.51.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@leverege/build-tools",
-  "version": "2.51.0",
+  "version": "2.51.2",
   "description": "A collection of build / support tools for Leverege developers",
   "main": "index.js",
   "repository": {
@@ -58,13 +58,13 @@
     "ansi-colors": "^4.1.3",
     "chalk": "^5.3.0",
     "command-line-args": "^6.0.0",
-    "command-line-usage": "^7.0.2",
+    "command-line-usage": "^7.0.3",
     "deepmerge": "^4.3.1",
     "enquirer": "^2.4.1",
-    "execa": "^9.3.0",
+    "execa": "^9.4.0",
     "glob": "^11.0.0",
     "handlebars": "^4.7.8",
-    "inquirer": "^10.0.1",
+    "inquirer": "^11.0.1",
     "js-yaml": "^4.1.0",
     "ms": "^2.1.3",
     "npm-registry-fetch": "^17.1.0",
@@ -72,12 +72,12 @@
     "parse-gitignore": "^2.0.0",
     "read-pkg": "^9.0.1",
     "readline-sync": "^1.4.10",
-    "semver": "^7.6.2",
-    "simple-git": "^3.25.0",
-    "zx": "^8.1.4"
+    "semver": "^7.6.3",
+    "simple-git": "^3.26.0",
+    "zx": "^8.1.6"
   },
   "devDependencies": {
     "@leverege/eslint-config-leverege": "^4.2.0",
-    "npm": "^10.8.2"
+    "npm": "^10.8.3"
   }
 }

package/src/helm-charts/cnpg-db-psql-stack/cluster.yaml.ovh

@@ -7,13 +7,16 @@ metadata:
   namespace: cnpg-operands
 spec:
   instances: 1
-# logLevel: debug
+  logLevel: info # warning debug
+
+  monitoring:
+    enablePodMonitor: false # set true once monitoring is setup
 
   description: "Leverege Stack PostgreSQL DB"
-  imageName: us-docker.pkg.dev/leverege-registry/system/images/leverege-pgsql:16.2-cnpg-tsdb2-beta.4
+  imageName: us-docker.pkg.dev/leverege-registry/system/images/leverege-pgsql:16.4-cnpg-tsdb2-rel.1
   bootstrap:
     initdb:
-#     database: models
+#     database: models # do not set - let it default to cnpg app db
       postInitSQL:
 #       - CREATE DATABASE app;       # operator expects this
 #       - CREATE ROLE app;           #   and this to exist on restore
@@ -23,9 +26,6 @@ spec:
         - CREATE DATABASE geotile;
         - CREATE DATABASE models;
         - CREATE DATABASE scheduler;
-#     owner: imagine
-#     secret:
-#       name: cnpg-db-psql-stack-imagine-pw # imagine db pw
 
   enableSuperuserAccess: true
   superuserSecret:
@@ -52,7 +52,7 @@ spec:
   # to the default ones to make the cluster work
   postgresql:
     parameters:
-      max_connections: "123"
+#     max_connections: "100"
 #     max_worker_processes: "60"
 #     max_worker_processes: "8"
 #     max_parallel_workers: "8"
@@ -81,7 +81,7 @@ spec:
 #   size: 1Gi
 
   backup:
-    retentionPolicy: 30d
+    retentionPolicy: 7d # don't think this does anything?
     volumeSnapshot:
       labels:
         snapshotOf: cnpg-db-psql-stack
@@ -106,6 +106,6 @@ spec:
   schedule: "0 0 */12 * * *"
   cluster:
     name: cnpg-db-psql-stack
-  method: barmanObjectStore # volumeSnapshot
+  method: volumeSnapshot # barmanObjectStore
   backupOwnerReference: self
   immediate: true

package/src/helm-charts/cnpg-db-tsdb-basic/cluster.yaml.ovh

@@ -13,18 +13,19 @@ spec:
     enablePodMonitor: false # set true once monitoring is setup
 
   description: "Leverege Plain Timescale DB"
-  imageName: us-docker.pkg.dev/leverege-registry/system/images/leverege-pgsql:16.2-cnpg-tsdb2-beta.4
+  imageName: us-docker.pkg.dev/leverege-registry/system/images/leverege-pgsql:16.3-cnpg-tsdb2-beta.2
   bootstrap:
     initdb:
+#     database: imagine # let it default to cnpg app db
       postInitTemplateSQL:
         - CREATE EXTENSION timescaledb;
         - CREATE EXTENSION jsquery;
-        - CREATE DATABASE app; # for the operator restores
-        - CREATE ROLE app;     # same
-      database: imagine
-      owner: imagine
-      secret:
-        name: cnpg-db-tsdb-basic-imagine-pw # imagine db pw
+        - CREATE DATABASE imagine
+#       - CREATE DATABASE app; # for the operator restores
+#       - CREATE ROLE app;     # same
+#     owner: imagine
+#     secret:
+#       name: cnpg-db-tsdb-basic-imagine-pw # imagine db pw
 
   enableSuperuserAccess: true
   superuserSecret:
@@ -54,7 +55,7 @@ spec:
       - timescaledb
       - jsquery
     parameters:
-      max_connections: "123"
+#     max_connections: "100"
 #     max_worker_processes: "60"
 #     max_worker_processes: "8"
 #     max_parallel_workers: "8"
@@ -108,6 +109,6 @@ spec:
   schedule: "0 0 */12 * * *"
   cluster:
     name: cnpg-db-tsdb-basic
-  method: volumeSnapshot
+  method: volumeSnapshot # barmanObjectStore
   backupOwnerReference: self
   immediate: true

package/src/helm-charts/cnpg-db-tsdb-dense/cluster.yaml.ovh

@@ -13,18 +13,15 @@ spec:
     enablePodMonitor: false # set true once monitoring is setup
 
   description: "Leverege Dense Timescale DB"
-  imageName: us-docker.pkg.dev/leverege-registry/system/images/leverege-pgsql:16.2-cnpg-tsdb2-beta.4
+  imageName: us-docker.pkg.dev/leverege-registry/system/images/leverege-pgsql:16.3-cnpg-tsdb2-beta.2
   bootstrap:
     initdb:
       postInitTemplateSQL:
         - CREATE EXTENSION timescaledb;
         - CREATE EXTENSION jsquery;
-        - CREATE DATABASE app; # for the operator restores
-        - CREATE ROLE app;     # same
-      database: imagine
-      owner: imagine
-      secret:
-        name: cnpg-db-tsdb-dense-imagine-pw # imagine db pw
+        - CREATE DATABASE imagine; # leverege tsdb db
+#     secret:
+#       name: cnpg-db-tsdb-dense-imagine-pw # imagine db pw
 
   enableSuperuserAccess: true
   superuserSecret:

package/src/helm-charts/cnpg-operator/helmup.plugin

@@ -1,10 +1,18 @@
 #!/bin/bash
 #
-# See => https://github.com/cloudnative-pg/charts
-addHelmRepo cnpg https://cloudnative-pg.github.io/charts
+# See => https://cloudnative-pg.io/documentation/current/installation_upgrade/
 
-helm upgrade --install cnpg cnpg/cloudnative-pg \
-  --namespace cnpg-system --create-namespace \
-  --set webhook.port="10250" $HELM_WHAT
+OPVER="1.24.0"
 
-removeHelmRepo cnpg
+kubectl apply --server-side -f \
+    https://raw.githubusercontent.com/cloudnative-pg/cloudnative-pg/release-${OPVER%.*}/releases/cnpg-${OPVER}.yaml $K8S_WHAT
+
+# Old approach used the helm chart.
+#
+#addHelmRepo cnpg https://cloudnative-pg.github.io/charts
+#
+#helm upgrade --install cnpg cnpg/cloudnative-pg \
+#  --namespace cnpg-system --create-namespace \
+#  --set webhook.port="10250" $HELM_WHAT
+#
+#removeHelmRepo cnpg

package/src/helm-charts/elasticsearch8/helmup.plugin

@@ -2,13 +2,14 @@
 #
 showInstalling "Elastic Search 8"
 
+[ -z "$ELASTIC_CHART_VERSION" ] && ELASTIC_CHART_VERSION="21.3.17"
+
 OCI_CHART="oci://registry-1.docker.io/bitnamicharts/elasticsearch"
 
 LOCAL_ES_VALUES="elasticsearch8/elasticsearch-local-dev.yaml"
 #LOCAL_ES_VALUES="elasticsearch8/elasticsearch-local-prd.yaml"
 [ ! -f "$LOCAL_ES_VALUES" ] && exitOnError 1 "Missing the local ES config file => `color y $LOCAL_ES_VALUES`"
 
-[ -z "$ELASTIC_CHART_VERSION" ] && ELASTIC_CHART_VERSION="19.21.2"
 helm upgrade --install elasticsearch8 $OCI_CHART \
              --namespace elastic --create-namespace \
              --values $LOCAL_ES_VALUES \
@@ -18,7 +19,7 @@ helm upgrade --install elasticsearch8 $OCI_CHART \
 addHelmRepo prometheus-community https://prometheus-community.github.io/helm-charts
 helm upgrade --install elasticsearch8-exporter prometheus-community/prometheus-elasticsearch-exporter \
              --namespace monitoring --create-namespace \
-             --version 5 \
+             --version 6 \
              -f - <<ELASTIC8_EXPORTER_CHART_MOD
 fullnameOverride: elasticsearch8-exporter
 

package/src/helm-charts/postgres/helmup.plugin

@@ -4,10 +4,13 @@ showInstalling "PostgreSQL Database"
 
 errorIfMissingSecret authz-postgres
 
-addBitnamiRepo # latest
+# addBitnamiRepo # latest
 
-[ -z "$POSTGRESQL_CHART_VERSION" ] && POSTGRESQL_CHART_VERSION="10"
-helm upgrade --install postgres bitnami/postgresql  \
+OCI_CHART="oci://registry-1.docker.io/bitnamicharts/postgresql"
+
+[ -z "$POSTGRESQL_CHART_VERSION" ] && POSTGRESQL_CHART_VERSION="15.5.22"
+
+helm upgrade --install postgres $OCI_CHART          \
              --values  postgres/postgres-local.yaml \
              --version $POSTGRESQL_CHART_VERSION $HELM_WHAT
 

package/src/helm-charts/{kube-prometheus-stack → prometheus-operator}/helmup.plugin

@@ -1,13 +1,14 @@
 #!/bin/bash
 #
-showInstalling "The Prometheus Kube Stack"
+showInstalling "The Prometheus Operator (kube-prometheus-stack)"
 
 addHelmRepo prometheus-community https://prometheus-community.github.io/helm-charts
 
-[ -z "$PROMETHEUS_STACK_CHART_VERSION" ] && PROMETHEUS_STACK_CHART_VERSION="57"
+[ -z "$PROMETHEUS_STACK_CHART_VERSION" ] && PROMETHEUS_STACK_CHART_VERSION="62"
 
 helm upgrade --install prometheus-stack prometheus-community/kube-prometheus-stack \
              --namespace prometheus --create-namespace \
+             --values  prometheus-operator/prometheus-stack.yaml \
              --version $PROMETHEUS_STACK_CHART_VERSION $HELM_WHAT
 
 removeHelmRepo prometheus-community

package/src/helm-charts/prometheus-operator/prometheus-stack.yaml

@@ -0,0 +1,59 @@
+# Bootstrapped from the cnpg monitoring setup
+enabled: true
+kubeControllerManager:
+  enabled: false
+nodeExporter:
+  enabled: false
+defaultRules:
+  create: true
+  rules:
+    alertmanager: false
+    etcd: false
+    configReloaders: false
+    general: false
+    k8s: true
+    kubeApiserver: false
+    kubeApiserverAvailability: false
+    kubeApiserverSlos: false
+    kubelet: true
+    kubeProxy: false
+    kubePrometheusGeneral: false
+    kubePrometheusNodeRecording: false
+    kubernetesApps: false
+    kubernetesResources: false
+    kubernetesStorage: false
+    kubernetesSystem: false
+    kubeScheduler: false
+    kubeStateMetrics: false
+    network: false
+    node: true
+    nodeExporterAlerting: false
+    nodeExporterRecording: true
+    prometheus: false
+    prometheusOperator: false
+
+#nodeSelector:
+  #workload: monitor
+prometheus:
+  prometheusSpec:
+    podMonitorSelectorNilUsesHelmValues: false
+    ruleSelectorNilUsesHelmValues: false
+    serviceMonitorSelectorNilUsesHelmValues: false
+    probeSelectorNilUsesHelmValues: false
+  #nodeSelector:
+    #workload: monitor
+grafana:
+  enabled: true
+  # -- the grafana admin password
+  adminPassword: prom-operator
+  defaultDashboardsEnabled: false
+  sidecar:
+      dashboards:
+        enabled: true
+  #nodeSelector:
+    #workload: monitor
+alertmanager:
+  enabled: true
+  #alertManagerSpec:
+    #nodeSelector:
+      #workload: monitor

package/src/helm-charts/redis/helmup.plugin

@@ -4,7 +4,7 @@ showInstalling "Redis"
 
 OCI_CHART="oci://registry-1.docker.io/bitnamicharts/redis"
 
-[ -z "$REDIS_CHART_VERSION" ] && REDIS_CHART_VERSION="19.6.1"
+[ -z "$REDIS_CHART_VERSION" ] && REDIS_CHART_VERSION="20.1.3"
 
 helm upgrade --install redis $OCI_CHART       \
              --values  redis/redis-local.yaml \

package/src/helm-charts/transponder-dh/values-local.yaml

@@ -10,8 +10,8 @@ config:
   DENSE_HISTORY_TDB_CACHE_KEY_PREFIX: "timescale-dh"
   DENSE_HISTORY_MIGRATE_PAGE_SIZE_LIMIT: "50"
 
-# PG_HOST: "cnpg-db-tsdb-dense-rw"
-  PG_HOST: "cnpg-db-tsdb-dense-pool-rw.cnpg-operands"
+  PG_HOST: "cnpg-db-tsdb-dense-rw.cnpg-operands"
+# PG_HOST: "cnpg-db-tsdb-dense-pool-rw.cnpg-operands"
 
   # password from secret
   PG_PASSWORD_SECRET: "cnpg-db-tsdb-dense-postgres-pw"

package/src/helm-charts/transponder-tsdb/values-local.yaml

@@ -15,6 +15,9 @@ config:
 
 # CNPG override
 # PG_HOST: "cnpg-db-tsdb-basic-rw.cnpg-operands"
+#
+# PG_PASSWORD_SECRET: "cnpg-db-tsdb-basic-postgres-pw"
+# PG_PASSWORD_KEY: "password"
 
   TRANSPORT_CONFIG: '{
     "type": "pubsub",

package/src/helmdn.sh

@@ -38,7 +38,6 @@ PLATFORM=(
 )
 
 SYSTEM=(
-  cert-manager
   elastic
   preemptible-killer
   redis

package/src/helmup.sh

@@ -47,13 +47,26 @@ AUXILIARY=(
   vin-decoder-server
 )
 
-# zombie-killer disabled 20231228
+# zombie-killer re-enabled 20240801
 SYSTEM=(
+  elasticsearch8
+  redis
+  cnpg-operator
+  preemptible-killer
+  zombie-killer
+  reflector
+  reloader
+  traefik
+  velero
+)
+
+SYSTEM_LEGACY=(
   elasticsearch8
   redis
   postgres
   timescale-db
   preemptible-killer
+  zombie-killer
   reflector
   reloader
   traefik
@@ -63,7 +76,7 @@ SYSTEM=(
 
 function isLeveregeService() {
   local answer=0
-  [[ ${PLATFORM[@]} =~ $1 || ${AUXILIARY[@]} =~ $1 || ${SYSTEM[@]} =~ $1 ]] && answer=1
+  [[ ${PLATFORM[@]} =~ $1 || ${AUXILIARY[@]} =~ $1 || ${SYSTEM_LEGACY[@]}} =~ 1 || ${SYSTEM[@]} =~ $1 ]] && answer=1
   echo $answer
 }
 
@@ -530,14 +543,15 @@ function initializeCnpgOperand() {
   local IMAGINE_PW_NAME="$CNPG_SERVICE-imagine-pw"
   local POSTGRES_PW_NAME="$CNPG_SERVICE-postgres-pw"
 
-  printf "\nCreating imagine db password for `color g $CNPG_SERVICE`\n"
-  kubectl create secret generic -n $CNPG_NAMESPACE "$IMAGINE_PW_NAME" \
-        --from-literal=username="imagine" \
-        --from-literal=password="$(generatePassword)" &> $DEVNULL
-  warnOnError $? "the `color y $IMAGINE_PW_NAME` secret may already exist"
+# TODO: we use postgres root password for access - that is bad - fix it?
+# printf "\nCreating imagine db password for `color g $CNPG_SERVICE`\n"
+# kubectl create secret generic -n $CNPG_NAMESPACE "$IMAGINE_PW_NAME" \
+#       --from-literal=username="imagine" \
+#       --from-literal=password="$(generatePassword)" &> $DEVNULL
+# warnOnError $? "the `color y $IMAGINE_PW_NAME` secret may already exist"
 
-  # reflect the secret to default namespace
-  reflectSecretIntoNamespaces $IMAGINE_PW_NAME $CNPG_NAMESPACE "default"
+# # reflect the secret to default namespace
+# reflectSecretIntoNamespaces $IMAGINE_PW_NAME $CNPG_NAMESPACE "default"
 
   printf "\nCreating postgres db password for `color g $CNPG_SERVICE`\n"
   kubectl create secret generic -n $CNPG_NAMESPACE "$POSTGRES_PW_NAME" \
@@ -880,6 +894,10 @@ CATBACKUP
         doInstall "${PLATFORM[@]}"
         ;;
 
+      "system-legacy")
+        doInstall "${SYSTEM_LEGACY[@]}"
+        ;;
+
       "system")
         doInstall "${SYSTEM[@]}"
         ;;
@@ -967,7 +985,7 @@ CATBACKUP
         bootstrapLocalSetup $SERVICE
         ;;
 
-      "prometheus")
+      "prometheus"|"prometheus-operator")
         bootstrapLocalSetup $SERVICE
         ;;
 

package/src/overwhelm.mjs

@@ -64,7 +64,7 @@ if ( !fs.existsSync( 'overwhelm.yaml' ) ) {
   process.exit( 1 )
 }
 
-if ( fs.existsSync( 'values-default.yaml' ) ) {
+if ( !args.key && fs.existsSync( 'values-default.yaml' ) ) {
   log( chalk.red.bold( `
 
 ***ERROR overwhelm => values-default.yaml is no longer used/supported and will