@leverege/build-tools 2.53.5 → 2.54.0-beta.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@leverege/build-tools",
-  "version": "2.53.5",
+  "version": "2.54.0-beta.1",
   "description": "A collection of build / support tools for Leverege developers",
   "main": "index.js",
   "repository": {
@@ -56,7 +56,6 @@
   "license": "SEE LICENSE IN LICENSE.md",
   "dependencies": {
     "@google-cloud/artifact-registry": "^3.4.0",
-    "@leverege/build-tools": "^2.53.2-beta.1",
     "ansi-colors": "^4.1.3",
     "chalk": "^5.3.0",
     "command-line-args": "^6.0.0",
@@ -66,20 +65,20 @@
     "execa": "^9.4.0",
     "glob": "^11.0.0",
     "handlebars": "^4.7.8",
-    "inquirer": "^11.1.0",
+    "inquirer": "^12.0.0",
     "js-yaml": "^4.1.0",
     "ms": "^2.1.3",
-    "npm-registry-fetch": "^18.0.0",
+    "npm-registry-fetch": "^18.0.1",
     "package-up": "^5.0.0",
     "parse-gitignore": "^2.0.0",
     "read-pkg": "^9.0.1",
     "readline-sync": "^1.4.10",
     "semver": "^7.6.3",
     "simple-git": "^3.27.0",
-    "zx": "^8.1.8"
+    "zx": "^8.1.9"
   },
   "devDependencies": {
     "@leverege/eslint-config-leverege": "^4.2.0",
-    "npm": "^10.8.3"
+    "npm": "^10.9.0"
   }
-}
+}

package/src/Docker.mjs

@@ -304,7 +304,6 @@ const buildContainerImage = async ( {
            ${gcloudLogs}  \\
            ${gcloudTags}
   ` ) )
-    
 
   if ( process.env.DRY_RUN === '1' ) {
     log( '\n***Exiting from DRY_RUN\n' )

package/src/build-and-push.sh

@@ -0,0 +1,83 @@
+#!/bin/bash
+
+application_name=$(awk -F'"' '/\[(tool\.poetry|project)\]/{flag=1;next} flag==1 && /^name = /{ print $2; exit }' ./pyproject.toml)
+application_version=$(awk -F'"' '/\[(tool\.poetry|project)\]/{flag=1;next} flag==1 && /^version = /{ print $2; exit }' ./pyproject.toml)
+gcp_project_id=$(awk -F'"' '/\[tool\.docker\]/{flag=1;next} flag==1 && /^gcp-project-id = /{ print $2; exit }' ./pyproject.toml)
+gcp_repository_name=$(awk -F'"' '/\[tool\.docker\]/{flag=1;next} flag==1 && /^gcp-repository-name = /{ print $2; exit }' ./pyproject.toml)
+platform=$(awk -F'[][]' '/\[tool\.docker\]/{flag=1;next} flag==1 && /^platform *=/ { gsub(/"/, "", $2); print $2; exit }' ./pyproject.toml)
+
+# Check if the platform is empty
+if [ -z "$platform" ]; then
+  platform="linux/amd64,linux/arm64"
+fi
+
+# Check if the version contains alpha, beta, rc, or a hyphen
+if [[ $application_version =~ (alpha|beta|rc|-) ]]; then
+  is_stable_release="false"
+else
+  is_stable_release="true"
+fi
+
+cache_path="$HOME/docker/$application_name"
+
+echo -e "\n\033[0;32mBuild Information:\033[0m\n"
+
+# List application name, application version, GCP project ID, GCP repository name, platform, and if it is a stable release.
+echo -e "\033[0;32mApplication Name:\033[0m $application_name"
+echo -e "\033[0;32mApplication Version:\033[0m $application_version"
+echo -e "\033[0;32mGCP Project ID:\033[0m $gcp_project_id"
+echo -e "\033[0;32mGCP Repository Name:\033[0m $gcp_repository_name"
+echo -e "\033[0;32mPlatform:\033[0m $platform"
+echo -e "\033[0;32mStable Release:\033[0m $is_stable_release"
+
+# Prompt for confirmation
+echo -e "\n"
+read -p "Do you wish to proceed? (y/n): " -n 1 -r
+echo -e "\n"
+
+if [[ ! $REPLY =~ ^[Yy]$ ]]; then
+  echo -e "\033[0;31mExiting...\033[0m\n\n"
+  exit 1
+fi
+
+
+echo -e "\n\033[0;32mBuilding, creating and pushing image(s)...\033[0m\n\n"
+
+export GCLOUD_ACCESS_TOKEN=$(gcloud auth print-access-token)
+
+if [ "$is_stable_release" = "true" ]; then
+  docker buildx build \
+    --platform $platform \
+    --cache-from type=local,src=$cache_path \
+    --cache-to type=local,dest=$cache_path \
+    --build-arg APPLICATION_NAME=$application_name \
+    --build-arg APPLICATION_VERSION=$application_version \
+    --secret id=token,env=GCLOUD_ACCESS_TOKEN \
+    --secret="id=creds,src=$HOME/.config/gcloud/application_default_credentials.json" \
+    --tag us-docker.pkg.dev/$gcp_project_id/$gcp_repository_name/${application_name}:${application_version} \
+    --tag us-docker.pkg.dev/$gcp_project_id/$gcp_repository_name/${application_name}:latest \
+    --push \
+    .
+else
+  docker buildx build \
+    --platform $platform \
+    --cache-from type=local,src=$cache_path \
+    --cache-to type=local,dest=$cache_path \
+    --build-arg APPLICATION_NAME=$application_name \
+    --build-arg APPLICATION_VERSION=$application_version \
+    --secret id=token,env=GCLOUD_ACCESS_TOKEN \
+    --secret="id=creds,src=$HOME/.config/gcloud/application_default_credentials.json" \
+    --tag us-docker.pkg.dev/$gcp_project_id/$gcp_repository_name/${application_name}:${application_version} \
+    --push \
+    .
+fi
+
+if [ $? -eq 0 ]; then
+    echo -e "\n\n\033[0;32mBuilt and pushed us-docker.pkg.dev/$gcp_project_id/$gcp_repository_name/$application_name:$application_version for $platform\033[0m"
+    if [ "$is_stable_release" = "true" ]; then
+        echo -e "\033[0;32mAlso tagged as 'latest'\033[0m"
+    fi
+    echo -e "\n\n"
+else
+    echo -e "\n\n\033[0;31mBuild and push failed!\033[0m\n\n"
+fi

package/src/docker-to-registry.mjs

@@ -33,7 +33,7 @@ for ( let n = 2; n < process.argv.length; n++ ) {
   if ( process.argv[n].startsWith( '-' ) ) {
     const str = process.argv[n].slice( 1 )
     if ( str.indexOf( '=' ) > 0 ) {
-      const [key, value] = str.split( '=' )
+      const [ key, value ] = str.split( '=' )
       options[key] = value
     } else {
       options[str] = true
@@ -165,8 +165,9 @@ if ( giveGuidance ) {
 }
 
 // do chart checks too
+let chart
 if ( options?.helmChecks !== 'false' ) {
-  const chart = helmChart?.yaml['Chart.yaml']
+  chart = helmChart?.yaml['Chart.yaml']
   const values = helmChart?.yaml['values.yaml']
   const expectedChartRegistry = `${artifactRegistry}/images`
 

package/src/helm-charts/api-server/values-local.yaml

@@ -5,7 +5,7 @@
 #     - node tools/UpdateESMappingV2.js
 #
 image:
-  tag: # v5.12.3
+  tag: # v5.16.0
 
 config:
   LOG_CONFIG: '{"type":"pino","level":"warn"}'

package/src/helm-charts/authz-server/values-local.yaml

@@ -4,7 +4,7 @@
 #     - npm run sequelizeRunMigrations
 #
 image:
-  tag: # v2.1.3
+  tag: # v2.2.0
 
 config:
   LOG_CONFIG: '{"type":"pino","level":"warn"}'

package/src/helm-charts/cnpg-db-recovery/cluster.yaml.ovh

@@ -0,0 +1,106 @@
+# Artifact Chart  => https://artifacthub.io/packages/helm/cloudnative-pg/cloudnative-pg
+# Examples Charts => https://cloudnative-pg.io/documentation/current/samples/
+apiVersion: postgresql.cnpg.io/v1
+kind: Cluster
+metadata:
+  name: cnpg-db-recovery
+  namespace: cnpg-operands
+spec:
+  instances: 1
+  logLevel: info # warning debug
+
+  monitoring:
+    enablePodMonitor: false # set true once monitoring is setup
+
+  description: "Leverege CNPG DB (recovered)"
+  imageName: us-docker.pkg.dev/leverege-registry/system/images/leverege-pgsql:16.4-cnpg-tsdb2-rel.1
+  bootstrap:
+    recovery:
+      volumeSnapshots:
+        storage:
+          name: <snapshot_name>
+          kind: VolumeSnapshot
+          apiGroup: snapshot.storage.k8s.io
+
+  enableSuperuserAccess: true
+  superuserSecret:
+    name: <postgres_password> # postgres db pw
+
+  affinity:
+    enablePodAntiAffinity: true
+    topologyKey: kubernetes.io/hostname # default value
+    nodeAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+        nodeSelectorTerms:
+        - matchExpressions:
+          - key: target-env
+            operator: In
+            values:
+            - database
+    tolerations:
+      - key: "database"
+        operator: "Equal"
+        value: "true"
+        effect: "NoSchedule"
+
+  # Parameters and pg_hba configuration will be append
+  # to the default ones to make the cluster work
+  postgresql:
+    parameters:
+#     max_connections: "100"
+#     max_worker_processes: "60"
+#     max_worker_processes: "8"
+#     max_parallel_workers: "8"
+#     shared_buffers: 2GB
+#     effective_cache_size: 6GB
+#     maintenance_work_mem: 1GB
+#     work_mem: 20MB
+
+  # - unsupervised: automated update of the primary once all
+  #                 replicas have been upgraded (default)
+  # - supervised: requires manual supervision to perform
+  #               the switchover of the primary
+  primaryUpdateStrategy: unsupervised
+
+  serviceAccountTemplate:
+    metadata:
+      annotations:
+        iam.gke.io/gcp-service-account: cnpg-operands-sa@OVH:<PROJECT_ID>.iam.gserviceaccount.com
+
+  # Require 50Gi of space per instance using ssd storage class
+  storage:
+    storageClass: premium-rwo
+    size: 50Gi
+# walStorage:
+#   storageClass: premium-rwo
+#   size: 1Gi
+
+  backup:
+    retentionPolicy: 7d # don't think this does anything?
+    volumeSnapshot:
+      labels:
+        snapshotOf: cnpg-db-recovery
+      className: cnpg-snapshotclass
+    barmanObjectStore:
+      destinationPath: gs://OVH:<PROJECT_ID>-barman
+      googleCredentials:
+        gkeEnvironment: true
+      tags:
+        cnpgCluster: cnpg-db-recovery
+      wal:
+        compression: gzip
+        maxParallel: 8
+---
+apiVersion: postgresql.cnpg.io/v1
+kind: ScheduledBackup
+metadata:
+  name: cnpg-db-recovery-backup
+  namespace: cnpg-operands
+spec:
+  # https://pkg.go.dev/github.com/robfig/cron#hdr-CRON_Expression_Format
+  schedule: "0 0 */12 * * *"
+  cluster:
+    name: cnpg-db-recovery
+  method: volumeSnapshot # barmanObjectStore
+  backupOwnerReference: self
+  immediate: true

package/src/helm-charts/cnpg-db-recovery/gitignore

@@ -0,0 +1,2 @@
+*.swp
+cluster.yaml

package/src/helm-charts/cnpg-db-recovery/helmdn.plugin

@@ -0,0 +1,12 @@
+#!/bin/bash
+#
+# We only want to hibernate unless the intent is to really delete EVERYTHING
+# which includes PVCs and volume snapshots.
+#
+printf "\nHibernating the cnpg-db-recovery cluster...\n"
+kubectl cnpg hibernate on -n cnpg-operands cnpg-db-recovery
+
+# *** DANGER ***
+# kubectl delete  will destroy the cluster
+# *** DANGER ***
+# kubectl delete -f cnpg-db-recovery/cluster.yaml $K8S_WHAT

package/src/helm-charts/cnpg-db-recovery/helmup.plugin

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+# ensures the PG_PASSWORD secret exists and correct
+# ensureCnpgSecret PG_PASSWORD $SERVICE-postgres-pw
+
+kubectl apply -f cnpg-db-recovery/cluster.yaml $K8S_WHAT

package/src/helm-charts/cnpg-db-recovery/pooler.yaml

@@ -0,0 +1,20 @@
+apiVersion: postgresql.cnpg.io/v1
+kind: Pooler
+metadata:
+  name: cnpg-db-recovery-pool-rw
+  namespace: cnpg-operands
+spec:
+  cluster:
+    name: cnpg-db-recovery
+
+  instances: 1
+  type: rw
+  pgbouncer:
+    poolMode: session
+    parameters:
+      # https://www.pgbouncer.org/config.html#generic-settings
+      max_client_conn: "500"
+      default_pool_size: "10"
+      # https://www.pgbouncer.org/config.html#log-settings
+      log_connections: "0"
+      log_disconnections: "0"

package/src/helm-charts/db-curator/values-local.yaml

@@ -1,5 +1,5 @@
 image:
-  tag: # v4.2.4
+  tag: # v4.3.0
 
 config:
   LOG_CONFIG: '{"type":"pino","level":"warn"}'

package/src/helm-charts/elasticsearch8/helmup.plugin

@@ -2,7 +2,7 @@
 #
 showInstalling "Elastic Search 8"
 
-[ -z "$ELASTIC_CHART_VERSION" ] && ELASTIC_CHART_VERSION="21.3.17"
+[ -z "$ELASTIC_CHART_VERSION" ] && ELASTIC_CHART_VERSION="21.3.20"
 
 OCI_CHART="oci://registry-1.docker.io/bitnamicharts/elasticsearch"
 

package/src/helm-charts/imagine/values-local.yaml

@@ -1,5 +1,5 @@
 mage:
-  tag: # v2.2.0
+  tag: # v2.4.1
 
 config:
   LOG_CONFIG: '{"type":"pino","level":"warn"}'

package/src/helm-charts/pubsub-pulse/values-local.yaml

@@ -1,6 +1,5 @@
 image:
-  tag: # v1.3.5
-# registry: gcr.io/leverege-docker-images # deprecated
+  tag: # v1.6.3
 # registry: us-docker.pkg.dev/leverege-registry/leverege/images
 
 config:

package/src/helm-charts/reason/values-local.yaml

@@ -1,5 +1,5 @@
 image:
-  tag: # v2.2.1
+  tag: # v2.3.0
 
 config:
   CF_TRANSPORT_CONFIG: '{

package/src/helm-charts/redis/helmup.plugin

@@ -4,7 +4,7 @@ showInstalling "Redis"
 
 OCI_CHART="oci://registry-1.docker.io/bitnamicharts/redis"
 
-[ -z "$REDIS_CHART_VERSION" ] && REDIS_CHART_VERSION="20.1.3"
+[ -z "$REDIS_CHART_VERSION" ] && REDIS_CHART_VERSION="20.1.7"
 
 helm upgrade --install redis $OCI_CHART       \
              --values  redis/redis-local.yaml \

package/src/helm-charts/resource-server/values-local.yaml

@@ -4,7 +4,7 @@
 #     - npm run sequelizeRunMigrations
 #
 image:
-  tag: # v1.2.2
+  tag: # v1.5.1
 
 config:
   LOG_CONFIG: '{"type":"pino","level":"warn"}'

package/src/helm-charts/rule-engine/values-local.yaml

@@ -1,5 +1,5 @@
 image:
-  tag: # v2.3.1
+  tag: # v2.6.0
 
 config:
   LOG_CONFIG: '{"type":"pino","level":"warn"}'

package/src/helm-charts/scheduler/values-local.yaml

@@ -1,5 +1,5 @@
 image:
-  tag: # v3.1.1
+  tag: # v3.1.2
 
 config:
   LOG_CONFIG: '{"type":"pino","level":"warn"}'

package/src/helm-charts/transponder-bq/values-local.yaml

@@ -1,7 +1,7 @@
 nameOverride: transponder-bq
 
 image:
-  tag: # v5.2.2
+  tag: # v5.2.4
 
 config:
   TRANSPONDER_RUN_MODE: "bigQueryJson"

package/src/helm-charts/transponder-dh/values-local.yaml

@@ -1,7 +1,7 @@
 nameOverride: transponder-dh
 
 image:
-  tag: # v5.2.2
+  tag: # v5.2.4
 
 config:
   TRANSPONDER_RUN_MODE: "denseHistoryCombined"

package/src/helm-charts/transponder-rt/values-local.yaml

@@ -1,7 +1,7 @@
 nameOverride: transponder-rt
 
 image:
-  tag: # v5.2.2
+  tag: # v5.2.4
 
 config:
   TRANSPONDER_RUN_MODE: "rt"

package/src/helm-charts/transponder-tsdb/values-local.yaml

@@ -7,7 +7,7 @@
 nameOverride: transponder-tsdb
 
 image:
-  tag: # v5.2.2
+  tag: # v5.2.4
 
 config:
   TRANSPONDER_RUN_MODE: "timescale"