@leverege/build-tools 2.48.6 → 2.49.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@leverege/build-tools",
-  "version": "2.48.6",
+  "version": "2.49.0",
   "description": "A collection of build / support tools for Leverege developers",
   "main": "index.js",
   "repository": {
@@ -62,7 +62,7 @@
     "deepmerge": "^4.3.1",
     "enquirer": "^2.4.1",
     "execa": "^8.0.1",
-    "glob": "^10.3.10",
+    "glob": "^10.3.12",
     "handlebars": "^4.7.8",
     "inquirer": "^9.2.16",
     "js-yaml": "^4.1.0",
@@ -73,8 +73,7 @@
     "read-pkg": "^9.0.1",
     "readline-sync": "^1.4.10",
     "semver": "^7.6.0",
-    "simple-git": "^3.22.0",
-    "tempy": "^3.1.0",
+    "simple-git": "^3.24.0",
     "zx": "^7.2.3"
   },
   "devDependencies": {

package/registry-compass.yaml

@@ -0,0 +1,20 @@
+registry:
+  - root: us-docker.pkg.dev
+  - repositories:
+    - name: stack
+      charts:
+        - api-server
+        - authz-server
+        - emailer
+        - message-processor
+    - name: leverege
+      charts:
+        - pubsub-pulse
+        - pusher
+        - overdose
+    - name: cox-health
+      charts:
+        - actions-server
+        - analytics-server
+        - centrak-healthz
+        - centrak-ingestor

package/src/chart-compass.mjs

@@ -0,0 +1,96 @@
+#!/usr/bin/env node
+/*
+ * chart-to-registry will...
+ */
+import fs from 'node:fs'
+
+import chalk from 'chalk'
+import commandLineArgs from 'command-line-args'
+import commandLineUsage from 'command-line-usage'
+import { lt as semverLt } from 'semver'
+import YAML from 'js-yaml'
+
+import {
+  condir,
+  debug,
+  errorExit,
+  log,
+  warning,
+  getGitRootDirectory,
+  parsePackageJson,
+  parseHelmChart,
+  shellCmd } from './Utils.mjs'
+
+const commandLineOptions = [ // Use commandLineOptions to tie into the Usage statements
+  /* eslint-disable max-len */
+  {
+    name : 'location',
+    type : String,
+    description : '{green the location of the artifact registry the chart will be pushed to (default us-docker.pkg.dev)}',
+  },
+  {
+    name : 'project',
+    type : String,
+    description : '{green the name of the google project containing the npmrc and slack config secrets (default leverege-registry)}',
+  },
+  {
+    name : 'repository',
+    type : String,
+    description : '{green the target repository to receive the pushed chart}',
+  },
+  {
+    name : 'dry-run',
+    type : Boolean,
+    description : '{yellow perform everything except the actual chart push}',
+  },
+  {
+    name : 'help',
+    type : Boolean,
+    description : '{green display this help screen}',
+  },
+  /* eslint-enable max-len */
+]
+
+const sections = [
+  {
+    header : 'Leverege Helm Chart Compass (for helmup)',
+    content : `{green This tool helps helmup navigate the helm charts stored in the
+artifact-registries.}`
+  },
+  { header : 'Options',
+    optionList : commandLineOptions,
+  },
+]
+
+const args = commandLineArgs( commandLineOptions, { camelCase : true, partial : true } )
+const usage = commandLineUsage( sections )
+
+if ( args.help ) {
+  log( usage )
+  process.exit( 0 )
+}
+
+/* eslint-disable no-underscore-dangle */
+if ( args._unknown ) {
+  log( usage )
+  log( `\nUnrecognized argument [${chalk.bold.red( args._unknown )}]\n` )
+  process.exit( 1 )
+}
+/* eslint-enable no-underscore-dangle */
+
+const minNodejsVersion = '18.0.0'
+if ( semverLt( process.version, minNodejsVersion ) ) {
+  errorExit( `\n***ERROR: must be running at least node ${minNodejsVersion}\n` )
+}
+
+// First of all, fail if we are not in a git repository
+let gitRoot
+try {
+  gitRoot = await getGitRootDirectory()
+} catch ( error ) {
+  errorExit( chalk.red.bold( error ), { errorCode : 5 } )
+}
+
+const chartCompass = YAML.load( fs.readFileSync( './registry-compass.yaml', 'utf8' ) )
+
+condir( { chartCompass }, '<==Navigation' )

package/src/cnpg-snapshot.mjs

@@ -1,38 +1,13 @@
 #!/usr/bin/env node
 
-import fs from 'node:fs'
-
-import handlebars from 'handlebars'
-import { temporaryFile } from 'tempy'
-
 import {
-  getDateTimestamp,
-  debug,
+  log,
   shellCmd,
 } from './Utils.mjs'
 
-const backup = `
-apiVersion: postgresql.cnpg.io/v1
-kind: Backup
-metadata:
-  name: {{clusterName}}-snapshot-{{dateTimestamp}}
-  namespace: cnpg-operands
-  labels:
-    cnpg-backup: manual
-    cnpg-operand: {{clusterName}}
-spec:
-  method: volumeSnapshot
-  cluster:
-    name: {{clusterName}}`
-
-const dateTimestamp = getDateTimestamp()
 const clusterName = process.argv[2]
-const compiled = handlebars.compile( backup )
-const replaced = compiled( { dateTimestamp, clusterName } )
-const tmpfile = temporaryFile()
 
-debug( { replaced, tmpfile }, '<==Apply That' )
-
-fs.writeFileSync( tmpfile, replaced ) // eslint-disable-line security/detect-non-literal-fs-filename
 await shellCmd( 'overwhelm' )
-await shellCmd( `kubectl apply -f ${tmpfile}` )
+await shellCmd( `kubectl cnpg backup -m volumeSnapshot -n cnpg-operands ${clusterName}` )
+
+log( '\nTo see backups run => kubectl get backups.postgresql.cnpg.io -n cnpg-operands\n' )

package/src/helm-charts/cnpg-db-pgbench/cnpg-db-pgbench.yaml.ovh

@@ -0,0 +1,100 @@
+# Artifact Chart  => https://artifacthub.io/packages/helm/cloudnative-pg/cloudnative-pg
+# Examples Charts => https://cloudnative-pg.io/documentation/current/samples/
+apiVersion: postgresql.cnpg.io/v1
+kind: Cluster
+metadata:
+  name: cnpg-db-pgbench
+  namespace: cnpg-operands
+spec:
+  instances: 1
+  logLevel: info # warning debug
+
+  description: "Leverege PG Bench Timescale DB"
+  imageName: us-docker.pkg.dev/leverege-registry/system/images/leverege-pgsql:16.1-cnpg-tsdb2-beta.6
+  bootstrap:
+    initdb:
+      postInitTemplateSQL:
+        - CREATE EXTENSION timescaledb;
+        - CREATE EXTENSION jsquery;
+      database: imagine
+      owner: imagine
+      secret:
+        name: cnpg-db-pgbench-imagine-pw # imagine db pw
+
+  enableSuperuserAccess: true
+  superuserSecret:
+    name: cnpg-db-pgbench-postgres-pw # postgres db pw
+
+  affinity:
+#   enablePodAntiAffinity: true
+#   topologyKey: failure-domain.beta.kubernetes.io/zone
+    tolerations:
+      - key: "database"
+        operator: "Equal"
+        value: "true"
+        effect: "NoSchedule"
+
+  # Parameters and pg_hba configuration will be append
+  # to the default ones to make the cluster work
+  postgresql:
+    shared_preload_libraries:
+      - timescaledb
+      - jsquery
+    parameters:
+#     max_worker_processes: "60"
+      max_worker_processes: "8"
+      max_parallel_workers: "8"
+      shared_buffers: 2GB
+      effective_cache_size: 6GB
+      maintenance_work_mem: 1GB
+      work_mem: 20MB
+    pg_hba:
+      # To access through TCP/IP you will need to get username
+      # and password from the secret cluster-example-custom-app
+      - host all all all md5
+
+
+  # Example of rolling update strategy:
+  # - unsupervised: automated update of the primary once all
+  #                 replicas have been upgraded (default)
+  # - supervised: requires manual supervision to perform
+  #               the switchover of the primary
+  primaryUpdateStrategy: unsupervised
+
+  serviceAccountTemplate:
+    metadata:
+      annotations:
+        iam.gke.io/gcp-service-account: cnpg-operands-sa@OVH:<PROJECT_ID>.iam.gserviceaccount.com
+
+  # Require 50Gi of space per instance using ssd storage class
+  storage:
+    storageClass: premium-rwo
+    size: 50Gi
+
+  backup:
+    volumeSnapshot:
+      labels:
+        snapshotOf: cnpg-db-pgbench
+      className: cnpg-snapshotclass
+    barmanObjectStore:
+      destinationPath: gs://OVH:<PROJECT_ID>-barman
+      googleCredentials:
+        gkeEnvironment: true
+      tags:
+        cnpgCluster: cnpg-db-pgbench
+      wal:
+        compression: gzip
+---
+apiVersion: postgresql.cnpg.io/v1
+kind: ScheduledBackup
+metadata:
+  name: cnpg-db-pgbench-backup
+  namespace: cnpg-operands
+spec:
+  # https://pkg.go.dev/github.com/robfig/cron#hdr-CRON_Expression_Format
+  schedule: "0 0 * * * *"
+  cluster:
+    name: cnpg-db-pgbench
+  method: volumeSnapshot
+  backupOwnerReference: self
+  immediate: true

package/src/helm-charts/cnpg-db-pgbench/gitignore

@@ -0,0 +1,2 @@
+*.swp
+cnpg-db-pgbench.yaml

package/src/helm-charts/cnpg-db-pgbench/helmdn.plugin

@@ -0,0 +1,12 @@
+#!/bin/bash
+#
+# We only want to hibernate unless the intent is to really delete EVERYTHING
+# which includes PVCs and volume snapshots.
+#
+printf "\nHibernating the cnpg-db-pgbench cluster...\n"
+kubectl cnpg hibernate on -n cnpg-operands cnpg-db-pgbench
+
+# *** DANGER ***
+# kubectl delete  will destroy the cluster pods and PVCs!!!!!
+# *** DANGER ***
+# kubectl do-not-do-this-delete -f cnpg-db-pgbench/cnpg-db-pgbench.yaml $K8S_WHAT

package/src/helm-charts/cnpg-db-pgbench/helmup.plugin

@@ -0,0 +1,3 @@
+#!/bin/bash
+#
+kubectl apply -f cnpg-db-pgbench/cnpg-db-pgbench.yaml $K8S_WHAT

package/src/helm-charts/cnpg-db-pgbench/pooler.yaml

@@ -0,0 +1,16 @@
+apiVersion: postgresql.cnpg.io/v1
+kind: Pooler
+metadata:
+  name: cnpg-db-pgbench-pool-rw
+  namespace: cnpg-operands
+spec:
+  cluster:
+    name: cnpg-db-pgbench
+
+  instances: 1
+  type: rw
+  pgbouncer:
+    poolMode: session
+    parameters:
+      max_client_conn: "500"
+      default_pool_size: "10"

package/src/helm-charts/{db-postgres-stack/db-postgres-stack.yaml.ovh → cnpg-db-psql-stack/cnpg-db-psql-stack.yaml.ovh}

@@ -3,7 +3,7 @@
 apiVersion: postgresql.cnpg.io/v1
 kind: Cluster
 metadata:
-  name: db-postgres-stack
+  name: cnpg-db-psql-stack
   namespace: cnpg-operands
 spec:
   instances: 1
@@ -16,11 +16,11 @@ spec:
       database: imagine
       owner: imagine
       secret:
-        name: db-postgres-stack-imagine-pw # imagine db pw
+        name: cnpg-db-psql-stack-imagine-pw # imagine db pw
 
   enableSuperuserAccess: true
   superuserSecret:
-    name: db-postgres-stack-postgres-pw # postgres db pw
+    name: cnpg-db-psql-stack-postgres-pw # postgres db pw
 
   affinity:
 #   enablePodAntiAffinity: true
@@ -67,44 +67,26 @@ spec:
     retentionPolicy: 30d
     volumeSnapshot:
       labels:
-        snapshotOf: db-postgres-stack
+        snapshotOf: cnpg-db-psql-stack
       className: cnpg-snapshotclass
     barmanObjectStore:
       destinationPath: gs://OVH:<PROJECT_ID>-barman
       googleCredentials:
         gkeEnvironment: true
       tags:
-        cnpgCluster: db-postgres-stack
+        cnpgCluster: cnpg-db-psql-stack
       wal:
         compression: gzip
-#---
-#apiVersion: postgresql.cnpg.io/v1
-#kind: Pooler
-#metadata:
-#  name: db-postgres-stack-pool-rw
-#  namespace: cnpg-operands
-#spec:
-#  cluster:
-#    name: db-postgres-stack
-#
-#  instances: 1
-#  type: rw
-#  pgbouncer:
-#    poolMode: session
-#    parameters:
-#      max_client_conn: "500"
-#      default_pool_size: "10"
 ---
 apiVersion: postgresql.cnpg.io/v1
 kind: ScheduledBackup
 metadata:
-  name: db-postgres-stack-backup
+  name: cnpg-db-psql-stack-backup
   namespace: cnpg-operands
 spec:
   # https://pkg.go.dev/github.com/robfig/cron#hdr-CRON_Expression_Format
   schedule: "0 0 * * * *"
   cluster:
-    name: db-postgres-stack
   method: volumeSnapshot
   backupOwnerReference: self
   immediate: true

package/src/helm-charts/cnpg-db-psql-stack/gitignore

@@ -0,0 +1,2 @@
+*.swp
+cnpg-db-psql-stack.yaml

package/src/helm-charts/cnpg-db-psql-stack/helmdn.plugin

@@ -0,0 +1,12 @@
+#!/bin/bash
+#
+# We only want to hibernate unless the intent is to really delete EVERYTHING
+# which includes PVCs and volume snapshots.
+#
+printf "\nHibernating the cnpg-db-psql-stack cluster...\n"
+kubectl cnpg hibernate on -n cnpg-operands cnpg-db-psql-stack
+
+# *** DANGER ***
+# kubectl delete  will destroy the cluster
+# *** DANGER ***
+# kubectl delete -f cnpg-db-psql-stack/cnpg-db-psql-stack.yaml $K8S_WHAT

package/src/helm-charts/cnpg-db-psql-stack/helmup.plugin

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+kubectl apply -f cnpg-db-psql-stack/cnpg-db-psql-stack.yaml $K8S_WHAT

package/src/helm-charts/cnpg-db-psql-stack/pooler.yaml

@@ -0,0 +1,16 @@
+apiVersion: postgresql.cnpg.io/v1
+kind: Pooler
+metadata:
+  name: cnpg-db-psql-stack-pool-rw
+  namespace: cnpg-operands
+spec:
+  cluster:
+    name: cnpg-db-psql-stack
+
+  instances: 1
+  type: rw
+  pgbouncer:
+    poolMode: session
+    parameters:
+      max_client_conn: "500"
+      default_pool_size: "10"

package/src/helm-charts/{db-timescale-plain/db-timescale-plain.yaml.ovh → cnpg-db-tsdb-basic/cnpg-db-tsdb-basic.yaml.ovh}

@@ -3,11 +3,11 @@
 apiVersion: postgresql.cnpg.io/v1
 kind: Cluster
 metadata:
-  name: db-timescale-plain
+  name: cnpg-db-tsdb-basic
   namespace: cnpg-operands
 spec:
   instances: 1
-  logLevel: debug
+  logLevel: info # warning debug
 
   description: "Leverege Plain Timescale DB"
   imageName: us-docker.pkg.dev/leverege-registry/system/images/leverege-pgsql:16.1-cnpg-tsdb2-beta.6
@@ -19,11 +19,11 @@ spec:
       database: imagine
       owner: imagine
       secret:
-        name: db-timescale-plain-imagine-pw # imagine db pw
+        name: cnpg-db-tsdb-basic-imagine-pw # imagine db pw
 
   enableSuperuserAccess: true
   superuserSecret:
-    name: db-timescale-plain-postgres-pw # postgres db pw
+    name: cnpg-db-tsdb-basic-postgres-pw # postgres db pw
 
   affinity:
 #   enablePodAntiAffinity: true
@@ -70,51 +70,31 @@ spec:
   storage:
     storageClass: premium-rwo
     size: 50Gi
-  walStorage:
-    storageClass: premium-rwo
-    size: 1Gi
 
   backup:
     volumeSnapshot:
       labels:
-        snapshotOf: db-timescale-plain
+        snapshotOf: cnpg-db-tsdb-basic
       className: cnpg-snapshotclass
     barmanObjectStore:
       destinationPath: gs://OVH:<PROJECT_ID>-barman
       googleCredentials:
         gkeEnvironment: true
       tags:
-        cnpgCluster: db-timescale-plain
+        cnpgCluster: cnpg-db-tsdb-basic
       wal:
         compression: gzip
 ---
 apiVersion: postgresql.cnpg.io/v1
-kind: Pooler
-metadata:
-  name: db-timescale-plain-pool-rw
-  namespace: cnpg-operands
-spec:
-  cluster:
-    name: db-timescale-plain
-
-  instances: 1
-  type: rw
-  pgbouncer:
-    poolMode: session
-    parameters:
-      max_client_conn: "500"
-      default_pool_size: "10"
----
-apiVersion: postgresql.cnpg.io/v1
 kind: ScheduledBackup
 metadata:
-  name: db-timescale-plain-backup
+  name: cnpg-db-tsdb-basic-backup
   namespace: cnpg-operands
 spec:
   # https://pkg.go.dev/github.com/robfig/cron#hdr-CRON_Expression_Format
   schedule: "0 0 * * * *"
   cluster:
-    name: db-timescale-plain
+    name: cnpg-db-tsdb-basic
   method: volumeSnapshot
   backupOwnerReference: self
   immediate: true

package/src/helm-charts/cnpg-db-tsdb-basic/gitignore

@@ -0,0 +1,2 @@
+*.swp
+cnpg-db-tsdb-basic.yaml

package/src/helm-charts/cnpg-db-tsdb-basic/helmdn.plugin

@@ -0,0 +1,12 @@
+#!/bin/bash
+#
+# We only want to hibernate unless the intent is to really delete EVERYTHING
+# which includes PVCs and volume snapshots.
+#
+printf "\nHibernating the cnpg-db-tsdb-basic cluster...\n"
+kubectl cnpg hibernate on -n cnpg-operands cnpg-db-tsdb-basic
+
+# *** DANGER ***
+# kubectl delete  will destroy the cluster pods and PVCs!!!!!
+# *** DANGER ***
+# kubectl do-not-do-this-delete -f cnpg-db-tsdb-basic/cnpg-db-tsdb-basic.yaml $K8S_WHAT

package/src/helm-charts/cnpg-db-tsdb-basic/helmup.plugin

@@ -0,0 +1,3 @@
+#!/bin/bash
+#
+kubectl apply -f cnpg-db-tsdb-basic/cnpg-db-tsdb-basic.yaml $K8S_WHAT

package/src/helm-charts/cnpg-db-tsdb-basic/pooler.yaml

@@ -0,0 +1,16 @@
+apiVersion: postgresql.cnpg.io/v1
+kind: Pooler
+metadata:
+  name: cnpg-db-tsdb-basic-pool-rw
+  namespace: cnpg-operands
+spec:
+  cluster:
+    name: cnpg-db-tsdb-basic
+
+  instances: 1
+  type: rw
+  pgbouncer:
+    poolMode: session
+    parameters:
+      max_client_conn: "500"
+      default_pool_size: "10"

package/src/helm-charts/{db-timescale-dense/db-timescale-dense.yaml.ovh → cnpg-db-tsdb-dense/cnpg-db-tsdb-dense.yaml.ovh}

@@ -3,7 +3,7 @@
 apiVersion: postgresql.cnpg.io/v1
 kind: Cluster
 metadata:
-  name: db-timescale-dense
+  name: cnpg-db-tsdb-dense
   namespace: cnpg-operands
 spec:
   instances: 1
@@ -19,11 +19,11 @@ spec:
       database: imagine
       owner: imagine
       secret:
-        name: db-timescale-dense-imagine-pw # imagine db pw
+        name: cnpg-db-tsdb-dense-imagine-pw # imagine db pw
 
   enableSuperuserAccess: true
   superuserSecret:
-    name: db-timescale-dense-postgres-pw # postgres db pw
+    name: cnpg-db-tsdb-dense-postgres-pw # postgres db pw
 
   affinity:
 #   enablePodAntiAffinity: true
@@ -70,51 +70,31 @@ spec:
   storage:
     storageClass: premium-rwo
     size: 50Gi
-  walStorage:
-    storageClass: premium-rwo
-    size: 1Gi
 
   backup:
     volumeSnapshot:
       labels:
-        snapshotOf: db-timescale-dense
+        snapshotOf: cnpg-db-tsdb-dense
       className: cnpg-snapshotclass
     barmanObjectStore:
       destinationPath: gs://OVH:<PROJECT_ID>-barman
       googleCredentials:
         gkeEnvironment: true
       tags:
-        cnpgCluster: db-timescale-dense
+        cnpgCluster: cnpg-db-tsdb-dense
       wal:
         compression: gzip
 ---
 apiVersion: postgresql.cnpg.io/v1
-kind: Pooler
-metadata:
-  name: db-timescale-dense-pool-rw
-  namespace: cnpg-operands
-spec:
-  cluster:
-    name: db-timescale-dense
-
-  instances: 1
-  type: rw
-  pgbouncer:
-    poolMode: session
-    parameters:
-      max_client_conn: "500"
-      default_pool_size: "10"
----
-apiVersion: postgresql.cnpg.io/v1
 kind: ScheduledBackup
 metadata:
-  name: db-timescale-dense-backup
+  name: cnpg-db-tsdb-dense-backup
   namespace: cnpg-operands
 spec:
   # https://pkg.go.dev/github.com/robfig/cron#hdr-CRON_Expression_Format
   schedule: "0 0 * * * *"
   cluster:
-    name: db-timescale-dense
+    name: cnpg-db-tsdb-dense
   method: volumeSnapshot
   backupOwnerReference: self
   immediate: true

package/src/helm-charts/cnpg-db-tsdb-dense/gitignore

@@ -0,0 +1,2 @@
+*.swp
+cnpg-db-tsdb-dense.yaml

package/src/helm-charts/cnpg-db-tsdb-dense/helmdn.plugin

@@ -0,0 +1,12 @@
+#!/bin/bash
+#
+# We only want to hibernate unless the intent is to really delete EVERYTHING
+# which includes PVCs and volume snapshots.
+#
+printf "\nHibernating the cnpg-db-tsdb-dense cluster...\n"
+kubectl cnpg hibernate on -n cnpg-operands cnpg-db-tsdb-dense
+
+# *** DANGER ***
+# kubectl delete  will destroy the cluster
+# *** DANGER ***
+# kubectl delete -f cnpg-db-tsdb-dense/cnpg-db-tsdb-dense.yaml $K8S_WHAT

package/src/helm-charts/cnpg-db-tsdb-dense/helmup.plugin

@@ -0,0 +1,5 @@
+#!/bin/bash
+#
+#bindWorkloadIdentity cnpg-operands-sa cnpg-operands cnpg-db-tsdb-dense
+
+kubectl apply -f cnpg-db-tsdb-dense/cnpg-db-tsdb-dense.yaml $K8S_WHAT