@leverege/build-tools 2.48.6 → 2.50.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@leverege/build-tools",
-  "version": "2.48.6",
+  "version": "2.50.0",
   "description": "A collection of build / support tools for Leverege developers",
   "main": "index.js",
   "repository": {
@@ -16,7 +16,7 @@
     "build-tools": "src/build-tools.mjs",
     "chart-to-registry": "src/chart-to-registry.mjs",
     "circleate": "src/circleate.sh",
-    "cnpg-snapshot": "src/cnpg-snapshot.mjs",
+    "cnpg-backup": "src/cnpg-backup.mjs",
     "decrypt-secrets": "src/decrypt-secrets.sh",
     "dirty-git": "src/dirty-git.sh",
     "dockreate": "src/dockreate.sh",
@@ -54,7 +54,7 @@
   "author": "Leverege Devs",
   "license": "SEE LICENSE IN LICENSE.md",
   "dependencies": {
-    "@google-cloud/artifact-registry": "^3.2.0",
+    "@google-cloud/artifact-registry": "^3.3.0",
     "ansi-colors": "^4.1.3",
     "chalk": "^5.3.0",
     "command-line-args": "^5.2.1",
@@ -62,23 +62,22 @@
     "deepmerge": "^4.3.1",
     "enquirer": "^2.4.1",
     "execa": "^8.0.1",
-    "glob": "^10.3.10",
+    "glob": "^10.3.12",
     "handlebars": "^4.7.8",
-    "inquirer": "^9.2.16",
+    "inquirer": "^9.2.18",
     "js-yaml": "^4.1.0",
     "ms": "^2.1.3",
-    "npm-registry-fetch": "^16.1.0",
+    "npm-registry-fetch": "^16.2.1",
     "package-up": "^5.0.0",
     "parse-gitignore": "^2.0.0",
     "read-pkg": "^9.0.1",
     "readline-sync": "^1.4.10",
     "semver": "^7.6.0",
-    "simple-git": "^3.22.0",
-    "tempy": "^3.1.0",
-    "zx": "^7.2.3"
+    "simple-git": "^3.24.0",
+    "zx": "^8.0.1"
   },
   "devDependencies": {
     "@leverege/eslint-config-leverege": "^4.2.0",
-    "npm": "^10.5.0"
+    "npm": "^10.5.2"
   }
 }

package/registry-compass.yaml

@@ -0,0 +1,20 @@
+registry:
+  - root: us-docker.pkg.dev
+  - repositories:
+    - name: stack
+      charts:
+        - api-server
+        - authz-server
+        - emailer
+        - message-processor
+    - name: leverege
+      charts:
+        - pubsub-pulse
+        - pusher
+        - overdose
+    - name: cox-health
+      charts:
+        - actions-server
+        - analytics-server
+        - centrak-healthz
+        - centrak-ingestor

package/src/chart-compass.mjs

@@ -0,0 +1,96 @@
+#!/usr/bin/env node
+/*
+ * chart-to-registry will...
+ */
+import fs from 'node:fs'
+
+import chalk from 'chalk'
+import commandLineArgs from 'command-line-args'
+import commandLineUsage from 'command-line-usage'
+import { lt as semverLt } from 'semver'
+import YAML from 'js-yaml'
+
+import {
+  condir,
+  debug,
+  errorExit,
+  log,
+  warning,
+  getGitRootDirectory,
+  parsePackageJson,
+  parseHelmChart,
+  shellCmd } from './Utils.mjs'
+
+const commandLineOptions = [ // Use commandLineOptions to tie into the Usage statements
+  /* eslint-disable max-len */
+  {
+    name : 'location',
+    type : String,
+    description : '{green the location of the artifact registry the chart will be pushed to (default us-docker.pkg.dev)}',
+  },
+  {
+    name : 'project',
+    type : String,
+    description : '{green the name of the google project containing the npmrc and slack config secrets (default leverege-registry)}',
+  },
+  {
+    name : 'repository',
+    type : String,
+    description : '{green the target repository to receive the pushed chart}',
+  },
+  {
+    name : 'dry-run',
+    type : Boolean,
+    description : '{yellow perform everything except the actual chart push}',
+  },
+  {
+    name : 'help',
+    type : Boolean,
+    description : '{green display this help screen}',
+  },
+  /* eslint-enable max-len */
+]
+
+const sections = [
+  {
+    header : 'Leverege Helm Chart Compass (for helmup)',
+    content : `{green This tool helps helmup navigate the helm charts stored in the
+artifact-registries.}`
+  },
+  { header : 'Options',
+    optionList : commandLineOptions,
+  },
+]
+
+const args = commandLineArgs( commandLineOptions, { camelCase : true, partial : true } )
+const usage = commandLineUsage( sections )
+
+if ( args.help ) {
+  log( usage )
+  process.exit( 0 )
+}
+
+/* eslint-disable no-underscore-dangle */
+if ( args._unknown ) {
+  log( usage )
+  log( `\nUnrecognized argument [${chalk.bold.red( args._unknown )}]\n` )
+  process.exit( 1 )
+}
+/* eslint-enable no-underscore-dangle */
+
+const minNodejsVersion = '18.0.0'
+if ( semverLt( process.version, minNodejsVersion ) ) {
+  errorExit( `\n***ERROR: must be running at least node ${minNodejsVersion}\n` )
+}
+
+// First of all, fail if we are not in a git repository
+let gitRoot
+try {
+  gitRoot = await getGitRootDirectory()
+} catch ( error ) {
+  errorExit( chalk.red.bold( error ), { errorCode : 5 } )
+}
+
+const chartCompass = YAML.load( fs.readFileSync( './registry-compass.yaml', 'utf8' ) )
+
+condir( { chartCompass }, '<==Navigation' )

package/src/cnpg-backup.mjs

@@ -0,0 +1,13 @@
+#!/usr/bin/env node
+
+import {
+  log,
+  shellCmd,
+} from './Utils.mjs'
+
+const clusterName = process.argv[2]
+
+await shellCmd( 'overwhelm' )
+await shellCmd( `kubectl cnpg backup -m volumeSnapshot -n cnpg-operands ${clusterName}` )
+
+log( '\nTo see backups run => kubectl get backups.postgresql.cnpg.io -n cnpg-operands\n' )

package/src/helm-charts/{db-timescale-dense/db-timescale-dense.yaml.ovh → cnpg-db-pgbench/cluster.yaml.ovh}

@@ -3,13 +3,13 @@
 apiVersion: postgresql.cnpg.io/v1
 kind: Cluster
 metadata:
-  name: db-timescale-dense
+  name: cnpg-db-pgbench
   namespace: cnpg-operands
 spec:
   instances: 1
-  logLevel: debug
+  logLevel: info # warning debug
 
-  description: "Leverege Dense Timescale DB"
+  description: "Leverege PG Bench Timescale DB"
   imageName: us-docker.pkg.dev/leverege-registry/system/images/leverege-pgsql:16.1-cnpg-tsdb2-beta.6
   bootstrap:
     initdb:
@@ -19,11 +19,11 @@ spec:
       database: imagine
       owner: imagine
       secret:
-        name: db-timescale-dense-imagine-pw # imagine db pw
+        name: cnpg-db-pgbench-imagine-pw # imagine db pw
 
   enableSuperuserAccess: true
   superuserSecret:
-    name: db-timescale-dense-postgres-pw # postgres db pw
+    name: cnpg-db-pgbench-postgres-pw # postgres db pw
 
   affinity:
 #   enablePodAntiAffinity: true
@@ -70,51 +70,31 @@ spec:
   storage:
     storageClass: premium-rwo
     size: 50Gi
-  walStorage:
-    storageClass: premium-rwo
-    size: 1Gi
 
   backup:
     volumeSnapshot:
       labels:
-        snapshotOf: db-timescale-dense
+        snapshotOf: cnpg-db-pgbench
       className: cnpg-snapshotclass
     barmanObjectStore:
       destinationPath: gs://OVH:<PROJECT_ID>-barman
       googleCredentials:
         gkeEnvironment: true
       tags:
-        cnpgCluster: db-timescale-dense
+        cnpgCluster: cnpg-db-pgbench
       wal:
         compression: gzip
 ---
 apiVersion: postgresql.cnpg.io/v1
-kind: Pooler
-metadata:
-  name: db-timescale-dense-pool-rw
-  namespace: cnpg-operands
-spec:
-  cluster:
-    name: db-timescale-dense
-
-  instances: 1
-  type: rw
-  pgbouncer:
-    poolMode: session
-    parameters:
-      max_client_conn: "500"
-      default_pool_size: "10"
----
-apiVersion: postgresql.cnpg.io/v1
 kind: ScheduledBackup
 metadata:
-  name: db-timescale-dense-backup
+  name: cnpg-db-pgbench-backup
   namespace: cnpg-operands
 spec:
   # https://pkg.go.dev/github.com/robfig/cron#hdr-CRON_Expression_Format
   schedule: "0 0 * * * *"
   cluster:
-    name: db-timescale-dense
+    name: cnpg-db-pgbench
   method: volumeSnapshot
   backupOwnerReference: self
   immediate: true

package/src/helm-charts/cnpg-db-pgbench/gitignore

@@ -0,0 +1,2 @@
+*.swp
+cnpg-db-pgbench.yaml

package/src/helm-charts/cnpg-db-pgbench/helmdn.plugin

@@ -0,0 +1,12 @@
+#!/bin/bash
+#
+# We only want to hibernate unless the intent is to really delete EVERYTHING
+# which includes PVCs and volume snapshots.
+#
+printf "\nHibernating the cnpg-db-pgbench cluster...\n"
+kubectl cnpg hibernate on -n cnpg-operands cnpg-db-pgbench
+
+# *** DANGER ***
+# kubectl delete  will destroy the cluster pods and PVCs!!!!!
+# *** DANGER ***
+# kubectl do-not-do-this-delete -f cnpg-db-pgbench/cnpg-db-pgbench.yaml $K8S_WHAT

package/src/helm-charts/cnpg-db-pgbench/helmup.plugin

@@ -0,0 +1,3 @@
+#!/bin/bash
+#
+kubectl apply -f cnpg-db-pgbench/cluster.yaml $K8S_WHAT

package/src/helm-charts/cnpg-db-pgbench/pooler.yaml

@@ -0,0 +1,16 @@
+apiVersion: postgresql.cnpg.io/v1
+kind: Pooler
+metadata:
+  name: cnpg-db-pgbench-pool-rw
+  namespace: cnpg-operands
+spec:
+  cluster:
+    name: cnpg-db-pgbench
+
+  instances: 1
+  type: rw
+  pgbouncer:
+    poolMode: session
+    parameters:
+      max_client_conn: "500"
+      default_pool_size: "10"

package/src/helm-charts/{db-postgres-stack/db-postgres-stack.yaml.ovh → cnpg-db-psql-stack/cluster.yaml.ovh}

@@ -3,24 +3,24 @@
 apiVersion: postgresql.cnpg.io/v1
 kind: Cluster
 metadata:
-  name: db-postgres-stack
+  name: cnpg-db-psql-stack
   namespace: cnpg-operands
 spec:
   instances: 1
 # logLevel: debug
 
   description: "Leverege Stack PostgreSQL DB"
-  imageName: us-docker.pkg.dev/leverege-registry/system/images/leverege-pgsql:16.1-cnpg-tsdb2-beta.10
+  imageName: us-docker.pkg.dev/leverege-registry/system/images/leverege-pgsql:16.2-cnpg-tsdb2-beta.4
   bootstrap:
     initdb:
       database: imagine
       owner: imagine
       secret:
-        name: db-postgres-stack-imagine-pw # imagine db pw
+        name: cnpg-db-psql-stack-imagine-pw # imagine db pw
 
   enableSuperuserAccess: true
   superuserSecret:
-    name: db-postgres-stack-postgres-pw # postgres db pw
+    name: cnpg-db-psql-stack-postgres-pw # postgres db pw
 
   affinity:
 #   enablePodAntiAffinity: true
@@ -35,10 +35,10 @@ spec:
   # to the default ones to make the cluster work
   postgresql:
     parameters:
+      max_connections: "123"
 #     max_worker_processes: "60"
 #     max_worker_processes: "8"
 #     max_parallel_workers: "8"
-      max_connections: "500"
 #     shared_buffers: 2GB
 #     effective_cache_size: 6GB
 #     maintenance_work_mem: 1GB
@@ -67,44 +67,40 @@ spec:
     retentionPolicy: 30d
     volumeSnapshot:
       labels:
-        snapshotOf: db-postgres-stack
+        snapshotOf: cnpg-db-psql-stack
       className: cnpg-snapshotclass
     barmanObjectStore:
       destinationPath: gs://OVH:<PROJECT_ID>-barman
       googleCredentials:
         gkeEnvironment: true
       tags:
-        cnpgCluster: db-postgres-stack
+        cnpgCluster: cnpg-db-psql-stack
       wal:
         compression: gzip
-#---
-#apiVersion: postgresql.cnpg.io/v1
-#kind: Pooler
-#metadata:
-#  name: db-postgres-stack-pool-rw
-#  namespace: cnpg-operands
-#spec:
-#  cluster:
-#    name: db-postgres-stack
-#
-#  instances: 1
-#  type: rw
-#  pgbouncer:
-#    poolMode: session
-#    parameters:
-#      max_client_conn: "500"
-#      default_pool_size: "10"
 ---
 apiVersion: postgresql.cnpg.io/v1
 kind: ScheduledBackup
 metadata:
-  name: db-postgres-stack-backup
+  name: cnpg-db-psql-stack-backup
   namespace: cnpg-operands
 spec:
   # https://pkg.go.dev/github.com/robfig/cron#hdr-CRON_Expression_Format
   schedule: "0 0 * * * *"
   cluster:
-    name: db-postgres-stack
+  method: volumeSnapshot
+  backupOwnerReference: self
+  immediate: true
+---
+apiVersion: postgresql.cnpg.io/v1
+kind: ScheduledBackup
+metadata:
+  name: cnpg-db-psql-stack-volsnap
+  namespace: cnpg-operands
+spec:
+  # https://pkg.go.dev/github.com/robfig/cron#hdr-CRON_Expression_Format
+  schedule: "0 0 */4 * * *"
+  cluster:
+    name: cnpg-db-psql-stack
   method: volumeSnapshot
   backupOwnerReference: self
   immediate: true

package/src/helm-charts/cnpg-db-psql-stack/gitignore

@@ -0,0 +1,2 @@
+*.swp
+cnpg-db-psql-stack.yaml

package/src/helm-charts/cnpg-db-psql-stack/helmdn.plugin

@@ -0,0 +1,12 @@
+#!/bin/bash
+#
+# We only want to hibernate unless the intent is to really delete EVERYTHING
+# which includes PVCs and volume snapshots.
+#
+printf "\nHibernating the cnpg-db-psql-stack cluster...\n"
+kubectl cnpg hibernate on -n cnpg-operands cnpg-db-psql-stack
+
+# *** DANGER ***
+# kubectl delete  will destroy the cluster
+# *** DANGER ***
+# kubectl delete -f cnpg-db-psql-stack/cnpg-db-psql-stack.yaml $K8S_WHAT

package/src/helm-charts/cnpg-db-psql-stack/helmup.plugin

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+kubectl apply -f cnpg-db-psql-stack/cluster.yaml $K8S_WHAT

package/src/helm-charts/cnpg-db-psql-stack/pooler.yaml

@@ -0,0 +1,16 @@
+apiVersion: postgresql.cnpg.io/v1
+kind: Pooler
+metadata:
+  name: cnpg-db-psql-stack-pool-rw
+  namespace: cnpg-operands
+spec:
+  cluster:
+    name: cnpg-db-psql-stack
+
+  instances: 1
+  type: rw
+  pgbouncer:
+    poolMode: session
+    parameters:
+      max_client_conn: "500"
+      default_pool_size: "10"

package/src/helm-charts/cnpg-db-tsdb-basic/cluster.yaml.ovh

@@ -0,0 +1,110 @@
+# Artifact Chart  => https://artifacthub.io/packages/helm/cloudnative-pg/cloudnative-pg
+# Examples Charts => https://cloudnative-pg.io/documentation/current/samples/
+apiVersion: postgresql.cnpg.io/v1
+kind: Cluster
+metadata:
+  name: cnpg-db-tsdb-basic
+  namespace: cnpg-operands
+spec:
+  instances: 1
+  logLevel: info # warning debug
+
+  monitoring:
+    enablePodMonitor: false # set true once monitoring is setup
+
+  description: "Leverege Plain Timescale DB"
+  imageName: us-docker.pkg.dev/leverege-registry/system/images/leverege-pgsql:16.2-cnpg-tsdb2-beta.4
+  bootstrap:
+    initdb:
+      postInitTemplateSQL:
+        - CREATE EXTENSION timescaledb;
+        - CREATE EXTENSION jsquery;
+      database: imagine
+      owner: imagine
+      secret:
+        name: cnpg-db-tsdb-basic-imagine-pw # imagine db pw
+
+  enableSuperuserAccess: true
+  superuserSecret:
+    name: cnpg-db-tsdb-basic-postgres-pw # postgres db pw
+
+  affinity:
+    enablePodAntiAffinity: true
+    topologyKey: failure-domain.beta.kubernetes.io/zone
+    nodeAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+        nodeSelectorTerms:
+        - matchExpressions:
+          - key: target-env
+            operator: In
+            values:
+            - database
+    tolerations:
+      - key: "database"
+        operator: "Equal"
+        value: "true"
+        effect: "NoSchedule"
+
+  # Parameters and pg_hba configuration will be append
+  # to the default ones to make the cluster work
+  postgresql:
+    shared_preload_libraries:
+      - timescaledb
+      - jsquery
+    parameters:
+      max_connections: "123"
+#     max_worker_processes: "60"
+#     max_worker_processes: "8"
+#     max_parallel_workers: "8"
+#     shared_buffers: 2GB
+#     effective_cache_size: 6GB
+#     maintenance_work_mem: 1GB
+#     work_mem: 20MB
+    pg_hba:
+      # To access through TCP/IP you will need to get username
+      # and password from the secret cluster-example-custom-app
+      - host all all all md5
+
+  # Example of rolling update strategy:
+  # - unsupervised: automated update of the primary once all
+  #                 replicas have been upgraded (default)
+  # - supervised: requires manual supervision to perform
+  #               the switchover of the primary
+  primaryUpdateStrategy: unsupervised
+
+  serviceAccountTemplate:
+    metadata:
+      annotations:
+        iam.gke.io/gcp-service-account: cnpg-operands-sa@OVH:<PROJECT_ID>.iam.gserviceaccount.com
+
+  storage:
+    storageClass: premium-rwo
+    size: 76Gi
+
+  backup:
+    volumeSnapshot:
+      labels:
+        snapshotOf: cnpg-db-tsdb-basic
+      className: cnpg-snapshotclass
+    barmanObjectStore:
+      destinationPath: gs://OVH:<PROJECT_ID>-barman
+      googleCredentials:
+        gkeEnvironment: true
+      tags:
+        cnpgCluster: cnpg-db-tsdb-basic
+      wal:
+        compression: gzip
+---
+apiVersion: postgresql.cnpg.io/v1
+kind: ScheduledBackup
+metadata:
+  name: cnpg-db-tsdb-basic-volsnap
+  namespace: cnpg-operands
+spec:
+  # https://pkg.go.dev/github.com/robfig/cron#hdr-CRON_Expression_Format
+  schedule: "0 0 */8 * * *"
+  cluster:
+    name: cnpg-db-tsdb-basic
+  method: volumeSnapshot
+  backupOwnerReference: self
+  immediate: true

package/src/helm-charts/cnpg-db-tsdb-basic/gitignore

@@ -0,0 +1,2 @@
+*.swp
+cnpg-db-tsdb-basic.yaml

package/src/helm-charts/cnpg-db-tsdb-basic/helmdn.plugin

@@ -0,0 +1,12 @@
+#!/bin/bash
+#
+# We only want to hibernate unless the intent is to really delete EVERYTHING
+# which includes PVCs and volume snapshots.
+#
+printf "\nHibernating the cnpg-db-tsdb-basic cluster...\n"
+kubectl cnpg hibernate on -n cnpg-operands cnpg-db-tsdb-basic
+
+# *** DANGER ***
+# kubectl delete  will destroy the cluster pods and PVCs!!!!!
+# *** DANGER ***
+# kubectl do-not-do-this-delete -f cnpg-db-tsdb-basic/cnpg-db-tsdb-basic.yaml $K8S_WHAT

package/src/helm-charts/cnpg-db-tsdb-basic/helmup.plugin

@@ -0,0 +1,3 @@
+#!/bin/bash
+#
+kubectl apply -f cnpg-db-tsdb-basic/cluster.yaml $K8S_WHAT

package/src/helm-charts/cnpg-db-tsdb-basic/pooler.yaml

@@ -0,0 +1,16 @@
+apiVersion: postgresql.cnpg.io/v1
+kind: Pooler
+metadata:
+  name: cnpg-db-tsdb-basic-pool-rw
+  namespace: cnpg-operands
+spec:
+  cluster:
+    name: cnpg-db-tsdb-basic
+
+  instances: 1
+  type: rw
+  pgbouncer:
+    poolMode: session
+    parameters:
+      max_client_conn: "500"
+      default_pool_size: "10"