@leverege/build-tools 2.51.0-beta.1 → 2.51.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@leverege/build-tools",
-  "version": "2.51.0-beta.1",
+  "version": "2.51.0",
   "description": "A collection of build / support tools for Leverege developers",
   "main": "index.js",
   "repository": {
@@ -20,7 +20,6 @@
     "decrypt-secrets": "src/decrypt-secrets.sh",
     "dirty-git": "src/dirty-git.sh",
     "dockreate": "src/dockreate.sh",
-    "documentate" : "src/documentate.sh",
     "docker-to-registry": "src/docker-to-registry.mjs",
     "docker-to-registry.sh": "src/docker-to-registry.sh",
     "encrypt-secrets": "src/encrypt-secrets.sh",
@@ -58,27 +57,27 @@
     "@google-cloud/artifact-registry": "^3.4.0",
     "ansi-colors": "^4.1.3",
     "chalk": "^5.3.0",
-    "command-line-args": "^5.2.1",
-    "command-line-usage": "^7.0.1",
+    "command-line-args": "^6.0.0",
+    "command-line-usage": "^7.0.2",
     "deepmerge": "^4.3.1",
     "enquirer": "^2.4.1",
-    "execa": "^9.1.0",
-    "glob": "^10.3.16",
+    "execa": "^9.3.0",
+    "glob": "^11.0.0",
     "handlebars": "^4.7.8",
-    "inquirer": "^9.2.22",
+    "inquirer": "^10.0.1",
     "js-yaml": "^4.1.0",
     "ms": "^2.1.3",
-    "npm-registry-fetch": "^17.0.1",
+    "npm-registry-fetch": "^17.1.0",
     "package-up": "^5.0.0",
     "parse-gitignore": "^2.0.0",
     "read-pkg": "^9.0.1",
     "readline-sync": "^1.4.10",
     "semver": "^7.6.2",
-    "simple-git": "^3.24.0",
-    "zx": "^8.1.1"
+    "simple-git": "^3.25.0",
+    "zx": "^8.1.4"
   },
   "devDependencies": {
     "@leverege/eslint-config-leverege": "^4.2.0",
-    "npm": "^10.8.0"
+    "npm": "^10.8.2"
   }
 }

package/registry-compass.yaml

@@ -0,0 +1,20 @@
+registry:
+  - root: us-docker.pkg.dev
+  - repositories:
+    - name: stack
+      charts:
+        - api-server
+        - authz-server
+        - emailer
+        - message-processor
+    - name: leverege
+      charts:
+        - pubsub-pulse
+        - pusher
+        - overdose
+    - name: cox-health
+      charts:
+        - actions-server
+        - analytics-server
+        - centrak-healthz
+        - centrak-ingestor

package/src/Utils.mjs

@@ -7,7 +7,9 @@ import { glob } from 'glob'
 import { packageUp } from 'package-up'
 import YAML from 'js-yaml'
 
-const debugEnabled = process.env.BUILD_TOOLS_DEBUG === '1'
+let debugEnabled = process.env.BUILD_TOOLS_DEBUG === '1'
+
+export const disableDebug = () => debugEnabled = false
 
 /* eslint-disable no-console */
 export const err = console.error

package/src/bash-funcs

@@ -29,8 +29,12 @@ RED_ERROR=`color r '***ERROR:'`
 YELO_WARN="`color y '***WARNING:'`"
 GREEN_INFO=`color g '***INFO:'`
 
+function clearScreen() {
+  [ "$BUILD_TOOLS_DEBUG" != "1" ] && clear
+}
+
 function clearlyWarn() {
-  [ -z "$1" ] && clear
+  clearScreen
   cat<<CLEARWARN
 
  `color y '*** WARNING *** WARNING *** WARNING *** WARNING *** WARNING *** WARNING ***'`

package/src/chart-compass.mjs

@@ -0,0 +1,96 @@
+#!/usr/bin/env node
+/*
+ * chart-to-registry will...
+ */
+import fs from 'node:fs'
+
+import chalk from 'chalk'
+import commandLineArgs from 'command-line-args'
+import commandLineUsage from 'command-line-usage'
+import { lt as semverLt } from 'semver'
+import YAML from 'js-yaml'
+
+import {
+  condir,
+  debug,
+  errorExit,
+  log,
+  warning,
+  getGitRootDirectory,
+  parsePackageJson,
+  parseHelmChart,
+  shellCmd } from './Utils.mjs'
+
+const commandLineOptions = [ // Use commandLineOptions to tie into the Usage statements
+  /* eslint-disable max-len */
+  {
+    name : 'location',
+    type : String,
+    description : '{green the location of the artifact registry the chart will be pushed to (default us-docker.pkg.dev)}',
+  },
+  {
+    name : 'project',
+    type : String,
+    description : '{green the name of the google project containing the npmrc and slack config secrets (default leverege-registry)}',
+  },
+  {
+    name : 'repository',
+    type : String,
+    description : '{green the target repository to receive the pushed chart}',
+  },
+  {
+    name : 'dry-run',
+    type : Boolean,
+    description : '{yellow perform everything except the actual chart push}',
+  },
+  {
+    name : 'help',
+    type : Boolean,
+    description : '{green display this help screen}',
+  },
+  /* eslint-enable max-len */
+]
+
+const sections = [
+  {
+    header : 'Leverege Helm Chart Compass (for helmup)',
+    content : `{green This tool helps helmup navigate the helm charts stored in the
+artifact-registries.}`
+  },
+  { header : 'Options',
+    optionList : commandLineOptions,
+  },
+]
+
+const args = commandLineArgs( commandLineOptions, { camelCase : true, partial : true } )
+const usage = commandLineUsage( sections )
+
+if ( args.help ) {
+  log( usage )
+  process.exit( 0 )
+}
+
+/* eslint-disable no-underscore-dangle */
+if ( args._unknown ) {
+  log( usage )
+  log( `\nUnrecognized argument [${chalk.bold.red( args._unknown )}]\n` )
+  process.exit( 1 )
+}
+/* eslint-enable no-underscore-dangle */
+
+const minNodejsVersion = '18.0.0'
+if ( semverLt( process.version, minNodejsVersion ) ) {
+  errorExit( `\n***ERROR: must be running at least node ${minNodejsVersion}\n` )
+}
+
+// First of all, fail if we are not in a git repository
+let gitRoot
+try {
+  gitRoot = await getGitRootDirectory()
+} catch ( error ) {
+  errorExit( chalk.red.bold( error ), { errorCode : 5 } )
+}
+
+const chartCompass = YAML.load( fs.readFileSync( './registry-compass.yaml', 'utf8' ) )
+
+condir( { chartCompass }, '<==Navigation' )

package/src/circleate.sh

@@ -10,7 +10,6 @@ if [ "$1" != "server" ] && [ "$1" != "ui" ] && [ "$1" != "lib" ]; then
 fi
 
 repo_type=$1
-NODE_VERSION="hydrogen"
 CIRCLE_NODE_IMG="cimg/node:lts"
 
 check_circle_config(){

package/src/dockreate.sh

@@ -98,7 +98,7 @@ DOCKERPLUGIN
 #         $3 = additional packages to install (htop redis-cli etc)
 create_Dockerfile() {
   local regvers="${1}"
-  local nodeimg="${2:-hydrogen-alpine}"
+  local nodeimg="${2:-iron-alpine}"
   local apkadds="${3}"
 
   create_DockerfilePlugin
@@ -236,7 +236,7 @@ fi
 
 # TODO: these should be combined into one require
 nodeimg=$(packageJson '.leverege.nodeimg')
-nodeimg="${nodeimg:-hydrogen-alpine}"
+nodeimg="${nodeimg:-iron-alpine}"
 apkadds=$(packageJson '.leverege.apkadds')
 runuser=$(packageJson '.leverege.runuser')
 [ -z "${runuser}" ] && runuser='node'

package/src/git-tar.sh

@@ -1,5 +1,8 @@
 #!/bin/bash
 
+# Load the standard helper functions
+. `build-tools --bashfun`
+
 # If it isn't already set use the default ~/work/.gitar
 #
 [ -z $GITAR ] && GITAR="${HOME}/.gitar"
@@ -65,12 +68,13 @@ REPO=`basename ${GTOP}`
 #
 DTS="`date +\"%Y%m%d-%H%M%S\"`"
 
-TARB=${GITAR}/${REPO}/${DTS}.tar # Craft the target tarball name.
+TARS=${GITAR}/${REPO}
+TARB=${TARS}/${DTS}.tar # Craft the target tarball name.
 
 # Now do the work
 #
 cd ${GTOP}/..
-printf "\n*** Tarring     $TARB\n"
+printf "\n*** Tarring     `color g $TARB`\n"
 tar cf $TARB -X $TARX $REPO
 chmod 0640 $TARB
 
@@ -84,6 +88,9 @@ then
   FIZ=xz
 fi
 
-printf   "*** Compressing $TARB.$FIZ (background)\n\n"
+printf   "\n*** Compressing `color y $TARB.$FIZ` (background)\n"
 $ZIPPER $TARB &
+
+printf "\n*** Tarballs in `color g ${TARS}`\n\n"
+ls -lh ${TARS}
 cd - 2>&1 >/dev/null

package/src/helm-charts/api-server/values-local.yaml

@@ -1,3 +1,9 @@
+# migrations
+#   v5.11.0+
+#     - k8x api-server pod
+#     - node tools/UpdateESComponentTemplateV2.js
+#     - node tools/UpdateESMappingV2.js
+#
 image:
   tag: # v5.12.3
 

package/src/helm-charts/authz-server/values-local.yaml

@@ -1,3 +1,8 @@
+# migrations
+#   v2.1.0+
+#     - k8x authz-server
+#     - npm run sequelizeRunMigrations
+#
 image:
   tag: # v2.1.3
 

package/src/helm-charts/cnpg-db-psql-stack/cluster.yaml.ovh

@@ -13,24 +13,35 @@ spec:
   imageName: us-docker.pkg.dev/leverege-registry/system/images/leverege-pgsql:16.2-cnpg-tsdb2-beta.4
   bootstrap:
     initdb:
-      database: models
+#     database: models
       postInitSQL:
+#       - CREATE DATABASE app;       # operator expects this
+#       - CREATE ROLE app;           #   and this to exist on restore
+        # create the stack databases
         - CREATE DATABASE authz;
         - CREATE DATABASE fota;
         - CREATE DATABASE geotile;
         - CREATE DATABASE models;
         - CREATE DATABASE scheduler;
-      owner: imagine
-      secret:
-        name: cnpg-db-psql-stack-imagine-pw # imagine db pw
+#     owner: imagine
+#     secret:
+#       name: cnpg-db-psql-stack-imagine-pw # imagine db pw
 
   enableSuperuserAccess: true
   superuserSecret:
     name: cnpg-db-psql-stack-postgres-pw # postgres db pw
 
   affinity:
-#   enablePodAntiAffinity: true
-#   topologyKey: kubernetes.io/hostname # default value
+    enablePodAntiAffinity: true
+    topologyKey: kubernetes.io/hostname # default value
+    nodeAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+        nodeSelectorTerms:
+        - matchExpressions:
+          - key: target-env
+            operator: In
+            values:
+            - database
     tolerations:
       - key: "database"
         operator: "Equal"
@@ -83,6 +94,7 @@ spec:
         cnpgCluster: cnpg-db-psql-stack
       wal:
         compression: gzip
+        maxParallel: 4
 ---
 apiVersion: postgresql.cnpg.io/v1
 kind: ScheduledBackup
@@ -91,8 +103,9 @@ metadata:
   namespace: cnpg-operands
 spec:
   # https://pkg.go.dev/github.com/robfig/cron#hdr-CRON_Expression_Format
-  schedule: "0 0 * * * *"
+  schedule: "0 0 */12 * * *"
   cluster:
-  method: volumeSnapshot
+    name: cnpg-db-psql-stack
+  method: barmanObjectStore # volumeSnapshot
   backupOwnerReference: self
   immediate: true

package/src/helm-charts/cnpg-db-tsdb-basic/cluster.yaml.ovh

@@ -19,6 +19,8 @@ spec:
       postInitTemplateSQL:
         - CREATE EXTENSION timescaledb;
         - CREATE EXTENSION jsquery;
+        - CREATE DATABASE app; # for the operator restores
+        - CREATE ROLE app;     # same
       database: imagine
       owner: imagine
       secret:
@@ -94,6 +96,7 @@ spec:
         cnpgCluster: cnpg-db-tsdb-basic
       wal:
         compression: gzip
+        maxParallel: 4
 ---
 apiVersion: postgresql.cnpg.io/v1
 kind: ScheduledBackup

package/src/helm-charts/cnpg-db-tsdb-dense/cluster.yaml.ovh

@@ -19,6 +19,8 @@ spec:
       postInitTemplateSQL:
         - CREATE EXTENSION timescaledb;
         - CREATE EXTENSION jsquery;
+        - CREATE DATABASE app; # for the operator restores
+        - CREATE ROLE app;     # same
       database: imagine
       owner: imagine
       secret:
@@ -93,6 +95,7 @@ spec:
         cnpgCluster: cnpg-db-tsdb-dense
       wal:
         compression: gzip
+        maxParallel: 4
 ---
 apiVersion: postgresql.cnpg.io/v1
 kind: ScheduledBackup

package/src/helm-charts/grafana/grafana-local.yaml.ovh

@@ -1,6 +1,5 @@
 # ONLY EDIT grafana-local.yaml.ovh / grafana-local.yaml is git ignored
 service:
-  type: "NodePort"
   port: "9000"
 
 datasources:
@@ -39,7 +38,7 @@ tolerations:
   effect: "NoSchedule"
 
 extraObjects:
-  - apiVersion: traefik.containo.us/v1alpha1
+  - apiVersion: traefik.io/v1alpha1
     kind: IngressRoute
     metadata:
       name: grafana

package/src/helm-charts/grafana/helmup.plugin

@@ -4,7 +4,7 @@ showInstalling "Grafana"
 
 addHelmRepo grafana https://grafana.github.io/helm-charts
 
-[ -z "$GRAFANA_CHART_VERSION" ] && GRAFANA_CHART_VERSION="7"
+[ -z "$GRAFANA_CHART_VERSION" ] && GRAFANA_CHART_VERSION="8"
 helm upgrade --install grafana grafana/grafana \
   --namespace monitoring --create-namespace \
   --values grafana/grafana-local.yaml \

package/src/helm-charts/redis/helmup.plugin

@@ -4,7 +4,7 @@ showInstalling "Redis"
 
 OCI_CHART="oci://registry-1.docker.io/bitnamicharts/redis"
 
-[ -z "$REDIS_CHART_VERSION" ] && REDIS_CHART_VERSION="19.5.0"
+[ -z "$REDIS_CHART_VERSION" ] && REDIS_CHART_VERSION="19.6.1"
 
 helm upgrade --install redis $OCI_CHART       \
              --values  redis/redis-local.yaml \

package/src/helm-charts/redis/redis-local.yaml

@@ -25,6 +25,11 @@ master:
   persistence:
     size: 8Gi # 8Gi is default
 
+  resources:
+    requests:
+      cpu: 250m
+      memory: 500Mi
+
 replica:
   affinity:
     nodeAffinity:
@@ -45,6 +50,11 @@ replica:
   persistence:
     size: 8Gi # 8Gi is default
 
+  resources:
+    requests:
+      cpu: 250m
+      memory: 500Mi
+
 architecture: replication
 
 sentinel:

package/src/helm-charts/resource-server/values-local.yaml

@@ -1,5 +1,7 @@
 # migrations
-#   v1.2+ run => npm run sequelizeRunMigrations
+#   v1.2+
+#     - k8x resource-server
+#     - npm run sequelizeRunMigrations
 #
 image:
   tag: # v1.2.2

package/src/helm-charts/rule-engine/values-local.yaml

@@ -1,5 +1,5 @@
 image:
-  tag: # v2.3.0
+  tag: # v2.3.1
 
 config:
   LOG_CONFIG: '{"type":"pino","level":"warn"}'

package/src/helm-charts/traefik/helmup.plugin

@@ -5,7 +5,11 @@
 # ***REMEMBER*** the instructions provided on the internet typically assume
 #                the default namespace not traefik
 #
-# ***DO NOT FORGET*** the -n traefik for kubectl commands
+# ***DO NOT FORGET*** the -n traefik for kubectl commands - so in order to
+#    move beyond v20 of the chart run the kubectl command below before upgrading
+#
+# kubectl apply --namespace traefik --server-side \
+#               --force-conflicts -k https://github.com/traefik/traefik-helm-chart/traefik/crds/\?ref\=v27
 #
 showInstalling "Traefik Load Balancer / Router"
 

package/src/helm-charts/velero/helmup.plugin

@@ -4,7 +4,7 @@ showInstalling "Velero Backup System"
 
 addHelmRepo vmware-tanzu https://vmware-tanzu.github.io/helm-charts
 
-[ -z "$VELERO_CHART_VERSION" ] && VELERO_CHART_VERSION="6"
+[ -z "$VELERO_CHART_VERSION" ] && VELERO_CHART_VERSION="7"
 #
 # Build the bucket, region and SA email variables and use --set to mod the
 # chart values as opposed to using an OVH:<label> approach. This eliminates

package/src/helm-charts/velero/velero-local.yaml

@@ -99,9 +99,10 @@ nodeSelector: {}
 
 # Init containers to add to the Velero deployment's pod spec. At least one plugin provider image is required.
 # If the value is a string then it is evaluated as a template.
+# See => https://github.com/vmware-tanzu/velero-plugin-for-gcp
 initContainers:
   - name: velero-plugin-for-gcp
-    image: velero/velero-plugin-for-gcp:v1.5.0
+    image: velero/velero-plugin-for-gcp:v1.10.0
     volumeMounts:
       - mountPath: /target
         name: plugins

package/src/helmup.sh

@@ -666,7 +666,7 @@ function installVeleroEnvironment() {
   gcloud config set project $GCP_PROJECT_ID
 
   ## The major chart version will determine the bucket suffix
-  [ -z "$VELERO_HELM_CHART" ] && VELERO_HELM_CHART="6"
+  [ -z "$VELERO_HELM_CHART" ] && VELERO_HELM_CHART="7"
 
   ## Create a bucket
   BUCKET="$GCP_PROJECT_ID-velero-$VELERO_HELM_CHART"
@@ -696,6 +696,7 @@ function installVeleroEnvironment() {
     compute.disks.get
     compute.disks.create
     compute.disks.createSnapshot
+    compute.projects.get
     compute.snapshots.get
     compute.snapshots.create
     compute.snapshots.useReadOnly
@@ -787,7 +788,27 @@ DEPRECATED_CHART
   fi
 }
 
-doHelmup() {
+function checkUpgradeNotes() {
+  UPNOTES="$1/helmup.upgrade"
+
+  [ ! -f "$UPNOTES" ] && return
+
+  cat<<UPGRADE_NOTES
+
+  `color g "*** $SERVICE upgrade instructions before moving forward ***"`
+
+UPGRADE_NOTES
+  cat $UPNOTES
+  yesToContinue
+  cat<<UPGRADES_DONE
+
+  `color y "*** Remember to remove $UPNOTES once the steps are performed ***"`
+
+UPGRADES_DONE
+  sleep 3
+}
+
+function doHelmup() {
   local helmup="$@"
   local doing="Helming => `color g \"$helmup\"`"
   local from=
@@ -1036,6 +1057,8 @@ NO_SERVICE_MAN
         fi
         service-man --from-helmup --create-service $SERVICE
 
+        checkUpgradeNotes $SERVICE
+
         # By default the museum chart name will be the same as the service, but
         # we have the flexibility to override the naming when the charts are
         # reference by the downstream repositories via the .helmup-charmuseum file.

package/src/k8scale.sh

@@ -19,8 +19,11 @@ case $DIRECTION in
     REPLICAS=0
     DIRECTION=down
     ;;
+  roll)
+    DIRECTION=roll
+    ;;
   *)
-    printf "Usage: `color g 'k8scale <up|dn|down> [deployment-list]'`\n\n" && exit 1
+    printf "Usage: `color g 'k8scale <up|dn|down|roll> [deployment-list]'`\n\n" && exit 1
 esac
 
 printf "   scaled => `color g $DIRECTION`\n\n"
@@ -28,7 +31,12 @@ for DEPLOYMENT in "$@";
 do
   IFS="/" read SERVICE <<< "$DEPLOYMENT" # strip trailing /
   RESULTS="`color g SUCCESS`"
-  kubectl scale --replicas=$REPLICAS deployment $SERVICE &>/dev/null
+  if [ "$DIRECTION" == "roll" ];
+  then
+    kubectl rollout restart deployment $SERVICE &> $DEVNULL
+  else
+    kubectl scale --replicas=$REPLICAS deployment $SERVICE &> $DEVNULL
+  fi
   [ $? -eq 1 ] && RESULTS="`color r FAILURE`"
   printf "  $RESULTS => $SERVICE\n"
   sendToSlack "k8scale $DIRECTION $SERVICE"

package/src/overwhelm.mjs

@@ -15,7 +15,7 @@ import parse from 'parse-gitignore'
 import ask from 'readline-sync'
 import YAML from 'js-yaml'
 
-import { debug, err, log, shellCmd } from './Utils.mjs'
+import { debug, disableDebug, err, log, shellCmd } from './Utils.mjs'
 
 const optionsDefinitions = [
   /* eslint-disable no-multi-spaces */
@@ -33,6 +33,9 @@ const optionsDefinitions = [
 ]
 const args = cliArgs( optionsDefinitions )
 
+// when using --key we only want the key contents w/o debug info
+if ( args.key ) { disableDebug() }
+
 // See if we are in a monorepo, which changes some of the rules.
 const gitTop = await shellCmd( 'git rev-parse --show-toplevel' )
 
@@ -64,10 +67,21 @@ if ( !fs.existsSync( 'overwhelm.yaml' ) ) {
 if ( fs.existsSync( 'values-default.yaml' ) ) {
   log( chalk.red.bold( `
 
-***ERROR overwhelm => values-default.yaml is no longer used/supported
-                      all defaults should be set in default-configs.yaml.ovh
+***ERROR overwhelm => values-default.yaml is no longer used/supported and will
+         be ignored - all defaults should be set in default-configs.yaml.ovh
   ` ) )
-  process.exit( 1 )
+
+  const ans = ask.question( `Enter "${chalk.bold.red( 'skip' )}" if you wish to ignore and continue => ` )
+  if ( ans !== 'skip' ) {
+    process.exit( 1 )
+  }
+}
+
+// Avoid race condition when running overwhelm for the first time on a new
+// cluster and the default-configs.yaml file has yet to be created.
+const defaultConfigs = 'default-configs.yaml'
+if ( !fs.existsSync( defaultConfigs ) ) {
+  fs.closeSync( fs.openSync( defaultConfigs, 'w' ) ) // same as unix touch
 }
 
 // Load the default values and then overlay with the project specific yaml
@@ -197,7 +211,7 @@ const doReplacements = ( yamls, replacements ) => {
 
 // Collect all of the yaml from the yams...
 const buildYaml = ( yamls, replacements ) => {
-  const always = [ 'default-configs.yaml' ]
+  const always = [ defaultConfigs ]
   const values = [ ...always, ...yamls ]
   const loaded = []
   let allYamls = ''

package/src/documentate.sh

@@ -1,38 +0,0 @@
-#!/bin/bash
-
-. `build-tools --bashfun`
-
-cwd=$(pwd)
-library=`basename $cwd`
-version=$(node -p "require('$cwd/package.json').version")
-
-echo $library
-echo $version
-
-bucket_name="leverege-registry-jsdocs"
-
-echo "deploying version $version docs to google cloud storage"
-
-echo "Check if build exists in root directory (current ver)"
-
-exists=`gsutil ls -b gs://$bucket_name/$library/$version`
-if [[ $exists == gs://* ]]; then
-  echo "documentation already exists"
-	exit 
-fi
-
-npm run clean-docs &> $DEVNULL
-errorExit $?
-npm run docs &> $DEVNULL
-errorExit $?
-
-TARGET_BUCKET="gs://$bucket_name/$library/$version"
-BUCKET_COPY="gsutil -m cp -r ./docs/** $TARGET_BUCKET"
-
-cat<<LOG_COPYING
-
-  Copying docs to bucket => `color g "$BUCKET_COPY"`
-
-LOG_COPYING
-
-$(BUCKET_COPY)