@leverege/build-tools 2.50.17 → 2.51.0-beta.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@leverege/build-tools",
-  "version": "2.50.17",
+  "version": "2.51.0-beta.1",
   "description": "A collection of build / support tools for Leverege developers",
   "main": "index.js",
   "repository": {
@@ -20,6 +20,7 @@
     "decrypt-secrets": "src/decrypt-secrets.sh",
     "dirty-git": "src/dirty-git.sh",
     "dockreate": "src/dockreate.sh",
+    "documentate" : "src/documentate.sh",
     "docker-to-registry": "src/docker-to-registry.mjs",
     "docker-to-registry.sh": "src/docker-to-registry.sh",
     "encrypt-secrets": "src/encrypt-secrets.sh",
@@ -61,23 +62,23 @@
     "command-line-usage": "^7.0.1",
     "deepmerge": "^4.3.1",
     "enquirer": "^2.4.1",
-    "execa": "^9.3.0",
-    "glob": "^10.4.2",
+    "execa": "^9.1.0",
+    "glob": "^10.3.16",
     "handlebars": "^4.7.8",
-    "inquirer": "^9.3.2",
+    "inquirer": "^9.2.22",
     "js-yaml": "^4.1.0",
     "ms": "^2.1.3",
-    "npm-registry-fetch": "^17.1.0",
+    "npm-registry-fetch": "^17.0.1",
     "package-up": "^5.0.0",
     "parse-gitignore": "^2.0.0",
     "read-pkg": "^9.0.1",
     "readline-sync": "^1.4.10",
     "semver": "^7.6.2",
-    "simple-git": "^3.25.0",
-    "zx": "^8.1.3"
+    "simple-git": "^3.24.0",
+    "zx": "^8.1.1"
   },
   "devDependencies": {
     "@leverege/eslint-config-leverege": "^4.2.0",
-    "npm": "^10.8.1"
+    "npm": "^10.8.0"
   }
 }

package/src/Utils.mjs

@@ -7,9 +7,7 @@ import { glob } from 'glob'
 import { packageUp } from 'package-up'
 import YAML from 'js-yaml'
 
-let debugEnabled = process.env.BUILD_TOOLS_DEBUG === '1'
-
-export const disableDebug = () => debugEnabled = false
+const debugEnabled = process.env.BUILD_TOOLS_DEBUG === '1'
 
 /* eslint-disable no-console */
 export const err = console.error

package/src/bash-funcs

@@ -29,12 +29,8 @@ RED_ERROR=`color r '***ERROR:'`
 YELO_WARN="`color y '***WARNING:'`"
 GREEN_INFO=`color g '***INFO:'`
 
-function clearScreen() {
-  [ "$BUILD_TOOLS_DEBUG" != "1" ] && clear
-}
-
 function clearlyWarn() {
-  clearScreen
+  [ -z "$1" ] && clear
   cat<<CLEARWARN
 
  `color y '*** WARNING *** WARNING *** WARNING *** WARNING *** WARNING *** WARNING ***'`

package/src/circleate.sh

@@ -10,6 +10,7 @@ if [ "$1" != "server" ] && [ "$1" != "ui" ] && [ "$1" != "lib" ]; then
 fi
 
 repo_type=$1
+NODE_VERSION="hydrogen"
 CIRCLE_NODE_IMG="cimg/node:lts"
 
 check_circle_config(){

package/src/dockreate.sh

@@ -98,7 +98,7 @@ DOCKERPLUGIN
 #         $3 = additional packages to install (htop redis-cli etc)
 create_Dockerfile() {
   local regvers="${1}"
-  local nodeimg="${2:-iron-alpine}"
+  local nodeimg="${2:-hydrogen-alpine}"
   local apkadds="${3}"
 
   create_DockerfilePlugin
@@ -236,7 +236,7 @@ fi
 
 # TODO: these should be combined into one require
 nodeimg=$(packageJson '.leverege.nodeimg')
-nodeimg="${nodeimg:-iron-alpine}"
+nodeimg="${nodeimg:-hydrogen-alpine}"
 apkadds=$(packageJson '.leverege.apkadds')
 runuser=$(packageJson '.leverege.runuser')
 [ -z "${runuser}" ] && runuser='node'

package/src/documentate.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+
+. `build-tools --bashfun`
+
+cwd=$(pwd)
+library=`basename $cwd`
+version=$(node -p "require('$cwd/package.json').version")
+
+echo $library
+echo $version
+
+bucket_name="leverege-registry-jsdocs"
+
+echo "deploying version $version docs to google cloud storage"
+
+echo "Check if build exists in root directory (current ver)"
+
+exists=`gsutil ls -b gs://$bucket_name/$library/$version`
+if [[ $exists == gs://* ]]; then
+  echo "documentation already exists"
+	exit 
+fi
+
+npm run clean-docs &> $DEVNULL
+errorExit $?
+npm run docs &> $DEVNULL
+errorExit $?
+
+TARGET_BUCKET="gs://$bucket_name/$library/$version"
+BUCKET_COPY="gsutil -m cp -r ./docs/** $TARGET_BUCKET"
+
+cat<<LOG_COPYING
+
+  Copying docs to bucket => `color g "$BUCKET_COPY"`
+
+LOG_COPYING
+
+$(BUCKET_COPY)

package/src/git-tar.sh

@@ -1,8 +1,5 @@
 #!/bin/bash
 
-# Load the standard helper functions
-. `build-tools --bashfun`
-
 # If it isn't already set use the default ~/work/.gitar
 #
 [ -z $GITAR ] && GITAR="${HOME}/.gitar"
@@ -68,13 +65,12 @@ REPO=`basename ${GTOP}`
 #
 DTS="`date +\"%Y%m%d-%H%M%S\"`"
 
-TARS=${GITAR}/${REPO}
-TARB=${TARS}/${DTS}.tar # Craft the target tarball name.
+TARB=${GITAR}/${REPO}/${DTS}.tar # Craft the target tarball name.
 
 # Now do the work
 #
 cd ${GTOP}/..
-printf "\n*** Tarring     `color g $TARB`\n"
+printf "\n*** Tarring     $TARB\n"
 tar cf $TARB -X $TARX $REPO
 chmod 0640 $TARB
 
@@ -88,9 +84,6 @@ then
   FIZ=xz
 fi
 
-printf   "\n*** Compressing `color y $TARB.$FIZ` (background)\n"
+printf   "*** Compressing $TARB.$FIZ (background)\n\n"
 $ZIPPER $TARB &
-
-printf "\n*** Tarballs in `color g ${TARS}`\n\n"
-ls -lh ${TARS}
 cd - 2>&1 >/dev/null

package/src/helm-charts/api-server/values-local.yaml

@@ -1,9 +1,3 @@
-# migrations
-#   v5.11.0+
-#     - k8x api-server pod
-#     - node tools/UpdateESComponentTemplateV2.js
-#     - node tools/UpdateESMappingV2.js
-#
 image:
   tag: # v5.12.3
 

package/src/helm-charts/authz-server/values-local.yaml

@@ -1,8 +1,3 @@
-# migrations
-#   v2.1.0+
-#     - k8x authz-server
-#     - npm run sequelizeRunMigrations
-#
 image:
   tag: # v2.1.3
 

package/src/helm-charts/cnpg-db-psql-stack/cluster.yaml.ovh

@@ -13,35 +13,24 @@ spec:
   imageName: us-docker.pkg.dev/leverege-registry/system/images/leverege-pgsql:16.2-cnpg-tsdb2-beta.4
   bootstrap:
     initdb:
-#     database: models
+      database: models
       postInitSQL:
-#       - CREATE DATABASE app;       # operator expects this
-#       - CREATE ROLE app;           #   and this to exist on restore
-        # create the stack databases
         - CREATE DATABASE authz;
         - CREATE DATABASE fota;
         - CREATE DATABASE geotile;
         - CREATE DATABASE models;
         - CREATE DATABASE scheduler;
-#     owner: imagine
-#     secret:
-#       name: cnpg-db-psql-stack-imagine-pw # imagine db pw
+      owner: imagine
+      secret:
+        name: cnpg-db-psql-stack-imagine-pw # imagine db pw
 
   enableSuperuserAccess: true
   superuserSecret:
     name: cnpg-db-psql-stack-postgres-pw # postgres db pw
 
   affinity:
-    enablePodAntiAffinity: true
-    topologyKey: kubernetes.io/hostname # default value
-    nodeAffinity:
-      requiredDuringSchedulingIgnoredDuringExecution:
-        nodeSelectorTerms:
-        - matchExpressions:
-          - key: target-env
-            operator: In
-            values:
-            - database
+#   enablePodAntiAffinity: true
+#   topologyKey: kubernetes.io/hostname # default value
     tolerations:
       - key: "database"
         operator: "Equal"
@@ -94,7 +83,6 @@ spec:
         cnpgCluster: cnpg-db-psql-stack
       wal:
         compression: gzip
-        maxParallel: 4
 ---
 apiVersion: postgresql.cnpg.io/v1
 kind: ScheduledBackup
@@ -103,9 +91,8 @@ metadata:
   namespace: cnpg-operands
 spec:
   # https://pkg.go.dev/github.com/robfig/cron#hdr-CRON_Expression_Format
-  schedule: "0 0 */12 * * *"
+  schedule: "0 0 * * * *"
   cluster:
-    name: cnpg-db-psql-stack
-  method: barmanObjectStore # volumeSnapshot
+  method: volumeSnapshot
   backupOwnerReference: self
   immediate: true

package/src/helm-charts/cnpg-db-tsdb-basic/cluster.yaml.ovh

@@ -19,8 +19,6 @@ spec:
       postInitTemplateSQL:
         - CREATE EXTENSION timescaledb;
         - CREATE EXTENSION jsquery;
-        - CREATE DATABASE app; # for the operator restores
-        - CREATE ROLE app;     # same
       database: imagine
       owner: imagine
       secret:
@@ -96,7 +94,6 @@ spec:
         cnpgCluster: cnpg-db-tsdb-basic
       wal:
         compression: gzip
-        maxParallel: 4
 ---
 apiVersion: postgresql.cnpg.io/v1
 kind: ScheduledBackup

package/src/helm-charts/cnpg-db-tsdb-dense/cluster.yaml.ovh

@@ -19,8 +19,6 @@ spec:
       postInitTemplateSQL:
         - CREATE EXTENSION timescaledb;
         - CREATE EXTENSION jsquery;
-        - CREATE DATABASE app; # for the operator restores
-        - CREATE ROLE app;     # same
       database: imagine
       owner: imagine
       secret:
@@ -95,7 +93,6 @@ spec:
         cnpgCluster: cnpg-db-tsdb-dense
       wal:
         compression: gzip
-        maxParallel: 4
 ---
 apiVersion: postgresql.cnpg.io/v1
 kind: ScheduledBackup

package/src/helm-charts/grafana/grafana-local.yaml.ovh

@@ -1,5 +1,6 @@
 # ONLY EDIT grafana-local.yaml.ovh / grafana-local.yaml is git ignored
 service:
+  type: "NodePort"
   port: "9000"
 
 datasources:
@@ -38,7 +39,7 @@ tolerations:
   effect: "NoSchedule"
 
 extraObjects:
-  - apiVersion: traefik.io/v1alpha1
+  - apiVersion: traefik.containo.us/v1alpha1
     kind: IngressRoute
     metadata:
       name: grafana

package/src/helm-charts/grafana/helmup.plugin

@@ -4,7 +4,7 @@ showInstalling "Grafana"
 
 addHelmRepo grafana https://grafana.github.io/helm-charts
 
-[ -z "$GRAFANA_CHART_VERSION" ] && GRAFANA_CHART_VERSION="8"
+[ -z "$GRAFANA_CHART_VERSION" ] && GRAFANA_CHART_VERSION="7"
 helm upgrade --install grafana grafana/grafana \
   --namespace monitoring --create-namespace \
   --values grafana/grafana-local.yaml \

package/src/helm-charts/redis/helmup.plugin

@@ -4,7 +4,7 @@ showInstalling "Redis"
 
 OCI_CHART="oci://registry-1.docker.io/bitnamicharts/redis"
 
-[ -z "$REDIS_CHART_VERSION" ] && REDIS_CHART_VERSION="19.6.0"
+[ -z "$REDIS_CHART_VERSION" ] && REDIS_CHART_VERSION="19.5.0"
 
 helm upgrade --install redis $OCI_CHART       \
              --values  redis/redis-local.yaml \

package/src/helm-charts/redis/redis-local.yaml

@@ -25,11 +25,6 @@ master:
   persistence:
     size: 8Gi # 8Gi is default
 
-  resources:
-    requests:
-      cpu: 250m
-      memory: 500Mi
-
 replica:
   affinity:
     nodeAffinity:
@@ -50,11 +45,6 @@ replica:
   persistence:
     size: 8Gi # 8Gi is default
 
-  resources:
-    requests:
-      cpu: 250m
-      memory: 500Mi
-
 architecture: replication
 
 sentinel:

package/src/helm-charts/resource-server/values-local.yaml

@@ -1,7 +1,5 @@
 # migrations
-#   v1.2+
-#     - k8x resource-server
-#     - npm run sequelizeRunMigrations
+#   v1.2+ run => npm run sequelizeRunMigrations
 #
 image:
   tag: # v1.2.2

package/src/helm-charts/rule-engine/values-local.yaml

@@ -1,5 +1,5 @@
 image:
-  tag: # v2.3.1
+  tag: # v2.3.0
 
 config:
   LOG_CONFIG: '{"type":"pino","level":"warn"}'

package/src/helm-charts/traefik/helmup.plugin

@@ -5,11 +5,7 @@
 # ***REMEMBER*** the instructions provided on the internet typically assume
 #                the default namespace not traefik
 #
-# ***DO NOT FORGET*** the -n traefik for kubectl commands - so in order to
-#    move beyond v20 of the chart run the kubectl command below before upgrading
-#
-# kubectl apply --namespace traefik --server-side \
-#               --force-conflicts -k https://github.com/traefik/traefik-helm-chart/traefik/crds/\?ref\=v27
+# ***DO NOT FORGET*** the -n traefik for kubectl commands
 #
 showInstalling "Traefik Load Balancer / Router"
 

package/src/helm-charts/velero/helmup.plugin

@@ -4,7 +4,7 @@ showInstalling "Velero Backup System"
 
 addHelmRepo vmware-tanzu https://vmware-tanzu.github.io/helm-charts
 
-[ -z "$VELERO_CHART_VERSION" ] && VELERO_CHART_VERSION="7"
+[ -z "$VELERO_CHART_VERSION" ] && VELERO_CHART_VERSION="6"
 #
 # Build the bucket, region and SA email variables and use --set to mod the
 # chart values as opposed to using an OVH:<label> approach. This eliminates

package/src/helmup.sh

@@ -666,7 +666,7 @@ function installVeleroEnvironment() {
   gcloud config set project $GCP_PROJECT_ID
 
   ## The major chart version will determine the bucket suffix
-  [ -z "$VELERO_HELM_CHART" ] && VELERO_HELM_CHART="7"
+  [ -z "$VELERO_HELM_CHART" ] && VELERO_HELM_CHART="6"
 
   ## Create a bucket
   BUCKET="$GCP_PROJECT_ID-velero-$VELERO_HELM_CHART"

package/src/k8scale.sh

@@ -19,11 +19,8 @@ case $DIRECTION in
     REPLICAS=0
     DIRECTION=down
     ;;
-  roll)
-    DIRECTION=roll
-    ;;
   *)
-    printf "Usage: `color g 'k8scale <up|dn|down|roll> [deployment-list]'`\n\n" && exit 1
+    printf "Usage: `color g 'k8scale <up|dn|down> [deployment-list]'`\n\n" && exit 1
 esac
 
 printf "   scaled => `color g $DIRECTION`\n\n"
@@ -31,12 +28,7 @@ for DEPLOYMENT in "$@";
 do
   IFS="/" read SERVICE <<< "$DEPLOYMENT" # strip trailing /
   RESULTS="`color g SUCCESS`"
-  if [ "$DIRECTION" == "roll" ];
-  then
-    kubectl rollout restart deployment $SERVICE &> $DEVNULL
-  else
-    kubectl scale --replicas=$REPLICAS deployment $SERVICE &> $DEVNULL
-  fi
+  kubectl scale --replicas=$REPLICAS deployment $SERVICE &>/dev/null
   [ $? -eq 1 ] && RESULTS="`color r FAILURE`"
   printf "  $RESULTS => $SERVICE\n"
   sendToSlack "k8scale $DIRECTION $SERVICE"

package/src/overwhelm.mjs

@@ -15,7 +15,7 @@ import parse from 'parse-gitignore'
 import ask from 'readline-sync'
 import YAML from 'js-yaml'
 
-import { debug, disableDebug, err, log, shellCmd } from './Utils.mjs'
+import { debug, err, log, shellCmd } from './Utils.mjs'
 
 const optionsDefinitions = [
   /* eslint-disable no-multi-spaces */
@@ -33,9 +33,6 @@ const optionsDefinitions = [
 ]
 const args = cliArgs( optionsDefinitions )
 
-// when using --key we only want the key contents w/o debug info
-if ( args.key ) { disableDebug() }
-
 // See if we are in a monorepo, which changes some of the rules.
 const gitTop = await shellCmd( 'git rev-parse --show-toplevel' )
 
@@ -67,21 +64,10 @@ if ( !fs.existsSync( 'overwhelm.yaml' ) ) {
 if ( fs.existsSync( 'values-default.yaml' ) ) {
   log( chalk.red.bold( `
 
-***ERROR overwhelm => values-default.yaml is no longer used/supported and will
-         be ignored - all defaults should be set in default-configs.yaml.ovh
+***ERROR overwhelm => values-default.yaml is no longer used/supported
+                      all defaults should be set in default-configs.yaml.ovh
   ` ) )
-
-  const ans = ask.question( `Enter "${chalk.bold.red( 'skip' )}" if you wish to ignore and continue => ` )
-  if ( ans !== 'skip' ) {
-    process.exit( 1 )
-  }
-}
-
-// Avoid race condition when running overwhelm for the first time on a new
-// cluster and the default-configs.yaml file has yet to be created.
-const defaultConfigs = 'default-configs.yaml'
-if ( !fs.existsSync( defaultConfigs ) ) {
-  fs.closeSync( fs.openSync( defaultConfigs, 'w' ) ) // same as unix touch
+  process.exit( 1 )
 }
 
 // Load the default values and then overlay with the project specific yaml
@@ -211,7 +197,7 @@ const doReplacements = ( yamls, replacements ) => {
 
 // Collect all of the yaml from the yams...
 const buildYaml = ( yamls, replacements ) => {
-  const always = [ defaultConfigs ]
+  const always = [ 'default-configs.yaml' ]
   const values = [ ...always, ...yamls ]
   const loaded = []
   let allYamls = ''

package/registry-compass.yaml

@@ -1,20 +0,0 @@
-registry:
-  - root: us-docker.pkg.dev
-  - repositories:
-    - name: stack
-      charts:
-        - api-server
-        - authz-server
-        - emailer
-        - message-processor
-    - name: leverege
-      charts:
-        - pubsub-pulse
-        - pusher
-        - overdose
-    - name: cox-health
-      charts:
-        - actions-server
-        - analytics-server
-        - centrak-healthz
-        - centrak-ingestor

package/src/chart-compass.mjs

@@ -1,96 +0,0 @@
-#!/usr/bin/env node
-/*
- * chart-to-registry will...
- */
-import fs from 'node:fs'
-
-import chalk from 'chalk'
-import commandLineArgs from 'command-line-args'
-import commandLineUsage from 'command-line-usage'
-import { lt as semverLt } from 'semver'
-import YAML from 'js-yaml'
-
-import {
-  condir,
-  debug,
-  errorExit,
-  log,
-  warning,
-  getGitRootDirectory,
-  parsePackageJson,
-  parseHelmChart,
-  shellCmd } from './Utils.mjs'
-
-const commandLineOptions = [ // Use commandLineOptions to tie into the Usage statements
-  /* eslint-disable max-len */
-  {
-    name : 'location',
-    type : String,
-    description : '{green the location of the artifact registry the chart will be pushed to (default us-docker.pkg.dev)}',
-  },
-  {
-    name : 'project',
-    type : String,
-    description : '{green the name of the google project containing the npmrc and slack config secrets (default leverege-registry)}',
-  },
-  {
-    name : 'repository',
-    type : String,
-    description : '{green the target repository to receive the pushed chart}',
-  },
-  {
-    name : 'dry-run',
-    type : Boolean,
-    description : '{yellow perform everything except the actual chart push}',
-  },
-  {
-    name : 'help',
-    type : Boolean,
-    description : '{green display this help screen}',
-  },
-  /* eslint-enable max-len */
-]
-
-const sections = [
-  {
-    header : 'Leverege Helm Chart Compass (for helmup)',
-    content : `{green This tool helps helmup navigate the helm charts stored in the
-artifact-registries.}`
-  },
-  { header : 'Options',
-    optionList : commandLineOptions,
-  },
-]
-
-const args = commandLineArgs( commandLineOptions, { camelCase : true, partial : true } )
-const usage = commandLineUsage( sections )
-
-if ( args.help ) {
-  log( usage )
-  process.exit( 0 )
-}
-
-/* eslint-disable no-underscore-dangle */
-if ( args._unknown ) {
-  log( usage )
-  log( `\nUnrecognized argument [${chalk.bold.red( args._unknown )}]\n` )
-  process.exit( 1 )
-}
-/* eslint-enable no-underscore-dangle */
-
-const minNodejsVersion = '18.0.0'
-if ( semverLt( process.version, minNodejsVersion ) ) {
-  errorExit( `\n***ERROR: must be running at least node ${minNodejsVersion}\n` )
-}
-
-// First of all, fail if we are not in a git repository
-let gitRoot
-try {
-  gitRoot = await getGitRootDirectory()
-} catch ( error ) {
-  errorExit( chalk.red.bold( error ), { errorCode : 5 } )
-}
-
-const chartCompass = YAML.load( fs.readFileSync( './registry-compass.yaml', 'utf8' ) )
-
-condir( { chartCompass }, '<==Navigation' )