@leverege/build-tools 2.50.9 → 2.50.10

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@leverege/build-tools",
-  "version": "2.50.9",
+  "version": "2.50.10",
   "description": "A collection of build / support tools for Leverege developers",
   "main": "index.js",
   "repository": {
@@ -54,7 +54,7 @@
   "author": "Leverege Devs",
   "license": "SEE LICENSE IN LICENSE.md",
   "dependencies": {
-    "@google-cloud/artifact-registry": "^3.3.0",
+    "@google-cloud/artifact-registry": "^3.4.0",
     "ansi-colors": "^4.1.3",
     "chalk": "^5.3.0",
     "command-line-args": "^5.2.1",
@@ -62,22 +62,22 @@
     "deepmerge": "^4.3.1",
     "enquirer": "^2.4.1",
     "execa": "^8.0.1",
-    "glob": "^10.3.12",
+    "glob": "^10.3.16",
     "handlebars": "^4.7.8",
-    "inquirer": "^9.2.20",
+    "inquirer": "^9.2.22",
     "js-yaml": "^4.1.0",
     "ms": "^2.1.3",
-    "npm-registry-fetch": "^17.0.0",
+    "npm-registry-fetch": "^17.0.1",
     "package-up": "^5.0.0",
     "parse-gitignore": "^2.0.0",
     "read-pkg": "^9.0.1",
     "readline-sync": "^1.4.10",
-    "semver": "^7.6.0",
+    "semver": "^7.6.2",
     "simple-git": "^3.24.0",
-    "zx": "^8.0.2"
+    "zx": "^8.1.0"
   },
   "devDependencies": {
     "@leverege/eslint-config-leverege": "^4.2.0",
-    "npm": "^10.7.0"
+    "npm": "^10.8.0"
   }
 }

package/registry-compass.yaml

@@ -0,0 +1,20 @@
+registry:
+  - root: us-docker.pkg.dev
+  - repositories:
+    - name: stack
+      charts:
+        - api-server
+        - authz-server
+        - emailer
+        - message-processor
+    - name: leverege
+      charts:
+        - pubsub-pulse
+        - pusher
+        - overdose
+    - name: cox-health
+      charts:
+        - actions-server
+        - analytics-server
+        - centrak-healthz
+        - centrak-ingestor

package/src/Utils.mjs

@@ -349,6 +349,10 @@ export const parseHelmChart = async ( helmroot = './helm' ) => {
   const chartVersion = chartYaml.version
   const chartPackage = `${chartName}-${chartVersion}.tgz`
   const valuesYaml = YAML.load( readFileSync( `${helmroot}/values.yaml`, 'utf8' ) )
+  debug( valuesYaml, '<== helm/values.yaml' )
+  if ( !valuesYaml.image ) {
+    errorExit( 'Error: legacy helm chart detected - must upgrade to latest ignition template to proceed' )
+  }
   const imageRegistry = valuesYaml.image?.registry
   const registryComponents = imageRegistry.split( '/' )
   /* eslint-enable security/detect-non-literal-fs-filename */

package/src/bash-funcs

@@ -169,13 +169,20 @@ function ifPluggedIn() {
   local INVOKED_AS=`basename $0`
   local PLUGIN=
   case "$INVOKED_AS" in
-    "helmup"|"helmwhat" )
+    "helmup"|"helmwhat"|"helmcycle" )
       PLUGIN="$1/helmup.plugin"
       ;;
     "helmdn" )
       PLUGIN="$1/helmdn.plugin"
       ;;
     *)
+      cat<<PLUGIN_INVOKED_AS_ERROR
+
+
+  $RED_ERROR bash-funcs ifPluggedIn: unknown INVOKED_AS=[`color r $INVOKED_AS`]
+
+PLUGIN_INVOKED_AS_ERROR
+      exit 1
       ;;
   esac
 

package/src/chart-compass.mjs

@@ -0,0 +1,96 @@
+#!/usr/bin/env node
+/*
+ * chart-to-registry will...
+ */
+import fs from 'node:fs'
+
+import chalk from 'chalk'
+import commandLineArgs from 'command-line-args'
+import commandLineUsage from 'command-line-usage'
+import { lt as semverLt } from 'semver'
+import YAML from 'js-yaml'
+
+import {
+  condir,
+  debug,
+  errorExit,
+  log,
+  warning,
+  getGitRootDirectory,
+  parsePackageJson,
+  parseHelmChart,
+  shellCmd } from './Utils.mjs'
+
+const commandLineOptions = [ // Use commandLineOptions to tie into the Usage statements
+  /* eslint-disable max-len */
+  {
+    name : 'location',
+    type : String,
+    description : '{green the location of the artifact registry the chart will be pushed to (default us-docker.pkg.dev)}',
+  },
+  {
+    name : 'project',
+    type : String,
+    description : '{green the name of the google project containing the npmrc and slack config secrets (default leverege-registry)}',
+  },
+  {
+    name : 'repository',
+    type : String,
+    description : '{green the target repository to receive the pushed chart}',
+  },
+  {
+    name : 'dry-run',
+    type : Boolean,
+    description : '{yellow perform everything except the actual chart push}',
+  },
+  {
+    name : 'help',
+    type : Boolean,
+    description : '{green display this help screen}',
+  },
+  /* eslint-enable max-len */
+]
+
+const sections = [
+  {
+    header : 'Leverege Helm Chart Compass (for helmup)',
+    content : `{green This tool helps helmup navigate the helm charts stored in the
+artifact-registries.}`
+  },
+  { header : 'Options',
+    optionList : commandLineOptions,
+  },
+]
+
+const args = commandLineArgs( commandLineOptions, { camelCase : true, partial : true } )
+const usage = commandLineUsage( sections )
+
+if ( args.help ) {
+  log( usage )
+  process.exit( 0 )
+}
+
+/* eslint-disable no-underscore-dangle */
+if ( args._unknown ) {
+  log( usage )
+  log( `\nUnrecognized argument [${chalk.bold.red( args._unknown )}]\n` )
+  process.exit( 1 )
+}
+/* eslint-enable no-underscore-dangle */
+
+const minNodejsVersion = '18.0.0'
+if ( semverLt( process.version, minNodejsVersion ) ) {
+  errorExit( `\n***ERROR: must be running at least node ${minNodejsVersion}\n` )
+}
+
+// First of all, fail if we are not in a git repository
+let gitRoot
+try {
+  gitRoot = await getGitRootDirectory()
+} catch ( error ) {
+  errorExit( chalk.red.bold( error ), { errorCode : 5 } )
+}
+
+const chartCompass = YAML.load( fs.readFileSync( './registry-compass.yaml', 'utf8' ) )
+
+condir( { chartCompass }, '<==Navigation' )

package/src/helm-charts/cnpg-db-pgbench/pooler.yaml

@@ -12,5 +12,9 @@ spec:
   pgbouncer:
     poolMode: session
     parameters:
+      # https://www.pgbouncer.org/config.html#generic-settings
       max_client_conn: "500"
       default_pool_size: "10"
+      # https://www.pgbouncer.org/config.html#log-settings
+      log_connections: "0"
+      log_disconnections: "0"

package/src/helm-charts/cnpg-db-psql-stack/cluster.yaml.ovh

@@ -13,24 +13,35 @@ spec:
   imageName: us-docker.pkg.dev/leverege-registry/system/images/leverege-pgsql:16.2-cnpg-tsdb2-beta.4
   bootstrap:
     initdb:
-      database: models
+#     database: models
       postInitSQL:
+#       - CREATE DATABASE app;       # operator expects this
+#       - CREATE ROLE app;           #   and this to exist on restore
+        # create the stack databases
         - CREATE DATABASE authz;
         - CREATE DATABASE fota;
         - CREATE DATABASE geotile;
         - CREATE DATABASE models;
         - CREATE DATABASE scheduler;
-      owner: imagine
-      secret:
-        name: cnpg-db-psql-stack-imagine-pw # imagine db pw
+#     owner: imagine
+#     secret:
+#       name: cnpg-db-psql-stack-imagine-pw # imagine db pw
 
   enableSuperuserAccess: true
   superuserSecret:
     name: cnpg-db-psql-stack-postgres-pw # postgres db pw
 
   affinity:
-#   enablePodAntiAffinity: true
-#   topologyKey: kubernetes.io/hostname # default value
+    enablePodAntiAffinity: true
+    topologyKey: kubernetes.io/hostname # default value
+    nodeAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+        nodeSelectorTerms:
+        - matchExpressions:
+          - key: target-env
+            operator: In
+            values:
+            - database
     tolerations:
       - key: "database"
         operator: "Equal"
@@ -83,6 +94,7 @@ spec:
         cnpgCluster: cnpg-db-psql-stack
       wal:
         compression: gzip
+        maxParallel: 4
 ---
 apiVersion: postgresql.cnpg.io/v1
 kind: ScheduledBackup
@@ -91,8 +103,9 @@ metadata:
   namespace: cnpg-operands
 spec:
   # https://pkg.go.dev/github.com/robfig/cron#hdr-CRON_Expression_Format
-  schedule: "0 0 * * * *"
+  schedule: "0 0 */12 * * *"
   cluster:
-  method: volumeSnapshot
+    name: cnpg-db-psql-stack
+  method: barmanObjectStore # volumeSnapshot
   backupOwnerReference: self
   immediate: true

package/src/helm-charts/cnpg-db-psql-stack/pooler.yaml

@@ -12,5 +12,9 @@ spec:
   pgbouncer:
     poolMode: session
     parameters:
+      # https://www.pgbouncer.org/config.html#generic-settings
       max_client_conn: "500"
       default_pool_size: "10"
+      # https://www.pgbouncer.org/config.html#log-settings
+      log_connections: "0"
+      log_disconnections: "0"

package/src/helm-charts/cnpg-db-tsdb-basic/cluster.yaml.ovh

@@ -19,6 +19,8 @@ spec:
       postInitTemplateSQL:
         - CREATE EXTENSION timescaledb;
         - CREATE EXTENSION jsquery;
+        - CREATE DATABASE app; # for the operator restores
+        - CREATE ROLE app;     # same
       database: imagine
       owner: imagine
       secret:
@@ -94,6 +96,7 @@ spec:
         cnpgCluster: cnpg-db-tsdb-basic
       wal:
         compression: gzip
+        maxParallel: 4
 ---
 apiVersion: postgresql.cnpg.io/v1
 kind: ScheduledBackup

package/src/helm-charts/cnpg-db-tsdb-basic/pooler.yaml

@@ -12,5 +12,9 @@ spec:
   pgbouncer:
     poolMode: session
     parameters:
+      # https://www.pgbouncer.org/config.html#generic-settings
       max_client_conn: "500"
       default_pool_size: "10"
+      # https://www.pgbouncer.org/config.html#log-settings
+      log_connections: "0"
+      log_disconnections: "0"

package/src/helm-charts/cnpg-db-tsdb-dense/cluster.yaml.ovh

@@ -19,6 +19,8 @@ spec:
       postInitTemplateSQL:
         - CREATE EXTENSION timescaledb;
         - CREATE EXTENSION jsquery;
+        - CREATE DATABASE app; # for the operator restores
+        - CREATE ROLE app;     # same
       database: imagine
       owner: imagine
       secret:
@@ -93,6 +95,7 @@ spec:
         cnpgCluster: cnpg-db-tsdb-dense
       wal:
         compression: gzip
+        maxParallel: 4
 ---
 apiVersion: postgresql.cnpg.io/v1
 kind: ScheduledBackup

package/src/helm-charts/cnpg-db-tsdb-dense/pooler.yaml

@@ -12,5 +12,9 @@ spec:
   pgbouncer:
     poolMode: session
     parameters:
+      # https://www.pgbouncer.org/config.html#generic-settings
       max_client_conn: "500"
       default_pool_size: "10"
+      # https://www.pgbouncer.org/config.html#log-settings
+      log_connections: "0"
+      log_disconnections: "0"

package/src/helmup.sh

@@ -794,7 +794,13 @@ doHelmup() {
   [ ! -z "$FROM_VERSIONS" ] && from="`color y \"<= from Versions.json\"`"
   printf "$doing $from\n"
   eval $helmup
-  sendToSlack "$helmup"
+  case "$INVOKED_AS" in
+    "helmup"|"helmdn")
+      sendToSlack "$helmup"
+      ;;
+    *)
+      ;;
+  esac
 }
 
 # not a fan of global vars but bash forced me to - set by getServiceAndChartVersion
@@ -814,7 +820,7 @@ function doInstall() {
     # slap additional helm syntax in there if the version is set
     [ ! -z "$CHARTVER" ] && CHARTVER="--version $CHARTVER"
 
-    local INVOKED_AS=`basename $0`
+    INVOKED_AS=`basename $0`
     # set the dry-run flag on what
     [ "$INVOKED_AS" == "helmwhat" ] && HELM_WHAT="--dry-run --debug" && K8S_WHAT="--dry-run=server -o yaml"
 

package/src/zx-testing

@@ -0,0 +1,216 @@
+#!/usr/bin/env node
+/* eslint-disable no-console */
+// https://github.com/google/zx
+/* eslint-disable max-len */
+import pkg from 'enquirer'
+import colors from 'ansi-colors'
+import { $, fs, argv, path, chalk } from 'zx'
+
+const { prompt } = pkg
+
+// This will preserve coloration from spawned child processes
+process.env.FORCE_COLORS = 3
+process.env.FORCE_COLOR = '1'
+
+let TARGET
+let CHANNEL
+
+// A single unnamed parameter is assumed to be a deploy target.
+// If named parameters exist, they may not be stored in argv,
+// so don't look for a larger length of argv to decide whether to look for named paramters
+if ( argv._.length === 1 ) {
+  TARGET = argv._[0]
+} else {
+  TARGET = argv.target
+
+  if ( argv.channelName ) {
+    CHANNEL = {}
+    CHANNEL.name = argv.channelName
+    CHANNEL.expiration = argv.channelExpiration ?? '7d'
+  }
+}
+
+// If target has not been defined on the command line,
+// then enter interactive mode
+const INTERACTIVE_MODE = !TARGET
+
+let exited = false
+
+const SECRETS_DIR = 'secrets'
+const EXIT_EVENTS = [ 'SIGINT', 'exit', 'uncaughException', 'unhandledRejection' ]
+
+const [
+  firebaserc,
+  firebaseJson,
+  gitBranch
+] = await Promise.all( [
+  fs.exists( path.join( process.cwd(), '.firebaserc' ) ),
+  fs.exists( path.join( process.cwd(), 'firebase.json' ) ),
+  $`git rev-parse --abbrev-ref HEAD -C ${process.cwd()}`
+    .then( ( { stdout } ) => {
+      return stdout.split( '\n' )[0]
+    } )
+] )
+
+if ( !firebaserc ) {
+  console.log( chalk.red( 'No .firebaserc file found. Make sure that you are executing firebaseDeploy from within a firebase project directory.' ) )
+}
+
+if ( !firebaseJson ) {
+  console.log( chalk.red( 'No firebase.json file found. Make sure that you are executing firebaseDeploy from within a firebase project directory.' ) )
+}
+
+if ( !firebaserc || !firebaseJson ) {
+  process.exit()
+}
+
+const firebasercContent = JSON.parse( await fs.readFile( path.join( process.cwd(), '.firebaserc' ), { encoding : 'utf-8' } ) )
+
+let maxSiteIdLength = 0
+
+const targets = Object
+  .entries( firebasercContent.targets )
+  .reduce( ( prev, [ projectId, config ] ) => {
+    Object
+      .entries( config.hosting )
+      .forEach( ( [ siteId, aliases ] ) => {
+        aliases.forEach( ( alias ) => {
+          // Store the longest length of site id for formatting later on
+          maxSiteIdLength = Math.max( siteId.length, maxSiteIdLength )
+
+          // eslint-disable-next-line no-param-reassign
+          prev[alias] = {
+            projectId,
+            siteId,
+            alias,
+            production : config.production
+          }
+        } )
+      } )
+
+    return prev
+  }, {} )
+
+if ( INTERACTIVE_MODE ) {
+  const { targetEnv } = await prompt( [ {
+    type : 'autocomplete',
+    name : 'targetEnv',
+    message : 'To which environment would you like to deploy?',
+    choices : Object.values( targets ).map( ( t ) => {
+      const productionLabel = t.production ? colors.bold.red( '--PRODUCTION--' ) : ''
+      return { message : `${colors.bold.cyan( 'Site: ' )}${t.siteId.padEnd( maxSiteIdLength )} ${colors.bold.green( 'Project: ' )}${t.projectId} ${productionLabel}`, value : t.alias }
+    } )
+  } ] )
+
+  TARGET = targets[targetEnv]
+
+  if ( TARGET.production ) {
+    const { confirmProductionDeploy } = await prompt( [ {
+      type : 'confirm',
+      name : 'confirmProductionDeploy',
+      message : `${colors.bold.red( 'WARNING: ' )} This is a ${colors.bold.yellow( 'PRODUCTION' )} environment. Are you sure you want to deploy here?`
+    } ] )
+    if ( !confirmProductionDeploy ) {
+      console.log( 'Cancelling deployment...' )
+      process.exit()
+    }
+  }
+
+  const { useChannel } = await prompt( [ {
+    type : 'confirm',
+    name : 'useChannel',
+    message : 'Do you want to deploy to a temporary preview channel?'
+  } ] )
+
+  if ( useChannel ) {
+    const { channelName, channelExpiration } = await prompt( [ {
+      type : 'input',
+      name : 'channelName',
+      message : 'To which channel would you like to deploy?',
+      initial : gitBranch ?? 'temp-deploy'
+    }, {
+      type : 'input',
+      name : 'channelExpiration',
+      message : 'When should the channel expire?',
+      initial : '7d'
+    } ] )
+
+    CHANNEL = { name : channelName, expiration : channelExpiration }
+  }
+} else {
+  TARGET = targets[TARGET]
+}
+
+if ( !TARGET ) {
+  console.log( chalk.red( 'No such target exists in .firebaserc config.' ) )
+  process.exit()
+}
+
+const [ aliasFileExists, siteIdFileExists, sharedFileExists ] = await Promise.all( [
+  fs.exists( path.join( process.cwd(), SECRETS_DIR, `${TARGET.alias}.env` ) ),
+  fs.exists( path.join( process.cwd(), SECRETS_DIR, `${TARGET.siteId}.env` ) ),
+  fs.exists( path.join( process.cwd(), SECRETS_DIR, 'shared.env' ) )
+] )
+
+if ( !aliasFileExists && !siteIdFileExists ) {
+  console.log( chalk.red( `No env file for ${TARGET.alias} exists. Check your secrets directory and try again` ) )
+  process.exit()
+}
+
+const exec = []
+
+if ( sharedFileExists ) {
+  exec.push( fs.readFile( path.join( process.cwd(), SECRETS_DIR, 'shared.env' ), { encoding : 'utf-8' } ) )
+}
+
+if ( aliasFileExists ) {
+  exec.push( fs.readFile( path.join( process.cwd(), SECRETS_DIR, `${TARGET.alias}.env` ), { encoding : 'utf-8' } ) )
+} else {
+  exec.push( fs.readFile( path.join( process.cwd(), SECRETS_DIR, `${TARGET.siteId}.env` ), { encoding : 'utf-8' } ) )
+}
+
+console.log( `\n\n\n${colors.bold.green( '====== Deployment Summary ======' )}` )
+console.log( `${colors.blue( 'Firebase project:' )} ${TARGET.projectId} ${TARGET.production ? colors.yellow( '--PRODUCTION--' ) : ''}` )
+console.log( `${colors.blue( 'Site ID:' )} ${TARGET.siteId}` )
+console.log( `${colors.blue( 'Channel Name:' )} ${CHANNEL?.name ?? colors.italic.gray( '(none)' )}` )
+console.log( `${colors.blue( 'Channel Expiration date:' )} ${CHANNEL?.name ? CHANNEL.expiration ?? colors.italic.gray( '(none)' ) : colors.italic.gray( '(n/a)' )}` ) 
+console.log( `${colors.blue( 'Shared .env file:' )} ${sharedFileExists ? `${SECRETS_DIR}/shared.env` : colors.italic.gray( '(none)' )}` )
+console.log( `${colors.blue( 'Additional .env overrides:' )} ${aliasFileExists ? `${SECRETS_DIR}/${TARGET.alias}.env` : `${SECRETS_DIR}/${TARGET.siteId}.env`}` )
+console.log( `${colors.bold.green( '===============================' )}` )
+
+if ( INTERACTIVE_MODE ) {
+  const { confirmSelections } = await prompt( [ {
+    type : 'confirm',
+    name : 'confirmSelections',
+    message : 'Proceed with this deployment?'
+  } ] )
+
+  if ( !confirmSelections ) {
+    console.log( 'Cancelling deployment...' )
+    process.exit()
+  }
+}
+const envFileContent = ( await Promise.all( exec ) ).join( '\n' )
+
+EXIT_EVENTS
+  .forEach( ( event ) => {
+    process.on( event, () => {
+      if ( !exited ) {
+        exited = true
+        fs.removeSync( path.join( process.cwd(), '.env.temp' ) )
+      }
+    } )
+  } )
+
+await fs.writeFile( path.join( process.cwd(), '.env.temp' ), envFileContent )
+
+try {
+
+  // deploying to a channel requires slightly different parameter structure 
+  const deployType = CHANNEL ? `hosting:channel:deploy ${CHANNEL.name} ${CHANNEL.expiration ? `--expires ${CHANNEL.expiration}` : ''}`.split( ' ' ) : 'deploy'
+  const hostingPrefix = CHANNEL ? '' : 'hosting:'
+
+  await $`npm run clean && DOTENV_CONFIG_PATH=${path.join( process.cwd(), '.env.temp' )} DEPLOYMENT_TARGET=${TARGET.siteId} npm run build && firebase use ${TARGET.projectId} && firebase ${deployType} --only ${hostingPrefix}${TARGET.siteId}`
+} catch ( err ) {
+  console.error( err )
+}