@leverege/build-tools 2.49.0 → 2.50.1

package/lib/web/unleash.js

@@ -0,0 +1,50 @@
+#!/usr/bin/env node
+"use strict";
+
+var _execa = _interopRequireDefault(require("execa"));
+var _simpleGit = _interopRequireDefault(require("simple-git"));
+function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
+const git = new _simpleGit.default();
+const getCurrentBranchName = async () => {
+  const branchInfo = await git.branch();
+
+  // first check to see if there are no branches - new repo?
+  if (Object.keys(branchInfo.branches).length === 0) {
+    return '';
+  }
+  return Object.entries(branchInfo.branches).find(([name, info]) => info.current)[0];
+};
+const isProtected = async () => {
+  const theBranch = await getCurrentBranchName();
+  const protectedBranches = ['main', 'master', 'development'];
+  const additionalProtectedBranches = [...process.argv.slice(2)];
+
+  // Check if the current branch starts with any of the additional
+  // protected branch name
+  let matchesAdditionalBranch = false;
+  for (let i = 0; i < additionalProtectedBranches.length; i++) {
+    matchesAdditionalBranch = matchesAdditionalBranch || theBranch.startsWith(additionalProtectedBranches[i]);
+  }
+  return protectedBranches.includes(theBranch) || matchesAdditionalBranch;
+};
+const npmRunner = async script => {
+  try {
+    const results = await (0, _execa.default)('npm', ['run', script], {
+      stdio: 'inherit'
+    });
+    return results;
+  } catch (err) {
+    process.exit(1);
+  }
+  return undefined;
+};
+(async () => {
+  if (await isProtected()) {
+    const script = ['test', 'lint'];
+    for (let i = 0; i < script.length; i++) {
+      await npmRunner(script[i]); // eslint-disable-line no-await-in-loop
+    }
+  } else {
+    process.exit(0);
+  }
+})();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@leverege/build-tools",
-  "version": "2.49.0",
+  "version": "2.50.1",
   "description": "A collection of build / support tools for Leverege developers",
   "main": "index.js",
   "repository": {
@@ -16,7 +16,7 @@
     "build-tools": "src/build-tools.mjs",
     "chart-to-registry": "src/chart-to-registry.mjs",
     "circleate": "src/circleate.sh",
-    "cnpg-snapshot": "src/cnpg-snapshot.mjs",
+    "cnpg-backup": "src/cnpg-backup.mjs",
     "decrypt-secrets": "src/decrypt-secrets.sh",
     "dirty-git": "src/dirty-git.sh",
     "dockreate": "src/dockreate.sh",
@@ -54,7 +54,7 @@
   "author": "Leverege Devs",
   "license": "SEE LICENSE IN LICENSE.md",
   "dependencies": {
-    "@google-cloud/artifact-registry": "^3.2.0",
+    "@google-cloud/artifact-registry": "^3.3.0",
     "ansi-colors": "^4.1.3",
     "chalk": "^5.3.0",
     "command-line-args": "^5.2.1",
@@ -64,20 +64,20 @@
     "execa": "^8.0.1",
     "glob": "^10.3.12",
     "handlebars": "^4.7.8",
-    "inquirer": "^9.2.16",
+    "inquirer": "^9.2.18",
     "js-yaml": "^4.1.0",
     "ms": "^2.1.3",
-    "npm-registry-fetch": "^16.1.0",
+    "npm-registry-fetch": "^16.2.1",
     "package-up": "^5.0.0",
     "parse-gitignore": "^2.0.0",
     "read-pkg": "^9.0.1",
     "readline-sync": "^1.4.10",
     "semver": "^7.6.0",
     "simple-git": "^3.24.0",
-    "zx": "^7.2.3"
+    "zx": "^8.0.1"
   },
   "devDependencies": {
     "@leverege/eslint-config-leverege": "^4.2.0",
-    "npm": "^10.5.0"
+    "npm": "^10.5.2"
   }
 }

package/src/firebaseDeploy.mjs

@@ -3,7 +3,6 @@
 // https://github.com/google/zx
 /* eslint-disable max-len */
 import pkg from 'enquirer'
-import colors from 'ansi-colors'
 import { $, fs, argv, path, chalk } from 'zx'
 
 const { prompt } = pkg
@@ -97,8 +96,8 @@ if ( INTERACTIVE_MODE ) {
     name : 'targetEnv',
     message : 'To which environment would you like to deploy?',
     choices : Object.values( targets ).map( ( t ) => {
-      const productionLabel = t.production ? colors.bold.red( '--PRODUCTION--' ) : ''
-      return { message : `${colors.bold.cyan( 'Site: ' )}${t.siteId.padEnd( maxSiteIdLength )} ${colors.bold.green( 'Project: ' )}${t.projectId} ${productionLabel}`, value : t.alias }
+      const productionLabel = t.production ? chalk.bold.red( '--PRODUCTION--' ) : ''
+      return { message : `${chalk.bold.cyan( 'Site: ' )}${t.siteId.padEnd( maxSiteIdLength )} ${chalk.bold.green( 'Project: ' )}${t.projectId} ${productionLabel}`, value : t.alias }
     } )
   } ] )
 
@@ -108,7 +107,7 @@ if ( INTERACTIVE_MODE ) {
     const { confirmProductionDeploy } = await prompt( [ {
       type : 'confirm',
       name : 'confirmProductionDeploy',
-      message : `${colors.bold.red( 'WARNING: ' )} This is a ${colors.bold.yellow( 'PRODUCTION' )} environment. Are you sure you want to deploy here?`
+      message : `${chalk.bold.red( 'WARNING: ' )} This is a ${chalk.bold.yellow( 'PRODUCTION' )} environment. Are you sure you want to deploy here?`
     } ] )
     if ( !confirmProductionDeploy ) {
       console.log( 'Cancelling deployment...' )
@@ -169,14 +168,14 @@ if ( aliasFileExists ) {
   exec.push( fs.readFile( path.join( process.cwd(), SECRETS_DIR, `${TARGET.siteId}.env` ), { encoding : 'utf-8' } ) )
 }
 
-console.log( `\n\n\n${colors.bold.green( '====== Deployment Summary ======' )}` )
-console.log( `${colors.blue( 'Firebase project:' )} ${TARGET.projectId} ${TARGET.production ? colors.yellow( '--PRODUCTION--' ) : ''}` )
-console.log( `${colors.blue( 'Site ID:' )} ${TARGET.siteId}` )
-console.log( `${colors.blue( 'Channel Name:' )} ${CHANNEL?.name ?? colors.italic.gray( '(none)' )}` )
-console.log( `${colors.blue( 'Channel Expiration date:' )} ${CHANNEL?.name ? CHANNEL.expiration ?? colors.italic.gray( '(none)' ) : colors.italic.gray( '(n/a)' )}` ) 
-console.log( `${colors.blue( 'Shared .env file:' )} ${sharedFileExists ? `${SECRETS_DIR}/shared.env` : colors.italic.gray( '(none)' )}` )
-console.log( `${colors.blue( 'Additional .env overrides:' )} ${aliasFileExists ? `${SECRETS_DIR}/${TARGET.alias}.env` : `${SECRETS_DIR}/${TARGET.siteId}.env`}` )
-console.log( `${colors.bold.green( '===============================' )}` )
+console.log( `\n\n\n${chalk.bold.green( '====== Deployment Summary ======' )}` )
+console.log( `${chalk.blue( 'Firebase project:' )} ${TARGET.projectId} ${TARGET.production ? chalk.yellow( '--PRODUCTION--' ) : ''}` )
+console.log( `${chalk.blue( 'Site ID:' )} ${TARGET.siteId}` )
+console.log( `${chalk.blue( 'Channel Name:' )} ${CHANNEL?.name ?? chalk.italic.gray( '(none)' )}` )
+console.log( `${chalk.blue( 'Channel Expiration date:' )} ${CHANNEL?.name ? CHANNEL.expiration ?? chalk.italic.gray( '(none)' ) : chalk.italic.gray( '(n/a)' )}` ) 
+console.log( `${chalk.blue( 'Shared .env file:' )} ${sharedFileExists ? `${SECRETS_DIR}/shared.env` : chalk.italic.gray( '(none)' )}` )
+console.log( `${chalk.blue( 'Additional .env overrides:' )} ${aliasFileExists ? `${SECRETS_DIR}/${TARGET.alias}.env` : `${SECRETS_DIR}/${TARGET.siteId}.env`}` )
+console.log( `${chalk.bold.green( '===============================' )}` )
 
 if ( INTERACTIVE_MODE ) {
   const { confirmSelections } = await prompt( [ {
@@ -192,12 +191,15 @@ if ( INTERACTIVE_MODE ) {
 }
 const envFileContent = ( await Promise.all( exec ) ).join( '\n' )
 
+const ac = new AbortController()
+
 EXIT_EVENTS
   .forEach( ( event ) => {
     process.on( event, () => {
       if ( !exited ) {
         exited = true
         fs.removeSync( path.join( process.cwd(), '.env.temp' ) )
+        ac.abort()
       }
     } )
   } )
@@ -210,7 +212,8 @@ try {
   const deployType = CHANNEL ? `hosting:channel:deploy ${CHANNEL.name} ${CHANNEL.expiration ? `--expires ${CHANNEL.expiration}` : ''}`.split( ' ' ) : 'deploy'
   const hostingPrefix = CHANNEL ? '' : 'hosting:'
 
-  await $`npm run clean && DOTENV_CONFIG_PATH=${path.join( process.cwd(), '.env.temp' )} DEPLOYMENT_TARGET=${TARGET.siteId} npm run build && firebase use ${TARGET.projectId} && firebase ${deployType} --only ${hostingPrefix}${TARGET.siteId}`
+  await $( { ac } )`npm run clean && DOTENV_CONFIG_PATH=${path.join( process.cwd(), '.env.temp' )} DEPLOYMENT_TARGET=${TARGET.siteId} npm run build && firebase use ${TARGET.projectId} && firebase ${deployType} --only ${hostingPrefix}${TARGET.siteId}`
+    .pipe( process.stdout )
 } catch ( err ) {
   console.error( err )
 }

package/src/firebaseServe.mjs

@@ -126,12 +126,15 @@ if ( aliasFileExists ) {
 
 const envFileContent = ( await Promise.all( exec ) ).join( '\n' )
 
+const ac = new AbortController()
+
 EXIT_EVENTS
   .forEach( ( event ) => {
     process.on( event, () => {
       if ( !exited ) {
         exited = true
         fs.removeSync( path.join( process.cwd(), '.env.temp' ) )
+        ac.abort()
       }
     } )
   } )
@@ -139,7 +142,7 @@ EXIT_EVENTS
 await fs.writeFile( path.join( process.cwd(), '.env.temp' ), envFileContent )
 
 try {
-  await $`DOTENV_CONFIG_PATH=${path.join( process.cwd(), '.env.temp' )} DEPLOYMENT_TARGET=${TARGET.siteId} npm run serve --colors`
+  await $( { ac } )`DOTENV_CONFIG_PATH=${path.join( process.cwd(), '.env.temp' )} DEPLOYMENT_TARGET=${TARGET.siteId} npm run serve --colors`.pipe( process.stdout )
 } catch ( err ) {
   console.error( err )
 }

package/src/helm-charts/cnpg-db-pgbench/helmup.plugin

@@ -1,3 +1,3 @@
 #!/bin/bash
 #
-kubectl apply -f cnpg-db-pgbench/cnpg-db-pgbench.yaml $K8S_WHAT
+kubectl apply -f cnpg-db-pgbench/cluster.yaml $K8S_WHAT

package/src/helm-charts/cnpg-db-psql-stack/{cnpg-db-psql-stack.yaml.ovh → cluster.yaml.ovh}

@@ -10,7 +10,7 @@ spec:
 # logLevel: debug
 
   description: "Leverege Stack PostgreSQL DB"
-  imageName: us-docker.pkg.dev/leverege-registry/system/images/leverege-pgsql:16.1-cnpg-tsdb2-beta.10
+  imageName: us-docker.pkg.dev/leverege-registry/system/images/leverege-pgsql:16.2-cnpg-tsdb2-beta.4
   bootstrap:
     initdb:
       database: imagine
@@ -35,10 +35,10 @@ spec:
   # to the default ones to make the cluster work
   postgresql:
     parameters:
+      max_connections: "123"
 #     max_worker_processes: "60"
 #     max_worker_processes: "8"
 #     max_parallel_workers: "8"
-      max_connections: "500"
 #     shared_buffers: 2GB
 #     effective_cache_size: 6GB
 #     maintenance_work_mem: 1GB
@@ -90,3 +90,17 @@ spec:
   method: volumeSnapshot
   backupOwnerReference: self
   immediate: true
+---
+apiVersion: postgresql.cnpg.io/v1
+kind: ScheduledBackup
+metadata:
+  name: cnpg-db-psql-stack-volsnap
+  namespace: cnpg-operands
+spec:
+  # https://pkg.go.dev/github.com/robfig/cron#hdr-CRON_Expression_Format
+  schedule: "0 0 */4 * * *"
+  cluster:
+    name: cnpg-db-psql-stack
+  method: volumeSnapshot
+  backupOwnerReference: self
+  immediate: true

package/src/helm-charts/cnpg-db-psql-stack/helmup.plugin

@@ -1,3 +1,3 @@
 #!/bin/bash
 
-kubectl apply -f cnpg-db-psql-stack/cnpg-db-psql-stack.yaml $K8S_WHAT
+kubectl apply -f cnpg-db-psql-stack/cluster.yaml $K8S_WHAT

package/src/helm-charts/cnpg-db-tsdb-basic/{cnpg-db-tsdb-basic.yaml.ovh → cluster.yaml.ovh}

@@ -9,8 +9,11 @@ spec:
   instances: 1
   logLevel: info # warning debug
 
+  monitoring:
+    enablePodMonitor: false # set true once monitoring is setup
+
   description: "Leverege Plain Timescale DB"
-  imageName: us-docker.pkg.dev/leverege-registry/system/images/leverege-pgsql:16.1-cnpg-tsdb2-beta.6
+  imageName: us-docker.pkg.dev/leverege-registry/system/images/leverege-pgsql:16.2-cnpg-tsdb2-beta.4
   bootstrap:
     initdb:
       postInitTemplateSQL:
@@ -26,8 +29,16 @@ spec:
     name: cnpg-db-tsdb-basic-postgres-pw # postgres db pw
 
   affinity:
-#   enablePodAntiAffinity: true
-#   topologyKey: failure-domain.beta.kubernetes.io/zone
+    enablePodAntiAffinity: true
+    topologyKey: failure-domain.beta.kubernetes.io/zone
+    nodeAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+        nodeSelectorTerms:
+        - matchExpressions:
+          - key: target-env
+            operator: In
+            values:
+            - database
     tolerations:
       - key: "database"
         operator: "Equal"
@@ -41,19 +52,19 @@ spec:
       - timescaledb
       - jsquery
     parameters:
+      max_connections: "123"
 #     max_worker_processes: "60"
-      max_worker_processes: "8"
-      max_parallel_workers: "8"
-      shared_buffers: 2GB
-      effective_cache_size: 6GB
-      maintenance_work_mem: 1GB
-      work_mem: 20MB
+#     max_worker_processes: "8"
+#     max_parallel_workers: "8"
+#     shared_buffers: 2GB
+#     effective_cache_size: 6GB
+#     maintenance_work_mem: 1GB
+#     work_mem: 20MB
     pg_hba:
       # To access through TCP/IP you will need to get username
       # and password from the secret cluster-example-custom-app
       - host all all all md5
 
-
   # Example of rolling update strategy:
   # - unsupervised: automated update of the primary once all
   #                 replicas have been upgraded (default)
@@ -66,10 +77,9 @@ spec:
       annotations:
         iam.gke.io/gcp-service-account: cnpg-operands-sa@OVH:<PROJECT_ID>.iam.gserviceaccount.com
 
-  # Require 50Gi of space per instance using ssd storage class
   storage:
     storageClass: premium-rwo
-    size: 50Gi
+    size: 76Gi
 
   backup:
     volumeSnapshot:
@@ -88,11 +98,11 @@ spec:
 apiVersion: postgresql.cnpg.io/v1
 kind: ScheduledBackup
 metadata:
-  name: cnpg-db-tsdb-basic-backup
+  name: cnpg-db-tsdb-basic-volsnap
   namespace: cnpg-operands
 spec:
   # https://pkg.go.dev/github.com/robfig/cron#hdr-CRON_Expression_Format
-  schedule: "0 0 * * * *"
+  schedule: "0 0 */8 * * *"
   cluster:
     name: cnpg-db-tsdb-basic
   method: volumeSnapshot

package/src/helm-charts/cnpg-db-tsdb-basic/helmup.plugin

@@ -1,3 +1,3 @@
 #!/bin/bash
 #
-kubectl apply -f cnpg-db-tsdb-basic/cnpg-db-tsdb-basic.yaml $K8S_WHAT
+kubectl apply -f cnpg-db-tsdb-basic/cluster.yaml $K8S_WHAT

package/src/helm-charts/cnpg-db-tsdb-dense/{cnpg-db-tsdb-dense.yaml.ovh → cluster.yalm.ovh}

@@ -7,10 +7,13 @@ metadata:
   namespace: cnpg-operands
 spec:
   instances: 1
-  logLevel: debug
+  logLevel: info # warning debug
+
+  monitoring:
+    enablePodMonitor: false # set true once monitoring is setup
 
   description: "Leverege Dense Timescale DB"
-  imageName: us-docker.pkg.dev/leverege-registry/system/images/leverege-pgsql:16.1-cnpg-tsdb2-beta.6
+  imageName: us-docker.pkg.dev/leverege-registry/system/images/leverege-pgsql:16.2-cnpg-tsdb2-beta.4
   bootstrap:
     initdb:
       postInitTemplateSQL:
@@ -26,8 +29,16 @@ spec:
     name: cnpg-db-tsdb-dense-postgres-pw # postgres db pw
 
   affinity:
-#   enablePodAntiAffinity: true
-#   topologyKey: failure-domain.beta.kubernetes.io/zone
+    enablePodAntiAffinity: true
+    topologyKey: failure-domain.beta.kubernetes.io/zone
+    nodeAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+        nodeSelectorTerms:
+        - matchExpressions:
+          - key: target-env
+            operator: In
+            values:
+            - database
     tolerations:
       - key: "database"
         operator: "Equal"
@@ -53,7 +64,6 @@ spec:
       # and password from the secret cluster-example-custom-app
       - host all all all md5
 
-
   # Example of rolling update strategy:
   # - unsupervised: automated update of the primary once all
   #                 replicas have been upgraded (default)
@@ -66,10 +76,9 @@ spec:
       annotations:
         iam.gke.io/gcp-service-account: cnpg-operands-sa@OVH:<PROJECT_ID>.iam.gserviceaccount.com
 
-  # Require 50Gi of space per instance using ssd storage class
   storage:
     storageClass: premium-rwo
-    size: 50Gi
+    size: 77Gi
 
   backup:
     volumeSnapshot:
@@ -88,11 +97,11 @@ spec:
 apiVersion: postgresql.cnpg.io/v1
 kind: ScheduledBackup
 metadata:
-  name: cnpg-db-tsdb-dense-backup
+  name: cnpg-db-tsdb-dense-volsnap
   namespace: cnpg-operands
 spec:
   # https://pkg.go.dev/github.com/robfig/cron#hdr-CRON_Expression_Format
-  schedule: "0 0 * * * *"
+  schedule: "0 0 */8 * * *"
   cluster:
     name: cnpg-db-tsdb-dense
   method: volumeSnapshot

package/src/helm-charts/cnpg-db-tsdb-dense/helmup.plugin

@@ -1,5 +1,3 @@
 #!/bin/bash
 #
-#bindWorkloadIdentity cnpg-operands-sa cnpg-operands cnpg-db-tsdb-dense
-
-kubectl apply -f cnpg-db-tsdb-dense/cnpg-db-tsdb-dense.yaml $K8S_WHAT
+kubectl apply -f cnpg-db-tsdb-dense/cluster.yaml $K8S_WHAT

package/registry-compass.yaml

@@ -1,20 +0,0 @@
-registry:
-  - root: us-docker.pkg.dev
-  - repositories:
-    - name: stack
-      charts:
-        - api-server
-        - authz-server
-        - emailer
-        - message-processor
-    - name: leverege
-      charts:
-        - pubsub-pulse
-        - pusher
-        - overdose
-    - name: cox-health
-      charts:
-        - actions-server
-        - analytics-server
-        - centrak-healthz
-        - centrak-ingestor

package/src/chart-compass.mjs

@@ -1,96 +0,0 @@
-#!/usr/bin/env node
-/*
- * chart-to-registry will...
- */
-import fs from 'node:fs'
-
-import chalk from 'chalk'
-import commandLineArgs from 'command-line-args'
-import commandLineUsage from 'command-line-usage'
-import { lt as semverLt } from 'semver'
-import YAML from 'js-yaml'
-
-import {
-  condir,
-  debug,
-  errorExit,
-  log,
-  warning,
-  getGitRootDirectory,
-  parsePackageJson,
-  parseHelmChart,
-  shellCmd } from './Utils.mjs'
-
-const commandLineOptions = [ // Use commandLineOptions to tie into the Usage statements
-  /* eslint-disable max-len */
-  {
-    name : 'location',
-    type : String,
-    description : '{green the location of the artifact registry the chart will be pushed to (default us-docker.pkg.dev)}',
-  },
-  {
-    name : 'project',
-    type : String,
-    description : '{green the name of the google project containing the npmrc and slack config secrets (default leverege-registry)}',
-  },
-  {
-    name : 'repository',
-    type : String,
-    description : '{green the target repository to receive the pushed chart}',
-  },
-  {
-    name : 'dry-run',
-    type : Boolean,
-    description : '{yellow perform everything except the actual chart push}',
-  },
-  {
-    name : 'help',
-    type : Boolean,
-    description : '{green display this help screen}',
-  },
-  /* eslint-enable max-len */
-]
-
-const sections = [
-  {
-    header : 'Leverege Helm Chart Compass (for helmup)',
-    content : `{green This tool helps helmup navigate the helm charts stored in the
-artifact-registries.}`
-  },
-  { header : 'Options',
-    optionList : commandLineOptions,
-  },
-]
-
-const args = commandLineArgs( commandLineOptions, { camelCase : true, partial : true } )
-const usage = commandLineUsage( sections )
-
-if ( args.help ) {
-  log( usage )
-  process.exit( 0 )
-}
-
-/* eslint-disable no-underscore-dangle */
-if ( args._unknown ) {
-  log( usage )
-  log( `\nUnrecognized argument [${chalk.bold.red( args._unknown )}]\n` )
-  process.exit( 1 )
-}
-/* eslint-enable no-underscore-dangle */
-
-const minNodejsVersion = '18.0.0'
-if ( semverLt( process.version, minNodejsVersion ) ) {
-  errorExit( `\n***ERROR: must be running at least node ${minNodejsVersion}\n` )
-}
-
-// First of all, fail if we are not in a git repository
-let gitRoot
-try {
-  gitRoot = await getGitRootDirectory()
-} catch ( error ) {
-  errorExit( chalk.red.bold( error ), { errorCode : 5 } )
-}
-
-const chartCompass = YAML.load( fs.readFileSync( './registry-compass.yaml', 'utf8' ) )
-
-condir( { chartCompass }, '<==Navigation' )