@leverege/build-tools 2.52.0-pedro.9 → 2.52.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@leverege/build-tools",
-  "version": "2.52.0-pedro.9",
+  "version": "2.52.0",
   "description": "A collection of build / support tools for Leverege developers",
   "main": "index.js",
   "repository": {
@@ -47,7 +47,6 @@
     "pull-git": "src/pull-git.sh",
     "push-my-chart": "src/push-my-chart.mjs",
     "refresh-npm-token": "src/refresh-npm-token.mjs",
-    "refresh-py-idx": "src/refresh-py-idx.mjs",
     "send-to-slack": "src/send-to-slack.mjs",
     "tag-release": "src/tag-release.mjs",
     "unleash": "src/unleash.mjs"
@@ -62,10 +61,10 @@
     "command-line-usage": "^7.0.3",
     "deepmerge": "^4.3.1",
     "enquirer": "^2.4.1",
-    "execa": "^9.3.1",
+    "execa": "^9.4.0",
     "glob": "^11.0.0",
     "handlebars": "^4.7.8",
-    "inquirer": "^10.2.2",
+    "inquirer": "^11.0.2",
     "js-yaml": "^4.1.0",
     "ms": "^2.1.3",
     "npm-registry-fetch": "^17.1.0",
@@ -74,11 +73,11 @@
     "read-pkg": "^9.0.1",
     "readline-sync": "^1.4.10",
     "semver": "^7.6.3",
-    "simple-git": "^3.26.0",
-    "zx": "^8.1.6"
+    "simple-git": "^3.27.0",
+    "zx": "^8.1.7"
   },
   "devDependencies": {
     "@leverege/eslint-config-leverege": "^4.2.0",
     "npm": "^10.8.3"
   }
-}
+}

package/src/Docker.mjs

@@ -40,8 +40,12 @@ COPY ./.npmrc /usr/src/app/.npmrc
 RUN apk --no-cache add openssh-client && \
     apk --update add --no-cache --virtual build-dep g++ gcc libgcc \\
       libstdc++ linux-headers make {{apkadds}} && \
-    npm install -g npm@10 && \
-    npm ci --only=production --ignore-scripts --no-optional {{npmlogging}} && \
+    npm install -g npm@10
+
+{{preInstallPluginfile}}
+
+# Removed --ignore-scripts from npm ci because it was causing issues with python native build
+RUN npm ci --only=production --no-optional {{npmlogging}} && \
     rm -f /usr/src/app/.npmrc /root/.ssh/*
 
 # --------------------------------------------------------------
@@ -67,7 +71,7 @@ COPY --chown=node:node --from=intermediate /usr/src/app /usr/src/app
 
 USER {{runuser}}
 COPY ./bashrc /home/node/.bashrc
-CMD [ "/bin/bash", "-c", "node index.js" ]`
+CMD [ "/bin/bash", "-c", "source /home/node/.bashrc && node index.js" ]`
 
 const pluginTemplate = `
 # Dockerfile.plugin - optionally extend the service Docker image
@@ -132,6 +136,19 @@ dunm() {
   du -sh /usr/src/app/node_modules/* | sort -sh
 }
 `
+const cloudBuildSteps = `
+steps:
+  - name: 'gcr.io/cloud-builders/docker'
+    args:
+      - 'build' 
+      - '--network=cloudbuild'
+      - '-t'
+      - '{{imageName}}:{{imageVersion}}'
+      - '.'
+    
+images:
+  - '{{imageName}}:{{imageVersion}}'
+`
 
 const defaultSettings = {
   apkadds : '',
@@ -139,6 +156,7 @@ const defaultSettings = {
   nodeimage : 'iron-alpine',
   npmlogging : '--silent',
   pluginfile : '# NO PLUGIN',
+  preInstallPluginfile : '# NO PRE-INSTALL PLUGIN',
   regvers : 'package.version',
   runuser : 'node',
 }
@@ -168,8 +186,14 @@ const generateDockerfile = ( settings = defaultSettings ) => {
 
   const bashrcFile = './docker/bashrc'
   if ( !fs.existsSync( bashrcFile ) ) {
+    let bf = bashrcTemplate
+    const bashrcFilePlugin = './docker/Bashrc.plugin'
+    if ( fs.existsSync( bashrcFilePlugin ) ) {
+      const bashrcAdditions = fs.readFileSync( bashrcFilePlugin )
+      bf = `${bf}\n${bashrcAdditions}`
+    }
     warning( `creating the missing ${bashrcFile} file in the docker directory` )
-    fs.writeFileSync( bashrcFile, bashrcTemplate )
+    fs.writeFileSync( bashrcFile, bf )
   }
 
   const dockerfilePlugin = './docker/Dockerfile.plugin'
@@ -194,7 +218,12 @@ const generateDockerfile = ( settings = defaultSettings ) => {
     errorExit( `***ERROR: something went wrong with ${dockerfilePlugin} plugin creation` )
   }
 
-  const compiled = handlebars.compile( dockerfileTemplate )
+  const dockerfilePrePlugin = './docker/DockerfilePreInstall.plugin'
+  if ( fs.existsSync( dockerfilePrePlugin ) ) {
+    defaultSettings.preInstallPluginfile = fs.readFileSync( dockerfilePrePlugin )
+  }
+
+  const compiled = handlebars.compile( dockerfileTemplate, { noEscape : true } )
   const replaced = compiled( settings )
   // temporarily write the Dockerfile to the docker subdir - the container
   // image builder will then move it over to the build subdir
@@ -226,8 +255,10 @@ const validateCloudBuildBucket = async ( artifactProject ) => {
 }
 
 const buildContainerImage = async ( {
-  artifactProject, artifactRegistry, containerName, imageVersion, isNpmWorkspace,
+  artifactProject, artifactRegistry, containerName, 
+  imageVersion, isNpmWorkspace, options
 } ) => {
+  const { useCloudBuildFile } = options || {}
   const buildWorkspace = './build/workspace' // passed to Cloud Build
   const imageName = `${artifactRegistry}/images/${containerName}`
 
@@ -256,14 +287,24 @@ const buildContainerImage = async ( {
   const { fullName : builderName } = await getGcpAccount()
   const gcloudBuild = `time gcloud builds submit --project ${artifactProject}`
   const gcloudLogs = `--gcs-log-dir ${formCloudBuildBucketName( artifactProject )}/log`
-  const gcloudTags = `--tag ${imageName}:${imageVersion}`
+  let gcloudTags = `--tag ${imageName}:${imageVersion}`
+  
+  if ( useCloudBuildFile ) {
+    const cloudBuildFile = './docker/cloudbuild.yaml'
+    gcloudTags = `--config=${cloudBuildFile}`
+    const cbst = handlebars.compile( cloudBuildSteps, { noEscape : true } )
+    const cbs = cbst( { artifactProject, artifactRegistry, imageName, imageVersion } )
+
+    fs.writeFileSync( cloudBuildFile, cbs )
+  }
 
   log( chalk.green.bold( `
   *** Submitting Build as ${builderName} ***
       ${gcloudBuild} \\
-           ${gcloudLogs}  \\
-           ${gcloudTags}
+          ${gcloudLogs}  \\
+          ${gcloudTags}
   ` ) )
+    
 
   if ( process.env.DRY_RUN === '1' ) {
     log( '\n***Exiting from DRY_RUN\n' )

package/src/Utils.mjs

@@ -9,7 +9,7 @@ import YAML from 'js-yaml'
 
 let debugEnabled = process.env.BUILD_TOOLS_DEBUG === '1'
 
-export const disableDebug = () => debugEnabled = false
+export const disableDebug = () => { debugEnabled = false }
 
 /* eslint-disable no-console */
 export const err = console.error

package/src/build-tools.mjs

@@ -135,11 +135,10 @@ const checkAndAnnounce = () => {
   checkForLatest()
     .then( ( latestVersion ) => {
       if ( semverLt( thisVersion, latestVersion ) ) {
-        console.log( chalk.white`
-  Update available ${chalk.grey( thisVersion )} \u2b62 ${chalk.green( latestVersion )}
-  Run ${chalk.cyan( 'npm i -g @leverege/build-tools' )} to update
-`
-        )
+        console.log( chalk.white( '\n' +
+          `    Update available ${chalk.yellow( thisVersion )} \u2b62 ${chalk.green( latestVersion )}\n` +
+          `    Run ${chalk.cyan( 'npm i -g @leverege/build-tools' )} to update\n`
+        ) )
         process.exit( 1 )
       } else {
         process.exit( 0 )

package/src/docker-to-registry.mjs

@@ -26,8 +26,22 @@ if ( refreshErr && !process.env.BUILD_TOOLS_DEBUG ) {
   log( refreshErr ) // most likely the token refreshed message
 }
 
+const options = {}
+let imageVersion = null
 // Expected to be invoked like docker-to-registry v1.2.3
-const imageVersion = process.argv[2]
+for ( let n = 2; n < process.argv.length; n++ ) {
+  if ( process.argv[n].startsWith( '-' ) ) {
+    const str = process.argv[n].slice( 1 )
+    if ( str.indexOf( '=' ) > 0 ) {
+      const [key, value] = str.split( '=' )
+      options[key] = value
+    } else {
+      options[str] = true
+    }
+  } else {
+    imageVersion = process.argv[n]
+  }
+}
 if ( !imageVersion ) {
   errorExit( '***Error: docker-to-registry requires an image version' )
 }
@@ -151,12 +165,13 @@ if ( giveGuidance ) {
 }
 
 // do chart checks too
-const chart = helmChart?.yaml['Chart.yaml']
-const values = helmChart?.yaml['values.yaml']
-const expectedChartRegistry = `${artifactRegistry}/images`
+if ( options?.helmChecks !== 'false' ) {
+  const chart = helmChart?.yaml['Chart.yaml']
+  const values = helmChart?.yaml['values.yaml']
+  const expectedChartRegistry = `${artifactRegistry}/images`
 
-if ( expectedChartRegistry !== values.image?.registry ) {
-  log( `
+  if ( expectedChartRegistry !== values.image?.registry ) {
+    log( `
   ${chalk.red.bold( '***Error: mismatched package.json registry and helm/values.yaml' )}
 
   The registry specified in the package.json leverege stanza does not line up
@@ -172,8 +187,8 @@ if ( expectedChartRegistry !== values.image?.registry ) {
       values.yaml  registry => ${chalk.green.bold( expectedChartRegistry )}
   ` )
 
-  if ( !values.image ) {
-    log( `
+    if ( !values.image ) {
+      log( `
   ${chalk.yellow.bold( '***DEPRECATED: legacy helm charts detected' )}
 
   The helm chart structure appears to be based on the pre-ignition helm chart
@@ -181,9 +196,10 @@ if ( expectedChartRegistry !== values.image?.registry ) {
   its job.
 
     ` )
-  }
+    }
 
-  process.exit( 1 )
+    process.exit( 1 )
+  }
 }
 
 // Give the summary and the user a chance to proceed or not
@@ -256,7 +272,7 @@ if ( isNpmWorkspace ) {
   await shellCmd( 'npm install --package-lock-only --workspaces false', { stdio : 'inherit' } )
 }
 
-await docker.buildContainerImage( { artifactProject, artifactRegistry, containerName, imageVersion, isNpmWorkspace } )
+await docker.buildContainerImage( { ...repoDescr, options, imageVersion } )
 
 if ( willGitTag ) {
   const tagDescription = `docker_${imageVersion}`

package/src/firebaseDeploy.mjs

@@ -213,7 +213,6 @@ try {
   const hostingPrefix = CHANNEL ? '' : 'hosting:'
 
   await $( { ac } )`npm run clean && DOTENV_CONFIG_PATH=${path.join( process.cwd(), '.env.temp' )} DEPLOYMENT_TARGET=${TARGET.siteId} npm run build && firebase use ${TARGET.projectId} && firebase ${deployType} --only ${hostingPrefix}${TARGET.siteId}`
-    .pipe( process.stdout )
 } catch ( err ) {
   console.error( err )
 }

package/src/firebaseServe.mjs

@@ -130,8 +130,9 @@ const ac = new AbortController()
 
 EXIT_EVENTS
   .forEach( ( event ) => {
-    process.on( event, () => {
+    process.on( event, ( d ) => {
       if ( !exited ) {
+        console.error( d )
         exited = true
         fs.removeSync( path.join( process.cwd(), '.env.temp' ) )
         ac.abort()
@@ -142,7 +143,7 @@ EXIT_EVENTS
 await fs.writeFile( path.join( process.cwd(), '.env.temp' ), envFileContent )
 
 try {
-  await $( { ac } )`DOTENV_CONFIG_PATH=${path.join( process.cwd(), '.env.temp' )} DEPLOYMENT_TARGET=${TARGET.siteId} npm run serve --colors`.pipe( process.stdout )
+  await $( { ac } )`DOTENV_CONFIG_PATH=${path.join( process.cwd(), '.env.temp' )} DEPLOYMENT_TARGET=${TARGET.siteId} npm run serve --colors`
 } catch ( err ) {
   console.error( err )
 }

package/src/helm-charts/api-server/values-local.yaml

@@ -17,6 +17,17 @@ config:
 # # timescale
 # PG_HOST: "cnpg-db-tsdb-basic-r.cnpg-operands"
 
+# The settings on the left are for the "standard" legacy chart settings which
+# specifiy the old k8s secrets setups and are the defaults values from the
+# api-server helm chart.
+psqlSecurity:                                   # v----- these are for CNPG -----v
+  PSQL_SECRET_NAME: "authz-postgres"            # "cnpg-db-psql-stack-postgres-pw"
+  PSQL_SECRET_KEY: "postgresql-password"        # "password"
+  TSDB_SECRET_NAME: "postgresql-password"       # "cnpg-db-tsdb-basic-postgres-pw"
+  TSDB_SECRET_KEY: "postgresql-password"        # "password"
+  TSDB_DENSE_SECRET_NAME: "postgresql-password" # "cnpg-db-tsdb-dense-postgres-pw"
+  TSDB_DENSE_SECRET_KEY: "postgresql-password"  # "password"
+
 autoscaling:
   minReplicas: 3
   maxReplicas: 12

package/src/helm-charts/cnpg-db-psql-stack/helmup.plugin

@@ -1,3 +1,6 @@
 #!/bin/bash
 
+# ensures the PG_PASSWORD secret exists and correct
+ensureCnpgSecret PG_PASSWORD $SERVICE-postgres-pw
+
 kubectl apply -f cnpg-db-psql-stack/cluster.yaml $K8S_WHAT

package/src/helm-charts/cnpg-db-tsdb-basic/helmup.plugin

@@ -1,3 +1,6 @@
 #!/bin/bash
-#
+
+# ensures the TSDB_PASSWORD secret exists and correct
+ensureCnpgSecret TSDB_PASSWORD $SERVICE-postgres-pw
+
 kubectl apply -f cnpg-db-tsdb-basic/cluster.yaml $K8S_WHAT

package/src/helm-charts/cnpg-db-tsdb-dense/helmup.plugin

@@ -1,3 +1,6 @@
 #!/bin/bash
-#
+
+# ensures the TSDB_DENSE_PASSWORD secret exists and correct
+ensureCnpgSecret TSDB_DENSE_PASSWORD $SERVICE-postgres-pw
+
 kubectl apply -f cnpg-db-tsdb-dense/cluster.yaml $K8S_WHAT

package/src/helm-charts/elasticsearch8/helmup.plugin

@@ -2,7 +2,7 @@
 #
 showInstalling "Elastic Search 8"
 
-[ -z "$ELASTIC_CHART_VERSION" ] && ELASTIC_CHART_VERSION="21.3.10"
+[ -z "$ELASTIC_CHART_VERSION" ] && ELASTIC_CHART_VERSION="21.3.17"
 
 OCI_CHART="oci://registry-1.docker.io/bitnamicharts/elasticsearch"
 

package/src/helm-charts/redis/helmup.plugin

@@ -4,7 +4,7 @@ showInstalling "Redis"
 
 OCI_CHART="oci://registry-1.docker.io/bitnamicharts/redis"
 
-[ -z "$REDIS_CHART_VERSION" ] && REDIS_CHART_VERSION="20.0.5"
+[ -z "$REDIS_CHART_VERSION" ] && REDIS_CHART_VERSION="20.1.3"
 
 helm upgrade --install redis $OCI_CHART       \
              --values  redis/redis-local.yaml \

package/src/helmdn.sh

@@ -38,7 +38,6 @@ PLATFORM=(
 )
 
 SYSTEM=(
-  cert-manager
   elastic
   preemptible-killer
   redis

package/src/helmup.sh

@@ -543,14 +543,15 @@ function initializeCnpgOperand() {
   local IMAGINE_PW_NAME="$CNPG_SERVICE-imagine-pw"
   local POSTGRES_PW_NAME="$CNPG_SERVICE-postgres-pw"
 
-  printf "\nCreating imagine db password for `color g $CNPG_SERVICE`\n"
-  kubectl create secret generic -n $CNPG_NAMESPACE "$IMAGINE_PW_NAME" \
-        --from-literal=username="imagine" \
-        --from-literal=password="$(generatePassword)" &> $DEVNULL
-  warnOnError $? "the `color y $IMAGINE_PW_NAME` secret may already exist"
+# TODO: we use postgres root password for access - that is bad - fix it?
+# printf "\nCreating imagine db password for `color g $CNPG_SERVICE`\n"
+# kubectl create secret generic -n $CNPG_NAMESPACE "$IMAGINE_PW_NAME" \
+#       --from-literal=username="imagine" \
+#       --from-literal=password="$(generatePassword)" &> $DEVNULL
+# warnOnError $? "the `color y $IMAGINE_PW_NAME` secret may already exist"
 
-  # reflect the secret to default namespace
-  reflectSecretIntoNamespaces $IMAGINE_PW_NAME $CNPG_NAMESPACE "default"
+# # reflect the secret to default namespace
+# reflectSecretIntoNamespaces $IMAGINE_PW_NAME $CNPG_NAMESPACE "default"
 
   printf "\nCreating postgres db password for `color g $CNPG_SERVICE`\n"
   kubectl create secret generic -n $CNPG_NAMESPACE "$POSTGRES_PW_NAME" \
@@ -670,6 +671,23 @@ EOSNAPSC
   sleep 2 # hold up processing for a moment to allow IAM mods to propagate
 }
 
+function ensureCnpgSecret() {
+  local mgrSecret=$1
+  local k8sSecret=$2
+
+  service-man --from-helmup --sync-secrets ${mgrSecret}=${k8sSecret}
+
+  if [ $? -ne 0 ];
+  then
+    cat<<BAD_SECRETS
+  `color r "There appears to be a problem with one, or both of the secrets. Please
+  rectify the situation and try again."`
+
+BAD_SECRETS
+    exit 1
+  fi
+}
+
 function installVeleroEnvironment() {
   # Follows instructions => https://github.com/vmware-tanzu/velero-plugin-for-gcp
   showInstalling "Velero Backup Management"

package/src/refresh-py-idx.mjs

@@ -1,124 +0,0 @@
-#!/usr/bin/env node
-/* eslint-disable max-len */
-/* eslint-disable security/detect-non-literal-fs-filename */
-/* eslint-disable no-console */
-
-import { execSync } from 'child_process';
-import fs from 'fs';
-import os from 'os';
-import path from 'path';
-
-// Execute gcloud command to get the token
-const TOKEN = execSync( 'gcloud auth print-access-token' ).toString().trim();
-
-// Write to pip.conf
-const pipConfigDir = path.join( os.homedir(), '.config', 'pip' );
-fs.mkdirSync( pipConfigDir, { recursive : true } );
-const pipConfigContent = `
-[global]
-extra-index-url = https://oauth2accesstoken:${TOKEN}@us-python.pkg.dev/leverege-registry/leverege-python-packages/simple/
-`;
-fs.writeFileSync( path.join( pipConfigDir, 'pip.conf' ), pipConfigContent );
-console.log( 'Updated ~/.config/pip/pip.conf' );
-
-// Write to pypirc
-const pypircContent = `
-[distutils]
-index-servers =
-    leverege-python-packages
-
-[leverege-python-packages]
-repository: https://us-python.pkg.dev/leverege-registry/leverege-python-packages/
-username: oauth2accesstoken
-password: ${TOKEN}
-`;
-fs.writeFileSync( path.join( os.homedir(), '.pypirc' ), pypircContent );
-console.log( 'Updated ~/.pypirc\n' );
-
-// Escape for sed
-
-const UV_EXTRA_INDEX_URL = `https://foo:${TOKEN}@us-python.pkg.dev/leverege-registry/leverege-python-packages/simple`;
-
-const SHELLS = [ 'bash', 'dash', 'zsh', 'fish' ];
-
-SHELLS.forEach( ( shell ) => {
-  try {
-    execSync( `command -v ${shell}` );
-    let configFile;
-    switch ( shell ) {
-      case 'bash':
-        configFile = path.join( os.homedir(), '.bashrc' );
-        break;
-      case 'dash':
-        configFile = path.join( os.homedir(), '.profile' );
-        break;
-      case 'zsh':
-        configFile = path.join( os.homedir(), '.zshrc' );
-        break;
-      case 'fish':
-        configFile = path.join( os.homedir(), '.config', 'fish', 'config.fish' );
-        break;
-      default:
-        console.log( `Shell ${shell} is not supported.` );
-        return;
-    }
-
-    console.log( `Updating shell configuration in ${configFile}` );
-
-    let configContent = fs.readFileSync( configFile, 'utf8' );
-    const regex = /export UV_EXTRA_INDEX_URL=.*/;
-    const newLine = `export UV_EXTRA_INDEX_URL=${UV_EXTRA_INDEX_URL}`;
-
-    if ( regex.test( configContent ) ) {
-      configContent = configContent.replace( regex, newLine );
-    } else {
-      configContent += `\n${newLine}\n`;
-    }
-
-    fs.writeFileSync( configFile, configContent );
-  } catch ( error ) {
-    console.log( `Shell ${shell} is not installed.` );
-  }
-} );
-
-function getParentShell() {
-  let ppid = execSync( 'ps -p $$ -o ppid=' ).toString().trim();
-  let shell = execSync( `ps -p ${ppid} -o comm=` ).toString().trim();
-
-  while ( ppid !== '1' ) {
-    ppid = execSync( `ps -p ${ppid} -o ppid=` ).toString().trim();
-    shell = execSync( `ps -p ${ppid} -o comm=` ).toString().trim();
-
-    const shellMatch = shell.match( /bash|dash|zsh|fish/ );
-    if ( shellMatch ) {
-      return shellMatch[0];
-    }
-  }
-
-  return null
-}
-
-const shell = getParentShell();
-
-if ( !shell ) {
-  console.log( '\nCould not determine parent shell. Please source your shell configuration to apply the changes.\n' );
-  console.log( 'Example (bash): source ~/.bashrc' );
-  console.log( 'Example (dash): source ~/.profile' );
-  console.log( 'Example (zsh): source ~/.zshrc' );
-  console.log( 'Example (fish): source ~/.config/fish/config.fish' );
-  process.exit( 1 );
-}
-
-console.log( `\nSourcing shell configuration for ${shell}` );
-
-switch ( shell ) {
-  case 'bash':
-  case 'dash':
-  case 'zsh':
-  case 'fish':
-    execSync( shell, { stdio : 'inherit' } );
-    console.log( `Sourced shell configuration for ${shell}` );
-    break;
-  default:
-    console.log( `Shell ${shell} is not supported.` );
-}