@leverege/build-tools 2.52.0-beta.2 → 2.52.0-pedro.10

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@leverege/build-tools",
-  "version": "2.52.0-beta.2",
+  "version": "2.52.0-pedro.10",
   "description": "A collection of build / support tools for Leverege developers",
   "main": "index.js",
   "repository": {
@@ -47,6 +47,7 @@
     "pull-git": "src/pull-git.sh",
     "push-my-chart": "src/push-my-chart.mjs",
     "refresh-npm-token": "src/refresh-npm-token.mjs",
+    "refresh-py-idx": "src/refresh-py-idx.mjs",
     "send-to-slack": "src/send-to-slack.mjs",
     "tag-release": "src/tag-release.mjs",
     "unleash": "src/unleash.mjs"
@@ -61,10 +62,10 @@
     "command-line-usage": "^7.0.3",
     "deepmerge": "^4.3.1",
     "enquirer": "^2.4.1",
-    "execa": "^9.4.0",
+    "execa": "^9.3.1",
     "glob": "^11.0.0",
     "handlebars": "^4.7.8",
-    "inquirer": "^11.0.2",
+    "inquirer": "^10.2.2",
     "js-yaml": "^4.1.0",
     "ms": "^2.1.3",
     "npm-registry-fetch": "^17.1.0",
@@ -73,11 +74,11 @@
     "read-pkg": "^9.0.1",
     "readline-sync": "^1.4.10",
     "semver": "^7.6.3",
-    "simple-git": "^3.27.0",
-    "zx": "^8.1.7"
+    "simple-git": "^3.26.0",
+    "zx": "^8.1.6"
   },
   "devDependencies": {
     "@leverege/eslint-config-leverege": "^4.2.0",
     "npm": "^10.8.3"
   }
-}
+}

package/src/Docker.mjs

@@ -40,12 +40,8 @@ COPY ./.npmrc /usr/src/app/.npmrc
 RUN apk --no-cache add openssh-client && \
     apk --update add --no-cache --virtual build-dep g++ gcc libgcc \\
       libstdc++ linux-headers make {{apkadds}} && \
-    npm install -g npm@10
-
-{{preInstallPluginfile}}
-
-# Removed --ignore-scripts from npm ci because it was causing issues with python native build
-RUN npm ci --only=production --no-optional {{npmlogging}} && \
+    npm install -g npm@10 && \
+    npm ci --only=production --ignore-scripts --no-optional {{npmlogging}} && \
     rm -f /usr/src/app/.npmrc /root/.ssh/*
 
 # --------------------------------------------------------------
@@ -71,7 +67,7 @@ COPY --chown=node:node --from=intermediate /usr/src/app /usr/src/app
 
 USER {{runuser}}
 COPY ./bashrc /home/node/.bashrc
-CMD [ "/bin/bash", "-c", "source /home/node/.bashrc && node index.js" ]`
+CMD [ "/bin/bash", "-c", "node index.js" ]`
 
 const pluginTemplate = `
 # Dockerfile.plugin - optionally extend the service Docker image
@@ -136,19 +132,6 @@ dunm() {
   du -sh /usr/src/app/node_modules/* | sort -sh
 }
 `
-const cloudBuildSteps = `
-steps:
-  - name: 'gcr.io/cloud-builders/docker'
-    args:
-      - 'build' 
-      - '--network=cloudbuild'
-      - '-t'
-      - '{{imageName}}:{{imageVersion}}'
-      - '.'
-    
-images:
-  - '{{imageName}}:{{imageVersion}}'
-`
 
 const defaultSettings = {
   apkadds : '',
@@ -156,7 +139,6 @@ const defaultSettings = {
   nodeimage : 'iron-alpine',
   npmlogging : '--silent',
   pluginfile : '# NO PLUGIN',
-  preInstallPluginfile : '# NO PRE-INSTALL PLUGIN',
   regvers : 'package.version',
   runuser : 'node',
 }
@@ -186,14 +168,8 @@ const generateDockerfile = ( settings = defaultSettings ) => {
 
   const bashrcFile = './docker/bashrc'
   if ( !fs.existsSync( bashrcFile ) ) {
-    let bf = bashrcTemplate
-    const bashrcFilePlugin = './docker/Bashrc.plugin'
-    if ( fs.existsSync( bashrcFilePlugin ) ) {
-      const bashrcAdditions = fs.readFileSync( bashrcFilePlugin )
-      bf = `${bf}\n${bashrcAdditions}`
-    }
     warning( `creating the missing ${bashrcFile} file in the docker directory` )
-    fs.writeFileSync( bashrcFile, bf )
+    fs.writeFileSync( bashrcFile, bashrcTemplate )
   }
 
   const dockerfilePlugin = './docker/Dockerfile.plugin'
@@ -218,12 +194,7 @@ const generateDockerfile = ( settings = defaultSettings ) => {
     errorExit( `***ERROR: something went wrong with ${dockerfilePlugin} plugin creation` )
   }
 
-  const dockerfilePrePlugin = './docker/DockerfilePreInstall.plugin'
-  if ( fs.existsSync( dockerfilePrePlugin ) ) {
-    defaultSettings.preInstallPluginfile = fs.readFileSync( dockerfilePrePlugin )
-  }
-
-  const compiled = handlebars.compile( dockerfileTemplate, { noEscape : true } )
+  const compiled = handlebars.compile( dockerfileTemplate )
   const replaced = compiled( settings )
   // temporarily write the Dockerfile to the docker subdir - the container
   // image builder will then move it over to the build subdir
@@ -255,10 +226,8 @@ const validateCloudBuildBucket = async ( artifactProject ) => {
 }
 
 const buildContainerImage = async ( {
-  artifactProject, artifactRegistry, containerName, 
-  imageVersion, isNpmWorkspace, options
+  artifactProject, artifactRegistry, containerName, imageVersion, isNpmWorkspace,
 } ) => {
-  const { useCloudBuildFile } = options || {}
   const buildWorkspace = './build/workspace' // passed to Cloud Build
   const imageName = `${artifactRegistry}/images/${containerName}`
 
@@ -287,24 +256,14 @@ const buildContainerImage = async ( {
   const { fullName : builderName } = await getGcpAccount()
   const gcloudBuild = `time gcloud builds submit --project ${artifactProject}`
   const gcloudLogs = `--gcs-log-dir ${formCloudBuildBucketName( artifactProject )}/log`
-  let gcloudTags = `--tag ${imageName}:${imageVersion}`
-  
-  if ( useCloudBuildFile ) {
-    const cloudBuildFile = './docker/cloudbuild.yaml'
-    gcloudTags = `--config=${cloudBuildFile}`
-    const cbst = handlebars.compile( cloudBuildSteps, { noEscape : true } )
-    const cbs = cbst( { artifactProject, artifactRegistry, imageName, imageVersion } )
-
-    fs.writeFileSync( cloudBuildFile, cbs )
-  }
+  const gcloudTags = `--tag ${imageName}:${imageVersion}`
 
   log( chalk.green.bold( `
-    *** Submitting Build as ${builderName} ***
-    ${gcloudBuild} \\
-    ${gcloudLogs}  \\
-    ${gcloudTags}
-    ` ) )
-    
+  *** Submitting Build as ${builderName} ***
+      ${gcloudBuild} \\
+           ${gcloudLogs}  \\
+           ${gcloudTags}
+  ` ) )
 
   if ( process.env.DRY_RUN === '1' ) {
     log( '\n***Exiting from DRY_RUN\n' )

package/src/Utils.mjs

@@ -9,7 +9,7 @@ import YAML from 'js-yaml'
 
 let debugEnabled = process.env.BUILD_TOOLS_DEBUG === '1'
 
-export const disableDebug = () => { debugEnabled = false }
+export const disableDebug = () => debugEnabled = false
 
 /* eslint-disable no-console */
 export const err = console.error

package/src/build-tools.mjs

@@ -135,10 +135,11 @@ const checkAndAnnounce = () => {
   checkForLatest()
     .then( ( latestVersion ) => {
       if ( semverLt( thisVersion, latestVersion ) ) {
-        console.log( chalk.white( '\n' +
-          `    Update available ${chalk.yellow( thisVersion )} \u2b62 ${chalk.green( latestVersion )}\n` +
-          `    Run ${chalk.cyan( 'npm i -g @leverege/build-tools' )} to update\n`
-        ) )
+        console.log( chalk.white`
+  Update available ${chalk.grey( thisVersion )} \u2b62 ${chalk.green( latestVersion )}
+  Run ${chalk.cyan( 'npm i -g @leverege/build-tools' )} to update
+`
+        )
         process.exit( 1 )
       } else {
         process.exit( 0 )

package/src/docker-to-registry.mjs

@@ -26,22 +26,8 @@ if ( refreshErr && !process.env.BUILD_TOOLS_DEBUG ) {
   log( refreshErr ) // most likely the token refreshed message
 }
 
-const options = {}
-let imageVersion = null
 // Expected to be invoked like docker-to-registry v1.2.3
-for ( let n = 2; n < process.argv.length; n++ ) {
-  if ( process.argv[n].startsWith( '-' ) ) {
-    const str = process.argv[n].slice( 1 )
-    if ( str.indexOf( '=' ) > 0 ) {
-      const [key, value] = str.split( '=' )
-      options[key] = value
-    } else {
-      options[str] = true
-    }
-  } else {
-    imageVersion = process.argv[n]
-  }
-}
+const imageVersion = process.argv[2]
 if ( !imageVersion ) {
   errorExit( '***Error: docker-to-registry requires an image version' )
 }
@@ -165,41 +151,39 @@ if ( giveGuidance ) {
 }
 
 // do chart checks too
-if ( options?.helmChecks !== 'false' ) {
-  const chart = helmChart?.yaml['Chart.yaml']
-  const values = helmChart?.yaml['values.yaml']
-  const expectedChartRegistry = `${artifactRegistry}/images`
-
-  if ( expectedChartRegistry !== values.image?.registry ) {
-    log( `
-    ${chalk.red.bold( '***Error: mismatched package.json registry and helm/values.yaml' )}
+const chart = helmChart?.yaml['Chart.yaml']
+const values = helmChart?.yaml['values.yaml']
+const expectedChartRegistry = `${artifactRegistry}/images`
 
-    The registry specified in the package.json leverege stanza does not line up
-    with the image registry in helm/values.yaml. The naming convention expects
-    the '/images' suffix to be added to the package.json registry string and then
-    stored as the image.registry in the helm/values.yaml file.
+if ( expectedChartRegistry !== values.image?.registry ) {
+  log( `
+  ${chalk.red.bold( '***Error: mismatched package.json registry and helm/values.yaml' )}
 
-      Current settings:
-        package.json registry => ${chalk.yellow.bold( artifactRegistry )}
-        values.yaml  registry => ${chalk.red.bold( values.image?.registry )}
+  The registry specified in the package.json leverege stanza does not line up
+  with the image registry in helm/values.yaml. The naming convention expects
+  the '/images' suffix to be added to the package.json registry string and then
+  stored as the image.registry in the helm/values.yaml file.
 
-      Expected settings:
-        values.yaml  registry => ${chalk.green.bold( expectedChartRegistry )}
-    ` )
+    Current settings:
+      package.json registry => ${chalk.yellow.bold( artifactRegistry )}
+      values.yaml  registry => ${chalk.red.bold( values.image?.registry )}
 
-    if ( !values.image ) {
-      log( `
-    ${chalk.yellow.bold( '***DEPRECATED: legacy helm charts detected' )}
+    Expected settings:
+      values.yaml  registry => ${chalk.green.bold( expectedChartRegistry )}
+  ` )
 
-    The helm chart structure appears to be based on the pre-ignition helm chart
-    layouts. The charts must be upgraded before docker-to-registry can complete
-    its job.
+  if ( !values.image ) {
+    log( `
+  ${chalk.yellow.bold( '***DEPRECATED: legacy helm charts detected' )}
 
-      ` )
-    }
+  The helm chart structure appears to be based on the pre-ignition helm chart
+  layouts. The charts must be upgraded before docker-to-registry can complete
+  its job.
 
-    process.exit( 1 )
+    ` )
   }
+
+  process.exit( 1 )
 }
 
 // Give the summary and the user a chance to proceed or not
@@ -272,7 +256,7 @@ if ( isNpmWorkspace ) {
   await shellCmd( 'npm install --package-lock-only --workspaces false', { stdio : 'inherit' } )
 }
 
-await docker.buildContainerImage( { ...repoDescr, options, imageVersion } )
+await docker.buildContainerImage( { artifactProject, artifactRegistry, containerName, imageVersion, isNpmWorkspace } )
 
 if ( willGitTag ) {
   const tagDescription = `docker_${imageVersion}`

package/src/firebaseDeploy.mjs

@@ -213,6 +213,7 @@ try {
   const hostingPrefix = CHANNEL ? '' : 'hosting:'
 
   await $( { ac } )`npm run clean && DOTENV_CONFIG_PATH=${path.join( process.cwd(), '.env.temp' )} DEPLOYMENT_TARGET=${TARGET.siteId} npm run build && firebase use ${TARGET.projectId} && firebase ${deployType} --only ${hostingPrefix}${TARGET.siteId}`
+    .pipe( process.stdout )
 } catch ( err ) {
   console.error( err )
 }

package/src/firebaseServe.mjs

@@ -130,9 +130,8 @@ const ac = new AbortController()
 
 EXIT_EVENTS
   .forEach( ( event ) => {
-    process.on( event, ( d ) => {
+    process.on( event, () => {
       if ( !exited ) {
-        console.error( d )
         exited = true
         fs.removeSync( path.join( process.cwd(), '.env.temp' ) )
         ac.abort()
@@ -143,7 +142,7 @@ EXIT_EVENTS
 await fs.writeFile( path.join( process.cwd(), '.env.temp' ), envFileContent )
 
 try {
-  await $( { ac } )`DOTENV_CONFIG_PATH=${path.join( process.cwd(), '.env.temp' )} DEPLOYMENT_TARGET=${TARGET.siteId} npm run serve --colors`
+  await $( { ac } )`DOTENV_CONFIG_PATH=${path.join( process.cwd(), '.env.temp' )} DEPLOYMENT_TARGET=${TARGET.siteId} npm run serve --colors`.pipe( process.stdout )
 } catch ( err ) {
   console.error( err )
 }

package/src/helm-charts/api-server/values-local.yaml

@@ -17,17 +17,6 @@ config:
 # # timescale
 # PG_HOST: "cnpg-db-tsdb-basic-r.cnpg-operands"
 
-# The settings on the left are for the "standard" legacy chart settings which
-# specifiy the old k8s secrets setups and are the defaults values from the
-# api-server helm chart.
-psqlSecurity:                                   # v----- these are for CNPG -----v
-  PSQL_SECRET_NAME: "authz-postgres"            # "cnpg-db-psql-stack-postgres-pw"
-  PSQL_SECRET_KEY: "postgresql-password"        # "password"
-  TSDB_SECRET_NAME: "postgresql-password"       # "cnpg-db-tsdb-basic-postgres-pw"
-  TSDB_SECRET_KEY: "postgresql-password"        # "password"
-  TSDB_DENSE_SECRET_NAME: "postgresql-password" # "cnpg-db-tsdb-dense-postgres-pw"
-  TSDB_DENSE_SECRET_KEY: "postgresql-password"  # "password"
-
 autoscaling:
   minReplicas: 3
   maxReplicas: 12

package/src/helm-charts/cnpg-db-psql-stack/helmup.plugin

@@ -1,6 +1,3 @@
 #!/bin/bash
 
-# ensures the PG_PASSWORD secret exists and correct
-ensureCnpgSecret PG_PASSWORD $SERVICE-postgres-pw
-
 kubectl apply -f cnpg-db-psql-stack/cluster.yaml $K8S_WHAT

package/src/helm-charts/cnpg-db-tsdb-basic/helmup.plugin

@@ -1,6 +1,3 @@
 #!/bin/bash
-
-# ensures the TSDB_PASSWORD secret exists and correct
-ensureCnpgSecret TSDB_PASSWORD $SERVICE-postgres-pw
-
+#
 kubectl apply -f cnpg-db-tsdb-basic/cluster.yaml $K8S_WHAT

package/src/helm-charts/cnpg-db-tsdb-dense/helmup.plugin

@@ -1,6 +1,3 @@
 #!/bin/bash
-
-# ensures the TSDB_DENSE_PASSWORD secret exists and correct
-ensureCnpgSecret TSDB_DENSE_PASSWORD $SERVICE-postgres-pw
-
+#
 kubectl apply -f cnpg-db-tsdb-dense/cluster.yaml $K8S_WHAT

package/src/helm-charts/elasticsearch8/helmup.plugin

@@ -2,7 +2,7 @@
 #
 showInstalling "Elastic Search 8"
 
-[ -z "$ELASTIC_CHART_VERSION" ] && ELASTIC_CHART_VERSION="21.3.17"
+[ -z "$ELASTIC_CHART_VERSION" ] && ELASTIC_CHART_VERSION="21.3.10"
 
 OCI_CHART="oci://registry-1.docker.io/bitnamicharts/elasticsearch"
 

package/src/helm-charts/redis/helmup.plugin

@@ -4,7 +4,7 @@ showInstalling "Redis"
 
 OCI_CHART="oci://registry-1.docker.io/bitnamicharts/redis"
 
-[ -z "$REDIS_CHART_VERSION" ] && REDIS_CHART_VERSION="20.1.3"
+[ -z "$REDIS_CHART_VERSION" ] && REDIS_CHART_VERSION="20.0.5"
 
 helm upgrade --install redis $OCI_CHART       \
              --values  redis/redis-local.yaml \

package/src/helmdn.sh

@@ -38,6 +38,7 @@ PLATFORM=(
 )
 
 SYSTEM=(
+  cert-manager
   elastic
   preemptible-killer
   redis

package/src/helmup.sh

@@ -543,15 +543,14 @@ function initializeCnpgOperand() {
   local IMAGINE_PW_NAME="$CNPG_SERVICE-imagine-pw"
   local POSTGRES_PW_NAME="$CNPG_SERVICE-postgres-pw"
 
-# TODO: we use postgres root password for access - that is bad - fix it?
-# printf "\nCreating imagine db password for `color g $CNPG_SERVICE`\n"
-# kubectl create secret generic -n $CNPG_NAMESPACE "$IMAGINE_PW_NAME" \
-#       --from-literal=username="imagine" \
-#       --from-literal=password="$(generatePassword)" &> $DEVNULL
-# warnOnError $? "the `color y $IMAGINE_PW_NAME` secret may already exist"
+  printf "\nCreating imagine db password for `color g $CNPG_SERVICE`\n"
+  kubectl create secret generic -n $CNPG_NAMESPACE "$IMAGINE_PW_NAME" \
+        --from-literal=username="imagine" \
+        --from-literal=password="$(generatePassword)" &> $DEVNULL
+  warnOnError $? "the `color y $IMAGINE_PW_NAME` secret may already exist"
 
-# # reflect the secret to default namespace
-# reflectSecretIntoNamespaces $IMAGINE_PW_NAME $CNPG_NAMESPACE "default"
+  # reflect the secret to default namespace
+  reflectSecretIntoNamespaces $IMAGINE_PW_NAME $CNPG_NAMESPACE "default"
 
   printf "\nCreating postgres db password for `color g $CNPG_SERVICE`\n"
   kubectl create secret generic -n $CNPG_NAMESPACE "$POSTGRES_PW_NAME" \
@@ -671,23 +670,6 @@ EOSNAPSC
   sleep 2 # hold up processing for a moment to allow IAM mods to propagate
 }
 
-function ensureCnpgSecret() {
-  local mgrSecret=$1
-  local k8sSecret=$2
-
-  service-man --from-helmup --sync-secrets ${mgrSecret}=${k8sSecret}
-
-  if [ $? -ne 0 ];
-  then
-    cat<<BAD_SECRETS
-  `color r "There appears to be a problem with one, or both of the secrets. Please
-  rectify the situation and try again."`
-
-BAD_SECRETS
-    exit 1
-  fi
-}
-
 function installVeleroEnvironment() {
   # Follows instructions => https://github.com/vmware-tanzu/velero-plugin-for-gcp
   showInstalling "Velero Backup Management"

package/src/refresh-py-idx.mjs

@@ -0,0 +1,123 @@
+#!/usr/bin/env node
+/* eslint-disable max-len */
+/* eslint-disable security/detect-non-literal-fs-filename */
+/* eslint-disable no-console */
+
+import { execSync } from 'child_process';
+import fs from 'fs';
+import os from 'os';
+import path from 'path';
+
+// Execute gcloud command to get the token
+const TOKEN = execSync( 'gcloud auth print-access-token' ).toString().trim();
+
+// Write to pip.conf
+const pipConfigDir = path.join( os.homedir(), '.config', 'pip' );
+fs.mkdirSync( pipConfigDir, { recursive : true } );
+const pipConfigContent = `
+[global]
+extra-index-url = https://oauth2accesstoken:${TOKEN}@us-python.pkg.dev/leverege-registry/leverege-python-packages/simple/
+`;
+fs.writeFileSync( path.join( pipConfigDir, 'pip.conf' ), pipConfigContent );
+console.log( 'Updated ~/.config/pip/pip.conf' );
+
+// Write to pypirc
+const pypircContent = `
+[distutils]
+index-servers =
+    leverege-python-packages
+
+[leverege-python-packages]
+repository: https://us-python.pkg.dev/leverege-registry/leverege-python-packages/
+username: oauth2accesstoken
+password: ${TOKEN}
+`;
+fs.writeFileSync( path.join( os.homedir(), '.pypirc' ), pypircContent );
+console.log( 'Updated ~/.pypirc\n' );
+
+// Escape for sed
+
+const UV_EXTRA_INDEX_URL = `https://foo:${TOKEN}@us-python.pkg.dev/leverege-registry/leverege-python-packages/simple`;
+
+const SHELLS = [ 'bash', 'dash', 'zsh', 'fish' ];
+
+SHELLS.forEach( ( shell ) => {
+  try {
+    execSync( `command -v ${shell}` );
+    let configFile;
+    switch ( shell ) {
+      case 'bash':
+        configFile = path.join( os.homedir(), '.bashrc' );
+        break;
+      case 'dash':
+        configFile = path.join( os.homedir(), '.profile' );
+        break;
+      case 'zsh':
+        configFile = path.join( os.homedir(), '.zshrc' );
+        break;
+      case 'fish':
+        configFile = path.join( os.homedir(), '.config', 'fish', 'config.fish' );
+        break;
+      default:
+        console.log( `Shell ${shell} is not supported.` );
+        return;
+    }
+
+    console.log( `Updating shell configuration in ${configFile}` );
+
+    let configContent = fs.readFileSync( configFile, 'utf8' );
+    const regex = /export UV_EXTRA_INDEX_URL=.*/;
+    const newLine = `export UV_EXTRA_INDEX_URL=${UV_EXTRA_INDEX_URL}`;
+
+    if ( regex.test( configContent ) ) {
+      configContent = configContent.replace( regex, newLine );
+    } else {
+      configContent += `\n${newLine}\n`;
+    }
+
+    fs.writeFileSync( configFile, configContent );
+  } catch ( error ) {
+    console.log( `Shell ${shell} is not installed.` );
+  }
+} );
+
+function getParentShell() {
+  let ppid = execSync( 'ps -p $$ -o ppid=' ).toString().trim();
+  let shell = execSync( `ps -p ${ppid} -o comm=` ).toString().trim();
+
+  while ( ppid !== '1' ) {
+    ppid = execSync( `ps -p ${ppid} -o ppid=` ).toString().trim();
+    shell = execSync( `ps -p ${ppid} -o comm=` ).toString().trim();
+
+    const shellMatch = shell.match( /bash|dash|zsh|fish/ );
+    if ( shellMatch ) {
+      return shellMatch[0];
+    }
+  }
+
+  return null
+}
+
+const shell = getParentShell();
+
+if ( !shell ) {
+  console.log( '\nCould not determine parent shell. Please source your shell configuration to apply the changes.\n' );
+  console.log( 'Example (bash): source ~/.bashrc' );
+  console.log( 'Example (dash): source ~/.profile' );
+  console.log( 'Example (zsh): source ~/.zshrc' );
+  console.log( 'Example (fish): source ~/.config/fish/config.fish' );
+  process.exit( 1 );
+}
+
+console.log( `\nReloading ${shell} shell with new configuration.\n` );
+
+switch ( shell ) {
+  case 'bash':
+  case 'dash':
+  case 'zsh':
+  case 'fish':
+    execSync( shell, { stdio : 'inherit' } );
+    break;
+  default:
+    console.log( `Shell ${shell} is not supported.\n` );
+}