@leverege/build-tools 2.66.7 → 2.67.0

package/.vscode/settings.json

@@ -0,0 +1,5 @@
+{
+  "files.associations": {
+    "*.yaml.ovh": "yaml"
+  }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@leverege/build-tools",
-  "version": "2.66.7",
+  "version": "2.67.0",
   "description": "A collection of build / support tools for Leverege developers",
   "main": "index.js",
   "repository": {
@@ -66,14 +66,14 @@
     "chalk": "^5.4.1",
     "command-line-args": "^6.0.1",
     "command-line-usage": "^7.0.3",
-    "commander": "^13.1.0",
+    "commander": "^14.0.0",
     "deepmerge": "^4.3.1",
     "enquirer": "^2.4.1",
-    "execa": "^9.5.3",
+    "execa": "^9.6.0",
     "glob": "^11.0.2",
     "handlebars": "^4.7.8",
-    "ignore": "^7.0.4",
-    "inquirer": "^12.6.0",
+    "ignore": "^7.0.5",
+    "inquirer": "^12.6.3",
     "js-yaml": "^4.1.0",
     "jsdoc": "^4.0.4",
     "ms": "^2.1.3",
@@ -82,14 +82,14 @@
     "p-limit": "^6.2.0",
     "package-up": "^5.0.0",
     "readline-sync": "^1.4.10",
-    "semver": "^7.7.1",
-    "shell-quote": "^1.8.2",
-    "simple-git": "^3.27.0",
+    "semver": "^7.7.2",
+    "shell-quote": "^1.8.3",
+    "simple-git": "^3.28.0",
     "toml": "^3.0.0",
-    "zx": "^8.5.4"
+    "zx": "^8.5.5"
   },
   "devDependencies": {
     "@leverege/eslint-config-leverege": "^5.0.1",
-    "npm": "^11.3.0"
+    "npm": "^11.4.1"
   }
 }

package/src/Docker.mjs

@@ -177,7 +177,6 @@ const buildContainerImage = async ( {
   artifactProject, artifactRegistry, containerName, 
   imageVersion, isNpmWorkspace, options
 } ) => {
-  const { useCloudBuildFile } = options || {}
   const buildWorkspace = './build/workspace' // passed to Cloud Build
   const imageName = `${artifactRegistry}/images/${containerName}`
 
@@ -213,41 +212,37 @@ const buildContainerImage = async ( {
 
   setPreviousBuild( imageVersion ) // update docker/.previous file
 
-  const { fullName : builderName } = await getGcpAccount()
-  const gcloudBuild = `gcloud builds submit --project ${artifactProject}`
+  const { fullName : imageBuilder } = await getGcpAccount()
+  const imageInfoTag = `zzinfo_${defaultSettings.date}_${defaultSettings.nodeimage}_${imageBuilder}`
   const gcloudProjectNumber = await shellCmd( `gcloud projects describe ${artifactProject} --format="value(projectNumber)"` )
+  
+  // build the cloudbuild.yaml from the handlebars template
+  const cloudBuildFile = './build/cloudbuild.yaml'
+  const cbst = handlebars.compile( cloudBuildSteps, { noEscape : true } )
+  const cbs = cbst( { artifactProject, artifactRegistry, imageName, imageVersion, imageInfoTag, } )
+  fs.writeFileSync( cloudBuildFile, cbs )
+
+  // build up the cloud build command and options
+  const cloudBuildCmd = `gcloud builds submit --project ${artifactProject}`
+  const cloudBuildLogs = `--gcs-log-dir ${formCloudBuildBucketName( artifactProject )}/log`
+  const cloudBuildCfg = `--config=${cloudBuildFile}`
   const cloudBuildSA = `--service-account=${gcloudProjectNumber}-compute@cloudbuild.gserviceaccount.com`
-  const gcloudLogs = `--gcs-log-dir ${formCloudBuildBucketName( artifactProject )}/log`
-  let gcloudTags = `--tag ${imageName}:${imageVersion}`
   
-  if ( useCloudBuildFile ) {
-    const cloudBuildFile = './docker/cloudbuild.yaml'
-    gcloudTags = `--config=${cloudBuildFile}`
-    const cbst = handlebars.compile( cloudBuildSteps, { noEscape : true } )
-    const cbs = cbst( { artifactProject, artifactRegistry, imageName, imageVersion } )
-
-    fs.writeFileSync( cloudBuildFile, cbs )
-  }
-
   log( chalk.green.bold( `
-  *** Submitting Build as ${builderName} ***
-      ${gcloudBuild} \\
-           ${gcloudLogs}  \\
-           ${gcloudTags} \\
+  *** Submitting Build as ${imageBuilder} ***
+      ${cloudBuildCmd} \\
+           ${cloudBuildLogs}  \\
+           ${cloudBuildCfg} \\
            ${cloudBuildSA}
   ` ) )
 
   if ( process.env.DRY_RUN === '1' ) {
+    log( `Would have run => ${cloudBuildCmd} ${cloudBuildLogs} ${cloudBuildCfg} ./build` )
     log( '\n***Exiting from DRY_RUN\n' )
     process.exit( 0 )
   }
 
-  await shellCmd( `${gcloudBuild} ${gcloudLogs} ${gcloudTags} ./build`, { stdio : 'inherit' } )
-
-  const imageTag = `${defaultSettings.date}-${defaultSettings.nodeimage}-${builderName}`
-  await shellCmd(
-    `gcloud container images add-tag --quiet ${imageName}:${imageVersion} ${imageName}:${imageTag} ${imageName}:latest`
-  )
+  await shellCmd( `${cloudBuildCmd} ${cloudBuildLogs} ${cloudBuildCfg} ./build`, { stdio : 'inherit' } )
 }
 
 const repoCleanup = async () => {

package/src/DockerPy.mjs

@@ -43,10 +43,8 @@ const bashrcPyTemplate = fs.readFileSync( path.join( DIRNAME, './templates/bashr
 bashrcTemplate = `${bashrcTemplate}\n${bashrcPyTemplate}`
 
 const defaultSettings = {
-  apkadds : '',
   date : getDateTimestamp(),
-  nodeimage : 'jod-alpine',
-  imageBase : 'node:jod-alpine',
+  imageBase : 'python:3.12.8-slim-bookworm',
   npmlogging : process.env.VERBOSE_NPM_LOGGING ? '--ddd' : '--silent',
   npmVersion : 'npm@11',
   pluginfile : '# NO PLUGIN',
@@ -55,6 +53,7 @@ const defaultSettings = {
   runuser : 'python',
 }
 
+// the previous build version is simply informational for the user
 const previousFile = './docker/.previous'
 
 function getPreviousBuild() {
@@ -65,16 +64,26 @@ const setPreviousBuild = ( version ) => {
   fs.writeFileSync( previousFile, version )
 }
 
+// the base image string to be in the form of an image name and a version
+// for example:
+//
+//   python:3.12.8-slim-bookworm becomes python-3.12.8
+//   nvcr.io/nvidia/cuda:12.3.1-base-ubuntu22.04 becomes cuda-12.3.1
+//
+// which will then be used to tag the image for information purposes only
+const extractBaseImageTag = ( imageStr ) => {
+  const match = imageStr.match( /(\w+:\d+(?:\.\d+)*)(?=-)/ ) // eslint-disable-line security/detect-unsafe-regex
+  return match ? match[1].replace( ':', '-' ) : null
+}
+
 function generateDockerfile( options ) {
 
   const settings = { ...defaultSettings, ...options }
 
-  if ( !settings.imageBase ) {
-    warning( 'using the default image python:3.12.8-slim-bookworm' )
-    settings.image = 'python:3.12.8-slim-bookworm'
-  } else {
-    settings.image = settings.imageBase
-  }
+  settings.image = settings.imageBase
+  // TODO: this is a janky way to pass the image tag back out
+  defaultSettings.buildTag = extractBaseImageTag( settings.image )
+
   // Ensure the presence of the docker dir with plugin and bashrc - this code
   // could probably be tightened up a little.
   try {
@@ -155,18 +164,13 @@ async function buildContainerImage( {
   artifactProject, artifactRegistry, containerName, 
   imageVersion, isNpmWorkspace, options
 } ) {
-
-  const useCloudBuildFile = true
   const buildWorkspace = './build/workspace' // passed to Cloud Build
   const imageName = `${artifactRegistry}/images/${containerName}`
 
+  // clean up any residual build cruft (should be clean though)
   if ( fs.existsSync( buildWorkspace ) ) {
     fs.renameSync( buildWorkspace, `${buildWorkspace}-junk` )
-    try {
-      fs.rmSync( `${buildWorkspace}-junk`, { recursive : true } )
-    } catch ( err ) {
-      log( `***Error: from Docker.buildContainerImage\n\n${chalk.bold.yellow( err )}` )
-    }
+    fs.rm( `${buildWorkspace}-junk`, { recursive : true }, ( err ) => {} ) // fire and forget
   }
 
   // TODO: This is janky - I'll clean it up later
@@ -176,48 +180,45 @@ async function buildContainerImage( {
   fs.mkdirSync( './build' )
   fs.renameSync( './workspace', buildWorkspace )
   fs.renameSync( './docker/Dockerfile', './build/Dockerfile' )
+  fs.renameSync( './docker/bashrc', './build/bashrc' )
   // fs.cpSync( `${process.env.HOME}/.npmrc`, './build/.npmrc' )
-  fs.cpSync( './docker/bashrc', './build/bashrc' )
   // fs.cpSync( './package.json', `${buildWorkspace}/package.json` )
   // fs.cpSync( './package-lock.json', `${buildWorkspace}/package-lock.json` )
 
   setPreviousBuild( imageVersion ) // update docker/.previous file
 
-  const { fullName : builderName } = await getGcpAccount()
-  const gcloudBuild = `gcloud builds submit --project ${artifactProject}`
+  const { fullName : imageBuilder } = await getGcpAccount()
+  const imageArch = defaultSettings.buildTag
+  const imageInfoTag = `zzinfo_${imageArch}_${imageBuilder}_${defaultSettings.date}`
   const gcloudProjectNumber = await shellCmd( `gcloud projects describe ${artifactProject} --format="value(projectNumber)"` )
+
+  // build the cloudbuild.yaml from the handlebars template
+  const cloudBuildFile = './build/cloudbuild.yaml'
+  const cbst = handlebars.compile( cloudBuildSteps, { noEscape : true } )
+  const cbs = cbst( { artifactProject, artifactRegistry, imageName, imageVersion, imageInfoTag, } )
+  fs.writeFileSync( cloudBuildFile, cbs )
+
+  // build up the cloud build command and options
+  const cloudBuildCmd = `gcloud builds submit --project ${artifactProject}`
+  const cloudBuildLogs = `--gcs-log-dir ${formCloudBuildBucketName( artifactProject )}/log`
+  const cloudBuildCfg = `--config=${cloudBuildFile}`
   const cloudBuildSA = `--service-account=${gcloudProjectNumber}-compute@cloudbuild.gserviceaccount.com`
-  const gcloudLogs = `--gcs-log-dir ${formCloudBuildBucketName( artifactProject )}/log`
-  let gcloudTags = `--tag ${imageName}:${imageVersion}`
-  
-  if ( useCloudBuildFile ) {
-    const cloudBuildFile = './docker/cloudbuild.yaml'
-    gcloudTags = `--config=${cloudBuildFile}`
-    const cbst = handlebars.compile( cloudBuildSteps, { noEscape : true } )
-    const cbs = cbst( { artifactProject, artifactRegistry, imageName, imageVersion } )
-
-    fs.writeFileSync( cloudBuildFile, cbs )
-  }
 
   log( chalk.green.bold( `
-  *** Submitting Build as ${builderName} ***
-      ${gcloudBuild} \\
-           ${gcloudLogs}  \\
-           ${gcloudTags} \\
+  *** Submitting Build as ${imageBuilder} ***
+      ${cloudBuildCmd} \\
+           ${cloudBuildLogs}  \\
+           ${cloudBuildCfg} \\
            ${cloudBuildSA}
   ` ) )
 
   if ( process.env.DRY_RUN === '1' ) {
+    log( `Would have run => ${cloudBuildCmd} ${cloudBuildLogs} ${cloudBuildCfg} ./build` )
     log( '\n***Exiting from DRY_RUN\n' )
     process.exit( 0 )
   }
 
-  await shellCmd( `${gcloudBuild} ${gcloudLogs} ${gcloudTags} ./build`, { stdio : 'inherit' } )
-
-  const imageTag = `${defaultSettings.date}-${defaultSettings.nodeimage}-${builderName}`
-  await shellCmd(
-    `gcloud container images add-tag --quiet ${imageName}:${imageVersion} ${imageName}:${imageTag} ${imageName}:latest`
-  )
+  await shellCmd( `${cloudBuildCmd} ${cloudBuildLogs} ${cloudBuildCfg} ./build`, { stdio : 'inherit' } )
 }
 
 async function repoCleanup() {

package/src/Utils.mjs

@@ -9,6 +9,7 @@ import inquirer from 'inquirer'
 import { glob } from 'glob'
 import { packageUp } from 'package-up'
 import { parse } from 'shell-quote'
+import toml from 'toml'
 import YAML from 'js-yaml'
 
 // __dirname goes away with ESM so we take this approach instead and
@@ -127,12 +128,6 @@ export const getDirectories = ( { path : srcPath = process.cwd(), respectGitigno
     .map( dirent => dirent.name )
 }
 
-export const xgetDirectories = ( srcPath = process.cwd() ) => {
-  return fs.readdirSync( srcPath, { withFileTypes : true } ) // eslint-disable-line security/detect-non-literal-fs-filename
-    .filter( dirent => dirent.isDirectory() )
-    .map( dirent => dirent.name )
-}
-
 // returns YYYMMDD-hhmm
 export const getDateTimestamp = () => {
   const fullDTS = new Date().toISOString().replace( /[-T:]/g, '' ).slice( 0, 14 )
@@ -170,7 +165,8 @@ export const getGcpSecret = async ( gcpProject, secretName, opts ) => {
   }
 }
 
-const runGitCommand = async ( cmd, debugLabel ) => {
+// just a wrapper to run command with try/catch
+const tryCommand = async ( cmd, debugLabel ) => {
   try {
     const result = await shellCmd( cmd )
     debug( { result }, `<==${debugLabel}` )
@@ -181,16 +177,16 @@ const runGitCommand = async ( cmd, debugLabel ) => {
   }
 }
 
-export const getGitBranch = async () => runGitCommand( 'git rev-parse --abbrev-ref HEAD', 'getGitCurrentBranch' )
-export const getGitDiff = async opts => runGitCommand( `git diff ${opts}`, 'getGitDiff' )
-export const getGitHooksPath = async () => runGitCommand( 'git config --get core.hooksPath', 'getGitHooksPath' )
-export const setGitHooksPath = async hookPath => runGitCommand( `git config core.hooksPath ${hookPath}`, 'setGitHooksPath' )
-export const getGitRootDirectory = async () => runGitCommand( 'git rev-parse --show-toplevel', 'getGitRootDirectory' )
+export const getGitBranch = async () => tryCommand( 'git rev-parse --abbrev-ref HEAD', 'getGitCurrentBranch' )
+export const getGitDiff = async opts => tryCommand( `git diff ${opts}`, 'getGitDiff' )
+export const getGitHooksPath = async () => tryCommand( 'git config --get core.hooksPath', 'getGitHooksPath' )
+export const setGitHooksPath = async hookPath => tryCommand( `git config core.hooksPath ${hookPath}`, 'setGitHooksPath' )
+export const getGitRootDirectory = async () => tryCommand( 'git rev-parse --show-toplevel', 'getGitRootDirectory' )
 export const getGitBranchAndUpstream = async () => {
   const branch = await getGitBranch() ?? 'detached'
   let upstream = 'no-upstream'
   try {
-    upstream = await runGitCommand( `git rev-parse --abbrev-ref ${branch}@{u}`, 'getGitUpstream' ) ?? 'no-upstream'
+    upstream = await tryCommand( `git rev-parse --abbrev-ref ${branch}@{u}`, 'getGitUpstream' ) ?? 'no-upstream'
   } catch ( error ) {}
   return { branch, upstream }
 }
@@ -201,6 +197,8 @@ export const gitRepoIsDirty = async () => {
 }
 
 export const parseJsonFile = async ( jsonFile ) => {
+  // TODO: rework this file exists logic / throw
+  if ( !fs.existsSync( jsonFile ) ) { return undefined } // eslint-disable-line security/detect-non-literal-fs-filename
   /* eslint-disable security/detect-non-literal-fs-filename */
   if ( fs.existsSync( jsonFile ) ) {
     try {
@@ -222,7 +220,9 @@ const deprecated = ( opts ) => {
   process.exit( 1 )
 }
 
-export const parsePackageJson = async ( packageFileName ) => {
+export const parsePackageJson = async ( packageFileName = './package.json' ) => {
+  if ( !fs.existsSync( packageFileName ) ) { return undefined } // eslint-disable-line security/detect-non-literal-fs-filename
+  
   const packageJson = await parseJsonFile( packageFileName )
 
   // deprecation checks are here to ease the pain of upgrading older repos
@@ -411,6 +411,13 @@ https://console.cloud.google.com/artifacts/browse/leverege-registry?project=leve
   return { artifactRegistry, ...packageJson }
 }
 
+export const parsePyProjectToml = async ( packageFileName = './pyproject.toml' ) => {
+  if ( !fs.existsSync( packageFileName ) ) { return undefined } // eslint-disable-line security/detect-non-literal-fs-filename
+  const tomlStr = fs.readFileSync( packageFileName ) // eslint-disable-line security/detect-non-literal-fs-filename
+  const pyproject = toml.parse( tomlStr )
+  return pyproject
+}
+
 export const parseHelmChart = async ( helmroot = './helm' ) => {
   /* eslint-disable security/detect-non-literal-fs-filename */
   if ( !fs.existsSync( helmroot ) ) { return undefined }
@@ -453,12 +460,8 @@ export const getRepositoryDescr = async () => {
   const subPkgs = await glob( `${gitRoot}/packages/**/package.json`, { ignore : '/**/node_modules/**' } )
   const helmChart = await parseHelmChart()
   const isMonoRepo = subPkgs?.length > 0
-  let rootPackageJson
-  try {
-    rootPackageJson = await parseJsonFile( `${gitRoot}/package.json` )
-  } catch ( error ) {
-    rootPackageJson = { isInvalid : true, error : 'invalid or non-existent root package.json file' }
-  }
+  const rootPackageJson = await parseJsonFile( `${gitRoot}/package.json` )
+  const rootPyProjectToml = await parsePyProjectToml( `${gitRoot}/pyproject.toml` )
   const isNpmWorkspace = rootPackageJson?.workspaces?.length > 0
 
   return {
@@ -473,6 +476,7 @@ export const getRepositoryDescr = async () => {
     isMonoRepo,
     isNpmWorkspace,
     rootPackageJson,
+    rootPyProjectToml,
     subPkgs,
   }
 }
@@ -484,6 +488,7 @@ export const analyzeRepository = async ( giveGuidance ) => {
     helmChart,
     isMonoRepo,
     rootPackageJson,
+    rootPyProjectToml,
     isNpmWorkspace,
   } = await getRepositoryDescr()
 
@@ -527,15 +532,12 @@ export const analyzeRepository = async ( giveGuidance ) => {
     await shellCmd( 'npm run build' )
   }
 
-  const buildToolsVersion = closestPackageJson.devDependencies['@leverege/build-tools'] || closestPackageJson.dependencies['@leverege/build-tools']
-  if ( !buildToolsVersion ) {
-    errorExit( 'Error: missing devDependency for @leverege/build-tools in package.json' )
-  }
+  const buildToolsVersion = closestPackageJson?.devDependencies['@leverege/build-tools'] || closestPackageJson?.dependencies['@leverege/build-tools']
 
-  const containerName = closestPackageJson.name.replace( /^@[^/]+\//, '' )
-  const artifactProject = closestPackageJson.artifactRegistry.project
-  const artifactRegistry = closestPackageJson.leverege.registry
-  const packageVersion = `v${closestPackageJson.version}`
+  const containerName = closestPackageJson?.name.replace( /^@[^/]+\//, '' ) || rootPyProjectToml?.project.name
+  const artifactProject = closestPackageJson?.artifactRegistry.project || 'unused'
+  const artifactRegistry = closestPackageJson?.leverege.registry || rootPyProjectToml?.registry
+  const packageVersion = `v${closestPackageJson?.version}` || rootPyProjectToml?.project.version
 
   const currentDir = process.cwd()
   const closestPkg = await packageUp()
@@ -567,6 +569,7 @@ export const analyzeRepository = async ( giveGuidance ) => {
     isMonoRepo,
     isNpmWorkspace,
     rootPackageJson,
+    rootPyProjectToml,
     closestPackageJson,
     packageVersion,
     subPkgs,

package/src/bash-funcs

@@ -187,7 +187,6 @@ function ifPluggedIn() {
     *)
       cat<<PLUGIN_INVOKED_AS_ERROR
 
-
   $RED_ERROR bash-funcs ifPluggedIn: unknown INVOKED_AS=[`color r $INVOKED_AS`]
 
 PLUGIN_INVOKED_AS_ERROR

package/src/docker-to-registry-py.mjs

@@ -219,13 +219,9 @@ log( `
 
   ${chalk.green.bold( 'Docker Information:' )}
     NodeImage:     ${chalk.green.bold( dockerInfo.imageBase || `node:${dockerInfo.nodeimage}` )}
-    Docker Env:    ${chalk.green.bold( dockerInfo.buildEnv || 'alpine' )}
-    AddedPkgs:     ${chalk.green.bold( dockerInfo.apkadds )}
     Run User:      ${chalk.green.bold( dockerInfo.runuser )}
     Previous:      ${chalk.yellow.bold( dockerInfo.previousBuild )}
     DateStamp:     ${chalk.green.bold( dockerInfo.date )}
-    NPM Logs:      ${chalk.green.bold( dockerInfo.npmlogging )}
-    NPM Version:   ${chalk.green.bold( dockerInfo.npmVersion )}
 ` )
 
 if ( imageVersion !== packageVersion ) {

package/src/helm-charts/cnpg-operator/helmup.plugin

@@ -2,26 +2,16 @@
 #
 # See => https://cloudnative-pg.io/documentation/current/installation_upgrade/
 #
-OPVER="1.25.1"
+OPVER="1.26.0" # 05/23/2025
 
 createNamespaceIfNeeded cnpg-system
 
 kubectl apply --server-side -f \
     https://raw.githubusercontent.com/cloudnative-pg/cloudnative-pg/release-${OPVER%.*}/releases/cnpg-${OPVER}.yaml $K8S_WHAT
 
-# Legacy terraformed clusters (like sandbox) may need a special firewall rule
-# added to the network layer on k8s. It should look like this:
+# NOTE: installing release candidates may reference something that looks like:
 #
-# Name   : cnpg-operator
-# Ports  : 8000,9443     <= kubectl cnpg status and webhooks
-# Filters: 172.16.0.0/28 <= k8s control plane
+# OPVER="1.26.0-rc1" # 03/28/2025
 #
-# Old approach used the helm chart.
-#
-#addHelmRepo cnpg https://cloudnative-pg.github.io/charts
-#
-#helm upgrade --install cnpg cnpg/cloudnative-pg \
-#  --namespace cnpg-system --create-namespace \
-#  --set webhook.port="10250" $HELM_WHAT
-#
-#removeHelmRepo cnpg
+# kubectl apply --server-side -f \
+#   https://raw.githubusercontent.com/cloudnative-pg/cloudnative-pg/main/releases/cnpg-${OPVER}.yaml $K8S_WHAT

package/src/helm-charts/prom-operator/helmup.plugin

@@ -6,7 +6,7 @@ addHelmRepo prometheus-community https://prometheus-community.github.io/helm-cha
 NS="--namespace prometheus"
 
 showInstalling "The Prometheus Operator (kube-prometheus-stack)"
-[ -z "$PROMETHEUS_STACK_CHART_VERSION" ] && PROMETHEUS_STACK_CHART_VERSION="72"
+[ -z "$PROMETHEUS_STACK_CHART_VERSION" ] && PROMETHEUS_STACK_CHART_VERSION="73"
 
 # https://artifacthub.io/packages/helm/prometheus-community/kube-prometheus-stack
 helm upgrade $NS --install prometheus-stack prometheus-community/kube-prometheus-stack \
@@ -34,4 +34,8 @@ helm upgrade $NS --install prometheus-adapter prometheus-community/prometheus-ad
                  --values prom-operator/prometheus-adapter.yaml \
                  --version $PROMETHEUS_ADAPTER_CHART_VERSION $HELM_WHAT
 
+# finally, apply the PrometheusRule to generate the GPU duty cycle metric
+showInstalling "The Prometheus Rule (custom GPU metrics rule)"
+kubectl apply -n prometheus -f prom-operator/stackdriver-exporter-gpu-rules.yaml
+
 removeHelmRepo prometheus-community

package/src/helm-charts/prom-operator/prometheus-stack.yaml.ovh

@@ -39,6 +39,11 @@ prometheus:
     ruleSelectorNilUsesHelmValues: false
     serviceMonitorSelectorNilUsesHelmValues: false
     probeSelectorNilUsesHelmValues: false
+    maximumStartupDurationSeconds: 60
+    resources:
+      requests:
+        cpu: 500m
+        memory: 2Gi
     storageSpec:
       volumeClaimTemplate:
         spec:
@@ -191,6 +196,7 @@ alertmanager:
                 *Description:* {{ .Annotations.description }}
                 *Dashboard:* {{ .Annotations.dashboard }}
               {{ end }}
+      - name: discard
 
     route:
       group_wait: 30s
@@ -198,12 +204,19 @@ alertmanager:
       group_by:
         - alertname
         - namespace
-      receiver: slack
+      receiver: slack # default for everything
       routes:
-      - match_re:
-          alertname: "Watchdog"
-        receiver: slack-watchdog
-      - match:
-          severity: page
-        receiver: opsgenie
+        - matchers:
+            - severity="page" # 📢 paging DevOps
+          receiver: opsgenie
+          continue: true # 👈 KEEP GOING after this match
+
+        - matchers:
+            - namespace="kube-system" # 🙈 ignore these alerts - GKE will handle them
+          receiver: discard
+
+        - matchers:
+            - alertname="Watchdog"
+          receiver: slack-watchdog
+
       repeat_interval: 6h

package/src/helm-charts/prom-operator/rules/cnpg-rules.yaml

@@ -84,7 +84,7 @@ spec:
 #
     - alert: PostgresqlDown
       expr: cnpg_collector_up == 0
-      for: 0m
+      for: 1m
       labels:
         severity: critical
       annotations:

package/src/helm-charts/prom-operator/rules/elasticsearch-rules.yaml

@@ -10,7 +10,7 @@ spec:
       rules:
         - alert: ElasticsearchHeapUsageTooHigh
           expr: (elasticsearch_jvm_memory_used_bytes{area="heap"} / elasticsearch_jvm_memory_max_bytes{area="heap"}) * 100 > 90
-          for: 2m
+          for: 5m # awesome was => 2m
           labels:
             severity: critical
           annotations:
@@ -21,7 +21,7 @@ spec:
                 LABELS = {{ $labels }}
         - alert: ElasticsearchHeapUsageWarning
           expr: (elasticsearch_jvm_memory_used_bytes{area="heap"} / elasticsearch_jvm_memory_max_bytes{area="heap"}) * 100 > 80
-          for: 2m
+          for: 5m # awesome was => 2m
           labels:
             severity: warning
           annotations:
@@ -43,7 +43,7 @@ spec:
                 LABELS = {{ $labels }}
         - alert: ElasticsearchDiskSpaceLow
           expr: elasticsearch_filesystem_data_available_bytes / elasticsearch_filesystem_data_size_bytes * 100 < 20
-          for: 2m
+          for: 5m # awesome was => 2m
           labels:
             severity: warning
           annotations:
@@ -54,7 +54,7 @@ spec:
                 LABELS = {{ $labels }}
         - alert: ElasticsearchClusterRed
           expr: elasticsearch_cluster_health_status{color="red"} == 1
-          for: 0m
+          for: 5m # awesome was => 0m
           labels:
             severity: critical
           annotations:
@@ -65,7 +65,7 @@ spec:
                 LABELS = {{ $labels }}
         - alert: ElasticsearchClusterYellow
           expr: elasticsearch_cluster_health_status{color="yellow"} == 1
-          for: 0m
+          for: 5m # awesome was => 0m
           labels:
             severity: warning
           annotations:
@@ -174,7 +174,7 @@ spec:
                 VALUE = {{ $value }}
                 LABELS = {{ $labels }}
         - alert: ElasticsearchHighIndexingLatency
-          expr: elasticsearch_indices_indexing_index_time_seconds_total / elasticsearch_indices_indexing_index_total > 0.002 # awesome was 0.0005
+          expr: elasticsearch_indices_indexing_index_time_seconds_total / elasticsearch_indices_indexing_index_total > 0.005 # awesome was 0.0005
           for: 10m
           labels:
             severity: warning

package/src/helm-charts/prom-operator/rules/gcp-rules.yaml

@@ -29,8 +29,7 @@ spec:
       expr: sum(stackdriver_pubsub_subscription_pubsub_googleapis_com_subscription_num_undelivered_messages) by (subscription_id) > 1000
       for: 5m 
       labels:
-        severity: page
-        priority: P3
+        severity: info # awesome was => page / priority: P3
       annotations:
         summary: "High Number of Undelivered Messages on PubSub"
         description: "Undelivered message count on topic {{$labels.subscription_id}} is greater than 1000"

package/src/helm-charts/prom-operator/rules/kubestate-rules.yaml

@@ -21,7 +21,7 @@ spec:
                 LABELS = {{ $labels }}
         - alert: KubernetesNodeMemoryPressure
           expr: kube_node_status_condition{condition="MemoryPressure",status="true"} == 1
-          for: 2m
+          for: 5m # awesome was => 2m
           labels:
             severity: critical
           annotations:
@@ -32,7 +32,7 @@ spec:
                 LABELS = {{ $labels }}
         - alert: KubernetesNodeDiskPressure
           expr: kube_node_status_condition{condition="DiskPressure",status="true",node!~".*preemptible.*"} == 1 # awesome mod => added preemptible check
-          for: 2m
+          for: 5m # awesome was => 2m
           labels:
             severity: critical
           annotations:
@@ -43,7 +43,7 @@ spec:
                 LABELS = {{ $labels }}
         - alert: KubernetesNodeNetworkUnavailable
           expr: kube_node_status_condition{condition="NetworkUnavailable",status="true"} == 1
-          for: 2m
+          for: 5m # awesome was => 2m
           labels:
             severity: critical
           annotations:
@@ -54,7 +54,7 @@ spec:
                 LABELS = {{ $labels }}
         - alert: KubernetesNodeOutOfPodCapacity
           expr: sum by (node) ((kube_pod_status_phase{phase="Running"} == 1) + on(uid, instance) group_left(node) (0 * kube_pod_info{pod_template_hash=""})) / sum by (node) (kube_node_status_allocatable{resource="pods"}) * 100 > 90
-          for: 2m
+          for: 5m # awesome was => 2m
           labels:
             severity: warning
           annotations:

package/src/helm-charts/prom-operator/rules/prometheus-rules.yaml

@@ -130,8 +130,8 @@ spec:
                 VALUE = {{ $value }}
                 LABELS = {{ $labels }}
         - alert: PrometheusRuleEvaluationFailures
-          expr: increase(prometheus_rule_evaluation_failures_total[3m]) > 0
-          for: 0m
+          expr: increase(prometheus_rule_evaluation_failures_total[5m]) > 0 # awesome was 3m
+          for: 5m # awesome was 0m
           labels:
             severity: critical
           annotations:
@@ -186,7 +186,7 @@ spec:
                 LABELS = {{ $labels }}
         - alert: PrometheusTargetEmpty
           expr: prometheus_sd_discovered_targets == 0
-          for: 0m
+          for: 3m # awesome was => 0m
           labels:
             severity: critical
           annotations:

package/src/helm-charts/prom-operator/stackdriver-exporter-gpu-rules.yaml

@@ -1,6 +1,6 @@
 #
 # these custom rules exist to essentially boil down the GPU information provided
-# by the stackdriver exporter into a form that is a little more humand readable
+# by the stackdriver exporter into a form that is a little more human readable
 # in an attempt to keep the grafana / alert manager rules a little less "verbose"
 #
 # In a nutshell, we take metrics in the form of:

package/src/helm-charts/redis/values-local.yaml

@@ -71,7 +71,7 @@ networkPolicy:
 metrics:
   enabled: true
   serviceMonitor:
-    enabled: false # set to true for prometheus operator
+    enabled: true # set to true for prometheus operator
     namespace: "prometheus"
     relabelings:
     - action: labelmap

package/src/helm-charts/traefik/helmup.plugin

@@ -17,7 +17,7 @@ TRAEFIK_NAMESPACE="traefik"
 
 addHelmRepo traefik https://helm.traefik.io/traefik
 
-[ -z "$TRAEFIK_CHART_VERSION" ] && TRAEFIK_CHART_VERSION="35"
+[ -z "$TRAEFIK_CHART_VERSION" ] && TRAEFIK_CHART_VERSION="36"
 helm upgrade --install traefik traefik/traefik \
              --namespace $TRAEFIK_NAMESPACE --create-namespace \
              --values  traefik/values.yaml \

package/src/helm-charts/velero/helmup.plugin

@@ -4,7 +4,7 @@ showInstalling "Velero Backup System"
 
 addHelmRepo vmware-tanzu https://vmware-tanzu.github.io/helm-charts
 
-[ -z "$VELERO_CHART_VERSION" ] && VELERO_CHART_VERSION="9"
+[ -z "$VELERO_CHART_VERSION" ] && VELERO_CHART_VERSION="10"
 #
 # Build the bucket, region and SA email variables and use --set to mod the
 # chart values as opposed to using an OVH:<label> approach. This eliminates

package/src/helm-charts/velero/values-local.yaml

@@ -71,7 +71,7 @@ resources:
     cpu: 250m
     memory: 128Mi
   limits:
-    cpu: 500m
+    cpu: 750m
     memory: 256Mi
 
 # Annotations to add to the Velero deployment's. Optional.

package/src/helmup.sh

@@ -12,8 +12,8 @@ REGISTRY_DOMAIN="us-docker.pkg.dev"
 # open container initiative (oci) => https://opencontainers.org/
 ARTIFACT_REGISTRY="oci://$REGISTRY_DOMAIN/leverege-registry"
 # login to the registry in the background
-gcloud auth print-access-token | helm registry login -u oauth2accesstoken \
-  --password-stdin https://$REGISTRY_DOMAIN &> $DEVNULL &
+gcloud auth print-access-token | helm registry login \
+  -u oauth2accesstoken --password-stdin $REGISTRY_DOMAIN &> $DEVNULL
 
 # make sure we have a current npmrc file
 refresh-npm-token
@@ -175,21 +175,21 @@ function createNamespaceIfNeeded() {
 function makeUniformBucketWithCors() {
   local BUCKET_NAME="gs://${GCP_PROJECT_ID}-${1}"
   printf "Creating/Updating bucket: `color g $BUCKET_NAME`\n"
-  gsutil ls -p $GCP_PROJECT_ID -b $BUCKET_NAME &> /dev/null
+  gsutil ls -p $GCP_PROJECT_ID -b $BUCKET_NAME &> $DEVNULL
   if [ "$?" -ne 0 ];
   then
-    gsutil mb -l $GCE_REGION -p $GCP_PROJECT_ID $BUCKET_NAME &> /dev/null
-    [ "$?" -ne 0 ] && errorExit "failed to create the bucket: `color y $BUCKET_NAME`"
+    gsutil mb -l $GCE_REGION -p $GCP_PROJECT_ID $BUCKET_NAME &> $DEVNULL
+    exitOnError $? "failed to create the bucket: `color y $BUCKET_NAME`"
   fi
   gcloud storage buckets update $BUCKET_NAME --uniform-bucket-level-access
   CORS_JSON_FILE="$(build-tools --reporoot)/src/helmup-cors-for-buckets.json"
   printf "  Applying CORS policy\n"
-  gsutil cors set $CORS_JSON_FILE $BUCKET_NAME &> /dev/null
+  gsutil cors set $CORS_JSON_FILE $BUCKET_NAME &> $DEVNULL
 
   if [ "$2" == "PUBLIC" ];
   then
     printf "  Making the bucket `color r PUBLIC`\n"
-    gsutil iam ch allUsers:objectViewer $BUCKET_NAME &> /dev/null
+    gsutil iam ch allUsers:objectViewer $BUCKET_NAME &> $DEVNULL
   fi
 }
 
@@ -394,7 +394,7 @@ function installPreemptibleKiller() {
   gcloud iam service-accounts create $SVC_ACCT \
     --project $GCP_PROJECT_ID \
     --description "Estafette Node Killer SA" \
-    --display-name "Estafette Node Killer SA"
+    --display-name "Estafette Node Killer SA" &> $DEVNULL
   warnOnError $? "SA creation may have failed or it already exists"
 
   printf "\nCreate the $SVC_ACCT IAM role\n"
@@ -402,7 +402,7 @@ function installPreemptibleKiller() {
     --project $GCP_PROJECT_ID \
     --title "Estafette Node Killer" \
     --description "Delete compute instances" \
-    --permissions compute.instances.delete
+    --permissions compute.instances.delete &> $DEVNULL
   warnOnError $? "Role creation may have failed or it already exists"
 
   local service_account_email=$(gcloud iam --project=$GCP_PROJECT_ID service-accounts list --filter $SVC_ACCT --format 'value([email])')
@@ -417,6 +417,7 @@ function installPreemptibleKiller() {
   helm upgrade --install estafette-gke-preemptible-killer \
                estafette/estafette-gke-preemptible-killer \
                --namespace ${NAMESPACE} \
+               --set resources.limits.cpu="100m" \
                --set secret.workloadIdentityServiceAccount="$SVC_ACCT@$GCP_PROJECT_ID.iam.gserviceaccount.com" \
                --set extraEnv.BLACKLIST_HOURS="10:00-02:00" # No killing from 5AM-9PM EST
 #              --set extraEnv.WHITELIST_HOURS="12:30-13:30"
@@ -478,7 +479,15 @@ spec:
               - "/bin/sh"
             args:
               - "-c"
-              - "echo 'Hunting Zombies' ; kubectl get pods --all-namespaces --field-selector=status.phase=Failed -o json | kubectl delete -f - 2> /dev/null"
+              - |
+                echo 'Hunting Zombies' ;
+                namespaces="default cnpg-operands elastic prometheus traefik velero"
+                for ns in \$namespaces; do
+                  pods=\$(kubectl get pods -n "\$ns" --field-selector=status.phase=Failed -o name 2>/dev/null)
+                  if [ -n "\$pods" ]; then
+                    echo "\$pods" | xargs -r kubectl delete -n "\$ns"
+                  fi
+                done
 status: {}
 END_OF_ZOMBIE
 }
@@ -792,7 +801,7 @@ function installVeleroEnvironment() {
   gcloud config set project $GCP_PROJECT_ID
 
   ## The major chart version will determine the bucket suffix
-  [ -z "$VELERO_CHART_VERSION" ] && VELERO_CHART_VERSION="9"
+  [ -z "$VELERO_CHART_VERSION" ] && VELERO_CHART_VERSION="10"
 
   ## Create a bucket
   BUCKET="$GCP_PROJECT_ID-velero-$VELERO_CHART_VERSION"

package/src/modify-gcs-access.mjs

@@ -0,0 +1,84 @@
+#!/usr/bin/env node
+
+import { Command } from 'commander'
+import { shellCmd, log, warning, errorExit } from './Utils.mjs'
+
+const program = new Command()
+
+program
+  .requiredOption( '--project <projectId>', 'GCP project ID of the GSA' )
+  .requiredOption( '--bucket <bucketName>', 'GCS bucket name (no gs:// prefix)' )
+  .requiredOption( '--gsa <gsaName>', 'Service account name, e.g. my-service' )
+  .requiredOption( '--role <roleId>', 'Custom role ID', 'leveregeStorageObjectReader' )
+  .option( '--org', 'Treat as  GCP organizational level role' )
+  .option( '--add', 'Add IAM binding (default)' )
+  .option( '--remove', 'Remove IAM binding' )
+  .option( '--dry-run', 'Show command without executing' )
+
+program.parse( process.argv )
+
+const opts = program.opts()
+
+// Validate mutually exclusive add/remove
+if ( opts.add && opts.remove ) {
+  errorExit( 'You cannot specify both --add and --remove.' )
+}
+
+const action = opts.remove ? 'remove' : 'add'
+
+const {
+  project,
+  org,
+  bucket,
+  gsa,
+  role,
+  dryRun
+} = opts
+
+const orgId = await shellCmd( 'gcloud organizations list --format="value(ID)"' )
+const gsaEmail = `${gsa}@${project}.iam.gserviceaccount.com`
+const fullRole = org ? `organizations/${orgId}/roles/${role}` : `roles/${role}`
+const bucketUri = `gs://${bucket}`
+
+const iamMember = `serviceAccount:${gsaEmail}:${fullRole}`
+
+const command = action === 'add' ?
+  [ 'gsutil', 'iam', 'ch', iamMember, bucketUri ] :
+  [ 'gsutil', 'iam', 'ch', '-d', iamMember, bucketUri ]
+
+log( `🔧 ${action === 'add' ? 'Granting' : 'Removing'} ${fullRole} for ${gsaEmail} on ${bucketUri}...` )
+
+if ( dryRun ) {
+  warning( '💡 DRY RUN mode — no commands executed.' )
+  log( `🔗 Command:\n$ ${command.join( ' ' )}` )
+  process.exit( 0 )
+}
+
+try {
+  const result = await shellCmd( command )
+  log( `✅ IAM ${action} successful:\n${result}` )
+} catch ( err ) {
+  errorExit( `❌ Failed to ${action} IAM binding: ${err.message}` )
+}
+
+// === Post-change audit ===
+log( '🔍 Verifying IAM policy for this GSA on the bucket...\n' )
+
+try {
+  const policyRaw = await shellCmd( [ 'gsutil', 'iam', 'get', bucketUri ] )
+  const policy = JSON.parse( policyRaw )
+
+  const matchingRoles = policy.bindings.filter(
+    b => b.members.includes( `serviceAccount:${gsaEmail}` )
+  )
+
+  if ( matchingRoles.length === 0 ) {
+    log( `ℹ️  No IAM bindings found for ${gsaEmail} on ${bucketUri}.` )
+  } else {
+    for ( const binding of matchingRoles ) {
+      log( `🔐 ${gsaEmail} has role: ${binding.role}` )
+    }
+  }
+} catch ( err ) {
+  warning( `⚠️ Failed to retrieve or parse IAM policy: ${err.message}` )
+}

package/src/refresh-py-idx.sh

@@ -1,5 +1,14 @@
 #!/bin/bash
 
+# Load the standard helper functions
+. `build-tools --bashfun`
+
+cat<<DEPRECATED
+
+  `color r "**DEPRECATED - refresh-py-idx is no longer used"`
+DEPRECATED
+exit 1
+
 function leveregeBuildToolsRefreshPyIdx() {
   # Execute gcloud command to get the token
   local TOKEN=$(gcloud auth print-access-token)

package/src/templates/cloudBuildSteps.hbs

@@ -1,3 +1,6 @@
+options:
+  machineType: E2_HIGHCPU_8
+
 steps:
   - name: 'gcr.io/cloud-builders/docker'
     entrypoint: 'bash'
@@ -12,6 +15,28 @@ steps:
       - '--cache-from'
       - '{{imageName}}:latest'
       - '.'
-    
+
+  - name: 'gcr.io/cloud-builders/docker'
+    args:
+      - 'tag'
+      - '{{imageName}}:{{imageVersion}}'
+      - '{{imageName}}:latest'
+
+  - name: 'gcr.io/cloud-builders/docker'
+    args:
+      - 'push'
+      - '{{imageName}}:latest'
+
+  - name: 'gcr.io/cloud-builders/docker'
+    args:
+      - 'tag'
+      - '{{imageName}}:{{imageVersion}}'
+      - '{{imageName}}:{{imageInfoTag}}'
+
+  - name: 'gcr.io/cloud-builders/docker'
+    args:
+      - 'push'
+      - '{{imageName}}:{{imageInfoTag}}'
+
 images:
   - '{{imageName}}:{{imageVersion}}'

package/src/templates/dockerfileTemplatePy.hbs

@@ -1,60 +1,64 @@
 # Version {{regvers}} @ {{date}}
 
-# The FROM directive sets the Base Image for subsequent instructions
 FROM {{image}}
 
 ARG BUILD_ENV="{{buildEnv}}"
 
 RUN useradd -ms /bin/bash {{runuser}} && \
-    mkdir -p /usr/src/app && \
-    chown {{runuser}} /usr/src/app && \
+    mkdir -p /usr/src/app /home/{{runuser}} && \
+    chown -R {{runuser}} /usr/src/app /home/{{runuser}} && \
     apt-get update && apt-get install -y \
-    wget git libgl1 libglib2.0-0 \
-    && rm -rf /var/lib/apt/lists/*
+      wget git libgl1 libglib2.0-0 && \
+    rm -rf /var/lib/apt/lists/*
 
-#    gcc \
-#    git \
-#    && rm -rf /var/lib/apt/lists/*
-
-# Copy uv binary from the official distroless Docker image.
+# Copy uv binary from distroless image
 COPY --from=ghcr.io/astral-sh/uv:latest /uv /bin/uv
 
-WORKDIR /usr/src/app
-
-# Install app dependencies
-COPY --chown={{runuser}}:{{runuser}} ./workspace/ /usr/src/app/
-ENV GRPC_VERBOSITY ERROR
-
-# Setup pip.conf for Google Artifact Registry.
-RUN mkdir -p ~/.config/pip && \
-    echo "[global]" >> ~/.config/pip/pip.conf && \
-    echo "break-system-packages = true" >> ~/.config/pip/pip.conf 
-
+# Copy bashrc early
+COPY --chown={{runuser}}:{{runuser}} ./bashrc /home/{{runuser}}/.bashrc
 
-# Set up the virtual environment.
+WORKDIR /usr/src/app
 USER {{runuser}}
-COPY --chown={{runuser}}:{{runuser}} ./bashrc /home/{{runuser}}/.bashrc
 
-RUN uv venv
+ENV GRPC_VERBOSITY=ERROR
 ENV VIRTUAL_ENV=/usr/src/app/.venv
 ENV PATH="$VIRTUAL_ENV/bin:$PATH:/home/{{runuser}}/.local/bin"
 
-# Setup keyring for Google Artifact Registry.
-RUN uv tool install keyring --with keyrings.google-artifactregistry-auth
-
-#RUN echo "extra-index-url = https://oauth2accesstoken@us-python.pkg.dev/leverege-registry/leverege-python-packages/simple" >> ~/.config/pip/pip.conf
-#ENV UV_EXTRA_INDEX_URL "https://oauth2accesstoken@us-python.pkg.dev/leverege-registry/leverege-python-packages/simple"
+# Setup pip.conf for GCP Artifact Registry in case pip is needed for building
+RUN mkdir -p /home/{{runuser}}/.config/pip && \
+    echo "[global]" >> /home/{{runuser}}/.config/pip/pip.conf && \
+    echo "break-system-packages = true" >> /home/{{runuser}}/.config/pip/pip.conf
 
+# Set up venv and keyring tools before app copy
+RUN uv venv && \
+    uv tool install keyring --with keyrings.google-artifactregistry-auth
 
+# Pre-plugin block: models or pre-install commands
 {{preInstallPluginfile}}
 
-RUN uv sync --keyring-provider subprocess --extra-index-url https://oauth2accesstoken@us-python.pkg.dev/leverege-registry/leverege-python-packages/simple
+# Copy only dependency definitions first
+COPY --chown=python:python ./workspace/pyproject.toml \
+    ./workspace/uv.lock \
+    ./workspace/.python-version \
+    ./workspace/README.md \
+    ./workspace/LICENSE.md \
+    /usr/src/app/
 
-# Pull in the Dockerfile.plugin file here
+# Sync dependencies
+RUN uv sync --keyring-provider subprocess \
+    --extra-index-url https://oauth2accesstoken@us-python.pkg.dev/leverege-registry/leverege-python-packages/simple \
+    && rm -rf /home/{{runuser}}/.cache
+
+# Copy application source AFTER venv + tool setup
+COPY --chown={{runuser}}:{{runuser}} ./workspace/ /usr/src/app/
+
+# Plugin block for additional pip packages
 {{pluginfile}}
 
-#USER {{runuser}}
-#COPY ./bashrc /home/{{runuser}}/.bashrc
+# Ensure src/ layout is importable from anywhere in the container
+ENV PYTHONPATH="/usr/src/app/src"
+
+# Show final venv state (optional)
 RUN ls /usr/src/app/.venv/bin
+
 CMD [ "/bin/bash", "-c", "source /home/{{runuser}}/.bashrc && python scripts/server.py" ]
-#CMD [ "/bin/bash" ]