@leverege/build-tools 2.101.1 → 2.102.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@leverege/build-tools",
-  "version": "2.101.1",
+  "version": "2.102.0",
   "description": "A collection of build / support tools for Leverege developers",
   "main": "index.js",
   "repository": {
@@ -75,8 +75,8 @@
   },
   "author": "Leverege Devs",
   "engines": {
-    "node": "^22.11 || ^24.11",
-    "npm": ">=11.6"
+    "node": "^22.11 || >=24",
+    "npm": ">=10.2 || >=11.6"
   },
   "license": "SEE LICENSE IN LICENSE.md",
   "dependencies": {
@@ -94,29 +94,29 @@
     "googleapis": "^171.4.0",
     "handlebars": "^4.7.9",
     "ignore": "^7.0.5",
-    "inquirer": "^13.4.1",
+    "inquirer": "^13.4.3",
     "js-yaml": "^4.1.1",
     "jsdoc": "^4.0.5",
     "ms": "^2.1.3",
     "npm-registry-fetch": "^19.1.1",
-    "ora": "^9.3.0",
+    "ora": "^9.4.0",
     "p-limit": "^7.3.0",
     "package-up": "^5.0.0",
     "readline-sync": "^1.4.10",
-    "semver": "^7.7.4",
+    "semver": "^7.8.0",
     "shell-quote": "^1.8.3",
     "simple-git": "^3.36.0",
     "sloc": "^0.3.2",
     "superstruct": "^2.0.2",
     "tmp": "^0.2.5",
     "toml": "^3.0.0",
-    "yaml": "^2.8.3",
+    "yaml": "^2.9.0",
     "zx": "^8.8.5"
   },
   "devDependencies": {
     "@leverege/eslint-config-leverege": "^5.1.2",
     "chai": "^6.2.2",
     "mocha": "^11.7.5",
-    "npm": "^11.12.1"
+    "npm": "^11.14.1"
   }
 }

package/src/Utils.mjs

@@ -560,6 +560,11 @@ export const parseHelmChart = async ( helmroot = './helm' ) => {
     } )
     
     depsOutOfSync = mismatches.length > 0
+
+    // One final check - if the helm/charts dir is missing then we also need to update dependencies
+    if ( hasDependencies && !chartFiles.includes( 'charts' ) ) {
+      depsOutOfSync = true
+    }
   }
   /* eslint-enable security/detect-non-literal-fs-filename */
 

package/src/bash-funcs

@@ -229,6 +229,33 @@ function waitForIAM() {
   done
 }
 
+# waitForSAIAM SVC_ACCT PROJECT ROLE MEMBER [TIMEOUT_SECONDS]
+# Like waitForIAM but checks the SA's own IAM policy (for bindings like workloadIdentityUser
+# that are set via `gcloud iam service-accounts add-iam-policy-binding`, not project-level).
+function waitForSAIAM() {
+  local svcAcct="$1" project="$2" role="$3" member="$4" timeout="${5:-120}"
+  local svcAcctEmail="${svcAcct}@${project}.iam.gserviceaccount.com"
+  local start=$(date +%s) delay=2
+
+  echo "⏳ waiting for SA IAM binding ${role} → ${member} on ${svcAcctEmail} (timeout ${timeout}s)..." >&2
+  while true; do
+    local found
+    found="$(gcloud iam service-accounts get-iam-policy "${svcAcctEmail}" \
+              --flatten="bindings[].members" \
+              --filter="bindings.role=${role} AND bindings.members=${member}" \
+              --format="value(bindings.role)" 2> "$DEVNULL")"
+    if [[ -n "$found" ]]; then
+      echo "✅ SA IAM binding active: ${role} → ${member}" >&2
+      return 0
+    fi
+    if (( $(date +%s) - start >= timeout )); then
+      echo "❌ timed out waiting for SA IAM binding ${role} → ${member}" >&2
+      return 1
+    fi
+    sleep "$delay"; (( delay = delay < 20 ? delay * 2 : 20 ))
+  done
+}
+
 function removeHelmRepo(){
   [ ! -z "$SKIP_RM_HELM_REPO" ] && return
   helm repo remove $1 &> $DEVNULL

package/src/dashboard/DashboardManager.mjs

@@ -9,16 +9,25 @@ import { debug, errorExit, log } from '../Utils.mjs'
 // const FIELD_MANAGER = 'dashboard'
 const GRAFANA_LABEL = 'grafana_dashboard'
 const GRAFANA_FOLDER = 'grafana_folder'
+const GRAFANA_LOCAL = 'grafana_dashboard_local'
 
 /* eslint-disable security/detect-non-literal-fs-filename */
 export default class DashboardManager {
   #api
   #namespace
   #dryRun
+  #prune
+  #folders
+  #localDir
+  #pruneLocal
 
-  constructor( { namespace = 'prometheus', dryRun = false } = {} ) {
+  constructor( { namespace = 'prometheus', dryRun = false, prune = false, folders = [], localDir = null, pruneLocal = false } = {} ) {
     this.#namespace = namespace
     this.#dryRun = dryRun
+    this.#prune = prune
+    this.#folders = folders
+    this.#localDir = localDir
+    this.#pruneLocal = pruneLocal
   }
 
   #getApi() {
@@ -31,21 +40,61 @@ export default class DashboardManager {
   }
 
   async apply( dir ) {
-    const folders = fs.readdirSync( dir, { withFileTypes : true } )
-      .filter( e => e.isDirectory() )
+    if ( !fs.existsSync( dir ) ) {
+      errorExit( `Dashboard directory not found: ${dir}` )
+    }
 
-    if ( folders.length === 0 ) {
-      log( chalk.yellow( `No dashboard subfolders found in ${dir}` ) )
+    const entries = fs.readdirSync( dir, { withFileTypes : true } )
+    const folders = entries.filter( e => e.isDirectory() )
+    const jsonFiles = entries.filter( e => e.isFile() && e.name.endsWith( '.json' ) )
+
+    if ( folders.length === 0 && jsonFiles.length === 0 ) {
+      log( chalk.yellow( `No dashboards or subfolders found in ${dir}` ) )
       return
     }
 
-    for ( const folder of folders ) {
-      const files = fs.readdirSync( path.join( dir, folder.name ) )
-        .filter( f => f.endsWith( '.json' ) )
+    const applied = new Set()
+
+    if ( folders.length === 0 ) {
+      // leaf directory — apply JSON files using basename as the Grafana folder name
+      const folderName = path.basename( dir )
+      for ( const file of jsonFiles ) {
+        const name = await this.#applyDashboard( folderName, path.join( dir, file.name ) )
+        if ( name ) applied.add( name )
+      }
+      if ( this.#prune ) await this.#pruneOrphans( applied, folderName )
+    } else {
+      // filter to configured folders if DASHBOARDS was set in overwhelm.yaml
+      const activeFolders = this.#folders.length > 0 ?
+        folders.filter( f => this.#folders.includes( f.name ) ) :
+        folders
+
+      if ( this.#folders.length > 0 && activeFolders.length < this.#folders.length ) {
+        const missing = this.#folders.filter( f => !folders.find( d => d.name === f ) )
+        missing.forEach( f => log( chalk.yellow( `  [warn] requested folder not found: ${f}` ) ) )
+      }
+
+      for ( const folder of activeFolders ) {
+        const files = fs.readdirSync( path.join( dir, folder.name ) )
+          .filter( f => f.endsWith( '.json' ) )
+
+        for ( const file of files ) {
+          const name = await this.#applyDashboard( folder.name, path.join( dir, folder.name, file ) )
+          if ( name ) applied.add( name )
+        }
+      }
+
+      if ( this.#prune ) await this.#pruneOrphans( applied )
+    }
 
-      for ( const file of files ) {
-        await this.#applyDashboard( folder.name, path.join( dir, folder.name, file ) )
+    // Apply local dashboards if --local-dir was specified
+    if ( this.#localDir ) {
+      if ( !fs.existsSync( this.#localDir ) ) {
+        errorExit( `Local dashboard directory not found: ${this.#localDir}` )
       }
+      log( chalk.bold( `Applying local dashboards from: ${this.#localDir}` ) )
+      const localApplied = await this.#applyDir( this.#localDir, true )
+      if ( this.#pruneLocal ) await this.#pruneLocalOrphans( localApplied )
     }
   }
 
@@ -64,7 +113,11 @@ export default class DashboardManager {
       return
     }
 
-    for ( const folder of folders ) {
+    const activeFolders = this.#folders.length > 0 ?
+      folders.filter( f => this.#folders.includes( f.name ) ) :
+      folders
+
+    for ( const folder of activeFolders ) {
       const files = fs.readdirSync( path.join( dir, folder.name ) )
         .filter( f => f.endsWith( '.json' ) )
 
@@ -75,20 +128,74 @@ export default class DashboardManager {
         files.forEach( f => log( `  ${chalk.cyan( path.basename( f, '.json' ) )}` ) )
       }
     }
+
+    if ( this.#localDir && fs.existsSync( this.#localDir ) ) {
+      log( chalk.bold( `\nLocal dashboards in: ${this.#localDir}` ) )
+      const localFolders = fs.readdirSync( this.#localDir, { withFileTypes : true } )
+        .filter( e => e.isDirectory() )
+
+      for ( const folder of localFolders ) {
+        const files = fs.readdirSync( path.join( this.#localDir, folder.name ) )
+          .filter( f => f.endsWith( '.json' ) )
+
+        log( chalk.bold( `\n[${folder.name}] (local)` ) )
+        if ( files.length === 0 ) {
+          log( chalk.dim( '  (empty)' ) )
+        } else {
+          files.forEach( f => log( `  ${chalk.cyan( path.basename( f, '.json' ) )}` ) )
+        }
+      }
+    }
   }
 
-  async #applyDashboard( folder, filePath ) {
+  // Scans a directory for JSON files (leaf mode) or subfolders and applies all dashboards.
+  // Returns the Set of ConfigMap names applied.
+  async #applyDir( dir, local = false ) {
+    const entries = fs.readdirSync( dir, { withFileTypes : true } )
+    const subDirs = entries.filter( e => e.isDirectory() )
+    const jsonFiles = entries.filter( e => e.isFile() && e.name.endsWith( '.json' ) )
+    const applied = new Set()
+
+    if ( subDirs.length === 0 ) {
+      const folderName = path.basename( dir )
+      for ( const file of jsonFiles ) {
+        const name = await this.#applyDashboard( folderName, path.join( dir, file.name ), local )
+        if ( name ) applied.add( name )
+      }
+    } else {
+      for ( const folder of subDirs ) {
+        const files = fs.readdirSync( path.join( dir, folder.name ) )
+          .filter( f => f.endsWith( '.json' ) )
+
+        for ( const file of files ) {
+          const name = await this.#applyDashboard( folder.name, path.join( dir, folder.name, file ), local )
+          if ( name ) applied.add( name )
+        }
+      }
+    }
+
+    return applied
+  }
+
+  async #applyDashboard( folder, filePath, local = false ) {
     const filename = path.basename( filePath )
-    const name = `dashboard-${path.basename( filePath, '.json' )}`
+    const slug = path.basename( filePath, '.json' )
+      .toLowerCase()
+      .replace( /[^a-z0-9.-]+/g, '-' )
+      .replace( /^-+|-+$/g, '' )
+    const name = `dashboard-${local ? 'local-' : ''}${slug}`
     const json = fs.readFileSync( filePath, 'utf8' )
 
+    const labels = { [GRAFANA_LABEL] : '1' }
+    if ( local ) labels[GRAFANA_LOCAL] = '1'
+
     const configMap = {
       apiVersion : 'v1',
       kind : 'ConfigMap',
       metadata : {
         name,
         namespace : this.#namespace,
-        labels : { [GRAFANA_LABEL] : '1' },
+        labels,
         annotations : { [GRAFANA_FOLDER] : folder },
       },
       data : { [filename] : json },
@@ -97,8 +204,8 @@ export default class DashboardManager {
     debug( configMap, `dashboard: ${name}` )
 
     if ( this.#dryRun ) {
-      log( chalk.yellow( `  [dry-run] ${filename} → [${folder}]` ) )
-      return
+      log( chalk.yellow( `  [dry-run] ${filename} → [${folder}]${local ? ' (local)' : ''}` ) )
+      return name
     }
 
     const api = this.#getApi()
@@ -117,10 +224,78 @@ export default class DashboardManager {
       } else {
         await api.createNamespacedConfigMap( { namespace : this.#namespace, body : configMap } )
       }
-      log( chalk.green( `  ✓ ${filename} → [${folder}]` ) )
+      log( chalk.green( `  ✓ ${filename} → [${folder}]${local ? ' (local)' : ''}` ) )
     } catch ( e ) {
       errorExit( `Failed to apply ${name}: ${e.message}` )
     }
+    return name
+  }
+
+  async #pruneOrphans( applied, folderFilter = null ) {
+    const api = this.#getApi()
+    let configMaps
+    try {
+      const res = await api.listNamespacedConfigMap( {
+        namespace : this.#namespace,
+        labelSelector : `${GRAFANA_LABEL}=1`,
+      } )
+      configMaps = res.items
+    } catch ( e ) {
+      log( chalk.yellow( `  [prune] could not list ConfigMaps: ${e.message}` ) )
+      return
+    }
+
+    const orphans = configMaps.filter( ( cm ) => {
+      if ( applied.has( cm.metadata.name ) ) return false
+      if ( cm.metadata.labels?.[GRAFANA_LOCAL] === '1' ) return false // leave local CMs to pruneLocal
+      if ( folderFilter && cm.metadata.annotations?.[GRAFANA_FOLDER] !== folderFilter ) return false
+      return true
+    } )
+
+    for ( const cm of orphans ) {
+      const name = cm.metadata.name
+      if ( this.#dryRun ) {
+        log( chalk.yellow( `  [dry-run] prune ${name}` ) )
+        continue
+      }
+      try {
+        await api.deleteNamespacedConfigMap( { name, namespace : this.#namespace } )
+        log( chalk.red( `  ✗ pruned ${name}` ) )
+      } catch ( e ) {
+        log( chalk.yellow( `  [prune] failed to delete ${name}: ${e.message}` ) )
+      }
+    }
+  }
+
+  async #pruneLocalOrphans( applied ) {
+    const api = this.#getApi()
+    let configMaps
+    try {
+      const res = await api.listNamespacedConfigMap( {
+        namespace : this.#namespace,
+        labelSelector : `${GRAFANA_LOCAL}=1`,
+      } )
+      configMaps = res.items
+    } catch ( e ) {
+      log( chalk.yellow( `  [prune-local] could not list ConfigMaps: ${e.message}` ) )
+      return
+    }
+
+    const orphans = configMaps.filter( cm => !applied.has( cm.metadata.name ) )
+
+    for ( const cm of orphans ) {
+      const name = cm.metadata.name
+      if ( this.#dryRun ) {
+        log( chalk.yellow( `  [dry-run] prune-local ${name}` ) )
+        continue
+      }
+      try {
+        await api.deleteNamespacedConfigMap( { name, namespace : this.#namespace } )
+        log( chalk.red( `  ✗ pruned (local) ${name}` ) )
+      } catch ( e ) {
+        log( chalk.yellow( `  [prune-local] failed to delete ${name}: ${e.message}` ) )
+      }
+    }
   }
 
 }

package/src/dashboard/dashboard.mjs

@@ -5,7 +5,7 @@ import path from 'node:path'
 import { program } from 'commander'
 import chalk from 'chalk'
 
-import { enableDebug, errorExit, log } from '../Utils.mjs'
+import { enableDebug, errorExit, log, shellCmd } from '../Utils.mjs'
 
 import DashboardManager from './DashboardManager.mjs'
 
@@ -17,16 +17,41 @@ program
   .argument( '[dir]', 'Directory of dashboard subfolders', defaultJsonDir )
   .option( '--list', 'List available dashboards without applying' )
   .option( '--dry-run', 'Show what would be applied without making changes' )
+  .option( '--prune', 'Delete ConfigMaps for dashboards no longer on disk' )
+  .option( '--local-dir <path>', 'Additional local directory of dashboards to apply alongside official ones' )
+  .option( '--prune-local', 'Delete ConfigMaps for local dashboards no longer on disk' )
   .option( '--debug', 'Enable debug logging' )
   .option( '-n, --namespace <ns>', 'Target Kubernetes namespace', 'prometheus' )
   .parse()
 
-const { list, dryRun, debug, namespace } = program.opts()
-const [ dir = defaultJsonDir ] = program.args
+const { list, dryRun, prune, localDir, pruneLocal, debug, namespace } = program.opts()
+const [ rawDir ] = program.args
+const dir = rawDir ?
+  path.isAbsolute( rawDir ) ? rawDir : path.join( defaultJsonDir, rawDir ) :
+  defaultJsonDir
 
 if ( debug ) enableDebug()
 
-const mgr = new DashboardManager( { namespace, dryRun } )
+// Read DASHBOARDS from overwhelm.yaml for the current context (exits early, no interactive prompts).
+// If the key is absent or overwhelm.yaml is not present, all folders are loaded.
+let folders = []
+try {
+  const raw = ( await shellCmd( 'overwhelm -k DASHBOARDS' ) ).trim()
+  if ( raw ) {
+    folders = raw.split( ',' ).map( f => f.trim() ).filter( Boolean )
+    log( chalk.dim( `Dashboard folders from overwhelm: ${folders.join( ', ' )}` ) )
+  }
+} catch {
+  // DASHBOARDS not configured — load all folders
+}
+
+try {
+  await shellCmd( 'overwhelm', { stdio : 'inherit' } )
+} catch ( err ) {
+  process.exit( err.exitCode )
+}
+
+const mgr = new DashboardManager( { namespace, dryRun, prune, folders, localDir, pruneLocal } )
 
 if ( list ) {
   await mgr.list( dir ).catch( e => errorExit( e ) )