@leverege/build-tools 2.97.1 → 2.98.1

package/.vscode/launch.json

@@ -18,6 +18,36 @@
       },
       "outputCapture": "std"
     },
+    {
+      "name": "push-my-chart rsrc-srvr",
+      "type": "node",
+      "request": "launch",
+      "program": "${workspaceFolder}/src/push-my-chart.mjs",
+      "args": [],
+      "cwd": "/Users/john/Work/leverege/cicd/@plt/resource-server",
+      "runtimeExecutable": "node",
+      "console": "integratedTerminal"
+    },
+    {
+      "name": "ingestor in monorepo",
+      "type": "node",
+      "request": "launch",
+      "program": "${workspaceFolder}/src/init-my-chart/init-my-chart.mjs",
+      "args": [],
+      "cwd": "/Users/john/Work/leverege/cicd/@kud/recovr-services/packages/recovr-ingestor",
+      "runtimeExecutable": "node",
+      "console": "integratedTerminal"
+    },
+    {
+      "name": "init-my-chart emailer legacy",
+      "type": "node",
+      "request": "launch",
+      "program": "${workspaceFolder}/src/init-my-chart/init-my-chart.mjs",
+      "args": [],
+      "cwd": "/Users/john/Work/leverege/cicd/@plt/emailer",
+      "runtimeExecutable": "node",
+      "console": "integratedTerminal"
+    },
     {
       "name": "init-my-chart pubsub-pulse",
       "type": "node",
@@ -38,6 +68,16 @@
       "runtimeExecutable": "node",
       "console": "integratedTerminal"
     },
+    {
+      "name": "init-my-chart in a lib",
+      "type": "node",
+      "request": "launch",
+      "program": "${workspaceFolder}/src/init-my-chart/init-my-chart.mjs",
+      "args": ["--dry-run"],
+      "cwd": "/Users/john/Work/leverege/cicd/@lev/secrets",
+      "runtimeExecutable": "node",
+      "console": "integratedTerminal"
+    },
     {
       "name": "service-man (launch)",
       "type": "node",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@leverege/build-tools",
-  "version": "2.97.1",
+  "version": "2.98.1",
   "description": "A collection of build / support tools for Leverege developers",
   "main": "index.js",
   "repository": {
@@ -21,7 +21,6 @@
     "build-tools": "src/build-tools.mjs",
     "change-up": "src/change-up.sh",
     "chart-compass": "src/chart-compass.mjs",
-    "chart-to-museum": "src/chart-to-museum.mjs",
     "chart-to-registry": "src/chart-to-registry.mjs",
     "circleate": "src/circleate.sh",
     "clone-cnpg-from-snapshot": "src/clone-cnpg-from-snapshot.mjs",
@@ -83,7 +82,7 @@
     "@leverege/jsdoc-template": "^1.0.1",
     "chalk": "^5.6.2",
     "command-line-args": "^6.0.1",
-    "command-line-usage": "^7.0.3",
+    "command-line-usage": "^7.0.4",
     "commander": "^14.0.3",
     "deepmerge": "^4.3.1",
     "enquirer": "^2.4.1",
@@ -104,17 +103,18 @@
     "readline-sync": "^1.4.10",
     "semver": "^7.7.4",
     "shell-quote": "^1.8.3",
-    "simple-git": "^3.32.2",
+    "simple-git": "^3.33.0",
     "sloc": "^0.3.2",
     "superstruct": "^2.0.2",
     "tmp": "^0.2.5",
     "toml": "^3.0.0",
+    "yaml": "^2.8.2",
     "zx": "^8.8.5"
   },
   "devDependencies": {
     "@leverege/eslint-config-leverege": "^5.1.2",
     "chai": "^6.2.2",
     "mocha": "^11.7.5",
-    "npm": "^11.10.1"
+    "npm": "^11.11.1"
   }
 }

package/src/Utils.mjs

@@ -10,6 +10,7 @@ import { glob } from 'glob'
 import { packageUp } from 'package-up'
 import { parse } from 'shell-quote'
 import toml from 'toml'
+import semver from 'semver'
 import YAML from 'js-yaml'
 
 // __dirname goes away with ESM so we take this approach instead and
@@ -530,7 +531,8 @@ export const parseHelmChart = async ( helmroot = './helm' ) => {
   const chartLibrary = chartYaml?.type === 'library'
   const chartVersion = chartYaml.version
   const chartPackage = `${chartName}-${chartVersion}.tgz`
-  const valuesYaml = YAML.load( fs.readFileSync( `${helmroot}/values.yaml`, 'utf8' ) )
+  const valuesText = fs.readFileSync( `${helmroot}/values.yaml`, 'utf8' )
+  const valuesYaml = YAML.load( valuesText )
   debug( valuesYaml, '<== helm/values.yaml' )
   if ( !valuesYaml.image ) {
     errorExit( 'Error: legacy helm chart detected - must upgrade to latest ignition template to proceed' )
@@ -543,15 +545,14 @@ export const parseHelmChart = async ( helmroot = './helm' ) => {
   let depsOutOfSync = hasDependencies // assume unsynced if dependencies
   if ( chartFiles.includes( 'Chart.lock' ) ) {
     const chartLock = YAML.load( fs.readFileSync( `${helmroot}/Chart.lock`, 'utf8' ) )
-    // Using Object.fromEntries will be able to handle multiple dependencies if ever needed
-    const chartDeps = Object.fromEntries(
-      chartYaml.dependencies.map( d => [ d.name, d.version ] )
-    )
+
     const lockDeps = Object.fromEntries(
       chartLock.dependencies.map( d => [ d.name, d.version ] )
     )
-    const mismatches = Object.entries( chartDeps ).filter( ( [ name, version ] ) => {
-      return lockDeps[name] !== version
+
+    const mismatches = chartYaml.dependencies.filter( ( { name, version } ) => {
+      const lockedVersion = lockDeps[name]
+      return !lockedVersion || !semver.satisfies( lockedVersion, version )
     } )
     
     depsOutOfSync = mismatches.length > 0
@@ -572,6 +573,7 @@ export const parseHelmChart = async ( helmroot = './helm' ) => {
     files : chartFiles,
     Chart : chartYaml,
     Values : valuesYaml,
+    valuesText, // in case we need to preserve formatting or comments for pushing to registry
     yaml : { // DEPRECATED
       'Chart.yaml' : chartYaml,
       'values.yaml' : valuesYaml,

package/src/chart-to-museum.mjs

@@ -1,5 +1,7 @@
 #!/usr/bin/env node
-
+// *** DEPRECATED: this goes away when we move away from the chart museum and
+// push-my-chart already handles this a artifact pushes
+//
 import chalk from 'chalk'
 import commandLineArgs from 'command-line-args'
 import commandLineUsage from 'command-line-usage'

package/src/helm-charts/prom-operator/helmup.plugin

@@ -8,7 +8,7 @@ showInstalling "The Prometheus Operator (kube-prometheus-stack)"
 
 # https://artifacthub.io/packages/helm/prometheus-community/kube-prometheus-stack
 OCI_CHART="oci://ghcr.io/prometheus-community/charts/kube-prometheus-stack"
-[ -z "$PROMETHEUS_STACK_CHART_VERSION" ] && PROMETHEUS_STACK_CHART_VERSION="82.1.0"
+[ -z "$PROMETHEUS_STACK_CHART_VERSION" ] && PROMETHEUS_STACK_CHART_VERSION="82.10.3"
 helm upgrade $NS --install prometheus-stack $OCI_CHART \
                  --values  prom-operator/prometheus-stack.yaml \
                  --version $PROMETHEUS_STACK_CHART_VERSION $HELM_WHAT

package/src/helm-charts/prom-operator/prometheus-stack.yaml.ovh

@@ -146,7 +146,7 @@ extraManifests:
         - websecure
       routes:
       - kind: Rule
-        match: Host(`${PROJECT_ID}-prometheus.${HOST}.com`)
+        match: Host(`${PROJECT_NAME}-prometheus.${HOST}.com`)
         middlewares:
           - name: basic-auth
         services:
@@ -224,7 +224,7 @@ alertmanager:
         slack_configs:
           - channel: "${SLACK_CHANNEL}"
             api_url: "${SLACK_HOOK_URL}"
-            username: "${PROJECT_ID}"
+            username: "${PROJECT_NAME}"
             send_resolved: true
             title: '{{ .CommonLabels.alertname }} alert'
             title_link: '#'
@@ -261,7 +261,7 @@ alertmanager:
               {{- end }}
 
               {{- if .Labels.pod }}
-              *Pod Link:* *<https://console.cloud.google.com/kubernetes/pod/${GCE_REGION}/${CLUSTER_NAME}/{{ .Labels.namespace }}/{{ .Labels.pod }}?project=${PROJECT_ID}|`{{ .Labels.pod }}`>*
+              *Pod Link:* *<https://console.cloud.google.com/kubernetes/pod/${GCE_REGION}/${CLUSTER_NAME}/{{ .Labels.namespace }}/{{ .Labels.pod }}?project=${PROJECT_NAME}|`{{ .Labels.pod }}`>*
               {{- end }}
 
               *Status:* `{{ .Status }}` | *Severity:* `{{ .Labels.severity }}` {{- if .Labels.container }} *Container:* `{{ .Labels.container }}` {{- end }}

package/src/helm-charts/prom-operator/stackdriver-exporter.yaml.ovh

@@ -5,7 +5,7 @@
 fullnameOverride: "prometheus-stack-stackdriver-metrics"
 
 stackdriver:
-  projectId: "${PROJECT_NAME}"
+  projectId: "${PROJECT_ID}"
   metrics:
     typePrefixes:  >
       agent.googleapis.com/cpu/utilization,

package/src/helm-charts/valkey/valkey-local.yaml

@@ -18,7 +18,7 @@ primary:
   replicaCount: 1
 
 # https://github.com/bitnami/charts/blob/main/bitnami/common/templates/_resources.tpl#L15
-  resourcesPreset: "small"
+  resourcesPreset: "medium"
 # resources:
 #   requests:
 #     cpu: 250m
@@ -54,7 +54,7 @@ replica:
   replicaCount: 1
 
 # https://github.com/bitnami/charts/blob/main/bitnami/common/templates/_resources.tpl#L15
-  resourcesPreset: "micro"
+  resourcesPreset: "medium"
 # resources:
 #   requests:
 #     cpu: 250m

package/src/helm-charts/velero/helmup.plugin

@@ -5,8 +5,8 @@ showInstalling "Velero Backup System"
 # We use a specific version for the velero chart because we need to control
 # the version of the locally installed velero client application.
 #
-VELERO_CHART_VERSION="11.3.2"
-EXPECTED_CLIENT_VERSION="v1.17.2"
+VELERO_CHART_VERSION="11.4.0"
+EXPECTED_CLIENT_VERSION="v1.18.0"
 
 addHelmRepo vmware-tanzu https://vmware-tanzu.github.io/helm-charts
 

package/src/helmup.sh

@@ -522,7 +522,6 @@ function getServiceAndChartInfo() {
     CHARTVER="$(jq -r --arg svc "$SERVICE" '.[$svc].chart // empty' "$versions_file")"
     [[ -n "$CHARTVER" ]] && FROM_VERSIONS=1
   fi
-
   return 0
 }
 
@@ -687,6 +686,7 @@ REBOOTSTRAP
       transponder-*)
         # transponders use the same chart but different deployment name
         [ ! -d $SERVICE ] && bootstrapLocalSetup $SERVICE
+        CHARTLOC="oci://us-docker.pkg.dev/leverege-registry/stack/charts/transponder" # always use art-reg
         HELMUP="helm upgrade --install $SERVICE $CHARTLOC --values $SERVICE/values.yaml $CHARTVER $HELM_WHAT"
         doHelmup "$HELMUP"
         ;;

package/src/init-my-chart/{leaf-chart-templates/Chart.yaml.hbs → Chart.yaml.hbs}

@@ -1,10 +1,10 @@
-name: {{containerName}}
-version: {{chartVersion}}
-appVersion: {{chartAppVersion}}
+name: {{hbsChartName}}
+version: {{hbsChartVersion}}
+appVersion: {{hbsChartAppVersion}}
 description: A leaf chart that depends on the Leverege base chart
 apiVersion: v2
 icon: https://raw.githubusercontent.com/kubernetes/kubernetes/master/logo/logo.svg
 dependencies:
   - name: leverege
-    version: "^1.0.0"
+    version: "^1.0.1"
     repository: "oci://us-docker.pkg.dev/leverege-registry/base-charts"

package/src/init-my-chart/Chartwright.mjs

@@ -1,69 +1,159 @@
 // Chartwright.mjs
+/* eslint-disable security/detect-non-literal-fs-filename */
+import fs from 'node:fs'
+
+import chalk from 'chalk'
+import Handlebars from 'handlebars'
+import YAML from 'yaml'
+
 import * as Utils from '../Utils.mjs'
 
-import {
-  BUILD_TOOLS_ROOT,
-} from '../Utils.mjs'
-
-const DEFAULT_LIVENESS = {
-  httpGet : {
-    path : '/livenessProbe',
-    port : 'http'
-  },
-  initialDelaySeconds : 30,
-  periodSeconds : 60
-} 
-const DEFAULT_READINESS = {
-  httpGet : {
-    path : '/healthz',
-    port : 'http'
-  },
-  initialDelaySeconds : 30,
-  periodSeconds : 10
+// handlebar variables available in the templates:
+const hbsContext = {
+  hbsChartName : '',
+  hbsChartVersion : '1.0.0',
+  hbsChartAppVersion : 'v1.0.0',
+  hbsArtifactRegistry : 'us-docker.pkg.dev/leverege-registry/FOOLDER/images',
+  hbsPartOf : '',
+  hbsServicePreemptible : true,
+  hbsRedisEnabled : true,
+  hbsAutoScalingEnabled : true
+}
+
+const PRESERVE_SECTIONS = new Set( [
+  'config',
+  'resources',
+  'autoscaling',
+  'staticscaling',
+  'pdb',
+  'probes',
+  'redis',
+] )
+
+// Helper functions
+const OBSOLETE_SECTIONS = [
+  'ingressRoute',
+  'loadbalancer',
+  'udploadbalancer',
+  'podAnnotations',
+  'workloadIdentity'
+]
+const warnDeprecatedSections = ( doc ) => {
+  let deprecationWarningShown = false
+  for ( const key of OBSOLETE_SECTIONS ) {
+    if ( !doc.has( key ) ) continue
+    Utils.log( `${chalk.yellow.bold( '⚠️  WARNING:' )} Deprecated values.yaml section detected => ${chalk.red.bold( key )}` )
+    deprecationWarningShown = true
+  }
+
+  if ( deprecationWarningShown ) {
+    Utils.log( chalk.yellow( 
+      '\nPlease remove any deprecated sections and migrate to the new configuration options as needed.'
+    ) )
+  }
 }
 
 export class Chartwright {
-  constructor( { cwd = process.cwd(), debug = false } = {} ) {
-    this.cwd = cwd
-    this.chartTemplate = `${BUILD_TOOLS_ROOT}/init-my-chart/leaf-chart-templates`
-    this.debug = debug
+  constructor( { Container, debug = false } = {} ) {
+    this.Container = Container
+    this.helmTemplates = `${Utils.BUILD_TOOLS_ROOT}/init-my-chart/leaf-chart-templates`
+    this.debug = debug    
+    this.templates = {}
+  }
 
-    this.pkg = null
-    this.existingChart = null
+  // This function takes a Handlebars template string and a context object,
+  // renders the template with the context, and parses the result as YAML,
+  // returning the resulting object.
+  async renderHbsTemplate( hbsTemplateFileName, context ) {
+    // TODO: better error handling here - if the template file doesn't exist,
+    // or if the rendered template isn't valid YAML, we should handle those
+    // errors gracefully and provide useful error messages.
+    const hbsTemplate = fs.readFileSync( hbsTemplateFileName, 'utf8' )
+    const compiled = Handlebars.compile( hbsTemplate )
+    const rendered = compiled( context )
+    return rendered
   }
 
-  async init() {
-    this.pkg = await Utils.parsePackageJson( { cwd : this.cwd } )
-    this.existingChart = await Utils.parseHelmChart( )
-    return this
+  // Load the handlebars templates for Chart.yaml and values.yaml into memory.
+  async loadTemplates() {
+    this.templates.Chart = await this.renderHbsTemplate( `${Utils.BUILD_TOOLS_ROOT}/init-my-chart/Chart.yaml.hbs`, hbsContext )
+    this.templates.Values = await this.renderHbsTemplate( `${Utils.BUILD_TOOLS_ROOT}/init-my-chart/values.yaml.hbs`, hbsContext )
   }
 
-  analyze() {
-    return {
-      serviceName : this.pkg?.name,
-      hasExistingChart : !!this.existingChart,
-      templateLoaded : !!this.templateChart
+  // Adds missing top-level keys from base into existing WITHOUT overwriting.
+  // IMPORTANT: skips PRESERVE_SECTIONS here; those are handled separately.
+  mergeTopLevelAddMissing( baseDoc, existingDoc ) {
+    const baseRoot = baseDoc.contents
+    const existingRoot = existingDoc.contents
+  
+    if ( !YAML.isMap( baseRoot ) || !YAML.isMap( existingRoot ) ) return
+  
+    for ( const pair of baseRoot.items ) {
+      const key = pair.key?.value ?? pair.key
+      if ( PRESERVE_SECTIONS.has( key ) ) continue
+  
+      if ( !existingRoot.has( key ) ) {
+        // Push the whole Pair node to preserve comments attached to it
+        existingRoot.items.push( pair )
+      }
     }
   }
 
-  bootstrap() {
-    // TODO: create new chart from template
-  }
+  // Preserve sections: if existing has it, keep it untouched.
+  // If existing is missing it and base has it, copy it in (with comments).
+  applyPreserveSections( baseDoc, existingDoc ) {
+    for ( const key of PRESERVE_SECTIONS ) {
+      const existingNode = existingDoc.get( key, true )
+      if ( existingNode ) continue // keep existing exactly
 
-  morph() {
-    // TODO: transform existing chart to new base structure
+      const baseNode = baseDoc.get( key, true )
+      if ( baseNode ) {
+        existingDoc.set( key, baseNode ) // brings comments from base node
+      }
+    }
   }
 
-  async scribe() {
-    // TODO: write resulting chart to disk
+  // Bootstraps a helm chart layout by copying the template structure and rendering the
+  // main files with the provided context. Optionally can suppress writing values.yaml
+  // if the intention is to preserve an existing one.
+  async bootstrap( { Container, suppressValues = false } ) {
+    // load the templates and render them with the context to get the initial 
+    // Chart.yaml and values.yaml content
+    await this.loadTemplates()
+
+    // replicate the chart structure from the templates directory
+    fs.mkdirSync( 'helm' )
+    fs.cpSync( `${Utils.BUILD_TOOLS_ROOT}/init-my-chart/leaf-chart-templates`, 'helm', { recursive : true } )
+    fs.writeFileSync( 'helm/Chart.yaml', this.templates.Chart )
+    if ( !suppressValues ) {
+      fs.writeFileSync( 'helm/values.yaml', this.templates.Values )
+    }
   }
 
-  dump() {
-    if ( !this.debug ) return
-    console.dir( {
-      pkg : this.pkg,
-      existingChart : this.existingChart,
-      templateChart : this.templateChart
-    }, { depth : 4 } )
+  // Scribes the new chart by bootstrapping the structure and then merging the existing
+  // values.yaml with the new template, preserving existing values and comments where possible.
+  // This allows us to update the chart structure and add any new configuration options without
+  // overwriting user customizations in values.yaml.
+  async scribe( { Container, Chart, Values, ValuesText, Registry } ) {
+    hbsContext.hbsChartName = Container
+    hbsContext.hbsChartVersion = Chart.version
+    hbsContext.hbsChartAppVersion = Chart.appVersion
+    hbsContext.hbsArtifactRegistry = Registry
+    hbsContext.hbsPartOf = Values.partOf
+    await this.bootstrap( { Container, suppressValues : true } )
+
+    // Now we need to smart merge the values.yaml template while preserving existing sections
+    const baseValuesDoc = YAML.parseDocument( this.templates.Values )
+    const existingValuesDoc = YAML.parseDocument( ValuesText )
+    // warn about obsolete sections
+    warnDeprecatedSections( existingValuesDoc )
+    
+    // Merge legacy values.yaml with new base, preserving formatting and comments
+    this.mergeTopLevelAddMissing( baseValuesDoc, existingValuesDoc )
+    this.applyPreserveSections( baseValuesDoc, existingValuesDoc )
+
+    // Final write of the merged values.yaml, preserving formatting and comments as much as possible
+    fs.writeFileSync( 'helm/values.yaml', existingValuesDoc.toString( { lineWidth : 0, doubleQuotedAsJSON : true } ) )
   }
-}
+
+}

package/src/init-my-chart/init-my-chart.mjs

@@ -2,16 +2,13 @@
 /* eslint-disable security/detect-non-literal-fs-filename */
 
 import fs from 'node:fs'
-import path from 'node:path'
 
 import { program } from 'commander'
 import chalk from 'chalk'
-import handlebars from 'handlebars'
 
 import {
   analyzeRepository,
   condir,
-  debug,
   enableDebug,
   errorExit,
   log, 
@@ -27,54 +24,52 @@ program
   .parse()
   
 const options = program.opts()
-const chartwright = await( new Chartwright( { debug : options.debug } ) ).init()
+if ( options.debug ) {
+  enableDebug() // enables debugging in the Utils functions
+}
 
 const repo = await analyzeRepository()
-
 if ( !repo.isGitRepo ) {
   errorExit( '***ERROR: This command must be run inside a git repository - run "git init" to proceed' )
 }
+const Container = repo.containerName
+const chartwright = new Chartwright( { Container, debug : options.debug } )
 
-if ( options.debug ) {
-  enableDebug()
-}
-
+// If there isn't an existing helm chart then we just bootstrap the new chart structure
+// from the current helm template and we're done. This will most likely be the case after
+// a new service has been bootstrapped with ignition.
 if ( !repo.helmChart ) {
-  log( '*** Bootstrap with the Default Helm Template' ) // TODO: finish this
-  process.exit( 1 )
+  await chartwright.bootstrap( { Container } )
+  process.exit( 0 )
 }
 
-const { Chart, Values, Registry, } = repo.helmChart
-const gitRoot = repo.gitRoot
-
-if ( fs.existsSync( `${gitRoot}/helm-old` ) ) {
+// Safety First: if there is already an older helm chart, we don't want to overwrite it - we
+// want the user to review / intervene before proceeding.
+const packageRoot = repo.currentDir
+if ( fs.existsSync( `${packageRoot}/helm-old` ) ) {
   errorExit( '***ERROR: An old helm chart structure already exists - cannot proceed' )
 }
 
 let movedOldHelm = false
-if ( fs.existsSync( `${gitRoot}/helm` ) && !options.dryRun ) {
-  fs.renameSync( `${gitRoot}/helm`, `${gitRoot}/helm-old` )
+if ( fs.existsSync( `${packageRoot}/helm` ) && !options.dryRun ) {
+  fs.renameSync( `${packageRoot}/helm`, `${packageRoot}/helm-old` )
   movedOldHelm = true
 }
 
-debug( { initMyChartRoot, movedOldHelm, repo }, '<==InitMyChart Root Variables' )
+const {
+  Chart,
+  Values,
+  valuesText : ValuesText,
+  imageRegistry : Registry,
+} = repo.helmChart
 
 if ( options.dryRun ) {
-  condir( { Chart, Values, Registry, }, chalk.green( '=== Analyzed Repository Helm Chart Info ===' ) )
-  condir( { chartOutput, valuesOutput }, chalk.green( '=== Generated Helm Chart Outputs ===' ) )
-  chartwright.dump()
-  log( { chartOutput }, chalk.green( '=== Chart.yaml ===' ) )
-  log( { valuesOutput }, chalk.green( '=== values.yaml ===' ) )
+  condir( { repo, Container, Chart, Values, ValuesText, Registry }, chalk.green( '<== Dry Run Mode - No Changes Made' ) )
   process.exit( 0 )
 }
 
-// TODO: I needed to reset this effort because I didn't like the direction it was
-//      going in - I want to focus on the transformation of the existing chart to
-//      the new structure, and less on the templating aspect. The templating can be
-//      a secondary step that we add after we have the transformation working. For
-//      now, I will focus on the analysis and transformation of the existing chart,
-//      and then we can add the templating aspect later.
+await chartwright.scribe( { Container, Chart, Values, ValuesText, Registry } )
 
 if ( movedOldHelm ) {
-  log( chalk.yellow( 'An existing helm chart was detected and moved to helm-old. Please review and migrate any customizations as needed.' ) )
+  log( chalk.yellow( '\nAn existing helm chart was detected and moved to helm-old. Please review and migrate any customizations as needed.' ) )
 }