@leverege/build-tools 2.97.0 → 2.98.0

package/.vscode/launch.json

@@ -18,6 +18,26 @@
       },
       "outputCapture": "std"
     },
+    {
+      "name": "ingestor in monorepo",
+      "type": "node",
+      "request": "launch",
+      "program": "${workspaceFolder}/src/init-my-chart/init-my-chart.mjs",
+      "args": [],
+      "cwd": "/Users/john/Work/leverege/cicd/@kud/recovr-services/packages/recovr-ingestor",
+      "runtimeExecutable": "node",
+      "console": "integratedTerminal"
+    },
+    {
+      "name": "init-my-chart emailer legacy",
+      "type": "node",
+      "request": "launch",
+      "program": "${workspaceFolder}/src/init-my-chart/init-my-chart.mjs",
+      "args": [],
+      "cwd": "/Users/john/Work/leverege/cicd/@plt/emailer",
+      "runtimeExecutable": "node",
+      "console": "integratedTerminal"
+    },
     {
       "name": "init-my-chart pubsub-pulse",
       "type": "node",
@@ -38,6 +58,16 @@
       "runtimeExecutable": "node",
       "console": "integratedTerminal"
     },
+    {
+      "name": "init-my-chart in a lib",
+      "type": "node",
+      "request": "launch",
+      "program": "${workspaceFolder}/src/init-my-chart/init-my-chart.mjs",
+      "args": ["--dry-run"],
+      "cwd": "/Users/john/Work/leverege/cicd/@lev/secrets",
+      "runtimeExecutable": "node",
+      "console": "integratedTerminal"
+    },
     {
       "name": "service-man (launch)",
       "type": "node",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@leverege/build-tools",
-  "version": "2.97.0",
+  "version": "2.98.0",
   "description": "A collection of build / support tools for Leverege developers",
   "main": "index.js",
   "repository": {
@@ -83,7 +83,7 @@
     "@leverege/jsdoc-template": "^1.0.1",
     "chalk": "^5.6.2",
     "command-line-args": "^6.0.1",
-    "command-line-usage": "^7.0.3",
+    "command-line-usage": "^7.0.4",
     "commander": "^14.0.3",
     "deepmerge": "^4.3.1",
     "enquirer": "^2.4.1",
@@ -93,7 +93,7 @@
     "googleapis": "^171.4.0",
     "handlebars": "^4.7.8",
     "ignore": "^7.0.5",
-    "inquirer": "^13.2.5",
+    "inquirer": "^13.3.0",
     "js-yaml": "^4.1.1",
     "jsdoc": "^4.0.5",
     "ms": "^2.1.3",
@@ -104,17 +104,18 @@
     "readline-sync": "^1.4.10",
     "semver": "^7.7.4",
     "shell-quote": "^1.8.3",
-    "simple-git": "^3.31.1",
+    "simple-git": "^3.32.3",
     "sloc": "^0.3.2",
     "superstruct": "^2.0.2",
     "tmp": "^0.2.5",
     "toml": "^3.0.0",
+    "yaml": "^2.8.2",
     "zx": "^8.8.5"
   },
   "devDependencies": {
     "@leverege/eslint-config-leverege": "^5.1.2",
     "chai": "^6.2.2",
     "mocha": "^11.7.5",
-    "npm": "^11.10.1"
+    "npm": "^11.11.0"
   }
 }

package/src/Utils.mjs

@@ -530,7 +530,8 @@ export const parseHelmChart = async ( helmroot = './helm' ) => {
   const chartLibrary = chartYaml?.type === 'library'
   const chartVersion = chartYaml.version
   const chartPackage = `${chartName}-${chartVersion}.tgz`
-  const valuesYaml = YAML.load( fs.readFileSync( `${helmroot}/values.yaml`, 'utf8' ) )
+  const valuesText = fs.readFileSync( `${helmroot}/values.yaml`, 'utf8' )
+  const valuesYaml = YAML.load( valuesText )
   debug( valuesYaml, '<== helm/values.yaml' )
   if ( !valuesYaml.image ) {
     errorExit( 'Error: legacy helm chart detected - must upgrade to latest ignition template to proceed' )
@@ -572,6 +573,7 @@ export const parseHelmChart = async ( helmroot = './helm' ) => {
     files : chartFiles,
     Chart : chartYaml,
     Values : valuesYaml,
+    valuesText, // in case we need to preserve formatting or comments for pushing to registry
     yaml : { // DEPRECATED
       'Chart.yaml' : chartYaml,
       'values.yaml' : valuesYaml,

package/src/helm-charts/prom-operator/helmup.plugin

@@ -8,7 +8,7 @@ showInstalling "The Prometheus Operator (kube-prometheus-stack)"
 
 # https://artifacthub.io/packages/helm/prometheus-community/kube-prometheus-stack
 OCI_CHART="oci://ghcr.io/prometheus-community/charts/kube-prometheus-stack"
-[ -z "$PROMETHEUS_STACK_CHART_VERSION" ] && PROMETHEUS_STACK_CHART_VERSION="82.1.0"
+[ -z "$PROMETHEUS_STACK_CHART_VERSION" ] && PROMETHEUS_STACK_CHART_VERSION="82.9.0"
 helm upgrade $NS --install prometheus-stack $OCI_CHART \
                  --values  prom-operator/prometheus-stack.yaml \
                  --version $PROMETHEUS_STACK_CHART_VERSION $HELM_WHAT

package/src/init-my-chart/{leaf-chart-templates/Chart.yaml.hbs → Chart.yaml.hbs}

@@ -1,10 +1,10 @@
-name: {{containerName}}
-version: {{chartVersion}}
-appVersion: {{chartAppVersion}}
+name: {{hbsChartName}}
+version: {{hbsChartVersion}}
+appVersion: {{hbsChartAppVersion}}
 description: A leaf chart that depends on the Leverege base chart
 apiVersion: v2
 icon: https://raw.githubusercontent.com/kubernetes/kubernetes/master/logo/logo.svg
 dependencies:
   - name: leverege
-    version: "^1.0.0"
+    version: "^1.0.1"
     repository: "oci://us-docker.pkg.dev/leverege-registry/base-charts"

package/src/init-my-chart/Chartwright.mjs

@@ -1,69 +1,152 @@
 // Chartwright.mjs
+/* eslint-disable security/detect-non-literal-fs-filename */
+import fs from 'node:fs'
+
+import chalk from 'chalk'
+import Handlebars from 'handlebars'
+import YAML from 'yaml'
+
 import * as Utils from '../Utils.mjs'
 
-import {
-  BUILD_TOOLS_ROOT,
-} from '../Utils.mjs'
-
-const DEFAULT_LIVENESS = {
-  httpGet : {
-    path : '/livenessProbe',
-    port : 'http'
-  },
-  initialDelaySeconds : 30,
-  periodSeconds : 60
-} 
-const DEFAULT_READINESS = {
-  httpGet : {
-    path : '/healthz',
-    port : 'http'
-  },
-  initialDelaySeconds : 30,
-  periodSeconds : 10
+// handlebar variables available in the templates:
+const hbsContext = {
+  hbsChartName : '',
+  hbsChartVersion : '1.0.0',
+  hbsChartAppVersion : 'v1.0.0',
+  hbsArtifactRegistry : 'us-docker.pkg.dev/leverege-registry/FOOLDER/images',
+  hbsPartOf : '',
+  hbsServicePreemptible : true,
+  hbsRedisEnabled : true,
+  hbsAutoScalingEnabled : true
+}
+
+const PRESERVE_SECTIONS = new Set( [
+  'config',
+  'resources',
+  'autoscaling',
+  'staticscaling',
+  'pdb',
+  'probes',
+  'redis',
+] )
+
+// Helper functions
+const warnDeprecatedSections = ( doc, sections ) => {
+  let deprecationWarningShown = false
+  for ( const key of sections ) {
+    if ( !doc.has( key ) ) continue
+    Utils.log( `${chalk.yellow.bold( '⚠️  WARNING:' )} Deprecated values.yaml section detected => ${chalk.red.bold( key )}` )
+    deprecationWarningShown = true
+  }
+
+  if ( deprecationWarningShown ) {
+    Utils.log( chalk.yellow( 
+      '\nPlease remove any deprecated sections and migrate to the new configuration options as needed.'
+    ) )
+  }
 }
 
 export class Chartwright {
-  constructor( { cwd = process.cwd(), debug = false } = {} ) {
-    this.cwd = cwd
-    this.chartTemplate = `${BUILD_TOOLS_ROOT}/init-my-chart/leaf-chart-templates`
-    this.debug = debug
+  constructor( { Container, debug = false } = {} ) {
+    this.Container = Container
+    this.helmTemplates = `${Utils.BUILD_TOOLS_ROOT}/init-my-chart/leaf-chart-templates`
+    this.debug = debug    
+    this.templates = {}
+  }
 
-    this.pkg = null
-    this.existingChart = null
+  // This function takes a Handlebars template string and a context object,
+  // renders the template with the context, and parses the result as YAML,
+  // returning the resulting object.
+  async renderHbsTemplate( hbsTemplateFileName, context ) {
+    // TODO: better error handling here - if the template file doesn't exist,
+    // or if the rendered template isn't valid YAML, we should handle those
+    // errors gracefully and provide useful error messages.
+    const hbsTemplate = fs.readFileSync( hbsTemplateFileName, 'utf8' )
+    const compiled = Handlebars.compile( hbsTemplate )
+    const rendered = compiled( context )
+    return rendered
   }
 
-  async init() {
-    this.pkg = await Utils.parsePackageJson( { cwd : this.cwd } )
-    this.existingChart = await Utils.parseHelmChart( )
-    return this
+  // Load the handlebars templates for Chart.yaml and values.yaml into memory.
+  async loadTemplates() {
+    this.templates.Chart = await this.renderHbsTemplate( `${Utils.BUILD_TOOLS_ROOT}/init-my-chart/Chart.yaml.hbs`, hbsContext )
+    this.templates.Values = await this.renderHbsTemplate( `${Utils.BUILD_TOOLS_ROOT}/init-my-chart/values.yaml.hbs`, hbsContext )
   }
 
-  analyze() {
-    return {
-      serviceName : this.pkg?.name,
-      hasExistingChart : !!this.existingChart,
-      templateLoaded : !!this.templateChart
+  // Adds missing top-level keys from base into existing WITHOUT overwriting.
+  // IMPORTANT: skips PRESERVE_SECTIONS here; those are handled separately.
+  mergeTopLevelAddMissing( baseDoc, existingDoc ) {
+    const baseRoot = baseDoc.contents
+    const existingRoot = existingDoc.contents
+  
+    if ( !YAML.isMap( baseRoot ) || !YAML.isMap( existingRoot ) ) return
+  
+    for ( const pair of baseRoot.items ) {
+      const key = pair.key?.value ?? pair.key
+      if ( PRESERVE_SECTIONS.has( key ) ) continue
+  
+      if ( !existingRoot.has( key ) ) {
+        // Push the whole Pair node to preserve comments attached to it
+        existingRoot.items.push( pair )
+      }
     }
   }
 
-  bootstrap() {
-    // TODO: create new chart from template
-  }
+  // Preserve sections: if existing has it, keep it untouched.
+  // If existing is missing it and base has it, copy it in (with comments).
+  applyPreserveSections( baseDoc, existingDoc ) {
+    for ( const key of PRESERVE_SECTIONS ) {
+      const existingNode = existingDoc.get( key, true )
+      if ( existingNode ) continue // keep existing exactly
 
-  morph() {
-    // TODO: transform existing chart to new base structure
+      const baseNode = baseDoc.get( key, true )
+      if ( baseNode ) {
+        existingDoc.set( key, baseNode ) // brings comments from base node
+      }
+    }
   }
 
-  async scribe() {
-    // TODO: write resulting chart to disk
+  // Bootstraps a helm chart layout by copying the template structure and rendering the
+  // main files with the provided context. Optionally can suppress writing values.yaml
+  // if the intention is to preserve an existing one.
+  async bootstrap( { Container, suppressValues = false } ) {
+    // load the templates and render them with the context to get the initial 
+    // Chart.yaml and values.yaml content
+    await this.loadTemplates()
+
+    // replicate the chart structure from the templates directory
+    fs.mkdirSync( 'helm' )
+    fs.cpSync( `${Utils.BUILD_TOOLS_ROOT}/init-my-chart/leaf-chart-templates`, 'helm', { recursive : true } )
+    fs.writeFileSync( 'helm/Chart.yaml', this.templates.Chart )
+    if ( !suppressValues ) {
+      fs.writeFileSync( 'helm/values.yaml', this.templates.Values )
+    }
   }
 
-  dump() {
-    if ( !this.debug ) return
-    console.dir( {
-      pkg : this.pkg,
-      existingChart : this.existingChart,
-      templateChart : this.templateChart
-    }, { depth : 4 } )
+  // Scribes the new chart by bootstrapping the structure and then merging the existing
+  // values.yaml with the new template, preserving existing values and comments where possible.
+  // This allows us to update the chart structure and add any new configuration options without
+  // overwriting user customizations in values.yaml.
+  async scribe( { Container, Chart, Values, ValuesText, Registry } ) {
+    hbsContext.hbsChartName = Container
+    hbsContext.hbsChartVersion = Chart.version
+    hbsContext.hbsChartAppVersion = Chart.appVersion
+    hbsContext.hbsArtifactRegistry = Registry
+    hbsContext.hbsPartOf = Values.partOf
+    await this.bootstrap( { Container, suppressValues : true } )
+
+    // Now we need to smart merge the values.yaml template while preserving existing sections
+    const baseValuesDoc = YAML.parseDocument( this.templates.Values )
+    const existingValuesDoc = YAML.parseDocument( ValuesText )
+    // prune obsolete sections and write the merged values.yaml back to disk, preserving formatting and comments
+    const OBSOLETE_SECTIONS = [ 'ingressRoute', 'loadbalancer', 'udploadbalancer' ]
+    warnDeprecatedSections( existingValuesDoc, OBSOLETE_SECTIONS )
+
+    this.mergeTopLevelAddMissing( baseValuesDoc, existingValuesDoc )
+    this.applyPreserveSections( baseValuesDoc, existingValuesDoc )
+
+    // Final write of the merged values.yaml, preserving formatting and comments as much as possible
+    fs.writeFileSync( 'helm/values.yaml', existingValuesDoc.toString( { lineWidth : 0, doubleQuotedAsJSON : true } ) )
   }
-}
+
+}

package/src/init-my-chart/init-my-chart.mjs

@@ -2,16 +2,13 @@
 /* eslint-disable security/detect-non-literal-fs-filename */
 
 import fs from 'node:fs'
-import path from 'node:path'
 
 import { program } from 'commander'
 import chalk from 'chalk'
-import handlebars from 'handlebars'
 
 import {
   analyzeRepository,
   condir,
-  debug,
   enableDebug,
   errorExit,
   log, 
@@ -27,54 +24,52 @@ program
   .parse()
   
 const options = program.opts()
-const chartwright = await( new Chartwright( { debug : options.debug } ) ).init()
+if ( options.debug ) {
+  enableDebug() // enables debugging in the Utils functions
+}
 
 const repo = await analyzeRepository()
-
 if ( !repo.isGitRepo ) {
   errorExit( '***ERROR: This command must be run inside a git repository - run "git init" to proceed' )
 }
+const Container = repo.containerName
+const chartwright = new Chartwright( { Container, debug : options.debug } )
 
-if ( options.debug ) {
-  enableDebug()
-}
-
+// If there isn't an existing helm chart then we just bootstrap the new chart structure
+// from the current helm template and we're done. This will most likely be the case after
+// a new service has been bootstrapped with ignition.
 if ( !repo.helmChart ) {
-  log( '*** Bootstrap with the Default Helm Template' ) // TODO: finish this
-  process.exit( 1 )
+  await chartwright.bootstrap( { Container } )
+  process.exit( 0 )
 }
 
-const { Chart, Values, Registry, } = repo.helmChart
-const gitRoot = repo.gitRoot
-
-if ( fs.existsSync( `${gitRoot}/helm-old` ) ) {
+// Safety First: if there is already an older helm chart, we don't want to overwrite it - we
+// want the user to review / intervene before proceeding.
+const packageRoot = repo.currentDir
+if ( fs.existsSync( `${packageRoot}/helm-old` ) ) {
   errorExit( '***ERROR: An old helm chart structure already exists - cannot proceed' )
 }
 
 let movedOldHelm = false
-if ( fs.existsSync( `${gitRoot}/helm` ) && !options.dryRun ) {
-  fs.renameSync( `${gitRoot}/helm`, `${gitRoot}/helm-old` )
+if ( fs.existsSync( `${packageRoot}/helm` ) && !options.dryRun ) {
+  fs.renameSync( `${packageRoot}/helm`, `${packageRoot}/helm-old` )
   movedOldHelm = true
 }
 
-debug( { initMyChartRoot, movedOldHelm, repo }, '<==InitMyChart Root Variables' )
+const {
+  Chart,
+  Values,
+  valuesText : ValuesText,
+  imageRegistry : Registry,
+} = repo.helmChart
 
 if ( options.dryRun ) {
-  condir( { Chart, Values, Registry, }, chalk.green( '=== Analyzed Repository Helm Chart Info ===' ) )
-  condir( { chartOutput, valuesOutput }, chalk.green( '=== Generated Helm Chart Outputs ===' ) )
-  chartwright.dump()
-  log( { chartOutput }, chalk.green( '=== Chart.yaml ===' ) )
-  log( { valuesOutput }, chalk.green( '=== values.yaml ===' ) )
+  condir( { repo, Container, Chart, Values, ValuesText, Registry }, chalk.green( '<== Dry Run Mode - No Changes Made' ) )
   process.exit( 0 )
 }
 
-// TODO: I needed to reset this effort because I didn't like the direction it was
-//      going in - I want to focus on the transformation of the existing chart to
-//      the new structure, and less on the templating aspect. The templating can be
-//      a secondary step that we add after we have the transformation working. For
-//      now, I will focus on the analysis and transformation of the existing chart,
-//      and then we can add the templating aspect later.
+await chartwright.scribe( { Container, Chart, Values, ValuesText, Registry } )
 
 if ( movedOldHelm ) {
-  log( chalk.yellow( 'An existing helm chart was detected and moved to helm-old. Please review and migrate any customizations as needed.' ) )
+  log( chalk.yellow( '\nAn existing helm chart was detected and moved to helm-old. Please review and migrate any customizations as needed.' ) )
 }