@leverege/build-tools 2.61.7 → 2.62.0-alpha.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@leverege/build-tools",
-  "version": "2.61.7",
+  "version": "2.62.0-alpha.1",
   "description": "A collection of build / support tools for Leverege developers",
   "main": "index.js",
   "repository": {
@@ -16,13 +16,14 @@
   "bin": {
     "build-tools": "src/build-tools.mjs",
     "build-and-push": "src/build-and-push.sh",
-    "build-cnpg-image": "src/build-cnpg-image.mjs",
     "chart-to-registry": "src/chart-to-registry.mjs",
     "circleate": "src/circleate.sh",
+    "cnpg-backup": "src/cnpg-backup.mjs",
     "decrypt-secrets": "src/decrypt-secrets.sh",
     "dirty-git": "src/dirty-git.sh",
     "dockreate": "src/dockreate.sh",
     "docker-to-registry": "src/docker-to-registry.mjs",
+    "docker-to-registry.sh": "src/docker-to-registry.sh",
     "encrypt-secrets": "src/encrypt-secrets.sh",
     "firebaseDeploy": "src/firebaseDeploy.mjs",
     "firebaseServe": "src/firebaseServe.mjs",
@@ -36,7 +37,9 @@
     "helminit": "src/helmup.sh",
     "helmwhat": "src/helmup.sh",
     "hook-and-release": "src/hook-and-release.mjs",
+    "k8cryo": "src/k8cryo.sh",
     "k8scale": "src/k8scale.sh",
+    "k8thaw": "src/k8cryo.sh",
     "k8x": "src/k8x.sh",
     "klog": "src/klog.sh",
     "npm-link": "src/npm-link.sh",
@@ -50,7 +53,6 @@
     "refresh-npm-token": "src/refresh-npm-token.mjs",
     "refresh-py-idx": "src/refresh-py-idx.sh",
     "send-to-slack": "src/send-to-slack.mjs",
-    "snapshot-cnpg": "src/snapshot-cnpg.mjs",
     "tag-release": "src/tag-release.mjs",
     "unleash": "src/unleash.mjs"
   },
@@ -59,28 +61,31 @@
   "dependencies": {
     "@google-cloud/artifact-registry": "^3.5.0",
     "@leverege/jsdoc-template": "^1.0.1",
+    "ansi-colors": "^4.1.3",
     "chalk": "^5.4.1",
     "command-line-args": "^6.0.1",
     "command-line-usage": "^7.0.3",
     "deepmerge": "^4.3.1",
     "enquirer": "^2.4.1",
     "execa": "^9.5.2",
-    "glob": "^11.0.1",
+    "glob": "^11.0.0",
     "handlebars": "^4.7.8",
-    "inquirer": "^12.4.1",
+    "inquirer": "^12.3.0",
     "js-yaml": "^4.1.0",
     "jsdoc": "^4.0.4",
     "ms": "^2.1.3",
     "npm-registry-fetch": "^18.0.2",
     "package-up": "^5.0.0",
     "parse-gitignore": "^2.0.0",
+    "read-pkg": "^9.0.1",
     "readline-sync": "^1.4.10",
-    "semver": "^7.7.1",
+    "semver": "^7.6.3",
     "simple-git": "^3.27.0",
-    "zx": "^8.3.2"
+    "toml": "^3.0.0",
+    "zx": "^8.3.0"
   },
   "devDependencies": {
     "@leverege/eslint-config-leverege": "^5.0.1",
-    "npm": "^11.1.0"
+    "npm": "^11.0.0"
   }
 }

package/src/Docker.mjs

@@ -24,7 +24,7 @@ const dockerfileNodeVersion = 'jod-alpine'
 // the build-tools repository, but it just became simpler to pull the contents
 // directly into these variables and forego the file loading.
 //
-const dockerfilePluginTemplate = `
+const pluginTemplate = `
 # Dockerfile.plugin - optionally extend the service Docker image
 #
 # This file may be used to run additional docker commands during the image
@@ -38,14 +38,6 @@ const dockerfilePluginTemplate = `
 # Keep in mind that the Alpine Linux base image is used to keep image
 # footprints small, so adding packages "just because" is not considered
 # a best practice.
-`
-const bashrcPluginTemplate = `
-
-# Bashrc.plugin - optionally extend the image .bashrc file
-#
-# This plugin file may contain additional .bashrc settigns / functions to
-# provide on the target image.
-
 `
 const dockerfileTemplate = fs.readFileSync( path.join( DIRNAME, './templates/dockerfileTemplate.hbs' ), 'utf8' )
 const bashrcTemplate = fs.readFileSync( path.join( DIRNAME, './templates/bashrcTemplate.hbs' ), 'utf8' )
@@ -74,10 +66,9 @@ const setPreviousBuild = ( version ) => {
   fs.writeFileSync( previousFile, version )
 }
 
-const generateDockerfile = async ( options ) => {
+const generateDockerfile = ( options ) => {
 
-  const gitReference = await shellCmd( 'git rev-parse --short HEAD' )
-  const settings = { gitReference, ...defaultSettings, ...options }
+  const settings = { ...defaultSettings, ...options }
 
   // defaults to the current "official" version of node unless it gets
   // overridden in the leverege.nodeimg package.json stanza
@@ -94,31 +85,24 @@ const generateDockerfile = async ( options ) => {
     fs.mkdirSync( './docker' ) // TODO: blindly assumes the mkdir succeeds
   }
 
-  // Create a bashrc plugin file if one does not already exist
-  const bashrcPluginFile = './docker/Bashrc.plugin'
-  if ( !fs.existsSync( bashrcPluginFile ) ) {
-    warning( `creating the missing ${bashrcPluginFile} file in the docker directory\n` )
-    fs.writeFileSync( bashrcPluginFile, bashrcPluginTemplate )
-  }
-  const bashrcPlugin = fs.readFileSync( bashrcPluginFile )
-
-  // Build the target bashrc file by using the template plus Bashrc.plugin
   const bashrcFile = './docker/bashrc'
-  if ( fs.existsSync( bashrcFile ) ) {
-    fs.rmSync( bashrcFile ) // start with a clean bashrc
-  }
-
-  try {
-    fs.writeFileSync( bashrcFile, bashrcTemplate )
-    fs.appendFileSync( bashrcFile, bashrcPlugin )
-  } catch ( err ) {
-    errorExit( `***ERROR: failed creation of the ${bashrcFile} file` )
+  // DISABLING exists check. With addition of Bashrc.plugin, the file needs to
+  // always be created or we encounter errors
+  if ( true ) { // !fs.existsSync( bashrcFile ) ) {
+    let bf = bashrcTemplate
+    const bashrcFilePlugin = './docker/Bashrc.plugin'
+    if ( fs.existsSync( bashrcFilePlugin ) ) {
+      const bashrcAdditions = fs.readFileSync( bashrcFilePlugin )
+      bf = `${bf}\n${bashrcAdditions}`
+    }
+    warning( `creating the missing ${bashrcFile} file in the docker directory` )
+    fs.writeFileSync( bashrcFile, bf )
   }
 
   const dockerfilePlugin = './docker/Dockerfile.plugin'
   if ( !fs.existsSync( dockerfilePlugin ) ) {
     warning( `creating the missing ${dockerfilePlugin} file in the docker directory` )
-    fs.writeFileSync( dockerfilePlugin, dockerfilePluginTemplate )
+    fs.writeFileSync( dockerfilePlugin, pluginTemplate )
   }
 
   try {
@@ -130,7 +114,7 @@ const generateDockerfile = async ( options ) => {
     errorExit( `***ERROR: expected the ./build directory to exist - ${chalk.yellow( 'npm run build' )}` )
   }
 
-  // At this point the docker plugin file better exist.
+  // At this point the plugin template better exist.
   if ( fs.existsSync( dockerfilePlugin ) ) {
     settings.pluginfile = fs.readFileSync( dockerfilePlugin )
   } else {
@@ -189,22 +173,12 @@ const buildContainerImage = async ( {
   // TODO: This is janky - I'll clean it up later
   // NOTE: this is dealing with raw filesystem interactions, meaning there
   //   isn't a shell performing file globbing and what not
-  //
-  // Steps taken here attempt to minimize impact on legacy build setups
-  //   1. rename build dir to workspace
-  //   2. create the actual build dir and move workspace into it
-  //   3. move the Dockerfile into build and make a copy for image root
-  //   4. copy the bashrc file into build and .bashrc on the image root
-  //   5. drop a copy of dev's npmrc to use for build process
-  //   6. drop a copy of package.json and the lock into the image root
-  //
   fs.renameSync( './build', './workspace' )
   fs.mkdirSync( './build' )
   fs.renameSync( './workspace', buildWorkspace )
   fs.renameSync( './docker/Dockerfile', './build/Dockerfile' )
-  fs.cpSync( './build/Dockerfile', `${buildWorkspace}/Dockerfile` )
-  fs.renameSync( './docker/bashrc', './build/bashrc' )
   fs.cpSync( `${process.env.HOME}/.npmrc`, './build/.npmrc' )
+  fs.cpSync( './docker/bashrc', './build/bashrc' )
   fs.cpSync( './package.json', `${buildWorkspace}/package.json` )
   fs.cpSync( './package-lock.json', `${buildWorkspace}/package-lock.json` )
   if ( isNpmWorkspace ) {
@@ -214,7 +188,7 @@ const buildContainerImage = async ( {
   setPreviousBuild( imageVersion ) // update docker/.previous file
 
   const { fullName : builderName } = await getGcpAccount()
-  const gcloudBuild = `gcloud builds submit --project ${artifactProject}`
+  const gcloudBuild = `time gcloud builds submit --project ${artifactProject}`
   const gcloudLogs = `--gcs-log-dir ${formCloudBuildBucketName( artifactProject )}/log`
   let gcloudTags = `--tag ${imageName}:${imageVersion}`
   

package/src/DockerPy.mjs

@@ -0,0 +1,257 @@
+import fs from 'node:fs'
+import path from 'node:path'
+import url from 'node:url'
+
+import chalk from 'chalk'
+import handlebars from 'handlebars'
+
+import {
+  errorExit,
+  getDateTimestamp,
+  getGcpAccount,
+  log,
+  proceed,
+  shellCmd,
+  warning
+} from './Utils.mjs'
+
+const DIRNAME = path.dirname( url.fileURLToPath( import.meta.url ) )
+
+// The contents of these variables were initially located in files that live in
+// the build-tools repository, but it just became simpler to pull the contents
+// directly into these variables and forego the file loading.
+//
+const pluginTemplate = `
+# Dockerfile.plugin - optionally extend the service Docker image
+#
+# This file may be used to run additional docker commands during the image
+# build process without needing to maintain a local custom Dockerfile. For
+# example, uncommenting the following docker RUN command will cause the
+# curl and vim packages to be added to the deployed image thus making them
+# available from the pod's command line on k8s:
+#
+# RUN apk update && apk add --no-cache curl vim
+#
+# Keep in mind that the Alpine Linux base image is used to keep image
+# footprints small, so adding packages "just because" is not considered
+# a best practice.
+`
+const dockerfileTemplate = fs.readFileSync( path.join( DIRNAME, './templates/dockerfileTemplatePy.hbs' ), 'utf8' )
+const cloudBuildSteps = fs.readFileSync( path.join( DIRNAME, './templates/cloudBuildSteps.hbs' ), 'utf8' )
+let bashrcTemplate = fs.readFileSync( path.join( DIRNAME, './templates/bashrcTemplate.hbs' ), 'utf8' )
+const bashrcPyTemplate = fs.readFileSync( path.join( DIRNAME, './templates/bashrcTemplatePy.hbs' ), 'utf8' )
+bashrcTemplate = `${bashrcTemplate}\n${bashrcPyTemplate}`
+
+const defaultSettings = {
+  apkadds : '',
+  date : getDateTimestamp(),
+  nodeimage : 'jod-alpine',
+  imageBase : 'node:jod-alpine',
+  npmlogging : process.env.VERBOSE_NPM_LOGGING ? '--ddd' : '--silent',
+  npmVersion : 'npm@11',
+  pluginfile : '# NO PLUGIN',
+  preInstallPluginfile : '# NO PRE-INSTALL PLUGIN',
+  regvers : 'package.version',
+  runuser : 'python',
+}
+
+const previousFile = './docker/.previous'
+
+function getPreviousBuild() {
+  return fs.existsSync( previousFile ) ? fs.readFileSync( previousFile ) : 'FIRST BUILD'
+}
+
+const setPreviousBuild = ( version ) => {
+  fs.writeFileSync( previousFile, version )
+}
+
+function generateDockerfile( options ) {
+
+  const settings = { ...defaultSettings, ...options }
+
+  if ( !settings.imageBase ) {
+    warning( 'using the default image python:3.12.8-slim-bookworm' )
+    settings.image = 'python:3.12.8-slim-bookworm'
+  } else {
+    settings.image = settings.imageBase
+  }
+  // Ensure the presence of the docker dir with plugin and bashrc - this code
+  // could probably be tightened up a little.
+  try {
+    fs.readdirSync( './docker' )
+  } catch ( err ) {
+    warning( 'creating the missing ./docker directory' )
+    fs.mkdirSync( './docker' ) // TODO: blindly assumes the mkdir succeeds
+  }
+
+  const bashrcFile = './docker/bashrc'
+  let bf = bashrcTemplate
+  const bashrcFilePlugin = './docker/Bashrc.plugin'
+  if ( fs.existsSync( bashrcFilePlugin ) ) {
+    const bashrcAdditions = fs.readFileSync( bashrcFilePlugin )
+    bf = `${bf}\n${bashrcAdditions}`
+  }
+  fs.writeFileSync( bashrcFile, bf )
+
+  const dockerfilePlugin = './docker/Dockerfile.plugin'
+  if ( !fs.existsSync( dockerfilePlugin ) ) {
+    warning( `creating the missing ${dockerfilePlugin} file in the docker directory` )
+    fs.writeFileSync( dockerfilePlugin, pluginTemplate )
+  }
+
+  try {
+    const build = fs.readdirSync( './build' )
+    if ( build.length === 0 ) {
+      errorExit( `***ERROR: the build dir is empty - ${chalk.yellow( 'npm run build' )}` )
+    }
+  } catch ( err ) {
+    errorExit( `***ERROR: expected the ./build directory to exist - ${chalk.yellow( 'npm run build' )}` )
+  }
+
+  // At this point the plugin template better exist.
+  if ( fs.existsSync( dockerfilePlugin ) ) {
+    settings.pluginfile = fs.readFileSync( dockerfilePlugin )
+  } else {
+    errorExit( `***ERROR: something went wrong with ${dockerfilePlugin} plugin creation` )
+  }
+
+  const dockerfilePrePlugin = './docker/DockerfilePreInstall.plugin'
+  if ( fs.existsSync( dockerfilePrePlugin ) ) {
+    settings.preInstallPluginfile = fs.readFileSync( dockerfilePrePlugin )
+  }
+
+  const compiled = handlebars.compile( dockerfileTemplate, { noEscape : true } )
+  const replaced = compiled( settings )
+  // temporarily write the Dockerfile to the docker subdir - the container
+  // image builder will then move it over to the build subdir
+  const dockerfile = './docker/Dockerfile'
+  try {
+    fs.writeFileSync( dockerfile, replaced )
+  } catch ( err ) {
+    errorExit( `***ERROR: failed to write ${dockerfile}\n${err}` )
+  }
+
+  const previousBuild = getPreviousBuild()
+  const dockerInfo = { dockerfile, previousBuild, ...settings }
+  delete dockerInfo.pluginfile
+  return dockerInfo
+}
+
+function formCloudBuildBucketName( artifactProject ) {
+  return `gs://${artifactProject}_cloudbuild`
+}
+
+async function validateCloudBuildBucket( artifactProject ) {
+  const bucketName = formCloudBuildBucketName( artifactProject )
+  log( chalk.green.bold( `Verifying the build bucket exists => ${bucketName}\n` ) )
+  try {
+    await shellCmd( `gsutil ls -p ${artifactProject} -b ${bucketName}` )
+  } catch ( error ) {
+    errorExit( `***Error: from Docker.validateCloudBuildBucket\n\n${chalk.bold.yellow( error )}` )
+  }
+}
+
+async function buildContainerImage( {
+  artifactProject, artifactRegistry, containerName, 
+  imageVersion, isNpmWorkspace, options
+} ) {
+
+  const useCloudBuildFile = true
+  const buildWorkspace = './build/workspace' // passed to Cloud Build
+  const imageName = `${artifactRegistry}/images/${containerName}`
+
+  if ( fs.existsSync( buildWorkspace ) ) {
+    fs.renameSync( buildWorkspace, `${buildWorkspace}-junk` )
+    try {
+      fs.rmSync( `${buildWorkspace}-junk`, { recursive : true } )
+    } catch ( err ) {
+      log( `***Error: from Docker.buildContainerImage\n\n${chalk.bold.yellow( err )}` )
+    }
+  }
+
+  // TODO: This is janky - I'll clean it up later
+  // NOTE: this is dealing with raw filesystem interactions, meaning there
+  //   isn't a shell performing file globbing and what not
+  fs.renameSync( './build', './workspace' )
+  fs.mkdirSync( './build' )
+  fs.renameSync( './workspace', buildWorkspace )
+  fs.renameSync( './docker/Dockerfile', './build/Dockerfile' )
+  // fs.cpSync( `${process.env.HOME}/.npmrc`, './build/.npmrc' )
+  fs.cpSync( './docker/bashrc', './build/bashrc' )
+  // fs.cpSync( './package.json', `${buildWorkspace}/package.json` )
+  // fs.cpSync( './package-lock.json', `${buildWorkspace}/package-lock.json` )
+
+  setPreviousBuild( imageVersion ) // update docker/.previous file
+
+  const { fullName : builderName } = await getGcpAccount()
+  const gcloudBuild = `time gcloud builds submit --project ${artifactProject}`
+  const gcloudLogs = `--gcs-log-dir ${formCloudBuildBucketName( artifactProject )}/log`
+  let gcloudTags = `--tag ${imageName}:${imageVersion}`
+  
+  if ( useCloudBuildFile ) {
+    const cloudBuildFile = './docker/cloudbuild.yaml'
+    gcloudTags = `--config=${cloudBuildFile}`
+    const cbst = handlebars.compile( cloudBuildSteps, { noEscape : true } )
+    const cbs = cbst( { artifactProject, artifactRegistry, imageName, imageVersion } )
+
+    fs.writeFileSync( cloudBuildFile, cbs )
+  }
+
+  log( chalk.green.bold( `
+  *** Submitting Build as ${builderName} ***
+      ${gcloudBuild} \\
+           ${gcloudLogs}  \\
+           ${gcloudTags}
+  ` ) )
+
+  if ( process.env.DRY_RUN === '1' ) {
+    log( '\n***Exiting from DRY_RUN\n' )
+    process.exit( 0 )
+  }
+
+  await shellCmd( `${gcloudBuild} ${gcloudLogs} ${gcloudTags} ./build`, { stdio : 'inherit' } )
+
+  const imageTag = `${defaultSettings.date}-${defaultSettings.nodeimage}-${builderName}`
+  await shellCmd(
+    `gcloud container images add-tag --quiet ${imageName}:${imageVersion} ${imageName}:${imageTag} ${imageName}:latest`
+  )
+}
+
+async function repoCleanup() {
+  // remove garbage from the repo
+  const garbage = [
+    'build',
+    'dist',
+    'docker/.keep',
+    'docker/.npmrc',
+    'docker/.gitignore',
+    'docker/.previous',
+    'docker/Dockerfile',
+    'docker/workspace',
+  ]
+  garbage.forEach( trash => fs.rmSync( trash, { recursive : true, force : true } ) )
+
+  fs.writeFileSync( 'docker/.gitignore', '.previous\nDockerfile' )
+
+  await shellCmd( 'git add -f docker' )
+
+  log( `
+  ${chalk.yellow.bold( '***IMPORTANT***' )}
+
+  Please make the following .gitignore modifications:
+    change /dist to ${chalk.green.bold( '/build' )}
+    remove ${chalk.red.bold( '/docker' )}
+` )
+
+  await proceed( 'Acknowledge the .gitignore mods were made?' )
+}
+
+export default {
+  getPreviousBuild,
+  setPreviousBuild,
+  generateDockerfile,
+  formCloudBuildBucketName,
+  validateCloudBuildBucket,
+  buildContainerImage,
+  repoCleanup,
+}

package/src/Utils.mjs

@@ -54,10 +54,6 @@ export const isString = ( s ) => {
  * General "yes" to proceed function.
  */
 export const proceed = async ( query = 'Do you wish to proceed?' ) => {
-  if ( process.env.CICD_PIPELINE === '1' ) {
-    log( '\n*** CICD Pipeline - skipping user prompts' )
-    return
-  }
   await inquirer.prompt( [
     {
       type : 'confirm',
@@ -403,7 +399,6 @@ export const getRepositoryDescr = async () => {
   const currentDir = process.cwd()
   const workingDir = process.env.PWD
   const gitRoot = await getGitRootDirectory()
-  const isGitRepo = gitRoot !== undefined
   const gitHooksPath = await getGitHooksPath()
   const { branch : gitBranch, upstream : gitUpstream } = await getGitBranchAndUpstream()
   const subPkgs = await glob( `${gitRoot}/packages/**/package.json`, { ignore : '/**/node_modules/**' } )
@@ -425,7 +420,6 @@ export const getRepositoryDescr = async () => {
     gitRoot,
     gitUpstream,
     helmChart,
-    isGitRepo,
     isMonoRepo,
     isNpmWorkspace,
     rootPackageJson,

package/src/bash-funcs

@@ -42,10 +42,6 @@ function clearlyWarn() {
 CLEARWARN
 }
 
-function warning() {
-  clearlyWarn DO_NOT_CLEAR
-}
-
 function warnOnError() {
   [ $1 -ne 0 ] && printf "\n$YELO_WARN $2\n"
 }