@leverege/build-tools 2.51.5 → 2.52.0-beta.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@leverege/build-tools",
-  "version": "2.51.5",
+  "version": "2.52.0-beta.1",
   "description": "A collection of build / support tools for Leverege developers",
   "main": "index.js",
   "repository": {

package/src/Docker.mjs

@@ -40,8 +40,12 @@ COPY ./.npmrc /usr/src/app/.npmrc
 RUN apk --no-cache add openssh-client && \
     apk --update add --no-cache --virtual build-dep g++ gcc libgcc \\
       libstdc++ linux-headers make {{apkadds}} && \
-    npm install -g npm@10 && \
-    npm ci --only=production --ignore-scripts --no-optional {{npmlogging}} && \
+    npm install -g npm@10
+
+{{preInstallPluginfile}}
+
+# Removed --ignore-scripts from npm ci because it was causing issues with python native build
+RUN npm ci --only=production --no-optional {{npmlogging}} && \
     rm -f /usr/src/app/.npmrc /root/.ssh/*
 
 # --------------------------------------------------------------
@@ -67,7 +71,7 @@ COPY --chown=node:node --from=intermediate /usr/src/app /usr/src/app
 
 USER {{runuser}}
 COPY ./bashrc /home/node/.bashrc
-CMD [ "/bin/bash", "-c", "node index.js" ]`
+CMD [ "/bin/bash", "-c", "source /home/node/.bashrc && node index.js" ]`
 
 const pluginTemplate = `
 # Dockerfile.plugin - optionally extend the service Docker image
@@ -132,6 +136,19 @@ dunm() {
   du -sh /usr/src/app/node_modules/* | sort -sh
 }
 `
+const cloudBuildSteps = `
+steps:
+  - name: 'gcr.io/cloud-builders/docker'
+    args:
+      - 'build' 
+      - '--network=cloudbuild'
+      - '-t'
+      - '{{imageName}}:{{imageVersion}}'
+      - '.'
+    
+images:
+  - '{{imageName}}:{{imageVersion}}'
+`
 
 const defaultSettings = {
   apkadds : '',
@@ -139,6 +156,7 @@ const defaultSettings = {
   nodeimage : 'iron-alpine',
   npmlogging : '--silent',
   pluginfile : '# NO PLUGIN',
+  preInstallPluginfile : '# NO PRE-INSTALL PLUGIN',
   regvers : 'package.version',
   runuser : 'node',
 }
@@ -168,8 +186,14 @@ const generateDockerfile = ( settings = defaultSettings ) => {
 
   const bashrcFile = './docker/bashrc'
   if ( !fs.existsSync( bashrcFile ) ) {
+    let bf = bashrcTemplate
+    const bashrcFilePlugin = './docker/Bashrc.plugin'
+    if ( fs.existsSync( bashrcFilePlugin ) ) {
+      const bashrcAdditions = fs.readFileSync( bashrcFilePlugin )
+      bf = `${bf}\n${bashrcAdditions}`
+    }
     warning( `creating the missing ${bashrcFile} file in the docker directory` )
-    fs.writeFileSync( bashrcFile, bashrcTemplate )
+    fs.writeFileSync( bashrcFile, bf )
   }
 
   const dockerfilePlugin = './docker/Dockerfile.plugin'
@@ -194,7 +218,12 @@ const generateDockerfile = ( settings = defaultSettings ) => {
     errorExit( `***ERROR: something went wrong with ${dockerfilePlugin} plugin creation` )
   }
 
-  const compiled = handlebars.compile( dockerfileTemplate )
+  const dockerfilePrePlugin = './docker/DockerfilePreInstall.plugin'
+  if ( fs.existsSync( dockerfilePrePlugin ) ) {
+    defaultSettings.preInstallPluginfile = fs.readFileSync( dockerfilePrePlugin )
+  }
+
+  const compiled = handlebars.compile( dockerfileTemplate, { noEscape : true } )
   const replaced = compiled( settings )
   // temporarily write the Dockerfile to the docker subdir - the container
   // image builder will then move it over to the build subdir
@@ -226,8 +255,10 @@ const validateCloudBuildBucket = async ( artifactProject ) => {
 }
 
 const buildContainerImage = async ( {
-  artifactProject, artifactRegistry, containerName, imageVersion, isNpmWorkspace,
+  artifactProject, artifactRegistry, containerName, 
+  imageVersion, isNpmWorkspace, options
 } ) => {
+  const { useCloudBuildFile } = options || {}
   const buildWorkspace = './build/workspace' // passed to Cloud Build
   const imageName = `${artifactRegistry}/images/${containerName}`
 
@@ -256,14 +287,24 @@ const buildContainerImage = async ( {
   const { fullName : builderName } = await getGcpAccount()
   const gcloudBuild = `time gcloud builds submit --project ${artifactProject}`
   const gcloudLogs = `--gcs-log-dir ${formCloudBuildBucketName( artifactProject )}/log`
-  const gcloudTags = `--tag ${imageName}:${imageVersion}`
+  let gcloudTags = `--tag ${imageName}:${imageVersion} `
+  
+  if ( useCloudBuildFile ) {
+    const cloudBuildFile = './docker/cloudbuild.yaml'
+    gcloudTags = `--config=${cloudBuildFile}`
+    const cbst = handlebars.compile( cloudBuildSteps, { noEscape : true } )
+    const cbs = cbst( { artifactProject, artifactRegistry, imageName, imageVersion } )
+
+    fs.writeFileSync( cloudBuildFile, cbs )
+  }
 
   log( chalk.green.bold( `
-  *** Submitting Build as ${builderName} ***
-      ${gcloudBuild} \\
-           ${gcloudLogs}  \\
-           ${gcloudTags}
-  ` ) )
+    *** Submitting Build as ${builderName} ***
+    ${gcloudBuild} \\
+    ${gcloudLogs}  \\
+    ${gcloudTags}
+    ` ) )
+    
 
   if ( process.env.DRY_RUN === '1' ) {
     log( '\n***Exiting from DRY_RUN\n' )

package/src/docker-to-registry.mjs

@@ -26,8 +26,22 @@ if ( refreshErr && !process.env.BUILD_TOOLS_DEBUG ) {
   log( refreshErr ) // most likely the token refreshed message
 }
 
+const options = {}
+let imageVersion = null
 // Expected to be invoked like docker-to-registry v1.2.3
-const imageVersion = process.argv[2]
+for ( let n = 2; n < process.argv.length; n++ ) {
+  if ( process.argv[n].startsWith( '-' ) ) {
+    const str = process.argv[n].slice( 1 )
+    if ( str.indexOf( '=' ) > 0 ) {
+      const [key, value] = str.split( '=' )
+      options[key] = value
+    } else {
+      options[str] = true
+    }
+  } else {
+    imageVersion = process.argv[n]
+  }
+}
 if ( !imageVersion ) {
   errorExit( '***Error: docker-to-registry requires an image version' )
 }
@@ -151,39 +165,41 @@ if ( giveGuidance ) {
 }
 
 // do chart checks too
-const chart = helmChart?.yaml['Chart.yaml']
-const values = helmChart?.yaml['values.yaml']
-const expectedChartRegistry = `${artifactRegistry}/images`
+if ( options?.helmChecks !== 'false' ) {
+  const chart = helmChart?.yaml['Chart.yaml']
+  const values = helmChart?.yaml['values.yaml']
+  const expectedChartRegistry = `${artifactRegistry}/images`
 
-if ( expectedChartRegistry !== values.image?.registry ) {
-  log( `
-  ${chalk.red.bold( '***Error: mismatched package.json registry and helm/values.yaml' )}
+  if ( expectedChartRegistry !== values.image?.registry ) {
+    log( `
+    ${chalk.red.bold( '***Error: mismatched package.json registry and helm/values.yaml' )}
 
-  The registry specified in the package.json leverege stanza does not line up
-  with the image registry in helm/values.yaml. The naming convention expects
-  the '/images' suffix to be added to the package.json registry string and then
-  stored as the image.registry in the helm/values.yaml file.
+    The registry specified in the package.json leverege stanza does not line up
+    with the image registry in helm/values.yaml. The naming convention expects
+    the '/images' suffix to be added to the package.json registry string and then
+    stored as the image.registry in the helm/values.yaml file.
 
-    Current settings:
-      package.json registry => ${chalk.yellow.bold( artifactRegistry )}
-      values.yaml  registry => ${chalk.red.bold( values.image?.registry )}
+      Current settings:
+        package.json registry => ${chalk.yellow.bold( artifactRegistry )}
+        values.yaml  registry => ${chalk.red.bold( values.image?.registry )}
 
-    Expected settings:
-      values.yaml  registry => ${chalk.green.bold( expectedChartRegistry )}
-  ` )
+      Expected settings:
+        values.yaml  registry => ${chalk.green.bold( expectedChartRegistry )}
+    ` )
 
-  if ( !values.image ) {
-    log( `
-  ${chalk.yellow.bold( '***DEPRECATED: legacy helm charts detected' )}
+    if ( !values.image ) {
+      log( `
+    ${chalk.yellow.bold( '***DEPRECATED: legacy helm charts detected' )}
 
-  The helm chart structure appears to be based on the pre-ignition helm chart
-  layouts. The charts must be upgraded before docker-to-registry can complete
-  its job.
+    The helm chart structure appears to be based on the pre-ignition helm chart
+    layouts. The charts must be upgraded before docker-to-registry can complete
+    its job.
 
-    ` )
-  }
+      ` )
+    }
 
-  process.exit( 1 )
+    process.exit( 1 )
+  }
 }
 
 // Give the summary and the user a chance to proceed or not
@@ -256,7 +272,7 @@ if ( isNpmWorkspace ) {
   await shellCmd( 'npm install --package-lock-only --workspaces false', { stdio : 'inherit' } )
 }
 
-await docker.buildContainerImage( { artifactProject, artifactRegistry, containerName, imageVersion, isNpmWorkspace } )
+await docker.buildContainerImage( { ...repoDescr, options, imageVersion } )
 
 if ( willGitTag ) {
   const tagDescription = `docker_${imageVersion}`

package/src/helm-charts/api-server/values-local.yaml

@@ -17,6 +17,17 @@ config:
 # # timescale
 # PG_HOST: "cnpg-db-tsdb-basic-r.cnpg-operands"
 
+# The settings on the left are for the "standard" legacy chart settings which
+# specifiy the old k8s secrets setups and are the defaults values from the
+# api-server helm chart.
+psqlSecurity:                                   # v----- these are for CNPG -----v
+  PSQL_SECRET_NAME: "authz-postgres"            # "cnpg-db-psql-stack-postgres-pw"
+  PSQL_SECRET_KEY: "postgresql-password"        # "password"
+  TSDB_SECRET_NAME: "postgresql-password"       # "cnpg-db-tsdb-basic-postgres-pw"
+  TSDB_SECRET_KEY: "postgresql-password"        # "password"
+  TSDB_DENSE_SECRET_NAME: "postgresql-password" # "cnpg-db-tsdb-dense-postgres-pw"
+  TSDB_DENSE_SECRET_KEY: "postgresql-password"  # "password"
+
 autoscaling:
   minReplicas: 3
   maxReplicas: 12

package/registry-compass.yaml

@@ -1,20 +0,0 @@
-registry:
-  - root: us-docker.pkg.dev
-  - repositories:
-    - name: stack
-      charts:
-        - api-server
-        - authz-server
-        - emailer
-        - message-processor
-    - name: leverege
-      charts:
-        - pubsub-pulse
-        - pusher
-        - overdose
-    - name: cox-health
-      charts:
-        - actions-server
-        - analytics-server
-        - centrak-healthz
-        - centrak-ingestor

package/src/chart-compass.mjs

@@ -1,96 +0,0 @@
-#!/usr/bin/env node
-/*
- * chart-to-registry will...
- */
-import fs from 'node:fs'
-
-import chalk from 'chalk'
-import commandLineArgs from 'command-line-args'
-import commandLineUsage from 'command-line-usage'
-import { lt as semverLt } from 'semver'
-import YAML from 'js-yaml'
-
-import {
-  condir,
-  debug,
-  errorExit,
-  log,
-  warning,
-  getGitRootDirectory,
-  parsePackageJson,
-  parseHelmChart,
-  shellCmd } from './Utils.mjs'
-
-const commandLineOptions = [ // Use commandLineOptions to tie into the Usage statements
-  /* eslint-disable max-len */
-  {
-    name : 'location',
-    type : String,
-    description : '{green the location of the artifact registry the chart will be pushed to (default us-docker.pkg.dev)}',
-  },
-  {
-    name : 'project',
-    type : String,
-    description : '{green the name of the google project containing the npmrc and slack config secrets (default leverege-registry)}',
-  },
-  {
-    name : 'repository',
-    type : String,
-    description : '{green the target repository to receive the pushed chart}',
-  },
-  {
-    name : 'dry-run',
-    type : Boolean,
-    description : '{yellow perform everything except the actual chart push}',
-  },
-  {
-    name : 'help',
-    type : Boolean,
-    description : '{green display this help screen}',
-  },
-  /* eslint-enable max-len */
-]
-
-const sections = [
-  {
-    header : 'Leverege Helm Chart Compass (for helmup)',
-    content : `{green This tool helps helmup navigate the helm charts stored in the
-artifact-registries.}`
-  },
-  { header : 'Options',
-    optionList : commandLineOptions,
-  },
-]
-
-const args = commandLineArgs( commandLineOptions, { camelCase : true, partial : true } )
-const usage = commandLineUsage( sections )
-
-if ( args.help ) {
-  log( usage )
-  process.exit( 0 )
-}
-
-/* eslint-disable no-underscore-dangle */
-if ( args._unknown ) {
-  log( usage )
-  log( `\nUnrecognized argument [${chalk.bold.red( args._unknown )}]\n` )
-  process.exit( 1 )
-}
-/* eslint-enable no-underscore-dangle */
-
-const minNodejsVersion = '18.0.0'
-if ( semverLt( process.version, minNodejsVersion ) ) {
-  errorExit( `\n***ERROR: must be running at least node ${minNodejsVersion}\n` )
-}
-
-// First of all, fail if we are not in a git repository
-let gitRoot
-try {
-  gitRoot = await getGitRootDirectory()
-} catch ( error ) {
-  errorExit( chalk.red.bold( error ), { errorCode : 5 } )
-}
-
-const chartCompass = YAML.load( fs.readFileSync( './registry-compass.yaml', 'utf8' ) )
-
-condir( { chartCompass }, '<==Navigation' )