@leverege/build-tools 2.43.5 → 2.43.7

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@leverege/build-tools",
-  "version": "2.43.5",
+  "version": "2.43.7",
   "description": "A collection of build / support tools for Leverege developers",
   "main": "index.js",
   "repository": {
@@ -64,7 +64,7 @@
     "ms": "^2.1.3",
     "npm-registry-fetch": "^16.1.0",
     "parse-gitignore": "^2.0.0",
-    "read-pkg": "^8.1.0",
+    "read-pkg": "^9.0.0",
     "readline-sync": "^1.4.10",
     "semver": "^7.5.4",
     "simple-git": "^3.20.0",
@@ -72,6 +72,6 @@
   },
   "devDependencies": {
     "@leverege/eslint-config-leverege": "^4.2.0",
-    "npm": "^10.2.0"
+    "npm": "^10.2.3"
   }
 }

package/src/Utils.mjs

@@ -1,19 +1,27 @@
+import { existsSync, readdirSync, readFileSync } from 'node:fs'
+
 import chalk from 'chalk'
 import { $ } from 'execa'
+import YAML from 'js-yaml'
 
 const debugEnabled = process.env.BUILD_TOOLS_DEBUG === '1'
 
 /* eslint-disable no-console */
 export const err = console.error
 export const log = console.log
+export const condir = ( obj, text = '' ) => {
+  console.log( `${chalk.yellow( '\n***DEBUG:' )} ${chalk.green( text )}` )
+  console.dir(
+    { obj, },
+    { depth : 8, colors : true },
+  )
+}
 export const debug = ( obj, text ) => {
-  if ( debugEnabled ) {
-    console.dir( {
-      obj,
-      text : `${chalk.yellow( '***DEBUG:' )} ${chalk.green( text )}`,
-    },
-    { depth : 8, colors : true }, )
-  }
+  if ( debugEnabled ) { condir( obj, text ) }
+}
+export const errorExit = ( error, opts = { errorCode : 1 } ) => {
+  console.error( `\n${chalk.red.bold( error )}\n` )
+  if ( opts?.errorCode ) { process.exit( opts.errorCode ) }
 }
 /* eslint-enable no-console */
 
@@ -30,7 +38,102 @@ export const shellCmd = async ( cmdstr, opts = {} ) => {
 
     return results.stdout
   } catch ( error ) {
-    debug( { error } )
+    debug( { error }, '<==shellCmd' )
+    throw error
+  }
+}
+
+export const getGcpSecret = async ( gcpProject, secretName, opts ) => {
+  const secretFetchCmd = `gcloud secrets versions access latest --project=${gcpProject} --secret=${secretName}`
+  let secretString
+  try {
+    secretString = await shellCmd( secretFetchCmd )
+  } catch ( error ) {
+    debug( { error }, '<==getGcpSecret' )
+    throw error
+  }
+
+  if ( !opts?.isJSON ) { return secretString }
+
+  try {
+    const secretJSON = JSON.parse( secretString )
+    return secretJSON
+  } catch ( error ) {
+    debug( { error }, '<==getGcpSecret JSON.parse' )
+    throw error
+  }
+}
+
+export const getGitRootDirectory = async () => {
+  try {
+    const gitRoot = await shellCmd( 'git rev-parse --show-toplevel' )
+    debug( { gitRoot }, '<==getGitRootDirectory' )
+    return gitRoot
+  } catch ( error ) {
+    debug( { error }, '<==getGitRootDirectory' )
     throw error
   }
 }
+
+export const parseJsonFile = async ( jsonFile ) => {
+  /* eslint-disable security/detect-non-literal-fs-filename */
+  if ( existsSync( jsonFile ) ) {
+    try {
+      const json = JSON.parse( readFileSync( jsonFile, 'utf8' ) )  
+      return json
+    } catch ( error ) {
+      throw new Error( `${error} in ${jsonFile}` )
+    }
+  } 
+  throw new Error( `File ${jsonFile} does not exist` )
+  /* eslint-enable security/detect-non-literal-fs-filename */
+}
+
+// Parses the ./package.json file and verifies there is a properly formatted
+// leverege.registry section present.
+export const parsePackageJson = async ( packageFileName ) => {
+  const packageJson = await parseJsonFile( packageFileName )
+
+  const leveregeClauseError = `leverege.registry statement, it should
+       resemble something like:
+
+    "leverege": {
+      "registry": "us-docker.pkg.dev/leverege-registry/<REPO NAME>"
+    },
+
+`
+
+  if ( !packageJson?.leverege?.registry ) {
+    errorExit( `Error: package.json is missing a proper ${leveregeClauseError}` )
+  }
+
+  const artifacts = packageJson.leverege.registry.split( '/' )
+  if ( artifacts.length !== 3 ) {
+    errorExit( `Error: package.json has a malformed ${leveregeClauseError}` )
+  }
+
+  return {
+    artifactRegistry : {
+      registry : packageJson.leverege.registry,
+      region : artifacts[0],
+      project : artifacts[1],
+      repository : artifacts[2],
+    },
+    ...packageJson,
+  }
+}
+
+export const parseHelmChart = async ( helmroot ) => {
+  /* eslint-disable security/detect-non-literal-fs-filename */
+  const chartFiles = readdirSync( helmroot, { encoding : 'utf8', recursive : true } )
+  const chartYaml = YAML.load( readFileSync( `${helmroot}/Chart.yaml`, 'utf-8' ) )
+  const valuesYaml = YAML.load( readFileSync( `${helmroot}/values.yaml`, 'utf-8' ) )
+  /* eslint-enable security/detect-non-literal-fs-filename */
+  return ( {
+    files : chartFiles,
+    yaml : {
+      'Chart.yaml' : chartYaml,
+      'values.yaml' : valuesYaml
+    }
+  } )
+}

package/src/bash-funcs

@@ -122,6 +122,7 @@ SKIPPING
 }
 
 function addHelmRepo() {
+  [ ! -z "$SKIP_ADD_HELM_REPO" ] && return
   local chart=$1
   local url=$2
 

package/src/chart-to-registry.mjs

@@ -8,20 +8,37 @@ import commandLineArgs from 'command-line-args'
 import commandLineUsage from 'command-line-usage'
 import { lt as semverLt } from 'semver'
 
-import { debug, log, shellCmd } from './Utils.mjs'
+import {
+  condir, debug,
+  errorExit,
+  log,
+  getGitRootDirectory,
+  parsePackageJson,
+  parseHelmChart,
+  shellCmd } from './Utils.mjs'
 
 const commandLineOptions = [ // Use commandLineOptions to tie into the Usage statements
   /* eslint-disable max-len */
   {
     name : 'project',
     type : String,
-    default : false,
+    defaultValue : process.env.HELM_CHART_REGISTRY || 'leverege-registry',
     description : '{green the name of the google project containing the npmrc and slack config secrets (default leverege-registry)}',
   },
+  {
+    name : 'region',
+    type : String,
+    defaultValue : 'us-docker.pkg.dev',
+    description : '{green the region of the artifact registry the chart will be pushed to (default us-docker.pkg.dev)}',
+  },
+  {
+    name : 'repository',
+    type : String,
+    description : '{green the target repository to receive the pushed chart}',
+  },
   {
     name : 'help',
     type : Boolean,
-    default : false,
     description : '{green display this help screen}',
   },
   /* eslint-enable max-len */
@@ -54,16 +71,52 @@ if ( args._unknown ) {
 }
 /* eslint-enable no-underscore-dangle */
 
-const gcpProject = args.project ? args.project : process.env.HELM_CHART_REGISTRY || 'leverege-registry'
-
-const minNodejsVersion = '18.13.0'
+const minNodejsVersion = '18.0.0'
 if ( semverLt( process.version, minNodejsVersion ) ) {
-  log( chalk.red( `\n\n***ERROR: must be running at least node ${minNodejsVersion}\n` ) )
-  process.exit( 1 )
+  errorExit( `\n***ERROR: must be running at least node ${minNodejsVersion}\n` )
 }
 
-const getGcpSecretCommand = ( secretName ) => {
-  return `gcloud secrets versions access latest --project=${gcpProject} --secret=${secretName}`
+log( { args }, chalk.yellow( '\n\n***WARNING: Under Construction' ) )
+
+// gcloud auth print-access-token | helm registry login -u oauth2accesstoken \
+// -p ${password} https://us-docker.pkg.dev
+//
+// Get the current access token and use it to login to the registry with helm.
+try {
+  const accessToken = await shellCmd( 'gcloud auth print-access-token' )
+  const registryLoginCommand = `helm registry login -u oauth2accesstoken -p ${accessToken}`
+
+  const loginResult = await shellCmd( `${registryLoginCommand} https://${args.region}` ) // TODO: cmd line param
+  debug( { loginResult }, '<==registryLogin' )
+} catch ( error ) {
+  errorExit( error )
+}
+
+// First of all, fail if we are not in a git repository
+let gitRoot
+try {
+  gitRoot = await getGitRootDirectory()
+} catch ( error ) {
+  errorExit( chalk.red.bold( error ), { errorCode : 5 } )
+}
+
+// Next, parse the package.json file looking for sections of interest
+let packageJson
+try {
+  packageJson = await parsePackageJson( './package.json' )
+} catch ( error ) {
+  errorExit( error )
+}
+
+// And then load up the helm chart too
+let helmChart
+try {
+  helmChart = await parseHelmChart( `${gitRoot}/helm` )
+} catch ( error ) {
+  errorExit( error )
 }
 
-log( chalk.yellow( '\n\n***WARNING: Under Construction' ) )
+condir( { packageJson, helmChart }, '<==Parsed package and chart' )
+
+// helm push chart-tarball oci://REGISTRY_NAME/REPOSITORY_NAME/charts/SERVICE
+// helm install my-release oci://REGISTRY_NAME/REPOSITORY_NAME/redis

package/src/circleate.sh

@@ -241,6 +241,7 @@ workflows:
           requires:
             - install
       - coverage:
+          context: npm
           requires:
             - install
       - docs:
@@ -413,6 +414,7 @@ workflows:
           requires:
             - install
       - coverage:
+          context: npm
           requires:
             - install
       - docs:

package/src/dirty-git.sh

@@ -8,9 +8,9 @@ CLEAN="\e[32m✔\e[0m"
 DIRTY="\e[31m✘\e[0m"
 STASH="\e[33;44m$\e[0m"
 AHEAD="\e[34;42m!\e[0m"
-#NOHED="\e[31m✖\e[0m"
 NOHED="\e[31;43m⚑\e[0m"
 LOCKD="\e[35m\e[0m"
+ELAS7="\e[33m7\e[0m"
 
 clear
 printf "${CLEAN} clean  ${DIRTY} dirty  ${STASH} stashed ${AHEAD} unpushed ${NOHED} no remote ${LOCKD} locked\n\n"
@@ -22,14 +22,21 @@ do
   [[ `git status --porcelain` ]] && status="$DIRTY" || status="$CLEAN"
   scount=`git stash list | wc -l`
   [ $scount -gt 0 ] && stash="\e[33;44m$\e[0m" || stash=" "
-  ahead=`git rev-list FETCH_HEAD..HEAD --count 2>/dev/null`
-  if [ -z "$ahead" ];
+  ahead=`git rev-parse --abbrev-ref $branch@{upstream} 2>/dev/null` # check on remote first
+  if [ $? -gt 0 ];
   then
     ahead="$NOHED" # "✖" # "❓"
   else
-    [ $ahead -gt 0 ] && ahead="$AHEAD" || ahead=" "
+    ahead=`git rev-list FETCH_HEAD..HEAD --count 2>/dev/null`
+    if [ -z "$ahead" ];
+    then
+      ahead="$NOHED" # "✖" # "❓"
+    else
+      [ $ahead -gt 0 ] && ahead="$AHEAD" || ahead=" "
+    fi
   fi
   [ -f "Versions.json" ] && locked="$LOCKD" || locked=" "
+  [ ! -d "elasticsearch8" ] && elastic="$ELAS7" || elastic=" "
   cd - &>/dev/null
-  printf "${status} %-28s ${stash}${ahead}${locked} %s\n" $dir $branch
+  printf "${status} %-28s ${stash}${ahead}${locked}${elastic} %s\n" $dir $branch
 done

package/src/helm-charts/api-server/values-local.yaml

@@ -1,5 +1,5 @@
 image:
-  tag: # v5.4.4
+  tag: # v5.1.2
 
 config:
   LOG_CONFIG: '{"type":"pino","level":"warn"}'

package/src/helm-charts/authz-server/values-local.yaml

@@ -3,3 +3,5 @@ image:
 
 config:
   LOG_CONFIG: '{"type":"pino","level":"warn"}'
+  IN_MEM_CONTROLLER_ACTIVE: "false"
+  USE_MEMORY_CONTROLLER_CHECK_MODE : "false"

package/src/helm-charts/db-curator-dh/values-local.yaml

@@ -0,0 +1,11 @@
+nameOverride: "db-curator-dh"
+
+image:
+  tag: # v4.2.0
+
+config:
+  LOG_CONFIG: '{"type":"pino","level":"warn"}'
+
+  TSDB_CLEANUP_ENABLED: "true"
+  FIREBASE_CLEANUP_ENABLED: "false"
+  TSDB_HOST: "timescale-dense-history-postgresql"

package/src/helm-charts/grafana/helmup.plugin

@@ -4,7 +4,7 @@ showInstalling "Grafana"
 
 addHelmRepo grafana https://grafana.github.io/helm-charts
 
-[ -z "$GRAFANA_CHART_VERSION" ] && GRAFANA_CHART_VERSION="6"
+[ -z "$GRAFANA_CHART_VERSION" ] && GRAFANA_CHART_VERSION="7"
 helm upgrade --install grafana grafana/grafana \
   --namespace monitoring --create-namespace \
   --values grafana/grafana-local.yaml \

package/src/helm-charts/messenger/values-local.yaml

@@ -1,5 +1,5 @@
 image:
- tag: # v2.7.0
+ tag: # v2.8.0
 
 config:
   LOG_CONFIG: '{"type":"pino","level":"warn"}'

package/src/helm-charts/overdose/values-local.yaml

@@ -0,0 +1,18 @@
+image:
+  tag: # 1.2.1
+
+config:
+  LOG_CONFIG: '{"type":"pino","level":"info"}'
+
+resources:
+  limits:
+    cpu: 500m
+    memory: 1024Mi
+  requests:
+    cpu: 500m
+    memory: 1024Mi
+
+autoscaling:
+  minReplicas: 6
+  maxReplicas: 16
+  averageCPU: 75

package/src/helm-charts/redis/helmup.plugin

@@ -2,11 +2,10 @@
 
 showInstalling "Redis"
 
-addBitnamiRepo latest
+OCI_CHART="oci://registry-1.docker.io/bitnamicharts/redis"
 
-[ -z "$REDIS_CHART_VERSION" ] && REDIS_CHART_VERSION="18"
-helm upgrade --install redis bitnami/redis    \
+[ -z "$REDIS_CHART_VERSION" ] && REDIS_CHART_VERSION="18.3.1"
+
+helm upgrade --install redis $OCI_CHART       \
              --values  redis/redis-local.yaml \
              --version $REDIS_CHART_VERSION $HELM_WHAT
-
-removeHelmRepo bitnami

package/src/helm-charts/resource-server/values-local.yaml

@@ -1,5 +1,5 @@
 image:
-  tag: # v1.0.1
+  tag: # v1.1.0
 
 config:
   LOG_CONFIG: '{"type":"pino","level":"warn"}'

package/src/helm-charts/traefik/helmup.plugin

@@ -6,7 +6,7 @@ TRAEFIK_NAMESPACE="traefik"
 
 addHelmRepo traefik https://helm.traefik.io/traefik
 
-[ -z "$TRAEFIK_CHART_VERSION" ] && TRAEFIK_CHART_VERSION="24"
+[ -z "$TRAEFIK_CHART_VERSION" ] && TRAEFIK_CHART_VERSION="25"
 helm upgrade --install traefik traefik/traefik \
              --namespace $TRAEFIK_NAMESPACE --create-namespace \
              --values  traefik/traefik-local.yaml \

package/src/helm-charts/transponder-bq/values-local.yaml

@@ -1,7 +1,7 @@
 nameOverride: transponder-bq
 
 image:
-  tag: # v5.0.1
+  tag: # v5.0.3
 
 config:
   TRANSPONDER_RUN_MODE: "bigQueryJson"

package/src/helm-charts/transponder-rt/values-local.yaml

@@ -1,7 +1,7 @@
 nameOverride: transponder-rt
 
 image:
-  tag: v5.0.1
+  tag: # v5.0.3
 
 config:
   TRANSPONDER_RUN_MODE: "rt"

package/src/helm-charts/transponder-tsdb/values-local.yaml

@@ -7,7 +7,7 @@
 nameOverride: transponder-tsdb
 
 image:
-  tag: # v5.0.1
+  tag: # v5.0.3
 
 config:
   TRANSPONDER_RUN_MODE: "timescale"
@@ -15,7 +15,7 @@ config:
 
   TRANSPORT_CONFIG: '{
     "type": "pubsub",
-    "projectId": "OVH:<PROJECT_NAME>",
+    "projectId": "OVH:<PROJECT_ID>",
     "publisher": {
         "batching": {
           "maxMilliseconds": 0,

package/src/helmup.sh

@@ -31,8 +31,10 @@ PLATFORM=(
 
 AUXILIARY=(
   geotile-server
+  overdose
   pubsub-pulse
   push-notifier
+  pusher
   vin-decoder-server
 )
 
@@ -690,8 +692,8 @@ CATBACKUP
         installDefaultConfigs
         ;;
 
-      db-curator*)
-        # db-curator / db-curator-dh diffs are defined in values-local.yaml
+      db-curator-dh)
+        # db-curator-dh is a variant of db-curator with diff controlled in values-local.yaml
         HELMUP="helm upgrade --install $SERVICE leverege/db-curator --values $SERVICE/values.yaml $CHARTVER $HELM_WHAT"
         doHelmup "$HELMUP"
         ;;

package/src/pkgck.sh

@@ -54,7 +54,7 @@ check_eslint () {
 
 check_license () {
   local C_stat=$(meh)
-  local lic="`node -p \"require('./package.json').scripts.licenseCheck !== undefined\"`"
+  local lic="`node -p \"require('./package.json').scripts.licenseCheck == undefined\"`"
   [ $lic == true ] && C_stat=$(good)
   printf $C_stat
 }

package/src/send-to-slack.mjs

@@ -42,7 +42,6 @@ const slackSendOptions = [ // Use slackSendOptions to tie into the Usage stateme
     default : false,
     description : '{green display this help screen}',
   },
-  /* eslint-enable max-len */
 ]
 
 const sections = [
@@ -100,14 +99,14 @@ try {
 }
 
 // slack out the details to the configured channel
-const account = await shellCmd( 'gcloud config get account' )
 const slackData = {
-  text : args.message.replace(/^\"|\"$/g,''), // strip pesky leading/trailing "
+  text : args.message.replace( /^"|"$/g, '' ), // strip pesky leading/trailing "
   channel : slackConfig.channel,
 }
 
 if ( process.env.BUILD_TOOLS_DEBUG || args.dryRun ) {
-  log( { slackConfig, slackData }, '<==Slack skipped' )
+  const account = await shellCmd( 'gcloud config get account' )
+  log( { account, slackConfig, slackData }, '<==Slack skipped' )
 } else {
   try {
     const fetchStatus = await fetch( slackConfig.url, {