@leverege/build-tools 2.39.3 → 2.40.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@leverege/build-tools",
-  "version": "2.39.3",
+  "version": "2.40.0",
   "description": "A collection of build / support tools for Leverege developers",
   "main": "index.js",
   "repository": {
@@ -43,6 +43,7 @@
     "pull-git": "src/pull-git.sh",
     "push-my-chart": "src/push-my-chart.mjs",
     "refresh-npm-token": "src/refresh-npm-token.mjs",
+    "send-to-slack": "src/send-to-slack.mjs",
     "tag-release": "src/tag-release.mjs",
     "unleash": "src/unleash.mjs"
   },
@@ -56,7 +57,7 @@
     "deepmerge": "^4.3.1",
     "enquirer": "^2.4.1",
     "execa": "^8.0.1",
-    "glob": "^10.3.4",
+    "glob": "^10.3.7",
     "js-yaml": "^4.1.0",
     "ms": "^2.1.3",
     "npm-registry-fetch": "^16.0.0",
@@ -64,7 +65,7 @@
     "read-pkg": "^8.1.0",
     "readline-sync": "^1.4.10",
     "semver": "^7.5.4",
-    "simple-git": "^3.19.1",
+    "simple-git": "^3.20.0",
     "zx": "^7.2.3"
   },
   "devDependencies": {

package/src/Utils.mjs

@@ -3,14 +3,19 @@ import { $ } from 'execa'
 
 const debugEnabled = process.env.BUILD_TOOLS_DEBUG === '1'
 
-// eslint-disable-next-line no-console
+/* eslint-disable no-console */
+export const err = console.error
 export const log = console.log
 export const debug = ( obj, text ) => {
   if ( debugEnabled ) {
-    log( obj, `${chalk.yellow( '***DEBUG:' )} ${chalk.green( text )}` )
+    console.dir( {
+      obj,
+      text : `${chalk.yellow( '***DEBUG:' )} ${chalk.green( text )}`,
+    },
+    { depth : 8, colors : true }, )
   }
 }
-export const err = console.error
+/* eslint-enable no-console */
 
 export const shellCmd = async ( cmdstr, opts = {} ) => {
   try {

package/src/bash-funcs

@@ -139,16 +139,18 @@ function removeHelmRepo(){
 }
 
 # initial use for logging helm activities
-function postToSlack() {
+function sendToSlack() {
   local toPost="$@"
   local running="`basename $0`"
   local dts="`date +\"%c\"`"
-  local slackURL="https://hooks.slack.com/services/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"
-  local channel="#dev-ops-helm"
   local account="`gcloud config get account`"
-  local slackmsg="\n--- $running by \`$account\` in \`$GCP_PROJECT_ID\` @ $dts\n \`$toPost\`\n node `node --version` / `build-tools --version` /  helm `helm version --short`\n"
-
-  curl -X POST -H 'Content-type: application/json' \
-           --data "{\"text\":\"$slackmsg\",\"channel\":\"$channel\"}" \
-           "$slackURL" &> $DEVNULL
+  read -rd '' slackmsg<<EOSLACK
+--- $running by \`$account\` in \`$GCP_PROJECT_ID\` @ $dts
+\`$toPost\`
+node `node --version` / `build-tools --version` /  helm `helm version --short`
+EOSLACK
+  send-to-slack --project leverege-registry \
+                --config REFRESH_NPM_SLACK_CONFIG \
+                --message "\"$slackmsg\""
+  # the "\"$var\"" escaped " is necessary for proper message encapsulation
 }

package/src/helm-charts/prometheus/helmup.plugin

@@ -2,9 +2,13 @@
 #
 showInstalling "Prometheus"
 
+# the following two lines may be removed after reaching prom chart 25
+kubectl delete deploy -n monitoring -l app=prometheus,component=server &> $DEVNULL
+kubectl delete deploy -n monitoring -l app.kubernetes.io/name=prometheus &> $DEVNULL
+
 addHelmRepo prometheus-community https://prometheus-community.github.io/helm-charts
 
-[ -z "$PROMETHEUS_CHART_VERSION" ] && PROMETHEUS_CHART_VERSION="15"
+[ -z "$PROMETHEUS_CHART_VERSION" ] && PROMETHEUS_CHART_VERSION="17"
 helm upgrade --install prometheus prometheus-community/prometheus \
              --namespace monitoring --create-namespace \
              --values prometheus/values.yaml \

package/src/helm-charts/prometheus/values-local.yaml

@@ -30,6 +30,10 @@ server:
 pushgateway:
   enabled: false
 
+# name change with chart 19+
+prometheus-pushgateway:
+  enabled: false
+
 # temp set to false if upgrade errors with something like:
 #   Error: UPGRADE FAILED: cannot patch "prometheus-kube-state-metrics" ... field is immutable
 #

package/src/helm-charts/redis/helmup.plugin

@@ -4,7 +4,7 @@ showInstalling "Redis"
 
 addBitnamiRepo latest
 
-[ -z "$REDIS_CHART_VERSION" ] && REDIS_CHART_VERSION="17" # latest chart 17 is redis v7
+[ -z "$REDIS_CHART_VERSION" ] && REDIS_CHART_VERSION="18"
 helm upgrade --install redis bitnami/redis    \
              --values  redis/redis-local.yaml \
              --version $REDIS_CHART_VERSION $HELM_WHAT

package/src/helm-charts/traefik/helmup.plugin

@@ -6,7 +6,7 @@ TRAEFIK_NAMESPACE="traefik"
 
 addHelmRepo traefik https://helm.traefik.io/traefik
 
-[ -z "$TRAEFIK_CHART_VERSION" ] && TRAEFIK_CHART_VERSION="22"
+[ -z "$TRAEFIK_CHART_VERSION" ] && TRAEFIK_CHART_VERSION="24"
 helm upgrade --install traefik traefik/traefik \
              --namespace $TRAEFIK_NAMESPACE --create-namespace \
              --values  traefik/traefik-local.yaml \

package/src/helmdn.sh

@@ -106,7 +106,7 @@ NOHELMDN
   local namespace=${2:-default}
   local helmdn="helm uninstall $1 -n $namespace"
   eval $helmdn
-  postToSlack $helmdn
+  sendToSlack $helmdn
 }
 
 function doNuking() {

package/src/helmup.sh

@@ -30,7 +30,7 @@ PLATFORM=(
 )
 
 SYSTEM=(
-  elasticsearch
+  elasticsearch8
   redis
   postgres
   timescale-db
@@ -172,7 +172,7 @@ INSTALLDEFAULTCONFIGS
 
   printf "\nRunning => `color y \"$kubecmd\"`\n\n"
   eval $kubecmd
-  postToSlack "$kubecmd"
+  sendToSlack "$kubecmd"
 }
 
 function errorIfMissingSecret() {
@@ -548,6 +548,7 @@ PLUGIN_NOEXEC
   printf "\n***Executing local plugin code => `color y $PLUGIN`\n\n" && sleep 2
   export HELM_WHAT
   . $PLUGIN # executing with . allows the plugin to use functions like color
+  sendToSlack "$PLUGIN"
   return 0
 }
 
@@ -595,7 +596,7 @@ doHelmup() {
   [ ! -z "$FROM_VERSIONS" ] && from="`color y \"<= from Versions.json\"`"
   printf "$doing $from\n"
   eval $helmup
-  postToSlack "$helmup"
+  sendToSlack "$helmup"
 }
 
 # not a fan of global vars but bash forced me to - set by getServiceAndChartVersion

package/src/k8scale.sh

@@ -31,6 +31,6 @@ do
   kubectl scale --replicas=$REPLICAS deployment $SERVICE &>/dev/null
   [ $? -eq 1 ] && RESULTS="`color r FAILURE`"
   printf "  $RESULTS => $SERVICE\n"
-  postToSlack "k8scale $DIRECTION $SERVICE"
+  sendToSlack "k8scale $DIRECTION $SERVICE"
 done
 printf "\n\n"

package/src/refresh-npm-token.mjs

@@ -253,13 +253,18 @@ const slackData = {
 if ( process.env.BUILD_TOOLS_DEBUG || args.dryRun ) {
   log( { slackConfig, slackData }, '<==Slack skipped' )
 } else {
-  fetch( slackConfig.url, {
-    method : 'POST',
-    body : JSON.stringify( slackData ),
-    headers : {
-      'Content-Type' : 'application/json',
-    },
-  } )
+  try {
+    const fetchStatus = await fetch( slackConfig.url, {
+      method : 'POST',
+      body : JSON.stringify( slackData ),
+      headers : {
+        'Content-Type' : 'application/json',
+      },
+    } )
+    debug( { fetchStatus }, '<==Fetch returned' )
+  } catch ( fetchError ) {
+    log( { fetchError }, chalk.red( '\n\n***ERROR: fetch call failed\n' ) )
+  }
 }
 
 if ( npmrcRo.exists ) {

package/src/send-to-slack.mjs

@@ -0,0 +1,124 @@
+#!/usr/bin/env node
+/*
+ * send-to-slack...
+ *
+ */
+
+import chalk from 'chalk'
+import commandLineArgs from 'command-line-args'
+import commandLineUsage from 'command-line-usage'
+import { lt as semverLt } from 'semver'
+
+import { debug, log, shellCmd } from './Utils.mjs'
+
+const slackSendOptions = [ // Use slackSendOptions to tie into the Usage statements
+  {
+    name : 'project',
+    type : String,
+    default : false,
+    description : '{green the name of the google project containing the slack config secret}',
+  },
+  {
+    name : 'config',
+    type : String,
+    default : false,
+    description : '{green the name of the JSON secret containing the slack URL and channel}',
+  },
+  {
+    name : 'message',
+    type : String,
+    default : false,
+    description : '{green the message to send to slack}',
+  },
+  {
+    name : 'dry-run',
+    type : Boolean,
+    default : false,
+    description : '{green do everything without actually sending to slack}',
+  },
+  {
+    name : 'help',
+    type : Boolean,
+    default : false,
+    description : '{green display this help screen}',
+  },
+  /* eslint-enable max-len */
+]
+
+const sections = [
+  {
+    header : 'Leverege Slack Sender',
+    content : '{green This tool helps you to send slack}'
+  },
+  { header : 'Options',
+    optionList : slackSendOptions,
+  },
+]
+
+const args = commandLineArgs( slackSendOptions, { camelCase : true, partial : true } )
+const usage = commandLineUsage( sections )
+
+if ( args.help ) {
+  log( usage )
+  process.exit( 0 )
+}
+
+/* eslint-disable no-underscore-dangle */
+if ( args._unknown ) {
+  log( usage )
+  log( `\nUnrecognized argument [${chalk.bold.red( args._unknown )}]\n` )
+  process.exit( 1 )
+}
+/* eslint-enable no-underscore-dangle */
+
+if ( semverLt( process.version, '18.0.0' ) ) {
+  log( chalk.red( '\n\n***ERROR: must be running at least node 18\n' ) )
+  process.exit( 1 )
+}
+
+const gcpProject = args.project
+
+const getGcpSecretCommand = ( secretName ) => {
+  return `gcloud secrets versions access latest --project=${gcpProject} --secret=${secretName}`
+}
+
+// fetch and parse the slack config for audits
+const slackSecretName = args.config
+let slackConfig
+try {
+  const slackConfigFetcher = getGcpSecretCommand( slackSecretName )
+  const slackSecret = await shellCmd( slackConfigFetcher ) 
+  slackConfig = JSON.parse( slackSecret )
+  debug( { slackSecretName, slackConfig, slackConfigFetcher }, `<==shellCmd Output ${slackSecretName}` )
+} catch ( err ) {
+  if ( err.escapedCommand && err.stderr ) {
+    log( chalk.yellow( `\nFailed Fetching Slack Secret => [${err.escapedCommand}]\n\n` ), chalk.red( err.stderr ) )
+  } else {
+    log( err, chalk.red( `\n\nFailed to fetch and / or parse secret ${slackSecretName} from ${gcpProject}` ) )
+  }
+  process.exit( 1 )
+}
+
+// slack out the details to the configured channel
+const account = await shellCmd( 'gcloud config get account' )
+const slackData = {
+  text : args.message.replace(/^\"|\"$/g,''), // strip pesky leading/trailing "
+  channel : slackConfig.channel,
+}
+
+if ( process.env.BUILD_TOOLS_DEBUG || args.dryRun ) {
+  log( { slackConfig, slackData }, '<==Slack skipped' )
+} else {
+  try {
+    const fetchStatus = await fetch( slackConfig.url, {
+      method : 'POST',
+      body : JSON.stringify( slackData ),
+      headers : {
+        'Content-Type' : 'application/json',
+      },
+    } )
+    debug( { fetchStatus }, '<==Fetch returned' )
+  } catch ( fetchError ) {
+    log( { fetchError }, chalk.red( '\n\n***ERROR: fetch call failed\n' ) )
+  }
+}

package/src/circle-lib.yml

@@ -1,150 +0,0 @@
-version: 2.1
-jobs:
-  install:
-    docker:
-      - image: $CIRCLE_NODE_IMG
-    steps:
-      - checkout
-      - restore_cache:
-          keys:
-            - v1-npm-deps-{{ checksum "package-lock.json" }}
-            - v1-npm-deps-
-
-      - run:
-          name: Setup @leverege Token
-          command: echo \${NPMRC} > ~/.npmrc
-
-      - run: npm install
-
-      - save_cache:
-          key: v1-npm-deps-{{ checksum "package-lock.json" }}
-          paths:
-            - ./node_modules
-
-      - persist_to_workspace:
-          root: /home/circleci/project
-          paths:
-            - ./node_modules
-
-  security-check:
-    docker:
-      - image: $CIRCLE_NODE_IMG
-    steps:
-      - checkout
-      - attach_workspace:
-          at: ~/project
-      - run:
-          name: Check for vulnerabilities
-          command: npm audit --production --audit-level=moderate
-
-  license-check:
-    docker:
-      - image: $CIRCLE_NODE_IMG
-    steps:
-      - checkout
-      - attach_workspace:
-          at: ~/project
-      - run:
-          name: Check for LICENSE and README files
-          command: npm run pkglint
-
-      - run:
-          name: Check for license violations (e.g. GPL)
-          command: npx license-checker --production --summary --failOn GPL
-
-  lint:
-    docker:
-      - image: $CIRCLE_NODE_IMG
-    steps:
-      - checkout
-      - attach_workspace:
-          at: ~/project
-      - run:
-          name: Lint and save results
-          command: npm run lint-ci
-
-      - store_test_results:
-          path: reports
-
-  docs:
-    docker:
-      - image: $CIRCLE_NODE_IMG
-    steps:
-      - checkout
-      - attach_workspace:
-          at: ~/project
-      - run:
-          name: Generate docs
-          command: npx jsdoc -r src -R README.md -d docs
-      - store_artifacts:
-          path: docs
-          prefix: docs
-
-  coverage:
-    # coverage runs tests, generates coverage, and saves docs
-    environment:
-      TZ: 'America/New_York'
-    docker:
-      - image: $CIRCLE_NODE_IMG
-      # IF YOU NEED ANY OF THESE, UNCOMMENT TO USE
-      # - image: redis
-      # - image: circleci/mysql:5.7
-      #   environment:
-      #     MYSQL_USER: root
-      #     MYSQL_ROOT_PASSWORD: root
-      #     MYSQL_ALLOW_EMPTY_PASSWORD: true
-      #     MYSQL_HOST: 127.0.0.1
-      # - image: nsqio/nsq
-      #   command: /nsqlookupd -broadcast-address localhost:4160 -tcp-address 0.0.0.0:4160 -http-address 0.0.0.0:4161
-
-      # - image: nsqio/nsq
-      #   command: >
-      #     /nsqd
-      #     -broadcast-address localhost:4150
-      #     -tcp-address 0.0.0.0:4150
-      #     -http-address 0.0.0.0:4151
-      #     -lookupd-tcp-address localhost:4160
-
-    steps:
-      - checkout
-      - attach_workspace:
-          at: ~/project
-
-      # UNCOMMENT IF YOU NEED DECRYPTING SECRETS
-      # - run:
-      #     name: Load secrets
-      #     command: npm run decrypt
-
-      - run:
-          name: Run Tests and Generate Code Coverage
-          command: npm run coverage
-
-      - run:
-          name: Check if coverage meets at least 70%
-          command: npx c8 check-coverage --functions 70 --lines 70 --statements 70
-
-      - store_artifacts:
-          path: coverage
-          prefix: coverage
-
-workflows:
-  version: 2.1
-  test-build-deploy:
-    jobs:
-      - install:
-          context: npm
-      - security-check:
-          requires:
-            - install
-      - license-check:
-          requires:
-            - install
-      - lint:
-          requires:
-            - install
-      - coverage:
-          requires:
-            - install
-      - docs:
-          requires:
-            - install

package/src/circle-srv.yml

@@ -1,213 +0,0 @@
-version: 2.1
-#orbs:
-#  gcp-gcr: circleci/gcp-gcr@0.6.1
-executors:
-  default:
-    description: >
-      A debian-based machine executor.
-    machine: true
-jobs:
-  install:
-    docker:
-      - image: $CIRCLE_NODE_IMG
-    steps:
-      - checkout
-      - restore_cache:
-          keys:
-            - v1-npm-deps-{{ checksum "package-lock.json" }}
-            - v1-npm-deps-
-
-      - run:
-          name: Setup @leverege Token
-          command: echo \${NPMRC} > ~/.npmrc
-
-      - run: npm install
-
-      - save_cache:
-          key: v1-npm-deps-{{ checksum "package-lock.json" }}
-          paths:
-            - ./node_modules
-
-      - persist_to_workspace:
-          root: /home/circleci/project
-          paths:
-            - ./node_modules
-
-  security-check:
-    docker:
-      - image: $CIRCLE_NODE_IMG
-    steps:
-      - checkout
-      - attach_workspace:
-          at: ~/project
-      - run:
-          name: Check for vulnerabilities
-          command: npm audit --production --audit-level=moderate
-
-  license-check:
-    docker:
-      - image: $CIRCLE_NODE_IMG
-    steps:
-      - checkout
-      - attach_workspace:
-          at: ~/project
-      - run:
-          name: Check for LICENSE and README files
-          command: npm run pkglint
-
-      - run:
-          name: Check for license violations (e.g. GPL)
-          command: npx license-checker --production --summary --failOn GPL
-
-  lint:
-    docker:
-      - image: $CIRCLE_NODE_IMG
-    steps:
-      - checkout
-      - attach_workspace:
-          at: ~/project
-      - run:
-          name: Lint and save results
-          command: npm run lint-ci
-
-      - store_test_results:
-          path: reports
-
-  docs:
-    docker:
-      - image: $CIRCLE_NODE_IMG
-    steps:
-      - checkout
-      - attach_workspace:
-          at: ~/project
-      - run:
-          name: Generate docs
-          command: npx jsdoc -r src -R README.md -d docs
-      - store_artifacts:
-          path: docs
-          prefix: docs
-
-  coverage:
-    # coverage runs tests, generates coverage, and saves docs
-    environment:
-      TZ: 'America/New_York'
-    docker:
-      - image: $CIRCLE_NODE_IMG
-      # IF YOU NEED ANY OF THESE, UNCOMMENT TO USE
-      # - image: redis
-      # - image: circleci/mysql:5.7
-      #   environment:
-      #     MYSQL_USER: root
-      #     MYSQL_ROOT_PASSWORD: root
-      #     MYSQL_ALLOW_EMPTY_PASSWORD: true
-      #     MYSQL_HOST: 127.0.0.1
-      # - image: nsqio/nsq
-      #   command: /nsqlookupd -broadcast-address localhost:4160 -tcp-address 0.0.0.0:4160 -http-address 0.0.0.0:4161
-
-      # - image: nsqio/nsq
-      #   command: >
-      #     /nsqd
-      #     -broadcast-address localhost:4150
-      #     -tcp-address 0.0.0.0:4150
-      #     -http-address 0.0.0.0:4151
-      #     -lookupd-tcp-address localhost:4160
-
-    steps:
-      - checkout
-      - attach_workspace:
-          at: ~/project
-
-      # UNCOMMENT IF YOU NEED DECRYPTING SECRETS
-      # - run:
-      #     name: Load secrets
-      #     command: npm run decrypt
-
-      - run:
-          name: Run Tests and Generate Code Coverage
-          command: npm run coverage
-
-      - run:
-          name: Check if coverage meets at least 70%
-          command: npx c8 check-coverage --functions 70 --lines 70 --statements 70
-
-      - store_artifacts:
-          path: coverage
-          prefix: coverage
-
-  build:
-    # build should make dist image and prepare docker file
-    docker:
-      - image: $CIRCLE_NODE_IMG
-
-    steps:
-      - checkout
-      - attach_workspace:
-          at: ~/project
-
-      - run:
-          name: Setup @leverege Token
-          command: echo \${NPMRC} > ~/.npmrc.ro
-
-      - run:
-          name: Determine version number and container name
-          command: |
-            container="\`node -p \"require('./package.json').leverege.container\"\`"
-            if [ "\$container" == "undefined" ];
-            then
-              container=\$CIRCLE_PROJECT_REPONAME
-            fi
-            NODE_IMAGE="\`node -p \"require('./package.json').leverege.nodeimg\"\`"
-
-            echo \$(git describe --tags) > package_version
-            echo \${container} > container_name
-            echo \${NODE_IMAGE} > node_img
-
-      - run:
-          name: Create Dockerfile and build dist directory
-          command: mkdir -p docker && npm run dockerize \$(git describe --tags)
-
-      - persist_to_workspace:
-          root: ~/project
-          paths:
-            - docker/*
-            - package_version
-            - container_name
-            - node_img
-
-workflows:
-  version: 2.1
-  test-build-deploy:
-    jobs:
-      - install:
-          context: npm
-      - security-check:
-          requires:
-            - install
-      - license-check:
-          requires:
-            - install
-      - lint:
-          requires:
-            - install
-      - coverage:
-          requires:
-            - install
-      - docs:
-          requires:
-            - install
-      - build:
-          context: npm
-          filters:
-            branches:
-              only:
-                - development
-                - /release-.*/
-                - master
-          requires:
-            - install
-            - security-check
-            - license-check
-            - lint
-            - coverage
-            - docs
-

package/src/helm-charts/geotile-server/gitignore

@@ -1 +0,0 @@
-geotile-local.yaml

package/src/helm-charts/geotile-server/helmup.plugin

@@ -1,6 +0,0 @@
-#!/bin/bash
-#
-showInstalling "Geotile Server"
-
-helm upgrade --install geotile-server leverege/geotile-server \
-             --values geotile-server/geotile-values.yaml