@leverege/build-tools 2.38.1 → 2.38.2

package/src/overwhelm.mjs

@@ -0,0 +1,397 @@
+#!/usr/bin/env node
+
+// XXX console.log( `TOP=>[${process.cwd()}] PWD=[${process.env.PWD}]` )
+/* eslint-disable no-console */
+/* eslint-disable security/detect-non-literal-fs-filename */
+
+import path from 'node:path'
+import { execSync as exec } from 'node:child_process'
+import fs from 'node:fs'
+
+import chalk from 'chalk'
+import cliArgs from 'command-line-args'
+// const cliHelp = require( 'command-line-usage' )
+import deepmerge from 'deepmerge'
+import glob from 'glob'
+import parse from 'parse-gitignore'
+import ask from 'readline-sync'
+import YAML from 'js-yaml'
+
+const optionsDefinitions = [
+  /* eslint-disable no-multi-spaces */
+  { name : 'context',  alias : 'x', type : String, default : '' },
+  { name : 'section',  alias : 's', type : String, default : 'default' },
+  { name : 'key',      alias : 'k', type : String },
+  { name : 'export',   alias : 'e', type : Boolean, default : false },
+  { name : 'continue', alias : 'c', type : Boolean, default : false },
+  { name : 'checkctx', alias : 't', type : Boolean, default : false }, // checks overwhelm context
+  { name : 'fetchctx', alias : 'f', type : Boolean, default : false }, // fetch and set k8s context
+  { name : 'genvals',  alias : 'g', type : Boolean, default : false }, // only generates values.yaml
+  { name : 'matchk8s', alias : 'm', type : Boolean, default : false }, // checks k8s context
+  { name : 'verbose',  alias : 'v', type : Boolean, default : false },
+  /* eslint-enable no-multi-spaces */
+]
+const args = cliArgs( optionsDefinitions )
+
+// See if we are in a monorepo, which changes some of the rules.
+const gitTop = exec(
+  'git rev-parse --show-toplevel', { stdio : [ undefined ] }
+).toString().trim()
+let monoTop
+if ( fs.existsSync( `${gitTop}/packages` ) ) {
+  monoTop = path.basename( path.resolve() ) // process.env.PWD
+  console.log( `MONO=[${monoTop}]` )
+}
+
+// The kubernetes context is equal to the git branch name.
+const gitBranch = exec(
+  'git rev-parse --abbrev-ref HEAD', { stdio : [ undefined ] }
+).toString().trim()
+
+const context = args.context || monoTop || gitBranch
+
+if ( context === 'master' ) {
+  console.log( '***ERROR: master context is forbidden; change to a different branch\n\n' )
+  process.exit( 1 )
+}
+
+if ( !fs.existsSync( 'overwhelm.yaml' ) ) {
+  console.log( chalk.red.bold( `
+
+***ERROR overwhelm => missing the overwhelm.yaml file - this command expects
+                      to execute from within a platform-k8s branch
+  ` ) )
+  process.exit( 1 )
+}
+
+// Load the default values and then overlay with the project specific yaml
+// values. This supports the notion of this type of approach in the
+// overwhelm yaml settings:
+//   default:
+//     GCE_MACHINE : "n1-standard-2"
+//
+//   my-project:
+//     GCE_MACHINE : "n1-highcpu-8"
+//
+// fashion, but it is advised not to set a default region with this
+// approach in order to keep us cognizant of exactly where a cluster is
+// being formed.
+//
+const yaml = YAML.load( fs.readFileSync( 'overwhelm.yaml', 'utf-8' ) )
+const over = yaml.default
+if ( process.env.OVERWHELM_DEBUG ) {
+  console.log( { over, yaml }, '<==Contents of the over and yaml structs' )
+}
+
+let cfgs = yaml[context]
+
+// If we just need to check context return 0 if cfgs exists, else error.
+if ( args.checkctx ) { process.exit( !cfgs ) }
+
+if ( !cfgs ) {
+  console.log( `***ERROR: Unknown context => [${context}]\n` )
+  console.log( '   Known k8s contexts are:' )
+  Object.keys( yaml ).forEach( ( key ) => {
+    if ( key !== 'default' ) { console.log( `             ${key}` ) }
+  } )
+  console.log( `Add a section into overwhelm.yaml for ${context}\n\n` )
+  process.exit( 1 )
+}
+
+// Check to see if we are deriving from another configuration.
+if ( cfgs.PROJECT_BASE ) {
+  const baseName = cfgs.PROJECT_BASE
+  const tempCfgs = cfgs
+  cfgs = yaml[baseName]
+  Object.keys( tempCfgs ).forEach( ( key ) => { cfgs[key] = tempCfgs[key] } )
+  console.log( 'PROJECT_BASE still under evaluation' )
+  console.log( '\n  cfgs=', cfgs )
+  process.exit( 1 ) // XXX HERE
+}
+
+Object.keys( cfgs ).forEach( ( key ) => { over[key] = cfgs[key] || over[key] } )
+
+// Expand any variables that reference other variable settings. A perfect
+// example of where this is useful is in the setting of a PROJECT_NAME and
+// a PROJECT_ID. Most of the time these values are identical, which is why
+// this setting exists in the default section:
+//   default:
+//     PROJECT_ID : ${PROJECT_NAME}
+//
+// However, in the case of projects like Siren Marine, this is the more
+// likely setup due to legacy naming schemes:
+//   sire-marine:
+//     PROJECT_NAME : siren-marine
+//     PROJECT_ID   : prod-rt-data
+//
+const varErrors = []
+
+Object.keys( over ).forEach( ( key ) => {
+  if ( !over[key] ) return // no value to work with - skip it
+
+  // Look for something in the form of "thing-${VAR1}-${VAR2}-${VARn}-stuff"
+  const matches = over[key].match( /\$\{\w+\}/g )
+  if ( !matches ) return
+
+  matches.forEach( ( replaceThis ) => {
+    // The matched variable names will be in the form of ${VARNAME} so drop
+    // the ${} characters before using the VARNAME to map into the
+    // overwhelm map containing the desired replacement value.
+    const withThat = over[replaceThis.replace( /\W+/g, '' )]
+    if ( !withThat ) {
+      varErrors.push( `${key} : ${over[key]}` )
+      return
+    }
+    // Store the replaced string back in the overwhelm value map.
+    over[key] = over[key].replace( replaceThis, withThat )
+  } )
+} )
+
+if ( varErrors.length > 0 ) {
+  console.log( '***ERROR: unkown variable reference(s) in overwhelm.yaml:\n' )
+  varErrors.forEach( ( error ) => {
+    console.log( `     ${error}` )
+  } )
+  console.log( '' )
+  process.exit( 1 )
+}
+
+// Grab a list of the existing yaml templates.
+const yams = glob.sync( 'values-*.yaml' )
+
+// A function to apply the replacements to all of the matching yaml which
+// will error out if it runs into an unknown OVH<tag>.
+const doReplacements = ( yamls, replacements ) => {
+  let replYaml = yamls
+
+  // wrap special chars with quotes if needed, like "#---incident-alerts---"
+  const quoteSpecials = ( raw ) => {
+    return /[#$?*]/g.test( raw ) ? `"${raw}"` : raw
+  }
+
+  Object.keys( replacements ).forEach( ( repl ) => {
+    const replaceThis = new RegExp( `OVH:<${repl}>`, 'g' )
+    const withThat = quoteSpecials( `${replacements[repl]}` )
+    replYaml = replYaml.replace( replaceThis, withThat )
+  } )
+
+  // Verify all replaceables have been handled, or error out.
+  const missed = replYaml.match( /OVH:<(.*)>/g )
+
+  if ( missed ) {
+    console.log( '\n***ERROR: missing replacements in overwhelm.yaml for these tags:\n' )
+    missed.forEach( ( miss ) => {
+      console.log( `     TAG: ${miss.match( /OVH:<(.*)>/ )[1]}` )
+    } )
+    console.log( '' )
+    process.exit( 1 )
+  }
+
+  return replYaml
+}
+
+// Collect all of the yaml from the yams...
+const buildYaml = ( yamls, replacements ) => {
+  const always = [ 'values-default.yaml' ]
+  const values = [ ...always, ...yamls ]
+  const loaded = []
+  let allYamls = ''
+
+  values.forEach( ( yaml ) => {
+    if ( !fs.existsSync( yaml ) ) {
+      console.log( `\n***ERROR: yaml file missing => ${yaml}\n` )
+      process.exit( 1 )
+    }
+
+    if ( !loaded.includes( yaml ) ) {
+      allYamls += `\n# AHOY => ${yaml}\n\n`
+      allYamls += fs.readFileSync( yaml, { encoding : 'utf-8' } )
+      loaded.push( yaml )
+    }
+  } )
+
+  // Now apply all of the replacements from the overwhelm file.
+  return doReplacements( allYamls, replacements )
+}
+
+// Call buildYaml to replace the OVH strings in the yaml files.
+const values = buildYaml( yams, over )
+
+// Check to see if someone is requesting a value from the yaml.
+if ( args.key ) {
+  const section = args.section || 'default'
+  const keys = yaml[section]
+  if ( args.verbose ) {
+    console.log( 'replacements' )
+    console.log( keys )
+  }
+  if ( args.export ) {
+    console.log( `export ${args.key}=${keys[args.key]}` )
+  } else {
+    console.log( keys[args.key] )
+  }
+  if ( !args.continue ) process.exit()
+}
+
+// Find all of the potential target dirs. Utilize .gitignore to ignore files
+const gitignore = parse( fs.readFileSync( '.gitignore' ) )
+const opt = {
+  ignore : gitignore.patterns,
+}
+
+const ahoy = glob.sync( '*/', opt )
+
+// .noahoy = skips generation of values.yaml but allows helmup to run
+// .nohelm = overwhelm and helmup will skip the deployment entirely
+ahoy.forEach( ( dir ) => {
+  if ( fs.existsSync( `${dir}/.noahoy` ) ) {
+    !args.fetchctx && console.log( chalk.yellow( `***WARNING: .noahoy detected, not regenerating ${dir}values.yaml` ) )
+  } else if ( !fs.existsSync( `${dir}/.nohelm` ) ) {
+    args.verbose && console.log( `Generating values in ${dir}` ) // eslint-disable-line no-unused-expressions
+
+    let valuesOut = [] // An accumulator for all of the output values.
+
+    // Add support for localized values for things like keel (or whatever)
+    const valuesLocal = `${dir}values-local.yaml`
+
+    if ( fs.existsSync( valuesLocal ) ) {
+      args.verbose && console.log( `     Local values in ${valuesLocal}` ) // eslint-disable-line no-unused-expressions
+      valuesOut += '\n# AHOY => values-local.yaml\n\n'
+      valuesOut += fs.readFileSync( valuesLocal, { encoding : 'utf-8' } )
+    }
+    valuesOut += `\n${values}`
+
+    // Augment the default service section of the YAML with settings that
+    // usually come from the top level overwhelm.yaml file.
+    //
+    const valuesAll = YAML.load( valuesOut ) || {}
+    const valuesSvc = {
+      service : {
+        project_id : 'OVH:<PROJECT_ID>',
+        project_name : 'OVH:<PROJECT_NAME>',
+        host : 'OVH:<HOST>',
+        ingress : 'OVH:<INGRESS>',
+      }
+    }
+    const valuesMerged = YAML.dump( deepmerge( valuesSvc, valuesAll ), { quotingType : '"' } )
+
+    valuesOut = doReplacements( valuesMerged, over )
+
+    // DEPRECATED once siren and manheim are platform 4.0
+    // If we are going to override then we need to convert it to YAML and
+    // do a deepmerge to combine the values.
+    const valuesOverride = `${dir}/values-override.yaml`
+
+    if ( fs.existsSync( valuesOverride ) ) {
+      console.log( `===>Applying Overrides=> ${valuesOverride}` )
+      const overrides = YAML.load( fs.readFileSync( valuesOverride, { encofind : 'utf-8' } ) )
+      const allValues = YAML.load( valuesOut )
+      valuesOut = YAML.dump( deepmerge( allValues, overrides ), { quotingType : '"' } )
+    }
+    // DEPRECATED
+
+    fs.writeFileSync( `${dir}/values.yaml`, valuesOut )
+  }
+} )
+
+// we can also apply replaceables to any top level .ovh files to support top level config maps
+glob.sync( '**/*.ovh' ).forEach( ( cfgmap ) => {
+  const target = cfgmap.replace( /.ovh$/, '' )
+  let valuesOut = fs.readFileSync( cfgmap, { encoding : 'utf-8' } )
+  valuesOut = doReplacements( valuesOut, over )
+  fs.writeFileSync( target, valuesOut )
+} )
+
+if ( args.genvals ) { process.exit( 0 ) }
+
+// Set the GCP context for safety!
+const clusterName = over.CLUSTER_NAME
+if ( over.PROJECT_NAME === 'minikube' ) {
+  process.exit( 0 )
+}
+const projectId = over.PROJECT_ID
+const zone = over.GCE_REGION + ( over.GCE_ZONE ? `-${over.GCE_ZONE}` : '' )
+
+if ( !clusterName ) {
+  console.log( chalk.yellow(
+    `
+
+    The overwhelm.yaml file in this branch is missing a CLUSTER_NAME setting,
+    which was added to support cluster names that do not match the project
+    name. Since the mismatch is rare, you can probably get by with simply
+    adding this line following PROJECT_NAME in the default section of
+    overwhelm.yaml:
+
+      ${chalk.bold.green( 'CLUSTER_NAME   : \'${PROJECT_NAME}\'' )}
+
+    However, if the CLUSTER_NAME is not the same as the PROJECT_NAME then go
+    ahead and add the CLUSTER_NAME setting to the ${chalk.bold.green( projectId )} section of the
+    overwhelm.yaml file.
+
+    ` ) )
+  process.exit( 1 )
+}
+
+let kubeout = '_no_current_context_'
+try {
+  kubeout = exec( 'kubectl config current-context', { stdio : [ undefined ] } )
+} catch ( err ) {
+  // err && console.log( `Caught Error =>[${err}]` )
+}
+
+const currentCtx = kubeout.toString().trim()
+const desiredCtx = `gke_${projectId}_${zone}_${clusterName}`
+const cmd = `gcloud container clusters get-credentials ${clusterName} --zone ${zone} --project ${projectId}`
+
+const matchedCtx = ( currentCtx === desiredCtx )
+
+if ( args.matchk8s ) {
+  process.exit( matchedCtx ? 0 : 1 )
+}
+
+// just fetch the context and exit if -f is specified
+if ( args.fetchctx ) {
+  if ( !matchedCtx ) {
+    try {
+      exec( `${cmd}`, { stdio : [ undefined ] } )
+    } catch ( err ) {
+      process.exit( 61 ) // ECONNREFUSED
+    }
+  }
+  process.exit( 0 )
+}
+
+if ( matchedCtx ) {
+  console.log( `
+
+  Current context matches => ${chalk.bold.green( currentCtx )}
+
+  ` )
+} else {
+  console.log( `
+  You are about to change your Kubernetes context to:
+      CLUSTER_NAME => ${chalk.bold.green( clusterName )}
+      PROJECT_ID   => ${chalk.bold.green( projectId )}
+      REGION       => ${chalk.bold.green( zone )}
+
+  ` )
+}
+const operation = matchedCtx ? 'refetch the credentials' : 'switch your context'
+const ans = ask.question( `Enter "${chalk.bold.green( 'yes' )}" if you wish to ${operation} => ` )
+if ( ans !== 'yes' ) {
+  if ( matchedCtx ) {
+    console.log( chalk.bold.yellow( '\n\nSkipping a refetch of your Kubernetes context and credentials\n\n' ) )
+    process.exit( 0 )
+  }
+
+  console.log( chalk.bold.red( '\n\nYour Kubernetes context and credentials do NOT match - this could be bad!\n\n' ) )
+  process.exit( 1 )
+}
+
+console.log( `\n\nExecuting: ${chalk.bold.yellow( cmd )}\n\n` )
+exec( `${cmd}`, ( error, out ) => {
+  if ( error ) {
+    console.log( `\n\n***ERROR ${error}\n\n` )
+    process.exit( 1 )
+  }
+} )

package/src/circle-lib.yml

@@ -1,150 +0,0 @@
-version: 2.1
-jobs:
-  install:
-    docker:
-      - image: $CIRCLE_NODE_IMG
-    steps:
-      - checkout
-      - restore_cache:
-          keys:
-            - v1-npm-deps-{{ checksum "package-lock.json" }}
-            - v1-npm-deps-
-
-      - run:
-          name: Setup @leverege Token
-          command: echo \${NPMRC} > ~/.npmrc
-
-      - run: npm install
-
-      - save_cache:
-          key: v1-npm-deps-{{ checksum "package-lock.json" }}
-          paths:
-            - ./node_modules
-
-      - persist_to_workspace:
-          root: /home/circleci/project
-          paths:
-            - ./node_modules
-
-  security-check:
-    docker:
-      - image: $CIRCLE_NODE_IMG
-    steps:
-      - checkout
-      - attach_workspace:
-          at: ~/project
-      - run:
-          name: Check for vulnerabilities
-          command: npm audit --production --audit-level=moderate
-
-  license-check:
-    docker:
-      - image: $CIRCLE_NODE_IMG
-    steps:
-      - checkout
-      - attach_workspace:
-          at: ~/project
-      - run:
-          name: Check for LICENSE and README files
-          command: npm run pkglint
-
-      - run:
-          name: Check for license violations (e.g. GPL)
-          command: npx license-checker --production --summary --failOn GPL
-
-  lint:
-    docker:
-      - image: $CIRCLE_NODE_IMG
-    steps:
-      - checkout
-      - attach_workspace:
-          at: ~/project
-      - run:
-          name: Lint and save results
-          command: npm run lint-ci
-
-      - store_test_results:
-          path: reports
-
-  docs:
-    docker:
-      - image: $CIRCLE_NODE_IMG
-    steps:
-      - checkout
-      - attach_workspace:
-          at: ~/project
-      - run:
-          name: Generate docs
-          command: npx jsdoc -r src -R README.md -d docs
-      - store_artifacts:
-          path: docs
-          prefix: docs
-
-  coverage:
-    # coverage runs tests, generates coverage, and saves docs
-    environment:
-      TZ: 'America/New_York'
-    docker:
-      - image: $CIRCLE_NODE_IMG
-      # IF YOU NEED ANY OF THESE, UNCOMMENT TO USE
-      # - image: redis
-      # - image: circleci/mysql:5.7
-      #   environment:
-      #     MYSQL_USER: root
-      #     MYSQL_ROOT_PASSWORD: root
-      #     MYSQL_ALLOW_EMPTY_PASSWORD: true
-      #     MYSQL_HOST: 127.0.0.1
-      # - image: nsqio/nsq
-      #   command: /nsqlookupd -broadcast-address localhost:4160 -tcp-address 0.0.0.0:4160 -http-address 0.0.0.0:4161
-
-      # - image: nsqio/nsq
-      #   command: >
-      #     /nsqd
-      #     -broadcast-address localhost:4150
-      #     -tcp-address 0.0.0.0:4150
-      #     -http-address 0.0.0.0:4151
-      #     -lookupd-tcp-address localhost:4160
-
-    steps:
-      - checkout
-      - attach_workspace:
-          at: ~/project
-
-      # UNCOMMENT IF YOU NEED DECRYPTING SECRETS
-      # - run:
-      #     name: Load secrets
-      #     command: npm run decrypt
-
-      - run:
-          name: Run Tests and Generate Code Coverage
-          command: npm run coverage
-
-      - run:
-          name: Check if coverage meets at least 70%
-          command: npx c8 check-coverage --functions 70 --lines 70 --statements 70
-
-      - store_artifacts:
-          path: coverage
-          prefix: coverage
-
-workflows:
-  version: 2.1
-  test-build-deploy:
-    jobs:
-      - install:
-          context: npm
-      - security-check:
-          requires:
-            - install
-      - license-check:
-          requires:
-            - install
-      - lint:
-          requires:
-            - install
-      - coverage:
-          requires:
-            - install
-      - docs:
-          requires:
-            - install