@leverege/build-tools 2.38.1 → 2.39.0-beta.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@leverege/build-tools",
-  "version": "2.38.1",
+  "version": "2.39.0-beta.1",
   "description": "A collection of build / support tools for Leverege developers",
   "main": "index.js",
   "repository": {
@@ -14,12 +14,12 @@
     "compile_server": "NODE_ENV=server babel -d lib/server src/",
     "compile_web": "NODE_ENV=web babel -d lib/web src/",
     "lint": "find ./src -name \\*.\\*js | xargs npx eslint --report-unused-disable-directives",
-    "prepublintOnly": "npm run lint && npm run build",
-    "prepublishOnly": "npm run compile",
+    "xprepublintOnly": "npm run lint && npm run build",
+    "xprepublishOnly": "npm run compile",
     "test": "echo \"Error: no test specified\" && exit 1"
   },
   "bin": {
-    "build-tools": "src/build-tools.js",
+    "build-tools": "src/build-tools.mjs",
     "circleate": "src/circleate.sh",
     "decrypt-secrets": "src/decrypt-secrets.sh",
     "dirty-git": "src/dirty-git.sh",
@@ -55,17 +55,16 @@
   "license": "SEE LICENSE IN LICENSE.md",
   "dependencies": {
     "ansi-colors": "^4.1.3",
-    "chalk": "^4.1.2",
+    "chalk": "^5.3.0",
     "command-line-args": "^5.2.1",
-    "command-line-usage": "^6.1.3",
+    "command-line-usage": "^7.0.1",
     "deepmerge": "^4.3.1",
-    "dotenv": "^16.3.1",
     "enquirer": "^2.4.1",
-    "execa": "^5.1.1",
-    "get-bin-path": "^10.0.0",
-    "glob": "^8.1.0",
+    "execa": "^8.0.1",
+    "glob": "^10.3.4",
     "js-yaml": "^4.1.0",
-    "npm-registry-fetch": "^13.3.1",
+    "ms": "^2.1.3",
+    "npm-registry-fetch": "^16.0.0",
     "parse-gitignore": "^2.0.0",
     "read-pkg": "^8.1.0",
     "readline-sync": "^1.4.10",
@@ -74,10 +73,7 @@
     "zx": "^7.2.3"
   },
   "devDependencies": {
-    "@babel/cli": "^7.22.10",
-    "@babel/core": "^7.22.11",
-    "@leverege/babel-preset-leverege-node": "^2.0.0",
-    "@leverege/eslint-config-leverege": "^3.0.1",
-    "npm": "^9.8.1"
+    "@leverege/eslint-config-leverege": "^4.2.0",
+    "npm": "^10.0.0"
   }
 }

package/src/Utils.mjs

@@ -0,0 +1,30 @@
+import chalk from 'chalk'
+import { $ } from 'execa'
+
+const debugEnabled = process.env.BUILD_TOOLS_DEBUG === '1'
+
+// eslint-disable-next-line no-console
+export const log = console.log
+export const debug = ( obj, text ) => {
+  if ( debugEnabled ) {
+    log( obj, `${chalk.yellow( '***DEBUG:' )} ${chalk.green( text )}` )
+  }
+}
+
+export const shellCmd = async ( cmdstr, opts = {} ) => {
+  try {
+    // due to the escaping rules we pull apart the cmdstr and invoke $ using
+    // the (command, argsArray) or the escaped command sent to spawn will have
+    // addition double quotes which causes fail
+    const [ command, ...args ] = cmdstr.split( ' ' )
+
+    debug( { command, args }, '<==execa $ invoke' )
+    const results = await $( opts )`${command} ${args}`
+    debug( { results }, '<==execa $ results' )
+
+    return results.stdout
+  } catch ( error ) {
+    debug( { error } )
+    throw error
+  }
+}

package/src/{build-tools.js → build-tools.mjs}

@@ -1,13 +1,31 @@
 #!/usr/bin/env node
+/* eslint-disable security/detect-non-literal-fs-filename */
+
+import fs from 'node:fs'
+import path from 'node:path'
+import url from 'node:url'
+
+import chalk from 'chalk'
+import cliArgs from 'command-line-args'
+import cliHelp from 'command-line-usage'
+import npmRegistryFetch from 'npm-registry-fetch'
+import { lt as semverLt } from 'semver'
+
+// because __dirname goes away with ESM
+const getFilename = () => {
+  const filename = url.fileURLToPath( import.meta.url )
+  return filename
+}
 
-const fs = require( 'node:fs' )
-const path = require( 'node:path' )
+const getDirname = () => {
+  const dirname = path.dirname( getFilename() )
+  return dirname
+}
 
-const chalk = require( 'chalk' )
-const cliArgs = require( 'command-line-args' )
-const cliHelp = require( 'command-line-usage' )
-const npmRegistryFetch = require( 'npm-registry-fetch' )
-const semverLt = require( 'semver/functions/lt' )
+const repoRoot = path.dirname( getDirname() )
+const thisVersion = JSON.parse(
+  fs.readFileSync( `${repoRoot}/package.json` )
+).version
 
 /* eslint-disable no-console */
 
@@ -65,10 +83,6 @@ const sections = [
 const args = cliArgs( optionList, { partial : true } )
 const help = cliHelp( sections )
 
-const repoRoot = path.dirname( __dirname )
-const thisPackage = require( `${repoRoot}/package.json` )
-const thisVersion = thisPackage.version
-
 if ( args.root ) {
   console.log( repoRoot )
   process.exit( 0 )
@@ -93,7 +107,7 @@ const checkForLatest = async ( pkg = '@leverege/build-tools' ) => {
     }
     try {
       const tokenLine = fs.readFileSync( tokenFile ).toString()
-      const tokenRegX = new RegExp( '.*registry.npmjs.org\\/:\\w+=+(.*)' )
+      const tokenRegX = '.*registry.npmjs.org\\/:\\w+=+(.*)' // old => new RegExp( '.*registry.npmjs.org\\/:\\w+=+(.*)' )
       const tokenStr = tokenLine.match( tokenRegX )
       if ( !tokenStr ) {
         console.error( `\n***ERROR: malformed npm token in ${tokenFile}\n` )
@@ -103,6 +117,7 @@ const checkForLatest = async ( pkg = '@leverege/build-tools' ) => {
     } catch ( err ) {
       console.error( err )
     }
+    return undefined
   }
 
   try {
@@ -113,6 +128,7 @@ const checkForLatest = async ( pkg = '@leverege/build-tools' ) => {
   } catch ( err ) {
     console.log( err )
   }
+  return undefined
 }
 
 const checkAndAnnounce = () => {

package/src/firebaseDeploy.mjs

@@ -78,6 +78,7 @@ const targets = Object
           // Store the longest length of site id for formatting later on
           maxSiteIdLength = Math.max( siteId.length, maxSiteIdLength )
 
+          // eslint-disable-next-line no-param-reassign
           prev[alias] = {
             projectId,
             siteId,

package/src/firebaseServe.mjs

@@ -61,6 +61,7 @@ const targets = Object
       .entries( config.hosting )
       .forEach( ( [ siteId, aliases ] ) => {
         aliases.forEach( ( alias ) => {
+          // eslint-disable-next-line no-param-reassign
           prev[alias] = {
             projectId,
             siteId,

package/src/getjson.mjs

@@ -18,7 +18,7 @@
 /**
  * Get a command line switch parser.
  */
-import fs from 'fs'
+import fs from 'node:fs'
 import cliArgs from 'command-line-args'
 import cliHelp from 'command-line-usage'
 
@@ -64,9 +64,9 @@ if ( !args.file ) {
 }
 
 try {
-  const json = JSON.parse( fs.readFileSync( args.file, 'utf8' ) )
+  const json = JSON.parse( fs.readFileSync( args.file, 'utf8' ) ) // eslint-disable-line security/detect-non-literal-fs-filename
   const value = json[args.key]
-  const match = new RegExp( args.regex || '.*' ).exec( value )
+  const match = new RegExp( args.regex || '.*' ).exec( value ) // eslint-disable-line security/detect-non-literal-regexp
 
   console.log( match ? match[match.length - 1] : '' )
 } catch ( e ) {

package/src/helm-charts/firebase-rules/firebase.json

@@ -0,0 +1,5 @@
+{
+  "database": [
+    {"target": "leverege", "rules": "leverege.rules.json"}
+  ]
+}

package/src/helm-charts/firebase-rules/firebaserc.ovh

@@ -0,0 +1,24 @@
+{
+  "projects": {
+    "default": "OVH:<PROJECT_ID>"
+  },
+  "targets": {
+    "OVH:<PROJECT_ID>": {
+      "database": {
+        "leverege": [
+          "OVH:<PROJECT_ID>",
+          "OVH:<PROJECT_ID>-rt-0",
+          "OVH:<PROJECT_ID>-rt-1",
+          "OVH:<PROJECT_ID>-rt-2",
+          "OVH:<PROJECT_ID>-rt-3",
+          "OVH:<PROJECT_ID>-rt-4",
+          "OVH:<PROJECT_ID>-rt-5",
+          "OVH:<PROJECT_ID>-rt-6",
+          "OVH:<PROJECT_ID>-rt-7",
+          "OVH:<PROJECT_ID>-rt-8",
+          "OVH:<PROJECT_ID>-rt-9"
+        ]
+      }
+    }
+  }
+}

package/src/helm-charts/firebase-rules/gitignore

@@ -0,0 +1,2 @@
+firebaserc
+.firebaserc

package/src/helm-charts/firebase-rules/helmup.plugin

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+# make sure we're running from the proper dir
+cd $GITTOP/firebase-rules
+
+# grab a copy of existing rules - date time stamp DTS comes from bash-funcs
+firebase --project $GCP_PROJECT_ID \
+                   database:get /.settings/rules > leverege.rules.previous.json
+
+# copy the current ovh generated rc file to where firebase will find it
+cp firebaserc .firebaserc
+
+# Note: GCP_PROJECT_ID is passed down from helmup's environment.
+firebase --project $GCP_PROJECT_ID deploy --only database

package/src/helm-charts/firebase-rules/leverege.rules.json

@@ -0,0 +1,65 @@
+{
+  "rules": {
+    "projectRefs": {
+      "$project_id": {
+        "member": {
+          ".indexOn": ["created"]
+        }
+      }
+    },
+    "blueprint": {
+      ".indexOn": "lastModified"
+    },
+    "blueprintChildren": {
+      "$bp_id": {
+        "attributes": {
+          ".indexOn": ".value"
+        }
+      }
+    },
+    "attribute":{
+      ".indexOn" : [ "path", "blueprintId" ]
+    },
+    "systemChildren": {
+      "$sys_id": {
+        "device": {
+          ".indexOn": ".value"
+        }
+      }
+    },
+    "device": {
+      ".indexOn": ["id", "networkId", "simNetworkId"]
+    },
+    "deviceRefs": {
+      "$dev_id": {
+        "member": {
+          ".indexOn": ["created"]
+        }
+      }
+    },
+    "user": {
+      ".indexOn": ["username", "email"]
+    },
+    "project": {
+      ".indexOn": ["id"]
+    },
+    "rt": {
+      "$sys_id": {
+        ".read": "auth.systemId === $sys_id",
+        "dev": {
+          "$dev_id": {
+            ".read": "auth.deviceId === $dev_id",
+            "outbound": {
+              ".indexOn": ["initiated", "sent", "ack"]
+            },
+            "events": {
+              "$evt_type": {
+                ".indexOn": ["time", "id"]
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}

package/src/helm-charts/velero/helmup.plugin

@@ -31,6 +31,22 @@ helm upgrade \
      --set configuration.backupStorageLocation[0].config.serviceAccount="$GCE_SA_EMAIL" \
      --set configuration.volumeSnapshotLocation[0].config.snapshotLocation="$GCE_REGION"
 
+if [ $? -ne 0 ];
+then
+  cat<<VELERROR
+
+  `color r ***ERROR:` helm upgrade has failed! If upgrading from an older chart there
+  may be residual velero components that need to be cleaned up. Try running the
+  following kubectl commands and then rerun helmup:
+
+      kubectl delete backupstoragelocations.velero.io -n velero default
+      kubectl delete volumesnapshotlocations.velero.io -n velero default
+      helmup velero
+
+VELERROR
+  exit 1
+fi
+
 velero schedule get
 cat<<DONE_VELERO
 

package/src/helmup.sh

@@ -152,10 +152,29 @@ MISSING_HELMUP_PLUGIN
   # because npm tends to not allow .gitinore files the easy way...
   [ -f "$SERVICE/gitignore" ] && mv $SERVICE/gitignore $SERVICE/.gitignore
   git add $SERVICE
-  overwhelm.js --genvals
+  overwhelm --genvals
   ifPluggedIn $SERVICE
 }
 
+function installDefaultConfigs() {
+  showInstalling "Default Configs"
+
+  clearlyWarn 'without clearing'
+  cat<<INSTALLDEFAULTCONFIGS
+  Continuing with applying the default-configs.yaml will cause all non-system
+  services to `color r restart`. Make certain people are made aware that this is about
+  to happen.
+
+INSTALLDEFAULTCONFIGS
+  yesToContinue "to apply the default-configs to the cluster"
+
+  kubecmd="kubectl apply -f default-configs.yaml"
+
+  printf "\nRunning => `color y \"$kubecmd\"`\n\n"
+  eval $kubecmd
+  postToSlack "$kubecmd"
+}
+
 function errorIfMissingSecret() {
   SECRET="$1"
   kubectl get secret $SECRET &> $DEVNULL
@@ -165,6 +184,24 @@ function errorIfMissingSecret() {
   fi
 }
 
+function installFirebaseDatabases() {
+  showInstalling "Firebase Realtime Databases"
+
+  open https://console.firebase.google.com/u/0/project/${GCP_PROJECT_ID}/authentication/users
+  yesToContinue "when you have pressed Get Started on the Firebase Authentication page"
+
+  # create the firebase primary db
+  firebase --project $GCP_PROJECT_ID database:instances:create ${PROJECT}
+
+  # TODO: should maybe be more flexible with number of shards in the future
+  for SHARD in {0..9}
+  do
+    SHARD="${GCP_PROJECT_ID}-rt-${SHARD}"
+    printf "Creating shard $SHARD\n"
+    firebase --project $GCP_PROJECT_ID database:instances:create ${SHARD}
+  done
+}
+
 function installStackdriverExporterEnvironment() {
   showInstalling "Stackdriver Exporter Service Account Binding"
 
@@ -627,6 +664,10 @@ CATBACKUP
         kubectl create -f https://raw.githubusercontent.com/GoogleCloudPlatform/k8s-stackdriver/master/custom-metrics-stackdriver-adapter/deploy/production/adapter.yaml
         ;;
 
+      default-configs*)
+        installDefaultConfigs
+        ;;
+
       db-curator*)
         # db-curator / db-curator-dh diffs are defined in values-local.yaml
         HELMUP="helm upgrade --install $SERVICE leverege/db-curator --values $SERVICE/values.yaml $CHARTVER $HELM_WHAT"
@@ -649,6 +690,11 @@ CATBACKUP
         installPreemptibleKiller # the gke preemptible node spreader outer
         ;;
 
+      "firebase")
+        installFirebaseDatabases
+        bootstrapLocalSetup firebase-rules
+        ;;
+
       "grafana")
         bootstrapLocalSetup $SERVICE
         ;;