@leverege/build-tools 2.94.0 → 2.95.0

package/src/service-man/config/CommandLine.mjs

@@ -0,0 +1,199 @@
+import fs from 'node:fs'
+
+import chalk from 'chalk'
+import commandLineArgs from 'command-line-args'
+import commandLineUsage from 'command-line-usage'
+import inquirer from 'inquirer'
+
+import { log, } from '../../Utils.mjs'
+
+import VersionCheck from './VersionCheck.mjs'
+
+const cliOptionList = [ // Use cliOptionList to tie into the Usage statements
+  {
+    name : 'fetch-versions',
+    type : Boolean,
+    default : false,
+    description : '{green create Versions.json from current k8s context}',
+  },
+  {
+    name : 'create-service',
+    type : String,
+    default : false,
+    description : '{green create a new service account and bind workload identity}',
+  },
+  {
+    name : 'create-sa',
+    type : String,
+    default : false,
+    description : '{green create a new service account without binding}',
+  },
+  {
+    name : 'namespace',
+    type : String,
+    default : 'default',
+    description : '{green the namespace to use if it is other than default}',
+  },
+  {
+    name : 'dump-configs',
+    type : String,
+    default : false,
+    description : '{green dumps the service-man configs to the specified directory}',
+  },
+  {
+    name : 'use-configs',
+    type : String,
+    default : false,
+    description : '{green overrides where service-man gets its configs}',
+  },
+  {
+    name : 'setup-secrets',
+    type : Boolean,
+    default : false,
+    description : '{red [re]generates the standard random secrets}',
+  },
+  {
+    name : 'setup-roles',
+    type : Boolean,
+    default : false,
+    description : '{yellow define the Leverege IAM roles}',
+  },
+  {
+    name : 'setup-services',
+    type : String,
+    default : false,
+    description : '{yellow create the platform Leverege Svc Accts}',
+  },
+  {
+    name : 'setup-services-dir',
+    type : String,
+    default : undefined,
+    description : '{yellow specify the Svc Accts directory}',
+  },
+  {
+    name : 'new-k8s',
+    type : Boolean,
+    default : false,
+    description : '{yellow shortcut for --setup-secrets --setup-roles --setup-services}',
+  },
+  {
+    name : 'sync-secrets',
+    type : String,
+    default : false,
+    description : '{yellow synchronize secret mgr secret to k8s secret <mgr_sec>=<k8s_sec>}',
+  },
+  {
+    name : 'from-helmup',
+    type : Boolean,
+    default : false,
+    description : '{red was invoked from helmup - cuts down on double k8s context set}',
+  },
+  {
+    name : 'help',
+    type : Boolean,
+    default : false,
+    description : '{green display this help screen}',
+  },
+  { name : 'version',
+    type : Boolean,
+    default : false,
+    description : '{green returns the build-tools repo version}',
+  },
+  { name : 'verbose',
+    alias : 'v',
+    type : Boolean,
+    default : false,
+    description : '{green emit additional info at run time}',
+  },
+]
+
+const sections = [
+  {
+    header : 'Leverege Service Management Tool',
+    content : `
+  {green This utility attempts to ease the pain of managing IAM roles and the creation of service accounts for new clusters and services.}
+`,
+  },
+  { header : 'Options',
+    optionList : cliOptionList,
+  },
+]
+
+const choices = {
+  setupSecrets : 'Generate random k8s secrets (authz-postgres, npmrc, etc)',
+  setupRoles : 'Install leverege custom IAM roles',
+  setupServices : 'Create platform service accounts (api-server, emailer, etc)',
+}
+
+const serviceManQs = [
+  {
+    type : 'checkbox',
+    name : 'sman.choices',
+    message : 'Select the service areas of interest',
+    choices : Object.values( choices )
+  },
+]
+
+export default class CommandLine {
+  constructor( options = {} ) {
+    /**
+     * Verify that node meets our minimum needs.
+     */
+    try {
+      VersionCheck.validate( {
+        current : process.version, // this is the node version
+        minimum : '20.12',
+        maximum : '25', // must be less than this
+        preferred : '24.12',
+      } )
+    } catch ( err ) {
+      log( err )
+      process.exit( 1 )
+    }
+
+    this.args = commandLineArgs( cliOptionList, { camelCase : true, partial : true } )
+    this.runInquiry = Object.keys( this.args ).length === 0
+
+    if ( this.args.newK8S ) {
+      this.args.setupSecrets = true
+      this.args.setupRoles = true
+      this.args.setupServices = true
+    }
+
+    if ( this.args.version ) {
+      const packageVersion = JSON.parse(
+        fs.readFileSync( `${options.sourceTop}/../../package.json` ) // eslint-disable-line security/detect-non-literal-fs-filename
+      ).version
+      log( `v${packageVersion}` )
+      process.exit( 0 )
+    }
+
+    /* eslint-disable no-underscore-dangle */
+    if ( this.args._unknown ) { 
+      log( commandLineUsage( sections ) )
+      log( chalk.red( `\n ***Error: unknown command line options: ${this.args._unknown}` ) )
+      process.exit( 1 )
+    }
+    /* eslint-enable no-underscore-dangle */
+
+    if ( this.args.help ) {
+      log( commandLineUsage( sections ) )
+      process.exit( 0 )
+    }
+  }
+
+  async init() {
+    if ( this.runInquiry ) {
+      const answers = await inquirer.prompt( serviceManQs )
+      answers.sman.choices.forEach( ( choice ) => {
+        // Do a reverse lookup of choices.key from user selections so a choice
+        // of 'IAM roles' from the choices object will yield an answer of
+        // 'setupRoles' in the args.
+        const answer = Object.keys( choices ).find( key => choices[key] === choice )
+        this.args[answer] = true
+      } )
+      this.answers = answers
+    }
+    return this
+  }
+}

package/src/service-man/config/Config.mjs

@@ -0,0 +1,203 @@
+import path from 'node:path'
+import fs from 'node:fs/promises' /* eslint-disable security/detect-non-literal-fs-filename */
+
+import chalk from 'chalk'
+import { execa } from 'execa'
+import inquirer from 'inquirer'
+import SimpleGit from 'simple-git'
+
+import { log, } from '../../Utils.mjs'
+
+export default class Config {
+  constructor( options ) {
+    // we cannot async in the constructor - so use init to do the rest
+    this.configRoot = `${options.sourceTop}/config`
+    this.git = new SimpleGit()
+  }
+
+  // expects a directory name containing text files with a .json extension
+  // returns an object with named entries based on the root file name
+  async loadJSON( jsonDirectory ) {
+    const jsonFiles = ( await fs.readdir( jsonDirectory ) )
+      .filter( el => path.extname( el ) === '.json' )
+
+    if ( !( jsonFiles?.length > 0 ) ) {
+      throw new Error( `missing JSON definition files in [${jsonDirectory}]` )
+    }
+
+    const jsonObject = {}
+    for ( let i = 0; i < jsonFiles.length; i++ ) {
+      const jsonFile = jsonFiles[i]
+      /* eslint-disable-next-line no-await-in-loop */
+      const fileName = await fs.readFile( `${jsonDirectory}/${jsonFile}`, { encoding : 'utf8', flag : 'r' } )
+      const roleName = jsonFile.split( '.' )[0]
+      const jsonRole = JSON.parse( fileName )
+      jsonObject[roleName] = jsonRole
+    }
+
+    return jsonObject
+  }
+
+  // Dumps the internal service-man config structures to the specified dir
+  // which may then be fed back into service-man via the --use-configs switch.
+  //
+  // Note: as of node 20 the node:fs.cp call was still experimental, but it
+  // has been around since node 16 so figured may as well use it.
+  async dump( dumpDir ) {
+    try {
+      const fileStat = await fs.stat( dumpDir )
+      if ( !fileStat.isDirectory() ) {
+        log( chalk.red.bold`
+  ***Error: File already exists and it is not a directory
+` + chalk.white`
+  In order to replicate service-man's internal configuration structure we need
+  to store the JSON files in a directory tree, but the specified dump target is
+  a flat file. Remove it and rerun the dump in order to proceed.
+
+  Remove => ` + chalk.yellow.bold( dumpDir ) )
+        process.exit( 1 )
+      }
+    } catch ( err ) {
+      // this file does not exist is a good thing
+    }
+
+    await fs.cp( // *Experimental*
+      this.configRoot,
+      dumpDir,
+      {
+        recursive : true,
+        filter : source => !source.endsWith( '.js' )
+      } )
+
+    const fh = await fs.open( `${dumpDir}/.nohelm`, 'a' )
+    await fh.close()
+  }
+
+  async init( CLI ) {
+    this.CLI = CLI
+    // allow the configRoot to be overridden from the CLI
+    const altConfigs = CLI.args.useConfigs
+    if ( altConfigs ) {
+      log( chalk.green`
+  *** Using alternate configs from => ` + chalk.yellow.bold( altConfigs ) )
+      this.configRoot = altConfigs
+    }
+    this.configDirs = ( await fs.readdir( this.configRoot, { withFileTypes : true } ) )
+      .filter( dirent => dirent.isDirectory( ) )
+      .map( dirent => dirent.name )
+
+    // get the top of the repo from git
+    this.gitTop = await this.git.revparse( [ '--show-toplevel' ] )
+
+    // verify this is executing from a platform-k8s branch
+    try {
+      this.overwhelm = `${this.gitTop}/overwhelm.yaml`
+      await fs.stat( this.overwhelm )
+    } catch ( err ) {
+      log( chalk.red.bold`
+  ***Error: This does not look like a platform-k8s branch
+` + chalk.yellow`
+  The service-man expects to operate from a platform-k8s branch to ensure
+  the proper handling of the kubernetes and gcloud context. Valid branches
+  of the platform-k8s repository should have an overwhelm.yaml file at their
+  root which absent from this location:
+
+  Missing => ` + chalk.red( err.path ) )
+      process.exit( 1 )
+    }
+
+    // load up the GCP role definitions
+    this.rolesDir = `${this.configRoot}/iam-roles`
+    this.roles = await this.loadJSON( this.rolesDir )
+
+    // load up the known platform service account definitions
+    this.serviceAccountDir = CLI?.args?.setupServicesDir ?? `${this.configRoot}/service-accounts`
+    this.serviceAccounts = await this.loadJSON( this.serviceAccountDir )
+
+    // load up the random secrets definitions
+    this.secretsDir = `${this.configRoot}/secrets`
+    this.secrets = await this.loadJSON( this.secretsDir )
+
+    // if we're not running from helmup we need to check context
+    if ( !CLI.args.fromHelmup && !CLI.args.dumpConfigs ) {
+      try {
+        await execa( 'overwhelm', { stdio : 'inherit' } )
+      } catch ( err ) {
+        process.exit( err.exitCode )
+      }
+    }
+
+    // if this is a new k8s then enable overwrites to avoid inquiry
+    this.overwrite = CLI.args.newK8S === true
+
+    // grab the current GCP config
+    const { stdout : gcloudConfig } = await execa( 'gcloud', [ 'config', 'list', '--format=json' ] )
+    this.gcloudConfig = JSON.parse( gcloudConfig )
+
+    // and the current k8s context which is gke_${project}_${region}_${cluster}
+    const { stdout : currentContext } = await execa( 'kubectl', [ 'config', 'current-context' ] )
+    const [ k8sPrefix, k8sProject, k8sRegion, k8sCluster ] = currentContext.split( '_' )
+
+    if ( k8sPrefix !== 'gke' ) {
+      throw Error( `Unknown cluster prefix [${k8sPrefix}] - expecting [gke]` )
+    }
+
+    this.k8sProject = k8sProject
+    this.k8sCluster = k8sCluster
+    this.k8sRegion = k8sRegion
+
+    this.initialized = true
+
+    return this
+  }
+
+  getRoles() { return this.roles }
+
+  getRoleNames() { return Object.keys( this.roles ) }
+
+  getRoleTitles() {
+    const roleNames = this.getRoleNames()
+    const titles = []
+    roleNames.forEach( name => titles.push( this.roles[name].title ) )
+    return titles
+  }
+
+  getRolesDir() { return this.rolesDir } // TODO: make smart look locally for existing roles in k8s project
+
+  getSecretsNames() { return Object.keys( this.secrets ) }
+
+  getServiceAccountNames() { return Object.keys( this.serviceAccounts ) }
+
+  getGcloudConfig() { return this.gcloudConfig }
+
+  getGcloudProject() { return this.gcloudConfig.core.project }
+
+  getGcloudCluster() { return this.gcloudConfig.container.cluster }
+
+  getK8sCluster() { return this.k8sCluster }
+
+  getK8sProject() { return this.k8sProject }
+
+  getK8sRegion() { return this.k8sRegion }
+
+  async askForRoles( serviceName ) {
+    const roleTitles = this.getRoleTitles()
+    const titlesInquiry = [
+      {
+        type : 'checkbox',
+        name : 'titles',
+        message : 'Select the roles necessary for the service',
+        choices : roleTitles,
+        pageSize : roleTitles.length,
+      }
+    ]
+    const selection = await inquirer.prompt( titlesInquiry )
+
+    const roleNames = this.getRoleNames()
+    const selectedRoles = selection.titles.map( ( title ) => {
+      return roleNames.find( name => this.roles[name].title === title )
+    } )
+
+    return selectedRoles
+  }
+}

package/src/service-man/config/VersionCheck.mjs

@@ -0,0 +1,79 @@
+import chalk from 'chalk'
+import coerce from 'semver/functions/coerce.js'
+import semverLt from 'semver/functions/lt.js'
+import satisfies from 'semver/functions/satisfies.js'
+
+import { err, log, } from '../../Utils.mjs'
+
+/**
+ * This function verifies the specified version falls in this range:
+ *
+ *    minimum >= current < maximum
+ *
+ * The params will be semver coerced where a version of 12 will be coerced
+ * to 12.0.0, a version of 12.18.4-beta.3 will be coerced to 12.18.4.
+ *
+ * Note that this function still has its references to node, but could be
+ * made entirely generic if need be.
+ *
+ * @param {Object} params
+ * @param {String} params.current - version to check
+ * @param {String} params.minimum - can be equal or greater to this
+ * @param {String} params.maximum - must be less than this
+ * @param {String} params.preferred - the most acceptable version
+ */
+function validate( params = {} ) {
+  const throwChalk = ( error ) => {
+    const errorStr = chalk.red.bold( '***ERROR service-man =>' )
+    err( chalk.yellow.bold( `\n${errorStr} ${error}` ) )
+    process.exit( 1 )
+  }
+
+  if ( !params || !params.current || !params.minimum || !params.maximum ) {
+    throwChalk(
+      'current, minimum and maximum parameters are required by validate'
+    )
+  }
+
+  const coercedVersions = ( sems ) => {
+    return Object.keys( sems ).reduce( ( obj, key ) => {
+      const coerced = obj // avoid eslint warning
+      coerced[key] = coerce( sems[key] ).version
+      return coerced
+    }, {} )
+  }
+
+  const { current, minimum, maximum, preferred } = coercedVersions( params )
+
+  /**
+   * Verify that the current version falls within the accepted ranges.
+   */
+  const rangeStr = `>=${minimum} <${maximum}`
+  if ( !satisfies( current, rangeStr ) ) {
+    throwChalk(
+      `Node version ${current} is outside the accepted range of ${rangeStr}`
+    )
+  }
+
+  /**
+   * Suggest upgrading to the currently recommended version.
+   */
+  if ( preferred && semverLt( current, preferred ) ) {
+    // the hex #FF8800 gives us Orange
+    log(
+      chalk.hex( '#FF8800' ).bold( `
+  ${chalk.yellow.bold( '        *** HEADS UP ***' )}
+  Running node version => ${chalk.yellow.bold( `v${current}` )} (which should work)
+  Preferred version is => ${chalk.green.bold( `v${preferred}` )}
+
+  Also, a reinstall of the build-tools may be necessary after any node upgrade
+          ${chalk.cyan.bold( 'npm i -g @leverege/build-tools' )}
+`
+      )
+    )
+  }
+}
+
+export default {
+  validate,
+}

package/src/service-man/config/iam-roles/leveregeBigQueryReader.json

@@ -0,0 +1,10 @@
+{
+  "description": "Leverege Big Query Data Accessor",
+  "includedPermissions": [
+    "bigquery.jobs.create",
+    "bigquery.tables.create",
+    "bigquery.tables.getData"
+  ],
+  "stage": "BETA",
+  "title": "Leverege Big Query Reader"
+}

package/src/service-man/config/iam-roles/leveregeBigQueryWriter.json

@@ -0,0 +1,13 @@
+{
+  "description": "Leverege Big Query Writer",
+  "includedPermissions": [
+    "bigquery.datasets.create",
+    "bigquery.datasets.get",
+    "bigquery.jobs.create",
+    "bigquery.tables.create",
+    "bigquery.tables.get",
+    "bigquery.tables.updateData"
+  ],
+  "stage": "BETA",
+  "title": "Leverege Big Query Writer"
+}

package/src/service-man/config/iam-roles/leveregeCloudFunctionsManager.json

@@ -0,0 +1,12 @@
+{
+  "description": "Leverege limited Cloud Functions Manager",
+  "includedPermissions": [
+    "cloudfunctions.functions.create",
+    "cloudfunctions.functions.delete",
+    "cloudfunctions.functions.get",
+    "cloudfunctions.functions.sourceCodeSet",
+    "cloudfunctions.functions.update"
+  ],
+  "stage": "BETA",
+  "title": "Leverege Cloud Functions Manager"
+}

package/src/service-man/config/iam-roles/leveregeFirebaseHosting.json

@@ -0,0 +1,9 @@
+{
+  "description": "Leverege Firebase Hosting Viewer",
+  "includedPermissions": [
+    "firebasehosting.sites.get",
+    "firebasehosting.sites.list"
+  ],
+  "stage": "BETA",
+  "title": "Leverege Firebase Hosting Access"
+}

package/src/service-man/config/iam-roles/leveregeFirebaseMessaging.json

@@ -0,0 +1,13 @@
+{
+  "description": "Leverege Firebase Messaging",
+  "includedPermissions": [
+    "cloudmessaging.messages.create",
+    "firebasenotifications.messages.create",
+    "firebasenotifications.messages.delete",
+    "firebasenotifications.messages.get",
+    "firebasenotifications.messages.list",
+    "firebasenotifications.messages.update"
+  ],
+  "stage": "BETA",
+  "title": "Leverege Firebase Messaging"
+}

package/src/service-man/config/iam-roles/leveregeFirebaseUpdates.json

@@ -0,0 +1,10 @@
+{
+  "description": "Leverege Firebase Updates Access",
+  "includedPermissions": [
+    "firebaseauth.configs.getHashConfig",
+    "firebaseauth.users.get",
+    "firebasedatabase.instances.update"
+  ],
+  "stage": "BETA",
+  "title": "Leverege Firebase Updates"
+}

package/src/service-man/config/iam-roles/leveregeLogsViewer.json

@@ -0,0 +1,8 @@
+{
+  "description": "Leverege view access to logs",
+  "includedPermissions": [
+    "logging.logEntries.list"
+  ],
+  "stage": "BETA",
+  "title": "Leverege Logs Viewer"
+}

package/src/service-man/config/iam-roles/leveregePubSub.json

@@ -0,0 +1,13 @@
+{
+  "description": "Leverege Pub/Sub",
+  "includedPermissions": [
+    "pubsub.subscriptions.consume",
+    "pubsub.subscriptions.create",
+    "pubsub.topics.attachSubscription",
+    "pubsub.topics.create",
+    "pubsub.topics.list",
+    "pubsub.topics.publish"
+  ],
+  "stage": "BETA",
+  "title": "Leverege Pub/Sub"
+}

package/src/service-man/config/iam-roles/leveregeSecretsViewer.json

@@ -0,0 +1,9 @@
+{
+  "description": "Leverege Secrets Viewer",
+  "includedPermissions": [
+    "secretmanager.secrets.get",
+    "secretmanager.versions.access"
+  ],
+  "stage": "BETA",
+  "title": "Leverege Secrets Viewer"
+}

package/src/service-man/config/iam-roles/leveregeStorageObjectAdmin.json

@@ -0,0 +1,21 @@
+{
+  "description": "Leverege read / write access to Storage Objects",
+  "includedPermissions": [
+    "storage.buckets.create",
+    "storage.buckets.get",
+    "storage.buckets.getIamPolicy",
+    "storage.buckets.list",
+    "storage.buckets.update",
+    "storage.multipartUploads.abort",
+    "storage.multipartUploads.create",
+    "storage.multipartUploads.list",
+    "storage.multipartUploads.listParts",
+    "storage.objects.create",
+    "storage.objects.delete",
+    "storage.objects.get",
+    "storage.objects.getIamPolicy",
+    "storage.objects.list"
+  ],
+  "stage": "BETA",
+  "title": "Leverege Storage Object Admin"
+}

package/src/service-man/config/iam-roles/leveregeStorageObjectReader.json

@@ -0,0 +1,13 @@
+{
+  "description": "Leverege read only access to Storage Objects",
+  "includedPermissions": [
+    "storage.buckets.get",
+    "storage.buckets.getIamPolicy",
+    "storage.buckets.list",
+    "storage.objects.get",
+    "storage.objects.getIamPolicy",
+    "storage.objects.list"
+  ],
+  "stage": "BETA",
+  "title": "Leverege Storage Object Reader"
+}

package/src/service-man/config/iam-roles/leveregeSvcAcctActor.json

@@ -0,0 +1,8 @@
+{
+  "description": "Leverege service account act as other user",
+  "includedPermissions": [
+    "iam.serviceAccounts.actAs"
+  ],
+  "stage": "BETA",
+  "title": "Leverege Svc Acct Impersonation"
+}

package/src/service-man/config/iam-roles/leveregeSvcAcctTokenCreator.json

@@ -0,0 +1,8 @@
+{
+  "description": "Leverege IAM blob signing access",
+  "includedPermissions": [
+    "iam.serviceAccounts.signBlob"
+  ],
+  "stage": "BETA",
+  "title": "Leverege Svc Acct Token Creator"
+}

package/src/service-man/config/index.mjs

@@ -0,0 +1,7 @@
+import CommandLine from './CommandLine.mjs'
+import Config from './Config.mjs'
+
+export {
+  CommandLine,
+  Config,
+}

package/src/service-man/config/secrets/authz-jwt.json

@@ -0,0 +1,6 @@
+{
+  "type": "random",
+  "content": {
+    "secret": 64
+  }
+}

package/src/service-man/config/secrets/authz-postgres.json

@@ -0,0 +1,7 @@
+{
+  "type": "random",
+  "immutable": true,
+  "content": {
+    "postgresql-password": 48
+  }
+}

package/src/service-man/config/secrets/db-timescale-dense.json

@@ -0,0 +1,8 @@
+{
+  "type": "random",
+  "immutable": true,
+  "content": {
+    "postgres": 48,
+    "imagine": 48
+  }
+}