@leverege/build-tools 2.91.1 → 2.93.0-beta.10

package/.vscode/settings.json

@@ -0,0 +1,3 @@
+{
+    "editor.formatOnSave": true
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@leverege/build-tools",
-  "version": "2.91.1",
+  "version": "2.93.0-beta.10",
   "description": "A collection of build / support tools for Leverege developers",
   "main": "index.js",
   "repository": {
@@ -11,7 +11,7 @@
     "clean": "rm -rf lib",
     "lint": "find ./src -name \\*.\\*js | xargs npx eslint --report-unused-disable-directives",
     "prepare": "[ $SKIP_PREPARE ] && exit 0 || ./src/hook-and-release.mjs",
-    "test": "echo \"no tests specified\" && exit 0",
+    "test": "mocha -t 60000 --exit 'test/**/*.mjs'",
     "prepack": "npm exec prepack"
   },
   "bin": {
@@ -54,7 +54,9 @@
     "pkglint": "src/pkglint.sh",
     "prepack": "src/prepack.mjs",
     "prune-git": "src/prune-git.sh",
+    "pull-artifact": "src/artifacts/pull-artifact.mjs",
     "pull-git": "src/pull-git.sh",
+    "push-artifact": "src/artifacts/push-artifact.mjs",
     "push-my-chart": "src/push-my-chart.mjs",
     "refresh-npm-token": "src/refresh-npm-token.mjs",
     "refresh-py-idx": "src/refresh-py-idx.sh",
@@ -91,11 +93,14 @@
     "shell-quote": "^1.8.3",
     "simple-git": "^3.30.0",
     "sloc": "^0.3.2",
+    "superstruct": "^2.0.2",
     "toml": "^3.0.0",
     "zx": "^8.8.5"
   },
   "devDependencies": {
     "@leverege/eslint-config-leverege": "^5.1.1",
+    "chai": "^6.2.1",
+    "mocha": "^11.7.5",
     "npm": "^11.6.2"
   }
-}
+}

package/src/Utils.mjs

@@ -224,6 +224,15 @@ export const gitRepoIsDirty = async () => {
   return cleanliness?.length > 0
 }
 
+export const parseYamlFile = async ( filePath ) => {
+  try {
+    const fileContent = await fs.promises.readFile( filePath, 'utf8' )
+    return YAML.load( fileContent )
+  } catch ( error ) {
+    throw new Error( `Failed to parse YAML file ${filePath}: ${error.message}` )
+  }
+}
+
 export const parseJsonFile = async ( jsonFile ) => {
   // TODO: rework this file exists logic / throw
   if ( !fs.existsSync( jsonFile ) ) { return undefined } // eslint-disable-line security/detect-non-literal-fs-filename

package/src/artifacts/ArtifactPuller.mjs

@@ -0,0 +1,21 @@
+import crypto from 'crypto'
+import fs from 'fs'
+
+class ArtifactPusher {
+  constructor( artifact ) {
+    this.artifact = artifact
+  }
+
+  async pull() {
+    throw new Error( 'Not implemented' )
+  }
+}
+
+class GenericArtifactPusher extends ArtifactPusher {
+
+  async pull() {
+    throw new Error( 'Not implemented' )
+  }
+}
+
+export { ArtifactPusher, GenericArtifactPusher }

package/src/artifacts/ArtifactPusher.mjs

@@ -0,0 +1,133 @@
+import crypto from 'crypto'
+import fs from 'fs'
+
+class ArtifactPusher {
+  constructor( artifact ) {
+    this.artifact = artifact
+  }
+  
+  calculateHash() {
+    try {
+      const hash = crypto.createHash( 'sha256' )
+      hash.update( fs.readFileSync( this.artifact.source_path ) )
+      return hash.digest( 'base64url' )
+    } catch ( error ) {
+      console.error( error )
+      throw new Error( `Failed to calculate hash for artifact ${this.artifact.name}: ${error.message}` )
+    }
+  }
+
+  async exists() {
+    throw new Error( 'Not implemented' )
+  }
+
+  async push() {
+    throw new Error( 'Not implemented' )
+  }
+}
+
+class GenericArtifactPusher extends ArtifactPusher {
+
+  async exists() {
+    const isDirectory = fs.statSync( this.artifact.source_path ).isDirectory()
+    
+    // If source path is a directory, check whether the destination path alreadu exists in the registry
+    if ( isDirectory ) {
+      // For the time being, return false, and let gcloud skip existing files in upload-time.
+      // It's pretty slow, might be worth to break down the directory into individual artifacts
+      // and manage each one with its own pusher.
+      return false
+    }
+
+    // Check if artifact already exists in the registry
+    console.log( 'Checking if artifact already exists in the registry...' )
+        
+    const artifactHash = this.calculateHash()
+        
+    const command = `gcloud artifacts files list \
+            --project=${this.artifact.gcp_project_id} \
+            --location=${this.artifact.gcp_location} \
+            --repository=${this.artifact.gcp_artifact_registry_repository_name} \
+            --package=${this.artifact.gcp_artifact_registry_package_name} \
+            --version ${this.artifact.gcp_artifact_registry_package_version} \
+            --filter="name:${this.artifact.destination_path}" \
+            --format="json"`
+
+    console.log( 'Command:', command.replace( /\s+/g, ' ' ).trim() )
+    
+    try {
+      const { execSync } = await import( 'child_process' )
+      const result = execSync( command, { encoding : 'utf8', stdio : 'pipe' } )
+
+      // Parse the JSON response
+      const files = JSON.parse( result )
+    
+      // Check if any files were found and match our hash
+      if ( files && files.length > 0 ) {
+        for ( const file of files ) {
+          for ( const hash of file.hashes ) {
+            // Compare without padding since base64url may omit trailing '=' 
+            const normalizedOurHash = artifactHash.replace( /=+$/, '' )
+            const normalizedRegistryHash = hash.value.replace( /=+$/, '' )
+            if ( hash.type === 'SHA256' && normalizedRegistryHash === normalizedOurHash ) {
+              console.log( `Artifact already exists in the registry: ${this.artifact.gcp_artifact_registry_package_name}:${this.artifact.gcp_artifact_registry_package_version}` )
+              return true
+            }
+          }
+        }
+      }
+      console.log( 'Artifact does not exist in the registry' )
+      return false
+          
+    } catch ( error ) {
+      console.error( 'Error checking artifact existence:', error.message )
+      throw error
+    }
+  }
+    
+  async push() {
+    console.log( 'Publishing artifact to registry...' )
+
+    if ( await this.exists() ) {
+      console.log( 'Artifact already exists in the registry' )
+      return false
+    }
+
+    console.log( 'Uploading artifact to registry...' )
+   
+    let command = ''
+
+    command = `gcloud artifacts generic upload \
+      --project=${this.artifact.gcp_project_id} \
+      --location=${this.artifact.gcp_location} \
+      --repository=${this.artifact.gcp_artifact_registry_repository_name} \
+      --package=${this.artifact.gcp_artifact_registry_package_name} \
+      --version=${this.artifact.gcp_artifact_registry_package_version}`
+
+    const isDirectory = fs.statSync( this.artifact.source_path ).isDirectory()
+    if ( isDirectory ) {
+      command += ` --source-directory=${this.artifact.source_path}`
+      command += ' --skip-existing'
+    } else {
+      command += ` --source=${this.artifact.source_path}`
+    }
+
+    if ( this.artifact.destination_path ) {
+      command += ` --destination-path=${this.artifact.destination_path}`
+    }
+    
+    console.log( 'Command:', command.replace( /\s+/g, ' ' ).trim() )
+        
+    try {
+      const { execSync } = await import( 'child_process' )
+      execSync( command, { encoding : 'utf8', stdio : 'pipe' } )
+      console.log( `Successfully published artifact: ${this.artifact.gcp_artifact_registry_package_name}:${this.artifact.gcp_artifact_registry_package_version}` )
+      return true
+    } catch ( error ) {
+      console.error( 'Error publishing artifact:', error.message )
+      return false
+    }
+  }
+}
+
+export { ArtifactPusher, GenericArtifactPusher }

package/src/artifacts/Structs.mjs

@@ -0,0 +1,63 @@
+import { assign, object, string, array, optional, enums, union } from 'superstruct'
+
+const ArtifactTypeEnum = {
+  GENERIC : 'generic',
+}
+
+// Superstruct schema for individual artifact configuration
+const PartialArtifactSchema = object( {
+  name : string(),
+  type : optional( enums( Object.values( ArtifactTypeEnum ) ) ),
+  gcp_project_id : optional( string() ),
+  gcp_location : optional( string() ),
+  gcp_artifact_registry_repository_name : optional( string() ),
+  gcp_artifact_registry_package_name : optional( string() ),
+  gcp_artifact_registry_package_version : optional( string() )
+} )
+
+const ArtifactSchema = assign( PartialArtifactSchema, object( {
+  name : string(),
+  type : enums( Object.values( ArtifactTypeEnum ) ),
+  gcp_project_id : string(),
+  gcp_location : string(),
+  gcp_artifact_registry_repository_name : string(),
+  gcp_artifact_registry_package_name : string(),
+  gcp_artifact_registry_package_version : string()
+} ) )
+
+const PartialGenericArtifactSchema = assign( PartialArtifactSchema, object( {
+  type : enums( [ ArtifactTypeEnum.GENERIC ] ),
+  source_path : string(),
+  destination_path : string()
+} ) )
+
+const GenericArtifactSchema = assign( ArtifactSchema, object( {
+  source_path : string(),
+  destination_path : optional( string() )
+} ) )
+
+// Superstruct schema for defaults configuration
+const DefaultsSchema = object( {
+  gcp_project_id : optional( string() ),
+  gcp_location : optional( string() ),
+  gcp_artifact_registry_repository_name : optional( string() ),
+  gcp_artifact_registry_package_name : optional( string() ),
+  gcp_artifact_registry_package_version : optional( string() )
+} )
+
+// Superstruct schema for the complete artifacts configuration with defaults
+const ArtifactsSchema = object( {
+  defaults : optional( DefaultsSchema ),
+  artifacts : array( union( [ PartialArtifactSchema, PartialGenericArtifactSchema ] ) )
+} )
+
+const FlattenedArtifactsSchema = array( assign( union( [ ArtifactSchema, GenericArtifactSchema ] ) ) )
+
+export {
+  ArtifactTypeEnum,
+  ArtifactsSchema,
+  FlattenedArtifactsSchema,
+  ArtifactSchema,
+  DefaultsSchema,
+  GenericArtifactSchema,
+}

package/src/artifacts/Utils.mjs

@@ -0,0 +1,49 @@
+import { validate } from 'superstruct'
+
+import { parseYamlFile, warning } from '../Utils.mjs'
+
+import { ArtifactTypeEnum, ArtifactsSchema, GenericArtifactSchema } from './Structs.mjs'
+
+const getArtifactsSchema = async ( artifactsFilePath = './artifacts.yml' ) => {
+  const artifactsConfig = await parseYamlFile( artifactsFilePath )
+  const [ error, result ] = validate( artifactsConfig, ArtifactsSchema )
+  if ( error ) {
+    throw new Error( `Invalid artifacts.yml file: ${error.message}` )
+  }
+  return result
+}
+
+const flattenArtifactsSchema = ( artifactsSchema ) => {
+  return artifactsSchema.artifacts.reduce( ( flattenedArtifacts, artifact ) => {
+    const mergedArtifact = {
+      ...artifactsSchema.defaults,
+      ...artifact
+    }
+    
+    let schema
+    switch ( mergedArtifact.type ) {
+      case ArtifactTypeEnum.GENERIC:
+        schema = GenericArtifactSchema
+        break
+      default:
+        warning( `Unsupported artifact type "${mergedArtifact.type}"` )
+        break
+    }
+
+    if ( !schema ) {
+      warning( `Invalid artifact type "${mergedArtifact.type}"` )
+      return flattenedArtifacts
+    }
+    
+    const [ error, validatedArtifact ] = validate( mergedArtifact, schema )
+    
+    if ( error ) {
+      warning( `Invalid artifact "${mergedArtifact.name}": ${error.message}` )
+      return flattenedArtifacts
+    }
+    
+    return [ ...flattenedArtifacts, validatedArtifact ]
+  }, [] )
+}
+
+export { getArtifactsSchema, flattenArtifactsSchema }

package/src/artifacts/pull-artifact.mjs

@@ -0,0 +1,7 @@
+#!/usr/bin/env node
+
+async function main() {
+  return null
+}
+
+main()

package/src/artifacts/push-artifact.mjs

@@ -0,0 +1,54 @@
+#!/usr/bin/env node
+
+import { warning } from '../Utils.mjs'
+
+import { getArtifactsSchema, flattenArtifactsSchema } from './Utils.mjs'
+import { ArtifactTypeEnum } from './Structs.mjs'
+import { GenericArtifactPusher } from './ArtifactPusher.mjs'
+
+async function main() {
+
+  // If an argument is provided, use it as the artifacts.yml file path
+  const artifactsYmlPath = process.argv[2]
+  const artifactPackageName = process.argv[3]
+  const artifactPackageVersion = process.argv[4]
+
+  if ( artifactsYmlPath ) {
+    const artifactsSchema = await getArtifactsSchema( artifactsYmlPath )
+    const flattenedArtifactsSchema = flattenArtifactsSchema( artifactsSchema )
+    
+    // Open for unexpected matches if we end up re-using the same package name and version for
+    // multiple artifacts spread across multiple repositories and/or projects.
+    const filteredArtifactSchemas = flattenedArtifactsSchema.filter(
+      artifact => artifact.gcp_artifact_registry_package_name === artifactPackageName &&
+      artifact.gcp_artifact_registry_package_version === artifactPackageVersion )
+    
+    if ( filteredArtifactSchemas.length === 0 ) {
+      console.error( `No artifacts found for ${artifactPackageName} version ${artifactPackageVersion} in ${artifactsYmlPath}` )
+      process.exit( 1 )
+    }
+
+    // TODO: Use concurrency to package artifacts and provide summary of results.
+    for ( const artifactSchema of filteredArtifactSchemas ) {
+      let packager
+      switch ( artifactSchema.type ) {
+        case ArtifactTypeEnum.GENERIC:
+          packager = new GenericArtifactPusher( artifactSchema )
+          break
+        default:
+          warning( `Unsupported artifact type "${artifactSchema.type}"` )
+          continue
+      }
+      
+      if ( !packager ) {
+        warning( `Unsupported artifact type "${artifactSchema.type}"` )
+        continue
+      }
+      
+      // eslint-disable-next-line no-await-in-loop
+      await packager.package()
+    }
+  }
+}
+
+main()

package/src/refresh-npm-token.mjs

@@ -25,9 +25,13 @@ import commandLineArgs from 'command-line-args'
 import commandLineUsage from 'command-line-usage'
 import ms from 'ms'
 import { lt as semverLt } from 'semver'
+import YAML from 'js-yaml'
 
 import { debug, log, shellCmd } from './Utils.mjs'
 
+const AUTH_TOKEN_REGEX = /_authToken=(.+)/
+const YARNRC_FILE_NAME = `${os.homedir()}/.yarnrc.yml`
+
 const refreshTokenOptions = [ // Use refreshTokenOptions to tie into the Usage statements
   {
     name : 'project',
@@ -59,6 +63,12 @@ const refreshTokenOptions = [ // Use refreshTokenOptions to tie into the Usage s
     default : false,
     description : '{green display this help screen}',
   },
+  {
+    name : 'update-yarnrc',
+    type : Boolean,
+    defaultValue : true,
+    description : '{green updates the npmAuthToken variable of your .yarnrc.yml file (in addition to .npmrc/.npmrc.ro)}'
+  }
 ]
 
 const sections = [
@@ -195,11 +205,14 @@ const getGcpSecretCommand = ( secretName ) => {
 
 // fetch the contents of the npmrc secret file stored on the specified project
 let npmrcFile
+let npmAuthToken
 try {
   const npmTokenFetcher = getGcpSecretCommand( 'REFRESH_NPM_NPMRC' )
   debug( { npmTokenFetcher }, '<==Secret Fetcher' )
 
   npmrcFile = await shellCmd( npmTokenFetcher )
+  npmAuthToken = npmrcFile.match( AUTH_TOKEN_REGEX )?.[1]
+  debug( { npmAuthToken }, '<==npmAuthToken' )
   debug( { npmrcFile }, '<==shellCmd Output npmrc file' )// debug
 } catch ( err ) {
   log( chalk.yellow( `\nFailed Command => [${err.escapedCommand}]\n\n` ), chalk.red( err.stderr ) )
@@ -234,6 +247,18 @@ try {
       fs.copyFileSync( filename, `${filename}-BAK` ) // make a backup for safety
     }
     fs.writeFileSync( filename, npmrcFile )
+
+    if ( args.updateYarnrc && npmAuthToken ) {
+      let yarnRcContent = { nodeLinker : 'node-modules' }
+      if ( fs.existsSync( YARNRC_FILE_NAME ) ) {
+        fs.copyFileSync( YARNRC_FILE_NAME, `${YARNRC_FILE_NAME}-BAK` )
+        yarnRcContent = YAML.load( fs.readFileSync( YARNRC_FILE_NAME, 'utf-8' ) )
+        debug( { yarnRcContent }, '<==yarnrc content' )
+        yarnRcContent.npmAuthToken = npmAuthToken
+      }
+
+      fs.writeFileSync( YARNRC_FILE_NAME, YAML.dump( yarnRcContent ) )
+    }
   }
 } catch ( fileErr ) {
   log( fileErr, 'fs failed' )

package/lib/server/build-tools.js

@@ -1,138 +0,0 @@
-#!/usr/bin/env node
-"use strict";
-
-const fs = require('node:fs');
-const path = require('node:path');
-const chalk = require('chalk');
-const cliArgs = require('command-line-args');
-const cliHelp = require('command-line-usage');
-const npmRegistryFetch = require('npm-registry-fetch');
-const semverLt = require('semver/functions/lt');
-
-/* eslint-disable no-console */
-
-const optionList = [
-// Use optionList to tie into the Usage statements
-{
-  name: 'root',
-  type: Boolean,
-  default: false,
-  description: 'returns the root of the build-tools installation'
-}, {
-  name: 'latest',
-  type: Boolean,
-  default: false,
-  description: 'verifies the latest build-tools are installed'
-}, {
-  name: 'bashfun',
-  type: Boolean,
-  default: false,
-  description: 'source the output to define common bash helper functions'
-}, {
-  name: 'reporoot',
-  type: Boolean,
-  default: false,
-  description: 'the root of the build-tools repo - for finding other config files'
-}, {
-  name: 'help',
-  type: Boolean,
-  default: false,
-  description: 'display this help screen'
-}, {
-  name: 'version',
-  type: Boolean,
-  default: false,
-  description: 'returns the build-tools repo version'
-}, {
-  name: 'verbose',
-  alias: 'v',
-  type: Boolean,
-  default: false,
-  description: 'emit additional info at run time'
-}];
-const sections = [{
-  header: 'A collection of build / support tools for all Leverege code',
-  content: `README: {green https://bitbucket.org/leverege/build-tools/src/development}
-`
-}, {
-  header: 'Options',
-  optionList
-}];
-const args = cliArgs(optionList, {
-  partial: true
-});
-const help = cliHelp(sections);
-const repoRoot = path.dirname(__dirname);
-const thisPackage = require(`${repoRoot}/package.json`);
-const thisVersion = thisPackage.version;
-if (args.root) {
-  console.log(repoRoot);
-  process.exit(0);
-}
-if (args.bashfun) {
-  console.log(`${repoRoot}/src/bash-funcs`);
-  process.exit(0);
-}
-if (args.reporoot) {
-  console.log(`${repoRoot}`);
-  process.exit(0);
-}
-const checkForLatest = async (pkg = '@leverege/build-tools') => {
-  const getToken = () => {
-    const tokenFile = `${process.env.HOME}/.npmrc`;
-    if (!fs.existsSync(tokenFile)) {
-      console.error(`Cannot find token file ${tokenFile}`);
-      process.exit(1);
-    }
-    try {
-      const tokenLine = fs.readFileSync(tokenFile).toString();
-      const tokenRegX = new RegExp('.*registry.npmjs.org\\/:\\w+=+(.*)');
-      const tokenStr = tokenLine.match(tokenRegX);
-      if (!tokenStr) {
-        console.error(`\n***ERROR: malformed npm token in ${tokenFile}\n`);
-        process.exit(1);
-      }
-      return tokenLine.match(tokenRegX)[1];
-    } catch (err) {
-      console.error(err);
-    }
-  };
-  try {
-    const list = await npmRegistryFetch.json(pkg, {
-      '//registry.npmjs.org/:_authToken': getToken()
-    });
-    return list['dist-tags'].latest;
-  } catch (err) {
-    console.log(err);
-  }
-};
-const checkAndAnnounce = () => {
-  checkForLatest().then(latestVersion => {
-    if (semverLt(thisVersion, latestVersion)) {
-      console.log(chalk.white`
-  Update available ${chalk.grey(thisVersion)} \u2b62 ${chalk.green(latestVersion)}
-  Run ${chalk.cyan('npm i -g @leverege/build-tools')} to update
-`);
-      process.exit(1);
-    } else {
-      process.exit(0);
-    }
-  }).catch(err => {});
-};
-if (args.latest) {
-  checkAndAnnounce();
-}
-if (args.version) {
-  console.log(`build-tools version ${thisVersion}`);
-  checkAndAnnounce();
-}
-if (args.help) {
-  console.log(help);
-  process.exit(0);
-}
-
-/* eslint-disable no-underscore-dangle */
-if (args._unknown) {
-  console.log(`\nUnrecognized argument [${args._unknown}] try --help\n`);
-  process.exit(1);
-}