@leverege/build-tools 2.92.0 → 2.93.0-beta.11

package/.vscode/settings.json

@@ -0,0 +1,3 @@
+{
+    "editor.formatOnSave": true
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@leverege/build-tools",
-  "version": "2.92.0",
+  "version": "2.93.0-beta.11",
   "description": "A collection of build / support tools for Leverege developers",
   "main": "index.js",
   "repository": {
@@ -11,7 +11,7 @@
     "clean": "rm -rf lib",
     "lint": "find ./src -name \\*.\\*js | xargs npx eslint --report-unused-disable-directives",
     "prepare": "[ $SKIP_PREPARE ] && exit 0 || ./src/hook-and-release.mjs",
-    "test": "echo \"no tests specified\" && exit 0",
+    "test": "mocha -t 60000 --exit 'test/**/*.mjs'",
     "prepack": "npm exec prepack"
   },
   "bin": {
@@ -54,7 +54,9 @@
     "pkglint": "src/pkglint.sh",
     "prepack": "src/prepack.mjs",
     "prune-git": "src/prune-git.sh",
+    "pull-artifact": "src/artifacts/pull-artifact.mjs",
     "pull-git": "src/pull-git.sh",
+    "push-artifact": "src/artifacts/push-artifact.mjs",
     "push-my-chart": "src/push-my-chart.mjs",
     "refresh-npm-token": "src/refresh-npm-token.mjs",
     "refresh-py-idx": "src/refresh-py-idx.sh",
@@ -91,11 +93,14 @@
     "shell-quote": "^1.8.3",
     "simple-git": "^3.30.0",
     "sloc": "^0.3.2",
+    "superstruct": "^2.0.2",
     "toml": "^3.0.0",
     "zx": "^8.8.5"
   },
   "devDependencies": {
     "@leverege/eslint-config-leverege": "^5.1.1",
+    "chai": "^6.2.1",
+    "mocha": "^11.7.5",
     "npm": "^11.6.2"
   }
-}
+}

package/src/Utils.mjs

@@ -224,6 +224,15 @@ export const gitRepoIsDirty = async () => {
   return cleanliness?.length > 0
 }
 
+export const parseYamlFile = async ( filePath ) => {
+  try {
+    const fileContent = await fs.promises.readFile( filePath, 'utf8' )
+    return YAML.load( fileContent )
+  } catch ( error ) {
+    throw new Error( `Failed to parse YAML file ${filePath}: ${error.message}` )
+  }
+}
+
 export const parseJsonFile = async ( jsonFile ) => {
   // TODO: rework this file exists logic / throw
   if ( !fs.existsSync( jsonFile ) ) { return undefined } // eslint-disable-line security/detect-non-literal-fs-filename

package/src/artifacts/ArtifactPuller.mjs

@@ -0,0 +1,148 @@
+import fs from 'fs'
+
+import { toExplicitPath } from './Utils.mjs'
+
+class ArtifactPuller {
+  constructor( artifact ) {
+    this.artifact = artifact
+  }
+
+  async pull() {
+    throw new Error( 'Not implemented' )
+  }
+}
+
+class GenericArtifactPuller extends ArtifactPuller {
+
+  async downloadFile( remoteFilePath, localFilePath ) {
+    const command = `gcloud artifacts generic download \
+      --project=${this.artifact.gcp_project_id} \
+      --location=${this.artifact.gcp_location} \
+      --repository=${this.artifact.gcp_artifact_registry_repository_name} \
+      --package=${this.artifact.gcp_artifact_registry_package_name} \
+      --version=${this.artifact.gcp_artifact_registry_package_version} \
+      --name="${remoteFilePath}" \
+      --destination=/tmp`
+
+    const oldPath = `/tmp/${remoteFilePath.split( '/' ).pop()}`
+    const newPath = localFilePath
+
+    // if file exists in the old path, delete it
+    if ( fs.existsSync( oldPath ) ) {
+      fs.unlinkSync( oldPath )
+    }
+
+    // if file exists in the new path, delete it
+    if ( fs.existsSync( newPath ) ) {
+      fs.unlinkSync( newPath )
+    }
+
+    // Ensure the directory exists
+    fs.mkdirSync( localFilePath.split( '/' ).slice( 0, -1 ).join( '/' ), { recursive : true } )
+
+    console.log( 'Command:', command.replace( /\s+/g, ' ' ).trim() )
+
+    const { execSync } = await import( 'child_process' )
+    const result = execSync( command, { encoding : 'utf8', stdio : 'pipe' } )
+    console.log( 'Result:', result )
+
+    // Check if file exists in the old path
+    if ( !fs.existsSync( oldPath ) ) {
+      console.error( `File does not exist in the old path: ${oldPath}` )
+      return false
+    }
+
+    console.log( `Copying file from ${oldPath} to ${newPath}` )
+
+    await fs.copyFileSync( oldPath, newPath )
+
+    // Check if file exists in the new path
+    if ( !fs.existsSync( newPath ) ) {
+      console.error( `File does not exist in the new path: ${newPath}` )
+      return false
+    }
+
+    return true
+  }
+
+  async downloadDirectory( localPath ) {
+    const command = `gcloud artifacts generic download \
+      --project=${this.artifact.gcp_project_id} \
+      --location=${this.artifact.gcp_location} \
+      --repository=${this.artifact.gcp_artifact_registry_repository_name} \
+      --package=${this.artifact.gcp_artifact_registry_package_name} \
+      --version=${this.artifact.gcp_artifact_registry_package_version} \
+      --destination=${localPath}`
+
+    console.log( 'Command:', command.replace( /\s+/g, ' ' ).trim() )
+
+    const { execSync } = await import( 'child_process' )
+    execSync( command, { encoding : 'utf8', stdio : 'pipe' } )
+
+    return true
+  }
+
+  async downloadNestedDirectory( remotePath, localPath ) {
+    const files = await this.listFiles( remotePath )
+    await Promise.all(
+      files
+        .filter( file => file.name.split( ':' ).pop().replaceAll( '%2F', '/' ).startsWith( remotePath ) )
+        .map( async ( file ) => {
+          const remoteFilePath = file.name.split( ':' ).pop().replaceAll( '%2F', '/' )
+          const localFilePath = `${localPath}${remoteFilePath.replaceAll( remotePath, '' )}`
+          await this.downloadFile( remoteFilePath, localFilePath )
+        } )
+    )
+    return true
+  }
+
+  async listFiles( ) {
+    const command = `gcloud artifacts files list \
+        --project=${this.artifact.gcp_project_id} \
+        --location=${this.artifact.gcp_location} \
+        --repository=${this.artifact.gcp_artifact_registry_repository_name} \
+        --package=${this.artifact.gcp_artifact_registry_package_name} \
+        --version ${this.artifact.gcp_artifact_registry_package_version} \
+        --format="json"`
+
+    console.log( 'Command:', command.replace( /\s+/g, ' ' ).trim() )
+      
+    const { execSync } = await import( 'child_process' )
+    const result = execSync( command, { encoding : 'utf8', stdio : 'pipe' } )
+  
+    // Parse the JSON response
+    const files = JSON.parse( result )
+
+    return files
+  }
+
+  async pull() {
+    console.log( 'Pulling artifact from registry...' )
+
+    if ( this.artifact.remote_file_path && this.artifact.local_file_path ) {
+      return this.downloadFile( this.artifact.remote_file_path, toExplicitPath( this.artifact.local_file_path ) )
+    }
+    if ( this.artifact.remote_file_path && this.artifact.local_path ) {
+      return this.downloadFile( this.artifact.remote_file_path, toExplicitPath( `${this.artifact.local_path}/${this.artifact.remote_file_path.split( '/' ).pop()}` ) )
+    }
+    if ( this.artifact.remote_path && this.artifact.local_path ) {
+      return this.downloadNestedDirectory( this.artifact.remote_path, toExplicitPath( this.artifact.local_path ) )
+    }
+    if ( this.artifact.remote_path && !this.artifact.local_path && !this.artifact.local_file_path ) {
+      return this.downloadDirectory( this.artifact.remote_path, '.' )
+    }
+    if ( !this.artifact.remote_path && !this.artifact.remote_file_path && this.artifact.local_path ) {
+      return this.downloadDirectory( toExplicitPath( this.artifact.local_path ) )
+    }
+    // if ( !this.artifact.remote_path && !this.artifact.remote_file_path && this.local_file_path ) {
+    //   throw new Error( '' )
+    // }
+    // if ( this.artifact.remote_path && this.artifact.local_file_path ) {
+    //   throw new Error( '' )
+    // }
+
+    return false
+  }
+}
+
+export { ArtifactPuller, GenericArtifactPuller }

package/src/artifacts/ArtifactPusher.mjs

@@ -0,0 +1,78 @@
+class ArtifactPusher {
+  constructor( artifact ) {
+    this.artifact = artifact
+  }
+
+  async push() {
+    throw new Error( 'Not implemented' )
+  }
+}
+
+class GenericArtifactPusher extends ArtifactPusher {
+
+  async listFiles( ) {
+    const command = `gcloud artifacts files list \
+        --project=${this.artifact.gcp_project_id} \
+        --location=${this.artifact.gcp_location} \
+        --repository=${this.artifact.gcp_artifact_registry_repository_name} \
+        --package=${this.artifact.gcp_artifact_registry_package_name} \
+        --version ${this.artifact.gcp_artifact_registry_package_version} \
+        --format="json"`
+
+    console.log( 'Command:', command.replace( /\s+/g, ' ' ).trim() )
+      
+    const { execSync } = await import( 'child_process' )
+    const result = execSync( command, { encoding : 'utf8', stdio : 'pipe' } )
+  
+    // Parse the JSON response
+    const files = JSON.parse( result )
+
+    return files
+  }
+    
+  async push() {
+    console.log( 'Pushing artifact to registry...' )
+
+    if ( this.artifact.remote_file_path ) {
+      return false
+    }
+
+    console.log( 'Uploading artifact to registry...' )
+   
+    let command = ''
+
+    command = `gcloud artifacts generic upload \
+      --project=${this.artifact.gcp_project_id} \
+      --location=${this.artifact.gcp_location} \
+      --repository=${this.artifact.gcp_artifact_registry_repository_name} \
+      --package=${this.artifact.gcp_artifact_registry_package_name} \
+      --version=${this.artifact.gcp_artifact_registry_package_version}`
+
+    if ( this.artifact.local_path ) {
+      command += ` --source-directory=${this.artifact.local_path}`
+      command += ' --skip-existing'
+    } else if ( this.artifact.local_file_path ) {
+      const files = await this.listFiles( this.artifact.remote_path )
+      if ( files.some( file => file.name.split( ':' ).pop().replaceAll( '%2F', '/' ) === `${this.artifact.remote_path}/${this.artifact.remote_file_path.split( '/' ).pop()}` ) ) {
+        return false
+      }
+      command += ` --source=${this.artifact.local_file_path}`
+    }
+
+    command += ` --destination-path=${this.artifact.remote_path}`
+
+    console.log( 'Command:', command.replace( /\s+/g, ' ' ).trim() )
+        
+    try {
+      const { execSync } = await import( 'child_process' )
+      execSync( command, { encoding : 'utf8', stdio : 'pipe' } )
+      console.log( `Successfully published artifact: ${this.artifact.gcp_artifact_registry_package_name}:${this.artifact.gcp_artifact_registry_package_version}` )
+      return true
+    } catch ( error ) {
+      console.error( 'Error publishing artifact:', error.message )
+      return false
+    }
+  }
+}
+
+export { ArtifactPusher, GenericArtifactPusher }

package/src/artifacts/Structs.mjs

@@ -0,0 +1,91 @@
+import { assign, object, string, array, optional, enums, union, refine } from 'superstruct'
+
+const ArtifactTypeEnum = {
+  GENERIC : 'generic',
+}
+
+// Superstruct schema for individual artifact configuration
+const PartialArtifactSchema = object( {
+  name : string(),
+  type : optional( enums( Object.values( ArtifactTypeEnum ) ) ),
+  gcp_project_id : optional( string() ),
+  gcp_location : optional( string() ),
+  gcp_artifact_registry_repository_name : optional( string() ),
+  gcp_artifact_registry_package_name : optional( string() ),
+  gcp_artifact_registry_package_version : optional( string() )
+} )
+
+const ArtifactSchema = assign( PartialArtifactSchema, object( {
+  name : string(),
+  type : enums( Object.values( ArtifactTypeEnum ) ),
+  gcp_project_id : string(),
+  gcp_location : string(),
+  gcp_artifact_registry_repository_name : string(),
+  gcp_artifact_registry_package_name : string(),
+  gcp_artifact_registry_package_version : string()
+} ) )
+
+const PartialGenericArtifactSchema = refine(
+  assign( PartialArtifactSchema, object( {
+    type : enums( [ ArtifactTypeEnum.GENERIC ] ),
+    local_file_path : optional( string() ),
+    local_path : optional( string() ),
+    remote_file_path : optional( string() ),
+    remote_path : optional( string() )
+  } ) ),
+  'ConflictingPaths',
+  ( value ) => {
+    if ( value.local_file_path && value.local_path ) {
+      return 'Only one of local_file_path or local_path must be provided'
+    }
+    if ( value.remote_file_path && value.remote_path ) {
+      return 'Only one of remote_file_path or remote_path must be provided'
+    }
+    return true
+  }
+)
+
+const GenericArtifactSchema = refine(
+  assign( ArtifactSchema, object( {
+    local_file_path : optional( string() ),
+    local_path : optional( string() ),
+    remote_file_path : optional( string() ),
+    remote_path : optional( string() )
+  } ) ),
+  'ConflictingPaths',
+  ( value ) => {
+    if ( value.local_file_path && value.local_path ) {
+      return 'Only one of local_file_path or local_path must be provided'
+    }
+    if ( value.remote_file_path && value.remote_path ) {
+      return 'Only one of remote_file_path or remote_path must be provided'
+    }
+    return true
+  }
+)
+
+// Superstruct schema for defaults configuration
+const DefaultsSchema = object( {
+  gcp_project_id : optional( string() ),
+  gcp_location : optional( string() ),
+  gcp_artifact_registry_repository_name : optional( string() ),
+  gcp_artifact_registry_package_name : optional( string() ),
+  gcp_artifact_registry_package_version : optional( string() )
+} )
+
+// Superstruct schema for the complete artifacts configuration with defaults
+const ArtifactsSchema = object( {
+  defaults : optional( DefaultsSchema ),
+  artifacts : array( union( [ PartialArtifactSchema, PartialGenericArtifactSchema ] ) )
+} )
+
+const FlattenedArtifactsSchema = array( assign( union( [ ArtifactSchema, GenericArtifactSchema ] ) ) )
+
+export {
+  ArtifactTypeEnum,
+  ArtifactsSchema,
+  FlattenedArtifactsSchema,
+  ArtifactSchema,
+  DefaultsSchema,
+  GenericArtifactSchema,
+}

package/src/artifacts/Utils.mjs

@@ -0,0 +1,113 @@
+import fs from 'fs'
+import crypto from 'crypto'
+
+import { validate } from 'superstruct'
+
+import { parseYamlFile, warning } from '../Utils.mjs'
+
+import { ArtifactTypeEnum, ArtifactsSchema, GenericArtifactSchema } from './Structs.mjs'
+
+const getArtifactsSchema = async ( artifactsFilePath = './artifacts.yml' ) => {
+  const artifactsConfig = await parseYamlFile( artifactsFilePath )
+  const [ error, result ] = validate( artifactsConfig, ArtifactsSchema )
+  if ( error ) {
+    throw new Error( `Invalid artifacts.yml file: ${error.message}` )
+  }
+  return result
+}
+
+const flattenArtifactsSchema = ( artifactsSchema ) => {
+  return artifactsSchema.artifacts.reduce( ( flattenedArtifacts, artifact ) => {
+    const mergedArtifact = {
+      ...artifactsSchema.defaults,
+      ...artifact
+    }
+    
+    let schema
+    switch ( mergedArtifact.type ) {
+      case ArtifactTypeEnum.GENERIC:
+        schema = GenericArtifactSchema
+        break
+      default:
+        warning( `Unsupported artifact type "${mergedArtifact.type}"` )
+        break
+    }
+
+    if ( !schema ) {
+      warning( `Invalid artifact type "${mergedArtifact.type}"` )
+      return flattenedArtifacts
+    }
+    
+    const [ error, validatedArtifact ] = validate( mergedArtifact, schema )
+    
+    if ( error ) {
+      warning( `Invalid artifact "${mergedArtifact.name}": ${error.message}` )
+      return flattenedArtifacts
+    }
+    
+    return [ ...flattenedArtifacts, validatedArtifact ]
+  }, [] )
+}
+
+const toImplicitPath = ( path ) => {
+  let implicitPath = path
+  // Remove leading './' if present
+  if ( implicitPath.startsWith( './' ) ) {
+    implicitPath = implicitPath.slice( 2 )
+  }
+  // Remove trailing slash
+  implicitPath = implicitPath.replace( /\/+$/, '' )
+  return implicitPath
+}
+
+const toExplicitPath = ( path ) => {
+  let explicitPath = path
+  // Keep absolute paths and paths already starting with '.' as-is
+  if ( !path.startsWith( '/' ) && !path.startsWith( '.' ) ) {
+    // Add './' to relative paths
+    explicitPath = `./${explicitPath}`
+  }
+  return explicitPath
+}
+
+const buildGenericArtifactName = ( artifact ) => {
+  const baseElements = [
+    'projects',
+    artifact.gcp_project_id,
+    'locations',
+    artifact.gcp_location,
+    'repositories',
+    artifact.gcp_artifact_registry_repository_name,
+    'files',
+    artifact.gcp_artifact_registry_package_name,
+  ]
+
+  const versionElement = artifact.gcp_artifact_registry_package_version
+
+  const remotePathElements = []
+
+  if ( artifact.remote_path || artifact.remote_file_path ) {
+    const explicitRemotePath = toExplicitPath( artifact.remote_path || artifact.remote_file_path )
+    const implicitRemotePath = toImplicitPath( explicitRemotePath )
+    if ( implicitRemotePath !== '.' ) {
+      remotePathElements.push( implicitRemotePath )
+    }
+  }
+
+  const name = [ baseElements.join( '/' ), versionElement, encodeURIComponent( remotePathElements.join( '/' ) ) ].join( ':' )
+
+  return name
+}
+
+const calculateHash = ( path ) => {
+  try {
+    const hash = crypto.createHash( 'sha256' )
+    hash.update( fs.readFileSync( path ) )
+    return hash.digest( 'base64url' )
+  } catch ( error ) {
+    console.error( error )
+    throw new Error( `Failed to calculate hash for artifact ${path}: ${error.message}` )
+  }
+}
+
+export { calculateHash, getArtifactsSchema, flattenArtifactsSchema, toImplicitPath, toExplicitPath, buildGenericArtifactName }

package/src/artifacts/pull-artifact.mjs

@@ -0,0 +1,68 @@
+#!/usr/bin/env node
+
+import { warning } from '../Utils.mjs'
+
+import { getArtifactsSchema, flattenArtifactsSchema } from './Utils.mjs'
+import { ArtifactTypeEnum } from './Structs.mjs'
+import { GenericArtifactPuller } from './ArtifactPuller.mjs'
+
+async function main() {
+
+  let artifactsYmlPath = null
+  let artifactName = null
+  let artifactPackageName = null
+  let artifactPackageVersion = null
+  let filteredArtifactSchema = null
+  
+  if ( process.argv.length === 3 ) {
+    artifactsYmlPath = process.argv[2]
+    filteredArtifactSchema = await getArtifactsSchema( artifactsYmlPath )
+    const flattenedArtifactsSchema = flattenArtifactsSchema( filteredArtifactSchema )
+    filteredArtifactSchema = flattenedArtifactsSchema
+  } else if ( process.argv.length === 4 ) {
+    artifactsYmlPath = process.argv[2]
+    filteredArtifactSchema = await getArtifactsSchema( artifactsYmlPath )
+    const flattenedArtifactsSchema = flattenArtifactsSchema( filteredArtifactSchema )
+    filteredArtifactSchema = flattenedArtifactsSchema.find(
+      artifact => artifact.name === artifactName )
+  } else if ( process.argv.length === 5 ) {
+    artifactsYmlPath = process.argv[2]
+    artifactName = process.argv[3]
+    artifactPackageName = process.argv[4]
+    artifactPackageVersion = process.argv[5]
+    filteredArtifactSchema = await getArtifactsSchema( artifactsYmlPath )
+    const flattenedArtifactsSchema = flattenArtifactsSchema( filteredArtifactSchema )
+    filteredArtifactSchema = flattenedArtifactsSchema.filter(
+      artifact => artifact.gcp_artifact_registry_package_name === artifactPackageName &&
+      artifact.gcp_artifact_registry_package_version === artifactPackageVersion )
+  } else {
+    console.error( 'Usage:' )
+    console.error( '  pull-artifact.mjs <artifacts.yml>' )
+    console.error( '  pull-artifact.mjs <artifacts.yml> <artifact name>' )
+    console.error( '  pull-artifact.mjs <artifacts.yml> <artifact package name> <artifact package version>' )
+    process.exit( 1 )
+  }
+
+  // TODO: Use concurrency to pull artifacts and provide summary of results.
+  for ( const artifactSchema of filteredArtifactSchema ) {
+    let puller
+    switch ( artifactSchema.type ) {
+      case ArtifactTypeEnum.GENERIC:
+        puller = new GenericArtifactPuller( artifactSchema )
+        break
+      default:
+        warning( `Unsupported artifact type "${artifactSchema.type}"` )
+        continue
+    }
+      
+    if ( !puller ) {
+      warning( `Unsupported artifact type "${artifactSchema.type}"` )
+      continue
+    }
+      
+    // eslint-disable-next-line no-await-in-loop
+    await puller.pull()
+  }
+}
+
+main()

package/src/artifacts/push-artifact.mjs

@@ -0,0 +1,68 @@
+#!/usr/bin/env node
+
+import { warning } from '../Utils.mjs'
+
+import { getArtifactsSchema, flattenArtifactsSchema } from './Utils.mjs'
+import { ArtifactTypeEnum } from './Structs.mjs'
+import { GenericArtifactPusher } from './ArtifactPusher.mjs'
+
+async function main() {
+
+  let artifactsYmlPath = null
+  let artifactName = null
+  let artifactPackageName = null
+  let artifactPackageVersion = null
+  let filteredArtifactSchema = null
+  
+  if ( process.argv.length === 3 ) {
+    artifactsYmlPath = process.argv[2]
+    filteredArtifactSchema = await getArtifactsSchema( artifactsYmlPath )
+    const flattenedArtifactsSchema = flattenArtifactsSchema( filteredArtifactSchema )
+    filteredArtifactSchema = flattenedArtifactsSchema
+  } else if ( process.argv.length === 4 ) {
+    artifactsYmlPath = process.argv[2]
+    filteredArtifactSchema = await getArtifactsSchema( artifactsYmlPath )
+    const flattenedArtifactsSchema = flattenArtifactsSchema( filteredArtifactSchema )
+    filteredArtifactSchema = flattenedArtifactsSchema.find(
+      artifact => artifact.name === artifactName )
+  } else if ( process.argv.length === 5 ) {
+    artifactsYmlPath = process.argv[2]
+    artifactName = process.argv[3]
+    artifactPackageName = process.argv[4]
+    artifactPackageVersion = process.argv[5]
+    filteredArtifactSchema = await getArtifactsSchema( artifactsYmlPath )
+    const flattenedArtifactsSchema = flattenArtifactsSchema( filteredArtifactSchema )
+    filteredArtifactSchema = flattenedArtifactsSchema.filter(
+      artifact => artifact.gcp_artifact_registry_package_name === artifactPackageName &&
+      artifact.gcp_artifact_registry_package_version === artifactPackageVersion )
+  } else {
+    console.error( 'Usage:' )
+    console.error( '  push-artifact.mjs <artifacts.yml>' )
+    console.error( '  push-artifact.mjs <artifacts.yml> <artifact name>' )
+    console.error( '  push-artifact.mjs <artifacts.yml> <artifact package name> <artifact package version>' )
+    process.exit( 1 )
+  }
+
+  // TODO: Use concurrency to push artifacts and provide summary of results.
+  for ( const artifactSchema of filteredArtifactSchema ) {
+    let pusher
+    switch ( artifactSchema.type ) {
+      case ArtifactTypeEnum.GENERIC:
+        pusher = new GenericArtifactPusher( artifactSchema )
+        break
+      default:
+        warning( `Unsupported artifact type "${artifactSchema.type}"` )
+        continue
+    }
+      
+    if ( !pusher ) {
+      warning( `Unsupported artifact type "${artifactSchema.type}"` )
+      continue
+    }
+      
+    // eslint-disable-next-line no-await-in-loop
+    await pusher.push()
+  }
+}
+
+main()