@leverege/build-tools 2.93.0-beta.10 → 2.93.0-beta.12

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@leverege/build-tools",
-  "version": "2.93.0-beta.10",
+  "version": "2.93.0-beta.12",
   "description": "A collection of build / support tools for Leverege developers",
   "main": "index.js",
   "repository": {

package/src/artifacts/ArtifactPuller.mjs

@@ -1,7 +1,11 @@
-import crypto from 'crypto'
+/* eslint-disable security/detect-non-literal-fs-filename */
 import fs from 'fs'
 
-class ArtifactPusher {
+import { shellCmd, log, debug, err } from '../Utils.mjs'
+
+import { toExplicitPath } from './Utils.mjs'
+
+class ArtifactPuller {
   constructor( artifact ) {
     this.artifact = artifact
   }
@@ -11,11 +15,141 @@ class ArtifactPusher {
   }
 }
 
-class GenericArtifactPusher extends ArtifactPusher {
+class GenericArtifactPuller extends ArtifactPuller {
+
+  async downloadFile( remoteFilePath, localFilePath ) {
+    debug( { remoteFilePath, localFilePath }, '<==Download File' )
+
+    const command = `gcloud artifacts generic download \
+      --project=${this.artifact.gcp_project_id} \
+      --location=${this.artifact.gcp_location} \
+      --repository=${this.artifact.gcp_artifact_registry_repository_name} \
+      --package=${this.artifact.gcp_artifact_registry_package_name} \
+      --version=${this.artifact.gcp_artifact_registry_package_version} \
+      --name="${remoteFilePath}" \
+      --destination=/tmp`
+
+    const oldPath = `/tmp/${remoteFilePath.split( '/' ).pop()}`
+    const newPath = localFilePath
+
+    // if file exists in the old path, delete it
+    if ( fs.existsSync( oldPath ) ) {
+      fs.unlinkSync( oldPath )
+    }
+
+    // if file exists in the new path, delete it
+    if ( fs.existsSync( newPath ) ) {
+      fs.unlinkSync( newPath )
+    }
+
+    // Ensure the directory exists
+    fs.mkdirSync( localFilePath.split( '/' ).slice( 0, -1 ).join( '/' ), { recursive : true } )
+
+    debug( { command }, '<==Download File Command' )
+
+    await shellCmd( command )
+
+    // Check if file exists in the old path
+    if ( !fs.existsSync( oldPath ) ) {
+      debug( { oldPath }, '<==File does not exist in the old path' )
+      return false
+    }
+
+    debug( { oldPath, newPath }, '<==Copying file' )
+
+    await fs.copyFileSync( oldPath, newPath )
+
+    // Check if file exists in the new path
+    if ( !fs.existsSync( newPath ) ) {
+      debug( { newPath }, '<==File does not exist in the new path' )
+      return false
+    }
+
+    return true
+  }
+
+  async downloadDirectory( localPath ) {
+    const command = `gcloud artifacts generic download \
+      --project=${this.artifact.gcp_project_id} \
+      --location=${this.artifact.gcp_location} \
+      --repository=${this.artifact.gcp_artifact_registry_repository_name} \
+      --package=${this.artifact.gcp_artifact_registry_package_name} \
+      --version=${this.artifact.gcp_artifact_registry_package_version} \
+      --destination=${localPath}`
+
+    debug( { command }, '<==Download Directory Command' )
+
+    await shellCmd( command )
+  }
+
+  async downloadNestedDirectory( remotePath, localPath ) {
+    debug( { remotePath, localPath }, '<==Download Nested Directory' )
+
+    const files = await this.listFiles( remotePath )
+    await Promise.all(
+      files
+        .filter( file => file.name.split( ':' ).pop().replaceAll( '%2F', '/' ).startsWith( remotePath ) )
+        .map( async ( file ) => {
+          const remoteFilePath = file.name.split( ':' ).pop().replaceAll( '%2F', '/' )
+          const localFilePath = `${localPath}${remoteFilePath.replaceAll( remotePath, '' )}`
+          await this.downloadFile( remoteFilePath, localFilePath )
+        } )
+    )
+  }
+
+  async listFiles( ) {
+    const command = `gcloud artifacts files list \
+        --project=${this.artifact.gcp_project_id} \
+        --location=${this.artifact.gcp_location} \
+        --repository=${this.artifact.gcp_artifact_registry_repository_name} \
+        --package=${this.artifact.gcp_artifact_registry_package_name} \
+        --version ${this.artifact.gcp_artifact_registry_package_version} \
+        --format="json"`
+
+    debug( { command }, '<==List Files Command' )
+      
+    const result = await shellCmd( command )
+  
+    // Parse the JSON response
+    const files = JSON.parse( result )
+    return files
+  }
 
   async pull() {
-    throw new Error( 'Not implemented' )
+    log( `Pulling artifact: ${this.artifact.name}` )
+    
+    debug( { artifact : this.artifact }, '<==Pulling artifact' )
+
+    try {
+      if ( this.artifact.remote_file_path && this.artifact.local_file_path ) {
+        await this.downloadFile( this.artifact.remote_file_path, toExplicitPath( this.artifact.local_file_path ) )
+      } else if ( this.artifact.remote_file_path && this.artifact.local_path ) {
+        await this.downloadFile( this.artifact.remote_file_path, toExplicitPath( `${this.artifact.local_path}/${this.artifact.remote_file_path.split( '/' ).pop()}` ) )
+      } else if ( this.artifact.remote_path && this.artifact.local_path ) {
+        await this.downloadNestedDirectory( this.artifact.remote_path, toExplicitPath( this.artifact.local_path ) )
+      } else if ( this.artifact.remote_path && !this.artifact.local_path && !this.artifact.local_file_path ) {
+        await this.downloadDirectory( this.artifact.remote_path, '.' )
+      } else if ( !this.artifact.remote_path && !this.artifact.remote_file_path && this.artifact.local_path ) {
+        await this.downloadDirectory( toExplicitPath( this.artifact.local_path ) )
+      } else {
+        throw new Error( 'No valid paths provided' )
+      }
+
+      log( `Successfully pulled artifact: ${this.artifact.name}` )
+
+      return true
+      
+      // if ( !this.artifact.remote_path && !this.artifact.remote_file_path && this.local_file_path ) {
+      //   throw new Error( '' )
+      // }
+      // if ( this.artifact.remote_path && this.artifact.local_file_path ) {
+      //   throw new Error( '' )
+      // }
+    } catch ( error ) {
+      err( `Error pulling artifact: ${this.artifact.name} ${error.message}` )
+    }
+    return false
   }
 }
 
-export { ArtifactPusher, GenericArtifactPusher }
+export { ArtifactPuller, GenericArtifactPuller }

package/src/artifacts/ArtifactPusher.mjs

@@ -1,25 +1,10 @@
-import crypto from 'crypto'
-import fs from 'fs'
+/* eslint-disable security/detect-non-literal-fs-filename */
+import { shellCmd, log, err, debug } from '../Utils.mjs'
 
 class ArtifactPusher {
   constructor( artifact ) {
     this.artifact = artifact
   }
-  
-  calculateHash() {
-    try {
-      const hash = crypto.createHash( 'sha256' )
-      hash.update( fs.readFileSync( this.artifact.source_path ) )
-      return hash.digest( 'base64url' )
-    } catch ( error ) {
-      console.error( error )
-      throw new Error( `Failed to calculate hash for artifact ${this.artifact.name}: ${error.message}` )
-    }
-  }
-
-  async exists() {
-    throw new Error( 'Not implemented' )
-  }
 
   async push() {
     throw new Error( 'Not implemented' )
@@ -28,103 +13,69 @@ class ArtifactPusher {
 
 class GenericArtifactPusher extends ArtifactPusher {
 
-  async exists() {
-    const isDirectory = fs.statSync( this.artifact.source_path ).isDirectory()
-    
-    // If source path is a directory, check whether the destination path alreadu exists in the registry
-    if ( isDirectory ) {
-      // For the time being, return false, and let gcloud skip existing files in upload-time.
-      // It's pretty slow, might be worth to break down the directory into individual artifacts
-      // and manage each one with its own pusher.
-      return false
-    }
-
-    // Check if artifact already exists in the registry
-    console.log( 'Checking if artifact already exists in the registry...' )
-        
-    const artifactHash = this.calculateHash()
-        
+  async listFiles( ) {
     const command = `gcloud artifacts files list \
-            --project=${this.artifact.gcp_project_id} \
-            --location=${this.artifact.gcp_location} \
-            --repository=${this.artifact.gcp_artifact_registry_repository_name} \
-            --package=${this.artifact.gcp_artifact_registry_package_name} \
-            --version ${this.artifact.gcp_artifact_registry_package_version} \
-            --filter="name:${this.artifact.destination_path}" \
-            --format="json"`
+        --project=${this.artifact.gcp_project_id} \
+        --location=${this.artifact.gcp_location} \
+        --repository=${this.artifact.gcp_artifact_registry_repository_name} \
+        --package=${this.artifact.gcp_artifact_registry_package_name} \
+        --version ${this.artifact.gcp_artifact_registry_package_version} \
+        --format="json"`
 
-    console.log( 'Command:', command.replace( /\s+/g, ' ' ).trim() )
-    
-    try {
-      const { execSync } = await import( 'child_process' )
-      const result = execSync( command, { encoding : 'utf8', stdio : 'pipe' } )
+    debug( { command }, '<==List Files Command' )
+      
+    const result = await shellCmd( command )
+  
+    // Parse the JSON response
+    const files = JSON.parse( result )
 
-      // Parse the JSON response
-      const files = JSON.parse( result )
-    
-      // Check if any files were found and match our hash
-      if ( files && files.length > 0 ) {
-        for ( const file of files ) {
-          for ( const hash of file.hashes ) {
-            // Compare without padding since base64url may omit trailing '=' 
-            const normalizedOurHash = artifactHash.replace( /=+$/, '' )
-            const normalizedRegistryHash = hash.value.replace( /=+$/, '' )
-            if ( hash.type === 'SHA256' && normalizedRegistryHash === normalizedOurHash ) {
-              console.log( `Artifact already exists in the registry: ${this.artifact.gcp_artifact_registry_package_name}:${this.artifact.gcp_artifact_registry_package_version}` )
-              return true
-            }
-          }
-        }
-      }
-      console.log( 'Artifact does not exist in the registry' )
-      return false
-          
-    } catch ( error ) {
-      console.error( 'Error checking artifact existence:', error.message )
-      throw error
-    }
+    return files
   }
     
   async push() {
-    console.log( 'Publishing artifact to registry...' )
+    log( `Pushing artifact: ${this.artifact.name}` )
 
-    if ( await this.exists() ) {
-      console.log( 'Artifact already exists in the registry' )
+    if ( this.artifact.remote_file_path ) {
+      err( 'Remote file path is not supported for pushing artifacts' )
       return false
     }
 
-    console.log( 'Uploading artifact to registry...' )
+    try {
    
-    let command = ''
+      let command = ''
 
-    command = `gcloud artifacts generic upload \
+      command = `gcloud artifacts generic upload \
       --project=${this.artifact.gcp_project_id} \
       --location=${this.artifact.gcp_location} \
       --repository=${this.artifact.gcp_artifact_registry_repository_name} \
       --package=${this.artifact.gcp_artifact_registry_package_name} \
       --version=${this.artifact.gcp_artifact_registry_package_version}`
 
-    const isDirectory = fs.statSync( this.artifact.source_path ).isDirectory()
-    if ( isDirectory ) {
-      command += ` --source-directory=${this.artifact.source_path}`
-      command += ' --skip-existing'
-    } else {
-      command += ` --source=${this.artifact.source_path}`
-    }
+      if ( this.artifact.local_path ) {
+        command += ` --source-directory=${this.artifact.local_path}`
+        command += ' --skip-existing'
+      } else if ( this.artifact.local_file_path ) {
+        const files = await this.listFiles( this.artifact.remote_path )
+        if ( files.some( file => file.name.split( ':' ).pop().replaceAll( '%2F', '/' ) === `${this.artifact.remote_path}/${this.artifact.remote_file_path.split( '/' ).pop()}` ) ) {
+          err( `File already exists in the artifact registry: ${this.artifact.local_file_path}` )
+          return false
+        }
+        command += ` --source=${this.artifact.local_file_path}`
+      }
 
-    if ( this.artifact.destination_path ) {
-      command += ` --destination-path=${this.artifact.destination_path}`
-    }
-    
-    console.log( 'Command:', command.replace( /\s+/g, ' ' ).trim() )
+      if ( this.artifact.remote_path ) {
+        command += ` --destination-path=${this.artifact.remote_path}`
+      }
+
+      debug( { command }, '<==Upload Command' )
         
-    try {
-      const { execSync } = await import( 'child_process' )
-      execSync( command, { encoding : 'utf8', stdio : 'pipe' } )
-      console.log( `Successfully published artifact: ${this.artifact.gcp_artifact_registry_package_name}:${this.artifact.gcp_artifact_registry_package_version}` )
+      await shellCmd( command )
+      
+      log( `Successfully pushed artifact: ${this.artifact.name}` )
+
       return true
     } catch ( error ) {
-      console.error( 'Error publishing artifact:', error.message )
+      err( `Error pushing artifact: ${this.artifact.name} ${error.message}` )
       return false
     }
   }

package/src/artifacts/Structs.mjs

@@ -1,4 +1,4 @@
-import { assign, object, string, array, optional, enums, union } from 'superstruct'
+import { assign, object, string, array, optional, enums, union, refine } from 'superstruct'
 
 const ArtifactTypeEnum = {
   GENERIC : 'generic',
@@ -25,16 +25,44 @@ const ArtifactSchema = assign( PartialArtifactSchema, object( {
   gcp_artifact_registry_package_version : string()
 } ) )
 
-const PartialGenericArtifactSchema = assign( PartialArtifactSchema, object( {
-  type : enums( [ ArtifactTypeEnum.GENERIC ] ),
-  source_path : string(),
-  destination_path : string()
-} ) )
+const PartialGenericArtifactSchema = refine(
+  assign( PartialArtifactSchema, object( {
+    type : enums( [ ArtifactTypeEnum.GENERIC ] ),
+    local_file_path : optional( string() ),
+    local_path : optional( string() ),
+    remote_file_path : optional( string() ),
+    remote_path : optional( string() )
+  } ) ),
+  'ConflictingPaths',
+  ( value ) => {
+    if ( value.local_file_path && value.local_path ) {
+      return 'Only one of local_file_path or local_path must be provided'
+    }
+    if ( value.remote_file_path && value.remote_path ) {
+      return 'Only one of remote_file_path or remote_path must be provided'
+    }
+    return true
+  }
+)
 
-const GenericArtifactSchema = assign( ArtifactSchema, object( {
-  source_path : string(),
-  destination_path : optional( string() )
-} ) )
+const GenericArtifactSchema = refine(
+  assign( ArtifactSchema, object( {
+    local_file_path : optional( string() ),
+    local_path : optional( string() ),
+    remote_file_path : optional( string() ),
+    remote_path : optional( string() )
+  } ) ),
+  'ConflictingPaths',
+  ( value ) => {
+    if ( value.local_file_path && value.local_path ) {
+      return 'Only one of local_file_path or local_path must be provided'
+    }
+    if ( value.remote_file_path && value.remote_path ) {
+      return 'Only one of remote_file_path or remote_path must be provided'
+    }
+    return true
+  }
+)
 
 // Superstruct schema for defaults configuration
 const DefaultsSchema = object( {

package/src/artifacts/Utils.mjs

@@ -1,3 +1,6 @@
+import fs from 'fs'
+import crypto from 'crypto'
+
 import { validate } from 'superstruct'
 
 import { parseYamlFile, warning } from '../Utils.mjs'
@@ -46,4 +49,36 @@ const flattenArtifactsSchema = ( artifactsSchema ) => {
   }, [] )
 }
 
-export { getArtifactsSchema, flattenArtifactsSchema }
+const toImplicitPath = ( path ) => {
+  let implicitPath = path
+  // Remove leading './' if present
+  if ( implicitPath.startsWith( './' ) ) {
+    implicitPath = implicitPath.slice( 2 )
+  }
+  // Remove trailing slash
+  implicitPath = implicitPath.replace( /\/+$/, '' )
+  return implicitPath
+}
+
+const toExplicitPath = ( path ) => {
+  let explicitPath = path
+  // Keep absolute paths and paths already starting with '.' as-is
+  if ( !path.startsWith( '/' ) && !path.startsWith( '.' ) ) {
+    // Add './' to relative paths
+    explicitPath = `./${explicitPath}`
+  }
+  return explicitPath
+}
+
+const calculateHash = ( path ) => {
+  try {
+    const hash = crypto.createHash( 'sha256' )
+    hash.update( fs.readFileSync( path ) )
+    return hash.digest( 'base64url' )
+  } catch ( error ) {
+    console.error( error )
+    throw new Error( `Failed to calculate hash for artifact ${path}: ${error.message}` )
+  }
+}
+
+export { calculateHash, getArtifactsSchema, flattenArtifactsSchema, toImplicitPath, toExplicitPath }

package/src/artifacts/pull-artifact.mjs

@@ -1,7 +1,71 @@
 #!/usr/bin/env node
 
+import { warning, err, log } from '../Utils.mjs'
+
+import { getArtifactsSchema, flattenArtifactsSchema } from './Utils.mjs'
+import { ArtifactTypeEnum } from './Structs.mjs'
+import { GenericArtifactPuller } from './ArtifactPuller.mjs'
+
 async function main() {
-  return null
+
+  if ( process.argv.length < 3 ) {
+    err( 'Usage:' )
+    err( '  pull-artifact.mjs <artifacts.yml>' )
+    err( '  pull-artifact.mjs <artifacts.yml> <artifact package name>' )
+    err( '  pull-artifact.mjs <artifacts.yml> <artifact package name> <artifact package version>' )
+    process.exit( 1 )
+  }
+
+  const artifactsYmlPath = process.argv[2]
+  const artifactPackageName = process.argv.length > 3 ? process.argv[3] : null
+  const artifactPackageVersion = process.argv.length > 4 ? process.argv[4] : null
+  
+  const artifactsSchema = await getArtifactsSchema( artifactsYmlPath )
+  const flattenedArtifactsSchema = flattenArtifactsSchema( artifactsSchema )
+  
+  let filteredArtifactsSchema
+  if ( artifactPackageName && artifactPackageVersion ) {
+    filteredArtifactsSchema = flattenedArtifactsSchema.filter(
+      artifact => artifact.gcp_artifact_registry_package_name === artifactPackageName &&
+      artifact.gcp_artifact_registry_package_version === artifactPackageVersion )
+  } else if ( artifactPackageName ) {
+    filteredArtifactsSchema = flattenedArtifactsSchema.filter(
+      artifact => artifact.gcp_artifact_registry_package_name === artifactPackageName )
+  } else {
+    filteredArtifactsSchema = flattenedArtifactsSchema
+  }
+  
+  let counter = 0
+
+  const pullResults = await Promise.all(
+    filteredArtifactsSchema.map( async ( artifactSchema ) => {
+      let puller
+      switch ( artifactSchema.type ) {
+        case ArtifactTypeEnum.GENERIC:
+          puller = new GenericArtifactPuller( artifactSchema )
+          break
+        default:
+          warning( `Unsupported artifact type "${artifactSchema.type}"` )
+          return false
+      }
+
+      if ( !puller ) {
+        warning( `Unsupported artifact type "${artifactSchema.type}"` )
+        return false
+      }
+
+      try {
+        return await puller.pull()
+      } catch ( error ) {
+        warning( `Error pulling artifact "${artifactSchema.name}": ${error.message}` )
+        return false
+      }
+    } )
+  )
+
+  counter = pullResults.filter( Boolean ).length
+
+  log( `Successfully pushed ${counter}/${filteredArtifactsSchema.length} artifacts` )
 }
 
 main()

package/src/artifacts/push-artifact.mjs

@@ -1,6 +1,6 @@
 #!/usr/bin/env node
 
-import { warning } from '../Utils.mjs'
+import { warning, err, log } from '../Utils.mjs'
 
 import { getArtifactsSchema, flattenArtifactsSchema } from './Utils.mjs'
 import { ArtifactTypeEnum } from './Structs.mjs'
@@ -8,47 +8,64 @@ import { GenericArtifactPusher } from './ArtifactPusher.mjs'
 
 async function main() {
 
-  // If an argument is provided, use it as the artifacts.yml file path
+  if ( process.argv.length < 3 ) {
+    err( 'Usage:' )
+    err( '  push-artifact.mjs <artifacts.yml>' )
+    err( '  push-artifact.mjs <artifacts.yml> <artifact package name>' )
+    err( '  push-artifact.mjs <artifacts.yml> <artifact package name> <artifact package version>' )
+    process.exit( 1 )
+  }
+
   const artifactsYmlPath = process.argv[2]
-  const artifactPackageName = process.argv[3]
-  const artifactPackageVersion = process.argv[4]
-
-  if ( artifactsYmlPath ) {
-    const artifactsSchema = await getArtifactsSchema( artifactsYmlPath )
-    const flattenedArtifactsSchema = flattenArtifactsSchema( artifactsSchema )
-    
-    // Open for unexpected matches if we end up re-using the same package name and version for
-    // multiple artifacts spread across multiple repositories and/or projects.
-    const filteredArtifactSchemas = flattenedArtifactsSchema.filter(
+  const artifactPackageName = process.argv.length > 3 ? process.argv[3] : null
+  const artifactPackageVersion = process.argv.length > 4 ? process.argv[4] : null
+  
+  const artifactsSchema = await getArtifactsSchema( artifactsYmlPath )
+  const flattenedArtifactsSchema = flattenArtifactsSchema( artifactsSchema )
+  
+  let filteredArtifactsSchema
+  if ( artifactPackageName && artifactPackageVersion ) {
+    filteredArtifactsSchema = flattenedArtifactsSchema.filter(
       artifact => artifact.gcp_artifact_registry_package_name === artifactPackageName &&
       artifact.gcp_artifact_registry_package_version === artifactPackageVersion )
-    
-    if ( filteredArtifactSchemas.length === 0 ) {
-      console.error( `No artifacts found for ${artifactPackageName} version ${artifactPackageVersion} in ${artifactsYmlPath}` )
-      process.exit( 1 )
-    }
-
-    // TODO: Use concurrency to package artifacts and provide summary of results.
-    for ( const artifactSchema of filteredArtifactSchemas ) {
-      let packager
+  } else if ( artifactPackageName ) {
+    filteredArtifactsSchema = flattenedArtifactsSchema.filter(
+      artifact => artifact.gcp_artifact_registry_package_name === artifactPackageName )
+  } else {
+    filteredArtifactsSchema = flattenedArtifactsSchema
+  }
+
+  let counter = 0
+
+  const pushResults = await Promise.all(
+    filteredArtifactsSchema.map( async ( artifactSchema ) => {
+      let pusher
       switch ( artifactSchema.type ) {
         case ArtifactTypeEnum.GENERIC:
-          packager = new GenericArtifactPusher( artifactSchema )
+          pusher = new GenericArtifactPusher( artifactSchema )
           break
         default:
           warning( `Unsupported artifact type "${artifactSchema.type}"` )
-          continue
+          return false
       }
-      
-      if ( !packager ) {
+
+      if ( !pusher ) {
         warning( `Unsupported artifact type "${artifactSchema.type}"` )
-        continue
+        return false
       }
-      
-      // eslint-disable-next-line no-await-in-loop
-      await packager.package()
-    }
-  }
+
+      try {
+        return await pusher.push()
+      } catch ( error ) {
+        warning( `Error pushing artifact "${artifactSchema.name}": ${error.message}` )
+        return false
+      }
+    } )
+  )
+
+  counter = pushResults.filter( Boolean ).length
+
+  log( `Successfully pushed ${counter}/${filteredArtifactsSchema.length} artifacts` )
 }
 
 main()