@leverege/build-tools 2.93.0-beta.10 → 2.93.0-beta.11

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@leverege/build-tools",
-  "version": "2.93.0-beta.10",
+  "version": "2.93.0-beta.11",
   "description": "A collection of build / support tools for Leverege developers",
   "main": "index.js",
   "repository": {

package/src/artifacts/ArtifactPuller.mjs

@@ -1,7 +1,8 @@
-import crypto from 'crypto'
 import fs from 'fs'
 
-class ArtifactPusher {
+import { toExplicitPath } from './Utils.mjs'
+
+class ArtifactPuller {
   constructor( artifact ) {
     this.artifact = artifact
   }
@@ -11,11 +12,137 @@ class ArtifactPusher {
   }
 }
 
-class GenericArtifactPusher extends ArtifactPusher {
+class GenericArtifactPuller extends ArtifactPuller {
+
+  async downloadFile( remoteFilePath, localFilePath ) {
+    const command = `gcloud artifacts generic download \
+      --project=${this.artifact.gcp_project_id} \
+      --location=${this.artifact.gcp_location} \
+      --repository=${this.artifact.gcp_artifact_registry_repository_name} \
+      --package=${this.artifact.gcp_artifact_registry_package_name} \
+      --version=${this.artifact.gcp_artifact_registry_package_version} \
+      --name="${remoteFilePath}" \
+      --destination=/tmp`
+
+    const oldPath = `/tmp/${remoteFilePath.split( '/' ).pop()}`
+    const newPath = localFilePath
+
+    // if file exists in the old path, delete it
+    if ( fs.existsSync( oldPath ) ) {
+      fs.unlinkSync( oldPath )
+    }
+
+    // if file exists in the new path, delete it
+    if ( fs.existsSync( newPath ) ) {
+      fs.unlinkSync( newPath )
+    }
+
+    // Ensure the directory exists
+    fs.mkdirSync( localFilePath.split( '/' ).slice( 0, -1 ).join( '/' ), { recursive : true } )
+
+    console.log( 'Command:', command.replace( /\s+/g, ' ' ).trim() )
+
+    const { execSync } = await import( 'child_process' )
+    const result = execSync( command, { encoding : 'utf8', stdio : 'pipe' } )
+    console.log( 'Result:', result )
+
+    // Check if file exists in the old path
+    if ( !fs.existsSync( oldPath ) ) {
+      console.error( `File does not exist in the old path: ${oldPath}` )
+      return false
+    }
+
+    console.log( `Copying file from ${oldPath} to ${newPath}` )
+
+    await fs.copyFileSync( oldPath, newPath )
+
+    // Check if file exists in the new path
+    if ( !fs.existsSync( newPath ) ) {
+      console.error( `File does not exist in the new path: ${newPath}` )
+      return false
+    }
+
+    return true
+  }
+
+  async downloadDirectory( localPath ) {
+    const command = `gcloud artifacts generic download \
+      --project=${this.artifact.gcp_project_id} \
+      --location=${this.artifact.gcp_location} \
+      --repository=${this.artifact.gcp_artifact_registry_repository_name} \
+      --package=${this.artifact.gcp_artifact_registry_package_name} \
+      --version=${this.artifact.gcp_artifact_registry_package_version} \
+      --destination=${localPath}`
+
+    console.log( 'Command:', command.replace( /\s+/g, ' ' ).trim() )
+
+    const { execSync } = await import( 'child_process' )
+    execSync( command, { encoding : 'utf8', stdio : 'pipe' } )
+
+    return true
+  }
+
+  async downloadNestedDirectory( remotePath, localPath ) {
+    const files = await this.listFiles( remotePath )
+    await Promise.all(
+      files
+        .filter( file => file.name.split( ':' ).pop().replaceAll( '%2F', '/' ).startsWith( remotePath ) )
+        .map( async ( file ) => {
+          const remoteFilePath = file.name.split( ':' ).pop().replaceAll( '%2F', '/' )
+          const localFilePath = `${localPath}${remoteFilePath.replaceAll( remotePath, '' )}`
+          await this.downloadFile( remoteFilePath, localFilePath )
+        } )
+    )
+    return true
+  }
+
+  async listFiles( ) {
+    const command = `gcloud artifacts files list \
+        --project=${this.artifact.gcp_project_id} \
+        --location=${this.artifact.gcp_location} \
+        --repository=${this.artifact.gcp_artifact_registry_repository_name} \
+        --package=${this.artifact.gcp_artifact_registry_package_name} \
+        --version ${this.artifact.gcp_artifact_registry_package_version} \
+        --format="json"`
+
+    console.log( 'Command:', command.replace( /\s+/g, ' ' ).trim() )
+      
+    const { execSync } = await import( 'child_process' )
+    const result = execSync( command, { encoding : 'utf8', stdio : 'pipe' } )
+  
+    // Parse the JSON response
+    const files = JSON.parse( result )
+
+    return files
+  }
 
   async pull() {
-    throw new Error( 'Not implemented' )
+    console.log( 'Pulling artifact from registry...' )
+
+    if ( this.artifact.remote_file_path && this.artifact.local_file_path ) {
+      return this.downloadFile( this.artifact.remote_file_path, toExplicitPath( this.artifact.local_file_path ) )
+    }
+    if ( this.artifact.remote_file_path && this.artifact.local_path ) {
+      return this.downloadFile( this.artifact.remote_file_path, toExplicitPath( `${this.artifact.local_path}/${this.artifact.remote_file_path.split( '/' ).pop()}` ) )
+    }
+    if ( this.artifact.remote_path && this.artifact.local_path ) {
+      return this.downloadNestedDirectory( this.artifact.remote_path, toExplicitPath( this.artifact.local_path ) )
+    }
+    if ( this.artifact.remote_path && !this.artifact.local_path && !this.artifact.local_file_path ) {
+      return this.downloadDirectory( this.artifact.remote_path, '.' )
+    }
+    if ( !this.artifact.remote_path && !this.artifact.remote_file_path && this.artifact.local_path ) {
+      return this.downloadDirectory( toExplicitPath( this.artifact.local_path ) )
+    }
+    // if ( !this.artifact.remote_path && !this.artifact.remote_file_path && this.local_file_path ) {
+    //   throw new Error( '' )
+    // }
+    // if ( this.artifact.remote_path && this.artifact.local_file_path ) {
+    //   throw new Error( '' )
+    // }
+
+    return false
   }
 }
 
-export { ArtifactPusher, GenericArtifactPusher }
+export { ArtifactPuller, GenericArtifactPuller }

package/src/artifacts/ArtifactPusher.mjs

@@ -1,25 +1,7 @@
-import crypto from 'crypto'
-import fs from 'fs'
-
 class ArtifactPusher {
   constructor( artifact ) {
     this.artifact = artifact
   }
-  
-  calculateHash() {
-    try {
-      const hash = crypto.createHash( 'sha256' )
-      hash.update( fs.readFileSync( this.artifact.source_path ) )
-      return hash.digest( 'base64url' )
-    } catch ( error ) {
-      console.error( error )
-      throw new Error( `Failed to calculate hash for artifact ${this.artifact.name}: ${error.message}` )
-    }
-  }
-
-  async exists() {
-    throw new Error( 'Not implemented' )
-  }
 
   async push() {
     throw new Error( 'Not implemented' )
@@ -28,68 +10,30 @@ class ArtifactPusher {
 
 class GenericArtifactPusher extends ArtifactPusher {
 
-  async exists() {
-    const isDirectory = fs.statSync( this.artifact.source_path ).isDirectory()
-    
-    // If source path is a directory, check whether the destination path alreadu exists in the registry
-    if ( isDirectory ) {
-      // For the time being, return false, and let gcloud skip existing files in upload-time.
-      // It's pretty slow, might be worth to break down the directory into individual artifacts
-      // and manage each one with its own pusher.
-      return false
-    }
-
-    // Check if artifact already exists in the registry
-    console.log( 'Checking if artifact already exists in the registry...' )
-        
-    const artifactHash = this.calculateHash()
-        
+  async listFiles( ) {
     const command = `gcloud artifacts files list \
-            --project=${this.artifact.gcp_project_id} \
-            --location=${this.artifact.gcp_location} \
-            --repository=${this.artifact.gcp_artifact_registry_repository_name} \
-            --package=${this.artifact.gcp_artifact_registry_package_name} \
-            --version ${this.artifact.gcp_artifact_registry_package_version} \
-            --filter="name:${this.artifact.destination_path}" \
-            --format="json"`
+        --project=${this.artifact.gcp_project_id} \
+        --location=${this.artifact.gcp_location} \
+        --repository=${this.artifact.gcp_artifact_registry_repository_name} \
+        --package=${this.artifact.gcp_artifact_registry_package_name} \
+        --version ${this.artifact.gcp_artifact_registry_package_version} \
+        --format="json"`
 
     console.log( 'Command:', command.replace( /\s+/g, ' ' ).trim() )
-    
-    try {
-      const { execSync } = await import( 'child_process' )
-      const result = execSync( command, { encoding : 'utf8', stdio : 'pipe' } )
+      
+    const { execSync } = await import( 'child_process' )
+    const result = execSync( command, { encoding : 'utf8', stdio : 'pipe' } )
+  
+    // Parse the JSON response
+    const files = JSON.parse( result )
 
-      // Parse the JSON response
-      const files = JSON.parse( result )
-    
-      // Check if any files were found and match our hash
-      if ( files && files.length > 0 ) {
-        for ( const file of files ) {
-          for ( const hash of file.hashes ) {
-            // Compare without padding since base64url may omit trailing '=' 
-            const normalizedOurHash = artifactHash.replace( /=+$/, '' )
-            const normalizedRegistryHash = hash.value.replace( /=+$/, '' )
-            if ( hash.type === 'SHA256' && normalizedRegistryHash === normalizedOurHash ) {
-              console.log( `Artifact already exists in the registry: ${this.artifact.gcp_artifact_registry_package_name}:${this.artifact.gcp_artifact_registry_package_version}` )
-              return true
-            }
-          }
-        }
-      }
-      console.log( 'Artifact does not exist in the registry' )
-      return false
-          
-    } catch ( error ) {
-      console.error( 'Error checking artifact existence:', error.message )
-      throw error
-    }
+    return files
   }
     
   async push() {
-    console.log( 'Publishing artifact to registry...' )
+    console.log( 'Pushing artifact to registry...' )
 
-    if ( await this.exists() ) {
-      console.log( 'Artifact already exists in the registry' )
+    if ( this.artifact.remote_file_path ) {
       return false
     }
 
@@ -104,18 +48,19 @@ class GenericArtifactPusher extends ArtifactPusher {
       --package=${this.artifact.gcp_artifact_registry_package_name} \
       --version=${this.artifact.gcp_artifact_registry_package_version}`
 
-    const isDirectory = fs.statSync( this.artifact.source_path ).isDirectory()
-    if ( isDirectory ) {
-      command += ` --source-directory=${this.artifact.source_path}`
+    if ( this.artifact.local_path ) {
+      command += ` --source-directory=${this.artifact.local_path}`
       command += ' --skip-existing'
-    } else {
-      command += ` --source=${this.artifact.source_path}`
+    } else if ( this.artifact.local_file_path ) {
+      const files = await this.listFiles( this.artifact.remote_path )
+      if ( files.some( file => file.name.split( ':' ).pop().replaceAll( '%2F', '/' ) === `${this.artifact.remote_path}/${this.artifact.remote_file_path.split( '/' ).pop()}` ) ) {
+        return false
+      }
+      command += ` --source=${this.artifact.local_file_path}`
     }
 
-    if ( this.artifact.destination_path ) {
-      command += ` --destination-path=${this.artifact.destination_path}`
-    }
-    
+    command += ` --destination-path=${this.artifact.remote_path}`
+
     console.log( 'Command:', command.replace( /\s+/g, ' ' ).trim() )
         
     try {

package/src/artifacts/Structs.mjs

@@ -1,4 +1,4 @@
-import { assign, object, string, array, optional, enums, union } from 'superstruct'
+import { assign, object, string, array, optional, enums, union, refine } from 'superstruct'
 
 const ArtifactTypeEnum = {
   GENERIC : 'generic',
@@ -25,16 +25,44 @@ const ArtifactSchema = assign( PartialArtifactSchema, object( {
   gcp_artifact_registry_package_version : string()
 } ) )
 
-const PartialGenericArtifactSchema = assign( PartialArtifactSchema, object( {
-  type : enums( [ ArtifactTypeEnum.GENERIC ] ),
-  source_path : string(),
-  destination_path : string()
-} ) )
+const PartialGenericArtifactSchema = refine(
+  assign( PartialArtifactSchema, object( {
+    type : enums( [ ArtifactTypeEnum.GENERIC ] ),
+    local_file_path : optional( string() ),
+    local_path : optional( string() ),
+    remote_file_path : optional( string() ),
+    remote_path : optional( string() )
+  } ) ),
+  'ConflictingPaths',
+  ( value ) => {
+    if ( value.local_file_path && value.local_path ) {
+      return 'Only one of local_file_path or local_path must be provided'
+    }
+    if ( value.remote_file_path && value.remote_path ) {
+      return 'Only one of remote_file_path or remote_path must be provided'
+    }
+    return true
+  }
+)
 
-const GenericArtifactSchema = assign( ArtifactSchema, object( {
-  source_path : string(),
-  destination_path : optional( string() )
-} ) )
+const GenericArtifactSchema = refine(
+  assign( ArtifactSchema, object( {
+    local_file_path : optional( string() ),
+    local_path : optional( string() ),
+    remote_file_path : optional( string() ),
+    remote_path : optional( string() )
+  } ) ),
+  'ConflictingPaths',
+  ( value ) => {
+    if ( value.local_file_path && value.local_path ) {
+      return 'Only one of local_file_path or local_path must be provided'
+    }
+    if ( value.remote_file_path && value.remote_path ) {
+      return 'Only one of remote_file_path or remote_path must be provided'
+    }
+    return true
+  }
+)
 
 // Superstruct schema for defaults configuration
 const DefaultsSchema = object( {

package/src/artifacts/Utils.mjs

@@ -1,3 +1,6 @@
+import fs from 'fs'
+import crypto from 'crypto'
+
 import { validate } from 'superstruct'
 
 import { parseYamlFile, warning } from '../Utils.mjs'
@@ -46,4 +49,65 @@ const flattenArtifactsSchema = ( artifactsSchema ) => {
   }, [] )
 }
 
-export { getArtifactsSchema, flattenArtifactsSchema }
+const toImplicitPath = ( path ) => {
+  let implicitPath = path
+  // Remove leading './' if present
+  if ( implicitPath.startsWith( './' ) ) {
+    implicitPath = implicitPath.slice( 2 )
+  }
+  // Remove trailing slash
+  implicitPath = implicitPath.replace( /\/+$/, '' )
+  return implicitPath
+}
+
+const toExplicitPath = ( path ) => {
+  let explicitPath = path
+  // Keep absolute paths and paths already starting with '.' as-is
+  if ( !path.startsWith( '/' ) && !path.startsWith( '.' ) ) {
+    // Add './' to relative paths
+    explicitPath = `./${explicitPath}`
+  }
+  return explicitPath
+}
+
+const buildGenericArtifactName = ( artifact ) => {
+  const baseElements = [
+    'projects',
+    artifact.gcp_project_id,
+    'locations',
+    artifact.gcp_location,
+    'repositories',
+    artifact.gcp_artifact_registry_repository_name,
+    'files',
+    artifact.gcp_artifact_registry_package_name,
+  ]
+
+  const versionElement = artifact.gcp_artifact_registry_package_version
+
+  const remotePathElements = []
+
+  if ( artifact.remote_path || artifact.remote_file_path ) {
+    const explicitRemotePath = toExplicitPath( artifact.remote_path || artifact.remote_file_path )
+    const implicitRemotePath = toImplicitPath( explicitRemotePath )
+    if ( implicitRemotePath !== '.' ) {
+      remotePathElements.push( implicitRemotePath )
+    }
+  }
+
+  const name = [ baseElements.join( '/' ), versionElement, encodeURIComponent( remotePathElements.join( '/' ) ) ].join( ':' )
+
+  return name
+}
+
+const calculateHash = ( path ) => {
+  try {
+    const hash = crypto.createHash( 'sha256' )
+    hash.update( fs.readFileSync( path ) )
+    return hash.digest( 'base64url' )
+  } catch ( error ) {
+    console.error( error )
+    throw new Error( `Failed to calculate hash for artifact ${path}: ${error.message}` )
+  }
+}
+
+export { calculateHash, getArtifactsSchema, flattenArtifactsSchema, toImplicitPath, toExplicitPath, buildGenericArtifactName }

package/src/artifacts/pull-artifact.mjs

@@ -1,7 +1,68 @@
 #!/usr/bin/env node
 
+import { warning } from '../Utils.mjs'
+
+import { getArtifactsSchema, flattenArtifactsSchema } from './Utils.mjs'
+import { ArtifactTypeEnum } from './Structs.mjs'
+import { GenericArtifactPuller } from './ArtifactPuller.mjs'
+
 async function main() {
-  return null
+
+  let artifactsYmlPath = null
+  let artifactName = null
+  let artifactPackageName = null
+  let artifactPackageVersion = null
+  let filteredArtifactSchema = null
+  
+  if ( process.argv.length === 3 ) {
+    artifactsYmlPath = process.argv[2]
+    filteredArtifactSchema = await getArtifactsSchema( artifactsYmlPath )
+    const flattenedArtifactsSchema = flattenArtifactsSchema( filteredArtifactSchema )
+    filteredArtifactSchema = flattenedArtifactsSchema
+  } else if ( process.argv.length === 4 ) {
+    artifactsYmlPath = process.argv[2]
+    filteredArtifactSchema = await getArtifactsSchema( artifactsYmlPath )
+    const flattenedArtifactsSchema = flattenArtifactsSchema( filteredArtifactSchema )
+    filteredArtifactSchema = flattenedArtifactsSchema.find(
+      artifact => artifact.name === artifactName )
+  } else if ( process.argv.length === 5 ) {
+    artifactsYmlPath = process.argv[2]
+    artifactName = process.argv[3]
+    artifactPackageName = process.argv[4]
+    artifactPackageVersion = process.argv[5]
+    filteredArtifactSchema = await getArtifactsSchema( artifactsYmlPath )
+    const flattenedArtifactsSchema = flattenArtifactsSchema( filteredArtifactSchema )
+    filteredArtifactSchema = flattenedArtifactsSchema.filter(
+      artifact => artifact.gcp_artifact_registry_package_name === artifactPackageName &&
+      artifact.gcp_artifact_registry_package_version === artifactPackageVersion )
+  } else {
+    console.error( 'Usage:' )
+    console.error( '  pull-artifact.mjs <artifacts.yml>' )
+    console.error( '  pull-artifact.mjs <artifacts.yml> <artifact name>' )
+    console.error( '  pull-artifact.mjs <artifacts.yml> <artifact package name> <artifact package version>' )
+    process.exit( 1 )
+  }
+
+  // TODO: Use concurrency to pull artifacts and provide summary of results.
+  for ( const artifactSchema of filteredArtifactSchema ) {
+    let puller
+    switch ( artifactSchema.type ) {
+      case ArtifactTypeEnum.GENERIC:
+        puller = new GenericArtifactPuller( artifactSchema )
+        break
+      default:
+        warning( `Unsupported artifact type "${artifactSchema.type}"` )
+        continue
+    }
+      
+    if ( !puller ) {
+      warning( `Unsupported artifact type "${artifactSchema.type}"` )
+      continue
+    }
+      
+    // eslint-disable-next-line no-await-in-loop
+    await puller.pull()
+  }
 }
 
 main()

package/src/artifacts/push-artifact.mjs

@@ -8,46 +8,60 @@ import { GenericArtifactPusher } from './ArtifactPusher.mjs'
 
 async function main() {
 
-  // If an argument is provided, use it as the artifacts.yml file path
-  const artifactsYmlPath = process.argv[2]
-  const artifactPackageName = process.argv[3]
-  const artifactPackageVersion = process.argv[4]
-
-  if ( artifactsYmlPath ) {
-    const artifactsSchema = await getArtifactsSchema( artifactsYmlPath )
-    const flattenedArtifactsSchema = flattenArtifactsSchema( artifactsSchema )
-    
-    // Open for unexpected matches if we end up re-using the same package name and version for
-    // multiple artifacts spread across multiple repositories and/or projects.
-    const filteredArtifactSchemas = flattenedArtifactsSchema.filter(
+  let artifactsYmlPath = null
+  let artifactName = null
+  let artifactPackageName = null
+  let artifactPackageVersion = null
+  let filteredArtifactSchema = null
+  
+  if ( process.argv.length === 3 ) {
+    artifactsYmlPath = process.argv[2]
+    filteredArtifactSchema = await getArtifactsSchema( artifactsYmlPath )
+    const flattenedArtifactsSchema = flattenArtifactsSchema( filteredArtifactSchema )
+    filteredArtifactSchema = flattenedArtifactsSchema
+  } else if ( process.argv.length === 4 ) {
+    artifactsYmlPath = process.argv[2]
+    filteredArtifactSchema = await getArtifactsSchema( artifactsYmlPath )
+    const flattenedArtifactsSchema = flattenArtifactsSchema( filteredArtifactSchema )
+    filteredArtifactSchema = flattenedArtifactsSchema.find(
+      artifact => artifact.name === artifactName )
+  } else if ( process.argv.length === 5 ) {
+    artifactsYmlPath = process.argv[2]
+    artifactName = process.argv[3]
+    artifactPackageName = process.argv[4]
+    artifactPackageVersion = process.argv[5]
+    filteredArtifactSchema = await getArtifactsSchema( artifactsYmlPath )
+    const flattenedArtifactsSchema = flattenArtifactsSchema( filteredArtifactSchema )
+    filteredArtifactSchema = flattenedArtifactsSchema.filter(
       artifact => artifact.gcp_artifact_registry_package_name === artifactPackageName &&
       artifact.gcp_artifact_registry_package_version === artifactPackageVersion )
-    
-    if ( filteredArtifactSchemas.length === 0 ) {
-      console.error( `No artifacts found for ${artifactPackageName} version ${artifactPackageVersion} in ${artifactsYmlPath}` )
-      process.exit( 1 )
-    }
+  } else {
+    console.error( 'Usage:' )
+    console.error( '  push-artifact.mjs <artifacts.yml>' )
+    console.error( '  push-artifact.mjs <artifacts.yml> <artifact name>' )
+    console.error( '  push-artifact.mjs <artifacts.yml> <artifact package name> <artifact package version>' )
+    process.exit( 1 )
+  }
 
-    // TODO: Use concurrency to package artifacts and provide summary of results.
-    for ( const artifactSchema of filteredArtifactSchemas ) {
-      let packager
-      switch ( artifactSchema.type ) {
-        case ArtifactTypeEnum.GENERIC:
-          packager = new GenericArtifactPusher( artifactSchema )
-          break
-        default:
-          warning( `Unsupported artifact type "${artifactSchema.type}"` )
-          continue
-      }
-      
-      if ( !packager ) {
+  // TODO: Use concurrency to push artifacts and provide summary of results.
+  for ( const artifactSchema of filteredArtifactSchema ) {
+    let pusher
+    switch ( artifactSchema.type ) {
+      case ArtifactTypeEnum.GENERIC:
+        pusher = new GenericArtifactPusher( artifactSchema )
+        break
+      default:
         warning( `Unsupported artifact type "${artifactSchema.type}"` )
         continue
-      }
+    }
       
-      // eslint-disable-next-line no-await-in-loop
-      await packager.package()
+    if ( !pusher ) {
+      warning( `Unsupported artifact type "${artifactSchema.type}"` )
+      continue
     }
+      
+    // eslint-disable-next-line no-await-in-loop
+    await pusher.push()
   }
 }