@leverege/build-tools 2.93.4 → 2.93.6

package/.vscode/settings.json

@@ -0,0 +1,3 @@
+{
+    "editor.formatOnSave": true
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@leverege/build-tools",
-  "version": "2.93.4",
+  "version": "2.93.6",
   "description": "A collection of build / support tools for Leverege developers",
   "main": "index.js",
   "repository": {
@@ -11,7 +11,8 @@
     "clean": "rm -rf lib",
     "lint": "find ./src -name \\*.\\*js | xargs npx eslint --report-unused-disable-directives",
     "prepare": "[ $SKIP_PREPARE ] && exit 0 || ./src/hook-and-release.mjs",
-    "test": "mocha -t 60000 --exit 'test/**/*.mjs'",
+    "test": "mocha -t 60000 --exit 'test/**/!(*e2e*).mjs'",
+    "test-all": "mocha -t 60000 --exit 'test/**/**.mjs'",
     "prepack": "npm exec prepack"
   },
   "bin": {
@@ -104,4 +105,4 @@
     "mocha": "^11.7.5",
     "npm": "^11.7.0"
   }
-}
+}

package/src/artifacts/ArtifactPuller.mjs

@@ -1,12 +1,14 @@
 /* eslint-disable security/detect-non-literal-fs-filename */
 import fs from 'node:fs'
+import path from 'node:path'
 
 import { shellCmd, log, debug, err } from '../Utils.mjs'
 
-import { toExplicitPath, deduplicatePath, convertGenericArtifactNameToPath } from './Utils.mjs'
+import { parseGenericArtifactSchemaToRemoteRelativePath, parseGenericArtifactSchemaToLocalRelativePath, listGenericArtifactFiles, parseGenericArtifactFileToRemoteRelativePath, parseGenericArtifactSchemaToArtifactName } from './Utils.mjs'
 
 class ArtifactPuller {
-  constructor( artifact ) {
+  constructor( path, artifact ) {
+    this.path = path
     this.artifact = artifact
   }
 
@@ -17,142 +19,101 @@ class ArtifactPuller {
 
 class GenericArtifactPuller extends ArtifactPuller {
 
-  async prepareFileDownload( tmpFilePath, finalFilePath ) {
-    // Ensure directories exist
-    fs.mkdirSync( tmpFilePath.split( '/' ).slice( 0, -1 ).join( '/' ), { recursive : true } )
-    fs.mkdirSync( finalFilePath.split( '/' ).slice( 0, -1 ).join( '/' ), { recursive : true } )
+  async downloadFile( remoteRelativePath, localRelativePath ) {
+    debug( { remoteRelativePath, localRelativePath }, '<==Download File' )
 
-    // Ensure no conflicting files exist
-    if ( fs.existsSync( tmpFilePath ) ) {
-      fs.unlinkSync( tmpFilePath )
-    }
-    if ( fs.existsSync( finalFilePath ) ) {
-      fs.unlinkSync( finalFilePath )
-    }
-  }
+    const isBasenameDifferent = path.basename( localRelativePath ) !== path.basename( remoteRelativePath )
 
-  async cleanupFileDownload( tmpFilePath, finalFilePath ) {
-    // Delete the tmp file
-    if ( fs.existsSync( tmpFilePath ) ) {
-      fs.unlinkSync( tmpFilePath )
+    let tmpLocalRelativePath
+    if ( isBasenameDifferent ) {
+      const artifactName = parseGenericArtifactSchemaToArtifactName( this.artifact )
+      const encodedArtifactName = artifactName.replaceAll( ':', '_' ).replaceAll( '/', '_' ).replaceAll( '%2F', '_' )
+      tmpLocalRelativePath = path.join( '/', 'tmp', encodedArtifactName )
+    } else {
+      await fs.promises.mkdir( path.dirname( localRelativePath ), { recursive : true } )
     }
-    // Delete the tmp directory
-    if ( fs.existsSync( tmpFilePath.split( '/' ).slice( 0, -1 ).join( '/' ) ) ) {
-      fs.rmSync( tmpFilePath.split( '/' ).slice( 0, -1 ).join( '/' ), { recursive : true } )
-    }
-  }
-
-  async downloadFile( remoteFilePath, localFilePath ) {
-    debug( { remoteFilePath, localFilePath }, '<==Download File' )
-
-    // 1 dir per file to avoid dir creation and deletion race conditions for files in the same directory
-    const tmpDirectoryPath = `/tmp/${remoteFilePath.replaceAll( '/', '_' ).replaceAll( '.', '_' )}`
-    const tmpFilePath = `${tmpDirectoryPath}/${remoteFilePath.split( '/' ).pop()}`
-
-    await this.prepareFileDownload( tmpFilePath, localFilePath )
 
     try {
       const command = `gcloud artifacts generic download \
-      --project=${this.artifact.gcp_project_id} \
-      --location=${this.artifact.gcp_location} \
-      --repository=${this.artifact.gcp_artifact_registry_repository_name} \
-      --package=${this.artifact.gcp_artifact_registry_package_name} \
-      --version=${this.artifact.gcp_artifact_registry_package_version} \
-      --name="${remoteFilePath}" \
-      --destination=${tmpDirectoryPath}`
+      --project=${this.artifact.project} \
+      --location=${this.artifact.location} \
+      --repository=${this.artifact.repository} \
+      --package=${this.artifact.package} \
+      --version=${this.artifact.version} \
+      --name="${remoteRelativePath}" \
+      --destination=${tmpLocalRelativePath || path.dirname( localRelativePath )}`
 
       debug( { command }, '<==Download File Command' )
 
       await shellCmd( command )
 
-      // Check if file exists in the tmp path
-      if ( !fs.existsSync( tmpFilePath ) ) {
-        debug( { tmpFilePath }, '<==File does not exist in the tmp path' )
-        return false
-      }
-
-      debug( { tmpFilePath, localFilePath }, '<==Copying file' )
-
-      await fs.copyFileSync( tmpFilePath, localFilePath )
-
-      // Check if file exists in the local path
-      if ( !fs.existsSync( localFilePath ) ) {
-        debug( { localFilePath }, '<==File does not exist in the local path' )
-        return false
+      if ( isBasenameDifferent ) {
+        await fs.promises.rename( tmpLocalRelativePath, localRelativePath )
       }
 
       return true
     } catch ( error ) {
-      err( `Error downloading file: ${remoteFilePath} ${error.message}` )
+      err( `Error downloading file: ${localRelativePath} ${error.message}` )
       return false
-    } finally {
-      await this.cleanupFileDownload( tmpFilePath, localFilePath )
     }
   }
 
-  async downloadDirectory( localPath ) {
+  async downloadDirectory( remoteRelativePath, localRelativePath ) {
+
+    await fs.promises.mkdir( localRelativePath, { recursive : true } )
+
     const command = `gcloud artifacts generic download \
-      --project=${this.artifact.gcp_project_id} \
-      --location=${this.artifact.gcp_location} \
-      --repository=${this.artifact.gcp_artifact_registry_repository_name} \
-      --package=${this.artifact.gcp_artifact_registry_package_name} \
-      --version=${this.artifact.gcp_artifact_registry_package_version} \
-      --destination=${localPath}`
+      --project=${this.artifact.project} \
+      --location=${this.artifact.location} \
+      --repository=${this.artifact.repository} \
+      --package=${this.artifact.package} \
+      --version=${this.artifact.version} \
+      --destination=${localRelativePath}`
 
     debug( { command }, '<==Download Directory Command' )
 
     await shellCmd( command )
   }
 
-  async downloadNestedDirectory( remotePath, localPath ) {
-    debug( { remotePath, localPath }, '<==Download Nested Directory' )
+  async downloadNestedDirectory( remoteRelativePath, localRelativePath ) {
+    debug( { remoteRelativePath, localRelativePath }, '<==Download Nested Directory' )
 
-    const files = await this.listFiles( remotePath )
+    const files = await listGenericArtifactFiles( this.artifact )
     await Promise.all(
       files
-        .filter( file => convertGenericArtifactNameToPath( file.name ).startsWith( remotePath ) )
-        .map( async ( file ) => {
-          const remoteFilePath = convertGenericArtifactNameToPath( file.name )
-          const localFilePath = deduplicatePath( `${localPath}/${remoteFilePath.replaceAll( remotePath, '' )}` )
-          await this.downloadFile( remoteFilePath, localFilePath )
-        } )
+        .reduce( ( acc, file ) => {
+          const remoteFilePath = parseGenericArtifactFileToRemoteRelativePath( file )
+          if ( remoteFilePath.startsWith( remoteRelativePath ) ) {
+            const remoteFilePathWithoutRemotePath = remoteFilePath.replace( remoteRelativePath, '' )
+            const localFilePath = path.join( localRelativePath, remoteFilePathWithoutRemotePath )
+            acc.push( this.downloadFile( remoteFilePath, localFilePath ) )
+          }
+          return acc
+        }, [] )
     )
   }
 
-  async listFiles( ) {
-    const command = `gcloud artifacts files list \
-        --project=${this.artifact.gcp_project_id} \
-        --location=${this.artifact.gcp_location} \
-        --repository=${this.artifact.gcp_artifact_registry_repository_name} \
-        --package=${this.artifact.gcp_artifact_registry_package_name} \
-        --version ${this.artifact.gcp_artifact_registry_package_version} \
-        --format="json"`
-
-    debug( { command }, '<==List Files Command' )
-      
-    const result = await shellCmd( command )
-  
-    // Parse the JSON response
-    const files = JSON.parse( result )
-    return files
-  }
-
   async pull() {
     log( `Pulling artifact: ${this.artifact.name}` )
     
     debug( { artifact : this.artifact }, '<==Pulling artifact' )
 
     try {
+      const remoteRelativePath = parseGenericArtifactSchemaToRemoteRelativePath( this.artifact )
+      const localRelativePath = parseGenericArtifactSchemaToLocalRelativePath( this.artifact )
+
       if ( this.artifact.remote_file_path && this.artifact.local_file_path ) {
-        await this.downloadFile( this.artifact.remote_file_path, toExplicitPath( this.artifact.local_file_path ) )
+        await this.downloadFile( remoteRelativePath, path.join( this.path, localRelativePath ) )
       } else if ( this.artifact.remote_file_path && this.artifact.local_path ) {
-        await this.downloadFile( this.artifact.remote_file_path, toExplicitPath( `${this.artifact.local_path}/${this.artifact.remote_file_path.split( '/' ).pop()}` ) )
+        await this.downloadFile( remoteRelativePath, path.join( this.path, localRelativePath ) )
+      } else if ( this.artifact.remote_file_path && !this.artifact.local_path && !this.artifact.local_file_path ) {
+        await this.downloadFile( remoteRelativePath, path.join( this.path, localRelativePath ) )
       } else if ( this.artifact.remote_path && this.artifact.local_path ) {
-        await this.downloadNestedDirectory( this.artifact.remote_path, toExplicitPath( this.artifact.local_path ) )
+        await this.downloadNestedDirectory( remoteRelativePath, path.join( this.path, localRelativePath ) )
       } else if ( this.artifact.remote_path && !this.artifact.local_path && !this.artifact.local_file_path ) {
-        await this.downloadDirectory( this.artifact.remote_path, '.' )
+        await this.downloadDirectory( remoteRelativePath, path.join( this.path, localRelativePath ) )
       } else if ( !this.artifact.remote_path && !this.artifact.remote_file_path && this.artifact.local_path ) {
-        await this.downloadDirectory( toExplicitPath( this.artifact.local_path ) )
+        await this.downloadDirectory( remoteRelativePath, path.join( this.path, localRelativePath ) )
       } else {
         throw new Error( 'No valid paths provided' )
       }
@@ -160,13 +121,6 @@ class GenericArtifactPuller extends ArtifactPuller {
       log( `Successfully pulled artifact: ${this.artifact.name}` )
 
       return true
-      
-      // if ( !this.artifact.remote_path && !this.artifact.remote_file_path && this.local_file_path ) {
-      //   throw new Error( '' )
-      // }
-      // if ( this.artifact.remote_path && this.artifact.local_file_path ) {
-      //   throw new Error( '' )
-      // }
     } catch ( error ) {
       err( `Error pulling artifact: ${this.artifact.name} ${error.message}` )
     }

package/src/artifacts/ArtifactPusher.mjs

@@ -1,9 +1,20 @@
+import path from 'node:path'
+import fs from 'node:fs'
+
 import { shellCmd, log, err, debug } from '../Utils.mjs'
 
-import { convertGenericArtifactNameToPath } from './Utils.mjs'
+import {
+  parseGenericArtifactFileToRemoteRelativePath,
+  parseGenericArtifactSchemaToRemoteRelativePath,
+  listGenericArtifactFiles,
+  shouldUseIgnoreConfiguration,
+  findIgnoreFilePath,
+  copyFilesWithIgnoreConfiguration
+} from './Utils.mjs'
 
 class ArtifactPusher {
-  constructor( artifact ) {
+  constructor( path, artifact ) {
+    this.path = path
     this.artifact = artifact
   }
 
@@ -13,25 +24,6 @@ class ArtifactPusher {
 }
 
 class GenericArtifactPusher extends ArtifactPusher {
-
-  async listFiles( ) {
-    const command = `gcloud artifacts files list \
-        --project=${this.artifact.gcp_project_id} \
-        --location=${this.artifact.gcp_location} \
-        --repository=${this.artifact.gcp_artifact_registry_repository_name} \
-        --package=${this.artifact.gcp_artifact_registry_package_name} \
-        --version ${this.artifact.gcp_artifact_registry_package_version} \
-        --format="json"`
-
-    debug( { command }, '<==List Files Command' )
-      
-    const result = await shellCmd( command )
-  
-    // Parse the JSON response
-    const files = JSON.parse( result )
-
-    return files
-  }
     
   async push() {
     log( `Pushing artifact: ${this.artifact.name}` )
@@ -41,41 +33,74 @@ class GenericArtifactPusher extends ArtifactPusher {
       return false
     }
 
-    try {
-   
+    if ( !this.artifact.local_path && !this.artifact.local_file_path ) {
+      err( 'Local path or local file path is required for pushing artifacts' )
+      return false
+    }
+
+    try {   
       let command = ''
 
       command = `gcloud artifacts generic upload \
-      --project=${this.artifact.gcp_project_id} \
-      --location=${this.artifact.gcp_location} \
-      --repository=${this.artifact.gcp_artifact_registry_repository_name} \
-      --package=${this.artifact.gcp_artifact_registry_package_name} \
-      --version=${this.artifact.gcp_artifact_registry_package_version}`
+      --project=${this.artifact.project} \
+      --location=${this.artifact.location} \
+      --repository=${this.artifact.repository} \
+      --package=${this.artifact.package} \
+      --version=${this.artifact.version}`
+
+      let pathToDelete
 
       if ( this.artifact.local_path ) {
-        command += ` --source-directory=${this.artifact.local_path}`
+        const originalLocalPath = path.join( this.path, this.artifact.local_path )
+
+        let localPath = originalLocalPath
+
+        const useIgnoreConfiguration = await shouldUseIgnoreConfiguration()
+
+        if ( useIgnoreConfiguration ) {
+          const ignoreFilePath = await findIgnoreFilePath( this.path )
+          if ( ignoreFilePath ) {
+            const tmpLocalPath = path.join( '/', 'tmp', `${this.artifact.project}_${this.artifact.location}_${this.artifact.repository}_${this.artifact.package}_${this.artifact.version}` )
+            localPath = tmpLocalPath
+            pathToDelete = tmpLocalPath
+            await copyFilesWithIgnoreConfiguration( originalLocalPath, tmpLocalPath, ignoreFilePath )
+          }
+        }
+
+        command += ` --source-directory=${localPath}`
         command += ' --skip-existing'
       } else if ( this.artifact.local_file_path ) {
-        const files = await this.listFiles( this.artifact.remote_path )
-        if ( files.some( file => convertGenericArtifactNameToPath( file.name ) === `${this.artifact.remote_path}/${this.artifact.remote_file_path.split( '/' ).pop()}` ) ) {
-          err( `File already exists in the artifact registry: ${this.artifact.local_file_path}` )
-          return false
+
+        const files = await listGenericArtifactFiles( this.artifact )
+
+        const newRemoteFilePath = parseGenericArtifactSchemaToRemoteRelativePath( this.artifact )
+
+        for ( const file of files ) {
+          const existingRemoteFilePath = parseGenericArtifactFileToRemoteRelativePath( file )
+          if ( existingRemoteFilePath === newRemoteFilePath ) {
+            err( `File already exists in the artifact registry:  ${existingRemoteFilePath}` )
+            return false
+          }
         }
-        command += ` --source=${this.artifact.local_file_path}`
+        const localPath = path.join( this.path, this.artifact.local_file_path )
+        command += ` --source=${localPath}`
       }
 
-      if ( this.artifact.remote_path ) {
-        command += ` --destination-path=${this.artifact.remote_path}`
-      }
+      command += ` --destination-path=${this.artifact.remote_path || '""'}`
 
       debug( { command }, '<==Upload Command' )
         
       await shellCmd( command )
-      
+
+      if ( pathToDelete ) {
+        await fs.promises.rm( pathToDelete, { recursive : true } )
+      }
+
       log( `Successfully pushed artifact: ${this.artifact.name}` )
 
       return true
     } catch ( error ) {
+      debug( { error }, '<==Error Pushing Artifact' )
       err( `Error pushing artifact: ${this.artifact.name} ${error.message}` )
       return false
     }

package/src/artifacts/Structs.mjs

@@ -1,37 +1,42 @@
-import { assign, object, string, array, optional, enums, union, refine } from 'superstruct'
+import path from 'node:path'
+
+import { assign, coerce, object, string, array, optional, enums, union, refine } from 'superstruct'
 
 const ArtifactTypeEnum = {
   GENERIC : 'generic',
 }
 
-// Superstruct schema for individual artifact configuration
+// Structs with the Partial prefix come from the need to validate the schema with defaults and without defaults merged together.
+
 const PartialArtifactSchema = object( {
   name : string(),
   type : optional( enums( Object.values( ArtifactTypeEnum ) ) ),
-  gcp_project_id : optional( string() ),
-  gcp_location : optional( string() ),
-  gcp_artifact_registry_repository_name : optional( string() ),
-  gcp_artifact_registry_package_name : optional( string() ),
-  gcp_artifact_registry_package_version : optional( string() )
+  project : optional( string() ),
+  location : optional( string() ),
+  repository : optional( string() ),
+  package : optional( string() ),
+  version : optional( string() )
 } )
 
 const ArtifactSchema = assign( PartialArtifactSchema, object( {
   name : string(),
   type : enums( Object.values( ArtifactTypeEnum ) ),
-  gcp_project_id : string(),
-  gcp_location : string(),
-  gcp_artifact_registry_repository_name : string(),
-  gcp_artifact_registry_package_name : string(),
-  gcp_artifact_registry_package_version : string()
+  project : string(),
+  location : string(),
+  repository : string(),
+  package : string(),
+  version : string()
 } ) )
 
+const normalizedPath = () => coerce( string(), string(), value => path.normalize( value ) )
+
 const PartialGenericArtifactSchema = refine(
   assign( PartialArtifactSchema, object( {
-    type : enums( [ ArtifactTypeEnum.GENERIC ] ),
-    local_file_path : optional( string() ),
-    local_path : optional( string() ),
-    remote_file_path : optional( string() ),
-    remote_path : optional( string() )
+    type : optional( enums( [ ArtifactTypeEnum.GENERIC ] ) ),
+    local_file_path : optional( normalizedPath() ),
+    local_path : optional( normalizedPath() ),
+    remote_file_path : optional( normalizedPath() ),
+    remote_path : optional( normalizedPath() )
   } ) ),
   'ConflictingPaths',
   ( value ) => {
@@ -47,10 +52,11 @@ const PartialGenericArtifactSchema = refine(
 
 const GenericArtifactSchema = refine(
   assign( ArtifactSchema, object( {
-    local_file_path : optional( string() ),
-    local_path : optional( string() ),
+    type : enums( [ ArtifactTypeEnum.GENERIC ] ),
+    local_file_path : optional( normalizedPath() ),
+    local_path : optional( normalizedPath() ),
     remote_file_path : optional( string() ),
-    remote_path : optional( string() )
+    remote_path : optional( normalizedPath() )
   } ) ),
   'ConflictingPaths',
   ( value ) => {
@@ -66,11 +72,12 @@ const GenericArtifactSchema = refine(
 
 // Superstruct schema for defaults configuration
 const DefaultsSchema = object( {
-  gcp_project_id : optional( string() ),
-  gcp_location : optional( string() ),
-  gcp_artifact_registry_repository_name : optional( string() ),
-  gcp_artifact_registry_package_name : optional( string() ),
-  gcp_artifact_registry_package_version : optional( string() )
+  type : optional( enums( Object.values( ArtifactTypeEnum ) ) ),
+  project : optional( string() ),
+  location : optional( string() ),
+  repository : optional( string() ),
+  package : optional( string() ),
+  version : optional( string() )
 } )
 
 // Superstruct schema for the complete artifacts configuration with defaults

package/src/artifacts/Utils.mjs

@@ -1,16 +1,19 @@
 import fs from 'node:fs'
+import path from 'node:path'
 import crypto from 'crypto'
 
 import { validate } from 'superstruct'
 
-import { parseYamlFile, warning } from '../Utils.mjs'
+import { parseYamlFile, err, warning, debug, shellCmd } from '../Utils.mjs'
 
 import { ArtifactTypeEnum, ArtifactsSchema, GenericArtifactSchema } from './Structs.mjs'
 
-const getArtifactsSchema = async ( artifactsFilePath = './artifacts.yml' ) => {
+const getArtifactsSchema = async ( artifactsFilePath = './artifacts.yaml' ) => {
   const artifactsConfig = await parseYamlFile( artifactsFilePath )
   const [ error, result ] = validate( artifactsConfig, ArtifactsSchema )
   if ( error ) {
+    err( `Invalid artifacts.yml file: ${error.message}` )
+    debug( { err : error }, '<==Invalid artifacts.yml file' )
     throw new Error( `Invalid artifacts.yml file: ${error.message}` )
   }
   return result
@@ -22,7 +25,7 @@ const flattenArtifactsSchema = ( artifactsSchema ) => {
       ...artifactsSchema.defaults,
       ...artifact
     }
-    
+
     let schema
     switch ( mergedArtifact.type ) {
       case ArtifactTypeEnum.GENERIC:
@@ -49,44 +52,230 @@ const flattenArtifactsSchema = ( artifactsSchema ) => {
   }, [] )
 }
 
-const toImplicitPath = ( path ) => {
-  let implicitPath = path
-  // Remove leading './' if present
-  if ( implicitPath.startsWith( './' ) ) {
-    implicitPath = implicitPath.slice( 2 )
+// Example of a file in a generic artifact registry repository:
+// [
+//   {
+//     "createTime": "2025-12-02T17:52:08.694541Z",
+//     "hashes": [
+//       {
+//         "type": "SHA256",
+//         "value": "wrBRqJOUjm-zy6B7YzgKlK0fv9Ad0EEjydBnsmc9Ruw="
+//       },
+//       {
+//         "type": "MD5",
+//         "value": "Ebp3fVphzrITJ461Mg7Kmg=="
+//       }
+//     ],
+//     "name": "projects/leverege-registry/locations/us-central1/repositories/test-generic-registry/files/sweet-poems-1-1764697925426:v1:test1.txt%2Ftest1.txt",
+//     "owner": "projects/leverege-registry/locations/us-central1/repositories/test-generic-registry/packages/sweet-poems-1-1764697925426/versions/v1",
+//     "sizeBytes": "40",
+//     "updateTime": "2025-12-02T17:52:08.694541Z"
+//   }
+// ]
+// Obtained with:gcloud artifacts files list --project=leverege-registry --location=us-central1 --repository=test-generic-registry --package=sweet-poems-1-1764697925426 --version v1 --format="json"
+
+const parseGenericArtifactFileToRemoteRelativePath = ( genericArtifactFile ) => {
+  // "projects/leverege-registry/locations/us-central1/repositories/test-generic-registry/files/sweet-poems-1-1764697925426:v1:test1.txt%2Ftest1.txt"
+  const artifactFileName = genericArtifactFile.name.split( '/' ).pop()
+  // sweet-poems-1-1764697925426:v1:test1.txt%2Ftest1.txt
+  const remoteRelativePath = artifactFileName.split( ':' ).pop().replaceAll( '%2F', '/' )
+  // test1.txt
+  return path.normalize( remoteRelativePath )
+}
+
+const parseGenericArtifactSchemaToRemoteRelativePath = ( genericArtifactSchema ) => {
+  if ( genericArtifactSchema.remote_file_path ) {
+    return path.normalize( genericArtifactSchema.remote_file_path )
+  }
+  if ( genericArtifactSchema.remote_path && genericArtifactSchema.local_file_path ) {
+    return path.join( genericArtifactSchema.remote_path, path.basename( genericArtifactSchema.local_file_path ) )
+  }
+  if ( !genericArtifactSchema.remote_file_path && !genericArtifactSchema.remote_path && genericArtifactSchema.local_file_path ) {
+    return path.basename( genericArtifactSchema.local_file_path )
   }
-  // Remove trailing slash
-  implicitPath = implicitPath.replace( /\/+$/, '' )
-  return implicitPath
+  if ( !genericArtifactSchema.remote_file_path && !genericArtifactSchema.remote_path && genericArtifactSchema.local_file_path ) {
+    return path.basename( genericArtifactSchema.local_file_path )
+  }
+  if ( genericArtifactSchema.remote_path && genericArtifactSchema.local_path ) {
+    return path.normalize( genericArtifactSchema.remote_path )
+  }
+  return path.normalize( '.' )
 }
 
-const toExplicitPath = ( path ) => {
-  let explicitPath = path
-  // Keep absolute paths and paths already starting with '.' as-is
-  if ( !path.startsWith( '/' ) && !path.startsWith( '.' ) ) {
-    // Add './' to relative paths
-    explicitPath = `./${explicitPath}`
+const parseGenericArtifactSchemaToLocalRelativePath = ( genericArtifactSchema ) => {
+  if ( genericArtifactSchema.local_file_path ) {
+    return path.normalize( genericArtifactSchema.local_file_path )
+  }
+  if ( genericArtifactSchema.local_path && genericArtifactSchema.remote_file_path ) {
+    return path.join( genericArtifactSchema.local_path, path.basename( genericArtifactSchema.remote_file_path ) )
   }
-  return explicitPath
+  if ( genericArtifactSchema.local_path && !genericArtifactSchema.remote_file_path && !genericArtifactSchema.remote_file_path ) {
+    return path.normalize( genericArtifactSchema.local_path )
+  }
+  if ( !genericArtifactSchema.local_file_path && !genericArtifactSchema.local_path && genericArtifactSchema.remote_file_path ) {
+    return path.basename( genericArtifactSchema.remote_file_path )
+  }
+  return path.normalize( '.' )
 }
 
-const deduplicatePath = ( path ) => {
-  return path.replaceAll( '//', '/' )
+const parseGenericArtifactSchemaToArtifactNameBase = ( genericArtifactSchema ) => {
+  const elements = [
+    'projecs',
+    genericArtifactSchema.project,
+    'locations',
+    genericArtifactSchema.location,
+    'repositories',
+    genericArtifactSchema.repository,
+    'files',
+  ]
+  return elements.join( '/' )
 }
 
-const convertGenericArtifactNameToPath = ( genericArtifactName ) => {
-  return genericArtifactName.split( ':' ).pop().replaceAll( '%2F', '/' ).replaceAll( '%3A', ':' )
+const parseGenericArtifactSchemaToArtifactNameFile = ( genericArtifactSchema ) => {
+  // //     "name": "projects/leverege-registry/locations/us-central1/repositories/test-generic-registry/files/sweet-poems-1-1764697925426:v1:test1.txt%2Ftest1.txt",
+
+  const remoteRelativePath = parseGenericArtifactSchemaToRemoteRelativePath( genericArtifactSchema )
+  const encodedRemoteRelativePath = encodeURIComponent( remoteRelativePath )
+
+  const elements = [
+    genericArtifactSchema.package,
+    genericArtifactSchema.version,
+    encodedRemoteRelativePath,
+  ]
+  return elements.join( ':' )
+}
+
+const parseGenericArtifactSchemaToArtifactName = ( genericArtifactSchema ) => {
+  const base = parseGenericArtifactSchemaToArtifactNameBase( genericArtifactSchema )
+  const file = parseGenericArtifactSchemaToArtifactNameFile( genericArtifactSchema )
+
+  const elements = [
+    base,
+    file,
+  ]
+
+  return elements.join( ':' )
 }
 
-const calculateHash = ( path ) => {
+const calculateHash = async ( path ) => {
   try {
     const hash = crypto.createHash( 'sha256' )
-    hash.update( fs.readFileSync( path ) )
-    return hash.digest( 'base64url' )
+    hash.update( await fs.promises.readFile( path ) )
+    return `${hash.digest( 'base64url' )}=`
   } catch ( error ) {
-    console.error( error )
     throw new Error( `Failed to calculate hash for artifact ${path}: ${error.message}` )
   }
 }
 
-export { calculateHash, getArtifactsSchema, flattenArtifactsSchema, toImplicitPath, toExplicitPath, deduplicatePath, convertGenericArtifactNameToPath }
+const listGenericArtifactFiles = async ( artifact ) => {
+  const command = `gcloud artifacts files list \
+      --project=${artifact.project} \
+      --location=${artifact.location} \
+      --repository=${artifact.repository} \
+      --package=${artifact.package} \
+      --version ${artifact.version} \
+      --format="json"`
+
+  debug( { command }, '<==List Files Command' )
+    
+  const result = await shellCmd( command )
+
+  // Parse the JSON response
+  const files = JSON.parse( result )
+  return files
+}
+
+const getMissingRemoteFiles = async ( genericArtifactFiles, genericArtifactSchemaPath, genericArtifactSchema ) => {
+  
+  const expectedRemoteRelativePath = parseGenericArtifactSchemaToRemoteRelativePath( genericArtifactSchema )
+  const localRelativePath = parseGenericArtifactSchemaToLocalRelativePath( genericArtifactSchema )
+  const expectedLocalRelativePath = path.join( genericArtifactSchemaPath, localRelativePath )
+
+  const actualFiles = new Set()
+  const expectedFiles = new Set()
+  
+  for ( const file of genericArtifactFiles ) {
+    const actualRemoteFilePath = parseGenericArtifactFileToRemoteRelativePath( file )
+    const actualRemoteFileHash = file.hashes.find( hash => hash.type === 'SHA256' ).value
+    actualFiles.add( { remoteFilePath : actualRemoteFilePath, remoteFileHash : actualRemoteFileHash } )
+  }
+
+  if ( ( await fs.promises.stat( expectedLocalRelativePath ) ).isDirectory() ) {
+    const localFiles = await fs.promises.readdir( expectedLocalRelativePath, { withFileTypes : true, recursive : true } )
+    for ( const localFile of localFiles ) {
+      if ( localFile.isDirectory() ) {
+        continue
+      }
+
+      const expectedRemoteFilePath = path.join( expectedRemoteRelativePath, localFile.path.replace( expectedLocalRelativePath, '' ), localFile.name )
+      const expectedLocalFilePath = path.join( localFile.path, localFile.name )
+      expectedFiles.add( { remoteFilePath : expectedRemoteFilePath, remoteFileHash : await calculateHash( expectedLocalFilePath ) } )
+    }
+  } else {
+    const expectedRemoteFilePath = expectedRemoteRelativePath
+    const expectedLocalFilePath = expectedLocalRelativePath
+    expectedFiles.add( { remoteFilePath : expectedRemoteFilePath, remoteFileHash : await calculateHash( expectedLocalFilePath ) } )
+  }
+
+  const actualArray = Array.from( actualFiles )
+  const expectedArray = Array.from( expectedFiles )
+
+  return expectedArray.filter( expected => !actualArray.some( actual => actual.remoteFilePath === expected.remoteFilePath && actual.remoteFileHash === expected.remoteFileHash ) )
+}
+
+const shouldUseIgnoreConfiguration = async () => {
+  let result
+  try {
+    result = await shellCmd( 'gcloud config get-value gcloudignore/enabled' )
+    if ( result.trim() === '' || result.includes( 'has no property' ) ) {
+      return false
+    }
+    return result.trim() === 'true'
+  } catch ( error ) {
+    if ( error.message && error.message.includes( 'Section [gcloudignore] has no property [enabled]' ) ) {
+      return false
+    }
+    // Re-throw other errors
+    throw error
+  }
+}
+
+const findIgnoreFilePath = async ( relativePath ) => {
+  if ( relativePath ) {
+    const ignoreFilePath = path.join( relativePath, '.gcloudignore' )
+    if ( fs.existsSync( ignoreFilePath ) ) {
+      return ignoreFilePath
+    }
+  }
+
+  const ignoreFilePath = path.join( process.cwd(), '.gcloudignore' )
+  if ( fs.existsSync( ignoreFilePath ) ) {
+    return ignoreFilePath
+  }
+
+  return null
+}
+
+const copyFilesWithIgnoreConfiguration = async ( sourcePath, destinationPath, ignoreFilePath ) => {
+  // Ensure sourcePath ends with / so rsync copies contents (not the folder itself) and exclude patterns work correctly
+  const normalizedSourcePath = sourcePath.endsWith( '/' ) ? sourcePath : `${sourcePath}/`
+  const command = `rsync -av --delete --exclude-from='${ignoreFilePath}' ${normalizedSourcePath} ${destinationPath}`
+  await shellCmd( command )
+}
+
+export {
+  calculateHash,
+  getArtifactsSchema,
+  flattenArtifactsSchema,
+  parseGenericArtifactFileToRemoteRelativePath,
+  parseGenericArtifactSchemaToRemoteRelativePath,
+  parseGenericArtifactSchemaToLocalRelativePath,
+  parseGenericArtifactSchemaToArtifactNameBase,
+  parseGenericArtifactSchemaToArtifactNameFile,
+  parseGenericArtifactSchemaToArtifactName,
+  getMissingRemoteFiles,
+  listGenericArtifactFiles,
+  shouldUseIgnoreConfiguration,
+  findIgnoreFilePath,
+  copyFilesWithIgnoreConfiguration
+}

package/src/artifacts/pull-artifact.mjs

@@ -1,5 +1,7 @@
 #!/usr/bin/env node
 
+import path from 'node:path'
+
 import { program } from 'commander'
 
 import { warning, log } from '../Utils.mjs'
@@ -11,33 +13,36 @@ import { GenericArtifactPuller } from './ArtifactPuller.mjs'
 program
   .name( 'pull-artifact' )
   .description( 'Pull artifacts from GCP Artifact Registry' )
-  .requiredOption( '-f, --file-path <path>', 'Path to artifacts.yaml file' )
-  .option( '-n, --package-name <name>', 'Filter by artifact package name' )
-  .option( '-v, --package-version <version>', 'Filter by artifact package version' )
+  .option( '-f, --file-path <file-path>', 'Path to artifacts.yaml file', './artifacts.yaml' )
+  .option( '-a, --artifact-path <artifact-path>', 'GCP artifact path' )
   .parse()
 
 const options = program.opts()
 
 async function main() {
-  const artifactsYmlPath = options.filePath
-  const artifactPackageName = options.packageName || null
-  const artifactPackageVersion = options.packageVersion || null
-  
-  const artifactsSchema = await getArtifactsSchema( artifactsYmlPath )
+  const artifactsSchema = await getArtifactsSchema( options.filePath )
   const flattenedArtifactsSchema = flattenArtifactsSchema( artifactsSchema )
-  
-  let filteredArtifactsSchema
-  if ( artifactPackageName && artifactPackageVersion ) {
-    filteredArtifactsSchema = flattenedArtifactsSchema.filter(
-      artifact => artifact.gcp_artifact_registry_package_name === artifactPackageName &&
-      artifact.gcp_artifact_registry_package_version === artifactPackageVersion )
-  } else if ( artifactPackageName ) {
+
+  let filteredArtifactsSchema = flattenedArtifactsSchema
+
+  if ( options.artifactPath ) {
+    const artifactPathElements = options.artifactPath.split( '/' )
+
+    if ( artifactPathElements.length !== 5 ) {
+      warning( 'Invalid artifact path, it must be in the format <project>/<location>/<repository>/<package>/<version>' )
+      return false
+    }
+
+    const [ project, location, repository, packageName, version ] = artifactPathElements
+
     filteredArtifactsSchema = flattenedArtifactsSchema.filter(
-      artifact => artifact.gcp_artifact_registry_package_name === artifactPackageName )
-  } else {
-    filteredArtifactsSchema = flattenedArtifactsSchema
+      artifact => artifact.project === project &&
+      artifact.location === location &&
+      artifact.repository === repository &&
+      artifact.package === packageName &&
+      artifact.version === version )
   }
-  
+
   let counter = 0
 
   const pullResults = await Promise.all(
@@ -45,7 +50,7 @@ async function main() {
       let puller
       switch ( artifactSchema.type ) {
         case ArtifactTypeEnum.GENERIC:
-          puller = new GenericArtifactPuller( artifactSchema )
+          puller = new GenericArtifactPuller( path.dirname( options.filePath ), artifactSchema )
           break
         default:
           warning( `Unsupported artifact type "${artifactSchema.type}"` )
@@ -69,6 +74,8 @@ async function main() {
   counter = pullResults.filter( Boolean ).length
 
   log( `Successfully pulled ${counter}/${filteredArtifactsSchema.length} artifacts` )
+
+  return counter === filteredArtifactsSchema.length
 }
 
 main()

package/src/artifacts/push-artifact.mjs

@@ -1,5 +1,7 @@
 #!/usr/bin/env node
 
+import path from 'node:path'
+
 import { program } from 'commander'
 
 import { warning, log } from '../Utils.mjs'
@@ -11,31 +13,33 @@ import { GenericArtifactPusher } from './ArtifactPusher.mjs'
 program
   .name( 'push-artifact' )
   .description( 'Push artifacts to GCP Artifact Registry' )
-  .requiredOption( '-f, --file-path <path>', 'Path to artifacts.yaml file' )
-  .option( '-n, --package-name <name>', 'Filter by artifact package name' )
-  .option( '-v, --package-version <version>', 'Filter by artifact package version' )
+  .option( '-f, --file-path <file-path>', 'Path to artifacts.yaml file', './artifacts.yaml' )
+  .option( '-a, --artifact-path <artifact-path>', 'GCP artifact path' )
   .parse()
 
 const options = program.opts()
 
 async function main() {
-  const artifactsYmlPath = options.filePath
-  const artifactPackageName = options.packageName || null
-  const artifactPackageVersion = options.packageVersion || null
-  
-  const artifactsSchema = await getArtifactsSchema( artifactsYmlPath )
+  const artifactsSchema = await getArtifactsSchema( options.filePath )
   const flattenedArtifactsSchema = flattenArtifactsSchema( artifactsSchema )
-  
-  let filteredArtifactsSchema
-  if ( artifactPackageName && artifactPackageVersion ) {
-    filteredArtifactsSchema = flattenedArtifactsSchema.filter(
-      artifact => artifact.gcp_artifact_registry_package_name === artifactPackageName &&
-      artifact.gcp_artifact_registry_package_version === artifactPackageVersion )
-  } else if ( artifactPackageName ) {
+
+  let filteredArtifactsSchema = flattenedArtifactsSchema
+
+  if ( options.artifactPath ) {
+    const artifactPathElements = options.artifactPath.split( '/' )
+    if ( artifactPathElements.length !== 5 ) {
+      warning( 'Invalid artifact path, it must be in the format <project>/<location>/<repository>/<package>/<version>' )
+      return false
+    }
+
+    const [ project, location, repository, packageName, version ] = artifactPathElements
+
     filteredArtifactsSchema = flattenedArtifactsSchema.filter(
-      artifact => artifact.gcp_artifact_registry_package_name === artifactPackageName )
-  } else {
-    filteredArtifactsSchema = flattenedArtifactsSchema
+      artifact => artifact.project === project &&
+      artifact.location === location &&
+      artifact.repository === repository &&
+      artifact.package === packageName &&
+      artifact.version === version )
   }
 
   let counter = 0
@@ -45,7 +49,7 @@ async function main() {
       let pusher
       switch ( artifactSchema.type ) {
         case ArtifactTypeEnum.GENERIC:
-          pusher = new GenericArtifactPusher( artifactSchema )
+          pusher = new GenericArtifactPusher( path.dirname( options.filePath ), artifactSchema )
           break
         default:
           warning( `Unsupported artifact type "${artifactSchema.type}"` )
@@ -69,6 +73,8 @@ async function main() {
   counter = pushResults.filter( Boolean ).length
 
   log( `Successfully pushed ${counter}/${filteredArtifactsSchema.length} artifacts` )
+  
+  return counter === filteredArtifactsSchema.length
 }
 
 main()