@leverege/build-tools 2.93.0-beta.14 → 2.93.0-beta.16

package/README.md

@@ -65,6 +65,15 @@ Usage: k8scale <up|dn|down|roll> [services]
 * **prune-git**: performs maintenance on a list of git repositories.
 * **pull-git**: walks a list of git repos, pulling each if they are clean.
 
+### Troubleshooting:
+
+For additional debug level logging, set the environment variable `BUILD_TOOLS_DEBUG=1` before running the tool.
+
+Example:
+```bash
+BUILD_TOOLS_DEBUG=1 docker-to-registry
+```
+
 ## Authors
 
 * **DevOps and Friends**

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@leverege/build-tools",
-  "version": "2.93.0-beta.14",
+  "version": "2.93.0-beta.16",
   "description": "A collection of build / support tools for Leverege developers",
   "main": "index.js",
   "repository": {

package/src/artifacts/ArtifactPuller.mjs

@@ -3,7 +3,7 @@ import fs from 'node:fs'
 
 import { shellCmd, log, debug, err } from '../Utils.mjs'
 
-import { toExplicitPath } from './Utils.mjs'
+import { toExplicitPath, deduplicatePath, convertGenericArtifactNameToPath } from './Utils.mjs'
 
 class ArtifactPuller {
   constructor( artifact ) {
@@ -17,55 +17,77 @@ class ArtifactPuller {
 
 class GenericArtifactPuller extends ArtifactPuller {
 
+  async prepareFileDownload( tmpFilePath, finalFilePath ) {
+    // Ensure directories exist
+    fs.mkdirSync( tmpFilePath.split( '/' ).slice( 0, -1 ).join( '/' ), { recursive : true } )
+    fs.mkdirSync( finalFilePath.split( '/' ).slice( 0, -1 ).join( '/' ), { recursive : true } )
+
+    // Ensure no conflicting files exist
+    if ( fs.existsSync( tmpFilePath ) ) {
+      fs.unlinkSync( tmpFilePath )
+    }
+    if ( fs.existsSync( finalFilePath ) ) {
+      fs.unlinkSync( finalFilePath )
+    }
+  }
+
+  async cleanupFileDownload( tmpFilePath, finalFilePath ) {
+    // Delete the tmp file
+    if ( fs.existsSync( tmpFilePath ) ) {
+      fs.unlinkSync( tmpFilePath )
+    }
+    // Delete the tmp directory
+    if ( fs.existsSync( tmpFilePath.split( '/' ).slice( 0, -1 ).join( '/' ) ) ) {
+      fs.rmdirSync( tmpFilePath.split( '/' ).slice( 0, -1 ).join( '/' ), { recursive : true } )
+    }
+  }
+
   async downloadFile( remoteFilePath, localFilePath ) {
     debug( { remoteFilePath, localFilePath }, '<==Download File' )
 
-    const command = `gcloud artifacts generic download \
+    // 1 dir per file to avoid dir creation and deletion race conditions for files in the same directory
+    const tmpDirectoryPath = `/tmp/${remoteFilePath.replaceAll( '/', '_' ).replaceAll( '.', '_' )}`
+    const tmpFilePath = `${tmpDirectoryPath}/${remoteFilePath.split( '/' ).pop()}`
+
+    await this.prepareFileDownload( tmpFilePath, localFilePath )
+
+    try {
+      const command = `gcloud artifacts generic download \
       --project=${this.artifact.gcp_project_id} \
       --location=${this.artifact.gcp_location} \
       --repository=${this.artifact.gcp_artifact_registry_repository_name} \
       --package=${this.artifact.gcp_artifact_registry_package_name} \
       --version=${this.artifact.gcp_artifact_registry_package_version} \
       --name="${remoteFilePath}" \
-      --destination=/tmp`
+      --destination=${tmpDirectoryPath}`
 
-    const oldPath = `/tmp/${remoteFilePath.split( '/' ).pop()}`
-    const newPath = localFilePath
+      debug( { command }, '<==Download File Command' )
 
-    // if file exists in the old path, delete it
-    if ( fs.existsSync( oldPath ) ) {
-      fs.unlinkSync( oldPath )
-    }
+      await shellCmd( command )
 
-    // if file exists in the new path, delete it
-    if ( fs.existsSync( newPath ) ) {
-      fs.unlinkSync( newPath )
-    }
+      // Check if file exists in the tmp path
+      if ( !fs.existsSync( tmpFilePath ) ) {
+        debug( { tmpFilePath }, '<==File does not exist in the tmp path' )
+        return false
+      }
 
-    // Ensure the directory exists
-    fs.mkdirSync( localFilePath.split( '/' ).slice( 0, -1 ).join( '/' ), { recursive : true } )
+      debug( { tmpFilePath, localFilePath }, '<==Copying file' )
 
-    debug( { command }, '<==Download File Command' )
+      await fs.copyFileSync( tmpFilePath, localFilePath )
 
-    await shellCmd( command )
-
-    // Check if file exists in the old path
-    if ( !fs.existsSync( oldPath ) ) {
-      debug( { oldPath }, '<==File does not exist in the old path' )
-      return false
-    }
-
-    debug( { oldPath, newPath }, '<==Copying file' )
-
-    await fs.copyFileSync( oldPath, newPath )
+      // Check if file exists in the local path
+      if ( !fs.existsSync( localFilePath ) ) {
+        debug( { localFilePath }, '<==File does not exist in the local path' )
+        return false
+      }
 
-    // Check if file exists in the new path
-    if ( !fs.existsSync( newPath ) ) {
-      debug( { newPath }, '<==File does not exist in the new path' )
+      return true
+    } catch ( error ) {
+      err( `Error downloading file: ${remoteFilePath} ${error.message}` )
       return false
+    } finally {
+      await this.cleanupFileDownload( tmpFilePath, localFilePath )
     }
-
-    return true
   }
 
   async downloadDirectory( localPath ) {
@@ -88,10 +110,10 @@ class GenericArtifactPuller extends ArtifactPuller {
     const files = await this.listFiles( remotePath )
     await Promise.all(
       files
-        .filter( file => file.name.split( ':' ).pop().replaceAll( '%2F', '/' ).startsWith( remotePath ) )
+        .filter( file => convertGenericArtifactNameToPath( file.name ).startsWith( remotePath ) )
         .map( async ( file ) => {
-          const remoteFilePath = file.name.split( ':' ).pop().replaceAll( '%2F', '/' )
-          const localFilePath = `${localPath}${remoteFilePath.replaceAll( remotePath, '' )}`
+          const remoteFilePath = convertGenericArtifactNameToPath( file.name )
+          const localFilePath = deduplicatePath( `${localPath}/${remoteFilePath.replaceAll( remotePath, '' )}` )
           await this.downloadFile( remoteFilePath, localFilePath )
         } )
     )

package/src/artifacts/ArtifactPusher.mjs

@@ -1,6 +1,8 @@
 /* eslint-disable security/detect-non-literal-fs-filename */
 import { shellCmd, log, err, debug } from '../Utils.mjs'
 
+import { convertGenericArtifactNameToPath } from './Utils.mjs'
+
 class ArtifactPusher {
   constructor( artifact ) {
     this.artifact = artifact
@@ -56,7 +58,7 @@ class GenericArtifactPusher extends ArtifactPusher {
         command += ' --skip-existing'
       } else if ( this.artifact.local_file_path ) {
         const files = await this.listFiles( this.artifact.remote_path )
-        if ( files.some( file => file.name.split( ':' ).pop().replaceAll( '%2F', '/' ) === `${this.artifact.remote_path}/${this.artifact.remote_file_path.split( '/' ).pop()}` ) ) {
+        if ( files.some( file => convertGenericArtifactNameToPath( file.name ) === `${this.artifact.remote_path}/${this.artifact.remote_file_path.split( '/' ).pop()}` ) ) {
           err( `File already exists in the artifact registry: ${this.artifact.local_file_path}` )
           return false
         }

package/src/artifacts/Utils.mjs

@@ -70,6 +70,14 @@ const toExplicitPath = ( path ) => {
   return explicitPath
 }
 
+const deduplicatePath = ( path ) => {
+  return path.replaceAll( '//', '/' )
+}
+
+const convertGenericArtifactNameToPath = ( genericArtifactName ) => {
+  return genericArtifactName.split( ':' ).pop().replaceAll( '%2F', '/' ).replaceAll( '%3A', ':' )
+}
+
 const calculateHash = ( path ) => {
   try {
     const hash = crypto.createHash( 'sha256' )
@@ -81,4 +89,4 @@ const calculateHash = ( path ) => {
   }
 }
 
-export { calculateHash, getArtifactsSchema, flattenArtifactsSchema, toImplicitPath, toExplicitPath }
+export { calculateHash, getArtifactsSchema, flattenArtifactsSchema, toImplicitPath, toExplicitPath, deduplicatePath, convertGenericArtifactNameToPath }

package/src/artifacts/pull-artifact.mjs

@@ -19,7 +19,7 @@ program
 const options = program.opts()
 
 async function main() {
-  const artifactsYmlPath = options.artifactsFilePath
+  const artifactsYmlPath = options.filePath
   const artifactPackageName = options.packageName || null
   const artifactPackageVersion = options.packageVersion || null
   

package/src/artifacts/push-artifact.mjs

@@ -19,7 +19,7 @@ program
 const options = program.opts()
 
 async function main() {
-  const artifactsYmlPath = options.artifactsFilePath
+  const artifactsYmlPath = options.filePath
   const artifactPackageName = options.packageName || null
   const artifactPackageVersion = options.packageVersion || null