@leverege/build-tools 2.93.3-beta.2 → 2.93.3-beta.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@leverege/build-tools",
-  "version": "2.93.3-beta.2",
+  "version": "2.93.3-beta.4",
   "description": "A collection of build / support tools for Leverege developers",
   "main": "index.js",
   "repository": {

package/src/artifacts/ArtifactPusher.mjs

@@ -1,11 +1,15 @@
 import path from 'node:path'
+import fs from 'node:fs'
 
 import { shellCmd, log, err, debug } from '../Utils.mjs'
 
 import {
   parseGenericArtifactFileToRemoteRelativePath,
   parseGenericArtifactSchemaToRemoteRelativePath,
-  listGenericArtifactFiles
+  listGenericArtifactFiles,
+  shouldUseIgnoreConfiguration,
+  findIgnoreFilePath,
+  copyFilesWithIgnoreConfiguration
 } from './Utils.mjs'
 
 class ArtifactPusher {
@@ -29,8 +33,12 @@ class GenericArtifactPusher extends ArtifactPusher {
       return false
     }
 
-    try {
-   
+    if ( !this.artifact.local_path && !this.artifact.local_file_path ) {
+      err( 'Local path or local file path is required for pushing artifacts' )
+      return false
+    }
+
+    try {   
       let command = ''
 
       command = `gcloud artifacts generic upload \
@@ -40,8 +48,25 @@ class GenericArtifactPusher extends ArtifactPusher {
       --package=${this.artifact.package} \
       --version=${this.artifact.version}`
 
+      let pathToDelete
+
       if ( this.artifact.local_path ) {
-        const localPath = path.join( this.path, this.artifact.local_path )
+        const originalLocalPath = path.join( this.path, this.artifact.local_path )
+
+        let localPath = originalLocalPath
+
+        const useIgnoreConfiguration = await shouldUseIgnoreConfiguration()
+
+        if ( useIgnoreConfiguration ) {
+          const ignoreFilePath = await findIgnoreFilePath( this.path )
+          if ( ignoreFilePath ) {
+            const tmpLocalPath = path.join( '/', 'tmp', `${this.artifact.project}_${this.artifact.location}_${this.artifact.repository}_${this.artifact.package}_${this.artifact.version}` )
+            localPath = tmpLocalPath
+            pathToDelete = tmpLocalPath
+            await copyFilesWithIgnoreConfiguration( originalLocalPath, tmpLocalPath, ignoreFilePath )
+          }
+        }
+
         command += ` --source-directory=${localPath}`
         command += ' --skip-existing'
       } else if ( this.artifact.local_file_path ) {
@@ -66,7 +91,11 @@ class GenericArtifactPusher extends ArtifactPusher {
       debug( { command }, '<==Upload Command' )
         
       await shellCmd( command )
-      
+
+      if ( pathToDelete ) {
+        await fs.promises.rm( pathToDelete, { recursive : true } )
+      }
+
       log( `Successfully pushed artifact: ${this.artifact.name}` )
 
       return true

package/src/artifacts/Utils.mjs

@@ -163,7 +163,6 @@ const calculateHash = async ( path ) => {
     hash.update( await fs.promises.readFile( path ) )
     return `${hash.digest( 'base64url' )}=`
   } catch ( error ) {
-    console.error( error )
     throw new Error( `Failed to calculate hash for artifact ${path}: ${error.message}` )
   }
 }
@@ -186,7 +185,7 @@ const listGenericArtifactFiles = async ( artifact ) => {
   return files
 }
 
-const checkGenericArtifactMatch = async ( genericArtifactFiles, genericArtifactSchemaPath, genericArtifactSchema ) => {
+const getMissingRemoteFiles = async ( genericArtifactFiles, genericArtifactSchemaPath, genericArtifactSchema ) => {
   
   const expectedRemoteRelativePath = parseGenericArtifactSchemaToRemoteRelativePath( genericArtifactSchema )
   const localRelativePath = parseGenericArtifactSchemaToLocalRelativePath( genericArtifactSchema )
@@ -221,8 +220,47 @@ const checkGenericArtifactMatch = async ( genericArtifactFiles, genericArtifactS
   const actualArray = Array.from( actualFiles )
   const expectedArray = Array.from( expectedFiles )
 
-  return actualArray.length === expectedArray.length && 
-    expectedArray.every( expected => actualArray.some( actual => actual.remoteFilePath === expected.remoteFilePath && actual.remoteFileHash === expected.remoteFileHash ) )
+  return expectedArray.filter( expected => !actualArray.some( actual => actual.remoteFilePath === expected.remoteFilePath && actual.remoteFileHash === expected.remoteFileHash ) )
+}
+
+const shouldUseIgnoreConfiguration = async () => {
+  let result
+  try {
+    result = await shellCmd( 'gcloud config get-value gcloudignore/enabled' )
+    if ( result.trim() === '' || result.includes( 'has no property' ) ) {
+      return false
+    }
+    return result.trim() === 'true'
+  } catch ( error ) {
+    if ( error.message && error.message.includes( 'Section [gcloudignore] has no property [enabled]' ) ) {
+      return false
+    }
+    // Re-throw other errors
+    throw error
+  }
+}
+
+const findIgnoreFilePath = async ( relativePath ) => {
+  if ( relativePath ) {
+    const ignoreFilePath = path.join( relativePath, '.gcloudignore' )
+    if ( fs.existsSync( ignoreFilePath ) ) {
+      return ignoreFilePath
+    }
+  }
+
+  const ignoreFilePath = path.join( process.cwd(), '.gcloudignore' )
+  if ( fs.existsSync( ignoreFilePath ) ) {
+    return ignoreFilePath
+  }
+
+  return null
+}
+
+const copyFilesWithIgnoreConfiguration = async ( sourcePath, destinationPath, ignoreFilePath ) => {
+  // Ensure sourcePath ends with / so rsync copies contents (not the folder itself) and exclude patterns work correctly
+  const normalizedSourcePath = sourcePath.endsWith( '/' ) ? sourcePath : `${sourcePath}/`
+  const command = `rsync -av --delete --exclude-from='${ignoreFilePath}' ${normalizedSourcePath} ${destinationPath}`
+  await shellCmd( command )
 }
 
 export {
@@ -232,7 +270,12 @@ export {
   parseGenericArtifactFileToRemoteRelativePath,
   parseGenericArtifactSchemaToRemoteRelativePath,
   parseGenericArtifactSchemaToLocalRelativePath,
+  parseGenericArtifactSchemaToArtifactNameBase,
+  parseGenericArtifactSchemaToArtifactNameFile,
   parseGenericArtifactSchemaToArtifactName,
-  checkGenericArtifactMatch,
-  listGenericArtifactFiles
+  getMissingRemoteFiles,
+  listGenericArtifactFiles,
+  shouldUseIgnoreConfiguration,
+  findIgnoreFilePath,
+  copyFilesWithIgnoreConfiguration
 }

package/src/artifacts/pull-artifact.mjs

@@ -13,12 +13,8 @@ import { GenericArtifactPuller } from './ArtifactPuller.mjs'
 program
   .name( 'pull-artifact' )
   .description( 'Pull artifacts from GCP Artifact Registry' )
-  .requiredOption( '-f, --file-path <path>', 'Path to artifacts.yaml file' )
-  .requiredOption( '-p, --project <project>', 'GCP project id' )
-  .requiredOption( '-l, --location <location>', 'GCP location' )
-  .requiredOption( '-r, --repository <repository>', 'GCP artifact registry repository' )
-  .requiredOption( '-n  --package <name>', 'GCP artifact registry package name' )
-  .requiredOption( '-v, --version <version>', 'GCP artifact registry package version' )
+  .option( '-f, --file <path>', 'Path to artifacts.yaml file', './artifacts.yaml' )
+  .option( '-a, --artifact <artifact-path>', 'GCP artifact path' )
   .parse()
 
 const options = program.opts()
@@ -26,13 +22,26 @@ const options = program.opts()
 async function main() {
   const artifactsSchema = await getArtifactsSchema( options.filePath )
   const flattenedArtifactsSchema = flattenArtifactsSchema( artifactsSchema )
-  
-  const filteredArtifactsSchema = flattenedArtifactsSchema.filter(
-    artifact => artifact.project === options.project &&
-    artifact.location === options.location &&
-    artifact.repository === options.repository &&
-    artifact.package === options.package &&
-    artifact.version === options.version )
+
+  let filteredArtifactsSchema = flattenedArtifactsSchema
+
+  if ( options.artifact ) {
+    const artifactPathElements = options.artifact.split( '/' )
+
+    if ( artifactPathElements.length !== 5 ) {
+      warning( 'Invalid artifact path, it must be in the format <project>/<location>/<repository>/<package>/<version>' )
+      return false
+    }
+
+    const [ project, location, repository, packageName, version ] = artifactPathElements
+
+    filteredArtifactsSchema = flattenedArtifactsSchema.filter(
+      artifact => artifact.project === project &&
+      artifact.location === location &&
+      artifact.repository === repository &&
+      artifact.package === packageName &&
+      artifact.version === version )
+  }
 
   let counter = 0
 
@@ -65,6 +74,8 @@ async function main() {
   counter = pullResults.filter( Boolean ).length
 
   log( `Successfully pulled ${counter}/${filteredArtifactsSchema.length} artifacts` )
+
+  return counter === filteredArtifactsSchema.length
 }
 
 main()

package/src/artifacts/push-artifact.mjs

@@ -13,12 +13,8 @@ import { GenericArtifactPusher } from './ArtifactPusher.mjs'
 program
   .name( 'push-artifact' )
   .description( 'Push artifacts to GCP Artifact Registry' )
-  .requiredOption( '-f, --file-path <path>', 'Path to artifacts.yaml file' )
-  .requiredOption( '-p, --project <project>', 'GCP project id' )
-  .requiredOption( '-l, --location <location>', 'GCP location' )
-  .requiredOption( '-r, --repository <repository>', 'GCP artifact registry repository' )
-  .requiredOption( '-n  --package <name>', 'GCP artifact registry package name' )
-  .requiredOption( '-v, --version <version>', 'GCP artifact registry package version' )
+  .option( '-f, --file <path>', 'Path to artifacts.yaml file', './artifacts.yaml' )
+  .option( '-a, --artifact <artifact-path>', 'GCP artifact path' )
   .parse()
 
 const options = program.opts()
@@ -26,13 +22,25 @@ const options = program.opts()
 async function main() {
   const artifactsSchema = await getArtifactsSchema( options.filePath )
   const flattenedArtifactsSchema = flattenArtifactsSchema( artifactsSchema )
-  
-  const filteredArtifactsSchema = flattenedArtifactsSchema.filter(
-    artifact => artifact.project === options.project &&
-    artifact.location === options.location &&
-    artifact.repository === options.repository &&
-    artifact.package === options.package &&
-    artifact.version === options.version )
+
+  let filteredArtifactsSchema = flattenedArtifactsSchema
+
+  if ( options.artifact ) {
+    const artifactPathElements = options.artifact.split( '/' )
+    if ( artifactPathElements.length !== 5 ) {
+      warning( 'Invalid artifact path, it must be in the format <project>/<location>/<repository>/<package>/<version>' )
+      return false
+    }
+
+    const [ project, location, repository, packageName, version ] = artifactPathElements
+
+    filteredArtifactsSchema = flattenedArtifactsSchema.filter(
+      artifact => artifact.project === project &&
+      artifact.location === location &&
+      artifact.repository === repository &&
+      artifact.package === packageName &&
+      artifact.version === version )
+  }
 
   let counter = 0
 
@@ -65,6 +73,8 @@ async function main() {
   counter = pushResults.filter( Boolean ).length
 
   log( `Successfully pushed ${counter}/${filteredArtifactsSchema.length} artifacts` )
+  
+  return counter === filteredArtifactsSchema.length
 }
 
 main()