@leverege/build-tools 2.94.0-quinn.9 → 2.95.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (97) hide show
  1. package/.vscode/launch.json +14 -0
  2. package/package.json +16 -8
  3. package/src/Docker.mjs +11 -4
  4. package/src/Utils.mjs +54 -7
  5. package/src/artifacts/ArtifactPuller.mjs +56 -102
  6. package/src/artifacts/ArtifactPusher.mjs +63 -38
  7. package/src/artifacts/Structs.mjs +32 -25
  8. package/src/artifacts/Utils.mjs +217 -28
  9. package/src/artifacts/pull-artifact.mjs +27 -20
  10. package/src/artifacts/push-artifact.mjs +25 -19
  11. package/src/bash-funcs +11 -2
  12. package/src/change-up.sh +27 -0
  13. package/src/chart-compass.mjs +22 -9
  14. package/src/chart-compass.yaml +23 -18
  15. package/src/git-tar.sh +1 -1
  16. package/src/helm-charts/emailer/values-local.yaml +1 -8
  17. package/src/helm-charts/prom-operator/helmup.plugin +9 -9
  18. package/src/helm-charts/traefik/helmup.plugin +1 -1
  19. package/src/helm-charts/velero/README.md +135 -0
  20. package/src/helm-charts/velero/helmup.bootstrap +1 -0
  21. package/src/helm-charts/velero/helmup.plugin +29 -2
  22. package/src/helm-charts/velero/values-local.yaml +1 -1
  23. package/src/helmup.sh +52 -41
  24. package/src/k8scale.sh +6 -2
  25. package/src/service-man/ServiceMan.mjs +75 -0
  26. package/src/service-man/config/CommandLine.mjs +199 -0
  27. package/src/service-man/config/Config.mjs +203 -0
  28. package/src/service-man/config/VersionCheck.mjs +79 -0
  29. package/src/service-man/config/iam-roles/leveregeBigQueryReader.json +10 -0
  30. package/src/service-man/config/iam-roles/leveregeBigQueryWriter.json +13 -0
  31. package/src/service-man/config/iam-roles/leveregeCloudFunctionsManager.json +12 -0
  32. package/src/service-man/config/iam-roles/leveregeFirebaseHosting.json +9 -0
  33. package/src/service-man/config/iam-roles/leveregeFirebaseMessaging.json +13 -0
  34. package/src/service-man/config/iam-roles/leveregeFirebaseUpdates.json +10 -0
  35. package/src/service-man/config/iam-roles/leveregeLogsViewer.json +8 -0
  36. package/src/service-man/config/iam-roles/leveregePubSub.json +13 -0
  37. package/src/service-man/config/iam-roles/leveregeSecretsViewer.json +9 -0
  38. package/src/service-man/config/iam-roles/leveregeStorageObjectAdmin.json +21 -0
  39. package/src/service-man/config/iam-roles/leveregeStorageObjectReader.json +13 -0
  40. package/src/service-man/config/iam-roles/leveregeSvcAcctActor.json +8 -0
  41. package/src/service-man/config/iam-roles/leveregeSvcAcctTokenCreator.json +8 -0
  42. package/src/service-man/config/index.mjs +7 -0
  43. package/src/service-man/config/secrets/authz-jwt.json +6 -0
  44. package/src/service-man/config/secrets/authz-postgres.json +7 -0
  45. package/src/service-man/config/secrets/db-timescale-dense.json +8 -0
  46. package/src/service-man/config/secrets/jwt-options.json +7 -0
  47. package/src/service-man/config/secrets/mail-gun-api-key.json +6 -0
  48. package/src/service-man/config/secrets/npmrc.json +6 -0
  49. package/src/service-man/config/secrets/oidc-client-secret.json +6 -0
  50. package/src/service-man/config/secrets/postgresql-password.json +8 -0
  51. package/src/service-man/config/secrets/session-secret.json +6 -0
  52. package/src/service-man/config/secrets/twilio.json +8 -0
  53. package/src/service-man/config/service-accounts/api-server.json +12 -0
  54. package/src/service-man/config/service-accounts/authz-server.json +5 -0
  55. package/src/service-man/config/service-accounts/db-curator.json +8 -0
  56. package/src/service-man/config/service-accounts/emailer.json +7 -0
  57. package/src/service-man/config/service-accounts/fota-server.json +6 -0
  58. package/src/service-man/config/service-accounts/geotile-server.json +8 -0
  59. package/src/service-man/config/service-accounts/imagine.json +6 -0
  60. package/src/service-man/config/service-accounts/message-processor.json +6 -0
  61. package/src/service-man/config/service-accounts/messenger.json +7 -0
  62. package/src/service-man/config/service-accounts/pubsub-pulse.json +6 -0
  63. package/src/service-man/config/service-accounts/push-notifier.json +8 -0
  64. package/src/service-man/config/service-accounts/reason.json +5 -0
  65. package/src/service-man/config/service-accounts/resource-server.json +9 -0
  66. package/src/service-man/config/service-accounts/rule-engine.json +7 -0
  67. package/src/service-man/config/service-accounts/scheduler.json +6 -0
  68. package/src/service-man/config/service-accounts/transponder.json +15 -0
  69. package/src/service-man/gcloud-k8s/RolesManager.mjs +41 -0
  70. package/src/service-man/gcloud-k8s/SecretsManager.mjs +182 -0
  71. package/src/service-man/gcloud-k8s/ServiceAccountManager.mjs +246 -0
  72. package/src/service-man/gcloud-k8s/VersionManager.mjs +114 -0
  73. package/src/service-man/gcloud-k8s/index.mjs +11 -0
  74. package/src/service-man/service-man.mjs +18 -0
  75. package/src/service-man/utils/index.mjs +83 -0
  76. package/src/yarn-build.mjs +60 -0
  77. package/leverege-build-tools-2.21.13.tgz +0 -0
  78. package/lib/server/build-tools.js +0 -138
  79. package/lib/server/firebaseDeploy.js +0 -195
  80. package/lib/server/firebaseServe.js +0 -116
  81. package/lib/server/getfbcfg.js +0 -25
  82. package/lib/server/getjson.js +0 -70
  83. package/lib/server/overwhelm.js +0 -447
  84. package/lib/server/push-my-chart.js +0 -18
  85. package/lib/server/refresh-npm-token.js +0 -135
  86. package/lib/server/tag-release.js +0 -681
  87. package/lib/server/unleash.js +0 -50
  88. package/lib/web/build-tools.js +0 -138
  89. package/lib/web/firebaseDeploy.js +0 -195
  90. package/lib/web/firebaseServe.js +0 -116
  91. package/lib/web/getfbcfg.js +0 -25
  92. package/lib/web/getjson.js +0 -70
  93. package/lib/web/overwhelm.js +0 -447
  94. package/lib/web/push-my-chart.js +0 -18
  95. package/lib/web/refresh-npm-token.js +0 -135
  96. package/lib/web/tag-release.js +0 -681
  97. package/lib/web/unleash.js +0 -50
@@ -17,6 +17,20 @@
17
17
  "env": {
18
18
  },
19
19
  "outputCapture": "std"
20
+ },
21
+ {
22
+ "name": "service-man (launch)",
23
+ "type": "node",
24
+ "request": "launch",
25
+ "program": "${workspaceFolder}/src/service-man/service-man.mjs",
26
+ "args": ["--dump-configs","dbg-cfgs"],
27
+ "cwd": "/Users/john/Work/leverege/k8s/tst-builder",
28
+ "runtimeExecutable": "node",
29
+ "console": "integratedTerminal",
30
+ "skipFiles": ["<node_internals>/**"],
31
+ "env": {
32
+ "NODE_ENV": "development"
33
+ }
20
34
  }
21
35
  ]
22
36
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@leverege/build-tools",
3
- "version": "2.94.0-quinn.9",
3
+ "version": "2.95.0",
4
4
  "description": "A collection of build / support tools for Leverege developers",
5
5
  "main": "index.js",
6
6
  "repository": {
@@ -11,13 +11,16 @@
11
11
  "clean": "rm -rf lib",
12
12
  "lint": "find ./src -name \\*.\\*js | xargs npx eslint --report-unused-disable-directives",
13
13
  "prepare": "[ $SKIP_PREPARE ] && exit 0 || ./src/hook-and-release.mjs",
14
- "test": "mocha -t 60000 --exit 'test/**/*.mjs'",
14
+ "test": "mocha -t 60000 --exit 'test/**/!(*e2e*).mjs'",
15
+ "test-all": "mocha -t 60000 --exit 'test/**/**.mjs'",
15
16
  "prepack": "npm exec prepack"
16
17
  },
17
18
  "bin": {
18
19
  "build-and-push": "src/build-and-push.sh",
19
20
  "build-cnpg-image": "src/build-cnpg-image.mjs",
20
21
  "build-tools": "src/build-tools.mjs",
22
+ "change-up": "src/change-up.sh",
23
+ "chart-compass": "src/chart-compass.mjs",
21
24
  "chart-to-museum": "src/chart-to-museum.mjs",
22
25
  "chart-to-registry": "src/chart-to-registry.mjs",
23
26
  "circleate": "src/circleate.sh",
@@ -37,6 +40,7 @@
37
40
  "getjson": "src/getjson.mjs",
38
41
  "git-tar": "src/git-tar.sh",
39
42
  "helm-audit": "src/helm-audit.mjs",
43
+ "helmboot": "src/helmup.sh",
40
44
  "helmcycle": "src/helmup.sh",
41
45
  "helmdn": "src/helmdn.sh",
42
46
  "helminit": "src/helmup.sh",
@@ -61,10 +65,12 @@
61
65
  "refresh-npm-token": "src/refresh-npm-token.mjs",
62
66
  "refresh-py-idx": "src/refresh-py-idx.sh",
63
67
  "send-to-slack": "src/send-to-slack.mjs",
68
+ "service-man": "src/service-man/service-man.mjs",
64
69
  "snapshot-cnpg": "src/snapshot-cnpg.mjs",
65
70
  "tag-release": "src/tag-release.mjs",
66
71
  "unleash": "src/unleash.mjs",
67
- "upload-model": "src/upload-model.sh"
72
+ "upload-model": "src/upload-model.sh",
73
+ "yarn-build": "src/yarn-build.mjs"
68
74
  },
69
75
  "author": "Leverege Devs",
70
76
  "license": "SEE LICENSE IN LICENSE.md",
@@ -76,11 +82,12 @@
76
82
  "commander": "^14.0.2",
77
83
  "deepmerge": "^4.3.1",
78
84
  "enquirer": "^2.4.1",
79
- "execa": "^9.6.0",
80
- "glob": "^11.0.3",
85
+ "execa": "^9.6.1",
86
+ "find-yarn-workspace-root": "^2.0.0",
87
+ "glob": "^13.0.0",
81
88
  "handlebars": "^4.7.8",
82
89
  "ignore": "^7.0.5",
83
- "inquirer": "^12.11.1",
90
+ "inquirer": "^13.1.0",
84
91
  "js-yaml": "^4.1.1",
85
92
  "jsdoc": "^4.0.5",
86
93
  "ms": "^2.1.3",
@@ -94,13 +101,14 @@
94
101
  "simple-git": "^3.30.0",
95
102
  "sloc": "^0.3.2",
96
103
  "superstruct": "^2.0.2",
104
+ "tmp": "^0.2.5",
97
105
  "toml": "^3.0.0",
98
106
  "zx": "^8.8.5"
99
107
  },
100
108
  "devDependencies": {
101
109
  "@leverege/eslint-config-leverege": "^5.1.1",
102
- "chai": "^6.2.1",
110
+ "chai": "^6.2.2",
103
111
  "mocha": "^11.7.5",
104
- "npm": "^11.6.2"
112
+ "npm": "^11.7.0"
105
113
  }
106
114
  }
package/src/Docker.mjs CHANGED
@@ -1,3 +1,4 @@
1
+ /* eslint-disable security/detect-non-literal-fs-filename */
1
2
  import fs from 'node:fs'
2
3
  import path from 'node:path'
3
4
  import url from 'node:url'
@@ -17,8 +18,14 @@ import {
17
18
 
18
19
  const DIRNAME = path.dirname( url.fileURLToPath( import.meta.url ) )
19
20
 
20
- // TODO: change default to node 22 (jod) after 01/01/25
21
- const dockerfileNodeVersion = 'jod-alpine'
21
+ // TODO: This is a temporary band-aid while we assess the impact of the yarn
22
+ // and/or CI pipeline transition. This will at least allow us to lock down the
23
+ // specific versions of node/npm and facilitate testing via the env vars.
24
+ //
25
+ // See the latest docker images here => https://hub.docker.com/_/node
26
+ //
27
+ const dockerfileNodeVersion = process.env.DOCKER_BUILD_NODE_VERSION || '24.12.0-alpine3.23'
28
+ const dockerfileNpmVersion = process.env.DOCKER_BUILD_NPM_VERSION || '11.6.0'
22
29
 
23
30
  // The contents of these variables were initially located in files that live in
24
31
  // the build-tools repository, but it just became simpler to pull the contents
@@ -55,7 +62,7 @@ const defaultSettings = {
55
62
  nodeimage : 'jod-alpine',
56
63
  imageBase : 'node:jod-alpine',
57
64
  npmlogging : process.env.VERBOSE_NPM_LOGGING ? '--ddd' : '--silent',
58
- npmVersion : 'npm@11',
65
+ npmVersion : `npm@${dockerfileNpmVersion}`,
59
66
  pluginfile : '# NO PLUGIN',
60
67
  preInstallPluginfile : '# NO PRE-INSTALL PLUGIN',
61
68
  regvers : 'package.version',
@@ -77,7 +84,7 @@ const generateDockerfile = async ( options ) => {
77
84
  const gitReference = await shellCmd( 'git rev-parse --short HEAD' )
78
85
  const settings = { gitReference, ...defaultSettings, ...options }
79
86
  const dockerTemplate = options.isNodeJsProject ? 'dockerfileTemplateNodeJs' : 'dockerfileTemplatePython'
80
- const dockerfileTemplate = fs.readFileSync( path.join( DIRNAME, `./templates/${dockerTemplate}.hbs` ), 'utf8' ) // eslint-disable-line security/detect-non-literal-fs-filename
87
+ const dockerfileTemplate = fs.readFileSync( path.join( DIRNAME, `./templates/${dockerTemplate}.hbs` ), 'utf8' )
81
88
 
82
89
  // defaults to the current "official" version of node unless it gets
83
90
  // overridden in the leverege.nodeimg package.json stanza
package/src/Utils.mjs CHANGED
@@ -39,11 +39,42 @@ export const warning = ( warningText ) => {
39
39
  console.error( `\n${chalk.yellow.bold( fullWarning )}` )
40
40
  }
41
41
 
42
- export const errorExit = ( error, opts = {} ) => {
43
- const { errorCode = 1 } = opts
44
- console.error( `\n${chalk.red.bold( error )}\n` )
45
- process.exit( errorCode )
42
+ export const errorExit = ( arg, opts = {} ) => {
43
+ const exitCode =
44
+ typeof opts.exitCode === 'number' ? opts.exitCode :
45
+ ( arg && typeof arg.exitCode === 'number' ) ? arg.exitCode : 1
46
+
47
+ // lets figure out what to print
48
+ let message
49
+ let details
50
+
51
+ if ( arg instanceof Error ) {
52
+ message = arg.message || 'Error'
53
+ details = arg.stack
54
+ } else if ( typeof arg === 'string' ) {
55
+ message = arg
56
+ } else if ( arg && typeof arg === 'object' ) {
57
+ message = 'Error'
58
+ details = JSON.stringify( arg, null, 2 )
59
+ } else if ( arg != null ) {
60
+ message = String( arg )
61
+ }
62
+
63
+ console.error( `\n${chalk.red.bold( message )}\n` )
64
+ if ( details && details !== message ) {
65
+ console.error( `${details}\n` )
66
+ }
67
+
68
+ process.exit( exitCode )
69
+ }
70
+
71
+ /**
72
+ * Clears the screen and optionally homes the cursor.
73
+ */
74
+ export const clear = ( home = true ) => {
75
+ log( `${home ? '\x1B[0f' : ''}\x1B[2J` )
46
76
  }
77
+
47
78
  /* eslint-enable no-console */
48
79
 
49
80
  export const isHex = ( hex ) => {
@@ -54,6 +85,22 @@ export const isString = ( s ) => {
54
85
  return typeof s === 'string' || s instanceof String
55
86
  }
56
87
 
88
+ /**
89
+ * * Sets first char to Upper
90
+ * */
91
+ export const capitalize = ( norm ) => {
92
+ return norm.charAt( 0 ).toUpperCase() + norm.slice( 1 )
93
+ }
94
+
95
+ /**
96
+ * A simple mS sleep delay.
97
+ */
98
+ export const sleep = ( ms = 0 ) => {
99
+ return new Promise( ( resolve ) => {
100
+ setTimeout( resolve, ms )
101
+ } )
102
+ }
103
+
57
104
  /**
58
105
  * General "yes" to proceed function.
59
106
  */
@@ -226,7 +273,7 @@ export const gitRepoIsDirty = async () => {
226
273
 
227
274
  export const parseYamlFile = async ( filePath ) => {
228
275
  try {
229
- const fileContent = await fs.promises.readFile( filePath, 'utf8' )
276
+ const fileContent = await fs.promises.readFile( filePath, 'utf8' ) /* eslint-disable-line security/detect-non-literal-fs-filename */
230
277
  return YAML.load( fileContent )
231
278
  } catch ( error ) {
232
279
  throw new Error( `Failed to parse YAML file ${filePath}: ${error.message}` )
@@ -493,7 +540,7 @@ export const parseHelmChart = async ( helmroot = './helm' ) => {
493
540
 
494
541
  // See if this is a leaf chart and whether or not dependencies need updating
495
542
  const hasDependencies = chartYaml.dependencies !== undefined
496
- let depsOutOfSync
543
+ let depsOutOfSync = hasDependencies // assume unsynced if dependencies
497
544
  if ( chartFiles.includes( 'Chart.lock' ) ) {
498
545
  const chartLock = YAML.load( fs.readFileSync( `${helmroot}/Chart.lock`, 'utf8' ) )
499
546
  // Using Object.fromEntries will be able to handle multiple dependencies if ever needed
@@ -538,7 +585,7 @@ export const getRepositoryDescr = async () => {
538
585
  const helmChart = await parseHelmChart()
539
586
  const rootPackageJson = await parseJsonFile( `${gitRoot}/package.json` )
540
587
  const rootPyProjectToml = await parsePyProjectToml( `${gitRoot}/pyproject.toml` )
541
- const isYarnProject = fs.existsSync( path.join( gitRoot, 'yarn.lock' ) ) || fs.existsSync( path.join( gitRoot, '.yarn' ) )
588
+ const isYarnProject = fs.existsSync( path.join( gitRoot, 'yarn.lock' ) ) || fs.existsSync( path.join( gitRoot, '.yarn' ) ) /* eslint-disable-line security/detect-non-literal-fs-filename */
542
589
 
543
590
  return {
544
591
  currentDir : process.cwd(),
@@ -1,12 +1,14 @@
1
1
  /* eslint-disable security/detect-non-literal-fs-filename */
2
2
  import fs from 'node:fs'
3
+ import path from 'node:path'
3
4
 
4
5
  import { shellCmd, log, debug, err } from '../Utils.mjs'
5
6
 
6
- import { toExplicitPath, deduplicatePath, convertGenericArtifactNameToPath } from './Utils.mjs'
7
+ import { parseGenericArtifactSchemaToRemoteRelativePath, parseGenericArtifactSchemaToLocalRelativePath, listGenericArtifactFiles, parseGenericArtifactFileToRemoteRelativePath, parseGenericArtifactSchemaToArtifactName } from './Utils.mjs'
7
8
 
8
9
  class ArtifactPuller {
9
- constructor( artifact ) {
10
+ constructor( path, artifact ) {
11
+ this.path = path
10
12
  this.artifact = artifact
11
13
  }
12
14
 
@@ -17,142 +19,101 @@ class ArtifactPuller {
17
19
 
18
20
  class GenericArtifactPuller extends ArtifactPuller {
19
21
 
20
- async prepareFileDownload( tmpFilePath, finalFilePath ) {
21
- // Ensure directories exist
22
- fs.mkdirSync( tmpFilePath.split( '/' ).slice( 0, -1 ).join( '/' ), { recursive : true } )
23
- fs.mkdirSync( finalFilePath.split( '/' ).slice( 0, -1 ).join( '/' ), { recursive : true } )
22
+ async downloadFile( remoteRelativePath, localRelativePath ) {
23
+ debug( { remoteRelativePath, localRelativePath }, '<==Download File' )
24
24
 
25
- // Ensure no conflicting files exist
26
- if ( fs.existsSync( tmpFilePath ) ) {
27
- fs.unlinkSync( tmpFilePath )
28
- }
29
- if ( fs.existsSync( finalFilePath ) ) {
30
- fs.unlinkSync( finalFilePath )
31
- }
32
- }
25
+ const isBasenameDifferent = path.basename( localRelativePath ) !== path.basename( remoteRelativePath )
33
26
 
34
- async cleanupFileDownload( tmpFilePath, finalFilePath ) {
35
- // Delete the tmp file
36
- if ( fs.existsSync( tmpFilePath ) ) {
37
- fs.unlinkSync( tmpFilePath )
27
+ let tmpLocalRelativePath
28
+ if ( isBasenameDifferent ) {
29
+ const artifactName = parseGenericArtifactSchemaToArtifactName( this.artifact )
30
+ const encodedArtifactName = artifactName.replaceAll( ':', '_' ).replaceAll( '/', '_' ).replaceAll( '%2F', '_' )
31
+ tmpLocalRelativePath = path.join( '/', 'tmp', encodedArtifactName )
32
+ } else {
33
+ await fs.promises.mkdir( path.dirname( localRelativePath ), { recursive : true } )
38
34
  }
39
- // Delete the tmp directory
40
- if ( fs.existsSync( tmpFilePath.split( '/' ).slice( 0, -1 ).join( '/' ) ) ) {
41
- fs.rmSync( tmpFilePath.split( '/' ).slice( 0, -1 ).join( '/' ), { recursive : true } )
42
- }
43
- }
44
-
45
- async downloadFile( remoteFilePath, localFilePath ) {
46
- debug( { remoteFilePath, localFilePath }, '<==Download File' )
47
-
48
- // 1 dir per file to avoid dir creation and deletion race conditions for files in the same directory
49
- const tmpDirectoryPath = `/tmp/${remoteFilePath.replaceAll( '/', '_' ).replaceAll( '.', '_' )}`
50
- const tmpFilePath = `${tmpDirectoryPath}/${remoteFilePath.split( '/' ).pop()}`
51
-
52
- await this.prepareFileDownload( tmpFilePath, localFilePath )
53
35
 
54
36
  try {
55
37
  const command = `gcloud artifacts generic download \
56
- --project=${this.artifact.gcp_project_id} \
57
- --location=${this.artifact.gcp_location} \
58
- --repository=${this.artifact.gcp_artifact_registry_repository_name} \
59
- --package=${this.artifact.gcp_artifact_registry_package_name} \
60
- --version=${this.artifact.gcp_artifact_registry_package_version} \
61
- --name="${remoteFilePath}" \
62
- --destination=${tmpDirectoryPath}`
38
+ --project=${this.artifact.project} \
39
+ --location=${this.artifact.location} \
40
+ --repository=${this.artifact.repository} \
41
+ --package=${this.artifact.package} \
42
+ --version=${this.artifact.version} \
43
+ --name="${remoteRelativePath}" \
44
+ --destination=${tmpLocalRelativePath || path.dirname( localRelativePath )}`
63
45
 
64
46
  debug( { command }, '<==Download File Command' )
65
47
 
66
48
  await shellCmd( command )
67
49
 
68
- // Check if file exists in the tmp path
69
- if ( !fs.existsSync( tmpFilePath ) ) {
70
- debug( { tmpFilePath }, '<==File does not exist in the tmp path' )
71
- return false
72
- }
73
-
74
- debug( { tmpFilePath, localFilePath }, '<==Copying file' )
75
-
76
- await fs.copyFileSync( tmpFilePath, localFilePath )
77
-
78
- // Check if file exists in the local path
79
- if ( !fs.existsSync( localFilePath ) ) {
80
- debug( { localFilePath }, '<==File does not exist in the local path' )
81
- return false
50
+ if ( isBasenameDifferent ) {
51
+ await fs.promises.rename( tmpLocalRelativePath, localRelativePath )
82
52
  }
83
53
 
84
54
  return true
85
55
  } catch ( error ) {
86
- err( `Error downloading file: ${remoteFilePath} ${error.message}` )
56
+ err( `Error downloading file: ${localRelativePath} ${error.message}` )
87
57
  return false
88
- } finally {
89
- await this.cleanupFileDownload( tmpFilePath, localFilePath )
90
58
  }
91
59
  }
92
60
 
93
- async downloadDirectory( localPath ) {
61
+ async downloadDirectory( remoteRelativePath, localRelativePath ) {
62
+
63
+ await fs.promises.mkdir( localRelativePath, { recursive : true } )
64
+
94
65
  const command = `gcloud artifacts generic download \
95
- --project=${this.artifact.gcp_project_id} \
96
- --location=${this.artifact.gcp_location} \
97
- --repository=${this.artifact.gcp_artifact_registry_repository_name} \
98
- --package=${this.artifact.gcp_artifact_registry_package_name} \
99
- --version=${this.artifact.gcp_artifact_registry_package_version} \
100
- --destination=${localPath}`
66
+ --project=${this.artifact.project} \
67
+ --location=${this.artifact.location} \
68
+ --repository=${this.artifact.repository} \
69
+ --package=${this.artifact.package} \
70
+ --version=${this.artifact.version} \
71
+ --destination=${localRelativePath}`
101
72
 
102
73
  debug( { command }, '<==Download Directory Command' )
103
74
 
104
75
  await shellCmd( command )
105
76
  }
106
77
 
107
- async downloadNestedDirectory( remotePath, localPath ) {
108
- debug( { remotePath, localPath }, '<==Download Nested Directory' )
78
+ async downloadNestedDirectory( remoteRelativePath, localRelativePath ) {
79
+ debug( { remoteRelativePath, localRelativePath }, '<==Download Nested Directory' )
109
80
 
110
- const files = await this.listFiles( remotePath )
81
+ const files = await listGenericArtifactFiles( this.artifact )
111
82
  await Promise.all(
112
83
  files
113
- .filter( file => convertGenericArtifactNameToPath( file.name ).startsWith( remotePath ) )
114
- .map( async ( file ) => {
115
- const remoteFilePath = convertGenericArtifactNameToPath( file.name )
116
- const localFilePath = deduplicatePath( `${localPath}/${remoteFilePath.replaceAll( remotePath, '' )}` )
117
- await this.downloadFile( remoteFilePath, localFilePath )
118
- } )
84
+ .reduce( ( acc, file ) => {
85
+ const remoteFilePath = parseGenericArtifactFileToRemoteRelativePath( file )
86
+ if ( remoteFilePath.startsWith( remoteRelativePath ) ) {
87
+ const remoteFilePathWithoutRemotePath = remoteFilePath.replace( remoteRelativePath, '' )
88
+ const localFilePath = path.join( localRelativePath, remoteFilePathWithoutRemotePath )
89
+ acc.push( this.downloadFile( remoteFilePath, localFilePath ) )
90
+ }
91
+ return acc
92
+ }, [] )
119
93
  )
120
94
  }
121
95
 
122
- async listFiles( ) {
123
- const command = `gcloud artifacts files list \
124
- --project=${this.artifact.gcp_project_id} \
125
- --location=${this.artifact.gcp_location} \
126
- --repository=${this.artifact.gcp_artifact_registry_repository_name} \
127
- --package=${this.artifact.gcp_artifact_registry_package_name} \
128
- --version ${this.artifact.gcp_artifact_registry_package_version} \
129
- --format="json"`
130
-
131
- debug( { command }, '<==List Files Command' )
132
-
133
- const result = await shellCmd( command )
134
-
135
- // Parse the JSON response
136
- const files = JSON.parse( result )
137
- return files
138
- }
139
-
140
96
  async pull() {
141
97
  log( `Pulling artifact: ${this.artifact.name}` )
142
98
 
143
99
  debug( { artifact : this.artifact }, '<==Pulling artifact' )
144
100
 
145
101
  try {
102
+ const remoteRelativePath = parseGenericArtifactSchemaToRemoteRelativePath( this.artifact )
103
+ const localRelativePath = parseGenericArtifactSchemaToLocalRelativePath( this.artifact )
104
+
146
105
  if ( this.artifact.remote_file_path && this.artifact.local_file_path ) {
147
- await this.downloadFile( this.artifact.remote_file_path, toExplicitPath( this.artifact.local_file_path ) )
106
+ await this.downloadFile( remoteRelativePath, path.join( this.path, localRelativePath ) )
148
107
  } else if ( this.artifact.remote_file_path && this.artifact.local_path ) {
149
- await this.downloadFile( this.artifact.remote_file_path, toExplicitPath( `${this.artifact.local_path}/${this.artifact.remote_file_path.split( '/' ).pop()}` ) )
108
+ await this.downloadFile( remoteRelativePath, path.join( this.path, localRelativePath ) )
109
+ } else if ( this.artifact.remote_file_path && !this.artifact.local_path && !this.artifact.local_file_path ) {
110
+ await this.downloadFile( remoteRelativePath, path.join( this.path, localRelativePath ) )
150
111
  } else if ( this.artifact.remote_path && this.artifact.local_path ) {
151
- await this.downloadNestedDirectory( this.artifact.remote_path, toExplicitPath( this.artifact.local_path ) )
112
+ await this.downloadNestedDirectory( remoteRelativePath, path.join( this.path, localRelativePath ) )
152
113
  } else if ( this.artifact.remote_path && !this.artifact.local_path && !this.artifact.local_file_path ) {
153
- await this.downloadDirectory( this.artifact.remote_path, '.' )
114
+ await this.downloadDirectory( remoteRelativePath, path.join( this.path, localRelativePath ) )
154
115
  } else if ( !this.artifact.remote_path && !this.artifact.remote_file_path && this.artifact.local_path ) {
155
- await this.downloadDirectory( toExplicitPath( this.artifact.local_path ) )
116
+ await this.downloadDirectory( remoteRelativePath, path.join( this.path, localRelativePath ) )
156
117
  } else {
157
118
  throw new Error( 'No valid paths provided' )
158
119
  }
@@ -160,13 +121,6 @@ class GenericArtifactPuller extends ArtifactPuller {
160
121
  log( `Successfully pulled artifact: ${this.artifact.name}` )
161
122
 
162
123
  return true
163
-
164
- // if ( !this.artifact.remote_path && !this.artifact.remote_file_path && this.local_file_path ) {
165
- // throw new Error( '' )
166
- // }
167
- // if ( this.artifact.remote_path && this.artifact.local_file_path ) {
168
- // throw new Error( '' )
169
- // }
170
124
  } catch ( error ) {
171
125
  err( `Error pulling artifact: ${this.artifact.name} ${error.message}` )
172
126
  }
@@ -1,9 +1,20 @@
1
+ import path from 'node:path'
2
+ import fs from 'node:fs'
3
+
1
4
  import { shellCmd, log, err, debug } from '../Utils.mjs'
2
5
 
3
- import { convertGenericArtifactNameToPath } from './Utils.mjs'
6
+ import {
7
+ parseGenericArtifactFileToRemoteRelativePath,
8
+ parseGenericArtifactSchemaToRemoteRelativePath,
9
+ listGenericArtifactFiles,
10
+ shouldUseIgnoreConfiguration,
11
+ findIgnoreFilePath,
12
+ copyFilesWithIgnoreConfiguration
13
+ } from './Utils.mjs'
4
14
 
5
15
  class ArtifactPusher {
6
- constructor( artifact ) {
16
+ constructor( path, artifact ) {
17
+ this.path = path
7
18
  this.artifact = artifact
8
19
  }
9
20
 
@@ -13,25 +24,6 @@ class ArtifactPusher {
13
24
  }
14
25
 
15
26
  class GenericArtifactPusher extends ArtifactPusher {
16
-
17
- async listFiles( ) {
18
- const command = `gcloud artifacts files list \
19
- --project=${this.artifact.gcp_project_id} \
20
- --location=${this.artifact.gcp_location} \
21
- --repository=${this.artifact.gcp_artifact_registry_repository_name} \
22
- --package=${this.artifact.gcp_artifact_registry_package_name} \
23
- --version ${this.artifact.gcp_artifact_registry_package_version} \
24
- --format="json"`
25
-
26
- debug( { command }, '<==List Files Command' )
27
-
28
- const result = await shellCmd( command )
29
-
30
- // Parse the JSON response
31
- const files = JSON.parse( result )
32
-
33
- return files
34
- }
35
27
 
36
28
  async push() {
37
29
  log( `Pushing artifact: ${this.artifact.name}` )
@@ -41,41 +33,74 @@ class GenericArtifactPusher extends ArtifactPusher {
41
33
  return false
42
34
  }
43
35
 
44
- try {
45
-
36
+ if ( !this.artifact.local_path && !this.artifact.local_file_path ) {
37
+ err( 'Local path or local file path is required for pushing artifacts' )
38
+ return false
39
+ }
40
+
41
+ try {
46
42
  let command = ''
47
43
 
48
44
  command = `gcloud artifacts generic upload \
49
- --project=${this.artifact.gcp_project_id} \
50
- --location=${this.artifact.gcp_location} \
51
- --repository=${this.artifact.gcp_artifact_registry_repository_name} \
52
- --package=${this.artifact.gcp_artifact_registry_package_name} \
53
- --version=${this.artifact.gcp_artifact_registry_package_version}`
45
+ --project=${this.artifact.project} \
46
+ --location=${this.artifact.location} \
47
+ --repository=${this.artifact.repository} \
48
+ --package=${this.artifact.package} \
49
+ --version=${this.artifact.version}`
50
+
51
+ let pathToDelete
54
52
 
55
53
  if ( this.artifact.local_path ) {
56
- command += ` --source-directory=${this.artifact.local_path}`
54
+ const originalLocalPath = path.join( this.path, this.artifact.local_path )
55
+
56
+ let localPath = originalLocalPath
57
+
58
+ const useIgnoreConfiguration = await shouldUseIgnoreConfiguration()
59
+
60
+ if ( useIgnoreConfiguration ) {
61
+ const ignoreFilePath = await findIgnoreFilePath( this.path )
62
+ if ( ignoreFilePath ) {
63
+ const tmpLocalPath = path.join( '/', 'tmp', `${this.artifact.project}_${this.artifact.location}_${this.artifact.repository}_${this.artifact.package}_${this.artifact.version}` )
64
+ localPath = tmpLocalPath
65
+ pathToDelete = tmpLocalPath
66
+ await copyFilesWithIgnoreConfiguration( originalLocalPath, tmpLocalPath, ignoreFilePath )
67
+ }
68
+ }
69
+
70
+ command += ` --source-directory=${localPath}`
57
71
  command += ' --skip-existing'
58
72
  } else if ( this.artifact.local_file_path ) {
59
- const files = await this.listFiles( this.artifact.remote_path )
60
- if ( files.some( file => convertGenericArtifactNameToPath( file.name ) === `${this.artifact.remote_path}/${this.artifact.remote_file_path.split( '/' ).pop()}` ) ) {
61
- err( `File already exists in the artifact registry: ${this.artifact.local_file_path}` )
62
- return false
73
+
74
+ const files = await listGenericArtifactFiles( this.artifact )
75
+
76
+ const newRemoteFilePath = parseGenericArtifactSchemaToRemoteRelativePath( this.artifact )
77
+
78
+ for ( const file of files ) {
79
+ const existingRemoteFilePath = parseGenericArtifactFileToRemoteRelativePath( file )
80
+ if ( existingRemoteFilePath === newRemoteFilePath ) {
81
+ err( `File already exists in the artifact registry: ${existingRemoteFilePath}` )
82
+ return false
83
+ }
63
84
  }
64
- command += ` --source=${this.artifact.local_file_path}`
85
+ const localPath = path.join( this.path, this.artifact.local_file_path )
86
+ command += ` --source=${localPath}`
65
87
  }
66
88
 
67
- if ( this.artifact.remote_path ) {
68
- command += ` --destination-path=${this.artifact.remote_path}`
69
- }
89
+ command += ` --destination-path=${this.artifact.remote_path || '""'}`
70
90
 
71
91
  debug( { command }, '<==Upload Command' )
72
92
 
73
93
  await shellCmd( command )
74
-
94
+
95
+ if ( pathToDelete ) {
96
+ await fs.promises.rm( pathToDelete, { recursive : true } )
97
+ }
98
+
75
99
  log( `Successfully pushed artifact: ${this.artifact.name}` )
76
100
 
77
101
  return true
78
102
  } catch ( error ) {
103
+ debug( { error }, '<==Error Pushing Artifact' )
79
104
  err( `Error pushing artifact: ${this.artifact.name} ${error.message}` )
80
105
  return false
81
106
  }