@leverege/build-tools 2.94.0-quinn.9 → 2.95.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.vscode/launch.json +14 -0
- package/package.json +16 -8
- package/src/Docker.mjs +11 -4
- package/src/Utils.mjs +54 -7
- package/src/artifacts/ArtifactPuller.mjs +56 -102
- package/src/artifacts/ArtifactPusher.mjs +63 -38
- package/src/artifacts/Structs.mjs +32 -25
- package/src/artifacts/Utils.mjs +217 -28
- package/src/artifacts/pull-artifact.mjs +27 -20
- package/src/artifacts/push-artifact.mjs +25 -19
- package/src/bash-funcs +11 -2
- package/src/change-up.sh +27 -0
- package/src/chart-compass.mjs +22 -9
- package/src/chart-compass.yaml +23 -18
- package/src/git-tar.sh +1 -1
- package/src/helm-charts/emailer/values-local.yaml +1 -8
- package/src/helm-charts/prom-operator/helmup.plugin +9 -9
- package/src/helm-charts/traefik/helmup.plugin +1 -1
- package/src/helm-charts/velero/README.md +135 -0
- package/src/helm-charts/velero/helmup.bootstrap +1 -0
- package/src/helm-charts/velero/helmup.plugin +29 -2
- package/src/helm-charts/velero/values-local.yaml +1 -1
- package/src/helmup.sh +52 -41
- package/src/k8scale.sh +6 -2
- package/src/service-man/ServiceMan.mjs +75 -0
- package/src/service-man/config/CommandLine.mjs +199 -0
- package/src/service-man/config/Config.mjs +203 -0
- package/src/service-man/config/VersionCheck.mjs +79 -0
- package/src/service-man/config/iam-roles/leveregeBigQueryReader.json +10 -0
- package/src/service-man/config/iam-roles/leveregeBigQueryWriter.json +13 -0
- package/src/service-man/config/iam-roles/leveregeCloudFunctionsManager.json +12 -0
- package/src/service-man/config/iam-roles/leveregeFirebaseHosting.json +9 -0
- package/src/service-man/config/iam-roles/leveregeFirebaseMessaging.json +13 -0
- package/src/service-man/config/iam-roles/leveregeFirebaseUpdates.json +10 -0
- package/src/service-man/config/iam-roles/leveregeLogsViewer.json +8 -0
- package/src/service-man/config/iam-roles/leveregePubSub.json +13 -0
- package/src/service-man/config/iam-roles/leveregeSecretsViewer.json +9 -0
- package/src/service-man/config/iam-roles/leveregeStorageObjectAdmin.json +21 -0
- package/src/service-man/config/iam-roles/leveregeStorageObjectReader.json +13 -0
- package/src/service-man/config/iam-roles/leveregeSvcAcctActor.json +8 -0
- package/src/service-man/config/iam-roles/leveregeSvcAcctTokenCreator.json +8 -0
- package/src/service-man/config/index.mjs +7 -0
- package/src/service-man/config/secrets/authz-jwt.json +6 -0
- package/src/service-man/config/secrets/authz-postgres.json +7 -0
- package/src/service-man/config/secrets/db-timescale-dense.json +8 -0
- package/src/service-man/config/secrets/jwt-options.json +7 -0
- package/src/service-man/config/secrets/mail-gun-api-key.json +6 -0
- package/src/service-man/config/secrets/npmrc.json +6 -0
- package/src/service-man/config/secrets/oidc-client-secret.json +6 -0
- package/src/service-man/config/secrets/postgresql-password.json +8 -0
- package/src/service-man/config/secrets/session-secret.json +6 -0
- package/src/service-man/config/secrets/twilio.json +8 -0
- package/src/service-man/config/service-accounts/api-server.json +12 -0
- package/src/service-man/config/service-accounts/authz-server.json +5 -0
- package/src/service-man/config/service-accounts/db-curator.json +8 -0
- package/src/service-man/config/service-accounts/emailer.json +7 -0
- package/src/service-man/config/service-accounts/fota-server.json +6 -0
- package/src/service-man/config/service-accounts/geotile-server.json +8 -0
- package/src/service-man/config/service-accounts/imagine.json +6 -0
- package/src/service-man/config/service-accounts/message-processor.json +6 -0
- package/src/service-man/config/service-accounts/messenger.json +7 -0
- package/src/service-man/config/service-accounts/pubsub-pulse.json +6 -0
- package/src/service-man/config/service-accounts/push-notifier.json +8 -0
- package/src/service-man/config/service-accounts/reason.json +5 -0
- package/src/service-man/config/service-accounts/resource-server.json +9 -0
- package/src/service-man/config/service-accounts/rule-engine.json +7 -0
- package/src/service-man/config/service-accounts/scheduler.json +6 -0
- package/src/service-man/config/service-accounts/transponder.json +15 -0
- package/src/service-man/gcloud-k8s/RolesManager.mjs +41 -0
- package/src/service-man/gcloud-k8s/SecretsManager.mjs +182 -0
- package/src/service-man/gcloud-k8s/ServiceAccountManager.mjs +246 -0
- package/src/service-man/gcloud-k8s/VersionManager.mjs +114 -0
- package/src/service-man/gcloud-k8s/index.mjs +11 -0
- package/src/service-man/service-man.mjs +18 -0
- package/src/service-man/utils/index.mjs +83 -0
- package/src/yarn-build.mjs +60 -0
- package/leverege-build-tools-2.21.13.tgz +0 -0
- package/lib/server/build-tools.js +0 -138
- package/lib/server/firebaseDeploy.js +0 -195
- package/lib/server/firebaseServe.js +0 -116
- package/lib/server/getfbcfg.js +0 -25
- package/lib/server/getjson.js +0 -70
- package/lib/server/overwhelm.js +0 -447
- package/lib/server/push-my-chart.js +0 -18
- package/lib/server/refresh-npm-token.js +0 -135
- package/lib/server/tag-release.js +0 -681
- package/lib/server/unleash.js +0 -50
- package/lib/web/build-tools.js +0 -138
- package/lib/web/firebaseDeploy.js +0 -195
- package/lib/web/firebaseServe.js +0 -116
- package/lib/web/getfbcfg.js +0 -25
- package/lib/web/getjson.js +0 -70
- package/lib/web/overwhelm.js +0 -447
- package/lib/web/push-my-chart.js +0 -18
- package/lib/web/refresh-npm-token.js +0 -135
- package/lib/web/tag-release.js +0 -681
- package/lib/web/unleash.js +0 -50
package/.vscode/launch.json
CHANGED
|
@@ -17,6 +17,20 @@
|
|
|
17
17
|
"env": {
|
|
18
18
|
},
|
|
19
19
|
"outputCapture": "std"
|
|
20
|
+
},
|
|
21
|
+
{
|
|
22
|
+
"name": "service-man (launch)",
|
|
23
|
+
"type": "node",
|
|
24
|
+
"request": "launch",
|
|
25
|
+
"program": "${workspaceFolder}/src/service-man/service-man.mjs",
|
|
26
|
+
"args": ["--dump-configs","dbg-cfgs"],
|
|
27
|
+
"cwd": "/Users/john/Work/leverege/k8s/tst-builder",
|
|
28
|
+
"runtimeExecutable": "node",
|
|
29
|
+
"console": "integratedTerminal",
|
|
30
|
+
"skipFiles": ["<node_internals>/**"],
|
|
31
|
+
"env": {
|
|
32
|
+
"NODE_ENV": "development"
|
|
33
|
+
}
|
|
20
34
|
}
|
|
21
35
|
]
|
|
22
36
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@leverege/build-tools",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.95.0",
|
|
4
4
|
"description": "A collection of build / support tools for Leverege developers",
|
|
5
5
|
"main": "index.js",
|
|
6
6
|
"repository": {
|
|
@@ -11,13 +11,16 @@
|
|
|
11
11
|
"clean": "rm -rf lib",
|
|
12
12
|
"lint": "find ./src -name \\*.\\*js | xargs npx eslint --report-unused-disable-directives",
|
|
13
13
|
"prepare": "[ $SKIP_PREPARE ] && exit 0 || ./src/hook-and-release.mjs",
|
|
14
|
-
"test": "mocha -t 60000 --exit 'test
|
|
14
|
+
"test": "mocha -t 60000 --exit 'test/**/!(*e2e*).mjs'",
|
|
15
|
+
"test-all": "mocha -t 60000 --exit 'test/**/**.mjs'",
|
|
15
16
|
"prepack": "npm exec prepack"
|
|
16
17
|
},
|
|
17
18
|
"bin": {
|
|
18
19
|
"build-and-push": "src/build-and-push.sh",
|
|
19
20
|
"build-cnpg-image": "src/build-cnpg-image.mjs",
|
|
20
21
|
"build-tools": "src/build-tools.mjs",
|
|
22
|
+
"change-up": "src/change-up.sh",
|
|
23
|
+
"chart-compass": "src/chart-compass.mjs",
|
|
21
24
|
"chart-to-museum": "src/chart-to-museum.mjs",
|
|
22
25
|
"chart-to-registry": "src/chart-to-registry.mjs",
|
|
23
26
|
"circleate": "src/circleate.sh",
|
|
@@ -37,6 +40,7 @@
|
|
|
37
40
|
"getjson": "src/getjson.mjs",
|
|
38
41
|
"git-tar": "src/git-tar.sh",
|
|
39
42
|
"helm-audit": "src/helm-audit.mjs",
|
|
43
|
+
"helmboot": "src/helmup.sh",
|
|
40
44
|
"helmcycle": "src/helmup.sh",
|
|
41
45
|
"helmdn": "src/helmdn.sh",
|
|
42
46
|
"helminit": "src/helmup.sh",
|
|
@@ -61,10 +65,12 @@
|
|
|
61
65
|
"refresh-npm-token": "src/refresh-npm-token.mjs",
|
|
62
66
|
"refresh-py-idx": "src/refresh-py-idx.sh",
|
|
63
67
|
"send-to-slack": "src/send-to-slack.mjs",
|
|
68
|
+
"service-man": "src/service-man/service-man.mjs",
|
|
64
69
|
"snapshot-cnpg": "src/snapshot-cnpg.mjs",
|
|
65
70
|
"tag-release": "src/tag-release.mjs",
|
|
66
71
|
"unleash": "src/unleash.mjs",
|
|
67
|
-
"upload-model": "src/upload-model.sh"
|
|
72
|
+
"upload-model": "src/upload-model.sh",
|
|
73
|
+
"yarn-build": "src/yarn-build.mjs"
|
|
68
74
|
},
|
|
69
75
|
"author": "Leverege Devs",
|
|
70
76
|
"license": "SEE LICENSE IN LICENSE.md",
|
|
@@ -76,11 +82,12 @@
|
|
|
76
82
|
"commander": "^14.0.2",
|
|
77
83
|
"deepmerge": "^4.3.1",
|
|
78
84
|
"enquirer": "^2.4.1",
|
|
79
|
-
"execa": "^9.6.
|
|
80
|
-
"
|
|
85
|
+
"execa": "^9.6.1",
|
|
86
|
+
"find-yarn-workspace-root": "^2.0.0",
|
|
87
|
+
"glob": "^13.0.0",
|
|
81
88
|
"handlebars": "^4.7.8",
|
|
82
89
|
"ignore": "^7.0.5",
|
|
83
|
-
"inquirer": "^
|
|
90
|
+
"inquirer": "^13.1.0",
|
|
84
91
|
"js-yaml": "^4.1.1",
|
|
85
92
|
"jsdoc": "^4.0.5",
|
|
86
93
|
"ms": "^2.1.3",
|
|
@@ -94,13 +101,14 @@
|
|
|
94
101
|
"simple-git": "^3.30.0",
|
|
95
102
|
"sloc": "^0.3.2",
|
|
96
103
|
"superstruct": "^2.0.2",
|
|
104
|
+
"tmp": "^0.2.5",
|
|
97
105
|
"toml": "^3.0.0",
|
|
98
106
|
"zx": "^8.8.5"
|
|
99
107
|
},
|
|
100
108
|
"devDependencies": {
|
|
101
109
|
"@leverege/eslint-config-leverege": "^5.1.1",
|
|
102
|
-
"chai": "^6.2.
|
|
110
|
+
"chai": "^6.2.2",
|
|
103
111
|
"mocha": "^11.7.5",
|
|
104
|
-
"npm": "^11.
|
|
112
|
+
"npm": "^11.7.0"
|
|
105
113
|
}
|
|
106
114
|
}
|
package/src/Docker.mjs
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
/* eslint-disable security/detect-non-literal-fs-filename */
|
|
1
2
|
import fs from 'node:fs'
|
|
2
3
|
import path from 'node:path'
|
|
3
4
|
import url from 'node:url'
|
|
@@ -17,8 +18,14 @@ import {
|
|
|
17
18
|
|
|
18
19
|
const DIRNAME = path.dirname( url.fileURLToPath( import.meta.url ) )
|
|
19
20
|
|
|
20
|
-
// TODO:
|
|
21
|
-
|
|
21
|
+
// TODO: This is a temporary band-aid while we assess the impact of the yarn
|
|
22
|
+
// and/or CI pipeline transition. This will at least allow us to lock down the
|
|
23
|
+
// specific versions of node/npm and facilitate testing via the env vars.
|
|
24
|
+
//
|
|
25
|
+
// See the latest docker images here => https://hub.docker.com/_/node
|
|
26
|
+
//
|
|
27
|
+
const dockerfileNodeVersion = process.env.DOCKER_BUILD_NODE_VERSION || '24.12.0-alpine3.23'
|
|
28
|
+
const dockerfileNpmVersion = process.env.DOCKER_BUILD_NPM_VERSION || '11.6.0'
|
|
22
29
|
|
|
23
30
|
// The contents of these variables were initially located in files that live in
|
|
24
31
|
// the build-tools repository, but it just became simpler to pull the contents
|
|
@@ -55,7 +62,7 @@ const defaultSettings = {
|
|
|
55
62
|
nodeimage : 'jod-alpine',
|
|
56
63
|
imageBase : 'node:jod-alpine',
|
|
57
64
|
npmlogging : process.env.VERBOSE_NPM_LOGGING ? '--ddd' : '--silent',
|
|
58
|
-
npmVersion :
|
|
65
|
+
npmVersion : `npm@${dockerfileNpmVersion}`,
|
|
59
66
|
pluginfile : '# NO PLUGIN',
|
|
60
67
|
preInstallPluginfile : '# NO PRE-INSTALL PLUGIN',
|
|
61
68
|
regvers : 'package.version',
|
|
@@ -77,7 +84,7 @@ const generateDockerfile = async ( options ) => {
|
|
|
77
84
|
const gitReference = await shellCmd( 'git rev-parse --short HEAD' )
|
|
78
85
|
const settings = { gitReference, ...defaultSettings, ...options }
|
|
79
86
|
const dockerTemplate = options.isNodeJsProject ? 'dockerfileTemplateNodeJs' : 'dockerfileTemplatePython'
|
|
80
|
-
const dockerfileTemplate = fs.readFileSync( path.join( DIRNAME, `./templates/${dockerTemplate}.hbs` ), 'utf8' )
|
|
87
|
+
const dockerfileTemplate = fs.readFileSync( path.join( DIRNAME, `./templates/${dockerTemplate}.hbs` ), 'utf8' )
|
|
81
88
|
|
|
82
89
|
// defaults to the current "official" version of node unless it gets
|
|
83
90
|
// overridden in the leverege.nodeimg package.json stanza
|
package/src/Utils.mjs
CHANGED
|
@@ -39,11 +39,42 @@ export const warning = ( warningText ) => {
|
|
|
39
39
|
console.error( `\n${chalk.yellow.bold( fullWarning )}` )
|
|
40
40
|
}
|
|
41
41
|
|
|
42
|
-
export const errorExit = (
|
|
43
|
-
const
|
|
44
|
-
|
|
45
|
-
|
|
42
|
+
export const errorExit = ( arg, opts = {} ) => {
|
|
43
|
+
const exitCode =
|
|
44
|
+
typeof opts.exitCode === 'number' ? opts.exitCode :
|
|
45
|
+
( arg && typeof arg.exitCode === 'number' ) ? arg.exitCode : 1
|
|
46
|
+
|
|
47
|
+
// lets figure out what to print
|
|
48
|
+
let message
|
|
49
|
+
let details
|
|
50
|
+
|
|
51
|
+
if ( arg instanceof Error ) {
|
|
52
|
+
message = arg.message || 'Error'
|
|
53
|
+
details = arg.stack
|
|
54
|
+
} else if ( typeof arg === 'string' ) {
|
|
55
|
+
message = arg
|
|
56
|
+
} else if ( arg && typeof arg === 'object' ) {
|
|
57
|
+
message = 'Error'
|
|
58
|
+
details = JSON.stringify( arg, null, 2 )
|
|
59
|
+
} else if ( arg != null ) {
|
|
60
|
+
message = String( arg )
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
console.error( `\n${chalk.red.bold( message )}\n` )
|
|
64
|
+
if ( details && details !== message ) {
|
|
65
|
+
console.error( `${details}\n` )
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
process.exit( exitCode )
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* Clears the screen and optionally homes the cursor.
|
|
73
|
+
*/
|
|
74
|
+
export const clear = ( home = true ) => {
|
|
75
|
+
log( `${home ? '\x1B[0f' : ''}\x1B[2J` )
|
|
46
76
|
}
|
|
77
|
+
|
|
47
78
|
/* eslint-enable no-console */
|
|
48
79
|
|
|
49
80
|
export const isHex = ( hex ) => {
|
|
@@ -54,6 +85,22 @@ export const isString = ( s ) => {
|
|
|
54
85
|
return typeof s === 'string' || s instanceof String
|
|
55
86
|
}
|
|
56
87
|
|
|
88
|
+
/**
|
|
89
|
+
* * Sets first char to Upper
|
|
90
|
+
* */
|
|
91
|
+
export const capitalize = ( norm ) => {
|
|
92
|
+
return norm.charAt( 0 ).toUpperCase() + norm.slice( 1 )
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/**
|
|
96
|
+
* A simple mS sleep delay.
|
|
97
|
+
*/
|
|
98
|
+
export const sleep = ( ms = 0 ) => {
|
|
99
|
+
return new Promise( ( resolve ) => {
|
|
100
|
+
setTimeout( resolve, ms )
|
|
101
|
+
} )
|
|
102
|
+
}
|
|
103
|
+
|
|
57
104
|
/**
|
|
58
105
|
* General "yes" to proceed function.
|
|
59
106
|
*/
|
|
@@ -226,7 +273,7 @@ export const gitRepoIsDirty = async () => {
|
|
|
226
273
|
|
|
227
274
|
export const parseYamlFile = async ( filePath ) => {
|
|
228
275
|
try {
|
|
229
|
-
const fileContent = await fs.promises.readFile( filePath, 'utf8' )
|
|
276
|
+
const fileContent = await fs.promises.readFile( filePath, 'utf8' ) /* eslint-disable-line security/detect-non-literal-fs-filename */
|
|
230
277
|
return YAML.load( fileContent )
|
|
231
278
|
} catch ( error ) {
|
|
232
279
|
throw new Error( `Failed to parse YAML file ${filePath}: ${error.message}` )
|
|
@@ -493,7 +540,7 @@ export const parseHelmChart = async ( helmroot = './helm' ) => {
|
|
|
493
540
|
|
|
494
541
|
// See if this is a leaf chart and whether or not dependencies need updating
|
|
495
542
|
const hasDependencies = chartYaml.dependencies !== undefined
|
|
496
|
-
let depsOutOfSync
|
|
543
|
+
let depsOutOfSync = hasDependencies // assume unsynced if dependencies
|
|
497
544
|
if ( chartFiles.includes( 'Chart.lock' ) ) {
|
|
498
545
|
const chartLock = YAML.load( fs.readFileSync( `${helmroot}/Chart.lock`, 'utf8' ) )
|
|
499
546
|
// Using Object.fromEntries will be able to handle multiple dependencies if ever needed
|
|
@@ -538,7 +585,7 @@ export const getRepositoryDescr = async () => {
|
|
|
538
585
|
const helmChart = await parseHelmChart()
|
|
539
586
|
const rootPackageJson = await parseJsonFile( `${gitRoot}/package.json` )
|
|
540
587
|
const rootPyProjectToml = await parsePyProjectToml( `${gitRoot}/pyproject.toml` )
|
|
541
|
-
const isYarnProject = fs.existsSync( path.join( gitRoot, 'yarn.lock' ) ) || fs.existsSync( path.join( gitRoot, '.yarn' ) )
|
|
588
|
+
const isYarnProject = fs.existsSync( path.join( gitRoot, 'yarn.lock' ) ) || fs.existsSync( path.join( gitRoot, '.yarn' ) ) /* eslint-disable-line security/detect-non-literal-fs-filename */
|
|
542
589
|
|
|
543
590
|
return {
|
|
544
591
|
currentDir : process.cwd(),
|
|
@@ -1,12 +1,14 @@
|
|
|
1
1
|
/* eslint-disable security/detect-non-literal-fs-filename */
|
|
2
2
|
import fs from 'node:fs'
|
|
3
|
+
import path from 'node:path'
|
|
3
4
|
|
|
4
5
|
import { shellCmd, log, debug, err } from '../Utils.mjs'
|
|
5
6
|
|
|
6
|
-
import {
|
|
7
|
+
import { parseGenericArtifactSchemaToRemoteRelativePath, parseGenericArtifactSchemaToLocalRelativePath, listGenericArtifactFiles, parseGenericArtifactFileToRemoteRelativePath, parseGenericArtifactSchemaToArtifactName } from './Utils.mjs'
|
|
7
8
|
|
|
8
9
|
class ArtifactPuller {
|
|
9
|
-
constructor( artifact ) {
|
|
10
|
+
constructor( path, artifact ) {
|
|
11
|
+
this.path = path
|
|
10
12
|
this.artifact = artifact
|
|
11
13
|
}
|
|
12
14
|
|
|
@@ -17,142 +19,101 @@ class ArtifactPuller {
|
|
|
17
19
|
|
|
18
20
|
class GenericArtifactPuller extends ArtifactPuller {
|
|
19
21
|
|
|
20
|
-
async
|
|
21
|
-
|
|
22
|
-
fs.mkdirSync( tmpFilePath.split( '/' ).slice( 0, -1 ).join( '/' ), { recursive : true } )
|
|
23
|
-
fs.mkdirSync( finalFilePath.split( '/' ).slice( 0, -1 ).join( '/' ), { recursive : true } )
|
|
22
|
+
async downloadFile( remoteRelativePath, localRelativePath ) {
|
|
23
|
+
debug( { remoteRelativePath, localRelativePath }, '<==Download File' )
|
|
24
24
|
|
|
25
|
-
|
|
26
|
-
if ( fs.existsSync( tmpFilePath ) ) {
|
|
27
|
-
fs.unlinkSync( tmpFilePath )
|
|
28
|
-
}
|
|
29
|
-
if ( fs.existsSync( finalFilePath ) ) {
|
|
30
|
-
fs.unlinkSync( finalFilePath )
|
|
31
|
-
}
|
|
32
|
-
}
|
|
25
|
+
const isBasenameDifferent = path.basename( localRelativePath ) !== path.basename( remoteRelativePath )
|
|
33
26
|
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
27
|
+
let tmpLocalRelativePath
|
|
28
|
+
if ( isBasenameDifferent ) {
|
|
29
|
+
const artifactName = parseGenericArtifactSchemaToArtifactName( this.artifact )
|
|
30
|
+
const encodedArtifactName = artifactName.replaceAll( ':', '_' ).replaceAll( '/', '_' ).replaceAll( '%2F', '_' )
|
|
31
|
+
tmpLocalRelativePath = path.join( '/', 'tmp', encodedArtifactName )
|
|
32
|
+
} else {
|
|
33
|
+
await fs.promises.mkdir( path.dirname( localRelativePath ), { recursive : true } )
|
|
38
34
|
}
|
|
39
|
-
// Delete the tmp directory
|
|
40
|
-
if ( fs.existsSync( tmpFilePath.split( '/' ).slice( 0, -1 ).join( '/' ) ) ) {
|
|
41
|
-
fs.rmSync( tmpFilePath.split( '/' ).slice( 0, -1 ).join( '/' ), { recursive : true } )
|
|
42
|
-
}
|
|
43
|
-
}
|
|
44
|
-
|
|
45
|
-
async downloadFile( remoteFilePath, localFilePath ) {
|
|
46
|
-
debug( { remoteFilePath, localFilePath }, '<==Download File' )
|
|
47
|
-
|
|
48
|
-
// 1 dir per file to avoid dir creation and deletion race conditions for files in the same directory
|
|
49
|
-
const tmpDirectoryPath = `/tmp/${remoteFilePath.replaceAll( '/', '_' ).replaceAll( '.', '_' )}`
|
|
50
|
-
const tmpFilePath = `${tmpDirectoryPath}/${remoteFilePath.split( '/' ).pop()}`
|
|
51
|
-
|
|
52
|
-
await this.prepareFileDownload( tmpFilePath, localFilePath )
|
|
53
35
|
|
|
54
36
|
try {
|
|
55
37
|
const command = `gcloud artifacts generic download \
|
|
56
|
-
--project=${this.artifact.
|
|
57
|
-
--location=${this.artifact.
|
|
58
|
-
--repository=${this.artifact.
|
|
59
|
-
--package=${this.artifact.
|
|
60
|
-
--version=${this.artifact.
|
|
61
|
-
--name="${
|
|
62
|
-
--destination=${
|
|
38
|
+
--project=${this.artifact.project} \
|
|
39
|
+
--location=${this.artifact.location} \
|
|
40
|
+
--repository=${this.artifact.repository} \
|
|
41
|
+
--package=${this.artifact.package} \
|
|
42
|
+
--version=${this.artifact.version} \
|
|
43
|
+
--name="${remoteRelativePath}" \
|
|
44
|
+
--destination=${tmpLocalRelativePath || path.dirname( localRelativePath )}`
|
|
63
45
|
|
|
64
46
|
debug( { command }, '<==Download File Command' )
|
|
65
47
|
|
|
66
48
|
await shellCmd( command )
|
|
67
49
|
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
debug( { tmpFilePath }, '<==File does not exist in the tmp path' )
|
|
71
|
-
return false
|
|
72
|
-
}
|
|
73
|
-
|
|
74
|
-
debug( { tmpFilePath, localFilePath }, '<==Copying file' )
|
|
75
|
-
|
|
76
|
-
await fs.copyFileSync( tmpFilePath, localFilePath )
|
|
77
|
-
|
|
78
|
-
// Check if file exists in the local path
|
|
79
|
-
if ( !fs.existsSync( localFilePath ) ) {
|
|
80
|
-
debug( { localFilePath }, '<==File does not exist in the local path' )
|
|
81
|
-
return false
|
|
50
|
+
if ( isBasenameDifferent ) {
|
|
51
|
+
await fs.promises.rename( tmpLocalRelativePath, localRelativePath )
|
|
82
52
|
}
|
|
83
53
|
|
|
84
54
|
return true
|
|
85
55
|
} catch ( error ) {
|
|
86
|
-
err( `Error downloading file: ${
|
|
56
|
+
err( `Error downloading file: ${localRelativePath} ${error.message}` )
|
|
87
57
|
return false
|
|
88
|
-
} finally {
|
|
89
|
-
await this.cleanupFileDownload( tmpFilePath, localFilePath )
|
|
90
58
|
}
|
|
91
59
|
}
|
|
92
60
|
|
|
93
|
-
async downloadDirectory(
|
|
61
|
+
async downloadDirectory( remoteRelativePath, localRelativePath ) {
|
|
62
|
+
|
|
63
|
+
await fs.promises.mkdir( localRelativePath, { recursive : true } )
|
|
64
|
+
|
|
94
65
|
const command = `gcloud artifacts generic download \
|
|
95
|
-
--project=${this.artifact.
|
|
96
|
-
--location=${this.artifact.
|
|
97
|
-
--repository=${this.artifact.
|
|
98
|
-
--package=${this.artifact.
|
|
99
|
-
--version=${this.artifact.
|
|
100
|
-
--destination=${
|
|
66
|
+
--project=${this.artifact.project} \
|
|
67
|
+
--location=${this.artifact.location} \
|
|
68
|
+
--repository=${this.artifact.repository} \
|
|
69
|
+
--package=${this.artifact.package} \
|
|
70
|
+
--version=${this.artifact.version} \
|
|
71
|
+
--destination=${localRelativePath}`
|
|
101
72
|
|
|
102
73
|
debug( { command }, '<==Download Directory Command' )
|
|
103
74
|
|
|
104
75
|
await shellCmd( command )
|
|
105
76
|
}
|
|
106
77
|
|
|
107
|
-
async downloadNestedDirectory(
|
|
108
|
-
debug( {
|
|
78
|
+
async downloadNestedDirectory( remoteRelativePath, localRelativePath ) {
|
|
79
|
+
debug( { remoteRelativePath, localRelativePath }, '<==Download Nested Directory' )
|
|
109
80
|
|
|
110
|
-
const files = await this.
|
|
81
|
+
const files = await listGenericArtifactFiles( this.artifact )
|
|
111
82
|
await Promise.all(
|
|
112
83
|
files
|
|
113
|
-
.
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
84
|
+
.reduce( ( acc, file ) => {
|
|
85
|
+
const remoteFilePath = parseGenericArtifactFileToRemoteRelativePath( file )
|
|
86
|
+
if ( remoteFilePath.startsWith( remoteRelativePath ) ) {
|
|
87
|
+
const remoteFilePathWithoutRemotePath = remoteFilePath.replace( remoteRelativePath, '' )
|
|
88
|
+
const localFilePath = path.join( localRelativePath, remoteFilePathWithoutRemotePath )
|
|
89
|
+
acc.push( this.downloadFile( remoteFilePath, localFilePath ) )
|
|
90
|
+
}
|
|
91
|
+
return acc
|
|
92
|
+
}, [] )
|
|
119
93
|
)
|
|
120
94
|
}
|
|
121
95
|
|
|
122
|
-
async listFiles( ) {
|
|
123
|
-
const command = `gcloud artifacts files list \
|
|
124
|
-
--project=${this.artifact.gcp_project_id} \
|
|
125
|
-
--location=${this.artifact.gcp_location} \
|
|
126
|
-
--repository=${this.artifact.gcp_artifact_registry_repository_name} \
|
|
127
|
-
--package=${this.artifact.gcp_artifact_registry_package_name} \
|
|
128
|
-
--version ${this.artifact.gcp_artifact_registry_package_version} \
|
|
129
|
-
--format="json"`
|
|
130
|
-
|
|
131
|
-
debug( { command }, '<==List Files Command' )
|
|
132
|
-
|
|
133
|
-
const result = await shellCmd( command )
|
|
134
|
-
|
|
135
|
-
// Parse the JSON response
|
|
136
|
-
const files = JSON.parse( result )
|
|
137
|
-
return files
|
|
138
|
-
}
|
|
139
|
-
|
|
140
96
|
async pull() {
|
|
141
97
|
log( `Pulling artifact: ${this.artifact.name}` )
|
|
142
98
|
|
|
143
99
|
debug( { artifact : this.artifact }, '<==Pulling artifact' )
|
|
144
100
|
|
|
145
101
|
try {
|
|
102
|
+
const remoteRelativePath = parseGenericArtifactSchemaToRemoteRelativePath( this.artifact )
|
|
103
|
+
const localRelativePath = parseGenericArtifactSchemaToLocalRelativePath( this.artifact )
|
|
104
|
+
|
|
146
105
|
if ( this.artifact.remote_file_path && this.artifact.local_file_path ) {
|
|
147
|
-
await this.downloadFile(
|
|
106
|
+
await this.downloadFile( remoteRelativePath, path.join( this.path, localRelativePath ) )
|
|
148
107
|
} else if ( this.artifact.remote_file_path && this.artifact.local_path ) {
|
|
149
|
-
await this.downloadFile(
|
|
108
|
+
await this.downloadFile( remoteRelativePath, path.join( this.path, localRelativePath ) )
|
|
109
|
+
} else if ( this.artifact.remote_file_path && !this.artifact.local_path && !this.artifact.local_file_path ) {
|
|
110
|
+
await this.downloadFile( remoteRelativePath, path.join( this.path, localRelativePath ) )
|
|
150
111
|
} else if ( this.artifact.remote_path && this.artifact.local_path ) {
|
|
151
|
-
await this.downloadNestedDirectory(
|
|
112
|
+
await this.downloadNestedDirectory( remoteRelativePath, path.join( this.path, localRelativePath ) )
|
|
152
113
|
} else if ( this.artifact.remote_path && !this.artifact.local_path && !this.artifact.local_file_path ) {
|
|
153
|
-
await this.downloadDirectory( this.
|
|
114
|
+
await this.downloadDirectory( remoteRelativePath, path.join( this.path, localRelativePath ) )
|
|
154
115
|
} else if ( !this.artifact.remote_path && !this.artifact.remote_file_path && this.artifact.local_path ) {
|
|
155
|
-
await this.downloadDirectory(
|
|
116
|
+
await this.downloadDirectory( remoteRelativePath, path.join( this.path, localRelativePath ) )
|
|
156
117
|
} else {
|
|
157
118
|
throw new Error( 'No valid paths provided' )
|
|
158
119
|
}
|
|
@@ -160,13 +121,6 @@ class GenericArtifactPuller extends ArtifactPuller {
|
|
|
160
121
|
log( `Successfully pulled artifact: ${this.artifact.name}` )
|
|
161
122
|
|
|
162
123
|
return true
|
|
163
|
-
|
|
164
|
-
// if ( !this.artifact.remote_path && !this.artifact.remote_file_path && this.local_file_path ) {
|
|
165
|
-
// throw new Error( '' )
|
|
166
|
-
// }
|
|
167
|
-
// if ( this.artifact.remote_path && this.artifact.local_file_path ) {
|
|
168
|
-
// throw new Error( '' )
|
|
169
|
-
// }
|
|
170
124
|
} catch ( error ) {
|
|
171
125
|
err( `Error pulling artifact: ${this.artifact.name} ${error.message}` )
|
|
172
126
|
}
|
|
@@ -1,9 +1,20 @@
|
|
|
1
|
+
import path from 'node:path'
|
|
2
|
+
import fs from 'node:fs'
|
|
3
|
+
|
|
1
4
|
import { shellCmd, log, err, debug } from '../Utils.mjs'
|
|
2
5
|
|
|
3
|
-
import {
|
|
6
|
+
import {
|
|
7
|
+
parseGenericArtifactFileToRemoteRelativePath,
|
|
8
|
+
parseGenericArtifactSchemaToRemoteRelativePath,
|
|
9
|
+
listGenericArtifactFiles,
|
|
10
|
+
shouldUseIgnoreConfiguration,
|
|
11
|
+
findIgnoreFilePath,
|
|
12
|
+
copyFilesWithIgnoreConfiguration
|
|
13
|
+
} from './Utils.mjs'
|
|
4
14
|
|
|
5
15
|
class ArtifactPusher {
|
|
6
|
-
constructor( artifact ) {
|
|
16
|
+
constructor( path, artifact ) {
|
|
17
|
+
this.path = path
|
|
7
18
|
this.artifact = artifact
|
|
8
19
|
}
|
|
9
20
|
|
|
@@ -13,25 +24,6 @@ class ArtifactPusher {
|
|
|
13
24
|
}
|
|
14
25
|
|
|
15
26
|
class GenericArtifactPusher extends ArtifactPusher {
|
|
16
|
-
|
|
17
|
-
async listFiles( ) {
|
|
18
|
-
const command = `gcloud artifacts files list \
|
|
19
|
-
--project=${this.artifact.gcp_project_id} \
|
|
20
|
-
--location=${this.artifact.gcp_location} \
|
|
21
|
-
--repository=${this.artifact.gcp_artifact_registry_repository_name} \
|
|
22
|
-
--package=${this.artifact.gcp_artifact_registry_package_name} \
|
|
23
|
-
--version ${this.artifact.gcp_artifact_registry_package_version} \
|
|
24
|
-
--format="json"`
|
|
25
|
-
|
|
26
|
-
debug( { command }, '<==List Files Command' )
|
|
27
|
-
|
|
28
|
-
const result = await shellCmd( command )
|
|
29
|
-
|
|
30
|
-
// Parse the JSON response
|
|
31
|
-
const files = JSON.parse( result )
|
|
32
|
-
|
|
33
|
-
return files
|
|
34
|
-
}
|
|
35
27
|
|
|
36
28
|
async push() {
|
|
37
29
|
log( `Pushing artifact: ${this.artifact.name}` )
|
|
@@ -41,41 +33,74 @@ class GenericArtifactPusher extends ArtifactPusher {
|
|
|
41
33
|
return false
|
|
42
34
|
}
|
|
43
35
|
|
|
44
|
-
|
|
45
|
-
|
|
36
|
+
if ( !this.artifact.local_path && !this.artifact.local_file_path ) {
|
|
37
|
+
err( 'Local path or local file path is required for pushing artifacts' )
|
|
38
|
+
return false
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
try {
|
|
46
42
|
let command = ''
|
|
47
43
|
|
|
48
44
|
command = `gcloud artifacts generic upload \
|
|
49
|
-
--project=${this.artifact.
|
|
50
|
-
--location=${this.artifact.
|
|
51
|
-
--repository=${this.artifact.
|
|
52
|
-
--package=${this.artifact.
|
|
53
|
-
--version=${this.artifact.
|
|
45
|
+
--project=${this.artifact.project} \
|
|
46
|
+
--location=${this.artifact.location} \
|
|
47
|
+
--repository=${this.artifact.repository} \
|
|
48
|
+
--package=${this.artifact.package} \
|
|
49
|
+
--version=${this.artifact.version}`
|
|
50
|
+
|
|
51
|
+
let pathToDelete
|
|
54
52
|
|
|
55
53
|
if ( this.artifact.local_path ) {
|
|
56
|
-
|
|
54
|
+
const originalLocalPath = path.join( this.path, this.artifact.local_path )
|
|
55
|
+
|
|
56
|
+
let localPath = originalLocalPath
|
|
57
|
+
|
|
58
|
+
const useIgnoreConfiguration = await shouldUseIgnoreConfiguration()
|
|
59
|
+
|
|
60
|
+
if ( useIgnoreConfiguration ) {
|
|
61
|
+
const ignoreFilePath = await findIgnoreFilePath( this.path )
|
|
62
|
+
if ( ignoreFilePath ) {
|
|
63
|
+
const tmpLocalPath = path.join( '/', 'tmp', `${this.artifact.project}_${this.artifact.location}_${this.artifact.repository}_${this.artifact.package}_${this.artifact.version}` )
|
|
64
|
+
localPath = tmpLocalPath
|
|
65
|
+
pathToDelete = tmpLocalPath
|
|
66
|
+
await copyFilesWithIgnoreConfiguration( originalLocalPath, tmpLocalPath, ignoreFilePath )
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
command += ` --source-directory=${localPath}`
|
|
57
71
|
command += ' --skip-existing'
|
|
58
72
|
} else if ( this.artifact.local_file_path ) {
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
73
|
+
|
|
74
|
+
const files = await listGenericArtifactFiles( this.artifact )
|
|
75
|
+
|
|
76
|
+
const newRemoteFilePath = parseGenericArtifactSchemaToRemoteRelativePath( this.artifact )
|
|
77
|
+
|
|
78
|
+
for ( const file of files ) {
|
|
79
|
+
const existingRemoteFilePath = parseGenericArtifactFileToRemoteRelativePath( file )
|
|
80
|
+
if ( existingRemoteFilePath === newRemoteFilePath ) {
|
|
81
|
+
err( `File already exists in the artifact registry: ${existingRemoteFilePath}` )
|
|
82
|
+
return false
|
|
83
|
+
}
|
|
63
84
|
}
|
|
64
|
-
|
|
85
|
+
const localPath = path.join( this.path, this.artifact.local_file_path )
|
|
86
|
+
command += ` --source=${localPath}`
|
|
65
87
|
}
|
|
66
88
|
|
|
67
|
-
|
|
68
|
-
command += ` --destination-path=${this.artifact.remote_path}`
|
|
69
|
-
}
|
|
89
|
+
command += ` --destination-path=${this.artifact.remote_path || '""'}`
|
|
70
90
|
|
|
71
91
|
debug( { command }, '<==Upload Command' )
|
|
72
92
|
|
|
73
93
|
await shellCmd( command )
|
|
74
|
-
|
|
94
|
+
|
|
95
|
+
if ( pathToDelete ) {
|
|
96
|
+
await fs.promises.rm( pathToDelete, { recursive : true } )
|
|
97
|
+
}
|
|
98
|
+
|
|
75
99
|
log( `Successfully pushed artifact: ${this.artifact.name}` )
|
|
76
100
|
|
|
77
101
|
return true
|
|
78
102
|
} catch ( error ) {
|
|
103
|
+
debug( { error }, '<==Error Pushing Artifact' )
|
|
79
104
|
err( `Error pushing artifact: ${this.artifact.name} ${error.message}` )
|
|
80
105
|
return false
|
|
81
106
|
}
|