@catladder/cli 0.0.0-main-c595e7af → 0.0.0-main-2f2f483e
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundles/catenv/index.js +70 -55
- package/dist/bundles/cli/index.js +70 -55
- package/dist/pipeline/src/deploy/cloudRun/createJobs/cloudRunJobs.js +70 -55
- package/dist/pipeline/src/deploy/cloudRun/createJobs/cloudRunJobs.js.map +1 -1
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +1 -1
|
@@ -3736,13 +3736,13 @@ exports.getDeleteUnusedRevisionsCommands = getDeleteUnusedRevisionsCommands;
|
|
|
3736
3736
|
|
|
3737
3737
|
Object.defineProperty(exports, "__esModule", ({ value: true }));
|
|
3738
3738
|
exports.getCreateScheduleScripts = exports.getJobCreateScripts = exports.getJobRunScripts = exports.getDeleteJobsScripts = exports.getDeleteSchedulesScripts = void 0;
|
|
3739
|
-
const gitlab_1 = __nccwpck_require__(8490);
|
|
3740
3739
|
const createArgsString_1 = __nccwpck_require__(9598);
|
|
3741
3740
|
const jobName_1 = __nccwpck_require__(9626);
|
|
3742
3741
|
const common_1 = __nccwpck_require__(3325);
|
|
3743
3742
|
const getLabels_1 = __nccwpck_require__(920);
|
|
3744
3743
|
const volumes_1 = __nccwpck_require__(847);
|
|
3745
3744
|
const constants_1 = __nccwpck_require__(8865);
|
|
3745
|
+
const utils_1 = __nccwpck_require__(2497);
|
|
3746
3746
|
const getJobRunScriptForJob = (context, jobName, wait) => {
|
|
3747
3747
|
const commonArgs = (0, common_1.getCommonCloudRunArgs)(context);
|
|
3748
3748
|
const commonArgsString = (0, createArgsString_1.createArgsString)(commonArgs);
|
|
@@ -3784,76 +3784,90 @@ const getJobRunScripts = (context, when) => {
|
|
|
3784
3784
|
["preStop", "postStop"].includes(when)));
|
|
3785
3785
|
};
|
|
3786
3786
|
exports.getJobRunScripts = getJobRunScripts;
|
|
3787
|
-
const getJobCreateScripts = (context) => {
|
|
3788
|
-
const
|
|
3789
|
-
|
|
3790
|
-
.
|
|
3791
|
-
|
|
3792
|
-
};
|
|
3793
|
-
exports.getJobCreateScripts = getJobCreateScripts;
|
|
3794
|
-
const getJobCreateScriptsForJob = (context, jobName, job) => {
|
|
3795
|
-
const commonDeployArgs = (0, common_1.getCommonDeployArgs)(context);
|
|
3796
|
-
// due to some oversight from google, jobs create does not yet accept `--add-volume` 🤦
|
|
3797
|
-
// lucky, update on the other hand accepts it... so let's just imediatly update it
|
|
3798
|
-
// we cannot upsert a job, so we have to create it and catch the error and then update
|
|
3799
|
-
const commandArray = Array.isArray(job.command)
|
|
3800
|
-
? job.command
|
|
3801
|
-
: job.command.split(" ");
|
|
3787
|
+
const getJobCreateScripts = (context) => getCloudRunJobsWithNames(context).map(({ job: { command, image, cpu, memory = "512Mi", timeout = "10m", parallelism = 1, volumes, }, jobName, }, jobIndex) => {
|
|
3788
|
+
const commandArray = Array.isArray(command)
|
|
3789
|
+
? command
|
|
3790
|
+
: command.split(" ");
|
|
3791
|
+
const { image: commonImage, project, region, ...deployArgs } = (0, common_1.getCommonDeployArgs)(context);
|
|
3802
3792
|
const commonDeployArgsString = (0, createArgsString_1.createArgsString)({
|
|
3803
|
-
command:
|
|
3804
|
-
|
|
3805
|
-
|
|
3806
|
-
|
|
3807
|
-
|
|
3808
|
-
|
|
3809
|
-
|
|
3810
|
-
|
|
3811
|
-
|
|
3812
|
-
"task-timeout": job.timeout || "10m",
|
|
3813
|
-
parallelism: job.parallelism || 1,
|
|
3793
|
+
command: `"${commandArray.join(",")}"`,
|
|
3794
|
+
labels: `"${(0, common_1.makeLabelString)((0, getLabels_1.getLabels)(context))},cloud-run-job-name=$current_job_name"`,
|
|
3795
|
+
image: `"${image !== null && image !== void 0 ? image : commonImage}"`,
|
|
3796
|
+
project,
|
|
3797
|
+
region,
|
|
3798
|
+
cpu,
|
|
3799
|
+
memory,
|
|
3800
|
+
parallelism,
|
|
3801
|
+
"task-timeout": timeout,
|
|
3814
3802
|
"env-vars-file": constants_1.ENV_VARS_FILENAME,
|
|
3815
3803
|
"max-retries": 0,
|
|
3804
|
+
...deployArgs,
|
|
3816
3805
|
});
|
|
3817
|
-
|
|
3818
|
-
|
|
3806
|
+
// due to some oversight from google, jobs create does not yet accept `--add-volume` 🤦
|
|
3807
|
+
// lucky, update on the other hand accepts it... so let's just imediatly update it
|
|
3808
|
+
// we cannot upsert a job, so we have to create it and catch the error and then update
|
|
3809
|
+
const hasVolumes = Object.keys(volumes || {}).length > 0;
|
|
3810
|
+
const needsBeta = hasVolumes ? "beta" : undefined;
|
|
3811
|
+
const volumeArgs = hasVolumes
|
|
3812
|
+
? (0, createArgsString_1.createArgsString)(...(0, volumes_1.createVolumeConfig)(volumes, "job"))
|
|
3813
|
+
: "";
|
|
3819
3814
|
return [
|
|
3820
|
-
|
|
3821
|
-
|
|
3822
|
-
|
|
3823
|
-
}
|
|
3815
|
+
jobIndex === 0
|
|
3816
|
+
? `exist_job_names="$(\n ${(0, common_1.gcloudRunCmd)()} jobs list --filter='metadata.name ~ ${context.env}.*${context.name}' --format='value(name)' --limit=999 --project='${project}' --region='${region}'\n)"`
|
|
3817
|
+
: null,
|
|
3818
|
+
`current_job_name="${jobName}"`,
|
|
3819
|
+
'if grep "$current_job_name" <<<"$exist_job_names" >/dev/null; then',
|
|
3820
|
+
` ${(0, common_1.gcloudRunCmd)(needsBeta)} jobs update "$current_job_name" ${commonDeployArgsString} ${volumeArgs}`,
|
|
3821
|
+
"else",
|
|
3822
|
+
` ${(0, common_1.gcloudRunCmd)()} jobs create "$current_job_name" ${commonDeployArgsString}`,
|
|
3823
|
+
"fi",
|
|
3824
|
+
]
|
|
3825
|
+
.filter(utils_1.notNil)
|
|
3826
|
+
.join("\n");
|
|
3827
|
+
});
|
|
3828
|
+
exports.getJobCreateScripts = getJobCreateScripts;
|
|
3824
3829
|
const getCreateScheduleScripts = (context) => {
|
|
3825
3830
|
const jobsWithSchedule = getCloudRunJobsWithSchedule(context);
|
|
3826
|
-
const
|
|
3827
|
-
|
|
3828
|
-
|
|
3829
|
-
|
|
3831
|
+
const { region: location, projectId: project } = (0, common_1.getCloudRunDeployConfig)(context);
|
|
3832
|
+
const uriBase = `https://${location}-run.googleapis.com/apis/run.googleapis.com/v1/namespaces/${project}/jobs`;
|
|
3833
|
+
const gcloudArgs = {
|
|
3834
|
+
project,
|
|
3835
|
+
location,
|
|
3836
|
+
uri: `"$current_job_uri"`,
|
|
3837
|
+
"http-method": "POST",
|
|
3838
|
+
"oauth-service-account-email": `"$GCLOUD_PROJECT_NUMBER-compute@developer.gserviceaccount.com"`,
|
|
3839
|
+
};
|
|
3840
|
+
return jobsWithSchedule.map(({ job: { maxRetryAttempts, schedule }, jobName, schedulerName }, jobIndex) => {
|
|
3830
3841
|
const argsString = (0, createArgsString_1.createArgsString)({
|
|
3831
|
-
|
|
3832
|
-
|
|
3833
|
-
|
|
3834
|
-
"max-retry-attempts": (_a = job.maxRetryAttempts) !== null && _a !== void 0 ? _a : 0,
|
|
3835
|
-
uri: `"https://${deployConfig.region}-run.googleapis.com/apis/run.googleapis.com/v1/namespaces/${deployConfig.projectId}/jobs/${jobName}:run"`,
|
|
3836
|
-
"http-method": "POST",
|
|
3837
|
-
"oauth-service-account-email": "$GCLOUD_PROJECT_NUMBER-compute@developer.gserviceaccount.com",
|
|
3842
|
+
...gcloudArgs,
|
|
3843
|
+
schedule: `"${schedule}"`,
|
|
3844
|
+
"max-retry-attempts": maxRetryAttempts !== null && maxRetryAttempts !== void 0 ? maxRetryAttempts : 0,
|
|
3838
3845
|
});
|
|
3839
|
-
const commonArgs = `http ${schedulerName} ${argsString}`;
|
|
3840
3846
|
return [
|
|
3841
|
-
|
|
3842
|
-
|
|
3843
|
-
|
|
3844
|
-
|
|
3845
|
-
|
|
3846
|
-
|
|
3847
|
-
|
|
3847
|
+
jobIndex === 0
|
|
3848
|
+
? `exist_scheduler_names="$(\n ${(0, common_1.gcloudSchedulerCmd)()} jobs list --filter='httpTarget.uri ~ ${context.env}.*${context.name}' --format='value(name)' --limit=999 --location='${location}' --project='${project}'\n)"`
|
|
3849
|
+
: null,
|
|
3850
|
+
`current_job_uri="${uriBase}/${jobName}:run"`,
|
|
3851
|
+
`current_scheduler_name="${schedulerName}"`,
|
|
3852
|
+
`if grep "$current_scheduler_name" <<<"$exist_scheduler_names" >/dev/null; then`,
|
|
3853
|
+
` ${(0, common_1.gcloudSchedulerCmd)()} jobs update http "$current_scheduler_name" ${argsString}`,
|
|
3854
|
+
`else`,
|
|
3855
|
+
` ${(0, common_1.gcloudSchedulerCmd)()} jobs create http "$current_scheduler_name" ${argsString}`,
|
|
3856
|
+
`fi`,
|
|
3857
|
+
]
|
|
3858
|
+
.filter(utils_1.notNil)
|
|
3859
|
+
.join("\n");
|
|
3860
|
+
});
|
|
3848
3861
|
};
|
|
3849
3862
|
exports.getCreateScheduleScripts = getCreateScheduleScripts;
|
|
3850
3863
|
const getCloudRunJobsWithSchedule = (context) => {
|
|
3851
3864
|
const jobsWithNames = getCloudRunJobsWithNames(context);
|
|
3852
3865
|
return jobsWithNames
|
|
3853
3866
|
.filter((entry) => entry.job.when === "schedule")
|
|
3854
|
-
.map(({ job, jobName }) => ({
|
|
3867
|
+
.map(({ job, jobName, jobKey }) => ({
|
|
3855
3868
|
job,
|
|
3856
3869
|
jobName,
|
|
3870
|
+
jobKey,
|
|
3857
3871
|
schedulerName: jobName.concat("-scheduler"),
|
|
3858
3872
|
}));
|
|
3859
3873
|
};
|
|
@@ -3864,9 +3878,10 @@ const getCloudRunJobsWithNames = (context) => {
|
|
|
3864
3878
|
const jobsWithNames = Object.entries((_a = deployConfig.jobs) !== null && _a !== void 0 ? _a : {})
|
|
3865
3879
|
// filter out disabled jobs
|
|
3866
3880
|
.filter((entry) => Boolean(entry[1]))
|
|
3867
|
-
.map(([
|
|
3868
|
-
jobName: getFullJobName(
|
|
3881
|
+
.map(([jobKey, job]) => ({
|
|
3882
|
+
jobName: getFullJobName(jobKey),
|
|
3869
3883
|
job,
|
|
3884
|
+
jobKey,
|
|
3870
3885
|
}));
|
|
3871
3886
|
return jobsWithNames;
|
|
3872
3887
|
};
|
|
@@ -6904,13 +6904,13 @@ exports.getDeleteUnusedRevisionsCommands = getDeleteUnusedRevisionsCommands;
|
|
|
6904
6904
|
|
|
6905
6905
|
Object.defineProperty(exports, "__esModule", ({ value: true }));
|
|
6906
6906
|
exports.getCreateScheduleScripts = exports.getJobCreateScripts = exports.getJobRunScripts = exports.getDeleteJobsScripts = exports.getDeleteSchedulesScripts = void 0;
|
|
6907
|
-
const gitlab_1 = __nccwpck_require__(38490);
|
|
6908
6907
|
const createArgsString_1 = __nccwpck_require__(49598);
|
|
6909
6908
|
const jobName_1 = __nccwpck_require__(9626);
|
|
6910
6909
|
const common_1 = __nccwpck_require__(73325);
|
|
6911
6910
|
const getLabels_1 = __nccwpck_require__(30920);
|
|
6912
6911
|
const volumes_1 = __nccwpck_require__(60847);
|
|
6913
6912
|
const constants_1 = __nccwpck_require__(38865);
|
|
6913
|
+
const utils_1 = __nccwpck_require__(32497);
|
|
6914
6914
|
const getJobRunScriptForJob = (context, jobName, wait) => {
|
|
6915
6915
|
const commonArgs = (0, common_1.getCommonCloudRunArgs)(context);
|
|
6916
6916
|
const commonArgsString = (0, createArgsString_1.createArgsString)(commonArgs);
|
|
@@ -6952,76 +6952,90 @@ const getJobRunScripts = (context, when) => {
|
|
|
6952
6952
|
["preStop", "postStop"].includes(when)));
|
|
6953
6953
|
};
|
|
6954
6954
|
exports.getJobRunScripts = getJobRunScripts;
|
|
6955
|
-
const getJobCreateScripts = (context) => {
|
|
6956
|
-
const
|
|
6957
|
-
|
|
6958
|
-
.
|
|
6959
|
-
|
|
6960
|
-
};
|
|
6961
|
-
exports.getJobCreateScripts = getJobCreateScripts;
|
|
6962
|
-
const getJobCreateScriptsForJob = (context, jobName, job) => {
|
|
6963
|
-
const commonDeployArgs = (0, common_1.getCommonDeployArgs)(context);
|
|
6964
|
-
// due to some oversight from google, jobs create does not yet accept `--add-volume` 🤦
|
|
6965
|
-
// lucky, update on the other hand accepts it... so let's just imediatly update it
|
|
6966
|
-
// we cannot upsert a job, so we have to create it and catch the error and then update
|
|
6967
|
-
const commandArray = Array.isArray(job.command)
|
|
6968
|
-
? job.command
|
|
6969
|
-
: job.command.split(" ");
|
|
6955
|
+
const getJobCreateScripts = (context) => getCloudRunJobsWithNames(context).map(({ job: { command, image, cpu, memory = "512Mi", timeout = "10m", parallelism = 1, volumes, }, jobName, }, jobIndex) => {
|
|
6956
|
+
const commandArray = Array.isArray(command)
|
|
6957
|
+
? command
|
|
6958
|
+
: command.split(" ");
|
|
6959
|
+
const { image: commonImage, project, region, ...deployArgs } = (0, common_1.getCommonDeployArgs)(context);
|
|
6970
6960
|
const commonDeployArgsString = (0, createArgsString_1.createArgsString)({
|
|
6971
|
-
command:
|
|
6972
|
-
|
|
6973
|
-
|
|
6974
|
-
|
|
6975
|
-
|
|
6976
|
-
|
|
6977
|
-
|
|
6978
|
-
|
|
6979
|
-
|
|
6980
|
-
"task-timeout": job.timeout || "10m",
|
|
6981
|
-
parallelism: job.parallelism || 1,
|
|
6961
|
+
command: `"${commandArray.join(",")}"`,
|
|
6962
|
+
labels: `"${(0, common_1.makeLabelString)((0, getLabels_1.getLabels)(context))},cloud-run-job-name=$current_job_name"`,
|
|
6963
|
+
image: `"${image !== null && image !== void 0 ? image : commonImage}"`,
|
|
6964
|
+
project,
|
|
6965
|
+
region,
|
|
6966
|
+
cpu,
|
|
6967
|
+
memory,
|
|
6968
|
+
parallelism,
|
|
6969
|
+
"task-timeout": timeout,
|
|
6982
6970
|
"env-vars-file": constants_1.ENV_VARS_FILENAME,
|
|
6983
6971
|
"max-retries": 0,
|
|
6972
|
+
...deployArgs,
|
|
6984
6973
|
});
|
|
6985
|
-
|
|
6986
|
-
|
|
6974
|
+
// due to some oversight from google, jobs create does not yet accept `--add-volume` 🤦
|
|
6975
|
+
// lucky, update on the other hand accepts it... so let's just imediatly update it
|
|
6976
|
+
// we cannot upsert a job, so we have to create it and catch the error and then update
|
|
6977
|
+
const hasVolumes = Object.keys(volumes || {}).length > 0;
|
|
6978
|
+
const needsBeta = hasVolumes ? "beta" : undefined;
|
|
6979
|
+
const volumeArgs = hasVolumes
|
|
6980
|
+
? (0, createArgsString_1.createArgsString)(...(0, volumes_1.createVolumeConfig)(volumes, "job"))
|
|
6981
|
+
: "";
|
|
6987
6982
|
return [
|
|
6988
|
-
|
|
6989
|
-
|
|
6990
|
-
|
|
6991
|
-
}
|
|
6983
|
+
jobIndex === 0
|
|
6984
|
+
? `exist_job_names="$(\n ${(0, common_1.gcloudRunCmd)()} jobs list --filter='metadata.name ~ ${context.env}.*${context.name}' --format='value(name)' --limit=999 --project='${project}' --region='${region}'\n)"`
|
|
6985
|
+
: null,
|
|
6986
|
+
`current_job_name="${jobName}"`,
|
|
6987
|
+
'if grep "$current_job_name" <<<"$exist_job_names" >/dev/null; then',
|
|
6988
|
+
` ${(0, common_1.gcloudRunCmd)(needsBeta)} jobs update "$current_job_name" ${commonDeployArgsString} ${volumeArgs}`,
|
|
6989
|
+
"else",
|
|
6990
|
+
` ${(0, common_1.gcloudRunCmd)()} jobs create "$current_job_name" ${commonDeployArgsString}`,
|
|
6991
|
+
"fi",
|
|
6992
|
+
]
|
|
6993
|
+
.filter(utils_1.notNil)
|
|
6994
|
+
.join("\n");
|
|
6995
|
+
});
|
|
6996
|
+
exports.getJobCreateScripts = getJobCreateScripts;
|
|
6992
6997
|
const getCreateScheduleScripts = (context) => {
|
|
6993
6998
|
const jobsWithSchedule = getCloudRunJobsWithSchedule(context);
|
|
6994
|
-
const
|
|
6995
|
-
|
|
6996
|
-
|
|
6997
|
-
|
|
6999
|
+
const { region: location, projectId: project } = (0, common_1.getCloudRunDeployConfig)(context);
|
|
7000
|
+
const uriBase = `https://${location}-run.googleapis.com/apis/run.googleapis.com/v1/namespaces/${project}/jobs`;
|
|
7001
|
+
const gcloudArgs = {
|
|
7002
|
+
project,
|
|
7003
|
+
location,
|
|
7004
|
+
uri: `"$current_job_uri"`,
|
|
7005
|
+
"http-method": "POST",
|
|
7006
|
+
"oauth-service-account-email": `"$GCLOUD_PROJECT_NUMBER-compute@developer.gserviceaccount.com"`,
|
|
7007
|
+
};
|
|
7008
|
+
return jobsWithSchedule.map(({ job: { maxRetryAttempts, schedule }, jobName, schedulerName }, jobIndex) => {
|
|
6998
7009
|
const argsString = (0, createArgsString_1.createArgsString)({
|
|
6999
|
-
|
|
7000
|
-
|
|
7001
|
-
|
|
7002
|
-
"max-retry-attempts": (_a = job.maxRetryAttempts) !== null && _a !== void 0 ? _a : 0,
|
|
7003
|
-
uri: `"https://${deployConfig.region}-run.googleapis.com/apis/run.googleapis.com/v1/namespaces/${deployConfig.projectId}/jobs/${jobName}:run"`,
|
|
7004
|
-
"http-method": "POST",
|
|
7005
|
-
"oauth-service-account-email": "$GCLOUD_PROJECT_NUMBER-compute@developer.gserviceaccount.com",
|
|
7010
|
+
...gcloudArgs,
|
|
7011
|
+
schedule: `"${schedule}"`,
|
|
7012
|
+
"max-retry-attempts": maxRetryAttempts !== null && maxRetryAttempts !== void 0 ? maxRetryAttempts : 0,
|
|
7006
7013
|
});
|
|
7007
|
-
const commonArgs = `http ${schedulerName} ${argsString}`;
|
|
7008
7014
|
return [
|
|
7009
|
-
|
|
7010
|
-
|
|
7011
|
-
|
|
7012
|
-
|
|
7013
|
-
|
|
7014
|
-
|
|
7015
|
-
|
|
7015
|
+
jobIndex === 0
|
|
7016
|
+
? `exist_scheduler_names="$(\n ${(0, common_1.gcloudSchedulerCmd)()} jobs list --filter='httpTarget.uri ~ ${context.env}.*${context.name}' --format='value(name)' --limit=999 --location='${location}' --project='${project}'\n)"`
|
|
7017
|
+
: null,
|
|
7018
|
+
`current_job_uri="${uriBase}/${jobName}:run"`,
|
|
7019
|
+
`current_scheduler_name="${schedulerName}"`,
|
|
7020
|
+
`if grep "$current_scheduler_name" <<<"$exist_scheduler_names" >/dev/null; then`,
|
|
7021
|
+
` ${(0, common_1.gcloudSchedulerCmd)()} jobs update http "$current_scheduler_name" ${argsString}`,
|
|
7022
|
+
`else`,
|
|
7023
|
+
` ${(0, common_1.gcloudSchedulerCmd)()} jobs create http "$current_scheduler_name" ${argsString}`,
|
|
7024
|
+
`fi`,
|
|
7025
|
+
]
|
|
7026
|
+
.filter(utils_1.notNil)
|
|
7027
|
+
.join("\n");
|
|
7028
|
+
});
|
|
7016
7029
|
};
|
|
7017
7030
|
exports.getCreateScheduleScripts = getCreateScheduleScripts;
|
|
7018
7031
|
const getCloudRunJobsWithSchedule = (context) => {
|
|
7019
7032
|
const jobsWithNames = getCloudRunJobsWithNames(context);
|
|
7020
7033
|
return jobsWithNames
|
|
7021
7034
|
.filter((entry) => entry.job.when === "schedule")
|
|
7022
|
-
.map(({ job, jobName }) => ({
|
|
7035
|
+
.map(({ job, jobName, jobKey }) => ({
|
|
7023
7036
|
job,
|
|
7024
7037
|
jobName,
|
|
7038
|
+
jobKey,
|
|
7025
7039
|
schedulerName: jobName.concat("-scheduler"),
|
|
7026
7040
|
}));
|
|
7027
7041
|
};
|
|
@@ -7032,9 +7046,10 @@ const getCloudRunJobsWithNames = (context) => {
|
|
|
7032
7046
|
const jobsWithNames = Object.entries((_a = deployConfig.jobs) !== null && _a !== void 0 ? _a : {})
|
|
7033
7047
|
// filter out disabled jobs
|
|
7034
7048
|
.filter((entry) => Boolean(entry[1]))
|
|
7035
|
-
.map(([
|
|
7036
|
-
jobName: getFullJobName(
|
|
7049
|
+
.map(([jobKey, job]) => ({
|
|
7050
|
+
jobName: getFullJobName(jobKey),
|
|
7037
7051
|
job,
|
|
7052
|
+
jobKey,
|
|
7038
7053
|
}));
|
|
7039
7054
|
return jobsWithNames;
|
|
7040
7055
|
};
|
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.getCreateScheduleScripts = exports.getJobCreateScripts = exports.getJobRunScripts = exports.getDeleteJobsScripts = exports.getDeleteSchedulesScripts = void 0;
|
|
4
|
-
const gitlab_1 = require("../../../utils/gitlab");
|
|
5
4
|
const createArgsString_1 = require("../utils/createArgsString");
|
|
6
5
|
const jobName_1 = require("../utils/jobName");
|
|
7
6
|
const common_1 = require("./common");
|
|
8
7
|
const getLabels_1 = require("../../../context/getLabels");
|
|
9
8
|
const volumes_1 = require("./volumes");
|
|
10
9
|
const constants_1 = require("./constants");
|
|
10
|
+
const utils_1 = require("../../../utils");
|
|
11
11
|
const getJobRunScriptForJob = (context, jobName, wait) => {
|
|
12
12
|
const commonArgs = (0, common_1.getCommonCloudRunArgs)(context);
|
|
13
13
|
const commonArgsString = (0, createArgsString_1.createArgsString)(commonArgs);
|
|
@@ -49,76 +49,90 @@ const getJobRunScripts = (context, when) => {
|
|
|
49
49
|
["preStop", "postStop"].includes(when)));
|
|
50
50
|
};
|
|
51
51
|
exports.getJobRunScripts = getJobRunScripts;
|
|
52
|
-
const getJobCreateScripts = (context) => {
|
|
53
|
-
const
|
|
54
|
-
|
|
55
|
-
.
|
|
56
|
-
|
|
57
|
-
};
|
|
58
|
-
exports.getJobCreateScripts = getJobCreateScripts;
|
|
59
|
-
const getJobCreateScriptsForJob = (context, jobName, job) => {
|
|
60
|
-
const commonDeployArgs = (0, common_1.getCommonDeployArgs)(context);
|
|
61
|
-
// due to some oversight from google, jobs create does not yet accept `--add-volume` 🤦
|
|
62
|
-
// lucky, update on the other hand accepts it... so let's just imediatly update it
|
|
63
|
-
// we cannot upsert a job, so we have to create it and catch the error and then update
|
|
64
|
-
const commandArray = Array.isArray(job.command)
|
|
65
|
-
? job.command
|
|
66
|
-
: job.command.split(" ");
|
|
52
|
+
const getJobCreateScripts = (context) => getCloudRunJobsWithNames(context).map(({ job: { command, image, cpu, memory = "512Mi", timeout = "10m", parallelism = 1, volumes, }, jobName, }, jobIndex) => {
|
|
53
|
+
const commandArray = Array.isArray(command)
|
|
54
|
+
? command
|
|
55
|
+
: command.split(" ");
|
|
56
|
+
const { image: commonImage, project, region, ...deployArgs } = (0, common_1.getCommonDeployArgs)(context);
|
|
67
57
|
const commonDeployArgsString = (0, createArgsString_1.createArgsString)({
|
|
68
|
-
command:
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
"task-timeout": job.timeout || "10m",
|
|
78
|
-
parallelism: job.parallelism || 1,
|
|
58
|
+
command: `"${commandArray.join(",")}"`,
|
|
59
|
+
labels: `"${(0, common_1.makeLabelString)((0, getLabels_1.getLabels)(context))},cloud-run-job-name=$current_job_name"`,
|
|
60
|
+
image: `"${image !== null && image !== void 0 ? image : commonImage}"`,
|
|
61
|
+
project,
|
|
62
|
+
region,
|
|
63
|
+
cpu,
|
|
64
|
+
memory,
|
|
65
|
+
parallelism,
|
|
66
|
+
"task-timeout": timeout,
|
|
79
67
|
"env-vars-file": constants_1.ENV_VARS_FILENAME,
|
|
80
68
|
"max-retries": 0,
|
|
69
|
+
...deployArgs,
|
|
81
70
|
});
|
|
82
|
-
|
|
83
|
-
|
|
71
|
+
// due to some oversight from google, jobs create does not yet accept `--add-volume` 🤦
|
|
72
|
+
// lucky, update on the other hand accepts it... so let's just imediatly update it
|
|
73
|
+
// we cannot upsert a job, so we have to create it and catch the error and then update
|
|
74
|
+
const hasVolumes = Object.keys(volumes || {}).length > 0;
|
|
75
|
+
const needsBeta = hasVolumes ? "beta" : undefined;
|
|
76
|
+
const volumeArgs = hasVolumes
|
|
77
|
+
? (0, createArgsString_1.createArgsString)(...(0, volumes_1.createVolumeConfig)(volumes, "job"))
|
|
78
|
+
: "";
|
|
84
79
|
return [
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
}
|
|
80
|
+
jobIndex === 0
|
|
81
|
+
? `exist_job_names="$(\n ${(0, common_1.gcloudRunCmd)()} jobs list --filter='metadata.name ~ ${context.env}.*${context.name}' --format='value(name)' --limit=999 --project='${project}' --region='${region}'\n)"`
|
|
82
|
+
: null,
|
|
83
|
+
`current_job_name="${jobName}"`,
|
|
84
|
+
'if grep "$current_job_name" <<<"$exist_job_names" >/dev/null; then',
|
|
85
|
+
` ${(0, common_1.gcloudRunCmd)(needsBeta)} jobs update "$current_job_name" ${commonDeployArgsString} ${volumeArgs}`,
|
|
86
|
+
"else",
|
|
87
|
+
` ${(0, common_1.gcloudRunCmd)()} jobs create "$current_job_name" ${commonDeployArgsString}`,
|
|
88
|
+
"fi",
|
|
89
|
+
]
|
|
90
|
+
.filter(utils_1.notNil)
|
|
91
|
+
.join("\n");
|
|
92
|
+
});
|
|
93
|
+
exports.getJobCreateScripts = getJobCreateScripts;
|
|
89
94
|
const getCreateScheduleScripts = (context) => {
|
|
90
95
|
const jobsWithSchedule = getCloudRunJobsWithSchedule(context);
|
|
91
|
-
const
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
96
|
+
const { region: location, projectId: project } = (0, common_1.getCloudRunDeployConfig)(context);
|
|
97
|
+
const uriBase = `https://${location}-run.googleapis.com/apis/run.googleapis.com/v1/namespaces/${project}/jobs`;
|
|
98
|
+
const gcloudArgs = {
|
|
99
|
+
project,
|
|
100
|
+
location,
|
|
101
|
+
uri: `"$current_job_uri"`,
|
|
102
|
+
"http-method": "POST",
|
|
103
|
+
"oauth-service-account-email": `"$GCLOUD_PROJECT_NUMBER-compute@developer.gserviceaccount.com"`,
|
|
104
|
+
};
|
|
105
|
+
return jobsWithSchedule.map(({ job: { maxRetryAttempts, schedule }, jobName, schedulerName }, jobIndex) => {
|
|
95
106
|
const argsString = (0, createArgsString_1.createArgsString)({
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
"max-retry-attempts": (_a = job.maxRetryAttempts) !== null && _a !== void 0 ? _a : 0,
|
|
100
|
-
uri: `"https://${deployConfig.region}-run.googleapis.com/apis/run.googleapis.com/v1/namespaces/${deployConfig.projectId}/jobs/${jobName}:run"`,
|
|
101
|
-
"http-method": "POST",
|
|
102
|
-
"oauth-service-account-email": "$GCLOUD_PROJECT_NUMBER-compute@developer.gserviceaccount.com",
|
|
107
|
+
...gcloudArgs,
|
|
108
|
+
schedule: `"${schedule}"`,
|
|
109
|
+
"max-retry-attempts": maxRetryAttempts !== null && maxRetryAttempts !== void 0 ? maxRetryAttempts : 0,
|
|
103
110
|
});
|
|
104
|
-
const commonArgs = `http ${schedulerName} ${argsString}`;
|
|
105
111
|
return [
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
112
|
+
jobIndex === 0
|
|
113
|
+
? `exist_scheduler_names="$(\n ${(0, common_1.gcloudSchedulerCmd)()} jobs list --filter='httpTarget.uri ~ ${context.env}.*${context.name}' --format='value(name)' --limit=999 --location='${location}' --project='${project}'\n)"`
|
|
114
|
+
: null,
|
|
115
|
+
`current_job_uri="${uriBase}/${jobName}:run"`,
|
|
116
|
+
`current_scheduler_name="${schedulerName}"`,
|
|
117
|
+
`if grep "$current_scheduler_name" <<<"$exist_scheduler_names" >/dev/null; then`,
|
|
118
|
+
` ${(0, common_1.gcloudSchedulerCmd)()} jobs update http "$current_scheduler_name" ${argsString}`,
|
|
119
|
+
`else`,
|
|
120
|
+
` ${(0, common_1.gcloudSchedulerCmd)()} jobs create http "$current_scheduler_name" ${argsString}`,
|
|
121
|
+
`fi`,
|
|
122
|
+
]
|
|
123
|
+
.filter(utils_1.notNil)
|
|
124
|
+
.join("\n");
|
|
125
|
+
});
|
|
113
126
|
};
|
|
114
127
|
exports.getCreateScheduleScripts = getCreateScheduleScripts;
|
|
115
128
|
const getCloudRunJobsWithSchedule = (context) => {
|
|
116
129
|
const jobsWithNames = getCloudRunJobsWithNames(context);
|
|
117
130
|
return jobsWithNames
|
|
118
131
|
.filter((entry) => entry.job.when === "schedule")
|
|
119
|
-
.map(({ job, jobName }) => ({
|
|
132
|
+
.map(({ job, jobName, jobKey }) => ({
|
|
120
133
|
job,
|
|
121
134
|
jobName,
|
|
135
|
+
jobKey,
|
|
122
136
|
schedulerName: jobName.concat("-scheduler"),
|
|
123
137
|
}));
|
|
124
138
|
};
|
|
@@ -129,9 +143,10 @@ const getCloudRunJobsWithNames = (context) => {
|
|
|
129
143
|
const jobsWithNames = Object.entries((_a = deployConfig.jobs) !== null && _a !== void 0 ? _a : {})
|
|
130
144
|
// filter out disabled jobs
|
|
131
145
|
.filter((entry) => Boolean(entry[1]))
|
|
132
|
-
.map(([
|
|
133
|
-
jobName: getFullJobName(
|
|
146
|
+
.map(([jobKey, job]) => ({
|
|
147
|
+
jobName: getFullJobName(jobKey),
|
|
134
148
|
job,
|
|
149
|
+
jobKey,
|
|
135
150
|
}));
|
|
136
151
|
return jobsWithNames;
|
|
137
152
|
};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cloudRunJobs.js","sourceRoot":"","sources":["../../../../../../../pipeline/src/deploy/cloudRun/createJobs/cloudRunJobs.ts"],"names":[],"mappings":";;;
|
|
1
|
+
{"version":3,"file":"cloudRunJobs.js","sourceRoot":"","sources":["../../../../../../../pipeline/src/deploy/cloudRun/createJobs/cloudRunJobs.ts"],"names":[],"mappings":";;;AAMA,gEAA6D;AAC7D,8CAAsD;AACtD,qCAOkB;AAClB,0DAAuD;AACvD,uCAA+C;AAK/C,2CAAgD;AAChD,0CAAwC;AAExC,MAAM,qBAAqB,GAAG,CAC5B,OAAyB,EACzB,OAA+B,EAC/B,IAAa,EACb,EAAE;IACF,MAAM,UAAU,GAAG,IAAA,8BAAqB,EAAC,OAAO,CAAC,CAAC;IAElD,MAAM,gBAAgB,GAAG,IAAA,mCAAgB,EAAC,UAAU,CAAC,CAAC;IACtD,OAAO,GAAG,IAAA,qBAAY,GAAE,iBAAiB,OAAO,CAAC,QAAQ,EAAE,IAAI,gBAAgB,GAC7E,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,EACrB,EAAE,CAAC;AACL,CAAC,CAAC;AAEK,MAAM,yBAAyB,GAAG,CAAC,OAAyB,EAAE,EAAE;IACrE,MAAM,YAAY,GAAG,IAAA,gCAAuB,EAAC,OAAO,CAAC,CAAC;IACtD,MAAM,gBAAgB,GAAG,2BAA2B,CAAC,OAAO,CAAC,CAAC;IAC9D,MAAM,UAAU,GAAG,IAAA,mCAAgB,EAAC;QAClC,OAAO,EAAE,YAAY,CAAC,SAAS;QAC/B,QAAQ,EAAE,YAAY,CAAC,MAAM;KAC9B,CAAC,CAAC;IACH,OAAO,gBAAgB;SACpB,GAAG,CAAC,CAAC,EAAE,aAAa,EAAE,EAAE,EAAE;QACzB,OAAO;YACL,GAAG,IAAA,2BAAkB,GAAE,gBAAgB,aAAa,IAAI,UAAU,EAAE;SACrE,CAAC;IACJ,CAAC,CAAC;SACD,IAAI,EAAE,CAAC;AACZ,CAAC,CAAC;AAdW,QAAA,yBAAyB,6BAcpC;AAEK,MAAM,oBAAoB,GAAG,CAAC,OAAyB,EAAE,EAAE;IAChE,MAAM,UAAU,GAAG,IAAA,8BAAqB,EAAC,OAAO,CAAC,CAAC;IAClD,MAAM,gBAAgB,GAAG,IAAA,mCAAgB,EAAC,UAAU,CAAC,CAAC;IACtD,MAAM,aAAa,GAAG,wBAAwB,CAAC,OAAO,CAAC,CAAC;IAExD,OAAO,aAAa,CAAC,OAAO,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC;QAC5C,gGAAgG;QAChG,GAAG,IAAA,qBAAY,GAAE,yBAAyB,gBAAgB,UAAU,OAAO,yCAAyC,IAAA,qBAAY,GAAE,uCAAuC,gBAAgB,EAAE;QAC3L,GAAG,IAAA,qBAAY,GAAE,gBAAgB,OAAO,IAAI,gBAAgB,EAAE;KAC/D,CAAC,CAAC;AACL,CAAC,CAAC;AAVW,QAAA,oBAAoB,wBAU/B;AAEK,MAAM,gBAAgB,GAAG,CAC9B,OAAyB,EACzB,IAAqC,EACrC,EAAE;IACF,MAAM,aAAa,GAAG,wBAAwB,CAAC,OAAO,CAAC,CAAC;IACxD,OAAO,aAAa;SACjB,MAAM,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,KAAK,IAAI,CAAC;SACtC,GAAG,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,CACnB,qBAAqB,CACnB,OAAO,EACP,OAAO;IACP,iGAAiG;IACjG,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CACvC,CACF,CAAC;AACN,CAAC,CAAC;AAfW,QAAA,gBAAgB,oBAe3B;AAEK,MAAM,mBAAmB,GAAG,CAAC,OAAyB,EAAY,EAAE,CACzE,wBAAwB,CAAC,OAAO,CAAC,CAAC,GAAG,CACnC,CACE,EACE,GAAG,EAAE,EACH,OAAO,EACP,KAAK,EACL,GAAG,EACH,MAAM,GAAG,OAAO,EAChB,OAAO,GAAG,KAAK,EACf,WAAW,GAAG,CAAC,EACf,OAAO,GACR,EACD,OAAO,GACR,EACD,QAAQ,EACA,EAAE;IACV,MAAM,YAAY,GAAG,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC;QACzC,CAAC,CAAC,OAAO;QACT,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAEvB,MAAM,EACJ,KAAK,EAAE,WAAW,EAClB,OAAO,EACP,MAAM,EACN,GAAG,UAAU,EACd,GAAG,IAAA,4BAAmB,EAAC,OAAO,CAAC,CAAC;IACjC,MAAM,sBAAsB,GAAG,IAAA,mCAAgB,EAAC;QAC9C,OAAO,EAAE,IAAI,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG;QACtC,MAAM,EAAE,IAAI,IAAA,wBAAe,EAAC,IAAA,qBAAS,EAAC,OAAO,CAAC,CAAC,wCAAwC;QACvF,KAAK,EAAE,IAAI,KAAK,aAAL,KAAK,cAAL,KAAK,GAAI,WAAW,GAAG;QAClC,OAAO;QACP,MAAM;QACN,GAAG;QACH,MAAM;QACN,WAAW;QACX,cAAc,EAAE,OAAO;QACvB,eAAe,EAAE,6BAAiB;QAClC,aAAa,EAAE,CAAC;QAChB,GAAG,UAAU;KACd,CAAC,CAAC;IAEH,uFAAuF;IACvF,kFAAkF;IAClF,sFAAsF;IACtF,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IACzD,MAAM,SAAS,GAAG,UAAU,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;IAClD,MAAM,UAAU,GAAG,UAAU;QAC3B,CAAC,CAAC,IAAA,mCAAgB,EAAC,GAAG,IAAA,4BAAkB,EAAC,OAAO,EAAE,KAAK,CAAC,CAAC;QACzD,CAAC,CAAC,EAAE,CAAC;IAEP,OAAO;QACL,QAAQ,KAAK,CAAC;YACZ,CAAC,CAAC,0BAA0B,IAAA,qBAAY,GAAE,wCAAwC,OAAO,CAAC,GAAG,KAAK,OAAO,CAAC,IAAI,mDAAmD,OAAO,eAAe,MAAM,OAAO;YACpM,CAAC,CAAC,IAAI;QACR,qBAAqB,OAAO,GAAG;QAC/B,oEAAoE;QACpE,KAAK,IAAA,qBAAY,EAAC,SAAS,CAAC,oCAAoC,sBAAsB,IAAI,UAAU,EAAE;QACtG,MAAM;QACN,KAAK,IAAA,qBAAY,GAAE,oCAAoC,sBAAsB,EAAE;QAC/E,IAAI;KACL;SACE,MAAM,CAAC,cAAM,CAAC;SACd,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC,CACF,CAAC;AAjES,QAAA,mBAAmB,uBAiE5B;AAEG,MAAM,wBAAwB,GAAG,CACtC,OAAyB,EACf,EAAE;IACZ,MAAM,gBAAgB,GAAG,2BAA2B,CAAC,OAAO,CAAC,CAAC;IAC9D,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,OAAO,EAAE,GAC5C,IAAA,gCAAuB,EAAC,OAAO,CAAC,CAAC;IAEnC,MAAM,OAAO,GAAG,WAAW,QAAQ,6DAA6D,OAAO,OAAO,CAAC;IAC/G,MAAM,UAAU,GAAG;QACjB,OAAO;QACP,QAAQ;QACR,GAAG,EAAE,oBAAoB;QACzB,aAAa,EAAE,MAAM;QACrB,6BAA6B,EAAE,gEAAgE;KAChG,CAAC;IAEF,OAAO,gBAAgB,CAAC,GAAG,CACzB,CACE,EAAE,GAAG,EAAE,EAAE,gBAAgB,EAAE,QAAQ,EAAE,EAAE,OAAO,EAAE,aAAa,EAAE,EAC/D,QAAQ,EACA,EAAE;QACV,MAAM,UAAU,GAAG,IAAA,mCAAgB,EAAC;YAClC,GAAG,UAAU;YACb,QAAQ,EAAE,IAAI,QAAQ,GAAG;YACzB,oBAAoB,EAAE,gBAAgB,aAAhB,gBAAgB,cAAhB,gBAAgB,GAAI,CAAC;SAC5C,CAAC,CAAC;QACH,OAAO;YACL,QAAQ,KAAK,CAAC;gBACZ,CAAC,CAAC,gCAAgC,IAAA,2BAAkB,GAAE,yCAAyC,OAAO,CAAC,GAAG,KAAK,OAAO,CAAC,IAAI,oDAAoD,QAAQ,gBAAgB,OAAO,OAAO;gBACrN,CAAC,CAAC,IAAI;YACR,oBAAoB,OAAO,IAAI,OAAO,OAAO;YAC7C,2BAA2B,aAAa,GAAG;YAC3C,gFAAgF;YAChF,KAAK,IAAA,2BAAkB,GAAE,+CAA+C,UAAU,EAAE;YACpF,MAAM;YACN,KAAK,IAAA,2BAAkB,GAAE,+CAA+C,UAAU,EAAE;YACpF,IAAI;SACL;aACE,MAAM,CAAC,cAAM,CAAC;aACd,IAAI,CAAC,IAAI,CAAC,CAAC;IAChB,CAAC,CACF,CAAC;AACJ,CAAC,CAAC;AA1CW,QAAA,wBAAwB,4BA0CnC;AAEF,MAAM,2BAA2B,GAAG,CAAC,OAAyB,EAAE,EAAE;IAChE,MAAM,aAAa,GAAG,wBAAwB,CAAC,OAAO,CAAC,CAAC;IAExD,OAAO,aAAa;SACjB,MAAM,CACL,CACE,KAAK,EAKL,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,KAAK,UAAU,CACnC;SACA,GAAG,CAAC,CAAC,EAAE,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,CAAC;QAClC,GAAG;QACH,OAAO;QACP,MAAM;QACN,aAAa,EAAE,OAAO,CAAC,MAAM,CAAC,YAAY,CAAC;KAC5C,CAAC,CAAC,CAAC;AACR,CAAC,CAAC;AAEF,MAAM,wBAAwB,GAAG,CAAC,OAAyB,EAAE,EAAE;;IAC7D,MAAM,YAAY,GAAG,IAAA,gCAAuB,EAAC,OAAO,CAAC,CAAC;IAEtD,MAAM,cAAc,GAAG,CAAC,IAAY,EAAE,EAAE,CACtC,IAAA,4BAAkB,EAAC,OAAO,CAAC,WAAW,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;IAEzD,MAAM,aAAa,GAAG,MAAM,CAAC,OAAO,CAAC,MAAA,YAAY,CAAC,IAAI,mCAAI,EAAE,CAAC;QAC3D,2BAA2B;SAC1B,MAAM,CAAC,CAAC,KAAK,EAA8C,EAAE,CAC5D,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAClB;SACA,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC;QACvB,OAAO,EAAE,cAAc,CAAC,MAAM,CAAC;QAC/B,GAAG;QACH,MAAM;KACP,CAAC,CAAC,CAAC;IACN,OAAO,aAAa,CAAC;AACvB,CAAC,CAAC"}
|