@catladder/pipeline 1.144.1 → 1.146.0

package/src/deploy/base/deploy.ts

@@ -15,11 +15,17 @@ export const DEPLOY_JOB_NAME = "🚀 Deploy";
 
 export type DeployJobDefinition = Pick<
   CatladderJob,
-  "script" | "variables" | "image" | "cache" | "artifacts" | "services"
+  | "script"
+  | "variables"
+  | "image"
+  | "cache"
+  | "artifacts"
+  | "services"
+  | "runnerVariables"
 >;
 export const createDeployJob = (
   context: Context,
-  jobDefinition: DeployJobDefinition
+  jobDefinition: DeployJobDefinition,
 ): CatladderJob => {
   const hasDocker = requiresDockerBuild(context);
   const isStoppable = contextIsStoppable(context);
@@ -28,8 +34,8 @@ export const createDeployJob = (
     context.environment.envType === "review"
       ? "1 week"
       : context.environment.envType === "dev"
-      ? "4 weeks"
-      : undefined;
+        ? "4 weeks"
+        : undefined;
 
   // if auto or manual is configured explicitly, use that
   const whenDeployDefined =
@@ -41,8 +47,8 @@ export const createDeployJob = (
     context.environment.envType !== "prod"
       ? "auto" // is not stage, auto deploy
       : context.componentConfig.env?.stage === false
-      ? "auto" // is prod, but no staging, auto deploy
-      : "manual"; // manually deploy
+        ? "auto" // is prod, but no staging, auto deploy
+        : "manual"; // manually deploy
   const whenDeploy = whenDeployDefined ? whenDeployDefined : whenDeployDefault;
 
   return {
@@ -77,19 +83,11 @@ export const createDeployJob = (
         artifacts: false,
       },
     ], // workaround for https://gitlab.com/gitlab-org/gitlab/-/issues/220758
-    rules: [
-      whenDeploy === "auto"
-        ? {
-            when: "on_success",
-          }
-        : {
-            when: "manual",
-          },
-    ],
+    when: whenDeploy === "auto" ? "on_success" : "manual",
+
     allow_failure: whenDeploy === "manual" ? true : false,
     stage: "deploy",
     variables: {
-      ...DEPLOY_RUNNER_VARIABLES,
       ...context.environment.envVars,
       ...(hasDocker ? getDockerImageVariables(context) : {}),
       ...context.environment.jobOnlyVars.deploy.envVars,
@@ -98,15 +96,19 @@ export const createDeployJob = (
         : {}),
       ...jobDefinition.variables,
     },
-    environment: {
-      ...context.environment.gitlabEnvironment,
-      ...(isStoppable
-        ? {
-            on_stop: STOP_JOB_NAME,
-            auto_stop_in: autoStop,
-          }
+    runnerVariables: {
+      ...DEPLOY_RUNNER_VARIABLES,
+      ...(jobDefinition.runnerVariables ?? {}),
+      ...(context.componentConfig.deploy
+        ? context.componentConfig.deploy.runnerVariables ?? {}
         : {}),
     },
+    environment: isStoppable
+      ? {
+          on_stop: STOP_JOB_NAME,
+          auto_stop_in: autoStop,
+        }
+      : undefined,
     jobTags: context.componentConfig.deploy
       ? context.componentConfig.deploy.jobTags
       : undefined,

package/src/deploy/base/index.ts

@@ -10,7 +10,7 @@ export const createDeployementJobs = (
     deploy: DeployJobDefinition;
     stop?: StopJobDefinition;
     rollback?: RollbackJobDefinition;
-  }
+  },
 ): CatladderJob[] => {
   return [
     createDeployJob(context, definitions.deploy),

package/src/deploy/base/rollback.ts

@@ -6,32 +6,30 @@ export const ROLLBACK_JOB_NAME = "↩️ Rollback ⚠️";
 
 export type RollbackJobDefinition = Pick<
   CatladderJob,
-  "script" | "variables" | "image"
+  "script" | "variables" | "runnerVariables" | "image"
 >;
 export const createRollbackJob = (
   context: Context,
-  jobDefinition: RollbackJobDefinition
+  jobDefinition: RollbackJobDefinition,
 ): CatladderJob => {
   return {
     name: ROLLBACK_JOB_NAME,
     image: jobDefinition.image,
     envMode: "stagePerEnv", // makes it easier to run manual tasks er env
     needs: [], // can be executed even if the deploy job failed
-    rules: [
-      {
-        when: "manual",
-        allow_failure: true,
-      },
-    ],
+    allow_failure: true,
+    when: "manual",
     variables: {
-      ...DEPLOY_RUNNER_VARIABLES,
       ...context.environment.jobOnlyVars.deploy.envVars,
-      GIT_STRATEGY: "none",
       ...jobDefinition.variables,
     },
+    runnerVariables: {
+      ...DEPLOY_RUNNER_VARIABLES,
+      GIT_STRATEGY: "none",
+      ...(jobDefinition.runnerVariables ?? {}),
+    },
     stage: "rollback",
     environment: {
-      ...context.environment.gitlabEnvironment,
       action: "access",
     },
 

package/src/deploy/base/stop.ts

@@ -6,37 +6,36 @@ export const STOP_JOB_NAME = "🛑 Stop ⚠️";
 
 export type StopJobDefinition = Pick<
   CatladderJob,
-  "script" | "variables" | "image"
+  "script" | "variables" | "image" | "runnerVariables"
 >;
 export const createStopJob = (
   context: Context,
-  jobDefinition: StopJobDefinition
+  jobDefinition: StopJobDefinition,
 ): CatladderJob => {
   return {
     name: STOP_JOB_NAME,
     image: jobDefinition.image,
     envMode: "stagePerEnv", // makes it easier to run manual tasks er env
     needs: [], // can be executed even if the deploy job failed
+    allow_failure: true,
+    when: "manual", // stop is always manual
     rules: [
       {
         if: "$CI_COMMIT_BRANCH =~ /^[0-9]+\\.([0-9]+|x)\\.x$/", // automatic on hotfix branches
         when: "on_success",
-        allow_failure: true,
-      },
-      {
-        when: "manual",
-        allow_failure: true,
       },
     ],
     variables: {
-      ...DEPLOY_RUNNER_VARIABLES,
       ...context.environment.jobOnlyVars.deploy.envVars,
-      GIT_STRATEGY: "none",
       ...jobDefinition.variables,
     },
+    runnerVariables: {
+      ...DEPLOY_RUNNER_VARIABLES,
+      GIT_STRATEGY: "none",
+      ...jobDefinition.runnerVariables,
+    },
     stage: "stop",
     environment: {
-      ...context.environment.gitlabEnvironment,
       action: "stop",
     },
     script: jobDefinition.script,

package/src/deploy/cloudRun/artifactsRegistry.ts

@@ -1,3 +1,5 @@
+import type { StringOrBashExpression } from "../../bash/BashExpression";
+import { joinBashExpressions } from "../../bash/BashExpression";
 import type { Context } from "../../types/context";
 import { allowFailureInScripts } from "../../utils/gitlab";
 import { isOfDeployType } from "../types";
@@ -34,7 +36,7 @@ export const getArtifactsRegistryDockerUrl = (context: Context) => {
  */
 export const getArtifactsRegistryImageName = (
   context: Context,
-  lecacyReviewImageName = false
+  lecacyReviewImageName = false,
 ) => {
   if (lecacyReviewImageName && context.environment.envType !== "review") {
     throw new Error("lecacyReviewImageName is only allowed for review app");
@@ -46,11 +48,11 @@ export const getArtifactsRegistryImageName = (
     context.environment.shortName,
     context.componentName,
 
-    ...(context.environment.envType === "review" && !lecacyReviewImageName
-      ? [context.commitInfo?.reviewSlug]
+    ...(context.environment.reviewSlug && !lecacyReviewImageName
+      ? [context.environment.reviewSlug]
       : []),
   ];
-  return gcloudImagePath.join("/");
+  return joinBashExpressions(gcloudImagePath, "/");
 };
 
 export const getArtifactsRegistryBuildCacheImage = (context: Context) => {
@@ -63,7 +65,10 @@ export const getArtifactsRegistryBuildCacheImage = (context: Context) => {
 export const getArtifactsRegistryImage = (context: Context) =>
   `${getArtifactsRegistryImageName(context)}:$DOCKER_IMAGE_TAG`;
 
-const getDeleteImageCommands = (fullImageName: string, keepNewest = 0) => {
+const getDeleteImageCommands = (
+  fullImageName: StringOrBashExpression,
+  keepNewest = 0,
+) => {
   if (keepNewest === 0) {
     // no need to list tags, we delete the whole thing
     return [
@@ -75,7 +80,7 @@ const getDeleteImageCommands = (fullImageName: string, keepNewest = 0) => {
 
   const listImagesToDeletecommand = removeFirstLinesFromCommandOutput(
     listAllImagesCommand,
-    keepNewest
+    keepNewest,
   );
   const deleteImageCommand = `gcloud artifacts docker images delete ${fullImageName}@$version --quiet --delete-tags`;
 
@@ -114,8 +119,8 @@ export const getDeleteUnusedImagesCommands = (context: Context, keep = 0) => {
       ? allowFailureInScripts(
           getDeleteImageCommands(
             getArtifactsRegistryImageName(context, true),
-            0
-          )
+            0,
+          ),
         )
       : []),
   ];

package/src/deploy/cloudRun/cleanup.ts

@@ -4,7 +4,7 @@ import { getDeleteUnusedRevisionsCommands } from "./cloudRunRevisions";
 
 export const getRemoveOldRevisionsAndImagesCommand = (
   context: Context,
-  when: "postDeploy" | "onStop"
+  when: "postDeploy" | "onStop",
 ) => {
   if (when === "onStop") {
     // service is already deleted, so we don't need to delete old revisions, just delete all images
@@ -19,11 +19,11 @@ export const getRemoveOldRevisionsAndImagesCommand = (
 
   const deleteOldRevisionsCommands = getDeleteUnusedRevisionsCommands(
     context,
-    revisionsToKeep
+    revisionsToKeep,
   );
   const deleteOldImagesCommands = getDeleteUnusedImagesCommands(
     context,
-    imagesToKeep
+    imagesToKeep,
   );
 
   return [...deleteOldRevisionsCommands, ...deleteOldImagesCommands];

package/src/deploy/cloudRun/cloudRunRevisions.ts

@@ -9,7 +9,7 @@ const getListRevisionsCommand = (
   args: {
     filter?: string;
     format: string;
-  }
+  },
 ) => {
   const serviceName = getServiceName(context);
   const deployConfig = context.componentConfig.deploy;
@@ -35,7 +35,7 @@ const getListRevisionsCommand = (
 
 export const getDeleteUnusedRevisionsCommands = (
   context: Context,
-  keep: number
+  keep: number,
 ) => {
   const deployConfig = context.componentConfig.deploy;
   if (deployConfig === false) {
@@ -54,7 +54,7 @@ export const getDeleteUnusedRevisionsCommands = (
   // this removes the newest `revisionsToKeep` images
   const listRevisionsToDeleteCmd = removeFirstLinesFromCommandOutput(
     listAllInactiveRevisionsCmd,
-    keep
+    keep,
   );
   const deleteRevisionCmd = `gcloud run revisions delete ${createArgsString({
     project: deployConfig.projectId,

package/src/deploy/cloudRun/createJobs/cloudRunJobs.ts

@@ -17,16 +17,21 @@ import {
 } from "./common";
 import { getLabels } from "../../../context/getLabels";
 import { createVolumeConfig } from "./volumes";
+import type {
+  BashExpression,
+  StringOrBashExpression,
+} from "../../../bash/BashExpression";
+import { ENV_VARS_FILENAME } from "./constants";
 
 const getJobRunScriptForJob = (
   context: Context,
-  jobName: string,
-  wait: boolean
+  jobName: StringOrBashExpression,
+  wait: boolean,
 ) => {
   const commonArgs = getCommonCloudRunArgs(context);
 
   const commonArgsString = createArgsString(commonArgs);
-  return `${gcloudRunCmd()} jobs execute ${jobName} ${commonArgsString}${
+  return `${gcloudRunCmd()} jobs execute ${jobName.toString()} ${commonArgsString}${
     wait ? " --wait" : ""
   }`;
 };
@@ -61,7 +66,7 @@ export const getDeleteJobsScripts = (context: Context) => {
 
 export const getJobRunScripts = (
   context: Context,
-  when: DeployConfigCloudRunJob["when"]
+  when: DeployConfigCloudRunJob["when"],
 ) => {
   const jobsWithNames = getCloudRunJobsWithNames(context);
   return jobsWithNames
@@ -71,8 +76,8 @@ export const getJobRunScripts = (
         context,
         jobName,
         // wait for completin on stop jobs, since stop will delete the jobs afterwards, so they will fail
-        ["preStop", "postStop"].includes(when)
-      )
+        ["preStop", "postStop"].includes(when),
+      ),
     );
 };
 
@@ -86,8 +91,8 @@ export const getJobCreateScripts = (context: Context) => {
 
 const getJobCreateScriptsForJob = (
   context: Context,
-  jobName: string,
-  job: DeployConfigCloudRunJob
+  jobName: StringOrBashExpression,
+  job: DeployConfigCloudRunJob,
 ) => {
   const commonDeployArgs = getCommonDeployArgs(context);
 
@@ -103,14 +108,14 @@ const getJobCreateScriptsForJob = (
     ...commonDeployArgs,
     labels: makeLabelString({
       ...getLabels(context),
-      "cloud-run-job-name": jobName,
+      "cloud-run-job-name": jobName.toString(),
     }),
     image: job.image || commonDeployArgs.image,
     cpu: job?.cpu,
     memory: job.memory || "512Mi",
     "task-timeout": job.timeout || "10m",
     parallelism: job.parallelism || 1,
-    "env-vars-file": "____envvars.yaml",
+    "env-vars-file": ENV_VARS_FILENAME,
     "max-retries": 0,
   });
 
@@ -120,9 +125,9 @@ const getJobCreateScriptsForJob = (
   return [
     ...allowFailureInScripts([`${gcloudRunCmd()} jobs create ${argsString}`]),
     `${gcloudRunCmd(
-      requiresBeta ? "beta" : undefined
+      requiresBeta ? "beta" : undefined,
     )} jobs update ${argsString} ${createArgsString(
-      ...createVolumeConfig(job?.volumes, "job")
+      ...createVolumeConfig(job?.volumes, "job"),
     )}`,
   ];
 };
@@ -161,16 +166,16 @@ const getCloudRunJobsWithSchedule = (context: Context) => {
   return jobsWithNames
     .filter(
       (
-        entry
+        entry,
       ): entry is {
-        jobName: string;
+        jobName: BashExpression;
         job: DeployConfigCloudRunJobWithSchedule;
-      } => entry.job.when === "schedule"
+      } => entry.job.when === "schedule",
     )
     .map(({ job, jobName }) => ({
       job,
       jobName,
-      schedulerName: jobName + "-scheduler",
+      schedulerName: jobName.concat("-scheduler"),
     }));
 };
 
@@ -183,7 +188,7 @@ const getCloudRunJobsWithNames = (context: Context) => {
   const jobsWithNames = Object.entries(deployConfig.jobs ?? {})
     // filter out disabled jobs
     .filter((entry): entry is [string, DeployConfigCloudRunJob] =>
-      Boolean(entry[1])
+      Boolean(entry[1]),
     )
     .map(([name, job]) => ({
       jobName: getFullJobName(name),

package/src/deploy/cloudRun/createJobs/cloudRunServices.ts

@@ -10,12 +10,13 @@ import {
   getCommonDeployArgs,
   makeLabelString,
 } from "./common";
+import { ENV_VARS_FILENAME } from "./constants";
 import { createVolumeConfig } from "./volumes";
 
 export const getServiceDeployScript = (
   context: Context,
   service: DeployConfigCloudRunService | true | undefined,
-  nameSuffix?: string
+  nameSuffix?: string,
 ) => {
   const commonDeployArgs = getCommonDeployArgs(context);
 
@@ -32,7 +33,7 @@ export const getServiceDeployScript = (
       ? command
       : command.split(" ")
     : undefined;
-  const fullServiceName = `${serviceName}${nameSuffix ?? ""}`;
+  const fullServiceName = serviceName.concat(nameSuffix ?? "");
   const argsString = createArgsString(
     {
       // command as empty string resets it to default (uses the image's entrypoint)
@@ -42,7 +43,7 @@ export const getServiceDeployScript = (
         ...getLabels(context),
         "cloud-run-service-name": fullServiceName,
       }),
-      "env-vars-file": "____envvars.yaml",
+      "env-vars-file": ENV_VARS_FILENAME,
       "min-instances": customConfig?.minInstances ?? 0,
       "max-instances": customConfig?.maxInstances ?? 100,
       "cpu-throttling": customConfig?.noCpuThrottling !== true,
@@ -53,20 +54,20 @@ export const getServiceDeployScript = (
       "cpu-boost": true,
       "execution-environment": customConfig?.executionEnvironment,
     },
-    ...createVolumeConfig(customConfig?.volumes, "service")
+    ...createVolumeConfig(customConfig?.volumes, "service"),
   );
   // volumes require beta
   const requiresBeta =
     customConfig?.volumes && Object.keys(customConfig?.volumes).length > 0;
 
   return `${gcloudRunCmd(
-    requiresBeta ? "beta" : undefined
+    requiresBeta ? "beta" : undefined,
   )} deploy ${fullServiceName} ${argsString}`;
 };
 
 export const getServiceDeleteScript = (
   context: Context,
-  serviceSuffix?: string
+  serviceSuffix?: string,
 ) => {
   const commonArgs = getCommonCloudRunArgs(context);
 

package/src/deploy/cloudRun/createJobs/common.ts

@@ -16,7 +16,7 @@ export const gcloudSchedulerCmd = (version?: "beta") => {
 };
 
 export const setGoogleProjectNumberScript = (
-  deployConfig: DeployConfigCloudRun
+  deployConfig: DeployConfigCloudRun,
 ) => [
   `export GCLOUD_PROJECT_NUMBER=$(${gcloudCmd()} projects describe ${
     deployConfig.projectId

package/src/deploy/cloudRun/createJobs/constants.ts

@@ -0,0 +1 @@
+export const ENV_VARS_FILENAME = "____envvars.yaml";

package/src/deploy/cloudRun/createJobs/getCloudRunDeployScripts.ts

@@ -1,4 +1,5 @@
-import type { Context } from "../../../types/context";
+import { isNil, omit } from "lodash";
+import type { Context, UnspecifiedEnvVars } from "../../../types/context";
 import { collapseableSection } from "../../../utils/gitlab";
 import { getDependencyTrackUploadScript } from "../../sbom";
 
@@ -15,22 +16,39 @@ import {
   getCloudRunDeployConfig,
   setGoogleProjectNumberScript,
 } from "./common";
+import { GCLOUD_DEPLOY_CREDENTIALS_KEY } from "..";
+import type { StringOrBashExpression } from "../../../bash/BashExpression";
+import { BashExpression, bashEscape } from "../../../bash/BashExpression";
+import { ENV_VARS_FILENAME } from "./constants";
+import {
+  writeBashYamlToFileScript,
+  yamlBashString,
+} from "../../../bash/bashYaml";
 
 export function getCloudRunDeployScripts(context: Context) {
   const deployConfig = getCloudRunDeployConfig(context);
+  const allEnvVars = omit(
+    context.environment.envVars,
+    GCLOUD_DEPLOY_CREDENTIALS_KEY,
+  );
+
   return [
     ...collapseableSection(
       "prepare",
-      "Prepare..."
+      "Prepare...",
     )([
       ...gcloudServiceAccountLoginCommands(context),
       ...setGoogleProjectNumberScript(deployConfig),
     ]),
+    ...collapseableSection(
+      "writeenvvars",
+      "Write env vars to file",
+    )(writeBashYamlToFileScript(allEnvVars, ENV_VARS_FILENAME)),
+
     ...collapseableSection(
       "deploy",
-      "Deploy to cloud run"
+      "Deploy to cloud run",
     )([
-      `echo "$ENV_VARS" > ____envvars.yaml`, // TODO: split secrets out
       ...(deployConfig.cloudSql
         ? getDatabaseCreateScript(context, deployConfig) // we create the db, so that we can also delete it afterwards
         : []),
@@ -43,15 +61,15 @@ export function getCloudRunDeployScripts(context: Context) {
         : []),
       ...Object.entries(deployConfig.additionalServices ?? {}).map(
         ([name, service]) =>
-          getServiceDeployScript(context, service, "-" + name)
+          getServiceDeployScript(context, service, "-" + name),
       ),
       ...getJobRunScripts(context, "postDeploy"),
     ]),
     ...collapseableSection(
       "cleanup",
-      "Cleanup"
+      "Cleanup",
     )(
-      getRemoveOldRevisionsAndImagesCommand(context, "postDeploy") // we cleanup inactive images both on deploy and stop
+      getRemoveOldRevisionsAndImagesCommand(context, "postDeploy"), // we cleanup inactive images both on deploy and stop
     ),
     ...getDependencyTrackUploadScript(context),
   ];

package/src/deploy/cloudRun/createJobs/getCloudRunStopScripts.ts

@@ -19,7 +19,7 @@ export function getCloudRunStopScripts(context: Context) {
     ...getJobRunScripts(context, "preStop"),
     ...(deployConfig.service !== false ? getServiceDeleteScript(context) : []),
     ...Object.entries(deployConfig.additionalServices ?? {}).flatMap(([name]) =>
-      getServiceDeleteScript(context, name)
+      getServiceDeleteScript(context, name),
     ),
     ...getJobRunScripts(context, "postStop"),
     ...getDeleteSchedulesScripts(context),

package/src/deploy/cloudRun/createJobs/index.ts

@@ -1,6 +1,3 @@
-import { merge } from "lodash";
-
-import { getDockerJobBaseProps } from "../../../build/docker";
 import { getRunnerImage } from "../../../runner";
 import type { Context } from "../../../types/context";
 import type { CatladderJob } from "../../../types/jobs";
@@ -8,12 +5,12 @@ import { allowFailureInScripts } from "../../../utils/gitlab";
 import { createDeployementJobs } from "../../base";
 
 import { isOfDeployType } from "../../types";
-import { getDeployJobVariables } from "./variables";
+import { ENV_VARS_FILENAME } from "./constants";
 import { getCloudRunDeployScripts } from "./getCloudRunDeployScripts";
 import { getCloudRunStopScripts } from "./getCloudRunStopScripts";
 
 export const createGoogleCloudRunDeployJobs = (
-  context: Context
+  context: Context,
 ): CatladderJob[] => {
   const deployConfig = context.componentConfig.deploy;
   if (deployConfig === false) {
@@ -28,12 +25,24 @@ export const createGoogleCloudRunDeployJobs = (
   const stopScripts = getCloudRunStopScripts(context);
 
   return createDeployementJobs(context, {
-    deploy: merge(getDockerJobBaseProps(context), {
-      artifacts: { paths: ["____envvars.yaml"] },
-      variables: getDeployJobVariables(context),
+    deploy: {
+      variables: {
+        CLOUDSDK_CORE_DISABLE_PROMPTS: "1",
+      },
+      ...(deployConfig.debug
+        ? {
+            artifacts: {
+              paths: [
+                ENV_VARS_FILENAME, // debug
+              ],
+              when: "always",
+            },
+          }
+        : {}),
+
       image: getRunnerImage("gcloud"),
       script: deployScripts,
-    }),
+    },
     stop: {
       image: getRunnerImage("gcloud"),
       variables: {

package/src/deploy/cloudRun/createJobs/volumes.ts

@@ -3,7 +3,7 @@ import type { keyValuesArg } from "../utils/createArgsString";
 
 export const createVolumeConfig = (
   volumes: DeployConfigCloudRunVolumes | undefined,
-  type: "service" | "job"
+  type: "service" | "job",
 ): keyValuesArg[] => {
   if (!volumes) {
     return [];
@@ -16,7 +16,7 @@ export const createVolumeConfig = (
           readonly ? ",readonly=true" : ""
         }`,
         "add-volume-mount": `volume=${volumeName},mount-path=${mountPath}`,
-      })
+      }),
     ),
     type === "service" ? { "execution-environment": "gen2" } : {},
   ];