@catladder/pipeline 0.0.0-refactore-catenv-command-validates-argument-78d05852 → 0.0.0-reference-database-url-1befbbee

package/src/deploy/base/rollback.ts

@@ -1,37 +1,32 @@
-import type { Context } from "../../types/context";
+import type { ComponentContext } from "../../types/context";
+import type { RollbackJobDefinition } from "../../types/jobDefinition";
 import type { CatladderJob } from "../../types/jobs";
 import { DEPLOY_RUNNER_VARIABLES } from "./variables";
 
 export const ROLLBACK_JOB_NAME = "↩️ Rollback ⚠️";
 
-export type RollbackJobDefinition = Pick<
-  CatladderJob,
-  "script" | "variables" | "image"
->;
 export const createRollbackJob = (
-  context: Context,
-  jobDefinition: RollbackJobDefinition
+  context: ComponentContext,
+  jobDefinition: RollbackJobDefinition,
 ): CatladderJob => {
   return {
     name: ROLLBACK_JOB_NAME,
     image: jobDefinition.image,
     envMode: "stagePerEnv", // makes it easier to run manual tasks er env
     needs: [], // can be executed even if the deploy job failed
-    rules: [
-      {
-        when: "manual",
-        allow_failure: true,
-      },
-    ],
+    allow_failure: true,
+    when: "manual",
     variables: {
-      ...DEPLOY_RUNNER_VARIABLES,
       ...context.environment.jobOnlyVars.deploy.envVars,
-      GIT_STRATEGY: "none",
       ...jobDefinition.variables,
     },
+    runnerVariables: {
+      ...DEPLOY_RUNNER_VARIABLES,
+      GIT_STRATEGY: "none",
+      ...(jobDefinition.runnerVariables ?? {}),
+    },
     stage: "rollback",
     environment: {
-      ...context.environment.gitlabEnvironment,
       action: "access",
     },
 

package/src/deploy/base/stop.ts

@@ -1,42 +1,38 @@
-import type { Context } from "../../types/context";
+import type { ComponentContext } from "../../types/context";
+import type { StopJobDefinition } from "../../types/jobDefinition";
 import type { CatladderJob } from "../../types/jobs";
 
 import { DEPLOY_RUNNER_VARIABLES } from "./variables";
 export const STOP_JOB_NAME = "🛑 Stop ⚠️";
 
-export type StopJobDefinition = Pick<
-  CatladderJob,
-  "script" | "variables" | "image"
->;
 export const createStopJob = (
-  context: Context,
-  jobDefinition: StopJobDefinition
+  context: ComponentContext,
+  jobDefinition: StopJobDefinition,
 ): CatladderJob => {
   return {
     name: STOP_JOB_NAME,
     image: jobDefinition.image,
     envMode: "stagePerEnv", // makes it easier to run manual tasks er env
     needs: [], // can be executed even if the deploy job failed
+    allow_failure: true,
+    when: "manual", // stop is always manual
     rules: [
       {
         if: "$CI_COMMIT_BRANCH =~ /^[0-9]+\\.([0-9]+|x)\\.x$/", // automatic on hotfix branches
         when: "on_success",
-        allow_failure: true,
-      },
-      {
-        when: "manual",
-        allow_failure: true,
       },
     ],
     variables: {
-      ...DEPLOY_RUNNER_VARIABLES,
       ...context.environment.jobOnlyVars.deploy.envVars,
-      GIT_STRATEGY: "none",
       ...jobDefinition.variables,
     },
+    runnerVariables: {
+      ...DEPLOY_RUNNER_VARIABLES,
+      GIT_STRATEGY: "none",
+      ...jobDefinition.runnerVariables,
+    },
     stage: "stop",
     environment: {
-      ...context.environment.gitlabEnvironment,
       action: "stop",
     },
     script: jobDefinition.script,

package/src/deploy/base/variables.ts

@@ -1,5 +1,5 @@
 export const DEPLOY_RUNNER_VARIABLES = {
-  KUBERNETES_CPU_REQUEST: "0.5",
+  KUBERNETES_CPU_REQUEST: "0.22",
   KUBERNETES_MEMORY_REQUEST: "200Mi",
   KUBERNETES_MEMORY_LIMIT: "400Mi",
 };

package/src/deploy/cloudRun/artifactsRegistry.ts

@@ -1,20 +1,20 @@
-import type { Context } from "../../types/context";
+import type { StringOrBashExpression } from "../../bash/BashExpression";
+import { joinBashExpressions } from "../../bash/BashExpression";
+import type { ComponentContext } from "../../types/context";
 import { allowFailureInScripts } from "../../utils/gitlab";
 import { isOfDeployType } from "../types";
 import { removeFirstLinesFromCommandOutput } from "./utils/removeFirstLinesFromCommandOutput";
 
-export const getArtifactsRegistryHost = ({
-  componentConfig: { deploy },
-}: Context) => {
-  if (!isOfDeployType(deploy, "google-cloudrun")) {
+export const getArtifactsRegistryHost = (context: ComponentContext) => {
+  if (!isOfDeployType(context.deploy?.config, "google-cloudrun")) {
     // should not happen
     throw new Error("deploy config is wrong");
   }
-  return `${deploy.region}-docker.pkg.dev`;
+  return `${context.deploy.config.region}-docker.pkg.dev`;
 };
 
-export const getArtifactsRegistryDockerUrl = (context: Context) => {
-  const deployConfig = context.componentConfig.deploy;
+export const getArtifactsRegistryDockerUrl = (context: ComponentContext) => {
+  const deployConfig = context.deploy?.config;
 
   if (!isOfDeployType(deployConfig, "google-cloudrun")) {
     // should not happen
@@ -33,8 +33,8 @@ export const getArtifactsRegistryDockerUrl = (context: Context) => {
  * lecacyReviewImageName is only temporary. In old versions the images had no reviewslug in review apps, which makes cleanup harder. We delete all those images now, but need the path
  */
 export const getArtifactsRegistryImageName = (
-  context: Context,
-  lecacyReviewImageName = false
+  context: ComponentContext,
+  lecacyReviewImageName = false,
 ) => {
   if (lecacyReviewImageName && context.environment.envType !== "review") {
     throw new Error("lecacyReviewImageName is only allowed for review app");
@@ -43,27 +43,31 @@ export const getArtifactsRegistryImageName = (
   const dockerUrl = getArtifactsRegistryDockerUrl(context);
   const gcloudImagePath = [
     dockerUrl,
-    context.environment.shortName,
-    context.componentName,
-
-    ...(context.environment.envType === "review" && !lecacyReviewImageName
-      ? [context.commitInfo?.reviewSlug]
+    context.env,
+    context.name,
+    ...(context.environment.reviewSlug && !lecacyReviewImageName
+      ? [context.environment.reviewSlug]
       : []),
   ];
-  return gcloudImagePath.join("/");
+  return joinBashExpressions(gcloudImagePath, "/");
 };
 
-export const getArtifactsRegistryBuildCacheImage = (context: Context) => {
+export const getArtifactsRegistryBuildCacheImage = (
+  context: ComponentContext,
+) => {
   const dockerUrl = getArtifactsRegistryDockerUrl(context);
   // does not include env, so that after merge, you might get more cache hits (review-->dev)
-  const gcloudImagePath = [dockerUrl, "caches", context.componentName];
+  const gcloudImagePath = [dockerUrl, "caches", context.name];
   return gcloudImagePath.join("/");
 };
 
-export const getArtifactsRegistryImage = (context: Context) =>
+export const getArtifactsRegistryImage = (context: ComponentContext) =>
   `${getArtifactsRegistryImageName(context)}:$DOCKER_IMAGE_TAG`;
 
-const getDeleteImageCommands = (fullImageName: string, keepNewest = 0) => {
+const getDeleteImageCommands = (
+  fullImageName: StringOrBashExpression,
+  keepNewest = 0,
+) => {
   if (keepNewest === 0) {
     // no need to list tags, we delete the whole thing
     return [
@@ -75,7 +79,7 @@ const getDeleteImageCommands = (fullImageName: string, keepNewest = 0) => {
 
   const listImagesToDeletecommand = removeFirstLinesFromCommandOutput(
     listAllImagesCommand,
-    keepNewest
+    keepNewest,
   );
   const deleteImageCommand = `gcloud artifacts docker images delete ${fullImageName}@$version --quiet --delete-tags`;
 
@@ -91,11 +95,11 @@ const getDeleteImageCommands = (fullImageName: string, keepNewest = 0) => {
  * @param keep how many of the newest images to keep
  * @returns
  */
-export const getDeleteUnusedImagesCommands = (context: Context, keep = 0) => {
-  const deployConfig = context.componentConfig.deploy;
-  if (deployConfig === false) {
-    return [];
-  }
+export const getDeleteUnusedImagesCommands = (
+  context: ComponentContext,
+  keep = 0,
+) => {
+  const deployConfig = context.deploy?.config;
   if (!isOfDeployType(deployConfig, "google-cloudrun")) {
     // should not happen
     throw new Error("deploy config is wrong");
@@ -114,8 +118,8 @@ export const getDeleteUnusedImagesCommands = (context: Context, keep = 0) => {
       ? allowFailureInScripts(
           getDeleteImageCommands(
             getArtifactsRegistryImageName(context, true),
-            0
-          )
+            0,
+          ),
         )
       : []),
   ];

package/src/deploy/cloudRun/cleanup.ts

@@ -1,10 +1,10 @@
-import type { Context } from "../../types/context";
+import type { ComponentContext } from "../../types/context";
 import { getDeleteUnusedImagesCommands } from "./artifactsRegistry";
 import { getDeleteUnusedRevisionsCommands } from "./cloudRunRevisions";
 
 export const getRemoveOldRevisionsAndImagesCommand = (
-  context: Context,
-  when: "postDeploy" | "onStop"
+  context: ComponentContext,
+  when: "postDeploy" | "onStop",
 ) => {
   if (when === "onStop") {
     // service is already deleted, so we don't need to delete old revisions, just delete all images
@@ -19,11 +19,11 @@ export const getRemoveOldRevisionsAndImagesCommand = (
 
   const deleteOldRevisionsCommands = getDeleteUnusedRevisionsCommands(
     context,
-    revisionsToKeep
+    revisionsToKeep,
   );
   const deleteOldImagesCommands = getDeleteUnusedImagesCommands(
     context,
-    imagesToKeep
+    imagesToKeep,
   );
 
   return [...deleteOldRevisionsCommands, ...deleteOldImagesCommands];

package/src/deploy/cloudRun/cloudRunRevisions.ts

@@ -1,18 +1,19 @@
-import type { Context } from "../../types/context";
+import type { ComponentContext } from "../../types/context";
 import { isOfDeployType } from "../types";
 import { createArgsString } from "./utils/createArgsString";
 import { getServiceName } from "./utils/getServiceName";
 import { removeFirstLinesFromCommandOutput } from "./utils/removeFirstLinesFromCommandOutput";
 
 const getListRevisionsCommand = (
-  context: Context,
+  context: ComponentContext,
   args: {
     filter?: string;
     format: string;
-  }
+  },
 ) => {
   const serviceName = getServiceName(context);
-  const deployConfig = context.componentConfig.deploy;
+
+  const deployConfig = context.deploy?.config;
 
   if (!deployConfig || !isOfDeployType(deployConfig, "google-cloudrun")) {
     // should not happen
@@ -34,13 +35,10 @@ const getListRevisionsCommand = (
 };
 
 export const getDeleteUnusedRevisionsCommands = (
-  context: Context,
-  keep: number
+  context: ComponentContext,
+  keep: number,
 ) => {
-  const deployConfig = context.componentConfig.deploy;
-  if (deployConfig === false) {
-    return [];
-  }
+  const deployConfig = context.deploy?.config;
   if (!isOfDeployType(deployConfig, "google-cloudrun")) {
     // should not happen
     throw new Error("deploy config is wrong");
@@ -54,7 +52,7 @@ export const getDeleteUnusedRevisionsCommands = (
   // this removes the newest `revisionsToKeep` images
   const listRevisionsToDeleteCmd = removeFirstLinesFromCommandOutput(
     listAllInactiveRevisionsCmd,
-    keep
+    keep,
   );
   const deleteRevisionCmd = `gcloud run revisions delete ${createArgsString({
     project: deployConfig.projectId,

package/src/deploy/cloudRun/createJobs/cloudRunJobs.ts

@@ -1,183 +1,101 @@
-import type { Context } from "../../../types/context";
-import { allowFailureInScripts } from "../../../utils/gitlab";
+import type { ComponentContext } from "../../../types/context";
 
-import type {
-  DeployConfigCloudRunJob,
-  DeployConfigCloudRunJobWithSchedule,
-} from "../../types/googleCloudRun";
+import { getLabels } from "../../../context/getLabels";
+import { notNil } from "../../../utils";
+import type { DeployConfigCloudRunJob } from "../../types/googleCloudRun";
 import { createArgsString } from "../utils/createArgsString";
-import { getCloudRunJobName } from "../utils/jobName";
+import { getFullJobName } from "../utils/jobName";
 import {
   gcloudRunCmd,
-  gcloudSchedulerCmd,
   getCloudRunDeployConfig,
   getCommonCloudRunArgs,
   getCommonDeployArgs,
   makeLabelString,
 } from "./common";
-import { getLabels } from "../../../context/getLabels";
+import { ENV_VARS_FILENAME } from "./constants";
+import { createVolumeConfig } from "./volumes";
+import { getCloudRunServiceOrJobArgsArg } from "../utils/getJobOrServiceArgs";
 
-const getJobRunScriptForJob = (
-  context: Context,
-  jobName: string,
-  wait: boolean
-) => {
-  const commonArgs = getCommonCloudRunArgs(context);
-
-  const commonArgsString = createArgsString(commonArgs);
-  return `${gcloudRunCmd()} jobs execute ${jobName} ${commonArgsString}${
-    wait ? " --wait" : ""
-  }`;
-};
-
-export const getDeleteSchedulesScripts = (context: Context) => {
-  const deployConfig = getCloudRunDeployConfig(context);
-  const jobsWithSchedule = getCloudRunJobsWithSchedule(context);
-  const argsString = createArgsString({
-    project: deployConfig.projectId,
-    location: deployConfig.region,
-  });
-  return jobsWithSchedule
-    .map(({ schedulerName }) => {
-      return [
-        `${gcloudSchedulerCmd()} jobs delete ${schedulerName} ${argsString}`,
-      ];
-    })
-    .flat();
-};
-
-export const getDeleteJobsScripts = (context: Context) => {
+export const getDeleteJobsScripts = (context: ComponentContext) => {
   const commonArgs = getCommonCloudRunArgs(context);
   const commonArgsString = createArgsString(commonArgs);
   const jobsWithNames = getCloudRunJobsWithNames(context);
 
-  return jobsWithNames.flatMap(({ jobName }) => [
+  return jobsWithNames.flatMap(({ fullJobName }) => [
     // first delete all job executions. Otherwise delete might fail if one of those is still running
-    `${gcloudRunCmd()} jobs executions list ${commonArgsString} --job ${jobName} --format="value(name)" | xargs -I {} ${gcloudRunCmd()} jobs executions delete {}  --quiet ${commonArgsString}`,
-    `${gcloudRunCmd()} jobs delete ${jobName} ${commonArgsString}`,
+    `${gcloudRunCmd()} jobs executions list ${commonArgsString} --job ${fullJobName} --format="value(name)" | xargs -I {} ${gcloudRunCmd()} jobs executions delete {}  --quiet ${commonArgsString}`,
+    `${gcloudRunCmd()} jobs delete ${fullJobName} ${commonArgsString}`,
   ]);
 };
 
-export const getJobRunScripts = (
-  context: Context,
-  when: DeployConfigCloudRunJob["when"]
-) => {
-  const jobsWithNames = getCloudRunJobsWithNames(context);
-  return jobsWithNames
-    .filter(({ job }) => job.when === when)
-    .map(({ jobName }) =>
-      getJobRunScriptForJob(
-        context,
-        jobName,
-        // wait for completin on stop jobs, since stop will delete the jobs afterwards, so they will fail
-        ["preStop", "postStop"].includes(when)
-      )
-    );
-};
-
-export const getJobCreateScripts = (context: Context) => {
-  const jobsWithNames = getCloudRunJobsWithNames(context);
-
-  return jobsWithNames
-    .map(({ job, jobName }) => getJobCreateScriptsForJob(context, jobName, job))
-    .flat();
-};
-
-const getJobCreateScriptsForJob = (
-  context: Context,
-  jobName: string,
-  job: DeployConfigCloudRunJob
-) => {
-  const commonDeployArgs = getCommonDeployArgs(context);
-
-  // we cannot upsert a job, so we have to create it and catch the error and then update
-  const commandArray = Array.isArray(job.command)
-    ? job.command
-    : job.command.split(" ");
+export const getJobCreateScripts = (context: ComponentContext): string[] =>
+  getCloudRunJobsWithNames(context).map(
+    ({ job, fullJobName }, jobIndex): string => {
+      const commandArray = Array.isArray(job.command)
+        ? job.command
+        : job.command.split(" ");
+
+      const {
+        image: commonImage,
+        project,
+        region,
+        ...deployArgs
+      } = getCommonDeployArgs(context);
+      const commonDeployArgsString = createArgsString(
+        {
+          command: `"${commandArray.join(",")}"`,
+          args: getCloudRunServiceOrJobArgsArg(job.args),
+          labels: `"${makeLabelString(getLabels(context))},cloud-run-job-name=$current_job_name"`,
+          image: `"${job.image ?? commonImage}"`,
+          project,
+          region,
+          cpu: job.cpu,
+          memory: job.memory ?? "512Mi",
+          parallelism: job.parallelism ?? 1,
+
+          "task-timeout": job.timeout ?? "10m",
+          "env-vars-file": ENV_VARS_FILENAME,
+          "max-retries": job.maxRetries ?? 0,
+
+          ...deployArgs,
+
+          // network
+          "vpc-connector": job?.vpcConnector,
+          "vpc-egress": job?.vpcEgress,
+          network: job?.network,
+          subnet: job?.subnet,
+        },
+        ...createVolumeConfig(job.volumes, "job"),
+      );
 
-  const commonDeployArgsString = createArgsString({
-    command: '"' + commandArray.join(",") + '"',
-    ...commonDeployArgs,
-    labels: makeLabelString({
-      ...getLabels(context),
-      "cloud-run-job-name": jobName,
-    }),
-    image: job.image || commonDeployArgs.image,
-    memory: job.memory || "512Mi",
-    "task-timeout": job.timeout || "10m",
-    parallelism: job.parallelism || 1,
-    "env-vars-file": "____envvars.yaml",
-    "max-retries": 0,
-  });
-
-  const argsString = `${jobName} ${commonDeployArgsString}`;
-  return [
-    ...allowFailureInScripts([`${gcloudRunCmd()} jobs create ${argsString}`]),
-    `${gcloudRunCmd()} jobs update ${argsString}`,
-  ];
-};
-
-export const getCreateScheduleScripts = (context: Context) => {
-  const jobsWithSchedule = getCloudRunJobsWithSchedule(context);
-  const deployConfig = getCloudRunDeployConfig(context);
-
-  return jobsWithSchedule
-    .map(({ job, jobName, schedulerName }) => {
-      const argsString = createArgsString({
-        project: deployConfig.projectId,
-        location: deployConfig.region,
-        schedule: `"${job.schedule}"`,
-        "max-retry-attempts": job.maxRetryAttempts ?? 0,
-
-        uri: `"https://${deployConfig.region}-run.googleapis.com/apis/run.googleapis.com/v1/namespaces/${deployConfig.projectId}/jobs/${jobName}:run"`,
-        "http-method": "POST",
-        "oauth-service-account-email":
-          "$GCLOUD_PROJECT_NUMBER-compute@developer.gserviceaccount.com",
-      });
-      const commonArgs = `http ${schedulerName} ${argsString}`;
       return [
-        ...allowFailureInScripts([
-          `${gcloudSchedulerCmd()} jobs create ${commonArgs}`,
-        ]),
-        `${gcloudSchedulerCmd()} jobs update ${commonArgs}`,
-      ];
-    })
-    .flat();
-};
-
-const getCloudRunJobsWithSchedule = (context: Context) => {
-  const jobsWithNames = getCloudRunJobsWithNames(context);
-
-  return jobsWithNames
-    .filter(
-      (
-        entry
-      ): entry is {
-        jobName: string;
-        job: DeployConfigCloudRunJobWithSchedule;
-      } => entry.job.when === "schedule"
-    )
-    .map(({ job, jobName }) => ({
-      job,
-      jobName,
-      schedulerName: jobName + "-scheduler",
-    }));
-};
-
-const getCloudRunJobsWithNames = (context: Context) => {
+        jobIndex === 0
+          ? `exist_job_names="$(\n  ${gcloudRunCmd()} jobs list --filter='metadata.name ~ ${context.env}.*${context.name}' --format='value(name)' --limit=999 --project='${project}' --region='${region}'\n)"`
+          : null,
+        `current_job_name="${fullJobName}"`,
+        'if grep "$current_job_name" <<<"$exist_job_names" >/dev/null; then',
+        `  ${gcloudRunCmd()} jobs update "$current_job_name" ${commonDeployArgsString}`,
+        "else",
+        `  ${gcloudRunCmd()} jobs create "$current_job_name" ${commonDeployArgsString}`,
+        "fi",
+      ]
+        .filter(notNil)
+        .join("\n");
+    },
+  );
+
+export const getCloudRunJobsWithNames = (context: ComponentContext) => {
   const deployConfig = getCloudRunDeployConfig(context);
 
-  const getFullJobName = (name: string) =>
-    getCloudRunJobName(context.environment.fullName, name);
-
   const jobsWithNames = Object.entries(deployConfig.jobs ?? {})
     // filter out disabled jobs
     .filter((entry): entry is [string, DeployConfigCloudRunJob] =>
-      Boolean(entry[1])
+      Boolean(entry[1]),
     )
-    .map(([name, job]) => ({
-      jobName: getFullJobName(name),
+    .map(([jobName, job]) => ({
+      fullJobName: getFullJobName(context, jobName),
       job,
+      jobName,
     }));
   return jobsWithNames;
 };