@catladder/pipeline 0.0.0-update-docker-for-apps-e047cc44 → 0.0.0-update-node-b0c7282d

package/src/deploy/cloudRun/artifactsRegistry.ts

@@ -1,9 +1,33 @@
-import { gitlabDockerLogin } from "../../build/docker";
 import type { Context } from "../../types/context";
 import { allowFailureInScripts } from "../../utils/gitlab";
 import { isOfDeployType } from "../types";
 import { removeFirstLinesFromCommandOutput } from "./utils/removeFirstLinesFromCommandOutput";
 
+export const getArtifactsRegistryHost = ({
+  componentConfig: { deploy },
+}: Context) => {
+  if (!isOfDeployType(deploy, "google-cloudrun")) {
+    // should not happen
+    throw new Error("deploy config is wrong");
+  }
+  return `${deploy.region}-docker.pkg.dev`;
+};
+
+export const getArtifactsRegistryDockerUrl = (context: Context) => {
+  const deployConfig = context.componentConfig.deploy;
+
+  if (!isOfDeployType(deployConfig, "google-cloudrun")) {
+    // should not happen
+    throw new Error("deploy config is wrong");
+  }
+
+  const fullAppName = `${context.fullConfig.customerName}-${context.fullConfig.appName}`;
+
+  return `${getArtifactsRegistryHost(context)}/${
+    deployConfig.projectId
+  }/catladder-deploy/${fullAppName}`;
+};
+
 /**
  *
  * lecacyReviewImageName is only temporary. In old versions the images had no reviewslug in review apps, which makes cleanup harder. We delete all those images now, but need the path
@@ -12,19 +36,11 @@ export const getArtifactsRegistryImageName = (
   context: Context,
   lecacyReviewImageName = false
 ) => {
-  const deployConfig = context.componentConfig.deploy;
-
-  if (!isOfDeployType(deployConfig, "google-cloudrun")) {
-    // should not happen
-    throw new Error("deploy config is wrong");
-  }
-
   if (lecacyReviewImageName && context.environment.envType !== "review") {
     throw new Error("lecacyReviewImageName is only allowed for review app");
   }
-  const fullAppName = `${context.fullConfig.customerName}-${context.fullConfig.appName}`;
 
-  const dockerUrl = `${deployConfig.region}-docker.pkg.dev/${deployConfig.projectId}/catladder-deploy/${fullAppName}`;
+  const dockerUrl = getArtifactsRegistryDockerUrl(context);
   const gcloudImagePath = [
     dockerUrl,
     context.environment.shortName,
@@ -37,31 +53,16 @@ export const getArtifactsRegistryImageName = (
   return gcloudImagePath.join("/");
 };
 
+export const getArtifactsRegistryBuildCacheImage = (context: Context) => {
+  const dockerUrl = getArtifactsRegistryDockerUrl(context);
+  // does not include env, so that after merge, you might get more cache hits (review-->dev)
+  const gcloudImagePath = [dockerUrl, "caches", context.componentName];
+  return gcloudImagePath.join("/");
+};
+
 export const getArtifactsRegistryImage = (context: Context) =>
   `${getArtifactsRegistryImageName(context)}:$DOCKER_IMAGE_TAG`;
 
-/**
- * commands to pull the image from the default registry and pushes it to google artifact registry. We might build and push directly to google artifact registry in the future
- */
-export const getPushToArtifactsRegistryCommands = (context: Context) => {
-  const deployConfig = context.componentConfig.deploy;
-
-  if (!isOfDeployType(deployConfig, "google-cloudrun")) {
-    // should not happen
-    throw new Error("deploy config is wrong");
-  }
-
-  const fullImageName = getArtifactsRegistryImageName(context);
-
-  return [
-    gitlabDockerLogin,
-    `gcloud auth configure-docker ${deployConfig.region}-docker.pkg.dev`,
-    `docker pull $DOCKER_IMAGE:$DOCKER_IMAGE_TAG`,
-    `docker tag $DOCKER_IMAGE:$DOCKER_IMAGE_TAG ${fullImageName}:$DOCKER_IMAGE_TAG`,
-    `docker push ${fullImageName}:$DOCKER_IMAGE_TAG`,
-  ];
-};
-
 const getDeleteImageCommands = (fullImageName: string, keepNewest = 0) => {
   if (keepNewest === 0) {
     // no need to list tags, we delete the whole thing
@@ -101,9 +102,11 @@ export const getDeleteUnusedImagesCommands = (context: Context, keep = 0) => {
   }
 
   const fullImageName = getArtifactsRegistryImageName(context);
+  const buildCacheImageName = getArtifactsRegistryBuildCacheImage(context);
 
   return [
     ...getDeleteImageCommands(fullImageName, keep),
+    ...getDeleteImageCommands(buildCacheImageName, 1),
     // because a recent version of catladder had no review-slug in the image name, we have to delete those as well
     // we can later remove this line in some months
 

package/src/deploy/cloudRun/deployJob.ts

@@ -26,14 +26,11 @@ import {
 } from "./utils/database";
 import { gcloudServiceAccountLoginCommands } from "./utils/gcloudServiceAccountLoginCommands";
 import { getCloudRunJobName } from "./utils/jobName";
-import {
-  getArtifactsRegistryImage,
-  getPushToArtifactsRegistryCommands,
-} from "./artifactsRegistry";
+import { getArtifactsRegistryImage } from "./artifactsRegistry";
 import { getServiceName } from "./utils/getServiceName";
 import { getRemoveOldRevisionsAndImagesCommand } from "./cleanup";
 
-const setExtraVarsScripts = (deployConfig: DeployConfigCloudRun) => [
+const setGoogleProjectNumberScript = (deployConfig: DeployConfigCloudRun) => [
   `export GCLOUD_PROJECT_NUMBER=$(gcloud projects describe ${deployConfig.projectId} --format="value(projectNumber)")`,
   `echo "GCLOUD_PROJECT_NUMBER: $GCLOUD_PROJECT_NUMBER"`,
 ];
@@ -112,9 +109,15 @@ export const createGoogleCloudRunDeployJobs = (
         ? service?.command ?? context.componentConfig.build.startCommand
         : undefined;
 
+    const commandArray = command
+      ? Array.isArray(command)
+        ? command
+        : command.split(" ")
+      : undefined;
+
     const argsString = createArgsString({
       // command as empty string resets it to default (uses the image's entrypoint)
-      command: command ? '"' + command.split(" ").join(",") + '"' : '""',
+      command: commandArray ? '"' + commandArray.join(",") + '"' : '""',
       ...commonDeployArgs,
       "env-vars-file": "____envvars.yaml",
       "min-instances": customConfig?.minInstances ?? 0,
@@ -137,9 +140,15 @@ export const createGoogleCloudRunDeployJobs = (
     // lucky, update on the other hand accepts it... so let's just imediatly update it
 
     // also we cannot upsert a job, so we have to create it and catch the error and then update
+
+    const commandArray = Array.isArray(job.command)
+      ? job.command
+      : job.command.split(" ");
+
     const commonDeployArgsString = createArgsString({
-      command: '"' + job.command.split(" ").join(",") + '"',
+      command: '"' + commandArray.join(",") + '"',
       ...commonDeployArgs,
+      image: job.image || commonDeployArgs.image,
       memory: job.memory || "512Mi",
       "task-timeout": job.timeout || "10m",
     });
@@ -158,14 +167,22 @@ export const createGoogleCloudRunDeployJobs = (
       .map(({ job, jobName }) => getJobCreateScriptsForJob(jobName, job))
       .flat();
 
-  const getJobRunScriptForJob = (jobName: string) => {
-    return `gcloud beta run jobs execute ${jobName} ${commonArgsString}`;
+  const getJobRunScriptForJob = (jobName: string, wait: boolean) => {
+    return `gcloud beta run jobs execute ${jobName} ${commonArgsString}${
+      wait ? " --wait" : ""
+    }`;
   };
 
   const getJobRunScripts = (when: DeployConfigCloudRunJob["when"]) =>
     jobsWithNames
       .filter(({ job }) => job.when === when)
-      .map(({ jobName }) => getJobRunScriptForJob(jobName));
+      .map(({ jobName }) =>
+        getJobRunScriptForJob(
+          jobName,
+          // wait for completin on stop jobs, since stop will delete the jobs afterwards, so they will fail
+          ["preStop", "postStop"].includes(when)
+        )
+      );
 
   const getCreateScheduleScripts = () => {
     return jobsWithSchedule
@@ -217,12 +234,8 @@ export const createGoogleCloudRunDeployJobs = (
       "Prepare..."
     )([
       ...gcloudServiceAccountLoginCommands(context),
-      ...setExtraVarsScripts(deployConfig),
+      ...setGoogleProjectNumberScript(deployConfig),
     ]),
-    ...collapseableSection(
-      "pushToArtifactRegistry",
-      "Pushing image to artifacts registry"
-    )(getPushToArtifactsRegistryCommands(context)),
     ...collapseableSection(
       "deploy",
       "Deploy to cloud run"
@@ -254,6 +267,7 @@ export const createGoogleCloudRunDeployJobs = (
 
   const stopScripts = [
     ...gcloudServiceAccountLoginCommands(context),
+    ...getJobRunScripts("preStop"),
     ...(deployConfig.service !== false
       ? [`gcloud run services delete ${serviceName} ${commonArgsString}`]
       : []),
@@ -261,6 +275,7 @@ export const createGoogleCloudRunDeployJobs = (
       ([name]) =>
         `gcloud run services delete ${serviceName}-${name} ${commonArgsString}`
     ),
+    ...getJobRunScripts("postStop"),
     ...getDeleteSchedulesScripts(),
     ...getDeleteJobsScripts(),
     ...(deployConfig.cloudSql && deployConfig.cloudSql.deleteDatabaseOnStop

package/src/deploy/dockerTag/deployJob.ts

@@ -19,7 +19,7 @@ export const createDockerTagDeployJobs = (context: Context): CatladderJob[] => {
     deploy: {
       ...getDockerJobBaseProps(context),
       script: [
-        gitlabDockerLogin,
+        ...gitlabDockerLogin(context),
         `docker pull $DOCKER_IMAGE:$DOCKER_IMAGE_TAG`,
         `docker tag $DOCKER_IMAGE:$DOCKER_IMAGE_TAG $DOCKER_IMAGE:${tag}`,
         `docker push $DOCKER_IMAGE:${tag}`,

package/src/deploy/types/googleCloudRun.ts

@@ -78,7 +78,7 @@ export type DeployConfigCloudRunService = {
   /**
    * command / entrypoint, fallsback to buildConfig.startcommand
    */
-  command?: string;
+  command?: string | string[];
 
   /**
    * how many instances to keep around when there are no requests. defaults to 0.
@@ -123,7 +123,12 @@ export type DeployConfigCloudRunJobBase = {
   /**
    * command
    */
-  command: string;
+  command: string | string[];
+
+  /**
+   * custom image to use. Defaults to the image from the build
+   */
+  image?: string;
 
   /**
    * memory limit of the job, defaults to 51Mi
@@ -151,7 +156,7 @@ export type DeployConfigCloudRunJobWithSchedule =
   };
 
 export type DeployConfigCloudRunJobNormal = DeployConfigCloudRunJobBase & {
-  when: "manual" | "preDeploy" | "postDeploy";
+  when: "manual" | "preDeploy" | "postDeploy" | "preStop" | "postStop";
 };
 
 export type DeployConfigCloudRunJob =
@@ -160,9 +165,9 @@ export type DeployConfigCloudRunJob =
 
 export type DeployConfigCloudRun = {
   /**
-   * EXPERIMENTAL cloud run deployment.
+   * cloud run deployment.
    *
-   * This will deploy a cloud run service.
+   * This will deploy a cloud run service. You can optionally define one-time and scheduled jobs running on cloud run
    *
    * Requires that cloud run is enabled on the project, as well as cloud run api and artifacts registry.
    */
@@ -182,7 +187,7 @@ export type DeployConfigCloudRun = {
   service?: boolean | DeployConfigCloudRunService;
 
   /**
-   * deploy additional services with different entry pointss
+   * deploy additional services with different entry points. 
    */
   additionalServices?: {
     [name: string]: DeployConfigCloudRunService;