@catladder/pipeline 1.124.0 → 1.124.2

package/src/build/docker.ts

@@ -6,6 +6,12 @@ import type { CatladderJob } from "../types/jobs";
 import { existsSync } from "fs";
 import path from "path";
 import { collapseableSection } from "../utils/gitlab";
+import {
+  getArtifactsRegistryBuildCacheImage,
+  getArtifactsRegistryHost,
+  getArtifactsRegistryImageName,
+} from "../deploy/cloudRun/artifactsRegistry";
+import { gcloudServiceAccountLoginCommands } from "../deploy/cloudRun/utils/gcloudServiceAccountLoginCommands";
 
 const DOCKER_RUNNER_BUILD_VARIABLES = {
   KUBERNETES_CPU_REQUEST: "0.5",
@@ -16,13 +22,24 @@ const DOCKER_RUNNER_BUILD_VARIABLES = {
 
 export const getDockerImageVariables = (context: Context) => {
   return {
-    DOCKER_REGISTRY: "$CI_REGISTRY",
-    DOCKER_REGISTRY_IMAGE_PATH: "$CI_REGISTRY_IMAGE",
-    DOCKER_CACHE_IMAGE:
-      "$DOCKER_REGISTRY_IMAGE_PATH/caches/" + context.componentName,
-    DOCKER_IMAGE_NAME:
-      context.environment.shortName + "/" + context.componentName,
-    DOCKER_IMAGE: "$DOCKER_REGISTRY_IMAGE_PATH/$DOCKER_IMAGE_NAME",
+    ...(isOfDeployType(context.componentConfig.deploy, "google-cloudrun")
+      ? {
+          DOCKER_REGISTRY: getArtifactsRegistryHost(context),
+          DOCKER_IMAGE: getArtifactsRegistryImageName(context),
+          DOCKER_CACHE_IMAGE: getArtifactsRegistryBuildCacheImage(context),
+        }
+      : // gitlab registry:
+        {
+          DOCKER_REGISTRY: "$CI_REGISTRY",
+
+          DOCKER_CACHE_IMAGE:
+            "$CI_REGISTRY_IMAGE/caches/" + context.componentName,
+          // ONLY USED IN KUBERNETES
+          DOCKER_IMAGE_NAME:
+            context.environment.shortName + "/" + context.componentName,
+          DOCKER_IMAGE: "$CI_REGISTRY_IMAGE/$DOCKER_IMAGE_NAME",
+        }),
+
     DOCKER_IMAGE_TAG: "$CI_COMMIT_SHA",
   };
 };
@@ -94,13 +111,26 @@ export const createDockerBuildJobBase = (
   );
 };
 
-export const gitlabDockerLogin =
-  "docker login --username gitlab-ci-token --password $CI_JOB_TOKEN $CI_REGISTRY";
+export const gitlabDockerLogin = (context: Context) =>
+  isOfDeployType(context.componentConfig.deploy, "google-cloudrun")
+    ? [
+        ...gcloudServiceAccountLoginCommands(context),
+        `gcloud auth configure-docker ${getArtifactsRegistryHost(context)}`,
+      ]
+    : [
+        "docker login --username gitlab-ci-token --password $CI_JOB_TOKEN $CI_REGISTRY",
+      ];
 
-export const getDockerBuildDefaultScript = (ensureDockerFileScript?: string) =>
+export const getDockerBuildDefaultScript = (
+  context: Context,
+  ensureDockerFileScript?: string
+) =>
   [
     ensureDockerFileScript,
-    ...collapseableSection("docker-login", "Docker Login")([gitlabDockerLogin]),
+    ...collapseableSection(
+      "docker-login",
+      "Docker Login"
+    )(gitlabDockerLogin(context)),
     ...collapseableSection(
       "docker-build",
       "Docker build"

package/src/build/node/buildJob.ts

@@ -60,6 +60,7 @@ export const createNodeBuildJobs = (context: Context): CatladderJob[] => {
 
     dockerBuild: {
       script: getDockerBuildDefaultScript(
+        context,
         buildConfig.type === "node-static" || buildConfig.type === "storybook"
           ? "ensureNginxDockerfile"
           : "ensureNodeDockerfile"

package/src/build/node/meteor.ts

@@ -64,7 +64,7 @@ export const createMeteorBuildJobs = (context: Context): CatladderJob[] => {
           }
         : undefined,
     dockerBuild: {
-      script: getDockerBuildDefaultScript("ensureMeteorDockerfile"),
+      script: getDockerBuildDefaultScript(context, "ensureMeteorDockerfile"),
       variables: {
         METEOR_INSTALL_SCRIPTS: buildConfig.installScripts ? "true" : "",
       },

package/src/build/rails/build.ts

@@ -19,7 +19,7 @@ export const createRailsBuildJobs = (context: Context): CatladderJob[] => {
     return createBuildJobs(context, {
       appBuild: undefined,
       dockerBuild: {
-        script: getDockerBuildDefaultScript(),
+        script: getDockerBuildDefaultScript(context),
         variables: {},
       },
     });
@@ -39,7 +39,7 @@ export const createRailsBuildJobs = (context: Context): CatladderJob[] => {
       },
       // custom script
       script: [
-        gitlabDockerLogin,
+        ...gitlabDockerLogin(context),
         `docker pull $DOCKER_CACHE_IMAGE || true`,
         `wget --output-document=- https://github.com/buildpacks/pack/releases/download/v${cnbConf?.packVersion}/pack-v${cnbConf?.packVersion}-linux.tgz | tar -zx --directory /usr/local/bin pack`,
         `chmod +x /usr/local/bin/pack`,

package/src/deploy/cloudRun/artifactsRegistry.ts

@@ -1,9 +1,33 @@
-import { gitlabDockerLogin } from "../../build/docker";
 import type { Context } from "../../types/context";
 import { allowFailureInScripts } from "../../utils/gitlab";
 import { isOfDeployType } from "../types";
 import { removeFirstLinesFromCommandOutput } from "./utils/removeFirstLinesFromCommandOutput";
 
+export const getArtifactsRegistryHost = ({
+  componentConfig: { deploy },
+}: Context) => {
+  if (!isOfDeployType(deploy, "google-cloudrun")) {
+    // should not happen
+    throw new Error("deploy config is wrong");
+  }
+  return `${deploy.region}-docker.pkg.dev`;
+};
+
+export const getArtifactsRegistryDockerUrl = (context: Context) => {
+  const deployConfig = context.componentConfig.deploy;
+
+  if (!isOfDeployType(deployConfig, "google-cloudrun")) {
+    // should not happen
+    throw new Error("deploy config is wrong");
+  }
+
+  const fullAppName = `${context.fullConfig.customerName}-${context.fullConfig.appName}`;
+
+  return `${getArtifactsRegistryHost(context)}/${
+    deployConfig.projectId
+  }/catladder-deploy/${fullAppName}`;
+};
+
 /**
  *
  * lecacyReviewImageName is only temporary. In old versions the images had no reviewslug in review apps, which makes cleanup harder. We delete all those images now, but need the path
@@ -12,19 +36,11 @@ export const getArtifactsRegistryImageName = (
   context: Context,
   lecacyReviewImageName = false
 ) => {
-  const deployConfig = context.componentConfig.deploy;
-
-  if (!isOfDeployType(deployConfig, "google-cloudrun")) {
-    // should not happen
-    throw new Error("deploy config is wrong");
-  }
-
   if (lecacyReviewImageName && context.environment.envType !== "review") {
     throw new Error("lecacyReviewImageName is only allowed for review app");
   }
-  const fullAppName = `${context.fullConfig.customerName}-${context.fullConfig.appName}`;
 
-  const dockerUrl = `${deployConfig.region}-docker.pkg.dev/${deployConfig.projectId}/catladder-deploy/${fullAppName}`;
+  const dockerUrl = getArtifactsRegistryDockerUrl(context);
   const gcloudImagePath = [
     dockerUrl,
     context.environment.shortName,
@@ -37,31 +53,16 @@ export const getArtifactsRegistryImageName = (
   return gcloudImagePath.join("/");
 };
 
+export const getArtifactsRegistryBuildCacheImage = (context: Context) => {
+  const dockerUrl = getArtifactsRegistryDockerUrl(context);
+  // does not include env, so that after merge, you might get more cache hits (review-->dev)
+  const gcloudImagePath = [dockerUrl, "caches", context.componentName];
+  return gcloudImagePath.join("/");
+};
+
 export const getArtifactsRegistryImage = (context: Context) =>
   `${getArtifactsRegistryImageName(context)}:$DOCKER_IMAGE_TAG`;
 
-/**
- * commands to pull the image from the default registry and pushes it to google artifact registry. We might build and push directly to google artifact registry in the future
- */
-export const getPushToArtifactsRegistryCommands = (context: Context) => {
-  const deployConfig = context.componentConfig.deploy;
-
-  if (!isOfDeployType(deployConfig, "google-cloudrun")) {
-    // should not happen
-    throw new Error("deploy config is wrong");
-  }
-
-  const fullImageName = getArtifactsRegistryImageName(context);
-
-  return [
-    gitlabDockerLogin,
-    `gcloud auth configure-docker ${deployConfig.region}-docker.pkg.dev`,
-    `docker pull $DOCKER_IMAGE:$DOCKER_IMAGE_TAG`,
-    `docker tag $DOCKER_IMAGE:$DOCKER_IMAGE_TAG ${fullImageName}:$DOCKER_IMAGE_TAG`,
-    `docker push ${fullImageName}:$DOCKER_IMAGE_TAG`,
-  ];
-};
-
 const getDeleteImageCommands = (fullImageName: string, keepNewest = 0) => {
   if (keepNewest === 0) {
     // no need to list tags, we delete the whole thing
@@ -101,9 +102,11 @@ export const getDeleteUnusedImagesCommands = (context: Context, keep = 0) => {
   }
 
   const fullImageName = getArtifactsRegistryImageName(context);
+  const buildCacheImageName = getArtifactsRegistryBuildCacheImage(context);
 
   return [
     ...getDeleteImageCommands(fullImageName, keep),
+    ...getDeleteImageCommands(buildCacheImageName, 1),
     // because a recent version of catladder had no review-slug in the image name, we have to delete those as well
     // we can later remove this line in some months
 

package/src/deploy/cloudRun/deployJob.ts

@@ -26,10 +26,7 @@ import {
 } from "./utils/database";
 import { gcloudServiceAccountLoginCommands } from "./utils/gcloudServiceAccountLoginCommands";
 import { getCloudRunJobName } from "./utils/jobName";
-import {
-  getArtifactsRegistryImage,
-  getPushToArtifactsRegistryCommands,
-} from "./artifactsRegistry";
+import { getArtifactsRegistryImage } from "./artifactsRegistry";
 import { getServiceName } from "./utils/getServiceName";
 import { getRemoveOldRevisionsAndImagesCommand } from "./cleanup";
 
@@ -219,10 +216,6 @@ export const createGoogleCloudRunDeployJobs = (
       ...gcloudServiceAccountLoginCommands(context),
       ...setExtraVarsScripts(deployConfig),
     ]),
-    ...collapseableSection(
-      "pushToArtifactRegistry",
-      "Pushing image to artifacts registry"
-    )(getPushToArtifactsRegistryCommands(context)),
     ...collapseableSection(
       "deploy",
       "Deploy to cloud run"

package/src/deploy/dockerTag/deployJob.ts

@@ -19,7 +19,7 @@ export const createDockerTagDeployJobs = (context: Context): CatladderJob[] => {
     deploy: {
       ...getDockerJobBaseProps(context),
       script: [
-        gitlabDockerLogin,
+        ...gitlabDockerLogin(context),
         `docker pull $DOCKER_IMAGE:$DOCKER_IMAGE_TAG`,
         `docker tag $DOCKER_IMAGE:$DOCKER_IMAGE_TAG $DOCKER_IMAGE:${tag}`,
         `docker push $DOCKER_IMAGE:${tag}`,