@catladder/pipeline 0.0.0-more-collapse-deeb54d7 → 0.0.0-multiple-gitlab-includes-3df5566d

package/src/context/getLabels.ts

@@ -1,5 +1,5 @@
 import slugify from "slugify";
-import type { Context } from "../types";
+import type { ComponentContext } from "../types";
 
 const sanitize = (value?: string) => {
   if (!value) return value;
@@ -7,14 +7,14 @@ const sanitize = (value?: string) => {
   // slugify should do the job
   return slugify(value).toLowerCase();
 };
-export const getLabels = (context: Context) => {
+export const getLabels = (context: ComponentContext) => {
   const labels = {
     "customer-name": sanitize(context.fullConfig.customerName),
-    "component-name": sanitize(context.componentName),
+    "component-name": sanitize(context.name),
     "app-name": sanitize(context.fullConfig.appName),
     "env-type": sanitize(context.environment.envType),
-    "env-name": sanitize(context.environment.shortName),
-    "build-type": sanitize(context.componentConfig.build?.type),
+    "env-name": sanitize(context.env),
+    "build-type": sanitize(context.build.buildType),
     ...(context.fullConfig.meta?.labels ?? {}),
   };
   return labels;

package/src/context/getReviewSlug.ts

@@ -0,0 +1,27 @@
+import {
+  BashExpression,
+  type StringOrBashExpression,
+} from "../bash/BashExpression";
+import type { BashExpressionPerPipelineType } from "../bash/bashExpressionPerPipelineType";
+import { getBashExpressionPerPipelineType } from "../bash/bashExpressionPerPipelineType";
+import type { PipelineType } from "../types";
+import type { EnvConfigWithComponent } from "../types/config";
+import { getEnvType } from "./getEnvType";
+const REVIEW_SLUG: BashExpressionPerPipelineType = {
+  default: "unknown-review-slug",
+  gitlab: new BashExpression(
+    `$([ -n "$CI_MERGE_REQUEST_IID" ] && echo "mr$CI_MERGE_REQUEST_IID" || { [ -n "$CI_COMMIT_REF_SLUG" ] && echo "$CI_COMMIT_REF_SLUG" || echo "unknown"; })`,
+  ),
+};
+
+export const getReviewSlug = (
+  envConfig: EnvConfigWithComponent,
+  env: string,
+  pipelineType?: PipelineType,
+): StringOrBashExpression | null => {
+  const envType = getEnvType(env, envConfig);
+  if (envType === "review") {
+    return getBashExpressionPerPipelineType(REVIEW_SLUG, pipelineType);
+  }
+  return null; // not a review app;
+};

package/src/context/index.ts

@@ -1,67 +1,3 @@
-import { BUILD_TYPES } from "../build";
-import type { BuildConfig, BuildConfigType } from "../build/types";
-import { DEPLOY_TYPES } from "../deploy";
-import type { DeployConfig, DeployConfigType } from "../deploy/types";
-import type { Config } from "../types/config";
-import type { CommitInfo, Context, PackageManagerInfo } from "../types/context";
-import type { PartialDeep } from "../types/utils";
-import { mergeWithMergingArrays } from "../utils";
-import { getEnvironment } from "./getEnvironment";
-import { getEnvironmentContext } from "./getEnvironmentContext";
-
+export * from "./createComponentContext";
 export * from "./getEnvironment";
 export * from "./getEnvironmentVariables";
-
-export const createContext = async (
-  config: Config,
-  componentName: string,
-  env: string,
-  commitInfo?: CommitInfo,
-  packageManagerInfo?: PackageManagerInfo
-): Promise<Context> => {
-  if (!/^[a-z0-9-]+$/.test(componentName)) {
-    throw new Error(
-      "componentName may only contain lower case letters, numbers and -"
-    );
-  }
-
-  const envContext = getEnvironmentContext(
-    config,
-    env,
-    componentName,
-    commitInfo
-  );
-
-  const componentConfigWithoutDefaults = envContext.envConfigRaw;
-  const defaults: {
-    build: PartialDeep<BuildConfig>;
-    deploy: PartialDeep<DeployConfig>;
-  } = componentConfigWithoutDefaults.deploy
-    ? {
-        build:
-          BUILD_TYPES[
-            componentConfigWithoutDefaults.build.type as BuildConfigType
-          ].defaults(envContext),
-        deploy:
-          DEPLOY_TYPES[
-            componentConfigWithoutDefaults.deploy.type as DeployConfigType
-          ].defaults(envContext),
-      }
-    : {
-        build: {},
-        deploy: {},
-      };
-  const componentConfig = mergeWithMergingArrays(
-    defaults,
-    componentConfigWithoutDefaults
-  );
-
-  return {
-    fullConfig: config,
-    componentConfig,
-    componentName,
-    environment: await getEnvironment(config, componentName, env, commitInfo),
-    commitInfo,
-    packageManagerInfo,
-  };
-};

package/src/context/transformJobOnlyVars.ts

@@ -1,4 +1,4 @@
-import type { EnvironmentEnvVars } from "..";
+import type { SecretEnvVar, UnspecifiedEnvVars } from "..";
 import type { EnvVars } from "../types/config";
 import {
   makeSecretEnvVarMapping,
@@ -16,8 +16,11 @@ import {
 export const transformJobOnlyVars = async (
   env: string,
   componentName: string,
-  vars: EnvVars | null
-): Promise<EnvironmentEnvVars> => {
+  vars: EnvVars | null,
+): Promise<{
+  envVars: UnspecifiedEnvVars;
+  secretEnvVarKeys: SecretEnvVar[];
+}> => {
   if (!vars) {
     return {
       envVars: {},

package/src/context/utils/envVars.ts

@@ -1,13 +1,14 @@
 import { isObject } from "lodash";
 import type { SecretEnvVar } from "../getEnvironmentVariables";
 import { getSecretVarName } from "../getEnvironmentVariables";
+import { getBashVariable } from "../../bash/BashExpression";
 
 export const stringifyValues = (obj: Record<string, unknown>) =>
   Object.fromEntries(
     Object.entries(obj).map(([key, value]) => [
       key,
       isObject(value) ? JSON.stringify(value) : `${value}`,
-    ])
+    ]),
   );
 
 export const stringListToSecreteEnvVarList = (keys: string[]): SecretEnvVar[] =>
@@ -15,12 +16,12 @@ export const stringListToSecreteEnvVarList = (keys: string[]): SecretEnvVar[] =>
 export const makeSecretEnvVarMapping = (
   env: string,
   componentName: string,
-  secretEnvVars: SecretEnvVar[]
+  secretEnvVars: SecretEnvVar[],
 ) => {
   return Object.fromEntries(
     secretEnvVars.map(({ key }) => [
       key,
-      `$${getSecretVarName(env, componentName, key)}`,
-    ])
+      getBashVariable(getSecretVarName(env, componentName, key)),
+    ]),
   );
 };

package/src/deploy/base/deploy.ts

@@ -1,10 +1,15 @@
+import { createJobCacheFromCacheConfigs } from "../../build/cache/createJobCache";
 import {
   getDockerImageVariables,
   requiresDockerBuild,
 } from "../../build/docker";
 import { SBOM_BUILD_JOB_NAME } from "../../build/sbom";
-import type { Context } from "../../types/context";
-import type { CatladderJob } from "../../types/jobs";
+import type { DeployJobDefinition } from "../../types";
+import {
+  componentContextHasWorkspaceBuild,
+  type ComponentContext,
+} from "../../types/context";
+import type { BaseStage, CatladderJob } from "../../types/jobs";
 import { sbomDeactivated } from "../sbom";
 import { contextIsStoppable } from "../utils";
 import { STOP_JOB_NAME } from "./stop";
@@ -13,43 +18,41 @@ import { DEPLOY_RUNNER_VARIABLES } from "./variables";
 
 export const DEPLOY_JOB_NAME = "🚀 Deploy";
 
-export type DeployJobDefinition = Pick<
-  CatladderJob,
-  "script" | "variables" | "image" | "cache" | "artifacts" | "services"
->;
 export const createDeployJob = (
-  context: Context,
-  jobDefinition: DeployJobDefinition
+  context: ComponentContext,
+  jobDefinition: DeployJobDefinition,
 ): CatladderJob => {
   const hasDocker = requiresDockerBuild(context);
   const isStoppable = contextIsStoppable(context);
 
   const autoStop =
     context.environment.envType === "review"
-      ? "2 week"
+      ? "1 week"
       : context.environment.envType === "dev"
-      ? "4 weeks"
-      : undefined;
+        ? "4 weeks"
+        : undefined;
+
+  const deployConfig = context.deploy?.config;
 
   // if auto or manual is configured explicitly, use that
   const whenDeployDefined =
-    context.componentConfig.deploy && context.componentConfig.deploy.when
-      ? context.componentConfig.deploy.when
-      : undefined;
+    deployConfig && deployConfig.when ? deployConfig.when : undefined;
   // otherwise auto deploy if env is not prod. If its prod, deploy automatically if stage is disabled
   const whenDeployDefault =
     context.environment.envType !== "prod"
       ? "auto" // is not stage, auto deploy
       : context.componentConfig.env?.stage === false
-      ? "auto" // is prod, but no staging, auto deploy
-      : "manual"; // manually deploy
+        ? "auto" // is prod, but no staging, auto deploy
+        : "manual"; // manually deploy
   const whenDeploy = whenDeployDefined ? whenDeployDefined : whenDeployDefault;
 
   return {
     name: DEPLOY_JOB_NAME,
     script: jobDefinition.script,
     image: jobDefinition.image,
-    cache: jobDefinition.cache,
+    cache: jobDefinition.cache
+      ? createJobCacheFromCacheConfigs(context, jobDefinition.cache)
+      : undefined,
     artifacts: jobDefinition.artifacts,
     services: jobDefinition.services,
     envMode: "stagePerEnv", // makes it easier to run manual tasks er env
@@ -58,57 +61,70 @@ export const createDeployJob = (
       ...(sbomDeactivated(context)
         ? []
         : [{ job: SBOM_BUILD_JOB_NAME, artifacts: true }]),
-      ...(context.componentConfig.deploy
-        ? context.componentConfig.deploy.waitFor?.map((c) => ({
+      ...(deployConfig
+        ? (deployConfig.waitFor?.map((c) => ({
             componentName: c,
             job: DEPLOY_JOB_NAME,
             artifacts: false,
-          })) ?? []
+          })) ?? [])
         : []),
     ],
     // we don't want to deploy when there is a broken test
     needsStages: [
-      {
-        stage: "build",
-        artifacts: hasDocker ? false : true, // we asume that no-docker deployments need build artifacts,
-      },
+      ...(componentContextHasWorkspaceBuild(context)
+        ? hasDocker // docker build is per component,
+          ? [
+              // we don't need artifacts, but have to wait for the component build
+              {
+                stage: "build" as BaseStage,
+                artifacts: false,
+              },
+            ]
+          : [
+              {
+                // pick build artifacts from workspace build
+                stage: "build" as BaseStage,
+                artifacts: true,
+                workspaceName: context.build.workspaceName,
+              },
+            ]
+        : [
+            {
+              stage: "build" as BaseStage,
+              artifacts: hasDocker ? false : true, // we asume that no-docker deployments need build artifacts,
+            },
+          ]),
       {
         stage: "test",
         artifacts: false,
+        // use test from workspace build
+        workspaceName: componentContextHasWorkspaceBuild(context)
+          ? context.build.workspaceName
+          : undefined,
       },
-    ], // workaround for https://gitlab.com/gitlab-org/gitlab/-/issues/220758
-    rules: [
-      whenDeploy === "auto"
-        ? {
-            when: "on_success",
-          }
-        : {
-            when: "manual",
-          },
     ],
+    when: whenDeploy === "auto" ? "on_success" : "manual",
+
     allow_failure: whenDeploy === "manual" ? true : false,
     stage: "deploy",
     variables: {
-      ...DEPLOY_RUNNER_VARIABLES,
       ...context.environment.envVars,
       ...(hasDocker ? getDockerImageVariables(context) : {}),
       ...context.environment.jobOnlyVars.deploy.envVars,
-      ...(context.componentConfig.deploy
-        ? context.componentConfig.deploy.extraVars ?? {}
-        : {}),
+
       ...jobDefinition.variables,
     },
-    environment: {
-      ...context.environment.gitlabEnvironment,
-      ...(isStoppable
-        ? {
-            on_stop: STOP_JOB_NAME,
-            auto_stop_in: autoStop,
-          }
-        : {}),
+    runnerVariables: {
+      ...DEPLOY_RUNNER_VARIABLES,
+      ...(jobDefinition.runnerVariables ?? {}),
+      ...(deployConfig ? (deployConfig.runnerVariables ?? {}) : {}),
     },
-    jobTags: context.componentConfig.deploy
-      ? context.componentConfig.deploy.jobTags
+    environment: isStoppable
+      ? {
+          on_stop: STOP_JOB_NAME,
+          auto_stop_in: autoStop,
+        }
       : undefined,
+    jobTags: deployConfig ? deployConfig.jobTags : undefined,
   };
 };

package/src/deploy/base/index.ts

@@ -1,16 +1,21 @@
-import type { Context } from "../../types/context";
+import type { ComponentContext } from "../../types/context";
+import type {
+  DeployJobDefinition,
+  RollbackJobDefinition,
+  StopJobDefinition,
+} from "../../types/jobDefinition";
 import type { CatladderJob } from "../../types/jobs";
-import { createDeployJob, type DeployJobDefinition } from "./deploy";
-import { createRollbackJob, type RollbackJobDefinition } from "./rollback";
-import { createStopJob, type StopJobDefinition } from "./stop";
+import { createDeployJob } from "./deploy";
+import { createRollbackJob } from "./rollback";
+import { createStopJob } from "./stop";
 
 export const createDeployementJobs = (
-  context: Context,
+  context: ComponentContext,
   definitions: {
     deploy: DeployJobDefinition;
     stop?: StopJobDefinition;
     rollback?: RollbackJobDefinition;
-  }
+  },
 ): CatladderJob[] => {
   return [
     createDeployJob(context, definitions.deploy),

package/src/deploy/base/rollback.ts

@@ -1,37 +1,32 @@
-import type { Context } from "../../types/context";
+import type { ComponentContext } from "../../types/context";
+import type { RollbackJobDefinition } from "../../types/jobDefinition";
 import type { CatladderJob } from "../../types/jobs";
 import { DEPLOY_RUNNER_VARIABLES } from "./variables";
 
 export const ROLLBACK_JOB_NAME = "↩️ Rollback ⚠️";
 
-export type RollbackJobDefinition = Pick<
-  CatladderJob,
-  "script" | "variables" | "image"
->;
 export const createRollbackJob = (
-  context: Context,
-  jobDefinition: RollbackJobDefinition
+  context: ComponentContext,
+  jobDefinition: RollbackJobDefinition,
 ): CatladderJob => {
   return {
     name: ROLLBACK_JOB_NAME,
     image: jobDefinition.image,
     envMode: "stagePerEnv", // makes it easier to run manual tasks er env
     needs: [], // can be executed even if the deploy job failed
-    rules: [
-      {
-        when: "manual",
-        allow_failure: true,
-      },
-    ],
+    allow_failure: true,
+    when: "manual",
     variables: {
-      ...DEPLOY_RUNNER_VARIABLES,
       ...context.environment.jobOnlyVars.deploy.envVars,
-      GIT_STRATEGY: "none",
       ...jobDefinition.variables,
     },
+    runnerVariables: {
+      ...DEPLOY_RUNNER_VARIABLES,
+      GIT_STRATEGY: "none",
+      ...(jobDefinition.runnerVariables ?? {}),
+    },
     stage: "rollback",
     environment: {
-      ...context.environment.gitlabEnvironment,
       action: "access",
     },
 

package/src/deploy/base/stop.ts

@@ -1,42 +1,38 @@
-import type { Context } from "../../types/context";
+import type { ComponentContext } from "../../types/context";
+import type { StopJobDefinition } from "../../types/jobDefinition";
 import type { CatladderJob } from "../../types/jobs";
 
 import { DEPLOY_RUNNER_VARIABLES } from "./variables";
 export const STOP_JOB_NAME = "🛑 Stop ⚠️";
 
-export type StopJobDefinition = Pick<
-  CatladderJob,
-  "script" | "variables" | "image"
->;
 export const createStopJob = (
-  context: Context,
-  jobDefinition: StopJobDefinition
+  context: ComponentContext,
+  jobDefinition: StopJobDefinition,
 ): CatladderJob => {
   return {
     name: STOP_JOB_NAME,
     image: jobDefinition.image,
     envMode: "stagePerEnv", // makes it easier to run manual tasks er env
     needs: [], // can be executed even if the deploy job failed
+    allow_failure: true,
+    when: "manual", // stop is always manual
     rules: [
       {
         if: "$CI_COMMIT_BRANCH =~ /^[0-9]+\\.([0-9]+|x)\\.x$/", // automatic on hotfix branches
         when: "on_success",
-        allow_failure: true,
-      },
-      {
-        when: "manual",
-        allow_failure: true,
       },
     ],
     variables: {
-      ...DEPLOY_RUNNER_VARIABLES,
       ...context.environment.jobOnlyVars.deploy.envVars,
-      GIT_STRATEGY: "none",
       ...jobDefinition.variables,
     },
+    runnerVariables: {
+      ...DEPLOY_RUNNER_VARIABLES,
+      GIT_STRATEGY: "none",
+      ...jobDefinition.runnerVariables,
+    },
     stage: "stop",
     environment: {
-      ...context.environment.gitlabEnvironment,
       action: "stop",
     },
     script: jobDefinition.script,

package/src/deploy/base/variables.ts

@@ -1,6 +1,5 @@
 export const DEPLOY_RUNNER_VARIABLES = {
-  KUBERNETES_CPU_REQUEST: "0.5",
-  KUBERNETES_CPU_LIMIT: "1",
+  KUBERNETES_CPU_REQUEST: "0.22",
   KUBERNETES_MEMORY_REQUEST: "200Mi",
   KUBERNETES_MEMORY_LIMIT: "400Mi",
 };

package/src/deploy/cloudRun/artifactsRegistry.ts

@@ -0,0 +1,126 @@
+import type { StringOrBashExpression } from "../../bash/BashExpression";
+import { joinBashExpressions } from "../../bash/BashExpression";
+import type { ComponentContext } from "../../types/context";
+import { allowFailureInScripts } from "../../utils/gitlab";
+import { isOfDeployType } from "../types";
+import { removeFirstLinesFromCommandOutput } from "./utils/removeFirstLinesFromCommandOutput";
+
+export const getArtifactsRegistryHost = (context: ComponentContext) => {
+  if (!isOfDeployType(context.deploy?.config, "google-cloudrun")) {
+    // should not happen
+    throw new Error("deploy config is wrong");
+  }
+  return `${context.deploy.config.region}-docker.pkg.dev`;
+};
+
+export const getArtifactsRegistryDockerUrl = (context: ComponentContext) => {
+  const deployConfig = context.deploy?.config;
+
+  if (!isOfDeployType(deployConfig, "google-cloudrun")) {
+    // should not happen
+    throw new Error("deploy config is wrong");
+  }
+
+  const fullAppName = `${context.fullConfig.customerName}-${context.fullConfig.appName}`;
+
+  return `${getArtifactsRegistryHost(context)}/${
+    deployConfig.projectId
+  }/catladder-deploy/${fullAppName}`;
+};
+
+/**
+ *
+ * lecacyReviewImageName is only temporary. In old versions the images had no reviewslug in review apps, which makes cleanup harder. We delete all those images now, but need the path
+ */
+export const getArtifactsRegistryImageName = (
+  context: ComponentContext,
+  lecacyReviewImageName = false,
+) => {
+  if (lecacyReviewImageName && context.environment.envType !== "review") {
+    throw new Error("lecacyReviewImageName is only allowed for review app");
+  }
+
+  const dockerUrl = getArtifactsRegistryDockerUrl(context);
+  const gcloudImagePath = [
+    dockerUrl,
+    context.env,
+    context.name,
+    ...(context.environment.reviewSlug && !lecacyReviewImageName
+      ? [context.environment.reviewSlug]
+      : []),
+  ];
+  return joinBashExpressions(gcloudImagePath, "/");
+};
+
+export const getArtifactsRegistryBuildCacheImage = (
+  context: ComponentContext,
+) => {
+  const dockerUrl = getArtifactsRegistryDockerUrl(context);
+  // does not include env, so that after merge, you might get more cache hits (review-->dev)
+  const gcloudImagePath = [dockerUrl, "caches", context.name];
+  return gcloudImagePath.join("/");
+};
+
+export const getArtifactsRegistryImage = (context: ComponentContext) =>
+  `${getArtifactsRegistryImageName(context)}:$DOCKER_IMAGE_TAG`;
+
+const getDeleteImageCommands = (
+  fullImageName: StringOrBashExpression,
+  keepNewest = 0,
+) => {
+  if (keepNewest === 0) {
+    // no need to list tags, we delete the whole thing
+    return [
+      `gcloud artifacts docker images delete ${fullImageName} --quiet --delete-tags`,
+    ];
+  }
+  // delete unused tags
+  const listAllImagesCommand = `gcloud artifacts docker images list ${fullImageName} --sort-by=~CREATE_TIME --format="value(version)"`;
+
+  const listImagesToDeletecommand = removeFirstLinesFromCommandOutput(
+    listAllImagesCommand,
+    keepNewest,
+  );
+  const deleteImageCommand = `gcloud artifacts docker images delete ${fullImageName}@$version --quiet --delete-tags`;
+
+  const deleteImagesCommand = `${listImagesToDeletecommand} | while read -r version; do ${deleteImageCommand}; done`;
+
+  return [deleteImagesCommand];
+};
+
+/**
+ * commands to delete unused images.
+ *
+ * @param context
+ * @param keep how many of the newest images to keep
+ * @returns
+ */
+export const getDeleteUnusedImagesCommands = (
+  context: ComponentContext,
+  keep = 0,
+) => {
+  const deployConfig = context.deploy?.config;
+  if (!isOfDeployType(deployConfig, "google-cloudrun")) {
+    // should not happen
+    throw new Error("deploy config is wrong");
+  }
+
+  const fullImageName = getArtifactsRegistryImageName(context);
+  const buildCacheImageName = getArtifactsRegistryBuildCacheImage(context);
+
+  return [
+    ...getDeleteImageCommands(fullImageName, keep),
+    ...getDeleteImageCommands(buildCacheImageName, 1),
+    // because a recent version of catladder had no review-slug in the image name, we have to delete those as well
+    // we can later remove this line in some months
+
+    ...(context.environment.envType === "review"
+      ? allowFailureInScripts(
+          getDeleteImageCommands(
+            getArtifactsRegistryImageName(context, true),
+            0,
+          ),
+        )
+      : []),
+  ];
+};

package/src/deploy/cloudRun/cleanup.ts

@@ -0,0 +1,30 @@
+import type { ComponentContext } from "../../types/context";
+import { getDeleteUnusedImagesCommands } from "./artifactsRegistry";
+import { getDeleteUnusedRevisionsCommands } from "./cloudRunRevisions";
+
+export const getRemoveOldRevisionsAndImagesCommand = (
+  context: ComponentContext,
+  when: "postDeploy" | "onStop",
+) => {
+  if (when === "onStop") {
+    // service is already deleted, so we don't need to delete old revisions, just delete all images
+    return getDeleteUnusedImagesCommands(context);
+  }
+
+  // this number only targets inactive revisions
+  const revisionsToKeep = context.environment.envType === "prod" ? 5 : 0;
+
+  // this number needs to be higher than inactive after deploy, so we add one
+  const imagesToKeep = revisionsToKeep + 1;
+
+  const deleteOldRevisionsCommands = getDeleteUnusedRevisionsCommands(
+    context,
+    revisionsToKeep,
+  );
+  const deleteOldImagesCommands = getDeleteUnusedImagesCommands(
+    context,
+    imagesToKeep,
+  );
+
+  return [...deleteOldRevisionsCommands, ...deleteOldImagesCommands];
+};