@catladder/pipeline 0.0.0-node16-ddfae617 → 0.0.0-perf-cli-reduce-bundle-size-with-tsconfig-optimization-3556aaa4

package/src/context/utils/envVars.ts

@@ -0,0 +1,27 @@
+import { isObject } from "lodash";
+import type { SecretEnvVar } from "../getEnvironmentVariables";
+import { getSecretVarName } from "../getEnvironmentVariables";
+import { getBashVariable } from "../../bash/BashExpression";
+
+export const stringifyValues = (obj: Record<string, unknown>) =>
+  Object.fromEntries(
+    Object.entries(obj).map(([key, value]) => [
+      key,
+      isObject(value) ? JSON.stringify(value) : `${value}`,
+    ]),
+  );
+
+export const stringListToSecreteEnvVarList = (keys: string[]): SecretEnvVar[] =>
+  keys.map((key) => ({ key }));
+export const makeSecretEnvVarMapping = (
+  env: string,
+  componentName: string,
+  secretEnvVars: SecretEnvVar[],
+) => {
+  return Object.fromEntries(
+    secretEnvVars.map(({ key }) => [
+      key,
+      getBashVariable(getSecretVarName(env, componentName, key)),
+    ]),
+  );
+};

package/src/defaults/index.ts

@@ -1,4 +1,4 @@
-import { Retry } from "../types";
+import type { Retry } from "../types";
 
 export const BASE_RETRY: Retry = {
   max: 2,

package/src/deploy/base/deploy.ts

@@ -0,0 +1,136 @@
+import {
+  getDockerImageVariables,
+  requiresDockerBuild,
+} from "../../build/docker";
+import { SBOM_BUILD_JOB_NAME } from "../../build/sbom";
+import {
+  componentContextHasWorkspaceBuild,
+  type ComponentContext,
+} from "../../types/context";
+import type { BaseStage, CatladderJob } from "../../types/jobs";
+import { sbomDeactivated } from "../sbom";
+import { contextIsStoppable } from "../utils";
+import { STOP_JOB_NAME } from "./stop";
+
+import { DEPLOY_RUNNER_VARIABLES } from "./variables";
+
+export const DEPLOY_JOB_NAME = "🚀 Deploy";
+
+export type DeployJobDefinition = Pick<
+  CatladderJob,
+  | "script"
+  | "variables"
+  | "image"
+  | "cache"
+  | "artifacts"
+  | "services"
+  | "runnerVariables"
+>;
+export const createDeployJob = (
+  context: ComponentContext,
+  jobDefinition: DeployJobDefinition,
+): CatladderJob => {
+  const hasDocker = requiresDockerBuild(context);
+  const isStoppable = contextIsStoppable(context);
+
+  const autoStop =
+    context.environment.envType === "review"
+      ? "1 week"
+      : context.environment.envType === "dev"
+        ? "4 weeks"
+        : undefined;
+
+  const deployConfig = context.deploy?.config;
+
+  // if auto or manual is configured explicitly, use that
+  const whenDeployDefined =
+    deployConfig && deployConfig.when ? deployConfig.when : undefined;
+  // otherwise auto deploy if env is not prod. If its prod, deploy automatically if stage is disabled
+  const whenDeployDefault =
+    context.environment.envType !== "prod"
+      ? "auto" // is not stage, auto deploy
+      : context.componentConfig.env?.stage === false
+        ? "auto" // is prod, but no staging, auto deploy
+        : "manual"; // manually deploy
+  const whenDeploy = whenDeployDefined ? whenDeployDefined : whenDeployDefault;
+
+  return {
+    name: DEPLOY_JOB_NAME,
+    script: jobDefinition.script,
+    image: jobDefinition.image,
+    cache: jobDefinition.cache,
+    artifacts: jobDefinition.artifacts,
+    services: jobDefinition.services,
+    envMode: "stagePerEnv", // makes it easier to run manual tasks er env
+
+    needs: [
+      ...(sbomDeactivated(context)
+        ? []
+        : [{ job: SBOM_BUILD_JOB_NAME, artifacts: true }]),
+      ...(deployConfig
+        ? deployConfig.waitFor?.map((c) => ({
+            componentName: c,
+            job: DEPLOY_JOB_NAME,
+            artifacts: false,
+          })) ?? []
+        : []),
+    ],
+    // we don't want to deploy when there is a broken test
+    needsStages: [
+      ...(componentContextHasWorkspaceBuild(context)
+        ? hasDocker // docker build is per component,
+          ? [
+              // we don't need artifacts, but have to wait for the component build
+              {
+                stage: "build" as BaseStage,
+                artifacts: false,
+              },
+            ]
+          : [
+              {
+                // pick build artifacts from workspace build
+                stage: "build" as BaseStage,
+                artifacts: true,
+                workspaceName: context.build.workspaceName,
+              },
+            ]
+        : [
+            {
+              stage: "build" as BaseStage,
+              artifacts: hasDocker ? false : true, // we asume that no-docker deployments need build artifacts,
+            },
+          ]),
+      {
+        stage: "test",
+        artifacts: false,
+        // use test from workspace build
+        workspaceName: componentContextHasWorkspaceBuild(context)
+          ? context.build.workspaceName
+          : undefined,
+      },
+    ],
+    when: whenDeploy === "auto" ? "on_success" : "manual",
+
+    allow_failure: whenDeploy === "manual" ? true : false,
+    stage: "deploy",
+    variables: {
+      ...context.environment.envVars,
+      ...(hasDocker ? getDockerImageVariables(context) : {}),
+      ...context.environment.jobOnlyVars.deploy.envVars,
+      ...(deployConfig ? deployConfig.extraVars ?? {} : {}),
+      ...jobDefinition.variables,
+    },
+    runnerVariables: {
+      ...DEPLOY_RUNNER_VARIABLES,
+      ...(jobDefinition.runnerVariables ?? {}),
+      ...(deployConfig ? deployConfig.runnerVariables ?? {} : {}),
+    },
+    environment: isStoppable
+      ? {
+          on_stop: STOP_JOB_NAME,
+          auto_stop_in: autoStop,
+        }
+      : undefined,
+    jobTags: deployConfig ? deployConfig.jobTags : undefined,
+  };
+};

package/src/deploy/base/index.ts

@@ -0,0 +1,22 @@
+import type { ComponentContext } from "../../types/context";
+import type { CatladderJob } from "../../types/jobs";
+import { createDeployJob, type DeployJobDefinition } from "./deploy";
+import { createRollbackJob, type RollbackJobDefinition } from "./rollback";
+import { createStopJob, type StopJobDefinition } from "./stop";
+
+export const createDeployementJobs = (
+  context: ComponentContext,
+  definitions: {
+    deploy: DeployJobDefinition;
+    stop?: StopJobDefinition;
+    rollback?: RollbackJobDefinition;
+  },
+): CatladderJob[] => {
+  return [
+    createDeployJob(context, definitions.deploy),
+    ...(definitions.stop ? [createStopJob(context, definitions.stop)] : []),
+    ...(definitions.rollback
+      ? [createRollbackJob(context, definitions.rollback)]
+      : []),
+  ];
+};

package/src/deploy/base/rollback.ts

@@ -0,0 +1,38 @@
+import type { ComponentContext } from "../../types/context";
+import type { CatladderJob } from "../../types/jobs";
+import { DEPLOY_RUNNER_VARIABLES } from "./variables";
+
+export const ROLLBACK_JOB_NAME = "↩️ Rollback ⚠️";
+
+export type RollbackJobDefinition = Pick<
+  CatladderJob,
+  "script" | "variables" | "runnerVariables" | "image"
+>;
+export const createRollbackJob = (
+  context: ComponentContext,
+  jobDefinition: RollbackJobDefinition,
+): CatladderJob => {
+  return {
+    name: ROLLBACK_JOB_NAME,
+    image: jobDefinition.image,
+    envMode: "stagePerEnv", // makes it easier to run manual tasks er env
+    needs: [], // can be executed even if the deploy job failed
+    allow_failure: true,
+    when: "manual",
+    variables: {
+      ...context.environment.jobOnlyVars.deploy.envVars,
+      ...jobDefinition.variables,
+    },
+    runnerVariables: {
+      ...DEPLOY_RUNNER_VARIABLES,
+      GIT_STRATEGY: "none",
+      ...(jobDefinition.runnerVariables ?? {}),
+    },
+    stage: "rollback",
+    environment: {
+      action: "access",
+    },
+
+    script: jobDefinition.script,
+  };
+};

package/src/deploy/base/stop.ts

@@ -0,0 +1,43 @@
+import type { ComponentContext } from "../../types/context";
+import type { CatladderJob } from "../../types/jobs";
+
+import { DEPLOY_RUNNER_VARIABLES } from "./variables";
+export const STOP_JOB_NAME = "🛑 Stop ⚠️";
+
+export type StopJobDefinition = Pick<
+  CatladderJob,
+  "script" | "variables" | "image" | "runnerVariables"
+>;
+export const createStopJob = (
+  context: ComponentContext,
+  jobDefinition: StopJobDefinition,
+): CatladderJob => {
+  return {
+    name: STOP_JOB_NAME,
+    image: jobDefinition.image,
+    envMode: "stagePerEnv", // makes it easier to run manual tasks er env
+    needs: [], // can be executed even if the deploy job failed
+    allow_failure: true,
+    when: "manual", // stop is always manual
+    rules: [
+      {
+        if: "$CI_COMMIT_BRANCH =~ /^[0-9]+\\.([0-9]+|x)\\.x$/", // automatic on hotfix branches
+        when: "on_success",
+      },
+    ],
+    variables: {
+      ...context.environment.jobOnlyVars.deploy.envVars,
+      ...jobDefinition.variables,
+    },
+    runnerVariables: {
+      ...DEPLOY_RUNNER_VARIABLES,
+      GIT_STRATEGY: "none",
+      ...jobDefinition.runnerVariables,
+    },
+    stage: "stop",
+    environment: {
+      action: "stop",
+    },
+    script: jobDefinition.script,
+  };
+};

package/src/deploy/base/types.ts

@@ -0,0 +1,3 @@
+import type { CatladderJob } from "../../types/jobs";
+
+export type JobWithoutScript = Omit<CatladderJob, "script">;

package/src/deploy/base/variables.ts

@@ -0,0 +1,5 @@
+export const DEPLOY_RUNNER_VARIABLES = {
+  KUBERNETES_CPU_REQUEST: "0.22",
+  KUBERNETES_MEMORY_REQUEST: "200Mi",
+  KUBERNETES_MEMORY_LIMIT: "400Mi",
+};

package/src/deploy/cloudRun/artifactsRegistry.ts

@@ -0,0 +1,126 @@
+import type { StringOrBashExpression } from "../../bash/BashExpression";
+import { joinBashExpressions } from "../../bash/BashExpression";
+import type { ComponentContext } from "../../types/context";
+import { allowFailureInScripts } from "../../utils/gitlab";
+import { isOfDeployType } from "../types";
+import { removeFirstLinesFromCommandOutput } from "./utils/removeFirstLinesFromCommandOutput";
+
+export const getArtifactsRegistryHost = (context: ComponentContext) => {
+  if (!isOfDeployType(context.deploy?.config, "google-cloudrun")) {
+    // should not happen
+    throw new Error("deploy config is wrong");
+  }
+  return `${context.deploy.config.region}-docker.pkg.dev`;
+};
+
+export const getArtifactsRegistryDockerUrl = (context: ComponentContext) => {
+  const deployConfig = context.deploy?.config;
+
+  if (!isOfDeployType(deployConfig, "google-cloudrun")) {
+    // should not happen
+    throw new Error("deploy config is wrong");
+  }
+
+  const fullAppName = `${context.fullConfig.customerName}-${context.fullConfig.appName}`;
+
+  return `${getArtifactsRegistryHost(context)}/${
+    deployConfig.projectId
+  }/catladder-deploy/${fullAppName}`;
+};
+
+/**
+ *
+ * lecacyReviewImageName is only temporary. In old versions the images had no reviewslug in review apps, which makes cleanup harder. We delete all those images now, but need the path
+ */
+export const getArtifactsRegistryImageName = (
+  context: ComponentContext,
+  lecacyReviewImageName = false,
+) => {
+  if (lecacyReviewImageName && context.environment.envType !== "review") {
+    throw new Error("lecacyReviewImageName is only allowed for review app");
+  }
+
+  const dockerUrl = getArtifactsRegistryDockerUrl(context);
+  const gcloudImagePath = [
+    dockerUrl,
+    context.env,
+    context.name,
+    ...(context.environment.reviewSlug && !lecacyReviewImageName
+      ? [context.environment.reviewSlug]
+      : []),
+  ];
+  return joinBashExpressions(gcloudImagePath, "/");
+};
+
+export const getArtifactsRegistryBuildCacheImage = (
+  context: ComponentContext,
+) => {
+  const dockerUrl = getArtifactsRegistryDockerUrl(context);
+  // does not include env, so that after merge, you might get more cache hits (review-->dev)
+  const gcloudImagePath = [dockerUrl, "caches", context.name];
+  return gcloudImagePath.join("/");
+};
+
+export const getArtifactsRegistryImage = (context: ComponentContext) =>
+  `${getArtifactsRegistryImageName(context)}:$DOCKER_IMAGE_TAG`;
+
+const getDeleteImageCommands = (
+  fullImageName: StringOrBashExpression,
+  keepNewest = 0,
+) => {
+  if (keepNewest === 0) {
+    // no need to list tags, we delete the whole thing
+    return [
+      `gcloud artifacts docker images delete ${fullImageName} --quiet --delete-tags`,
+    ];
+  }
+  // delete unused tags
+  const listAllImagesCommand = `gcloud artifacts docker images list ${fullImageName} --sort-by=~CREATE_TIME --format="value(version)"`;
+
+  const listImagesToDeletecommand = removeFirstLinesFromCommandOutput(
+    listAllImagesCommand,
+    keepNewest,
+  );
+  const deleteImageCommand = `gcloud artifacts docker images delete ${fullImageName}@$version --quiet --delete-tags`;
+
+  const deleteImagesCommand = `${listImagesToDeletecommand} | while read -r version; do ${deleteImageCommand}; done`;
+
+  return [deleteImagesCommand];
+};
+
+/**
+ * commands to delete unused images.
+ *
+ * @param context
+ * @param keep how many of the newest images to keep
+ * @returns
+ */
+export const getDeleteUnusedImagesCommands = (
+  context: ComponentContext,
+  keep = 0,
+) => {
+  const deployConfig = context.deploy?.config;
+  if (!isOfDeployType(deployConfig, "google-cloudrun")) {
+    // should not happen
+    throw new Error("deploy config is wrong");
+  }
+
+  const fullImageName = getArtifactsRegistryImageName(context);
+  const buildCacheImageName = getArtifactsRegistryBuildCacheImage(context);
+
+  return [
+    ...getDeleteImageCommands(fullImageName, keep),
+    ...getDeleteImageCommands(buildCacheImageName, 1),
+    // because a recent version of catladder had no review-slug in the image name, we have to delete those as well
+    // we can later remove this line in some months
+
+    ...(context.environment.envType === "review"
+      ? allowFailureInScripts(
+          getDeleteImageCommands(
+            getArtifactsRegistryImageName(context, true),
+            0,
+          ),
+        )
+      : []),
+  ];
+};

package/src/deploy/cloudRun/cleanup.ts

@@ -0,0 +1,30 @@
+import type { ComponentContext } from "../../types/context";
+import { getDeleteUnusedImagesCommands } from "./artifactsRegistry";
+import { getDeleteUnusedRevisionsCommands } from "./cloudRunRevisions";
+
+export const getRemoveOldRevisionsAndImagesCommand = (
+  context: ComponentContext,
+  when: "postDeploy" | "onStop",
+) => {
+  if (when === "onStop") {
+    // service is already deleted, so we don't need to delete old revisions, just delete all images
+    return getDeleteUnusedImagesCommands(context);
+  }
+
+  // this number only targets inactive revisions
+  const revisionsToKeep = context.environment.envType === "prod" ? 5 : 0;
+
+  // this number needs to be higher than inactive after deploy, so we add one
+  const imagesToKeep = revisionsToKeep + 1;
+
+  const deleteOldRevisionsCommands = getDeleteUnusedRevisionsCommands(
+    context,
+    revisionsToKeep,
+  );
+  const deleteOldImagesCommands = getDeleteUnusedImagesCommands(
+    context,
+    imagesToKeep,
+  );
+
+  return [...deleteOldRevisionsCommands, ...deleteOldImagesCommands];
+};

package/src/deploy/cloudRun/cloudRunRevisions.ts

@@ -0,0 +1,65 @@
+import type { ComponentContext } from "../../types/context";
+import { isOfDeployType } from "../types";
+import { createArgsString } from "./utils/createArgsString";
+import { getServiceName } from "./utils/getServiceName";
+import { removeFirstLinesFromCommandOutput } from "./utils/removeFirstLinesFromCommandOutput";
+
+const getListRevisionsCommand = (
+  context: ComponentContext,
+  args: {
+    filter?: string;
+    format: string;
+  },
+) => {
+  const serviceName = getServiceName(context);
+
+  const deployConfig = context.deploy?.config;
+
+  if (!deployConfig || !isOfDeployType(deployConfig, "google-cloudrun")) {
+    // should not happen
+    throw new Error("deploy config is wrong");
+  }
+  const filterRevisionArgs = {
+    project: deployConfig.projectId,
+    region: deployConfig.region,
+    // only show inactive revisions
+    service: serviceName,
+
+    limit: "unlimited", // is actualy the default
+    "sort-by": "metadata.creationTimestamp",
+    ...args,
+  };
+
+  // this prints out all inactive images of the given app
+  return `gcloud run revisions list ${createArgsString(filterRevisionArgs)}`;
+};
+
+export const getDeleteUnusedRevisionsCommands = (
+  context: ComponentContext,
+  keep: number,
+) => {
+  const deployConfig = context.deploy?.config;
+  if (!isOfDeployType(deployConfig, "google-cloudrun")) {
+    // should not happen
+    throw new Error("deploy config is wrong");
+  }
+
+  // this prints out all inactive images of the given app
+  const listAllInactiveRevisionsCmd = getListRevisionsCommand(context, {
+    format: `"value(name)"`,
+    filter: `'(status.conditions.status=False OR status.conditions.status=Unknown)'`,
+  });
+  // this removes the newest `revisionsToKeep` images
+  const listRevisionsToDeleteCmd = removeFirstLinesFromCommandOutput(
+    listAllInactiveRevisionsCmd,
+    keep,
+  );
+  const deleteRevisionCmd = `gcloud run revisions delete ${createArgsString({
+    project: deployConfig.projectId,
+    region: deployConfig.region,
+    quiet: true,
+  })} $revisionname `;
+
+  const deleteRevisionsCmd = `${listRevisionsToDeleteCmd} | while read -r revisionname; do ${deleteRevisionCmd}; done`;
+  return [deleteRevisionsCmd];
+};