@catladder/pipeline 0.0.0-node16-ddfae617 → 0.0.0-perf-cli-reduce-bundle-size-with-tsconfig-optimization-3556aaa4

package/src/deploy/kubernetes/index.ts

@@ -1 +1,50 @@
-export * from "./deployJob";
+import slugify from "slugify";
+import type { DeployConfigKubernetes, DeployTypeDefinition } from "..";
+import { getKubernetesNamespace } from "..";
+import { joinBashExpressions } from "../../bash/BashExpression";
+import { additionalKubernetesSecretKeys } from "./additionalSecretKeys";
+import { createKubernetesDeployJobs } from "./deployJob";
+export * from "./cloudSql";
+export const KUBERNETES_DEPLOY_TYPE: DeployTypeDefinition<DeployConfigKubernetes> =
+  {
+    jobs: createKubernetesDeployJobs,
+    defaults: () => ({}),
+    additionalSecretKeys: additionalKubernetesSecretKeys,
+    getAdditionalEnvVars: ({
+      componentName,
+      fullConfig,
+      deployConfigRaw,
+      env,
+      reviewSlug,
+      envType,
+    }) => {
+      const KUBE_APP_NAME_PREFIX =
+        envType === "review" && reviewSlug ? reviewSlug.concat("-") : "";
+      const KUBE_APP_NAME = KUBE_APP_NAME_PREFIX.concat(componentName);
+      const KUBE_NAMESPACE = getKubernetesNamespace(fullConfig, env);
+      const componentSlug = slugify(componentName);
+
+      const domainCanonical =
+        (deployConfigRaw && deployConfigRaw.cluster?.domainCanonical) || // for convenience, we allow clusters to define a canonical domain, because a cluster has a fixed ip and you will usually have a domain pointing to that cluster
+        fullConfig.domainCanonical ||
+        "panter.cloud";
+
+      const HOST_INTERNAL = joinBashExpressions(
+        [
+          componentSlug,
+          ...(envType === "review" && reviewSlug ? [reviewSlug] : []),
+          env,
+          fullConfig.appName,
+          fullConfig.customerName,
+          domainCanonical,
+        ],
+        ".",
+      ); // default for kubernetes and rest
+      return {
+        KUBE_NAMESPACE,
+        KUBE_APP_NAME,
+        KUBE_APP_NAME_PREFIX,
+        HOST_INTERNAL,
+      };
+    },
+  };

package/src/deploy/kubernetes/kubeEnv.ts

@@ -0,0 +1,50 @@
+import type { StringOrBashExpression } from "../../bash/BashExpression";
+import type { ComponentContext } from "../../types";
+import { isOfDeployType } from "../types";
+
+const shouldGoIntoSecrets = (key: string, value: string | undefined) => {
+  if (!value) return false;
+  if (String(value)?.includes("$CL_")) {
+    return true;
+  }
+  return false;
+};
+
+/**
+ * separate by secrets and public.
+ * we evalulate the actual values later, but want to store the secrets in kubernetes secrets
+ */
+export const createKubeEnv = (context: ComponentContext) => {
+  if (!isOfDeployType(context.deploy?.config, "kubernetes")) {
+    // should not happen
+    throw new Error("deploy config is not kubernetes");
+  }
+
+  const allEnvVars = context.environment.envVars;
+
+  const env = Object.entries(allEnvVars).reduce<{
+    secret: Record<string, StringOrBashExpression>;
+    public: Record<string, StringOrBashExpression>;
+  }>(
+    (acc, [key, value]) => {
+      if (shouldGoIntoSecrets(key, value?.toString())) {
+        acc.secret = {
+          ...acc.secret,
+          [key]: value ?? "",
+        };
+        return acc;
+      }
+      acc.public = {
+        ...acc.public,
+        [key]: value ?? "",
+      };
+      return acc;
+    },
+    {
+      secret: {},
+      public: {},
+    },
+  );
+
+  return env;
+};

package/src/deploy/kubernetes/kubeValues.ts

@@ -0,0 +1,100 @@
+import { merge } from "lodash";
+
+import type { ComponentContext } from "../../types/context";
+import { mergeWithMergingArrays } from "../../utils";
+import type { DeployConfigKubernetesValues } from "../types";
+import { isOfDeployType } from "../types";
+import {
+  hasKubernetesCloudSQL,
+  createKubernetesCloudsqlBaseValues,
+} from "./cloudSql";
+
+import { createKubeEnv } from "./kubeEnv";
+import { createMongodbBaseConfig } from "./mongodb";
+import { processSecretsAsFiles } from "./processSecretsAsFiles";
+
+const createAppConfig = (
+  context: ComponentContext,
+  application: DeployConfigKubernetesValues["application"],
+): DeployConfigKubernetesValues["application"] => {
+  if (application === false) {
+    return {
+      enabled: false,
+    };
+  }
+
+  const { healthRoute, command, ...rest } = application ?? {};
+
+  return mergeWithMergingArrays(
+    {
+      host: context.environment.host,
+      command: command ?? context.build.config.startCommand,
+      livenessProbe: {
+        httpGet: {
+          path: healthRoute ?? "__health",
+        },
+      },
+      readinessProbe: {
+        httpGet: {
+          path: healthRoute ?? "__health",
+        },
+      },
+      startupProbe: {
+        httpGet: {
+          path: healthRoute ?? "__health",
+        },
+      },
+    }, // default
+    rest, // merge rest in
+  );
+};
+
+const removeFalsy = <T>(record?: Record<string, false | T>) => {
+  if (!record) return undefined;
+  return Object.fromEntries(
+    Object.entries(record).filter(([, value]) => value !== false),
+  );
+};
+
+export const createKubeValues = (context: ComponentContext) => {
+  const deployConfig = context.deploy?.config;
+  if (!deployConfig) {
+    return [];
+  }
+  if (!isOfDeployType(deployConfig, "kubernetes")) {
+    // should not happen
+    throw new Error("deploy config is not kubernetes");
+  }
+
+  const { values } = deployConfig;
+
+  // we remove the application config because it can be just the value `false` which is a convenience feature, but not supported in the helm chart
+  // we only merge the rest of the values in
+  const { application, jobs, cronjobs, ...rest } = values ?? {};
+
+  const env = createKubeEnv(context);
+
+  const defaultKubeValues = merge(
+    {
+      env,
+      application: createAppConfig(context, application),
+    },
+    hasKubernetesCloudSQL(context)
+      ? createKubernetesCloudsqlBaseValues(context)
+      : {},
+    deployConfig.values?.mongodb?.enabled
+      ? createMongodbBaseConfig(context)
+      : {},
+  );
+
+  const kubeValues = processSecretsAsFiles(
+    mergeWithMergingArrays(defaultKubeValues, {
+      jobs: removeFalsy(jobs),
+      cronjobs: removeFalsy(cronjobs),
+
+      ...rest,
+    }),
+  );
+
+  return kubeValues;
+};

package/src/deploy/kubernetes/mongodb.ts

@@ -1,17 +1,115 @@
-import { Context } from "../..";
+import { range } from "lodash";
+import { getSecretVarNameForContext } from "../../context/getEnvironmentVariables";
+import type { ComponentContext } from "../../types/context";
+import { isOfDeployType } from "../types";
+
+const getCredentialString = (context: ComponentContext) =>
+  `root:$${getSecretVarNameForContext(context, "MONGODB_ROOT_PASSWORD")}@`;
+const getMongodbHost = (context: ComponentContext, name: string) => {
+  const namespace = context.environment.envVars.KUBE_NAMESPACE;
+  return `${name}.${namespace}.svc.cluster.local:27017`;
+};
+
+const getMongodbStandaloneHost = (context: ComponentContext) => {
+  const fullAppname = context.environment.envVars.KUBE_APP_NAME;
+  return getMongodbHost(context, `${fullAppname}-mongodb`);
+};
+
+const getMongodbReplicasetHost = (context: ComponentContext, index: number) => {
+  const fullAppname = context.environment.envVars.KUBE_APP_NAME;
+  return getMongodbHost(
+    context,
+    `${fullAppname}-mongodb-${index}.${fullAppname}-mongodb-headless`,
+  );
+};
+const createMongodbUrl = (context: ComponentContext, dbName: string) => {
+  if (!isOfDeployType(context.deploy?.config, "kubernetes")) {
+    throw new Error("can only createMongodbUrl on supported deploys");
+  }
+  const mongodbConfig = context.deploy?.config.values?.mongodb;
+
+  let queryParams: string | undefined = undefined;
+
+  let hosts = "";
+  if (mongodbConfig?.architecture === "replicaset") {
+    hosts = range(0, mongodbConfig?.replicaCount ?? 2)
+      .map((i) => getMongodbReplicasetHost(context, i))
+      .join(",");
+
+    queryParams = "replicaSet=rs0&authSource=admin";
+  } else {
+    hosts = getMongodbStandaloneHost(context);
+    queryParams = "authSource=admin";
+  }
+
+  return `mongodb://${getCredentialString(context)}${hosts}/${dbName}${
+    queryParams ? `?${queryParams}` : ""
+  }`;
+};
+const createMongoBackupDefaultConfig = (context: ComponentContext) => {
+  if (!isOfDeployType(context.deploy?.config, "kubernetes")) {
+    throw new Error("can only create mongodb base config on supported deploys");
+  }
+  const mongodbConfig = context.deploy?.config.values?.mongodb;
+  const fullAppName = context.environment.envVars.KUBE_APP_NAME;
+  const backupEnabled = ["prod", "stage"].includes(context.environment.envType);
+
+  let hostToBackup: string;
+  let pvcToBackup: string;
+
+  if (mongodbConfig?.architecture === "replicaset") {
+    // bit quirky, we need to specify the host and its volume
+    // on replicaset its probably best to not use the first one, whcih usually starts as master, so we take the second (last one would also be ok)
+    const backupHostIndex =
+      mongodbConfig?.architecture === "replicaset" &&
+      mongodbConfig.replicaCount &&
+      mongodbConfig.replicaCount > 1
+        ? 1
+        : 0;
+
+    hostToBackup = getMongodbReplicasetHost(context, backupHostIndex);
+    pvcToBackup = `datadir-${fullAppName}-mongodb-${backupHostIndex}`;
+  } else {
+    hostToBackup = getMongodbStandaloneHost(context);
+    pvcToBackup = `${fullAppName}-mongodb`;
+  }
+
+  return {
+    enabled: backupEnabled,
+    hostToBackup,
+    pvcToBackup,
+    image: "mrelite/kubectlmongoshell:v1.0",
+    schedule: "0 4 * * *",
+    volumeSnapshotClass: "snapshotclass",
+  };
+};
+export const createMongodbBaseConfig = (context: ComponentContext) => {
+  if (!isOfDeployType(context.deploy?.config, "kubernetes")) {
+    throw new Error("can only create mongodb base config on supported deploys");
+  }
+  const mongodbConfig = context.deploy?.config.values?.mongodb;
 
-export const createMongodbBaseConfig = (context: Context) => {
   return {
     mongodb: {
       enabled: true,
-      backup: {
-        enabled: ["prod", "stage"].includes(context.environment.envType),
+      auth: {
+        enabled: true,
+        rootPassword:
+          "$" + getSecretVarNameForContext(context, "MONGODB_ROOT_PASSWORD"),
+        replicaSetKey:
+          "$" + getSecretVarNameForContext(context, "MONGODB_REPLICASET_KEY"),
+      },
+      persistence: {
+        storageClass: "standard-rwo",
       },
+      backup: createMongoBackupDefaultConfig(context),
     },
-    "mongodb-replicaset": {
-      replicas: 1,
-      persistentVolume: {
-        storageClass: "standard",
+    env: {
+      secret: {
+        MONGO_URL: createMongodbUrl(context, mongodbConfig?.dbName ?? "app"),
+        ...(mongodbConfig?.architecture === "replicaset"
+          ? { MONGO_OPLOG_URL: createMongodbUrl(context, "local") } // oplog only works with replicasets
+          : {}),
       },
     },
   };

package/src/deploy/kubernetes/processSecretsAsFiles.ts

@@ -1,14 +1,15 @@
 import { omit } from "lodash";
-import { DeployConfigKubernetesValues } from "../types";
+import type { DeployConfigKubernetesValues } from "../types";
+import type { StringOrBashExpression } from "../../bash/BashExpression";
 
 export const processSecretsAsFiles = <
   T extends {
     env: {
-      secret: Record<string, string>;
+      secret: Record<string, StringOrBashExpression>;
     };
-  } & DeployConfigKubernetesValues
+  } & DeployConfigKubernetesValues,
 >(
-  values: T
+  values: T,
 ): T => {
   if (!values.secretsAsFile || values.secretsAsFile.length === 0) {
     return values;
@@ -26,7 +27,7 @@ export const processSecretsAsFiles = <
           values.secretsAsFile.map((key) => [
             key + "_content",
             values.env.secret[key],
-          ])
+          ]),
         ),
       },
     },

package/src/deploy/sbom.ts

@@ -0,0 +1,32 @@
+import { SBOM_FILE } from "../build/sbom";
+import {
+  componentContextIsStandaloneBuild,
+  type ComponentContext,
+} from "../types/context";
+
+export const sbomDeactivated = (context: ComponentContext) =>
+  componentContextIsStandaloneBuild(context) &&
+  context.build.config.type === "custom" &&
+  context.build.config.sbom === false;
+
+export const getDependencyTrackUploadScript = (
+  context: ComponentContext,
+): string[] => {
+  return sbomDeactivated(context)
+    ? []
+    : [
+        "echo 'Uploading SBOM to Dependency Track'",
+        `/dtrackuploader https://dep.panter.swiss/ "$DT_KEY_PROD" upload "${context.fullConfig.customerName}-${context.fullConfig.appName}/${context.name}" "${context.environment.url}" "${SBOM_FILE}" vex.json || true`,
+      ];
+};
+
+export const getDependencyTrackDeleteScript = (
+  context: ComponentContext,
+): string[] => {
+  return sbomDeactivated(context)
+    ? []
+    : [
+        "echo 'Disabling component in Dependency Track'",
+        `/dtrackuploader https://dep.panter.swiss/ "$DT_KEY_PROD" disable "${context.fullConfig.customerName}-${context.fullConfig.appName}/${context.name}" "${context.environment.url}" || true`,
+      ];
+};

package/src/deploy/types/base.ts

@@ -0,0 +1,43 @@
+import type { EnvVars } from "../../types/config";
+
+export type DeployConfigBase = {
+  /**
+   * whether to deploy automatically or manual. If not defined, these rules apply:
+   * - prod: manual
+   * - all other envs: auto
+   */
+  when?: "manual" | "auto";
+
+  /**
+   * EXPERIMENTAL
+   * wait for other components to deploy first, before doing this deployment
+   */
+  waitFor?: string[];
+
+  /**
+   * tags for the underlying job runner (e.g gitlab)
+   */
+  jobTags?: string[];
+
+  /**
+   * vars that only exist in the deploy job.
+   * Can curently not reference other variables from other components
+   */
+  jobVars?: EnvVars;
+
+  /**
+   * additional env vars for the deploy job
+   *
+   * @deprecated use deploy.jobVars for deploy job specific variables
+   */
+  extraVars?: Record<string, string>;
+  /**
+   * additional vars only for the runner.
+   * Also if you use services: that require env vars, you need to set them here.
+   *
+   */
+  runnerVariables?: Record<string, string>;
+};
+
+export type AllowUnknownProps<T extends Record<string, unknown>> = T &
+  Record<string, unknown>;

package/src/deploy/types/custom.ts

@@ -0,0 +1,36 @@
+import type { GitlabJobImage } from "../../types";
+import type { CatladderJob } from "../../types/jobs";
+import type { DeployConfigBase } from "./base";
+
+export type DeployConfigCustom = {
+  /**
+   * custom deploy types allow you to specify custom commands to deploy your app
+   */
+  type: "custom";
+  /**
+   * whether the deploy script requires a docker image to be built
+   */
+  requiresDocker: boolean;
+  /**
+   * whether your custom script requires yarn install
+   */
+  requiresYarnInstall?: boolean;
+  /**
+   * your custom script
+   */
+  script: string[];
+  /**
+   * script to run to stop the environment
+   */
+  stopScript?: string[];
+
+  /**
+   * image to use
+   */
+  jobImage?: GitlabJobImage;
+
+  /**
+   * customize cache for the job
+   */
+  jobCache?: CatladderJob["cache"];
+} & DeployConfigBase;

package/src/deploy/types/dockerTag.ts

@@ -0,0 +1,14 @@
+import type { DeployConfigBase } from "./base";
+
+export type DeployConfigDockerTag = {
+  /**
+   * adds a custom tag on the repo to deploy it.
+   *
+   * only used in veloplus atm, generally not recommended to use
+   * because run-time environment variables have to be coordinated
+   * manually with the platform running the container.
+   */
+  type: "dockerTag";
+
+  tag: string;
+} & DeployConfigBase;