@catladder/pipeline 0.0.0-node16-ddfae617 → 0.0.0-prettier-cbceeed9

package/src/deploy/kubernetes/deployJob.ts

@@ -1,16 +1,16 @@
 import { dump } from "js-yaml";
 import { merge } from "lodash";
-import { DeployConfigKubernetesValues } from "..";
 import { getSecretVarNameForContext } from "../..";
 import { getRunnerImage } from "../../runner";
-import { Context } from "../../types/context";
-import { CatladderJob } from "../../types/jobs";
-import { mergeWithMergingArrays } from "../../utils";
-import { getBaseDeploymentJob, getBaseDeploymentStopJob } from "../base";
+import type { Context } from "../../types/context";
+import type { CatladderJob } from "../../types/jobs";
+import {
+  getBaseDeploymentJob,
+  getBaseDeploymentStopJob,
+  getBaseRollbackJob,
+} from "../base";
 import { isOfDeployType } from "../types";
-import { createCloudsqlBaseConfig } from "./cloudsql";
-import { createMongodbBaseConfig } from "./mongodb";
-import { processSecretsAsFiles } from "./processSecretsAsFiles";
+import { createKubeValues } from "./kubeValues";
 
 export const createKubernetesDeployJobs = (
   context: Context
@@ -24,71 +24,7 @@ export const createKubernetesDeployJobs = (
     throw new Error("deploy config is not kubernetes");
   }
 
-  const allEnvVars = context.environment.envVars;
-  /**
-   * separate by secrets and public.
-   * we evalulate the actual values later, but want to store the secrets in kubernetes secrets
-   */
-  const env = Object.entries(allEnvVars).reduce<{
-    secret: Record<string, string>;
-    public: Record<string, string>;
-  }>(
-    (acc, [key, value]) => {
-      if (String(value)?.startsWith("$CL_")) {
-        acc.secret = {
-          ...acc.secret,
-          [key]: value,
-        };
-        return acc;
-      }
-      acc.public = {
-        ...acc.public,
-        [key]: value,
-      };
-      return acc;
-    },
-    {
-      secret: {},
-      public: {},
-    }
-  );
-
-  const defaultAppConfig: DeployConfigKubernetesValues["application"] = {
-    host: context.environment.host,
-    command: context.componentConfig.build.startCommand,
-    livenessProbe: {
-      httpGet: {
-        path: deployConfig.values?.application?.healthRoute ?? "__health",
-      },
-    },
-    readinessProbe: {
-      httpGet: {
-        path: deployConfig.values?.application?.healthRoute ?? "__health",
-      },
-    },
-    startupProbe: {
-      httpGet: {
-        path: deployConfig.values?.application?.healthRoute ?? "__health",
-      },
-    },
-  };
-
-  const defaultKubeValues = {
-    application: defaultAppConfig,
-
-    env: env,
-    ...(deployConfig.values?.cloudsql?.enabled
-      ? createCloudsqlBaseConfig(context)
-      : {}),
-    ...(deployConfig.values?.mongodb?.enabled
-      ? createMongodbBaseConfig(context)
-      : {}),
-  };
-
-  const kubeValues = processSecretsAsFiles(
-    mergeWithMergingArrays(defaultKubeValues, deployConfig.values)
-  );
-
+  const kubeValues = createKubeValues(context);
   const kubernetesEnvironment = {
     namespace: context.environment.envVars.KUBE_NAMESPACE,
   };
@@ -96,28 +32,28 @@ export const createKubernetesDeployJobs = (
     image: getRunnerImage("kubernetes"),
     variables: {
       ...context.environment.envVars,
+      RELEASE_NAME: context.environment.fullName,
       HELM_EXPERIMENTAL_OCI: "1",
-      IMAGE_PULL_SECRET: `gitlab-registry-${context.componentName}`,
+      KUBE_DOCKER_IMAGE_PULL_SECRET: `gitlab-registry-${context.componentName}`,
       KUBE_VALUES: dump(kubeValues, {
         lineWidth: -1,
         quotingType: "'",
         forceQuotes: true,
       }),
-      HELM_GITLAB_CHART_NAME: "the-panter-chart",
+      HELM_GITLAB_CHART_NAME:
+        deployConfig.chartName ?? "/helm-charts/the-panter-chart",
       HELM_ARGS: [
         ...(deployConfig.debug ? ["--debug"] : []),
         ...(deployConfig.additionalHelmArgs ?? []),
       ].join(" "),
       COMPONENT_NAME: context.componentName,
       BUILD_ID: context.commitInfo?.buildId,
-      // TODO: unify with docker build stage
-      IMAGE_NAME: context.environment.shortName + "/" + context.componentName,
-      IMAGE_TAG: "$CI_COMMIT_SHA",
     },
   };
 
   const baseDeploymentJob = getBaseDeploymentJob(context);
   const baseStopJob = getBaseDeploymentStopJob(context);
+  const baseRollbackJob = getBaseRollbackJob(context);
   const clusterName = `kube-${context.environment.fullName}`;
   const connectContext = [
     `kubectl config set-cluster "${clusterName}" --server="$${getSecretVarNameForContext(
@@ -152,5 +88,12 @@ export const createKubernetesDeployJobs = (
         kubernetes: kubernetesEnvironment,
       },
     }),
+
+    merge({}, baseRollbackJob, shared, {
+      script: [...connectContext, "kubernetesRollback"],
+      environment: {
+        kubernetes: kubernetesEnvironment,
+      },
+    }),
   ];
 };

package/src/deploy/kubernetes/index.ts

@@ -1 +1,42 @@
-export * from "./deployJob";
+import slugify from "slugify";
+import type { DeployTypeDefinition } from "..";
+import { getKubernetesNamespace } from "..";
+import { additionalKubernetesSecretKeys } from "./additionalSecretKeys";
+import { createKubernetesDeployJobs } from "./deployJob";
+export * from "./cloudSql";
+export const KUBERNETES_DEPLOY_TYPE: DeployTypeDefinition<"kubernetes"> = {
+  jobs: createKubernetesDeployJobs,
+  defaults: () => ({}),
+  additionalSecretKeys: additionalKubernetesSecretKeys,
+  getAdditionalEnvVars: ({
+    componentName,
+    fullConfig,
+    deployConfigRaw,
+    env,
+    envType,
+    commitInfo,
+  }) => {
+    const KUBE_APP_NAME_PREFIX =
+      envType === "review" && commitInfo ? `${commitInfo.reviewSlug}-` : "";
+    const KUBE_APP_NAME = `${KUBE_APP_NAME_PREFIX}${componentName}`;
+    const KUBE_NAMESPACE = getKubernetesNamespace(fullConfig, env);
+    const componentSlug = slugify(componentName);
+    const envInUrl =
+      envType === "review" && commitInfo
+        ? `${commitInfo.reviewSlug}.${env}`
+        : env;
+
+    const domainCanonical =
+      (deployConfigRaw && deployConfigRaw.cluster?.domainCanonical) || // for convenience, we allow clusters to define a canonical domain, because a cluster has a fixed ip and you will usually have a domain pointing to that cluster
+      fullConfig.domainCanonical ||
+      "panter.cloud";
+    const HOST_CANONICAL = `${componentSlug}.${envInUrl}.${fullConfig.appName}.${fullConfig.customerName}.${domainCanonical}`; // default for kubernetes and rest
+
+    return {
+      KUBE_NAMESPACE,
+      KUBE_APP_NAME,
+      KUBE_APP_NAME_PREFIX,
+      HOST_CANONICAL,
+    };
+  },
+};

package/src/deploy/kubernetes/kubeEnv.ts

@@ -0,0 +1,48 @@
+import type { Context } from "../../types";
+import { isOfDeployType } from "../types";
+
+const shouldGoIntoSecrets = (key: string, value: string) => {
+  if (String(value)?.includes("$CL_")) {
+    return true;
+  }
+  return false;
+};
+
+/**
+ * separate by secrets and public.
+ * we evalulate the actual values later, but want to store the secrets in kubernetes secrets
+ */
+export const createKubeEnv = (context: Context) => {
+  if (!isOfDeployType(context.componentConfig.deploy, "kubernetes")) {
+    // should not happen
+    throw new Error("deploy config is not kubernetes");
+  }
+
+  const allEnvVars = context.environment.envVars;
+
+  const env = Object.entries(allEnvVars).reduce<{
+    secret: Record<string, string>;
+    public: Record<string, string>;
+  }>(
+    (acc, [key, value]) => {
+      if (shouldGoIntoSecrets(key, value)) {
+        acc.secret = {
+          ...acc.secret,
+          [key]: value,
+        };
+        return acc;
+      }
+      acc.public = {
+        ...acc.public,
+        [key]: value,
+      };
+      return acc;
+    },
+    {
+      secret: {},
+      public: {},
+    }
+  );
+
+  return env;
+};

package/src/deploy/kubernetes/kubeValues.ts

@@ -0,0 +1,96 @@
+import { merge } from "lodash";
+
+import type { Context } from "../../types/context";
+import { mergeWithMergingArrays } from "../../utils";
+import type { DeployConfigKubernetesValues } from "../types";
+import { isOfDeployType } from "../types";
+import { hasKubernetesCloudSQL, createCloudsqlBaseConfig } from "./cloudSql";
+
+import { createKubeEnv } from "./kubeEnv";
+import { createMongodbBaseConfig } from "./mongodb";
+import { processSecretsAsFiles } from "./processSecretsAsFiles";
+
+const createAppConfig = (
+  context: Context,
+  application: DeployConfigKubernetesValues["application"]
+): DeployConfigKubernetesValues["application"] => {
+  if (application === false) {
+    return {
+      enabled: false,
+    };
+  }
+
+  const { healthRoute, command, ...rest } = application ?? {};
+
+  return mergeWithMergingArrays(
+    {
+      host: context.environment.host,
+      command: command ?? context.componentConfig.build.startCommand,
+      livenessProbe: {
+        httpGet: {
+          path: healthRoute ?? "__health",
+        },
+      },
+      readinessProbe: {
+        httpGet: {
+          path: healthRoute ?? "__health",
+        },
+      },
+      startupProbe: {
+        httpGet: {
+          path: healthRoute ?? "__health",
+        },
+      },
+    }, // default
+    rest // merge rest in
+  );
+};
+
+const removeFalsy = <T>(record?: Record<string, false | T>) => {
+  if (!record) return undefined;
+  return Object.fromEntries(
+    Object.entries(record).filter(([, value]) => value !== false)
+  );
+};
+
+export const createKubeValues = (context: Context) => {
+  const deployConfig = context.componentConfig.deploy;
+  if (deployConfig === false) {
+    return [];
+  }
+  if (!isOfDeployType(deployConfig, "kubernetes")) {
+    // should not happen
+    throw new Error("deploy config is not kubernetes");
+  }
+
+  const { values } = deployConfig;
+
+  // we remove the application config because it can be just the value `false` which is a convenience feature, but not supported in the helm chart
+  // we only merge the rest of the values in
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  const { application, jobs, cronjobs, ...rest } = values ?? {};
+
+  const env = createKubeEnv(context);
+
+  const defaultKubeValues = merge(
+    {
+      env,
+      application: createAppConfig(context, application),
+    },
+    hasKubernetesCloudSQL(context) ? createCloudsqlBaseConfig(context) : {},
+    deployConfig.values?.mongodb?.enabled
+      ? createMongodbBaseConfig(context)
+      : {}
+  );
+
+  const kubeValues = processSecretsAsFiles(
+    mergeWithMergingArrays(defaultKubeValues, {
+      jobs: removeFalsy(jobs),
+      cronjobs: removeFalsy(cronjobs),
+
+      ...rest,
+    })
+  );
+
+  return kubeValues;
+};

package/src/deploy/kubernetes/mongodb.ts

@@ -1,17 +1,114 @@
-import { Context } from "../..";
+import { range } from "lodash";
+import type { Context } from "../..";
+import { getSecretVarNameForContext, isOfDeployType } from "../..";
 
+const getCredentialString = (context: Context) =>
+  `root:$${getSecretVarNameForContext(context, "MONGODB_ROOT_PASSWORD")}@`;
+const getMongodbHost = (context: Context, name: string) => {
+  const namespace = context.environment.envVars.KUBE_NAMESPACE;
+  return `${name}.${namespace}.svc.cluster.local:27017`;
+};
+
+const getMongodbStandaloneHost = (context: Context) => {
+  const fullAppname = context.environment.envVars.KUBE_APP_NAME;
+  return getMongodbHost(context, `${fullAppname}-mongodb`);
+};
+
+const getMongodbReplicasetHost = (context: Context, index: number) => {
+  const fullAppname = context.environment.envVars.KUBE_APP_NAME;
+  return getMongodbHost(
+    context,
+    `${fullAppname}-mongodb-${index}.${fullAppname}-mongodb-headless`
+  );
+};
+const createMongodbUrl = (context: Context, dbName: string) => {
+  if (!isOfDeployType(context.componentConfig.deploy, "kubernetes")) {
+    throw new Error("can only createMongodbUrl on supported deploys");
+  }
+  const mongodbConfig = context.componentConfig.deploy.values?.mongodb;
+
+  let queryParams: string | undefined = undefined;
+
+  let hosts = "";
+  if (mongodbConfig?.architecture === "replicaset") {
+    hosts = range(0, mongodbConfig?.replicaCount ?? 2)
+      .map((i) => getMongodbReplicasetHost(context, i))
+      .join(",");
+
+    queryParams = "replicaSet=rs0&authSource=admin";
+  } else {
+    hosts = getMongodbStandaloneHost(context);
+    queryParams = "authSource=admin";
+  }
+
+  return `mongodb://${getCredentialString(context)}${hosts}/${dbName}${
+    queryParams ? `?${queryParams}` : ""
+  }`;
+};
+const createMongoBackupDefaultConfig = (context: Context) => {
+  if (!isOfDeployType(context.componentConfig.deploy, "kubernetes")) {
+    throw new Error("can only create mongodb base config on supported deploys");
+  }
+  const mongodbConfig = context.componentConfig.deploy.values?.mongodb;
+  const fullAppName = context.environment.envVars.KUBE_APP_NAME;
+  const backupEnabled = ["prod", "stage"].includes(context.environment.envType);
+
+  let hostToBackup: string;
+  let pvcToBackup: string;
+
+  if (mongodbConfig?.architecture === "replicaset") {
+    // bit quirky, we need to specify the host and its volume
+    // on replicaset its probably best to not use the first one, whcih usually starts as master, so we take the second (last one would also be ok)
+    const backupHostIndex =
+      mongodbConfig?.architecture === "replicaset" &&
+      mongodbConfig.replicaCount &&
+      mongodbConfig.replicaCount > 1
+        ? 1
+        : 0;
+
+    hostToBackup = getMongodbReplicasetHost(context, backupHostIndex);
+    pvcToBackup = `datadir-${fullAppName}-mongodb-${backupHostIndex}`;
+  } else {
+    hostToBackup = getMongodbStandaloneHost(context);
+    pvcToBackup = `${fullAppName}-mongodb`;
+  }
+
+  return {
+    enabled: backupEnabled,
+    hostToBackup,
+    pvcToBackup,
+    image: "mrelite/kubectlmongoshell:v1.0",
+    schedule: "0 4 * * *",
+    volumeSnapshotClass: "snapshotclass",
+  };
+};
 export const createMongodbBaseConfig = (context: Context) => {
+  if (!isOfDeployType(context.componentConfig.deploy, "kubernetes")) {
+    throw new Error("can only create mongodb base config on supported deploys");
+  }
+  const mongodbConfig = context.componentConfig.deploy.values?.mongodb;
+
   return {
     mongodb: {
       enabled: true,
-      backup: {
-        enabled: ["prod", "stage"].includes(context.environment.envType),
+      auth: {
+        enabled: true,
+        rootPassword:
+          "$" + getSecretVarNameForContext(context, "MONGODB_ROOT_PASSWORD"),
+        replicaSetKey:
+          "$" + getSecretVarNameForContext(context, "MONGODB_REPLICASET_KEY"),
+      },
+      persistence: {
+        storageClass: "standard-rwo",
       },
+      backup: createMongoBackupDefaultConfig(context),
     },
-    "mongodb-replicaset": {
-      replicas: 1,
-      persistentVolume: {
-        storageClass: "standard",
+    env: {
+      secret: {
+        MONGO_URL: createMongodbUrl(context, mongodbConfig?.dbName ?? "app"),
+        ...(mongodbConfig?.architecture === "replicaset"
+          ? { MONGO_OPLOG_URL: createMongodbUrl(context, "local") } // oplog only works with replicasets
+          : {}),
       },
     },
   };

package/src/deploy/kubernetes/processSecretsAsFiles.ts

@@ -1,5 +1,5 @@
 import { omit } from "lodash";
-import { DeployConfigKubernetesValues } from "../types";
+import type { DeployConfigKubernetesValues } from "../types";
 
 export const processSecretsAsFiles = <
   T extends {

package/src/deploy/types/base.ts

@@ -0,0 +1,17 @@
+export type DeployConfigBase = {
+  /**
+   * whether to deploy automatically or manual. If not defined, these rules apply:
+   * - prod: manual
+   * - all other envs: auto
+   */
+  when?: "manual" | "auto";
+
+  /**
+   * EXPERIMENTAL
+   * wait for other components to deploy first, before doing this deployment
+   */
+  waitFor?: string[];
+};
+
+export type AllowUnknownProps<T extends Record<string, unknown>> = T &
+  Record<string, unknown>;

package/src/deploy/types/custom.ts

@@ -0,0 +1,36 @@
+import type { GitlabJobImage } from "../../types";
+import type { CatladderJob } from "../../types/jobs";
+import type { DeployConfigBase } from "./base";
+
+export type DeployConfigCustom = {
+  /**
+   * custom deploy types allow you to specify custom commands to deploy your app
+   */
+  type: "custom";
+  /**
+   * whether the deploy script requires a docker image to be built
+   */
+  requiresDocker: boolean;
+  /**
+   * whether your custom script requires yarn install
+   */
+  requiresYarnInstall?: boolean;
+  /**
+   * your custom script
+   */
+  script: string[];
+  /**
+   * script to run to stop the environment
+   */
+  stopScript?: string[];
+
+  /**
+   * image to use
+   */
+  jobImage?: GitlabJobImage;
+
+  /**
+   * customize cache for the job
+   */
+  jobCache?: CatladderJob["cache"];
+} & DeployConfigBase;

package/src/deploy/types/dockerTag.ts

@@ -0,0 +1,12 @@
+import type { DeployConfigBase } from "./base";
+
+export type DeployConfigDockerTag = {
+  /**
+   * adds a custom tag on the repo to deploy it.
+   *
+   * only used in veloplus atm, generally not recommended to use
+   */
+  type: "dockerTag";
+
+  tag: string;
+} & DeployConfigBase;