@catladder/pipeline 0.0.0-more-collapse-deeb54d7 → 0.0.0-multiple-gitlab-includes-1a40efc0

package/src/deploy/kubernetes/deployJob.ts

@@ -1,6 +1,6 @@
-import { dump } from "js-yaml";
+import { getSecretVarNameForContext } from "../../context/getEnvironmentVariables";
 import { getRunnerImage } from "../../runner";
-import type { Context } from "../../types/context";
+import type { ComponentContext } from "../../types/context";
 import type { CatladderJob } from "../../types/jobs";
 import { createDeployementJobs } from "../base";
 import {
@@ -9,41 +9,44 @@ import {
 } from "../sbom";
 import { isOfDeployType } from "../types";
 import { createKubeValues } from "./kubeValues";
-import { getSecretVarNameForContext } from "../../context/getEnvironmentVariables";
 
+import { writeBashYamlToFileScript } from "../../bash/bashYaml";
+import { collapseableSection } from "../../utils/gitlab";
+
+const ALL_VALUES_FILE = "__all_values.yml";
 export const createKubernetesDeployJobs = (
-  context: Context
+  context: ComponentContext,
 ): CatladderJob[] => {
-  const deployConfig = context.componentConfig.deploy;
-  if (deployConfig === false) {
-    return [];
-  }
+  const deployConfig = context.deploy?.config;
   if (!isOfDeployType(deployConfig, "kubernetes")) {
     // should not happen
     throw new Error("deploy config is not kubernetes");
   }
 
-  const kubeValues = createKubeValues(context);
-  const shared: Pick<CatladderJob, "image" | "variables"> = {
+  const shared: Pick<CatladderJob, "image" | "variables" | "artifacts"> = {
     image: getRunnerImage("kubernetes"),
+    ...(deployConfig.debug
+      ? {
+          artifacts: {
+            paths: [
+              ALL_VALUES_FILE, // debug
+            ],
+            when: "always",
+          },
+        }
+      : {}),
     variables: {
       ...context.environment.envVars,
       RELEASE_NAME: context.environment.fullName,
       HELM_EXPERIMENTAL_OCI: "1",
-      KUBE_DOCKER_IMAGE_PULL_SECRET: `gitlab-registry-${context.componentName}`,
-      KUBE_VALUES: dump(kubeValues, {
-        lineWidth: -1,
-        quotingType: "'",
-        forceQuotes: true,
-      }),
+      KUBE_DOCKER_IMAGE_PULL_SECRET: `gitlab-registry-${context.name}`,
       HELM_GITLAB_CHART_NAME:
         deployConfig.chartName ?? "/helm-charts/the-panter-chart",
       HELM_ARGS: [
         ...(deployConfig.debug ? ["--debug"] : []),
         ...(deployConfig.additionalHelmArgs ?? []),
       ].join(" "),
-      COMPONENT_NAME: context.componentName,
-      BUILD_ID: context.commitInfo?.buildId,
+      COMPONENT_NAME: context.name,
     },
   };
 
@@ -51,14 +54,14 @@ export const createKubernetesDeployJobs = (
   const connectContext = [
     `kubectl config set-cluster "${clusterName}" --server="$${getSecretVarNameForContext(
       context,
-      "KUBE_URL"
+      "KUBE_URL",
     )}" --certificate-authority <(echo $${getSecretVarNameForContext(
       context,
-      "KUBE_CA_PEM"
+      "KUBE_CA_PEM",
     )} | base64 -d) --embed-certs=true`,
     `kubectl config set-credentials "${clusterName}" --token="$${getSecretVarNameForContext(
       context,
-      "KUBE_TOKEN"
+      "KUBE_TOKEN",
     )}"`,
     `kubectl config set-context "${clusterName}" --cluster="${clusterName}" --user="${clusterName}" --namespace="${context.environment.envVars.KUBE_NAMESPACE}"`,
     `kubectl config use-context "${clusterName}"`,
@@ -69,6 +72,12 @@ export const createKubernetesDeployJobs = (
       ...shared,
       script: [
         ...connectContext,
+        ...collapseableSection(
+          "writeallvalues",
+          "Write " + ALL_VALUES_FILE + " for helm deployment",
+        )(
+          writeBashYamlToFileScript(createKubeValues(context), ALL_VALUES_FILE),
+        ),
         "kubernetesCreateSecret",
         "kubernetesDeploy",
         ...getDependencyTrackUploadScript(context),

package/src/deploy/kubernetes/index.ts

@@ -1,42 +1,50 @@
 import slugify from "slugify";
-import type { DeployTypeDefinition } from "..";
+import type { DeployConfigKubernetes, DeployTypeDefinition } from "..";
 import { getKubernetesNamespace } from "..";
+import { joinBashExpressions } from "../../bash/BashExpression";
 import { additionalKubernetesSecretKeys } from "./additionalSecretKeys";
 import { createKubernetesDeployJobs } from "./deployJob";
 export * from "./cloudSql";
-export const KUBERNETES_DEPLOY_TYPE: DeployTypeDefinition<"kubernetes"> = {
-  jobs: createKubernetesDeployJobs,
-  defaults: () => ({}),
-  additionalSecretKeys: additionalKubernetesSecretKeys,
-  getAdditionalEnvVars: ({
-    componentName,
-    fullConfig,
-    deployConfigRaw,
-    env,
-    envType,
-    commitInfo,
-  }) => {
-    const KUBE_APP_NAME_PREFIX =
-      envType === "review" && commitInfo ? `${commitInfo.reviewSlug}-` : "";
-    const KUBE_APP_NAME = `${KUBE_APP_NAME_PREFIX}${componentName}`;
-    const KUBE_NAMESPACE = getKubernetesNamespace(fullConfig, env);
-    const componentSlug = slugify(componentName);
-    const envInUrl =
-      envType === "review" && commitInfo
-        ? `${commitInfo.reviewSlug}.${env}`
-        : env;
+export const KUBERNETES_DEPLOY_TYPE: DeployTypeDefinition<DeployConfigKubernetes> =
+  {
+    jobs: createKubernetesDeployJobs,
+    defaults: () => ({}),
+    additionalSecretKeys: additionalKubernetesSecretKeys,
+    getAdditionalEnvVars: ({
+      componentName,
+      fullConfig,
+      deployConfigRaw,
+      env,
+      reviewSlug,
+      envType,
+    }) => {
+      const KUBE_APP_NAME_PREFIX =
+        envType === "review" && reviewSlug ? reviewSlug.concat("-") : "";
+      const KUBE_APP_NAME = KUBE_APP_NAME_PREFIX.concat(componentName);
+      const KUBE_NAMESPACE = getKubernetesNamespace(fullConfig, env);
+      const componentSlug = slugify(componentName);
 
-    const domainCanonical =
-      (deployConfigRaw && deployConfigRaw.cluster?.domainCanonical) || // for convenience, we allow clusters to define a canonical domain, because a cluster has a fixed ip and you will usually have a domain pointing to that cluster
-      fullConfig.domainCanonical ||
-      "panter.cloud";
-    const HOST_CANONICAL = `${componentSlug}.${envInUrl}.${fullConfig.appName}.${fullConfig.customerName}.${domainCanonical}`; // default for kubernetes and rest
+      const domainCanonical =
+        (deployConfigRaw && deployConfigRaw.cluster?.domainCanonical) || // for convenience, we allow clusters to define a canonical domain, because a cluster has a fixed ip and you will usually have a domain pointing to that cluster
+        fullConfig.domainCanonical ||
+        "panter.cloud";
 
-    return {
-      KUBE_NAMESPACE,
-      KUBE_APP_NAME,
-      KUBE_APP_NAME_PREFIX,
-      HOST_CANONICAL,
-    };
-  },
-};
+      const HOSTNAME_INTERNAL = joinBashExpressions(
+        [
+          componentSlug,
+          ...(envType === "review" && reviewSlug ? [reviewSlug] : []),
+          env,
+          fullConfig.appName,
+          fullConfig.customerName,
+          domainCanonical,
+        ],
+        ".",
+      ); // default for kubernetes and rest
+      return {
+        KUBE_NAMESPACE,
+        KUBE_APP_NAME,
+        KUBE_APP_NAME_PREFIX,
+        HOSTNAME_INTERNAL,
+      };
+    },
+  };

package/src/deploy/kubernetes/kubeEnv.ts

@@ -1,7 +1,9 @@
-import type { Context } from "../../types";
+import type { ComponentContext } from "../../types";
+import type { VariableValue } from "../../variables/VariableValue";
 import { isOfDeployType } from "../types";
 
-const shouldGoIntoSecrets = (key: string, value: string) => {
+const shouldGoIntoSecrets = (key: string, value: string | undefined) => {
+  if (!value) return false;
   if (String(value)?.includes("$CL_")) {
     return true;
   }
@@ -12,8 +14,8 @@ const shouldGoIntoSecrets = (key: string, value: string) => {
  * separate by secrets and public.
  * we evalulate the actual values later, but want to store the secrets in kubernetes secrets
  */
-export const createKubeEnv = (context: Context) => {
-  if (!isOfDeployType(context.componentConfig.deploy, "kubernetes")) {
+export const createKubeEnv = (context: ComponentContext) => {
+  if (!isOfDeployType(context.deploy?.config, "kubernetes")) {
     // should not happen
     throw new Error("deploy config is not kubernetes");
   }
@@ -21,27 +23,27 @@ export const createKubeEnv = (context: Context) => {
   const allEnvVars = context.environment.envVars;
 
   const env = Object.entries(allEnvVars).reduce<{
-    secret: Record<string, string>;
-    public: Record<string, string>;
+    secret: Record<string, VariableValue>;
+    public: Record<string, VariableValue>;
   }>(
     (acc, [key, value]) => {
-      if (shouldGoIntoSecrets(key, value)) {
+      if (shouldGoIntoSecrets(key, value?.toString())) {
         acc.secret = {
           ...acc.secret,
-          [key]: value,
+          [key]: value ?? "",
         };
         return acc;
       }
       acc.public = {
         ...acc.public,
-        [key]: value,
+        [key]: value ?? "",
       };
       return acc;
     },
     {
       secret: {},
       public: {},
-    }
+    },
   );
 
   return env;

package/src/deploy/kubernetes/kubeValues.ts

@@ -1,6 +1,6 @@
 import { merge } from "lodash";
 
-import type { Context } from "../../types/context";
+import type { ComponentContext } from "../../types/context";
 import { mergeWithMergingArrays } from "../../utils";
 import type { DeployConfigKubernetesValues } from "../types";
 import { isOfDeployType } from "../types";
@@ -11,11 +11,10 @@ import {
 
 import { createKubeEnv } from "./kubeEnv";
 import { createMongodbBaseConfig } from "./mongodb";
-import { processSecretsAsFiles } from "./processSecretsAsFiles";
 
 const createAppConfig = (
-  context: Context,
-  application: DeployConfigKubernetesValues["application"]
+  context: ComponentContext,
+  application: DeployConfigKubernetesValues["application"],
 ): DeployConfigKubernetesValues["application"] => {
   if (application === false) {
     return {
@@ -28,7 +27,7 @@ const createAppConfig = (
   return mergeWithMergingArrays(
     {
       host: context.environment.host,
-      command: command ?? context.componentConfig.build.startCommand,
+      command: command ?? context.build.config.startCommand,
       livenessProbe: {
         httpGet: {
           path: healthRoute ?? "__health",
@@ -45,20 +44,20 @@ const createAppConfig = (
         },
       },
     }, // default
-    rest // merge rest in
+    rest, // merge rest in
   );
 };
 
 const removeFalsy = <T>(record?: Record<string, false | T>) => {
   if (!record) return undefined;
   return Object.fromEntries(
-    Object.entries(record).filter(([, value]) => value !== false)
+    Object.entries(record).filter(([, value]) => value !== false),
   );
 };
 
-export const createKubeValues = (context: Context) => {
-  const deployConfig = context.componentConfig.deploy;
-  if (deployConfig === false) {
+export const createKubeValues = (context: ComponentContext) => {
+  const deployConfig = context.deploy?.config;
+  if (!deployConfig) {
     return [];
   }
   if (!isOfDeployType(deployConfig, "kubernetes")) {
@@ -70,7 +69,6 @@ export const createKubeValues = (context: Context) => {
 
   // we remove the application config because it can be just the value `false` which is a convenience feature, but not supported in the helm chart
   // we only merge the rest of the values in
-  // eslint-disable-next-line @typescript-eslint/no-unused-vars
   const { application, jobs, cronjobs, ...rest } = values ?? {};
 
   const env = createKubeEnv(context);
@@ -85,17 +83,15 @@ export const createKubeValues = (context: Context) => {
       : {},
     deployConfig.values?.mongodb?.enabled
       ? createMongodbBaseConfig(context)
-      : {}
+      : {},
   );
 
-  const kubeValues = processSecretsAsFiles(
-    mergeWithMergingArrays(defaultKubeValues, {
-      jobs: removeFalsy(jobs),
-      cronjobs: removeFalsy(cronjobs),
+  const kubeValues = mergeWithMergingArrays(defaultKubeValues, {
+    jobs: removeFalsy(jobs),
+    cronjobs: removeFalsy(cronjobs),
 
-      ...rest,
-    })
-  );
+    ...rest,
+  });
 
   return kubeValues;
 };

package/src/deploy/kubernetes/mongodb.ts

@@ -1,32 +1,32 @@
 import { range } from "lodash";
 import { getSecretVarNameForContext } from "../../context/getEnvironmentVariables";
-import type { Context } from "../../types/context";
+import type { ComponentContext } from "../../types/context";
 import { isOfDeployType } from "../types";
 
-const getCredentialString = (context: Context) =>
+const getCredentialString = (context: ComponentContext) =>
   `root:$${getSecretVarNameForContext(context, "MONGODB_ROOT_PASSWORD")}@`;
-const getMongodbHost = (context: Context, name: string) => {
+const getMongodbHost = (context: ComponentContext, name: string) => {
   const namespace = context.environment.envVars.KUBE_NAMESPACE;
   return `${name}.${namespace}.svc.cluster.local:27017`;
 };
 
-const getMongodbStandaloneHost = (context: Context) => {
+const getMongodbStandaloneHost = (context: ComponentContext) => {
   const fullAppname = context.environment.envVars.KUBE_APP_NAME;
   return getMongodbHost(context, `${fullAppname}-mongodb`);
 };
 
-const getMongodbReplicasetHost = (context: Context, index: number) => {
+const getMongodbReplicasetHost = (context: ComponentContext, index: number) => {
   const fullAppname = context.environment.envVars.KUBE_APP_NAME;
   return getMongodbHost(
     context,
-    `${fullAppname}-mongodb-${index}.${fullAppname}-mongodb-headless`
+    `${fullAppname}-mongodb-${index}.${fullAppname}-mongodb-headless`,
   );
 };
-const createMongodbUrl = (context: Context, dbName: string) => {
-  if (!isOfDeployType(context.componentConfig.deploy, "kubernetes")) {
+const createMongodbUrl = (context: ComponentContext, dbName: string) => {
+  if (!isOfDeployType(context.deploy?.config, "kubernetes")) {
     throw new Error("can only createMongodbUrl on supported deploys");
   }
-  const mongodbConfig = context.componentConfig.deploy.values?.mongodb;
+  const mongodbConfig = context.deploy?.config.values?.mongodb;
 
   let queryParams: string | undefined = undefined;
 
@@ -46,11 +46,11 @@ const createMongodbUrl = (context: Context, dbName: string) => {
     queryParams ? `?${queryParams}` : ""
   }`;
 };
-const createMongoBackupDefaultConfig = (context: Context) => {
-  if (!isOfDeployType(context.componentConfig.deploy, "kubernetes")) {
+const createMongoBackupDefaultConfig = (context: ComponentContext) => {
+  if (!isOfDeployType(context.deploy?.config, "kubernetes")) {
     throw new Error("can only create mongodb base config on supported deploys");
   }
-  const mongodbConfig = context.componentConfig.deploy.values?.mongodb;
+  const mongodbConfig = context.deploy?.config.values?.mongodb;
   const fullAppName = context.environment.envVars.KUBE_APP_NAME;
   const backupEnabled = ["prod", "stage"].includes(context.environment.envType);
 
@@ -83,11 +83,11 @@ const createMongoBackupDefaultConfig = (context: Context) => {
     volumeSnapshotClass: "snapshotclass",
   };
 };
-export const createMongodbBaseConfig = (context: Context) => {
-  if (!isOfDeployType(context.componentConfig.deploy, "kubernetes")) {
+export const createMongodbBaseConfig = (context: ComponentContext) => {
+  if (!isOfDeployType(context.deploy?.config, "kubernetes")) {
     throw new Error("can only create mongodb base config on supported deploys");
   }
-  const mongodbConfig = context.componentConfig.deploy.values?.mongodb;
+  const mongodbConfig = context.deploy?.config.values?.mongodb;
 
   return {
     mongodb: {

package/src/deploy/sbom.ts

@@ -1,24 +1,32 @@
-import type { Context } from "../types/context";
 import { SBOM_FILE } from "../build/sbom";
+import {
+  componentContextIsStandaloneBuild,
+  type ComponentContext,
+} from "../types/context";
 
-export const sbomDeactivated = (context: Context) =>
-  context.componentConfig.build.type === "custom" &&
-  context.componentConfig.build.sbom === false;
+export const sbomDeactivated = (context: ComponentContext) =>
+  componentContextIsStandaloneBuild(context) &&
+  context.build.config.type === "custom" &&
+  context.build.config.sbom === false;
 
-export const getDependencyTrackUploadScript = (context: Context): string[] => {
+export const getDependencyTrackUploadScript = (
+  context: ComponentContext,
+): string[] => {
   return sbomDeactivated(context)
     ? []
     : [
-        "echo Uploading SBOM to Dependency Track",
-        `/dtrackuploader https://dep.panter.swiss/ "$DT_KEY_PROD" upload "${context.fullConfig.customerName}-${context.fullConfig.appName}/${context.componentName}" "${context.environment.url}" "${SBOM_FILE}" vex.json || true`,
+        "echo 'Uploading SBOM to Dependency Track'",
+        `/dtrackuploader https://dep.panter.swiss/ "$DT_KEY_PROD" upload "${context.fullConfig.customerName}-${context.fullConfig.appName}/${context.name}" "${context.environment.url}" "${SBOM_FILE}" vex.json || true`,
       ];
 };
 
-export const getDependencyTrackDeleteScript = (context: Context): string[] => {
+export const getDependencyTrackDeleteScript = (
+  context: ComponentContext,
+): string[] => {
   return sbomDeactivated(context)
     ? []
     : [
-        "echo Disabling component in Dependency Track",
-        `/dtrackuploader https://dep.panter.swiss/ "$DT_KEY_PROD" disable "${context.fullConfig.customerName}-${context.fullConfig.appName}/${context.componentName}" "${context.environment.url}" || true`,
+        "echo 'Disabling component in Dependency Track'",
+        `/dtrackuploader https://dep.panter.swiss/ "$DT_KEY_PROD" disable "${context.fullConfig.customerName}-${context.fullConfig.appName}/${context.name}" "${context.environment.url}" || true`,
       ];
 };

package/src/deploy/types/base.ts

@@ -26,11 +26,11 @@ export type DeployConfigBase = {
   jobVars?: EnvVars;
 
   /**
-   * additional env vars for the deploy job
+   * additional vars only for the runner.
+   * Also if you use services: that require env vars, you need to set them here.
    *
-   * @deprecated use deploy.jobVars for deploy job specific variables
    */
-  extraVars?: Record<string, string>;
+  runnerVariables?: Record<string, string>;
 };
 
 export type AllowUnknownProps<T extends Record<string, unknown>> = T &

package/src/deploy/types/custom.ts

@@ -1,5 +1,5 @@
+import type { WithCacheConfig } from "../../build";
 import type { GitlabJobImage } from "../../types";
-import type { CatladderJob } from "../../types/jobs";
 import type { DeployConfigBase } from "./base";
 
 export type DeployConfigCustom = {
@@ -28,9 +28,5 @@ export type DeployConfigCustom = {
    * image to use
    */
   jobImage?: GitlabJobImage;
-
-  /**
-   * customize cache for the job
-   */
-  jobCache?: CatladderJob["cache"];
-} & DeployConfigBase;
+} & DeployConfigBase &
+  WithCacheConfig;

package/src/deploy/types/dockerTag.ts

@@ -5,6 +5,8 @@ export type DeployConfigDockerTag = {
    * adds a custom tag on the repo to deploy it.
    *
    * only used in veloplus atm, generally not recommended to use
+   * because run-time environment variables have to be coordinated
+   * manually with the platform running the container.
    */
   type: "dockerTag";