@catladder/pipeline 1.144.1 → 1.146.0

package/src/deploy/cloudRun/index.ts

@@ -1,4 +1,9 @@
 import type { DeployTypeDefinition } from "..";
+import {
+  BashExpression,
+  getBashVariable,
+  joinBashExpressions,
+} from "../../bash/BashExpression";
 import { getSecretVarName } from "../../context";
 import type { EnvironmentContext } from "../../types/environmentContext";
 import { sanitizeForBashVariable } from "../../utils/gitlab";
@@ -28,11 +33,11 @@ const getCloudSqlVariables = ({
       deployConfigRaw.cloudSql,
       fullConfig,
       environmentSlugPrefix,
-      componentName
+      componentName,
     );
 
     const additionalQueryParamsString = Object.entries(
-      deployConfigRaw.cloudSql.dbAdditionalQueryParams ?? {}
+      deployConfigRaw.cloudSql.dbAdditionalQueryParams ?? {},
     )
       .map(([key, value]) => `&${key}=${value}`)
       .join("");
@@ -44,7 +49,7 @@ const getCloudSqlVariables = ({
       DB_USER: deployConfigRaw.cloudSql.dbUser ?? "postgres",
       DB_PASSWORD: "$" + getSecretVarName(env, componentName, "DB_PASSWORD"),
       DATABASE_URL: `${getDatabaseConnectionString(
-        deployConfigRaw.cloudSql
+        deployConfigRaw.cloudSql,
       )}${additionalQueryParamsString}`,
       DATABASE_JDBC_URL: DATABASE_JDBC_URL,
     };
@@ -78,13 +83,20 @@ export const GCLOUD_RUN_DEPLOY_TYPE: DeployTypeDefinition<"google-cloudrun"> = {
   ],
   getAdditionalEnvVars: (ctx) => {
     const { fullName, env, componentName, deployConfigRaw } = ctx;
-    const HOST_INTERNAL =
-      fullName.toLowerCase() +
-      "-" +
-      (process.env[
-        getSecretVarName(env, componentName, GCLOUD_RUN_CANONICAL_HOST_SUFFIX)
-      ] ?? "unknown-host.example.com");
 
+    const HOST_INTERNAL = joinBashExpressions(
+      [
+        fullName,
+        getBashVariable(
+          getSecretVarName(
+            env,
+            componentName,
+            GCLOUD_RUN_CANONICAL_HOST_SUFFIX,
+          ),
+        ),
+      ],
+      "-",
+    ).toLowerCase();
     const jobTriggers =
       deployConfigRaw && deployConfigRaw.jobs
         ? Object.fromEntries(
@@ -95,7 +107,7 @@ export const GCLOUD_RUN_DEPLOY_TYPE: DeployTypeDefinition<"google-cloudrun"> = {
               }-run.googleapis.com/apis/run.googleapis.com/v1/namespaces/${
                 deployConfigRaw.projectId
               }/jobs/${getCloudRunJobName(fullName, name)}:run`,
-            ])
+            ]),
           )
         : {};
 

package/src/deploy/cloudRun/utils/createArgsString.ts

@@ -1,4 +1,5 @@
 import { isNil } from "lodash";
+import type { BashExpression } from "../../../bash/BashExpression";
 
 /**
  * creates arguments string:
@@ -22,12 +23,12 @@ export const createArgsString = (...args: keyValuesArg[]) =>
             return `--no-${key}`;
           }
           return `--${key}=${value}`;
-        })
+        }),
     )
     .flat()
     .join(" ");
 
 export type keyValuesArg = Record<
   string,
-  string | number | true | false | undefined
+  string | number | true | false | undefined | BashExpression
 >;

package/src/deploy/cloudRun/utils/database.ts

@@ -7,13 +7,13 @@ import type {
 
 export const getDatabaseDeleteScript = (
   context: Context,
-  deployConfig: DeployConfigCloudRun
+  deployConfig: DeployConfigCloudRun,
 ): string[] => {
   if (!deployConfig.cloudSql || !deployConfig.cloudSql.deleteDatabaseOnStop) {
     throw new Error("not possible");
   }
+  const DB_NAME = context.environment.envVars["DB_NAME"];
 
-  const { DB_NAME } = context.environment.envVars ?? {};
   if (!DB_NAME) {
     throw new Error("error: env vars did not specify DB_NAME");
   }
@@ -29,20 +29,20 @@ export const getDatabaseDeleteScript = (
       `gcloud sql databases delete ${DB_NAME} --instance=${instanceId} --project ${projectId}`,
       {
         pauseInSeconds: 10,
-      }
+      },
     ),
   ];
 };
 
 export const getDatabaseCreateScript = (
   context: Context,
-  deployConfig: DeployConfigCloudRun
+  deployConfig: DeployConfigCloudRun,
 ): string[] => {
   if (!deployConfig.cloudSql) {
     throw new Error("not possible");
   }
 
-  const { DB_NAME } = context.environment.envVars ?? {};
+  const DB_NAME = context.environment.envVars["DB_NAME"];
 
   if (!DB_NAME) {
     throw new Error("error: env vars did not specify DB_NAME");
@@ -61,14 +61,14 @@ export const DATABASE_JDBC_URL =
   "jdbc:postgresql:///$DB_NAME?cloudSqlInstance=$CLOUD_SQL_INSTANCE_CONNECTION_NAME&socketFactory=com.google.cloud.sql.postgres.SocketFactory&user=$DB_USER&password=$DB_PASSWORD";
 
 export const getDatabaseConnectionString = (
-  config: DeployConfigCloudRunCloudSql
+  config: DeployConfigCloudRunCloudSql,
 ): string => {
   switch (config.dbConnectionStringFormat) {
     case "jdbc":
       return DATABASE_JDBC_URL;
     case "rails":
       return `postgresql://$DB_USER:$DB_PASSWORD@${encodeURIComponent(
-        `/cloudsql/${config.instanceConnectionName}`
+        `/cloudsql/${config.instanceConnectionName}`,
       )}/$DB_NAME?`;
     default:
       // prisma

package/src/deploy/cloudRun/utils/gcloudServiceAccountLoginCommands.ts

@@ -5,6 +5,6 @@ import type { Context } from "../../../types";
 export const gcloudServiceAccountLoginCommands = (context: Context) => [
   `gcloud auth activate-service-account --key-file=<(echo "$${getSecretVarNameForContext(
     context,
-    GCLOUD_DEPLOY_CREDENTIALS_KEY
+    GCLOUD_DEPLOY_CREDENTIALS_KEY,
   )}")`,
 ];

package/src/deploy/cloudRun/utils/jobName.ts

@@ -1,2 +1,7 @@
-export const getCloudRunJobName = (fullAppName: string, jobName: string) =>
-  fullAppName.toLowerCase() + "-" + jobName.toLowerCase();
+import type { StringOrBashExpression } from "../../../bash/BashExpression";
+
+export const getCloudRunJobName = (
+  fullAppName: StringOrBashExpression,
+  jobName: string,
+): StringOrBashExpression =>
+  fullAppName.toLowerCase().concat("-" + jobName.toLowerCase());

package/src/deploy/cloudRun/utils/removeFirstLinesFromCommandOutput.ts

@@ -8,5 +8,5 @@
  */
 export const removeFirstLinesFromCommandOutput = (
   cmd: string,
-  removeFirstN: number
+  removeFirstN: number,
 ) => (removeFirstN === 0 ? cmd : `${cmd} | tail -n +${removeFirstN + 1}`);

package/src/deploy/cloudSql/utils.ts

@@ -1,4 +1,6 @@
 import type { Config } from "../..";
+import type { StringOrBashExpression } from "../../bash/BashExpression";
+import { joinBashExpressions } from "../../bash/BashExpression";
 import type {
   DeployConfigCloudRunCloudSql,
   DeployConfigKubernetesValuesCloudSQLUnmanaged,
@@ -9,15 +11,16 @@ export const getFullDbName = (
     | DeployConfigCloudRunCloudSql
     | DeployConfigKubernetesValuesCloudSQLUnmanaged,
   fullConfig: Config<never>,
-  environmentSlugPrefix: string,
-  componentName: string
+  environmentSlugPrefix: StringOrBashExpression,
+  componentName: string,
 ) => {
-  return [
-    cloudSqlConfig.dbNamePrefix ??
-      `${fullConfig.customerName}-${fullConfig.appName}`,
-    environmentSlugPrefix,
-    cloudSqlConfig.dbBaseName ?? componentName,
-  ]
-    .filter(Boolean)
-    .join("-");
+  return joinBashExpressions(
+    [
+      cloudSqlConfig.dbNamePrefix ??
+        `${fullConfig.customerName}-${fullConfig.appName}`,
+      environmentSlugPrefix,
+      cloudSqlConfig.dbBaseName ?? componentName,
+    ].flatMap((part) => (part ? [part] : [])),
+    "-",
+  );
 };

package/src/deploy/index.ts

@@ -1,4 +1,5 @@
 import type { SecretEnvVar } from "..";
+import type { BashExpression } from "../bash/BashExpression";
 import type { Context } from "../types/context";
 import type { EnvironmentContext } from "../types/environmentContext";
 import type { CatladderJob } from "../types/jobs";
@@ -16,14 +17,14 @@ export * from "./utils";
 export type DeployTypeDefinition<T extends DeployConfigType> = {
   jobs: (context: Context) => CatladderJob[];
   defaults: (
-    envContext: EnvironmentContext<any, T>
+    envContext: EnvironmentContext<any, T>,
   ) => PartialDeep<DeployConfigGeneric<T>>;
   additionalSecretKeys: (
-    envContext: EnvironmentContext<any, T>
+    envContext: EnvironmentContext<any, T>,
   ) => SecretEnvVar[];
   getAdditionalEnvVars: (
-    envContext: EnvironmentContext<any, T>
-  ) => Record<string, string | undefined | null>;
+    envContext: EnvironmentContext<any, T>,
+  ) => Record<string, string | BashExpression | undefined | null>;
 };
 export type DeployTypes = {
   [T in DeployConfigType]: DeployTypeDefinition<T>;

package/src/deploy/kubernetes/cloudSql/index.ts

@@ -1,3 +1,4 @@
+import type { StringOrBashExpression } from "../../../bash/BashExpression";
 import { getSecretVarNameForContext } from "../../../context";
 import type { Context } from "../../../types";
 import { getFullDbName } from "../../cloudSql/utils";
@@ -15,11 +16,11 @@ type CloudSqlValues = {
   proxyCredentials: string;
 
   instanceConnectionName: string;
-  fullDbName: string;
+  fullDbName: StringOrBashExpression;
   dbUser: string;
 };
 export const createKubernetesCloudsqlBaseValues = (
-  context: Context
+  context: Context,
 ): {
   cloudsql: CloudSqlValues;
 } => {
@@ -35,7 +36,7 @@ export const createKubernetesCloudsqlBaseValues = (
 
   const proxyCredentials = `$${getSecretVarNameForContext(
     context,
-    "cloudsqlProxyCredentials"
+    "cloudsqlProxyCredentials",
   )}`;
   if (config?.type !== "unmanaged") {
     const instanceConnectionName = `${config.projectId ?? "skynet-164509"}:${
@@ -48,7 +49,7 @@ export const createKubernetesCloudsqlBaseValues = (
         dbUser: "postgres",
         instanceConnectionName,
         proxyCredentials,
-        fullDbName: context.environment.envVars.KUBE_APP_NAME,
+        fullDbName: context.environment.envVars.KUBE_APP_NAME ?? "",
       },
     };
   } else if (config.type === "unmanaged") {
@@ -62,7 +63,7 @@ export const createKubernetesCloudsqlBaseValues = (
           config,
           context.fullConfig,
           context.environment.slugPrefix,
-          context.componentName
+          context.componentName,
         ),
       },
     };

package/src/deploy/kubernetes/deployJob.ts

@@ -1,4 +1,4 @@
-import { dump } from "js-yaml";
+import { getSecretVarNameForContext } from "../../context/getEnvironmentVariables";
 import { getRunnerImage } from "../../runner";
 import type { Context } from "../../types/context";
 import type { CatladderJob } from "../../types/jobs";
@@ -9,10 +9,13 @@ import {
 } from "../sbom";
 import { isOfDeployType } from "../types";
 import { createKubeValues } from "./kubeValues";
-import { getSecretVarNameForContext } from "../../context/getEnvironmentVariables";
 
+import { writeBashYamlToFileScript } from "../../bash/bashYaml";
+import { collapseableSection } from "../../utils/gitlab";
+
+const ALL_VALUES_FILE = "__all_values.yml";
 export const createKubernetesDeployJobs = (
-  context: Context
+  context: Context,
 ): CatladderJob[] => {
   const deployConfig = context.componentConfig.deploy;
   if (deployConfig === false) {
@@ -23,19 +26,23 @@ export const createKubernetesDeployJobs = (
     throw new Error("deploy config is not kubernetes");
   }
 
-  const kubeValues = createKubeValues(context);
-  const shared: Pick<CatladderJob, "image" | "variables"> = {
+  const shared: Pick<CatladderJob, "image" | "variables" | "artifacts"> = {
     image: getRunnerImage("kubernetes"),
+    ...(deployConfig.debug
+      ? {
+          artifacts: {
+            paths: [
+              ALL_VALUES_FILE, // debug
+            ],
+            when: "always",
+          },
+        }
+      : {}),
     variables: {
       ...context.environment.envVars,
       RELEASE_NAME: context.environment.fullName,
       HELM_EXPERIMENTAL_OCI: "1",
       KUBE_DOCKER_IMAGE_PULL_SECRET: `gitlab-registry-${context.componentName}`,
-      KUBE_VALUES: dump(kubeValues, {
-        lineWidth: -1,
-        quotingType: "'",
-        forceQuotes: true,
-      }),
       HELM_GITLAB_CHART_NAME:
         deployConfig.chartName ?? "/helm-charts/the-panter-chart",
       HELM_ARGS: [
@@ -43,7 +50,8 @@ export const createKubernetesDeployJobs = (
         ...(deployConfig.additionalHelmArgs ?? []),
       ].join(" "),
       COMPONENT_NAME: context.componentName,
-      BUILD_ID: context.commitInfo?.buildId,
+      /** @deprecated */
+      BUILD_ID: context.environment.envVars.BUILD_INFO_BUILD_ID,
     },
   };
 
@@ -51,14 +59,14 @@ export const createKubernetesDeployJobs = (
   const connectContext = [
     `kubectl config set-cluster "${clusterName}" --server="$${getSecretVarNameForContext(
       context,
-      "KUBE_URL"
+      "KUBE_URL",
     )}" --certificate-authority <(echo $${getSecretVarNameForContext(
       context,
-      "KUBE_CA_PEM"
+      "KUBE_CA_PEM",
     )} | base64 -d) --embed-certs=true`,
     `kubectl config set-credentials "${clusterName}" --token="$${getSecretVarNameForContext(
       context,
-      "KUBE_TOKEN"
+      "KUBE_TOKEN",
     )}"`,
     `kubectl config set-context "${clusterName}" --cluster="${clusterName}" --user="${clusterName}" --namespace="${context.environment.envVars.KUBE_NAMESPACE}"`,
     `kubectl config use-context "${clusterName}"`,
@@ -69,6 +77,12 @@ export const createKubernetesDeployJobs = (
       ...shared,
       script: [
         ...connectContext,
+        ...collapseableSection(
+          "writeallvalues",
+          "Write " + ALL_VALUES_FILE + " for helm deployment",
+        )(
+          writeBashYamlToFileScript(createKubeValues(context), ALL_VALUES_FILE),
+        ),
         "kubernetesCreateSecret",
         "kubernetesDeploy",
         ...getDependencyTrackUploadScript(context),

package/src/deploy/kubernetes/index.ts

@@ -1,6 +1,7 @@
 import slugify from "slugify";
 import type { DeployTypeDefinition } from "..";
 import { getKubernetesNamespace } from "..";
+import { joinBashExpressions } from "../../bash/BashExpression";
 import { additionalKubernetesSecretKeys } from "./additionalSecretKeys";
 import { createKubernetesDeployJobs } from "./deployJob";
 export * from "./cloudSql";
@@ -13,25 +14,31 @@ export const KUBERNETES_DEPLOY_TYPE: DeployTypeDefinition<"kubernetes"> = {
     fullConfig,
     deployConfigRaw,
     env,
+    reviewSlug,
     envType,
-    commitInfo,
   }) => {
     const KUBE_APP_NAME_PREFIX =
-      envType === "review" && commitInfo ? `${commitInfo.reviewSlug}-` : "";
-    const KUBE_APP_NAME = `${KUBE_APP_NAME_PREFIX}${componentName}`;
+      envType === "review" && reviewSlug ? reviewSlug.concat("-") : "";
+    const KUBE_APP_NAME = KUBE_APP_NAME_PREFIX.concat(componentName);
     const KUBE_NAMESPACE = getKubernetesNamespace(fullConfig, env);
     const componentSlug = slugify(componentName);
-    const envInUrl =
-      envType === "review" && commitInfo
-        ? `${commitInfo.reviewSlug}.${env}`
-        : env;
 
     const domainCanonical =
       (deployConfigRaw && deployConfigRaw.cluster?.domainCanonical) || // for convenience, we allow clusters to define a canonical domain, because a cluster has a fixed ip and you will usually have a domain pointing to that cluster
       fullConfig.domainCanonical ||
       "panter.cloud";
-    const HOST_INTERNAL = `${componentSlug}.${envInUrl}.${fullConfig.appName}.${fullConfig.customerName}.${domainCanonical}`; // default for kubernetes and rest
 
+    const HOST_INTERNAL = joinBashExpressions(
+      [
+        componentSlug,
+        ...(envType === "review" && reviewSlug ? [reviewSlug] : []),
+        env,
+        fullConfig.appName,
+        fullConfig.customerName,
+        domainCanonical,
+      ],
+      ".",
+    ); // default for kubernetes and rest
     return {
       KUBE_NAMESPACE,
       KUBE_APP_NAME,

package/src/deploy/kubernetes/kubeEnv.ts

@@ -1,7 +1,9 @@
+import type { StringOrBashExpression } from "../../bash/BashExpression";
 import type { Context } from "../../types";
 import { isOfDeployType } from "../types";
 
-const shouldGoIntoSecrets = (key: string, value: string) => {
+const shouldGoIntoSecrets = (key: string, value: string | undefined) => {
+  if (!value) return false;
   if (String(value)?.includes("$CL_")) {
     return true;
   }
@@ -21,27 +23,27 @@ export const createKubeEnv = (context: Context) => {
   const allEnvVars = context.environment.envVars;
 
   const env = Object.entries(allEnvVars).reduce<{
-    secret: Record<string, string>;
-    public: Record<string, string>;
+    secret: Record<string, StringOrBashExpression>;
+    public: Record<string, StringOrBashExpression>;
   }>(
     (acc, [key, value]) => {
-      if (shouldGoIntoSecrets(key, value)) {
+      if (shouldGoIntoSecrets(key, value?.toString())) {
         acc.secret = {
           ...acc.secret,
-          [key]: value,
+          [key]: value ?? "",
         };
         return acc;
       }
       acc.public = {
         ...acc.public,
-        [key]: value,
+        [key]: value ?? "",
       };
       return acc;
     },
     {
       secret: {},
       public: {},
-    }
+    },
   );
 
   return env;

package/src/deploy/kubernetes/kubeValues.ts

@@ -15,7 +15,7 @@ import { processSecretsAsFiles } from "./processSecretsAsFiles";
 
 const createAppConfig = (
   context: Context,
-  application: DeployConfigKubernetesValues["application"]
+  application: DeployConfigKubernetesValues["application"],
 ): DeployConfigKubernetesValues["application"] => {
   if (application === false) {
     return {
@@ -45,14 +45,14 @@ const createAppConfig = (
         },
       },
     }, // default
-    rest // merge rest in
+    rest, // merge rest in
   );
 };
 
 const removeFalsy = <T>(record?: Record<string, false | T>) => {
   if (!record) return undefined;
   return Object.fromEntries(
-    Object.entries(record).filter(([, value]) => value !== false)
+    Object.entries(record).filter(([, value]) => value !== false),
   );
 };
 
@@ -70,7 +70,6 @@ export const createKubeValues = (context: Context) => {
 
   // we remove the application config because it can be just the value `false` which is a convenience feature, but not supported in the helm chart
   // we only merge the rest of the values in
-  // eslint-disable-next-line @typescript-eslint/no-unused-vars
   const { application, jobs, cronjobs, ...rest } = values ?? {};
 
   const env = createKubeEnv(context);
@@ -85,7 +84,7 @@ export const createKubeValues = (context: Context) => {
       : {},
     deployConfig.values?.mongodb?.enabled
       ? createMongodbBaseConfig(context)
-      : {}
+      : {},
   );
 
   const kubeValues = processSecretsAsFiles(
@@ -94,7 +93,7 @@ export const createKubeValues = (context: Context) => {
       cronjobs: removeFalsy(cronjobs),
 
       ...rest,
-    })
+    }),
   );
 
   return kubeValues;

package/src/deploy/kubernetes/mongodb.ts

@@ -19,7 +19,7 @@ const getMongodbReplicasetHost = (context: Context, index: number) => {
   const fullAppname = context.environment.envVars.KUBE_APP_NAME;
   return getMongodbHost(
     context,
-    `${fullAppname}-mongodb-${index}.${fullAppname}-mongodb-headless`
+    `${fullAppname}-mongodb-${index}.${fullAppname}-mongodb-headless`,
   );
 };
 const createMongodbUrl = (context: Context, dbName: string) => {

package/src/deploy/kubernetes/processSecretsAsFiles.ts

@@ -1,14 +1,15 @@
 import { omit } from "lodash";
 import type { DeployConfigKubernetesValues } from "../types";
+import type { StringOrBashExpression } from "../../bash/BashExpression";
 
 export const processSecretsAsFiles = <
   T extends {
     env: {
-      secret: Record<string, string>;
+      secret: Record<string, StringOrBashExpression>;
     };
-  } & DeployConfigKubernetesValues
+  } & DeployConfigKubernetesValues,
 >(
-  values: T
+  values: T,
 ): T => {
   if (!values.secretsAsFile || values.secretsAsFile.length === 0) {
     return values;
@@ -26,7 +27,7 @@ export const processSecretsAsFiles = <
           values.secretsAsFile.map((key) => [
             key + "_content",
             values.env.secret[key],
-          ])
+          ]),
         ),
       },
     },

package/src/deploy/types/base.ts

@@ -31,6 +31,12 @@ export type DeployConfigBase = {
    * @deprecated use deploy.jobVars for deploy job specific variables
    */
   extraVars?: Record<string, string>;
+  /**
+   * additional vars only for the runner.
+   * Also if you use services: that require env vars, you need to set them here.
+   *
+   */
+  runnerVariables?: Record<string, string>;
 };
 
 export type AllowUnknownProps<T extends Record<string, unknown>> = T &

package/src/deploy/types/googleCloudRun.ts

@@ -256,6 +256,8 @@ export type DeployConfigCloudRun = {
    * add cloudSql
    */
   cloudSql?: DeployConfigCloudRunCloudSql | false;
+
+  debug?: boolean;
 } & DeployConfigBase;
 
 export type DeployConfigCloudRunVolumes = Record<

package/src/deploy/utils.ts

@@ -3,7 +3,7 @@ import type { DeployConfigKubernetesCluster } from "./types";
 import { isOfDeployType } from "./types";
 
 export const getFullKubernetesClusterName = (
-  cluster: DeployConfigKubernetesCluster
+  cluster: DeployConfigKubernetesCluster,
 ) => {
   if (cluster.type === "gcloud") {
     return `gke_${cluster.projectId}_${cluster.region}_${cluster.name}`;
@@ -12,7 +12,7 @@ export const getFullKubernetesClusterName = (
 
 export const getKubernetesNamespace = (
   config: Pick<Config, "customerName" | "appName">,
-  env: string
+  env: string,
 ) => {
   return `${config.customerName}-${config.appName}-${env}`;
 };

package/src/index.ts

@@ -1,4 +1,4 @@
-export * from "./pipeline/";
+export * from "./pipeline";
 export * from "./types";
 export * from "./rules";
 export * from "./runner";
@@ -6,3 +6,4 @@ export * from "./config";
 export * from "./context";
 export * from "./build";
 export * from "./deploy";
+export * from "./utils/writeFiles";

package/src/packageInfos.ts

@@ -0,0 +1,10 @@
+// we can't import the package.json directly, because the bundling process will inline it
+// but the version is only set during deployement, when its already bundled
+// instead we need to use require.main.path to get the path of the main module, which is actually the file in the bin
+// this will point to the root package.json
+
+const path = require.main?.path + "/../package.json";
+// eslint-disable-next-line @typescript-eslint/no-var-requires
+const ownPkg = require(path);
+
+export default ownPkg;