@catladder/pipeline 0.0.0-vars-07a720d0 → 0.0.0-waitfor-deploy-e64a33ba

package/src/context/resolveReferences.ts

@@ -6,32 +6,36 @@ export const resolveReferences = (
   vars: Record<string, string>,
   getOtherVariables?: (
     componentName: string,
-    variableName: string
-  ) => string | null
+    variableName: string,
+    alreadyVisited: Record<string, Record<string, boolean>>
+  ) => string | null,
+  alreadyVisitedBase: Record<string, Record<string, boolean>> = {}
 ) => {
   const replaceSingleValue = (
     value: string,
-    alreadyVisited: Record<string, Record<string, boolean>> = {}
+    alreadyVisited: Record<string, Record<string, boolean>> = alreadyVisitedBase
   ): string => {
     if (REGEX.test(value)) {
       return value.replace(REGEX, (match, _, componentName, variableName) => {
         if (alreadyVisited[componentName]?.[variableName]) {
           return match; // prevent endless loop
         }
+        const newAlreadyVisited = merge({}, alreadyVisited, {
+          [componentName]: {
+            [variableName]: true,
+          },
+        });
         const result = componentName
-          ? getOtherVariables?.(componentName, variableName) ?? null
+          ? getOtherVariables?.(
+              componentName,
+              variableName,
+              newAlreadyVisited
+            ) ?? null
           : vars[variableName]; // is self reference
 
         const replaced =
           result !== null && result !== undefined
-            ? replaceSingleValue(
-                result,
-                merge({}, alreadyVisited, {
-                  [componentName]: {
-                    [variableName]: true,
-                  },
-                })
-              )
+            ? replaceSingleValue(result, newAlreadyVisited)
             : match;
 
         return replaced;

package/src/deploy/base.ts

@@ -1,5 +1,7 @@
+import { requiresDockerBuild } from "../build/docker";
 import { Context } from "../types";
 import { CatladderJob } from "../types/jobs";
+import { contextIsStoppable } from "./utils";
 
 export const DEPLOY_JOB_NAME = "🚀 Deploy";
 export const STOP_JOB_NAME = "🛑 Stop ⚠️";
@@ -17,36 +19,53 @@ export const getBaseDeploymentJob = (context: Context): JobWithoutScript => {
     url: context.environment.url,
   };
 
+  const hasDocker = requiresDockerBuild(context);
+  const isStoppable = contextIsStoppable(context);
+
   const autoStop =
     context.environment.envType === "review"
-      ? "2 weeks"
+      ? "3 days" // auto stop is not working properly when branch is merged, so we do
       : context.environment.envType === "dev"
       ? "3 weeks"
       : undefined;
-  // automatically run deploy job, unless its prod and stage is active
-  const autoDeploy =
+  // if auto or manual is configured explicitly, use that
+  const whenDeployDefined =
+    context.componentConfig.deploy && context.componentConfig.deploy.when
+      ? context.componentConfig.deploy.when
+      : undefined;
+  // otherwise auto deploy if env is not prod. If its prod, deploy automatically if stage is disabled
+  const whenDeployDefault =
     context.environment.envType !== "prod"
-      ? true // is not stage, auto deploy
+      ? "auto" // is not stage, auto deploy
       : context.componentConfig.env?.stage === false
-      ? true // is prod, but no staging, auto deploy
-      : false; // manually deploy
+      ? "auto" // is prod, but no staging, auto deploy
+      : "manual"; // manually deploy
+  const whenDeploy = whenDeployDefined ? whenDeployDefined : whenDeployDefault;
+
   return {
     name: DEPLOY_JOB_NAME,
     envMode: "stagePerEnv", // makes it easier to run manual tasks er env
+
+    needsOtherComponent: context.componentConfig.deploy
+      ? context.componentConfig.deploy.waitFor?.map((c) => ({
+          componentName: c,
+          job: DEPLOY_JOB_NAME,
+          artifacts: false,
+        })) ?? undefined
+      : undefined,
     // we don't want to deploy when there is a broken test
     needsStages: [
       {
         stage: "build",
-        artifacts: false,
+        artifacts: hasDocker ? false : true, // we asume that no-docker deployments need build artifacts,
       },
       {
         stage: "test",
         artifacts: false,
       },
     ], // workaround for https://gitlab.com/gitlab-org/gitlab/-/issues/220758
-
     rules: [
-      autoDeploy
+      whenDeploy === "auto"
         ? {
             when: "on_success",
           }
@@ -54,14 +73,20 @@ export const getBaseDeploymentJob = (context: Context): JobWithoutScript => {
             when: "manual",
           },
     ],
+    allow_failure: whenDeploy === "manual" ? true : false,
     stage: "deploy",
     variables: {
       ...DEPLOY_RUNNER_VARIABLES,
+      ...context.environment.envVars,
     },
     environment: {
       ...environment,
-      on_stop: STOP_JOB_NAME,
-      auto_stop_in: autoStop,
+      ...(isStoppable
+        ? {
+            on_stop: STOP_JOB_NAME,
+            auto_stop_in: autoStop,
+          }
+        : {}),
     },
   };
 };

package/src/deploy/custom/deployJob.ts

@@ -0,0 +1,23 @@
+import { merge } from "lodash";
+import { Context } from "../../types/context";
+import { CatladderJob } from "../../types/jobs";
+import { getBaseDeploymentJob } from "../base";
+import { isOfDeployType } from "../types";
+
+export const createCustomDeployJobs = (context: Context): CatladderJob[] => {
+  const deployConfig = context.componentConfig.deploy;
+  if (deployConfig === false) {
+    return [];
+  }
+  if (!isOfDeployType(deployConfig, "custom")) {
+    // should not happen
+    throw new Error("deploy config is not custom");
+  }
+  const baseDeploymentJob = getBaseDeploymentJob(context);
+
+  return [
+    merge({}, baseDeploymentJob, {
+      script: [`cd ${context.componentConfig.dir}`, ...deployConfig.script],
+    }),
+  ];
+};

package/src/deploy/custom/index.ts

@@ -0,0 +1 @@
+export * from "./deployJob";

package/src/deploy/index.ts

@@ -1,5 +1,6 @@
 import { Context } from "../types/context";
 import { CatladderJob } from "../types/jobs";
+import { createCustomDeployJobs } from "./custom";
 import { createKubernetesDeployJobs } from "./kubernetes";
 import { DeployConfig } from "./types";
 export * from "./kubernetes";
@@ -17,4 +18,8 @@ export const DEPLOY_TYPES: DeployTypes = {
     jobs: createKubernetesDeployJobs,
     defaults: () => ({}),
   },
+  custom: {
+    jobs: createCustomDeployJobs,
+    defaults: () => ({}),
+  },
 };

package/src/deploy/kubernetes/deployJob.ts

@@ -10,6 +10,7 @@ import { getBaseDeploymentJob, getBaseDeploymentStopJob } from "../base";
 import { isOfDeployType } from "../types";
 import { createCloudsqlBaseConfig } from "./cloudsql";
 import { createMongodbBaseConfig } from "./mongodb";
+import { processSecretsAsFiles } from "./processSecretsAsFiles";
 
 export const createKubernetesDeployJobs = (
   context: Context
@@ -74,6 +75,7 @@ export const createKubernetesDeployJobs = (
 
   const defaultKubeValues = {
     application: defaultAppConfig,
+
     env: env,
     ...(deployConfig.values?.cloudsql?.enabled
       ? createCloudsqlBaseConfig(context)
@@ -83,9 +85,8 @@ export const createKubernetesDeployJobs = (
       : {}),
   };
 
-  const kubeValues = mergeWithMergingArrays(
-    defaultKubeValues,
-    deployConfig.values
+  const kubeValues = processSecretsAsFiles(
+    mergeWithMergingArrays(defaultKubeValues, deployConfig.values)
   );
 
   const kubernetesEnvironment = {

package/src/deploy/kubernetes/processSecretsAsFiles.ts

@@ -0,0 +1,34 @@
+import { omit } from "lodash";
+import { DeployConfigKubernetesValues } from "../types";
+
+export const processSecretsAsFiles = <
+  T extends {
+    env: {
+      secret: Record<string, string>;
+    };
+  } & DeployConfigKubernetesValues
+>(
+  values: T
+): T => {
+  if (!values.secretsAsFile || values.secretsAsFile.length === 0) {
+    return values;
+  }
+  // the initial idea was that we "shadow" secrets that we mount as files,
+  // but there is a weird kubernetes  issue with that (https://github.com/kubernetes/kubernetes/issues/46861),
+  // so instead we rename the secrets and append _content, so that inside kuberentes, we can use the original name to contain the path and name+_content as the actual content
+  return {
+    ...values,
+    env: {
+      ...values.env,
+      secret: {
+        ...omit(values.env.secret, values.secretsAsFile),
+        ...Object.fromEntries(
+          values.secretsAsFile.map((key) => [
+            key + "_content",
+            values.env.secret[key],
+          ])
+        ),
+      },
+    },
+  };
+};

package/src/deploy/types.ts

@@ -1,5 +1,18 @@
 // eslint-disable-next-line @typescript-eslint/ban-types
-export type DeployConfigBase = {};
+export type DeployConfigBase = {
+  /**
+   * whether to deploy automatically or manual. If not defined, these rules apply:
+   * - prod: manual
+   * - all other envs: auto
+   */
+  when?: "manual" | "auto";
+
+  /**
+   * EXPERIMENTAL
+   * wait for other components to deploy first, before doing this deployment
+   */
+  waitFor?: string[];
+};
 type AllowUnknownProps<T extends Record<string, unknown>> = T &
   Record<string, unknown>;
 
@@ -24,6 +37,30 @@ export type DeployConfigKubernetesClusterGCloud = {
 };
 export type DeployConfigKubernetesCluster = DeployConfigKubernetesClusterGCloud; // currently only this
 
+export type KubernetesAutoscaleMetricResourceTarget =
+  | {
+      type: "Utilization";
+      averageUtilization: number;
+    }
+  | {
+      type: "AverageValue";
+      averageValue: number | string;
+    };
+export type KubernetesAutoscaleMetric = {
+  type: "Resource";
+  resource: {
+    name: "cpu" | "memory";
+    target: KubernetesAutoscaleMetricResourceTarget;
+  };
+};
+export type KubernetesAutoscale = {
+  minReplicas: number;
+  maxReplicas: number;
+  /**
+   * declare a number of metrics (at least one). Usually you would use cpu as a metric
+   */
+  metrics: KubernetesAutoscaleMetric[];
+};
 type KubernetesHealthDef = {
   httpGet?: {
     path?: string;
@@ -94,6 +131,11 @@ export type DeployConfigKubernetesValues = AllowUnknownProps<{
      * how many pods will be started
      */
     replicas?: number;
+
+    /**
+     * autoscale the pods horizontally
+     */
+    autoscale?: KubernetesAutoscale;
     /**
      * kubernetes resources
      */
@@ -112,6 +154,11 @@ export type DeployConfigKubernetesValues = AllowUnknownProps<{
     readinessProbe?: KubernetesHealthDef;
     livenessProbe?: KubernetesHealthDef;
   }>;
+  /**
+   * Mount secrets as files in the filesystem.
+   * These secrets are still available as env vars, but will contain the path of the file instead
+   */
+  secretsAsFile?: string[];
 }>;
 
 export type DeployConfigKubernetes = {
@@ -131,12 +178,14 @@ export type DeployConfigKubernetes = {
   values?: DeployConfigKubernetesValues;
 } & DeployConfigBase;
 
-type CustomDeployConfig = {
+type DeployConfigCustom = {
   type: "custom";
+  requiresDocker: boolean;
   script: string[];
-};
+  stopScript?: string[];
+} & DeployConfigBase;
 
-export type DeployConfig = DeployConfigKubernetes;
+export type DeployConfig = DeployConfigKubernetes | DeployConfigCustom;
 
 export const isOfDeployType = <T extends Array<DeployConfig["type"]>>(
   t: DeployConfig | false,

package/src/deploy/utils.ts

@@ -1,5 +1,5 @@
-import { Config } from "../types";
-import type { DeployConfigKubernetesCluster } from "./types";
+import { Config, Context } from "../types";
+import { DeployConfigKubernetesCluster, isOfDeployType } from "./types";
 
 export const getFullKubernetesClusterName = (
   cluster: DeployConfigKubernetesCluster
@@ -15,3 +15,16 @@ export const getKubernetesNamespace = (
 ) => {
   return `${config.customerName}-${config.appName}-${env}`;
 };
+
+export const contextIsStoppable = (context: Context) => {
+  const deployConfig = context.componentConfig.deploy;
+  if (isOfDeployType(deployConfig, "kubernetes")) {
+    return true;
+  }
+
+  if (isOfDeployType(deployConfig, "custom") && deployConfig.stopScript) {
+    return true;
+  }
+
+  return false;
+};

package/src/pipeline/createJobs.ts

@@ -1,5 +1,5 @@
 import { exec } from "child-process-promise";
-import { isObject } from "lodash";
+import { isFunction, isObject } from "lodash";
 import { BUILD_TYPES } from "../build";
 import { createContext } from "../context";
 import { DEPLOY_TYPES } from "../deploy";
@@ -11,6 +11,26 @@ import { notNil } from "../utils";
 import { makeGitlabJob } from "./gitlab/makeGitlabJob";
 import { getPackageManagerInfo } from "./packageManager";
 
+const injectDefaultVarsInCustomJobs = (
+  context: Context,
+  jobs: CatladderJob[]
+) =>
+  jobs.map(({ variables, ...job }) => ({
+    variables: {
+      ...(context.environment.envVars ?? {}),
+      ...(variables ?? {}),
+    },
+    ...job,
+  }));
+const getCustomJobs = (context: Context) => {
+  if (!context.componentConfig.customJobs) {
+    return [];
+  }
+  const rawJobs = isFunction(context.componentConfig.customJobs)
+    ? context.componentConfig.customJobs(context)
+    : context.componentConfig.customJobs;
+  return injectDefaultVarsInCustomJobs(context, rawJobs);
+};
 const createRawJobs = (context: Context): CatladderJob[] => {
   if (context.componentConfig.deploy === false) {
     return [];
@@ -20,7 +40,8 @@ const createRawJobs = (context: Context): CatladderJob[] => {
   const deployJobs =
     DEPLOY_TYPES[context.componentConfig.deploy.type].jobs(context);
 
-  return [...buildJobs, ...deployJobs];
+  const customJobs = getCustomJobs(context);
+  return [...buildJobs, ...deployJobs, ...customJobs];
 };
 const getFullJobName = (
   name: string,
@@ -56,22 +77,33 @@ const replaceReferences = (
 ): CatladderJob<string> => {
   const stage =
     job.envMode === "stagePerEnv" ? `${job.stage} ${env}` : job.stage;
+
+  const needsFromNeeds: CatladderJob["needs"] = job.needs?.map((n) =>
+    isObject(n)
+      ? {
+          job: getFullReferencedJobName(n.job, componentName, env, allRawJobs),
+          artifacts: n.artifacts,
+        }
+      : getFullReferencedJobName(n, componentName, env, allRawJobs)
+  );
+  const needsFromOtherComponents: CatladderJob["needs"] =
+    job.needsOtherComponent?.map((other) => ({
+      artifacts: other.artifacts,
+      job: getFullReferencedJobName(
+        other.job,
+        other.componentName,
+        env,
+        allRawJobs
+      ),
+    }));
+  const needs = [
+    ...(needsFromNeeds ?? []),
+    ...(needsFromOtherComponents ?? []),
+  ];
   return {
     ...job,
     stage,
-    needs: job.needs?.map((n) =>
-      isObject(n)
-        ? {
-            job: getFullReferencedJobName(
-              n.job,
-              componentName,
-              env,
-              allRawJobs
-            ),
-            artifacts: n.artifacts,
-          }
-        : getFullReferencedJobName(n, componentName, env, allRawJobs)
-    ),
+    needs,
     environment: job.environment?.on_stop
       ? {
           ...job.environment,
@@ -122,7 +154,9 @@ export const createJobs = async <T extends PipelineType>(
 ): Promise<Record<string, PipelineJob<T>>> => {
   const commitInfo: CommitInfo = {
     refName: process.env.CI_COMMIT_REF_NAME ?? "unknown",
-    refSlug: process.env.CI_COMMIT_REF_SLUG ?? "unknown",
+    reviewSlug: process.env.CI_MERGE_REQUEST_IID
+      ? `mr${process.env.CI_MERGE_REQUEST_IID}`
+      : "unknown",
     buildTime: new Date().toISOString(),
     buildId: await exec("git describe --tags || git rev-parse HEAD").then((s) =>
       s.stdout.trim()

package/src/types/config.ts

@@ -1,6 +1,8 @@
 import { BuildConfig } from "../build/types";
 import { DeployConfig } from "../deploy/types";
 import { PartialDeep } from "type-fest";
+import { CatladderJob } from "./jobs";
+import { Context } from "./context";
 export type PipelineTrigger = "mainBranch" | "mr" | "taggedRelease";
 
 /**
@@ -123,6 +125,16 @@ export type ComponentConfig = {
    * the directory of the component (e.g. where the package.json or similar is located). You can set "." if you only have one app.
    */
   dir: string;
+
+  /**
+   * add custom jobs. Can be an array or a function that receives the context and returns an array of jobs.
+   * For convenience, all env vars are injectd as variables
+   *
+   * Please raise an issue on https://git.panter.ch/catladder/catladder
+   * to let us know about why you need to use custom jobs.
+   * This feedback will help us to generalize use cases
+   */
+  customJobs?: CatladderJob[] | ((context: Context) => CatladderJob[]);
 } & DefaultEnvConfig;
 
 export type Config = {

package/src/types/context.ts

@@ -16,7 +16,7 @@ export type Environment = {
 
 export type CommitInfo = {
   refName: string;
-  refSlug: string;
+  reviewSlug: string;
   buildTime: string;
   buildId: string;
   trigger: PipelineTrigger;

package/src/types/jobs.ts

@@ -49,9 +49,15 @@ export type CatladderJob<S = BaseStage> = {
   }[];
 
   /**
-   * does this require another job?
+   * does this require another job (from the same component)?
    */
   needs?: Array<string | { job: string; artifacts: boolean }>;
+
+  needsOtherComponent?: Array<{
+    componentName: string;
+    job: string;
+    artifacts: boolean;
+  }>;
   /**
    * cache config, we use here the same shape as gitlab itself
    */