@catladder/pipeline 0.0.0-node16-ddfae617 → 0.0.0-prettier-cbceeed9

package/src/context/index.ts

@@ -1,216 +1,51 @@
-import { isObject, merge } from "lodash";
-import slugify from "slugify";
 import { BUILD_TYPES } from "../build";
-import { BuildConfig } from "../build/types";
-import { DEPLOY_TYPES, getKubernetesNamespace } from "../deploy";
-import { DeployConfig } from "../deploy/types";
-import { Config, isKnowEnvType, DevLocalEnvConfig } from "../types/config";
-import {
-  CommitInfo,
-  Context,
-  Environment,
-  PackageManagerInfo,
-} from "../types/context";
+import type { BuildConfig, BuildConfigType } from "../build/types";
+import { DEPLOY_TYPES } from "../deploy";
+import type { DeployConfig, DeployConfigType } from "../deploy/types";
+import type { Config } from "../types/config";
+import type { CommitInfo, Context, PackageManagerInfo } from "../types/context";
+import type { PartialDeep } from "../types/utils";
 import { mergeWithMergingArrays } from "../utils";
-import {
-  resolveReferences,
-  translateLegacyFromComponents,
-} from "./resolveReferences";
+import { getEnvironment } from "./getEnvironment";
+import { getEnvironmentContext } from "./getEnvironmentContext";
 
-const sanitizeForEnVar = (s: string) => s.replace(/-/g, "_");
-export const getSecretVarName = (
-  env: string,
-  componentName: string,
-  key: string
-) => `CL_${sanitizeForEnVar(env)}_${sanitizeForEnVar(componentName)}_${key}`; // remove dash from component name
-
-export const getSecretVarNameForContext = (context: Context, key: string) =>
-  getSecretVarName(context.environment.shortName, context.componentName, key);
-export const getEnvironment = (
-  config: Config,
-  componentName: string,
-  env: string,
-  commitInfo?: CommitInfo,
-  alreadyVisited: Record<string, Record<string, boolean>> = {} // to prevent endless loop
-): Environment => {
-  const componentConfig = config.components[componentName];
-  if (!componentConfig) {
-    throw new Error("unknown component " + componentName);
-  }
-
-  const defaultConfig = componentConfig;
-
-  const envConfig = componentConfig.env?.[env] ?? {};
-  if (envConfig === false) {
-    throw new Error("env is disabled: " + env);
-  }
-
-  const mergedConfig = mergeWithMergingArrays(defaultConfig, envConfig);
-
-  // env type: if its set manually, use that, otherwise use the known env types
-
-  const envType = envConfig?.type ?? (isKnowEnvType(env) ? env : null);
-
-  if (!envType) {
-    throw new Error(
-      "Missing type in environment " + env + " in component " + componentName
-    );
-  }
-
-  const basePredefinedVariables = {
-    ENV_SHORT: env,
-    APP_DIR: componentConfig.dir,
-    ENV_TYPE: envType,
-  };
-
-  const environmentName =
-    envType === "review" && commitInfo
-      ? `${env}/${commitInfo.refName}/${componentName}`
-      : `${env}/${componentName}`;
-
-  const environmentSlug =
-    envType === "review" && commitInfo
-      ? `${env}-${commitInfo.reviewSlug}-${componentName}`
-      : `${env}-${componentName}`;
+export * from "./getEnvironment";
+export * from "./getEnvironmentVariables";
 
-  let predefinedVariables: Record<string, string>;
-  let host: string;
-  let url: string;
-  if (envType === "local") {
-    const devLocalConfig: DevLocalEnvConfig = mergedConfig;
-    const port = devLocalConfig.port ?? 3000;
-    host = "localhost:" + port;
-    url = "http://" + host;
-    predefinedVariables = {
-      ENV_SHORT: "local",
-      ROOT_URL: url,
-      PORT: port.toString(),
-    };
-  } else {
-    const RELEASE_NAME = `${config.customerName}-${config.appName}-${environmentSlug}`;
-
-    const componentSlug = slugify(componentName);
-    const envInUrl =
-      envType === "review" && commitInfo ? `${commitInfo.reviewSlug}.${env}` : env;
-
-    const domainCanonical =
-      (mergedConfig?.deploy &&
-        mergedConfig?.deploy.type === "kubernetes" &&
-        mergedConfig.deploy.cluster?.domainCanonical) || // for convenience, we allow clusters to define a canonical domain, because a cluster has a fixed ip and you will usually have a domain pointing to that cluster
-      config.domainCanonical ||
-      "panter.cloud";
-
-    const HOST_CANONICAL = `${componentSlug}.${envInUrl}.${config.appName}.${config.customerName}.${domainCanonical}`;
-    host = mergedConfig?.host ?? HOST_CANONICAL;
-    const url = `https://${host}`;
-
-    // FIXME: move to kube specific jobs
-    const KUBE_APP_NAME =
-      envType === "review" && commitInfo
-        ? `${commitInfo.reviewSlug}-${componentName}`
-        : componentName;
-    const KUBE_NAMESPACE = getKubernetesNamespace(config, env);
-
-    predefinedVariables = {
-      ...basePredefinedVariables,
-      HOST_CANONICAL,
-      ROOT_URL: url,
-      KUBE_NAMESPACE,
-      KUBE_APP_NAME,
-      RELEASE_NAME,
-    };
-  }
-  const publicEnvVarsRaw = mergedConfig.vars?.public ?? {};
-  const secretEnvVarKeys = mergedConfig.vars?.secret ?? [];
-  const secretEnvVars = Object.fromEntries(
-    secretEnvVarKeys.map((key) => [
-      key,
-      `$${getSecretVarName(env, componentName, key)}`,
-    ])
-  );
-
-  // this is deprecated, we now support: $componentname:FOO
-  const legacyFromComponents = mergedConfig.vars?.fromComponents ?? {};
-  const publicEnvVarsRawWithLegasyFromComponents = merge(
-    {},
-    translateLegacyFromComponents(legacyFromComponents),
-    publicEnvVarsRaw
-  );
-
-  const publicEnvVarsRawSanitized = Object.fromEntries(
-    Object.entries(publicEnvVarsRawWithLegasyFromComponents).map(
-      ([key, value]) => [
-        key,
-        isObject(value) ? JSON.stringify(value) : `${value}`,
-      ]
-    )
-  );
-
-  const envVarsRaw = {
-    ...predefinedVariables,
-    ...secretEnvVars,
-    ...publicEnvVarsRawSanitized,
-  };
-
-  const envVars = resolveReferences(
-    envVarsRaw,
-    (componentName, variableName, alreadyVisited) => {
-      const { envVars } = getEnvironment(
-        config,
-        componentName,
-        env,
-        commitInfo,
-        alreadyVisited
-      );
-      return envVars[variableName];
-    },
-    alreadyVisited
-  );
-  return {
-    envType,
-    host,
-    fullName: environmentName,
-    slug: environmentSlug,
-    shortName: env,
-    url: predefinedVariables.ROOT_URL,
-    envVars,
-    secretEnvVarKeys,
-  };
-};
-
-export const createContext = (
+export const createContext = async (
   config: Config,
   componentName: string,
   env: string,
   commitInfo?: CommitInfo,
   packageManagerInfo?: PackageManagerInfo
-): Context => {
+): Promise<Context> => {
   if (!/^[a-z0-9-]+$/.test(componentName)) {
     throw new Error(
       "componentName may only contain lower case letters, numbers and -"
     );
   }
-  const rawConfig = config.components[componentName];
-  if (!rawConfig) {
-    throw new Error("unknown component " + componentName);
-  }
-  // envs can override the config
-  const envConfig = rawConfig.env?.[env] ?? {};
-  const componentConfigWithoutDefaults = mergeWithMergingArrays(
-    rawConfig,
-    envConfig
+
+  const envContext = getEnvironmentContext(
+    config,
+    env,
+    componentName,
+    commitInfo
   );
 
-  // fill in defaults of build and deploy
+  const componentConfigWithoutDefaults = envContext.envConfigRaw;
   const defaults: {
-    build: Partial<BuildConfig>;
-    deploy: Partial<DeployConfig>;
+    build: PartialDeep<BuildConfig>;
+    deploy: PartialDeep<DeployConfig>;
   } = componentConfigWithoutDefaults.deploy
     ? {
         build:
-          BUILD_TYPES[componentConfigWithoutDefaults.build.type].defaults(),
+          BUILD_TYPES[
+            componentConfigWithoutDefaults.build.type as BuildConfigType
+          ].defaults(envContext),
         deploy:
-          DEPLOY_TYPES[componentConfigWithoutDefaults.deploy.type].defaults(),
+          DEPLOY_TYPES[
+            componentConfigWithoutDefaults.deploy.type as DeployConfigType
+          ].defaults(envContext),
       }
     : {
         build: {},
@@ -225,7 +60,7 @@ export const createContext = (
     fullConfig: config,
     componentConfig,
     componentName,
-    environment: getEnvironment(config, componentName, env, commitInfo),
+    environment: await getEnvironment(config, componentName, env, commitInfo),
     commitInfo,
     packageManagerInfo,
   };

package/src/context/resolveReferences.ts

@@ -1,55 +1,59 @@
 import { merge } from "lodash";
+import replaceAsync from "string-replace-async";
 
 const REGEX = /\$\{(([^:]+):)?([^}]+)}/gm;
 
-export const resolveReferences = (
+export const resolveReferences = async (
   vars: Record<string, string>,
   getOtherVariables?: (
     componentName: string,
-    variableName: string,
     alreadyVisited: Record<string, Record<string, boolean>>
-  ) => string | null,
+  ) => Promise<Record<string, string | null>>,
   alreadyVisitedBase: Record<string, Record<string, boolean>> = {}
 ) => {
-  const replaceSingleValue = (
+  const replaceSingleValue = async (
     value: string,
     alreadyVisited: Record<string, Record<string, boolean>> = alreadyVisitedBase
-  ): string => {
+  ): Promise<string> => {
     if (REGEX.test(value)) {
-      return value.replace(REGEX, (match, _, componentName, variableName) => {
-        if (alreadyVisited[componentName]?.[variableName]) {
-          return match; // prevent endless loop
-        }
-        const newAlreadyVisited = merge({}, alreadyVisited, {
-          [componentName]: {
-            [variableName]: true,
-          },
-        });
-        const result = componentName
-          ? getOtherVariables?.(
-              componentName,
-              variableName,
-              newAlreadyVisited
-            ) ?? null
-          : vars[variableName]; // is self reference
+      return await replaceAsync(
+        value,
+        REGEX,
+        async (match, _, componentName, variableName) => {
+          if (alreadyVisited[componentName]?.[variableName]) {
+            return match; // prevent endless loop
+          }
+          const newAlreadyVisited = merge({}, alreadyVisited, {
+            [componentName]: {
+              [variableName]: true,
+            },
+          });
+          const result = componentName
+            ? (await getOtherVariables?.(componentName, newAlreadyVisited).then(
+                (r) => r?.[variableName]
+              )) ?? null
+            : vars[variableName]; // is self reference
 
-        const replaced =
-          result !== null && result !== undefined
-            ? replaceSingleValue(result, newAlreadyVisited)
-            : match;
+          const replaced =
+            result !== null && result !== undefined
+              ? await replaceSingleValue(result, newAlreadyVisited)
+              : match;
 
-        return replaced;
-      });
+          return replaced;
+        }
+      );
     } else {
       return value;
     }
   };
 
   return Object.fromEntries(
-    Object.entries(vars).map(([key, value]) => {
-      return [key, replaceSingleValue(value)];
-    })
-  );
+    await Promise.all(
+      Object.entries(vars).map(async ([key, value]) => {
+        return [key, await replaceSingleValue(value)];
+      })
+    )
+  ) as Record<string, string>;
 };
 
 export const translateLegacyFromComponents = (

package/src/defaults/index.ts

@@ -1,4 +1,4 @@
-import { Retry } from "../types";
+import type { Retry } from "../types";
 
 export const BASE_RETRY: Retry = {
   max: 2,

package/src/deploy/base.ts

@@ -1,10 +1,12 @@
-import { requiresDockerBuild } from "../build/docker";
-import { Context } from "../types";
-import { CatladderJob } from "../types/jobs";
+import { getDockerImageVariables, requiresDockerBuild } from "../build/docker";
+import type { Context } from "../types";
+import type { CatladderJob } from "../types/jobs";
+import { isOfDeployType } from "./types";
 import { contextIsStoppable } from "./utils";
 
 export const DEPLOY_JOB_NAME = "🚀 Deploy";
 export const STOP_JOB_NAME = "🛑 Stop ⚠️";
+export const ROLLBACK_JOB_NAME = "↩️ Rollback ⚠️";
 
 const DEPLOY_RUNNER_VARIABLES = {
   KUBERNETES_CPU_REQUEST: "0.5",
@@ -14,20 +16,16 @@ const DEPLOY_RUNNER_VARIABLES = {
 };
 type JobWithoutScript = Omit<CatladderJob, "script">;
 export const getBaseDeploymentJob = (context: Context): JobWithoutScript => {
-  const environment = {
-    name: context.environment.fullName,
-    url: context.environment.url,
-  };
-
   const hasDocker = requiresDockerBuild(context);
   const isStoppable = contextIsStoppable(context);
 
   const autoStop =
     context.environment.envType === "review"
-      ? "3 days" // auto stop is not working properly when branch is merged, so we do
+      ? "2 week"
       : context.environment.envType === "dev"
-      ? "3 weeks"
+      ? "4 weeks"
       : undefined;
+
   // if auto or manual is configured explicitly, use that
   const whenDeployDefined =
     context.componentConfig.deploy && context.componentConfig.deploy.when
@@ -45,6 +43,14 @@ export const getBaseDeploymentJob = (context: Context): JobWithoutScript => {
   return {
     name: DEPLOY_JOB_NAME,
     envMode: "stagePerEnv", // makes it easier to run manual tasks er env
+
+    needs: context.componentConfig.deploy
+      ? context.componentConfig.deploy.waitFor?.map((c) => ({
+          componentName: c,
+          job: DEPLOY_JOB_NAME,
+          artifacts: false,
+        })) ?? undefined
+      : undefined,
     // we don't want to deploy when there is a broken test
     needsStages: [
       {
@@ -56,7 +62,6 @@ export const getBaseDeploymentJob = (context: Context): JobWithoutScript => {
         artifacts: false,
       },
     ], // workaround for https://gitlab.com/gitlab-org/gitlab/-/issues/220758
-
     rules: [
       whenDeploy === "auto"
         ? {
@@ -71,9 +76,10 @@ export const getBaseDeploymentJob = (context: Context): JobWithoutScript => {
     variables: {
       ...DEPLOY_RUNNER_VARIABLES,
       ...context.environment.envVars,
+      ...(hasDocker ? getDockerImageVariables(context) : {}),
     },
     environment: {
-      ...environment,
+      ...context.environment.gitlabEnvironment,
       ...(isStoppable
         ? {
             on_stop: STOP_JOB_NAME,
@@ -87,15 +93,10 @@ export const getBaseDeploymentJob = (context: Context): JobWithoutScript => {
 export const getBaseDeploymentStopJob = (
   context: Context
 ): JobWithoutScript => {
-  const environment = {
-    name: context.environment.fullName,
-    url: context.environment.url,
-  };
   return {
     name: STOP_JOB_NAME,
     envMode: "stagePerEnv", // makes it easier to run manual tasks er env
-
-    needs: [DEPLOY_JOB_NAME],
+    needs: [], // can be executed even if the deploy job failed
     rules: [
       {
         if: "$CI_COMMIT_BRANCH =~ /^[0-9]+\\.([0-9]+|x)\\.x$/", // automatic on hotfix branches
@@ -113,8 +114,31 @@ export const getBaseDeploymentStopJob = (
     },
     stage: "stop",
     environment: {
-      ...environment,
+      ...context.environment.gitlabEnvironment,
       action: "stop",
     },
   };
 };
+
+export const getBaseRollbackJob = (context: Context): JobWithoutScript => {
+  return {
+    name: ROLLBACK_JOB_NAME,
+    envMode: "stagePerEnv", // makes it easier to run manual tasks er env
+    needs: [], // can be executed even if the deploy job failed
+    rules: [
+      {
+        when: "manual",
+        allow_failure: true,
+      },
+    ],
+    variables: {
+      ...DEPLOY_RUNNER_VARIABLES,
+      GIT_STRATEGY: "none",
+    },
+    stage: "rollback",
+    environment: {
+      ...context.environment.gitlabEnvironment,
+      action: "access",
+    },
+  };
+};