@catladder/pipeline 0.0.0-node16-ddfae617 → 0.0.0-refactor-env-context-9e652b63

package/src/context/index.ts

@@ -1,216 +1,50 @@
-import { isObject, merge } from "lodash";
-import slugify from "slugify";
 import { BUILD_TYPES } from "../build";
-import { BuildConfig } from "../build/types";
-import { DEPLOY_TYPES, getKubernetesNamespace } from "../deploy";
-import { DeployConfig } from "../deploy/types";
-import { Config, isKnowEnvType, DevLocalEnvConfig } from "../types/config";
-import {
-  CommitInfo,
-  Context,
-  Environment,
-  PackageManagerInfo,
-} from "../types/context";
+import type { BuildConfig, BuildConfigType } from "../build/types";
+import { DEPLOY_TYPES } from "../deploy";
+import type { DeployConfig, DeployConfigType } from "../deploy/types";
+import type { Config } from "../types/config";
+import type { CommitInfo, Context, PackageManagerInfo } from "../types/context";
 import { mergeWithMergingArrays } from "../utils";
-import {
-  resolveReferences,
-  translateLegacyFromComponents,
-} from "./resolveReferences";
+import { getEnvironment } from "./getEnvironment";
+import { getEnvironmentContext } from "./getEnvironmentContext";
 
-const sanitizeForEnVar = (s: string) => s.replace(/-/g, "_");
-export const getSecretVarName = (
-  env: string,
-  componentName: string,
-  key: string
-) => `CL_${sanitizeForEnVar(env)}_${sanitizeForEnVar(componentName)}_${key}`; // remove dash from component name
-
-export const getSecretVarNameForContext = (context: Context, key: string) =>
-  getSecretVarName(context.environment.shortName, context.componentName, key);
-export const getEnvironment = (
-  config: Config,
-  componentName: string,
-  env: string,
-  commitInfo?: CommitInfo,
-  alreadyVisited: Record<string, Record<string, boolean>> = {} // to prevent endless loop
-): Environment => {
-  const componentConfig = config.components[componentName];
-  if (!componentConfig) {
-    throw new Error("unknown component " + componentName);
-  }
-
-  const defaultConfig = componentConfig;
-
-  const envConfig = componentConfig.env?.[env] ?? {};
-  if (envConfig === false) {
-    throw new Error("env is disabled: " + env);
-  }
-
-  const mergedConfig = mergeWithMergingArrays(defaultConfig, envConfig);
-
-  // env type: if its set manually, use that, otherwise use the known env types
-
-  const envType = envConfig?.type ?? (isKnowEnvType(env) ? env : null);
-
-  if (!envType) {
-    throw new Error(
-      "Missing type in environment " + env + " in component " + componentName
-    );
-  }
-
-  const basePredefinedVariables = {
-    ENV_SHORT: env,
-    APP_DIR: componentConfig.dir,
-    ENV_TYPE: envType,
-  };
-
-  const environmentName =
-    envType === "review" && commitInfo
-      ? `${env}/${commitInfo.refName}/${componentName}`
-      : `${env}/${componentName}`;
-
-  const environmentSlug =
-    envType === "review" && commitInfo
-      ? `${env}-${commitInfo.reviewSlug}-${componentName}`
-      : `${env}-${componentName}`;
+export * from "./getEnvironment";
+export * from "./getEnvironmentVariables";
 
-  let predefinedVariables: Record<string, string>;
-  let host: string;
-  let url: string;
-  if (envType === "local") {
-    const devLocalConfig: DevLocalEnvConfig = mergedConfig;
-    const port = devLocalConfig.port ?? 3000;
-    host = "localhost:" + port;
-    url = "http://" + host;
-    predefinedVariables = {
-      ENV_SHORT: "local",
-      ROOT_URL: url,
-      PORT: port.toString(),
-    };
-  } else {
-    const RELEASE_NAME = `${config.customerName}-${config.appName}-${environmentSlug}`;
-
-    const componentSlug = slugify(componentName);
-    const envInUrl =
-      envType === "review" && commitInfo ? `${commitInfo.reviewSlug}.${env}` : env;
-
-    const domainCanonical =
-      (mergedConfig?.deploy &&
-        mergedConfig?.deploy.type === "kubernetes" &&
-        mergedConfig.deploy.cluster?.domainCanonical) || // for convenience, we allow clusters to define a canonical domain, because a cluster has a fixed ip and you will usually have a domain pointing to that cluster
-      config.domainCanonical ||
-      "panter.cloud";
-
-    const HOST_CANONICAL = `${componentSlug}.${envInUrl}.${config.appName}.${config.customerName}.${domainCanonical}`;
-    host = mergedConfig?.host ?? HOST_CANONICAL;
-    const url = `https://${host}`;
-
-    // FIXME: move to kube specific jobs
-    const KUBE_APP_NAME =
-      envType === "review" && commitInfo
-        ? `${commitInfo.reviewSlug}-${componentName}`
-        : componentName;
-    const KUBE_NAMESPACE = getKubernetesNamespace(config, env);
-
-    predefinedVariables = {
-      ...basePredefinedVariables,
-      HOST_CANONICAL,
-      ROOT_URL: url,
-      KUBE_NAMESPACE,
-      KUBE_APP_NAME,
-      RELEASE_NAME,
-    };
-  }
-  const publicEnvVarsRaw = mergedConfig.vars?.public ?? {};
-  const secretEnvVarKeys = mergedConfig.vars?.secret ?? [];
-  const secretEnvVars = Object.fromEntries(
-    secretEnvVarKeys.map((key) => [
-      key,
-      `$${getSecretVarName(env, componentName, key)}`,
-    ])
-  );
-
-  // this is deprecated, we now support: $componentname:FOO
-  const legacyFromComponents = mergedConfig.vars?.fromComponents ?? {};
-  const publicEnvVarsRawWithLegasyFromComponents = merge(
-    {},
-    translateLegacyFromComponents(legacyFromComponents),
-    publicEnvVarsRaw
-  );
-
-  const publicEnvVarsRawSanitized = Object.fromEntries(
-    Object.entries(publicEnvVarsRawWithLegasyFromComponents).map(
-      ([key, value]) => [
-        key,
-        isObject(value) ? JSON.stringify(value) : `${value}`,
-      ]
-    )
-  );
-
-  const envVarsRaw = {
-    ...predefinedVariables,
-    ...secretEnvVars,
-    ...publicEnvVarsRawSanitized,
-  };
-
-  const envVars = resolveReferences(
-    envVarsRaw,
-    (componentName, variableName, alreadyVisited) => {
-      const { envVars } = getEnvironment(
-        config,
-        componentName,
-        env,
-        commitInfo,
-        alreadyVisited
-      );
-      return envVars[variableName];
-    },
-    alreadyVisited
-  );
-  return {
-    envType,
-    host,
-    fullName: environmentName,
-    slug: environmentSlug,
-    shortName: env,
-    url: predefinedVariables.ROOT_URL,
-    envVars,
-    secretEnvVarKeys,
-  };
-};
-
-export const createContext = (
+export const createContext = async (
   config: Config,
   componentName: string,
   env: string,
   commitInfo?: CommitInfo,
   packageManagerInfo?: PackageManagerInfo
-): Context => {
+): Promise<Context> => {
   if (!/^[a-z0-9-]+$/.test(componentName)) {
     throw new Error(
       "componentName may only contain lower case letters, numbers and -"
     );
   }
-  const rawConfig = config.components[componentName];
-  if (!rawConfig) {
-    throw new Error("unknown component " + componentName);
-  }
-  // envs can override the config
-  const envConfig = rawConfig.env?.[env] ?? {};
-  const componentConfigWithoutDefaults = mergeWithMergingArrays(
-    rawConfig,
-    envConfig
+
+  const envContext = getEnvironmentContext(
+    config,
+    env,
+    componentName,
+    commitInfo
   );
 
-  // fill in defaults of build and deploy
+  const componentConfigWithoutDefaults = envContext.envConfigRaw;
   const defaults: {
     build: Partial<BuildConfig>;
     deploy: Partial<DeployConfig>;
   } = componentConfigWithoutDefaults.deploy
     ? {
         build:
-          BUILD_TYPES[componentConfigWithoutDefaults.build.type].defaults(),
+          BUILD_TYPES[
+            componentConfigWithoutDefaults.build.type as BuildConfigType
+          ].defaults(envContext),
         deploy:
-          DEPLOY_TYPES[componentConfigWithoutDefaults.deploy.type].defaults(),
+          DEPLOY_TYPES[
+            componentConfigWithoutDefaults.deploy.type as DeployConfigType
+          ].defaults(envContext),
       }
     : {
         build: {},
@@ -225,7 +59,7 @@ export const createContext = (
     fullConfig: config,
     componentConfig,
     componentName,
-    environment: getEnvironment(config, componentName, env, commitInfo),
+    environment: await getEnvironment(config, componentName, env, commitInfo),
     commitInfo,
     packageManagerInfo,
   };

package/src/context/resolveReferences.test.ts

@@ -4,7 +4,7 @@ import {
 } from "./resolveReferences";
 
 describe("resolveReferences", () => {
-  it("replaces occurences of ${componentName:VARIABLE_NAME}", () => {
+  it("replaces occurences of ${componentName:VARIABLE_NAME}", async () => {
     const variables = {
       a: "hello world",
       b: "a replaced value looks like this: '${api:FOO}', nice!",
@@ -17,9 +17,9 @@ describe("resolveReferences", () => {
         FOO: "foo from frontend",
       },
     };
-    const result = resolveReferences(
+    const result = await resolveReferences(
       variables,
-      (componentName, variableName) => {
+      async (componentName, variableName) => {
         return otherVariables[componentName]?.[variableName];
       }
     );
@@ -43,7 +43,7 @@ describe("resolveReferences", () => {
     });
   });
 
-  it("replaces self references mixed with other vars", () => {
+  it("replaces self references mixed with other vars", async () => {
     const variables = {
       FOO: "hello from ${api:FOO}",
       BAR: "this: ${FOO}!",
@@ -53,9 +53,9 @@ describe("resolveReferences", () => {
         FOO: "foo from api",
       },
     };
-    const result = resolveReferences(
+    const result = await resolveReferences(
       variables,
-      (componentName, variableName) => {
+      async (componentName, variableName) => {
         return otherVariables[componentName]?.[variableName];
       }
     );
@@ -66,15 +66,15 @@ describe("resolveReferences", () => {
     });
   });
 
-  it("keeps variables as is if not found (when null is returned)", () => {
+  it("keeps variables as is if not found (when null is returned)", async () => {
     const variables = {
       a: "hello world",
       b: "a not found value looks like this: '${api:FOO}'",
     };
     const otherVariables: Record<string, Record<string, string>> = {};
-    const result = resolveReferences(
+    const result = await resolveReferences(
       variables,
-      (componentName, variableName) => {
+      async (componentName, variableName) => {
         return otherVariables[componentName]?.[variableName];
       }
     );
@@ -85,7 +85,7 @@ describe("resolveReferences", () => {
     });
   });
 
-  it("replaces mulitple levels deep", () => {
+  it("replaces mulitple levels deep", async () => {
     const variables = {
       a: "value is ${frontend:FOO}!",
     };
@@ -97,9 +97,9 @@ describe("resolveReferences", () => {
         FOO: "hi, ${api:FOO}",
       },
     };
-    const result = resolveReferences(
+    const result = await resolveReferences(
       variables,
-      (componentName, variableName) => {
+      async (componentName, variableName) => {
         return otherVariables[componentName]?.[variableName] ?? "";
       }
     );
@@ -109,7 +109,7 @@ describe("resolveReferences", () => {
     });
   });
 
-  it("prevents endless loops", () => {
+  it("prevents endless loops", async () => {
     const variables = {
       a: "value is ${frontend:FOO}!",
     };
@@ -121,9 +121,9 @@ describe("resolveReferences", () => {
         FOO: "frontend ${api:FOO}",
       },
     };
-    const result = resolveReferences(
+    const result = await resolveReferences(
       variables,
-      (componentName, variableName) => {
+      async (componentName, variableName) => {
         return otherVariables[componentName]?.[variableName] ?? "";
       }
     );

package/src/context/resolveReferences.ts

@@ -1,55 +1,62 @@
 import { merge } from "lodash";
+import replaceAsync from "string-replace-async";
 
 const REGEX = /\$\{(([^:]+):)?([^}]+)}/gm;
 
-export const resolveReferences = (
+export const resolveReferences = async (
   vars: Record<string, string>,
   getOtherVariables?: (
     componentName: string,
     variableName: string,
     alreadyVisited: Record<string, Record<string, boolean>>
-  ) => string | null,
+  ) => Promise<string | null>,
   alreadyVisitedBase: Record<string, Record<string, boolean>> = {}
 ) => {
-  const replaceSingleValue = (
+  const replaceSingleValue = async (
     value: string,
     alreadyVisited: Record<string, Record<string, boolean>> = alreadyVisitedBase
-  ): string => {
+  ): Promise<string> => {
     if (REGEX.test(value)) {
-      return value.replace(REGEX, (match, _, componentName, variableName) => {
-        if (alreadyVisited[componentName]?.[variableName]) {
-          return match; // prevent endless loop
-        }
-        const newAlreadyVisited = merge({}, alreadyVisited, {
-          [componentName]: {
-            [variableName]: true,
-          },
-        });
-        const result = componentName
-          ? getOtherVariables?.(
-              componentName,
-              variableName,
-              newAlreadyVisited
-            ) ?? null
-          : vars[variableName]; // is self reference
+      return await replaceAsync(
+        value,
+        REGEX,
+        async (match, _, componentName, variableName) => {
+          if (alreadyVisited[componentName]?.[variableName]) {
+            return match; // prevent endless loop
+          }
+          const newAlreadyVisited = merge({}, alreadyVisited, {
+            [componentName]: {
+              [variableName]: true,
+            },
+          });
+          const result = componentName
+            ? (await getOtherVariables?.(
+                componentName,
+                variableName,
+                newAlreadyVisited
+              )) ?? null
+            : vars[variableName]; // is self reference
 
-        const replaced =
-          result !== null && result !== undefined
-            ? replaceSingleValue(result, newAlreadyVisited)
-            : match;
+          const replaced =
+            result !== null && result !== undefined
+              ? await replaceSingleValue(result, newAlreadyVisited)
+              : match;
 
-        return replaced;
-      });
+          return replaced;
+        }
+      );
     } else {
       return value;
     }
   };
 
   return Object.fromEntries(
-    Object.entries(vars).map(([key, value]) => {
-      return [key, replaceSingleValue(value)];
-    })
-  );
+    await Promise.all(
+      Object.entries(vars).map(async ([key, value]) => {
+        return [key, await replaceSingleValue(value)];
+      })
+    )
+  ) as Record<string, string>;
 };
 
 export const translateLegacyFromComponents = (

package/src/defaults/index.ts

@@ -1,4 +1,4 @@
-import { Retry } from "../types";
+import type { Retry } from "../types";
 
 export const BASE_RETRY: Retry = {
   max: 2,

package/src/deploy/base.ts

@@ -1,10 +1,12 @@
-import { requiresDockerBuild } from "../build/docker";
-import { Context } from "../types";
-import { CatladderJob } from "../types/jobs";
+import { getDockerImageVariables, requiresDockerBuild } from "../build/docker";
+import type { Context } from "../types";
+import type { CatladderJob } from "../types/jobs";
+import { isOfDeployType } from "./types";
 import { contextIsStoppable } from "./utils";
 
 export const DEPLOY_JOB_NAME = "🚀 Deploy";
 export const STOP_JOB_NAME = "🛑 Stop ⚠️";
+export const ROLLBACK_JOB_NAME = "↩️ Rollback ⚠️";
 
 const DEPLOY_RUNNER_VARIABLES = {
   KUBERNETES_CPU_REQUEST: "0.5",
@@ -14,20 +16,16 @@ const DEPLOY_RUNNER_VARIABLES = {
 };
 type JobWithoutScript = Omit<CatladderJob, "script">;
 export const getBaseDeploymentJob = (context: Context): JobWithoutScript => {
-  const environment = {
-    name: context.environment.fullName,
-    url: context.environment.url,
-  };
-
   const hasDocker = requiresDockerBuild(context);
   const isStoppable = contextIsStoppable(context);
 
   const autoStop =
     context.environment.envType === "review"
-      ? "3 days" // auto stop is not working properly when branch is merged, so we do
+      ? "2 week"
       : context.environment.envType === "dev"
-      ? "3 weeks"
+      ? "4 weeks"
       : undefined;
+
   // if auto or manual is configured explicitly, use that
   const whenDeployDefined =
     context.componentConfig.deploy && context.componentConfig.deploy.when
@@ -45,6 +43,14 @@ export const getBaseDeploymentJob = (context: Context): JobWithoutScript => {
   return {
     name: DEPLOY_JOB_NAME,
     envMode: "stagePerEnv", // makes it easier to run manual tasks er env
+
+    needsOtherComponent: context.componentConfig.deploy
+      ? context.componentConfig.deploy.waitFor?.map((c) => ({
+          componentName: c,
+          job: DEPLOY_JOB_NAME,
+          artifacts: false,
+        })) ?? undefined
+      : undefined,
     // we don't want to deploy when there is a broken test
     needsStages: [
       {
@@ -56,7 +62,6 @@ export const getBaseDeploymentJob = (context: Context): JobWithoutScript => {
         artifacts: false,
       },
     ], // workaround for https://gitlab.com/gitlab-org/gitlab/-/issues/220758
-
     rules: [
       whenDeploy === "auto"
         ? {
@@ -71,9 +76,10 @@ export const getBaseDeploymentJob = (context: Context): JobWithoutScript => {
     variables: {
       ...DEPLOY_RUNNER_VARIABLES,
       ...context.environment.envVars,
+      ...(hasDocker ? getDockerImageVariables(context) : {}),
     },
     environment: {
-      ...environment,
+      ...context.environment.gitlabEnvironment,
       ...(isStoppable
         ? {
             on_stop: STOP_JOB_NAME,
@@ -87,15 +93,10 @@ export const getBaseDeploymentJob = (context: Context): JobWithoutScript => {
 export const getBaseDeploymentStopJob = (
   context: Context
 ): JobWithoutScript => {
-  const environment = {
-    name: context.environment.fullName,
-    url: context.environment.url,
-  };
   return {
     name: STOP_JOB_NAME,
     envMode: "stagePerEnv", // makes it easier to run manual tasks er env
-
-    needs: [DEPLOY_JOB_NAME],
+    needs: [], // can be executed even if the deploy job failed
     rules: [
       {
         if: "$CI_COMMIT_BRANCH =~ /^[0-9]+\\.([0-9]+|x)\\.x$/", // automatic on hotfix branches
@@ -113,8 +114,31 @@ export const getBaseDeploymentStopJob = (
     },
     stage: "stop",
     environment: {
-      ...environment,
+      ...context.environment.gitlabEnvironment,
       action: "stop",
     },
   };
 };
+
+export const getBaseRollbackJob = (context: Context): JobWithoutScript => {
+  return {
+    name: ROLLBACK_JOB_NAME,
+    envMode: "stagePerEnv", // makes it easier to run manual tasks er env
+    needs: [], // can be executed even if the deploy job failed
+    rules: [
+      {
+        when: "manual",
+        allow_failure: true,
+      },
+    ],
+    variables: {
+      ...DEPLOY_RUNNER_VARIABLES,
+      GIT_STRATEGY: "none",
+    },
+    stage: "rollback",
+    environment: {
+      ...context.environment.gitlabEnvironment,
+      action: "access",
+    },
+  };
+};

package/src/deploy/cloudRun/deployJob.ts

@@ -0,0 +1,104 @@
+import { dump } from "js-yaml";
+import { merge, omit } from "lodash";
+import { GCLOUD_DEPLOY_CREDENTIALS_KEY } from ".";
+import { getDockerJobBaseProps, gitlabDockerLogin } from "../../build/docker";
+import { getLabels } from "../../context/getLabels";
+import { getRunnerImage } from "../../runner";
+import type { Context } from "../../types/context";
+import type { CatladderJob } from "../../types/jobs";
+import { getBaseDeploymentJob, getBaseDeploymentStopJob } from "../base";
+import { isOfDeployType } from "../types";
+import { gcloudServiceAccountLoginCommands } from "./utils/gcloudServiceAccountLoginCommands";
+
+export const createGoogleCloudRunDeployJobs = (
+  context: Context
+): CatladderJob[] => {
+  const deployConfig = context.componentConfig.deploy;
+  if (deployConfig === false) {
+    return [];
+  }
+  if (!isOfDeployType(deployConfig, "google-cloudrun", "google-cloudrun-job")) {
+    // should not happen
+    throw new Error("deploy config is wrong");
+  }
+  const baseDeploymentJob = getBaseDeploymentJob(context);
+
+  const fullAppName = `${context.fullConfig.customerName}-${context.fullConfig.appName}`;
+  const dockerUrl = `${deployConfig.region}-docker.pkg.dev/${deployConfig.projectId}/catladder-deploy/${fullAppName}`;
+  const gcloudImageName = `${dockerUrl}/$DOCKER_IMAGE_NAME:$DOCKER_IMAGE_TAG`;
+
+  const pushImageToArtifactsRegistry = [
+    gitlabDockerLogin,
+    ...gcloudServiceAccountLoginCommands(context),
+    `gcloud auth configure-docker ${deployConfig.region}-docker.pkg.dev`,
+    `docker pull $DOCKER_IMAGE:$DOCKER_IMAGE_TAG`,
+    `docker tag $DOCKER_IMAGE:$DOCKER_IMAGE_TAG ${gcloudImageName}`,
+    `docker push ${gcloudImageName}`,
+  ];
+
+  const allEnvVars = omit(
+    context.environment.envVars,
+    GCLOUD_DEPLOY_CREDENTIALS_KEY
+  );
+
+  const labelsString = Object.entries(getLabels(context))
+    .map(([key, value]) => `${key}=${value}`)
+    .join(",");
+
+  const commonArgs = `--project ${deployConfig.projectId} --region=${deployConfig.region}`;
+  const getDeployCommand = () => {
+    const name = context.environment.fullName.toLowerCase();
+    const command =
+      deployConfig.command ?? context.componentConfig.build.startCommand;
+    const commandArg = command
+      ? `--command="${command.split(" ").join(",")}"`
+      : "";
+
+    const commonDeployArgs = `${commandArg} --image ${gcloudImageName} ${commonArgs} --env-vars-file=____envvars.yaml --labels ${labelsString}`;
+    if (deployConfig.type === "google-cloudrun") {
+      return `gcloud run deploy ${name} ${commonDeployArgs} --allow-unauthenticated`;
+    }
+    if (deployConfig.type === "google-cloudrun-job") {
+      return `gcloud beta run jobs create ${name} ${commonDeployArgs}`;
+    }
+  };
+
+  const getStopCommand = () => {
+    const name = context.environment.fullName.toLowerCase();
+    const commonStopArgs = `${commonArgs} --quiet`;
+    if (deployConfig.type === "google-cloudrun") {
+      return `gcloud run services delete ${name} ${commonStopArgs}`;
+    }
+    if (deployConfig.type === "google-cloudrun-job") {
+      return `gcloud beta run jobs delete ${name} ${commonStopArgs}`;
+    }
+  };
+
+  const cloudRunDeploy = [
+    `echo "$ENV_VARS" > ____envvars.yaml`, // TODO: split secrets out
+    getDeployCommand(),
+    `docker image rm ${gcloudImageName}`,
+  ];
+
+  const baseStopJob = getBaseDeploymentStopJob(context);
+
+  return [
+    merge({}, baseDeploymentJob, getDockerJobBaseProps(context), {
+      artifacts: { paths: ["____envvars.yaml"] },
+      variables: {
+        ENV_VARS: dump(allEnvVars, {
+          lineWidth: -1,
+          quotingType: "'",
+          forceQuotes: true,
+        }),
+      },
+      image: getRunnerImage("gcloud"),
+      script: [...pushImageToArtifactsRegistry, ...cloudRunDeploy],
+    }),
+
+    merge({}, baseStopJob, {
+      image: getRunnerImage("gcloud"),
+      script: [...gcloudServiceAccountLoginCommands(context), getStopCommand()],
+    }),
+  ];
+};